@sixb/sync-worker 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -26,6 +26,21 @@ its attempts.
26
26
  One run of a sync runs at a time: a queued run waits for the one running, whose end wakes it, and
27
27
  triggers join it.
28
28
 
29
+ ## Connections
30
+
31
+ A sync of an OAuth connector runs once per connection: dispatch queues a run for each, and each run
32
+ names its `connectionId`. They still run one at a time, since they commit to one dataset. `resolveSyncRunSource` finds that one connection among those the run may
33
+ read. Its checkpoint is the last succeeded run of the same sync and connection, bound to the account
34
+ it read, so a replaced account starts over. A connection's first run of its own starts from its
35
+ entry in the last run that read every connection, from before runs were per connection.
36
+
37
+ | What happens | The run |
38
+ | --- | --- |
39
+ | The connection's grant can no longer give a token | Fails with `connector.authorization_required`. A person's own connection isn't reported to `onError` |
40
+ | The connection is gone, or its owner suspended, since it was queued | Ends `cancelled` without reading |
41
+ | It was queued before runs were per connection | Ends `cancelled` without reading; the next trigger runs each connection |
42
+ | Any other failure | Fails that run alone |
43
+
29
44
  ## Recovery
30
45
 
31
46
  Nothing of a partial run exists until its commit: rows stay in a session no reader sees, and a crash
@@ -1,24 +1,17 @@
1
1
  import { type JsonValue, type SyncConnectorConnection } from "@sixb/core";
2
- type SyncCheckpointStrategy = "single" | "connections";
3
- export interface SyncCheckpointCodec {
4
- read(value: JsonValue | undefined): JsonValue | undefined;
5
- serialize(value: JsonValue): JsonValue;
6
- }
7
- /** Bind opaque handler checkpoints to the framework source contract stored beside the run. */
8
- export declare function createSyncCheckpointCodec(options: {
2
+ interface SyncCheckpointIdentity {
9
3
  readonly syncId: string;
10
4
  readonly connectorId: string;
11
- readonly strategy: SyncCheckpointStrategy;
12
- }): SyncCheckpointCodec;
13
- /** Per-connection cursor state carried inside a framework-managed checkpoint envelope. */
14
- export declare class ConnectionCheckpoints {
15
- private readonly entries;
16
- private constructor();
17
- static from(syncId: string, value: JsonValue | undefined): ConnectionCheckpoints;
18
- reconcile(connections: readonly SyncConnectorConnection[]): void;
19
- get(connection: SyncConnectorConnection): JsonValue | undefined;
20
- set(connection: SyncConnectorConnection, checkpoint: JsonValue): void;
21
- serialize(): JsonValue;
22
5
  }
6
+ /** A static connector's checkpoint: the handler's value inside the framework envelope. */
7
+ export declare const singleCheckpoint: {
8
+ read(identity: SyncCheckpointIdentity, value: JsonValue | undefined): JsonValue | undefined;
9
+ serialize(identity: SyncCheckpointIdentity, value: JsonValue): JsonValue;
10
+ };
11
+ /** One connection's checkpoint. A replaced account starts over. */
12
+ export declare const connectionCheckpoint: {
13
+ read(identity: SyncCheckpointIdentity, connection: SyncConnectorConnection, value: JsonValue | undefined): JsonValue | undefined;
14
+ serialize(identity: SyncCheckpointIdentity, connection: SyncConnectorConnection, checkpoint: JsonValue | undefined): JsonValue;
15
+ };
23
16
  export {};
24
17
  //# sourceMappingURL=checkpoints.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,uBAAuB,EAC7B,MAAM,YAAY,CAAA;AAMnB,KAAK,sBAAsB,GAAG,QAAQ,GAAG,aAAa,CAAA;AAQtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAA;IACzD,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,CAAA;CACvC;AAED,8FAA8F;AAC9F,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAA;CAC1C,GAAG,mBAAmB,CAoCtB;AAED,0FAA0F;AAC1F,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAEhE,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,qBAAqB;IA8BhF,SAAS,CAAC,WAAW,EAAE,SAAS,uBAAuB,EAAE,GAAG,IAAI;IAShE,GAAG,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,GAAG,SAAS;IAQ/D,GAAG,CAAC,UAAU,EAAE,uBAAuB,EAAE,UAAU,EAAE,SAAS,GAAG,IAAI;IAQrE,SAAS,IAAI,SAAS;CAevB"}
1
+ {"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,uBAAuB,EAC7B,MAAM,YAAY,CAAA;AAanB,UAAU,sBAAsB;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAUD,0FAA0F;AAC1F,eAAO,MAAM,gBAAgB;mBACZ,sBAAsB,SAAS,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS;wBAOvE,sBAAsB,SAAS,SAAS,GAAG,SAAS;CAGzE,CAAA;AAED,mEAAmE;AACnE,eAAO,MAAM,oBAAoB;mBAEnB,sBAAsB,cACpB,uBAAuB,SAC5B,SAAS,GAAG,SAAS,GAC3B,SAAS,GAAG,SAAS;wBAsBZ,sBAAsB,cACpB,uBAAuB,cACvB,SAAS,GAAG,SAAS,GAChC,SAAS;CAMb,CAAA"}
package/dist/index.js CHANGED
@@ -125,226 +125,166 @@ import {
125
125
  import { createSixbError } from "@sixb/core/internal/errors";
126
126
  var SYNC_CHECKPOINT_KIND = "sync_checkpoint";
127
127
  var SYNC_CHECKPOINT_VERSION = 1;
128
- function createSyncCheckpointCodec(options) {
129
- return {
130
- read(value) {
131
- if (value === undefined)
132
- return;
133
- if (!isSyncCheckpoint(value)) {
134
- if (options.strategy === "single")
135
- return cloneJsonValue(value);
136
- throw incompatibleCheckpoint(options, "format");
137
- }
138
- if (value.version !== SYNC_CHECKPOINT_VERSION) {
139
- throw incompatibleCheckpoint(options, "version");
140
- }
141
- if (value.connectorId !== options.connectorId) {
142
- throw incompatibleCheckpoint(options, "connector");
143
- }
144
- if (value.strategy !== options.strategy) {
145
- throw incompatibleCheckpoint(options, "strategy");
146
- }
147
- if (!("value" in value) || !isJsonValue(value.value)) {
148
- throw invalidFrameworkCheckpoint(options.syncId);
149
- }
150
- return cloneJsonValue(value.value);
151
- },
152
- serialize(value) {
153
- const envelope = {
154
- kind: SYNC_CHECKPOINT_KIND,
155
- version: SYNC_CHECKPOINT_VERSION,
156
- connectorId: options.connectorId,
157
- strategy: options.strategy,
158
- value: cloneJsonValue(value)
159
- };
160
- assertJsonValue(envelope, "Sync checkpoint envelope");
161
- return envelope;
162
- }
163
- };
164
- }
165
-
166
- class ConnectionCheckpoints {
167
- entries;
168
- constructor(entries) {
169
- this.entries = entries;
170
- }
171
- static from(syncId, value) {
128
+ var singleCheckpoint = {
129
+ read(identity, value) {
172
130
  if (value === undefined)
173
- return new ConnectionCheckpoints(new Map);
174
- if (!isConnectionCheckpoint(value))
175
- throw invalidConnectionCheckpoint(syncId);
176
- const entries = new Map;
177
- for (const candidate of value.entries) {
178
- if (!isRecord(candidate) || typeof candidate.connectionId !== "string" || !candidate.connectionId.trim() || typeof candidate.accountId !== "string" || !candidate.accountId.trim() || entries.has(candidate.connectionId)) {
179
- throw invalidConnectionCheckpoint(syncId);
180
- }
181
- if ("checkpoint" in candidate && !isJsonValue(candidate.checkpoint)) {
182
- throw invalidConnectionCheckpoint(syncId);
183
- }
184
- entries.set(candidate.connectionId, {
185
- connectionId: candidate.connectionId,
186
- accountId: candidate.accountId,
187
- ...candidate.checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(candidate.checkpoint) }
188
- });
189
- }
190
- return new ConnectionCheckpoints(entries);
131
+ return;
132
+ if (!isSyncCheckpoint(value))
133
+ return cloneJsonValue(value);
134
+ return openEnvelope(identity, value, ["single"]).value;
135
+ },
136
+ serialize(identity, value) {
137
+ return envelope(identity, "single", value);
191
138
  }
192
- reconcile(connections) {
193
- for (const connection of connections) {
194
- const existing = this.entries.get(connection.id);
195
- if (existing && existing.accountId !== connection.account.id) {
196
- this.entries.delete(connection.id);
197
- }
139
+ };
140
+ var connectionCheckpoint = {
141
+ read(identity, connection, value) {
142
+ if (value === undefined)
143
+ return;
144
+ if (!isSyncCheckpoint(value))
145
+ throw incompatibleCheckpoint(identity, "format");
146
+ const opened = openEnvelope(identity, value, ["connection", "connections"]);
147
+ const entry = opened.strategy === "connection" ? opened.value : legacyEntries(identity.syncId, opened.value).find((candidate) => candidate.connectionId === connection.id);
148
+ if (entry === undefined)
149
+ return;
150
+ if (!isRecord(entry) || typeof entry.accountId !== "string") {
151
+ throw invalidCheckpoint(identity.syncId);
198
152
  }
199
- }
200
- get(connection) {
201
- const entry = this.entries.get(connection.id);
202
- if (entry?.accountId !== connection.account.id || entry.checkpoint === undefined) {
153
+ if (entry.accountId !== connection.account.id || entry.checkpoint === undefined) {
203
154
  return;
204
155
  }
156
+ if (!isJsonValue(entry.checkpoint))
157
+ throw invalidCheckpoint(identity.syncId);
205
158
  return cloneJsonValue(entry.checkpoint);
206
- }
207
- set(connection, checkpoint) {
208
- this.entries.set(connection.id, {
209
- connectionId: connection.id,
159
+ },
160
+ serialize(identity, connection, checkpoint) {
161
+ return envelope(identity, "connection", {
210
162
  accountId: connection.account.id,
211
- checkpoint: cloneJsonValue(checkpoint)
163
+ ...checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(checkpoint) }
212
164
  });
213
165
  }
214
- serialize() {
215
- const value = {
216
- entries: [...this.entries.values()].sort((left, right) => left.connectionId.localeCompare(right.connectionId)).map((entry) => ({
217
- connectionId: entry.connectionId,
218
- accountId: entry.accountId,
219
- ...entry.checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(entry.checkpoint) }
220
- }))
221
- };
222
- assertJsonValue(value, "Managed connector Sync checkpoint");
223
- return value;
224
- }
166
+ };
167
+ function openEnvelope(identity, value, strategies) {
168
+ if (value.version !== SYNC_CHECKPOINT_VERSION)
169
+ throw incompatibleCheckpoint(identity, "version");
170
+ if (value.connectorId !== identity.connectorId) {
171
+ throw incompatibleCheckpoint(identity, "connector");
172
+ }
173
+ const strategy = strategies.find((candidate) => candidate === value.strategy);
174
+ if (!strategy)
175
+ throw incompatibleCheckpoint(identity, "strategy");
176
+ if (!("value" in value) || !isJsonValue(value.value))
177
+ throw invalidCheckpoint(identity.syncId);
178
+ return { strategy, value: cloneJsonValue(value.value) };
179
+ }
180
+ function envelope(identity, strategy, value) {
181
+ const serialized = {
182
+ kind: SYNC_CHECKPOINT_KIND,
183
+ version: SYNC_CHECKPOINT_VERSION,
184
+ connectorId: identity.connectorId,
185
+ strategy,
186
+ value: cloneJsonValue(value)
187
+ };
188
+ assertJsonValue(serialized, "Sync checkpoint envelope");
189
+ return serialized;
225
190
  }
226
- function incompatibleCheckpoint(options, reason) {
227
- return createSixbError("sync.execution_failed", `[SixbSyncWorker] Sync '${options.syncId}' has a checkpoint incompatible with connector '${options.connectorId}'. Use a new Sync id to reset its ingestion state.`, {
191
+ function legacyEntries(syncId, value) {
192
+ if (!isRecord(value) || !Array.isArray(value.entries))
193
+ throw invalidCheckpoint(syncId);
194
+ return value.entries.flatMap((entry) => isRecord(entry) ? [entry] : []);
195
+ }
196
+ function incompatibleCheckpoint(identity, reason) {
197
+ return createSixbError("sync.execution_failed", `[SixbSyncWorker] Sync '${identity.syncId}' has a checkpoint incompatible with connector '${identity.connectorId}'. Use a new Sync id to reset its ingestion state.`, {
228
198
  details: {
229
- syncId: options.syncId,
230
- connectorId: options.connectorId,
199
+ syncId: identity.syncId,
200
+ connectorId: identity.connectorId,
231
201
  reason: `checkpoint_${reason}`
232
202
  }
233
203
  });
234
204
  }
235
- function invalidFrameworkCheckpoint(syncId) {
205
+ function invalidCheckpoint(syncId) {
236
206
  return createSixbError("internal.unexpected", `[SixbSyncWorker] Sync '${syncId}' has an invalid checkpoint envelope.`, { details: { syncId } });
237
207
  }
238
- function invalidConnectionCheckpoint(syncId) {
239
- return createSixbError("internal.unexpected", `[SixbSyncWorker] Sync '${syncId}' has an invalid managed connector checkpoint.`, { details: { syncId } });
240
- }
241
208
  function isSyncCheckpoint(value) {
242
209
  return isRecord(value) && value.kind === SYNC_CHECKPOINT_KIND;
243
210
  }
244
- function isConnectionCheckpoint(value) {
245
- return isRecord(value) && Array.isArray(value.entries);
246
- }
247
211
  function isRecord(value) {
248
212
  return typeof value === "object" && value !== null && !Array.isArray(value);
249
213
  }
250
214
 
251
215
  // src/source-read.ts
252
- async function readSyncValues(options) {
253
- const sources = await options.runtime.connectorSources.list(options.sync.connector);
254
- if (!isOAuthConnectorDefinition(options.sync.connector)) {
255
- if (sources.length !== 1 || sources[0]?.connection !== undefined) {
256
- throw sourceInvariant(options.sync.id, "Static connectors must resolve exactly one source.");
216
+ class SyncRunSourceGoneError extends Error {
217
+ name = "SyncRunSourceGoneError";
218
+ }
219
+ async function resolveSyncRunSource(runtime, sync, run) {
220
+ const sources = await runtime.connectorSources.list(sync.connector);
221
+ if (!isOAuthConnectorDefinition(sync.connector)) {
222
+ if (run.connectionId !== undefined) {
223
+ throw sourceInvariant(sync.id, `Static connector run '${run.id}' names a connection.`);
257
224
  }
258
- const checkpointCodec = createSyncCheckpointCodec({
259
- syncId: options.sync.id,
260
- connectorId: options.sync.connector.id,
261
- strategy: "single"
262
- });
263
- return readStaticSource(options, sources[0], checkpointCodec);
264
- }
265
- const connectionIds = new Set;
266
- const managedSources = sources.map((source) => {
267
- if (!source.connection) {
268
- throw sourceInvariant(options.sync.id, "OAuth connector sources must include connection metadata.");
269
- }
270
- const connection = source.connection;
271
- if (connection.connectorId !== options.sync.connector.id) {
272
- throw sourceInvariant(options.sync.id, `Connection '${connection.id}' belongs to connector '${connection.connectorId}', not '${options.sync.connector.id}'.`);
225
+ const [source] = sources;
226
+ if (sources.length !== 1 || !source || source.connection !== undefined) {
227
+ throw sourceInvariant(sync.id, "Static connectors must resolve exactly one source.");
273
228
  }
274
- if (connectionIds.has(connection.id)) {
275
- throw sourceInvariant(options.sync.id, `Connection '${connection.id}' was resolved twice.`);
276
- }
277
- connectionIds.add(connection.id);
278
- return {
279
- connection,
280
- connect: (signal) => source.connect(signal)
281
- };
282
- });
283
- managedSources.sort((left, right) => left.connection.id.localeCompare(right.connection.id));
284
- const checkpointCodec = createSyncCheckpointCodec({
285
- syncId: options.sync.id,
286
- connectorId: options.sync.connector.id,
287
- strategy: "connections"
288
- });
289
- const checkpoints = ConnectionCheckpoints.from(options.sync.id, checkpointCodec.read(options.previousCheckpoint));
290
- checkpoints.reconcile(managedSources.map((source) => source.connection));
291
- const storeCheckpoint = () => options.setCheckpoint(checkpointCodec.serialize(checkpoints.serialize()));
292
- storeCheckpoint();
293
- return readManagedSources(options, managedSources, checkpoints, storeCheckpoint);
294
- }
295
- async function readStaticSource(options, source, checkpointCodec) {
296
- const previousCheckpoint = checkpointCodec.read(options.previousCheckpoint);
229
+ return { kind: "static", source };
230
+ }
231
+ if (run.connectionId === undefined) {
232
+ throw new SyncRunSourceGoneError(`[SixbSyncWorker] Sync run '${run.id}' was queued before a sync of an OAuth connector ran once per connection. The sync's next trigger runs each connection.`);
233
+ }
234
+ const source = sources.find((candidate) => candidate.connection?.id === run.connectionId);
235
+ if (!source?.connection) {
236
+ throw new SyncRunSourceGoneError(`[SixbSyncWorker] Connection '${run.connectionId}' of sync '${sync.id}' is no longer connected, or its owner is suspended.`);
237
+ }
238
+ if (source.connection.connectorId !== sync.connector.id) {
239
+ throw sourceInvariant(sync.id, `Connection '${source.connection.id}' belongs to connector '${source.connection.connectorId}', not '${sync.connector.id}'.`);
240
+ }
241
+ return { kind: "connection", source, connection: source.connection };
242
+ }
243
+ async function readSyncValues(options) {
244
+ const { source } = options;
245
+ return source.kind === "static" ? readStaticSource(options, source.source) : readConnectionSource(options, source.source, source.connection);
246
+ }
247
+ async function readStaticSource(options, source) {
248
+ const identity = checkpointIdentity(options.sync);
249
+ const previousCheckpoint = singleCheckpoint.read(identity, options.previousCheckpoint);
297
250
  if (previousCheckpoint !== undefined) {
298
- options.setCheckpoint(checkpointCodec.serialize(previousCheckpoint));
251
+ options.setCheckpoint(singleCheckpoint.serialize(identity, previousCheckpoint));
299
252
  }
300
253
  const client = await runAbortable(options.signal, () => source.connect(options.signal));
301
254
  throwIfAborted(options.signal);
302
255
  const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
303
- projectId: options.runtime.id,
304
- syncId: options.sync.id,
305
- signal: options.signal,
306
- blobs: options.blobStorage,
307
- logger: options.logger,
256
+ ...readContext(options),
308
257
  checkpoint: previousCheckpoint === undefined ? undefined : cloneJsonValue2(previousCheckpoint),
309
258
  setCheckpoint(next) {
310
259
  assertJsonValue2(next, `Sync '${options.sync.id}' checkpoint`);
311
- options.setCheckpoint(checkpointCodec.serialize(cloneJsonValue2(next)));
260
+ options.setCheckpoint(singleCheckpoint.serialize(identity, cloneJsonValue2(next)));
312
261
  }
313
262
  }));
314
263
  return sourceValues(normalizeReadResult(readResult, options.sync.id, options.signal));
315
264
  }
316
- async function* readManagedSources(options, sources, checkpoints, storeCheckpoint) {
317
- for (const source of sources) {
318
- yield* readManagedSource(options, source, checkpoints, storeCheckpoint);
319
- }
320
- }
321
- async function* readManagedSource(options, source, checkpoints, storeCheckpoint) {
322
- const { connection } = source;
265
+ async function* readConnectionSource(options, source, connection) {
266
+ const identity = checkpointIdentity(options.sync);
267
+ const previous = connectionCheckpoint.read(identity, connection, options.previousCheckpoint);
268
+ options.setCheckpoint(connectionCheckpoint.serialize(identity, connection, previous));
323
269
  try {
324
270
  throwIfAborted(options.signal);
325
271
  let client;
326
272
  try {
327
273
  client = await runAbortable(options.signal, () => source.connect(options.signal));
328
274
  } catch (error) {
329
- if (!skipsUnauthorizedConnection(options.sync, connection, error))
330
- throw error;
331
- console.warn(`[SixbSyncWorker] Sync '${options.sync.id}' skipped connection '${connection.id}' (${connection.account.label}): it needs reconnecting.`);
332
- return;
275
+ if (isSixbError(error) && error.code === "connector.authorization_required") {
276
+ throw needsReconnecting(options, connection, error);
277
+ }
278
+ throw error;
333
279
  }
334
280
  throwIfAborted(options.signal);
335
- const previous = checkpoints.get(connection);
336
281
  const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
337
- projectId: options.runtime.id,
338
- syncId: options.sync.id,
339
- signal: options.signal,
340
- blobs: options.blobStorage,
341
- logger: options.logger,
282
+ ...readContext(options),
342
283
  connection,
343
284
  checkpoint: previous === undefined ? undefined : cloneJsonValue2(previous),
344
285
  setCheckpoint(next) {
345
286
  assertJsonValue2(next, `Sync '${options.sync.id}' connection '${connection.id}' checkpoint`);
346
- checkpoints.set(connection, cloneJsonValue2(next));
347
- storeCheckpoint();
287
+ options.setCheckpoint(connectionCheckpoint.serialize(identity, connection, cloneJsonValue2(next)));
348
288
  }
349
289
  }));
350
290
  for await (const value of normalizeReadResult(readResult, options.sync.id, options.signal)) {
@@ -353,20 +293,40 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
353
293
  } catch (error) {
354
294
  if (options.signal.aborted)
355
295
  throw error;
356
- throw createSixbError2("sync.execution_failed", summarizeErrorMessage(error, "Managed connector Sync source failed."), {
357
- cause: error,
358
- details: {
359
- syncId: options.sync.id,
360
- runId: options.runId,
361
- datasetId: options.datasetId,
362
- connectionId: connection.id,
363
- accountId: connection.account.id
364
- }
365
- });
296
+ if (isSixbError(error) && error.code === "connector.authorization_required")
297
+ throw error;
298
+ throw createSixbError2("sync.execution_failed", summarizeErrorMessage(error, "Connector connection Sync source failed."), { cause: error, details: connectionDetails(options, connection) });
366
299
  }
367
300
  }
368
- function skipsUnauthorizedConnection(sync, connection, error) {
369
- return sync.config.mode !== "snapshot" && connection.owner.type === "user" && isSixbError(error) && error.code === "connector.authorization_required";
301
+ function needsReconnecting(options, connection, cause) {
302
+ if (connection.owner.type === "user") {
303
+ console.warn(`[SixbSyncWorker] Sync '${options.sync.id}' could not read connection '${connection.id}': it needs reconnecting.`);
304
+ }
305
+ return createSixbError2("connector.authorization_required", `[SixbSyncWorker] Connection '${connection.id}' of sync '${options.sync.id}' needs reconnecting.`, {
306
+ cause,
307
+ details: { ...connectionDetails(options, connection), ownerType: connection.owner.type }
308
+ });
309
+ }
310
+ function readContext(options) {
311
+ return {
312
+ projectId: options.runtime.id,
313
+ syncId: options.sync.id,
314
+ signal: options.signal,
315
+ blobs: options.blobStorage,
316
+ logger: options.logger
317
+ };
318
+ }
319
+ function checkpointIdentity(sync) {
320
+ return { syncId: sync.id, connectorId: sync.connector.id };
321
+ }
322
+ function connectionDetails(options, connection) {
323
+ return {
324
+ syncId: options.sync.id,
325
+ runId: options.runId,
326
+ datasetId: options.datasetId,
327
+ connectionId: connection.id,
328
+ accountId: connection.account.id
329
+ };
370
330
  }
371
331
  function sourceInvariant(syncId, message) {
372
332
  return createSixbError2("internal.unexpected", `[SixbSyncWorker] ${message}`, {
@@ -409,7 +369,7 @@ function requireFinishedAt(input) {
409
369
  throw createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync run '${input.runId}' finished without a finishedAt timestamp.`, { details: { syncId: input.syncId, runId: input.runId } });
410
370
  }
411
371
  function translateSyncExecutionError(error, input) {
412
- if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed")) {
372
+ if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed" || error.code === "connector.authorization_required")) {
413
373
  return error;
414
374
  }
415
375
  return createSixbError3("sync.execution_failed", summarizeErrorMessage2(error, "Sync execution failed."), {
@@ -483,7 +443,8 @@ async function commitRun(execution, logger) {
483
443
  const { lakeStorage, blobs: blobStorage } = runtime;
484
444
  const sync = requireSync(runtime, run);
485
445
  const dataset = requireDataset(runtime, sync, run);
486
- const previousCheckpoint = await readPreviousCheckpoint(runtime, sync);
446
+ const source = await resolveSyncRunSource(runtime, sync, run);
447
+ const previousCheckpoint = await readPreviousCheckpoint(runtime, sync, run);
487
448
  execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint);
488
449
  const { outcome, version } = await writeDataset({
489
450
  lakeStorage,
@@ -495,6 +456,7 @@ async function commitRun(execution, logger) {
495
456
  readValues: () => readSyncValues({
496
457
  runtime,
497
458
  sync,
459
+ source,
498
460
  runId: run.id,
499
461
  datasetId: dataset.id,
500
462
  signal,
@@ -598,6 +560,10 @@ async function settleFailure(execution, caught) {
598
560
  const landed = committedVersion ? null : await landedCommitBehind(execution, caught);
599
561
  if (landed)
600
562
  return finishLanded(execution, landed, landedRead(execution, caught));
563
+ if (caught instanceof SyncRunSourceGoneError && !signal.aborted) {
564
+ await recordCancelled(execution, caught);
565
+ throw caught;
566
+ }
601
567
  const error = translateLostRace(run, caught);
602
568
  if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
603
569
  await recordFailed(execution, error);
@@ -634,18 +600,42 @@ async function recordFailed(execution, error) {
634
600
  onRunFailed?.(error, failedRun, failure);
635
601
  } catch {}
636
602
  }
603
+ async function recordCancelled(execution, reason) {
604
+ const { run } = execution;
605
+ const { runtime, claim, onRunFinished } = execution.input;
606
+ try {
607
+ const cancelled = await runtime.syncRunsStorage.finish({
608
+ projectId: runtime.id,
609
+ id: run.id,
610
+ executionToken: claim.execution.token,
611
+ status: "cancelled",
612
+ rowsRead: 0,
613
+ error: captureSixbFailure(createSixbError3("runtime.cancelled", reason.message), {
614
+ allowedCodes: SYNC_RUN_FAILURE_CODES,
615
+ defaultCode: "runtime.cancelled"
616
+ })
617
+ });
618
+ await notifyRunFinished(onRunFinished, cancelled);
619
+ } catch {}
620
+ }
637
621
  function cloneCheckpoint(checkpoint) {
638
622
  return checkpoint === undefined ? undefined : cloneJsonValue3(checkpoint);
639
623
  }
640
- async function readPreviousCheckpoint(runtime, sync) {
641
- const { runs } = await runtime.syncRunsStorage.list({
624
+ async function readPreviousCheckpoint(runtime, sync, run) {
625
+ const latest = (connectionId) => runtime.syncRunsStorage.list({
642
626
  projectId: runtime.id,
643
627
  syncId: sync.id,
628
+ ...connectionId === undefined ? {} : { connectionId },
644
629
  statuses: ["succeeded"],
645
630
  limit: 1,
646
631
  order: "desc"
647
- });
648
- return runs[0]?.checkpoint;
632
+ }).then(({ runs }) => runs[0]);
633
+ if (run.connectionId === undefined)
634
+ return (await latest())?.checkpoint;
635
+ const own = await latest(run.connectionId);
636
+ if (own)
637
+ return own.checkpoint;
638
+ return (await latest(null))?.checkpoint;
649
639
  }
650
640
  function requireDataset(runtime, sync, run) {
651
641
  const dataset = runtime.datasets.getById(run.datasetId);
@@ -744,6 +734,8 @@ class SyncWorker extends RunWorker {
744
734
  await this.wakeNextRun(finished);
745
735
  },
746
736
  onRunFailed: (error, run, failure) => {
737
+ if (isPersonalReconnect(failure))
738
+ return;
747
739
  reportRunFailure(this.host, error, {
748
740
  projectId: this.host.id,
749
741
  attempt: run.attempt,
@@ -795,6 +787,7 @@ async function emitSyncRunStarted(host, run, correlationId) {
795
787
  payload: {
796
788
  syncId: run.syncId,
797
789
  runId: run.id,
790
+ ...run.connectionId === undefined ? {} : { connectionId: run.connectionId },
798
791
  startedAt: run.startedAt.toISOString()
799
792
  }
800
793
  }
@@ -827,6 +820,7 @@ async function emitSyncRunFinishedEvents(host, run, correlationId, createdVersio
827
820
  payload: {
828
821
  syncId: run.syncId,
829
822
  runId: run.id,
823
+ ...run.connectionId === undefined ? {} : { connectionId: run.connectionId },
830
824
  status: requireTerminalStatus(run.status, `Sync run '${run.id}'`),
831
825
  datasetId: run.datasetId,
832
826
  ...run.output ? { versionId: run.output.versionId } : {},
@@ -837,6 +831,10 @@ async function emitSyncRunFinishedEvents(host, run, correlationId, createdVersio
837
831
  correlationId
838
832
  }, { source: SOURCE });
839
833
  }
834
+ function isPersonalReconnect(failure) {
835
+ const details = failure.details;
836
+ return failure.code === "connector.authorization_required" && isRecord2(details) && details.ownerType === "user";
837
+ }
840
838
  function requireTerminalStatus(status, context) {
841
839
  if (status === "queued" || status === "running") {
842
840
  throw new Error(`[SixbSyncWorker] ${context} is not terminal.`);
@@ -863,8 +861,11 @@ function buildSyncContext(host, sixb) {
863
861
  }
864
862
  };
865
863
  }
864
+ function isRecord2(value) {
865
+ return typeof value === "object" && value !== null && !Array.isArray(value);
866
+ }
866
867
  export {
867
868
  SyncWorker
868
869
  };
869
870
 
870
- //# debugId=734C88A91BA0AEF864756E2164756E21
871
+ //# debugId=FDCAB685EF4DF87464756E2164756E21