@sixb/sync-worker 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,11 @@
1
- import { cloneJsonValue, type DatasetDefinition, type JsonValue } from "@sixb/core"
2
- import { writeDataset } from "@sixb/core/internal/datasets"
1
+ import {
2
+ cloneJsonValue,
3
+ type DatasetDefinition,
4
+ type JsonValue,
5
+ type Logger,
6
+ type SyncDefinition,
7
+ } from "@sixb/core"
8
+ import { findProducedVersion, writeDataset } from "@sixb/core/internal/datasets"
3
9
  import {
4
10
  captureSixbFailure,
5
11
  createSixbError,
@@ -7,30 +13,45 @@ import {
7
13
  summarizeErrorMessage,
8
14
  } from "@sixb/core/internal/errors"
9
15
  import { resolveLoggingService } from "@sixb/core/internal/logging"
10
- import type { DatasetVersion } from "@sixb/core/lake-storage"
16
+ import { type DatasetVersion, LakeConcurrencyError } from "@sixb/core/lake-storage"
11
17
  import { SYNC_RUN_FAILURE_CODES, type SyncRunRecord } from "@sixb/core/storage"
12
18
  import { throwIfAborted } from "./normalize"
13
19
  import { readSyncValues, type SyncSourceValue } from "./source-read"
14
20
  import type { RunSyncJobInput, SyncRunFinishedHandler, SyncRunResult } from "./types"
15
21
 
16
- export class SyncRunAlreadyStartedError extends Error {
17
- override readonly name = "SyncRunAlreadyStartedError"
22
+ /** What the execution behind a commit read, which the run records once it finds that commit. */
23
+ interface LandedRead {
24
+ readonly rowsRead: number
25
+ readonly checkpoint: JsonValue | undefined
26
+ }
27
+
28
+ /** What a run found just before committing: an earlier execution's commit landed after all. */
29
+ class CommitAlreadyLandedError extends Error {
30
+ override readonly name = "CommitAlreadyLandedError"
18
31
 
19
- constructor(readonly run: SyncRunRecord) {
20
- super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`)
32
+ constructor(readonly version: DatasetVersion) {
33
+ super(`[SixbSyncWorker] Version '${version.versionId}' already holds this run's rows.`)
21
34
  }
22
35
  }
23
36
 
37
+ /**
38
+ * Another writer moved the dataset between this run's guard and its commit. Nothing was written, so
39
+ * the run stays running and the next delivery reads its source again.
40
+ */
41
+ class CommitLostRaceError extends Error {
42
+ override readonly name = "CommitLostRaceError"
43
+ }
44
+
24
45
  function createBookkeepingError(options: {
25
46
  syncId: string
26
47
  runId: string
27
48
  version: DatasetVersion
28
49
  cause: unknown
29
50
  }): Error {
30
- // The lake commit is already durable here, so we surface an explicit repair-needed error.
51
+ // The lake commit is already durable here: the run stays running, and the next delivery finds it.
31
52
  return createSixbError(
32
53
  "internal.unexpected",
33
- `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`,
54
+ `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'.`,
34
55
  {
35
56
  cause: options.cause,
36
57
  details: {
@@ -113,22 +134,34 @@ function sourceValidationError(
113
134
  })
114
135
  }
115
136
 
137
+ /** What one execution of a run learns as it goes, for the steps below to record. */
138
+ interface SyncExecution {
139
+ readonly input: RunSyncJobInput
140
+ readonly run: SyncRunRecord
141
+ readonly signal: AbortSignal
142
+ rowsRead: number
143
+ nextCheckpoint: JsonValue | undefined
144
+ /** The version this execution knows the run has, once the lake and the run agree on it. */
145
+ committedVersion: DatasetVersion | undefined
146
+ /** When this run first reached a commit: the earliest a version of it can have landed. */
147
+ commitPreparedAt: Date | undefined
148
+ /** Whether a commit of this very execution is the one that may have landed. */
149
+ reachedCommit: boolean
150
+ }
151
+
116
152
  /**
117
- * Run one already-persisted Sync run end to end.
153
+ * Run what is left of one Sync run this delivery holds, and record how it ended.
118
154
  *
119
- * Dispatch owns identity and queueing. The worker only transitions the durable run from queued to
120
- * running, streams source values, and records its terminal outcome.
155
+ * Dispatch owns identity and queueing, and the worker owns the claim. A run reads its whole source
156
+ * into one lake commit, so what is left is either that commit, from the last succeeded run's
157
+ * checkpoint, or nothing but its record: a run that resumes asks the lake which.
158
+ *
159
+ * A stopped worker writes nothing here: the run stays running, and the next delivery takes it up.
121
160
  */
122
161
  export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {
123
- const { runtime, run } = input
124
- const job = {
125
- id: run.id,
126
- syncId: run.syncId,
127
- expectedLatestVersionId: run.expectedLatestVersionId,
128
- commitMessage: run.commitMessage,
129
- }
162
+ const { runtime, claim } = input
163
+ const { run } = claim
130
164
  const signal = input.signal ?? new AbortController().signal
131
- const { syncRunsStorage, lakeStorage, blobs: blobStorage } = runtime
132
165
 
133
166
  if (run.projectId !== runtime.id) {
134
167
  throw new Error(
@@ -138,183 +171,344 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
138
171
 
139
172
  throwIfAborted(signal)
140
173
 
141
- let startedRun: SyncRunRecord
142
- try {
143
- startedRun = await syncRunsStorage.start({
144
- projectId: runtime.id,
145
- id: job.id,
146
- })
147
- } catch (error) {
148
- const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })
149
- // A run still queued did not start elsewhere: the start itself failed.
150
- if (
151
- existing?.syncId === run.syncId &&
152
- existing.executionId === run.executionId &&
153
- existing.status !== "queued"
154
- ) {
155
- throw new SyncRunAlreadyStartedError(existing)
156
- }
157
- throw error
158
- }
159
- await input.onRunStarted?.(startedRun)
174
+ if (claim.outcome === "started") await input.onRunStarted?.(run)
160
175
 
161
176
  const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({
162
177
  kind: "sync",
163
- id: job.id,
178
+ id: run.id,
164
179
  })
165
- const logger = logSession.logger
166
- let rowsRead = 0
167
- let committedVersion: DatasetVersion | undefined
180
+
181
+ const execution: SyncExecution = {
182
+ input,
183
+ run,
184
+ signal,
185
+ rowsRead: 0,
186
+ nextCheckpoint: undefined,
187
+ committedVersion: undefined,
188
+ commitPreparedAt: run.commitPreparedAt,
189
+ reachedCommit: false,
190
+ }
168
191
 
169
192
  try {
170
- const sync = runtime.syncs.getById(run.syncId)
171
- if (!sync) {
172
- throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`)
173
- }
174
- if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
175
- throw new Error(
176
- `[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`
177
- )
193
+ if (claim.outcome === "resumed") {
194
+ const landed = await findLandedCommit(execution)
195
+ if (landed) return await finishLanded(execution, landed, readByEarlierExecution(run))
178
196
  }
179
197
 
180
- const dataset = runtime.datasets.getById(run.datasetId)
181
- if (!dataset) {
182
- throw new Error(
183
- `[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`
184
- )
185
- }
198
+ if (claim.givenUp) throw claim.givenUp
186
199
 
187
- const latestSuccessfulRuns = await syncRunsStorage.list({
188
- projectId: runtime.id,
189
- syncId: sync.id,
190
- statuses: ["succeeded"],
191
- limit: 1,
192
- order: "desc",
193
- })
194
- const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint
195
- let nextCheckpoint: JsonValue | undefined =
196
- previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined
197
-
198
- const producer = {
199
- kind: "sync" as const,
200
- id: sync.id,
201
- runId: job.id,
202
- }
203
- const readValues = () =>
200
+ return await commitRun(execution, logSession.logger)
201
+ } catch (caught) {
202
+ return await settleFailure(execution, caught)
203
+ } finally {
204
+ await logSession.flush()
205
+ }
206
+ }
207
+
208
+ /** Reads the source from the last succeeded run's checkpoint into one commit. */
209
+ async function commitRun(execution: SyncExecution, logger: Logger): Promise<SyncRunResult> {
210
+ const { run, signal } = execution
211
+ const { runtime } = execution.input
212
+ const { lakeStorage, blobs: blobStorage } = runtime
213
+
214
+ const sync = requireSync(runtime, run)
215
+ const dataset = requireDataset(runtime, sync, run)
216
+
217
+ const previousCheckpoint = await readPreviousCheckpoint(runtime, sync)
218
+ execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint)
219
+
220
+ const { outcome, version } = await writeDataset({
221
+ lakeStorage,
222
+ blobStorage,
223
+ dataset,
224
+ mode: sync.config.mode,
225
+ signal,
226
+ producer: { kind: "sync", id: sync.id, runId: run.id },
227
+ readValues: () =>
204
228
  readSyncValues({
205
229
  runtime,
206
230
  sync,
207
- runId: job.id,
231
+ runId: run.id,
208
232
  datasetId: dataset.id,
209
233
  signal,
210
234
  blobStorage,
211
235
  logger,
212
236
  previousCheckpoint,
213
237
  setCheckpoint(next) {
214
- nextCheckpoint = next === undefined ? undefined : cloneJsonValue(next)
238
+ execution.nextCheckpoint = cloneCheckpoint(next)
215
239
  },
216
- })
217
- const { outcome, version } = await writeDataset({
218
- lakeStorage,
219
- blobStorage,
220
- dataset,
221
- mode: sync.config.mode,
222
- signal,
223
- producer,
224
- readValues,
225
- expectedLatestVersionId: job.expectedLatestVersionId,
226
- commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,
227
- sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
228
- onRead(count) {
229
- rowsRead = count
230
- },
231
- mapValidationError: (error, value, itemIndex) =>
232
- sourceValidationError(error, value, { syncId: sync.id, runId: job.id, dataset }, itemIndex),
233
- })
234
- if (outcome === "created") {
235
- committedVersion = version
236
- }
237
- let finishedRun: SyncRunRecord
238
-
239
- try {
240
- finishedRun = await syncRunsStorage.finish({
241
- projectId: runtime.id,
242
- id: job.id,
243
- status: "succeeded",
244
- rowsRead,
245
- ...(version
246
- ? { output: { datasetId: version.datasetId, versionId: version.versionId } }
247
- : {}),
248
- checkpoint: nextCheckpoint,
249
- })
250
- } catch (error) {
251
- if (committedVersion) {
252
- throw createBookkeepingError({
253
- syncId: sync.id,
254
- runId: job.id,
255
- version: committedVersion,
256
- cause: error,
257
- })
258
- }
259
- throw error
260
- }
240
+ }),
241
+ expectedLatestVersionId: run.expectedLatestVersionId,
242
+ commitMessage: run.commitMessage ?? `sync ${sync.id} run ${run.id}`,
243
+ sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
244
+ onRead(count) {
245
+ execution.rowsRead = count
246
+ },
247
+ mapValidationError: (error, value, itemIndex) =>
248
+ sourceValidationError(error, value, { syncId: sync.id, runId: run.id, dataset }, itemIndex),
249
+ guardLatestVersion: true,
250
+ beforeCommit: () => prepareCommit(execution),
251
+ })
261
252
 
262
- const finishedAt = requireFinishedAt({
263
- syncId: sync.id,
264
- runId: job.id,
265
- finishedAt: finishedRun.finishedAt,
266
- })
267
- await notifyRunFinished(input.onRunFinished, finishedRun, committedVersion)
268
- const result = {
269
- id: job.id,
270
- syncId: sync.id,
271
- datasetId: dataset.id,
272
- mode: sync.config.mode,
273
- startedAt: requireStartedAt(job.id, startedRun.startedAt),
274
- finishedAt,
253
+ if (outcome === "created") execution.committedVersion = version
254
+
255
+ return finishSucceeded(execution, {
256
+ version,
257
+ versionCreated: outcome === "created",
258
+ rowsRead: execution.rowsRead,
259
+ checkpoint: execution.nextCheckpoint,
260
+ })
261
+ }
262
+
263
+ /**
264
+ * A crash right after the commit would lose the checkpoint it paid for, so it goes first. A stale
265
+ * execution stops here; one that slept past this point meets the guard.
266
+ */
267
+ async function prepareCommit(execution: SyncExecution): Promise<void> {
268
+ const { run } = execution
269
+ const { runtime, claim } = execution.input
270
+
271
+ const landed = await findLandedCommit(execution)
272
+ if (landed) throw new CommitAlreadyLandedError(landed)
273
+
274
+ const prepared = await runtime.syncRunsStorage.prepareCommit({
275
+ projectId: runtime.id,
276
+ id: run.id,
277
+ executionToken: claim.execution.token,
278
+ rowsRead: execution.rowsRead,
279
+ checkpoint: execution.nextCheckpoint,
280
+ })
281
+
282
+ execution.commitPreparedAt ??= prepared.commitPreparedAt
283
+ execution.reachedCommit = true
284
+ }
285
+
286
+ /** The lake and the run agree from here: what is left is only telling others. */
287
+ async function finishSucceeded(
288
+ execution: SyncExecution,
289
+ result: LandedRead & {
290
+ readonly version: DatasetVersion | null | undefined
291
+ readonly versionCreated: boolean
292
+ }
293
+ ): Promise<SyncRunResult> {
294
+ const { run, committedVersion } = execution
295
+ const { version, versionCreated, rowsRead } = result
296
+
297
+ const finished = await recordSucceeded(execution, result)
298
+
299
+ const finishedAt = requireFinishedAt({
300
+ syncId: run.syncId,
301
+ runId: run.id,
302
+ finishedAt: finished.finishedAt,
303
+ })
304
+
305
+ await notifyRunFinished(execution.input.onRunFinished, finished, committedVersion)
306
+
307
+ const summary = {
308
+ id: run.id,
309
+ syncId: run.syncId,
310
+ datasetId: run.datasetId,
311
+ mode: run.mode,
312
+ startedAt: requireStartedAt(run.id, run.startedAt),
313
+ finishedAt,
314
+ rowsRead,
315
+ }
316
+
317
+ return version ? { ...summary, version, versionCreated } : { ...summary, versionCreated: false }
318
+ }
319
+
320
+ async function recordSucceeded(
321
+ execution: SyncExecution,
322
+ result: LandedRead & { readonly version: DatasetVersion | null | undefined }
323
+ ): Promise<SyncRunRecord> {
324
+ const { run, committedVersion } = execution
325
+ const { runtime, claim } = execution.input
326
+ const { version, rowsRead, checkpoint } = result
327
+
328
+ try {
329
+ return await runtime.syncRunsStorage.finish({
330
+ projectId: runtime.id,
331
+ id: run.id,
332
+ executionToken: claim.execution.token,
333
+ status: "succeeded",
275
334
  rowsRead,
276
- }
277
- return version
278
- ? { ...result, version, versionCreated: outcome === "created" }
279
- : { ...result, versionCreated: false }
335
+ ...(version
336
+ ? { output: { datasetId: version.datasetId, versionId: version.versionId } }
337
+ : {}),
338
+ checkpoint,
339
+ })
280
340
  } catch (error) {
281
- if (!committedVersion) {
282
- // The shared writer has already cleaned up its session. Finalize the failed sync run.
283
- const status = signal.aborted ? "cancelled" : "failed"
284
- try {
285
- const failureError =
286
- status === "failed"
287
- ? translateSyncExecutionError(error, {
288
- syncId: run.syncId,
289
- runId: job.id,
290
- datasetId: run.datasetId,
291
- })
292
- : error
293
- const failure = captureSixbFailure(failureError, {
294
- allowedCodes: SYNC_RUN_FAILURE_CODES,
295
- defaultCode: status === "cancelled" ? "runtime.cancelled" : "internal.unexpected",
296
- details: { syncId: run.syncId, runId: job.id, datasetId: run.datasetId },
297
- })
298
- const failedRun = await syncRunsStorage.finish({
299
- projectId: runtime.id,
300
- id: job.id,
301
- status,
302
- rowsRead,
303
- error: failure,
304
- })
305
- await notifyRunFinished(input.onRunFinished, failedRun)
306
- if (status === "failed" && failedRun.status === "failed") {
307
- input.onRunFailed?.(error, failedRun, failure)
308
- }
309
- } catch {
310
- // The run did not transition to the requested terminal status.
341
+ if (!committedVersion) throw error
342
+
343
+ throw createBookkeepingError({
344
+ syncId: run.syncId,
345
+ runId: run.id,
346
+ version: committedVersion,
347
+ cause: error,
348
+ })
349
+ }
350
+ }
351
+
352
+ /** Records a commit this run made, with what the execution that made it read. */
353
+ function finishLanded(
354
+ execution: SyncExecution,
355
+ version: DatasetVersion,
356
+ read: LandedRead
357
+ ): Promise<SyncRunResult> {
358
+ execution.committedVersion = version
359
+
360
+ return finishSucceeded(execution, { version, versionCreated: true, ...read })
361
+ }
362
+
363
+ /** Only an execution holding the run could prepare a commit, so none has moved these since. */
364
+ function readByEarlierExecution(run: SyncRunRecord): LandedRead {
365
+ return { rowsRead: run.rowsRead ?? 0, checkpoint: run.checkpoint }
366
+ }
367
+
368
+ /**
369
+ * A commit can land and still answer with an error, so only the lake can say what happened. If it
370
+ * cannot say either, this throws, and the run stays open for the next delivery to ask.
371
+ */
372
+ async function settleFailure(execution: SyncExecution, caught: unknown): Promise<SyncRunResult> {
373
+ const { run, signal, committedVersion } = execution
374
+
375
+ const landed = committedVersion ? null : await landedCommitBehind(execution, caught)
376
+ if (landed) return finishLanded(execution, landed, landedRead(execution, caught))
377
+
378
+ const error = translateLostRace(run, caught)
379
+
380
+ if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
381
+ await recordFailed(execution, error)
382
+ }
383
+
384
+ throw error
385
+ }
386
+
387
+ /** What the execution behind a landed commit read: this one, if the commit is its own. */
388
+ function landedRead(execution: SyncExecution, caught: unknown): LandedRead {
389
+ const lostItsRace = caught instanceof LakeConcurrencyError
390
+ const own =
391
+ execution.reachedCommit && !lostItsRace && !(caught instanceof CommitAlreadyLandedError)
392
+
393
+ return own
394
+ ? { rowsRead: execution.rowsRead, checkpoint: execution.nextCheckpoint }
395
+ : readByEarlierExecution(execution.run)
396
+ }
397
+
398
+ /** The shared writer has already cleaned up its session. Finalize the failed sync run. */
399
+ async function recordFailed(execution: SyncExecution, error: unknown): Promise<void> {
400
+ const { run, rowsRead } = execution
401
+ const { runtime, claim, onRunFinished, onRunFailed } = execution.input
402
+
403
+ try {
404
+ const failure = captureSixbFailure(
405
+ translateSyncExecutionError(error, {
406
+ syncId: run.syncId,
407
+ runId: run.id,
408
+ datasetId: run.datasetId,
409
+ }),
410
+ {
411
+ allowedCodes: SYNC_RUN_FAILURE_CODES,
412
+ defaultCode: "internal.unexpected",
413
+ details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId },
311
414
  }
312
- }
415
+ )
313
416
 
314
- throw error
315
- } finally {
316
- await logSession.flush()
417
+ const failedRun = await runtime.syncRunsStorage.finish({
418
+ projectId: runtime.id,
419
+ id: run.id,
420
+ executionToken: claim.execution.token,
421
+ status: "failed",
422
+ rowsRead,
423
+ error: failure,
424
+ })
425
+
426
+ await notifyRunFinished(onRunFinished, failedRun)
427
+ onRunFailed?.(error, failedRun, failure)
428
+ } catch {
429
+ // The run did not transition: a later delivery, or the execution that holds it, ends it.
430
+ }
431
+ }
432
+
433
+ function cloneCheckpoint(checkpoint: JsonValue | undefined): JsonValue | undefined {
434
+ return checkpoint === undefined ? undefined : cloneJsonValue(checkpoint)
435
+ }
436
+
437
+ async function readPreviousCheckpoint(
438
+ runtime: RunSyncJobInput["runtime"],
439
+ sync: SyncDefinition
440
+ ): Promise<JsonValue | undefined> {
441
+ const { runs } = await runtime.syncRunsStorage.list({
442
+ projectId: runtime.id,
443
+ syncId: sync.id,
444
+ statuses: ["succeeded"],
445
+ limit: 1,
446
+ order: "desc",
447
+ })
448
+
449
+ return runs[0]?.checkpoint
450
+ }
451
+
452
+ function requireDataset(
453
+ runtime: RunSyncJobInput["runtime"],
454
+ sync: SyncDefinition,
455
+ run: SyncRunRecord
456
+ ): DatasetDefinition {
457
+ const dataset = runtime.datasets.getById(run.datasetId)
458
+ if (dataset) return dataset
459
+
460
+ throw new Error(`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`)
461
+ }
462
+
463
+ function requireSync(runtime: RunSyncJobInput["runtime"], run: SyncRunRecord): SyncDefinition {
464
+ const sync = runtime.syncs.getById(run.syncId)
465
+ if (!sync) {
466
+ throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`)
467
+ }
468
+ if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
469
+ throw new Error(
470
+ `[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`
471
+ )
317
472
  }
473
+
474
+ return sync
475
+ }
476
+
477
+ /** The commit an error says landed, or that the lake holds in spite of the error. */
478
+ async function landedCommitBehind(
479
+ execution: SyncExecution,
480
+ error: unknown
481
+ ): Promise<DatasetVersion | null> {
482
+ if (error instanceof CommitAlreadyLandedError) return error.version
483
+
484
+ return findLandedCommit(execution)
485
+ }
486
+
487
+ /** A guard this run asked for failed on purpose. Its own guard only lost to another writer. */
488
+ function translateLostRace(run: SyncRunRecord, error: unknown): unknown {
489
+ if (!(error instanceof LakeConcurrencyError) || run.expectedLatestVersionId !== undefined) {
490
+ return error
491
+ }
492
+
493
+ return new CommitLostRaceError(`[SixbSyncWorker] Sync run '${run.id}' lost its commit race.`, {
494
+ cause: error,
495
+ })
496
+ }
497
+
498
+ /**
499
+ * The version this run committed, if one did. A run that never prepared a commit has none: nothing
500
+ * of a partial run exists until then, rows stay in a session no reader sees, and it need not ask.
501
+ */
502
+ async function findLandedCommit(execution: SyncExecution): Promise<DatasetVersion | null> {
503
+ const { run, commitPreparedAt, input } = execution
504
+ if (!commitPreparedAt) return null
505
+
506
+ return findProducedVersion({
507
+ lakeStorage: input.runtime.lakeStorage,
508
+ datasetId: run.datasetId,
509
+ producer: { kind: "sync", runId: run.id },
510
+ since: commitPreparedAt,
511
+ })
318
512
  }
319
513
 
320
514
  async function notifyRunFinished(
@@ -8,7 +8,7 @@ import {
8
8
  type SyncDefinition,
9
9
  } from "@sixb/core"
10
10
  import { isOAuthConnectorDefinition } from "@sixb/core/internal/connector-connections"
11
- import { createSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors"
11
+ import { createSixbError, isSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors"
12
12
  import type { SyncConnectorSource } from "@sixb/core/internal/syncs"
13
13
  import {
14
14
  ConnectionCheckpoints,
@@ -149,7 +149,17 @@ async function* readManagedSource(
149
149
  const { connection } = source
150
150
  try {
151
151
  throwIfAborted(options.signal)
152
- const client = await runAbortable(options.signal, () => source.connect(options.signal))
152
+ let client: unknown
153
+ try {
154
+ client = await runAbortable(options.signal, () => source.connect(options.signal))
155
+ } catch (error) {
156
+ if (!skipsUnauthorizedConnection(options.sync, connection, error)) throw error
157
+ // The connection keeps its checkpoint and its rows; its owner sees it needs reconnecting.
158
+ console.warn(
159
+ `[SixbSyncWorker] Sync '${options.sync.id}' skipped connection '${connection.id}' (${connection.account.label}): it needs reconnecting.`
160
+ )
161
+ return
162
+ }
153
163
  throwIfAborted(options.signal)
154
164
  const previous = checkpoints.get(connection)
155
165
  const readResult = await runAbortable(options.signal, () =>
@@ -196,6 +206,24 @@ async function* readManagedSource(
196
206
  }
197
207
  }
198
208
 
209
+ /**
210
+ * One person's account that needs reconnecting must not fail everyone's run. A snapshot run still
211
+ * fails: leaving the account out would drop its rows from the dataset. A project connection still
212
+ * fails the run, so the project's own expired grant reaches `onError` as it always has.
213
+ */
214
+ function skipsUnauthorizedConnection(
215
+ sync: SyncDefinition,
216
+ connection: SyncConnectorConnection,
217
+ error: unknown
218
+ ): boolean {
219
+ return (
220
+ sync.config.mode !== "snapshot" &&
221
+ connection.owner.type === "user" &&
222
+ isSixbError(error) &&
223
+ error.code === "connector.authorization_required"
224
+ )
225
+ }
226
+
199
227
  function sourceInvariant(syncId: string, message: string): Error {
200
228
  return createSixbError("internal.unexpected", `[SixbSyncWorker] ${message}`, {
201
229
  details: { syncId },
package/src/types.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import type { BlobStorage, LakeStorage, SixbDefinitions, SixbFailure, SyncMode } from "@sixb/core"
2
2
  import type { LoggingService } from "@sixb/core/internal/logging"
3
3
  import type { SyncConnectorSourceResolver } from "@sixb/core/internal/syncs"
4
+ import type { HeldRunClaim } from "@sixb/core/internal/workers"
4
5
  import type { DatasetVersion } from "@sixb/core/lake-storage"
5
6
  import type { SyncRunFailureCode, SyncRunRecord, SyncRunStorage } from "@sixb/core/storage"
6
7
 
@@ -18,8 +19,8 @@ export interface SyncWorkerContext {
18
19
 
19
20
  export interface RunSyncJobInput {
20
21
  readonly runtime: SyncWorkerContext
21
- /** Durable run loaded before the execution scope is restored. */
22
- readonly run: SyncRunRecord
22
+ /** The run this delivery holds, and whether an earlier execution already worked on it. */
23
+ readonly claim: HeldRunClaim<SyncRunRecord>
23
24
  readonly signal?: AbortSignal
24
25
  readonly onRunStarted?: SyncRunStartedHandler
25
26
  readonly onRunFinished?: SyncRunFinishedHandler