@sixb/sync-worker 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,26 +1,28 @@
1
1
  // @bun
2
2
  // src/worker.ts
3
3
  import { reportRunFailure } from "@sixb/core/internal/error-reporting";
4
- import { captureSixbFailure as captureSixbFailure2 } from "@sixb/core/internal/errors";
5
4
  import {
6
5
  bindDurablePrimitiveExecution
7
6
  } from "@sixb/core/internal/primitive-execution";
8
7
  import { resolveSyncConnectorSources } from "@sixb/core/internal/syncs";
9
- import { QueueWorker, retryUnstartedRun } from "@sixb/core/internal/workers";
8
+ import { RunWorker, wakeJobId } from "@sixb/core/internal/workers";
10
9
  import {
11
10
  SYNC_RUN_FAILURE_CODES as SYNC_RUN_FAILURE_CODES2
12
11
  } from "@sixb/core/storage";
13
12
 
14
13
  // src/run-sync-job.ts
15
- import { cloneJsonValue as cloneJsonValue3 } from "@sixb/core";
16
- import { writeDataset } from "@sixb/core/internal/datasets";
14
+ import {
15
+ cloneJsonValue as cloneJsonValue3
16
+ } from "@sixb/core";
17
+ import { findProducedVersion, writeDataset } from "@sixb/core/internal/datasets";
17
18
  import {
18
19
  captureSixbFailure,
19
20
  createSixbError as createSixbError3,
20
- isSixbError,
21
+ isSixbError as isSixbError2,
21
22
  summarizeErrorMessage as summarizeErrorMessage2
22
23
  } from "@sixb/core/internal/errors";
23
24
  import { resolveLoggingService } from "@sixb/core/internal/logging";
25
+ import { LakeConcurrencyError } from "@sixb/core/lake-storage";
24
26
  import { SYNC_RUN_FAILURE_CODES } from "@sixb/core/storage";
25
27
 
26
28
  // src/normalize.ts
@@ -112,7 +114,7 @@ import {
112
114
  cloneJsonValue as cloneJsonValue2
113
115
  } from "@sixb/core";
114
116
  import { isOAuthConnectorDefinition } from "@sixb/core/internal/connector-connections";
115
- import { createSixbError as createSixbError2, summarizeErrorMessage } from "@sixb/core/internal/errors";
117
+ import { createSixbError as createSixbError2, isSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors";
116
118
 
117
119
  // src/checkpoints.ts
118
120
  import {
@@ -320,7 +322,15 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
320
322
  const { connection } = source;
321
323
  try {
322
324
  throwIfAborted(options.signal);
323
- const client = await runAbortable(options.signal, () => source.connect(options.signal));
325
+ let client;
326
+ try {
327
+ client = await runAbortable(options.signal, () => source.connect(options.signal));
328
+ } catch (error) {
329
+ if (!skipsUnauthorizedConnection(options.sync, connection, error))
330
+ throw error;
331
+ console.warn(`[SixbSyncWorker] Sync '${options.sync.id}' skipped connection '${connection.id}' (${connection.account.label}): it needs reconnecting.`);
332
+ return;
333
+ }
324
334
  throwIfAborted(options.signal);
325
335
  const previous = checkpoints.get(connection);
326
336
  const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
@@ -355,6 +365,9 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
355
365
  });
356
366
  }
357
367
  }
368
+ function skipsUnauthorizedConnection(sync, connection, error) {
369
+ return sync.config.mode !== "snapshot" && connection.owner.type === "user" && isSixbError(error) && error.code === "connector.authorization_required";
370
+ }
358
371
  function sourceInvariant(syncId, message) {
359
372
  return createSixbError2("internal.unexpected", `[SixbSyncWorker] ${message}`, {
360
373
  details: { syncId }
@@ -366,16 +379,20 @@ async function* sourceValues(values) {
366
379
  }
367
380
 
368
381
  // src/run-sync-job.ts
369
- class SyncRunAlreadyStartedError extends Error {
370
- run;
371
- name = "SyncRunAlreadyStartedError";
372
- constructor(run) {
373
- super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`);
374
- this.run = run;
382
+ class CommitAlreadyLandedError extends Error {
383
+ version;
384
+ name = "CommitAlreadyLandedError";
385
+ constructor(version) {
386
+ super(`[SixbSyncWorker] Version '${version.versionId}' already holds this run's rows.`);
387
+ this.version = version;
375
388
  }
376
389
  }
390
+
391
+ class CommitLostRaceError extends Error {
392
+ name = "CommitLostRaceError";
393
+ }
377
394
  function createBookkeepingError(options) {
378
- return createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`, {
395
+ return createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'.`, {
379
396
  cause: options.cause,
380
397
  details: {
381
398
  syncId: options.syncId,
@@ -392,7 +409,7 @@ function requireFinishedAt(input) {
392
409
  throw createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync run '${input.runId}' finished without a finishedAt timestamp.`, { details: { syncId: input.syncId, runId: input.runId } });
393
410
  }
394
411
  function translateSyncExecutionError(error, input) {
395
- if (isSixbError(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed")) {
412
+ if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed")) {
396
413
  return error;
397
414
  }
398
415
  return createSixbError3("sync.execution_failed", summarizeErrorMessage2(error, "Sync execution failed."), {
@@ -408,7 +425,7 @@ function sourceValidationError(error, sourceValue, options, itemIndex) {
408
425
  const connection = sourceValue.connection;
409
426
  if (!connection)
410
427
  return error;
411
- const code = isSixbError(error) && error.code === "internal.unexpected" ? "internal.unexpected" : "sync.execution_failed";
428
+ const code = isSixbError2(error) && error.code === "internal.unexpected" ? "internal.unexpected" : "sync.execution_failed";
412
429
  return createSixbError3(code, summarizeErrorMessage2(error, "Sync source validation failed."), {
413
430
  cause: error,
414
431
  details: {
@@ -422,167 +439,254 @@ function sourceValidationError(error, sourceValue, options, itemIndex) {
422
439
  });
423
440
  }
424
441
  async function runSyncJob(input) {
425
- const { runtime, run } = input;
426
- const job = {
427
- id: run.id,
428
- syncId: run.syncId,
429
- expectedLatestVersionId: run.expectedLatestVersionId,
430
- commitMessage: run.commitMessage
431
- };
442
+ const { runtime, claim } = input;
443
+ const { run } = claim;
432
444
  const signal = input.signal ?? new AbortController().signal;
433
- const { syncRunsStorage, lakeStorage, blobs: blobStorage } = runtime;
434
445
  if (run.projectId !== runtime.id) {
435
446
  throw new Error(`[SixbSyncWorker] Sync run '${run.id}' belongs to project '${run.projectId}', not '${runtime.id}'.`);
436
447
  }
437
448
  throwIfAborted(signal);
438
- let startedRun;
439
- try {
440
- startedRun = await syncRunsStorage.start({
441
- projectId: runtime.id,
442
- id: job.id
443
- });
444
- } catch (error) {
445
- const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id });
446
- if (existing?.syncId === run.syncId && existing.executionId === run.executionId && existing.status !== "queued") {
447
- throw new SyncRunAlreadyStartedError(existing);
448
- }
449
- throw error;
450
- }
451
- await input.onRunStarted?.(startedRun);
449
+ if (claim.outcome === "started")
450
+ await input.onRunStarted?.(run);
452
451
  const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({
453
452
  kind: "sync",
454
- id: job.id
453
+ id: run.id
455
454
  });
456
- const logger = logSession.logger;
457
- let rowsRead = 0;
458
- let committedVersion;
455
+ const execution = {
456
+ input,
457
+ run,
458
+ signal,
459
+ rowsRead: 0,
460
+ nextCheckpoint: undefined,
461
+ committedVersion: undefined,
462
+ commitPreparedAt: run.commitPreparedAt,
463
+ reachedCommit: false
464
+ };
459
465
  try {
460
- const sync = runtime.syncs.getById(run.syncId);
461
- if (!sync) {
462
- throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`);
463
- }
464
- if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
465
- throw new Error(`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`);
466
+ if (claim.outcome === "resumed") {
467
+ const landed = await findLandedCommit(execution);
468
+ if (landed)
469
+ return await finishLanded(execution, landed, readByEarlierExecution(run));
466
470
  }
467
- const dataset = runtime.datasets.getById(run.datasetId);
468
- if (!dataset) {
469
- throw new Error(`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`);
470
- }
471
- const latestSuccessfulRuns = await syncRunsStorage.list({
472
- projectId: runtime.id,
473
- syncId: sync.id,
474
- statuses: ["succeeded"],
475
- limit: 1,
476
- order: "desc"
477
- });
478
- const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint;
479
- let nextCheckpoint = previousCheckpoint !== undefined ? cloneJsonValue3(previousCheckpoint) : undefined;
480
- const producer = {
481
- kind: "sync",
482
- id: sync.id,
483
- runId: job.id
484
- };
485
- const readValues = () => readSyncValues({
471
+ if (claim.givenUp)
472
+ throw claim.givenUp;
473
+ return await commitRun(execution, logSession.logger);
474
+ } catch (caught) {
475
+ return await settleFailure(execution, caught);
476
+ } finally {
477
+ await logSession.flush();
478
+ }
479
+ }
480
+ async function commitRun(execution, logger) {
481
+ const { run, signal } = execution;
482
+ const { runtime } = execution.input;
483
+ const { lakeStorage, blobs: blobStorage } = runtime;
484
+ const sync = requireSync(runtime, run);
485
+ const dataset = requireDataset(runtime, sync, run);
486
+ const previousCheckpoint = await readPreviousCheckpoint(runtime, sync);
487
+ execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint);
488
+ const { outcome, version } = await writeDataset({
489
+ lakeStorage,
490
+ blobStorage,
491
+ dataset,
492
+ mode: sync.config.mode,
493
+ signal,
494
+ producer: { kind: "sync", id: sync.id, runId: run.id },
495
+ readValues: () => readSyncValues({
486
496
  runtime,
487
497
  sync,
488
- runId: job.id,
498
+ runId: run.id,
489
499
  datasetId: dataset.id,
490
500
  signal,
491
501
  blobStorage,
492
502
  logger,
493
503
  previousCheckpoint,
494
504
  setCheckpoint(next) {
495
- nextCheckpoint = next === undefined ? undefined : cloneJsonValue3(next);
505
+ execution.nextCheckpoint = cloneCheckpoint(next);
496
506
  }
507
+ }),
508
+ expectedLatestVersionId: run.expectedLatestVersionId,
509
+ commitMessage: run.commitMessage ?? `sync ${sync.id} run ${run.id}`,
510
+ sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
511
+ onRead(count) {
512
+ execution.rowsRead = count;
513
+ },
514
+ mapValidationError: (error, value, itemIndex) => sourceValidationError(error, value, { syncId: sync.id, runId: run.id, dataset }, itemIndex),
515
+ guardLatestVersion: true,
516
+ beforeCommit: () => prepareCommit(execution)
517
+ });
518
+ if (outcome === "created")
519
+ execution.committedVersion = version;
520
+ return finishSucceeded(execution, {
521
+ version,
522
+ versionCreated: outcome === "created",
523
+ rowsRead: execution.rowsRead,
524
+ checkpoint: execution.nextCheckpoint
525
+ });
526
+ }
527
+ async function prepareCommit(execution) {
528
+ const { run } = execution;
529
+ const { runtime, claim } = execution.input;
530
+ const landed = await findLandedCommit(execution);
531
+ if (landed)
532
+ throw new CommitAlreadyLandedError(landed);
533
+ const prepared = await runtime.syncRunsStorage.prepareCommit({
534
+ projectId: runtime.id,
535
+ id: run.id,
536
+ executionToken: claim.execution.token,
537
+ rowsRead: execution.rowsRead,
538
+ checkpoint: execution.nextCheckpoint
539
+ });
540
+ execution.commitPreparedAt ??= prepared.commitPreparedAt;
541
+ execution.reachedCommit = true;
542
+ }
543
+ async function finishSucceeded(execution, result) {
544
+ const { run, committedVersion } = execution;
545
+ const { version, versionCreated, rowsRead } = result;
546
+ const finished = await recordSucceeded(execution, result);
547
+ const finishedAt = requireFinishedAt({
548
+ syncId: run.syncId,
549
+ runId: run.id,
550
+ finishedAt: finished.finishedAt
551
+ });
552
+ await notifyRunFinished(execution.input.onRunFinished, finished, committedVersion);
553
+ const summary = {
554
+ id: run.id,
555
+ syncId: run.syncId,
556
+ datasetId: run.datasetId,
557
+ mode: run.mode,
558
+ startedAt: requireStartedAt(run.id, run.startedAt),
559
+ finishedAt,
560
+ rowsRead
561
+ };
562
+ return version ? { ...summary, version, versionCreated } : { ...summary, versionCreated: false };
563
+ }
564
+ async function recordSucceeded(execution, result) {
565
+ const { run, committedVersion } = execution;
566
+ const { runtime, claim } = execution.input;
567
+ const { version, rowsRead, checkpoint } = result;
568
+ try {
569
+ return await runtime.syncRunsStorage.finish({
570
+ projectId: runtime.id,
571
+ id: run.id,
572
+ executionToken: claim.execution.token,
573
+ status: "succeeded",
574
+ rowsRead,
575
+ ...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
576
+ checkpoint
497
577
  });
498
- const { outcome, version } = await writeDataset({
499
- lakeStorage,
500
- blobStorage,
501
- dataset,
502
- mode: sync.config.mode,
503
- signal,
504
- producer,
505
- readValues,
506
- expectedLatestVersionId: job.expectedLatestVersionId,
507
- commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,
508
- sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
509
- onRead(count) {
510
- rowsRead = count;
511
- },
512
- mapValidationError: (error, value, itemIndex) => sourceValidationError(error, value, { syncId: sync.id, runId: job.id, dataset }, itemIndex)
513
- });
514
- if (outcome === "created") {
515
- committedVersion = version;
516
- }
517
- let finishedRun;
518
- try {
519
- finishedRun = await syncRunsStorage.finish({
520
- projectId: runtime.id,
521
- id: job.id,
522
- status: "succeeded",
523
- rowsRead,
524
- ...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
525
- checkpoint: nextCheckpoint
526
- });
527
- } catch (error) {
528
- if (committedVersion) {
529
- throw createBookkeepingError({
530
- syncId: sync.id,
531
- runId: job.id,
532
- version: committedVersion,
533
- cause: error
534
- });
535
- }
578
+ } catch (error) {
579
+ if (!committedVersion)
536
580
  throw error;
537
- }
538
- const finishedAt = requireFinishedAt({
539
- syncId: sync.id,
540
- runId: job.id,
541
- finishedAt: finishedRun.finishedAt
581
+ throw createBookkeepingError({
582
+ syncId: run.syncId,
583
+ runId: run.id,
584
+ version: committedVersion,
585
+ cause: error
542
586
  });
543
- await notifyRunFinished(input.onRunFinished, finishedRun, committedVersion);
544
- const result = {
545
- id: job.id,
546
- syncId: sync.id,
547
- datasetId: dataset.id,
548
- mode: sync.config.mode,
549
- startedAt: requireStartedAt(job.id, startedRun.startedAt),
550
- finishedAt,
551
- rowsRead
552
- };
553
- return version ? { ...result, version, versionCreated: outcome === "created" } : { ...result, versionCreated: false };
554
- } catch (error) {
555
- if (!committedVersion) {
556
- const status = signal.aborted ? "cancelled" : "failed";
557
- try {
558
- const failureError = status === "failed" ? translateSyncExecutionError(error, {
559
- syncId: run.syncId,
560
- runId: job.id,
561
- datasetId: run.datasetId
562
- }) : error;
563
- const failure = captureSixbFailure(failureError, {
564
- allowedCodes: SYNC_RUN_FAILURE_CODES,
565
- defaultCode: status === "cancelled" ? "runtime.cancelled" : "internal.unexpected",
566
- details: { syncId: run.syncId, runId: job.id, datasetId: run.datasetId }
567
- });
568
- const failedRun = await syncRunsStorage.finish({
569
- projectId: runtime.id,
570
- id: job.id,
571
- status,
572
- rowsRead,
573
- error: failure
574
- });
575
- await notifyRunFinished(input.onRunFinished, failedRun);
576
- if (status === "failed" && failedRun.status === "failed") {
577
- input.onRunFailed?.(error, failedRun, failure);
578
- }
579
- } catch {}
580
- }
581
- throw error;
582
- } finally {
583
- await logSession.flush();
584
587
  }
585
588
  }
589
+ function finishLanded(execution, version, read) {
590
+ execution.committedVersion = version;
591
+ return finishSucceeded(execution, { version, versionCreated: true, ...read });
592
+ }
593
+ function readByEarlierExecution(run) {
594
+ return { rowsRead: run.rowsRead ?? 0, checkpoint: run.checkpoint };
595
+ }
596
+ async function settleFailure(execution, caught) {
597
+ const { run, signal, committedVersion } = execution;
598
+ const landed = committedVersion ? null : await landedCommitBehind(execution, caught);
599
+ if (landed)
600
+ return finishLanded(execution, landed, landedRead(execution, caught));
601
+ const error = translateLostRace(run, caught);
602
+ if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
603
+ await recordFailed(execution, error);
604
+ }
605
+ throw error;
606
+ }
607
+ function landedRead(execution, caught) {
608
+ const lostItsRace = caught instanceof LakeConcurrencyError;
609
+ const own = execution.reachedCommit && !lostItsRace && !(caught instanceof CommitAlreadyLandedError);
610
+ return own ? { rowsRead: execution.rowsRead, checkpoint: execution.nextCheckpoint } : readByEarlierExecution(execution.run);
611
+ }
612
+ async function recordFailed(execution, error) {
613
+ const { run, rowsRead } = execution;
614
+ const { runtime, claim, onRunFinished, onRunFailed } = execution.input;
615
+ try {
616
+ const failure = captureSixbFailure(translateSyncExecutionError(error, {
617
+ syncId: run.syncId,
618
+ runId: run.id,
619
+ datasetId: run.datasetId
620
+ }), {
621
+ allowedCodes: SYNC_RUN_FAILURE_CODES,
622
+ defaultCode: "internal.unexpected",
623
+ details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId }
624
+ });
625
+ const failedRun = await runtime.syncRunsStorage.finish({
626
+ projectId: runtime.id,
627
+ id: run.id,
628
+ executionToken: claim.execution.token,
629
+ status: "failed",
630
+ rowsRead,
631
+ error: failure
632
+ });
633
+ await notifyRunFinished(onRunFinished, failedRun);
634
+ onRunFailed?.(error, failedRun, failure);
635
+ } catch {}
636
+ }
637
+ function cloneCheckpoint(checkpoint) {
638
+ return checkpoint === undefined ? undefined : cloneJsonValue3(checkpoint);
639
+ }
640
+ async function readPreviousCheckpoint(runtime, sync) {
641
+ const { runs } = await runtime.syncRunsStorage.list({
642
+ projectId: runtime.id,
643
+ syncId: sync.id,
644
+ statuses: ["succeeded"],
645
+ limit: 1,
646
+ order: "desc"
647
+ });
648
+ return runs[0]?.checkpoint;
649
+ }
650
+ function requireDataset(runtime, sync, run) {
651
+ const dataset = runtime.datasets.getById(run.datasetId);
652
+ if (dataset)
653
+ return dataset;
654
+ throw new Error(`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`);
655
+ }
656
+ function requireSync(runtime, run) {
657
+ const sync = runtime.syncs.getById(run.syncId);
658
+ if (!sync) {
659
+ throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`);
660
+ }
661
+ if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
662
+ throw new Error(`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`);
663
+ }
664
+ return sync;
665
+ }
666
+ async function landedCommitBehind(execution, error) {
667
+ if (error instanceof CommitAlreadyLandedError)
668
+ return error.version;
669
+ return findLandedCommit(execution);
670
+ }
671
+ function translateLostRace(run, error) {
672
+ if (!(error instanceof LakeConcurrencyError) || run.expectedLatestVersionId !== undefined) {
673
+ return error;
674
+ }
675
+ return new CommitLostRaceError(`[SixbSyncWorker] Sync run '${run.id}' lost its commit race.`, {
676
+ cause: error
677
+ });
678
+ }
679
+ async function findLandedCommit(execution) {
680
+ const { run, commitPreparedAt, input } = execution;
681
+ if (!commitPreparedAt)
682
+ return null;
683
+ return findProducedVersion({
684
+ lakeStorage: input.runtime.lakeStorage,
685
+ datasetId: run.datasetId,
686
+ producer: { kind: "sync", runId: run.id },
687
+ since: commitPreparedAt
688
+ });
689
+ }
586
690
  async function notifyRunFinished(handler, run, createdVersion) {
587
691
  try {
588
692
  await handler?.(run, createdVersion);
@@ -599,7 +703,7 @@ function requireStartedAt(runId, startedAt) {
599
703
  // src/worker.ts
600
704
  var SOURCE = "SixbSyncWorker";
601
705
 
602
- class SyncWorker extends QueueWorker {
706
+ class SyncWorker extends RunWorker {
603
707
  host;
604
708
  constructor(host, options = {}) {
605
709
  if (host.definitions.syncs.list().length === 0) {
@@ -610,98 +714,74 @@ class SyncWorker extends QueueWorker {
610
714
  queue: host.queues.syncRuns,
611
715
  failureCodes: SYNC_RUN_FAILURE_CODES2,
612
716
  workerId: `sync-worker-${host.id}`,
613
- claimLimit: options.concurrency
717
+ claimLimit: options.concurrency,
718
+ runs: requireSyncRuns(host),
719
+ executions: host.storage.executions
614
720
  });
615
- assertSyncStorage(host);
616
721
  this.host = host;
617
722
  }
618
- async execute(claimed, signal) {
619
- const { job } = claimed;
620
- const syncRuns = this.host.storage.syncRuns;
621
- if (!syncRuns) {
622
- throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
623
- }
624
- const run = await syncRuns.getById({ projectId: this.host.id, id: job.payload.runId });
625
- if (!run) {
626
- throw new Error(`[SixbSyncWorker] Sync run '${job.payload.runId}' was not found.`);
627
- }
628
- if (run.status !== "queued")
629
- return;
630
- const durableExecution = await this.host.storage.executions.getById({
723
+ claimRun(claimed, execution) {
724
+ return requireSyncRuns(this.host).startOrReclaim({
631
725
  projectId: this.host.id,
632
- id: run.executionId
726
+ id: claimed.job.payload.runId,
727
+ execution
633
728
  });
634
- if (!durableExecution) {
635
- throw new Error(`[SixbSyncWorker] Sync run '${run.id}' references missing execution '${run.executionId}'.`);
636
- }
729
+ }
730
+ async executeRun(claim, _claimed, signal) {
731
+ const { run } = claim;
732
+ const durableExecution = await this.requireExecution(run);
637
733
  const execution = bindDurablePrimitiveExecution(this.host, {
638
734
  execution: durableExecution,
639
735
  primitive: { kind: "sync", id: run.syncId, runId: run.id }
640
736
  });
641
- const context = buildSyncContext(this.host, execution.sixb);
737
+ await runSyncJob({
738
+ runtime: buildSyncContext(this.host, execution.sixb),
739
+ claim,
740
+ signal,
741
+ onRunStarted: (started) => emitSyncRunStarted(this.host, started, durableExecution.correlationId),
742
+ onRunFinished: async (finished, createdVersion) => {
743
+ await emitSyncRunFinishedEvents(this.host, finished, durableExecution.correlationId, createdVersion);
744
+ await this.wakeNextRun(finished);
745
+ },
746
+ onRunFailed: (error, run, failure) => {
747
+ reportRunFailure(this.host, error, {
748
+ projectId: this.host.id,
749
+ attempt: run.attempt,
750
+ runKind: "sync",
751
+ run: {
752
+ runId: run.id,
753
+ syncId: run.syncId
754
+ },
755
+ failure
756
+ });
757
+ }
758
+ });
759
+ }
760
+ async wakeNextRun(finished) {
642
761
  try {
643
- await runSyncJob({
644
- runtime: context,
645
- run,
646
- signal,
647
- onRunStarted: (started) => emitSyncRunStarted(this.host, started, durableExecution.correlationId),
648
- onRunFinished: (finished, createdVersion) => emitSyncRunFinishedEvents(this.host, finished, durableExecution.correlationId, createdVersion),
649
- onRunFailed: (error, run, failure) => {
650
- reportRunFailure(this.host, error, {
651
- projectId: this.host.id,
652
- attempt: job.attempt,
653
- runKind: "sync",
654
- run: {
655
- runId: run.id,
656
- syncId: run.syncId
657
- },
658
- failure
659
- });
660
- }
762
+ const { runs } = await requireSyncRuns(this.host).list({
763
+ projectId: this.host.id,
764
+ syncId: finished.syncId,
765
+ statuses: ["queued"],
766
+ order: "asc",
767
+ limit: 1
661
768
  });
662
- } catch (error) {
663
- if (error instanceof SyncRunAlreadyStartedError)
769
+ const [next] = runs;
770
+ if (!next)
664
771
  return;
665
- throw error;
666
- }
667
- }
668
- async onExecutionError(claimed, error) {
669
- const run = await this.host.storage.syncRuns?.getById({
670
- projectId: this.host.id,
671
- id: claimed.job.payload.runId
672
- });
673
- if (run?.status === "queued") {
674
- const retry = retryUnstartedRun(claimed.job.attempt);
675
- if (retry)
676
- return retry;
677
- return { kind: "fail", failure: await this.failUnstartedRun(claimed, run, error) };
678
- }
679
- return run?.status === "failed" && run.error ? { kind: "fail", failure: run.error } : { kind: "fail" };
680
- }
681
- async failUnstartedRun(claimed, run, error) {
682
- const syncRuns = this.host.storage.syncRuns;
683
- if (!syncRuns) {
684
- throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
772
+ await this.host.queues.syncRuns.enqueue({
773
+ projectId: this.host.id,
774
+ jobs: [
775
+ {
776
+ id: wakeJobId(next.id, finished.id),
777
+ type: "sync.run.requested",
778
+ payload: { runId: next.id }
779
+ }
780
+ ]
781
+ });
782
+ } catch (error) {
783
+ console.error(`[SixbSyncWorker] Could not wake the run after '${finished.id}'.`, error);
685
784
  }
686
- const failure = captureSixbFailure2(error, {
687
- allowedCodes: SYNC_RUN_FAILURE_CODES2,
688
- defaultCode: "internal.unexpected",
689
- details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId }
690
- });
691
- await syncRuns.finish({
692
- projectId: this.host.id,
693
- id: run.id,
694
- status: "failed",
695
- error: failure
696
- });
697
- reportRunFailure(this.host, error, {
698
- projectId: this.host.id,
699
- attempt: claimed.job.attempt,
700
- runKind: "sync",
701
- run: { runId: run.id, syncId: run.syncId },
702
- failure
703
- });
704
- return failure;
705
785
  }
706
786
  }
707
787
  async function emitSyncRunStarted(host, run, correlationId) {
@@ -763,19 +843,16 @@ function requireTerminalStatus(status, context) {
763
843
  }
764
844
  return status;
765
845
  }
766
- function assertSyncStorage(host) {
846
+ function requireSyncRuns(host) {
767
847
  if (!host.storage.syncRuns) {
768
848
  throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
769
849
  }
850
+ return host.storage.syncRuns;
770
851
  }
771
852
  function buildSyncContext(host, sixb) {
772
- const syncRunsStorage = host.storage.syncRuns;
773
- if (!syncRunsStorage) {
774
- throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
775
- }
776
853
  return {
777
854
  id: host.id,
778
- syncRunsStorage,
855
+ syncRunsStorage: requireSyncRuns(host),
779
856
  lakeStorage: host.lakeStorage,
780
857
  blobs: sixb.blobs,
781
858
  logging: host.logging,
@@ -790,4 +867,4 @@ export {
790
867
  SyncWorker
791
868
  };
792
869
 
793
- //# debugId=2134D7A3DD1DBC1164756E2164756E21
870
+ //# debugId=734C88A91BA0AEF864756E2164756E21