@sixb/sync-worker 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -46
- package/dist/index.js +317 -240
- package/dist/index.js.map +5 -5
- package/dist/run-sync-job.d.ts +6 -9
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/types.d.ts +3 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +7 -11
- package/dist/worker.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/run-sync-job.ts +366 -172
- package/src/source-read.ts +30 -2
- package/src/types.ts +3 -2
- package/src/worker.ts +84 -131
package/dist/index.js
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/worker.ts
|
|
3
3
|
import { reportRunFailure } from "@sixb/core/internal/error-reporting";
|
|
4
|
-
import { captureSixbFailure as captureSixbFailure2 } from "@sixb/core/internal/errors";
|
|
5
4
|
import {
|
|
6
5
|
bindDurablePrimitiveExecution
|
|
7
6
|
} from "@sixb/core/internal/primitive-execution";
|
|
8
7
|
import { resolveSyncConnectorSources } from "@sixb/core/internal/syncs";
|
|
9
|
-
import {
|
|
8
|
+
import { RunWorker, wakeJobId } from "@sixb/core/internal/workers";
|
|
10
9
|
import {
|
|
11
10
|
SYNC_RUN_FAILURE_CODES as SYNC_RUN_FAILURE_CODES2
|
|
12
11
|
} from "@sixb/core/storage";
|
|
13
12
|
|
|
14
13
|
// src/run-sync-job.ts
|
|
15
|
-
import {
|
|
16
|
-
|
|
14
|
+
import {
|
|
15
|
+
cloneJsonValue as cloneJsonValue3
|
|
16
|
+
} from "@sixb/core";
|
|
17
|
+
import { findProducedVersion, writeDataset } from "@sixb/core/internal/datasets";
|
|
17
18
|
import {
|
|
18
19
|
captureSixbFailure,
|
|
19
20
|
createSixbError as createSixbError3,
|
|
20
|
-
isSixbError,
|
|
21
|
+
isSixbError as isSixbError2,
|
|
21
22
|
summarizeErrorMessage as summarizeErrorMessage2
|
|
22
23
|
} from "@sixb/core/internal/errors";
|
|
23
24
|
import { resolveLoggingService } from "@sixb/core/internal/logging";
|
|
25
|
+
import { LakeConcurrencyError } from "@sixb/core/lake-storage";
|
|
24
26
|
import { SYNC_RUN_FAILURE_CODES } from "@sixb/core/storage";
|
|
25
27
|
|
|
26
28
|
// src/normalize.ts
|
|
@@ -112,7 +114,7 @@ import {
|
|
|
112
114
|
cloneJsonValue as cloneJsonValue2
|
|
113
115
|
} from "@sixb/core";
|
|
114
116
|
import { isOAuthConnectorDefinition } from "@sixb/core/internal/connector-connections";
|
|
115
|
-
import { createSixbError as createSixbError2, summarizeErrorMessage } from "@sixb/core/internal/errors";
|
|
117
|
+
import { createSixbError as createSixbError2, isSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors";
|
|
116
118
|
|
|
117
119
|
// src/checkpoints.ts
|
|
118
120
|
import {
|
|
@@ -320,7 +322,15 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
|
|
|
320
322
|
const { connection } = source;
|
|
321
323
|
try {
|
|
322
324
|
throwIfAborted(options.signal);
|
|
323
|
-
|
|
325
|
+
let client;
|
|
326
|
+
try {
|
|
327
|
+
client = await runAbortable(options.signal, () => source.connect(options.signal));
|
|
328
|
+
} catch (error) {
|
|
329
|
+
if (!skipsUnauthorizedConnection(options.sync, connection, error))
|
|
330
|
+
throw error;
|
|
331
|
+
console.warn(`[SixbSyncWorker] Sync '${options.sync.id}' skipped connection '${connection.id}' (${connection.account.label}): it needs reconnecting.`);
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
324
334
|
throwIfAborted(options.signal);
|
|
325
335
|
const previous = checkpoints.get(connection);
|
|
326
336
|
const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
|
|
@@ -355,6 +365,9 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
|
|
|
355
365
|
});
|
|
356
366
|
}
|
|
357
367
|
}
|
|
368
|
+
function skipsUnauthorizedConnection(sync, connection, error) {
|
|
369
|
+
return sync.config.mode !== "snapshot" && connection.owner.type === "user" && isSixbError(error) && error.code === "connector.authorization_required";
|
|
370
|
+
}
|
|
358
371
|
function sourceInvariant(syncId, message) {
|
|
359
372
|
return createSixbError2("internal.unexpected", `[SixbSyncWorker] ${message}`, {
|
|
360
373
|
details: { syncId }
|
|
@@ -366,16 +379,20 @@ async function* sourceValues(values) {
|
|
|
366
379
|
}
|
|
367
380
|
|
|
368
381
|
// src/run-sync-job.ts
|
|
369
|
-
class
|
|
370
|
-
|
|
371
|
-
name = "
|
|
372
|
-
constructor(
|
|
373
|
-
super(`[SixbSyncWorker]
|
|
374
|
-
this.
|
|
382
|
+
class CommitAlreadyLandedError extends Error {
|
|
383
|
+
version;
|
|
384
|
+
name = "CommitAlreadyLandedError";
|
|
385
|
+
constructor(version) {
|
|
386
|
+
super(`[SixbSyncWorker] Version '${version.versionId}' already holds this run's rows.`);
|
|
387
|
+
this.version = version;
|
|
375
388
|
}
|
|
376
389
|
}
|
|
390
|
+
|
|
391
|
+
class CommitLostRaceError extends Error {
|
|
392
|
+
name = "CommitLostRaceError";
|
|
393
|
+
}
|
|
377
394
|
function createBookkeepingError(options) {
|
|
378
|
-
return createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'
|
|
395
|
+
return createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'.`, {
|
|
379
396
|
cause: options.cause,
|
|
380
397
|
details: {
|
|
381
398
|
syncId: options.syncId,
|
|
@@ -392,7 +409,7 @@ function requireFinishedAt(input) {
|
|
|
392
409
|
throw createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync run '${input.runId}' finished without a finishedAt timestamp.`, { details: { syncId: input.syncId, runId: input.runId } });
|
|
393
410
|
}
|
|
394
411
|
function translateSyncExecutionError(error, input) {
|
|
395
|
-
if (
|
|
412
|
+
if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed")) {
|
|
396
413
|
return error;
|
|
397
414
|
}
|
|
398
415
|
return createSixbError3("sync.execution_failed", summarizeErrorMessage2(error, "Sync execution failed."), {
|
|
@@ -408,7 +425,7 @@ function sourceValidationError(error, sourceValue, options, itemIndex) {
|
|
|
408
425
|
const connection = sourceValue.connection;
|
|
409
426
|
if (!connection)
|
|
410
427
|
return error;
|
|
411
|
-
const code =
|
|
428
|
+
const code = isSixbError2(error) && error.code === "internal.unexpected" ? "internal.unexpected" : "sync.execution_failed";
|
|
412
429
|
return createSixbError3(code, summarizeErrorMessage2(error, "Sync source validation failed."), {
|
|
413
430
|
cause: error,
|
|
414
431
|
details: {
|
|
@@ -422,167 +439,254 @@ function sourceValidationError(error, sourceValue, options, itemIndex) {
|
|
|
422
439
|
});
|
|
423
440
|
}
|
|
424
441
|
async function runSyncJob(input) {
|
|
425
|
-
const { runtime,
|
|
426
|
-
const
|
|
427
|
-
id: run.id,
|
|
428
|
-
syncId: run.syncId,
|
|
429
|
-
expectedLatestVersionId: run.expectedLatestVersionId,
|
|
430
|
-
commitMessage: run.commitMessage
|
|
431
|
-
};
|
|
442
|
+
const { runtime, claim } = input;
|
|
443
|
+
const { run } = claim;
|
|
432
444
|
const signal = input.signal ?? new AbortController().signal;
|
|
433
|
-
const { syncRunsStorage, lakeStorage, blobs: blobStorage } = runtime;
|
|
434
445
|
if (run.projectId !== runtime.id) {
|
|
435
446
|
throw new Error(`[SixbSyncWorker] Sync run '${run.id}' belongs to project '${run.projectId}', not '${runtime.id}'.`);
|
|
436
447
|
}
|
|
437
448
|
throwIfAborted(signal);
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
startedRun = await syncRunsStorage.start({
|
|
441
|
-
projectId: runtime.id,
|
|
442
|
-
id: job.id
|
|
443
|
-
});
|
|
444
|
-
} catch (error) {
|
|
445
|
-
const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id });
|
|
446
|
-
if (existing?.syncId === run.syncId && existing.executionId === run.executionId && existing.status !== "queued") {
|
|
447
|
-
throw new SyncRunAlreadyStartedError(existing);
|
|
448
|
-
}
|
|
449
|
-
throw error;
|
|
450
|
-
}
|
|
451
|
-
await input.onRunStarted?.(startedRun);
|
|
449
|
+
if (claim.outcome === "started")
|
|
450
|
+
await input.onRunStarted?.(run);
|
|
452
451
|
const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({
|
|
453
452
|
kind: "sync",
|
|
454
|
-
id:
|
|
453
|
+
id: run.id
|
|
455
454
|
});
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
|
|
455
|
+
const execution = {
|
|
456
|
+
input,
|
|
457
|
+
run,
|
|
458
|
+
signal,
|
|
459
|
+
rowsRead: 0,
|
|
460
|
+
nextCheckpoint: undefined,
|
|
461
|
+
committedVersion: undefined,
|
|
462
|
+
commitPreparedAt: run.commitPreparedAt,
|
|
463
|
+
reachedCommit: false
|
|
464
|
+
};
|
|
459
465
|
try {
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
|
|
465
|
-
throw new Error(`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`);
|
|
466
|
+
if (claim.outcome === "resumed") {
|
|
467
|
+
const landed = await findLandedCommit(execution);
|
|
468
|
+
if (landed)
|
|
469
|
+
return await finishLanded(execution, landed, readByEarlierExecution(run));
|
|
466
470
|
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
471
|
+
if (claim.givenUp)
|
|
472
|
+
throw claim.givenUp;
|
|
473
|
+
return await commitRun(execution, logSession.logger);
|
|
474
|
+
} catch (caught) {
|
|
475
|
+
return await settleFailure(execution, caught);
|
|
476
|
+
} finally {
|
|
477
|
+
await logSession.flush();
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
async function commitRun(execution, logger) {
|
|
481
|
+
const { run, signal } = execution;
|
|
482
|
+
const { runtime } = execution.input;
|
|
483
|
+
const { lakeStorage, blobs: blobStorage } = runtime;
|
|
484
|
+
const sync = requireSync(runtime, run);
|
|
485
|
+
const dataset = requireDataset(runtime, sync, run);
|
|
486
|
+
const previousCheckpoint = await readPreviousCheckpoint(runtime, sync);
|
|
487
|
+
execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint);
|
|
488
|
+
const { outcome, version } = await writeDataset({
|
|
489
|
+
lakeStorage,
|
|
490
|
+
blobStorage,
|
|
491
|
+
dataset,
|
|
492
|
+
mode: sync.config.mode,
|
|
493
|
+
signal,
|
|
494
|
+
producer: { kind: "sync", id: sync.id, runId: run.id },
|
|
495
|
+
readValues: () => readSyncValues({
|
|
486
496
|
runtime,
|
|
487
497
|
sync,
|
|
488
|
-
runId:
|
|
498
|
+
runId: run.id,
|
|
489
499
|
datasetId: dataset.id,
|
|
490
500
|
signal,
|
|
491
501
|
blobStorage,
|
|
492
502
|
logger,
|
|
493
503
|
previousCheckpoint,
|
|
494
504
|
setCheckpoint(next) {
|
|
495
|
-
nextCheckpoint =
|
|
505
|
+
execution.nextCheckpoint = cloneCheckpoint(next);
|
|
496
506
|
}
|
|
507
|
+
}),
|
|
508
|
+
expectedLatestVersionId: run.expectedLatestVersionId,
|
|
509
|
+
commitMessage: run.commitMessage ?? `sync ${sync.id} run ${run.id}`,
|
|
510
|
+
sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
|
|
511
|
+
onRead(count) {
|
|
512
|
+
execution.rowsRead = count;
|
|
513
|
+
},
|
|
514
|
+
mapValidationError: (error, value, itemIndex) => sourceValidationError(error, value, { syncId: sync.id, runId: run.id, dataset }, itemIndex),
|
|
515
|
+
guardLatestVersion: true,
|
|
516
|
+
beforeCommit: () => prepareCommit(execution)
|
|
517
|
+
});
|
|
518
|
+
if (outcome === "created")
|
|
519
|
+
execution.committedVersion = version;
|
|
520
|
+
return finishSucceeded(execution, {
|
|
521
|
+
version,
|
|
522
|
+
versionCreated: outcome === "created",
|
|
523
|
+
rowsRead: execution.rowsRead,
|
|
524
|
+
checkpoint: execution.nextCheckpoint
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
async function prepareCommit(execution) {
|
|
528
|
+
const { run } = execution;
|
|
529
|
+
const { runtime, claim } = execution.input;
|
|
530
|
+
const landed = await findLandedCommit(execution);
|
|
531
|
+
if (landed)
|
|
532
|
+
throw new CommitAlreadyLandedError(landed);
|
|
533
|
+
const prepared = await runtime.syncRunsStorage.prepareCommit({
|
|
534
|
+
projectId: runtime.id,
|
|
535
|
+
id: run.id,
|
|
536
|
+
executionToken: claim.execution.token,
|
|
537
|
+
rowsRead: execution.rowsRead,
|
|
538
|
+
checkpoint: execution.nextCheckpoint
|
|
539
|
+
});
|
|
540
|
+
execution.commitPreparedAt ??= prepared.commitPreparedAt;
|
|
541
|
+
execution.reachedCommit = true;
|
|
542
|
+
}
|
|
543
|
+
async function finishSucceeded(execution, result) {
|
|
544
|
+
const { run, committedVersion } = execution;
|
|
545
|
+
const { version, versionCreated, rowsRead } = result;
|
|
546
|
+
const finished = await recordSucceeded(execution, result);
|
|
547
|
+
const finishedAt = requireFinishedAt({
|
|
548
|
+
syncId: run.syncId,
|
|
549
|
+
runId: run.id,
|
|
550
|
+
finishedAt: finished.finishedAt
|
|
551
|
+
});
|
|
552
|
+
await notifyRunFinished(execution.input.onRunFinished, finished, committedVersion);
|
|
553
|
+
const summary = {
|
|
554
|
+
id: run.id,
|
|
555
|
+
syncId: run.syncId,
|
|
556
|
+
datasetId: run.datasetId,
|
|
557
|
+
mode: run.mode,
|
|
558
|
+
startedAt: requireStartedAt(run.id, run.startedAt),
|
|
559
|
+
finishedAt,
|
|
560
|
+
rowsRead
|
|
561
|
+
};
|
|
562
|
+
return version ? { ...summary, version, versionCreated } : { ...summary, versionCreated: false };
|
|
563
|
+
}
|
|
564
|
+
async function recordSucceeded(execution, result) {
|
|
565
|
+
const { run, committedVersion } = execution;
|
|
566
|
+
const { runtime, claim } = execution.input;
|
|
567
|
+
const { version, rowsRead, checkpoint } = result;
|
|
568
|
+
try {
|
|
569
|
+
return await runtime.syncRunsStorage.finish({
|
|
570
|
+
projectId: runtime.id,
|
|
571
|
+
id: run.id,
|
|
572
|
+
executionToken: claim.execution.token,
|
|
573
|
+
status: "succeeded",
|
|
574
|
+
rowsRead,
|
|
575
|
+
...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
|
|
576
|
+
checkpoint
|
|
497
577
|
});
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
blobStorage,
|
|
501
|
-
dataset,
|
|
502
|
-
mode: sync.config.mode,
|
|
503
|
-
signal,
|
|
504
|
-
producer,
|
|
505
|
-
readValues,
|
|
506
|
-
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
507
|
-
commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,
|
|
508
|
-
sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
|
|
509
|
-
onRead(count) {
|
|
510
|
-
rowsRead = count;
|
|
511
|
-
},
|
|
512
|
-
mapValidationError: (error, value, itemIndex) => sourceValidationError(error, value, { syncId: sync.id, runId: job.id, dataset }, itemIndex)
|
|
513
|
-
});
|
|
514
|
-
if (outcome === "created") {
|
|
515
|
-
committedVersion = version;
|
|
516
|
-
}
|
|
517
|
-
let finishedRun;
|
|
518
|
-
try {
|
|
519
|
-
finishedRun = await syncRunsStorage.finish({
|
|
520
|
-
projectId: runtime.id,
|
|
521
|
-
id: job.id,
|
|
522
|
-
status: "succeeded",
|
|
523
|
-
rowsRead,
|
|
524
|
-
...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
|
|
525
|
-
checkpoint: nextCheckpoint
|
|
526
|
-
});
|
|
527
|
-
} catch (error) {
|
|
528
|
-
if (committedVersion) {
|
|
529
|
-
throw createBookkeepingError({
|
|
530
|
-
syncId: sync.id,
|
|
531
|
-
runId: job.id,
|
|
532
|
-
version: committedVersion,
|
|
533
|
-
cause: error
|
|
534
|
-
});
|
|
535
|
-
}
|
|
578
|
+
} catch (error) {
|
|
579
|
+
if (!committedVersion)
|
|
536
580
|
throw error;
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
581
|
+
throw createBookkeepingError({
|
|
582
|
+
syncId: run.syncId,
|
|
583
|
+
runId: run.id,
|
|
584
|
+
version: committedVersion,
|
|
585
|
+
cause: error
|
|
542
586
|
});
|
|
543
|
-
await notifyRunFinished(input.onRunFinished, finishedRun, committedVersion);
|
|
544
|
-
const result = {
|
|
545
|
-
id: job.id,
|
|
546
|
-
syncId: sync.id,
|
|
547
|
-
datasetId: dataset.id,
|
|
548
|
-
mode: sync.config.mode,
|
|
549
|
-
startedAt: requireStartedAt(job.id, startedRun.startedAt),
|
|
550
|
-
finishedAt,
|
|
551
|
-
rowsRead
|
|
552
|
-
};
|
|
553
|
-
return version ? { ...result, version, versionCreated: outcome === "created" } : { ...result, versionCreated: false };
|
|
554
|
-
} catch (error) {
|
|
555
|
-
if (!committedVersion) {
|
|
556
|
-
const status = signal.aborted ? "cancelled" : "failed";
|
|
557
|
-
try {
|
|
558
|
-
const failureError = status === "failed" ? translateSyncExecutionError(error, {
|
|
559
|
-
syncId: run.syncId,
|
|
560
|
-
runId: job.id,
|
|
561
|
-
datasetId: run.datasetId
|
|
562
|
-
}) : error;
|
|
563
|
-
const failure = captureSixbFailure(failureError, {
|
|
564
|
-
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
565
|
-
defaultCode: status === "cancelled" ? "runtime.cancelled" : "internal.unexpected",
|
|
566
|
-
details: { syncId: run.syncId, runId: job.id, datasetId: run.datasetId }
|
|
567
|
-
});
|
|
568
|
-
const failedRun = await syncRunsStorage.finish({
|
|
569
|
-
projectId: runtime.id,
|
|
570
|
-
id: job.id,
|
|
571
|
-
status,
|
|
572
|
-
rowsRead,
|
|
573
|
-
error: failure
|
|
574
|
-
});
|
|
575
|
-
await notifyRunFinished(input.onRunFinished, failedRun);
|
|
576
|
-
if (status === "failed" && failedRun.status === "failed") {
|
|
577
|
-
input.onRunFailed?.(error, failedRun, failure);
|
|
578
|
-
}
|
|
579
|
-
} catch {}
|
|
580
|
-
}
|
|
581
|
-
throw error;
|
|
582
|
-
} finally {
|
|
583
|
-
await logSession.flush();
|
|
584
587
|
}
|
|
585
588
|
}
|
|
589
|
+
function finishLanded(execution, version, read) {
|
|
590
|
+
execution.committedVersion = version;
|
|
591
|
+
return finishSucceeded(execution, { version, versionCreated: true, ...read });
|
|
592
|
+
}
|
|
593
|
+
function readByEarlierExecution(run) {
|
|
594
|
+
return { rowsRead: run.rowsRead ?? 0, checkpoint: run.checkpoint };
|
|
595
|
+
}
|
|
596
|
+
async function settleFailure(execution, caught) {
|
|
597
|
+
const { run, signal, committedVersion } = execution;
|
|
598
|
+
const landed = committedVersion ? null : await landedCommitBehind(execution, caught);
|
|
599
|
+
if (landed)
|
|
600
|
+
return finishLanded(execution, landed, landedRead(execution, caught));
|
|
601
|
+
const error = translateLostRace(run, caught);
|
|
602
|
+
if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
|
|
603
|
+
await recordFailed(execution, error);
|
|
604
|
+
}
|
|
605
|
+
throw error;
|
|
606
|
+
}
|
|
607
|
+
function landedRead(execution, caught) {
|
|
608
|
+
const lostItsRace = caught instanceof LakeConcurrencyError;
|
|
609
|
+
const own = execution.reachedCommit && !lostItsRace && !(caught instanceof CommitAlreadyLandedError);
|
|
610
|
+
return own ? { rowsRead: execution.rowsRead, checkpoint: execution.nextCheckpoint } : readByEarlierExecution(execution.run);
|
|
611
|
+
}
|
|
612
|
+
async function recordFailed(execution, error) {
|
|
613
|
+
const { run, rowsRead } = execution;
|
|
614
|
+
const { runtime, claim, onRunFinished, onRunFailed } = execution.input;
|
|
615
|
+
try {
|
|
616
|
+
const failure = captureSixbFailure(translateSyncExecutionError(error, {
|
|
617
|
+
syncId: run.syncId,
|
|
618
|
+
runId: run.id,
|
|
619
|
+
datasetId: run.datasetId
|
|
620
|
+
}), {
|
|
621
|
+
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
622
|
+
defaultCode: "internal.unexpected",
|
|
623
|
+
details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId }
|
|
624
|
+
});
|
|
625
|
+
const failedRun = await runtime.syncRunsStorage.finish({
|
|
626
|
+
projectId: runtime.id,
|
|
627
|
+
id: run.id,
|
|
628
|
+
executionToken: claim.execution.token,
|
|
629
|
+
status: "failed",
|
|
630
|
+
rowsRead,
|
|
631
|
+
error: failure
|
|
632
|
+
});
|
|
633
|
+
await notifyRunFinished(onRunFinished, failedRun);
|
|
634
|
+
onRunFailed?.(error, failedRun, failure);
|
|
635
|
+
} catch {}
|
|
636
|
+
}
|
|
637
|
+
function cloneCheckpoint(checkpoint) {
|
|
638
|
+
return checkpoint === undefined ? undefined : cloneJsonValue3(checkpoint);
|
|
639
|
+
}
|
|
640
|
+
async function readPreviousCheckpoint(runtime, sync) {
|
|
641
|
+
const { runs } = await runtime.syncRunsStorage.list({
|
|
642
|
+
projectId: runtime.id,
|
|
643
|
+
syncId: sync.id,
|
|
644
|
+
statuses: ["succeeded"],
|
|
645
|
+
limit: 1,
|
|
646
|
+
order: "desc"
|
|
647
|
+
});
|
|
648
|
+
return runs[0]?.checkpoint;
|
|
649
|
+
}
|
|
650
|
+
function requireDataset(runtime, sync, run) {
|
|
651
|
+
const dataset = runtime.datasets.getById(run.datasetId);
|
|
652
|
+
if (dataset)
|
|
653
|
+
return dataset;
|
|
654
|
+
throw new Error(`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`);
|
|
655
|
+
}
|
|
656
|
+
function requireSync(runtime, run) {
|
|
657
|
+
const sync = runtime.syncs.getById(run.syncId);
|
|
658
|
+
if (!sync) {
|
|
659
|
+
throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`);
|
|
660
|
+
}
|
|
661
|
+
if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
|
|
662
|
+
throw new Error(`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`);
|
|
663
|
+
}
|
|
664
|
+
return sync;
|
|
665
|
+
}
|
|
666
|
+
async function landedCommitBehind(execution, error) {
|
|
667
|
+
if (error instanceof CommitAlreadyLandedError)
|
|
668
|
+
return error.version;
|
|
669
|
+
return findLandedCommit(execution);
|
|
670
|
+
}
|
|
671
|
+
function translateLostRace(run, error) {
|
|
672
|
+
if (!(error instanceof LakeConcurrencyError) || run.expectedLatestVersionId !== undefined) {
|
|
673
|
+
return error;
|
|
674
|
+
}
|
|
675
|
+
return new CommitLostRaceError(`[SixbSyncWorker] Sync run '${run.id}' lost its commit race.`, {
|
|
676
|
+
cause: error
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
async function findLandedCommit(execution) {
|
|
680
|
+
const { run, commitPreparedAt, input } = execution;
|
|
681
|
+
if (!commitPreparedAt)
|
|
682
|
+
return null;
|
|
683
|
+
return findProducedVersion({
|
|
684
|
+
lakeStorage: input.runtime.lakeStorage,
|
|
685
|
+
datasetId: run.datasetId,
|
|
686
|
+
producer: { kind: "sync", runId: run.id },
|
|
687
|
+
since: commitPreparedAt
|
|
688
|
+
});
|
|
689
|
+
}
|
|
586
690
|
async function notifyRunFinished(handler, run, createdVersion) {
|
|
587
691
|
try {
|
|
588
692
|
await handler?.(run, createdVersion);
|
|
@@ -599,7 +703,7 @@ function requireStartedAt(runId, startedAt) {
|
|
|
599
703
|
// src/worker.ts
|
|
600
704
|
var SOURCE = "SixbSyncWorker";
|
|
601
705
|
|
|
602
|
-
class SyncWorker extends
|
|
706
|
+
class SyncWorker extends RunWorker {
|
|
603
707
|
host;
|
|
604
708
|
constructor(host, options = {}) {
|
|
605
709
|
if (host.definitions.syncs.list().length === 0) {
|
|
@@ -610,98 +714,74 @@ class SyncWorker extends QueueWorker {
|
|
|
610
714
|
queue: host.queues.syncRuns,
|
|
611
715
|
failureCodes: SYNC_RUN_FAILURE_CODES2,
|
|
612
716
|
workerId: `sync-worker-${host.id}`,
|
|
613
|
-
claimLimit: options.concurrency
|
|
717
|
+
claimLimit: options.concurrency,
|
|
718
|
+
runs: requireSyncRuns(host),
|
|
719
|
+
executions: host.storage.executions
|
|
614
720
|
});
|
|
615
|
-
assertSyncStorage(host);
|
|
616
721
|
this.host = host;
|
|
617
722
|
}
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
const syncRuns = this.host.storage.syncRuns;
|
|
621
|
-
if (!syncRuns) {
|
|
622
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
|
|
623
|
-
}
|
|
624
|
-
const run = await syncRuns.getById({ projectId: this.host.id, id: job.payload.runId });
|
|
625
|
-
if (!run) {
|
|
626
|
-
throw new Error(`[SixbSyncWorker] Sync run '${job.payload.runId}' was not found.`);
|
|
627
|
-
}
|
|
628
|
-
if (run.status !== "queued")
|
|
629
|
-
return;
|
|
630
|
-
const durableExecution = await this.host.storage.executions.getById({
|
|
723
|
+
claimRun(claimed, execution) {
|
|
724
|
+
return requireSyncRuns(this.host).startOrReclaim({
|
|
631
725
|
projectId: this.host.id,
|
|
632
|
-
id:
|
|
726
|
+
id: claimed.job.payload.runId,
|
|
727
|
+
execution
|
|
633
728
|
});
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
}
|
|
729
|
+
}
|
|
730
|
+
async executeRun(claim, _claimed, signal) {
|
|
731
|
+
const { run } = claim;
|
|
732
|
+
const durableExecution = await this.requireExecution(run);
|
|
637
733
|
const execution = bindDurablePrimitiveExecution(this.host, {
|
|
638
734
|
execution: durableExecution,
|
|
639
735
|
primitive: { kind: "sync", id: run.syncId, runId: run.id }
|
|
640
736
|
});
|
|
641
|
-
|
|
737
|
+
await runSyncJob({
|
|
738
|
+
runtime: buildSyncContext(this.host, execution.sixb),
|
|
739
|
+
claim,
|
|
740
|
+
signal,
|
|
741
|
+
onRunStarted: (started) => emitSyncRunStarted(this.host, started, durableExecution.correlationId),
|
|
742
|
+
onRunFinished: async (finished, createdVersion) => {
|
|
743
|
+
await emitSyncRunFinishedEvents(this.host, finished, durableExecution.correlationId, createdVersion);
|
|
744
|
+
await this.wakeNextRun(finished);
|
|
745
|
+
},
|
|
746
|
+
onRunFailed: (error, run, failure) => {
|
|
747
|
+
reportRunFailure(this.host, error, {
|
|
748
|
+
projectId: this.host.id,
|
|
749
|
+
attempt: run.attempt,
|
|
750
|
+
runKind: "sync",
|
|
751
|
+
run: {
|
|
752
|
+
runId: run.id,
|
|
753
|
+
syncId: run.syncId
|
|
754
|
+
},
|
|
755
|
+
failure
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
async wakeNextRun(finished) {
|
|
642
761
|
try {
|
|
643
|
-
await
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
onRunFailed: (error, run, failure) => {
|
|
650
|
-
reportRunFailure(this.host, error, {
|
|
651
|
-
projectId: this.host.id,
|
|
652
|
-
attempt: job.attempt,
|
|
653
|
-
runKind: "sync",
|
|
654
|
-
run: {
|
|
655
|
-
runId: run.id,
|
|
656
|
-
syncId: run.syncId
|
|
657
|
-
},
|
|
658
|
-
failure
|
|
659
|
-
});
|
|
660
|
-
}
|
|
762
|
+
const { runs } = await requireSyncRuns(this.host).list({
|
|
763
|
+
projectId: this.host.id,
|
|
764
|
+
syncId: finished.syncId,
|
|
765
|
+
statuses: ["queued"],
|
|
766
|
+
order: "asc",
|
|
767
|
+
limit: 1
|
|
661
768
|
});
|
|
662
|
-
|
|
663
|
-
if (
|
|
769
|
+
const [next] = runs;
|
|
770
|
+
if (!next)
|
|
664
771
|
return;
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
return { kind: "fail", failure: await this.failUnstartedRun(claimed, run, error) };
|
|
678
|
-
}
|
|
679
|
-
return run?.status === "failed" && run.error ? { kind: "fail", failure: run.error } : { kind: "fail" };
|
|
680
|
-
}
|
|
681
|
-
async failUnstartedRun(claimed, run, error) {
|
|
682
|
-
const syncRuns = this.host.storage.syncRuns;
|
|
683
|
-
if (!syncRuns) {
|
|
684
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
|
|
772
|
+
await this.host.queues.syncRuns.enqueue({
|
|
773
|
+
projectId: this.host.id,
|
|
774
|
+
jobs: [
|
|
775
|
+
{
|
|
776
|
+
id: wakeJobId(next.id, finished.id),
|
|
777
|
+
type: "sync.run.requested",
|
|
778
|
+
payload: { runId: next.id }
|
|
779
|
+
}
|
|
780
|
+
]
|
|
781
|
+
});
|
|
782
|
+
} catch (error) {
|
|
783
|
+
console.error(`[SixbSyncWorker] Could not wake the run after '${finished.id}'.`, error);
|
|
685
784
|
}
|
|
686
|
-
const failure = captureSixbFailure2(error, {
|
|
687
|
-
allowedCodes: SYNC_RUN_FAILURE_CODES2,
|
|
688
|
-
defaultCode: "internal.unexpected",
|
|
689
|
-
details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId }
|
|
690
|
-
});
|
|
691
|
-
await syncRuns.finish({
|
|
692
|
-
projectId: this.host.id,
|
|
693
|
-
id: run.id,
|
|
694
|
-
status: "failed",
|
|
695
|
-
error: failure
|
|
696
|
-
});
|
|
697
|
-
reportRunFailure(this.host, error, {
|
|
698
|
-
projectId: this.host.id,
|
|
699
|
-
attempt: claimed.job.attempt,
|
|
700
|
-
runKind: "sync",
|
|
701
|
-
run: { runId: run.id, syncId: run.syncId },
|
|
702
|
-
failure
|
|
703
|
-
});
|
|
704
|
-
return failure;
|
|
705
785
|
}
|
|
706
786
|
}
|
|
707
787
|
async function emitSyncRunStarted(host, run, correlationId) {
|
|
@@ -763,19 +843,16 @@ function requireTerminalStatus(status, context) {
|
|
|
763
843
|
}
|
|
764
844
|
return status;
|
|
765
845
|
}
|
|
766
|
-
function
|
|
846
|
+
function requireSyncRuns(host) {
|
|
767
847
|
if (!host.storage.syncRuns) {
|
|
768
848
|
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
|
|
769
849
|
}
|
|
850
|
+
return host.storage.syncRuns;
|
|
770
851
|
}
|
|
771
852
|
function buildSyncContext(host, sixb) {
|
|
772
|
-
const syncRunsStorage = host.storage.syncRuns;
|
|
773
|
-
if (!syncRunsStorage) {
|
|
774
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.");
|
|
775
|
-
}
|
|
776
853
|
return {
|
|
777
854
|
id: host.id,
|
|
778
|
-
syncRunsStorage,
|
|
855
|
+
syncRunsStorage: requireSyncRuns(host),
|
|
779
856
|
lakeStorage: host.lakeStorage,
|
|
780
857
|
blobs: sixb.blobs,
|
|
781
858
|
logging: host.logging,
|
|
@@ -790,4 +867,4 @@ export {
|
|
|
790
867
|
SyncWorker
|
|
791
868
|
};
|
|
792
869
|
|
|
793
|
-
//# debugId=
|
|
870
|
+
//# debugId=734C88A91BA0AEF864756E2164756E21
|