@sixb/sync-worker 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/checkpoints.d.ts +11 -18
- package/dist/checkpoints.d.ts.map +1 -1
- package/dist/index.js +188 -187
- package/dist/index.js.map +6 -6
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/source-read.d.ts +23 -1
- package/dist/source-read.d.ts.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/checkpoints.ts +98 -145
- package/src/run-sync-job.ts +65 -13
- package/src/source-read.ts +133 -123
- package/src/worker.ts +31 -3
package/src/checkpoints.ts
CHANGED
|
@@ -10,160 +10,133 @@ import { createSixbError } from "@sixb/core/internal/errors"
|
|
|
10
10
|
const SYNC_CHECKPOINT_KIND = "sync_checkpoint"
|
|
11
11
|
const SYNC_CHECKPOINT_VERSION = 1
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
export interface SyncCheckpointCodec {
|
|
22
|
-
read(value: JsonValue | undefined): JsonValue | undefined
|
|
23
|
-
serialize(value: JsonValue): JsonValue
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/** Bind opaque handler checkpoints to the framework source contract stored beside the run. */
|
|
27
|
-
export function createSyncCheckpointCodec(options: {
|
|
13
|
+
/**
|
|
14
|
+
* `single` is a static connector's one cursor. `connection` is one connection's cursor, bound to
|
|
15
|
+
* the account it read. `connections` held every connection's cursor when one run read them all:
|
|
16
|
+
* a connection's first run of its own still starts from its entry there.
|
|
17
|
+
*/
|
|
18
|
+
type SyncCheckpointStrategy = "single" | "connection" | "connections"
|
|
19
|
+
|
|
20
|
+
interface SyncCheckpointIdentity {
|
|
28
21
|
readonly syncId: string
|
|
29
22
|
readonly connectorId: string
|
|
30
|
-
readonly strategy: SyncCheckpointStrategy
|
|
31
|
-
}): SyncCheckpointCodec {
|
|
32
|
-
return {
|
|
33
|
-
read(value) {
|
|
34
|
-
if (value === undefined) return undefined
|
|
35
|
-
if (!isSyncCheckpoint(value)) {
|
|
36
|
-
// Static Sync checkpoints predate the framework envelope. Preserve them in place and
|
|
37
|
-
// migrate the next successful run; managed fan-out has no compatible legacy raw format.
|
|
38
|
-
if (options.strategy === "single") return cloneJsonValue(value)
|
|
39
|
-
throw incompatibleCheckpoint(options, "format")
|
|
40
|
-
}
|
|
41
|
-
if (value.version !== SYNC_CHECKPOINT_VERSION) {
|
|
42
|
-
throw incompatibleCheckpoint(options, "version")
|
|
43
|
-
}
|
|
44
|
-
if (value.connectorId !== options.connectorId) {
|
|
45
|
-
throw incompatibleCheckpoint(options, "connector")
|
|
46
|
-
}
|
|
47
|
-
if (value.strategy !== options.strategy) {
|
|
48
|
-
throw incompatibleCheckpoint(options, "strategy")
|
|
49
|
-
}
|
|
50
|
-
if (!("value" in value) || !isJsonValue(value.value)) {
|
|
51
|
-
throw invalidFrameworkCheckpoint(options.syncId)
|
|
52
|
-
}
|
|
53
|
-
return cloneJsonValue(value.value)
|
|
54
|
-
},
|
|
55
|
-
serialize(value) {
|
|
56
|
-
const envelope: unknown = {
|
|
57
|
-
kind: SYNC_CHECKPOINT_KIND,
|
|
58
|
-
version: SYNC_CHECKPOINT_VERSION,
|
|
59
|
-
connectorId: options.connectorId,
|
|
60
|
-
strategy: options.strategy,
|
|
61
|
-
value: cloneJsonValue(value),
|
|
62
|
-
}
|
|
63
|
-
assertJsonValue(envelope, "Sync checkpoint envelope")
|
|
64
|
-
return envelope
|
|
65
|
-
},
|
|
66
|
-
}
|
|
67
23
|
}
|
|
68
24
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
static from(syncId: string, value: JsonValue | undefined): ConnectionCheckpoints {
|
|
78
|
-
if (value === undefined) return new ConnectionCheckpoints(new Map())
|
|
79
|
-
if (!isConnectionCheckpoint(value)) throw invalidConnectionCheckpoint(syncId)
|
|
25
|
+
type SyncCheckpointEnvelope = JsonValue & {
|
|
26
|
+
readonly kind: typeof SYNC_CHECKPOINT_KIND
|
|
27
|
+
readonly version?: JsonValue
|
|
28
|
+
readonly connectorId?: JsonValue
|
|
29
|
+
readonly strategy?: JsonValue
|
|
30
|
+
readonly value?: JsonValue
|
|
31
|
+
}
|
|
80
32
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
throw invalidConnectionCheckpoint(syncId)
|
|
95
|
-
}
|
|
96
|
-
entries.set(candidate.connectionId, {
|
|
97
|
-
connectionId: candidate.connectionId,
|
|
98
|
-
accountId: candidate.accountId,
|
|
99
|
-
...(candidate.checkpoint === undefined
|
|
100
|
-
? {}
|
|
101
|
-
: { checkpoint: cloneJsonValue(candidate.checkpoint as JsonValue) }),
|
|
102
|
-
})
|
|
103
|
-
}
|
|
104
|
-
return new ConnectionCheckpoints(entries)
|
|
105
|
-
}
|
|
33
|
+
/** A static connector's checkpoint: the handler's value inside the framework envelope. */
|
|
34
|
+
export const singleCheckpoint = {
|
|
35
|
+
read(identity: SyncCheckpointIdentity, value: JsonValue | undefined): JsonValue | undefined {
|
|
36
|
+
if (value === undefined) return undefined
|
|
37
|
+
// Static Sync checkpoints predate the framework envelope. Preserve them in place and migrate
|
|
38
|
+
// the next successful run.
|
|
39
|
+
if (!isSyncCheckpoint(value)) return cloneJsonValue(value)
|
|
40
|
+
return openEnvelope(identity, value, ["single"]).value
|
|
41
|
+
},
|
|
42
|
+
serialize(identity: SyncCheckpointIdentity, value: JsonValue): JsonValue {
|
|
43
|
+
return envelope(identity, "single", value)
|
|
44
|
+
},
|
|
45
|
+
}
|
|
106
46
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
47
|
+
/** One connection's checkpoint. A replaced account starts over. */
|
|
48
|
+
export const connectionCheckpoint = {
|
|
49
|
+
read(
|
|
50
|
+
identity: SyncCheckpointIdentity,
|
|
51
|
+
connection: SyncConnectorConnection,
|
|
52
|
+
value: JsonValue | undefined
|
|
53
|
+
): JsonValue | undefined {
|
|
54
|
+
if (value === undefined) return undefined
|
|
55
|
+
if (!isSyncCheckpoint(value)) throw incompatibleCheckpoint(identity, "format")
|
|
56
|
+
|
|
57
|
+
const opened = openEnvelope(identity, value, ["connection", "connections"])
|
|
58
|
+
const entry =
|
|
59
|
+
opened.strategy === "connection"
|
|
60
|
+
? opened.value
|
|
61
|
+
: legacyEntries(identity.syncId, opened.value).find(
|
|
62
|
+
(candidate) => candidate.connectionId === connection.id
|
|
63
|
+
)
|
|
64
|
+
if (entry === undefined) return undefined
|
|
65
|
+
if (!isRecord(entry) || typeof entry.accountId !== "string") {
|
|
66
|
+
throw invalidCheckpoint(identity.syncId)
|
|
113
67
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
get(connection: SyncConnectorConnection): JsonValue | undefined {
|
|
117
|
-
const entry = this.entries.get(connection.id)
|
|
118
|
-
if (entry?.accountId !== connection.account.id || entry.checkpoint === undefined) {
|
|
68
|
+
if (entry.accountId !== connection.account.id || entry.checkpoint === undefined) {
|
|
119
69
|
return undefined
|
|
120
70
|
}
|
|
71
|
+
if (!isJsonValue(entry.checkpoint)) throw invalidCheckpoint(identity.syncId)
|
|
121
72
|
return cloneJsonValue(entry.checkpoint)
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
73
|
+
},
|
|
74
|
+
serialize(
|
|
75
|
+
identity: SyncCheckpointIdentity,
|
|
76
|
+
connection: SyncConnectorConnection,
|
|
77
|
+
checkpoint: JsonValue | undefined
|
|
78
|
+
): JsonValue {
|
|
79
|
+
return envelope(identity, "connection", {
|
|
127
80
|
accountId: connection.account.id,
|
|
128
|
-
checkpoint: cloneJsonValue(checkpoint),
|
|
81
|
+
...(checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(checkpoint) }),
|
|
129
82
|
})
|
|
83
|
+
},
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function openEnvelope(
|
|
87
|
+
identity: SyncCheckpointIdentity,
|
|
88
|
+
value: SyncCheckpointEnvelope,
|
|
89
|
+
strategies: readonly SyncCheckpointStrategy[]
|
|
90
|
+
): { readonly strategy: SyncCheckpointStrategy; readonly value: JsonValue } {
|
|
91
|
+
if (value.version !== SYNC_CHECKPOINT_VERSION) throw incompatibleCheckpoint(identity, "version")
|
|
92
|
+
if (value.connectorId !== identity.connectorId) {
|
|
93
|
+
throw incompatibleCheckpoint(identity, "connector")
|
|
130
94
|
}
|
|
95
|
+
const strategy = strategies.find((candidate) => candidate === value.strategy)
|
|
96
|
+
if (!strategy) throw incompatibleCheckpoint(identity, "strategy")
|
|
97
|
+
if (!("value" in value) || !isJsonValue(value.value)) throw invalidCheckpoint(identity.syncId)
|
|
98
|
+
return { strategy, value: cloneJsonValue(value.value) }
|
|
99
|
+
}
|
|
131
100
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
}
|
|
144
|
-
assertJsonValue(value, "Managed connector Sync checkpoint")
|
|
145
|
-
return value
|
|
101
|
+
function envelope(
|
|
102
|
+
identity: SyncCheckpointIdentity,
|
|
103
|
+
strategy: SyncCheckpointStrategy,
|
|
104
|
+
value: JsonValue
|
|
105
|
+
): JsonValue {
|
|
106
|
+
const serialized: unknown = {
|
|
107
|
+
kind: SYNC_CHECKPOINT_KIND,
|
|
108
|
+
version: SYNC_CHECKPOINT_VERSION,
|
|
109
|
+
connectorId: identity.connectorId,
|
|
110
|
+
strategy,
|
|
111
|
+
value: cloneJsonValue(value),
|
|
146
112
|
}
|
|
113
|
+
assertJsonValue(serialized, "Sync checkpoint envelope")
|
|
114
|
+
return serialized
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function legacyEntries(syncId: string, value: JsonValue): readonly Record<string, unknown>[] {
|
|
118
|
+
if (!isRecord(value) || !Array.isArray(value.entries)) throw invalidCheckpoint(syncId)
|
|
119
|
+
return value.entries.flatMap((entry) => (isRecord(entry) ? [entry] : []))
|
|
147
120
|
}
|
|
148
121
|
|
|
149
122
|
function incompatibleCheckpoint(
|
|
150
|
-
|
|
123
|
+
identity: SyncCheckpointIdentity,
|
|
151
124
|
reason: "format" | "version" | "connector" | "strategy"
|
|
152
125
|
): Error {
|
|
153
126
|
return createSixbError(
|
|
154
127
|
"sync.execution_failed",
|
|
155
|
-
`[SixbSyncWorker] Sync '${
|
|
128
|
+
`[SixbSyncWorker] Sync '${identity.syncId}' has a checkpoint incompatible with connector '${identity.connectorId}'. Use a new Sync id to reset its ingestion state.`,
|
|
156
129
|
{
|
|
157
130
|
details: {
|
|
158
|
-
syncId:
|
|
159
|
-
connectorId:
|
|
131
|
+
syncId: identity.syncId,
|
|
132
|
+
connectorId: identity.connectorId,
|
|
160
133
|
reason: `checkpoint_${reason}`,
|
|
161
134
|
},
|
|
162
135
|
}
|
|
163
136
|
)
|
|
164
137
|
}
|
|
165
138
|
|
|
166
|
-
function
|
|
139
|
+
function invalidCheckpoint(syncId: string): Error {
|
|
167
140
|
return createSixbError(
|
|
168
141
|
"internal.unexpected",
|
|
169
142
|
`[SixbSyncWorker] Sync '${syncId}' has an invalid checkpoint envelope.`,
|
|
@@ -171,30 +144,10 @@ function invalidFrameworkCheckpoint(syncId: string): Error {
|
|
|
171
144
|
)
|
|
172
145
|
}
|
|
173
146
|
|
|
174
|
-
function
|
|
175
|
-
return createSixbError(
|
|
176
|
-
"internal.unexpected",
|
|
177
|
-
`[SixbSyncWorker] Sync '${syncId}' has an invalid managed connector checkpoint.`,
|
|
178
|
-
{ details: { syncId } }
|
|
179
|
-
)
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
function isSyncCheckpoint(value: JsonValue): value is JsonValue & {
|
|
183
|
-
readonly kind: typeof SYNC_CHECKPOINT_KIND
|
|
184
|
-
readonly version?: JsonValue
|
|
185
|
-
readonly connectorId?: JsonValue
|
|
186
|
-
readonly strategy?: JsonValue
|
|
187
|
-
readonly value?: JsonValue
|
|
188
|
-
} {
|
|
147
|
+
function isSyncCheckpoint(value: JsonValue): value is SyncCheckpointEnvelope {
|
|
189
148
|
return isRecord(value) && value.kind === SYNC_CHECKPOINT_KIND
|
|
190
149
|
}
|
|
191
150
|
|
|
192
|
-
function isConnectionCheckpoint(value: JsonValue): value is JsonValue & {
|
|
193
|
-
readonly entries: JsonValue[]
|
|
194
|
-
} {
|
|
195
|
-
return isRecord(value) && Array.isArray(value.entries)
|
|
196
|
-
}
|
|
197
|
-
|
|
198
151
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
199
152
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
200
153
|
}
|
package/src/run-sync-job.ts
CHANGED
|
@@ -16,7 +16,12 @@ import { resolveLoggingService } from "@sixb/core/internal/logging"
|
|
|
16
16
|
import { type DatasetVersion, LakeConcurrencyError } from "@sixb/core/lake-storage"
|
|
17
17
|
import { SYNC_RUN_FAILURE_CODES, type SyncRunRecord } from "@sixb/core/storage"
|
|
18
18
|
import { throwIfAborted } from "./normalize"
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
readSyncValues,
|
|
21
|
+
resolveSyncRunSource,
|
|
22
|
+
SyncRunSourceGoneError,
|
|
23
|
+
type SyncSourceValue,
|
|
24
|
+
} from "./source-read"
|
|
20
25
|
import type { RunSyncJobInput, SyncRunFinishedHandler, SyncRunResult } from "./types"
|
|
21
26
|
|
|
22
27
|
/** What the execution behind a commit read, which the run records once it finds that commit. */
|
|
@@ -90,7 +95,9 @@ function translateSyncExecutionError(
|
|
|
90
95
|
): unknown {
|
|
91
96
|
if (
|
|
92
97
|
isSixbError(error) &&
|
|
93
|
-
(error.code === "internal.unexpected" ||
|
|
98
|
+
(error.code === "internal.unexpected" ||
|
|
99
|
+
error.code === "sync.execution_failed" ||
|
|
100
|
+
error.code === "connector.authorization_required")
|
|
94
101
|
) {
|
|
95
102
|
return error
|
|
96
103
|
}
|
|
@@ -214,7 +221,8 @@ async function commitRun(execution: SyncExecution, logger: Logger): Promise<Sync
|
|
|
214
221
|
const sync = requireSync(runtime, run)
|
|
215
222
|
const dataset = requireDataset(runtime, sync, run)
|
|
216
223
|
|
|
217
|
-
const
|
|
224
|
+
const source = await resolveSyncRunSource(runtime, sync, run)
|
|
225
|
+
const previousCheckpoint = await readPreviousCheckpoint(runtime, sync, run)
|
|
218
226
|
execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint)
|
|
219
227
|
|
|
220
228
|
const { outcome, version } = await writeDataset({
|
|
@@ -228,6 +236,7 @@ async function commitRun(execution: SyncExecution, logger: Logger): Promise<Sync
|
|
|
228
236
|
readSyncValues({
|
|
229
237
|
runtime,
|
|
230
238
|
sync,
|
|
239
|
+
source,
|
|
231
240
|
runId: run.id,
|
|
232
241
|
datasetId: dataset.id,
|
|
233
242
|
signal,
|
|
@@ -375,6 +384,11 @@ async function settleFailure(execution: SyncExecution, caught: unknown): Promise
|
|
|
375
384
|
const landed = committedVersion ? null : await landedCommitBehind(execution, caught)
|
|
376
385
|
if (landed) return finishLanded(execution, landed, landedRead(execution, caught))
|
|
377
386
|
|
|
387
|
+
if (caught instanceof SyncRunSourceGoneError && !signal.aborted) {
|
|
388
|
+
await recordCancelled(execution, caught)
|
|
389
|
+
throw caught
|
|
390
|
+
}
|
|
391
|
+
|
|
378
392
|
const error = translateLostRace(run, caught)
|
|
379
393
|
|
|
380
394
|
if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
|
|
@@ -430,23 +444,61 @@ async function recordFailed(execution: SyncExecution, error: unknown): Promise<v
|
|
|
430
444
|
}
|
|
431
445
|
}
|
|
432
446
|
|
|
447
|
+
/** The run had nothing left to read: nothing failed, so nothing is reported. */
|
|
448
|
+
async function recordCancelled(execution: SyncExecution, reason: Error): Promise<void> {
|
|
449
|
+
const { run } = execution
|
|
450
|
+
const { runtime, claim, onRunFinished } = execution.input
|
|
451
|
+
|
|
452
|
+
try {
|
|
453
|
+
const cancelled = await runtime.syncRunsStorage.finish({
|
|
454
|
+
projectId: runtime.id,
|
|
455
|
+
id: run.id,
|
|
456
|
+
executionToken: claim.execution.token,
|
|
457
|
+
status: "cancelled",
|
|
458
|
+
rowsRead: 0,
|
|
459
|
+
error: captureSixbFailure(createSixbError("runtime.cancelled", reason.message), {
|
|
460
|
+
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
461
|
+
defaultCode: "runtime.cancelled",
|
|
462
|
+
}),
|
|
463
|
+
})
|
|
464
|
+
|
|
465
|
+
await notifyRunFinished(onRunFinished, cancelled)
|
|
466
|
+
} catch {
|
|
467
|
+
// The run did not transition: a later delivery, or the execution that holds it, ends it.
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
433
471
|
function cloneCheckpoint(checkpoint: JsonValue | undefined): JsonValue | undefined {
|
|
434
472
|
return checkpoint === undefined ? undefined : cloneJsonValue(checkpoint)
|
|
435
473
|
}
|
|
436
474
|
|
|
475
|
+
/**
|
|
476
|
+
* The checkpoint of the last succeeded run of this sync, and of this run's connection. A
|
|
477
|
+
* connection's first run of its own starts from the last succeeded run that read every connection,
|
|
478
|
+
* whose checkpoint holds an entry for each.
|
|
479
|
+
*/
|
|
437
480
|
async function readPreviousCheckpoint(
|
|
438
481
|
runtime: RunSyncJobInput["runtime"],
|
|
439
|
-
sync: SyncDefinition
|
|
482
|
+
sync: SyncDefinition,
|
|
483
|
+
run: SyncRunRecord
|
|
440
484
|
): Promise<JsonValue | undefined> {
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
485
|
+
const latest = (connectionId?: string | null) =>
|
|
486
|
+
runtime.syncRunsStorage
|
|
487
|
+
.list({
|
|
488
|
+
projectId: runtime.id,
|
|
489
|
+
syncId: sync.id,
|
|
490
|
+
...(connectionId === undefined ? {} : { connectionId }),
|
|
491
|
+
statuses: ["succeeded"],
|
|
492
|
+
limit: 1,
|
|
493
|
+
order: "desc",
|
|
494
|
+
})
|
|
495
|
+
.then(({ runs }) => runs[0])
|
|
496
|
+
|
|
497
|
+
if (run.connectionId === undefined) return (await latest())?.checkpoint
|
|
498
|
+
|
|
499
|
+
const own = await latest(run.connectionId)
|
|
500
|
+
if (own) return own.checkpoint
|
|
501
|
+
return (await latest(null))?.checkpoint
|
|
450
502
|
}
|
|
451
503
|
|
|
452
504
|
function requireDataset(
|