@sixb/sync-worker 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,160 +10,133 @@ import { createSixbError } from "@sixb/core/internal/errors"
10
10
  const SYNC_CHECKPOINT_KIND = "sync_checkpoint"
11
11
  const SYNC_CHECKPOINT_VERSION = 1
12
12
 
13
- type SyncCheckpointStrategy = "single" | "connections"
14
-
15
- interface ConnectionCheckpointEntry {
16
- readonly connectionId: string
17
- readonly accountId: string
18
- readonly checkpoint?: JsonValue
19
- }
20
-
21
- export interface SyncCheckpointCodec {
22
- read(value: JsonValue | undefined): JsonValue | undefined
23
- serialize(value: JsonValue): JsonValue
24
- }
25
-
26
- /** Bind opaque handler checkpoints to the framework source contract stored beside the run. */
27
- export function createSyncCheckpointCodec(options: {
13
+ /**
14
+ * `single` is a static connector's one cursor. `connection` is one connection's cursor, bound to
15
+ * the account it read. `connections` held every connection's cursor when one run read them all:
16
+ * a connection's first run of its own still starts from its entry there.
17
+ */
18
+ type SyncCheckpointStrategy = "single" | "connection" | "connections"
19
+
20
+ interface SyncCheckpointIdentity {
28
21
  readonly syncId: string
29
22
  readonly connectorId: string
30
- readonly strategy: SyncCheckpointStrategy
31
- }): SyncCheckpointCodec {
32
- return {
33
- read(value) {
34
- if (value === undefined) return undefined
35
- if (!isSyncCheckpoint(value)) {
36
- // Static Sync checkpoints predate the framework envelope. Preserve them in place and
37
- // migrate the next successful run; managed fan-out has no compatible legacy raw format.
38
- if (options.strategy === "single") return cloneJsonValue(value)
39
- throw incompatibleCheckpoint(options, "format")
40
- }
41
- if (value.version !== SYNC_CHECKPOINT_VERSION) {
42
- throw incompatibleCheckpoint(options, "version")
43
- }
44
- if (value.connectorId !== options.connectorId) {
45
- throw incompatibleCheckpoint(options, "connector")
46
- }
47
- if (value.strategy !== options.strategy) {
48
- throw incompatibleCheckpoint(options, "strategy")
49
- }
50
- if (!("value" in value) || !isJsonValue(value.value)) {
51
- throw invalidFrameworkCheckpoint(options.syncId)
52
- }
53
- return cloneJsonValue(value.value)
54
- },
55
- serialize(value) {
56
- const envelope: unknown = {
57
- kind: SYNC_CHECKPOINT_KIND,
58
- version: SYNC_CHECKPOINT_VERSION,
59
- connectorId: options.connectorId,
60
- strategy: options.strategy,
61
- value: cloneJsonValue(value),
62
- }
63
- assertJsonValue(envelope, "Sync checkpoint envelope")
64
- return envelope
65
- },
66
- }
67
23
  }
68
24
 
69
- /** Per-connection cursor state carried inside a framework-managed checkpoint envelope. */
70
- export class ConnectionCheckpoints {
71
- private readonly entries: Map<string, ConnectionCheckpointEntry>
72
-
73
- private constructor(entries: Map<string, ConnectionCheckpointEntry>) {
74
- this.entries = entries
75
- }
76
-
77
- static from(syncId: string, value: JsonValue | undefined): ConnectionCheckpoints {
78
- if (value === undefined) return new ConnectionCheckpoints(new Map())
79
- if (!isConnectionCheckpoint(value)) throw invalidConnectionCheckpoint(syncId)
25
+ type SyncCheckpointEnvelope = JsonValue & {
26
+ readonly kind: typeof SYNC_CHECKPOINT_KIND
27
+ readonly version?: JsonValue
28
+ readonly connectorId?: JsonValue
29
+ readonly strategy?: JsonValue
30
+ readonly value?: JsonValue
31
+ }
80
32
 
81
- const entries = new Map<string, ConnectionCheckpointEntry>()
82
- for (const candidate of value.entries) {
83
- if (
84
- !isRecord(candidate) ||
85
- typeof candidate.connectionId !== "string" ||
86
- !candidate.connectionId.trim() ||
87
- typeof candidate.accountId !== "string" ||
88
- !candidate.accountId.trim() ||
89
- entries.has(candidate.connectionId)
90
- ) {
91
- throw invalidConnectionCheckpoint(syncId)
92
- }
93
- if ("checkpoint" in candidate && !isJsonValue(candidate.checkpoint)) {
94
- throw invalidConnectionCheckpoint(syncId)
95
- }
96
- entries.set(candidate.connectionId, {
97
- connectionId: candidate.connectionId,
98
- accountId: candidate.accountId,
99
- ...(candidate.checkpoint === undefined
100
- ? {}
101
- : { checkpoint: cloneJsonValue(candidate.checkpoint as JsonValue) }),
102
- })
103
- }
104
- return new ConnectionCheckpoints(entries)
105
- }
33
+ /** A static connector's checkpoint: the handler's value inside the framework envelope. */
34
+ export const singleCheckpoint = {
35
+ read(identity: SyncCheckpointIdentity, value: JsonValue | undefined): JsonValue | undefined {
36
+ if (value === undefined) return undefined
37
+ // Static Sync checkpoints predate the framework envelope. Preserve them in place and migrate
38
+ // the next successful run.
39
+ if (!isSyncCheckpoint(value)) return cloneJsonValue(value)
40
+ return openEnvelope(identity, value, ["single"]).value
41
+ },
42
+ serialize(identity: SyncCheckpointIdentity, value: JsonValue): JsonValue {
43
+ return envelope(identity, "single", value)
44
+ },
45
+ }
106
46
 
107
- reconcile(connections: readonly SyncConnectorConnection[]): void {
108
- for (const connection of connections) {
109
- const existing = this.entries.get(connection.id)
110
- if (existing && existing.accountId !== connection.account.id) {
111
- this.entries.delete(connection.id)
112
- }
47
+ /** One connection's checkpoint. A replaced account starts over. */
48
+ export const connectionCheckpoint = {
49
+ read(
50
+ identity: SyncCheckpointIdentity,
51
+ connection: SyncConnectorConnection,
52
+ value: JsonValue | undefined
53
+ ): JsonValue | undefined {
54
+ if (value === undefined) return undefined
55
+ if (!isSyncCheckpoint(value)) throw incompatibleCheckpoint(identity, "format")
56
+
57
+ const opened = openEnvelope(identity, value, ["connection", "connections"])
58
+ const entry =
59
+ opened.strategy === "connection"
60
+ ? opened.value
61
+ : legacyEntries(identity.syncId, opened.value).find(
62
+ (candidate) => candidate.connectionId === connection.id
63
+ )
64
+ if (entry === undefined) return undefined
65
+ if (!isRecord(entry) || typeof entry.accountId !== "string") {
66
+ throw invalidCheckpoint(identity.syncId)
113
67
  }
114
- }
115
-
116
- get(connection: SyncConnectorConnection): JsonValue | undefined {
117
- const entry = this.entries.get(connection.id)
118
- if (entry?.accountId !== connection.account.id || entry.checkpoint === undefined) {
68
+ if (entry.accountId !== connection.account.id || entry.checkpoint === undefined) {
119
69
  return undefined
120
70
  }
71
+ if (!isJsonValue(entry.checkpoint)) throw invalidCheckpoint(identity.syncId)
121
72
  return cloneJsonValue(entry.checkpoint)
122
- }
123
-
124
- set(connection: SyncConnectorConnection, checkpoint: JsonValue): void {
125
- this.entries.set(connection.id, {
126
- connectionId: connection.id,
73
+ },
74
+ serialize(
75
+ identity: SyncCheckpointIdentity,
76
+ connection: SyncConnectorConnection,
77
+ checkpoint: JsonValue | undefined
78
+ ): JsonValue {
79
+ return envelope(identity, "connection", {
127
80
  accountId: connection.account.id,
128
- checkpoint: cloneJsonValue(checkpoint),
81
+ ...(checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(checkpoint) }),
129
82
  })
83
+ },
84
+ }
85
+
86
+ function openEnvelope(
87
+ identity: SyncCheckpointIdentity,
88
+ value: SyncCheckpointEnvelope,
89
+ strategies: readonly SyncCheckpointStrategy[]
90
+ ): { readonly strategy: SyncCheckpointStrategy; readonly value: JsonValue } {
91
+ if (value.version !== SYNC_CHECKPOINT_VERSION) throw incompatibleCheckpoint(identity, "version")
92
+ if (value.connectorId !== identity.connectorId) {
93
+ throw incompatibleCheckpoint(identity, "connector")
130
94
  }
95
+ const strategy = strategies.find((candidate) => candidate === value.strategy)
96
+ if (!strategy) throw incompatibleCheckpoint(identity, "strategy")
97
+ if (!("value" in value) || !isJsonValue(value.value)) throw invalidCheckpoint(identity.syncId)
98
+ return { strategy, value: cloneJsonValue(value.value) }
99
+ }
131
100
 
132
- serialize(): JsonValue {
133
- const value: unknown = {
134
- entries: [...this.entries.values()]
135
- .sort((left, right) => left.connectionId.localeCompare(right.connectionId))
136
- .map((entry) => ({
137
- connectionId: entry.connectionId,
138
- accountId: entry.accountId,
139
- ...(entry.checkpoint === undefined
140
- ? {}
141
- : { checkpoint: cloneJsonValue(entry.checkpoint) }),
142
- })),
143
- }
144
- assertJsonValue(value, "Managed connector Sync checkpoint")
145
- return value
101
+ function envelope(
102
+ identity: SyncCheckpointIdentity,
103
+ strategy: SyncCheckpointStrategy,
104
+ value: JsonValue
105
+ ): JsonValue {
106
+ const serialized: unknown = {
107
+ kind: SYNC_CHECKPOINT_KIND,
108
+ version: SYNC_CHECKPOINT_VERSION,
109
+ connectorId: identity.connectorId,
110
+ strategy,
111
+ value: cloneJsonValue(value),
146
112
  }
113
+ assertJsonValue(serialized, "Sync checkpoint envelope")
114
+ return serialized
115
+ }
116
+
117
+ function legacyEntries(syncId: string, value: JsonValue): readonly Record<string, unknown>[] {
118
+ if (!isRecord(value) || !Array.isArray(value.entries)) throw invalidCheckpoint(syncId)
119
+ return value.entries.flatMap((entry) => (isRecord(entry) ? [entry] : []))
147
120
  }
148
121
 
149
122
  function incompatibleCheckpoint(
150
- options: { readonly syncId: string; readonly connectorId: string },
123
+ identity: SyncCheckpointIdentity,
151
124
  reason: "format" | "version" | "connector" | "strategy"
152
125
  ): Error {
153
126
  return createSixbError(
154
127
  "sync.execution_failed",
155
- `[SixbSyncWorker] Sync '${options.syncId}' has a checkpoint incompatible with connector '${options.connectorId}'. Use a new Sync id to reset its ingestion state.`,
128
+ `[SixbSyncWorker] Sync '${identity.syncId}' has a checkpoint incompatible with connector '${identity.connectorId}'. Use a new Sync id to reset its ingestion state.`,
156
129
  {
157
130
  details: {
158
- syncId: options.syncId,
159
- connectorId: options.connectorId,
131
+ syncId: identity.syncId,
132
+ connectorId: identity.connectorId,
160
133
  reason: `checkpoint_${reason}`,
161
134
  },
162
135
  }
163
136
  )
164
137
  }
165
138
 
166
- function invalidFrameworkCheckpoint(syncId: string): Error {
139
+ function invalidCheckpoint(syncId: string): Error {
167
140
  return createSixbError(
168
141
  "internal.unexpected",
169
142
  `[SixbSyncWorker] Sync '${syncId}' has an invalid checkpoint envelope.`,
@@ -171,30 +144,10 @@ function invalidFrameworkCheckpoint(syncId: string): Error {
171
144
  )
172
145
  }
173
146
 
174
- function invalidConnectionCheckpoint(syncId: string): Error {
175
- return createSixbError(
176
- "internal.unexpected",
177
- `[SixbSyncWorker] Sync '${syncId}' has an invalid managed connector checkpoint.`,
178
- { details: { syncId } }
179
- )
180
- }
181
-
182
- function isSyncCheckpoint(value: JsonValue): value is JsonValue & {
183
- readonly kind: typeof SYNC_CHECKPOINT_KIND
184
- readonly version?: JsonValue
185
- readonly connectorId?: JsonValue
186
- readonly strategy?: JsonValue
187
- readonly value?: JsonValue
188
- } {
147
+ function isSyncCheckpoint(value: JsonValue): value is SyncCheckpointEnvelope {
189
148
  return isRecord(value) && value.kind === SYNC_CHECKPOINT_KIND
190
149
  }
191
150
 
192
- function isConnectionCheckpoint(value: JsonValue): value is JsonValue & {
193
- readonly entries: JsonValue[]
194
- } {
195
- return isRecord(value) && Array.isArray(value.entries)
196
- }
197
-
198
151
  function isRecord(value: unknown): value is Record<string, unknown> {
199
152
  return typeof value === "object" && value !== null && !Array.isArray(value)
200
153
  }
@@ -16,7 +16,12 @@ import { resolveLoggingService } from "@sixb/core/internal/logging"
16
16
  import { type DatasetVersion, LakeConcurrencyError } from "@sixb/core/lake-storage"
17
17
  import { SYNC_RUN_FAILURE_CODES, type SyncRunRecord } from "@sixb/core/storage"
18
18
  import { throwIfAborted } from "./normalize"
19
- import { readSyncValues, type SyncSourceValue } from "./source-read"
19
+ import {
20
+ readSyncValues,
21
+ resolveSyncRunSource,
22
+ SyncRunSourceGoneError,
23
+ type SyncSourceValue,
24
+ } from "./source-read"
20
25
  import type { RunSyncJobInput, SyncRunFinishedHandler, SyncRunResult } from "./types"
21
26
 
22
27
  /** What the execution behind a commit read, which the run records once it finds that commit. */
@@ -90,7 +95,9 @@ function translateSyncExecutionError(
90
95
  ): unknown {
91
96
  if (
92
97
  isSixbError(error) &&
93
- (error.code === "internal.unexpected" || error.code === "sync.execution_failed")
98
+ (error.code === "internal.unexpected" ||
99
+ error.code === "sync.execution_failed" ||
100
+ error.code === "connector.authorization_required")
94
101
  ) {
95
102
  return error
96
103
  }
@@ -214,7 +221,8 @@ async function commitRun(execution: SyncExecution, logger: Logger): Promise<Sync
214
221
  const sync = requireSync(runtime, run)
215
222
  const dataset = requireDataset(runtime, sync, run)
216
223
 
217
- const previousCheckpoint = await readPreviousCheckpoint(runtime, sync)
224
+ const source = await resolveSyncRunSource(runtime, sync, run)
225
+ const previousCheckpoint = await readPreviousCheckpoint(runtime, sync, run)
218
226
  execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint)
219
227
 
220
228
  const { outcome, version } = await writeDataset({
@@ -228,6 +236,7 @@ async function commitRun(execution: SyncExecution, logger: Logger): Promise<Sync
228
236
  readSyncValues({
229
237
  runtime,
230
238
  sync,
239
+ source,
231
240
  runId: run.id,
232
241
  datasetId: dataset.id,
233
242
  signal,
@@ -375,6 +384,11 @@ async function settleFailure(execution: SyncExecution, caught: unknown): Promise
375
384
  const landed = committedVersion ? null : await landedCommitBehind(execution, caught)
376
385
  if (landed) return finishLanded(execution, landed, landedRead(execution, caught))
377
386
 
387
+ if (caught instanceof SyncRunSourceGoneError && !signal.aborted) {
388
+ await recordCancelled(execution, caught)
389
+ throw caught
390
+ }
391
+
378
392
  const error = translateLostRace(run, caught)
379
393
 
380
394
  if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
@@ -430,23 +444,61 @@ async function recordFailed(execution: SyncExecution, error: unknown): Promise<v
430
444
  }
431
445
  }
432
446
 
447
+ /** The run had nothing left to read: nothing failed, so nothing is reported. */
448
+ async function recordCancelled(execution: SyncExecution, reason: Error): Promise<void> {
449
+ const { run } = execution
450
+ const { runtime, claim, onRunFinished } = execution.input
451
+
452
+ try {
453
+ const cancelled = await runtime.syncRunsStorage.finish({
454
+ projectId: runtime.id,
455
+ id: run.id,
456
+ executionToken: claim.execution.token,
457
+ status: "cancelled",
458
+ rowsRead: 0,
459
+ error: captureSixbFailure(createSixbError("runtime.cancelled", reason.message), {
460
+ allowedCodes: SYNC_RUN_FAILURE_CODES,
461
+ defaultCode: "runtime.cancelled",
462
+ }),
463
+ })
464
+
465
+ await notifyRunFinished(onRunFinished, cancelled)
466
+ } catch {
467
+ // The run did not transition: a later delivery, or the execution that holds it, ends it.
468
+ }
469
+ }
470
+
433
471
  function cloneCheckpoint(checkpoint: JsonValue | undefined): JsonValue | undefined {
434
472
  return checkpoint === undefined ? undefined : cloneJsonValue(checkpoint)
435
473
  }
436
474
 
475
+ /**
476
+ * The checkpoint of the last succeeded run of this sync, and of this run's connection. A
477
+ * connection's first run of its own starts from the last succeeded run that read every connection,
478
+ * whose checkpoint holds an entry for each.
479
+ */
437
480
  async function readPreviousCheckpoint(
438
481
  runtime: RunSyncJobInput["runtime"],
439
- sync: SyncDefinition
482
+ sync: SyncDefinition,
483
+ run: SyncRunRecord
440
484
  ): Promise<JsonValue | undefined> {
441
- const { runs } = await runtime.syncRunsStorage.list({
442
- projectId: runtime.id,
443
- syncId: sync.id,
444
- statuses: ["succeeded"],
445
- limit: 1,
446
- order: "desc",
447
- })
448
-
449
- return runs[0]?.checkpoint
485
+ const latest = (connectionId?: string | null) =>
486
+ runtime.syncRunsStorage
487
+ .list({
488
+ projectId: runtime.id,
489
+ syncId: sync.id,
490
+ ...(connectionId === undefined ? {} : { connectionId }),
491
+ statuses: ["succeeded"],
492
+ limit: 1,
493
+ order: "desc",
494
+ })
495
+ .then(({ runs }) => runs[0])
496
+
497
+ if (run.connectionId === undefined) return (await latest())?.checkpoint
498
+
499
+ const own = await latest(run.connectionId)
500
+ if (own) return own.checkpoint
501
+ return (await latest(null))?.checkpoint
450
502
  }
451
503
 
452
504
  function requireDataset(