@sixb/sync-worker 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  isFileRef
12
12
  } from "@sixb/core";
13
13
  import { resolveLogsRuntime } from "@sixb/core/internal/logging";
14
+ import { getDatasetMergeChangeValidationError } from "@sixb/core/lake-storage";
14
15
 
15
16
  // src/normalize.ts
16
17
  function isPlainObject(value) {
@@ -122,6 +123,66 @@ async function verifyRowFileRefs(options) {
122
123
  });
123
124
  }
124
125
  }
126
+ function assertMergeChange(value, syncId, dataset, itemIndex) {
127
+ const validationError = getDatasetMergeChangeValidationError(value, dataset);
128
+ if (validationError) {
129
+ throw new Error(`[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`);
130
+ }
131
+ return value;
132
+ }
133
+ async function readSyncValues(options) {
134
+ const client = await options.runtime.connector(options.sync.connector);
135
+ const readResult = await options.sync.read(client, {
136
+ projectId: options.runtime.id,
137
+ syncId: options.sync.id,
138
+ signal: options.signal,
139
+ blobs: options.blobStorage,
140
+ logger: options.logger,
141
+ checkpoint: options.previousCheckpoint !== undefined ? cloneJsonValue(options.previousCheckpoint) : undefined,
142
+ setCheckpoint: options.setCheckpoint
143
+ });
144
+ return normalizeReadResult(readResult, options.sync.id);
145
+ }
146
+ async function* validatedDatasetRows(options) {
147
+ let itemIndex = 0;
148
+ for await (const value of options.values) {
149
+ throwIfAborted(options.signal);
150
+ itemIndex += 1;
151
+ const row = assertDatasetRow(value, options.syncId, itemIndex);
152
+ const validationError = getDatasetRowValidationError(row, options.dataset);
153
+ if (validationError) {
154
+ throw new Error(`[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`);
155
+ }
156
+ await verifyRowFileRefs({
157
+ blobStorage: options.blobStorage,
158
+ syncId: options.syncId,
159
+ dataset: options.dataset,
160
+ row,
161
+ itemIndex
162
+ });
163
+ options.onRead();
164
+ yield row;
165
+ }
166
+ }
167
+ async function* validatedMergeChanges(options) {
168
+ let itemIndex = 0;
169
+ for await (const value of options.values) {
170
+ throwIfAborted(options.signal);
171
+ itemIndex += 1;
172
+ const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex);
173
+ if (mergeChange.kind === "upsert") {
174
+ await verifyRowFileRefs({
175
+ blobStorage: options.blobStorage,
176
+ syncId: options.syncId,
177
+ dataset: options.dataset,
178
+ row: mergeChange.row,
179
+ itemIndex
180
+ });
181
+ }
182
+ options.onRead();
183
+ yield mergeChange;
184
+ }
185
+ }
125
186
  async function runSyncJob(input) {
126
187
  const { runtime, job } = input;
127
188
  const signal = input.signal ?? new AbortController().signal;
@@ -159,7 +220,7 @@ async function runSyncJob(input) {
159
220
  id: job.id
160
221
  });
161
222
  const logger = logSession.logger;
162
- let write;
223
+ let abortWrite;
163
224
  let rowsRead = 0;
164
225
  let committedVersion;
165
226
  try {
@@ -174,82 +235,110 @@ async function runSyncJob(input) {
174
235
  });
175
236
  const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint;
176
237
  let nextCheckpoint = previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined;
177
- const client = await runtime.connector(sync.connector);
178
- const readResult = await sync.read(client, {
179
- projectId: runtime.id,
180
- syncId: sync.id,
238
+ const producer = {
239
+ kind: "sync",
240
+ id: sync.id,
241
+ runId: job.id
242
+ };
243
+ const readValues = () => readSyncValues({
244
+ runtime,
245
+ sync,
181
246
  signal,
182
- blobs: blobStorage,
247
+ blobStorage,
183
248
  logger,
184
- checkpoint: previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined,
249
+ previousCheckpoint,
185
250
  setCheckpoint(next) {
186
251
  assertJsonValue(next, `Sync '${sync.id}' checkpoint`);
187
252
  nextCheckpoint = cloneJsonValue(next);
188
253
  }
189
254
  });
190
- const rows = normalizeReadResult(readResult, sync.id);
191
- throwIfAborted(signal);
192
- write = await lakeStorage.beginWrite({
193
- dataset,
194
- mode: sync.config.mode,
195
- producer: {
196
- kind: "sync",
197
- id: sync.id,
198
- runId: job.id
199
- }
200
- });
201
- await write.writeRows(async function* () {
202
- let itemIndex = 0;
203
- for await (const value of rows) {
204
- throwIfAborted(signal);
205
- itemIndex += 1;
206
- const row = assertDatasetRow(value, sync.id, itemIndex);
207
- const validationError = getDatasetRowValidationError(row, dataset);
208
- if (validationError) {
209
- throw new Error(`[SixbSyncWorker] Sync '${sync.id}' returned an invalid row at item ${itemIndex}. ${validationError}`);
210
- }
211
- await verifyRowFileRefs({
212
- blobStorage,
213
- syncId: sync.id,
214
- dataset,
215
- row,
216
- itemIndex
217
- });
218
- rowsRead += 1;
219
- yield row;
220
- }
221
- }());
222
- throwIfAborted(signal);
223
- if (sync.config.mode === "append" && rowsRead === 0) {
224
- await write.abort();
225
- write = undefined;
226
- const version2 = await lakeStorage.getLatestVersion(dataset.id);
227
- const finishedRun2 = await syncRunsStorage.finish({
228
- projectId: runtime.id,
229
- id: job.id,
230
- status: "succeeded",
231
- rowsRead,
232
- ...version2 ? { output: { datasetId: version2.datasetId, versionId: version2.versionId } } : {},
233
- checkpoint: nextCheckpoint
255
+ const onRead = () => {
256
+ rowsRead += 1;
257
+ };
258
+ const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`;
259
+ let commitResult;
260
+ if (sync.config.mode === "merge") {
261
+ const mergeWrite = await lakeStorage.beginMerge({
262
+ dataset,
263
+ expectedLatestVersionId: job.expectedLatestVersionId,
264
+ producer
234
265
  });
235
- return {
236
- id: job.id,
266
+ abortWrite = () => mergeWrite.abort();
267
+ const values = await readValues();
268
+ throwIfAborted(signal);
269
+ await mergeWrite.writeChanges(validatedMergeChanges({
270
+ values,
271
+ signal,
272
+ blobStorage,
237
273
  syncId: sync.id,
238
- datasetId: dataset.id,
239
- mode: sync.config.mode,
240
- startedAt: startedRun.startedAt,
241
- finishedAt: requireFinishedAt(job.id, finishedRun2.finishedAt),
242
- rowsRead,
243
- ...version2 ? { version: version2 } : {},
244
- versionCreated: false
274
+ dataset,
275
+ onRead
276
+ }));
277
+ throwIfAborted(signal);
278
+ const commit = await mergeWrite.commit({ commitMessage });
279
+ abortWrite = undefined;
280
+ commitResult = {
281
+ outcome: commit.outcome,
282
+ ...commit.version ? { version: commit.version } : {}
245
283
  };
284
+ } else {
285
+ const values = await readValues();
286
+ throwIfAborted(signal);
287
+ const rowWrite = await lakeStorage.beginWrite({
288
+ dataset,
289
+ mode: sync.config.mode,
290
+ producer
291
+ });
292
+ abortWrite = () => rowWrite.abort();
293
+ await rowWrite.writeRows(validatedDatasetRows({
294
+ values,
295
+ signal,
296
+ blobStorage,
297
+ syncId: sync.id,
298
+ dataset,
299
+ onRead
300
+ }));
301
+ throwIfAborted(signal);
302
+ if (rowsRead === 0) {
303
+ const version3 = await lakeStorage.getLatestVersion(dataset.id);
304
+ if (sync.config.mode === "append" || !version3) {
305
+ await rowWrite.abort();
306
+ abortWrite = undefined;
307
+ const finishedRun2 = await syncRunsStorage.finish({
308
+ projectId: runtime.id,
309
+ id: job.id,
310
+ status: "succeeded",
311
+ rowsRead,
312
+ ...version3 ? { output: { datasetId: version3.datasetId, versionId: version3.versionId } } : {},
313
+ checkpoint: nextCheckpoint
314
+ });
315
+ return {
316
+ id: job.id,
317
+ syncId: sync.id,
318
+ datasetId: dataset.id,
319
+ mode: sync.config.mode,
320
+ startedAt: startedRun.startedAt,
321
+ finishedAt: requireFinishedAt(job.id, finishedRun2.finishedAt),
322
+ rowsRead,
323
+ ...version3 ? { version: version3 } : {},
324
+ versionCreated: false
325
+ };
326
+ }
327
+ }
328
+ const { outcome: outcome2, ...version2 } = await rowWrite.commit({
329
+ expectedLatestVersionId: job.expectedLatestVersionId,
330
+ commitMessage
331
+ });
332
+ abortWrite = undefined;
333
+ commitResult = { outcome: outcome2, version: version2 };
334
+ }
335
+ const { outcome, version } = commitResult;
336
+ if (outcome === "created") {
337
+ if (!version) {
338
+ throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`);
339
+ }
340
+ committedVersion = version;
246
341
  }
247
- const commit = await write.commit({
248
- expectedLatestVersionId: job.expectedLatestVersionId,
249
- commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`
250
- });
251
- const { outcome, ...version } = commit;
252
- committedVersion = version;
253
342
  let finishedRun;
254
343
  try {
255
344
  finishedRun = await syncRunsStorage.finish({
@@ -257,34 +346,33 @@ async function runSyncJob(input) {
257
346
  id: job.id,
258
347
  status: "succeeded",
259
348
  rowsRead,
260
- output: {
261
- datasetId: version.datasetId,
262
- versionId: version.versionId
263
- },
349
+ ...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
264
350
  checkpoint: nextCheckpoint
265
351
  });
266
352
  } catch (error) {
267
- throw createBookkeepingError({
268
- syncId: sync.id,
269
- runId: job.id,
270
- version,
271
- cause: error
272
- });
353
+ if (committedVersion) {
354
+ throw createBookkeepingError({
355
+ syncId: sync.id,
356
+ runId: job.id,
357
+ version: committedVersion,
358
+ cause: error
359
+ });
360
+ }
361
+ throw error;
273
362
  }
274
- return {
363
+ const result = {
275
364
  id: job.id,
276
365
  syncId: sync.id,
277
366
  datasetId: dataset.id,
278
367
  mode: sync.config.mode,
279
368
  startedAt: startedRun.startedAt,
280
369
  finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
281
- rowsRead,
282
- version,
283
- versionCreated: outcome === "created"
370
+ rowsRead
284
371
  };
372
+ return version ? { ...result, version, versionCreated: outcome === "created" } : { ...result, versionCreated: false };
285
373
  } catch (error) {
286
374
  if (!committedVersion) {
287
- await write?.abort().catch(() => {});
375
+ await abortWrite?.().catch(() => {});
288
376
  const status = signal.aborted ? "cancelled" : "failed";
289
377
  try {
290
378
  const run = await syncRunsStorage.finish({
@@ -451,4 +539,4 @@ export {
451
539
  SyncWorker
452
540
  };
453
541
 
454
- //# debugId=33375B9F4568F6E364756E2164756E21
542
+ //# debugId=3B01117F7588DCC564756E2164756E21
package/dist/index.js.map CHANGED
@@ -3,10 +3,10 @@
3
3
  "sources": ["../src/worker.ts", "../src/run-sync-job.ts", "../src/normalize.ts"],
4
4
  "sourcesContent": [
5
5
  "import type {\n BlobStorage,\n ConnectorAdapter,\n ConnectorClient,\n ConnectorDefinition,\n DatasetDefinition,\n DomainEventLog,\n LakeStorage,\n Queues,\n Storage,\n SyncDefinition,\n} from \"@sixb/core\"\nimport { reportRunFailure } from \"@sixb/core/internal/error-reporting\"\nimport type { LogsRuntime } from \"@sixb/core/internal/logging\"\nimport type { QueueWorkerFailureDecision } from \"@sixb/core/internal/workers\"\nimport { QueueWorker } from \"@sixb/core/internal/workers\"\nimport type { ClaimedQueueJob, SyncRunRequestedQueueJob } from \"@sixb/core/queues\"\nimport type { SyncRunRecord } from \"@sixb/core/storage\"\nimport { runSyncJob, SyncRunAlreadyStartedError } from \"./run-sync-job\"\nimport type { SyncJob, SyncRunResult, SyncWorkerContext } from \"./types\"\n\nconst SOURCE = \"SixbSyncWorker\"\n\nexport interface SyncWorkerSixb {\n readonly id: string\n readonly events?: DomainEventLog\n readonly logs?: LogsRuntime\n readonly lakeStorage: LakeStorage\n readonly blobStorage: BlobStorage\n readonly queues: Queues\n readonly storage: Storage\n listSyncs(): readonly SyncDefinition[]\n getSyncById(syncId: string): SyncDefinition | null\n getDatasetById(datasetId: string): DatasetDefinition | null\n connector<TAdapter extends ConnectorAdapter>(\n definition: ConnectorDefinition<string, TAdapter>\n ): Promise<ConnectorClient<TAdapter>>\n}\n\nexport class SyncWorker extends QueueWorker<SyncRunRequestedQueueJob> {\n private readonly context: SyncWorkerContext\n private readonly sixb: SyncWorkerSixb\n\n constructor(sixb: SyncWorkerSixb) {\n if (sixb.listSyncs().length === 0) {\n throw new Error(\"[SixbSyncWorker] No sync definitions are registered.\")\n }\n\n super({\n projectId: sixb.id,\n queue: sixb.queues.syncRuns,\n workerId: `sync-worker-${sixb.id}`,\n })\n\n this.context = buildSyncContext(sixb)\n this.sixb = sixb\n }\n\n protected async execute(\n claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,\n signal: AbortSignal\n ): Promise<void> {\n const { job } = claimed\n const syncJob: SyncJob = {\n id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n syncId: job.payload.syncId,\n expectedLatestVersionId: job.payload.expectedLatestVersionId,\n commitMessage: job.payload.commitMessage,\n }\n\n let result: SyncRunResult\n try {\n result = await runSyncJob({\n runtime: this.context,\n job: syncJob,\n signal,\n onRunStarted: (run) => emitSyncRunStarted(this.sixb, run),\n onRunFailed: (error, run) => {\n reportRunFailure(this.sixb, error, {\n projectId: this.sixb.id,\n occurredAt: run.finishedAt,\n attempt: job.attempt,\n run: {\n kind: \"sync\",\n runId: run.id,\n syncId: run.syncId,\n },\n })\n },\n })\n } catch (error) {\n if (error instanceof SyncRunAlreadyStartedError) return\n throw error\n }\n\n await emitSyncSucceededEvents(this.sixb, syncJob, result)\n }\n\n protected override async onExecutionError(\n claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,\n _error: unknown\n ): Promise<QueueWorkerFailureDecision> {\n const { job } = claimed\n await emitSyncRunFinished(this.sixb, {\n id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n syncId: job.payload.syncId,\n })\n\n return { kind: \"fail\" }\n }\n}\n\nasync function emitSyncRunStarted(\n sixb: SyncWorkerSixb,\n run: Pick<SyncRunRecord, \"id\" | \"syncId\" | \"startedAt\">\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n {\n type: \"sync.run.started\",\n payload: {\n syncId: run.syncId,\n runId: run.id,\n startedAt: run.startedAt.toISOString(),\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nasync function emitSyncSucceededEvents(\n sixb: SyncWorkerSixb,\n job: Pick<SyncJob, \"id\" | \"syncId\">,\n result: SyncRunResult\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n ...(result.versionCreated\n ? [\n {\n type: \"dataset.version.committed\" as const,\n payload: {\n datasetId: result.datasetId,\n versionId: result.version.versionId,\n createdAt: result.version.createdAt.toISOString(),\n producer: {\n kind: \"sync\" as const,\n id: job.syncId,\n runId: job.id,\n },\n },\n },\n ]\n : []),\n {\n type: \"sync.run.finished\",\n payload: {\n syncId: job.syncId,\n runId: job.id,\n status: \"succeeded\",\n datasetId: result.datasetId,\n ...(result.version ? { versionId: result.version.versionId } : {}),\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nasync function emitSyncRunFinished(\n sixb: SyncWorkerSixb,\n job: Pick<SyncJob, \"id\" | \"syncId\">\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n {\n type: \"sync.run.finished\",\n payload: {\n syncId: job.syncId,\n runId: job.id,\n status: \"failed\",\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nfunction buildSyncContext(sixb: SyncWorkerSixb): SyncWorkerContext {\n const syncRunsStorage = sixb.storage.syncRuns\n if (!syncRunsStorage) {\n throw new Error(\"[SixbSyncWorker] Sync workers require storage.syncRuns support.\")\n }\n\n return {\n id: sixb.id,\n syncRunsStorage,\n lakeStorage: sixb.lakeStorage,\n blobStorage: sixb.blobStorage,\n logs: sixb.logs,\n getSyncById(syncId) {\n return sixb.getSyncById(syncId)\n },\n getDatasetById(datasetId) {\n return sixb.getDatasetById(datasetId)\n },\n connector(definition) {\n return sixb.connector(definition)\n },\n }\n}\n",
6
- "import {\n assertJsonValue,\n type BlobStorage,\n cloneJsonValue,\n type FileRef,\n getDatasetRowValidationError,\n isFileRef,\n type JsonValue,\n} from \"@sixb/core\"\nimport { resolveLogsRuntime } from \"@sixb/core/internal/logging\"\nimport type { DatasetVersion, LakeWriteSession } from \"@sixb/core/lake-storage\"\nimport type { SyncRunFailure, SyncRunRecord } from \"@sixb/core/storage\"\nimport { assertDatasetRow, normalizeReadResult, throwIfAborted } from \"./normalize\"\nimport type { RunSyncJobInput, SyncRunResult } from \"./types\"\n\nexport class SyncRunAlreadyStartedError extends Error {\n override readonly name = \"SyncRunAlreadyStartedError\"\n\n constructor(readonly run: SyncRunRecord) {\n super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`)\n }\n}\n\nfunction toSyncRunFailure(error: unknown): SyncRunFailure {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n }\n }\n\n return {\n message: String(error),\n }\n}\n\nfunction createBookkeepingError(options: {\n syncId: string\n runId: string\n version: DatasetVersion\n cause: unknown\n}): Error {\n // The lake commit is already durable here, so we surface an explicit repair-needed error.\n return new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`,\n { cause: options.cause }\n )\n}\n\nfunction requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {\n if (finishedAt) {\n return finishedAt\n }\n\n throw new Error(`[SixbSyncWorker] Sync run '${runId}' finished without a finishedAt timestamp.`)\n}\n\nasync function verifyFileRef(options: {\n blobStorage: BlobStorage\n syncId: string\n datasetId: string\n columnName: string\n itemIndex: number\n fileRef: FileRef\n}): Promise<void> {\n const blobInfo = await options.blobStorage.stat(options.fileRef.blobId)\n if (!blobInfo) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing unknown blob '${options.fileRef.blobId}'.`\n )\n }\n\n if (blobInfo.digest !== options.fileRef.digest) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with digest '${options.fileRef.digest}', but blob storage has '${blobInfo.digest}'.`\n )\n }\n\n if (blobInfo.sizeBytes !== options.fileRef.sizeBytes) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with size ${options.fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`\n )\n }\n}\n\nasync function verifyRowFileRefs(options: {\n blobStorage: BlobStorage\n syncId: string\n dataset: { id: string; schema: { columns: readonly { name: string; type: string }[] } }\n row: Record<string, unknown>\n itemIndex: number\n}): Promise<void> {\n const fileRefColumns = options.dataset.schema.columns.filter(\n (column) => column.type === \"fileRef\"\n )\n for (const column of fileRefColumns) {\n const value = options.row[column.name]\n if (!isFileRef(value)) {\n continue\n }\n\n await verifyFileRef({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n datasetId: options.dataset.id,\n columnName: column.name,\n itemIndex: options.itemIndex,\n fileRef: value,\n })\n }\n}\n\n/**\n * Run one sync job end to end using a caller-supplied run id.\n *\n * The worker creates the operational sync-run record, streams rows into lake\n * storage, commits one dataset version, and then finalizes the run record with\n * the outcome. It does not own scheduling, retries, or run-id generation.\n */\nexport async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {\n const { runtime, job } = input\n const signal = input.signal ?? new AbortController().signal\n const { syncRunsStorage, lakeStorage, blobStorage } = runtime\n\n const sync = runtime.getSyncById(job.syncId)\n if (!sync) {\n throw new Error(`[SixbSyncWorker] Unknown sync '${job.syncId}'.`)\n }\n\n const dataset = runtime.getDatasetById(sync.target.dataset.id)\n if (!dataset) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${sync.target.dataset.id}'.`\n )\n }\n\n throwIfAborted(signal)\n\n let startedRun: SyncRunRecord\n try {\n startedRun = await syncRunsStorage.start({\n projectId: runtime.id,\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage,\n })\n } catch (error) {\n const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })\n if (existing?.syncId === sync.id && existing.datasetId === dataset.id) {\n throw new SyncRunAlreadyStartedError(existing)\n }\n throw error\n }\n await input.onRunStarted?.(startedRun)\n\n const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({\n kind: \"sync\",\n id: job.id,\n })\n const logger = logSession.logger\n let write: LakeWriteSession | undefined\n let rowsRead = 0\n let committedVersion: DatasetVersion | undefined\n\n try {\n throwIfAborted(signal)\n await lakeStorage.createDataset(dataset)\n\n const latestSuccessfulRuns = await syncRunsStorage.list({\n projectId: runtime.id,\n syncId: sync.id,\n statuses: [\"succeeded\"],\n limit: 1,\n order: \"desc\",\n })\n const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint\n let nextCheckpoint: JsonValue | undefined =\n previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined\n\n const client = await runtime.connector(sync.connector)\n const readResult = await sync.read(client, {\n projectId: runtime.id,\n syncId: sync.id,\n signal,\n blobs: blobStorage,\n logger,\n checkpoint: previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined,\n setCheckpoint(next: unknown) {\n assertJsonValue(next, `Sync '${sync.id}' checkpoint`)\n nextCheckpoint = cloneJsonValue(next)\n },\n })\n const rows = normalizeReadResult(readResult, sync.id)\n\n throwIfAborted(signal)\n\n write = await lakeStorage.beginWrite({\n dataset,\n mode: sync.config.mode,\n producer: {\n kind: \"sync\",\n id: sync.id,\n runId: job.id,\n },\n })\n\n await write.writeRows(\n (async function* () {\n let itemIndex = 0\n\n for await (const value of rows) {\n throwIfAborted(signal)\n itemIndex += 1\n\n const row = assertDatasetRow(value, sync.id, itemIndex)\n // Validate before handing rows to lake storage so sync failures include\n // the source item index; lake storage still re-validates as the final boundary.\n const validationError = getDatasetRowValidationError(row, dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' returned an invalid row at item ${itemIndex}. ${validationError}`\n )\n }\n\n // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.\n await verifyRowFileRefs({\n blobStorage,\n syncId: sync.id,\n dataset,\n row,\n itemIndex,\n })\n\n rowsRead += 1\n yield row\n }\n })()\n )\n\n throwIfAborted(signal)\n\n if (sync.config.mode === \"append\" && rowsRead === 0) {\n await write.abort()\n write = undefined\n const version = await lakeStorage.getLatestVersion(dataset.id)\n const finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n ...(version ? { version } : {}),\n versionCreated: false,\n }\n }\n\n const commit = await write.commit({\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,\n })\n const { outcome, ...version } = commit\n committedVersion = version\n let finishedRun: SyncRunRecord\n\n try {\n finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n output: {\n datasetId: version.datasetId,\n versionId: version.versionId,\n },\n checkpoint: nextCheckpoint,\n })\n } catch (error) {\n throw createBookkeepingError({\n syncId: sync.id,\n runId: job.id,\n version,\n cause: error,\n })\n }\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n version,\n versionCreated: outcome === \"created\",\n }\n } catch (error) {\n if (!committedVersion) {\n // Before commit succeeds we can still best-effort clean up both the write session and run record.\n await write?.abort().catch(() => {})\n\n const status = signal.aborted ? \"cancelled\" : \"failed\"\n try {\n const run = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status,\n rowsRead,\n error: toSyncRunFailure(error),\n })\n if (status === \"failed\" && run.status === \"failed\") input.onRunFailed?.(error, run)\n } catch {\n // The run did not transition to the requested terminal status.\n }\n }\n\n throw error\n } finally {\n await logSession.flush()\n }\n}\n",
6
+ "import {\n assertJsonValue,\n type BlobStorage,\n cloneJsonValue,\n type DatasetDefinition,\n type DatasetRow,\n type FileRef,\n getDatasetRowValidationError,\n isFileRef,\n type JsonValue,\n type Logger,\n type MergeChange,\n type SyncDefinition,\n} from \"@sixb/core\"\nimport { resolveLogsRuntime } from \"@sixb/core/internal/logging\"\nimport { type DatasetVersion, getDatasetMergeChangeValidationError } from \"@sixb/core/lake-storage\"\nimport type { SyncRunFailure, SyncRunRecord } from \"@sixb/core/storage\"\nimport { assertDatasetRow, normalizeReadResult, throwIfAborted } from \"./normalize\"\nimport type { RunSyncJobInput, SyncRunResult, SyncWorkerContext } from \"./types\"\n\nexport class SyncRunAlreadyStartedError extends Error {\n override readonly name = \"SyncRunAlreadyStartedError\"\n\n constructor(readonly run: SyncRunRecord) {\n super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`)\n }\n}\n\nfunction toSyncRunFailure(error: unknown): SyncRunFailure {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n }\n }\n\n return {\n message: String(error),\n }\n}\n\nfunction createBookkeepingError(options: {\n syncId: string\n runId: string\n version: DatasetVersion\n cause: unknown\n}): Error {\n // The lake commit is already durable here, so we surface an explicit repair-needed error.\n return new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`,\n { cause: options.cause }\n )\n}\n\nfunction requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {\n if (finishedAt) {\n return finishedAt\n }\n\n throw new Error(`[SixbSyncWorker] Sync run '${runId}' finished without a finishedAt timestamp.`)\n}\n\nasync function verifyFileRef(options: {\n blobStorage: BlobStorage\n syncId: string\n datasetId: string\n columnName: string\n itemIndex: number\n fileRef: FileRef\n}): Promise<void> {\n const blobInfo = await options.blobStorage.stat(options.fileRef.blobId)\n if (!blobInfo) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing unknown blob '${options.fileRef.blobId}'.`\n )\n }\n\n if (blobInfo.digest !== options.fileRef.digest) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with digest '${options.fileRef.digest}', but blob storage has '${blobInfo.digest}'.`\n )\n }\n\n if (blobInfo.sizeBytes !== options.fileRef.sizeBytes) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with size ${options.fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`\n )\n }\n}\n\nasync function verifyRowFileRefs(options: {\n blobStorage: BlobStorage\n syncId: string\n dataset: { id: string; schema: { columns: readonly { name: string; type: string }[] } }\n row: Record<string, unknown>\n itemIndex: number\n}): Promise<void> {\n const fileRefColumns = options.dataset.schema.columns.filter(\n (column) => column.type === \"fileRef\"\n )\n for (const column of fileRefColumns) {\n const value = options.row[column.name]\n if (!isFileRef(value)) {\n continue\n }\n\n await verifyFileRef({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n datasetId: options.dataset.id,\n columnName: column.name,\n itemIndex: options.itemIndex,\n fileRef: value,\n })\n }\n}\n\nfunction assertMergeChange(\n value: unknown,\n syncId: string,\n dataset: DatasetDefinition,\n itemIndex: number\n): MergeChange<DatasetRow, DatasetRow> {\n const validationError = getDatasetMergeChangeValidationError(value, dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`\n )\n }\n return value as MergeChange<DatasetRow, DatasetRow>\n}\n\nasync function readSyncValues(options: {\n runtime: SyncWorkerContext\n sync: SyncDefinition\n signal: AbortSignal\n blobStorage: BlobStorage\n logger: Logger\n previousCheckpoint: JsonValue | undefined\n setCheckpoint(next: unknown): void\n}): Promise<AsyncIterable<unknown>> {\n const client = await options.runtime.connector(options.sync.connector)\n const readResult = await options.sync.read(client, {\n projectId: options.runtime.id,\n syncId: options.sync.id,\n signal: options.signal,\n blobs: options.blobStorage,\n logger: options.logger,\n checkpoint:\n options.previousCheckpoint !== undefined\n ? cloneJsonValue(options.previousCheckpoint)\n : undefined,\n setCheckpoint: options.setCheckpoint,\n })\n return normalizeReadResult(readResult, options.sync.id)\n}\n\nasync function* validatedDatasetRows(options: {\n values: AsyncIterable<unknown>\n signal: AbortSignal\n blobStorage: BlobStorage\n syncId: string\n dataset: DatasetDefinition\n onRead(): void\n}): AsyncIterable<DatasetRow> {\n let itemIndex = 0\n for await (const value of options.values) {\n throwIfAborted(options.signal)\n itemIndex += 1\n\n const row = assertDatasetRow(value, options.syncId, itemIndex)\n // Validate before handing rows to lake storage so sync failures include\n // the source item index; lake storage still re-validates as the final boundary.\n const validationError = getDatasetRowValidationError(row, options.dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`\n )\n }\n\n // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.\n await verifyRowFileRefs({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n dataset: options.dataset,\n row,\n itemIndex,\n })\n\n options.onRead()\n yield row\n }\n}\n\nasync function* validatedMergeChanges(options: {\n values: AsyncIterable<unknown>\n signal: AbortSignal\n blobStorage: BlobStorage\n syncId: string\n dataset: DatasetDefinition\n onRead(): void\n}): AsyncIterable<MergeChange<DatasetRow, DatasetRow>> {\n let itemIndex = 0\n for await (const value of options.values) {\n throwIfAborted(options.signal)\n itemIndex += 1\n const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex)\n if (mergeChange.kind === \"upsert\") {\n await verifyRowFileRefs({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n dataset: options.dataset,\n row: mergeChange.row,\n itemIndex,\n })\n }\n options.onRead()\n yield mergeChange\n }\n}\n\ninterface SyncCommitResult {\n readonly outcome: \"created\" | \"unchanged\"\n readonly version?: DatasetVersion\n}\n\n/**\n * Run one sync job end to end using a caller-supplied run id.\n *\n * The worker creates the operational sync-run record, streams source values into lake storage,\n * commits or reuses a dataset version when one exists, and then finalizes the run record. It does\n * not own scheduling, retries, or run-id generation.\n */\nexport async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {\n const { runtime, job } = input\n const signal = input.signal ?? new AbortController().signal\n const { syncRunsStorage, lakeStorage, blobStorage } = runtime\n\n const sync = runtime.getSyncById(job.syncId)\n if (!sync) {\n throw new Error(`[SixbSyncWorker] Unknown sync '${job.syncId}'.`)\n }\n\n const dataset = runtime.getDatasetById(sync.target.dataset.id)\n if (!dataset) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${sync.target.dataset.id}'.`\n )\n }\n\n throwIfAborted(signal)\n\n let startedRun: SyncRunRecord\n try {\n startedRun = await syncRunsStorage.start({\n projectId: runtime.id,\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage,\n })\n } catch (error) {\n const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })\n if (existing?.syncId === sync.id && existing.datasetId === dataset.id) {\n throw new SyncRunAlreadyStartedError(existing)\n }\n throw error\n }\n await input.onRunStarted?.(startedRun)\n\n const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({\n kind: \"sync\",\n id: job.id,\n })\n const logger = logSession.logger\n let abortWrite: (() => Promise<void>) | undefined\n let rowsRead = 0\n let committedVersion: DatasetVersion | undefined\n\n try {\n throwIfAborted(signal)\n await lakeStorage.createDataset(dataset)\n\n const latestSuccessfulRuns = await syncRunsStorage.list({\n projectId: runtime.id,\n syncId: sync.id,\n statuses: [\"succeeded\"],\n limit: 1,\n order: \"desc\",\n })\n const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint\n let nextCheckpoint: JsonValue | undefined =\n previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined\n\n const producer = {\n kind: \"sync\" as const,\n id: sync.id,\n runId: job.id,\n }\n const readValues = () =>\n readSyncValues({\n runtime,\n sync,\n signal,\n blobStorage,\n logger,\n previousCheckpoint,\n setCheckpoint(next) {\n assertJsonValue(next, `Sync '${sync.id}' checkpoint`)\n nextCheckpoint = cloneJsonValue(next)\n },\n })\n const onRead = () => {\n rowsRead += 1\n }\n const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`\n let commitResult: SyncCommitResult\n\n if (sync.config.mode === \"merge\") {\n const mergeWrite = await lakeStorage.beginMerge({\n dataset,\n expectedLatestVersionId: job.expectedLatestVersionId,\n producer,\n })\n abortWrite = () => mergeWrite.abort()\n\n const values = await readValues()\n throwIfAborted(signal)\n await mergeWrite.writeChanges(\n validatedMergeChanges({\n values,\n signal,\n blobStorage,\n syncId: sync.id,\n dataset,\n onRead,\n })\n )\n throwIfAborted(signal)\n\n const commit = await mergeWrite.commit({ commitMessage })\n abortWrite = undefined\n commitResult = {\n outcome: commit.outcome,\n ...(commit.version ? { version: commit.version } : {}),\n }\n } else {\n const values = await readValues()\n throwIfAborted(signal)\n\n const rowWrite = await lakeStorage.beginWrite({\n dataset,\n mode: sync.config.mode,\n producer,\n })\n abortWrite = () => rowWrite.abort()\n await rowWrite.writeRows(\n validatedDatasetRows({\n values,\n signal,\n blobStorage,\n syncId: sync.id,\n dataset,\n onRead,\n })\n )\n throwIfAborted(signal)\n\n if (rowsRead === 0) {\n const version = await lakeStorage.getLatestVersion(dataset.id)\n if (sync.config.mode === \"append\" || !version) {\n await rowWrite.abort()\n abortWrite = undefined\n const finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n ...(version ? { version } : {}),\n versionCreated: false,\n }\n }\n }\n\n const { outcome, ...version } = await rowWrite.commit({\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage,\n })\n abortWrite = undefined\n commitResult = { outcome, version }\n }\n\n const { outcome, version } = commitResult\n if (outcome === \"created\") {\n if (!version) {\n throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`)\n }\n committedVersion = version\n }\n let finishedRun: SyncRunRecord\n\n try {\n finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n } catch (error) {\n if (committedVersion) {\n throw createBookkeepingError({\n syncId: sync.id,\n runId: job.id,\n version: committedVersion,\n cause: error,\n })\n }\n throw error\n }\n\n const result = {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n }\n return version\n ? { ...result, version, versionCreated: outcome === \"created\" }\n : { ...result, versionCreated: false }\n } catch (error) {\n if (!committedVersion) {\n // Without a created commit, best-effort clean up any open session and the run record.\n await abortWrite?.().catch(() => {})\n\n const status = signal.aborted ? \"cancelled\" : \"failed\"\n try {\n const run = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status,\n rowsRead,\n error: toSyncRunFailure(error),\n })\n if (status === \"failed\" && run.status === \"failed\") input.onRunFailed?.(error, run)\n } catch {\n // The run did not transition to the requested terminal status.\n }\n }\n\n throw error\n } finally {\n await logSession.flush()\n }\n}\n",
7
7
  "import type { DatasetRow, SyncReadResult } from \"@sixb/core\"\n\nfunction isPlainObject(value: unknown): value is DatasetRow {\n if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n return false\n }\n\n const prototype = Object.getPrototypeOf(value)\n return prototype === Object.prototype || prototype === null\n}\n\nfunction isIterable(value: unknown): value is Iterable<unknown> {\n return (\n value !== null &&\n value !== undefined &&\n typeof (value as { [Symbol.iterator]?: unknown })[Symbol.iterator] === \"function\"\n )\n}\n\nfunction isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {\n return (\n value !== null &&\n value !== undefined &&\n typeof (value as { [Symbol.asyncIterator]?: unknown })[Symbol.asyncIterator] === \"function\"\n )\n}\n\nfunction toAbortError(signal: AbortSignal): Error {\n if (signal.reason instanceof Error) {\n return signal.reason\n }\n\n const message =\n typeof signal.reason === \"string\" && signal.reason.length > 0\n ? signal.reason\n : \"[SixbSyncWorker] Operation was cancelled.\"\n\n const error = new Error(message)\n error.name = \"AbortError\"\n return error\n}\n\nexport function throwIfAborted(signal: AbortSignal): void {\n if (signal.aborted) {\n throw toAbortError(signal)\n }\n}\n\nexport function assertDatasetRow(value: unknown, syncId: string, itemIndex: number): DatasetRow {\n if (isPlainObject(value)) {\n return value\n }\n\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an invalid row at item ${itemIndex}. Dataset rows must be plain objects.`\n )\n}\n\nexport async function* normalizeReadResult(\n readResult: SyncReadResult,\n syncId: string\n): AsyncIterable<unknown> {\n if (isAsyncIterable(readResult)) {\n yield* readResult\n return\n }\n\n if (isIterable(readResult)) {\n for (const item of readResult) {\n yield item\n }\n return\n }\n\n if (isPlainObject(readResult)) {\n yield readResult\n return\n }\n\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an unsupported read result. Expected a row object, iterable, or async iterable.`\n )\n}\n"
8
8
  ],
9
- "mappings": ";;AAYA;AAGA;;;ACfA;AAAA;AAAA;AAAA;AAAA;AAAA;AASA;;;ACPA,SAAS,aAAa,CAAC,OAAqC;AAAA,EAC1D,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AAAA,IACvE,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,OAAO,eAAe,KAAK;AAAA,EAC7C,OAAO,cAAc,OAAO,aAAa,cAAc;AAAA;AAGzD,SAAS,UAAU,CAAC,OAA4C;AAAA,EAC9D,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA0C,OAAO,cAAc;AAAA;AAI3E,SAAS,eAAe,CAAC,OAAiD;AAAA,EACxE,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA+C,OAAO,mBAAmB;AAAA;AAIrF,SAAS,YAAY,CAAC,QAA4B;AAAA,EAChD,IAAI,OAAO,kBAAkB,OAAO;AAAA,IAClC,OAAO,OAAO;AAAA,EAChB;AAAA,EAEA,MAAM,UACJ,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,SAAS,IACxD,OAAO,SACP;AAAA,EAEN,MAAM,QAAQ,IAAI,MAAM,OAAO;AAAA,EAC/B,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGF,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,aAAa,MAAM;AAAA,EAC3B;AAAA;AAGK,SAAS,gBAAgB,CAAC,OAAgB,QAAgB,WAA+B;AAAA,EAC9F,IAAI,cAAc,KAAK,GAAG;AAAA,IACxB,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,2CAA2C,gDACvE;AAAA;AAGF,gBAAuB,mBAAmB,CACxC,YACA,QACwB;AAAA,EACxB,IAAI,gBAAgB,UAAU,GAAG;AAAA,IAC/B,OAAO;AAAA,IACP;AAAA,EACF;AAAA,EAEA,IAAI,WAAW,UAAU,GAAG;AAAA,IAC1B,WAAW,QAAQ,YAAY;AAAA,MAC7B,MAAM;AAAA,IACR;AAAA,IACA;AAAA,EACF;AAAA,EAEA,IAAI,cAAc,UAAU,GAAG;AAAA,IAC7B,MAAM;AAAA,IACN;AAAA,EACF;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,kGAC5B;AAAA;;;ADlEK,MAAM,mCAAmC,MAAM;AAAA,EAG/B;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAoB;AAAA,IACvC,MAAM,8BAA8B,IAAI,0BAA0B;AAAA,IAD/C;AAAA;AAGvB;AAEA,SAAS,gBAAgB,CAAC,OAAgC;AAAA,EACxD,IAAI,iBAAiB,OAAO;AAAA,IAC1B,OAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,OAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,EACvB;AAAA;AAGF,SAAS,sBAAsB,CAAC,SAKtB;AAAA,EAER,OAAO,IAAI,MACT,0BAA0B,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,kGAChJ,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGF,SAAS,iBAAiB,CAAC,OAAe,YAAoC;AAAA,EAC5E,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,8BAA8B,iDAAiD;AAAA;AAGjG,eAAe,aAAa,CAAC,SAOX;AAAA,EAChB,MAAM,WAAW,MAAM,QAAQ,YAAY,KAAK,QAAQ,QAAQ,MAAM;AAAA,EACtE,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,yCAAyC,QAAQ,QAAQ,UAC9L;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,WAAW,QAAQ,QAAQ,QAAQ;AAAA,IAC9C,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,wBAAwB,QAAQ,QAAQ,kCAAkC,SAAS,UACzQ;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,cAAc,QAAQ,QAAQ,WAAW;AAAA,IACpD,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,qBAAqB,QAAQ,QAAQ,mCAAmC,SAAS,YACvQ;AAAA,EACF;AAAA;AAGF,eAAe,iBAAiB,CAAC,SAMf;AAAA,EAChB,MAAM,iBAAiB,QAAQ,QAAQ,OAAO,QAAQ,OACpD,CAAC,WAAW,OAAO,SAAS,SAC9B;AAAA,EACA,WAAW,UAAU,gBAAgB;AAAA,IACnC,MAAM,QAAQ,QAAQ,IAAI,OAAO;AAAA,IACjC,IAAI,CAAC,UAAU,KAAK,GAAG;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,cAAc;AAAA,MAClB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,YAAY,OAAO;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA;AAUF,eAAsB,UAAU,CAAC,OAAgD;AAAA,EAC/E,QAAQ,SAAS,QAAQ;AAAA,EACzB,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,QAAQ,iBAAiB,aAAa,gBAAgB;AAAA,EAEtD,MAAM,OAAO,QAAQ,YAAY,IAAI,MAAM;AAAA,EAC3C,IAAI,CAAC,MAAM;AAAA,IACT,MAAM,IAAI,MAAM,kCAAkC,IAAI,UAAU;AAAA,EAClE;AAAA,EAEA,MAAM,UAAU,QAAQ,eAAe,KAAK,OAAO,QAAQ,EAAE;AAAA,EAC7D,IAAI,CAAC,SAAS;AAAA,IACZ,MAAM,IAAI,MACR,0BAA0B,KAAK,gCAAgC,KAAK,OAAO,QAAQ,MACrF;AAAA,EACF;AAAA,EAEA,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,aAAa,MAAM,gBAAgB,MAAM;AAAA,MACvC,WAAW,QAAQ;AAAA,MACnB,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,MAAM,WAAW,MAAM,gBAAgB,QAAQ,EAAE,WAAW,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC;AAAA,IACpF,IAAI,UAAU,WAAW,KAAK,MAAM,SAAS,cAAc,QAAQ,IAAI;AAAA,MACrE,MAAM,IAAI,2BAA2B,QAAQ;AAAA,IAC/C;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,MAAM,eAAe,UAAU;AAAA,EAErC,MAAM,aAAa,mBAAmB,QAAQ,IAAI,QAAQ,IAAI,EAAE,eAAe;AAAA,IAC7E,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,SAAS,WAAW;AAAA,EAC1B,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EACf,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,eAAe,MAAM;AAAA,IACrB,MAAM,YAAY,cAAc,OAAO;AAAA,IAEvC,MAAM,uBAAuB,MAAM,gBAAgB,KAAK;AAAA,MACtD,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,UAAU,CAAC,WAAW;AAAA,MACtB,OAAO;AAAA,MACP,OAAO;AAAA,IACT,CAAC;AAAA,IACD,MAAM,qBAAqB,qBAAqB,KAAK,IAAI;AAAA,IACzD,IAAI,iBACF,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,IAE1E,MAAM,SAAS,MAAM,QAAQ,UAAU,KAAK,SAAS;AAAA,IACrD,MAAM,aAAa,MAAM,KAAK,KAAK,QAAQ;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA,YAAY,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,MACpF,aAAa,CAAC,MAAe;AAAA,QAC3B,gBAAgB,MAAM,SAAS,KAAK,gBAAgB;AAAA,QACpD,iBAAiB,eAAe,IAAI;AAAA;AAAA,IAExC,CAAC;AAAA,IACD,MAAM,OAAO,oBAAoB,YAAY,KAAK,EAAE;AAAA,IAEpD,eAAe,MAAM;AAAA,IAErB,QAAQ,MAAM,YAAY,WAAW;AAAA,MACnC;AAAA,MACA,MAAM,KAAK,OAAO;AAAA,MAClB,UAAU;AAAA,QACR,MAAM;AAAA,QACN,IAAI,KAAK;AAAA,QACT,OAAO,IAAI;AAAA,MACb;AAAA,IACF,CAAC;AAAA,IAED,MAAM,MAAM,UACT,gBAAgB,GAAG;AAAA,MAClB,IAAI,YAAY;AAAA,MAEhB,iBAAiB,SAAS,MAAM;AAAA,QAC9B,eAAe,MAAM;AAAA,QACrB,aAAa;AAAA,QAEb,MAAM,MAAM,iBAAiB,OAAO,KAAK,IAAI,SAAS;AAAA,QAGtD,MAAM,kBAAkB,6BAA6B,KAAK,OAAO;AAAA,QACjE,IAAI,iBAAiB;AAAA,UACnB,MAAM,IAAI,MACR,0BAA0B,KAAK,uCAAuC,cAAc,iBACtF;AAAA,QACF;AAAA,QAGA,MAAM,kBAAkB;AAAA,UACtB;AAAA,UACA,QAAQ,KAAK;AAAA,UACb;AAAA,UACA;AAAA,UACA;AAAA,QACF,CAAC;AAAA,QAED,YAAY;AAAA,QACZ,MAAM;AAAA,MACR;AAAA,MACC,CACL;AAAA,IAEA,eAAe,MAAM;AAAA,IAErB,IAAI,KAAK,OAAO,SAAS,YAAY,aAAa,GAAG;AAAA,MACnD,MAAM,MAAM,MAAM;AAAA,MAClB,QAAQ;AAAA,MACR,MAAM,WAAU,MAAM,YAAY,iBAAiB,QAAQ,EAAE;AAAA,MAC7D,MAAM,eAAc,MAAM,gBAAgB,OAAO;AAAA,QAC/C,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,WACI,WACA,EAAE,QAAQ,EAAE,WAAW,SAAQ,WAAW,WAAW,SAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,QACL,YAAY;AAAA,MACd,CAAC;AAAA,MAED,OAAO;AAAA,QACL,IAAI,IAAI;AAAA,QACR,QAAQ,KAAK;AAAA,QACb,WAAW,QAAQ;AAAA,QACnB,MAAM,KAAK,OAAO;AAAA,QAClB,WAAW,WAAW;AAAA,QACtB,YAAY,kBAAkB,IAAI,IAAI,aAAY,UAAU;AAAA,QAC5D;AAAA,WACI,WAAU,EAAE,kBAAQ,IAAI,CAAC;AAAA,QAC7B,gBAAgB;AAAA,MAClB;AAAA,IACF;AAAA,IAEA,MAAM,SAAS,MAAM,MAAM,OAAO;AAAA,MAChC,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI,iBAAiB,QAAQ,KAAK,UAAU,IAAI;AAAA,IACjE,CAAC;AAAA,IACD,QAAQ,YAAY,YAAY;AAAA,IAChC,mBAAmB;AAAA,IACnB,IAAI;AAAA,IAEJ,IAAI;AAAA,MACF,cAAc,MAAM,gBAAgB,OAAO;AAAA,QACzC,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,QACA,QAAQ;AAAA,UACN,WAAW,QAAQ;AAAA,UACnB,WAAW,QAAQ;AAAA,QACrB;AAAA,QACA,YAAY;AAAA,MACd,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,uBAAuB;AAAA,QAC3B,QAAQ,KAAK;AAAA,QACb,OAAO,IAAI;AAAA,QACX;AAAA,QACA,OAAO;AAAA,MACT,CAAC;AAAA;AAAA,IAGH,OAAO;AAAA,MACL,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,YAAY,kBAAkB,IAAI,IAAI,YAAY,UAAU;AAAA,MAC5D;AAAA,MACA;AAAA,MACA,gBAAgB,YAAY;AAAA,IAC9B;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MAErB,MAAM,OAAO,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,MAEnC,MAAM,SAAS,OAAO,UAAU,cAAc;AAAA,MAC9C,IAAI;AAAA,QACF,MAAM,MAAM,MAAM,gBAAgB,OAAO;AAAA,UACvC,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,KAAK;AAAA,QAC/B,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW;AAAA,UAAU,MAAM,cAAc,OAAO,GAAG;AAAA,QAClF,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;;;ADzT3B,IAAM,SAAS;AAAA;AAkBR,MAAM,mBAAmB,YAAsC;AAAA,EACnD;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAAsB;AAAA,IAChC,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;AAAA,MACjC,MAAM,IAAI,MAAM,sDAAsD;AAAA,IACxE;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,UAAU,eAAe,KAAK;AAAA,IAChC,CAAC;AAAA,IAED,KAAK,UAAU,iBAAiB,IAAI;AAAA,IACpC,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,UAAmB;AAAA,MACvB,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,MACpB,yBAAyB,IAAI,QAAQ;AAAA,MACrC,eAAe,IAAI,QAAQ;AAAA,IAC7B;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,SAAS,MAAM,WAAW;AAAA,QACxB,SAAS,KAAK;AAAA,QACd,KAAK;AAAA,QACL;AAAA,QACA,cAAc,CAAC,QAAQ,mBAAmB,KAAK,MAAM,GAAG;AAAA,QACxD,aAAa,CAAC,OAAO,QAAQ;AAAA,UAC3B,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,YAAY,IAAI;AAAA,YAChB,SAAS,IAAI;AAAA,YACb,KAAK;AAAA,cACH,MAAM;AAAA,cACN,OAAO,IAAI;AAAA,cACX,QAAQ,IAAI;AAAA,YACd;AAAA,UACF,CAAC;AAAA;AAAA,MAEL,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAA4B;AAAA,MACjD,MAAM;AAAA;AAAA,IAGR,MAAM,wBAAwB,KAAK,MAAM,SAAS,MAAM;AAAA;AAAA,OAGjC,iBAAgB,CACvC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,oBAAoB,KAAK,MAAM;AAAA,MACnC,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,OAAO,EAAE,MAAM,OAAO;AAAA;AAE1B;AAEA,eAAe,kBAAkB,CAC/B,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,uBAAuB,CACpC,MACA,KACA,QACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN,GAAI,OAAO,iBACP;AAAA,QACE;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,WAAW,OAAO,QAAQ;AAAA,YAC1B,WAAW,OAAO,QAAQ,UAAU,YAAY;AAAA,YAChD,UAAU;AAAA,cACR,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,OAAO,IAAI;AAAA,YACb;AAAA,UACF;AAAA,QACF;AAAA,MACF,IACA,CAAC;AAAA,MACL;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,UACR,WAAW,OAAO;AAAA,aACd,OAAO,UAAU,EAAE,WAAW,OAAO,QAAQ,UAAU,IAAI,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,mBAAmB,CAChC,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,gBAAgB,CAAC,MAAyC;AAAA,EACjE,MAAM,kBAAkB,KAAK,QAAQ;AAAA,EACrC,IAAI,CAAC,iBAAiB;AAAA,IACpB,MAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AAAA,EAEA,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK;AAAA,IACX,WAAW,CAAC,QAAQ;AAAA,MAClB,OAAO,KAAK,YAAY,MAAM;AAAA;AAAA,IAEhC,cAAc,CAAC,WAAW;AAAA,MACxB,OAAO,KAAK,eAAe,SAAS;AAAA;AAAA,IAEtC,SAAS,CAAC,YAAY;AAAA,MACpB,OAAO,KAAK,UAAU,UAAU;AAAA;AAAA,EAEpC;AAAA;",
10
- "debugId": "33375B9F4568F6E364756E2164756E21",
9
+ "mappings": ";;AAYA;AAGA;;;ACfA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcA;AACA;;;ACbA,SAAS,aAAa,CAAC,OAAqC;AAAA,EAC1D,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AAAA,IACvE,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,OAAO,eAAe,KAAK;AAAA,EAC7C,OAAO,cAAc,OAAO,aAAa,cAAc;AAAA;AAGzD,SAAS,UAAU,CAAC,OAA4C;AAAA,EAC9D,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA0C,OAAO,cAAc;AAAA;AAI3E,SAAS,eAAe,CAAC,OAAiD;AAAA,EACxE,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA+C,OAAO,mBAAmB;AAAA;AAIrF,SAAS,YAAY,CAAC,QAA4B;AAAA,EAChD,IAAI,OAAO,kBAAkB,OAAO;AAAA,IAClC,OAAO,OAAO;AAAA,EAChB;AAAA,EAEA,MAAM,UACJ,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,SAAS,IACxD,OAAO,SACP;AAAA,EAEN,MAAM,QAAQ,IAAI,MAAM,OAAO;AAAA,EAC/B,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGF,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,aAAa,MAAM;AAAA,EAC3B;AAAA;AAGK,SAAS,gBAAgB,CAAC,OAAgB,QAAgB,WAA+B;AAAA,EAC9F,IAAI,cAAc,KAAK,GAAG;AAAA,IACxB,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,2CAA2C,gDACvE;AAAA;AAGF,gBAAuB,mBAAmB,CACxC,YACA,QACwB;AAAA,EACxB,IAAI,gBAAgB,UAAU,GAAG;AAAA,IAC/B,OAAO;AAAA,IACP;AAAA,EACF;AAAA,EAEA,IAAI,WAAW,UAAU,GAAG;AAAA,IAC1B,WAAW,QAAQ,YAAY;AAAA,MAC7B,MAAM;AAAA,IACR;AAAA,IACA;AAAA,EACF;AAAA,EAEA,IAAI,cAAc,UAAU,GAAG;AAAA,IAC7B,MAAM;AAAA,IACN;AAAA,EACF;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,kGAC5B;AAAA;;;AD7DK,MAAM,mCAAmC,MAAM;AAAA,EAG/B;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAoB;AAAA,IACvC,MAAM,8BAA8B,IAAI,0BAA0B;AAAA,IAD/C;AAAA;AAGvB;AAEA,SAAS,gBAAgB,CAAC,OAAgC;AAAA,EACxD,IAAI,iBAAiB,OAAO;AAAA,IAC1B,OAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,OAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,EACvB;AAAA;AAGF,SAAS,sBAAsB,CAAC,SAKtB;AAAA,EAER,OAAO,IAAI,MACT,0BAA0B,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,kGAChJ,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGF,SAAS,iBAAiB,CAAC,OAAe,YAAoC;AAAA,EAC5E,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,8BAA8B,iDAAiD;AAAA;AAGjG,eAAe,aAAa,CAAC,SAOX;AAAA,EAChB,MAAM,WAAW,MAAM,QAAQ,YAAY,KAAK,QAAQ,QAAQ,MAAM;AAAA,EACtE,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,yCAAyC,QAAQ,QAAQ,UAC9L;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,WAAW,QAAQ,QAAQ,QAAQ;AAAA,IAC9C,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,wBAAwB,QAAQ,QAAQ,kCAAkC,SAAS,UACzQ;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,cAAc,QAAQ,QAAQ,WAAW;AAAA,IACpD,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,qBAAqB,QAAQ,QAAQ,mCAAmC,SAAS,YACvQ;AAAA,EACF;AAAA;AAGF,eAAe,iBAAiB,CAAC,SAMf;AAAA,EAChB,MAAM,iBAAiB,QAAQ,QAAQ,OAAO,QAAQ,OACpD,CAAC,WAAW,OAAO,SAAS,SAC9B;AAAA,EACA,WAAW,UAAU,gBAAgB;AAAA,IACnC,MAAM,QAAQ,QAAQ,IAAI,OAAO;AAAA,IACjC,IAAI,CAAC,UAAU,KAAK,GAAG;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,cAAc;AAAA,MAClB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,YAAY,OAAO;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA;AAGF,SAAS,iBAAiB,CACxB,OACA,QACA,SACA,WACqC;AAAA,EACrC,MAAM,kBAAkB,qCAAqC,OAAO,OAAO;AAAA,EAC3E,IAAI,iBAAiB;AAAA,IACnB,MAAM,IAAI,MACR,0BAA0B,oDAAoD,cAAc,iBAC9F;AAAA,EACF;AAAA,EACA,OAAO;AAAA;AAGT,eAAe,cAAc,CAAC,SAQM;AAAA,EAClC,MAAM,SAAS,MAAM,QAAQ,QAAQ,UAAU,QAAQ,KAAK,SAAS;AAAA,EACrE,MAAM,aAAa,MAAM,QAAQ,KAAK,KAAK,QAAQ;AAAA,IACjD,WAAW,QAAQ,QAAQ;AAAA,IAC3B,QAAQ,QAAQ,KAAK;AAAA,IACrB,QAAQ,QAAQ;AAAA,IAChB,OAAO,QAAQ;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,YACE,QAAQ,uBAAuB,YAC3B,eAAe,QAAQ,kBAAkB,IACzC;AAAA,IACN,eAAe,QAAQ;AAAA,EACzB,CAAC;AAAA,EACD,OAAO,oBAAoB,YAAY,QAAQ,KAAK,EAAE;AAAA;AAGxD,gBAAgB,oBAAoB,CAAC,SAOP;AAAA,EAC5B,IAAI,YAAY;AAAA,EAChB,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,IACxC,eAAe,QAAQ,MAAM;AAAA,IAC7B,aAAa;AAAA,IAEb,MAAM,MAAM,iBAAiB,OAAO,QAAQ,QAAQ,SAAS;AAAA,IAG7D,MAAM,kBAAkB,6BAA6B,KAAK,QAAQ,OAAO;AAAA,IACzE,IAAI,iBAAiB;AAAA,MACnB,MAAM,IAAI,MACR,0BAA0B,QAAQ,2CAA2C,cAAc,iBAC7F;AAAA,IACF;AAAA,IAGA,MAAM,kBAAkB;AAAA,MACtB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IAED,QAAQ,OAAO;AAAA,IACf,MAAM;AAAA,EACR;AAAA;AAGF,gBAAgB,qBAAqB,CAAC,SAOiB;AAAA,EACrD,IAAI,YAAY;AAAA,EAChB,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,IACxC,eAAe,QAAQ,MAAM;AAAA,IAC7B,aAAa;AAAA,IACb,MAAM,cAAc,kBAAkB,OAAO,QAAQ,QAAQ,QAAQ,SAAS,SAAS;AAAA,IACvF,IAAI,YAAY,SAAS,UAAU;AAAA,MACjC,MAAM,kBAAkB;AAAA,QACtB,aAAa,QAAQ;AAAA,QACrB,QAAQ,QAAQ;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,KAAK,YAAY;AAAA,QACjB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,QAAQ,OAAO;AAAA,IACf,MAAM;AAAA,EACR;AAAA;AAeF,eAAsB,UAAU,CAAC,OAAgD;AAAA,EAC/E,QAAQ,SAAS,QAAQ;AAAA,EACzB,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,QAAQ,iBAAiB,aAAa,gBAAgB;AAAA,EAEtD,MAAM,OAAO,QAAQ,YAAY,IAAI,MAAM;AAAA,EAC3C,IAAI,CAAC,MAAM;AAAA,IACT,MAAM,IAAI,MAAM,kCAAkC,IAAI,UAAU;AAAA,EAClE;AAAA,EAEA,MAAM,UAAU,QAAQ,eAAe,KAAK,OAAO,QAAQ,EAAE;AAAA,EAC7D,IAAI,CAAC,SAAS;AAAA,IACZ,MAAM,IAAI,MACR,0BAA0B,KAAK,gCAAgC,KAAK,OAAO,QAAQ,MACrF;AAAA,EACF;AAAA,EAEA,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,aAAa,MAAM,gBAAgB,MAAM;AAAA,MACvC,WAAW,QAAQ;AAAA,MACnB,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,MAAM,WAAW,MAAM,gBAAgB,QAAQ,EAAE,WAAW,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC;AAAA,IACpF,IAAI,UAAU,WAAW,KAAK,MAAM,SAAS,cAAc,QAAQ,IAAI;AAAA,MACrE,MAAM,IAAI,2BAA2B,QAAQ;AAAA,IAC/C;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,MAAM,eAAe,UAAU;AAAA,EAErC,MAAM,aAAa,mBAAmB,QAAQ,IAAI,QAAQ,IAAI,EAAE,eAAe;AAAA,IAC7E,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,SAAS,WAAW;AAAA,EAC1B,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EACf,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,eAAe,MAAM;AAAA,IACrB,MAAM,YAAY,cAAc,OAAO;AAAA,IAEvC,MAAM,uBAAuB,MAAM,gBAAgB,KAAK;AAAA,MACtD,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,UAAU,CAAC,WAAW;AAAA,MACtB,OAAO;AAAA,MACP,OAAO;AAAA,IACT,CAAC;AAAA,IACD,MAAM,qBAAqB,qBAAqB,KAAK,IAAI;AAAA,IACzD,IAAI,iBACF,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,IAE1E,MAAM,WAAW;AAAA,MACf,MAAM;AAAA,MACN,IAAI,KAAK;AAAA,MACT,OAAO,IAAI;AAAA,IACb;AAAA,IACA,MAAM,aAAa,MACjB,eAAe;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,CAAC,MAAM;AAAA,QAClB,gBAAgB,MAAM,SAAS,KAAK,gBAAgB;AAAA,QACpD,iBAAiB,eAAe,IAAI;AAAA;AAAA,IAExC,CAAC;AAAA,IACH,MAAM,SAAS,MAAM;AAAA,MACnB,YAAY;AAAA;AAAA,IAEd,MAAM,gBAAgB,IAAI,iBAAiB,QAAQ,KAAK,UAAU,IAAI;AAAA,IACtE,IAAI;AAAA,IAEJ,IAAI,KAAK,OAAO,SAAS,SAAS;AAAA,MAChC,MAAM,aAAa,MAAM,YAAY,WAAW;AAAA,QAC9C;AAAA,QACA,yBAAyB,IAAI;AAAA,QAC7B;AAAA,MACF,CAAC;AAAA,MACD,aAAa,MAAM,WAAW,MAAM;AAAA,MAEpC,MAAM,SAAS,MAAM,WAAW;AAAA,MAChC,eAAe,MAAM;AAAA,MACrB,MAAM,WAAW,aACf,sBAAsB;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,KAAK;AAAA,QACb;AAAA,QACA;AAAA,MACF,CAAC,CACH;AAAA,MACA,eAAe,MAAM;AAAA,MAErB,MAAM,SAAS,MAAM,WAAW,OAAO,EAAE,cAAc,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,eAAe;AAAA,QACb,SAAS,OAAO;AAAA,WACZ,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,MACtD;AAAA,IACF,EAAO;AAAA,MACL,MAAM,SAAS,MAAM,WAAW;AAAA,MAChC,eAAe,MAAM;AAAA,MAErB,MAAM,WAAW,MAAM,YAAY,WAAW;AAAA,QAC5C;AAAA,QACA,MAAM,KAAK,OAAO;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,MACD,aAAa,MAAM,SAAS,MAAM;AAAA,MAClC,MAAM,SAAS,UACb,qBAAqB;AAAA,QACnB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,KAAK;AAAA,QACb;AAAA,QACA;AAAA,MACF,CAAC,CACH;AAAA,MACA,eAAe,MAAM;AAAA,MAErB,IAAI,aAAa,GAAG;AAAA,QAClB,MAAM,WAAU,MAAM,YAAY,iBAAiB,QAAQ,EAAE;AAAA,QAC7D,IAAI,KAAK,OAAO,SAAS,YAAY,CAAC,UAAS;AAAA,UAC7C,MAAM,SAAS,MAAM;AAAA,UACrB,aAAa;AAAA,UACb,MAAM,eAAc,MAAM,gBAAgB,OAAO;AAAA,YAC/C,WAAW,QAAQ;AAAA,YACnB,IAAI,IAAI;AAAA,YACR,QAAQ;AAAA,YACR;AAAA,eACI,WACA,EAAE,QAAQ,EAAE,WAAW,SAAQ,WAAW,WAAW,SAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,YACL,YAAY;AAAA,UACd,CAAC;AAAA,UAED,OAAO;AAAA,YACL,IAAI,IAAI;AAAA,YACR,QAAQ,KAAK;AAAA,YACb,WAAW,QAAQ;AAAA,YACnB,MAAM,KAAK,OAAO;AAAA,YAClB,WAAW,WAAW;AAAA,YACtB,YAAY,kBAAkB,IAAI,IAAI,aAAY,UAAU;AAAA,YAC5D;AAAA,eACI,WAAU,EAAE,kBAAQ,IAAI,CAAC;AAAA,YAC7B,gBAAgB;AAAA,UAClB;AAAA,QACF;AAAA,MACF;AAAA,MAEA,QAAQ,sBAAY,aAAY,MAAM,SAAS,OAAO;AAAA,QACpD,yBAAyB,IAAI;AAAA,QAC7B;AAAA,MACF,CAAC;AAAA,MACD,aAAa;AAAA,MACb,eAAe,EAAE,mBAAS,kBAAQ;AAAA;AAAA,IAGpC,QAAQ,SAAS,YAAY;AAAA,IAC7B,IAAI,YAAY,WAAW;AAAA,MACzB,IAAI,CAAC,SAAS;AAAA,QACZ,MAAM,IAAI,MAAM,0BAA0B,KAAK,iCAAiC;AAAA,MAClF;AAAA,MACA,mBAAmB;AAAA,IACrB;AAAA,IACA,IAAI;AAAA,IAEJ,IAAI;AAAA,MACF,cAAc,MAAM,gBAAgB,OAAO;AAAA,QACzC,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,WACI,UACA,EAAE,QAAQ,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,QACL,YAAY;AAAA,MACd,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,kBAAkB;AAAA,QACpB,MAAM,uBAAuB;AAAA,UAC3B,QAAQ,KAAK;AAAA,UACb,OAAO,IAAI;AAAA,UACX,SAAS;AAAA,UACT,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,MACA,MAAM;AAAA;AAAA,IAGR,MAAM,SAAS;AAAA,MACb,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,YAAY,kBAAkB,IAAI,IAAI,YAAY,UAAU;AAAA,MAC5D;AAAA,IACF;AAAA,IACA,OAAO,UACH,KAAK,QAAQ,SAAS,gBAAgB,YAAY,UAAU,IAC5D,KAAK,QAAQ,gBAAgB,MAAM;AAAA,IACvC,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MAErB,MAAM,aAAa,EAAE,MAAM,MAAM,EAAE;AAAA,MAEnC,MAAM,SAAS,OAAO,UAAU,cAAc;AAAA,MAC9C,IAAI;AAAA,QACF,MAAM,MAAM,MAAM,gBAAgB,OAAO;AAAA,UACvC,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,KAAK;AAAA,QAC/B,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW;AAAA,UAAU,MAAM,cAAc,OAAO,GAAG;AAAA,QAClF,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;;;ADrc3B,IAAM,SAAS;AAAA;AAkBR,MAAM,mBAAmB,YAAsC;AAAA,EACnD;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAAsB;AAAA,IAChC,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;AAAA,MACjC,MAAM,IAAI,MAAM,sDAAsD;AAAA,IACxE;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,UAAU,eAAe,KAAK;AAAA,IAChC,CAAC;AAAA,IAED,KAAK,UAAU,iBAAiB,IAAI;AAAA,IACpC,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,UAAmB;AAAA,MACvB,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,MACpB,yBAAyB,IAAI,QAAQ;AAAA,MACrC,eAAe,IAAI,QAAQ;AAAA,IAC7B;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,SAAS,MAAM,WAAW;AAAA,QACxB,SAAS,KAAK;AAAA,QACd,KAAK;AAAA,QACL;AAAA,QACA,cAAc,CAAC,QAAQ,mBAAmB,KAAK,MAAM,GAAG;AAAA,QACxD,aAAa,CAAC,OAAO,QAAQ;AAAA,UAC3B,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,YAAY,IAAI;AAAA,YAChB,SAAS,IAAI;AAAA,YACb,KAAK;AAAA,cACH,MAAM;AAAA,cACN,OAAO,IAAI;AAAA,cACX,QAAQ,IAAI;AAAA,YACd;AAAA,UACF,CAAC;AAAA;AAAA,MAEL,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAA4B;AAAA,MACjD,MAAM;AAAA;AAAA,IAGR,MAAM,wBAAwB,KAAK,MAAM,SAAS,MAAM;AAAA;AAAA,OAGjC,iBAAgB,CACvC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,oBAAoB,KAAK,MAAM;AAAA,MACnC,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,OAAO,EAAE,MAAM,OAAO;AAAA;AAE1B;AAEA,eAAe,kBAAkB,CAC/B,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,uBAAuB,CACpC,MACA,KACA,QACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN,GAAI,OAAO,iBACP;AAAA,QACE;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,WAAW,OAAO,QAAQ;AAAA,YAC1B,WAAW,OAAO,QAAQ,UAAU,YAAY;AAAA,YAChD,UAAU;AAAA,cACR,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,OAAO,IAAI;AAAA,YACb;AAAA,UACF;AAAA,QACF;AAAA,MACF,IACA,CAAC;AAAA,MACL;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,UACR,WAAW,OAAO;AAAA,aACd,OAAO,UAAU,EAAE,WAAW,OAAO,QAAQ,UAAU,IAAI,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,mBAAmB,CAChC,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,gBAAgB,CAAC,MAAyC;AAAA,EACjE,MAAM,kBAAkB,KAAK,QAAQ;AAAA,EACrC,IAAI,CAAC,iBAAiB;AAAA,IACpB,MAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AAAA,EAEA,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK;AAAA,IACX,WAAW,CAAC,QAAQ;AAAA,MAClB,OAAO,KAAK,YAAY,MAAM;AAAA;AAAA,IAEhC,cAAc,CAAC,WAAW;AAAA,MACxB,OAAO,KAAK,eAAe,SAAS;AAAA;AAAA,IAEtC,SAAS,CAAC,YAAY;AAAA,MACpB,OAAO,KAAK,UAAU,UAAU;AAAA;AAAA,EAEpC;AAAA;",
10
+ "debugId": "3B01117F7588DCC564756E2164756E21",
11
11
  "names": []
12
12
  }
@@ -8,9 +8,9 @@ export declare class SyncRunAlreadyStartedError extends Error {
8
8
  /**
9
9
  * Run one sync job end to end using a caller-supplied run id.
10
10
  *
11
- * The worker creates the operational sync-run record, streams rows into lake
12
- * storage, commits one dataset version, and then finalizes the run record with
13
- * the outcome. It does not own scheduling, retries, or run-id generation.
11
+ * The worker creates the operational sync-run record, streams source values into lake storage,
12
+ * commits or reuses a dataset version when one exists, and then finalizes the run record. It does
13
+ * not own scheduling, retries, or run-id generation.
14
14
  */
15
15
  export declare function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>;
16
16
  //# sourceMappingURL=run-sync-job.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"run-sync-job.d.ts","sourceRoot":"","sources":["../src/run-sync-job.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAkB,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAEvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAE7D,qBAAa,0BAA2B,SAAQ,KAAK;IAGvC,QAAQ,CAAC,GAAG,EAAE,aAAa;IAFvC,SAAkB,IAAI,gCAA+B;gBAEhC,GAAG,EAAE,aAAa;CAGxC;AA2FD;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAyN/E"}
1
+ {"version":3,"file":"run-sync-job.d.ts","sourceRoot":"","sources":["../src/run-sync-job.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAkB,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAEvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAqB,MAAM,SAAS,CAAA;AAEhF,qBAAa,0BAA2B,SAAQ,KAAK;IAGvC,QAAQ,CAAC,GAAG,EAAE,aAAa;IAFvC,SAAkB,IAAI,gCAA+B;gBAEhC,GAAG,EAAE,aAAa;CAGxC;AAwMD;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAmP/E"}
package/dist/types.d.ts CHANGED
@@ -1,6 +1,6 @@
1
- import type { BlobStorage, ConnectorAdapter, ConnectorClient, ConnectorDefinition, DatasetDefinition, LakeStorage, SyncDefinition } from "@sixb/core";
1
+ import type { BlobStorage, ConnectorAdapter, ConnectorClient, ConnectorDefinition, DatasetDefinition, LakeStorage, SyncDefinition, SyncMode } from "@sixb/core";
2
2
  import type { LogsRuntime } from "@sixb/core/internal/logging";
3
- import type { DatasetVersion, DatasetWriteMode } from "@sixb/core/lake-storage";
3
+ import type { DatasetVersion } from "@sixb/core/lake-storage";
4
4
  import type { SyncRunRecord, SyncRunStorage } from "@sixb/core/storage";
5
5
  export interface SyncWorkerContext {
6
6
  readonly id: string;
@@ -31,9 +31,10 @@ interface SyncRunResultBase {
31
31
  readonly id: string;
32
32
  readonly syncId: string;
33
33
  readonly datasetId: string;
34
- readonly mode: DatasetWriteMode;
34
+ readonly mode: SyncMode;
35
35
  readonly startedAt: Date;
36
36
  readonly finishedAt: Date;
37
+ /** Source items successfully consumed. Merge runs count both upserts and deletes. */
37
38
  readonly rowsRead: number;
38
39
  }
39
40
  export type SyncRunResult = SyncRunResultBase & ({
@@ -41,7 +42,7 @@ export type SyncRunResult = SyncRunResultBase & ({
41
42
  /** True only when this run created the returned dataset version. */
42
43
  readonly versionCreated: boolean;
43
44
  } | {
44
- /** A first empty append has no dataset version yet. */
45
+ /** A successful initial no-op has no dataset version yet. */
45
46
  readonly version?: undefined;
46
47
  readonly versionCreated: false;
47
48
  });
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,cAAc,EACf,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAEvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAA;IACxC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAE3B,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;IAC3D,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAAA;IAElD,SAAS,CAAC,QAAQ,SAAS,gBAAgB,EACzC,UAAU,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,GAChD,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAA;CACtC;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAA;IACzC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,qBAAqB,CAAA;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,oBAAoB,CAAA;CAC5C;AAED,MAAM,MAAM,qBAAqB,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEhF,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAA;AAE/E,UAAU,iBAAiB;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAC3C,CACI;IACE,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,oEAAoE;IACpE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC,GACD;IACE,uDAAuD;IACvD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAA;IAC5B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;CAC/B,CACJ,CAAA"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,cAAc,EACd,QAAQ,EACT,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAEvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAA;IACxC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAE3B,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;IAC3D,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAAA;IAElD,SAAS,CAAC,QAAQ,SAAS,gBAAgB,EACzC,UAAU,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,GAChD,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAA;CACtC;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAA;IACzC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,qBAAqB,CAAA;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,oBAAoB,CAAA;CAC5C;AAED,MAAM,MAAM,qBAAqB,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEhF,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAA;AAE/E,UAAU,iBAAiB;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAA;IACzB,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAC3C,CACI;IACE,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,oEAAoE;IACpE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC,GACD;IACE,6DAA6D;IAC7D,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAA;IAC5B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;CAC/B,CACJ,CAAA"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sixb/sync-worker",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Worker that runs Sixb syncs",
5
5
  "keywords": [
6
6
  "sixb",
@@ -34,11 +34,12 @@
34
34
  "test": "bun test",
35
35
  "prepack": "bun run build"
36
36
  },
37
- "dependencies": {
38
- "@sixb/core": "0.1.0"
37
+ "peerDependencies": {
38
+ "@sixb/core": "0.1.1"
39
39
  },
40
40
  "devDependencies": {
41
- "@sixb/lake-local": "0.1.0",
41
+ "@sixb/core": "0.1.1",
42
+ "@sixb/lake-local": "0.1.1",
42
43
  "@types/bun": "latest",
43
44
  "typescript": "^5.7.0"
44
45
  },
@@ -2,16 +2,21 @@ import {
2
2
  assertJsonValue,
3
3
  type BlobStorage,
4
4
  cloneJsonValue,
5
+ type DatasetDefinition,
6
+ type DatasetRow,
5
7
  type FileRef,
6
8
  getDatasetRowValidationError,
7
9
  isFileRef,
8
10
  type JsonValue,
11
+ type Logger,
12
+ type MergeChange,
13
+ type SyncDefinition,
9
14
  } from "@sixb/core"
10
15
  import { resolveLogsRuntime } from "@sixb/core/internal/logging"
11
- import type { DatasetVersion, LakeWriteSession } from "@sixb/core/lake-storage"
16
+ import { type DatasetVersion, getDatasetMergeChangeValidationError } from "@sixb/core/lake-storage"
12
17
  import type { SyncRunFailure, SyncRunRecord } from "@sixb/core/storage"
13
18
  import { assertDatasetRow, normalizeReadResult, throwIfAborted } from "./normalize"
14
- import type { RunSyncJobInput, SyncRunResult } from "./types"
19
+ import type { RunSyncJobInput, SyncRunResult, SyncWorkerContext } from "./types"
15
20
 
16
21
  export class SyncRunAlreadyStartedError extends Error {
17
22
  override readonly name = "SyncRunAlreadyStartedError"
@@ -110,12 +115,121 @@ async function verifyRowFileRefs(options: {
110
115
  }
111
116
  }
112
117
 
118
+ function assertMergeChange(
119
+ value: unknown,
120
+ syncId: string,
121
+ dataset: DatasetDefinition,
122
+ itemIndex: number
123
+ ): MergeChange<DatasetRow, DatasetRow> {
124
+ const validationError = getDatasetMergeChangeValidationError(value, dataset)
125
+ if (validationError) {
126
+ throw new Error(
127
+ `[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`
128
+ )
129
+ }
130
+ return value as MergeChange<DatasetRow, DatasetRow>
131
+ }
132
+
133
+ async function readSyncValues(options: {
134
+ runtime: SyncWorkerContext
135
+ sync: SyncDefinition
136
+ signal: AbortSignal
137
+ blobStorage: BlobStorage
138
+ logger: Logger
139
+ previousCheckpoint: JsonValue | undefined
140
+ setCheckpoint(next: unknown): void
141
+ }): Promise<AsyncIterable<unknown>> {
142
+ const client = await options.runtime.connector(options.sync.connector)
143
+ const readResult = await options.sync.read(client, {
144
+ projectId: options.runtime.id,
145
+ syncId: options.sync.id,
146
+ signal: options.signal,
147
+ blobs: options.blobStorage,
148
+ logger: options.logger,
149
+ checkpoint:
150
+ options.previousCheckpoint !== undefined
151
+ ? cloneJsonValue(options.previousCheckpoint)
152
+ : undefined,
153
+ setCheckpoint: options.setCheckpoint,
154
+ })
155
+ return normalizeReadResult(readResult, options.sync.id)
156
+ }
157
+
158
+ async function* validatedDatasetRows(options: {
159
+ values: AsyncIterable<unknown>
160
+ signal: AbortSignal
161
+ blobStorage: BlobStorage
162
+ syncId: string
163
+ dataset: DatasetDefinition
164
+ onRead(): void
165
+ }): AsyncIterable<DatasetRow> {
166
+ let itemIndex = 0
167
+ for await (const value of options.values) {
168
+ throwIfAborted(options.signal)
169
+ itemIndex += 1
170
+
171
+ const row = assertDatasetRow(value, options.syncId, itemIndex)
172
+ // Validate before handing rows to lake storage so sync failures include
173
+ // the source item index; lake storage still re-validates as the final boundary.
174
+ const validationError = getDatasetRowValidationError(row, options.dataset)
175
+ if (validationError) {
176
+ throw new Error(
177
+ `[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`
178
+ )
179
+ }
180
+
181
+ // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.
182
+ await verifyRowFileRefs({
183
+ blobStorage: options.blobStorage,
184
+ syncId: options.syncId,
185
+ dataset: options.dataset,
186
+ row,
187
+ itemIndex,
188
+ })
189
+
190
+ options.onRead()
191
+ yield row
192
+ }
193
+ }
194
+
195
+ async function* validatedMergeChanges(options: {
196
+ values: AsyncIterable<unknown>
197
+ signal: AbortSignal
198
+ blobStorage: BlobStorage
199
+ syncId: string
200
+ dataset: DatasetDefinition
201
+ onRead(): void
202
+ }): AsyncIterable<MergeChange<DatasetRow, DatasetRow>> {
203
+ let itemIndex = 0
204
+ for await (const value of options.values) {
205
+ throwIfAborted(options.signal)
206
+ itemIndex += 1
207
+ const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex)
208
+ if (mergeChange.kind === "upsert") {
209
+ await verifyRowFileRefs({
210
+ blobStorage: options.blobStorage,
211
+ syncId: options.syncId,
212
+ dataset: options.dataset,
213
+ row: mergeChange.row,
214
+ itemIndex,
215
+ })
216
+ }
217
+ options.onRead()
218
+ yield mergeChange
219
+ }
220
+ }
221
+
222
+ interface SyncCommitResult {
223
+ readonly outcome: "created" | "unchanged"
224
+ readonly version?: DatasetVersion
225
+ }
226
+
113
227
  /**
114
228
  * Run one sync job end to end using a caller-supplied run id.
115
229
  *
116
- * The worker creates the operational sync-run record, streams rows into lake
117
- * storage, commits one dataset version, and then finalizes the run record with
118
- * the outcome. It does not own scheduling, retries, or run-id generation.
230
+ * The worker creates the operational sync-run record, streams source values into lake storage,
231
+ * commits or reuses a dataset version when one exists, and then finalizes the run record. It does
232
+ * not own scheduling, retries, or run-id generation.
119
233
  */
120
234
  export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {
121
235
  const { runtime, job } = input
@@ -161,7 +275,7 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
161
275
  id: job.id,
162
276
  })
163
277
  const logger = logSession.logger
164
- let write: LakeWriteSession | undefined
278
+ let abortWrite: (() => Promise<void>) | undefined
165
279
  let rowsRead = 0
166
280
  let committedVersion: DatasetVersion | undefined
167
281
 
@@ -180,102 +294,125 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
180
294
  let nextCheckpoint: JsonValue | undefined =
181
295
  previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined
182
296
 
183
- const client = await runtime.connector(sync.connector)
184
- const readResult = await sync.read(client, {
185
- projectId: runtime.id,
186
- syncId: sync.id,
187
- signal,
188
- blobs: blobStorage,
189
- logger,
190
- checkpoint: previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined,
191
- setCheckpoint(next: unknown) {
192
- assertJsonValue(next, `Sync '${sync.id}' checkpoint`)
193
- nextCheckpoint = cloneJsonValue(next)
194
- },
195
- })
196
- const rows = normalizeReadResult(readResult, sync.id)
197
-
198
- throwIfAborted(signal)
199
-
200
- write = await lakeStorage.beginWrite({
201
- dataset,
202
- mode: sync.config.mode,
203
- producer: {
204
- kind: "sync",
205
- id: sync.id,
206
- runId: job.id,
207
- },
208
- })
209
-
210
- await write.writeRows(
211
- (async function* () {
212
- let itemIndex = 0
213
-
214
- for await (const value of rows) {
215
- throwIfAborted(signal)
216
- itemIndex += 1
217
-
218
- const row = assertDatasetRow(value, sync.id, itemIndex)
219
- // Validate before handing rows to lake storage so sync failures include
220
- // the source item index; lake storage still re-validates as the final boundary.
221
- const validationError = getDatasetRowValidationError(row, dataset)
222
- if (validationError) {
223
- throw new Error(
224
- `[SixbSyncWorker] Sync '${sync.id}' returned an invalid row at item ${itemIndex}. ${validationError}`
225
- )
226
- }
297
+ const producer = {
298
+ kind: "sync" as const,
299
+ id: sync.id,
300
+ runId: job.id,
301
+ }
302
+ const readValues = () =>
303
+ readSyncValues({
304
+ runtime,
305
+ sync,
306
+ signal,
307
+ blobStorage,
308
+ logger,
309
+ previousCheckpoint,
310
+ setCheckpoint(next) {
311
+ assertJsonValue(next, `Sync '${sync.id}' checkpoint`)
312
+ nextCheckpoint = cloneJsonValue(next)
313
+ },
314
+ })
315
+ const onRead = () => {
316
+ rowsRead += 1
317
+ }
318
+ const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`
319
+ let commitResult: SyncCommitResult
320
+
321
+ if (sync.config.mode === "merge") {
322
+ const mergeWrite = await lakeStorage.beginMerge({
323
+ dataset,
324
+ expectedLatestVersionId: job.expectedLatestVersionId,
325
+ producer,
326
+ })
327
+ abortWrite = () => mergeWrite.abort()
328
+
329
+ const values = await readValues()
330
+ throwIfAborted(signal)
331
+ await mergeWrite.writeChanges(
332
+ validatedMergeChanges({
333
+ values,
334
+ signal,
335
+ blobStorage,
336
+ syncId: sync.id,
337
+ dataset,
338
+ onRead,
339
+ })
340
+ )
341
+ throwIfAborted(signal)
342
+
343
+ const commit = await mergeWrite.commit({ commitMessage })
344
+ abortWrite = undefined
345
+ commitResult = {
346
+ outcome: commit.outcome,
347
+ ...(commit.version ? { version: commit.version } : {}),
348
+ }
349
+ } else {
350
+ const values = await readValues()
351
+ throwIfAborted(signal)
227
352
 
228
- // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.
229
- await verifyRowFileRefs({
230
- blobStorage,
231
- syncId: sync.id,
232
- dataset,
233
- row,
234
- itemIndex,
353
+ const rowWrite = await lakeStorage.beginWrite({
354
+ dataset,
355
+ mode: sync.config.mode,
356
+ producer,
357
+ })
358
+ abortWrite = () => rowWrite.abort()
359
+ await rowWrite.writeRows(
360
+ validatedDatasetRows({
361
+ values,
362
+ signal,
363
+ blobStorage,
364
+ syncId: sync.id,
365
+ dataset,
366
+ onRead,
367
+ })
368
+ )
369
+ throwIfAborted(signal)
370
+
371
+ if (rowsRead === 0) {
372
+ const version = await lakeStorage.getLatestVersion(dataset.id)
373
+ if (sync.config.mode === "append" || !version) {
374
+ await rowWrite.abort()
375
+ abortWrite = undefined
376
+ const finishedRun = await syncRunsStorage.finish({
377
+ projectId: runtime.id,
378
+ id: job.id,
379
+ status: "succeeded",
380
+ rowsRead,
381
+ ...(version
382
+ ? { output: { datasetId: version.datasetId, versionId: version.versionId } }
383
+ : {}),
384
+ checkpoint: nextCheckpoint,
235
385
  })
236
386
 
237
- rowsRead += 1
238
- yield row
387
+ return {
388
+ id: job.id,
389
+ syncId: sync.id,
390
+ datasetId: dataset.id,
391
+ mode: sync.config.mode,
392
+ startedAt: startedRun.startedAt,
393
+ finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
394
+ rowsRead,
395
+ ...(version ? { version } : {}),
396
+ versionCreated: false,
397
+ }
239
398
  }
240
- })()
241
- )
242
-
243
- throwIfAborted(signal)
399
+ }
244
400
 
245
- if (sync.config.mode === "append" && rowsRead === 0) {
246
- await write.abort()
247
- write = undefined
248
- const version = await lakeStorage.getLatestVersion(dataset.id)
249
- const finishedRun = await syncRunsStorage.finish({
250
- projectId: runtime.id,
251
- id: job.id,
252
- status: "succeeded",
253
- rowsRead,
254
- ...(version
255
- ? { output: { datasetId: version.datasetId, versionId: version.versionId } }
256
- : {}),
257
- checkpoint: nextCheckpoint,
401
+ const { outcome, ...version } = await rowWrite.commit({
402
+ expectedLatestVersionId: job.expectedLatestVersionId,
403
+ commitMessage,
258
404
  })
405
+ abortWrite = undefined
406
+ commitResult = { outcome, version }
407
+ }
259
408
 
260
- return {
261
- id: job.id,
262
- syncId: sync.id,
263
- datasetId: dataset.id,
264
- mode: sync.config.mode,
265
- startedAt: startedRun.startedAt,
266
- finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
267
- rowsRead,
268
- ...(version ? { version } : {}),
269
- versionCreated: false,
409
+ const { outcome, version } = commitResult
410
+ if (outcome === "created") {
411
+ if (!version) {
412
+ throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`)
270
413
  }
414
+ committedVersion = version
271
415
  }
272
-
273
- const commit = await write.commit({
274
- expectedLatestVersionId: job.expectedLatestVersionId,
275
- commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,
276
- })
277
- const { outcome, ...version } = commit
278
- committedVersion = version
279
416
  let finishedRun: SyncRunRecord
280
417
 
281
418
  try {
@@ -284,22 +421,24 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
284
421
  id: job.id,
285
422
  status: "succeeded",
286
423
  rowsRead,
287
- output: {
288
- datasetId: version.datasetId,
289
- versionId: version.versionId,
290
- },
424
+ ...(version
425
+ ? { output: { datasetId: version.datasetId, versionId: version.versionId } }
426
+ : {}),
291
427
  checkpoint: nextCheckpoint,
292
428
  })
293
429
  } catch (error) {
294
- throw createBookkeepingError({
295
- syncId: sync.id,
296
- runId: job.id,
297
- version,
298
- cause: error,
299
- })
430
+ if (committedVersion) {
431
+ throw createBookkeepingError({
432
+ syncId: sync.id,
433
+ runId: job.id,
434
+ version: committedVersion,
435
+ cause: error,
436
+ })
437
+ }
438
+ throw error
300
439
  }
301
440
 
302
- return {
441
+ const result = {
303
442
  id: job.id,
304
443
  syncId: sync.id,
305
444
  datasetId: dataset.id,
@@ -307,13 +446,14 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
307
446
  startedAt: startedRun.startedAt,
308
447
  finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
309
448
  rowsRead,
310
- version,
311
- versionCreated: outcome === "created",
312
449
  }
450
+ return version
451
+ ? { ...result, version, versionCreated: outcome === "created" }
452
+ : { ...result, versionCreated: false }
313
453
  } catch (error) {
314
454
  if (!committedVersion) {
315
- // Before commit succeeds we can still best-effort clean up both the write session and run record.
316
- await write?.abort().catch(() => {})
455
+ // Without a created commit, best-effort clean up any open session and the run record.
456
+ await abortWrite?.().catch(() => {})
317
457
 
318
458
  const status = signal.aborted ? "cancelled" : "failed"
319
459
  try {
package/src/types.ts CHANGED
@@ -6,9 +6,10 @@ import type {
6
6
  DatasetDefinition,
7
7
  LakeStorage,
8
8
  SyncDefinition,
9
+ SyncMode,
9
10
  } from "@sixb/core"
10
11
  import type { LogsRuntime } from "@sixb/core/internal/logging"
11
- import type { DatasetVersion, DatasetWriteMode } from "@sixb/core/lake-storage"
12
+ import type { DatasetVersion } from "@sixb/core/lake-storage"
12
13
  import type { SyncRunRecord, SyncRunStorage } from "@sixb/core/storage"
13
14
 
14
15
  export interface SyncWorkerContext {
@@ -49,9 +50,10 @@ interface SyncRunResultBase {
49
50
  readonly id: string
50
51
  readonly syncId: string
51
52
  readonly datasetId: string
52
- readonly mode: DatasetWriteMode
53
+ readonly mode: SyncMode
53
54
  readonly startedAt: Date
54
55
  readonly finishedAt: Date
56
+ /** Source items successfully consumed. Merge runs count both upserts and deletes. */
55
57
  readonly rowsRead: number
56
58
  }
57
59
 
@@ -63,7 +65,7 @@ export type SyncRunResult = SyncRunResultBase &
63
65
  readonly versionCreated: boolean
64
66
  }
65
67
  | {
66
- /** A first empty append has no dataset version yet. */
68
+ /** A successful initial no-op has no dataset version yet. */
67
69
  readonly version?: undefined
68
70
  readonly versionCreated: false
69
71
  }