@sixb/sync-worker 0.0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +170 -82
- package/dist/index.js.map +3 -3
- package/dist/run-sync-job.d.ts +3 -3
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/types.d.ts +5 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -4
- package/src/run-sync-job.ts +248 -108
- package/src/types.ts +5 -3
package/dist/index.js
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
isFileRef
|
|
12
12
|
} from "@sixb/core";
|
|
13
13
|
import { resolveLogsRuntime } from "@sixb/core/internal/logging";
|
|
14
|
+
import { getDatasetMergeChangeValidationError } from "@sixb/core/lake-storage";
|
|
14
15
|
|
|
15
16
|
// src/normalize.ts
|
|
16
17
|
function isPlainObject(value) {
|
|
@@ -122,6 +123,66 @@ async function verifyRowFileRefs(options) {
|
|
|
122
123
|
});
|
|
123
124
|
}
|
|
124
125
|
}
|
|
126
|
+
function assertMergeChange(value, syncId, dataset, itemIndex) {
|
|
127
|
+
const validationError = getDatasetMergeChangeValidationError(value, dataset);
|
|
128
|
+
if (validationError) {
|
|
129
|
+
throw new Error(`[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`);
|
|
130
|
+
}
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
async function readSyncValues(options) {
|
|
134
|
+
const client = await options.runtime.connector(options.sync.connector);
|
|
135
|
+
const readResult = await options.sync.read(client, {
|
|
136
|
+
projectId: options.runtime.id,
|
|
137
|
+
syncId: options.sync.id,
|
|
138
|
+
signal: options.signal,
|
|
139
|
+
blobs: options.blobStorage,
|
|
140
|
+
logger: options.logger,
|
|
141
|
+
checkpoint: options.previousCheckpoint !== undefined ? cloneJsonValue(options.previousCheckpoint) : undefined,
|
|
142
|
+
setCheckpoint: options.setCheckpoint
|
|
143
|
+
});
|
|
144
|
+
return normalizeReadResult(readResult, options.sync.id);
|
|
145
|
+
}
|
|
146
|
+
async function* validatedDatasetRows(options) {
|
|
147
|
+
let itemIndex = 0;
|
|
148
|
+
for await (const value of options.values) {
|
|
149
|
+
throwIfAborted(options.signal);
|
|
150
|
+
itemIndex += 1;
|
|
151
|
+
const row = assertDatasetRow(value, options.syncId, itemIndex);
|
|
152
|
+
const validationError = getDatasetRowValidationError(row, options.dataset);
|
|
153
|
+
if (validationError) {
|
|
154
|
+
throw new Error(`[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`);
|
|
155
|
+
}
|
|
156
|
+
await verifyRowFileRefs({
|
|
157
|
+
blobStorage: options.blobStorage,
|
|
158
|
+
syncId: options.syncId,
|
|
159
|
+
dataset: options.dataset,
|
|
160
|
+
row,
|
|
161
|
+
itemIndex
|
|
162
|
+
});
|
|
163
|
+
options.onRead();
|
|
164
|
+
yield row;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async function* validatedMergeChanges(options) {
|
|
168
|
+
let itemIndex = 0;
|
|
169
|
+
for await (const value of options.values) {
|
|
170
|
+
throwIfAborted(options.signal);
|
|
171
|
+
itemIndex += 1;
|
|
172
|
+
const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex);
|
|
173
|
+
if (mergeChange.kind === "upsert") {
|
|
174
|
+
await verifyRowFileRefs({
|
|
175
|
+
blobStorage: options.blobStorage,
|
|
176
|
+
syncId: options.syncId,
|
|
177
|
+
dataset: options.dataset,
|
|
178
|
+
row: mergeChange.row,
|
|
179
|
+
itemIndex
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
options.onRead();
|
|
183
|
+
yield mergeChange;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
125
186
|
async function runSyncJob(input) {
|
|
126
187
|
const { runtime, job } = input;
|
|
127
188
|
const signal = input.signal ?? new AbortController().signal;
|
|
@@ -159,7 +220,7 @@ async function runSyncJob(input) {
|
|
|
159
220
|
id: job.id
|
|
160
221
|
});
|
|
161
222
|
const logger = logSession.logger;
|
|
162
|
-
let
|
|
223
|
+
let abortWrite;
|
|
163
224
|
let rowsRead = 0;
|
|
164
225
|
let committedVersion;
|
|
165
226
|
try {
|
|
@@ -174,82 +235,110 @@ async function runSyncJob(input) {
|
|
|
174
235
|
});
|
|
175
236
|
const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint;
|
|
176
237
|
let nextCheckpoint = previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined;
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
238
|
+
const producer = {
|
|
239
|
+
kind: "sync",
|
|
240
|
+
id: sync.id,
|
|
241
|
+
runId: job.id
|
|
242
|
+
};
|
|
243
|
+
const readValues = () => readSyncValues({
|
|
244
|
+
runtime,
|
|
245
|
+
sync,
|
|
181
246
|
signal,
|
|
182
|
-
|
|
247
|
+
blobStorage,
|
|
183
248
|
logger,
|
|
184
|
-
|
|
249
|
+
previousCheckpoint,
|
|
185
250
|
setCheckpoint(next) {
|
|
186
251
|
assertJsonValue(next, `Sync '${sync.id}' checkpoint`);
|
|
187
252
|
nextCheckpoint = cloneJsonValue(next);
|
|
188
253
|
}
|
|
189
254
|
});
|
|
190
|
-
const
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
});
|
|
201
|
-
await write.writeRows(async function* () {
|
|
202
|
-
let itemIndex = 0;
|
|
203
|
-
for await (const value of rows) {
|
|
204
|
-
throwIfAborted(signal);
|
|
205
|
-
itemIndex += 1;
|
|
206
|
-
const row = assertDatasetRow(value, sync.id, itemIndex);
|
|
207
|
-
const validationError = getDatasetRowValidationError(row, dataset);
|
|
208
|
-
if (validationError) {
|
|
209
|
-
throw new Error(`[SixbSyncWorker] Sync '${sync.id}' returned an invalid row at item ${itemIndex}. ${validationError}`);
|
|
210
|
-
}
|
|
211
|
-
await verifyRowFileRefs({
|
|
212
|
-
blobStorage,
|
|
213
|
-
syncId: sync.id,
|
|
214
|
-
dataset,
|
|
215
|
-
row,
|
|
216
|
-
itemIndex
|
|
217
|
-
});
|
|
218
|
-
rowsRead += 1;
|
|
219
|
-
yield row;
|
|
220
|
-
}
|
|
221
|
-
}());
|
|
222
|
-
throwIfAborted(signal);
|
|
223
|
-
if (sync.config.mode === "append" && rowsRead === 0) {
|
|
224
|
-
await write.abort();
|
|
225
|
-
write = undefined;
|
|
226
|
-
const version2 = await lakeStorage.getLatestVersion(dataset.id);
|
|
227
|
-
const finishedRun2 = await syncRunsStorage.finish({
|
|
228
|
-
projectId: runtime.id,
|
|
229
|
-
id: job.id,
|
|
230
|
-
status: "succeeded",
|
|
231
|
-
rowsRead,
|
|
232
|
-
...version2 ? { output: { datasetId: version2.datasetId, versionId: version2.versionId } } : {},
|
|
233
|
-
checkpoint: nextCheckpoint
|
|
255
|
+
const onRead = () => {
|
|
256
|
+
rowsRead += 1;
|
|
257
|
+
};
|
|
258
|
+
const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`;
|
|
259
|
+
let commitResult;
|
|
260
|
+
if (sync.config.mode === "merge") {
|
|
261
|
+
const mergeWrite = await lakeStorage.beginMerge({
|
|
262
|
+
dataset,
|
|
263
|
+
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
264
|
+
producer
|
|
234
265
|
});
|
|
235
|
-
|
|
236
|
-
|
|
266
|
+
abortWrite = () => mergeWrite.abort();
|
|
267
|
+
const values = await readValues();
|
|
268
|
+
throwIfAborted(signal);
|
|
269
|
+
await mergeWrite.writeChanges(validatedMergeChanges({
|
|
270
|
+
values,
|
|
271
|
+
signal,
|
|
272
|
+
blobStorage,
|
|
237
273
|
syncId: sync.id,
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
274
|
+
dataset,
|
|
275
|
+
onRead
|
|
276
|
+
}));
|
|
277
|
+
throwIfAborted(signal);
|
|
278
|
+
const commit = await mergeWrite.commit({ commitMessage });
|
|
279
|
+
abortWrite = undefined;
|
|
280
|
+
commitResult = {
|
|
281
|
+
outcome: commit.outcome,
|
|
282
|
+
...commit.version ? { version: commit.version } : {}
|
|
245
283
|
};
|
|
284
|
+
} else {
|
|
285
|
+
const values = await readValues();
|
|
286
|
+
throwIfAborted(signal);
|
|
287
|
+
const rowWrite = await lakeStorage.beginWrite({
|
|
288
|
+
dataset,
|
|
289
|
+
mode: sync.config.mode,
|
|
290
|
+
producer
|
|
291
|
+
});
|
|
292
|
+
abortWrite = () => rowWrite.abort();
|
|
293
|
+
await rowWrite.writeRows(validatedDatasetRows({
|
|
294
|
+
values,
|
|
295
|
+
signal,
|
|
296
|
+
blobStorage,
|
|
297
|
+
syncId: sync.id,
|
|
298
|
+
dataset,
|
|
299
|
+
onRead
|
|
300
|
+
}));
|
|
301
|
+
throwIfAborted(signal);
|
|
302
|
+
if (rowsRead === 0) {
|
|
303
|
+
const version3 = await lakeStorage.getLatestVersion(dataset.id);
|
|
304
|
+
if (sync.config.mode === "append" || !version3) {
|
|
305
|
+
await rowWrite.abort();
|
|
306
|
+
abortWrite = undefined;
|
|
307
|
+
const finishedRun2 = await syncRunsStorage.finish({
|
|
308
|
+
projectId: runtime.id,
|
|
309
|
+
id: job.id,
|
|
310
|
+
status: "succeeded",
|
|
311
|
+
rowsRead,
|
|
312
|
+
...version3 ? { output: { datasetId: version3.datasetId, versionId: version3.versionId } } : {},
|
|
313
|
+
checkpoint: nextCheckpoint
|
|
314
|
+
});
|
|
315
|
+
return {
|
|
316
|
+
id: job.id,
|
|
317
|
+
syncId: sync.id,
|
|
318
|
+
datasetId: dataset.id,
|
|
319
|
+
mode: sync.config.mode,
|
|
320
|
+
startedAt: startedRun.startedAt,
|
|
321
|
+
finishedAt: requireFinishedAt(job.id, finishedRun2.finishedAt),
|
|
322
|
+
rowsRead,
|
|
323
|
+
...version3 ? { version: version3 } : {},
|
|
324
|
+
versionCreated: false
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
const { outcome: outcome2, ...version2 } = await rowWrite.commit({
|
|
329
|
+
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
330
|
+
commitMessage
|
|
331
|
+
});
|
|
332
|
+
abortWrite = undefined;
|
|
333
|
+
commitResult = { outcome: outcome2, version: version2 };
|
|
334
|
+
}
|
|
335
|
+
const { outcome, version } = commitResult;
|
|
336
|
+
if (outcome === "created") {
|
|
337
|
+
if (!version) {
|
|
338
|
+
throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`);
|
|
339
|
+
}
|
|
340
|
+
committedVersion = version;
|
|
246
341
|
}
|
|
247
|
-
const commit = await write.commit({
|
|
248
|
-
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
249
|
-
commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`
|
|
250
|
-
});
|
|
251
|
-
const { outcome, ...version } = commit;
|
|
252
|
-
committedVersion = version;
|
|
253
342
|
let finishedRun;
|
|
254
343
|
try {
|
|
255
344
|
finishedRun = await syncRunsStorage.finish({
|
|
@@ -257,34 +346,33 @@ async function runSyncJob(input) {
|
|
|
257
346
|
id: job.id,
|
|
258
347
|
status: "succeeded",
|
|
259
348
|
rowsRead,
|
|
260
|
-
output: {
|
|
261
|
-
datasetId: version.datasetId,
|
|
262
|
-
versionId: version.versionId
|
|
263
|
-
},
|
|
349
|
+
...version ? { output: { datasetId: version.datasetId, versionId: version.versionId } } : {},
|
|
264
350
|
checkpoint: nextCheckpoint
|
|
265
351
|
});
|
|
266
352
|
} catch (error) {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
353
|
+
if (committedVersion) {
|
|
354
|
+
throw createBookkeepingError({
|
|
355
|
+
syncId: sync.id,
|
|
356
|
+
runId: job.id,
|
|
357
|
+
version: committedVersion,
|
|
358
|
+
cause: error
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
throw error;
|
|
273
362
|
}
|
|
274
|
-
|
|
363
|
+
const result = {
|
|
275
364
|
id: job.id,
|
|
276
365
|
syncId: sync.id,
|
|
277
366
|
datasetId: dataset.id,
|
|
278
367
|
mode: sync.config.mode,
|
|
279
368
|
startedAt: startedRun.startedAt,
|
|
280
369
|
finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
|
|
281
|
-
rowsRead
|
|
282
|
-
version,
|
|
283
|
-
versionCreated: outcome === "created"
|
|
370
|
+
rowsRead
|
|
284
371
|
};
|
|
372
|
+
return version ? { ...result, version, versionCreated: outcome === "created" } : { ...result, versionCreated: false };
|
|
285
373
|
} catch (error) {
|
|
286
374
|
if (!committedVersion) {
|
|
287
|
-
await
|
|
375
|
+
await abortWrite?.().catch(() => {});
|
|
288
376
|
const status = signal.aborted ? "cancelled" : "failed";
|
|
289
377
|
try {
|
|
290
378
|
const run = await syncRunsStorage.finish({
|
|
@@ -451,4 +539,4 @@ export {
|
|
|
451
539
|
SyncWorker
|
|
452
540
|
};
|
|
453
541
|
|
|
454
|
-
//# debugId=
|
|
542
|
+
//# debugId=3B01117F7588DCC564756E2164756E21
|
package/dist/index.js.map
CHANGED
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
"sources": ["../src/worker.ts", "../src/run-sync-job.ts", "../src/normalize.ts"],
|
|
4
4
|
"sourcesContent": [
|
|
5
5
|
"import type {\n BlobStorage,\n ConnectorAdapter,\n ConnectorClient,\n ConnectorDefinition,\n DatasetDefinition,\n DomainEventLog,\n LakeStorage,\n Queues,\n Storage,\n SyncDefinition,\n} from \"@sixb/core\"\nimport { reportRunFailure } from \"@sixb/core/internal/error-reporting\"\nimport type { LogsRuntime } from \"@sixb/core/internal/logging\"\nimport type { QueueWorkerFailureDecision } from \"@sixb/core/internal/workers\"\nimport { QueueWorker } from \"@sixb/core/internal/workers\"\nimport type { ClaimedQueueJob, SyncRunRequestedQueueJob } from \"@sixb/core/queues\"\nimport type { SyncRunRecord } from \"@sixb/core/storage\"\nimport { runSyncJob, SyncRunAlreadyStartedError } from \"./run-sync-job\"\nimport type { SyncJob, SyncRunResult, SyncWorkerContext } from \"./types\"\n\nconst SOURCE = \"SixbSyncWorker\"\n\nexport interface SyncWorkerSixb {\n readonly id: string\n readonly events?: DomainEventLog\n readonly logs?: LogsRuntime\n readonly lakeStorage: LakeStorage\n readonly blobStorage: BlobStorage\n readonly queues: Queues\n readonly storage: Storage\n listSyncs(): readonly SyncDefinition[]\n getSyncById(syncId: string): SyncDefinition | null\n getDatasetById(datasetId: string): DatasetDefinition | null\n connector<TAdapter extends ConnectorAdapter>(\n definition: ConnectorDefinition<string, TAdapter>\n ): Promise<ConnectorClient<TAdapter>>\n}\n\nexport class SyncWorker extends QueueWorker<SyncRunRequestedQueueJob> {\n private readonly context: SyncWorkerContext\n private readonly sixb: SyncWorkerSixb\n\n constructor(sixb: SyncWorkerSixb) {\n if (sixb.listSyncs().length === 0) {\n throw new Error(\"[SixbSyncWorker] No sync definitions are registered.\")\n }\n\n super({\n projectId: sixb.id,\n queue: sixb.queues.syncRuns,\n workerId: `sync-worker-${sixb.id}`,\n })\n\n this.context = buildSyncContext(sixb)\n this.sixb = sixb\n }\n\n protected async execute(\n claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,\n signal: AbortSignal\n ): Promise<void> {\n const { job } = claimed\n const syncJob: SyncJob = {\n id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n syncId: job.payload.syncId,\n expectedLatestVersionId: job.payload.expectedLatestVersionId,\n commitMessage: job.payload.commitMessage,\n }\n\n let result: SyncRunResult\n try {\n result = await runSyncJob({\n runtime: this.context,\n job: syncJob,\n signal,\n onRunStarted: (run) => emitSyncRunStarted(this.sixb, run),\n onRunFailed: (error, run) => {\n reportRunFailure(this.sixb, error, {\n projectId: this.sixb.id,\n occurredAt: run.finishedAt,\n attempt: job.attempt,\n run: {\n kind: \"sync\",\n runId: run.id,\n syncId: run.syncId,\n },\n })\n },\n })\n } catch (error) {\n if (error instanceof SyncRunAlreadyStartedError) return\n throw error\n }\n\n await emitSyncSucceededEvents(this.sixb, syncJob, result)\n }\n\n protected override async onExecutionError(\n claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,\n _error: unknown\n ): Promise<QueueWorkerFailureDecision> {\n const { job } = claimed\n await emitSyncRunFinished(this.sixb, {\n id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n syncId: job.payload.syncId,\n })\n\n return { kind: \"fail\" }\n }\n}\n\nasync function emitSyncRunStarted(\n sixb: SyncWorkerSixb,\n run: Pick<SyncRunRecord, \"id\" | \"syncId\" | \"startedAt\">\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n {\n type: \"sync.run.started\",\n payload: {\n syncId: run.syncId,\n runId: run.id,\n startedAt: run.startedAt.toISOString(),\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nasync function emitSyncSucceededEvents(\n sixb: SyncWorkerSixb,\n job: Pick<SyncJob, \"id\" | \"syncId\">,\n result: SyncRunResult\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n ...(result.versionCreated\n ? [\n {\n type: \"dataset.version.committed\" as const,\n payload: {\n datasetId: result.datasetId,\n versionId: result.version.versionId,\n createdAt: result.version.createdAt.toISOString(),\n producer: {\n kind: \"sync\" as const,\n id: job.syncId,\n runId: job.id,\n },\n },\n },\n ]\n : []),\n {\n type: \"sync.run.finished\",\n payload: {\n syncId: job.syncId,\n runId: job.id,\n status: \"succeeded\",\n datasetId: result.datasetId,\n ...(result.version ? { versionId: result.version.versionId } : {}),\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nasync function emitSyncRunFinished(\n sixb: SyncWorkerSixb,\n job: Pick<SyncJob, \"id\" | \"syncId\">\n): Promise<void> {\n await sixb.events?.emit(\n {\n events: [\n {\n type: \"sync.run.finished\",\n payload: {\n syncId: job.syncId,\n runId: job.id,\n status: \"failed\",\n },\n },\n ],\n },\n { source: SOURCE }\n )\n}\n\nfunction buildSyncContext(sixb: SyncWorkerSixb): SyncWorkerContext {\n const syncRunsStorage = sixb.storage.syncRuns\n if (!syncRunsStorage) {\n throw new Error(\"[SixbSyncWorker] Sync workers require storage.syncRuns support.\")\n }\n\n return {\n id: sixb.id,\n syncRunsStorage,\n lakeStorage: sixb.lakeStorage,\n blobStorage: sixb.blobStorage,\n logs: sixb.logs,\n getSyncById(syncId) {\n return sixb.getSyncById(syncId)\n },\n getDatasetById(datasetId) {\n return sixb.getDatasetById(datasetId)\n },\n connector(definition) {\n return sixb.connector(definition)\n },\n }\n}\n",
|
|
6
|
-
"import {\n assertJsonValue,\n type BlobStorage,\n cloneJsonValue,\n type FileRef,\n getDatasetRowValidationError,\n isFileRef,\n type JsonValue,\n} from \"@sixb/core\"\nimport { resolveLogsRuntime } from \"@sixb/core/internal/logging\"\nimport type { DatasetVersion, LakeWriteSession } from \"@sixb/core/lake-storage\"\nimport type { SyncRunFailure, SyncRunRecord } from \"@sixb/core/storage\"\nimport { assertDatasetRow, normalizeReadResult, throwIfAborted } from \"./normalize\"\nimport type { RunSyncJobInput, SyncRunResult } from \"./types\"\n\nexport class SyncRunAlreadyStartedError extends Error {\n override readonly name = \"SyncRunAlreadyStartedError\"\n\n constructor(readonly run: SyncRunRecord) {\n super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`)\n }\n}\n\nfunction toSyncRunFailure(error: unknown): SyncRunFailure {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n }\n }\n\n return {\n message: String(error),\n }\n}\n\nfunction createBookkeepingError(options: {\n syncId: string\n runId: string\n version: DatasetVersion\n cause: unknown\n}): Error {\n // The lake commit is already durable here, so we surface an explicit repair-needed error.\n return new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`,\n { cause: options.cause }\n )\n}\n\nfunction requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {\n if (finishedAt) {\n return finishedAt\n }\n\n throw new Error(`[SixbSyncWorker] Sync run '${runId}' finished without a finishedAt timestamp.`)\n}\n\nasync function verifyFileRef(options: {\n blobStorage: BlobStorage\n syncId: string\n datasetId: string\n columnName: string\n itemIndex: number\n fileRef: FileRef\n}): Promise<void> {\n const blobInfo = await options.blobStorage.stat(options.fileRef.blobId)\n if (!blobInfo) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing unknown blob '${options.fileRef.blobId}'.`\n )\n }\n\n if (blobInfo.digest !== options.fileRef.digest) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with digest '${options.fileRef.digest}', but blob storage has '${blobInfo.digest}'.`\n )\n }\n\n if (blobInfo.sizeBytes !== options.fileRef.sizeBytes) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with size ${options.fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`\n )\n }\n}\n\nasync function verifyRowFileRefs(options: {\n blobStorage: BlobStorage\n syncId: string\n dataset: { id: string; schema: { columns: readonly { name: string; type: string }[] } }\n row: Record<string, unknown>\n itemIndex: number\n}): Promise<void> {\n const fileRefColumns = options.dataset.schema.columns.filter(\n (column) => column.type === \"fileRef\"\n )\n for (const column of fileRefColumns) {\n const value = options.row[column.name]\n if (!isFileRef(value)) {\n continue\n }\n\n await verifyFileRef({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n datasetId: options.dataset.id,\n columnName: column.name,\n itemIndex: options.itemIndex,\n fileRef: value,\n })\n }\n}\n\n/**\n * Run one sync job end to end using a caller-supplied run id.\n *\n * The worker creates the operational sync-run record, streams rows into lake\n * storage, commits one dataset version, and then finalizes the run record with\n * the outcome. It does not own scheduling, retries, or run-id generation.\n */\nexport async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {\n const { runtime, job } = input\n const signal = input.signal ?? new AbortController().signal\n const { syncRunsStorage, lakeStorage, blobStorage } = runtime\n\n const sync = runtime.getSyncById(job.syncId)\n if (!sync) {\n throw new Error(`[SixbSyncWorker] Unknown sync '${job.syncId}'.`)\n }\n\n const dataset = runtime.getDatasetById(sync.target.dataset.id)\n if (!dataset) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${sync.target.dataset.id}'.`\n )\n }\n\n throwIfAborted(signal)\n\n let startedRun: SyncRunRecord\n try {\n startedRun = await syncRunsStorage.start({\n projectId: runtime.id,\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage,\n })\n } catch (error) {\n const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })\n if (existing?.syncId === sync.id && existing.datasetId === dataset.id) {\n throw new SyncRunAlreadyStartedError(existing)\n }\n throw error\n }\n await input.onRunStarted?.(startedRun)\n\n const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({\n kind: \"sync\",\n id: job.id,\n })\n const logger = logSession.logger\n let write: LakeWriteSession | undefined\n let rowsRead = 0\n let committedVersion: DatasetVersion | undefined\n\n try {\n throwIfAborted(signal)\n await lakeStorage.createDataset(dataset)\n\n const latestSuccessfulRuns = await syncRunsStorage.list({\n projectId: runtime.id,\n syncId: sync.id,\n statuses: [\"succeeded\"],\n limit: 1,\n order: \"desc\",\n })\n const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint\n let nextCheckpoint: JsonValue | undefined =\n previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined\n\n const client = await runtime.connector(sync.connector)\n const readResult = await sync.read(client, {\n projectId: runtime.id,\n syncId: sync.id,\n signal,\n blobs: blobStorage,\n logger,\n checkpoint: previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined,\n setCheckpoint(next: unknown) {\n assertJsonValue(next, `Sync '${sync.id}' checkpoint`)\n nextCheckpoint = cloneJsonValue(next)\n },\n })\n const rows = normalizeReadResult(readResult, sync.id)\n\n throwIfAborted(signal)\n\n write = await lakeStorage.beginWrite({\n dataset,\n mode: sync.config.mode,\n producer: {\n kind: \"sync\",\n id: sync.id,\n runId: job.id,\n },\n })\n\n await write.writeRows(\n (async function* () {\n let itemIndex = 0\n\n for await (const value of rows) {\n throwIfAborted(signal)\n itemIndex += 1\n\n const row = assertDatasetRow(value, sync.id, itemIndex)\n // Validate before handing rows to lake storage so sync failures include\n // the source item index; lake storage still re-validates as the final boundary.\n const validationError = getDatasetRowValidationError(row, dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' returned an invalid row at item ${itemIndex}. ${validationError}`\n )\n }\n\n // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.\n await verifyRowFileRefs({\n blobStorage,\n syncId: sync.id,\n dataset,\n row,\n itemIndex,\n })\n\n rowsRead += 1\n yield row\n }\n })()\n )\n\n throwIfAborted(signal)\n\n if (sync.config.mode === \"append\" && rowsRead === 0) {\n await write.abort()\n write = undefined\n const version = await lakeStorage.getLatestVersion(dataset.id)\n const finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n ...(version ? { version } : {}),\n versionCreated: false,\n }\n }\n\n const commit = await write.commit({\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,\n })\n const { outcome, ...version } = commit\n committedVersion = version\n let finishedRun: SyncRunRecord\n\n try {\n finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n output: {\n datasetId: version.datasetId,\n versionId: version.versionId,\n },\n checkpoint: nextCheckpoint,\n })\n } catch (error) {\n throw createBookkeepingError({\n syncId: sync.id,\n runId: job.id,\n version,\n cause: error,\n })\n }\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n version,\n versionCreated: outcome === \"created\",\n }\n } catch (error) {\n if (!committedVersion) {\n // Before commit succeeds we can still best-effort clean up both the write session and run record.\n await write?.abort().catch(() => {})\n\n const status = signal.aborted ? \"cancelled\" : \"failed\"\n try {\n const run = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status,\n rowsRead,\n error: toSyncRunFailure(error),\n })\n if (status === \"failed\" && run.status === \"failed\") input.onRunFailed?.(error, run)\n } catch {\n // The run did not transition to the requested terminal status.\n }\n }\n\n throw error\n } finally {\n await logSession.flush()\n }\n}\n",
|
|
6
|
+
"import {\n assertJsonValue,\n type BlobStorage,\n cloneJsonValue,\n type DatasetDefinition,\n type DatasetRow,\n type FileRef,\n getDatasetRowValidationError,\n isFileRef,\n type JsonValue,\n type Logger,\n type MergeChange,\n type SyncDefinition,\n} from \"@sixb/core\"\nimport { resolveLogsRuntime } from \"@sixb/core/internal/logging\"\nimport { type DatasetVersion, getDatasetMergeChangeValidationError } from \"@sixb/core/lake-storage\"\nimport type { SyncRunFailure, SyncRunRecord } from \"@sixb/core/storage\"\nimport { assertDatasetRow, normalizeReadResult, throwIfAborted } from \"./normalize\"\nimport type { RunSyncJobInput, SyncRunResult, SyncWorkerContext } from \"./types\"\n\nexport class SyncRunAlreadyStartedError extends Error {\n override readonly name = \"SyncRunAlreadyStartedError\"\n\n constructor(readonly run: SyncRunRecord) {\n super(`[SixbSyncWorker] Sync run '${run.id}' has already started.`)\n }\n}\n\nfunction toSyncRunFailure(error: unknown): SyncRunFailure {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n }\n }\n\n return {\n message: String(error),\n }\n}\n\nfunction createBookkeepingError(options: {\n syncId: string\n runId: string\n version: DatasetVersion\n cause: unknown\n}): Error {\n // The lake commit is already durable here, so we surface an explicit repair-needed error.\n return new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'. The dataset commit may already have succeeded and the sync run record may need repair.`,\n { cause: options.cause }\n )\n}\n\nfunction requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {\n if (finishedAt) {\n return finishedAt\n }\n\n throw new Error(`[SixbSyncWorker] Sync run '${runId}' finished without a finishedAt timestamp.`)\n}\n\nasync function verifyFileRef(options: {\n blobStorage: BlobStorage\n syncId: string\n datasetId: string\n columnName: string\n itemIndex: number\n fileRef: FileRef\n}): Promise<void> {\n const blobInfo = await options.blobStorage.stat(options.fileRef.blobId)\n if (!blobInfo) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing unknown blob '${options.fileRef.blobId}'.`\n )\n }\n\n if (blobInfo.digest !== options.fileRef.digest) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with digest '${options.fileRef.digest}', but blob storage has '${blobInfo.digest}'.`\n )\n }\n\n if (blobInfo.sizeBytes !== options.fileRef.sizeBytes) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned row ${options.itemIndex} with dataset '${options.datasetId}' column '${options.columnName}' referencing blob '${options.fileRef.blobId}' with size ${options.fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`\n )\n }\n}\n\nasync function verifyRowFileRefs(options: {\n blobStorage: BlobStorage\n syncId: string\n dataset: { id: string; schema: { columns: readonly { name: string; type: string }[] } }\n row: Record<string, unknown>\n itemIndex: number\n}): Promise<void> {\n const fileRefColumns = options.dataset.schema.columns.filter(\n (column) => column.type === \"fileRef\"\n )\n for (const column of fileRefColumns) {\n const value = options.row[column.name]\n if (!isFileRef(value)) {\n continue\n }\n\n await verifyFileRef({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n datasetId: options.dataset.id,\n columnName: column.name,\n itemIndex: options.itemIndex,\n fileRef: value,\n })\n }\n}\n\nfunction assertMergeChange(\n value: unknown,\n syncId: string,\n dataset: DatasetDefinition,\n itemIndex: number\n): MergeChange<DatasetRow, DatasetRow> {\n const validationError = getDatasetMergeChangeValidationError(value, dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`\n )\n }\n return value as MergeChange<DatasetRow, DatasetRow>\n}\n\nasync function readSyncValues(options: {\n runtime: SyncWorkerContext\n sync: SyncDefinition\n signal: AbortSignal\n blobStorage: BlobStorage\n logger: Logger\n previousCheckpoint: JsonValue | undefined\n setCheckpoint(next: unknown): void\n}): Promise<AsyncIterable<unknown>> {\n const client = await options.runtime.connector(options.sync.connector)\n const readResult = await options.sync.read(client, {\n projectId: options.runtime.id,\n syncId: options.sync.id,\n signal: options.signal,\n blobs: options.blobStorage,\n logger: options.logger,\n checkpoint:\n options.previousCheckpoint !== undefined\n ? cloneJsonValue(options.previousCheckpoint)\n : undefined,\n setCheckpoint: options.setCheckpoint,\n })\n return normalizeReadResult(readResult, options.sync.id)\n}\n\nasync function* validatedDatasetRows(options: {\n values: AsyncIterable<unknown>\n signal: AbortSignal\n blobStorage: BlobStorage\n syncId: string\n dataset: DatasetDefinition\n onRead(): void\n}): AsyncIterable<DatasetRow> {\n let itemIndex = 0\n for await (const value of options.values) {\n throwIfAborted(options.signal)\n itemIndex += 1\n\n const row = assertDatasetRow(value, options.syncId, itemIndex)\n // Validate before handing rows to lake storage so sync failures include\n // the source item index; lake storage still re-validates as the final boundary.\n const validationError = getDatasetRowValidationError(row, options.dataset)\n if (validationError) {\n throw new Error(\n `[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`\n )\n }\n\n // Lake storage validates fileRef shape; the worker owns existence checks against blob storage.\n await verifyRowFileRefs({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n dataset: options.dataset,\n row,\n itemIndex,\n })\n\n options.onRead()\n yield row\n }\n}\n\nasync function* validatedMergeChanges(options: {\n values: AsyncIterable<unknown>\n signal: AbortSignal\n blobStorage: BlobStorage\n syncId: string\n dataset: DatasetDefinition\n onRead(): void\n}): AsyncIterable<MergeChange<DatasetRow, DatasetRow>> {\n let itemIndex = 0\n for await (const value of options.values) {\n throwIfAborted(options.signal)\n itemIndex += 1\n const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex)\n if (mergeChange.kind === \"upsert\") {\n await verifyRowFileRefs({\n blobStorage: options.blobStorage,\n syncId: options.syncId,\n dataset: options.dataset,\n row: mergeChange.row,\n itemIndex,\n })\n }\n options.onRead()\n yield mergeChange\n }\n}\n\ninterface SyncCommitResult {\n readonly outcome: \"created\" | \"unchanged\"\n readonly version?: DatasetVersion\n}\n\n/**\n * Run one sync job end to end using a caller-supplied run id.\n *\n * The worker creates the operational sync-run record, streams source values into lake storage,\n * commits or reuses a dataset version when one exists, and then finalizes the run record. It does\n * not own scheduling, retries, or run-id generation.\n */\nexport async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {\n const { runtime, job } = input\n const signal = input.signal ?? new AbortController().signal\n const { syncRunsStorage, lakeStorage, blobStorage } = runtime\n\n const sync = runtime.getSyncById(job.syncId)\n if (!sync) {\n throw new Error(`[SixbSyncWorker] Unknown sync '${job.syncId}'.`)\n }\n\n const dataset = runtime.getDatasetById(sync.target.dataset.id)\n if (!dataset) {\n throw new Error(\n `[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${sync.target.dataset.id}'.`\n )\n }\n\n throwIfAborted(signal)\n\n let startedRun: SyncRunRecord\n try {\n startedRun = await syncRunsStorage.start({\n projectId: runtime.id,\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage: job.commitMessage,\n })\n } catch (error) {\n const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })\n if (existing?.syncId === sync.id && existing.datasetId === dataset.id) {\n throw new SyncRunAlreadyStartedError(existing)\n }\n throw error\n }\n await input.onRunStarted?.(startedRun)\n\n const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({\n kind: \"sync\",\n id: job.id,\n })\n const logger = logSession.logger\n let abortWrite: (() => Promise<void>) | undefined\n let rowsRead = 0\n let committedVersion: DatasetVersion | undefined\n\n try {\n throwIfAborted(signal)\n await lakeStorage.createDataset(dataset)\n\n const latestSuccessfulRuns = await syncRunsStorage.list({\n projectId: runtime.id,\n syncId: sync.id,\n statuses: [\"succeeded\"],\n limit: 1,\n order: \"desc\",\n })\n const previousCheckpoint = latestSuccessfulRuns.runs[0]?.checkpoint\n let nextCheckpoint: JsonValue | undefined =\n previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined\n\n const producer = {\n kind: \"sync\" as const,\n id: sync.id,\n runId: job.id,\n }\n const readValues = () =>\n readSyncValues({\n runtime,\n sync,\n signal,\n blobStorage,\n logger,\n previousCheckpoint,\n setCheckpoint(next) {\n assertJsonValue(next, `Sync '${sync.id}' checkpoint`)\n nextCheckpoint = cloneJsonValue(next)\n },\n })\n const onRead = () => {\n rowsRead += 1\n }\n const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`\n let commitResult: SyncCommitResult\n\n if (sync.config.mode === \"merge\") {\n const mergeWrite = await lakeStorage.beginMerge({\n dataset,\n expectedLatestVersionId: job.expectedLatestVersionId,\n producer,\n })\n abortWrite = () => mergeWrite.abort()\n\n const values = await readValues()\n throwIfAborted(signal)\n await mergeWrite.writeChanges(\n validatedMergeChanges({\n values,\n signal,\n blobStorage,\n syncId: sync.id,\n dataset,\n onRead,\n })\n )\n throwIfAborted(signal)\n\n const commit = await mergeWrite.commit({ commitMessage })\n abortWrite = undefined\n commitResult = {\n outcome: commit.outcome,\n ...(commit.version ? { version: commit.version } : {}),\n }\n } else {\n const values = await readValues()\n throwIfAborted(signal)\n\n const rowWrite = await lakeStorage.beginWrite({\n dataset,\n mode: sync.config.mode,\n producer,\n })\n abortWrite = () => rowWrite.abort()\n await rowWrite.writeRows(\n validatedDatasetRows({\n values,\n signal,\n blobStorage,\n syncId: sync.id,\n dataset,\n onRead,\n })\n )\n throwIfAborted(signal)\n\n if (rowsRead === 0) {\n const version = await lakeStorage.getLatestVersion(dataset.id)\n if (sync.config.mode === \"append\" || !version) {\n await rowWrite.abort()\n abortWrite = undefined\n const finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n\n return {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n ...(version ? { version } : {}),\n versionCreated: false,\n }\n }\n }\n\n const { outcome, ...version } = await rowWrite.commit({\n expectedLatestVersionId: job.expectedLatestVersionId,\n commitMessage,\n })\n abortWrite = undefined\n commitResult = { outcome, version }\n }\n\n const { outcome, version } = commitResult\n if (outcome === \"created\") {\n if (!version) {\n throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`)\n }\n committedVersion = version\n }\n let finishedRun: SyncRunRecord\n\n try {\n finishedRun = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n rowsRead,\n ...(version\n ? { output: { datasetId: version.datasetId, versionId: version.versionId } }\n : {}),\n checkpoint: nextCheckpoint,\n })\n } catch (error) {\n if (committedVersion) {\n throw createBookkeepingError({\n syncId: sync.id,\n runId: job.id,\n version: committedVersion,\n cause: error,\n })\n }\n throw error\n }\n\n const result = {\n id: job.id,\n syncId: sync.id,\n datasetId: dataset.id,\n mode: sync.config.mode,\n startedAt: startedRun.startedAt,\n finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),\n rowsRead,\n }\n return version\n ? { ...result, version, versionCreated: outcome === \"created\" }\n : { ...result, versionCreated: false }\n } catch (error) {\n if (!committedVersion) {\n // Without a created commit, best-effort clean up any open session and the run record.\n await abortWrite?.().catch(() => {})\n\n const status = signal.aborted ? \"cancelled\" : \"failed\"\n try {\n const run = await syncRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status,\n rowsRead,\n error: toSyncRunFailure(error),\n })\n if (status === \"failed\" && run.status === \"failed\") input.onRunFailed?.(error, run)\n } catch {\n // The run did not transition to the requested terminal status.\n }\n }\n\n throw error\n } finally {\n await logSession.flush()\n }\n}\n",
|
|
7
7
|
"import type { DatasetRow, SyncReadResult } from \"@sixb/core\"\n\nfunction isPlainObject(value: unknown): value is DatasetRow {\n if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n return false\n }\n\n const prototype = Object.getPrototypeOf(value)\n return prototype === Object.prototype || prototype === null\n}\n\nfunction isIterable(value: unknown): value is Iterable<unknown> {\n return (\n value !== null &&\n value !== undefined &&\n typeof (value as { [Symbol.iterator]?: unknown })[Symbol.iterator] === \"function\"\n )\n}\n\nfunction isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {\n return (\n value !== null &&\n value !== undefined &&\n typeof (value as { [Symbol.asyncIterator]?: unknown })[Symbol.asyncIterator] === \"function\"\n )\n}\n\nfunction toAbortError(signal: AbortSignal): Error {\n if (signal.reason instanceof Error) {\n return signal.reason\n }\n\n const message =\n typeof signal.reason === \"string\" && signal.reason.length > 0\n ? signal.reason\n : \"[SixbSyncWorker] Operation was cancelled.\"\n\n const error = new Error(message)\n error.name = \"AbortError\"\n return error\n}\n\nexport function throwIfAborted(signal: AbortSignal): void {\n if (signal.aborted) {\n throw toAbortError(signal)\n }\n}\n\nexport function assertDatasetRow(value: unknown, syncId: string, itemIndex: number): DatasetRow {\n if (isPlainObject(value)) {\n return value\n }\n\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an invalid row at item ${itemIndex}. Dataset rows must be plain objects.`\n )\n}\n\nexport async function* normalizeReadResult(\n readResult: SyncReadResult,\n syncId: string\n): AsyncIterable<unknown> {\n if (isAsyncIterable(readResult)) {\n yield* readResult\n return\n }\n\n if (isIterable(readResult)) {\n for (const item of readResult) {\n yield item\n }\n return\n }\n\n if (isPlainObject(readResult)) {\n yield readResult\n return\n }\n\n throw new Error(\n `[SixbSyncWorker] Sync '${syncId}' returned an unsupported read result. Expected a row object, iterable, or async iterable.`\n )\n}\n"
|
|
8
8
|
],
|
|
9
|
-
"mappings": ";;AAYA;AAGA;;;ACfA;AAAA;AAAA;AAAA;AAAA;AAAA;AASA;;;ACPA,SAAS,aAAa,CAAC,OAAqC;AAAA,EAC1D,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AAAA,IACvE,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,OAAO,eAAe,KAAK;AAAA,EAC7C,OAAO,cAAc,OAAO,aAAa,cAAc;AAAA;AAGzD,SAAS,UAAU,CAAC,OAA4C;AAAA,EAC9D,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA0C,OAAO,cAAc;AAAA;AAI3E,SAAS,eAAe,CAAC,OAAiD;AAAA,EACxE,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA+C,OAAO,mBAAmB;AAAA;AAIrF,SAAS,YAAY,CAAC,QAA4B;AAAA,EAChD,IAAI,OAAO,kBAAkB,OAAO;AAAA,IAClC,OAAO,OAAO;AAAA,EAChB;AAAA,EAEA,MAAM,UACJ,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,SAAS,IACxD,OAAO,SACP;AAAA,EAEN,MAAM,QAAQ,IAAI,MAAM,OAAO;AAAA,EAC/B,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGF,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,aAAa,MAAM;AAAA,EAC3B;AAAA;AAGK,SAAS,gBAAgB,CAAC,OAAgB,QAAgB,WAA+B;AAAA,EAC9F,IAAI,cAAc,KAAK,GAAG;AAAA,IACxB,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,2CAA2C,gDACvE;AAAA;AAGF,gBAAuB,mBAAmB,CACxC,YACA,QACwB;AAAA,EACxB,IAAI,gBAAgB,UAAU,GAAG;AAAA,IAC/B,OAAO;AAAA,IACP;AAAA,EACF;AAAA,EAEA,IAAI,WAAW,UAAU,GAAG;AAAA,IAC1B,WAAW,QAAQ,YAAY;AAAA,MAC7B,MAAM;AAAA,IACR;AAAA,IACA;AAAA,EACF;AAAA,EAEA,IAAI,cAAc,UAAU,GAAG;AAAA,IAC7B,MAAM;AAAA,IACN;AAAA,EACF;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,kGAC5B;AAAA;;;ADlEK,MAAM,mCAAmC,MAAM;AAAA,EAG/B;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAoB;AAAA,IACvC,MAAM,8BAA8B,IAAI,0BAA0B;AAAA,IAD/C;AAAA;AAGvB;AAEA,SAAS,gBAAgB,CAAC,OAAgC;AAAA,EACxD,IAAI,iBAAiB,OAAO;AAAA,IAC1B,OAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,OAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,EACvB;AAAA;AAGF,SAAS,sBAAsB,CAAC,SAKtB;AAAA,EAER,OAAO,IAAI,MACT,0BAA0B,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,kGAChJ,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGF,SAAS,iBAAiB,CAAC,OAAe,YAAoC;AAAA,EAC5E,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,8BAA8B,iDAAiD;AAAA;AAGjG,eAAe,aAAa,CAAC,SAOX;AAAA,EAChB,MAAM,WAAW,MAAM,QAAQ,YAAY,KAAK,QAAQ,QAAQ,MAAM;AAAA,EACtE,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,yCAAyC,QAAQ,QAAQ,UAC9L;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,WAAW,QAAQ,QAAQ,QAAQ;AAAA,IAC9C,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,wBAAwB,QAAQ,QAAQ,kCAAkC,SAAS,UACzQ;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,cAAc,QAAQ,QAAQ,WAAW;AAAA,IACpD,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,qBAAqB,QAAQ,QAAQ,mCAAmC,SAAS,YACvQ;AAAA,EACF;AAAA;AAGF,eAAe,iBAAiB,CAAC,SAMf;AAAA,EAChB,MAAM,iBAAiB,QAAQ,QAAQ,OAAO,QAAQ,OACpD,CAAC,WAAW,OAAO,SAAS,SAC9B;AAAA,EACA,WAAW,UAAU,gBAAgB;AAAA,IACnC,MAAM,QAAQ,QAAQ,IAAI,OAAO;AAAA,IACjC,IAAI,CAAC,UAAU,KAAK,GAAG;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,cAAc;AAAA,MAClB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,YAAY,OAAO;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA;AAUF,eAAsB,UAAU,CAAC,OAAgD;AAAA,EAC/E,QAAQ,SAAS,QAAQ;AAAA,EACzB,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,QAAQ,iBAAiB,aAAa,gBAAgB;AAAA,EAEtD,MAAM,OAAO,QAAQ,YAAY,IAAI,MAAM;AAAA,EAC3C,IAAI,CAAC,MAAM;AAAA,IACT,MAAM,IAAI,MAAM,kCAAkC,IAAI,UAAU;AAAA,EAClE;AAAA,EAEA,MAAM,UAAU,QAAQ,eAAe,KAAK,OAAO,QAAQ,EAAE;AAAA,EAC7D,IAAI,CAAC,SAAS;AAAA,IACZ,MAAM,IAAI,MACR,0BAA0B,KAAK,gCAAgC,KAAK,OAAO,QAAQ,MACrF;AAAA,EACF;AAAA,EAEA,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,aAAa,MAAM,gBAAgB,MAAM;AAAA,MACvC,WAAW,QAAQ;AAAA,MACnB,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,MAAM,WAAW,MAAM,gBAAgB,QAAQ,EAAE,WAAW,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC;AAAA,IACpF,IAAI,UAAU,WAAW,KAAK,MAAM,SAAS,cAAc,QAAQ,IAAI;AAAA,MACrE,MAAM,IAAI,2BAA2B,QAAQ;AAAA,IAC/C;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,MAAM,eAAe,UAAU;AAAA,EAErC,MAAM,aAAa,mBAAmB,QAAQ,IAAI,QAAQ,IAAI,EAAE,eAAe;AAAA,IAC7E,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,SAAS,WAAW;AAAA,EAC1B,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EACf,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,eAAe,MAAM;AAAA,IACrB,MAAM,YAAY,cAAc,OAAO;AAAA,IAEvC,MAAM,uBAAuB,MAAM,gBAAgB,KAAK;AAAA,MACtD,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,UAAU,CAAC,WAAW;AAAA,MACtB,OAAO;AAAA,MACP,OAAO;AAAA,IACT,CAAC;AAAA,IACD,MAAM,qBAAqB,qBAAqB,KAAK,IAAI;AAAA,IACzD,IAAI,iBACF,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,IAE1E,MAAM,SAAS,MAAM,QAAQ,UAAU,KAAK,SAAS;AAAA,IACrD,MAAM,aAAa,MAAM,KAAK,KAAK,QAAQ;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA,YAAY,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,MACpF,aAAa,CAAC,MAAe;AAAA,QAC3B,gBAAgB,MAAM,SAAS,KAAK,gBAAgB;AAAA,QACpD,iBAAiB,eAAe,IAAI;AAAA;AAAA,IAExC,CAAC;AAAA,IACD,MAAM,OAAO,oBAAoB,YAAY,KAAK,EAAE;AAAA,IAEpD,eAAe,MAAM;AAAA,IAErB,QAAQ,MAAM,YAAY,WAAW;AAAA,MACnC;AAAA,MACA,MAAM,KAAK,OAAO;AAAA,MAClB,UAAU;AAAA,QACR,MAAM;AAAA,QACN,IAAI,KAAK;AAAA,QACT,OAAO,IAAI;AAAA,MACb;AAAA,IACF,CAAC;AAAA,IAED,MAAM,MAAM,UACT,gBAAgB,GAAG;AAAA,MAClB,IAAI,YAAY;AAAA,MAEhB,iBAAiB,SAAS,MAAM;AAAA,QAC9B,eAAe,MAAM;AAAA,QACrB,aAAa;AAAA,QAEb,MAAM,MAAM,iBAAiB,OAAO,KAAK,IAAI,SAAS;AAAA,QAGtD,MAAM,kBAAkB,6BAA6B,KAAK,OAAO;AAAA,QACjE,IAAI,iBAAiB;AAAA,UACnB,MAAM,IAAI,MACR,0BAA0B,KAAK,uCAAuC,cAAc,iBACtF;AAAA,QACF;AAAA,QAGA,MAAM,kBAAkB;AAAA,UACtB;AAAA,UACA,QAAQ,KAAK;AAAA,UACb;AAAA,UACA;AAAA,UACA;AAAA,QACF,CAAC;AAAA,QAED,YAAY;AAAA,QACZ,MAAM;AAAA,MACR;AAAA,MACC,CACL;AAAA,IAEA,eAAe,MAAM;AAAA,IAErB,IAAI,KAAK,OAAO,SAAS,YAAY,aAAa,GAAG;AAAA,MACnD,MAAM,MAAM,MAAM;AAAA,MAClB,QAAQ;AAAA,MACR,MAAM,WAAU,MAAM,YAAY,iBAAiB,QAAQ,EAAE;AAAA,MAC7D,MAAM,eAAc,MAAM,gBAAgB,OAAO;AAAA,QAC/C,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,WACI,WACA,EAAE,QAAQ,EAAE,WAAW,SAAQ,WAAW,WAAW,SAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,QACL,YAAY;AAAA,MACd,CAAC;AAAA,MAED,OAAO;AAAA,QACL,IAAI,IAAI;AAAA,QACR,QAAQ,KAAK;AAAA,QACb,WAAW,QAAQ;AAAA,QACnB,MAAM,KAAK,OAAO;AAAA,QAClB,WAAW,WAAW;AAAA,QACtB,YAAY,kBAAkB,IAAI,IAAI,aAAY,UAAU;AAAA,QAC5D;AAAA,WACI,WAAU,EAAE,kBAAQ,IAAI,CAAC;AAAA,QAC7B,gBAAgB;AAAA,MAClB;AAAA,IACF;AAAA,IAEA,MAAM,SAAS,MAAM,MAAM,OAAO;AAAA,MAChC,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI,iBAAiB,QAAQ,KAAK,UAAU,IAAI;AAAA,IACjE,CAAC;AAAA,IACD,QAAQ,YAAY,YAAY;AAAA,IAChC,mBAAmB;AAAA,IACnB,IAAI;AAAA,IAEJ,IAAI;AAAA,MACF,cAAc,MAAM,gBAAgB,OAAO;AAAA,QACzC,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,QACA,QAAQ;AAAA,UACN,WAAW,QAAQ;AAAA,UACnB,WAAW,QAAQ;AAAA,QACrB;AAAA,QACA,YAAY;AAAA,MACd,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,uBAAuB;AAAA,QAC3B,QAAQ,KAAK;AAAA,QACb,OAAO,IAAI;AAAA,QACX;AAAA,QACA,OAAO;AAAA,MACT,CAAC;AAAA;AAAA,IAGH,OAAO;AAAA,MACL,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,YAAY,kBAAkB,IAAI,IAAI,YAAY,UAAU;AAAA,MAC5D;AAAA,MACA;AAAA,MACA,gBAAgB,YAAY;AAAA,IAC9B;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MAErB,MAAM,OAAO,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,MAEnC,MAAM,SAAS,OAAO,UAAU,cAAc;AAAA,MAC9C,IAAI;AAAA,QACF,MAAM,MAAM,MAAM,gBAAgB,OAAO;AAAA,UACvC,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,KAAK;AAAA,QAC/B,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW;AAAA,UAAU,MAAM,cAAc,OAAO,GAAG;AAAA,QAClF,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;;;ADzT3B,IAAM,SAAS;AAAA;AAkBR,MAAM,mBAAmB,YAAsC;AAAA,EACnD;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAAsB;AAAA,IAChC,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;AAAA,MACjC,MAAM,IAAI,MAAM,sDAAsD;AAAA,IACxE;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,UAAU,eAAe,KAAK;AAAA,IAChC,CAAC;AAAA,IAED,KAAK,UAAU,iBAAiB,IAAI;AAAA,IACpC,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,UAAmB;AAAA,MACvB,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,MACpB,yBAAyB,IAAI,QAAQ;AAAA,MACrC,eAAe,IAAI,QAAQ;AAAA,IAC7B;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,SAAS,MAAM,WAAW;AAAA,QACxB,SAAS,KAAK;AAAA,QACd,KAAK;AAAA,QACL;AAAA,QACA,cAAc,CAAC,QAAQ,mBAAmB,KAAK,MAAM,GAAG;AAAA,QACxD,aAAa,CAAC,OAAO,QAAQ;AAAA,UAC3B,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,YAAY,IAAI;AAAA,YAChB,SAAS,IAAI;AAAA,YACb,KAAK;AAAA,cACH,MAAM;AAAA,cACN,OAAO,IAAI;AAAA,cACX,QAAQ,IAAI;AAAA,YACd;AAAA,UACF,CAAC;AAAA;AAAA,MAEL,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAA4B;AAAA,MACjD,MAAM;AAAA;AAAA,IAGR,MAAM,wBAAwB,KAAK,MAAM,SAAS,MAAM;AAAA;AAAA,OAGjC,iBAAgB,CACvC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,oBAAoB,KAAK,MAAM;AAAA,MACnC,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,OAAO,EAAE,MAAM,OAAO;AAAA;AAE1B;AAEA,eAAe,kBAAkB,CAC/B,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,uBAAuB,CACpC,MACA,KACA,QACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN,GAAI,OAAO,iBACP;AAAA,QACE;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,WAAW,OAAO,QAAQ;AAAA,YAC1B,WAAW,OAAO,QAAQ,UAAU,YAAY;AAAA,YAChD,UAAU;AAAA,cACR,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,OAAO,IAAI;AAAA,YACb;AAAA,UACF;AAAA,QACF;AAAA,MACF,IACA,CAAC;AAAA,MACL;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,UACR,WAAW,OAAO;AAAA,aACd,OAAO,UAAU,EAAE,WAAW,OAAO,QAAQ,UAAU,IAAI,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,mBAAmB,CAChC,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,gBAAgB,CAAC,MAAyC;AAAA,EACjE,MAAM,kBAAkB,KAAK,QAAQ;AAAA,EACrC,IAAI,CAAC,iBAAiB;AAAA,IACpB,MAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AAAA,EAEA,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK;AAAA,IACX,WAAW,CAAC,QAAQ;AAAA,MAClB,OAAO,KAAK,YAAY,MAAM;AAAA;AAAA,IAEhC,cAAc,CAAC,WAAW;AAAA,MACxB,OAAO,KAAK,eAAe,SAAS;AAAA;AAAA,IAEtC,SAAS,CAAC,YAAY;AAAA,MACpB,OAAO,KAAK,UAAU,UAAU;AAAA;AAAA,EAEpC;AAAA;",
|
|
10
|
-
"debugId": "
|
|
9
|
+
"mappings": ";;AAYA;AAGA;;;ACfA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcA;AACA;;;ACbA,SAAS,aAAa,CAAC,OAAqC;AAAA,EAC1D,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AAAA,IACvE,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,OAAO,eAAe,KAAK;AAAA,EAC7C,OAAO,cAAc,OAAO,aAAa,cAAc;AAAA;AAGzD,SAAS,UAAU,CAAC,OAA4C;AAAA,EAC9D,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA0C,OAAO,cAAc;AAAA;AAI3E,SAAS,eAAe,CAAC,OAAiD;AAAA,EACxE,OACE,UAAU,QACV,UAAU,aACV,OAAQ,MAA+C,OAAO,mBAAmB;AAAA;AAIrF,SAAS,YAAY,CAAC,QAA4B;AAAA,EAChD,IAAI,OAAO,kBAAkB,OAAO;AAAA,IAClC,OAAO,OAAO;AAAA,EAChB;AAAA,EAEA,MAAM,UACJ,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,SAAS,IACxD,OAAO,SACP;AAAA,EAEN,MAAM,QAAQ,IAAI,MAAM,OAAO;AAAA,EAC/B,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGF,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,aAAa,MAAM;AAAA,EAC3B;AAAA;AAGK,SAAS,gBAAgB,CAAC,OAAgB,QAAgB,WAA+B;AAAA,EAC9F,IAAI,cAAc,KAAK,GAAG;AAAA,IACxB,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,2CAA2C,gDACvE;AAAA;AAGF,gBAAuB,mBAAmB,CACxC,YACA,QACwB;AAAA,EACxB,IAAI,gBAAgB,UAAU,GAAG;AAAA,IAC/B,OAAO;AAAA,IACP;AAAA,EACF;AAAA,EAEA,IAAI,WAAW,UAAU,GAAG;AAAA,IAC1B,WAAW,QAAQ,YAAY;AAAA,MAC7B,MAAM;AAAA,IACR;AAAA,IACA;AAAA,EACF;AAAA,EAEA,IAAI,cAAc,UAAU,GAAG;AAAA,IAC7B,MAAM;AAAA,IACN;AAAA,EACF;AAAA,EAEA,MAAM,IAAI,MACR,0BAA0B,kGAC5B;AAAA;;;AD7DK,MAAM,mCAAmC,MAAM;AAAA,EAG/B;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAoB;AAAA,IACvC,MAAM,8BAA8B,IAAI,0BAA0B;AAAA,IAD/C;AAAA;AAGvB;AAEA,SAAS,gBAAgB,CAAC,OAAgC;AAAA,EACxD,IAAI,iBAAiB,OAAO;AAAA,IAC1B,OAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,OAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,EACvB;AAAA;AAGF,SAAS,sBAAsB,CAAC,SAKtB;AAAA,EAER,OAAO,IAAI,MACT,0BAA0B,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,kGAChJ,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGF,SAAS,iBAAiB,CAAC,OAAe,YAAoC;AAAA,EAC5E,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,8BAA8B,iDAAiD;AAAA;AAGjG,eAAe,aAAa,CAAC,SAOX;AAAA,EAChB,MAAM,WAAW,MAAM,QAAQ,YAAY,KAAK,QAAQ,QAAQ,MAAM;AAAA,EACtE,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,yCAAyC,QAAQ,QAAQ,UAC9L;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,WAAW,QAAQ,QAAQ,QAAQ;AAAA,IAC9C,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,wBAAwB,QAAQ,QAAQ,kCAAkC,SAAS,UACzQ;AAAA,EACF;AAAA,EAEA,IAAI,SAAS,cAAc,QAAQ,QAAQ,WAAW;AAAA,IACpD,MAAM,IAAI,MACR,0BAA0B,QAAQ,wBAAwB,QAAQ,2BAA2B,QAAQ,sBAAsB,QAAQ,iCAAiC,QAAQ,QAAQ,qBAAqB,QAAQ,QAAQ,mCAAmC,SAAS,YACvQ;AAAA,EACF;AAAA;AAGF,eAAe,iBAAiB,CAAC,SAMf;AAAA,EAChB,MAAM,iBAAiB,QAAQ,QAAQ,OAAO,QAAQ,OACpD,CAAC,WAAW,OAAO,SAAS,SAC9B;AAAA,EACA,WAAW,UAAU,gBAAgB;AAAA,IACnC,MAAM,QAAQ,QAAQ,IAAI,OAAO;AAAA,IACjC,IAAI,CAAC,UAAU,KAAK,GAAG;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,cAAc;AAAA,MAClB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,YAAY,OAAO;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA;AAGF,SAAS,iBAAiB,CACxB,OACA,QACA,SACA,WACqC;AAAA,EACrC,MAAM,kBAAkB,qCAAqC,OAAO,OAAO;AAAA,EAC3E,IAAI,iBAAiB;AAAA,IACnB,MAAM,IAAI,MACR,0BAA0B,oDAAoD,cAAc,iBAC9F;AAAA,EACF;AAAA,EACA,OAAO;AAAA;AAGT,eAAe,cAAc,CAAC,SAQM;AAAA,EAClC,MAAM,SAAS,MAAM,QAAQ,QAAQ,UAAU,QAAQ,KAAK,SAAS;AAAA,EACrE,MAAM,aAAa,MAAM,QAAQ,KAAK,KAAK,QAAQ;AAAA,IACjD,WAAW,QAAQ,QAAQ;AAAA,IAC3B,QAAQ,QAAQ,KAAK;AAAA,IACrB,QAAQ,QAAQ;AAAA,IAChB,OAAO,QAAQ;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,YACE,QAAQ,uBAAuB,YAC3B,eAAe,QAAQ,kBAAkB,IACzC;AAAA,IACN,eAAe,QAAQ;AAAA,EACzB,CAAC;AAAA,EACD,OAAO,oBAAoB,YAAY,QAAQ,KAAK,EAAE;AAAA;AAGxD,gBAAgB,oBAAoB,CAAC,SAOP;AAAA,EAC5B,IAAI,YAAY;AAAA,EAChB,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,IACxC,eAAe,QAAQ,MAAM;AAAA,IAC7B,aAAa;AAAA,IAEb,MAAM,MAAM,iBAAiB,OAAO,QAAQ,QAAQ,SAAS;AAAA,IAG7D,MAAM,kBAAkB,6BAA6B,KAAK,QAAQ,OAAO;AAAA,IACzE,IAAI,iBAAiB;AAAA,MACnB,MAAM,IAAI,MACR,0BAA0B,QAAQ,2CAA2C,cAAc,iBAC7F;AAAA,IACF;AAAA,IAGA,MAAM,kBAAkB;AAAA,MACtB,aAAa,QAAQ;AAAA,MACrB,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IAED,QAAQ,OAAO;AAAA,IACf,MAAM;AAAA,EACR;AAAA;AAGF,gBAAgB,qBAAqB,CAAC,SAOiB;AAAA,EACrD,IAAI,YAAY;AAAA,EAChB,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,IACxC,eAAe,QAAQ,MAAM;AAAA,IAC7B,aAAa;AAAA,IACb,MAAM,cAAc,kBAAkB,OAAO,QAAQ,QAAQ,QAAQ,SAAS,SAAS;AAAA,IACvF,IAAI,YAAY,SAAS,UAAU;AAAA,MACjC,MAAM,kBAAkB;AAAA,QACtB,aAAa,QAAQ;AAAA,QACrB,QAAQ,QAAQ;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,KAAK,YAAY;AAAA,QACjB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,QAAQ,OAAO;AAAA,IACf,MAAM;AAAA,EACR;AAAA;AAeF,eAAsB,UAAU,CAAC,OAAgD;AAAA,EAC/E,QAAQ,SAAS,QAAQ;AAAA,EACzB,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,QAAQ,iBAAiB,aAAa,gBAAgB;AAAA,EAEtD,MAAM,OAAO,QAAQ,YAAY,IAAI,MAAM;AAAA,EAC3C,IAAI,CAAC,MAAM;AAAA,IACT,MAAM,IAAI,MAAM,kCAAkC,IAAI,UAAU;AAAA,EAClE;AAAA,EAEA,MAAM,UAAU,QAAQ,eAAe,KAAK,OAAO,QAAQ,EAAE;AAAA,EAC7D,IAAI,CAAC,SAAS;AAAA,IACZ,MAAM,IAAI,MACR,0BAA0B,KAAK,gCAAgC,KAAK,OAAO,QAAQ,MACrF;AAAA,EACF;AAAA,EAEA,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,aAAa,MAAM,gBAAgB,MAAM;AAAA,MACvC,WAAW,QAAQ;AAAA,MACnB,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,yBAAyB,IAAI;AAAA,MAC7B,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,MAAM,WAAW,MAAM,gBAAgB,QAAQ,EAAE,WAAW,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC;AAAA,IACpF,IAAI,UAAU,WAAW,KAAK,MAAM,SAAS,cAAc,QAAQ,IAAI;AAAA,MACrE,MAAM,IAAI,2BAA2B,QAAQ;AAAA,IAC/C;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,MAAM,eAAe,UAAU;AAAA,EAErC,MAAM,aAAa,mBAAmB,QAAQ,IAAI,QAAQ,IAAI,EAAE,eAAe;AAAA,IAC7E,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,SAAS,WAAW;AAAA,EAC1B,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EACf,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,eAAe,MAAM;AAAA,IACrB,MAAM,YAAY,cAAc,OAAO;AAAA,IAEvC,MAAM,uBAAuB,MAAM,gBAAgB,KAAK;AAAA,MACtD,WAAW,QAAQ;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,UAAU,CAAC,WAAW;AAAA,MACtB,OAAO;AAAA,MACP,OAAO;AAAA,IACT,CAAC;AAAA,IACD,MAAM,qBAAqB,qBAAqB,KAAK,IAAI;AAAA,IACzD,IAAI,iBACF,uBAAuB,YAAY,eAAe,kBAAkB,IAAI;AAAA,IAE1E,MAAM,WAAW;AAAA,MACf,MAAM;AAAA,MACN,IAAI,KAAK;AAAA,MACT,OAAO,IAAI;AAAA,IACb;AAAA,IACA,MAAM,aAAa,MACjB,eAAe;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,CAAC,MAAM;AAAA,QAClB,gBAAgB,MAAM,SAAS,KAAK,gBAAgB;AAAA,QACpD,iBAAiB,eAAe,IAAI;AAAA;AAAA,IAExC,CAAC;AAAA,IACH,MAAM,SAAS,MAAM;AAAA,MACnB,YAAY;AAAA;AAAA,IAEd,MAAM,gBAAgB,IAAI,iBAAiB,QAAQ,KAAK,UAAU,IAAI;AAAA,IACtE,IAAI;AAAA,IAEJ,IAAI,KAAK,OAAO,SAAS,SAAS;AAAA,MAChC,MAAM,aAAa,MAAM,YAAY,WAAW;AAAA,QAC9C;AAAA,QACA,yBAAyB,IAAI;AAAA,QAC7B;AAAA,MACF,CAAC;AAAA,MACD,aAAa,MAAM,WAAW,MAAM;AAAA,MAEpC,MAAM,SAAS,MAAM,WAAW;AAAA,MAChC,eAAe,MAAM;AAAA,MACrB,MAAM,WAAW,aACf,sBAAsB;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,KAAK;AAAA,QACb;AAAA,QACA;AAAA,MACF,CAAC,CACH;AAAA,MACA,eAAe,MAAM;AAAA,MAErB,MAAM,SAAS,MAAM,WAAW,OAAO,EAAE,cAAc,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,eAAe;AAAA,QACb,SAAS,OAAO;AAAA,WACZ,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,MACtD;AAAA,IACF,EAAO;AAAA,MACL,MAAM,SAAS,MAAM,WAAW;AAAA,MAChC,eAAe,MAAM;AAAA,MAErB,MAAM,WAAW,MAAM,YAAY,WAAW;AAAA,QAC5C;AAAA,QACA,MAAM,KAAK,OAAO;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,MACD,aAAa,MAAM,SAAS,MAAM;AAAA,MAClC,MAAM,SAAS,UACb,qBAAqB;AAAA,QACnB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,KAAK;AAAA,QACb;AAAA,QACA;AAAA,MACF,CAAC,CACH;AAAA,MACA,eAAe,MAAM;AAAA,MAErB,IAAI,aAAa,GAAG;AAAA,QAClB,MAAM,WAAU,MAAM,YAAY,iBAAiB,QAAQ,EAAE;AAAA,QAC7D,IAAI,KAAK,OAAO,SAAS,YAAY,CAAC,UAAS;AAAA,UAC7C,MAAM,SAAS,MAAM;AAAA,UACrB,aAAa;AAAA,UACb,MAAM,eAAc,MAAM,gBAAgB,OAAO;AAAA,YAC/C,WAAW,QAAQ;AAAA,YACnB,IAAI,IAAI;AAAA,YACR,QAAQ;AAAA,YACR;AAAA,eACI,WACA,EAAE,QAAQ,EAAE,WAAW,SAAQ,WAAW,WAAW,SAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,YACL,YAAY;AAAA,UACd,CAAC;AAAA,UAED,OAAO;AAAA,YACL,IAAI,IAAI;AAAA,YACR,QAAQ,KAAK;AAAA,YACb,WAAW,QAAQ;AAAA,YACnB,MAAM,KAAK,OAAO;AAAA,YAClB,WAAW,WAAW;AAAA,YACtB,YAAY,kBAAkB,IAAI,IAAI,aAAY,UAAU;AAAA,YAC5D;AAAA,eACI,WAAU,EAAE,kBAAQ,IAAI,CAAC;AAAA,YAC7B,gBAAgB;AAAA,UAClB;AAAA,QACF;AAAA,MACF;AAAA,MAEA,QAAQ,sBAAY,aAAY,MAAM,SAAS,OAAO;AAAA,QACpD,yBAAyB,IAAI;AAAA,QAC7B;AAAA,MACF,CAAC;AAAA,MACD,aAAa;AAAA,MACb,eAAe,EAAE,mBAAS,kBAAQ;AAAA;AAAA,IAGpC,QAAQ,SAAS,YAAY;AAAA,IAC7B,IAAI,YAAY,WAAW;AAAA,MACzB,IAAI,CAAC,SAAS;AAAA,QACZ,MAAM,IAAI,MAAM,0BAA0B,KAAK,iCAAiC;AAAA,MAClF;AAAA,MACA,mBAAmB;AAAA,IACrB;AAAA,IACA,IAAI;AAAA,IAEJ,IAAI;AAAA,MACF,cAAc,MAAM,gBAAgB,OAAO;AAAA,QACzC,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,WACI,UACA,EAAE,QAAQ,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU,EAAE,IACzE,CAAC;AAAA,QACL,YAAY;AAAA,MACd,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,kBAAkB;AAAA,QACpB,MAAM,uBAAuB;AAAA,UAC3B,QAAQ,KAAK;AAAA,UACb,OAAO,IAAI;AAAA,UACX,SAAS;AAAA,UACT,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,MACA,MAAM;AAAA;AAAA,IAGR,MAAM,SAAS;AAAA,MACb,IAAI,IAAI;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,WAAW,QAAQ;AAAA,MACnB,MAAM,KAAK,OAAO;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,YAAY,kBAAkB,IAAI,IAAI,YAAY,UAAU;AAAA,MAC5D;AAAA,IACF;AAAA,IACA,OAAO,UACH,KAAK,QAAQ,SAAS,gBAAgB,YAAY,UAAU,IAC5D,KAAK,QAAQ,gBAAgB,MAAM;AAAA,IACvC,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MAErB,MAAM,aAAa,EAAE,MAAM,MAAM,EAAE;AAAA,MAEnC,MAAM,SAAS,OAAO,UAAU,cAAc;AAAA,MAC9C,IAAI;AAAA,QACF,MAAM,MAAM,MAAM,gBAAgB,OAAO;AAAA,UACvC,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,KAAK;AAAA,QAC/B,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW;AAAA,UAAU,MAAM,cAAc,OAAO,GAAG;AAAA,QAClF,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;;;ADrc3B,IAAM,SAAS;AAAA;AAkBR,MAAM,mBAAmB,YAAsC;AAAA,EACnD;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAAsB;AAAA,IAChC,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;AAAA,MACjC,MAAM,IAAI,MAAM,sDAAsD;AAAA,IACxE;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,UAAU,eAAe,KAAK;AAAA,IAChC,CAAC;AAAA,IAED,KAAK,UAAU,iBAAiB,IAAI;AAAA,IACpC,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,UAAmB;AAAA,MACvB,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,MACpB,yBAAyB,IAAI,QAAQ;AAAA,MACrC,eAAe,IAAI,QAAQ;AAAA,IAC7B;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,SAAS,MAAM,WAAW;AAAA,QACxB,SAAS,KAAK;AAAA,QACd,KAAK;AAAA,QACL;AAAA,QACA,cAAc,CAAC,QAAQ,mBAAmB,KAAK,MAAM,GAAG;AAAA,QACxD,aAAa,CAAC,OAAO,QAAQ;AAAA,UAC3B,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,YAAY,IAAI;AAAA,YAChB,SAAS,IAAI;AAAA,YACb,KAAK;AAAA,cACH,MAAM;AAAA,cACN,OAAO,IAAI;AAAA,cACX,QAAQ,IAAI;AAAA,YACd;AAAA,UACF,CAAC;AAAA;AAAA,MAEL,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAA4B;AAAA,MACjD,MAAM;AAAA;AAAA,IAGR,MAAM,wBAAwB,KAAK,MAAM,SAAS,MAAM;AAAA;AAAA,OAGjC,iBAAgB,CACvC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,oBAAoB,KAAK,MAAM;AAAA,MACnC,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,QAAQ,IAAI,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,OAAO,EAAE,MAAM,OAAO;AAAA;AAE1B;AAEA,eAAe,kBAAkB,CAC/B,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,uBAAuB,CACpC,MACA,KACA,QACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN,GAAI,OAAO,iBACP;AAAA,QACE;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,WAAW,OAAO,QAAQ;AAAA,YAC1B,WAAW,OAAO,QAAQ,UAAU,YAAY;AAAA,YAChD,UAAU;AAAA,cACR,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,OAAO,IAAI;AAAA,YACb;AAAA,UACF;AAAA,QACF;AAAA,MACF,IACA,CAAC;AAAA,MACL;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,UACR,WAAW,OAAO;AAAA,aACd,OAAO,UAAU,EAAE,WAAW,OAAO,QAAQ,UAAU,IAAI,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAe,mBAAmB,CAChC,MACA,KACe;AAAA,EACf,MAAM,KAAK,QAAQ,KACjB;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,QAAQ,IAAI;AAAA,UACZ,OAAO,IAAI;AAAA,UACX,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,gBAAgB,CAAC,MAAyC;AAAA,EACjE,MAAM,kBAAkB,KAAK,QAAQ;AAAA,EACrC,IAAI,CAAC,iBAAiB;AAAA,IACpB,MAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AAAA,EAEA,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK;AAAA,IACX,WAAW,CAAC,QAAQ;AAAA,MAClB,OAAO,KAAK,YAAY,MAAM;AAAA;AAAA,IAEhC,cAAc,CAAC,WAAW;AAAA,MACxB,OAAO,KAAK,eAAe,SAAS;AAAA;AAAA,IAEtC,SAAS,CAAC,YAAY;AAAA,MACpB,OAAO,KAAK,UAAU,UAAU;AAAA;AAAA,EAEpC;AAAA;",
|
|
10
|
+
"debugId": "3B01117F7588DCC564756E2164756E21",
|
|
11
11
|
"names": []
|
|
12
12
|
}
|
package/dist/run-sync-job.d.ts
CHANGED
|
@@ -8,9 +8,9 @@ export declare class SyncRunAlreadyStartedError extends Error {
|
|
|
8
8
|
/**
|
|
9
9
|
* Run one sync job end to end using a caller-supplied run id.
|
|
10
10
|
*
|
|
11
|
-
* The worker creates the operational sync-run record, streams
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* The worker creates the operational sync-run record, streams source values into lake storage,
|
|
12
|
+
* commits or reuses a dataset version when one exists, and then finalizes the run record. It does
|
|
13
|
+
* not own scheduling, retries, or run-id generation.
|
|
14
14
|
*/
|
|
15
15
|
export declare function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>;
|
|
16
16
|
//# sourceMappingURL=run-sync-job.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-sync-job.d.ts","sourceRoot":"","sources":["../src/run-sync-job.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-sync-job.d.ts","sourceRoot":"","sources":["../src/run-sync-job.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAkB,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAEvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAqB,MAAM,SAAS,CAAA;AAEhF,qBAAa,0BAA2B,SAAQ,KAAK;IAGvC,QAAQ,CAAC,GAAG,EAAE,aAAa;IAFvC,SAAkB,IAAI,gCAA+B;gBAEhC,GAAG,EAAE,aAAa;CAGxC;AAwMD;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAmP/E"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { BlobStorage, ConnectorAdapter, ConnectorClient, ConnectorDefinition, DatasetDefinition, LakeStorage, SyncDefinition } from "@sixb/core";
|
|
1
|
+
import type { BlobStorage, ConnectorAdapter, ConnectorClient, ConnectorDefinition, DatasetDefinition, LakeStorage, SyncDefinition, SyncMode } from "@sixb/core";
|
|
2
2
|
import type { LogsRuntime } from "@sixb/core/internal/logging";
|
|
3
|
-
import type { DatasetVersion
|
|
3
|
+
import type { DatasetVersion } from "@sixb/core/lake-storage";
|
|
4
4
|
import type { SyncRunRecord, SyncRunStorage } from "@sixb/core/storage";
|
|
5
5
|
export interface SyncWorkerContext {
|
|
6
6
|
readonly id: string;
|
|
@@ -31,9 +31,10 @@ interface SyncRunResultBase {
|
|
|
31
31
|
readonly id: string;
|
|
32
32
|
readonly syncId: string;
|
|
33
33
|
readonly datasetId: string;
|
|
34
|
-
readonly mode:
|
|
34
|
+
readonly mode: SyncMode;
|
|
35
35
|
readonly startedAt: Date;
|
|
36
36
|
readonly finishedAt: Date;
|
|
37
|
+
/** Source items successfully consumed. Merge runs count both upserts and deletes. */
|
|
37
38
|
readonly rowsRead: number;
|
|
38
39
|
}
|
|
39
40
|
export type SyncRunResult = SyncRunResultBase & ({
|
|
@@ -41,7 +42,7 @@ export type SyncRunResult = SyncRunResultBase & ({
|
|
|
41
42
|
/** True only when this run created the returned dataset version. */
|
|
42
43
|
readonly versionCreated: boolean;
|
|
43
44
|
} | {
|
|
44
|
-
/** A
|
|
45
|
+
/** A successful initial no-op has no dataset version yet. */
|
|
45
46
|
readonly version?: undefined;
|
|
46
47
|
readonly versionCreated: false;
|
|
47
48
|
});
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,cAAc,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,cAAc,EACd,QAAQ,EACT,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAEvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAA;IACxC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAE3B,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;IAC3D,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAAA;IAElD,SAAS,CAAC,QAAQ,SAAS,gBAAgB,EACzC,UAAU,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,GAChD,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAA;CACtC;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAA;IACzC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,qBAAqB,CAAA;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,oBAAoB,CAAA;CAC5C;AAED,MAAM,MAAM,qBAAqB,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEhF,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAA;AAE/E,UAAU,iBAAiB;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAA;IACzB,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAC3C,CACI;IACE,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,oEAAoE;IACpE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC,GACD;IACE,6DAA6D;IAC7D,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAA;IAC5B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;CAC/B,CACJ,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sixb/sync-worker",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Worker that runs Sixb syncs",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sixb",
|
|
@@ -34,11 +34,12 @@
|
|
|
34
34
|
"test": "bun test",
|
|
35
35
|
"prepack": "bun run build"
|
|
36
36
|
},
|
|
37
|
-
"
|
|
38
|
-
"@sixb/core": "0.
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"@sixb/core": "0.1.1"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
|
-
"@sixb/
|
|
41
|
+
"@sixb/core": "0.1.1",
|
|
42
|
+
"@sixb/lake-local": "0.1.1",
|
|
42
43
|
"@types/bun": "latest",
|
|
43
44
|
"typescript": "^5.7.0"
|
|
44
45
|
},
|
package/src/run-sync-job.ts
CHANGED
|
@@ -2,16 +2,21 @@ import {
|
|
|
2
2
|
assertJsonValue,
|
|
3
3
|
type BlobStorage,
|
|
4
4
|
cloneJsonValue,
|
|
5
|
+
type DatasetDefinition,
|
|
6
|
+
type DatasetRow,
|
|
5
7
|
type FileRef,
|
|
6
8
|
getDatasetRowValidationError,
|
|
7
9
|
isFileRef,
|
|
8
10
|
type JsonValue,
|
|
11
|
+
type Logger,
|
|
12
|
+
type MergeChange,
|
|
13
|
+
type SyncDefinition,
|
|
9
14
|
} from "@sixb/core"
|
|
10
15
|
import { resolveLogsRuntime } from "@sixb/core/internal/logging"
|
|
11
|
-
import type
|
|
16
|
+
import { type DatasetVersion, getDatasetMergeChangeValidationError } from "@sixb/core/lake-storage"
|
|
12
17
|
import type { SyncRunFailure, SyncRunRecord } from "@sixb/core/storage"
|
|
13
18
|
import { assertDatasetRow, normalizeReadResult, throwIfAborted } from "./normalize"
|
|
14
|
-
import type { RunSyncJobInput, SyncRunResult } from "./types"
|
|
19
|
+
import type { RunSyncJobInput, SyncRunResult, SyncWorkerContext } from "./types"
|
|
15
20
|
|
|
16
21
|
export class SyncRunAlreadyStartedError extends Error {
|
|
17
22
|
override readonly name = "SyncRunAlreadyStartedError"
|
|
@@ -110,12 +115,121 @@ async function verifyRowFileRefs(options: {
|
|
|
110
115
|
}
|
|
111
116
|
}
|
|
112
117
|
|
|
118
|
+
function assertMergeChange(
|
|
119
|
+
value: unknown,
|
|
120
|
+
syncId: string,
|
|
121
|
+
dataset: DatasetDefinition,
|
|
122
|
+
itemIndex: number
|
|
123
|
+
): MergeChange<DatasetRow, DatasetRow> {
|
|
124
|
+
const validationError = getDatasetMergeChangeValidationError(value, dataset)
|
|
125
|
+
if (validationError) {
|
|
126
|
+
throw new Error(
|
|
127
|
+
`[SixbSyncWorker] Sync '${syncId}' returned an invalid merge change at item ${itemIndex}. ${validationError}`
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
return value as MergeChange<DatasetRow, DatasetRow>
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function readSyncValues(options: {
|
|
134
|
+
runtime: SyncWorkerContext
|
|
135
|
+
sync: SyncDefinition
|
|
136
|
+
signal: AbortSignal
|
|
137
|
+
blobStorage: BlobStorage
|
|
138
|
+
logger: Logger
|
|
139
|
+
previousCheckpoint: JsonValue | undefined
|
|
140
|
+
setCheckpoint(next: unknown): void
|
|
141
|
+
}): Promise<AsyncIterable<unknown>> {
|
|
142
|
+
const client = await options.runtime.connector(options.sync.connector)
|
|
143
|
+
const readResult = await options.sync.read(client, {
|
|
144
|
+
projectId: options.runtime.id,
|
|
145
|
+
syncId: options.sync.id,
|
|
146
|
+
signal: options.signal,
|
|
147
|
+
blobs: options.blobStorage,
|
|
148
|
+
logger: options.logger,
|
|
149
|
+
checkpoint:
|
|
150
|
+
options.previousCheckpoint !== undefined
|
|
151
|
+
? cloneJsonValue(options.previousCheckpoint)
|
|
152
|
+
: undefined,
|
|
153
|
+
setCheckpoint: options.setCheckpoint,
|
|
154
|
+
})
|
|
155
|
+
return normalizeReadResult(readResult, options.sync.id)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function* validatedDatasetRows(options: {
|
|
159
|
+
values: AsyncIterable<unknown>
|
|
160
|
+
signal: AbortSignal
|
|
161
|
+
blobStorage: BlobStorage
|
|
162
|
+
syncId: string
|
|
163
|
+
dataset: DatasetDefinition
|
|
164
|
+
onRead(): void
|
|
165
|
+
}): AsyncIterable<DatasetRow> {
|
|
166
|
+
let itemIndex = 0
|
|
167
|
+
for await (const value of options.values) {
|
|
168
|
+
throwIfAborted(options.signal)
|
|
169
|
+
itemIndex += 1
|
|
170
|
+
|
|
171
|
+
const row = assertDatasetRow(value, options.syncId, itemIndex)
|
|
172
|
+
// Validate before handing rows to lake storage so sync failures include
|
|
173
|
+
// the source item index; lake storage still re-validates as the final boundary.
|
|
174
|
+
const validationError = getDatasetRowValidationError(row, options.dataset)
|
|
175
|
+
if (validationError) {
|
|
176
|
+
throw new Error(
|
|
177
|
+
`[SixbSyncWorker] Sync '${options.syncId}' returned an invalid row at item ${itemIndex}. ${validationError}`
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Lake storage validates fileRef shape; the worker owns existence checks against blob storage.
|
|
182
|
+
await verifyRowFileRefs({
|
|
183
|
+
blobStorage: options.blobStorage,
|
|
184
|
+
syncId: options.syncId,
|
|
185
|
+
dataset: options.dataset,
|
|
186
|
+
row,
|
|
187
|
+
itemIndex,
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
options.onRead()
|
|
191
|
+
yield row
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async function* validatedMergeChanges(options: {
|
|
196
|
+
values: AsyncIterable<unknown>
|
|
197
|
+
signal: AbortSignal
|
|
198
|
+
blobStorage: BlobStorage
|
|
199
|
+
syncId: string
|
|
200
|
+
dataset: DatasetDefinition
|
|
201
|
+
onRead(): void
|
|
202
|
+
}): AsyncIterable<MergeChange<DatasetRow, DatasetRow>> {
|
|
203
|
+
let itemIndex = 0
|
|
204
|
+
for await (const value of options.values) {
|
|
205
|
+
throwIfAborted(options.signal)
|
|
206
|
+
itemIndex += 1
|
|
207
|
+
const mergeChange = assertMergeChange(value, options.syncId, options.dataset, itemIndex)
|
|
208
|
+
if (mergeChange.kind === "upsert") {
|
|
209
|
+
await verifyRowFileRefs({
|
|
210
|
+
blobStorage: options.blobStorage,
|
|
211
|
+
syncId: options.syncId,
|
|
212
|
+
dataset: options.dataset,
|
|
213
|
+
row: mergeChange.row,
|
|
214
|
+
itemIndex,
|
|
215
|
+
})
|
|
216
|
+
}
|
|
217
|
+
options.onRead()
|
|
218
|
+
yield mergeChange
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
interface SyncCommitResult {
|
|
223
|
+
readonly outcome: "created" | "unchanged"
|
|
224
|
+
readonly version?: DatasetVersion
|
|
225
|
+
}
|
|
226
|
+
|
|
113
227
|
/**
|
|
114
228
|
* Run one sync job end to end using a caller-supplied run id.
|
|
115
229
|
*
|
|
116
|
-
* The worker creates the operational sync-run record, streams
|
|
117
|
-
*
|
|
118
|
-
*
|
|
230
|
+
* The worker creates the operational sync-run record, streams source values into lake storage,
|
|
231
|
+
* commits or reuses a dataset version when one exists, and then finalizes the run record. It does
|
|
232
|
+
* not own scheduling, retries, or run-id generation.
|
|
119
233
|
*/
|
|
120
234
|
export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {
|
|
121
235
|
const { runtime, job } = input
|
|
@@ -161,7 +275,7 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
|
|
|
161
275
|
id: job.id,
|
|
162
276
|
})
|
|
163
277
|
const logger = logSession.logger
|
|
164
|
-
let
|
|
278
|
+
let abortWrite: (() => Promise<void>) | undefined
|
|
165
279
|
let rowsRead = 0
|
|
166
280
|
let committedVersion: DatasetVersion | undefined
|
|
167
281
|
|
|
@@ -180,102 +294,125 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
|
|
|
180
294
|
let nextCheckpoint: JsonValue | undefined =
|
|
181
295
|
previousCheckpoint !== undefined ? cloneJsonValue(previousCheckpoint) : undefined
|
|
182
296
|
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
297
|
+
const producer = {
|
|
298
|
+
kind: "sync" as const,
|
|
299
|
+
id: sync.id,
|
|
300
|
+
runId: job.id,
|
|
301
|
+
}
|
|
302
|
+
const readValues = () =>
|
|
303
|
+
readSyncValues({
|
|
304
|
+
runtime,
|
|
305
|
+
sync,
|
|
306
|
+
signal,
|
|
307
|
+
blobStorage,
|
|
308
|
+
logger,
|
|
309
|
+
previousCheckpoint,
|
|
310
|
+
setCheckpoint(next) {
|
|
311
|
+
assertJsonValue(next, `Sync '${sync.id}' checkpoint`)
|
|
312
|
+
nextCheckpoint = cloneJsonValue(next)
|
|
313
|
+
},
|
|
314
|
+
})
|
|
315
|
+
const onRead = () => {
|
|
316
|
+
rowsRead += 1
|
|
317
|
+
}
|
|
318
|
+
const commitMessage = job.commitMessage ?? `sync ${sync.id} run ${job.id}`
|
|
319
|
+
let commitResult: SyncCommitResult
|
|
320
|
+
|
|
321
|
+
if (sync.config.mode === "merge") {
|
|
322
|
+
const mergeWrite = await lakeStorage.beginMerge({
|
|
323
|
+
dataset,
|
|
324
|
+
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
325
|
+
producer,
|
|
326
|
+
})
|
|
327
|
+
abortWrite = () => mergeWrite.abort()
|
|
328
|
+
|
|
329
|
+
const values = await readValues()
|
|
330
|
+
throwIfAborted(signal)
|
|
331
|
+
await mergeWrite.writeChanges(
|
|
332
|
+
validatedMergeChanges({
|
|
333
|
+
values,
|
|
334
|
+
signal,
|
|
335
|
+
blobStorage,
|
|
336
|
+
syncId: sync.id,
|
|
337
|
+
dataset,
|
|
338
|
+
onRead,
|
|
339
|
+
})
|
|
340
|
+
)
|
|
341
|
+
throwIfAborted(signal)
|
|
342
|
+
|
|
343
|
+
const commit = await mergeWrite.commit({ commitMessage })
|
|
344
|
+
abortWrite = undefined
|
|
345
|
+
commitResult = {
|
|
346
|
+
outcome: commit.outcome,
|
|
347
|
+
...(commit.version ? { version: commit.version } : {}),
|
|
348
|
+
}
|
|
349
|
+
} else {
|
|
350
|
+
const values = await readValues()
|
|
351
|
+
throwIfAborted(signal)
|
|
227
352
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
353
|
+
const rowWrite = await lakeStorage.beginWrite({
|
|
354
|
+
dataset,
|
|
355
|
+
mode: sync.config.mode,
|
|
356
|
+
producer,
|
|
357
|
+
})
|
|
358
|
+
abortWrite = () => rowWrite.abort()
|
|
359
|
+
await rowWrite.writeRows(
|
|
360
|
+
validatedDatasetRows({
|
|
361
|
+
values,
|
|
362
|
+
signal,
|
|
363
|
+
blobStorage,
|
|
364
|
+
syncId: sync.id,
|
|
365
|
+
dataset,
|
|
366
|
+
onRead,
|
|
367
|
+
})
|
|
368
|
+
)
|
|
369
|
+
throwIfAborted(signal)
|
|
370
|
+
|
|
371
|
+
if (rowsRead === 0) {
|
|
372
|
+
const version = await lakeStorage.getLatestVersion(dataset.id)
|
|
373
|
+
if (sync.config.mode === "append" || !version) {
|
|
374
|
+
await rowWrite.abort()
|
|
375
|
+
abortWrite = undefined
|
|
376
|
+
const finishedRun = await syncRunsStorage.finish({
|
|
377
|
+
projectId: runtime.id,
|
|
378
|
+
id: job.id,
|
|
379
|
+
status: "succeeded",
|
|
380
|
+
rowsRead,
|
|
381
|
+
...(version
|
|
382
|
+
? { output: { datasetId: version.datasetId, versionId: version.versionId } }
|
|
383
|
+
: {}),
|
|
384
|
+
checkpoint: nextCheckpoint,
|
|
235
385
|
})
|
|
236
386
|
|
|
237
|
-
|
|
238
|
-
|
|
387
|
+
return {
|
|
388
|
+
id: job.id,
|
|
389
|
+
syncId: sync.id,
|
|
390
|
+
datasetId: dataset.id,
|
|
391
|
+
mode: sync.config.mode,
|
|
392
|
+
startedAt: startedRun.startedAt,
|
|
393
|
+
finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
|
|
394
|
+
rowsRead,
|
|
395
|
+
...(version ? { version } : {}),
|
|
396
|
+
versionCreated: false,
|
|
397
|
+
}
|
|
239
398
|
}
|
|
240
|
-
}
|
|
241
|
-
)
|
|
242
|
-
|
|
243
|
-
throwIfAborted(signal)
|
|
399
|
+
}
|
|
244
400
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
const version = await lakeStorage.getLatestVersion(dataset.id)
|
|
249
|
-
const finishedRun = await syncRunsStorage.finish({
|
|
250
|
-
projectId: runtime.id,
|
|
251
|
-
id: job.id,
|
|
252
|
-
status: "succeeded",
|
|
253
|
-
rowsRead,
|
|
254
|
-
...(version
|
|
255
|
-
? { output: { datasetId: version.datasetId, versionId: version.versionId } }
|
|
256
|
-
: {}),
|
|
257
|
-
checkpoint: nextCheckpoint,
|
|
401
|
+
const { outcome, ...version } = await rowWrite.commit({
|
|
402
|
+
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
403
|
+
commitMessage,
|
|
258
404
|
})
|
|
405
|
+
abortWrite = undefined
|
|
406
|
+
commitResult = { outcome, version }
|
|
407
|
+
}
|
|
259
408
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
mode: sync.config.mode,
|
|
265
|
-
startedAt: startedRun.startedAt,
|
|
266
|
-
finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
|
|
267
|
-
rowsRead,
|
|
268
|
-
...(version ? { version } : {}),
|
|
269
|
-
versionCreated: false,
|
|
409
|
+
const { outcome, version } = commitResult
|
|
410
|
+
if (outcome === "created") {
|
|
411
|
+
if (!version) {
|
|
412
|
+
throw new Error(`[SixbSyncWorker] Sync '${sync.id}' created no dataset version.`)
|
|
270
413
|
}
|
|
414
|
+
committedVersion = version
|
|
271
415
|
}
|
|
272
|
-
|
|
273
|
-
const commit = await write.commit({
|
|
274
|
-
expectedLatestVersionId: job.expectedLatestVersionId,
|
|
275
|
-
commitMessage: job.commitMessage ?? `sync ${sync.id} run ${job.id}`,
|
|
276
|
-
})
|
|
277
|
-
const { outcome, ...version } = commit
|
|
278
|
-
committedVersion = version
|
|
279
416
|
let finishedRun: SyncRunRecord
|
|
280
417
|
|
|
281
418
|
try {
|
|
@@ -284,22 +421,24 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
|
|
|
284
421
|
id: job.id,
|
|
285
422
|
status: "succeeded",
|
|
286
423
|
rowsRead,
|
|
287
|
-
|
|
288
|
-
datasetId: version.datasetId,
|
|
289
|
-
|
|
290
|
-
},
|
|
424
|
+
...(version
|
|
425
|
+
? { output: { datasetId: version.datasetId, versionId: version.versionId } }
|
|
426
|
+
: {}),
|
|
291
427
|
checkpoint: nextCheckpoint,
|
|
292
428
|
})
|
|
293
429
|
} catch (error) {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
430
|
+
if (committedVersion) {
|
|
431
|
+
throw createBookkeepingError({
|
|
432
|
+
syncId: sync.id,
|
|
433
|
+
runId: job.id,
|
|
434
|
+
version: committedVersion,
|
|
435
|
+
cause: error,
|
|
436
|
+
})
|
|
437
|
+
}
|
|
438
|
+
throw error
|
|
300
439
|
}
|
|
301
440
|
|
|
302
|
-
|
|
441
|
+
const result = {
|
|
303
442
|
id: job.id,
|
|
304
443
|
syncId: sync.id,
|
|
305
444
|
datasetId: dataset.id,
|
|
@@ -307,13 +446,14 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
|
|
|
307
446
|
startedAt: startedRun.startedAt,
|
|
308
447
|
finishedAt: requireFinishedAt(job.id, finishedRun.finishedAt),
|
|
309
448
|
rowsRead,
|
|
310
|
-
version,
|
|
311
|
-
versionCreated: outcome === "created",
|
|
312
449
|
}
|
|
450
|
+
return version
|
|
451
|
+
? { ...result, version, versionCreated: outcome === "created" }
|
|
452
|
+
: { ...result, versionCreated: false }
|
|
313
453
|
} catch (error) {
|
|
314
454
|
if (!committedVersion) {
|
|
315
|
-
//
|
|
316
|
-
await
|
|
455
|
+
// Without a created commit, best-effort clean up any open session and the run record.
|
|
456
|
+
await abortWrite?.().catch(() => {})
|
|
317
457
|
|
|
318
458
|
const status = signal.aborted ? "cancelled" : "failed"
|
|
319
459
|
try {
|
package/src/types.ts
CHANGED
|
@@ -6,9 +6,10 @@ import type {
|
|
|
6
6
|
DatasetDefinition,
|
|
7
7
|
LakeStorage,
|
|
8
8
|
SyncDefinition,
|
|
9
|
+
SyncMode,
|
|
9
10
|
} from "@sixb/core"
|
|
10
11
|
import type { LogsRuntime } from "@sixb/core/internal/logging"
|
|
11
|
-
import type { DatasetVersion
|
|
12
|
+
import type { DatasetVersion } from "@sixb/core/lake-storage"
|
|
12
13
|
import type { SyncRunRecord, SyncRunStorage } from "@sixb/core/storage"
|
|
13
14
|
|
|
14
15
|
export interface SyncWorkerContext {
|
|
@@ -49,9 +50,10 @@ interface SyncRunResultBase {
|
|
|
49
50
|
readonly id: string
|
|
50
51
|
readonly syncId: string
|
|
51
52
|
readonly datasetId: string
|
|
52
|
-
readonly mode:
|
|
53
|
+
readonly mode: SyncMode
|
|
53
54
|
readonly startedAt: Date
|
|
54
55
|
readonly finishedAt: Date
|
|
56
|
+
/** Source items successfully consumed. Merge runs count both upserts and deletes. */
|
|
55
57
|
readonly rowsRead: number
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -63,7 +65,7 @@ export type SyncRunResult = SyncRunResultBase &
|
|
|
63
65
|
readonly versionCreated: boolean
|
|
64
66
|
}
|
|
65
67
|
| {
|
|
66
|
-
/** A
|
|
68
|
+
/** A successful initial no-op has no dataset version yet. */
|
|
67
69
|
readonly version?: undefined
|
|
68
70
|
readonly versionCreated: false
|
|
69
71
|
}
|