@sixb/sync-worker 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -46
- package/dist/index.js +317 -240
- package/dist/index.js.map +5 -5
- package/dist/run-sync-job.d.ts +6 -9
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/types.d.ts +3 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +7 -11
- package/dist/worker.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/run-sync-job.ts +366 -172
- package/src/source-read.ts +30 -2
- package/src/types.ts +3 -2
- package/src/worker.ts +84 -131
package/src/run-sync-job.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
cloneJsonValue,
|
|
3
|
+
type DatasetDefinition,
|
|
4
|
+
type JsonValue,
|
|
5
|
+
type Logger,
|
|
6
|
+
type SyncDefinition,
|
|
7
|
+
} from "@sixb/core"
|
|
8
|
+
import { findProducedVersion, writeDataset } from "@sixb/core/internal/datasets"
|
|
3
9
|
import {
|
|
4
10
|
captureSixbFailure,
|
|
5
11
|
createSixbError,
|
|
@@ -7,30 +13,45 @@ import {
|
|
|
7
13
|
summarizeErrorMessage,
|
|
8
14
|
} from "@sixb/core/internal/errors"
|
|
9
15
|
import { resolveLoggingService } from "@sixb/core/internal/logging"
|
|
10
|
-
import type
|
|
16
|
+
import { type DatasetVersion, LakeConcurrencyError } from "@sixb/core/lake-storage"
|
|
11
17
|
import { SYNC_RUN_FAILURE_CODES, type SyncRunRecord } from "@sixb/core/storage"
|
|
12
18
|
import { throwIfAborted } from "./normalize"
|
|
13
19
|
import { readSyncValues, type SyncSourceValue } from "./source-read"
|
|
14
20
|
import type { RunSyncJobInput, SyncRunFinishedHandler, SyncRunResult } from "./types"
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
|
|
22
|
+
/** What the execution behind a commit read, which the run records once it finds that commit. */
|
|
23
|
+
interface LandedRead {
|
|
24
|
+
readonly rowsRead: number
|
|
25
|
+
readonly checkpoint: JsonValue | undefined
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What a run found just before committing: an earlier execution's commit landed after all. */
|
|
29
|
+
class CommitAlreadyLandedError extends Error {
|
|
30
|
+
override readonly name = "CommitAlreadyLandedError"
|
|
18
31
|
|
|
19
|
-
constructor(readonly
|
|
20
|
-
super(`[SixbSyncWorker]
|
|
32
|
+
constructor(readonly version: DatasetVersion) {
|
|
33
|
+
super(`[SixbSyncWorker] Version '${version.versionId}' already holds this run's rows.`)
|
|
21
34
|
}
|
|
22
35
|
}
|
|
23
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Another writer moved the dataset between this run's guard and its commit. Nothing was written, so
|
|
39
|
+
* the run stays running and the next delivery reads its source again.
|
|
40
|
+
*/
|
|
41
|
+
class CommitLostRaceError extends Error {
|
|
42
|
+
override readonly name = "CommitLostRaceError"
|
|
43
|
+
}
|
|
44
|
+
|
|
24
45
|
function createBookkeepingError(options: {
|
|
25
46
|
syncId: string
|
|
26
47
|
runId: string
|
|
27
48
|
version: DatasetVersion
|
|
28
49
|
cause: unknown
|
|
29
50
|
}): Error {
|
|
30
|
-
// The lake commit is already durable here,
|
|
51
|
+
// The lake commit is already durable here: the run stays running, and the next delivery finds it.
|
|
31
52
|
return createSixbError(
|
|
32
53
|
"internal.unexpected",
|
|
33
|
-
`[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'
|
|
54
|
+
`[SixbSyncWorker] Sync '${options.syncId}' committed dataset version '${options.version.versionId}', but failed to finalize sync run '${options.runId}'.`,
|
|
34
55
|
{
|
|
35
56
|
cause: options.cause,
|
|
36
57
|
details: {
|
|
@@ -113,22 +134,34 @@ function sourceValidationError(
|
|
|
113
134
|
})
|
|
114
135
|
}
|
|
115
136
|
|
|
137
|
+
/** What one execution of a run learns as it goes, for the steps below to record. */
|
|
138
|
+
interface SyncExecution {
|
|
139
|
+
readonly input: RunSyncJobInput
|
|
140
|
+
readonly run: SyncRunRecord
|
|
141
|
+
readonly signal: AbortSignal
|
|
142
|
+
rowsRead: number
|
|
143
|
+
nextCheckpoint: JsonValue | undefined
|
|
144
|
+
/** The version this execution knows the run has, once the lake and the run agree on it. */
|
|
145
|
+
committedVersion: DatasetVersion | undefined
|
|
146
|
+
/** When this run first reached a commit: the earliest a version of it can have landed. */
|
|
147
|
+
commitPreparedAt: Date | undefined
|
|
148
|
+
/** Whether a commit of this very execution is the one that may have landed. */
|
|
149
|
+
reachedCommit: boolean
|
|
150
|
+
}
|
|
151
|
+
|
|
116
152
|
/**
|
|
117
|
-
* Run one
|
|
153
|
+
* Run what is left of one Sync run this delivery holds, and record how it ended.
|
|
118
154
|
*
|
|
119
|
-
* Dispatch owns identity and queueing
|
|
120
|
-
*
|
|
155
|
+
* Dispatch owns identity and queueing, and the worker owns the claim. A run reads its whole source
|
|
156
|
+
* into one lake commit, so what is left is either that commit, from the last succeeded run's
|
|
157
|
+
* checkpoint, or nothing but its record: a run that resumes asks the lake which.
|
|
158
|
+
*
|
|
159
|
+
* A stopped worker writes nothing here: the run stays running, and the next delivery takes it up.
|
|
121
160
|
*/
|
|
122
161
|
export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult> {
|
|
123
|
-
const { runtime,
|
|
124
|
-
const
|
|
125
|
-
id: run.id,
|
|
126
|
-
syncId: run.syncId,
|
|
127
|
-
expectedLatestVersionId: run.expectedLatestVersionId,
|
|
128
|
-
commitMessage: run.commitMessage,
|
|
129
|
-
}
|
|
162
|
+
const { runtime, claim } = input
|
|
163
|
+
const { run } = claim
|
|
130
164
|
const signal = input.signal ?? new AbortController().signal
|
|
131
|
-
const { syncRunsStorage, lakeStorage, blobs: blobStorage } = runtime
|
|
132
165
|
|
|
133
166
|
if (run.projectId !== runtime.id) {
|
|
134
167
|
throw new Error(
|
|
@@ -138,183 +171,344 @@ export async function runSyncJob(input: RunSyncJobInput): Promise<SyncRunResult>
|
|
|
138
171
|
|
|
139
172
|
throwIfAborted(signal)
|
|
140
173
|
|
|
141
|
-
|
|
142
|
-
try {
|
|
143
|
-
startedRun = await syncRunsStorage.start({
|
|
144
|
-
projectId: runtime.id,
|
|
145
|
-
id: job.id,
|
|
146
|
-
})
|
|
147
|
-
} catch (error) {
|
|
148
|
-
const existing = await syncRunsStorage.getById({ projectId: runtime.id, id: job.id })
|
|
149
|
-
// A run still queued did not start elsewhere: the start itself failed.
|
|
150
|
-
if (
|
|
151
|
-
existing?.syncId === run.syncId &&
|
|
152
|
-
existing.executionId === run.executionId &&
|
|
153
|
-
existing.status !== "queued"
|
|
154
|
-
) {
|
|
155
|
-
throw new SyncRunAlreadyStartedError(existing)
|
|
156
|
-
}
|
|
157
|
-
throw error
|
|
158
|
-
}
|
|
159
|
-
await input.onRunStarted?.(startedRun)
|
|
174
|
+
if (claim.outcome === "started") await input.onRunStarted?.(run)
|
|
160
175
|
|
|
161
176
|
const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({
|
|
162
177
|
kind: "sync",
|
|
163
|
-
id:
|
|
178
|
+
id: run.id,
|
|
164
179
|
})
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
180
|
+
|
|
181
|
+
const execution: SyncExecution = {
|
|
182
|
+
input,
|
|
183
|
+
run,
|
|
184
|
+
signal,
|
|
185
|
+
rowsRead: 0,
|
|
186
|
+
nextCheckpoint: undefined,
|
|
187
|
+
committedVersion: undefined,
|
|
188
|
+
commitPreparedAt: run.commitPreparedAt,
|
|
189
|
+
reachedCommit: false,
|
|
190
|
+
}
|
|
168
191
|
|
|
169
192
|
try {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
|
|
175
|
-
throw new Error(
|
|
176
|
-
`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`
|
|
177
|
-
)
|
|
193
|
+
if (claim.outcome === "resumed") {
|
|
194
|
+
const landed = await findLandedCommit(execution)
|
|
195
|
+
if (landed) return await finishLanded(execution, landed, readByEarlierExecution(run))
|
|
178
196
|
}
|
|
179
197
|
|
|
180
|
-
|
|
181
|
-
if (!dataset) {
|
|
182
|
-
throw new Error(
|
|
183
|
-
`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`
|
|
184
|
-
)
|
|
185
|
-
}
|
|
198
|
+
if (claim.givenUp) throw claim.givenUp
|
|
186
199
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
200
|
+
return await commitRun(execution, logSession.logger)
|
|
201
|
+
} catch (caught) {
|
|
202
|
+
return await settleFailure(execution, caught)
|
|
203
|
+
} finally {
|
|
204
|
+
await logSession.flush()
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Reads the source from the last succeeded run's checkpoint into one commit. */
|
|
209
|
+
async function commitRun(execution: SyncExecution, logger: Logger): Promise<SyncRunResult> {
|
|
210
|
+
const { run, signal } = execution
|
|
211
|
+
const { runtime } = execution.input
|
|
212
|
+
const { lakeStorage, blobs: blobStorage } = runtime
|
|
213
|
+
|
|
214
|
+
const sync = requireSync(runtime, run)
|
|
215
|
+
const dataset = requireDataset(runtime, sync, run)
|
|
216
|
+
|
|
217
|
+
const previousCheckpoint = await readPreviousCheckpoint(runtime, sync)
|
|
218
|
+
execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint)
|
|
219
|
+
|
|
220
|
+
const { outcome, version } = await writeDataset({
|
|
221
|
+
lakeStorage,
|
|
222
|
+
blobStorage,
|
|
223
|
+
dataset,
|
|
224
|
+
mode: sync.config.mode,
|
|
225
|
+
signal,
|
|
226
|
+
producer: { kind: "sync", id: sync.id, runId: run.id },
|
|
227
|
+
readValues: () =>
|
|
204
228
|
readSyncValues({
|
|
205
229
|
runtime,
|
|
206
230
|
sync,
|
|
207
|
-
runId:
|
|
231
|
+
runId: run.id,
|
|
208
232
|
datasetId: dataset.id,
|
|
209
233
|
signal,
|
|
210
234
|
blobStorage,
|
|
211
235
|
logger,
|
|
212
236
|
previousCheckpoint,
|
|
213
237
|
setCheckpoint(next) {
|
|
214
|
-
nextCheckpoint =
|
|
238
|
+
execution.nextCheckpoint = cloneCheckpoint(next)
|
|
215
239
|
},
|
|
216
|
-
})
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
onRead(count) {
|
|
229
|
-
rowsRead = count
|
|
230
|
-
},
|
|
231
|
-
mapValidationError: (error, value, itemIndex) =>
|
|
232
|
-
sourceValidationError(error, value, { syncId: sync.id, runId: job.id, dataset }, itemIndex),
|
|
233
|
-
})
|
|
234
|
-
if (outcome === "created") {
|
|
235
|
-
committedVersion = version
|
|
236
|
-
}
|
|
237
|
-
let finishedRun: SyncRunRecord
|
|
238
|
-
|
|
239
|
-
try {
|
|
240
|
-
finishedRun = await syncRunsStorage.finish({
|
|
241
|
-
projectId: runtime.id,
|
|
242
|
-
id: job.id,
|
|
243
|
-
status: "succeeded",
|
|
244
|
-
rowsRead,
|
|
245
|
-
...(version
|
|
246
|
-
? { output: { datasetId: version.datasetId, versionId: version.versionId } }
|
|
247
|
-
: {}),
|
|
248
|
-
checkpoint: nextCheckpoint,
|
|
249
|
-
})
|
|
250
|
-
} catch (error) {
|
|
251
|
-
if (committedVersion) {
|
|
252
|
-
throw createBookkeepingError({
|
|
253
|
-
syncId: sync.id,
|
|
254
|
-
runId: job.id,
|
|
255
|
-
version: committedVersion,
|
|
256
|
-
cause: error,
|
|
257
|
-
})
|
|
258
|
-
}
|
|
259
|
-
throw error
|
|
260
|
-
}
|
|
240
|
+
}),
|
|
241
|
+
expectedLatestVersionId: run.expectedLatestVersionId,
|
|
242
|
+
commitMessage: run.commitMessage ?? `sync ${sync.id} run ${run.id}`,
|
|
243
|
+
sourceLabel: `[SixbSyncWorker] Sync '${sync.id}'`,
|
|
244
|
+
onRead(count) {
|
|
245
|
+
execution.rowsRead = count
|
|
246
|
+
},
|
|
247
|
+
mapValidationError: (error, value, itemIndex) =>
|
|
248
|
+
sourceValidationError(error, value, { syncId: sync.id, runId: run.id, dataset }, itemIndex),
|
|
249
|
+
guardLatestVersion: true,
|
|
250
|
+
beforeCommit: () => prepareCommit(execution),
|
|
251
|
+
})
|
|
261
252
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
253
|
+
if (outcome === "created") execution.committedVersion = version
|
|
254
|
+
|
|
255
|
+
return finishSucceeded(execution, {
|
|
256
|
+
version,
|
|
257
|
+
versionCreated: outcome === "created",
|
|
258
|
+
rowsRead: execution.rowsRead,
|
|
259
|
+
checkpoint: execution.nextCheckpoint,
|
|
260
|
+
})
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* A crash right after the commit would lose the checkpoint it paid for, so it goes first. A stale
|
|
265
|
+
* execution stops here; one that slept past this point meets the guard.
|
|
266
|
+
*/
|
|
267
|
+
async function prepareCommit(execution: SyncExecution): Promise<void> {
|
|
268
|
+
const { run } = execution
|
|
269
|
+
const { runtime, claim } = execution.input
|
|
270
|
+
|
|
271
|
+
const landed = await findLandedCommit(execution)
|
|
272
|
+
if (landed) throw new CommitAlreadyLandedError(landed)
|
|
273
|
+
|
|
274
|
+
const prepared = await runtime.syncRunsStorage.prepareCommit({
|
|
275
|
+
projectId: runtime.id,
|
|
276
|
+
id: run.id,
|
|
277
|
+
executionToken: claim.execution.token,
|
|
278
|
+
rowsRead: execution.rowsRead,
|
|
279
|
+
checkpoint: execution.nextCheckpoint,
|
|
280
|
+
})
|
|
281
|
+
|
|
282
|
+
execution.commitPreparedAt ??= prepared.commitPreparedAt
|
|
283
|
+
execution.reachedCommit = true
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** The lake and the run agree from here: what is left is only telling others. */
|
|
287
|
+
async function finishSucceeded(
|
|
288
|
+
execution: SyncExecution,
|
|
289
|
+
result: LandedRead & {
|
|
290
|
+
readonly version: DatasetVersion | null | undefined
|
|
291
|
+
readonly versionCreated: boolean
|
|
292
|
+
}
|
|
293
|
+
): Promise<SyncRunResult> {
|
|
294
|
+
const { run, committedVersion } = execution
|
|
295
|
+
const { version, versionCreated, rowsRead } = result
|
|
296
|
+
|
|
297
|
+
const finished = await recordSucceeded(execution, result)
|
|
298
|
+
|
|
299
|
+
const finishedAt = requireFinishedAt({
|
|
300
|
+
syncId: run.syncId,
|
|
301
|
+
runId: run.id,
|
|
302
|
+
finishedAt: finished.finishedAt,
|
|
303
|
+
})
|
|
304
|
+
|
|
305
|
+
await notifyRunFinished(execution.input.onRunFinished, finished, committedVersion)
|
|
306
|
+
|
|
307
|
+
const summary = {
|
|
308
|
+
id: run.id,
|
|
309
|
+
syncId: run.syncId,
|
|
310
|
+
datasetId: run.datasetId,
|
|
311
|
+
mode: run.mode,
|
|
312
|
+
startedAt: requireStartedAt(run.id, run.startedAt),
|
|
313
|
+
finishedAt,
|
|
314
|
+
rowsRead,
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return version ? { ...summary, version, versionCreated } : { ...summary, versionCreated: false }
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
async function recordSucceeded(
|
|
321
|
+
execution: SyncExecution,
|
|
322
|
+
result: LandedRead & { readonly version: DatasetVersion | null | undefined }
|
|
323
|
+
): Promise<SyncRunRecord> {
|
|
324
|
+
const { run, committedVersion } = execution
|
|
325
|
+
const { runtime, claim } = execution.input
|
|
326
|
+
const { version, rowsRead, checkpoint } = result
|
|
327
|
+
|
|
328
|
+
try {
|
|
329
|
+
return await runtime.syncRunsStorage.finish({
|
|
330
|
+
projectId: runtime.id,
|
|
331
|
+
id: run.id,
|
|
332
|
+
executionToken: claim.execution.token,
|
|
333
|
+
status: "succeeded",
|
|
275
334
|
rowsRead,
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
335
|
+
...(version
|
|
336
|
+
? { output: { datasetId: version.datasetId, versionId: version.versionId } }
|
|
337
|
+
: {}),
|
|
338
|
+
checkpoint,
|
|
339
|
+
})
|
|
280
340
|
} catch (error) {
|
|
281
|
-
if (!committedVersion)
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
341
|
+
if (!committedVersion) throw error
|
|
342
|
+
|
|
343
|
+
throw createBookkeepingError({
|
|
344
|
+
syncId: run.syncId,
|
|
345
|
+
runId: run.id,
|
|
346
|
+
version: committedVersion,
|
|
347
|
+
cause: error,
|
|
348
|
+
})
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Records a commit this run made, with what the execution that made it read. */
|
|
353
|
+
function finishLanded(
|
|
354
|
+
execution: SyncExecution,
|
|
355
|
+
version: DatasetVersion,
|
|
356
|
+
read: LandedRead
|
|
357
|
+
): Promise<SyncRunResult> {
|
|
358
|
+
execution.committedVersion = version
|
|
359
|
+
|
|
360
|
+
return finishSucceeded(execution, { version, versionCreated: true, ...read })
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** Only an execution holding the run could prepare a commit, so none has moved these since. */
|
|
364
|
+
function readByEarlierExecution(run: SyncRunRecord): LandedRead {
|
|
365
|
+
return { rowsRead: run.rowsRead ?? 0, checkpoint: run.checkpoint }
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* A commit can land and still answer with an error, so only the lake can say what happened. If it
|
|
370
|
+
* cannot say either, this throws, and the run stays open for the next delivery to ask.
|
|
371
|
+
*/
|
|
372
|
+
async function settleFailure(execution: SyncExecution, caught: unknown): Promise<SyncRunResult> {
|
|
373
|
+
const { run, signal, committedVersion } = execution
|
|
374
|
+
|
|
375
|
+
const landed = committedVersion ? null : await landedCommitBehind(execution, caught)
|
|
376
|
+
if (landed) return finishLanded(execution, landed, landedRead(execution, caught))
|
|
377
|
+
|
|
378
|
+
const error = translateLostRace(run, caught)
|
|
379
|
+
|
|
380
|
+
if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
|
|
381
|
+
await recordFailed(execution, error)
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
throw error
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** What the execution behind a landed commit read: this one, if the commit is its own. */
|
|
388
|
+
function landedRead(execution: SyncExecution, caught: unknown): LandedRead {
|
|
389
|
+
const lostItsRace = caught instanceof LakeConcurrencyError
|
|
390
|
+
const own =
|
|
391
|
+
execution.reachedCommit && !lostItsRace && !(caught instanceof CommitAlreadyLandedError)
|
|
392
|
+
|
|
393
|
+
return own
|
|
394
|
+
? { rowsRead: execution.rowsRead, checkpoint: execution.nextCheckpoint }
|
|
395
|
+
: readByEarlierExecution(execution.run)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** The shared writer has already cleaned up its session. Finalize the failed sync run. */
|
|
399
|
+
async function recordFailed(execution: SyncExecution, error: unknown): Promise<void> {
|
|
400
|
+
const { run, rowsRead } = execution
|
|
401
|
+
const { runtime, claim, onRunFinished, onRunFailed } = execution.input
|
|
402
|
+
|
|
403
|
+
try {
|
|
404
|
+
const failure = captureSixbFailure(
|
|
405
|
+
translateSyncExecutionError(error, {
|
|
406
|
+
syncId: run.syncId,
|
|
407
|
+
runId: run.id,
|
|
408
|
+
datasetId: run.datasetId,
|
|
409
|
+
}),
|
|
410
|
+
{
|
|
411
|
+
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
412
|
+
defaultCode: "internal.unexpected",
|
|
413
|
+
details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId },
|
|
311
414
|
}
|
|
312
|
-
|
|
415
|
+
)
|
|
313
416
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
417
|
+
const failedRun = await runtime.syncRunsStorage.finish({
|
|
418
|
+
projectId: runtime.id,
|
|
419
|
+
id: run.id,
|
|
420
|
+
executionToken: claim.execution.token,
|
|
421
|
+
status: "failed",
|
|
422
|
+
rowsRead,
|
|
423
|
+
error: failure,
|
|
424
|
+
})
|
|
425
|
+
|
|
426
|
+
await notifyRunFinished(onRunFinished, failedRun)
|
|
427
|
+
onRunFailed?.(error, failedRun, failure)
|
|
428
|
+
} catch {
|
|
429
|
+
// The run did not transition: a later delivery, or the execution that holds it, ends it.
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function cloneCheckpoint(checkpoint: JsonValue | undefined): JsonValue | undefined {
|
|
434
|
+
return checkpoint === undefined ? undefined : cloneJsonValue(checkpoint)
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
async function readPreviousCheckpoint(
|
|
438
|
+
runtime: RunSyncJobInput["runtime"],
|
|
439
|
+
sync: SyncDefinition
|
|
440
|
+
): Promise<JsonValue | undefined> {
|
|
441
|
+
const { runs } = await runtime.syncRunsStorage.list({
|
|
442
|
+
projectId: runtime.id,
|
|
443
|
+
syncId: sync.id,
|
|
444
|
+
statuses: ["succeeded"],
|
|
445
|
+
limit: 1,
|
|
446
|
+
order: "desc",
|
|
447
|
+
})
|
|
448
|
+
|
|
449
|
+
return runs[0]?.checkpoint
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function requireDataset(
|
|
453
|
+
runtime: RunSyncJobInput["runtime"],
|
|
454
|
+
sync: SyncDefinition,
|
|
455
|
+
run: SyncRunRecord
|
|
456
|
+
): DatasetDefinition {
|
|
457
|
+
const dataset = runtime.datasets.getById(run.datasetId)
|
|
458
|
+
if (dataset) return dataset
|
|
459
|
+
|
|
460
|
+
throw new Error(`[SixbSyncWorker] Sync '${sync.id}' targets unknown dataset '${run.datasetId}'.`)
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
function requireSync(runtime: RunSyncJobInput["runtime"], run: SyncRunRecord): SyncDefinition {
|
|
464
|
+
const sync = runtime.syncs.getById(run.syncId)
|
|
465
|
+
if (!sync) {
|
|
466
|
+
throw new Error(`[SixbSyncWorker] Unknown sync '${run.syncId}'.`)
|
|
467
|
+
}
|
|
468
|
+
if (sync.target.dataset.id !== run.datasetId || sync.config.mode !== run.mode) {
|
|
469
|
+
throw new Error(
|
|
470
|
+
`[SixbSyncWorker] Sync run '${run.id}' no longer matches Sync '${sync.id}' configuration.`
|
|
471
|
+
)
|
|
317
472
|
}
|
|
473
|
+
|
|
474
|
+
return sync
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** The commit an error says landed, or that the lake holds in spite of the error. */
|
|
478
|
+
async function landedCommitBehind(
|
|
479
|
+
execution: SyncExecution,
|
|
480
|
+
error: unknown
|
|
481
|
+
): Promise<DatasetVersion | null> {
|
|
482
|
+
if (error instanceof CommitAlreadyLandedError) return error.version
|
|
483
|
+
|
|
484
|
+
return findLandedCommit(execution)
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** A guard this run asked for failed on purpose. Its own guard only lost to another writer. */
|
|
488
|
+
function translateLostRace(run: SyncRunRecord, error: unknown): unknown {
|
|
489
|
+
if (!(error instanceof LakeConcurrencyError) || run.expectedLatestVersionId !== undefined) {
|
|
490
|
+
return error
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
return new CommitLostRaceError(`[SixbSyncWorker] Sync run '${run.id}' lost its commit race.`, {
|
|
494
|
+
cause: error,
|
|
495
|
+
})
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* The version this run committed, if one did. A run that never prepared a commit has none: nothing
|
|
500
|
+
* of a partial run exists until then, rows stay in a session no reader sees, and it need not ask.
|
|
501
|
+
*/
|
|
502
|
+
async function findLandedCommit(execution: SyncExecution): Promise<DatasetVersion | null> {
|
|
503
|
+
const { run, commitPreparedAt, input } = execution
|
|
504
|
+
if (!commitPreparedAt) return null
|
|
505
|
+
|
|
506
|
+
return findProducedVersion({
|
|
507
|
+
lakeStorage: input.runtime.lakeStorage,
|
|
508
|
+
datasetId: run.datasetId,
|
|
509
|
+
producer: { kind: "sync", runId: run.id },
|
|
510
|
+
since: commitPreparedAt,
|
|
511
|
+
})
|
|
318
512
|
}
|
|
319
513
|
|
|
320
514
|
async function notifyRunFinished(
|
package/src/source-read.ts
CHANGED
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
type SyncDefinition,
|
|
9
9
|
} from "@sixb/core"
|
|
10
10
|
import { isOAuthConnectorDefinition } from "@sixb/core/internal/connector-connections"
|
|
11
|
-
import { createSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors"
|
|
11
|
+
import { createSixbError, isSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors"
|
|
12
12
|
import type { SyncConnectorSource } from "@sixb/core/internal/syncs"
|
|
13
13
|
import {
|
|
14
14
|
ConnectionCheckpoints,
|
|
@@ -149,7 +149,17 @@ async function* readManagedSource(
|
|
|
149
149
|
const { connection } = source
|
|
150
150
|
try {
|
|
151
151
|
throwIfAborted(options.signal)
|
|
152
|
-
|
|
152
|
+
let client: unknown
|
|
153
|
+
try {
|
|
154
|
+
client = await runAbortable(options.signal, () => source.connect(options.signal))
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (!skipsUnauthorizedConnection(options.sync, connection, error)) throw error
|
|
157
|
+
// The connection keeps its checkpoint and its rows; its owner sees it needs reconnecting.
|
|
158
|
+
console.warn(
|
|
159
|
+
`[SixbSyncWorker] Sync '${options.sync.id}' skipped connection '${connection.id}' (${connection.account.label}): it needs reconnecting.`
|
|
160
|
+
)
|
|
161
|
+
return
|
|
162
|
+
}
|
|
153
163
|
throwIfAborted(options.signal)
|
|
154
164
|
const previous = checkpoints.get(connection)
|
|
155
165
|
const readResult = await runAbortable(options.signal, () =>
|
|
@@ -196,6 +206,24 @@ async function* readManagedSource(
|
|
|
196
206
|
}
|
|
197
207
|
}
|
|
198
208
|
|
|
209
|
+
/**
|
|
210
|
+
* One person's account that needs reconnecting must not fail everyone's run. A snapshot run still
|
|
211
|
+
* fails: leaving the account out would drop its rows from the dataset. A project connection still
|
|
212
|
+
* fails the run, so the project's own expired grant reaches `onError` as it always has.
|
|
213
|
+
*/
|
|
214
|
+
function skipsUnauthorizedConnection(
|
|
215
|
+
sync: SyncDefinition,
|
|
216
|
+
connection: SyncConnectorConnection,
|
|
217
|
+
error: unknown
|
|
218
|
+
): boolean {
|
|
219
|
+
return (
|
|
220
|
+
sync.config.mode !== "snapshot" &&
|
|
221
|
+
connection.owner.type === "user" &&
|
|
222
|
+
isSixbError(error) &&
|
|
223
|
+
error.code === "connector.authorization_required"
|
|
224
|
+
)
|
|
225
|
+
}
|
|
226
|
+
|
|
199
227
|
function sourceInvariant(syncId: string, message: string): Error {
|
|
200
228
|
return createSixbError("internal.unexpected", `[SixbSyncWorker] ${message}`, {
|
|
201
229
|
details: { syncId },
|
package/src/types.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BlobStorage, LakeStorage, SixbDefinitions, SixbFailure, SyncMode } from "@sixb/core"
|
|
2
2
|
import type { LoggingService } from "@sixb/core/internal/logging"
|
|
3
3
|
import type { SyncConnectorSourceResolver } from "@sixb/core/internal/syncs"
|
|
4
|
+
import type { HeldRunClaim } from "@sixb/core/internal/workers"
|
|
4
5
|
import type { DatasetVersion } from "@sixb/core/lake-storage"
|
|
5
6
|
import type { SyncRunFailureCode, SyncRunRecord, SyncRunStorage } from "@sixb/core/storage"
|
|
6
7
|
|
|
@@ -18,8 +19,8 @@ export interface SyncWorkerContext {
|
|
|
18
19
|
|
|
19
20
|
export interface RunSyncJobInput {
|
|
20
21
|
readonly runtime: SyncWorkerContext
|
|
21
|
-
/**
|
|
22
|
-
readonly
|
|
22
|
+
/** The run this delivery holds, and whether an earlier execution already worked on it. */
|
|
23
|
+
readonly claim: HeldRunClaim<SyncRunRecord>
|
|
23
24
|
readonly signal?: AbortSignal
|
|
24
25
|
readonly onRunStarted?: SyncRunStartedHandler
|
|
25
26
|
readonly onRunFinished?: SyncRunFinishedHandler
|