@sixb/sync-worker 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -46
- package/dist/index.js +317 -240
- package/dist/index.js.map +5 -5
- package/dist/run-sync-job.d.ts +6 -9
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/types.d.ts +3 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +7 -11
- package/dist/worker.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/run-sync-job.ts +366 -172
- package/src/source-read.ts +30 -2
- package/src/types.ts +3 -2
- package/src/worker.ts +84 -131
package/src/worker.ts
CHANGED
|
@@ -1,30 +1,23 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
DomainEventLog,
|
|
3
|
-
LakeStorage,
|
|
4
|
-
Queues,
|
|
5
|
-
SixbDefinitions,
|
|
6
|
-
SixbFailure,
|
|
7
|
-
Storage,
|
|
8
|
-
} from "@sixb/core"
|
|
1
|
+
import type { DomainEventLog, LakeStorage, Queues, SixbDefinitions, Storage } from "@sixb/core"
|
|
9
2
|
import { reportRunFailure } from "@sixb/core/internal/error-reporting"
|
|
10
|
-
import { captureSixbFailure } from "@sixb/core/internal/errors"
|
|
11
3
|
import type { LoggingService } from "@sixb/core/internal/logging"
|
|
12
4
|
import {
|
|
13
5
|
bindDurablePrimitiveExecution,
|
|
14
6
|
type PrimitiveExecutionHost,
|
|
15
7
|
} from "@sixb/core/internal/primitive-execution"
|
|
16
8
|
import { resolveSyncConnectorSources } from "@sixb/core/internal/syncs"
|
|
17
|
-
import type
|
|
18
|
-
import { QueueWorker, retryUnstartedRun } from "@sixb/core/internal/workers"
|
|
9
|
+
import { type HeldRunClaim, RunWorker, wakeJobId } from "@sixb/core/internal/workers"
|
|
19
10
|
import type { DatasetVersion } from "@sixb/core/lake-storage"
|
|
20
11
|
import type { ClaimedQueueJob, SyncRunRequestedQueueJob } from "@sixb/core/queues"
|
|
21
12
|
import {
|
|
13
|
+
type RunClaim,
|
|
14
|
+
type RunExecution,
|
|
22
15
|
SYNC_RUN_FAILURE_CODES,
|
|
23
|
-
type SyncRunFailureCode,
|
|
24
16
|
type SyncRunRecord,
|
|
25
17
|
type SyncRunStatus,
|
|
18
|
+
type SyncRunStorage,
|
|
26
19
|
} from "@sixb/core/storage"
|
|
27
|
-
import { runSyncJob
|
|
20
|
+
import { runSyncJob } from "./run-sync-job"
|
|
28
21
|
import type { SyncWorkerContext } from "./types"
|
|
29
22
|
|
|
30
23
|
const SOURCE = "SixbSyncWorker"
|
|
@@ -43,9 +36,10 @@ export interface SyncWorkerOptions {
|
|
|
43
36
|
readonly concurrency?: number
|
|
44
37
|
}
|
|
45
38
|
|
|
46
|
-
export class SyncWorker extends
|
|
39
|
+
export class SyncWorker extends RunWorker<
|
|
47
40
|
SyncRunRequestedQueueJob,
|
|
48
|
-
typeof SYNC_RUN_FAILURE_CODES
|
|
41
|
+
typeof SYNC_RUN_FAILURE_CODES,
|
|
42
|
+
SyncRunRecord
|
|
49
43
|
> {
|
|
50
44
|
private readonly host: SyncWorkerHost
|
|
51
45
|
|
|
@@ -60,134 +54,96 @@ export class SyncWorker extends QueueWorker<
|
|
|
60
54
|
failureCodes: SYNC_RUN_FAILURE_CODES,
|
|
61
55
|
workerId: `sync-worker-${host.id}`,
|
|
62
56
|
claimLimit: options.concurrency,
|
|
57
|
+
runs: requireSyncRuns(host),
|
|
58
|
+
executions: host.storage.executions,
|
|
63
59
|
})
|
|
64
60
|
|
|
65
|
-
assertSyncStorage(host)
|
|
66
61
|
this.host = host
|
|
67
62
|
}
|
|
68
63
|
|
|
69
|
-
protected
|
|
64
|
+
protected claimRun(
|
|
70
65
|
claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,
|
|
66
|
+
execution: RunExecution
|
|
67
|
+
): Promise<RunClaim<SyncRunRecord>> {
|
|
68
|
+
return requireSyncRuns(this.host).startOrReclaim({
|
|
69
|
+
projectId: this.host.id,
|
|
70
|
+
id: claimed.job.payload.runId,
|
|
71
|
+
execution,
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
protected async executeRun(
|
|
76
|
+
claim: HeldRunClaim<SyncRunRecord>,
|
|
77
|
+
_claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,
|
|
71
78
|
signal: AbortSignal
|
|
72
79
|
): Promise<void> {
|
|
73
|
-
const {
|
|
74
|
-
const syncRuns = this.host.storage.syncRuns
|
|
75
|
-
if (!syncRuns) {
|
|
76
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.")
|
|
77
|
-
}
|
|
78
|
-
const run = await syncRuns.getById({ projectId: this.host.id, id: job.payload.runId })
|
|
79
|
-
if (!run) {
|
|
80
|
-
throw new Error(`[SixbSyncWorker] Sync run '${job.payload.runId}' was not found.`)
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// Each trigger that joined this run published its own job for it. Once one of them has started
|
|
84
|
-
// the run, the others have nothing left to do.
|
|
85
|
-
if (run.status !== "queued") return
|
|
80
|
+
const { run } = claim
|
|
86
81
|
|
|
87
|
-
const durableExecution = await this.
|
|
88
|
-
projectId: this.host.id,
|
|
89
|
-
id: run.executionId,
|
|
90
|
-
})
|
|
91
|
-
if (!durableExecution) {
|
|
92
|
-
throw new Error(
|
|
93
|
-
`[SixbSyncWorker] Sync run '${run.id}' references missing execution '${run.executionId}'.`
|
|
94
|
-
)
|
|
95
|
-
}
|
|
82
|
+
const durableExecution = await this.requireExecution(run)
|
|
96
83
|
|
|
97
84
|
const execution = bindDurablePrimitiveExecution(this.host, {
|
|
98
85
|
execution: durableExecution,
|
|
99
86
|
primitive: { kind: "sync", id: run.syncId, runId: run.id },
|
|
100
87
|
})
|
|
101
|
-
const context = buildSyncContext(this.host, execution.sixb)
|
|
102
88
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
} catch (error) {
|
|
131
|
-
if (error instanceof SyncRunAlreadyStartedError) return
|
|
132
|
-
throw error
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
protected override async onExecutionError(
|
|
137
|
-
claimed: ClaimedQueueJob<SyncRunRequestedQueueJob>,
|
|
138
|
-
error: unknown
|
|
139
|
-
): Promise<QueueWorkerFailureDecision<(typeof SYNC_RUN_FAILURE_CODES)[number]>> {
|
|
140
|
-
const run = await this.host.storage.syncRuns?.getById({
|
|
141
|
-
projectId: this.host.id,
|
|
142
|
-
id: claimed.job.payload.runId,
|
|
89
|
+
await runSyncJob({
|
|
90
|
+
runtime: buildSyncContext(this.host, execution.sixb),
|
|
91
|
+
claim,
|
|
92
|
+
signal,
|
|
93
|
+
onRunStarted: (started) =>
|
|
94
|
+
emitSyncRunStarted(this.host, started, durableExecution.correlationId),
|
|
95
|
+
onRunFinished: async (finished, createdVersion) => {
|
|
96
|
+
await emitSyncRunFinishedEvents(
|
|
97
|
+
this.host,
|
|
98
|
+
finished,
|
|
99
|
+
durableExecution.correlationId,
|
|
100
|
+
createdVersion
|
|
101
|
+
)
|
|
102
|
+
await this.wakeNextRun(finished)
|
|
103
|
+
},
|
|
104
|
+
onRunFailed: (error, run, failure) => {
|
|
105
|
+
reportRunFailure(this.host, error, {
|
|
106
|
+
projectId: this.host.id,
|
|
107
|
+
attempt: run.attempt,
|
|
108
|
+
runKind: "sync",
|
|
109
|
+
run: {
|
|
110
|
+
runId: run.id,
|
|
111
|
+
syncId: run.syncId,
|
|
112
|
+
},
|
|
113
|
+
failure,
|
|
114
|
+
})
|
|
115
|
+
},
|
|
143
116
|
})
|
|
144
|
-
if (run?.status === "queued") {
|
|
145
|
-
const retry = retryUnstartedRun(claimed.job.attempt)
|
|
146
|
-
if (retry) return retry
|
|
147
|
-
|
|
148
|
-
return { kind: "fail", failure: await this.failUnstartedRun(claimed, run, error) }
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
return run?.status === "failed" && run.error
|
|
152
|
-
? { kind: "fail", failure: run.error }
|
|
153
|
-
: { kind: "fail" }
|
|
154
117
|
}
|
|
155
118
|
|
|
156
|
-
/**
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if (!syncRuns) {
|
|
167
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.")
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
const failure = captureSixbFailure(error, {
|
|
171
|
-
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
172
|
-
defaultCode: "internal.unexpected",
|
|
173
|
-
details: { syncId: run.syncId, runId: run.id, datasetId: run.datasetId },
|
|
174
|
-
})
|
|
119
|
+
/** A run of a Sync that waited for this one need not wait for its next look. */
|
|
120
|
+
private async wakeNextRun(finished: SyncRunRecord): Promise<void> {
|
|
121
|
+
try {
|
|
122
|
+
const { runs } = await requireSyncRuns(this.host).list({
|
|
123
|
+
projectId: this.host.id,
|
|
124
|
+
syncId: finished.syncId,
|
|
125
|
+
statuses: ["queued"],
|
|
126
|
+
order: "asc",
|
|
127
|
+
limit: 1,
|
|
128
|
+
})
|
|
175
129
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
id: run.id,
|
|
179
|
-
status: "failed",
|
|
180
|
-
error: failure,
|
|
181
|
-
})
|
|
130
|
+
const [next] = runs
|
|
131
|
+
if (!next) return
|
|
182
132
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
133
|
+
await this.host.queues.syncRuns.enqueue({
|
|
134
|
+
projectId: this.host.id,
|
|
135
|
+
jobs: [
|
|
136
|
+
{
|
|
137
|
+
id: wakeJobId(next.id, finished.id),
|
|
138
|
+
type: "sync.run.requested",
|
|
139
|
+
payload: { runId: next.id },
|
|
140
|
+
},
|
|
141
|
+
],
|
|
142
|
+
})
|
|
143
|
+
} catch (error) {
|
|
144
|
+
// It looks again by itself, a little later.
|
|
145
|
+
console.error(`[SixbSyncWorker] Could not wake the run after '${finished.id}'.`, error)
|
|
146
|
+
}
|
|
191
147
|
}
|
|
192
148
|
}
|
|
193
149
|
|
|
@@ -224,7 +180,8 @@ async function emitSyncRunFinishedEvents(
|
|
|
224
180
|
createdVersion?: DatasetVersion
|
|
225
181
|
): Promise<void> {
|
|
226
182
|
// Sync owns notification after its run is finalized; the shared dataset writer returns the
|
|
227
|
-
// created version but does not publish events.
|
|
183
|
+
// created version but does not publish events. If these are lost, the `projections` and
|
|
184
|
+
// `triggers` passes find the version in the lake.
|
|
228
185
|
await host.events?.emit(
|
|
229
186
|
{
|
|
230
187
|
events: [
|
|
@@ -276,24 +233,20 @@ function requireTerminalStatus(
|
|
|
276
233
|
return status
|
|
277
234
|
}
|
|
278
235
|
|
|
279
|
-
function
|
|
236
|
+
function requireSyncRuns(host: SyncWorkerHost): SyncRunStorage {
|
|
280
237
|
if (!host.storage.syncRuns) {
|
|
281
238
|
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.")
|
|
282
239
|
}
|
|
240
|
+
return host.storage.syncRuns
|
|
283
241
|
}
|
|
284
242
|
|
|
285
243
|
function buildSyncContext(
|
|
286
244
|
host: SyncWorkerHost,
|
|
287
245
|
sixb: ReturnType<typeof bindDurablePrimitiveExecution>["sixb"]
|
|
288
246
|
): SyncWorkerContext {
|
|
289
|
-
const syncRunsStorage = host.storage.syncRuns
|
|
290
|
-
if (!syncRunsStorage) {
|
|
291
|
-
throw new Error("[SixbSyncWorker] Sync workers require storage.syncRuns support.")
|
|
292
|
-
}
|
|
293
|
-
|
|
294
247
|
return {
|
|
295
248
|
id: host.id,
|
|
296
|
-
syncRunsStorage,
|
|
249
|
+
syncRunsStorage: requireSyncRuns(host),
|
|
297
250
|
lakeStorage: host.lakeStorage,
|
|
298
251
|
blobs: sixb.blobs,
|
|
299
252
|
logging: host.logging,
|