@sixb/sync-worker 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/checkpoints.d.ts +11 -18
- package/dist/checkpoints.d.ts.map +1 -1
- package/dist/index.js +188 -187
- package/dist/index.js.map +6 -6
- package/dist/run-sync-job.d.ts.map +1 -1
- package/dist/source-read.d.ts +23 -1
- package/dist/source-read.d.ts.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/checkpoints.ts +98 -145
- package/src/run-sync-job.ts +65 -13
- package/src/source-read.ts +133 -123
- package/src/worker.ts +31 -3
package/README.md
CHANGED
|
@@ -26,6 +26,21 @@ its attempts.
|
|
|
26
26
|
One run of a sync runs at a time: a queued run waits for the one running, whose end wakes it, and
|
|
27
27
|
triggers join it.
|
|
28
28
|
|
|
29
|
+
## Connections
|
|
30
|
+
|
|
31
|
+
A sync of an OAuth connector runs once per connection: dispatch queues a run for each, and each run
|
|
32
|
+
names its `connectionId`. They still run one at a time, since they commit to one dataset. `resolveSyncRunSource` finds that one connection among those the run may
|
|
33
|
+
read. Its checkpoint is the last succeeded run of the same sync and connection, bound to the account
|
|
34
|
+
it read, so a replaced account starts over. A connection's first run of its own starts from its
|
|
35
|
+
entry in the last run that read every connection, from before runs were per connection.
|
|
36
|
+
|
|
37
|
+
| What happens | The run |
|
|
38
|
+
| --- | --- |
|
|
39
|
+
| The connection's grant can no longer give a token | Fails with `connector.authorization_required`. A person's own connection isn't reported to `onError` |
|
|
40
|
+
| The connection is gone, or its owner suspended, since it was queued | Ends `cancelled` without reading |
|
|
41
|
+
| It was queued before runs were per connection | Ends `cancelled` without reading; the next trigger runs each connection |
|
|
42
|
+
| Any other failure | Fails that run alone |
|
|
43
|
+
|
|
29
44
|
## Recovery
|
|
30
45
|
|
|
31
46
|
Nothing of a partial run exists until its commit: rows stay in a session no reader sees, and a crash
|
package/dist/checkpoints.d.ts
CHANGED
|
@@ -1,24 +1,17 @@
|
|
|
1
1
|
import { type JsonValue, type SyncConnectorConnection } from "@sixb/core";
|
|
2
|
-
|
|
3
|
-
export interface SyncCheckpointCodec {
|
|
4
|
-
read(value: JsonValue | undefined): JsonValue | undefined;
|
|
5
|
-
serialize(value: JsonValue): JsonValue;
|
|
6
|
-
}
|
|
7
|
-
/** Bind opaque handler checkpoints to the framework source contract stored beside the run. */
|
|
8
|
-
export declare function createSyncCheckpointCodec(options: {
|
|
2
|
+
interface SyncCheckpointIdentity {
|
|
9
3
|
readonly syncId: string;
|
|
10
4
|
readonly connectorId: string;
|
|
11
|
-
readonly strategy: SyncCheckpointStrategy;
|
|
12
|
-
}): SyncCheckpointCodec;
|
|
13
|
-
/** Per-connection cursor state carried inside a framework-managed checkpoint envelope. */
|
|
14
|
-
export declare class ConnectionCheckpoints {
|
|
15
|
-
private readonly entries;
|
|
16
|
-
private constructor();
|
|
17
|
-
static from(syncId: string, value: JsonValue | undefined): ConnectionCheckpoints;
|
|
18
|
-
reconcile(connections: readonly SyncConnectorConnection[]): void;
|
|
19
|
-
get(connection: SyncConnectorConnection): JsonValue | undefined;
|
|
20
|
-
set(connection: SyncConnectorConnection, checkpoint: JsonValue): void;
|
|
21
|
-
serialize(): JsonValue;
|
|
22
5
|
}
|
|
6
|
+
/** A static connector's checkpoint: the handler's value inside the framework envelope. */
|
|
7
|
+
export declare const singleCheckpoint: {
|
|
8
|
+
read(identity: SyncCheckpointIdentity, value: JsonValue | undefined): JsonValue | undefined;
|
|
9
|
+
serialize(identity: SyncCheckpointIdentity, value: JsonValue): JsonValue;
|
|
10
|
+
};
|
|
11
|
+
/** One connection's checkpoint. A replaced account starts over. */
|
|
12
|
+
export declare const connectionCheckpoint: {
|
|
13
|
+
read(identity: SyncCheckpointIdentity, connection: SyncConnectorConnection, value: JsonValue | undefined): JsonValue | undefined;
|
|
14
|
+
serialize(identity: SyncCheckpointIdentity, connection: SyncConnectorConnection, checkpoint: JsonValue | undefined): JsonValue;
|
|
15
|
+
};
|
|
23
16
|
export {};
|
|
24
17
|
//# sourceMappingURL=checkpoints.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,uBAAuB,EAC7B,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../src/checkpoints.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,uBAAuB,EAC7B,MAAM,YAAY,CAAA;AAanB,UAAU,sBAAsB;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAUD,0FAA0F;AAC1F,eAAO,MAAM,gBAAgB;mBACZ,sBAAsB,SAAS,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS;wBAOvE,sBAAsB,SAAS,SAAS,GAAG,SAAS;CAGzE,CAAA;AAED,mEAAmE;AACnE,eAAO,MAAM,oBAAoB;mBAEnB,sBAAsB,cACpB,uBAAuB,SAC5B,SAAS,GAAG,SAAS,GAC3B,SAAS,GAAG,SAAS;wBAsBZ,sBAAsB,cACpB,uBAAuB,cACvB,SAAS,GAAG,SAAS,GAChC,SAAS;CAMb,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -125,226 +125,166 @@ import {
|
|
|
125
125
|
import { createSixbError } from "@sixb/core/internal/errors";
|
|
126
126
|
var SYNC_CHECKPOINT_KIND = "sync_checkpoint";
|
|
127
127
|
var SYNC_CHECKPOINT_VERSION = 1;
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
read(value) {
|
|
131
|
-
if (value === undefined)
|
|
132
|
-
return;
|
|
133
|
-
if (!isSyncCheckpoint(value)) {
|
|
134
|
-
if (options.strategy === "single")
|
|
135
|
-
return cloneJsonValue(value);
|
|
136
|
-
throw incompatibleCheckpoint(options, "format");
|
|
137
|
-
}
|
|
138
|
-
if (value.version !== SYNC_CHECKPOINT_VERSION) {
|
|
139
|
-
throw incompatibleCheckpoint(options, "version");
|
|
140
|
-
}
|
|
141
|
-
if (value.connectorId !== options.connectorId) {
|
|
142
|
-
throw incompatibleCheckpoint(options, "connector");
|
|
143
|
-
}
|
|
144
|
-
if (value.strategy !== options.strategy) {
|
|
145
|
-
throw incompatibleCheckpoint(options, "strategy");
|
|
146
|
-
}
|
|
147
|
-
if (!("value" in value) || !isJsonValue(value.value)) {
|
|
148
|
-
throw invalidFrameworkCheckpoint(options.syncId);
|
|
149
|
-
}
|
|
150
|
-
return cloneJsonValue(value.value);
|
|
151
|
-
},
|
|
152
|
-
serialize(value) {
|
|
153
|
-
const envelope = {
|
|
154
|
-
kind: SYNC_CHECKPOINT_KIND,
|
|
155
|
-
version: SYNC_CHECKPOINT_VERSION,
|
|
156
|
-
connectorId: options.connectorId,
|
|
157
|
-
strategy: options.strategy,
|
|
158
|
-
value: cloneJsonValue(value)
|
|
159
|
-
};
|
|
160
|
-
assertJsonValue(envelope, "Sync checkpoint envelope");
|
|
161
|
-
return envelope;
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
class ConnectionCheckpoints {
|
|
167
|
-
entries;
|
|
168
|
-
constructor(entries) {
|
|
169
|
-
this.entries = entries;
|
|
170
|
-
}
|
|
171
|
-
static from(syncId, value) {
|
|
128
|
+
var singleCheckpoint = {
|
|
129
|
+
read(identity, value) {
|
|
172
130
|
if (value === undefined)
|
|
173
|
-
return
|
|
174
|
-
if (!
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
}
|
|
181
|
-
if ("checkpoint" in candidate && !isJsonValue(candidate.checkpoint)) {
|
|
182
|
-
throw invalidConnectionCheckpoint(syncId);
|
|
183
|
-
}
|
|
184
|
-
entries.set(candidate.connectionId, {
|
|
185
|
-
connectionId: candidate.connectionId,
|
|
186
|
-
accountId: candidate.accountId,
|
|
187
|
-
...candidate.checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(candidate.checkpoint) }
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
return new ConnectionCheckpoints(entries);
|
|
131
|
+
return;
|
|
132
|
+
if (!isSyncCheckpoint(value))
|
|
133
|
+
return cloneJsonValue(value);
|
|
134
|
+
return openEnvelope(identity, value, ["single"]).value;
|
|
135
|
+
},
|
|
136
|
+
serialize(identity, value) {
|
|
137
|
+
return envelope(identity, "single", value);
|
|
191
138
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
139
|
+
};
|
|
140
|
+
var connectionCheckpoint = {
|
|
141
|
+
read(identity, connection, value) {
|
|
142
|
+
if (value === undefined)
|
|
143
|
+
return;
|
|
144
|
+
if (!isSyncCheckpoint(value))
|
|
145
|
+
throw incompatibleCheckpoint(identity, "format");
|
|
146
|
+
const opened = openEnvelope(identity, value, ["connection", "connections"]);
|
|
147
|
+
const entry = opened.strategy === "connection" ? opened.value : legacyEntries(identity.syncId, opened.value).find((candidate) => candidate.connectionId === connection.id);
|
|
148
|
+
if (entry === undefined)
|
|
149
|
+
return;
|
|
150
|
+
if (!isRecord(entry) || typeof entry.accountId !== "string") {
|
|
151
|
+
throw invalidCheckpoint(identity.syncId);
|
|
198
152
|
}
|
|
199
|
-
|
|
200
|
-
get(connection) {
|
|
201
|
-
const entry = this.entries.get(connection.id);
|
|
202
|
-
if (entry?.accountId !== connection.account.id || entry.checkpoint === undefined) {
|
|
153
|
+
if (entry.accountId !== connection.account.id || entry.checkpoint === undefined) {
|
|
203
154
|
return;
|
|
204
155
|
}
|
|
156
|
+
if (!isJsonValue(entry.checkpoint))
|
|
157
|
+
throw invalidCheckpoint(identity.syncId);
|
|
205
158
|
return cloneJsonValue(entry.checkpoint);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
connectionId: connection.id,
|
|
159
|
+
},
|
|
160
|
+
serialize(identity, connection, checkpoint) {
|
|
161
|
+
return envelope(identity, "connection", {
|
|
210
162
|
accountId: connection.account.id,
|
|
211
|
-
checkpoint: cloneJsonValue(checkpoint)
|
|
163
|
+
...checkpoint === undefined ? {} : { checkpoint: cloneJsonValue(checkpoint) }
|
|
212
164
|
});
|
|
213
165
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
166
|
+
};
|
|
167
|
+
function openEnvelope(identity, value, strategies) {
|
|
168
|
+
if (value.version !== SYNC_CHECKPOINT_VERSION)
|
|
169
|
+
throw incompatibleCheckpoint(identity, "version");
|
|
170
|
+
if (value.connectorId !== identity.connectorId) {
|
|
171
|
+
throw incompatibleCheckpoint(identity, "connector");
|
|
172
|
+
}
|
|
173
|
+
const strategy = strategies.find((candidate) => candidate === value.strategy);
|
|
174
|
+
if (!strategy)
|
|
175
|
+
throw incompatibleCheckpoint(identity, "strategy");
|
|
176
|
+
if (!("value" in value) || !isJsonValue(value.value))
|
|
177
|
+
throw invalidCheckpoint(identity.syncId);
|
|
178
|
+
return { strategy, value: cloneJsonValue(value.value) };
|
|
179
|
+
}
|
|
180
|
+
function envelope(identity, strategy, value) {
|
|
181
|
+
const serialized = {
|
|
182
|
+
kind: SYNC_CHECKPOINT_KIND,
|
|
183
|
+
version: SYNC_CHECKPOINT_VERSION,
|
|
184
|
+
connectorId: identity.connectorId,
|
|
185
|
+
strategy,
|
|
186
|
+
value: cloneJsonValue(value)
|
|
187
|
+
};
|
|
188
|
+
assertJsonValue(serialized, "Sync checkpoint envelope");
|
|
189
|
+
return serialized;
|
|
225
190
|
}
|
|
226
|
-
function
|
|
227
|
-
|
|
191
|
+
function legacyEntries(syncId, value) {
|
|
192
|
+
if (!isRecord(value) || !Array.isArray(value.entries))
|
|
193
|
+
throw invalidCheckpoint(syncId);
|
|
194
|
+
return value.entries.flatMap((entry) => isRecord(entry) ? [entry] : []);
|
|
195
|
+
}
|
|
196
|
+
function incompatibleCheckpoint(identity, reason) {
|
|
197
|
+
return createSixbError("sync.execution_failed", `[SixbSyncWorker] Sync '${identity.syncId}' has a checkpoint incompatible with connector '${identity.connectorId}'. Use a new Sync id to reset its ingestion state.`, {
|
|
228
198
|
details: {
|
|
229
|
-
syncId:
|
|
230
|
-
connectorId:
|
|
199
|
+
syncId: identity.syncId,
|
|
200
|
+
connectorId: identity.connectorId,
|
|
231
201
|
reason: `checkpoint_${reason}`
|
|
232
202
|
}
|
|
233
203
|
});
|
|
234
204
|
}
|
|
235
|
-
function
|
|
205
|
+
function invalidCheckpoint(syncId) {
|
|
236
206
|
return createSixbError("internal.unexpected", `[SixbSyncWorker] Sync '${syncId}' has an invalid checkpoint envelope.`, { details: { syncId } });
|
|
237
207
|
}
|
|
238
|
-
function invalidConnectionCheckpoint(syncId) {
|
|
239
|
-
return createSixbError("internal.unexpected", `[SixbSyncWorker] Sync '${syncId}' has an invalid managed connector checkpoint.`, { details: { syncId } });
|
|
240
|
-
}
|
|
241
208
|
function isSyncCheckpoint(value) {
|
|
242
209
|
return isRecord(value) && value.kind === SYNC_CHECKPOINT_KIND;
|
|
243
210
|
}
|
|
244
|
-
function isConnectionCheckpoint(value) {
|
|
245
|
-
return isRecord(value) && Array.isArray(value.entries);
|
|
246
|
-
}
|
|
247
211
|
function isRecord(value) {
|
|
248
212
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
249
213
|
}
|
|
250
214
|
|
|
251
215
|
// src/source-read.ts
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
216
|
+
class SyncRunSourceGoneError extends Error {
|
|
217
|
+
name = "SyncRunSourceGoneError";
|
|
218
|
+
}
|
|
219
|
+
async function resolveSyncRunSource(runtime, sync, run) {
|
|
220
|
+
const sources = await runtime.connectorSources.list(sync.connector);
|
|
221
|
+
if (!isOAuthConnectorDefinition(sync.connector)) {
|
|
222
|
+
if (run.connectionId !== undefined) {
|
|
223
|
+
throw sourceInvariant(sync.id, `Static connector run '${run.id}' names a connection.`);
|
|
257
224
|
}
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
strategy: "single"
|
|
262
|
-
});
|
|
263
|
-
return readStaticSource(options, sources[0], checkpointCodec);
|
|
264
|
-
}
|
|
265
|
-
const connectionIds = new Set;
|
|
266
|
-
const managedSources = sources.map((source) => {
|
|
267
|
-
if (!source.connection) {
|
|
268
|
-
throw sourceInvariant(options.sync.id, "OAuth connector sources must include connection metadata.");
|
|
269
|
-
}
|
|
270
|
-
const connection = source.connection;
|
|
271
|
-
if (connection.connectorId !== options.sync.connector.id) {
|
|
272
|
-
throw sourceInvariant(options.sync.id, `Connection '${connection.id}' belongs to connector '${connection.connectorId}', not '${options.sync.connector.id}'.`);
|
|
225
|
+
const [source] = sources;
|
|
226
|
+
if (sources.length !== 1 || !source || source.connection !== undefined) {
|
|
227
|
+
throw sourceInvariant(sync.id, "Static connectors must resolve exactly one source.");
|
|
273
228
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
};
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
async function readStaticSource(options, source, checkpointCodec) {
|
|
296
|
-
const previousCheckpoint = checkpointCodec.read(options.previousCheckpoint);
|
|
229
|
+
return { kind: "static", source };
|
|
230
|
+
}
|
|
231
|
+
if (run.connectionId === undefined) {
|
|
232
|
+
throw new SyncRunSourceGoneError(`[SixbSyncWorker] Sync run '${run.id}' was queued before a sync of an OAuth connector ran once per connection. The sync's next trigger runs each connection.`);
|
|
233
|
+
}
|
|
234
|
+
const source = sources.find((candidate) => candidate.connection?.id === run.connectionId);
|
|
235
|
+
if (!source?.connection) {
|
|
236
|
+
throw new SyncRunSourceGoneError(`[SixbSyncWorker] Connection '${run.connectionId}' of sync '${sync.id}' is no longer connected, or its owner is suspended.`);
|
|
237
|
+
}
|
|
238
|
+
if (source.connection.connectorId !== sync.connector.id) {
|
|
239
|
+
throw sourceInvariant(sync.id, `Connection '${source.connection.id}' belongs to connector '${source.connection.connectorId}', not '${sync.connector.id}'.`);
|
|
240
|
+
}
|
|
241
|
+
return { kind: "connection", source, connection: source.connection };
|
|
242
|
+
}
|
|
243
|
+
async function readSyncValues(options) {
|
|
244
|
+
const { source } = options;
|
|
245
|
+
return source.kind === "static" ? readStaticSource(options, source.source) : readConnectionSource(options, source.source, source.connection);
|
|
246
|
+
}
|
|
247
|
+
async function readStaticSource(options, source) {
|
|
248
|
+
const identity = checkpointIdentity(options.sync);
|
|
249
|
+
const previousCheckpoint = singleCheckpoint.read(identity, options.previousCheckpoint);
|
|
297
250
|
if (previousCheckpoint !== undefined) {
|
|
298
|
-
options.setCheckpoint(
|
|
251
|
+
options.setCheckpoint(singleCheckpoint.serialize(identity, previousCheckpoint));
|
|
299
252
|
}
|
|
300
253
|
const client = await runAbortable(options.signal, () => source.connect(options.signal));
|
|
301
254
|
throwIfAborted(options.signal);
|
|
302
255
|
const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
|
|
303
|
-
|
|
304
|
-
syncId: options.sync.id,
|
|
305
|
-
signal: options.signal,
|
|
306
|
-
blobs: options.blobStorage,
|
|
307
|
-
logger: options.logger,
|
|
256
|
+
...readContext(options),
|
|
308
257
|
checkpoint: previousCheckpoint === undefined ? undefined : cloneJsonValue2(previousCheckpoint),
|
|
309
258
|
setCheckpoint(next) {
|
|
310
259
|
assertJsonValue2(next, `Sync '${options.sync.id}' checkpoint`);
|
|
311
|
-
options.setCheckpoint(
|
|
260
|
+
options.setCheckpoint(singleCheckpoint.serialize(identity, cloneJsonValue2(next)));
|
|
312
261
|
}
|
|
313
262
|
}));
|
|
314
263
|
return sourceValues(normalizeReadResult(readResult, options.sync.id, options.signal));
|
|
315
264
|
}
|
|
316
|
-
async function*
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
}
|
|
321
|
-
async function* readManagedSource(options, source, checkpoints, storeCheckpoint) {
|
|
322
|
-
const { connection } = source;
|
|
265
|
+
async function* readConnectionSource(options, source, connection) {
|
|
266
|
+
const identity = checkpointIdentity(options.sync);
|
|
267
|
+
const previous = connectionCheckpoint.read(identity, connection, options.previousCheckpoint);
|
|
268
|
+
options.setCheckpoint(connectionCheckpoint.serialize(identity, connection, previous));
|
|
323
269
|
try {
|
|
324
270
|
throwIfAborted(options.signal);
|
|
325
271
|
let client;
|
|
326
272
|
try {
|
|
327
273
|
client = await runAbortable(options.signal, () => source.connect(options.signal));
|
|
328
274
|
} catch (error) {
|
|
329
|
-
if (
|
|
330
|
-
throw error;
|
|
331
|
-
|
|
332
|
-
|
|
275
|
+
if (isSixbError(error) && error.code === "connector.authorization_required") {
|
|
276
|
+
throw needsReconnecting(options, connection, error);
|
|
277
|
+
}
|
|
278
|
+
throw error;
|
|
333
279
|
}
|
|
334
280
|
throwIfAborted(options.signal);
|
|
335
|
-
const previous = checkpoints.get(connection);
|
|
336
281
|
const readResult = await runAbortable(options.signal, () => options.sync.read(client, {
|
|
337
|
-
|
|
338
|
-
syncId: options.sync.id,
|
|
339
|
-
signal: options.signal,
|
|
340
|
-
blobs: options.blobStorage,
|
|
341
|
-
logger: options.logger,
|
|
282
|
+
...readContext(options),
|
|
342
283
|
connection,
|
|
343
284
|
checkpoint: previous === undefined ? undefined : cloneJsonValue2(previous),
|
|
344
285
|
setCheckpoint(next) {
|
|
345
286
|
assertJsonValue2(next, `Sync '${options.sync.id}' connection '${connection.id}' checkpoint`);
|
|
346
|
-
|
|
347
|
-
storeCheckpoint();
|
|
287
|
+
options.setCheckpoint(connectionCheckpoint.serialize(identity, connection, cloneJsonValue2(next)));
|
|
348
288
|
}
|
|
349
289
|
}));
|
|
350
290
|
for await (const value of normalizeReadResult(readResult, options.sync.id, options.signal)) {
|
|
@@ -353,20 +293,40 @@ async function* readManagedSource(options, source, checkpoints, storeCheckpoint)
|
|
|
353
293
|
} catch (error) {
|
|
354
294
|
if (options.signal.aborted)
|
|
355
295
|
throw error;
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
syncId: options.sync.id,
|
|
360
|
-
runId: options.runId,
|
|
361
|
-
datasetId: options.datasetId,
|
|
362
|
-
connectionId: connection.id,
|
|
363
|
-
accountId: connection.account.id
|
|
364
|
-
}
|
|
365
|
-
});
|
|
296
|
+
if (isSixbError(error) && error.code === "connector.authorization_required")
|
|
297
|
+
throw error;
|
|
298
|
+
throw createSixbError2("sync.execution_failed", summarizeErrorMessage(error, "Connector connection Sync source failed."), { cause: error, details: connectionDetails(options, connection) });
|
|
366
299
|
}
|
|
367
300
|
}
|
|
368
|
-
function
|
|
369
|
-
|
|
301
|
+
function needsReconnecting(options, connection, cause) {
|
|
302
|
+
if (connection.owner.type === "user") {
|
|
303
|
+
console.warn(`[SixbSyncWorker] Sync '${options.sync.id}' could not read connection '${connection.id}': it needs reconnecting.`);
|
|
304
|
+
}
|
|
305
|
+
return createSixbError2("connector.authorization_required", `[SixbSyncWorker] Connection '${connection.id}' of sync '${options.sync.id}' needs reconnecting.`, {
|
|
306
|
+
cause,
|
|
307
|
+
details: { ...connectionDetails(options, connection), ownerType: connection.owner.type }
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
function readContext(options) {
|
|
311
|
+
return {
|
|
312
|
+
projectId: options.runtime.id,
|
|
313
|
+
syncId: options.sync.id,
|
|
314
|
+
signal: options.signal,
|
|
315
|
+
blobs: options.blobStorage,
|
|
316
|
+
logger: options.logger
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
function checkpointIdentity(sync) {
|
|
320
|
+
return { syncId: sync.id, connectorId: sync.connector.id };
|
|
321
|
+
}
|
|
322
|
+
function connectionDetails(options, connection) {
|
|
323
|
+
return {
|
|
324
|
+
syncId: options.sync.id,
|
|
325
|
+
runId: options.runId,
|
|
326
|
+
datasetId: options.datasetId,
|
|
327
|
+
connectionId: connection.id,
|
|
328
|
+
accountId: connection.account.id
|
|
329
|
+
};
|
|
370
330
|
}
|
|
371
331
|
function sourceInvariant(syncId, message) {
|
|
372
332
|
return createSixbError2("internal.unexpected", `[SixbSyncWorker] ${message}`, {
|
|
@@ -409,7 +369,7 @@ function requireFinishedAt(input) {
|
|
|
409
369
|
throw createSixbError3("internal.unexpected", `[SixbSyncWorker] Sync run '${input.runId}' finished without a finishedAt timestamp.`, { details: { syncId: input.syncId, runId: input.runId } });
|
|
410
370
|
}
|
|
411
371
|
function translateSyncExecutionError(error, input) {
|
|
412
|
-
if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed")) {
|
|
372
|
+
if (isSixbError2(error) && (error.code === "internal.unexpected" || error.code === "sync.execution_failed" || error.code === "connector.authorization_required")) {
|
|
413
373
|
return error;
|
|
414
374
|
}
|
|
415
375
|
return createSixbError3("sync.execution_failed", summarizeErrorMessage2(error, "Sync execution failed."), {
|
|
@@ -483,7 +443,8 @@ async function commitRun(execution, logger) {
|
|
|
483
443
|
const { lakeStorage, blobs: blobStorage } = runtime;
|
|
484
444
|
const sync = requireSync(runtime, run);
|
|
485
445
|
const dataset = requireDataset(runtime, sync, run);
|
|
486
|
-
const
|
|
446
|
+
const source = await resolveSyncRunSource(runtime, sync, run);
|
|
447
|
+
const previousCheckpoint = await readPreviousCheckpoint(runtime, sync, run);
|
|
487
448
|
execution.nextCheckpoint = cloneCheckpoint(previousCheckpoint);
|
|
488
449
|
const { outcome, version } = await writeDataset({
|
|
489
450
|
lakeStorage,
|
|
@@ -495,6 +456,7 @@ async function commitRun(execution, logger) {
|
|
|
495
456
|
readValues: () => readSyncValues({
|
|
496
457
|
runtime,
|
|
497
458
|
sync,
|
|
459
|
+
source,
|
|
498
460
|
runId: run.id,
|
|
499
461
|
datasetId: dataset.id,
|
|
500
462
|
signal,
|
|
@@ -598,6 +560,10 @@ async function settleFailure(execution, caught) {
|
|
|
598
560
|
const landed = committedVersion ? null : await landedCommitBehind(execution, caught);
|
|
599
561
|
if (landed)
|
|
600
562
|
return finishLanded(execution, landed, landedRead(execution, caught));
|
|
563
|
+
if (caught instanceof SyncRunSourceGoneError && !signal.aborted) {
|
|
564
|
+
await recordCancelled(execution, caught);
|
|
565
|
+
throw caught;
|
|
566
|
+
}
|
|
601
567
|
const error = translateLostRace(run, caught);
|
|
602
568
|
if (!committedVersion && !signal.aborted && !(error instanceof CommitLostRaceError)) {
|
|
603
569
|
await recordFailed(execution, error);
|
|
@@ -634,18 +600,42 @@ async function recordFailed(execution, error) {
|
|
|
634
600
|
onRunFailed?.(error, failedRun, failure);
|
|
635
601
|
} catch {}
|
|
636
602
|
}
|
|
603
|
+
async function recordCancelled(execution, reason) {
|
|
604
|
+
const { run } = execution;
|
|
605
|
+
const { runtime, claim, onRunFinished } = execution.input;
|
|
606
|
+
try {
|
|
607
|
+
const cancelled = await runtime.syncRunsStorage.finish({
|
|
608
|
+
projectId: runtime.id,
|
|
609
|
+
id: run.id,
|
|
610
|
+
executionToken: claim.execution.token,
|
|
611
|
+
status: "cancelled",
|
|
612
|
+
rowsRead: 0,
|
|
613
|
+
error: captureSixbFailure(createSixbError3("runtime.cancelled", reason.message), {
|
|
614
|
+
allowedCodes: SYNC_RUN_FAILURE_CODES,
|
|
615
|
+
defaultCode: "runtime.cancelled"
|
|
616
|
+
})
|
|
617
|
+
});
|
|
618
|
+
await notifyRunFinished(onRunFinished, cancelled);
|
|
619
|
+
} catch {}
|
|
620
|
+
}
|
|
637
621
|
function cloneCheckpoint(checkpoint) {
|
|
638
622
|
return checkpoint === undefined ? undefined : cloneJsonValue3(checkpoint);
|
|
639
623
|
}
|
|
640
|
-
async function readPreviousCheckpoint(runtime, sync) {
|
|
641
|
-
const
|
|
624
|
+
async function readPreviousCheckpoint(runtime, sync, run) {
|
|
625
|
+
const latest = (connectionId) => runtime.syncRunsStorage.list({
|
|
642
626
|
projectId: runtime.id,
|
|
643
627
|
syncId: sync.id,
|
|
628
|
+
...connectionId === undefined ? {} : { connectionId },
|
|
644
629
|
statuses: ["succeeded"],
|
|
645
630
|
limit: 1,
|
|
646
631
|
order: "desc"
|
|
647
|
-
});
|
|
648
|
-
|
|
632
|
+
}).then(({ runs }) => runs[0]);
|
|
633
|
+
if (run.connectionId === undefined)
|
|
634
|
+
return (await latest())?.checkpoint;
|
|
635
|
+
const own = await latest(run.connectionId);
|
|
636
|
+
if (own)
|
|
637
|
+
return own.checkpoint;
|
|
638
|
+
return (await latest(null))?.checkpoint;
|
|
649
639
|
}
|
|
650
640
|
function requireDataset(runtime, sync, run) {
|
|
651
641
|
const dataset = runtime.datasets.getById(run.datasetId);
|
|
@@ -744,6 +734,8 @@ class SyncWorker extends RunWorker {
|
|
|
744
734
|
await this.wakeNextRun(finished);
|
|
745
735
|
},
|
|
746
736
|
onRunFailed: (error, run, failure) => {
|
|
737
|
+
if (isPersonalReconnect(failure))
|
|
738
|
+
return;
|
|
747
739
|
reportRunFailure(this.host, error, {
|
|
748
740
|
projectId: this.host.id,
|
|
749
741
|
attempt: run.attempt,
|
|
@@ -795,6 +787,7 @@ async function emitSyncRunStarted(host, run, correlationId) {
|
|
|
795
787
|
payload: {
|
|
796
788
|
syncId: run.syncId,
|
|
797
789
|
runId: run.id,
|
|
790
|
+
...run.connectionId === undefined ? {} : { connectionId: run.connectionId },
|
|
798
791
|
startedAt: run.startedAt.toISOString()
|
|
799
792
|
}
|
|
800
793
|
}
|
|
@@ -827,6 +820,7 @@ async function emitSyncRunFinishedEvents(host, run, correlationId, createdVersio
|
|
|
827
820
|
payload: {
|
|
828
821
|
syncId: run.syncId,
|
|
829
822
|
runId: run.id,
|
|
823
|
+
...run.connectionId === undefined ? {} : { connectionId: run.connectionId },
|
|
830
824
|
status: requireTerminalStatus(run.status, `Sync run '${run.id}'`),
|
|
831
825
|
datasetId: run.datasetId,
|
|
832
826
|
...run.output ? { versionId: run.output.versionId } : {},
|
|
@@ -837,6 +831,10 @@ async function emitSyncRunFinishedEvents(host, run, correlationId, createdVersio
|
|
|
837
831
|
correlationId
|
|
838
832
|
}, { source: SOURCE });
|
|
839
833
|
}
|
|
834
|
+
function isPersonalReconnect(failure) {
|
|
835
|
+
const details = failure.details;
|
|
836
|
+
return failure.code === "connector.authorization_required" && isRecord2(details) && details.ownerType === "user";
|
|
837
|
+
}
|
|
840
838
|
function requireTerminalStatus(status, context) {
|
|
841
839
|
if (status === "queued" || status === "running") {
|
|
842
840
|
throw new Error(`[SixbSyncWorker] ${context} is not terminal.`);
|
|
@@ -863,8 +861,11 @@ function buildSyncContext(host, sixb) {
|
|
|
863
861
|
}
|
|
864
862
|
};
|
|
865
863
|
}
|
|
864
|
+
function isRecord2(value) {
|
|
865
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
866
|
+
}
|
|
866
867
|
export {
|
|
867
868
|
SyncWorker
|
|
868
869
|
};
|
|
869
870
|
|
|
870
|
-
//# debugId=
|
|
871
|
+
//# debugId=FDCAB685EF4DF87464756E2164756E21
|