@context-use/open-sync 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/db/client.ts +3 -3
- package/src/db/migrate.ts +48 -0
- package/src/db/migrations/0001-centralize-polling.sql +3 -0
- package/src/db/migrations/0001-centralize-polling.sql.d.ts +2 -0
- package/src/execution/worker.ts +11 -2
- package/src/http/controller.ts +0 -1
- package/src/models/definition.ts +1 -1
- package/src/models/sync.ts +0 -7
- package/src/repositories/acquisition/contract.ts +1 -3
- package/src/repositories/acquisition/lease.ts +9 -13
- package/src/repositories/acquisition/sqlite.ts +9 -4
- package/src/repositories/assets/sqlite.ts +2 -2
- package/src/repositories/catalog/contract.ts +1 -1
- package/src/repositories/catalog/sqlite.ts +27 -19
- package/src/repositories/delivery/sqlite.ts +2 -2
- package/src/repositories/rows.ts +0 -2
- package/src/services/acquisition.ts +8 -13
- package/src/services/management.ts +9 -6
- /package/src/db/{schema.sql → migrations/0000-initial.sql} +0 -0
- /package/src/db/{schema.sql.d.ts → migrations/0000-initial.sql.d.ts} +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@context-use/open-sync",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/open-sync.ts",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"proper-lockfile": "4.1.2"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
+
"open-sync-previous": "https://registry.npmjs.org/@context-use/open-sync/-/open-sync-0.3.1.tgz",
|
|
35
36
|
"typescript": "6.0.3"
|
|
36
37
|
},
|
|
37
38
|
"engines": {
|
package/src/db/client.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
|
-
import
|
|
2
|
+
import { runMigrations } from './migrate';
|
|
3
3
|
|
|
4
4
|
export function openDatabase(path: string): Database {
|
|
5
5
|
const db = new Database(path, { create: true, strict: true });
|
|
6
6
|
try {
|
|
7
|
-
db.exec('PRAGMA foreign_keys=ON; PRAGMA
|
|
8
|
-
|
|
7
|
+
db.exec('PRAGMA foreign_keys=ON; PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;');
|
|
8
|
+
runMigrations(db);
|
|
9
9
|
return db;
|
|
10
10
|
} catch (error) {
|
|
11
11
|
db.close();
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import initialSchema from './migrations/0000-initial.sql' with { type: 'text' };
|
|
3
|
+
import centralizePolling from './migrations/0001-centralize-polling.sql' with { type: 'text' };
|
|
4
|
+
|
|
5
|
+
// Static imports keep migration history available in both installed packages and compiled hosts.
|
|
6
|
+
const migrations = [
|
|
7
|
+
{ name: '0000-initial.sql', sql: initialSchema },
|
|
8
|
+
{ name: '0001-centralize-polling.sql', sql: centralizePolling },
|
|
9
|
+
] as const;
|
|
10
|
+
|
|
11
|
+
export function runMigrations(db: Database): void {
|
|
12
|
+
db.transaction(() => {
|
|
13
|
+
db.query(`CREATE TABLE IF NOT EXISTS __migrations (
|
|
14
|
+
name TEXT PRIMARY KEY, applied_at TEXT NOT NULL
|
|
15
|
+
)`).run();
|
|
16
|
+
const applied = db
|
|
17
|
+
.query<{ name: string }, []>('SELECT name FROM __migrations ORDER BY name')
|
|
18
|
+
.all();
|
|
19
|
+
for (const [index, { name }] of applied.entries()) {
|
|
20
|
+
if (name !== migrations[index]?.name) {
|
|
21
|
+
throw new Error('Unsupported sync database migration history');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
for (const migration of migrations.slice(applied.length)) {
|
|
25
|
+
executeSql({ db, sql: migration.sql });
|
|
26
|
+
db.query('INSERT INTO __migrations (name,applied_at) VALUES (?,?)').run(
|
|
27
|
+
migration.name,
|
|
28
|
+
new Date().toISOString(),
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}).immediate();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function executeSql({ db, sql }: { db: Database; sql: string }): void {
|
|
35
|
+
// Bun 1.4's exec() can hide failures in non-final statements. Let SQLite parse each
|
|
36
|
+
// statement instead, preserving trigger bodies, comments and quoted semicolons.
|
|
37
|
+
// A final no-op also lets prepare() consume files ending in comments or whitespace.
|
|
38
|
+
let remaining = `${sql}\n;SELECT 1;`;
|
|
39
|
+
while (remaining.length) {
|
|
40
|
+
using statement = db.prepare(remaining);
|
|
41
|
+
const parsed = statement.toString();
|
|
42
|
+
if (!parsed || !remaining.startsWith(parsed)) {
|
|
43
|
+
throw new Error('Migration SQL must contain complete statements without parameters');
|
|
44
|
+
}
|
|
45
|
+
statement.run();
|
|
46
|
+
remaining = remaining.slice(parsed.length);
|
|
47
|
+
}
|
|
48
|
+
}
|
package/src/execution/worker.ts
CHANGED
|
@@ -27,6 +27,7 @@ export class Worker implements WorkerControl {
|
|
|
27
27
|
#capacityReleased = false;
|
|
28
28
|
#closing?: Promise<void>;
|
|
29
29
|
#draining?: Promise<void>;
|
|
30
|
+
#polling = false;
|
|
30
31
|
#drainRequested = false;
|
|
31
32
|
#resume?: () => void;
|
|
32
33
|
constructor(
|
|
@@ -55,7 +56,7 @@ export class Worker implements WorkerControl {
|
|
|
55
56
|
if ((!this.#started && !this.#draining) || this.#lifetime.signal.aborted) {
|
|
56
57
|
return;
|
|
57
58
|
}
|
|
58
|
-
void this.
|
|
59
|
+
void this.requestDrain().catch(() => this.input.log({ code: 'runtime_failed' }));
|
|
59
60
|
}
|
|
60
61
|
/** A deterministic dispatch round for hosts/tests that do not start the background worker. */
|
|
61
62
|
tick(): Promise<void> {
|
|
@@ -65,9 +66,16 @@ export class Worker implements WorkerControl {
|
|
|
65
66
|
this.cleanup(),
|
|
66
67
|
);
|
|
67
68
|
}
|
|
68
|
-
/**
|
|
69
|
+
/** Poll enabled syncs and drain runnable work, sharing a round across concurrent host triggers. */
|
|
69
70
|
runDue(): Promise<void> {
|
|
70
71
|
this.ensureOpen();
|
|
72
|
+
if (!this.#polling) {
|
|
73
|
+
this.input.acquisition.poll();
|
|
74
|
+
this.#polling = true;
|
|
75
|
+
}
|
|
76
|
+
return this.requestDrain();
|
|
77
|
+
}
|
|
78
|
+
private requestDrain(): Promise<void> {
|
|
71
79
|
this.#drainRequested = true;
|
|
72
80
|
this.#resume?.();
|
|
73
81
|
this.#draining ??= Promise.resolve().then(() => this.drain());
|
|
@@ -98,6 +106,7 @@ export class Worker implements WorkerControl {
|
|
|
98
106
|
} finally {
|
|
99
107
|
this.#resume = undefined;
|
|
100
108
|
this.#draining = undefined;
|
|
109
|
+
this.#polling = false;
|
|
101
110
|
}
|
|
102
111
|
}
|
|
103
112
|
private dispatch(): void {
|
package/src/http/controller.ts
CHANGED
|
@@ -15,7 +15,6 @@ const syncBody = t.Object({
|
|
|
15
15
|
config,
|
|
16
16
|
destination: t.Object({ type: identifier, input: config }),
|
|
17
17
|
connection: t.Optional(t.Object({ id: identifier, service: identifier })),
|
|
18
|
-
intervalMs: t.Optional(t.Integer({ minimum: 1 })),
|
|
19
18
|
enabled: t.Optional(t.Boolean()),
|
|
20
19
|
});
|
|
21
20
|
const resourceParams = { params: t.Object({ id: identifier }) };
|
package/src/models/definition.ts
CHANGED
|
@@ -54,7 +54,7 @@ export interface SyncStep {
|
|
|
54
54
|
* Include any cross-iteration update boundary as well as the current page position.
|
|
55
55
|
*/
|
|
56
56
|
checkpoint: JsonValue;
|
|
57
|
-
/** False continues this iteration immediately; true
|
|
57
|
+
/** False continues this iteration immediately; true waits for the shared cron or a manual run.
|
|
58
58
|
* Completion retains the returned checkpoint; the source clears any exhausted page cursor.
|
|
59
59
|
*/
|
|
60
60
|
complete: boolean;
|
package/src/models/sync.ts
CHANGED
|
@@ -11,8 +11,6 @@ export interface Sync {
|
|
|
11
11
|
destination: { type: string; config: JsonObject };
|
|
12
12
|
enabled: boolean;
|
|
13
13
|
checkpoint: JsonValue;
|
|
14
|
-
intervalMs: number;
|
|
15
|
-
nextDueAt: number;
|
|
16
14
|
status: SyncStatus;
|
|
17
15
|
errorCode: string | null;
|
|
18
16
|
}
|
|
@@ -21,7 +19,6 @@ export interface CreateSync extends Scope {
|
|
|
21
19
|
connection?: ConnectionRef;
|
|
22
20
|
config: JsonObject;
|
|
23
21
|
destination: { type: string; input: JsonObject };
|
|
24
|
-
intervalMs?: number;
|
|
25
22
|
enabled?: boolean;
|
|
26
23
|
}
|
|
27
24
|
|
|
@@ -51,8 +48,6 @@ export interface SyncSummary {
|
|
|
51
48
|
destinationType: string;
|
|
52
49
|
connection?: ConnectionRef;
|
|
53
50
|
enabled: boolean;
|
|
54
|
-
intervalMs: number;
|
|
55
|
-
nextDueAt: number;
|
|
56
51
|
status: SyncStatus;
|
|
57
52
|
errorCode: string | null;
|
|
58
53
|
}
|
|
@@ -63,8 +58,6 @@ export function summarizeSync(sync: Sync): SyncSummary {
|
|
|
63
58
|
destinationType: sync.destination.type,
|
|
64
59
|
connection: sync.connection,
|
|
65
60
|
enabled: sync.enabled,
|
|
66
|
-
intervalMs: sync.intervalMs,
|
|
67
|
-
nextDueAt: sync.nextDueAt,
|
|
68
61
|
status: sync.status,
|
|
69
62
|
errorCode: sync.errorCode,
|
|
70
63
|
};
|
|
@@ -6,9 +6,9 @@ export interface AcquisitionLease extends Scope {
|
|
|
6
6
|
sync: Sync;
|
|
7
7
|
generation: number;
|
|
8
8
|
force: boolean;
|
|
9
|
-
failureCount: number;
|
|
10
9
|
}
|
|
11
10
|
export interface AcquisitionRepository {
|
|
11
|
+
poll(): void;
|
|
12
12
|
capacityReleased(): void;
|
|
13
13
|
claim(leaseMs: number): AcquisitionLease | undefined;
|
|
14
14
|
hasCapacity(): boolean;
|
|
@@ -17,8 +17,6 @@ export interface AcquisitionRepository {
|
|
|
17
17
|
lease: AcquisitionLease;
|
|
18
18
|
state: Exclude<SyncStatus, 'running' | 'disabled'>;
|
|
19
19
|
errorCode?: string;
|
|
20
|
-
delay: number;
|
|
21
|
-
failureCount?: number;
|
|
22
20
|
pause?: boolean;
|
|
23
21
|
}): void;
|
|
24
22
|
}
|
|
@@ -30,8 +30,8 @@ export function claimAcquisition({
|
|
|
30
30
|
WHERE completed_at IS NULL AND EXISTS (SELECT 1 FROM syncs
|
|
31
31
|
WHERE syncs.owner_id=sync_polls.owner_id AND syncs.id=sync_polls.sync_id
|
|
32
32
|
AND enabled=1 AND expires_at<=?)`).run(now);
|
|
33
|
-
db.query(`UPDATE syncs SET
|
|
34
|
-
WHERE enabled=1 AND expires_at<=?`).run(now
|
|
33
|
+
db.query(`UPDATE syncs SET status='ready',error_code='lease_expired',expires_at=NULL
|
|
34
|
+
WHERE enabled=1 AND expires_at<=?`).run(now);
|
|
35
35
|
const row = db
|
|
36
36
|
.query<
|
|
37
37
|
{
|
|
@@ -39,13 +39,12 @@ export function claimAcquisition({
|
|
|
39
39
|
owner_id: string;
|
|
40
40
|
generation: number;
|
|
41
41
|
resync: number;
|
|
42
|
-
failure_count: number;
|
|
43
42
|
},
|
|
44
|
-
[
|
|
45
|
-
>(`SELECT id,owner_id,generation,resync
|
|
46
|
-
WHERE enabled=1 AND expires_at IS NULL
|
|
47
|
-
ORDER BY
|
|
48
|
-
.get(
|
|
43
|
+
[]
|
|
44
|
+
>(`SELECT id,owner_id,generation,resync FROM syncs
|
|
45
|
+
WHERE enabled=1 AND status='ready' AND expires_at IS NULL
|
|
46
|
+
ORDER BY generation,id LIMIT 1`)
|
|
47
|
+
.get();
|
|
49
48
|
if (!row) {
|
|
50
49
|
return;
|
|
51
50
|
}
|
|
@@ -69,7 +68,6 @@ export function claimAcquisition({
|
|
|
69
68
|
sync: readSync({ db, scope: { ...scope, id: row.id } }),
|
|
70
69
|
generation: row.generation + 1,
|
|
71
70
|
force: row.resync === 1,
|
|
72
|
-
failureCount: row.failure_count,
|
|
73
71
|
};
|
|
74
72
|
})
|
|
75
73
|
.immediate();
|
|
@@ -88,15 +86,13 @@ export function finishAcquisition(
|
|
|
88
86
|
lease.ownerId,
|
|
89
87
|
lease.sync.id,
|
|
90
88
|
);
|
|
91
|
-
db.query(`UPDATE syncs SET status=?,error_code=?,
|
|
89
|
+
db.query(`UPDATE syncs SET status=?,error_code=?,expires_at=NULL,
|
|
92
90
|
enabled=CASE WHEN ? THEN 0 ELSE enabled END,
|
|
93
|
-
|
|
91
|
+
resync=CASE WHEN ? THEN 0 ELSE resync END
|
|
94
92
|
WHERE owner_id=? AND id=?`).run(
|
|
95
93
|
input.pause ? 'disabled' : input.state,
|
|
96
94
|
input.errorCode ?? null,
|
|
97
|
-
now + input.delay,
|
|
98
95
|
Number(input.pause ?? false),
|
|
99
|
-
input.failureCount ?? null,
|
|
100
96
|
Number(input.state === 'succeeded'),
|
|
101
97
|
lease.ownerId,
|
|
102
98
|
lease.sync.id,
|
|
@@ -15,10 +15,17 @@ import { writeRecord } from './records';
|
|
|
15
15
|
|
|
16
16
|
export class SqliteAcquisition implements AcquisitionRepository {
|
|
17
17
|
constructor(private readonly input: { db: Database; limits: QueueLimits }) {}
|
|
18
|
+
poll(): void {
|
|
19
|
+
this.input.db
|
|
20
|
+
.query(
|
|
21
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status IN ('succeeded','retrying','interrupted','waiting_for_capacity')",
|
|
22
|
+
)
|
|
23
|
+
.run();
|
|
24
|
+
}
|
|
18
25
|
capacityReleased(): void {
|
|
19
26
|
this.input.db
|
|
20
|
-
.query("UPDATE syncs SET
|
|
21
|
-
.run(
|
|
27
|
+
.query("UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'")
|
|
28
|
+
.run();
|
|
22
29
|
}
|
|
23
30
|
claim(leaseMs: number) {
|
|
24
31
|
return claimAcquisition({ ...this.input, leaseMs });
|
|
@@ -52,8 +59,6 @@ export class SqliteAcquisition implements AcquisitionRepository {
|
|
|
52
59
|
db,
|
|
53
60
|
lease: input.lease,
|
|
54
61
|
state: page.complete ? 'succeeded' : 'ready',
|
|
55
|
-
delay: page.complete ? sync.intervalMs : 0,
|
|
56
|
-
failureCount: page.complete ? 0 : undefined,
|
|
57
62
|
});
|
|
58
63
|
}).immediate();
|
|
59
64
|
}
|
|
@@ -103,9 +103,9 @@ export class SqliteAssets implements AssetRepository {
|
|
|
103
103
|
if (deleted) {
|
|
104
104
|
this.input.db
|
|
105
105
|
.query(
|
|
106
|
-
"UPDATE syncs SET
|
|
106
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity' AND NOT (owner_id=? AND id=?)",
|
|
107
107
|
)
|
|
108
|
-
.run(
|
|
108
|
+
.run(deleted.owner_id, deleted.sync_id);
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
garbage(): string[] {
|
|
@@ -13,7 +13,7 @@ export interface CatalogRepository {
|
|
|
13
13
|
sync(input: Resource): Sync;
|
|
14
14
|
syncs(scope: Scope): Sync[];
|
|
15
15
|
polls(input: Resource & { before?: number }): { polls: SyncPoll[]; nextCursor: number | null };
|
|
16
|
-
connectSync(input: Resource & { connection: ConnectionRef }): Sync;
|
|
16
|
+
connectSync(input: Resource & { connection: ConnectionRef; checkpoint: JsonValue }): Sync;
|
|
17
17
|
setEnabled(input: Resource & { enabled: boolean }): Sync;
|
|
18
18
|
runNow(input: Resource): void;
|
|
19
19
|
resync(input: Resource & { checkpoint: JsonValue }): void;
|
|
@@ -7,8 +7,6 @@ import type { CreateSync, SyncPoll } from '../../models/sync';
|
|
|
7
7
|
import { readSync } from '../rows';
|
|
8
8
|
import type { CatalogRepository } from './contract';
|
|
9
9
|
|
|
10
|
-
const defaultIntervalMs = 60_000;
|
|
11
|
-
|
|
12
10
|
export class SqliteCatalog implements CatalogRepository {
|
|
13
11
|
constructor(private readonly db: Database) {}
|
|
14
12
|
createSync(
|
|
@@ -19,8 +17,8 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
19
17
|
) {
|
|
20
18
|
const id = `sync_${crypto.randomUUID()}`;
|
|
21
19
|
this.db
|
|
22
|
-
.query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,
|
|
23
|
-
VALUES (
|
|
20
|
+
.query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,status)
|
|
21
|
+
VALUES (?,?,?,?,?,?,?,?,?,?)`)
|
|
24
22
|
.run(
|
|
25
23
|
input.ownerId,
|
|
26
24
|
id,
|
|
@@ -31,8 +29,6 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
31
29
|
canonicalJson(input.destination.config).json,
|
|
32
30
|
Number(input.enabled ?? true),
|
|
33
31
|
canonicalJson(input.initialCheckpoint).json,
|
|
34
|
-
input.intervalMs ?? defaultIntervalMs,
|
|
35
|
-
Date.now(),
|
|
36
32
|
input.enabled === false ? 'disabled' : 'ready',
|
|
37
33
|
);
|
|
38
34
|
return this.sync({ ...input, id });
|
|
@@ -60,17 +56,28 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
60
56
|
const polls = rows.slice(0, pageSize);
|
|
61
57
|
return { polls, nextCursor: rows.length > pageSize ? polls.at(-1)!.id : null };
|
|
62
58
|
}
|
|
63
|
-
connectSync(input: Resource & { connection: ConnectionRef }) {
|
|
59
|
+
connectSync(input: Resource & { connection: ConnectionRef; checkpoint: JsonValue }) {
|
|
64
60
|
return this.db
|
|
65
61
|
.transaction(() => {
|
|
66
62
|
const sync = this.sync(input);
|
|
67
|
-
if (sync.
|
|
63
|
+
if (sync.enabled) {
|
|
68
64
|
fail('already_connected');
|
|
69
65
|
}
|
|
70
66
|
this.db
|
|
71
|
-
.query(`UPDATE
|
|
72
|
-
|
|
73
|
-
.run(
|
|
67
|
+
.query(`UPDATE sync_polls SET state='interrupted',error_code='connection_changed',completed_at=?
|
|
68
|
+
WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`)
|
|
69
|
+
.run(Date.now(), input.ownerId, input.id);
|
|
70
|
+
this.db
|
|
71
|
+
.query(`UPDATE syncs SET connection=?,checkpoint=?,enabled=?,
|
|
72
|
+
status=?,error_code=NULL,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`)
|
|
73
|
+
.run(
|
|
74
|
+
canonicalJson(input.connection).json,
|
|
75
|
+
canonicalJson(input.checkpoint).json,
|
|
76
|
+
Number(!sync.connection),
|
|
77
|
+
sync.connection ? 'disabled' : 'ready',
|
|
78
|
+
input.ownerId,
|
|
79
|
+
input.id,
|
|
80
|
+
);
|
|
74
81
|
return this.sync(input);
|
|
75
82
|
})
|
|
76
83
|
.immediate();
|
|
@@ -88,12 +95,11 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
88
95
|
.run(input.enabled ? 'ready' : 'disabled', input.ownerId, input.id);
|
|
89
96
|
this.db
|
|
90
97
|
.query(
|
|
91
|
-
`UPDATE syncs SET enabled=?,status=?,error_code=NULL,
|
|
98
|
+
`UPDATE syncs SET enabled=?,status=?,error_code=NULL,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
|
|
92
99
|
)
|
|
93
100
|
.run(
|
|
94
101
|
Number(input.enabled),
|
|
95
102
|
input.enabled ? 'ready' : 'disabled',
|
|
96
|
-
Date.now(),
|
|
97
103
|
input.ownerId,
|
|
98
104
|
input.id,
|
|
99
105
|
);
|
|
@@ -115,8 +121,8 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
115
121
|
fail('busy');
|
|
116
122
|
}
|
|
117
123
|
this.db
|
|
118
|
-
.query(`UPDATE syncs SET
|
|
119
|
-
.run(
|
|
124
|
+
.query(`UPDATE syncs SET status='ready' WHERE owner_id=? AND id=?`)
|
|
125
|
+
.run(input.ownerId, input.id);
|
|
120
126
|
})
|
|
121
127
|
.immediate();
|
|
122
128
|
}
|
|
@@ -132,9 +138,9 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
132
138
|
.run(Date.now(), input.ownerId, input.id);
|
|
133
139
|
this.db
|
|
134
140
|
.query(
|
|
135
|
-
`UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,
|
|
141
|
+
`UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,resync=1,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
|
|
136
142
|
)
|
|
137
|
-
.run(canonicalJson(input.checkpoint).json,
|
|
143
|
+
.run(canonicalJson(input.checkpoint).json, input.ownerId, input.id);
|
|
138
144
|
})
|
|
139
145
|
.immediate();
|
|
140
146
|
}
|
|
@@ -150,8 +156,10 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
150
156
|
.run(input.ownerId, input.id);
|
|
151
157
|
this.db.query('DELETE FROM syncs WHERE owner_id=? AND id=?').run(input.ownerId, input.id);
|
|
152
158
|
this.db
|
|
153
|
-
.query(
|
|
154
|
-
|
|
159
|
+
.query(
|
|
160
|
+
`UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'`,
|
|
161
|
+
)
|
|
162
|
+
.run();
|
|
155
163
|
})
|
|
156
164
|
.immediate();
|
|
157
165
|
}
|
|
@@ -59,9 +59,9 @@ export class SqliteDeliveries implements DeliveryRepository {
|
|
|
59
59
|
// Capacity is global; all paused acquisitions can compete for the released budget.
|
|
60
60
|
this.db
|
|
61
61
|
.query(
|
|
62
|
-
"UPDATE syncs SET
|
|
62
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'",
|
|
63
63
|
)
|
|
64
|
-
.run(
|
|
64
|
+
.run();
|
|
65
65
|
} else {
|
|
66
66
|
this.db
|
|
67
67
|
.query(
|
package/src/repositories/rows.ts
CHANGED
|
@@ -23,8 +23,6 @@ export function readSync(input: { db: Database; scope: Resource }): Sync {
|
|
|
23
23
|
},
|
|
24
24
|
enabled: row.enabled === 1,
|
|
25
25
|
checkpoint: JSON.parse(String(row.checkpoint)),
|
|
26
|
-
intervalMs: Number(row.interval_ms),
|
|
27
|
-
nextDueAt: Number(row.next_due_at),
|
|
28
26
|
status: row.status as SyncStatus,
|
|
29
27
|
errorCode: row.error_code === null ? null : String(row.error_code),
|
|
30
28
|
};
|
|
@@ -4,7 +4,7 @@ import { bindProvider, type ProviderGateway } from '../execution/provider';
|
|
|
4
4
|
import { assetKey, type SourceAssets } from '../models/asset';
|
|
5
5
|
import { SyncError } from '../models/error';
|
|
6
6
|
import { canonicalJson } from '../models/json';
|
|
7
|
-
import {
|
|
7
|
+
import type { Timing } from '../models/limits';
|
|
8
8
|
import type { Registry } from '../models/registry';
|
|
9
9
|
import { retryableStatus } from '../models/source-http-error';
|
|
10
10
|
import { identifier } from '../models/validation';
|
|
@@ -24,6 +24,9 @@ export class AcquisitionService {
|
|
|
24
24
|
log: Logger;
|
|
25
25
|
},
|
|
26
26
|
) {}
|
|
27
|
+
poll() {
|
|
28
|
+
this.input.repository.poll();
|
|
29
|
+
}
|
|
27
30
|
capacityReleased() {
|
|
28
31
|
this.input.repository.capacityReleased();
|
|
29
32
|
}
|
|
@@ -31,11 +34,11 @@ export class AcquisitionService {
|
|
|
31
34
|
return this.input.repository.claim(this.input.timing.leaseMs);
|
|
32
35
|
}
|
|
33
36
|
async execute(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
|
|
34
|
-
const { repository
|
|
37
|
+
const { repository } = this.input;
|
|
35
38
|
const { lease } = input;
|
|
36
39
|
try {
|
|
37
40
|
if (!repository.hasCapacity()) {
|
|
38
|
-
this.finish({ lease, state: 'waiting_for_capacity'
|
|
41
|
+
this.finish({ lease, state: 'waiting_for_capacity' });
|
|
39
42
|
return;
|
|
40
43
|
}
|
|
41
44
|
await abortable({ signal: input.signal, run: () => this.consume(input) });
|
|
@@ -44,20 +47,15 @@ export class AcquisitionService {
|
|
|
44
47
|
}
|
|
45
48
|
}
|
|
46
49
|
private failed(input: { lease: AcquisitionLease; signal: AbortSignal; error: unknown }): void {
|
|
47
|
-
const {
|
|
50
|
+
const { log } = this.input;
|
|
48
51
|
const { lease, signal, error } = input;
|
|
49
52
|
const code = signal.aborted
|
|
50
53
|
? abortCode(signal)
|
|
51
54
|
: error instanceof SyncError
|
|
52
55
|
? error.code
|
|
53
56
|
: 'execution_failed';
|
|
54
|
-
const failed = !['paused', 'interrupted', 'waiting_for_capacity'].includes(code);
|
|
55
|
-
const failureCount = failed ? lease.failureCount + 1 : lease.failureCount;
|
|
56
57
|
const status = error instanceof SyncError ? error.status : undefined;
|
|
57
58
|
const pause = !signal.aborted && status !== undefined && !retryableStatus(status);
|
|
58
|
-
const delay = failed
|
|
59
|
-
? retryDelay({ attempt: failureCount, retryMs: timing.retryMs })
|
|
60
|
-
: timing.retryMs;
|
|
61
59
|
log({
|
|
62
60
|
code,
|
|
63
61
|
ownerId: lease.ownerId,
|
|
@@ -65,8 +63,7 @@ export class AcquisitionService {
|
|
|
65
63
|
fields: {
|
|
66
64
|
...(error instanceof SyncError ? error.diagnostics : {}),
|
|
67
65
|
...(status === undefined ? {} : { httpStatus: status }),
|
|
68
|
-
|
|
69
|
-
...(pause ? { paused: true } : { retryAfterMs: delay }),
|
|
66
|
+
...(pause ? { paused: true } : {}),
|
|
70
67
|
},
|
|
71
68
|
});
|
|
72
69
|
this.finish({
|
|
@@ -78,8 +75,6 @@ export class AcquisitionService {
|
|
|
78
75
|
? 'interrupted'
|
|
79
76
|
: 'retrying',
|
|
80
77
|
errorCode: code === 'waiting_for_capacity' ? undefined : code,
|
|
81
|
-
delay,
|
|
82
|
-
failureCount,
|
|
83
78
|
pause,
|
|
84
79
|
});
|
|
85
80
|
}
|
|
@@ -4,7 +4,7 @@ import type { ConnectionRef } from '../models/definition';
|
|
|
4
4
|
import { fail } from '../models/error';
|
|
5
5
|
import type { Resource, Scope } from '../models/identity';
|
|
6
6
|
import type { JsonObject } from '../models/json';
|
|
7
|
-
import {
|
|
7
|
+
import type { QueueLimits } from '../models/limits';
|
|
8
8
|
import type { Registry } from '../models/registry';
|
|
9
9
|
import { type CreateSync, summarizeSync } from '../models/sync';
|
|
10
10
|
import { identifier, validate } from '../models/validation';
|
|
@@ -67,9 +67,6 @@ export class SyncManagement {
|
|
|
67
67
|
this.guard(input);
|
|
68
68
|
const { definition } = this.input.registry.definition(input.definition);
|
|
69
69
|
const config = validate({ value: input.config, schema: definition.configSchema }) as JsonObject;
|
|
70
|
-
if (input.intervalMs !== undefined) {
|
|
71
|
-
positive(input.intervalMs);
|
|
72
|
-
}
|
|
73
70
|
if (definition.provider && (input.connection || input.enabled !== false)) {
|
|
74
71
|
await bindProvider({
|
|
75
72
|
actorId: input.actorId,
|
|
@@ -107,10 +104,13 @@ export class SyncManagement {
|
|
|
107
104
|
validateCursor(input.before);
|
|
108
105
|
return this.input.catalog.polls(input);
|
|
109
106
|
}
|
|
107
|
+
/** Bind a waiting sync, or replace a paused sync's connection and restart acquisition.
|
|
108
|
+
* Replacement keeps the sync paused and preserves delivered records and queued deliveries.
|
|
109
|
+
*/
|
|
110
110
|
async connectSync(input: Resource & { connection: ConnectionRef }) {
|
|
111
111
|
this.guard(input);
|
|
112
112
|
const sync = this.input.catalog.sync(input);
|
|
113
|
-
if (sync.
|
|
113
|
+
if (sync.enabled) {
|
|
114
114
|
fail('already_connected');
|
|
115
115
|
}
|
|
116
116
|
const { definition } = this.input.registry.definition(sync.definition);
|
|
@@ -124,7 +124,10 @@ export class SyncManagement {
|
|
|
124
124
|
signal: AbortSignal.timeout(this.input.timeoutMs),
|
|
125
125
|
});
|
|
126
126
|
this.guard(input);
|
|
127
|
-
const connected = this.input.catalog.connectSync(
|
|
127
|
+
const connected = this.input.catalog.connectSync({
|
|
128
|
+
...input,
|
|
129
|
+
checkpoint: definition.initialCheckpoint,
|
|
130
|
+
});
|
|
128
131
|
this.input.worker.wake();
|
|
129
132
|
return summarizeSync(connected);
|
|
130
133
|
}
|
|
File without changes
|
|
File without changes
|