@context-use/open-sync 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -2
- package/src/cron.ts +2 -0
- package/src/db/client.ts +3 -3
- package/src/db/migrate.ts +48 -0
- package/src/db/migrations/0001-centralize-polling.sql +3 -0
- package/src/db/migrations/0001-centralize-polling.sql.d.ts +2 -0
- package/src/execution/cron-client.ts +47 -0
- package/src/execution/cron-invocation.ts +23 -0
- package/src/execution/cron.ts +137 -0
- package/src/execution/schedule.ts +57 -0
- package/src/execution/worker.ts +56 -54
- package/src/http/controller.ts +0 -1
- package/src/models/definition.ts +1 -1
- package/src/models/sync.ts +0 -7
- package/src/open-sync.ts +7 -1
- package/src/repositories/acquisition/contract.ts +1 -4
- package/src/repositories/acquisition/lease.ts +9 -13
- package/src/repositories/acquisition/sqlite.ts +9 -12
- package/src/repositories/assets/sqlite.ts +2 -2
- package/src/repositories/catalog/sqlite.ts +13 -16
- package/src/repositories/delivery/contract.ts +0 -1
- package/src/repositories/delivery/sqlite.ts +2 -11
- package/src/repositories/rows.ts +0 -2
- package/src/runtime.ts +32 -2
- package/src/services/acquisition.ts +8 -16
- package/src/services/delivery.ts +0 -3
- package/src/services/management.ts +1 -4
- /package/src/db/{schema.sql → migrations/0000-initial.sql} +0 -0
- /package/src/db/{schema.sql.d.ts → migrations/0000-initial.sql.d.ts} +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@context-use/open-sync",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/open-sync.ts",
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"./delivery": "./src/models/delivery.ts",
|
|
10
10
|
"./json": "./src/models/json.ts",
|
|
11
11
|
"./http": "./src/http/index.ts",
|
|
12
|
+
"./cron": "./src/cron.ts",
|
|
12
13
|
"./engine": "./src/runtime.ts",
|
|
13
14
|
"./build": "./src/build.ts",
|
|
14
15
|
"./assets": "./src/models/asset.ts"
|
|
@@ -25,10 +26,13 @@
|
|
|
25
26
|
"dependencies": {
|
|
26
27
|
"@cfworker/json-schema": "4.1.1",
|
|
27
28
|
"@oomol-lab/open-connector": "1.7.0",
|
|
29
|
+
"@types/proper-lockfile": "4.1.4",
|
|
28
30
|
"elysia": "^1.4.30",
|
|
29
|
-
"minisearch": "^7.2.0"
|
|
31
|
+
"minisearch": "^7.2.0",
|
|
32
|
+
"proper-lockfile": "4.1.2"
|
|
30
33
|
},
|
|
31
34
|
"devDependencies": {
|
|
35
|
+
"open-sync-previous": "https://registry.npmjs.org/@context-use/open-sync/-/open-sync-0.3.1.tgz",
|
|
32
36
|
"typescript": "6.0.3"
|
|
33
37
|
},
|
|
34
38
|
"engines": {
|
package/src/cron.ts
ADDED
package/src/db/client.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
|
-
import
|
|
2
|
+
import { runMigrations } from './migrate';
|
|
3
3
|
|
|
4
4
|
export function openDatabase(path: string): Database {
|
|
5
5
|
const db = new Database(path, { create: true, strict: true });
|
|
6
6
|
try {
|
|
7
|
-
db.exec('PRAGMA foreign_keys=ON; PRAGMA
|
|
8
|
-
|
|
7
|
+
db.exec('PRAGMA foreign_keys=ON; PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;');
|
|
8
|
+
runMigrations(db);
|
|
9
9
|
return db;
|
|
10
10
|
} catch (error) {
|
|
11
11
|
db.close();
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import initialSchema from './migrations/0000-initial.sql' with { type: 'text' };
|
|
3
|
+
import centralizePolling from './migrations/0001-centralize-polling.sql' with { type: 'text' };
|
|
4
|
+
|
|
5
|
+
// Static imports keep migration history available in both installed packages and compiled hosts.
|
|
6
|
+
const migrations = [
|
|
7
|
+
{ name: '0000-initial.sql', sql: initialSchema },
|
|
8
|
+
{ name: '0001-centralize-polling.sql', sql: centralizePolling },
|
|
9
|
+
] as const;
|
|
10
|
+
|
|
11
|
+
export function runMigrations(db: Database): void {
|
|
12
|
+
db.transaction(() => {
|
|
13
|
+
db.query(`CREATE TABLE IF NOT EXISTS __migrations (
|
|
14
|
+
name TEXT PRIMARY KEY, applied_at TEXT NOT NULL
|
|
15
|
+
)`).run();
|
|
16
|
+
const applied = db
|
|
17
|
+
.query<{ name: string }, []>('SELECT name FROM __migrations ORDER BY name')
|
|
18
|
+
.all();
|
|
19
|
+
for (const [index, { name }] of applied.entries()) {
|
|
20
|
+
if (name !== migrations[index]?.name) {
|
|
21
|
+
throw new Error('Unsupported sync database migration history');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
for (const migration of migrations.slice(applied.length)) {
|
|
25
|
+
executeSql({ db, sql: migration.sql });
|
|
26
|
+
db.query('INSERT INTO __migrations (name,applied_at) VALUES (?,?)').run(
|
|
27
|
+
migration.name,
|
|
28
|
+
new Date().toISOString(),
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}).immediate();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function executeSql({ db, sql }: { db: Database; sql: string }): void {
|
|
35
|
+
// Bun 1.4's exec() can hide failures in non-final statements. Let SQLite parse each
|
|
36
|
+
// statement instead, preserving trigger bodies, comments and quoted semicolons.
|
|
37
|
+
// A final no-op also lets prepare() consume files ending in comments or whitespace.
|
|
38
|
+
let remaining = `${sql}\n;SELECT 1;`;
|
|
39
|
+
while (remaining.length) {
|
|
40
|
+
using statement = db.prepare(remaining);
|
|
41
|
+
const parsed = statement.toString();
|
|
42
|
+
if (!parsed || !remaining.startsWith(parsed)) {
|
|
43
|
+
throw new Error('Migration SQL must contain complete statements without parameters');
|
|
44
|
+
}
|
|
45
|
+
statement.run();
|
|
46
|
+
remaining = remaining.slice(parsed.length);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { request } from 'node:http';
|
|
2
|
+
|
|
3
|
+
const STARTUP_TIMEOUT_MS = 30_000;
|
|
4
|
+
const RETRY_INTERVAL_MS = 100;
|
|
5
|
+
const HTTP_OK = 200;
|
|
6
|
+
|
|
7
|
+
/** Wait for the existing worker to finish, retrying only while its socket is unavailable. */
|
|
8
|
+
export default async function requestCron(socketPath: string): Promise<void> {
|
|
9
|
+
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
|
|
10
|
+
for (;;) {
|
|
11
|
+
try {
|
|
12
|
+
await run(socketPath);
|
|
13
|
+
return;
|
|
14
|
+
} catch (error) {
|
|
15
|
+
const unavailable =
|
|
16
|
+
error instanceof Error &&
|
|
17
|
+
'code' in error &&
|
|
18
|
+
(error.code === 'ECONNREFUSED' || error.code === 'ENOENT');
|
|
19
|
+
if (!unavailable || Date.now() >= deadline) {
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
22
|
+
await Bun.sleep(RETRY_INTERVAL_MS);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function run(socketPath: string): Promise<void> {
|
|
28
|
+
// biome-ignore lint/complexity/useMaxParams: Native Promise executor signature.
|
|
29
|
+
return new Promise((resolve, reject) => {
|
|
30
|
+
const client = request(
|
|
31
|
+
{ socketPath, path: '/run', method: 'POST', agent: false },
|
|
32
|
+
(response) => {
|
|
33
|
+
response.on('error', reject);
|
|
34
|
+
response.on('end', () => {
|
|
35
|
+
if (response.statusCode === HTTP_OK) {
|
|
36
|
+
resolve();
|
|
37
|
+
} else {
|
|
38
|
+
reject(new Error(`Open Sync cron failed: ${response.statusCode}`));
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
response.resume();
|
|
42
|
+
},
|
|
43
|
+
);
|
|
44
|
+
client.on('error', reject);
|
|
45
|
+
client.end();
|
|
46
|
+
});
|
|
47
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import requestCron from './cron-client';
|
|
2
|
+
|
|
3
|
+
const TITLE_PREFIX = 'open-sync-';
|
|
4
|
+
|
|
5
|
+
/** Handle Bun's compiled cron invocation before loading application configuration or services. */
|
|
6
|
+
export async function runOpenSyncCron(): Promise<boolean> {
|
|
7
|
+
const argument = process.argv.find((value) => value.startsWith(`--cron-title=${TITLE_PREFIX}`));
|
|
8
|
+
if (!argument) {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
11
|
+
const encoded = argument.slice(`--cron-title=${TITLE_PREFIX}`.length);
|
|
12
|
+
const socket = Buffer.from(encoded, 'base64url').toString();
|
|
13
|
+
if (!socket.startsWith('/') || Buffer.from(socket).toString('base64url') !== encoded) {
|
|
14
|
+
throw new Error('Invalid Open Sync cron invocation.');
|
|
15
|
+
}
|
|
16
|
+
await requestCron(socket);
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function cronTitle(socket: string): string {
|
|
21
|
+
// The command must find its worker even when Unix cron supplies a different cwd/environment.
|
|
22
|
+
return `${TITLE_PREFIX}${Buffer.from(socket).toString('base64url')}`;
|
|
23
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { chmod, lstat, mkdir, unlink, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { request } from 'node:http';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { cronTitle } from './cron-invocation';
|
|
5
|
+
import { updateSchedule } from './schedule';
|
|
6
|
+
|
|
7
|
+
const PRIVATE_DIRECTORY_MODE = 0o700;
|
|
8
|
+
const PRIVATE_SOCKET_MODE = 0o600;
|
|
9
|
+
const HTTP_OK = 200;
|
|
10
|
+
const HTTP_NOT_FOUND = 404;
|
|
11
|
+
const HTTP_UNAVAILABLE = 503;
|
|
12
|
+
const PROBE_TIMEOUT_MS = 1_000;
|
|
13
|
+
|
|
14
|
+
/** One half-hourly check for every sync owned by this runtime. */
|
|
15
|
+
export async function startCron(input: {
|
|
16
|
+
runtime: { runDue(): Promise<void> };
|
|
17
|
+
/** A private, stable directory owned by this runtime. */
|
|
18
|
+
directory: string;
|
|
19
|
+
/** Report a failed run; the recurring job remains registered for the next invocation. */
|
|
20
|
+
onError(error: unknown): void;
|
|
21
|
+
}) {
|
|
22
|
+
const directory = resolve(input.directory);
|
|
23
|
+
await mkdir(directory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
|
|
24
|
+
await chmod(directory, PRIVATE_DIRECTORY_MODE);
|
|
25
|
+
const socket = join(directory, 'worker.sock');
|
|
26
|
+
await removeStaleSocket(socket);
|
|
27
|
+
const title = cronTitle(socket);
|
|
28
|
+
let entrypoint = Bun.main;
|
|
29
|
+
if (!Bun.isStandaloneExecutable) {
|
|
30
|
+
// Bun consumes cron arguments in source mode and invokes default.scheduled().
|
|
31
|
+
// Only the absolute import and socket path are generated; the client remains ordinary code.
|
|
32
|
+
entrypoint = join(directory, 'scheduled.ts');
|
|
33
|
+
await writeFile(
|
|
34
|
+
entrypoint,
|
|
35
|
+
`import requestCron from ${JSON.stringify(import.meta.resolve('./cron-client.ts'))};\n` +
|
|
36
|
+
`export default { scheduled: () => requestCron(${JSON.stringify(socket)}) };\n`,
|
|
37
|
+
{ mode: PRIVATE_SOCKET_MODE },
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
let closed = false;
|
|
41
|
+
const server = Bun.serve({
|
|
42
|
+
unix: socket,
|
|
43
|
+
async fetch(request) {
|
|
44
|
+
this.timeout(request, 0);
|
|
45
|
+
const path = new URL(request.url).pathname;
|
|
46
|
+
if (path === '/health') {
|
|
47
|
+
return new Response(null, { status: HTTP_OK });
|
|
48
|
+
}
|
|
49
|
+
if (path !== '/run' || request.method !== 'POST') {
|
|
50
|
+
return new Response(null, { status: HTTP_NOT_FOUND });
|
|
51
|
+
}
|
|
52
|
+
if (closed) {
|
|
53
|
+
return new Response(null, { status: HTTP_UNAVAILABLE });
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
await input.runtime.runDue();
|
|
57
|
+
return new Response(null, { status: HTTP_OK });
|
|
58
|
+
} catch (error) {
|
|
59
|
+
input.onError(error);
|
|
60
|
+
return new Response(null, { status: HTTP_UNAVAILABLE });
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
let closing: Promise<void> | undefined;
|
|
65
|
+
const close = () => {
|
|
66
|
+
closing ??= (async () => {
|
|
67
|
+
closed = true;
|
|
68
|
+
try {
|
|
69
|
+
await updateSchedule(() => Bun.cron.remove(title));
|
|
70
|
+
} finally {
|
|
71
|
+
await server.stop(true);
|
|
72
|
+
}
|
|
73
|
+
})();
|
|
74
|
+
return closing;
|
|
75
|
+
};
|
|
76
|
+
try {
|
|
77
|
+
await chmod(socket, PRIVATE_SOCKET_MODE);
|
|
78
|
+
await updateSchedule(() => Bun.cron(entrypoint, '*/30 * * * *', title));
|
|
79
|
+
return { close, [Symbol.asyncDispose]: close };
|
|
80
|
+
} catch (error) {
|
|
81
|
+
await server.stop(true);
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function removeStaleSocket(socket: string): Promise<void> {
|
|
87
|
+
const existing = await lstat(socket).catch((error: NodeJS.ErrnoException) => {
|
|
88
|
+
if (error.code !== 'ENOENT') {
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
if (!existing) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (!existing.isSocket()) {
|
|
96
|
+
throw new Error('The cron worker socket path is occupied by another file.');
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
await probeWorker(socket);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
if (unavailableSocket(error)) {
|
|
102
|
+
await unlink(socket);
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
throw error;
|
|
106
|
+
}
|
|
107
|
+
throw new Error('An Open Sync cron worker already owns this directory.');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function unavailableSocket(error: unknown): boolean {
|
|
111
|
+
return (
|
|
112
|
+
error instanceof Error &&
|
|
113
|
+
'code' in error &&
|
|
114
|
+
['ECONNREFUSED', 'ENOENT'].includes(String(error.code))
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function probeWorker(socket: string): Promise<number> {
|
|
119
|
+
// biome-ignore lint/complexity/useMaxParams: Native Promise executor signature.
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const client = request(
|
|
122
|
+
{
|
|
123
|
+
socketPath: socket,
|
|
124
|
+
path: '/health',
|
|
125
|
+
agent: false,
|
|
126
|
+
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
|
|
127
|
+
},
|
|
128
|
+
(response) => {
|
|
129
|
+
response.on('error', reject);
|
|
130
|
+
response.on('end', () => resolve(response.statusCode!));
|
|
131
|
+
response.resume();
|
|
132
|
+
},
|
|
133
|
+
);
|
|
134
|
+
client.on('error', reject);
|
|
135
|
+
client.end();
|
|
136
|
+
});
|
|
137
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { tmpdir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { lock } from 'proper-lockfile';
|
|
4
|
+
|
|
5
|
+
const LOCK_STALE_MS = 30_000;
|
|
6
|
+
const LOCK_RETRIES = 40;
|
|
7
|
+
const LOCK_RETRY_MS = 1_000;
|
|
8
|
+
let updating = Promise.resolve();
|
|
9
|
+
|
|
10
|
+
/** Bun replaces the user's entire crontab; serialize changes across Open Sync hosts. */
|
|
11
|
+
export function updateSchedule(operation: () => Promise<void>): Promise<void> {
|
|
12
|
+
const pending = updating.then(async () => {
|
|
13
|
+
const release = await lock(join(tmpdir(), `open-sync-crontab-${process.getuid!()}`), {
|
|
14
|
+
realpath: false,
|
|
15
|
+
stale: LOCK_STALE_MS,
|
|
16
|
+
retries: { retries: LOCK_RETRIES, minTimeout: LOCK_RETRY_MS, maxTimeout: LOCK_RETRY_MS },
|
|
17
|
+
});
|
|
18
|
+
try {
|
|
19
|
+
if (process.platform === 'linux') {
|
|
20
|
+
await verifyCrontabReadable();
|
|
21
|
+
}
|
|
22
|
+
await operation();
|
|
23
|
+
} finally {
|
|
24
|
+
await release();
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
updating = pending.catch(() => undefined);
|
|
28
|
+
return pending;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Bun 1.4 treats every `crontab -l` exit 1 as an empty table. Reject permission/service errors
|
|
32
|
+
// before asking it to replace the table; only the documented missing-table responses are safe.
|
|
33
|
+
async function verifyCrontabReadable(): Promise<void> {
|
|
34
|
+
const child = Bun.spawn(['crontab', '-l'], {
|
|
35
|
+
env: { ...process.env, LC_ALL: 'C' },
|
|
36
|
+
stdout: 'pipe',
|
|
37
|
+
stderr: 'pipe',
|
|
38
|
+
timeout: 10_000,
|
|
39
|
+
});
|
|
40
|
+
const [code, output, error] = await Promise.all([
|
|
41
|
+
child.exited,
|
|
42
|
+
new Response(child.stdout).text(),
|
|
43
|
+
new Response(child.stderr).text(),
|
|
44
|
+
]);
|
|
45
|
+
if (code === 0) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (
|
|
49
|
+
code === 1 &&
|
|
50
|
+
!output &&
|
|
51
|
+
(/^no crontab for [^\r\n]+$/i.test(error.trim()) ||
|
|
52
|
+
/^crontab: can't open '[^/'\r\n]+': No such file or directory$/.test(error.trim()))
|
|
53
|
+
) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
throw new Error(`Cannot read crontab: ${error.trim() || `exit ${code}`}`);
|
|
57
|
+
}
|
package/src/execution/worker.ts
CHANGED
|
@@ -13,7 +13,6 @@ export interface WorkerControl {
|
|
|
13
13
|
}
|
|
14
14
|
export class Worker implements WorkerControl {
|
|
15
15
|
readonly #lifetime = new AbortController();
|
|
16
|
-
#timer?: ReturnType<typeof setTimeout>;
|
|
17
16
|
#started = false;
|
|
18
17
|
readonly #acquisitions = new Map<
|
|
19
18
|
Promise<void>,
|
|
@@ -26,8 +25,11 @@ export class Worker implements WorkerControl {
|
|
|
26
25
|
#cleaning?: Promise<void>;
|
|
27
26
|
#cleanupRequested = false;
|
|
28
27
|
#capacityReleased = false;
|
|
29
|
-
#cleanupTimer?: ReturnType<typeof setTimeout>;
|
|
30
28
|
#closing?: Promise<void>;
|
|
29
|
+
#draining?: Promise<void>;
|
|
30
|
+
#polling = false;
|
|
31
|
+
#drainRequested = false;
|
|
32
|
+
#resume?: () => void;
|
|
31
33
|
constructor(
|
|
32
34
|
private readonly input: {
|
|
33
35
|
acquisition: AcquisitionService;
|
|
@@ -48,30 +50,13 @@ export class Worker implements WorkerControl {
|
|
|
48
50
|
return;
|
|
49
51
|
}
|
|
50
52
|
this.#started = true;
|
|
51
|
-
this.
|
|
53
|
+
this.wake();
|
|
52
54
|
}
|
|
53
55
|
wake(): void {
|
|
54
|
-
this.
|
|
55
|
-
}
|
|
56
|
-
private schedule(delay: number): void {
|
|
57
|
-
if (!this.#started || this.#lifetime.signal.aborted) {
|
|
56
|
+
if ((!this.#started && !this.#draining) || this.#lifetime.signal.aborted) {
|
|
58
57
|
return;
|
|
59
58
|
}
|
|
60
|
-
|
|
61
|
-
// Yield to the event loop even when another step is immediately runnable.
|
|
62
|
-
const maxTimerDelay = 2_147_483_647;
|
|
63
|
-
this.#timer = setTimeout(
|
|
64
|
-
() => {
|
|
65
|
-
this.#timer = undefined;
|
|
66
|
-
try {
|
|
67
|
-
this.dispatch();
|
|
68
|
-
} catch {
|
|
69
|
-
this.input.log({ code: 'runtime_failed' });
|
|
70
|
-
this.schedule(this.input.timing.retryMs);
|
|
71
|
-
}
|
|
72
|
-
},
|
|
73
|
-
Math.min(delay, maxTimerDelay),
|
|
74
|
-
);
|
|
59
|
+
void this.requestDrain().catch(() => this.input.log({ code: 'runtime_failed' }));
|
|
75
60
|
}
|
|
76
61
|
/** A deterministic dispatch round for hosts/tests that do not start the background worker. */
|
|
77
62
|
tick(): Promise<void> {
|
|
@@ -81,6 +66,49 @@ export class Worker implements WorkerControl {
|
|
|
81
66
|
this.cleanup(),
|
|
82
67
|
);
|
|
83
68
|
}
|
|
69
|
+
/** Poll enabled syncs and drain runnable work, sharing a round across concurrent host triggers. */
|
|
70
|
+
runDue(): Promise<void> {
|
|
71
|
+
this.ensureOpen();
|
|
72
|
+
if (!this.#polling) {
|
|
73
|
+
this.input.acquisition.poll();
|
|
74
|
+
this.#polling = true;
|
|
75
|
+
}
|
|
76
|
+
return this.requestDrain();
|
|
77
|
+
}
|
|
78
|
+
private requestDrain(): Promise<void> {
|
|
79
|
+
this.#drainRequested = true;
|
|
80
|
+
this.#resume?.();
|
|
81
|
+
this.#draining ??= Promise.resolve().then(() => this.drain());
|
|
82
|
+
return this.#draining;
|
|
83
|
+
}
|
|
84
|
+
private async drain(): Promise<void> {
|
|
85
|
+
try {
|
|
86
|
+
while (!this.#lifetime.signal.aborted) {
|
|
87
|
+
this.#drainRequested = false;
|
|
88
|
+
const changed = Promise.withResolvers<void>();
|
|
89
|
+
this.#resume = changed.resolve;
|
|
90
|
+
this.dispatch();
|
|
91
|
+
const active = this.#acquisitions.size + this.#deliveries.size;
|
|
92
|
+
if (active) {
|
|
93
|
+
await changed.promise;
|
|
94
|
+
}
|
|
95
|
+
await this.cleanup();
|
|
96
|
+
if (this.#lifetime.signal.aborted) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (active || this.#drainRequested) {
|
|
100
|
+
// Continue pages immediately, while allowing cancellation and host requests to run.
|
|
101
|
+
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
} finally {
|
|
107
|
+
this.#resume = undefined;
|
|
108
|
+
this.#draining = undefined;
|
|
109
|
+
this.#polling = false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
84
112
|
private dispatch(): void {
|
|
85
113
|
this.ensureOpen();
|
|
86
114
|
const { acquisition, delivery, timing } = this.input;
|
|
@@ -95,7 +123,7 @@ export class Worker implements WorkerControl {
|
|
|
95
123
|
.catch(() => this.input.log({ code: 'acquisition_failed' }))
|
|
96
124
|
.finally(() => {
|
|
97
125
|
this.#acquisitions.delete(task);
|
|
98
|
-
this.
|
|
126
|
+
this.wake();
|
|
99
127
|
});
|
|
100
128
|
this.#acquisitions.set(task, { ownerId: lease.ownerId, syncId: lease.sync.id, abort });
|
|
101
129
|
}
|
|
@@ -113,34 +141,10 @@ export class Worker implements WorkerControl {
|
|
|
113
141
|
.catch(() => this.input.log({ code: 'delivery_failed' }))
|
|
114
142
|
.finally(() => {
|
|
115
143
|
this.#deliveries.delete(task);
|
|
116
|
-
this.
|
|
144
|
+
this.wake();
|
|
117
145
|
});
|
|
118
146
|
this.#deliveries.set(task, { ownerId: lease.ownerId, syncId: lease.delivery.syncId, abort });
|
|
119
147
|
}
|
|
120
|
-
const due = [
|
|
121
|
-
this.#acquisitions.size < timing.sourceConcurrency ? acquisition.nextDue() : undefined,
|
|
122
|
-
this.#deliveries.size < timing.deliveryConcurrency ? delivery.nextDue() : undefined,
|
|
123
|
-
].filter((value): value is number => value !== undefined);
|
|
124
|
-
if (due.length) {
|
|
125
|
-
this.schedule(Math.max(0, Math.min(...due) - Date.now()));
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
private completed(): void {
|
|
129
|
-
this.wake();
|
|
130
|
-
if (this.#started) {
|
|
131
|
-
void this.cleanup().then(
|
|
132
|
-
() => this.wake(),
|
|
133
|
-
() => {
|
|
134
|
-
this.input.log({ code: 'cleanup_failed' });
|
|
135
|
-
if (!this.#lifetime.signal.aborted && !this.#cleanupTimer) {
|
|
136
|
-
this.#cleanupTimer = setTimeout(() => {
|
|
137
|
-
this.#cleanupTimer = undefined;
|
|
138
|
-
this.completed();
|
|
139
|
-
}, this.input.timing.retryMs);
|
|
140
|
-
}
|
|
141
|
-
},
|
|
142
|
-
);
|
|
143
|
-
}
|
|
144
148
|
}
|
|
145
149
|
private cleanup(): Promise<void> {
|
|
146
150
|
this.#cleanupRequested = true;
|
|
@@ -157,8 +161,6 @@ export class Worker implements WorkerControl {
|
|
|
157
161
|
this.#capacityReleased = false;
|
|
158
162
|
this.input.acquisition.capacityReleased();
|
|
159
163
|
}
|
|
160
|
-
clearTimeout(this.#cleanupTimer);
|
|
161
|
-
this.#cleanupTimer = undefined;
|
|
162
164
|
} finally {
|
|
163
165
|
this.#cleaning = undefined;
|
|
164
166
|
}
|
|
@@ -172,7 +174,7 @@ export class Worker implements WorkerControl {
|
|
|
172
174
|
}
|
|
173
175
|
async cancelAcquisition(input: Resource): Promise<void> {
|
|
174
176
|
await this.cancelTasks({ scope: input, tasks: this.#acquisitions });
|
|
175
|
-
this.
|
|
177
|
+
this.wake();
|
|
176
178
|
}
|
|
177
179
|
async cancelSync(input: Resource): Promise<void> {
|
|
178
180
|
await Promise.all([
|
|
@@ -180,7 +182,7 @@ export class Worker implements WorkerControl {
|
|
|
180
182
|
this.cancelTasks({ scope: input, tasks: this.#deliveries }),
|
|
181
183
|
]);
|
|
182
184
|
this.#capacityReleased = true;
|
|
183
|
-
this.
|
|
185
|
+
this.wake();
|
|
184
186
|
await this.cleanup().catch(() => this.input.log({ code: 'cleanup_failed' }));
|
|
185
187
|
}
|
|
186
188
|
private async cancelTasks({
|
|
@@ -204,10 +206,10 @@ export class Worker implements WorkerControl {
|
|
|
204
206
|
return this.#closing;
|
|
205
207
|
}
|
|
206
208
|
private async stop(): Promise<void> {
|
|
207
|
-
clearTimeout(this.#timer);
|
|
208
|
-
clearTimeout(this.#cleanupTimer);
|
|
209
209
|
this.#lifetime.abort('interrupted');
|
|
210
|
+
this.#resume?.();
|
|
210
211
|
await Promise.allSettled([...this.#acquisitions.keys(), ...this.#deliveries.keys()]);
|
|
212
|
+
await this.#draining?.catch(() => undefined);
|
|
211
213
|
await this.cleanup();
|
|
212
214
|
}
|
|
213
215
|
}
|
package/src/http/controller.ts
CHANGED
|
@@ -15,7 +15,6 @@ const syncBody = t.Object({
|
|
|
15
15
|
config,
|
|
16
16
|
destination: t.Object({ type: identifier, input: config }),
|
|
17
17
|
connection: t.Optional(t.Object({ id: identifier, service: identifier })),
|
|
18
|
-
intervalMs: t.Optional(t.Integer({ minimum: 1 })),
|
|
19
18
|
enabled: t.Optional(t.Boolean()),
|
|
20
19
|
});
|
|
21
20
|
const resourceParams = { params: t.Object({ id: identifier }) };
|
package/src/models/definition.ts
CHANGED
|
@@ -54,7 +54,7 @@ export interface SyncStep {
|
|
|
54
54
|
* Include any cross-iteration update boundary as well as the current page position.
|
|
55
55
|
*/
|
|
56
56
|
checkpoint: JsonValue;
|
|
57
|
-
/** False continues this iteration immediately; true
|
|
57
|
+
/** False continues this iteration immediately; true waits for the shared cron or a manual run.
|
|
58
58
|
* Completion retains the returned checkpoint; the source clears any exhausted page cursor.
|
|
59
59
|
*/
|
|
60
60
|
complete: boolean;
|
package/src/models/sync.ts
CHANGED
|
@@ -11,8 +11,6 @@ export interface Sync {
|
|
|
11
11
|
destination: { type: string; config: JsonObject };
|
|
12
12
|
enabled: boolean;
|
|
13
13
|
checkpoint: JsonValue;
|
|
14
|
-
intervalMs: number;
|
|
15
|
-
nextDueAt: number;
|
|
16
14
|
status: SyncStatus;
|
|
17
15
|
errorCode: string | null;
|
|
18
16
|
}
|
|
@@ -21,7 +19,6 @@ export interface CreateSync extends Scope {
|
|
|
21
19
|
connection?: ConnectionRef;
|
|
22
20
|
config: JsonObject;
|
|
23
21
|
destination: { type: string; input: JsonObject };
|
|
24
|
-
intervalMs?: number;
|
|
25
22
|
enabled?: boolean;
|
|
26
23
|
}
|
|
27
24
|
|
|
@@ -51,8 +48,6 @@ export interface SyncSummary {
|
|
|
51
48
|
destinationType: string;
|
|
52
49
|
connection?: ConnectionRef;
|
|
53
50
|
enabled: boolean;
|
|
54
|
-
intervalMs: number;
|
|
55
|
-
nextDueAt: number;
|
|
56
51
|
status: SyncStatus;
|
|
57
52
|
errorCode: string | null;
|
|
58
53
|
}
|
|
@@ -63,8 +58,6 @@ export function summarizeSync(sync: Sync): SyncSummary {
|
|
|
63
58
|
destinationType: sync.destination.type,
|
|
64
59
|
connection: sync.connection,
|
|
65
60
|
enabled: sync.enabled,
|
|
66
|
-
intervalMs: sync.intervalMs,
|
|
67
|
-
nextDueAt: sync.nextDueAt,
|
|
68
61
|
status: sync.status,
|
|
69
62
|
errorCode: sync.errorCode,
|
|
70
63
|
};
|
package/src/open-sync.ts
CHANGED
|
@@ -158,6 +158,7 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
|
|
|
158
158
|
api,
|
|
159
159
|
providers: providerApi(providers),
|
|
160
160
|
start: () => runtime.start(),
|
|
161
|
+
runDue: () => runtime.runDue(),
|
|
161
162
|
async fetch(request: Request): Promise<Response> {
|
|
162
163
|
if (lifetime.signal.aborted) {
|
|
163
164
|
return new Response(null, { status: 503 });
|
|
@@ -205,7 +206,12 @@ export interface OpenSyncRuntime {
|
|
|
205
206
|
api: SyncApi;
|
|
206
207
|
providers: ProviderApi;
|
|
207
208
|
fetch(request: Request): Promise<Response>;
|
|
208
|
-
|
|
209
|
+
/**
|
|
210
|
+
* Register one half-hourly OS job and start processing work.
|
|
211
|
+
* Compiled hosts must call runOpenSyncCron() from @context-use/open-sync/cron before initialization.
|
|
212
|
+
*/
|
|
213
|
+
start(): Promise<void>;
|
|
214
|
+
runDue(): Promise<void>;
|
|
209
215
|
close(): Promise<void>;
|
|
210
216
|
}
|
|
211
217
|
export type { ProviderApi, SyncApi } from './api';
|
|
@@ -6,11 +6,10 @@ export interface AcquisitionLease extends Scope {
|
|
|
6
6
|
sync: Sync;
|
|
7
7
|
generation: number;
|
|
8
8
|
force: boolean;
|
|
9
|
-
failureCount: number;
|
|
10
9
|
}
|
|
11
10
|
export interface AcquisitionRepository {
|
|
11
|
+
poll(): void;
|
|
12
12
|
capacityReleased(): void;
|
|
13
|
-
nextDue(): number | undefined;
|
|
14
13
|
claim(leaseMs: number): AcquisitionLease | undefined;
|
|
15
14
|
hasCapacity(): boolean;
|
|
16
15
|
commit(input: { lease: AcquisitionLease; page: SyncStep; definition: SyncDefinition }): void;
|
|
@@ -18,8 +17,6 @@ export interface AcquisitionRepository {
|
|
|
18
17
|
lease: AcquisitionLease;
|
|
19
18
|
state: Exclude<SyncStatus, 'running' | 'disabled'>;
|
|
20
19
|
errorCode?: string;
|
|
21
|
-
delay: number;
|
|
22
|
-
failureCount?: number;
|
|
23
20
|
pause?: boolean;
|
|
24
21
|
}): void;
|
|
25
22
|
}
|
|
@@ -30,8 +30,8 @@ export function claimAcquisition({
|
|
|
30
30
|
WHERE completed_at IS NULL AND EXISTS (SELECT 1 FROM syncs
|
|
31
31
|
WHERE syncs.owner_id=sync_polls.owner_id AND syncs.id=sync_polls.sync_id
|
|
32
32
|
AND enabled=1 AND expires_at<=?)`).run(now);
|
|
33
|
-
db.query(`UPDATE syncs SET
|
|
34
|
-
WHERE enabled=1 AND expires_at<=?`).run(now
|
|
33
|
+
db.query(`UPDATE syncs SET status='ready',error_code='lease_expired',expires_at=NULL
|
|
34
|
+
WHERE enabled=1 AND expires_at<=?`).run(now);
|
|
35
35
|
const row = db
|
|
36
36
|
.query<
|
|
37
37
|
{
|
|
@@ -39,13 +39,12 @@ export function claimAcquisition({
|
|
|
39
39
|
owner_id: string;
|
|
40
40
|
generation: number;
|
|
41
41
|
resync: number;
|
|
42
|
-
failure_count: number;
|
|
43
42
|
},
|
|
44
|
-
[
|
|
45
|
-
>(`SELECT id,owner_id,generation,resync
|
|
46
|
-
WHERE enabled=1 AND expires_at IS NULL
|
|
47
|
-
ORDER BY
|
|
48
|
-
.get(
|
|
43
|
+
[]
|
|
44
|
+
>(`SELECT id,owner_id,generation,resync FROM syncs
|
|
45
|
+
WHERE enabled=1 AND status='ready' AND expires_at IS NULL
|
|
46
|
+
ORDER BY generation,id LIMIT 1`)
|
|
47
|
+
.get();
|
|
49
48
|
if (!row) {
|
|
50
49
|
return;
|
|
51
50
|
}
|
|
@@ -69,7 +68,6 @@ export function claimAcquisition({
|
|
|
69
68
|
sync: readSync({ db, scope: { ...scope, id: row.id } }),
|
|
70
69
|
generation: row.generation + 1,
|
|
71
70
|
force: row.resync === 1,
|
|
72
|
-
failureCount: row.failure_count,
|
|
73
71
|
};
|
|
74
72
|
})
|
|
75
73
|
.immediate();
|
|
@@ -88,15 +86,13 @@ export function finishAcquisition(
|
|
|
88
86
|
lease.ownerId,
|
|
89
87
|
lease.sync.id,
|
|
90
88
|
);
|
|
91
|
-
db.query(`UPDATE syncs SET status=?,error_code=?,
|
|
89
|
+
db.query(`UPDATE syncs SET status=?,error_code=?,expires_at=NULL,
|
|
92
90
|
enabled=CASE WHEN ? THEN 0 ELSE enabled END,
|
|
93
|
-
|
|
91
|
+
resync=CASE WHEN ? THEN 0 ELSE resync END
|
|
94
92
|
WHERE owner_id=? AND id=?`).run(
|
|
95
93
|
input.pause ? 'disabled' : input.state,
|
|
96
94
|
input.errorCode ?? null,
|
|
97
|
-
now + input.delay,
|
|
98
95
|
Number(input.pause ?? false),
|
|
99
|
-
input.failureCount ?? null,
|
|
100
96
|
Number(input.state === 'succeeded'),
|
|
101
97
|
lease.ownerId,
|
|
102
98
|
lease.sync.id,
|
|
@@ -15,18 +15,17 @@ import { writeRecord } from './records';
|
|
|
15
15
|
|
|
16
16
|
export class SqliteAcquisition implements AcquisitionRepository {
|
|
17
17
|
constructor(private readonly input: { db: Database; limits: QueueLimits }) {}
|
|
18
|
-
|
|
18
|
+
poll(): void {
|
|
19
19
|
this.input.db
|
|
20
|
-
.query(
|
|
21
|
-
|
|
20
|
+
.query(
|
|
21
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status IN ('succeeded','retrying','interrupted','waiting_for_capacity')",
|
|
22
|
+
)
|
|
23
|
+
.run();
|
|
22
24
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
SELECT MIN(COALESCE(expires_at,next_due_at)) AS due FROM syncs WHERE enabled=1`)
|
|
28
|
-
.get()?.due ?? undefined
|
|
29
|
-
);
|
|
25
|
+
capacityReleased(): void {
|
|
26
|
+
this.input.db
|
|
27
|
+
.query("UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'")
|
|
28
|
+
.run();
|
|
30
29
|
}
|
|
31
30
|
claim(leaseMs: number) {
|
|
32
31
|
return claimAcquisition({ ...this.input, leaseMs });
|
|
@@ -60,8 +59,6 @@ export class SqliteAcquisition implements AcquisitionRepository {
|
|
|
60
59
|
db,
|
|
61
60
|
lease: input.lease,
|
|
62
61
|
state: page.complete ? 'succeeded' : 'ready',
|
|
63
|
-
delay: page.complete ? sync.intervalMs : 0,
|
|
64
|
-
failureCount: page.complete ? 0 : undefined,
|
|
65
62
|
});
|
|
66
63
|
}).immediate();
|
|
67
64
|
}
|
|
@@ -103,9 +103,9 @@ export class SqliteAssets implements AssetRepository {
|
|
|
103
103
|
if (deleted) {
|
|
104
104
|
this.input.db
|
|
105
105
|
.query(
|
|
106
|
-
"UPDATE syncs SET
|
|
106
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity' AND NOT (owner_id=? AND id=?)",
|
|
107
107
|
)
|
|
108
|
-
.run(
|
|
108
|
+
.run(deleted.owner_id, deleted.sync_id);
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
garbage(): string[] {
|
|
@@ -7,8 +7,6 @@ import type { CreateSync, SyncPoll } from '../../models/sync';
|
|
|
7
7
|
import { readSync } from '../rows';
|
|
8
8
|
import type { CatalogRepository } from './contract';
|
|
9
9
|
|
|
10
|
-
const defaultIntervalMs = 60_000;
|
|
11
|
-
|
|
12
10
|
export class SqliteCatalog implements CatalogRepository {
|
|
13
11
|
constructor(private readonly db: Database) {}
|
|
14
12
|
createSync(
|
|
@@ -19,8 +17,8 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
19
17
|
) {
|
|
20
18
|
const id = `sync_${crypto.randomUUID()}`;
|
|
21
19
|
this.db
|
|
22
|
-
.query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,
|
|
23
|
-
VALUES (
|
|
20
|
+
.query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,status)
|
|
21
|
+
VALUES (?,?,?,?,?,?,?,?,?,?)`)
|
|
24
22
|
.run(
|
|
25
23
|
input.ownerId,
|
|
26
24
|
id,
|
|
@@ -31,8 +29,6 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
31
29
|
canonicalJson(input.destination.config).json,
|
|
32
30
|
Number(input.enabled ?? true),
|
|
33
31
|
canonicalJson(input.initialCheckpoint).json,
|
|
34
|
-
input.intervalMs ?? defaultIntervalMs,
|
|
35
|
-
Date.now(),
|
|
36
32
|
input.enabled === false ? 'disabled' : 'ready',
|
|
37
33
|
);
|
|
38
34
|
return this.sync({ ...input, id });
|
|
@@ -69,8 +65,8 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
69
65
|
}
|
|
70
66
|
this.db
|
|
71
67
|
.query(`UPDATE syncs SET connection=?,enabled=1,
|
|
72
|
-
status='ready',error_code=NULL
|
|
73
|
-
.run(canonicalJson(input.connection).json,
|
|
68
|
+
status='ready',error_code=NULL WHERE owner_id=? AND id=?`)
|
|
69
|
+
.run(canonicalJson(input.connection).json, input.ownerId, input.id);
|
|
74
70
|
return this.sync(input);
|
|
75
71
|
})
|
|
76
72
|
.immediate();
|
|
@@ -88,12 +84,11 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
88
84
|
.run(input.enabled ? 'ready' : 'disabled', input.ownerId, input.id);
|
|
89
85
|
this.db
|
|
90
86
|
.query(
|
|
91
|
-
`UPDATE syncs SET enabled=?,status=?,error_code=NULL,
|
|
87
|
+
`UPDATE syncs SET enabled=?,status=?,error_code=NULL,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
|
|
92
88
|
)
|
|
93
89
|
.run(
|
|
94
90
|
Number(input.enabled),
|
|
95
91
|
input.enabled ? 'ready' : 'disabled',
|
|
96
|
-
Date.now(),
|
|
97
92
|
input.ownerId,
|
|
98
93
|
input.id,
|
|
99
94
|
);
|
|
@@ -115,8 +110,8 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
115
110
|
fail('busy');
|
|
116
111
|
}
|
|
117
112
|
this.db
|
|
118
|
-
.query(`UPDATE syncs SET
|
|
119
|
-
.run(
|
|
113
|
+
.query(`UPDATE syncs SET status='ready' WHERE owner_id=? AND id=?`)
|
|
114
|
+
.run(input.ownerId, input.id);
|
|
120
115
|
})
|
|
121
116
|
.immediate();
|
|
122
117
|
}
|
|
@@ -132,9 +127,9 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
132
127
|
.run(Date.now(), input.ownerId, input.id);
|
|
133
128
|
this.db
|
|
134
129
|
.query(
|
|
135
|
-
`UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,
|
|
130
|
+
`UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,resync=1,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
|
|
136
131
|
)
|
|
137
|
-
.run(canonicalJson(input.checkpoint).json,
|
|
132
|
+
.run(canonicalJson(input.checkpoint).json, input.ownerId, input.id);
|
|
138
133
|
})
|
|
139
134
|
.immediate();
|
|
140
135
|
}
|
|
@@ -150,8 +145,10 @@ export class SqliteCatalog implements CatalogRepository {
|
|
|
150
145
|
.run(input.ownerId, input.id);
|
|
151
146
|
this.db.query('DELETE FROM syncs WHERE owner_id=? AND id=?').run(input.ownerId, input.id);
|
|
152
147
|
this.db
|
|
153
|
-
.query(
|
|
154
|
-
|
|
148
|
+
.query(
|
|
149
|
+
`UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'`,
|
|
150
|
+
)
|
|
151
|
+
.run();
|
|
155
152
|
})
|
|
156
153
|
.immediate();
|
|
157
154
|
}
|
|
@@ -9,7 +9,6 @@ export interface DeliveryLease extends Scope {
|
|
|
9
9
|
attempt: number;
|
|
10
10
|
}
|
|
11
11
|
export interface DeliveryRepository {
|
|
12
|
-
nextDue(): number | undefined;
|
|
13
12
|
claim(leaseMs: number): DeliveryLease | undefined;
|
|
14
13
|
complete(input: { lease: DeliveryLease; result: DeliveryResult; delay: number }): void;
|
|
15
14
|
status(scope: Scope): QueueStatus;
|
|
@@ -13,15 +13,6 @@ const headOfQueue = `NOT EXISTS (SELECT 1 FROM deliveries prior
|
|
|
13
13
|
|
|
14
14
|
export class SqliteDeliveries implements DeliveryRepository {
|
|
15
15
|
constructor(private readonly db: Database) {}
|
|
16
|
-
nextDue(): number | undefined {
|
|
17
|
-
return (
|
|
18
|
-
this.db
|
|
19
|
-
.query<{ due: number | null }, []>(`SELECT MIN(
|
|
20
|
-
CASE WHEN d.state='leased' THEN MAX(d.due_at,d.expires_at) ELSE d.due_at END) AS due
|
|
21
|
-
FROM deliveries d WHERE d.state!='blocked' AND ${headOfQueue}`)
|
|
22
|
-
.get()?.due ?? undefined
|
|
23
|
-
);
|
|
24
|
-
}
|
|
25
16
|
claim(leaseMs: number): DeliveryLease | undefined {
|
|
26
17
|
return this.db
|
|
27
18
|
.transaction(() => {
|
|
@@ -68,9 +59,9 @@ export class SqliteDeliveries implements DeliveryRepository {
|
|
|
68
59
|
// Capacity is global; all paused acquisitions can compete for the released budget.
|
|
69
60
|
this.db
|
|
70
61
|
.query(
|
|
71
|
-
"UPDATE syncs SET
|
|
62
|
+
"UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'",
|
|
72
63
|
)
|
|
73
|
-
.run(
|
|
64
|
+
.run();
|
|
74
65
|
} else {
|
|
75
66
|
this.db
|
|
76
67
|
.query(
|
package/src/repositories/rows.ts
CHANGED
|
@@ -23,8 +23,6 @@ export function readSync(input: { db: Database; scope: Resource }): Sync {
|
|
|
23
23
|
},
|
|
24
24
|
enabled: row.enabled === 1,
|
|
25
25
|
checkpoint: JSON.parse(String(row.checkpoint)),
|
|
26
|
-
intervalMs: Number(row.interval_ms),
|
|
27
|
-
nextDueAt: Number(row.next_due_at),
|
|
28
26
|
status: row.status as SyncStatus,
|
|
29
27
|
errorCode: row.error_code === null ? null : String(row.error_code),
|
|
30
28
|
};
|
package/src/runtime.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { syncApi } from './api';
|
|
2
2
|
import { openDatabase } from './db/client';
|
|
3
|
+
import { startCron } from './execution/cron';
|
|
3
4
|
import { type Logger, safeLogger } from './execution/diagnostics';
|
|
4
5
|
import type { ProviderGateway } from './execution/provider';
|
|
5
6
|
import { Worker } from './execution/worker';
|
|
@@ -93,13 +94,42 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
|
|
|
93
94
|
limits,
|
|
94
95
|
timeoutMs: timing.timeoutMs,
|
|
95
96
|
});
|
|
97
|
+
let starting: Promise<void> | undefined;
|
|
98
|
+
let cron: Awaited<ReturnType<typeof startCron>> | undefined;
|
|
96
99
|
let closing: Promise<void> | undefined;
|
|
97
100
|
return {
|
|
98
101
|
api: syncApi(api),
|
|
99
|
-
|
|
102
|
+
/** Requires a working OS scheduler supported by Bun.cron. Registration failures reject startup. */
|
|
103
|
+
start(): Promise<void> {
|
|
104
|
+
worker.ensureOpen();
|
|
105
|
+
if (closing) {
|
|
106
|
+
fail('closed');
|
|
107
|
+
}
|
|
108
|
+
starting ??= (async () => {
|
|
109
|
+
cron = await startCron({
|
|
110
|
+
runtime: worker,
|
|
111
|
+
directory: `${options.databasePath}.cron`,
|
|
112
|
+
onError: () => log({ code: 'runtime_failed' }),
|
|
113
|
+
});
|
|
114
|
+
worker.start();
|
|
115
|
+
})();
|
|
116
|
+
return starting;
|
|
117
|
+
},
|
|
118
|
+
runDue: () => worker.runDue(),
|
|
100
119
|
tick: () => worker.tick(),
|
|
101
120
|
close: () => {
|
|
102
|
-
closing ??=
|
|
121
|
+
closing ??= (async () => {
|
|
122
|
+
try {
|
|
123
|
+
await starting?.catch(() => undefined);
|
|
124
|
+
await worker.close();
|
|
125
|
+
} finally {
|
|
126
|
+
try {
|
|
127
|
+
await cron?.close();
|
|
128
|
+
} finally {
|
|
129
|
+
db.close();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
})();
|
|
103
133
|
return closing;
|
|
104
134
|
},
|
|
105
135
|
};
|
|
@@ -4,7 +4,7 @@ import { bindProvider, type ProviderGateway } from '../execution/provider';
|
|
|
4
4
|
import { assetKey, type SourceAssets } from '../models/asset';
|
|
5
5
|
import { SyncError } from '../models/error';
|
|
6
6
|
import { canonicalJson } from '../models/json';
|
|
7
|
-
import {
|
|
7
|
+
import type { Timing } from '../models/limits';
|
|
8
8
|
import type { Registry } from '../models/registry';
|
|
9
9
|
import { retryableStatus } from '../models/source-http-error';
|
|
10
10
|
import { identifier } from '../models/validation';
|
|
@@ -24,21 +24,21 @@ export class AcquisitionService {
|
|
|
24
24
|
log: Logger;
|
|
25
25
|
},
|
|
26
26
|
) {}
|
|
27
|
+
poll() {
|
|
28
|
+
this.input.repository.poll();
|
|
29
|
+
}
|
|
27
30
|
capacityReleased() {
|
|
28
31
|
this.input.repository.capacityReleased();
|
|
29
32
|
}
|
|
30
|
-
nextDue() {
|
|
31
|
-
return this.input.repository.nextDue();
|
|
32
|
-
}
|
|
33
33
|
claim() {
|
|
34
34
|
return this.input.repository.claim(this.input.timing.leaseMs);
|
|
35
35
|
}
|
|
36
36
|
async execute(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
|
|
37
|
-
const { repository
|
|
37
|
+
const { repository } = this.input;
|
|
38
38
|
const { lease } = input;
|
|
39
39
|
try {
|
|
40
40
|
if (!repository.hasCapacity()) {
|
|
41
|
-
this.finish({ lease, state: 'waiting_for_capacity'
|
|
41
|
+
this.finish({ lease, state: 'waiting_for_capacity' });
|
|
42
42
|
return;
|
|
43
43
|
}
|
|
44
44
|
await abortable({ signal: input.signal, run: () => this.consume(input) });
|
|
@@ -47,20 +47,15 @@ export class AcquisitionService {
|
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
49
|
private failed(input: { lease: AcquisitionLease; signal: AbortSignal; error: unknown }): void {
|
|
50
|
-
const {
|
|
50
|
+
const { log } = this.input;
|
|
51
51
|
const { lease, signal, error } = input;
|
|
52
52
|
const code = signal.aborted
|
|
53
53
|
? abortCode(signal)
|
|
54
54
|
: error instanceof SyncError
|
|
55
55
|
? error.code
|
|
56
56
|
: 'execution_failed';
|
|
57
|
-
const failed = !['paused', 'interrupted', 'waiting_for_capacity'].includes(code);
|
|
58
|
-
const failureCount = failed ? lease.failureCount + 1 : lease.failureCount;
|
|
59
57
|
const status = error instanceof SyncError ? error.status : undefined;
|
|
60
58
|
const pause = !signal.aborted && status !== undefined && !retryableStatus(status);
|
|
61
|
-
const delay = failed
|
|
62
|
-
? retryDelay({ attempt: failureCount, retryMs: timing.retryMs })
|
|
63
|
-
: timing.retryMs;
|
|
64
59
|
log({
|
|
65
60
|
code,
|
|
66
61
|
ownerId: lease.ownerId,
|
|
@@ -68,8 +63,7 @@ export class AcquisitionService {
|
|
|
68
63
|
fields: {
|
|
69
64
|
...(error instanceof SyncError ? error.diagnostics : {}),
|
|
70
65
|
...(status === undefined ? {} : { httpStatus: status }),
|
|
71
|
-
|
|
72
|
-
...(pause ? { paused: true } : { retryAfterMs: delay }),
|
|
66
|
+
...(pause ? { paused: true } : {}),
|
|
73
67
|
},
|
|
74
68
|
});
|
|
75
69
|
this.finish({
|
|
@@ -81,8 +75,6 @@ export class AcquisitionService {
|
|
|
81
75
|
? 'interrupted'
|
|
82
76
|
: 'retrying',
|
|
83
77
|
errorCode: code === 'waiting_for_capacity' ? undefined : code,
|
|
84
|
-
delay,
|
|
85
|
-
failureCount,
|
|
86
78
|
pause,
|
|
87
79
|
});
|
|
88
80
|
}
|
package/src/services/delivery.ts
CHANGED
|
@@ -20,9 +20,6 @@ export class DeliveryService {
|
|
|
20
20
|
claim() {
|
|
21
21
|
return this.input.repository.claim(this.input.timing.leaseMs);
|
|
22
22
|
}
|
|
23
|
-
nextDue() {
|
|
24
|
-
return this.input.repository.nextDue();
|
|
25
|
-
}
|
|
26
23
|
async execute(input: { lease: DeliveryLease; signal: AbortSignal }): Promise<boolean> {
|
|
27
24
|
const { repository, registry, timing } = this.input;
|
|
28
25
|
const { lease, signal } = input;
|
|
@@ -4,7 +4,7 @@ import type { ConnectionRef } from '../models/definition';
|
|
|
4
4
|
import { fail } from '../models/error';
|
|
5
5
|
import type { Resource, Scope } from '../models/identity';
|
|
6
6
|
import type { JsonObject } from '../models/json';
|
|
7
|
-
import {
|
|
7
|
+
import type { QueueLimits } from '../models/limits';
|
|
8
8
|
import type { Registry } from '../models/registry';
|
|
9
9
|
import { type CreateSync, summarizeSync } from '../models/sync';
|
|
10
10
|
import { identifier, validate } from '../models/validation';
|
|
@@ -67,9 +67,6 @@ export class SyncManagement {
|
|
|
67
67
|
this.guard(input);
|
|
68
68
|
const { definition } = this.input.registry.definition(input.definition);
|
|
69
69
|
const config = validate({ value: input.config, schema: definition.configSchema }) as JsonObject;
|
|
70
|
-
if (input.intervalMs !== undefined) {
|
|
71
|
-
positive(input.intervalMs);
|
|
72
|
-
}
|
|
73
70
|
if (definition.provider && (input.connection || input.enabled !== false)) {
|
|
74
71
|
await bindProvider({
|
|
75
72
|
actorId: input.actorId,
|
|
File without changes
|
|
File without changes
|