@context-use/open-sync 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@context-use/open-sync",
3
- "version": "0.3.1",
3
+ "version": "0.5.0",
4
4
  "type": "module",
5
5
  "exports": {
6
6
  ".": "./src/open-sync.ts",
@@ -9,6 +9,7 @@
9
9
  "./delivery": "./src/models/delivery.ts",
10
10
  "./json": "./src/models/json.ts",
11
11
  "./http": "./src/http/index.ts",
12
+ "./cron": "./src/cron.ts",
12
13
  "./engine": "./src/runtime.ts",
13
14
  "./build": "./src/build.ts",
14
15
  "./assets": "./src/models/asset.ts"
@@ -25,10 +26,13 @@
25
26
  "dependencies": {
26
27
  "@cfworker/json-schema": "4.1.1",
27
28
  "@oomol-lab/open-connector": "1.7.0",
29
+ "@types/proper-lockfile": "4.1.4",
28
30
  "elysia": "^1.4.30",
29
- "minisearch": "^7.2.0"
31
+ "minisearch": "^7.2.0",
32
+ "proper-lockfile": "4.1.2"
30
33
  },
31
34
  "devDependencies": {
35
+ "open-sync-previous": "https://registry.npmjs.org/@context-use/open-sync/-/open-sync-0.3.1.tgz",
32
36
  "typescript": "6.0.3"
33
37
  },
34
38
  "engines": {
package/src/cron.ts ADDED
@@ -0,0 +1,2 @@
1
+ // biome-ignore lint/performance/noBarrelFile: Public entrypoint exposes only the host startup hook.
2
+ export { runOpenSyncCron } from './execution/cron-invocation';
package/src/db/client.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  import { Database } from 'bun:sqlite';
2
- import schema from './schema.sql' with { type: 'text' };
2
+ import { runMigrations } from './migrate';
3
3
 
4
4
  export function openDatabase(path: string): Database {
5
5
  const db = new Database(path, { create: true, strict: true });
6
6
  try {
7
- db.exec('PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;');
8
- db.transaction(() => db.exec(schema)).immediate();
7
+ db.exec('PRAGMA foreign_keys=ON; PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;');
8
+ runMigrations(db);
9
9
  return db;
10
10
  } catch (error) {
11
11
  db.close();
@@ -0,0 +1,48 @@
1
+ import type { Database } from 'bun:sqlite';
2
+ import initialSchema from './migrations/0000-initial.sql' with { type: 'text' };
3
+ import centralizePolling from './migrations/0001-centralize-polling.sql' with { type: 'text' };
4
+
5
+ // Static imports keep migration history available in both installed packages and compiled hosts.
6
+ const migrations = [
7
+ { name: '0000-initial.sql', sql: initialSchema },
8
+ { name: '0001-centralize-polling.sql', sql: centralizePolling },
9
+ ] as const;
10
+
11
+ export function runMigrations(db: Database): void {
12
+ db.transaction(() => {
13
+ db.query(`CREATE TABLE IF NOT EXISTS __migrations (
14
+ name TEXT PRIMARY KEY, applied_at TEXT NOT NULL
15
+ )`).run();
16
+ const applied = db
17
+ .query<{ name: string }, []>('SELECT name FROM __migrations ORDER BY name')
18
+ .all();
19
+ for (const [index, { name }] of applied.entries()) {
20
+ if (name !== migrations[index]?.name) {
21
+ throw new Error('Unsupported sync database migration history');
22
+ }
23
+ }
24
+ for (const migration of migrations.slice(applied.length)) {
25
+ executeSql({ db, sql: migration.sql });
26
+ db.query('INSERT INTO __migrations (name,applied_at) VALUES (?,?)').run(
27
+ migration.name,
28
+ new Date().toISOString(),
29
+ );
30
+ }
31
+ }).immediate();
32
+ }
33
+
34
+ export function executeSql({ db, sql }: { db: Database; sql: string }): void {
35
+ // Bun 1.4's exec() can hide failures in non-final statements. Let SQLite parse each
36
+ // statement instead, preserving trigger bodies, comments and quoted semicolons.
37
+ // A final no-op also lets prepare() consume files ending in comments or whitespace.
38
+ let remaining = `${sql}\n;SELECT 1;`;
39
+ while (remaining.length) {
40
+ using statement = db.prepare(remaining);
41
+ const parsed = statement.toString();
42
+ if (!parsed || !remaining.startsWith(parsed)) {
43
+ throw new Error('Migration SQL must contain complete statements without parameters');
44
+ }
45
+ statement.run();
46
+ remaining = remaining.slice(parsed.length);
47
+ }
48
+ }
@@ -0,0 +1,3 @@
1
+ ALTER TABLE syncs DROP COLUMN interval_ms;
2
+ ALTER TABLE syncs DROP COLUMN next_due_at;
3
+ ALTER TABLE syncs DROP COLUMN failure_count;
@@ -0,0 +1,2 @@
1
+ declare const sql: string;
2
+ export default sql;
@@ -0,0 +1,47 @@
1
+ import { request } from 'node:http';
2
+
3
+ const STARTUP_TIMEOUT_MS = 30_000;
4
+ const RETRY_INTERVAL_MS = 100;
5
+ const HTTP_OK = 200;
6
+
7
+ /** Wait for the existing worker to finish, retrying only while its socket is unavailable. */
8
+ export default async function requestCron(socketPath: string): Promise<void> {
9
+ const deadline = Date.now() + STARTUP_TIMEOUT_MS;
10
+ for (;;) {
11
+ try {
12
+ await run(socketPath);
13
+ return;
14
+ } catch (error) {
15
+ const unavailable =
16
+ error instanceof Error &&
17
+ 'code' in error &&
18
+ (error.code === 'ECONNREFUSED' || error.code === 'ENOENT');
19
+ if (!unavailable || Date.now() >= deadline) {
20
+ throw error;
21
+ }
22
+ await Bun.sleep(RETRY_INTERVAL_MS);
23
+ }
24
+ }
25
+ }
26
+
27
+ function run(socketPath: string): Promise<void> {
28
+ // biome-ignore lint/complexity/useMaxParams: Native Promise executor signature.
29
+ return new Promise((resolve, reject) => {
30
+ const client = request(
31
+ { socketPath, path: '/run', method: 'POST', agent: false },
32
+ (response) => {
33
+ response.on('error', reject);
34
+ response.on('end', () => {
35
+ if (response.statusCode === HTTP_OK) {
36
+ resolve();
37
+ } else {
38
+ reject(new Error(`Open Sync cron failed: ${response.statusCode}`));
39
+ }
40
+ });
41
+ response.resume();
42
+ },
43
+ );
44
+ client.on('error', reject);
45
+ client.end();
46
+ });
47
+ }
@@ -0,0 +1,23 @@
1
+ import requestCron from './cron-client';
2
+
3
+ const TITLE_PREFIX = 'open-sync-';
4
+
5
+ /** Handle Bun's compiled cron invocation before loading application configuration or services. */
6
+ export async function runOpenSyncCron(): Promise<boolean> {
7
+ const argument = process.argv.find((value) => value.startsWith(`--cron-title=${TITLE_PREFIX}`));
8
+ if (!argument) {
9
+ return false;
10
+ }
11
+ const encoded = argument.slice(`--cron-title=${TITLE_PREFIX}`.length);
12
+ const socket = Buffer.from(encoded, 'base64url').toString();
13
+ if (!socket.startsWith('/') || Buffer.from(socket).toString('base64url') !== encoded) {
14
+ throw new Error('Invalid Open Sync cron invocation.');
15
+ }
16
+ await requestCron(socket);
17
+ return true;
18
+ }
19
+
20
+ export function cronTitle(socket: string): string {
21
+ // The command must find its worker even when Unix cron supplies a different cwd/environment.
22
+ return `${TITLE_PREFIX}${Buffer.from(socket).toString('base64url')}`;
23
+ }
@@ -0,0 +1,137 @@
1
+ import { chmod, lstat, mkdir, unlink, writeFile } from 'node:fs/promises';
2
+ import { request } from 'node:http';
3
+ import { join, resolve } from 'node:path';
4
+ import { cronTitle } from './cron-invocation';
5
+ import { updateSchedule } from './schedule';
6
+
7
+ const PRIVATE_DIRECTORY_MODE = 0o700;
8
+ const PRIVATE_SOCKET_MODE = 0o600;
9
+ const HTTP_OK = 200;
10
+ const HTTP_NOT_FOUND = 404;
11
+ const HTTP_UNAVAILABLE = 503;
12
+ const PROBE_TIMEOUT_MS = 1_000;
13
+
14
+ /** One half-hourly check for every sync owned by this runtime. */
15
+ export async function startCron(input: {
16
+ runtime: { runDue(): Promise<void> };
17
+ /** A private, stable directory owned by this runtime. */
18
+ directory: string;
19
+ /** Report a failed run; the recurring job remains registered for the next invocation. */
20
+ onError(error: unknown): void;
21
+ }) {
22
+ const directory = resolve(input.directory);
23
+ await mkdir(directory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
24
+ await chmod(directory, PRIVATE_DIRECTORY_MODE);
25
+ const socket = join(directory, 'worker.sock');
26
+ await removeStaleSocket(socket);
27
+ const title = cronTitle(socket);
28
+ let entrypoint = Bun.main;
29
+ if (!Bun.isStandaloneExecutable) {
30
+ // Bun consumes cron arguments in source mode and invokes default.scheduled().
31
+ // Only the absolute import and socket path are generated; the client remains ordinary code.
32
+ entrypoint = join(directory, 'scheduled.ts');
33
+ await writeFile(
34
+ entrypoint,
35
+ `import requestCron from ${JSON.stringify(import.meta.resolve('./cron-client.ts'))};\n` +
36
+ `export default { scheduled: () => requestCron(${JSON.stringify(socket)}) };\n`,
37
+ { mode: PRIVATE_SOCKET_MODE },
38
+ );
39
+ }
40
+ let closed = false;
41
+ const server = Bun.serve({
42
+ unix: socket,
43
+ async fetch(request) {
44
+ this.timeout(request, 0);
45
+ const path = new URL(request.url).pathname;
46
+ if (path === '/health') {
47
+ return new Response(null, { status: HTTP_OK });
48
+ }
49
+ if (path !== '/run' || request.method !== 'POST') {
50
+ return new Response(null, { status: HTTP_NOT_FOUND });
51
+ }
52
+ if (closed) {
53
+ return new Response(null, { status: HTTP_UNAVAILABLE });
54
+ }
55
+ try {
56
+ await input.runtime.runDue();
57
+ return new Response(null, { status: HTTP_OK });
58
+ } catch (error) {
59
+ input.onError(error);
60
+ return new Response(null, { status: HTTP_UNAVAILABLE });
61
+ }
62
+ },
63
+ });
64
+ let closing: Promise<void> | undefined;
65
+ const close = () => {
66
+ closing ??= (async () => {
67
+ closed = true;
68
+ try {
69
+ await updateSchedule(() => Bun.cron.remove(title));
70
+ } finally {
71
+ await server.stop(true);
72
+ }
73
+ })();
74
+ return closing;
75
+ };
76
+ try {
77
+ await chmod(socket, PRIVATE_SOCKET_MODE);
78
+ await updateSchedule(() => Bun.cron(entrypoint, '*/30 * * * *', title));
79
+ return { close, [Symbol.asyncDispose]: close };
80
+ } catch (error) {
81
+ await server.stop(true);
82
+ throw error;
83
+ }
84
+ }
85
+
86
+ async function removeStaleSocket(socket: string): Promise<void> {
87
+ const existing = await lstat(socket).catch((error: NodeJS.ErrnoException) => {
88
+ if (error.code !== 'ENOENT') {
89
+ throw error;
90
+ }
91
+ });
92
+ if (!existing) {
93
+ return;
94
+ }
95
+ if (!existing.isSocket()) {
96
+ throw new Error('The cron worker socket path is occupied by another file.');
97
+ }
98
+ try {
99
+ await probeWorker(socket);
100
+ } catch (error) {
101
+ if (unavailableSocket(error)) {
102
+ await unlink(socket);
103
+ return;
104
+ }
105
+ throw error;
106
+ }
107
+ throw new Error('An Open Sync cron worker already owns this directory.');
108
+ }
109
+
110
+ function unavailableSocket(error: unknown): boolean {
111
+ return (
112
+ error instanceof Error &&
113
+ 'code' in error &&
114
+ ['ECONNREFUSED', 'ENOENT'].includes(String(error.code))
115
+ );
116
+ }
117
+
118
+ function probeWorker(socket: string): Promise<number> {
119
+ // biome-ignore lint/complexity/useMaxParams: Native Promise executor signature.
120
+ return new Promise((resolve, reject) => {
121
+ const client = request(
122
+ {
123
+ socketPath: socket,
124
+ path: '/health',
125
+ agent: false,
126
+ signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
127
+ },
128
+ (response) => {
129
+ response.on('error', reject);
130
+ response.on('end', () => resolve(response.statusCode!));
131
+ response.resume();
132
+ },
133
+ );
134
+ client.on('error', reject);
135
+ client.end();
136
+ });
137
+ }
@@ -0,0 +1,57 @@
1
+ import { tmpdir } from 'node:os';
2
+ import { join } from 'node:path';
3
+ import { lock } from 'proper-lockfile';
4
+
5
+ const LOCK_STALE_MS = 30_000;
6
+ const LOCK_RETRIES = 40;
7
+ const LOCK_RETRY_MS = 1_000;
8
+ let updating = Promise.resolve();
9
+
10
+ /** Bun replaces the user's entire crontab; serialize changes across Open Sync hosts. */
11
+ export function updateSchedule(operation: () => Promise<void>): Promise<void> {
12
+ const pending = updating.then(async () => {
13
+ const release = await lock(join(tmpdir(), `open-sync-crontab-${process.getuid!()}`), {
14
+ realpath: false,
15
+ stale: LOCK_STALE_MS,
16
+ retries: { retries: LOCK_RETRIES, minTimeout: LOCK_RETRY_MS, maxTimeout: LOCK_RETRY_MS },
17
+ });
18
+ try {
19
+ if (process.platform === 'linux') {
20
+ await verifyCrontabReadable();
21
+ }
22
+ await operation();
23
+ } finally {
24
+ await release();
25
+ }
26
+ });
27
+ updating = pending.catch(() => undefined);
28
+ return pending;
29
+ }
30
+
31
+ // Bun 1.4 treats every `crontab -l` exit 1 as an empty table. Reject permission/service errors
32
+ // before asking it to replace the table; only the documented missing-table responses are safe.
33
+ async function verifyCrontabReadable(): Promise<void> {
34
+ const child = Bun.spawn(['crontab', '-l'], {
35
+ env: { ...process.env, LC_ALL: 'C' },
36
+ stdout: 'pipe',
37
+ stderr: 'pipe',
38
+ timeout: 10_000,
39
+ });
40
+ const [code, output, error] = await Promise.all([
41
+ child.exited,
42
+ new Response(child.stdout).text(),
43
+ new Response(child.stderr).text(),
44
+ ]);
45
+ if (code === 0) {
46
+ return;
47
+ }
48
+ if (
49
+ code === 1 &&
50
+ !output &&
51
+ (/^no crontab for [^\r\n]+$/i.test(error.trim()) ||
52
+ /^crontab: can't open '[^/'\r\n]+': No such file or directory$/.test(error.trim()))
53
+ ) {
54
+ return;
55
+ }
56
+ throw new Error(`Cannot read crontab: ${error.trim() || `exit ${code}`}`);
57
+ }
@@ -13,7 +13,6 @@ export interface WorkerControl {
13
13
  }
14
14
  export class Worker implements WorkerControl {
15
15
  readonly #lifetime = new AbortController();
16
- #timer?: ReturnType<typeof setTimeout>;
17
16
  #started = false;
18
17
  readonly #acquisitions = new Map<
19
18
  Promise<void>,
@@ -26,8 +25,11 @@ export class Worker implements WorkerControl {
26
25
  #cleaning?: Promise<void>;
27
26
  #cleanupRequested = false;
28
27
  #capacityReleased = false;
29
- #cleanupTimer?: ReturnType<typeof setTimeout>;
30
28
  #closing?: Promise<void>;
29
+ #draining?: Promise<void>;
30
+ #polling = false;
31
+ #drainRequested = false;
32
+ #resume?: () => void;
31
33
  constructor(
32
34
  private readonly input: {
33
35
  acquisition: AcquisitionService;
@@ -48,30 +50,13 @@ export class Worker implements WorkerControl {
48
50
  return;
49
51
  }
50
52
  this.#started = true;
51
- this.completed();
53
+ this.wake();
52
54
  }
53
55
  wake(): void {
54
- this.schedule(0);
55
- }
56
- private schedule(delay: number): void {
57
- if (!this.#started || this.#lifetime.signal.aborted) {
56
+ if ((!this.#started && !this.#draining) || this.#lifetime.signal.aborted) {
58
57
  return;
59
58
  }
60
- clearTimeout(this.#timer);
61
- // Yield to the event loop even when another step is immediately runnable.
62
- const maxTimerDelay = 2_147_483_647;
63
- this.#timer = setTimeout(
64
- () => {
65
- this.#timer = undefined;
66
- try {
67
- this.dispatch();
68
- } catch {
69
- this.input.log({ code: 'runtime_failed' });
70
- this.schedule(this.input.timing.retryMs);
71
- }
72
- },
73
- Math.min(delay, maxTimerDelay),
74
- );
59
+ void this.requestDrain().catch(() => this.input.log({ code: 'runtime_failed' }));
75
60
  }
76
61
  /** A deterministic dispatch round for hosts/tests that do not start the background worker. */
77
62
  tick(): Promise<void> {
@@ -81,6 +66,49 @@ export class Worker implements WorkerControl {
81
66
  this.cleanup(),
82
67
  );
83
68
  }
69
+ /** Poll enabled syncs and drain runnable work, sharing a round across concurrent host triggers. */
70
+ runDue(): Promise<void> {
71
+ this.ensureOpen();
72
+ if (!this.#polling) {
73
+ this.input.acquisition.poll();
74
+ this.#polling = true;
75
+ }
76
+ return this.requestDrain();
77
+ }
78
+ private requestDrain(): Promise<void> {
79
+ this.#drainRequested = true;
80
+ this.#resume?.();
81
+ this.#draining ??= Promise.resolve().then(() => this.drain());
82
+ return this.#draining;
83
+ }
84
+ private async drain(): Promise<void> {
85
+ try {
86
+ while (!this.#lifetime.signal.aborted) {
87
+ this.#drainRequested = false;
88
+ const changed = Promise.withResolvers<void>();
89
+ this.#resume = changed.resolve;
90
+ this.dispatch();
91
+ const active = this.#acquisitions.size + this.#deliveries.size;
92
+ if (active) {
93
+ await changed.promise;
94
+ }
95
+ await this.cleanup();
96
+ if (this.#lifetime.signal.aborted) {
97
+ return;
98
+ }
99
+ if (active || this.#drainRequested) {
100
+ // Continue pages immediately, while allowing cancellation and host requests to run.
101
+ await new Promise<void>((resolve) => setImmediate(resolve));
102
+ continue;
103
+ }
104
+ return;
105
+ }
106
+ } finally {
107
+ this.#resume = undefined;
108
+ this.#draining = undefined;
109
+ this.#polling = false;
110
+ }
111
+ }
84
112
  private dispatch(): void {
85
113
  this.ensureOpen();
86
114
  const { acquisition, delivery, timing } = this.input;
@@ -95,7 +123,7 @@ export class Worker implements WorkerControl {
95
123
  .catch(() => this.input.log({ code: 'acquisition_failed' }))
96
124
  .finally(() => {
97
125
  this.#acquisitions.delete(task);
98
- this.completed();
126
+ this.wake();
99
127
  });
100
128
  this.#acquisitions.set(task, { ownerId: lease.ownerId, syncId: lease.sync.id, abort });
101
129
  }
@@ -113,34 +141,10 @@ export class Worker implements WorkerControl {
113
141
  .catch(() => this.input.log({ code: 'delivery_failed' }))
114
142
  .finally(() => {
115
143
  this.#deliveries.delete(task);
116
- this.completed();
144
+ this.wake();
117
145
  });
118
146
  this.#deliveries.set(task, { ownerId: lease.ownerId, syncId: lease.delivery.syncId, abort });
119
147
  }
120
- const due = [
121
- this.#acquisitions.size < timing.sourceConcurrency ? acquisition.nextDue() : undefined,
122
- this.#deliveries.size < timing.deliveryConcurrency ? delivery.nextDue() : undefined,
123
- ].filter((value): value is number => value !== undefined);
124
- if (due.length) {
125
- this.schedule(Math.max(0, Math.min(...due) - Date.now()));
126
- }
127
- }
128
- private completed(): void {
129
- this.wake();
130
- if (this.#started) {
131
- void this.cleanup().then(
132
- () => this.wake(),
133
- () => {
134
- this.input.log({ code: 'cleanup_failed' });
135
- if (!this.#lifetime.signal.aborted && !this.#cleanupTimer) {
136
- this.#cleanupTimer = setTimeout(() => {
137
- this.#cleanupTimer = undefined;
138
- this.completed();
139
- }, this.input.timing.retryMs);
140
- }
141
- },
142
- );
143
- }
144
148
  }
145
149
  private cleanup(): Promise<void> {
146
150
  this.#cleanupRequested = true;
@@ -157,8 +161,6 @@ export class Worker implements WorkerControl {
157
161
  this.#capacityReleased = false;
158
162
  this.input.acquisition.capacityReleased();
159
163
  }
160
- clearTimeout(this.#cleanupTimer);
161
- this.#cleanupTimer = undefined;
162
164
  } finally {
163
165
  this.#cleaning = undefined;
164
166
  }
@@ -172,7 +174,7 @@ export class Worker implements WorkerControl {
172
174
  }
173
175
  async cancelAcquisition(input: Resource): Promise<void> {
174
176
  await this.cancelTasks({ scope: input, tasks: this.#acquisitions });
175
- this.completed();
177
+ this.wake();
176
178
  }
177
179
  async cancelSync(input: Resource): Promise<void> {
178
180
  await Promise.all([
@@ -180,7 +182,7 @@ export class Worker implements WorkerControl {
180
182
  this.cancelTasks({ scope: input, tasks: this.#deliveries }),
181
183
  ]);
182
184
  this.#capacityReleased = true;
183
- this.completed();
185
+ this.wake();
184
186
  await this.cleanup().catch(() => this.input.log({ code: 'cleanup_failed' }));
185
187
  }
186
188
  private async cancelTasks({
@@ -204,10 +206,10 @@ export class Worker implements WorkerControl {
204
206
  return this.#closing;
205
207
  }
206
208
  private async stop(): Promise<void> {
207
- clearTimeout(this.#timer);
208
- clearTimeout(this.#cleanupTimer);
209
209
  this.#lifetime.abort('interrupted');
210
+ this.#resume?.();
210
211
  await Promise.allSettled([...this.#acquisitions.keys(), ...this.#deliveries.keys()]);
212
+ await this.#draining?.catch(() => undefined);
211
213
  await this.cleanup();
212
214
  }
213
215
  }
@@ -15,7 +15,6 @@ const syncBody = t.Object({
15
15
  config,
16
16
  destination: t.Object({ type: identifier, input: config }),
17
17
  connection: t.Optional(t.Object({ id: identifier, service: identifier })),
18
- intervalMs: t.Optional(t.Integer({ minimum: 1 })),
19
18
  enabled: t.Optional(t.Boolean()),
20
19
  });
21
20
  const resourceParams = { params: t.Object({ id: identifier }) };
@@ -54,7 +54,7 @@ export interface SyncStep {
54
54
  * Include any cross-iteration update boundary as well as the current page position.
55
55
  */
56
56
  checkpoint: JsonValue;
57
- /** False continues this iteration immediately; true schedules the next poll.
57
+ /** False continues this iteration immediately; true waits for the shared cron or a manual run.
58
58
  * Completion retains the returned checkpoint; the source clears any exhausted page cursor.
59
59
  */
60
60
  complete: boolean;
@@ -11,8 +11,6 @@ export interface Sync {
11
11
  destination: { type: string; config: JsonObject };
12
12
  enabled: boolean;
13
13
  checkpoint: JsonValue;
14
- intervalMs: number;
15
- nextDueAt: number;
16
14
  status: SyncStatus;
17
15
  errorCode: string | null;
18
16
  }
@@ -21,7 +19,6 @@ export interface CreateSync extends Scope {
21
19
  connection?: ConnectionRef;
22
20
  config: JsonObject;
23
21
  destination: { type: string; input: JsonObject };
24
- intervalMs?: number;
25
22
  enabled?: boolean;
26
23
  }
27
24
 
@@ -51,8 +48,6 @@ export interface SyncSummary {
51
48
  destinationType: string;
52
49
  connection?: ConnectionRef;
53
50
  enabled: boolean;
54
- intervalMs: number;
55
- nextDueAt: number;
56
51
  status: SyncStatus;
57
52
  errorCode: string | null;
58
53
  }
@@ -63,8 +58,6 @@ export function summarizeSync(sync: Sync): SyncSummary {
63
58
  destinationType: sync.destination.type,
64
59
  connection: sync.connection,
65
60
  enabled: sync.enabled,
66
- intervalMs: sync.intervalMs,
67
- nextDueAt: sync.nextDueAt,
68
61
  status: sync.status,
69
62
  errorCode: sync.errorCode,
70
63
  };
package/src/open-sync.ts CHANGED
@@ -158,6 +158,7 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
158
158
  api,
159
159
  providers: providerApi(providers),
160
160
  start: () => runtime.start(),
161
+ runDue: () => runtime.runDue(),
161
162
  async fetch(request: Request): Promise<Response> {
162
163
  if (lifetime.signal.aborted) {
163
164
  return new Response(null, { status: 503 });
@@ -205,7 +206,12 @@ export interface OpenSyncRuntime {
205
206
  api: SyncApi;
206
207
  providers: ProviderApi;
207
208
  fetch(request: Request): Promise<Response>;
208
- start(): void;
209
+ /**
210
+ * Register one half-hourly OS job and start processing work.
211
+ * Compiled hosts must call runOpenSyncCron() from @context-use/open-sync/cron before initialization.
212
+ */
213
+ start(): Promise<void>;
214
+ runDue(): Promise<void>;
209
215
  close(): Promise<void>;
210
216
  }
211
217
  export type { ProviderApi, SyncApi } from './api';
@@ -6,11 +6,10 @@ export interface AcquisitionLease extends Scope {
6
6
  sync: Sync;
7
7
  generation: number;
8
8
  force: boolean;
9
- failureCount: number;
10
9
  }
11
10
  export interface AcquisitionRepository {
11
+ poll(): void;
12
12
  capacityReleased(): void;
13
- nextDue(): number | undefined;
14
13
  claim(leaseMs: number): AcquisitionLease | undefined;
15
14
  hasCapacity(): boolean;
16
15
  commit(input: { lease: AcquisitionLease; page: SyncStep; definition: SyncDefinition }): void;
@@ -18,8 +17,6 @@ export interface AcquisitionRepository {
18
17
  lease: AcquisitionLease;
19
18
  state: Exclude<SyncStatus, 'running' | 'disabled'>;
20
19
  errorCode?: string;
21
- delay: number;
22
- failureCount?: number;
23
20
  pause?: boolean;
24
21
  }): void;
25
22
  }
@@ -30,8 +30,8 @@ export function claimAcquisition({
30
30
  WHERE completed_at IS NULL AND EXISTS (SELECT 1 FROM syncs
31
31
  WHERE syncs.owner_id=sync_polls.owner_id AND syncs.id=sync_polls.sync_id
32
32
  AND enabled=1 AND expires_at<=?)`).run(now);
33
- db.query(`UPDATE syncs SET next_due_at=?,status='interrupted',error_code='lease_expired',expires_at=NULL
34
- WHERE enabled=1 AND expires_at<=?`).run(now, now);
33
+ db.query(`UPDATE syncs SET status='ready',error_code='lease_expired',expires_at=NULL
34
+ WHERE enabled=1 AND expires_at<=?`).run(now);
35
35
  const row = db
36
36
  .query<
37
37
  {
@@ -39,13 +39,12 @@ export function claimAcquisition({
39
39
  owner_id: string;
40
40
  generation: number;
41
41
  resync: number;
42
- failure_count: number;
43
42
  },
44
- [number]
45
- >(`SELECT id,owner_id,generation,resync,failure_count FROM syncs
46
- WHERE enabled=1 AND expires_at IS NULL AND next_due_at<=?
47
- ORDER BY next_due_at,generation,id LIMIT 1`)
48
- .get(now);
43
+ []
44
+ >(`SELECT id,owner_id,generation,resync FROM syncs
45
+ WHERE enabled=1 AND status='ready' AND expires_at IS NULL
46
+ ORDER BY generation,id LIMIT 1`)
47
+ .get();
49
48
  if (!row) {
50
49
  return;
51
50
  }
@@ -69,7 +68,6 @@ export function claimAcquisition({
69
68
  sync: readSync({ db, scope: { ...scope, id: row.id } }),
70
69
  generation: row.generation + 1,
71
70
  force: row.resync === 1,
72
- failureCount: row.failure_count,
73
71
  };
74
72
  })
75
73
  .immediate();
@@ -88,15 +86,13 @@ export function finishAcquisition(
88
86
  lease.ownerId,
89
87
  lease.sync.id,
90
88
  );
91
- db.query(`UPDATE syncs SET status=?,error_code=?,next_due_at=?,expires_at=NULL,
89
+ db.query(`UPDATE syncs SET status=?,error_code=?,expires_at=NULL,
92
90
  enabled=CASE WHEN ? THEN 0 ELSE enabled END,
93
- failure_count=COALESCE(?,failure_count),resync=CASE WHEN ? THEN 0 ELSE resync END
91
+ resync=CASE WHEN ? THEN 0 ELSE resync END
94
92
  WHERE owner_id=? AND id=?`).run(
95
93
  input.pause ? 'disabled' : input.state,
96
94
  input.errorCode ?? null,
97
- now + input.delay,
98
95
  Number(input.pause ?? false),
99
- input.failureCount ?? null,
100
96
  Number(input.state === 'succeeded'),
101
97
  lease.ownerId,
102
98
  lease.sync.id,
@@ -15,18 +15,17 @@ import { writeRecord } from './records';
15
15
 
16
16
  export class SqliteAcquisition implements AcquisitionRepository {
17
17
  constructor(private readonly input: { db: Database; limits: QueueLimits }) {}
18
- capacityReleased(): void {
18
+ poll(): void {
19
19
  this.input.db
20
- .query("UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'")
21
- .run(Date.now());
20
+ .query(
21
+ "UPDATE syncs SET status='ready' WHERE enabled=1 AND status IN ('succeeded','retrying','interrupted','waiting_for_capacity')",
22
+ )
23
+ .run();
22
24
  }
23
- nextDue(): number | undefined {
24
- return (
25
- this.input.db
26
- .query<{ due: number | null }, []>(`
27
- SELECT MIN(COALESCE(expires_at,next_due_at)) AS due FROM syncs WHERE enabled=1`)
28
- .get()?.due ?? undefined
29
- );
25
+ capacityReleased(): void {
26
+ this.input.db
27
+ .query("UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'")
28
+ .run();
30
29
  }
31
30
  claim(leaseMs: number) {
32
31
  return claimAcquisition({ ...this.input, leaseMs });
@@ -60,8 +59,6 @@ export class SqliteAcquisition implements AcquisitionRepository {
60
59
  db,
61
60
  lease: input.lease,
62
61
  state: page.complete ? 'succeeded' : 'ready',
63
- delay: page.complete ? sync.intervalMs : 0,
64
- failureCount: page.complete ? 0 : undefined,
65
62
  });
66
63
  }).immediate();
67
64
  }
@@ -103,9 +103,9 @@ export class SqliteAssets implements AssetRepository {
103
103
  if (deleted) {
104
104
  this.input.db
105
105
  .query(
106
- "UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity' AND NOT (owner_id=? AND id=?)",
106
+ "UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity' AND NOT (owner_id=? AND id=?)",
107
107
  )
108
- .run(Date.now(), deleted.owner_id, deleted.sync_id);
108
+ .run(deleted.owner_id, deleted.sync_id);
109
109
  }
110
110
  }
111
111
  garbage(): string[] {
@@ -7,8 +7,6 @@ import type { CreateSync, SyncPoll } from '../../models/sync';
7
7
  import { readSync } from '../rows';
8
8
  import type { CatalogRepository } from './contract';
9
9
 
10
- const defaultIntervalMs = 60_000;
11
-
12
10
  export class SqliteCatalog implements CatalogRepository {
13
11
  constructor(private readonly db: Database) {}
14
12
  createSync(
@@ -19,8 +17,8 @@ export class SqliteCatalog implements CatalogRepository {
19
17
  ) {
20
18
  const id = `sync_${crypto.randomUUID()}`;
21
19
  this.db
22
- .query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,interval_ms,next_due_at,status)
23
- VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`)
20
+ .query(`INSERT INTO syncs (owner_id,id,definition_id,connection,config,destination_type,destination_config,enabled,checkpoint,status)
21
+ VALUES (?,?,?,?,?,?,?,?,?,?)`)
24
22
  .run(
25
23
  input.ownerId,
26
24
  id,
@@ -31,8 +29,6 @@ export class SqliteCatalog implements CatalogRepository {
31
29
  canonicalJson(input.destination.config).json,
32
30
  Number(input.enabled ?? true),
33
31
  canonicalJson(input.initialCheckpoint).json,
34
- input.intervalMs ?? defaultIntervalMs,
35
- Date.now(),
36
32
  input.enabled === false ? 'disabled' : 'ready',
37
33
  );
38
34
  return this.sync({ ...input, id });
@@ -69,8 +65,8 @@ export class SqliteCatalog implements CatalogRepository {
69
65
  }
70
66
  this.db
71
67
  .query(`UPDATE syncs SET connection=?,enabled=1,
72
- status='ready',error_code=NULL,next_due_at=? WHERE owner_id=? AND id=?`)
73
- .run(canonicalJson(input.connection).json, Date.now(), input.ownerId, input.id);
68
+ status='ready',error_code=NULL WHERE owner_id=? AND id=?`)
69
+ .run(canonicalJson(input.connection).json, input.ownerId, input.id);
74
70
  return this.sync(input);
75
71
  })
76
72
  .immediate();
@@ -88,12 +84,11 @@ export class SqliteCatalog implements CatalogRepository {
88
84
  .run(input.enabled ? 'ready' : 'disabled', input.ownerId, input.id);
89
85
  this.db
90
86
  .query(
91
- `UPDATE syncs SET enabled=?,status=?,error_code=NULL,next_due_at=?,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
87
+ `UPDATE syncs SET enabled=?,status=?,error_code=NULL,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
92
88
  )
93
89
  .run(
94
90
  Number(input.enabled),
95
91
  input.enabled ? 'ready' : 'disabled',
96
- Date.now(),
97
92
  input.ownerId,
98
93
  input.id,
99
94
  );
@@ -115,8 +110,8 @@ export class SqliteCatalog implements CatalogRepository {
115
110
  fail('busy');
116
111
  }
117
112
  this.db
118
- .query(`UPDATE syncs SET next_due_at=? WHERE owner_id=? AND id=?`)
119
- .run(Date.now(), input.ownerId, input.id);
113
+ .query(`UPDATE syncs SET status='ready' WHERE owner_id=? AND id=?`)
114
+ .run(input.ownerId, input.id);
120
115
  })
121
116
  .immediate();
122
117
  }
@@ -132,9 +127,9 @@ export class SqliteCatalog implements CatalogRepository {
132
127
  .run(Date.now(), input.ownerId, input.id);
133
128
  this.db
134
129
  .query(
135
- `UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,next_due_at=?,resync=1,failure_count=0,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
130
+ `UPDATE syncs SET checkpoint=?,status='ready',error_code=NULL,resync=1,generation=generation+1,expires_at=NULL WHERE owner_id=? AND id=?`,
136
131
  )
137
- .run(canonicalJson(input.checkpoint).json, Date.now(), input.ownerId, input.id);
132
+ .run(canonicalJson(input.checkpoint).json, input.ownerId, input.id);
138
133
  })
139
134
  .immediate();
140
135
  }
@@ -150,8 +145,10 @@ export class SqliteCatalog implements CatalogRepository {
150
145
  .run(input.ownerId, input.id);
151
146
  this.db.query('DELETE FROM syncs WHERE owner_id=? AND id=?').run(input.ownerId, input.id);
152
147
  this.db
153
- .query(`UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'`)
154
- .run(Date.now());
148
+ .query(
149
+ `UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'`,
150
+ )
151
+ .run();
155
152
  })
156
153
  .immediate();
157
154
  }
@@ -9,7 +9,6 @@ export interface DeliveryLease extends Scope {
9
9
  attempt: number;
10
10
  }
11
11
  export interface DeliveryRepository {
12
- nextDue(): number | undefined;
13
12
  claim(leaseMs: number): DeliveryLease | undefined;
14
13
  complete(input: { lease: DeliveryLease; result: DeliveryResult; delay: number }): void;
15
14
  status(scope: Scope): QueueStatus;
@@ -13,15 +13,6 @@ const headOfQueue = `NOT EXISTS (SELECT 1 FROM deliveries prior
13
13
 
14
14
  export class SqliteDeliveries implements DeliveryRepository {
15
15
  constructor(private readonly db: Database) {}
16
- nextDue(): number | undefined {
17
- return (
18
- this.db
19
- .query<{ due: number | null }, []>(`SELECT MIN(
20
- CASE WHEN d.state='leased' THEN MAX(d.due_at,d.expires_at) ELSE d.due_at END) AS due
21
- FROM deliveries d WHERE d.state!='blocked' AND ${headOfQueue}`)
22
- .get()?.due ?? undefined
23
- );
24
- }
25
16
  claim(leaseMs: number): DeliveryLease | undefined {
26
17
  return this.db
27
18
  .transaction(() => {
@@ -68,9 +59,9 @@ export class SqliteDeliveries implements DeliveryRepository {
68
59
  // Capacity is global; all paused acquisitions can compete for the released budget.
69
60
  this.db
70
61
  .query(
71
- "UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'",
62
+ "UPDATE syncs SET status='ready' WHERE enabled=1 AND status='waiting_for_capacity'",
72
63
  )
73
- .run(Date.now());
64
+ .run();
74
65
  } else {
75
66
  this.db
76
67
  .query(
@@ -23,8 +23,6 @@ export function readSync(input: { db: Database; scope: Resource }): Sync {
23
23
  },
24
24
  enabled: row.enabled === 1,
25
25
  checkpoint: JSON.parse(String(row.checkpoint)),
26
- intervalMs: Number(row.interval_ms),
27
- nextDueAt: Number(row.next_due_at),
28
26
  status: row.status as SyncStatus,
29
27
  errorCode: row.error_code === null ? null : String(row.error_code),
30
28
  };
package/src/runtime.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { syncApi } from './api';
2
2
  import { openDatabase } from './db/client';
3
+ import { startCron } from './execution/cron';
3
4
  import { type Logger, safeLogger } from './execution/diagnostics';
4
5
  import type { ProviderGateway } from './execution/provider';
5
6
  import { Worker } from './execution/worker';
@@ -93,13 +94,42 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
93
94
  limits,
94
95
  timeoutMs: timing.timeoutMs,
95
96
  });
97
+ let starting: Promise<void> | undefined;
98
+ let cron: Awaited<ReturnType<typeof startCron>> | undefined;
96
99
  let closing: Promise<void> | undefined;
97
100
  return {
98
101
  api: syncApi(api),
99
- start: () => worker.start(),
102
+ /** Requires a working OS scheduler supported by Bun.cron. Registration failures reject startup. */
103
+ start(): Promise<void> {
104
+ worker.ensureOpen();
105
+ if (closing) {
106
+ fail('closed');
107
+ }
108
+ starting ??= (async () => {
109
+ cron = await startCron({
110
+ runtime: worker,
111
+ directory: `${options.databasePath}.cron`,
112
+ onError: () => log({ code: 'runtime_failed' }),
113
+ });
114
+ worker.start();
115
+ })();
116
+ return starting;
117
+ },
118
+ runDue: () => worker.runDue(),
100
119
  tick: () => worker.tick(),
101
120
  close: () => {
102
- closing ??= worker.close().finally(() => db.close());
121
+ closing ??= (async () => {
122
+ try {
123
+ await starting?.catch(() => undefined);
124
+ await worker.close();
125
+ } finally {
126
+ try {
127
+ await cron?.close();
128
+ } finally {
129
+ db.close();
130
+ }
131
+ }
132
+ })();
103
133
  return closing;
104
134
  },
105
135
  };
@@ -4,7 +4,7 @@ import { bindProvider, type ProviderGateway } from '../execution/provider';
4
4
  import { assetKey, type SourceAssets } from '../models/asset';
5
5
  import { SyncError } from '../models/error';
6
6
  import { canonicalJson } from '../models/json';
7
- import { retryDelay, type Timing } from '../models/limits';
7
+ import type { Timing } from '../models/limits';
8
8
  import type { Registry } from '../models/registry';
9
9
  import { retryableStatus } from '../models/source-http-error';
10
10
  import { identifier } from '../models/validation';
@@ -24,21 +24,21 @@ export class AcquisitionService {
24
24
  log: Logger;
25
25
  },
26
26
  ) {}
27
+ poll() {
28
+ this.input.repository.poll();
29
+ }
27
30
  capacityReleased() {
28
31
  this.input.repository.capacityReleased();
29
32
  }
30
- nextDue() {
31
- return this.input.repository.nextDue();
32
- }
33
33
  claim() {
34
34
  return this.input.repository.claim(this.input.timing.leaseMs);
35
35
  }
36
36
  async execute(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
37
- const { repository, timing } = this.input;
37
+ const { repository } = this.input;
38
38
  const { lease } = input;
39
39
  try {
40
40
  if (!repository.hasCapacity()) {
41
- this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
41
+ this.finish({ lease, state: 'waiting_for_capacity' });
42
42
  return;
43
43
  }
44
44
  await abortable({ signal: input.signal, run: () => this.consume(input) });
@@ -47,20 +47,15 @@ export class AcquisitionService {
47
47
  }
48
48
  }
49
49
  private failed(input: { lease: AcquisitionLease; signal: AbortSignal; error: unknown }): void {
50
- const { timing, log } = this.input;
50
+ const { log } = this.input;
51
51
  const { lease, signal, error } = input;
52
52
  const code = signal.aborted
53
53
  ? abortCode(signal)
54
54
  : error instanceof SyncError
55
55
  ? error.code
56
56
  : 'execution_failed';
57
- const failed = !['paused', 'interrupted', 'waiting_for_capacity'].includes(code);
58
- const failureCount = failed ? lease.failureCount + 1 : lease.failureCount;
59
57
  const status = error instanceof SyncError ? error.status : undefined;
60
58
  const pause = !signal.aborted && status !== undefined && !retryableStatus(status);
61
- const delay = failed
62
- ? retryDelay({ attempt: failureCount, retryMs: timing.retryMs })
63
- : timing.retryMs;
64
59
  log({
65
60
  code,
66
61
  ownerId: lease.ownerId,
@@ -68,8 +63,7 @@ export class AcquisitionService {
68
63
  fields: {
69
64
  ...(error instanceof SyncError ? error.diagnostics : {}),
70
65
  ...(status === undefined ? {} : { httpStatus: status }),
71
- failureCount,
72
- ...(pause ? { paused: true } : { retryAfterMs: delay }),
66
+ ...(pause ? { paused: true } : {}),
73
67
  },
74
68
  });
75
69
  this.finish({
@@ -81,8 +75,6 @@ export class AcquisitionService {
81
75
  ? 'interrupted'
82
76
  : 'retrying',
83
77
  errorCode: code === 'waiting_for_capacity' ? undefined : code,
84
- delay,
85
- failureCount,
86
78
  pause,
87
79
  });
88
80
  }
@@ -20,9 +20,6 @@ export class DeliveryService {
20
20
  claim() {
21
21
  return this.input.repository.claim(this.input.timing.leaseMs);
22
22
  }
23
- nextDue() {
24
- return this.input.repository.nextDue();
25
- }
26
23
  async execute(input: { lease: DeliveryLease; signal: AbortSignal }): Promise<boolean> {
27
24
  const { repository, registry, timing } = this.input;
28
25
  const { lease, signal } = input;
@@ -4,7 +4,7 @@ import type { ConnectionRef } from '../models/definition';
4
4
  import { fail } from '../models/error';
5
5
  import type { Resource, Scope } from '../models/identity';
6
6
  import type { JsonObject } from '../models/json';
7
- import { positive, type QueueLimits } from '../models/limits';
7
+ import type { QueueLimits } from '../models/limits';
8
8
  import type { Registry } from '../models/registry';
9
9
  import { type CreateSync, summarizeSync } from '../models/sync';
10
10
  import { identifier, validate } from '../models/validation';
@@ -67,9 +67,6 @@ export class SyncManagement {
67
67
  this.guard(input);
68
68
  const { definition } = this.input.registry.definition(input.definition);
69
69
  const config = validate({ value: input.config, schema: definition.configSchema }) as JsonObject;
70
- if (input.intervalMs !== undefined) {
71
- positive(input.intervalMs);
72
- }
73
70
  if (definition.provider && (input.connection || input.enabled !== false)) {
74
71
  await bindProvider({
75
72
  actorId: input.actorId,