cairnq 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +28 -0
  2. package/dist/_protocol/migrations/postgres/0001_init.sql +3 -1
  3. package/dist/_protocol/migrations/postgres/0002_purge_index.sql +6 -0
  4. package/dist/_protocol/migrations/postgres/0003_notify.sql +38 -0
  5. package/dist/_protocol/migrations/postgres/0004_lease_index.sql +16 -0
  6. package/dist/_protocol/migrations/postgres/0005_clear_terminal_lease.sql +17 -0
  7. package/dist/_protocol/migrations/sqlite/0001_init.sql +3 -1
  8. package/dist/_protocol/migrations/sqlite/0002_purge_index.sql +6 -0
  9. package/dist/_protocol/migrations/sqlite/0004_lease_index.sql +22 -0
  10. package/dist/_protocol/migrations/sqlite/0005_clear_terminal_lease.sql +17 -0
  11. package/dist/_protocol/sql/postgres/claim.sql +18 -5
  12. package/dist/_protocol/sql/postgres/claim_one_queue.sql +35 -0
  13. package/dist/_protocol/sql/postgres/complete.sql +3 -0
  14. package/dist/_protocol/sql/postgres/fail.sql +30 -8
  15. package/dist/_protocol/sql/postgres/insert_task.sql +6 -3
  16. package/dist/_protocol/sql/postgres/list.sql +3 -1
  17. package/dist/_protocol/sql/postgres/lock_key.sql +9 -0
  18. package/dist/_protocol/sql/postgres/progress.sql +4 -3
  19. package/dist/_protocol/sql/postgres/protocol_version.sql +4 -0
  20. package/dist/_protocol/sql/postgres/purge.sql +25 -0
  21. package/dist/_protocol/sql/postgres/recover_leases.sql +49 -14
  22. package/dist/_protocol/sql/postgres/retry.sql +3 -0
  23. package/dist/_protocol/sql/postgres/stats.sql +8 -0
  24. package/dist/_protocol/sql/postgres/succeed.sql +4 -0
  25. package/dist/_protocol/sql/sqlite/claim.sql +13 -2
  26. package/dist/_protocol/sql/sqlite/claim_one_queue.sql +36 -0
  27. package/dist/_protocol/sql/sqlite/claimable_probe.sql +6 -2
  28. package/dist/_protocol/sql/sqlite/complete.sql +3 -0
  29. package/dist/_protocol/sql/sqlite/fail.sql +32 -8
  30. package/dist/_protocol/sql/sqlite/list.sql +3 -1
  31. package/dist/_protocol/sql/sqlite/lock_key.sql +5 -0
  32. package/dist/_protocol/sql/sqlite/progress.sql +6 -2
  33. package/dist/_protocol/sql/sqlite/protocol_version.sql +4 -0
  34. package/dist/_protocol/sql/sqlite/purge.sql +18 -0
  35. package/dist/_protocol/sql/sqlite/recover_leases.sql +25 -7
  36. package/dist/_protocol/sql/sqlite/retry.sql +3 -0
  37. package/dist/_protocol/sql/sqlite/stats.sql +8 -0
  38. package/dist/_protocol/sql/sqlite/succeed.sql +4 -0
  39. package/dist/client.d.ts +10 -2
  40. package/dist/client.js +12 -0
  41. package/dist/context.d.ts +17 -1
  42. package/dist/context.js +60 -6
  43. package/dist/errors.d.ts +18 -2
  44. package/dist/errors.js +49 -3
  45. package/dist/index.d.ts +3 -2
  46. package/dist/index.js +2 -1
  47. package/dist/sql.js +16 -9
  48. package/dist/store/base.d.ts +114 -9
  49. package/dist/store/base.js +376 -1
  50. package/dist/store/postgres.d.ts +62 -63
  51. package/dist/store/postgres.js +245 -222
  52. package/dist/store/sqlite.d.ts +83 -60
  53. package/dist/store/sqlite.js +370 -234
  54. package/dist/wait.d.ts +15 -2
  55. package/dist/wait.js +23 -5
  56. package/dist/worker.d.ts +53 -1
  57. package/dist/worker.js +202 -42
  58. package/package.json +9 -2
  59. package/src/client.ts +16 -2
  60. package/src/context.ts +70 -13
  61. package/src/errors.ts +59 -4
  62. package/src/index.ts +3 -1
  63. package/src/sql.ts +15 -8
  64. package/src/store/base.ts +443 -27
  65. package/src/store/postgres.ts +243 -267
  66. package/src/store/sqlite.ts +378 -263
  67. package/src/wait.ts +28 -5
  68. package/src/worker.ts +242 -42
package/src/context.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { LostLease } from "./errors.js";
1
2
  import { cancelRequested, type Task } from "./models.js";
2
3
  import type { SubmitOptions } from "./client.js";
3
4
  import type { TaskStore } from "./store/base.js";
@@ -6,6 +7,12 @@ import { pollWait } from "./wait.js";
6
7
 
7
8
  /** Handed to a task handler. Worker-side capabilities mirror the Python SDK. */
8
9
  export class TaskContext {
10
+ private readonly abort = new AbortController();
11
+ private leaseLost = false;
12
+ // Cancellation is monotonic: once the DB has told us a cancel was requested it
13
+ // can't be taken back, so canceled() can answer from this without a re-read.
14
+ private cancelSeen = false;
15
+
9
16
  constructor(
10
17
  private readonly store: TaskStore,
11
18
  private readonly task: Task,
@@ -38,28 +45,78 @@ export class TaskContext {
38
45
  return this.task.payload;
39
46
  }
40
47
 
48
+ /**
49
+ * True once this worker has lost the task's lease — it expired and another
50
+ * worker reclaimed it. Nothing this handler writes will be recorded any more
51
+ * and the task is already running elsewhere, so a long handler should check
52
+ * this (or `signal`) and bail out instead of continuing to do side effects.
53
+ */
54
+ get lostLease(): boolean {
55
+ return this.leaseLost;
56
+ }
57
+
58
+ /** Aborts when the lease is lost. Pass it to fetch / any AbortSignal-aware API. */
59
+ get signal(): AbortSignal {
60
+ return this.abort.signal;
61
+ }
62
+
63
+ /** @internal Called by the worker when an owned write reports a lost lease. */
64
+ markLeaseLost(): void {
65
+ if (this.leaseLost) return;
66
+ this.leaseLost = true;
67
+ this.abort.abort(new LostLease(this.task.id));
68
+ }
69
+
70
+ // Every owned write returns the current row, so cancellation and lease loss
71
+ // ride along on writes the handler was making anyway.
72
+ private observe(task: Task): Task {
73
+ if (cancelRequested(task)) this.cancelSeen = true;
74
+ return task;
75
+ }
76
+
77
+ private async owned(write: () => Promise<Task>): Promise<Task> {
78
+ // Short-circuit once the lease is known lost: nothing this context writes
79
+ // may be recorded any more. Locally, not just via the store's ownership
80
+ // check — after an abandoned (timed-out) attempt the same worker may
81
+ // re-claim this task under the same workerId, and a zombie handler's write
82
+ // would then pass ownership against the NEW attempt.
83
+ if (this.leaseLost) throw new LostLease(this.task.id);
84
+ try {
85
+ return this.observe(await write());
86
+ } catch (err) {
87
+ if (err instanceof LostLease) this.markLeaseLost();
88
+ throw err;
89
+ }
90
+ }
91
+
41
92
  async progress(value: number | null, message: string | null = null): Promise<Task> {
42
- return this.store.progress({
43
- taskId: this.task.id,
44
- workerId: this.workerId,
45
- progress: value,
46
- message,
47
- });
93
+ return this.owned(() =>
94
+ this.store.progress({
95
+ taskId: this.task.id,
96
+ workerId: this.workerId,
97
+ progress: value,
98
+ message,
99
+ }),
100
+ );
48
101
  }
49
102
 
50
103
  async heartbeat(): Promise<Task> {
51
- return this.store.heartbeat({
52
- taskId: this.task.id,
53
- workerId: this.workerId,
54
- leaseMs: this.leaseMs,
55
- });
104
+ return this.owned(() =>
105
+ this.store.heartbeat({
106
+ taskId: this.task.id,
107
+ workerId: this.workerId,
108
+ leaseMs: this.leaseMs,
109
+ }),
110
+ );
56
111
  }
57
112
 
58
- /** Cooperative cancel check. */
113
+ /** Cooperative cancel check. Free once a heartbeat has already seen the flag. */
59
114
  async canceled(): Promise<boolean> {
115
+ if (this.cancelSeen) return true;
60
116
  const t = await this.store.get(this.task.id);
61
117
  if (!t) return true;
62
- return cancelRequested(t) || t.status === "canceled";
118
+ if (cancelRequested(t)) this.cancelSeen = true;
119
+ return this.cancelSeen || t.status === "canceled";
63
120
  }
64
121
 
65
122
  /** Submit a child task; parent/root/correlation are wired automatically. */
package/src/errors.ts CHANGED
@@ -1,3 +1,6 @@
1
+ import { nowMs } from "./ids.js";
2
+ import { cancelRequested, isQueued, type Task } from "./models.js";
3
+
1
4
  /** The single shape of the JSON error envelope (see PROTOCOL.md). Everything that
2
5
  * records an error — a handler exception, a missing handler, lease expiry, a thrown
3
6
  * TaskError — builds it here, so the contract's fields live in one place. */
@@ -17,7 +20,15 @@ export function errorEnvelope(e: {
17
20
  };
18
21
  }
19
22
 
20
- export class CairnQError extends Error {}
23
+ export class CairnQError extends Error {
24
+ constructor(message?: string) {
25
+ super(message);
26
+ // Subclasses each set their own; without this a bare CairnQError reports
27
+ // "Error", and `err.name` is how callers (and the conformance runner) tell
28
+ // one apart from another.
29
+ this.name = "CairnQError";
30
+ }
31
+ }
21
32
 
22
33
  export class AlreadyExists extends CairnQError {
23
34
  constructor(public key: string) {
@@ -26,11 +37,44 @@ export class AlreadyExists extends CairnQError {
26
37
  }
27
38
  }
28
39
 
29
- /** wait/call did not reach a terminal status in time. The task keeps running. */
40
+ /** One line of "why hasn't this finished" from the last snapshot wait()
41
+ * observed. No worker running, no handler for the name, wrong queue, and two
42
+ * processes on different database files all look identical from the API side —
43
+ * queued, never claimed — so that case names the likely causes. */
44
+ function timeoutDetail(task: Task | null): string {
45
+ if (!task) return "task not found — wrong database file, or already purged?";
46
+ if (isQueued(task)) {
47
+ const delayMs = task.run_at_ms - nowMs();
48
+ if (task.attempt === 0 && delayMs <= 0) {
49
+ return (
50
+ `never claimed by a worker — is a worker running with a handler for ` +
51
+ `'${task.name}' on queue '${task.queue}', against this same database?`
52
+ );
53
+ }
54
+ const next = delayMs > 0 ? `, next run in ~${delayMs}ms` : "";
55
+ return `still queued (attempt ${task.attempt}/${task.max_attempts})${next}`;
56
+ }
57
+ if (cancelRequested(task)) return "cancel requested, waiting for the handler to observe it";
58
+ return `still running (attempt ${task.attempt}/${task.max_attempts})`;
59
+ }
60
+
61
+ /** wait/call did not reach a terminal status in time. The task keeps running.
62
+ * `task` is the last snapshot wait() observed (null if get() found nothing), and
63
+ * the message says what state it was stuck in — a queued-never-claimed task is
64
+ * the classic first-run failure (no worker, no handler, wrong queue or file). */
30
65
  export class TaskTimeout extends CairnQError {
31
- constructor(public taskId: string) {
32
- super(`task ${taskId} did not finish in time`);
66
+ readonly task: Task | null;
67
+ constructor(
68
+ public taskId: string,
69
+ opts: { timeoutMs?: number; task?: Task | null } = {},
70
+ ) {
71
+ super(
72
+ opts.timeoutMs == null
73
+ ? `task ${taskId} did not finish in time`
74
+ : `task ${taskId} did not finish within ${opts.timeoutMs}ms: ${timeoutDetail(opts.task ?? null)}`,
75
+ );
33
76
  this.name = "TaskTimeout";
77
+ this.task = opts.task ?? null;
34
78
  }
35
79
  }
36
80
 
@@ -81,6 +125,17 @@ export class ProtocolVersionMismatch extends CairnQError {
81
125
  }
82
126
  }
83
127
 
128
+ /** A value could not be encoded for a protocol JSON column (non-finite number,
129
+ * BigInt, circular structure, …). Raised at the boundary — submit rejects with
130
+ * it, and a worker records a handler result that triggers it as a permanent
131
+ * `unserializable_result` failure. The Python SDK raises the same named error. */
132
+ export class SerializationError extends CairnQError {
133
+ constructor(message: string) {
134
+ super(message);
135
+ this.name = "SerializationError";
136
+ }
137
+ }
138
+
84
139
  /** Throw inside a handler to control how the failure is recorded. Defaults to
85
140
  * non-retryable so deterministic errors fail fast instead of burning retries.
86
141
  * Any other thrown value is treated as retryable. */
package/src/index.ts CHANGED
@@ -7,7 +7,8 @@ export { defineTask } from "./task.js";
7
7
  export type { TaskDef } from "./task.js";
8
8
  export { SQLiteStore } from "./store/sqlite.js";
9
9
  export { PostgresStore } from "./store/postgres.js";
10
- export type { ListInput, SubmitInput, TaskStore, Conflict } from "./store/base.js";
10
+ export { TaskStore } from "./store/base.js";
11
+ export type { ListInput, PurgeInput, SubmitInput, Conflict } from "./store/base.js";
11
12
  export type { Task, TaskStatus } from "./models.js";
12
13
  export {
13
14
  STATUSES,
@@ -28,4 +29,5 @@ export {
28
29
  TaskError,
29
30
  LostLease,
30
31
  ProtocolVersionMismatch,
32
+ SerializationError,
31
33
  } from "./errors.js";
package/src/sql.ts CHANGED
@@ -2,17 +2,22 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
2
2
  import { dirname, join } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
 
5
- // Locate the shared cairnq-protocol dir. Resolution: $CAIRNQ_PROTOCOL_DIR ->
6
- // vendored `_protocol/` next to this module -> walk up to `cairnq-protocol/`
7
- // (monorepo dev). Both SDKs load the SAME .sql strings (zero-drift guarantee).
8
- // The dir is laid out per-dialect (sql/<dialect>/*.sql, migrations/<dialect>/*.sql)
9
- // so a second backend (Postgres) slots in beside sqlite; `dialect` picks the subtree.
5
+ // Locate the shared cairnq-protocol dir. Resolution: $CAIRNQ_PROTOCOL_DIR -> walk
6
+ // up to `cairnq-protocol/` (monorepo dev) -> vendored `_protocol/` next to this
7
+ // module (written at publish time). Both SDKs load the SAME .sql strings
8
+ // (zero-drift guarantee). The dir is laid out per-dialect (sql/<dialect>/*.sql,
9
+ // migrations/<dialect>/*.sql) so a second backend (Postgres) slots in beside
10
+ // sqlite; `dialect` picks the subtree.
11
+ //
12
+ // The source tree wins over the vendored copy on purpose: vendoring is a publish
13
+ // step that also runs locally, and a stale `_protocol/` shadowing the canonical
14
+ // SQL means edits to cairnq-protocol/ are silently not under test. An installed
15
+ // package has no repo above it, so it falls through to the vendored copy.
10
16
  export function findProtocolRoot(): string {
11
17
  const env = process.env.CAIRNQ_PROTOCOL_DIR;
12
18
  if (env) return env;
13
- let dir = dirname(fileURLToPath(import.meta.url));
14
- const vendored = join(dir, "_protocol");
15
- if (existsSync(join(vendored, "sql"))) return vendored;
19
+ const start = dirname(fileURLToPath(import.meta.url));
20
+ let dir = start;
16
21
  for (let i = 0; i < 10; i++) {
17
22
  const candidate = join(dir, "cairnq-protocol");
18
23
  if (existsSync(join(candidate, "sql"))) return candidate;
@@ -20,6 +25,8 @@ export function findProtocolRoot(): string {
20
25
  if (parent === dir) break;
21
26
  dir = parent;
22
27
  }
28
+ const vendored = join(start, "_protocol");
29
+ if (existsSync(join(vendored, "sql"))) return vendored;
23
30
  throw new Error("cannot locate cairnq-protocol; set CAIRNQ_PROTOCOL_DIR");
24
31
  }
25
32