cairnq 0.1.0

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "cairnq",
+  "version": "0.1.0",
+  "description": "SQLite-first, cross-language, storage-centered durable task runtime",
+  "license": "MIT",
+  "author": "Jannchie <jannchie@gmail.com>",
+  "keywords": [
+    "sqlite",
+    "task-queue",
+    "job-queue",
+    "durable",
+    "task-runtime",
+    "worker",
+    "cross-language",
+    "better-sqlite3"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Jannchie/cairnq.git",
+    "directory": "cairnq-node"
+  },
+  "homepage": "https://github.com/Jannchie/cairnq#readme",
+  "bugs": "https://github.com/Jannchie/cairnq/issues",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": ["dist", "src"],
+  "engines": { "node": ">=20" },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "vitest run",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.3.0"
+  },
+  "optionalDependencies": {
+    "pg": "^8.13.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/node": "^22.7.0",
+    "@types/pg": "^8.11.10",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.1.0"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": ["better-sqlite3", "esbuild"]
+  }
+}

package/src/client.ts

@@ -0,0 +1,98 @@
+import { TaskCanceled, TaskFailed } from "./errors.js";
+import { isFailed, isSucceeded, type Task } from "./models.js";
+import { SQLiteStore } from "./store/sqlite.js";
+import { PostgresStore } from "./store/postgres.js";
+import type { ListInput, SubmitInput, TaskStore } from "./store/base.js";
+import { type TaskDef, taskName } from "./task.js";
+import { pollWait } from "./wait.js";
+
+export type SubmitOptions = Omit<SubmitInput, "name" | "payload">;
+export interface CallOptions extends SubmitOptions {
+  waitTimeoutMs?: number;
+  pollMs?: number;
+}
+
+/** API-side handle. Thin wrapper over a TaskStore + SDK-orchestrated wait/call. */
+export class CairnQ {
+  constructor(private readonly _store: TaskStore) {}
+
+  static sqlite(path: string, opts?: { busyTimeoutMs?: number }): CairnQ {
+    return new CairnQ(new SQLiteStore(path, opts));
+  }
+
+  /** Multi-host backend. `dsn` is a libpq connection string; requires the
+   * optional `pg` package. */
+  static postgres(dsn: string, opts?: { max?: number }): CairnQ {
+    return new CairnQ(new PostgresStore(dsn, opts));
+  }
+
+  get store(): TaskStore {
+    return this._store;
+  }
+
+  connect(): Promise<void> {
+    return this._store.connect();
+  }
+
+  close(): Promise<void> {
+    return this._store.close();
+  }
+
+  submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
+  submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;
+  submit(task: string | TaskDef, payload?: unknown, opts: SubmitOptions = {}): Promise<Task> {
+    return this._store.submit({ name: taskName(task), payload, ...opts });
+  }
+
+  get(taskId: string): Promise<Task | null> {
+    return this._store.get(taskId);
+  }
+
+  getByKey(key: string): Promise<Task | null> {
+    return this._store.getByKey(key);
+  }
+
+  list(input?: ListInput): Promise<Task[]> {
+    return this._store.list(input);
+  }
+
+  cancel(taskId: string): Promise<Task | null> {
+    return this._store.cancel(taskId);
+  }
+
+  cancelByKey(key: string): Promise<Task | null> {
+    return this._store.cancelByKey(key);
+  }
+
+  retry(taskId: string, opts?: { resetAttempt?: boolean }): Promise<Task | null> {
+    return this._store.retry(taskId, opts);
+  }
+
+  retryByKey(key: string, opts?: { resetAttempt?: boolean }): Promise<Task | null> {
+    return this._store.retryByKey(key, opts);
+  }
+
+  wait(
+    taskId: string,
+    opts: { timeoutMs?: number; pollMs?: number } = {},
+  ): Promise<Task> {
+    return pollWait(this._store, taskId, {
+      timeoutMs: opts.timeoutMs ?? 30_000,
+      pollMs: opts.pollMs,
+    });
+  }
+
+  /** submit + wait. Resolves with the result on success; rejects with
+   * TaskFailed / TaskCanceled / TaskTimeout otherwise. Pass a TaskDef and the
+   * resolved value is typed as its Result. */
+  async call(name: string, payload?: unknown, opts?: CallOptions): Promise<unknown>;
+  async call<P, R>(task: TaskDef<P, R>, payload?: P, opts?: CallOptions): Promise<R>;
+  async call(task: string | TaskDef, payload?: unknown, opts: CallOptions = {}): Promise<unknown> {
+    const { waitTimeoutMs = 30_000, pollMs, ...submit } = opts;
+    const created = await this.submit(taskName(task), payload, submit);
+    const final = await pollWait(this._store, created.id, { timeoutMs: waitTimeoutMs, pollMs });
+    if (isSucceeded(final)) return final.result;
+    if (isFailed(final)) throw new TaskFailed(final.error);
+    throw new TaskCanceled(final.id);
+  }
+}

package/src/context.ts

@@ -0,0 +1,85 @@
+import { cancelRequested, type Task } from "./models.js";
+import type { SubmitOptions } from "./client.js";
+import type { TaskStore } from "./store/base.js";
+import { type TaskDef, taskName } from "./task.js";
+import { pollWait } from "./wait.js";
+
+/** Handed to a task handler. Worker-side capabilities mirror the Python SDK. */
+export class TaskContext {
+  constructor(
+    private readonly store: TaskStore,
+    private readonly task: Task,
+    public readonly workerId: string,
+    private readonly leaseMs: number,
+  ) {}
+
+  get taskId(): string {
+    return this.task.id;
+  }
+  get name(): string {
+    return this.task.name;
+  }
+  get queue(): string {
+    return this.task.queue;
+  }
+  get attempt(): number {
+    return this.task.attempt;
+  }
+  get metadata(): any {
+    return this.task.metadata;
+  }
+  get rootId(): string | null {
+    return this.task.root_id;
+  }
+  get correlationId(): string | null {
+    return this.task.correlation_id;
+  }
+  get payload(): any {
+    return this.task.payload;
+  }
+
+  async progress(value: number | null, message: string | null = null): Promise<Task> {
+    return this.store.progress({
+      taskId: this.task.id,
+      workerId: this.workerId,
+      progress: value,
+      message,
+    });
+  }
+
+  async heartbeat(): Promise<Task> {
+    return this.store.heartbeat({
+      taskId: this.task.id,
+      workerId: this.workerId,
+      leaseMs: this.leaseMs,
+    });
+  }
+
+  /** Cooperative cancel check. */
+  async canceled(): Promise<boolean> {
+    const t = await this.store.get(this.task.id);
+    if (!t) return true;
+    return cancelRequested(t) || t.status === "canceled";
+  }
+
+  /** Submit a child task; parent/root/correlation are wired automatically. */
+  submit(name: string, payload?: unknown, opts?: SubmitOptions): Promise<Task>;
+  submit<P, R>(task: TaskDef<P, R>, payload?: P, opts?: SubmitOptions): Promise<Task>;
+  async submit(task: string | TaskDef, payload?: unknown, opts: SubmitOptions = {}): Promise<Task> {
+    return this.store.submit({
+      name: taskName(task),
+      payload,
+      parentId: this.task.id,
+      rootId: this.task.root_id,
+      correlationId: this.task.correlation_id,
+      ...opts,
+    });
+  }
+
+  async wait(taskId: string, opts: { timeoutMs?: number; pollMs?: number } = {}): Promise<Task> {
+    return pollWait(this.store, taskId, {
+      timeoutMs: opts.timeoutMs ?? 30_000,
+      pollMs: opts.pollMs,
+    });
+  }
+}

package/src/errors.ts

@@ -0,0 +1,112 @@
+/** The single shape of the JSON error envelope (see PROTOCOL.md). Everything that
+ * records an error — a handler exception, a missing handler, lease expiry, a thrown
+ * TaskError — builds it here, so the contract's fields live in one place. */
+export function errorEnvelope(e: {
+  type: string;
+  code: string;
+  message: string;
+  retryable: boolean;
+  details?: Record<string, unknown>;
+}): Record<string, unknown> {
+  return {
+    type: e.type,
+    code: e.code,
+    message: e.message,
+    retryable: e.retryable,
+    details: e.details ?? {},
+  };
+}
+
+export class CairnQError extends Error {}
+
+export class AlreadyExists extends CairnQError {
+  constructor(public key: string) {
+    super(`task with key ${key} already exists`);
+    this.name = "AlreadyExists";
+  }
+}
+
+/** wait/call did not reach a terminal status in time. The task keeps running. */
+export class TaskTimeout extends CairnQError {
+  constructor(public taskId: string) {
+    super(`task ${taskId} did not finish in time`);
+    this.name = "TaskTimeout";
+  }
+}
+
+/** A waited-on task ended in `failed`. The envelope's fields are unpacked onto the
+ * error — read `e.code` / `e.message` / `e.retryable` / `e.details` instead of
+ * digging into `e.error` (the raw envelope stays available on `e.error`). */
+export class TaskFailed extends CairnQError {
+  readonly type: string;
+  readonly code: string;
+  readonly retryable: boolean;
+  readonly details: Record<string, unknown>;
+  constructor(public error: unknown) {
+    const env = (error ?? {}) as {
+      type?: string;
+      code?: string;
+      message?: string;
+      retryable?: boolean;
+      details?: Record<string, unknown>;
+    };
+    super(env.message ?? "task failed");
+    this.name = "TaskFailed";
+    this.type = env.type ?? "TaskError";
+    this.code = env.code ?? "task_error";
+    this.retryable = env.retryable ?? false;
+    this.details = env.details ?? {};
+  }
+}
+
+export class TaskCanceled extends CairnQError {
+  constructor(public taskId: string) {
+    super(`task ${taskId} was canceled`);
+    this.name = "TaskCanceled";
+  }
+}
+
+/** A worker write affected 0 rows: the lease expired and was reclaimed. */
+export class LostLease extends CairnQError {
+  constructor(public taskId: string) {
+    super(`lost lease on task ${taskId}`);
+    this.name = "LostLease";
+  }
+}
+
+export class ProtocolVersionMismatch extends CairnQError {
+  constructor(message: string) {
+    super(message);
+    this.name = "ProtocolVersionMismatch";
+  }
+}
+
+/** Throw inside a handler to control how the failure is recorded. Defaults to
+ * non-retryable so deterministic errors fail fast instead of burning retries.
+ * Any other thrown value is treated as retryable. */
+export class TaskError extends CairnQError {
+  code: string;
+  retryable: boolean;
+  type: string;
+  details: Record<string, unknown>;
+  constructor(
+    message: string,
+    opts: { code?: string; retryable?: boolean; type?: string; details?: Record<string, unknown> } = {},
+  ) {
+    super(message);
+    this.name = "TaskError";
+    this.code = opts.code ?? "task_error";
+    this.retryable = opts.retryable ?? false;
+    this.type = opts.type ?? "TaskError";
+    this.details = opts.details ?? {};
+  }
+  envelope(): Record<string, unknown> {
+    return errorEnvelope({
+      type: this.type,
+      code: this.code,
+      message: this.message,
+      retryable: this.retryable,
+      details: this.details,
+    });
+  }
+}

package/src/ids.ts

@@ -0,0 +1,23 @@
+import { randomBytes } from "node:crypto";
+
+// ULID-style id. Must match the Python SDK byte-for-byte in format (PROTOCOL.md):
+// <prefix>_ + 26-char Crockford base32 of (48-bit ms timestamp << 80 | 80-bit random).
+const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+
+export function newUlid(tsMs: number = Date.now()): string {
+  let value = (BigInt(tsMs) << 80n) | BigInt("0x" + randomBytes(10).toString("hex"));
+  const chars: string[] = [];
+  for (let i = 0; i < 26; i++) {
+    chars.push(CROCKFORD[Number(value & 31n)]);
+    value >>= 5n;
+  }
+  return chars.reverse().join("");
+}
+
+export function newId(prefix = "task"): string {
+  return `${prefix}_${newUlid()}`;
+}
+
+export function nowMs(): number {
+  return Date.now();
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+export { CairnQ } from "./client.js";
+export type { CallOptions, SubmitOptions } from "./client.js";
+export { Worker } from "./worker.js";
+export type { Handler, TypedHandler, WorkerOptions } from "./worker.js";
+export { TaskContext } from "./context.js";
+export { defineTask } from "./task.js";
+export type { TaskDef } from "./task.js";
+export { SQLiteStore } from "./store/sqlite.js";
+export { PostgresStore } from "./store/postgres.js";
+export type { ListInput, SubmitInput, TaskStore, Conflict } from "./store/base.js";
+export type { Task, TaskStatus } from "./models.js";
+export {
+  STATUSES,
+  isTerminal,
+  cancelRequested,
+  isQueued,
+  isRunning,
+  isSucceeded,
+  isFailed,
+  isCanceled,
+} from "./models.js";
+export {
+  CairnQError,
+  AlreadyExists,
+  TaskTimeout,
+  TaskFailed,
+  TaskCanceled,
+  TaskError,
+  LostLease,
+  ProtocolVersionMismatch,
+} from "./errors.js";

package/src/models.ts

@@ -0,0 +1,63 @@
+// STATUSES is the canonical declaration within the TS SDK; TaskStatus derives from
+// it so the type and the runtime set can't drift apart. The cross-language source of
+// truth is the status CHECK constraint in cairnq-protocol's migration, which the
+// conformance suite pins this set against.
+export const STATUSES = ["queued", "running", "succeeded", "failed", "canceled"] as const;
+export type TaskStatus = (typeof STATUSES)[number];
+
+export interface Task {
+  id: string;
+  name: string;
+  queue: string;
+  status: TaskStatus;
+  payload: any;
+  metadata: any;
+  result: any | null;
+  error: any | null;
+  progress: number | null;
+  message: string | null;
+  attempt: number;
+  max_attempts: number;
+  priority: number;
+  worker_id: string | null;
+  lease_until_ms: number | null;
+  run_at_ms: number;
+  cancel_requested_at_ms: number | null;
+  parent_id: string | null;
+  root_id: string | null;
+  correlation_id: string | null;
+  created_at_ms: number;
+  updated_at_ms: number;
+  completed_at_ms: number | null;
+}
+
+const JSON_COLUMNS = ["payload", "result", "error", "metadata"] as const;
+export const TERMINAL: TaskStatus[] = ["succeeded", "failed", "canceled"];
+
+export function rowToTask(row: Record<string, unknown>): Task {
+  const t: Record<string, unknown> = { ...row };
+  for (const col of JSON_COLUMNS) {
+    const v = row[col];
+    // The driver decides a JSON column's wire form: SQLite (TEXT) hands back a
+    // string to parse; a jsonb-aware driver (Postgres `pg`) hands back an
+    // already-decoded object. Parse only a string — never assume one backend.
+    t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
+  }
+  return t as unknown as Task;
+}
+
+export function isTerminal(task: Task): boolean {
+  return TERMINAL.includes(task.status);
+}
+
+export function cancelRequested(task: Task): boolean {
+  return task.cancel_requested_at_ms != null;
+}
+
+// Status predicates — mirror the Python `task.succeeded` properties so callers
+// don't compare status strings by hand.
+export const isQueued = (task: Task): boolean => task.status === "queued";
+export const isRunning = (task: Task): boolean => task.status === "running";
+export const isSucceeded = (task: Task): boolean => task.status === "succeeded";
+export const isFailed = (task: Task): boolean => task.status === "failed";
+export const isCanceled = (task: Task): boolean => task.status === "canceled";

package/src/sql.ts

@@ -0,0 +1,49 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// Locate the shared cairnq-protocol dir. Resolution: $CAIRNQ_PROTOCOL_DIR ->
+// vendored `_protocol/` next to this module -> walk up to `cairnq-protocol/`
+// (monorepo dev). Both SDKs load the SAME .sql strings (zero-drift guarantee).
+// The dir is laid out per-dialect (sql/<dialect>/*.sql, migrations/<dialect>/*.sql)
+// so a second backend (Postgres) slots in beside sqlite; `dialect` picks the subtree.
+export function findProtocolRoot(): string {
+  const env = process.env.CAIRNQ_PROTOCOL_DIR;
+  if (env) return env;
+  let dir = dirname(fileURLToPath(import.meta.url));
+  const vendored = join(dir, "_protocol");
+  if (existsSync(join(vendored, "sql"))) return vendored;
+  for (let i = 0; i < 10; i++) {
+    const candidate = join(dir, "cairnq-protocol");
+    if (existsSync(join(candidate, "sql"))) return candidate;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error("cannot locate cairnq-protocol; set CAIRNQ_PROTOCOL_DIR");
+}
+
+export function loadStatements(
+  dialect = "sqlite",
+  root = findProtocolRoot(),
+): Record<string, string> {
+  const dir = join(root, "sql", dialect);
+  const out: Record<string, string> = {};
+  for (const file of readdirSync(dir).sort()) {
+    if (file.endsWith(".sql")) {
+      out[file.slice(0, -4)] = readFileSync(join(dir, file), "utf-8");
+    }
+  }
+  return out;
+}
+
+export function loadMigrations(
+  dialect = "sqlite",
+  root = findProtocolRoot(),
+): { name: string; sql: string }[] {
+  const dir = join(root, "migrations", dialect);
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".sql"))
+    .sort()
+    .map((f) => ({ name: f, sql: readFileSync(join(dir, f), "utf-8") }));
+}

package/src/store/base.ts

@@ -0,0 +1,67 @@
+import type { Task } from "../models.js";
+
+export type Conflict = "reuse" | "reject" | "replace";
+
+export interface SubmitInput {
+  name: string;
+  payload: unknown;
+  queue?: string;
+  key?: string | null;
+  conflict?: Conflict;
+  maxAttempts?: number;
+  priority?: number;
+  metadata?: unknown;
+  parentId?: string | null;
+  rootId?: string | null;
+  correlationId?: string | null;
+  runAtDelayMs?: number;
+}
+
+export interface ListInput {
+  status?: string | null;
+  queue?: string | null;
+  name?: string | null;
+  rootId?: string | null;
+  correlationId?: string | null;
+  limit?: number;
+  offset?: number;
+}
+
+/** The storage seam. SQLiteStore is the only MVP implementation. */
+export interface TaskStore {
+  connect(): Promise<void>;
+  close(): Promise<void>;
+  protocolVersion(): Promise<number>;
+
+  submit(input: SubmitInput): Promise<Task>;
+  get(taskId: string): Promise<Task | null>;
+  getByKey(key: string): Promise<Task | null>;
+  list(input?: ListInput): Promise<Task[]>;
+  cancel(taskId: string): Promise<Task | null>;
+  cancelByKey(key: string): Promise<Task | null>;
+  retry(taskId: string, opts?: { resetAttempt?: boolean }): Promise<Task | null>;
+  retryByKey(key: string, opts?: { resetAttempt?: boolean }): Promise<Task | null>;
+
+  claim(input: {
+    queues: string[];
+    workerId: string;
+    leaseMs?: number;
+    limit?: number;
+  }): Promise<Task[]>;
+  heartbeat(input: { taskId: string; workerId: string; leaseMs?: number }): Promise<Task>;
+  progress(input: {
+    taskId: string;
+    workerId: string;
+    progress: number | null;
+    message: string | null;
+  }): Promise<Task>;
+  succeed(input: { taskId: string; workerId: string; result: unknown }): Promise<Task>;
+  complete(input: { taskId: string; workerId: string; result: unknown }): Promise<Task>;
+  fail(input: {
+    taskId: string;
+    workerId: string;
+    error: unknown;
+    retryable?: boolean;
+    delayMs?: number;
+  }): Promise<Task>;
+}