@workit-dev/core 0.0.1 → 1.0.0

package/src/store/event-log.ts

@@ -0,0 +1,217 @@
+// The append-only event log of one task: `events.jsonl`, one JSON event per
+// line, plus `snapshot.json`, a rebuildable cache of the reduced state
+// (design §4.1).
+//
+// - A writer holds the task lock, appends one complete line with O_APPEND in
+//   a single write, and fsyncs. A crash can therefore only leave a torn last
+//   line (no trailing newline); readers ignore it and the next locked append
+//   truncates it away first. Any other unreadable line is real damage.
+// - Readers are lenient (D17): unknown event keys and unknown event types are
+//   ignored, unless the event says `critical: true`, in which case a reader
+//   that does not know the type fails closed with an upgrade message.
+// - The snapshot names the event it was taken after (seq, id, byte range). A
+//   snapshot that does not match the log is ignored and the log is replayed.
+import { randomUUID } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+export const EVENT_VERSION = 1;
+
+export type EventActor = { host: string; session: string | null; agentId: string | null };
+
+export type StoreEvent = {
+  v: number;
+  seq: number;
+  at: string;
+  id: string;
+  task: string;
+  actor: EventActor | null;
+  type: string;
+  data: Record<string, unknown>;
+  critical?: boolean;
+};
+
+export type LogRead = {
+  events: StoreEvent[];
+  /** Byte offset just past the last complete event line. */
+  end: number;
+  /** Byte offset where the last complete event line starts. */
+  lastStart: number;
+  /** File size; larger than `end` when a torn line trails the log. */
+  size: number;
+};
+
+export class LogDamage extends Error {
+  constructor(
+    message: string,
+    readonly upgrade: boolean = false,
+  ) {
+    super(message);
+  }
+}
+
+const isObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const parseEvent = (line: string, file: string, offset: number): StoreEvent => {
+  let value: unknown;
+  try {
+    value = JSON.parse(line);
+  } catch {
+    throw new LogDamage(`${file}: unreadable event at byte ${offset}`);
+  }
+  if (!isObject(value) || typeof value.v !== "number")
+    throw new LogDamage(`${file}: malformed event at byte ${offset}`);
+  if (value.v > EVENT_VERSION)
+    throw new LogDamage(
+      `${file}: event format v${value.v} was written by a newer Workit; upgrade Workit to read this task`,
+      true,
+    );
+  if (
+    typeof value.seq !== "number" ||
+    !Number.isInteger(value.seq) ||
+    typeof value.type !== "string" ||
+    typeof value.id !== "string" ||
+    typeof value.task !== "string" ||
+    !isObject(value.data)
+  )
+    throw new LogDamage(`${file}: malformed event at byte ${offset}`);
+  return value as StoreEvent;
+};
+
+/**
+ * Read the events of `file` starting at byte `from` (a line start). A torn
+ * trailing line is excluded; `expectSeq` is the seq the first event must have.
+ * Returns null when the file does not exist.
+ */
+export function readLog(file: string, from = 0, expectSeq: number | null = null): LogRead | null {
+  let fd: number;
+  try {
+    fd = fs.openSync(file, "r");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+  let bytes: Buffer;
+  let size: number;
+  try {
+    size = fs.fstatSync(fd).size;
+    if (from > size) throw new LogDamage(`${file}: shorter than its snapshot`);
+    bytes = Buffer.alloc(size - from);
+    let read = 0;
+    while (read < bytes.length) {
+      const got = fs.readSync(fd, bytes, read, bytes.length - read, from + read);
+      if (got === 0) break;
+      read += got;
+    }
+    bytes = bytes.subarray(0, read);
+    size = from + read;
+  } finally {
+    fs.closeSync(fd);
+  }
+  const events: StoreEvent[] = [];
+  let start = 0;
+  let lastStart = from;
+  let seq = expectSeq;
+  for (;;) {
+    const newline = bytes.indexOf(0x0a, start);
+    if (newline < 0) break;
+    const line = bytes.subarray(start, newline).toString("utf8");
+    if (line.trim() !== "") {
+      const event = parseEvent(line, file, from + start);
+      if (seq !== null && event.seq !== seq)
+        throw new LogDamage(`${file}: event seq ${event.seq} where ${seq} was expected`);
+      seq = event.seq + 1;
+      events.push(event);
+      lastStart = from + start;
+    }
+    start = newline + 1;
+  }
+  return { events, end: from + start, lastStart, size };
+}
+
+/** Durably write `dir`'s entries (no-op where directories cannot be fsynced). */
+export const fsyncDirectory = (dir: string): void => {
+  try {
+    const fd = fs.openSync(dir, "r");
+    try {
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch (error) {
+    if (process.platform !== "win32") throw error;
+  }
+};
+
+/**
+ * Append one event line. `validEnd` is where the last complete line ends: a
+ * torn tail beyond it is truncated first. Returns the byte offset the line
+ * starts at.
+ */
+export function appendEvent(file: string, event: StoreEvent, validEnd: number): number {
+  const line = Buffer.from(`${JSON.stringify(event)}\n`, "utf8");
+  const created = !fs.existsSync(file);
+  // Cut a torn tail first, through a writable handle: an append-only handle
+  // cannot truncate on Windows.
+  if (!created && fs.statSync(file).size !== validEnd) {
+    const cut = fs.openSync(file, "r+");
+    try {
+      fs.ftruncateSync(cut, validEnd);
+      fs.fsyncSync(cut);
+    } finally {
+      fs.closeSync(cut);
+    }
+  }
+  const fd = fs.openSync(file, "a", 0o600);
+  try {
+    let written = 0;
+    while (written < line.length) written += fs.writeSync(fd, line, written, line.length - written);
+    fs.fsyncSync(fd);
+  } finally {
+    fs.closeSync(fd);
+  }
+  if (created) fsyncDirectory(path.dirname(file));
+  return validEnd;
+}
+
+/** Replace `file` with `bytes` atomically (temp file + rename). */
+export function replaceFile(file: string, bytes: string, durable: boolean): void {
+  const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    const fd = fs.openSync(temporary, "wx", 0o600);
+    try {
+      fs.writeSync(fd, bytes);
+      if (durable) fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+    renameWithRetry(temporary, file);
+    if (durable) fsyncDirectory(path.dirname(file));
+  } catch (error) {
+    try {
+      fs.unlinkSync(temporary);
+    } catch {}
+    throw error;
+  }
+}
+
+const TRANSIENT_WINDOWS_CODES = new Set(["EPERM", "EACCES", "EBUSY"]);
+/** Windows briefly refuses to replace a file another process is reading. */
+export const isTransientWindowsError = (error: unknown): boolean => {
+  if (process.platform !== "win32") return false;
+  const code = (error as { code?: unknown } | null)?.code;
+  return typeof code === "string" && TRANSIENT_WINDOWS_CODES.has(code);
+};
+export const retryTransient = <T>(run: () => T): T => {
+  if (process.platform !== "win32") return run();
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      return run();
+    } catch (error) {
+      if (attempt >= 20 || !isTransientWindowsError(error)) throw error;
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5 * (attempt + 1));
+    }
+  }
+};
+const renameWithRetry = (from: string, to: string) => retryTransient(() => fs.renameSync(from, to));

package/src/store/notes.ts

@@ -0,0 +1,29 @@
+// One-time notes from the store to the user. A host installs a reporter on
+// the global `Symbol.for("workit.migrationReporter")` slot (the CLI router
+// does; it must not import workit-core on its cold path). Without one, the
+// note is silent: hooks and in-process hosts own their output channels.
+
+export type MigrationReport = {
+  /** The 2.x store that was migrated. */
+  from: string;
+  /** The store it now lives in. */
+  to: string;
+  /** Where the 2.x files were backed up. */
+  backup: string;
+  tasks: number;
+  skipped: number;
+  workspace: boolean;
+};
+
+const SLOT = Symbol.for("workit.migrationReporter");
+
+export const reportMigration = (report: MigrationReport): void => {
+  try {
+    const reporter = (globalThis as Record<symbol, unknown>)[SLOT];
+    if (typeof reporter === "function") reporter(report);
+  } catch {}
+};
+
+/** Whether a host installed a reporter (it can tell the user about a migration). */
+export const migrationReporterInstalled = (): boolean =>
+  typeof (globalThis as Record<symbol, unknown>)[SLOT] === "function";

package/src/store/patch.ts

@@ -0,0 +1,135 @@
+// Structural JSON diff for the task event log (design §4.1). One mutation
+// becomes the smallest set of operations that turns the previous record into
+// the next one, so the log grows with the change, never with the record:
+//
+//   set     replace the value at `path`
+//   del     remove the key at `path`
+//   push    append `values` to the array at `path`
+//   cut     truncate the array at `path` to `length`
+//
+// Arrays are compared element by element, so appending an entry (evidence,
+// findings, decisions) is one `push`, and editing one entry in place is a
+// diff inside that entry. Paths are arrays of keys and indices.
+
+export type PatchPath = (string | number)[];
+export type PatchOp =
+  | { op: "set"; path: PatchPath; value: unknown }
+  | { op: "del"; path: PatchPath }
+  | { op: "push"; path: PatchPath; values: unknown[] }
+  | { op: "cut"; path: PatchPath; length: number };
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const equal = (left: unknown, right: unknown): boolean => {
+  if (left === right) return true;
+  if (typeof left !== "object" || typeof right !== "object" || left === null || right === null)
+    return false;
+  if (Array.isArray(left) !== Array.isArray(right)) return false;
+  if (Array.isArray(left)) {
+    const other = right as unknown[];
+    return left.length === other.length && left.every((item, index) => equal(item, other[index]));
+  }
+  const leftKeys = Object.keys(left).filter(
+    (key) => (left as Record<string, unknown>)[key] !== undefined,
+  );
+  const rightKeys = Object.keys(right).filter(
+    (key) => (right as Record<string, unknown>)[key] !== undefined,
+  );
+  return (
+    leftKeys.length === rightKeys.length &&
+    leftKeys.every((key) =>
+      equal((left as Record<string, unknown>)[key], (right as Record<string, unknown>)[key]),
+    )
+  );
+};
+
+/** Operations that turn `before` into `after` (JSON values only). */
+export function diff(before: unknown, after: unknown, path: PatchPath = []): PatchOp[] {
+  if (equal(before, after)) return [];
+  if (isPlainObject(before) && isPlainObject(after)) {
+    const ops: PatchOp[] = [];
+    for (const key of Object.keys(before))
+      if (before[key] !== undefined && after[key] === undefined)
+        ops.push({ op: "del", path: [...path, key] });
+    for (const key of Object.keys(after)) {
+      if (after[key] === undefined) continue;
+      if (before[key] === undefined)
+        ops.push({ op: "set", path: [...path, key], value: after[key] });
+      else ops.push(...diff(before[key], after[key], [...path, key]));
+    }
+    return ops;
+  }
+  if (Array.isArray(before) && Array.isArray(after)) {
+    const ops: PatchOp[] = [];
+    if (after.length < before.length) ops.push({ op: "cut", path, length: after.length });
+    const shared = Math.min(before.length, after.length);
+    // A rewrite of most of the array is cheaper as one value.
+    let changed = 0;
+    for (let index = 0; index < shared; index += 1)
+      if (!equal(before[index], after[index])) changed += 1;
+    if (shared > 0 && changed > shared / 2 && changed > 4)
+      return [{ op: "set", path, value: after }];
+    for (let index = 0; index < shared; index += 1)
+      ops.push(...diff(before[index], after[index], [...path, index]));
+    if (after.length > before.length)
+      ops.push({ op: "push", path, values: after.slice(before.length) });
+    return ops;
+  }
+  return [{ op: "set", path, value: after }];
+}
+
+const clone = <T>(value: T): T => structuredClone(value);
+
+/** Apply `ops` to `target` in place; throws on a path that does not exist. */
+export function apply(target: unknown, ops: readonly PatchOp[]): unknown {
+  let root = target;
+  for (const op of ops) {
+    if (op.path.length === 0) {
+      if (op.op === "set") root = clone(op.value);
+      else if (op.op === "push" && Array.isArray(root)) root.push(...clone(op.values));
+      else if (op.op === "cut" && Array.isArray(root)) root.length = op.length;
+      else throw new Error(`patch: ${op.op} at the root is invalid`);
+      continue;
+    }
+    let parent: unknown = root;
+    for (const key of op.path.slice(0, -1)) {
+      if (typeof parent !== "object" || parent === null)
+        throw new Error(`patch: missing path ${op.path.join(".")}`);
+      parent = (parent as Record<string | number, unknown>)[key];
+    }
+    if (typeof parent !== "object" || parent === null)
+      throw new Error(`patch: missing path ${op.path.join(".")}`);
+    const key = op.path.at(-1)!;
+    const container = parent as Record<string | number, unknown>;
+    if (op.op === "set") container[key] = clone(op.value);
+    else if (op.op === "del") delete container[key];
+    else {
+      const array = container[key];
+      if (!Array.isArray(array)) throw new Error(`patch: ${op.path.join(".")} is not an array`);
+      if (op.op === "push") array.push(...clone(op.values));
+      else array.length = op.length;
+    }
+  }
+  return root;
+}
+
+const isPath = (value: unknown): value is PatchPath =>
+  Array.isArray(value) &&
+  value.every(
+    (key) =>
+      typeof key === "string" || (typeof key === "number" && Number.isInteger(key) && key >= 0),
+  );
+
+/** Structural check of untrusted ops (from disk). */
+export const isPatch = (value: unknown): value is PatchOp[] =>
+  Array.isArray(value) &&
+  value.every((op) => {
+    if (!isPlainObject(op) || !isPath(op.path)) return false;
+    if (op.op === "set") return "value" in op;
+    if (op.op === "del") return op.path.length > 0;
+    if (op.op === "push") return Array.isArray(op.values);
+    if (op.op === "cut")
+      return typeof op.length === "number" && Number.isInteger(op.length) && op.length >= 0;
+    return false;
+  });