@isparling/engram-cli 0.1.0

package/src/submit.ts

@@ -0,0 +1,228 @@
+// The one operation:
+//   submit a candidate against an existing Markdown record
+//     -> classify the change as additive or non-additive
+//     -> render the complete diff
+//     -> gate on approval when non-additive
+//     -> write atomically
+//     -> refresh qmd once
+//     -> report fresh or index-stale
+//
+// Markdown is authoritative, qmd is derived. A refresh failure after a valid
+// write never rolls back that write; it is reported as index-stale and the
+// committed Markdown stands.
+//
+// Approval integrity: `approval_required` carries a `plan_hash` covering
+// BOTH endpoints of the diff — the record bytes read and the bytes that
+// would replace them. A non-additive candidate can only be committed with
+// `approve: true` if `expectHash` matches the plan recomputed at commit
+// time. Anything that changes the mutation invalidates the approval: the
+// record changing on disk, a different candidate being submitted, or a
+// submission date that alters the result.
+//
+// Hashing the record alone was insufficient and shipped as a defect: it
+// caught a changed record but let a caller preview candidate A and then
+// approve candidate B using A's hash, committing a diff never shown.
+
+import { lstat, realpath, readFile } from "node:fs/promises";
+import type { Stats } from "node:fs";
+import { sep } from "node:path";
+import { AtomicWriteDirectorySyncError, atomicWriteFile } from "./atomicWrite.ts";
+import { validateCandidate, type Candidate } from "./candidate.ts";
+import type { Classification } from "./classify.ts";
+import { planMutation } from "./classify.ts";
+import { hashPlan } from "./contentHash.ts";
+import { parseRecord } from "./markdownRecord.ts";
+import { REFRESH_NOT_ATTEMPTED, refreshQmdCollection, type FreshnessRefreshReport, type RefreshReport, type SpawnFn } from "./qmdRunner.ts";
+import { resolveRecordPath, type SpaceBinding } from "./spaceBinding.ts";
+
+/** Same shape as atomicWriteFile. Overridable so tests can simulate a
+ * post-rename directory-fsync failure (AtomicWriteDirectorySyncError)
+ * deterministically, the same way qmdRunner's spawnFn lets tests simulate
+ * a spawn failure — both are OS-boundary failures that are impractical to
+ * trigger reliably through real fault injection in a portable test. */
+export type WriteRecordFn = (targetPath: string, content: string) => Promise<void>;
+
+export type SubmitOutcome =
+  | {
+      schema_version: 0;
+      status: "invalid";
+      errors: string[];
+      refresh: RefreshReport;
+    }
+  | {
+      schema_version: 0;
+      status: "approval_required";
+      record_id: string;
+      classification: "non-additive";
+      diff: string;
+      plan_hash: string;
+      refresh: RefreshReport;
+    }
+  | {
+      schema_version: 0;
+      status: "stale_approval";
+      record_id: string;
+      classification: "non-additive";
+      diff: string;
+      expected_plan_hash: string;
+      actual_plan_hash: string;
+      reason: string;
+      refresh: RefreshReport;
+    }
+  | {
+      schema_version: 0;
+      status: "committed";
+      record_id: string;
+      classification: Classification;
+      diff: string;
+      written_path: string;
+      refresh: FreshnessRefreshReport;
+    };
+
+export type SubmitInput = {
+  binding: SpaceBinding;
+  candidateInput: unknown;
+  approve: boolean;
+  /** Required when approve is true and the candidate turns out
+   * non-additive: must equal the plan_hash from a prior approval_required
+   * result, or the commit is refused as stale. */
+  expectHash?: string;
+  /** YYYY-MM-DD. Passed explicitly (never sampled internally) so results are deterministic under test. */
+  submittedAt: string;
+  /** Testing seam: overrides the real atomic write. Defaults to atomicWriteFile. */
+  writeRecord?: WriteRecordFn;
+  /** Testing seam: overrides the real qmd spawn function used for refresh. Defaults to the real child_process.spawn. */
+  spawnFn?: SpawnFn;
+};
+
+function invalid(errors: string[]): SubmitOutcome {
+  return { schema_version: 0, status: "invalid", errors, refresh: REFRESH_NOT_ATTEMPTED };
+}
+
+export async function submitCandidate(input: SubmitInput): Promise<SubmitOutcome> {
+  const candidateResult = validateCandidate(input.candidateInput);
+  if (!candidateResult.ok) {
+    return invalid(candidateResult.errors);
+  }
+  const candidate: Candidate = candidateResult.value;
+
+  const pathResult = resolveRecordPath(input.binding, candidate.target_id);
+  if (!pathResult.ok) {
+    return invalid(pathResult.errors);
+  }
+  const recordPath = pathResult.value;
+
+  let recordsRoot: string;
+  let recordStat: Stats;
+  try {
+    [recordsRoot, recordStat] = await Promise.all([realpath(input.binding.recordsRoot), lstat(recordPath)]);
+  } catch {
+    return invalid([`record not found for target_id: ${candidate.target_id}`]);
+  }
+  if (recordStat.isSymbolicLink()) {
+    return invalid([`record target must not be a symbolic link: ${candidate.target_id}`]);
+  }
+  let canonicalRecordPath: string;
+  try {
+    canonicalRecordPath = await realpath(recordPath);
+  } catch {
+    return invalid([`record target could not be resolved: ${candidate.target_id}`]);
+  }
+  const rootPrefix = recordsRoot.endsWith(sep) ? recordsRoot : recordsRoot + sep;
+  if (!canonicalRecordPath.startsWith(rootPrefix)) {
+    return invalid([`record target resolves outside the bound records root: ${candidate.target_id}`]);
+  }
+
+  // Read as bytes: the plan hash covers the record exactly as it is on disk,
+  // and decoding first would collapse distinct invalid UTF-8 sequences.
+  const recordBytes = await readFile(recordPath);
+  const recordText = recordBytes.toString("utf8");
+
+  const recordResult = parseRecord(recordText);
+  if (!recordResult.ok) {
+    return invalid(recordResult.errors);
+  }
+
+  const planResult = planMutation(recordText, recordResult.value, candidate, input.submittedAt);
+  if (!planResult.ok) {
+    return invalid(planResult.errors);
+  }
+  const plan = planResult.value;
+
+  // Covers destination and both endpoints of the diff, so approval cannot
+  // carry across to a different space, a different candidate, a changed
+  // record, or a different submission date.
+  const planHash = hashPlan(recordPath, recordBytes, plan.afterText);
+
+  if (plan.classification === "non-additive") {
+    if (!input.approve) {
+      return {
+        schema_version: 0,
+        status: "approval_required",
+        record_id: candidate.target_id,
+        classification: "non-additive",
+        diff: plan.diff,
+        plan_hash: planHash,
+        refresh: REFRESH_NOT_ATTEMPTED,
+      };
+    }
+
+    if (input.expectHash === undefined) {
+      return invalid([
+        "--approve on a non-additive candidate requires --expect <hash>, using the plan_hash from a prior approval_required result",
+      ]);
+    }
+
+    if (input.expectHash !== planHash) {
+      return {
+        schema_version: 0,
+        status: "stale_approval",
+        record_id: candidate.target_id,
+        classification: "non-additive",
+        diff: plan.diff,
+        expected_plan_hash: input.expectHash,
+        actual_plan_hash: planHash,
+        reason:
+          "the mutation is not the one that was approved; the record changed on disk, " +
+          "the candidate differs from the one previewed, or the submission date alters the result",
+        refresh: REFRESH_NOT_ATTEMPTED,
+      };
+    }
+  }
+
+  const writeRecord = input.writeRecord ?? atomicWriteFile;
+
+  try {
+    await writeRecord(recordPath, plan.afterText);
+  } catch (error) {
+    if (error instanceof AtomicWriteDirectorySyncError) {
+      return {
+        schema_version: 0,
+        status: "committed",
+        record_id: candidate.target_id,
+        classification: plan.classification,
+        diff: plan.diff,
+        written_path: recordPath,
+        refresh: {
+          attempted: false,
+          count: 0,
+          state: "index-stale",
+          detail: `refresh skipped: ${error.message}`,
+        },
+      };
+    }
+    throw error;
+  }
+
+  const refresh = await refreshQmdCollection(input.binding, input.spawnFn);
+
+  return {
+    schema_version: 0,
+    status: "committed",
+    record_id: candidate.target_id,
+    classification: plan.classification,
+    diff: plan.diff,
+    written_path: recordPath,
+    refresh,
+  };
+}

package/src/symlinkGuard.ts

@@ -0,0 +1,71 @@
+// Symlink guard: a symlink inside the records root that resolves outside it is
+// refused before any scanning qmd command runs.
+//
+// qmd scans collections with `followSymlinks: true` (qmd.ts's indexFiles,
+// via Glob.scan). A symlink inside the bound records root that resolves
+// outside it would therefore get indexed into the bound space's qmd
+// collection — silently widening what "the bound space" actually covers.
+// This walk runs before any qmd command that scans the filesystem
+// (`collection add`, `update`) and refuses if it finds one. Records roots
+// are small synthetic fixtures/spaces, so a full walk on every such call is
+// cheap; this is not meant to scale to large corpora.
+
+import { readdir, realpath } from "node:fs/promises";
+import { join, sep } from "node:path";
+import { realOrResolvedPath } from "./realPath.ts";
+import { err, ok, type Result } from "./types.ts";
+
+export async function verifyNoSymlinkEscape(recordsRoot: string): Promise<Result<void>> {
+  // A symlink target is always fully resolved by realpath(), so the root
+  // it's compared against must be too, or a symlinked path component
+  // (macOS's /var -> /private/var, which every OS temp directory lives
+  // under) makes an entirely-internal symlink look like an escape.
+  const resolvedRoot = await realOrResolvedPath(recordsRoot);
+  const rootWithSep = resolvedRoot.endsWith(sep) ? resolvedRoot : resolvedRoot + sep;
+  const errors: string[] = [];
+
+  async function walk(dir: string): Promise<void> {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      errors.push(`failed to read directory while checking for escaping symlinks: ${dir} (${describeError(error)})`);
+      return;
+    }
+
+    for (const entry of entries) {
+      const entryPath = join(dir, entry.name);
+
+      if (entry.isSymbolicLink()) {
+        let target: string;
+        try {
+          target = await realpath(entryPath);
+        } catch (error) {
+          errors.push(`symlink could not be resolved: ${entryPath} (${describeError(error)})`);
+          continue;
+        }
+        if (target !== resolvedRoot && !target.startsWith(rootWithSep)) {
+          errors.push(`symlink escapes the bound records root: ${entryPath} -> ${target}`);
+        }
+        // Do not follow into (or through) a symlink even if it happens to
+        // resolve inside the root — only plain directories are walked
+        // further, so a symlink cannot be used to reintroduce a deeper
+        // escaping symlink under a name this walk already accepted.
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        await walk(entryPath);
+      }
+    }
+  }
+
+  await walk(resolvedRoot);
+
+  if (errors.length > 0) return err(errors);
+  return ok(undefined);
+}
+
+function describeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/transactionLock.ts

@@ -0,0 +1,188 @@
+import { link, mkdir, readFile, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import type { ActiveSpace } from "./spaceRegistry.ts";
+import type { KnowledgeResult } from "./knowledgeTypes.ts";
+
+type LockOwner = {
+  schema_version: 0;
+  pid: number;
+  hostname: string;
+  token: string;
+};
+
+export type TransactionLock = {
+  state: "acquired" | "recovered";
+  release: () => Promise<void>;
+};
+
+export type TransactionLockHooks = {
+  afterExistingOwnerRead?: () => Promise<void>;
+};
+
+export function transactionLockDirectory(binding: ActiveSpace): string {
+  return join(binding.spaceRoot, ".engram-knowledge-transaction.lock");
+}
+
+function lockError<T>(code: string, message: string): KnowledgeResult<T> {
+  return { ok: false, errors: [{ kind: "lock", code, message }] };
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function errorCode(error: unknown): string | undefined {
+  return isObject(error) && typeof error.code === "string" ? error.code : undefined;
+}
+
+async function readOwner(path: string): Promise<KnowledgeResult<LockOwner>> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (!isObject(parsed) || parsed.schema_version !== 0 || typeof parsed.pid !== "number" || !Number.isInteger(parsed.pid) || parsed.pid <= 0 || typeof parsed.hostname !== "string" || parsed.hostname.length === 0 || typeof parsed.token !== "string" || parsed.token.length === 0) {
+      return { ok: false, errors: [{ kind: "lock", code: "lock_owner_unverifiable", message: "transaction lock owner metadata is malformed" }] };
+    }
+    return { ok: true, value: { schema_version: 0, pid: parsed.pid, hostname: parsed.hostname, token: parsed.token } };
+  } catch (error) {
+    return { ok: false, errors: [{ kind: "lock", code: "lock_owner_unverifiable", message: `transaction lock owner metadata could not be read: ${error instanceof Error ? error.message : String(error)}` }] };
+  }
+}
+
+function processState(pid: number): "live" | "absent" | "unknown" {
+  try {
+    process.kill(pid, 0);
+    return "live";
+  } catch (error) {
+    return errorCode(error) === "ESRCH" ? "absent" : "unknown";
+  }
+}
+
+async function installExclusiveOwnerMetadata(ownerPath: string, owner: LockOwner): Promise<KnowledgeResult<boolean>> {
+  const candidatePath = `${ownerPath}.candidate-${owner.pid}-${owner.token}`;
+  try {
+    await writeFile(candidatePath, JSON.stringify(owner), { encoding: "utf8", flag: "wx", mode: 0o600 });
+    try {
+      await link(candidatePath, ownerPath);
+      return { ok: true, value: true };
+    } catch (error) {
+      if (errorCode(error) === "EEXIST") return { ok: true, value: false };
+      return { ok: false, errors: [{ kind: "lock", code: "lock_install_failed", message: `transaction lock owner metadata could not be installed: ${error instanceof Error ? error.message : String(error)}` }] };
+    }
+  } catch (error) {
+    return { ok: false, errors: [{ kind: "lock", code: "lock_install_failed", message: `transaction lock owner metadata could not be prepared: ${error instanceof Error ? error.message : String(error)}` }] };
+  } finally {
+    await unlink(candidatePath).catch(() => {});
+  }
+}
+
+async function installLock(path: string, state: "acquired" | "recovered"): Promise<KnowledgeResult<TransactionLock>> {
+  const owner: LockOwner = { schema_version: 0, pid: process.pid, hostname: hostname(), token: randomUUID() };
+  const installed = await installExclusiveOwnerMetadata(join(path, "owner.json"), owner);
+  if (!installed.ok) return installed;
+  if (!installed.value) return lockError("lock_install_failed", "transaction lock owner metadata already exists");
+  return {
+    ok: true,
+    value: {
+      state,
+      release: async () => {
+        const current = await readOwner(join(path, "owner.json"));
+        if (!current.ok || current.value.token !== owner.token) return;
+        await rm(path, { recursive: true, force: false }).catch(() => {});
+      },
+    },
+  };
+}
+
+async function clearStaleRecoveryMarker(path: string): Promise<KnowledgeResult<void>> {
+  try {
+    await stat(path);
+  } catch (error) {
+    if (errorCode(error) === "ENOENT") return { ok: true, value: undefined };
+    return { ok: false, errors: [{ kind: "lock", code: "lock_recovery_failed", message: "transaction lock recovery marker could not be inspected" }] };
+  }
+
+  const owner = await readOwner(path);
+  if (!owner.ok) return { ok: false, errors: owner.errors };
+  if (owner.value.hostname !== hostname()) {
+    return lockError("lock_conflict", `transaction lock recovery is held by another host (${owner.value.hostname})`);
+  }
+  const state = processState(owner.value.pid);
+  if (state === "live") return lockError("lock_conflict", `transaction lock recovery is held by process ${owner.value.pid}`);
+  if (state === "unknown") return lockError("lock_owner_unverifiable", `transaction lock recovery owner process ${owner.value.pid} cannot be verified as absent`);
+  try {
+    await unlink(path);
+    return { ok: true, value: undefined };
+  } catch (error) {
+    if (errorCode(error) === "ENOENT") return { ok: true, value: undefined };
+    return { ok: false, errors: [{ kind: "lock", code: "lock_recovery_failed", message: "transaction lock recovery marker changed while it was being inspected" }] };
+  }
+}
+
+async function recoverProvenStaleLock(
+  path: string,
+  recoveryPath: string,
+  expectedOwner: LockOwner,
+): Promise<KnowledgeResult<void>> {
+  const recoveryOwner: LockOwner = { schema_version: 0, pid: process.pid, hostname: hostname(), token: randomUUID() };
+  const installed = await installExclusiveOwnerMetadata(recoveryPath, recoveryOwner);
+  if (!installed.ok) return { ok: false, errors: installed.errors };
+  if (!installed.value) return { ok: false, errors: [{ kind: "lock", code: "lock_conflict", message: "transaction lock stale recovery is already in progress" }] };
+
+  try {
+    const current = await readOwner(join(path, "owner.json"));
+    if (!current.ok) return { ok: false, errors: current.errors };
+    if (current.value.pid !== expectedOwner.pid || current.value.hostname !== expectedOwner.hostname || current.value.token !== expectedOwner.token) {
+      return { ok: false, errors: [{ kind: "lock", code: "lock_conflict", message: "transaction lock ownership changed during stale recovery" }] };
+    }
+    if (current.value.hostname !== hostname()) {
+      return { ok: false, errors: [{ kind: "lock", code: "lock_conflict", message: `transaction lock changed to another host (${current.value.hostname}) during stale recovery` }] };
+    }
+    const state = processState(current.value.pid);
+    if (state === "live") return { ok: false, errors: [{ kind: "lock", code: "lock_conflict", message: `transaction lock owner process ${current.value.pid} became live during stale recovery` }] };
+    if (state === "unknown") return { ok: false, errors: [{ kind: "lock", code: "lock_owner_unverifiable", message: `transaction lock owner process ${current.value.pid} cannot be verified as absent` }] };
+    try {
+      await rm(path, { recursive: true, force: false });
+      return { ok: true, value: undefined };
+    } catch (error) {
+      return { ok: false, errors: [{ kind: "lock", code: "lock_recovery_failed", message: `transaction lock stale recovery failed: ${error instanceof Error ? error.message : String(error)}` }] };
+    }
+  } finally {
+    const currentRecoveryOwner = await readOwner(recoveryPath);
+    if (currentRecoveryOwner.ok && currentRecoveryOwner.value.token === recoveryOwner.token) await unlink(recoveryPath).catch(() => {});
+  }
+}
+
+export async function acquireTransactionLock(binding: ActiveSpace, hooks?: TransactionLockHooks): Promise<KnowledgeResult<TransactionLock>> {
+  const path = transactionLockDirectory(binding);
+  const recoveryPath = `${path}.recovery`;
+  try {
+    await mkdir(path);
+    return installLock(path, "acquired");
+  } catch (error) {
+    if (errorCode(error) !== "EEXIST") {
+      return lockError("lock_acquire_failed", `transaction lock could not be acquired: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  const cleared = await clearStaleRecoveryMarker(recoveryPath);
+  if (!cleared.ok) return cleared;
+  const owner = await readOwner(join(path, "owner.json"));
+  if (!owner.ok) return owner;
+  if (hooks?.afterExistingOwnerRead !== undefined) await hooks.afterExistingOwnerRead();
+  if (owner.value.hostname !== hostname()) {
+    return lockError("lock_conflict", `transaction lock is held by another host (${owner.value.hostname})`);
+  }
+  const state = processState(owner.value.pid);
+  if (state === "live") return lockError("lock_conflict", `transaction lock is held by process ${owner.value.pid}`);
+  if (state === "unknown") return lockError("lock_owner_unverifiable", `transaction lock owner process ${owner.value.pid} cannot be verified as absent`);
+
+  const recovered = await recoverProvenStaleLock(path, recoveryPath, owner.value);
+  if (!recovered.ok) return recovered;
+  try {
+    await mkdir(path);
+  } catch (error) {
+    return lockError("lock_conflict", `transaction lock changed during stale recovery: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  return installLock(path, "recovered");
+}

package/src/types.ts

@@ -0,0 +1,38 @@
+// Shared result, error, and helper types for the harness runtime.
+//
+// Shared hand-written validation result type. No schema library is used
+// (zero runtime dependencies); every validating function in this package
+// returns one of these instead of throwing on bad input.
+
+export type Result<T> = { ok: true; value: T } | { ok: false; errors: string[] };
+
+export function ok<T>(value: T): Result<T> {
+  return { ok: true, value };
+}
+
+export function err<T>(errors: string[]): Result<T> {
+  return { ok: false, errors };
+}
+
+/**
+ * Structural stand-in for NodeJS.ProcessEnv that doesn't require importing
+ * Node's ambient global type or casting object literals to it in tests.
+ * `process.env` is structurally assignable to this without a cast.
+ */
+export type EnvLike = Record<string, string | undefined>;
+
+/**
+ * Use in place of the non-null assertion operator (`!`) wherever a value is
+ * known by loop/validation invariant to be defined but the type checker
+ * cannot prove it (array indexing without noUncheckedIndexedAccess, regex
+ * capture groups, Map.get after a prior .has check, etc.). Unlike `!`,
+ * this fails loudly with a descriptive error if the invariant is ever
+ * actually violated, instead of silently producing `undefined` typed as
+ * the non-optional type.
+ */
+export function requireDefined<T>(value: T | undefined, message: string): T {
+  if (value === undefined) {
+    throw new Error(`internal invariant violated: ${message}`);
+  }
+  return value;
+}