@isparling/engram-cli 0.1.0

package/src/atomicWrite.ts

@@ -0,0 +1,80 @@
+// Atomic file replacement that preserves the target's permission bits.
+//
+// Atomic write: temp file in the same directory as the
+// target -> fsync the file descriptor -> rename -> fsync the containing
+// directory. The rename is what makes the write atomic from the point of
+// view of any reader; the two fsyncs are what make it durable across a
+// crash rather than merely atomic in memory.
+//
+// The temp file's mode is set to match the target's existing mode before
+// the rename, so a record that was e.g. 0600 doesn't silently become
+// 0644 (open()'s default, subject to umask) after a routine edit.
+//
+// Concurrency note (explicitly out of scope, not solved here): the write
+// path is single-writer. Two concurrent submitCandidate calls against the
+// same record both read-plan-write independently; the second rename simply
+// wins and the first writer's change is lost without conflict detection.
+// That is accepted for the single-user core, not fixed.
+
+import { open, rename, stat } from "node:fs/promises";
+import { basename, dirname, join } from "node:path";
+import { randomBytes } from "node:crypto";
+
+/**
+ * Thrown when the temp file was written, fsynced, and successfully
+ * renamed over the target — the new content is live and readable — but
+ * the final fsync of the containing directory failed. The write is not
+ * rolled back (a valid write is never undone to make something else
+ * look healthy); callers should treat the record as committed but treat
+ * anything depending on this write's crash-durability, including the
+ * subsequent qmd refresh, as unsafe to attempt.
+ */
+export class AtomicWriteDirectorySyncError extends Error {
+  readonly cause: unknown;
+
+  constructor(message: string, cause: unknown) {
+    super(message);
+    this.name = "AtomicWriteDirectorySyncError";
+    this.cause = cause;
+  }
+}
+
+export async function atomicWriteFile(targetPath: string, content: string): Promise<void> {
+  const dir = dirname(targetPath);
+  const tmpPath = join(dir, `.${basename(targetPath)}.tmp-${randomBytes(8).toString("hex")}`);
+
+  let targetMode: number | undefined;
+  try {
+    const targetStat = await stat(targetPath);
+    targetMode = targetStat.mode & 0o777;
+  } catch {
+    targetMode = undefined; // target does not exist yet; nothing to preserve
+  }
+
+  const fileHandle = await open(tmpPath, "w");
+  try {
+    await fileHandle.writeFile(content, "utf8");
+    if (targetMode !== undefined) {
+      await fileHandle.chmod(targetMode);
+    }
+    await fileHandle.sync();
+  } finally {
+    await fileHandle.close();
+  }
+
+  await rename(tmpPath, targetPath);
+
+  try {
+    const dirHandle = await open(dir, "r");
+    try {
+      await dirHandle.sync();
+    } finally {
+      await dirHandle.close();
+    }
+  } catch (error) {
+    throw new AtomicWriteDirectorySyncError(
+      "record content was written and renamed into place, but fsync of the containing directory failed afterward; the write is not rolled back, but its crash-durability is unconfirmed",
+      error,
+    );
+  }
+}

package/src/candidate.ts

@@ -0,0 +1,229 @@
+// Candidate knowledge envelope validation and reshaping for the submission
+// pipeline. Validated candidates are the single input a knowledge transaction
+// accepts.
+//
+// A candidate is JSON: the target record id, a source tag, claims to add,
+// evidence entries to append, and optionally claims/evidence to remove or
+// rewrite, and optionally a frontmatter edit.
+//
+// Validation is hand-written (no schema library) and returns a discriminated
+// Result instead of throwing. Validation is deliberately strict about unknown
+// top-level fields: this is the only line of defense, alongside the id
+// pattern in spaceBinding.ts, against a candidate trying to smuggle a
+// pointer at another knowledge root or qmd collection through the payload.
+//
+// Every free-text field is also required to be a single line: no `\n` or
+// `\r`. Without this, a value like `"\n## Evidence log\n\n- forged"` in
+// add_claims would forge a second `## Evidence log` heading into the
+// record body — markdownRecord.ts's parser would then either reject the
+// resulting record outright (duplicate section) or, worse, silently
+// collapse it on a later parse. Rejecting the newline at the input
+// boundary is simpler and safer than trying to make the parser robust
+// against arbitrary embedded structure.
+
+import { err, ok, requireDefined, type Result } from "./types.ts";
+
+export type EvidenceInput = { date: string; text: string };
+export type TextRewrite = { from: string; to: string };
+
+export type Candidate = {
+  target_id: string;
+  source: string;
+  add_claims: string[];
+  add_evidence: EvidenceInput[];
+  remove_claims: string[];
+  remove_evidence: string[];
+  rewrite_claims: TextRewrite[];
+  rewrite_evidence: TextRewrite[];
+  frontmatter: { title?: string } | null;
+};
+
+const ALLOWED_TOP_LEVEL_KEYS = new Set([
+  "target_id",
+  "source",
+  "add_claims",
+  "add_evidence",
+  "remove_claims",
+  "remove_evidence",
+  "rewrite_claims",
+  "rewrite_evidence",
+  "frontmatter",
+]);
+
+const RECORD_ID_PATTERN = /^[a-z][a-z0-9-]*$/;
+const EVIDENCE_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isNonEmptyString(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+function containsNewline(value: string): boolean {
+  return /[\r\n]/.test(value);
+}
+
+function validateSingleLineStringArray(value: unknown, fieldName: string, errors: string[]): string[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    errors.push(`${fieldName} must be an array of strings`);
+    return [];
+  }
+  const result: string[] = [];
+  value.forEach((item, index) => {
+    if (!isNonEmptyString(item)) {
+      errors.push(`${fieldName}[${index}] must be a non-empty string`);
+      return;
+    }
+    if (containsNewline(item)) {
+      errors.push(`${fieldName}[${index}] must not contain newlines`);
+      return;
+    }
+    result.push(item);
+  });
+  return result;
+}
+
+function validateRewriteArray(value: unknown, fieldName: string, errors: string[]): TextRewrite[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    errors.push(`${fieldName} must be an array of {from, to} objects`);
+    return [];
+  }
+  const result: TextRewrite[] = [];
+  value.forEach((item, index) => {
+    if (!isPlainObject(item)) {
+      errors.push(`${fieldName}[${index}] must be an object with "from" and "to"`);
+      return;
+    }
+    const keys = Object.keys(item);
+    const unexpected = keys.filter((k) => k !== "from" && k !== "to");
+    if (unexpected.length > 0) {
+      errors.push(`${fieldName}[${index}] has unexpected field(s): ${unexpected.join(", ")}`);
+      return;
+    }
+    if (!isNonEmptyString(item.from) || !isNonEmptyString(item.to)) {
+      errors.push(`${fieldName}[${index}] must have non-empty string "from" and "to"`);
+      return;
+    }
+    if (containsNewline(item.from) || containsNewline(item.to)) {
+      errors.push(`${fieldName}[${index}] "from"/"to" must not contain newlines`);
+      return;
+    }
+    result.push({ from: item.from, to: item.to });
+  });
+  return result;
+}
+
+function validateEvidenceArray(value: unknown, fieldName: string, errors: string[]): EvidenceInput[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    errors.push(`${fieldName} must be an array of {date, text} objects`);
+    return [];
+  }
+  const result: EvidenceInput[] = [];
+  value.forEach((item, index) => {
+    if (!isPlainObject(item)) {
+      errors.push(`${fieldName}[${index}] must be an object with "date" and "text"`);
+      return;
+    }
+    const keys = Object.keys(item);
+    const unexpected = keys.filter((k) => k !== "date" && k !== "text");
+    if (unexpected.length > 0) {
+      errors.push(`${fieldName}[${index}] has unexpected field(s): ${unexpected.join(", ")}`);
+      return;
+    }
+    if (typeof item.date !== "string" || !EVIDENCE_DATE_PATTERN.test(item.date) || containsNewline(item.date)) {
+      errors.push(`${fieldName}[${index}].date must match YYYY-MM-DD`);
+      return;
+    }
+    if (!isNonEmptyString(item.text)) {
+      errors.push(`${fieldName}[${index}].text must be a non-empty string`);
+      return;
+    }
+    if (containsNewline(item.text)) {
+      errors.push(`${fieldName}[${index}].text must not contain newlines`);
+      return;
+    }
+    result.push({ date: item.date, text: item.text });
+  });
+  return result;
+}
+
+export function validateCandidate(raw: unknown): Result<Candidate> {
+  const errors: string[] = [];
+
+  if (!isPlainObject(raw)) {
+    return err(["candidate must be a JSON object"]);
+  }
+
+  const unexpectedKeys = Object.keys(raw).filter((k) => !ALLOWED_TOP_LEVEL_KEYS.has(k));
+  if (unexpectedKeys.length > 0) {
+    errors.push(`candidate has unexpected field(s): ${unexpectedKeys.join(", ")}`);
+  }
+
+  let targetId: string | undefined;
+  if (!isNonEmptyString(raw.target_id)) {
+    errors.push("target_id is required and must be a non-empty string");
+  } else if (!RECORD_ID_PATTERN.test(raw.target_id)) {
+    errors.push(`target_id must match ${RECORD_ID_PATTERN} (lowercase letters, digits, hyphens; no path separators)`);
+  } else {
+    targetId = raw.target_id;
+  }
+
+  let source: string | undefined;
+  if (!isNonEmptyString(raw.source)) {
+    errors.push("source is required and must be a non-empty string");
+  } else if (containsNewline(raw.source)) {
+    errors.push("source must not contain newlines");
+  } else {
+    source = raw.source;
+  }
+
+  const addClaims = validateSingleLineStringArray(raw.add_claims, "add_claims", errors);
+  const removeClaims = validateSingleLineStringArray(raw.remove_claims, "remove_claims", errors);
+  const addEvidence = validateEvidenceArray(raw.add_evidence, "add_evidence", errors);
+  const removeEvidence = validateSingleLineStringArray(raw.remove_evidence, "remove_evidence", errors);
+  const rewriteClaims = validateRewriteArray(raw.rewrite_claims, "rewrite_claims", errors);
+  const rewriteEvidence = validateRewriteArray(raw.rewrite_evidence, "rewrite_evidence", errors);
+
+  let frontmatter: { title?: string } | null = null;
+  if (raw.frontmatter !== undefined && raw.frontmatter !== null) {
+    if (!isPlainObject(raw.frontmatter)) {
+      errors.push("frontmatter must be an object");
+    } else {
+      const keys = Object.keys(raw.frontmatter);
+      const unexpected = keys.filter((k) => k !== "title");
+      if (unexpected.length > 0) {
+        errors.push(`frontmatter has unexpected field(s): ${unexpected.join(", ")}`);
+      }
+      if (raw.frontmatter.title !== undefined) {
+        if (!isNonEmptyString(raw.frontmatter.title)) {
+          errors.push("frontmatter.title must be a non-empty string");
+        } else if (containsNewline(raw.frontmatter.title)) {
+          errors.push("frontmatter.title must not contain newlines");
+        } else {
+          frontmatter = { title: raw.frontmatter.title };
+        }
+      }
+    }
+  }
+
+  if (errors.length > 0) {
+    return err(errors);
+  }
+
+  return ok({
+    target_id: requireDefined(targetId, "target_id validated but not captured"),
+    source: requireDefined(source, "source validated but not captured"),
+    add_claims: addClaims,
+    add_evidence: addEvidence,
+    remove_claims: removeClaims,
+    remove_evidence: removeEvidence,
+    rewrite_claims: rewriteClaims,
+    rewrite_evidence: rewriteEvidence,
+    frontmatter,
+  });
+}

package/src/classify.ts

@@ -0,0 +1,275 @@
+// Mutation classification: a planned delta is additive, non-additive, or
+// no-change, derived from the assembled plan rather than from intent.
+//
+// Classifies a candidate against a parsed record and plans the resulting
+// Markdown mutation.
+//
+// Classification rule: additive means only appending
+// evidence and/or adding new claims. Non-additive means removing or
+// rewriting an existing claim or evidence entry, or altering frontmatter.
+//
+// ASSUMPTION (documented, not litigated further — see harness/README.md):
+// every commit mechanically stamps `updated` to the submission date. That
+// stamp is not, by itself, a classification-relevant frontmatter change;
+// only a candidate that explicitly requests a frontmatter edit (currently
+// just `title`) makes the candidate non-additive. Otherwise a harmless
+// bookkeeping timestamp would force every candidate through the approval
+// gate, which defeats the additive path entirely.
+//
+// Every remove/rewrite directive is resolved against the ORIGINAL record
+// state, not applied as a sequential pipeline: a rewrite whose `to` value
+// happens to match some other original entry's text must not make that
+// unrelated entry vanish just because a later step compares against
+// already-mutated state. Directives that are individually unambiguous but
+// jointly conflicting (the same original text named by both a remove and
+// a rewrite, or by two different rewrites) are rejected outright rather
+// than resolved by pipeline order, which is exactly what "order" was
+// silently deciding before.
+//
+// Classification is NOT trusted from candidate directives alone. A
+// candidate that only names add_claims/add_evidence LOOKS additive, but
+// the actual planned mutation is what can destroy content — through a
+// latent bug in this file, in withMutatedContent, or in the parser's
+// round-trip fidelity (preamble loss was exactly such a bug: it discarded
+// content that no candidate ever asked to touch, and would have kept
+// classifying as additive forever, because the classification check never
+// looked at the record). `mutationPreservesAllContent` compares the
+// planned `after` against the pristine `before` directly — preamble,
+// every "other" section, and every original claim/evidence entry must
+// still be present — and classification is non-additive if EITHER the
+// candidate declared a destructive directive OR the actual plan drops or
+// alters anything, whichever fires. This makes classification a property
+// of the mutation, not a report of intent.
+
+import type { Candidate, TextRewrite } from "./candidate.ts";
+import { renderUnifiedDiff } from "./diff.ts";
+import {
+  getActiveClaims,
+  getEvidenceLog,
+  withMutatedContent,
+  type OtherSection,
+  type ParsedRecord,
+} from "./markdownRecord.ts";
+import { serializeRecord } from "./markdownRecord.ts";
+import { err, ok, type Result } from "./types.ts";
+
+export type Classification = "additive" | "non-additive";
+
+export type MutationPlan = {
+  classification: Classification;
+  before: ParsedRecord;
+  after: ParsedRecord;
+  beforeText: string;
+  afterText: string;
+  diff: string;
+};
+
+/**
+ * Resolves remove/rewrite/add directives for a single flat text list
+ * (active claims, or evidence log entries) against its original state.
+ * Returns errors for: a remove/rewrite target that doesn't exist in the
+ * original list, two rewrite directives naming the same original text, or
+ * an original text named by both a remove and a rewrite directive.
+ */
+function resolveTextList(
+  original: string[],
+  removeList: string[],
+  rewriteList: TextRewrite[],
+  additions: string[],
+  fieldLabel: string,
+): Result<string[]> {
+  const errors: string[] = [];
+  const originalSet = new Set(original);
+
+  for (const text of removeList) {
+    if (!originalSet.has(text)) {
+      errors.push(`remove target not found in ${fieldLabel}: ${JSON.stringify(text)}`);
+    }
+  }
+
+  const rewriteFromCounts = new Map<string, number>();
+  for (const rewrite of rewriteList) {
+    if (!originalSet.has(rewrite.from)) {
+      errors.push(`rewrite target not found in ${fieldLabel}: ${JSON.stringify(rewrite.from)}`);
+    }
+    rewriteFromCounts.set(rewrite.from, (rewriteFromCounts.get(rewrite.from) ?? 0) + 1);
+  }
+
+  for (const [from, count] of rewriteFromCounts) {
+    if (count > 1) {
+      errors.push(`${fieldLabel} has ${count} conflicting rewrite directives for the same original text: ${JSON.stringify(from)}`);
+    }
+  }
+
+  const removeSet = new Set(removeList);
+  for (const from of rewriteFromCounts.keys()) {
+    if (removeSet.has(from)) {
+      errors.push(`${fieldLabel} text is targeted by both a remove and a rewrite directive: ${JSON.stringify(from)}`);
+    }
+  }
+
+  if (errors.length > 0) {
+    return err(errors);
+  }
+
+  const rewriteMap = new Map(rewriteList.map((rewrite) => [rewrite.from, rewrite.to]));
+  const resolved: string[] = [];
+  for (const item of original) {
+    if (removeSet.has(item)) continue;
+    const rewritten = rewriteMap.get(item);
+    resolved.push(rewritten !== undefined ? rewritten : item);
+  }
+  for (const addition of additions) {
+    resolved.push(addition);
+  }
+
+  return ok(resolved);
+}
+
+function arraysEqual(a: string[], b: string[]): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+
+/** before ⊆ after as a multiset: every original entry (counting
+ * duplicates) must still appear in after at least as many times. A
+ * rewrite or removal makes the original text's count drop, which this
+ * catches regardless of what the candidate's directives claimed to do;
+ * pure additions only ever raise counts, so they always pass. */
+function isTextMultisetSubset(before: string[], after: string[]): boolean {
+  const remaining = new Map<string, number>();
+  for (const item of after) {
+    remaining.set(item, (remaining.get(item) ?? 0) + 1);
+  }
+  for (const item of before) {
+    const count = remaining.get(item) ?? 0;
+    if (count <= 0) return false;
+    remaining.set(item, count - 1);
+  }
+  return true;
+}
+
+/**
+ * True iff `after` preserves everything `before` had: the same id and
+ * title (the mechanical `updated` bump is expected and excluded), the
+ * same preamble, every "other" section unchanged in place, and every
+ * original active-claims/evidence-log entry still present at least as
+ * many times as it originally appeared. False means the plan destroys or
+ * alters pre-existing content, independent of what the candidate declared.
+ *
+ * Exported so this safety net can be unit-tested directly against
+ * synthetic before/after pairs. Under the current (correct)
+ * resolveTextList/withMutatedContent implementation, there is no
+ * candidate reachable through the public submitCandidate/planMutation API
+ * that makes this return false for a directive-declared-additive
+ * candidate — which is the point: it is a structural backstop against a
+ * FUTURE bug in this file or in markdownRecord.ts's round-trip fidelity
+ * (preamble loss was exactly such a bug), not a path exercised by valid
+ * input today.
+ */
+export function mutationPreservesAllContent(before: ParsedRecord, after: ParsedRecord): boolean {
+  if (before.frontmatter.id !== after.frontmatter.id) return false;
+  if (before.frontmatter.title !== after.frontmatter.title) return false;
+
+  if (!arraysEqual(before.preambleLines, after.preambleLines)) return false;
+
+  const beforeOthers = before.sections.filter((s): s is OtherSection => s.kind === "other");
+  const afterOthers = after.sections.filter((s): s is OtherSection => s.kind === "other");
+  if (beforeOthers.length !== afterOthers.length) return false;
+  for (let i = 0; i < beforeOthers.length; i++) {
+    const beforeSection = beforeOthers[i];
+    const afterSection = afterOthers[i];
+    if (beforeSection === undefined || afterSection === undefined) return false;
+    if (beforeSection.heading !== afterSection.heading) return false;
+    if (!arraysEqual(beforeSection.lines, afterSection.lines)) return false;
+  }
+
+  if (!isTextMultisetSubset(getActiveClaims(before), getActiveClaims(after))) return false;
+  if (!isTextMultisetSubset(getEvidenceLog(before), getEvidenceLog(after))) return false;
+
+  return true;
+}
+
+export type ContentPreservationCheck = (before: ParsedRecord, after: ParsedRecord) => boolean;
+
+export function planMutation(
+  recordText: string,
+  record: ParsedRecord,
+  candidate: Candidate,
+  submittedAt: string,
+  /** Testing seam, defaulting to the real mutationPreservesAllContent.
+   * Mirrors the spawnFn/writeRecord injection pattern used elsewhere:
+   * lets a test prove the classification WIRING itself combines this
+   * check with the directive-declared signal, independent of whether a
+   * real candidate can currently drive the real check to false. */
+  contentCheck: ContentPreservationCheck = mutationPreservesAllContent,
+): Result<MutationPlan> {
+  const claimsResult = resolveTextList(
+    getActiveClaims(record),
+    candidate.remove_claims,
+    candidate.rewrite_claims,
+    candidate.add_claims.map((text) => `${text} [source: ${candidate.source}]`),
+    "active claims",
+  );
+  const evidenceResult = resolveTextList(
+    getEvidenceLog(record),
+    candidate.remove_evidence,
+    candidate.rewrite_evidence,
+    candidate.add_evidence.map((evidence) => `${evidence.date} — ${evidence.text} [source: ${candidate.source}]`),
+    "evidence log",
+  );
+
+  const errors: string[] = [];
+  if (!claimsResult.ok) errors.push(...claimsResult.errors);
+  if (!evidenceResult.ok) errors.push(...evidenceResult.errors);
+  if (errors.length > 0) {
+    return err(errors);
+  }
+  // Narrowed by the checks above, but the checks live on separate Result
+  // values so TypeScript can't see that from `errors.length === 0` alone.
+  if (!claimsResult.ok || !evidenceResult.ok) {
+    return err(["internal invariant violated: resolved list marked ok=false without contributing errors"]);
+  }
+
+  const after = withMutatedContent(record, {
+    frontmatter: {
+      id: record.frontmatter.id,
+      title: candidate.frontmatter?.title ?? record.frontmatter.title,
+      updated: submittedAt,
+    },
+    activeClaims: claimsResult.value,
+    evidenceLog: evidenceResult.value,
+  });
+
+  const declaredNonAdditive =
+    candidate.remove_claims.length > 0 ||
+    candidate.remove_evidence.length > 0 ||
+    candidate.rewrite_claims.length > 0 ||
+    candidate.rewrite_evidence.length > 0 ||
+    candidate.frontmatter !== null;
+
+  // The authoritative signal: does the actual planned mutation drop or
+  // alter any pre-existing content? This can fire even when
+  // declaredNonAdditive is false — e.g. a parser/serializer round-trip
+  // bug that silently drops content the candidate never asked to touch —
+  // which is exactly the property "no commit may destroy content the
+  // candidate did not ask to change" requires.
+  const contentPreserved = contentCheck(record, after);
+
+  const classification: Classification = declaredNonAdditive || !contentPreserved ? "non-additive" : "additive";
+
+  const afterText = serializeRecord(after);
+  const diff = renderUnifiedDiff(record.frontmatter.id, recordText, afterText);
+
+  return ok({
+    classification,
+    before: record,
+    after,
+    beforeText: recordText,
+    afterText,
+    diff,
+  });
+}