@openparachute/vault 0.7.9-rc.7 → 0.7.9-rc.9

package/src/history-import.ts

@@ -1,10 +1,11 @@
 /** Offline git-history import. Git objects are data; no working tree is checked out. */
-import { Database } from "bun:sqlite";
+import { Database, SQLiteError } from "bun:sqlite";
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { chmodSync, closeSync, copyFileSync, existsSync, fsyncSync, mkdirSync, mkdtempSync, openSync, readSync, readFileSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
+import { isNoteIdShape } from "../core/src/ulid.ts";
 import { applyImportedNote, beginImportRun, canonicalJson, importObservationDigest, importTargetDigest, readImportReceipt, type ImportedObservation, type ImportRun } from "../core/src/history-import.ts";
 import { hashContent, resolveHistoryPolicy, VERSION_MAX_BYTES, type HistoryPolicy } from "../core/src/history.ts";
 import { applyConnectionPragmas, initSchema } from "../core/src/schema.ts";
@@ -16,7 +17,6 @@ const MAX_OUTPUT = 32 * 1024 * 1024;
 const MAX_COMMITS = 100_000;
 const MAX_OBJECTS = 10_000_000;
 const MAX_BYTES = 2 * 1024 * 1024 * 1024;
-const ID = /^[0-9A-HJKMNP-TV-Z]{26}$/;
 const decoder = new TextDecoder("utf-8", { fatal: true });
 class GitArchiveError extends Error {}
 function git(repo: string, args: string[]): Buffer {
@@ -47,7 +47,7 @@ function inline(raw: string): { meta: Record<string, unknown>; content: string }
   return { meta: yaml(match[1]!), content: raw.slice(match[0].length) };
 }
 function observation(meta: Record<string, unknown>, content: string, commit: string, blob: string, time: string): { id: string; row: ImportedObservation } {
-  if (typeof meta.id !== "string" || !ID.test(meta.id)) throw new Error("invalid_id");
+  if (!isNoteIdShape(meta.id)) throw new Error("invalid_id");
   if (Buffer.byteLength(content) > VERSION_MAX_BYTES) throw new Error("oversized_body");
   const path = meta.path ?? null, extension = meta.extension ?? "md", created = meta.created_at ?? null;
   if (path !== null && (typeof path !== "string" || path.includes("\0") || path.split("/").includes(".."))) throw new Error("invalid_path");
@@ -88,7 +88,7 @@ export function normalizeHistorySelections(input: unknown): HistorySelections {
   const seen = new Set<string>();
   const selections = value.selections.map(raw => {
     const entry = selectionObject(raw, ["note_id", "commit", "selected", "rejected", "reason"]);
-    if (typeof entry.note_id !== "string" || !ID.test(entry.note_id) || typeof entry.commit !== "string" || !OBJECT_ID.test(entry.commit) || !Array.isArray(entry.rejected) || !entry.rejected.length) throw new Error("Invalid selection entry");
+    if (!isNoteIdShape(entry.note_id) || typeof entry.commit !== "string" || !OBJECT_ID.test(entry.commit) || !Array.isArray(entry.rejected) || !entry.rejected.length) throw new Error("Invalid selection entry");
     const key = `${entry.note_id}:${entry.commit}`;
     if (seen.has(key)) throw new Error("Duplicate selection note/commit");
     seen.add(key);
@@ -182,10 +182,13 @@ export function stageArchive(repo: string, tip: string, stage: Database, inputSe
       collect(id, state, () => JSON.parse((parsedBody.get(cacheKey) as { row_json: string }).row_json), { git_path: entry.path, blob: entry.hash, sidecar });
     };
     const failure = (entry: typeof entries[number], error: unknown, id?: unknown) => {
+      // Staging/cache failures are operational, not evidence of malformed source.
+      // Abort so no diagnostic manifest can bless an incomplete archive walk.
+      if (error instanceof SQLiteError) throw error;
       const message = error instanceof Error ? error.message : "invalid_observation";
       const reason = /^[a-z_]+$/.test(message) ? message : "invalid_metadata_or_utf8";
       if (error instanceof GitArchiveError || message === "Archive byte limit exceeded" || message === "Archive note limit exceeded") throw error;
-      if (typeof id === "string" && ID.test(id)) stage.prepare("INSERT OR REPLACE INTO quarantine VALUES(?,?)").run(id, reason);
+      if (isNoteIdShape(id)) stage.prepare("INSERT OR REPLACE INTO quarantine VALUES(?,?)").run(id, reason);
       else stage.prepare("INSERT INTO issues VALUES(?,?,?)").run(commit, entry.path, reason);
     };
     const format = files.get(".parachute/vault.yaml");
@@ -195,7 +198,7 @@ export function stageArchive(repo: string, tip: string, stage: Database, inputSe
       try {
         meta = yaml(read(entry));
         const extension = meta.extension;
-        if (typeof meta.id !== "string" || !ID.test(meta.id) || typeof extension !== "string") throw new Error("invalid_sidecar");
+        if (!isNoteIdShape(meta.id) || typeof extension !== "string") throw new Error("invalid_sidecar");
         const contentPath = `${meta.path ?? `_unpathed/${meta.id}`}.${extension}`;
         const body = files.get(contentPath);
         if (!body) throw new Error("missing_sidecar_body");
@@ -318,7 +321,7 @@ function project(vault: string, repo: string, tip: string, dir: string, opts: {
     const selections = opts.selections ?? null;
     const source = stageArchive(repo, tip, stage, selections), policy = resolveHistoryPolicy(config.history);
     const waivers = opts.waivers ?? {};
-    for (const [id, reason] of Object.entries(waivers)) if (!ID.test(id) || typeof reason !== "string" || !reason.trim()) throw new Error("Waivers require note IDs and reasons");
+    for (const [id, reason] of Object.entries(waivers)) if (!isNoteIdShape(id) || typeof reason !== "string" || !reason.trim()) throw new Error("Waivers require note IDs and reasons");
     const options_digest = optionsDigest(policy, waivers, selections);
     const run = { run_id: "projection", source_fingerprint: source.source_fingerprint, tip, options_digest };
     beginImportRun(db, run);

package/src/history-rebuild-cli.test.ts

@@ -0,0 +1,38 @@
+import { test, expect } from "bun:test";
+import { Database } from "bun:sqlite";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { writeVaultConfig, vaultDbPath } from "./config.ts";
+import { BunSqliteStore } from "../core/src/store.ts";
+import { runDoctorScan } from "../core/src/doctor.ts";
+
+test("history rebuild-state repairs hints only, without requiring a Git mirror", async () => {
+  const taskDir = mkdtempSync(join(tmpdir(), "history-rebuild-"));
+  const saved = process.env.PARACHUTE_HOME;
+  process.env.PARACHUTE_HOME = taskDir;
+  let db: Database | undefined;
+  try {
+    writeVaultConfig({ name: "repair", api_keys: [], created_at: new Date().toISOString() });
+    db = new Database(vaultDbPath("repair"), { create: true });
+    const store = new BunSqliteStore(db);
+    const n = await store.createNote("before");
+    await store.updateNote(n.id, { content: "after" });
+    db.exec("UPDATE history_compact_state SET stored=999,refused=1");
+    expect(runDoctorScan(db).findings.some(f=>f.type==="history_compact_state_drift")).toBe(true);
+    const notes = db.query("SELECT * FROM notes").all(), versions = db.query("SELECT * FROM note_versions").all(), blobs = db.query("SELECT * FROM note_blobs").all();
+    db.close(); db=undefined;
+    const proc = Bun.spawnSync({ cmd: [process.execPath, join(import.meta.dir,"cli.ts"),"history","rebuild-state","--vault","repair","--json"], env: {...process.env, PARACHUTE_HOME: taskDir}, stdout:"pipe",stderr:"pipe" });
+    expect(proc.exitCode, new TextDecoder().decode(proc.stderr)).toBe(0);
+    expect(JSON.parse(new TextDecoder().decode(proc.stdout))).toEqual({ vault: "repair", rebuilt_notes: 1 });
+    db = new Database(vaultDbPath("repair"));
+    expect(runDoctorScan(db).findings.some(f=>f.type==="history_compact_state_drift")).toBe(false);
+    expect(db.query("SELECT * FROM notes").all()).toEqual(notes);
+    expect(db.query("SELECT * FROM note_versions").all()).toEqual(versions);
+    expect(db.query("SELECT * FROM note_blobs").all()).toEqual(blobs);
+  } finally {
+    db?.close();
+    if(saved===undefined)delete process.env.PARACHUTE_HOME;else process.env.PARACHUTE_HOME=saved;
+    rmSync(taskDir,{recursive:true,force:true});
+  }
+});

package/src/mcp-tools.ts

@@ -6,6 +6,7 @@
  */
 
 import { generateMcpTools, resolveNote } from "../core/src/mcp.ts";
+import { projectHistoryProvenance } from "../core/src/history-visibility.js";
 import type { McpToolDef, GenerateMcpToolsOpts } from "../core/src/mcp.ts";
 import { getNote, getNoteTags, getVaultMap } from "../core/src/notes.ts";
 import { narrowLinkWarningsForVisibility } from "../core/src/wikilinks.ts";
@@ -579,8 +580,13 @@ function applyTagScopeWrappers(
     )) {
       const id = (params as any).versions.note_id;
       const note = resolveNote(store.db, id);
-      return note && noteWithinTagScope(note, allowed, rawTags)
-        ? result : { error: "Note not found", error_type: "not_found", id };
+      if (!note || !noteWithinTagScope(note, allowed, rawTags)) {
+        return { error: "Note not found", error_type: "not_found", id };
+      }
+      if ("versions" in result && Array.isArray(result.versions)) {
+        return { ...result, versions: result.versions.map(row => projectHistoryProvenance(row, true)) };
+      }
+      return projectHistoryProvenance(result, true);
     }
     // Possible response shapes (vault#550 added the `warnings` variants):
     //   - Array (legacy list, no cursor, no warnings)
@@ -905,6 +911,7 @@ function applyTagScopeWrappers(
   wrapReadTool(tools, "doctor", async (orig, params) => {
     const allowed = await getAllowed();
     if (!allowed) return await orig(params);
+    if (params.deep) return forbidden("deep history audit requires an unrestricted session");
     return runDoctorScan(store.db, { allowedTags: allowed });
   });
 }

package/src/routes.ts

@@ -13,7 +13,8 @@ import { getImportedVersion, importStorageIndex, parseHistorySelector, projectHi
  */
 
 import { HistoryNotFoundError, latestTombstone } from "../core/src/history.js";
-import { ULID_REGEX } from "../core/src/ulid.js";
+import { projectHistoryProvenance } from "../core/src/history-visibility.js";
+import { isNoteIdShape } from "../core/src/ulid.js";
 import type { Database } from "bun:sqlite";
 import type { Store, Note, QueryOpts, AggregateSpec } from "../core/src/types.ts";
 import { TAG_EXPAND_MODES, stripTagHash, suggestSimilarTag, type TagExpandMode } from "../core/src/tag-hierarchy.ts";
@@ -268,6 +269,7 @@ function conflictResponse(e: any): Response | null {
       400,
     );
   }
+  if (e && e.code === "PRECONDITION_REQUIRED") return json({ error_type: "precondition_required", note_id: e.note_id, message: e.message }, 428);
   if (e && e.code === "HISTORY_NOT_FOUND") return json({ error: e.message, error_type: "not_found" }, 404);
   if (e && e.code === "HISTORY_OVERFLOW") return json({
     error_type: "history_overflow", note_id: e.note_id, byte_size: e.byte_size, limit: e.limit, message: e.message,
@@ -2930,7 +2932,7 @@ async function handleNotesInner(
     if (note && !noteWithinTagScope(note, tagScope.allowed, tagScope.raw)) {
       return json({ error: "Not found", error_type: "not_found" }, 404);
     }
-    if (!note && (tagScope.raw !== null || !ULID_REGEX.test(idOrPath) ||
+    if (!note && (tagScope.raw !== null || !isNoteIdShape(idOrPath) ||
       !(await store.listNoteVersions(idOrPath, { limit: 1 })).length)) {
       return json({ error: "Not found", error_type: "not_found" }, 404);
     }
@@ -2942,7 +2944,7 @@ async function handleNotesInner(
         const offset = Math.max(0, Number.parseInt(url.searchParams.get("offset") ?? "0", 10) || 0);
         const versions = await store.listNoteVersions(id, { limit, offset });
         const total = await store.countNoteVersions(id);
-        return json({ versions: versions.map(projectHistoryRow), total });
+        return json({ versions: versions.map(row => projectHistoryProvenance(projectHistoryRow(row), tagScope.allowed !== null)), total });
       }
       if (method === "DELETE") {
         const result = await store.eraseNoteHistory(id);
@@ -2953,7 +2955,7 @@ async function handleNotesInner(
       if (!/^\d+$/.test(importMatch[1]!) || !Number.isSafeInteger(index) || index < 0) return json({ error: "Invalid import_ix", error_type: "invalid_request" }, 400);
       try {
         const version = getImportedVersion(db, id, index);
-        return version ? json(projectHistoryRow(version)) : json({ error: "Not found", error_type: "not_found", origin: "git-import", import_ix: index }, 404);
+        return version ? json(projectHistoryProvenance(projectHistoryRow(version), tagScope.allowed !== null)) : json({ error: "Not found", error_type: "not_found", origin: "git-import", import_ix: index }, 404);
       } catch (e) {
         const response = conflictResponse(e);
         if (response) return response;
@@ -2964,7 +2966,7 @@ async function handleNotesInner(
       if (!/^\d+$/.test(verMatch[1]!) || !Number.isInteger(ix) || ix < 0) return json({ error: "Invalid version_ix", error_type: "invalid_request" }, 400);
       try {
         const version = await store.getNoteVersion(id, ix);
-        return version ? json(version) : json({ error: "Not found", error_type: "not_found" }, 404);
+        return version ? json(projectHistoryProvenance(version, tagScope.allowed !== null)) : json({ error: "Not found", error_type: "not_found" }, 404);
       } catch (e) {
         const response = conflictResponse(e);
         if (response) return response;
@@ -4451,11 +4453,21 @@ export function handleUnresolvedWikilinks(
  * catalog.
  */
 export async function handleDoctor(
-  _req: Request,
+  req: Request,
   store: Store,
   tagScope: TagScopeCtx = NO_TAG_SCOPE,
 ): Promise<Response> {
-  const report = await store.doctor({ allowedTags: tagScope.allowed });
+  const q = new URL(req.url).searchParams;
+  if (q.has("deep") && !["true", "false"].includes(q.get("deep")!)) return json({ error: "deep must be true or false" }, 400);
+  const deep = q.get("deep") === "true";
+  if (deep && tagScope.allowed !== null) return json({ error: "deep history audit requires an unrestricted session" }, 403);
+  const after = q.get("history_after") ?? undefined;
+  const maxBlobs = q.has("history_max_blobs") ? Number(q.get("history_max_blobs")) : undefined;
+  const budgetMs = q.has("history_budget_ms") ? Number(q.get("history_budget_ms")) : undefined;
+  if ((after !== undefined && !/^[a-f0-9]{64}$/.test(after)) ||
+      (maxBlobs !== undefined && (!Number.isInteger(maxBlobs) || maxBlobs < 1 || maxBlobs > 500)) ||
+      (budgetMs !== undefined && (!Number.isInteger(budgetMs) || budgetMs < 1 || budgetMs > 1000))) return json({ error: "invalid history audit bounds or cursor" }, 400);
+  const report = await store.doctor({ allowedTags: tagScope.allowed, deep, history_after: after, history_max_blobs: maxBlobs, history_budget_ms: budgetMs });
   return json(report);
 }
 

package/src/tag-integrity-mcp.test.ts

@@ -65,6 +65,24 @@ async function toolsFor(vaultName: string, scopedTags: string[] | null, verb: "r
   return generateScopedMcpTools(vaultName, authFor(vaultName, scopedTags, verb));
 }
 
+test("deep doctor REST/MCP parity and scoped refusal", async () => {
+  seedVault("journal");
+  const store = getVaultStore("journal");
+  await store.upsertTagRecord("mine", {});
+  const doctor = (await toolsFor("journal", null, "read")).find(t => t.name === "doctor")!;
+  const mcp: any = await doctor.execute({ deep: true, history_max_blobs: 1 });
+  const rest = await handleDoctor(new Request("http://localhost/api/doctor?deep=true&history_max_blobs=1"), store);
+  expect(rest.status).toBe(200);
+  expect((await rest.json() as any).history_audit).toEqual(mcp.history_audit);
+  expect(mcp.history_audit.complete).toBe(true);
+  const scoped = (await toolsFor("journal", ["mine"], "admin")).find(t => t.name === "doctor")!;
+  expect((await scoped.execute({ deep: true }) as any).error_type).toBe("tag_scope_violation");
+  expect((await handleDoctor(new Request("http://localhost/api/doctor?deep=true"), store, { allowed: new Set(["mine"]), raw: ["mine"] })).status).toBe(403);
+  for (const q of ["deep=1", "deep=true&history_after=oops", "deep=true&history_max_blobs=501", "deep=true&history_budget_ms=0"]) {
+    expect((await handleDoctor(new Request("http://localhost/api/doctor?" + q), store)).status).toBe(400);
+  }
+});
+
 describe("rename-tag / merge-tags — tag-scope gating (vault#552)", () => {
   test("rename-tag: old_name or new_name outside the allowlist is refused before reaching core", async () => {
     seedVault("journal");

package/src/vault-compact-routes.test.ts

@@ -64,7 +64,7 @@ test("P9/P18 compacted REST and MCP versions roundtrip with exact keys", async (
   expect(mcp).toEqual(get.body);
   const ml = await query()({ versions: { note_id: n.id, limit: 2 } }) as any;
   expect(ml.total).toBe(list.body.total);
-  const restore = await call(`/notes/${n.id}/restore`, "POST", { version_ix: 0 });
+  const restore = await call(`/notes/${n.id}/restore`, "POST", { version_ix: 0, if_updated_at: (await store.getNote(n.id))!.updatedAt });
   expect(restore.status).toBe(200);
   expect((await store.getNote(n.id))!.content).toBe(before.content);
 });

package/src/vault-history-routes.test.ts

@@ -11,6 +11,8 @@ import {
 import { getVaultStore, clearVaultStoreCache } from "./vault-store.ts";
 import { generateScopedMcpTools } from "./mcp-tools.ts";
 import { runDoctorScan } from "../core/src/doctor.ts";
+import { applyImportedNote, beginImportRun, importTargetDigest } from "../core/src/history-import.ts";
+import { resolveHistoryPolicy } from "../core/src/history.ts";
 let taskDir: string,
   savedHome: string | undefined,
   store: ReturnType<typeof getVaultStore>;
@@ -78,9 +80,56 @@ async function fixture() {
   await store.updateNote(n.id, { content: "two" });
   return n;
 }
+test("tag-scoped REST and MCP history omit provenance without changing stored rows", async () => {
+  const n = await store.createNote("original", { path: "history-visible", tags: ["journal"] });
+  await store.updateNote(n.id, { content: "current", actor: "PRIVATE_EDITOR", via: "PRIVATE_INTERFACE" });
+  const run = { run_id: "visibility", source_fingerprint: "source", tip: "tip", options_digest: "options" };
+  beginImportRun(store.db, run);
+  applyImportedNote(store.db, { run, noteId: n.id, targetDigest: importTargetDigest(store.db, n.id),
+    policy: resolveHistoryPolicy({ min_versions: 20 }), now: Date.now(),
+    observations: [{ content: "archive", path: "old/path", metadata: {}, extension: "md", created_at: null,
+      observed_at: new Date().toISOString(), commit: "commit", blob: "blob" }] });
+  const before = await store.listNoteVersions(n.id);
+  const full = (await call(n.id, "/versions")).body;
+  expect(full.versions[0]).toMatchObject({ actor: "PRIVATE_EDITOR", via: "PRIVATE_INTERFACE" });
+  for (const scoped of [false, true]) {
+    const rest = (await call(n.id, "/versions", "GET", undefined, scoped)).body;
+    const mcp = await query(scoped ? ["journal"] : null)({ versions: { note_id: n.id } });
+    expect(mcp).toEqual(rest);
+    expect(rest.total).toBe(2);
+    expect(rest.versions[0].version_ix).toBe(0);
+    expect(rest.versions[1]).toMatchObject({ origin: "git-import", import_ix: 0 });
+    for (const row of rest.versions) {
+      expect(Object.hasOwn(row, "actor")).toBe(!scoped);
+      expect(Object.hasOwn(row, "via")).toBe(!scoped);
+    }
+    for (const [path, selector, content] of [
+      ["/versions/0", { version_ix: 0 }, "original"],
+      ["/imports/0", { origin: "git-import", import_ix: 0 }, "archive"],
+    ] as const) {
+      const one = await call(n.id, path, "GET", undefined, scoped);
+      expect(one.status).toBe(200);
+      expect(one.body.content).toBe(content);
+      expect(await query(scoped ? ["journal"] : null)({ versions: { note_id: n.id, ...selector } })).toEqual(one.body);
+      expect(Object.hasOwn(one.body, "actor")).toBe(!scoped);
+      expect(Object.hasOwn(one.body, "via")).toBe(!scoped);
+    }
+  }
+  expect(await store.listNoteVersions(n.id)).toEqual(before);
+  expect((await store.getNote(n.id))!.content).toBe("current");
+  // Empty tag lists mean unrestricted in the existing authorization contract.
+  expect(await query([])({ versions: { note_id: n.id } })).toEqual(full);
+  const denied = await query(["unrelated"])({ versions: { note_id: n.id } });
+  expect(denied).toMatchObject({ error_type: "not_found" });
+});
 test("P5 REST restore rejects stale optimistic tokens without capture", async () => {
   const n = await fixture();
   const before = await store.listNoteVersions(n.id);
+  for (const extra of [{}, { force: true }]) {
+    const missing = await call(n.id, "/restore", "POST", { version_ix: 0, ...extra });
+    expect(missing.status).toBe(428);
+    expect(missing.body.error_type).toBe("precondition_required");
+  }
   const r = await call(n.id, "/restore", "POST", {
     version_ix: 0,
     if_updated_at: "1999-01-01T00:00:00.000Z",
@@ -320,6 +369,21 @@ test("P19 overflow deletion remains possible but its tombstone cannot restore",
   expect(r.status).toBe(409);
   expect(r.body.error_type).toBe("history_unrecoverable");
 });
+test("legacy deleted IDs resolve exact history and restore without exposing scoped tombstones", async () => {
+  for (const id of ["2020-01-02-03-04-05", "legacy:abc123", "shortid"]) {
+    await store.createNote("original", { id, path: `legacy/${id}` });
+    await store.updateNote(id, { content: "current" });
+    await store.deleteNote(id);
+    expect((await call(id, "/versions", "GET", undefined, true)).status).toBe(404);
+    expect((await call(id, "/restore", "POST", { version_ix: 0 }, true)).status).toBe(404);
+    expect((await call(id, "/versions")).status).toBe(200);
+    const restored = await call(id, "/restore", "POST", { version_ix: 0 });
+    expect(restored.status).toBe(200);
+    expect(restored.body).toMatchObject({ id, content: "original", recreated: true });
+  }
+  expect((await call("not-a-tombstone", "/versions")).status).toBe(404);
+  expect((await call("not-a-tombstone", "/restore", "POST", { version_ix: 0 })).status).toBe(404);
+});
 test("P20 REST recreates untagged at the tombstone path and guards conflicts", async () => {
   const n = await fixture();
   await store.deleteNote(n.id);
@@ -393,7 +457,7 @@ test("P24 migration bypass restores, captures once, and emits the bypass log", a
   const n = await strictFixture();
   const log = spyOn(console, "warn").mockImplementation(() => {});
   try {
-    const r = await call(n.id, "/restore", "POST", { version_ix: 0 }, false, {
+    const r = await call(n.id, "/restore", "POST", { version_ix: 0, if_updated_at: (await store.getNote(n.id))!.updatedAt }, false, {
       actor: "migrator",
       via: "api",
       bypassStrict: true,