@openparachute/vault 0.7.1 → 0.7.2-rc.4

package/core/src/wikilinks.ts

@@ -1,7 +1,7 @@
 import { Database } from "bun:sqlite";
 import type { Note } from "./types.js";
 import * as linkOps from "./links.js";
-import { getNote, findNotesByTitle } from "./notes.js";
+import { getNote, findNotesByTitle, extractH1Title } from "./notes.js";
 import { chunkForInClause } from "./sql-in.js";
 import { transaction } from "./txn.js";
 import type { QueryWarning } from "./query-warnings.js";
@@ -619,11 +619,32 @@ function syncUnresolvedWikilinks(
 
 /**
  * Try to resolve pending wikilinks AND pending structured-link forward-refs
- * (vault#555) that point to a given path. Called when a note is created or
+ * (vault#555) that point to a given note. Called when a note is created or
  * its path changes. Each pending row materializes with ITS OWN relationship
  * (a structured link queued via {@link queueUnresolvedLink} backfills with
  * the caller's original relationship, not "wikilink").
  *
+ * Deferred resolution now covers all four legs of
+ * {@link resolveWikilinkDetailed}: exact path, basename, H1 title, and the
+ * explicit `path.ext` form. (Caveat: the candidate pre-filter below matches the
+ * title/ext legs via SQL `COLLATE NOCASE`, which case-folds ASCII only — a
+ * non-ASCII title differing from the target only in letter case is missed at
+ * the candidate stage and re-heals on the source's next save instead. Rare;
+ * tracked as a follow-up.) Before this, the sweep matched a pending row to
+ * the new note by PATH TEXT ONLY (`target_path = path OR path LIKE
+ * '%/'||target_path`) — so a `[[John Doe]]` that resolved at write time via
+ * the H1-title fallback (its note's displayed title differs from its path,
+ * e.g. `people/jdoe`), or a `[[Foo.csv]]` that resolved via the extension
+ * leg, silently never re-healed on a delete→recreate (the exact LB6 gap) and
+ * more broadly never backfilled when the target note was created AFTER the
+ * referencing note. Each candidate pending row is now VERIFIED through
+ * `resolveWikilinkDetailed` against the current DB — a row is healed only
+ * when its target string actually resolves to THIS note. An AMBIGUOUS target
+ * (≥2 notes now share the path/title) resolves to neither and stays queued,
+ * identical to write-time's "don't guess" contract — this also closes the
+ * pre-existing asymmetry where the path-only sweep would link an ambiguous
+ * `[[Foo]]` to whichever colliding note happened to be created.
+ *
  * Returns the number of links resolved.
  */
 export function resolveUnresolvedWikilinks(
@@ -632,13 +653,32 @@ export function resolveUnresolvedWikilinks(
   noteId: string,
 ): number {
   ensureRelationshipColumn(db);
-  let rows: { source_id: string; relationship: string }[];
+
+  // The newly-created / newly-repathed note supplies the two resolution keys
+  // the pending row's `target_path` (a bare path/basename string) can't carry
+  // on its own: the note's H1 title (title-fallback leg) and its extension
+  // (explicit `path.ext` leg). `getNote` is one indexed PK lookup; the title
+  // is parsed from content in JS (no separate query).
+  const note = getNote(db, noteId);
+  const h1Title = note?.content ? extractH1Title(note.content) : null;
+  const pathDotExt = note?.extension ? `${notePath}.${note.extension}` : null;
+
+  let rows: { source_id: string; target_path: string; relationship: string }[];
   try {
+    // Candidate pre-filter: every pending row whose `target_path` COULD
+    // resolve to this note under any resolveWikilinkDetailed leg — exact
+    // path, basename (target is the last path segment), H1 title, or the
+    // `path.ext` form. A `null` bind (no H1 heading / no extension) makes its
+    // clause never match (`target_path = NULL` is NULL, i.e. falsy in SQL).
+    // The verify step below is what enforces correctness; this clause only
+    // BOUNDS how many rows reach the (title-fallback-scanning) resolver.
     rows = db.prepare(`
-      SELECT source_id, relationship FROM unresolved_wikilinks
+      SELECT source_id, target_path, relationship FROM unresolved_wikilinks
       WHERE target_path = ? COLLATE NOCASE
          OR ? LIKE '%/' || target_path
-    `).all(notePath, notePath) as { source_id: string; relationship: string }[];
+         OR target_path = ? COLLATE NOCASE
+         OR target_path = ? COLLATE NOCASE
+    `).all(notePath, notePath, h1Title, pathDotExt) as typeof rows;
   } catch {
     return 0; // Table doesn't exist
   }
@@ -649,16 +689,24 @@ export function resolveUnresolvedWikilinks(
   for (const row of rows) {
     if (row.source_id === noteId) continue; // Skip self-links
 
+    // Verify against the SAME resolver write-time uses, so deferred
+    // resolution can't diverge from it: heal a pending row ONLY when its
+    // target string actually resolves to THIS note now. A miss or an
+    // ambiguous result leaves the row queued (surfaced as a visible broken
+    // link, and re-tried on the next matching note create).
+    const detail = resolveWikilinkDetailed(db, row.target_path);
+    if (!detail.resolved || detail.note_id !== noteId) continue;
+
     const relationship = row.relationship || WIKILINK_REL;
     linkOps.createLink(db, row.source_id, noteId, relationship);
     resolved++;
 
-    // Remove the unresolved entry (this exact relationship only — a
-    // source may have BOTH a wikilink and a structured-link forward-ref
-    // pending against the same target_path).
+    // Remove exactly this pending row — a source may have BOTH a wikilink and
+    // a structured-link forward-ref pending against the same target_path
+    // (distinct relationships), so scope the delete to all three PK columns.
     db.prepare(
-      "DELETE FROM unresolved_wikilinks WHERE source_id = ? AND relationship = ? AND (target_path = ? COLLATE NOCASE OR ? LIKE '%/' || target_path)",
-    ).run(row.source_id, relationship, notePath, notePath);
+      "DELETE FROM unresolved_wikilinks WHERE source_id = ? AND target_path = ? AND relationship = ?",
+    ).run(row.source_id, row.target_path, relationship);
   }
 
   return resolved;
@@ -840,3 +888,96 @@ export function getContentWikilinkWarnings(
 
   return warnings;
 }
+
+// ---------------------------------------------------------------------------
+// Delete-time re-queue (LB6) — a deleted note's INBOUND wikilink edges must
+// come back to life if a note is later recreated at the same path/title.
+// ---------------------------------------------------------------------------
+
+/**
+ * Before a note is deleted, re-queue every INBOUND `wikilink`-relationship
+ * edge pointing at it into `unresolved_wikilinks`, so that recreating a note
+ * matching the original `[[target]]` text (same path, basename, H1 title, or
+ * `path.ext` form — anything {@link resolveWikilinkDetailed} would have
+ * matched) auto-heals the edge via {@link resolveUnresolvedWikilinks} — which
+ * now verifies each pending row through that same resolver — exactly as if
+ * the link had never resolved in the first place.
+ *
+ * Without this, `deleteNote`'s `DELETE FROM notes` cascades the `links` row
+ * away (FK `ON DELETE CASCADE`) but leaves `unresolved_wikilinks` untouched —
+ * a note recreated at the deleted note's path never gets linked back to,
+ * because nothing is pending resolution for it. Only a re-save of the
+ * SOURCE note (which reparses its own content from scratch) would recover
+ * it; a fresh `[[Foo]]` reference elsewhere would work, but the ORIGINAL
+ * source note's edge stayed dead despite its unchanged `[[Foo]]` text.
+ *
+ * MUST be called BEFORE the note row is deleted — it needs both the `links`
+ * row (about to cascade away) and, per source, the CURRENT content (to
+ * recover the raw `[[target]]` text the resolver actually matched on: a
+ * resolved link's row retains the target NOTE id, not the original bracket
+ * text, and title-fallback / basename resolution mean that text can differ
+ * from the deleted note's own path).
+ *
+ * Scope: only `relationship = "wikilink"` inbound edges are re-queued.
+ * Explicit typed `links` (structured `links: [{target, relationship}]`
+ * entries, vault#555) are left alone — those are hand-authored associations
+ * the caller owns, not content-derived, so silently resurrecting them as a
+ * forward-ref on an unrelated future note would be surprising. A source
+ * that links to itself is never queued (wikilinks never create self-links,
+ * so no such inbound row can exist) and is skipped defensively anyway.
+ */
+export function requeueInboundWikilinksForDelete(db: Database, noteId: string): void {
+  let inbound: { source_id: string }[];
+  try {
+    inbound = db.prepare(
+      "SELECT DISTINCT source_id FROM links WHERE target_id = ? AND relationship = ?",
+    ).all(noteId, WIKILINK_REL) as { source_id: string }[];
+  } catch {
+    return; // links table missing (shouldn't happen post-schema-init, but never block a delete on this)
+  }
+  if (inbound.length === 0) return;
+
+  // Cheap string pre-filter (perf): a wikilink can only have resolved to the
+  // note being deleted if its raw target text equals one of THIS note's own
+  // resolution keys — its path, basename, H1 title, or `path.ext` form (the
+  // four legs {@link resolveWikilinkDetailed} matches on; every leg produces a
+  // target string equal to one of these, so the key set is a complete
+  // necessary-condition superset). Computed ONCE, then each source wikilink's
+  // target is gated on set membership BEFORE the expensive resolver call
+  // (whose title-fallback leg scans every note's content). Without this, a hub
+  // note with hundreds of inbound sources would fire hundreds of full-vault
+  // scans in one delete. The resolver still CONFIRMS each survivor — the
+  // pre-filter narrows, it doesn't decide.
+  const deleted = getNote(db, noteId);
+  const keys = new Set<string>();
+  const addKey = (s: string | null | undefined): void => {
+    const k = s?.trim().toLowerCase();
+    if (k) keys.add(k);
+  };
+  if (deleted?.path) {
+    addKey(deleted.path);
+    const slash = deleted.path.lastIndexOf("/");
+    addKey(slash >= 0 ? deleted.path.slice(slash + 1) : deleted.path); // basename
+    if (deleted.extension) addKey(`${deleted.path}.${deleted.extension}`);
+  }
+  if (deleted?.content) addKey(extractH1Title(deleted.content));
+  if (keys.size === 0) return; // no key any wikilink could have matched on
+
+  for (const { source_id } of inbound) {
+    if (source_id === noteId) continue; // defensive — wikilinks never self-link
+
+    const source = getNote(db, source_id);
+    if (!source || !source.content) continue;
+
+    const targetMap = dedupeWikilinkTargets(parseWikilinks(source.content));
+    for (const [, wl] of targetMap) {
+      // Skip the resolver unless this target text could match the deleted
+      // note by one of its resolution keys (necessary-condition pre-filter).
+      if (!keys.has(wl.target.trim().toLowerCase())) continue;
+      const detail = resolveWikilinkDetailed(db, wl.target);
+      if (detail.resolved && detail.note_id === noteId) {
+        queueUnresolvedLink(db, source_id, wl.target, WIKILINK_REL);
+      }
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/vault",
-  "version": "0.7.1",
+  "version": "0.7.2-rc.4",
   "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
   "module": "src/cli.ts",
   "type": "module",

package/src/cli.ts

@@ -2983,6 +2983,12 @@ async function cmdImport(args: string[]) {
     if (stats.skipped_attachments.length > 0) {
       console.log(`Skipped ${stats.skipped_attachments.length} attachment(s) — see warnings above.`);
     }
+    if (stats.skipped_duplicate_ids.length > 0) {
+      console.log(
+        `Skipped ${stats.skipped_duplicate_ids.length} file(s) with duplicate/blank note id(s) — ` +
+        `kept the first of each collision; see warnings above.`,
+      );
+    }
     return;
   }
 

package/src/mirror-import.ts

@@ -523,6 +523,11 @@ function importResultFromStats(
       `Dropped parent_names on tag "${sp.tag}": ${sp.reason}`,
     );
   }
+  for (const sd of stats.skipped_duplicate_ids) {
+    warnings.push(
+      `Skipped file (id "${sd.id}", path=${sd.path ?? "—"}): ${sd.reason}`,
+    );
+  }
   const result: ImportResult = {
     notes_imported: stats.notes_created + stats.notes_updated,
     tags_imported: stats.schemas_restored,

package/src/routes.ts

@@ -31,7 +31,7 @@ import {
   getContentWikilinkWarnings,
 } from "../core/src/wikilinks.ts";
 import { transactionAsync } from "../core/src/txn.ts";
-import { getNote, getNotes, getNoteTags, getNoteByTitle, toNoteIndex, filterMetadata, mergeMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError, PathConflictError, getVaultMap } from "../core/src/notes.ts";
+import { getNote, getNotes, getNoteTags, getNoteByTitle, toNoteIndex, filterMetadata, mergeMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError, PathConflictError, validatePath, PathValidationError, getVaultMap } from "../core/src/notes.ts";
 import { normalizePath } from "../core/src/paths.ts";
 import {
   parseContentRange,
@@ -178,6 +178,105 @@ function jsonWithWarnings(data: unknown, warnings: QueryWarning[], status = 200)
   return res;
 }
 
+/**
+ * Sane upper bound on a mutating-route JSON request body (LB7c). Without
+ * this, a giant field (most commonly a note's `content`) sails straight
+ * through `req.json()` and into wikilink parsing + the store write with no
+ * gate at all — reproduced at 50MB `content` -> 201 Created, RSS 92MB ->
+ * 663MB (memory-DoS via a single request). 10MB is generous for
+ * hand-authored or transcribed note content (multiple hours of dense
+ * transcript) while bounding the failure mode. Mirrors `/upload`'s
+ * `MAX_UPLOAD_BYTES` file-size gate for the JSON-body transport.
+ */
+export const MAX_JSON_BODY_BYTES = 10 * 1024 * 1024;
+
+function payloadTooLargeResponse(bytes: number): Response {
+  return json(
+    {
+      error: `Request body too large (${(bytes / 1024 / 1024).toFixed(1)}MB). Max: ${Math.round(MAX_JSON_BODY_BYTES / 1024 / 1024)}MB`,
+      error_type: "payload_too_large",
+      limit: MAX_JSON_BODY_BYTES,
+      got: bytes,
+    },
+    413,
+  );
+}
+
+/**
+ * Single choke point for every mutating route that reads a JSON request
+ * body (LB7 — door asymmetry fix). Three failure modes, all returning a
+ * clean 400/413 instead of falling through to server.ts's generic
+ * `{ error: "Internal server error" }` 500-with-no-`error_type` (LB7a), or
+ * silently succeeding on a wrong-shape body (LB7b — `null` used to throw a
+ * TypeError deep in the handler; `42`/`[]` used to sail through property
+ * access as `undefined` and create a blank note/no-op update):
+ *
+ *   1. **Oversize** — `Content-Length` over {@link MAX_JSON_BODY_BYTES}
+ *      rejects BEFORE `req.json()` even buffers/parses the body (the cheap
+ *      path). When the header is absent or understates the true size
+ *      (chunked transfer, a lying client), the parsed body's own
+ *      JSON-serialized size is checked as a backstop — one extra parse, but
+ *      still cheaper than letting the oversized value reach the store and
+ *      get re-processed (wikilink scan, indexing, etc.).
+ *   2. **Malformed JSON** — `req.json()` rejects; caught here rather than
+ *      propagating to the top-level handler.
+ *   3. **Wrong shape** — valid JSON, but not a plain object (`null`, an
+ *      array, or a bare primitive). Gated by `requireObject` (default
+ *      `true`); every current caller wants this — the parameter exists so a
+ *      future non-object-bodied route doesn't have to bypass the helper
+ *      entirely to opt out.
+ *
+ * Callers: check `.ok` and return `.response` on failure; `.body` is the
+ * parsed JSON value (still `any` — this validates SHAPE, not the specific
+ * fields a given route requires, which callers keep checking themselves).
+ */
+async function parseJsonBody(
+  req: Request,
+  opts?: { requireObject?: boolean },
+): Promise<{ ok: true; body: any } | { ok: false; response: Response }> {
+  const contentLengthHeader = req.headers.get("content-length");
+  if (contentLengthHeader) {
+    const declaredBytes = Number(contentLengthHeader);
+    if (Number.isFinite(declaredBytes) && declaredBytes > MAX_JSON_BODY_BYTES) {
+      return { ok: false, response: payloadTooLargeResponse(declaredBytes) };
+    }
+  }
+
+  const INVALID_JSON = Symbol("invalid-json");
+  const parsed = await req.json().catch(() => INVALID_JSON);
+  if (parsed === INVALID_JSON) {
+    return { ok: false, response: json({ error: "Invalid JSON body", error_type: "invalid_json" }, 400) };
+  }
+
+  if (!contentLengthHeader) {
+    const approxBytes = Buffer.byteLength(JSON.stringify(parsed) ?? "", "utf8");
+    if (approxBytes > MAX_JSON_BODY_BYTES) {
+      return { ok: false, response: payloadTooLargeResponse(approxBytes) };
+    }
+  }
+
+  const requireObject = opts?.requireObject ?? true;
+  if (requireObject && (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))) {
+    // The body PARSED as JSON but is the wrong SHAPE (a primitive/array/null,
+    // not an object) — a `invalid_request` per the taxonomy (parallel to
+    // `/tags/merge`'s `sources`/`target` shape errors). `invalid_json` is
+    // reserved for a genuine parse failure (the branch above).
+    return {
+      ok: false,
+      response: json(
+        {
+          error: "Request body must be a JSON object",
+          error_type: "invalid_request",
+          hint: `expected a JSON object body — got ${parsed === null ? "null" : Array.isArray(parsed) ? "an array" : typeof parsed}`,
+        },
+        400,
+      ),
+    };
+  }
+
+  return { ok: true, body: parsed };
+}
+
 /**
  * REST-only `removed_param` warning (vault#550) — the flat `date_field` /
  * `date_from` / `date_to` query-string params were removed at 0.6.4
@@ -1647,8 +1746,39 @@ async function handleNotesInner(
 
     // POST /notes — create (single or batch)
     if (method === "POST") {
-      const body = await req.json() as any;
+      const parsedBody = await parseJsonBody(req);
+      if (!parsedBody.ok) return parsedBody.response;
+      const body = parsedBody.body as any;
+      // `notes` (when present) must actually be an array — otherwise
+      // `items` below iterates whatever `body.notes` happens to be (e.g. a
+      // STRING iterates as one-character items), silently creating a batch
+      // of blank notes (LB7b).
+      if (body.notes !== undefined && !Array.isArray(body.notes)) {
+        return json(
+          {
+            error: "notes must be an array",
+            error_type: "invalid_request",
+            field: "notes",
+            hint: 'pass { "notes": [...] } for a batch, or a single note object with no `notes` key',
+          },
+          400,
+        );
+      }
       const items: any[] = body.notes ?? [body];
+      // Every batch item must itself be a plain object — a primitive/array
+      // entry would otherwise read as an all-`undefined` note (LB7b).
+      for (const item of items) {
+        if (item === null || typeof item !== "object" || Array.isArray(item)) {
+          return json(
+            {
+              error: "each note must be a JSON object",
+              error_type: "invalid_request",
+              hint: 'each entry in "notes" (or the top-level body, for a single-note POST) must be an object, e.g. {"content": "..."}',
+            },
+            400,
+          );
+        }
+      }
 
       // Batch cap (#213): refuse oversized batches before doing any work. 500
       // is the cap (Benjamin's number) — tighter blast radius than 1000 for
@@ -1811,6 +1941,9 @@ async function handleNotesInner(
           const extension = item.extension !== undefined
             ? validateExtension(item.extension)
             : undefined;
+          // Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
+          // Same batch-transaction throw semantics as extension validation.
+          validatePath(item.path);
           const effectiveExtension = extension ?? "md";
           const ifExists: string = item.if_exists ?? "error";
           const upsertMode = ifExists === "ignore" || ifExists === "update" || ifExists === "replace";
@@ -1958,6 +2091,13 @@ async function handleNotesInner(
             400,
           );
         }
+        // Bad `path` value (vault#589 / FIX 2) — NUL byte or `..` segment.
+        if (e && e.code === "INVALID_PATH") {
+          return json(
+            { error_type: "invalid_path", error: "invalid_path", path: e.path, reason: e.reason, message: e.message },
+            400,
+          );
+        }
         throw e;
       }
 
@@ -2042,7 +2182,9 @@ async function handleNotesInner(
       if (!noteWithinTagScope(note, tagScope.allowed, tagScope.raw)) {
         return json({ error: "Not found", error_type: "not_found" }, 404);
       }
-      const body = await req.json() as { path: string; mimeType: string; transcribe?: boolean };
+      const parsedBody = await parseJsonBody(req);
+      if (!parsedBody.ok) return parsedBody.response;
+      const body = parsedBody.body as { path: string; mimeType: string; transcribe?: boolean };
       if (!body.path || !body.mimeType) {
         return json(
           { error: "path and mimeType are required", error_type: "missing_required_field", hint: "pass both `path` and `mimeType`" },
@@ -2225,7 +2367,9 @@ async function handleNotesInner(
       // Body is parsed up front so the `if_missing: "create"` branch
       // (vault#309) can fire when the note doesn't exist. Pre-#309
       // shape parsed the body only after the not-found check.
-      const body = await req.json() as any;
+      const parsedBody = await parseJsonBody(req);
+      if (!parsedBody.ok) return parsedBody.response;
+      const body = parsedBody.body as any;
       const note = await resolveNote(store, idOrPath);
       if (!note) {
         // vault#309 — `if_missing: "create"` turns this PATCH into a
@@ -2253,6 +2397,9 @@ async function handleNotesInner(
           const createExt = body.extension !== undefined
             ? validateExtension(body.extension)
             : undefined;
+          // Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
+          // Covers both the explicit `path` and the idOrPath-as-path shape.
+          validatePath(explicitPath ?? (idLooksLikePath ? idOrPathStr : undefined));
           const createOpts: Parameters<Store["createNote"]>[1] = {
             ...(idLooksLikePath ? { path: explicitPath ?? idOrPathStr } : { id: idOrPathStr, ...(explicitPath !== undefined ? { path: explicitPath } : {}) }),
             ...(tagsArr.length > 0 ? { tags: tagsArr } : {}),
@@ -2499,7 +2646,12 @@ async function handleNotesInner(
         if (body.append !== undefined) updates.append = body.append;
         if (body.prepend !== undefined) updates.prepend = body.prepend;
       }
-      if (body.path !== undefined) updates.path = body.path;
+      if (body.path !== undefined) {
+        // Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
+        // Throws PathValidationError → 400 in the outer catch.
+        validatePath(body.path);
+        updates.path = body.path;
+      }
       if (body.extension !== undefined) {
         // Validate up front (vault#328). Throws ExtensionValidationError
         // which the outer catch converts to a 400.
@@ -2756,6 +2908,13 @@ async function handleNotesInner(
           400,
         );
       }
+      // Bad `path` value (vault#589 / FIX 2) — NUL byte or `..` segment.
+      if (e && e.code === "INVALID_PATH") {
+        return json(
+          { error_type: "invalid_path", error: "invalid_path", path: e.path, reason: e.reason, message: e.message },
+          400,
+        );
+      }
       throw e;
     }
   }
@@ -3075,7 +3234,9 @@ export async function handleTags(
     if (tagScope.allowed && !tagScope.allowed.has(putTagName)) {
       return tagScopeForbidden(tagScope.raw ?? []);
     }
-    const body = (await req.json()) as {
+    const parsedBody = await parseJsonBody(req);
+    if (!parsedBody.ok) return parsedBody.response;
+    const body = parsedBody.body as {
       description?: string | null;
       fields?: Record<string, unknown> | null;
       relationships?: Record<string, unknown> | null;
@@ -3489,7 +3650,9 @@ export async function handleVault(
   }
 
   if (req.method === "PATCH") {
-    const body = await req.json() as {
+    const parsedBody = await parseJsonBody(req);
+    if (!parsedBody.ok) return parsedBody.response;
+    const body = parsedBody.body as {
       description?: string;
       config?: { audio_retention?: string; auto_transcribe?: { enabled?: unknown } };
     };