@openparachute/vault 0.7.0-rc.8 → 0.7.1

package/core/src/notes.ts

@@ -1,5 +1,5 @@
 import { Database, type SQLQueryBindings } from "bun:sqlite";
-import type { Note, NoteIndex, QueryOpts, QueryNotesPage, VaultStats } from "./types.js";
+import type { Note, NoteIndex, QueryOpts, QueryNotesPage, VaultStats, VaultMap, AggregateSpec, AggregateRow } from "./types.js";
 import { normalizePath } from "./paths.js";
 import { transaction } from "./txn.js";
 import {
@@ -28,8 +28,7 @@ import {
   SEARCH_WEIGHT_CONTENT,
   type SearchMode,
 } from "./search-query.js";
-
-let idCounter = 0;
+import { generateUlid } from "./ulid.js";
 
 /**
  * Write-attribution context (vault#298) — the two axes of provenance threaded
@@ -61,20 +60,25 @@ function attrValue(v: string | null | undefined): string | null {
   return t.length === 0 ? null : t;
 }
 
-/** Generate a timestamp-based ID: YYYY-MM-DD-HH-MM-SS-ffffff */
+/**
+ * Generate a new note/attachment ID.
+ *
+ * As of vault#ulid-ids this returns a ULID (see `ulid.ts`) — monotonic,
+ * lexicographically time-sortable, Crockford base32, opaque, and
+ * collision-resistant. Previously this returned a timestamp-format ID
+ * (`YYYY-MM-DD-HH-MM-SS-ffffff`).
+ *
+ * IMPORTANT: existing notes are NOT migrated. Old timestamp-format IDs
+ * stay exactly as they are — only newly-generated IDs use the ULID
+ * format, so a vault's `id` column is (and must remain) a mix of both
+ * shapes indefinitely. Nothing may assume a uniform id format, and
+ * nothing may parse a note's ID to recover its creation time — that's
+ * what the `created_at` column is for. The cursor-pagination tiebreaker
+ * (`cursor.ts`) treats `id` as an opaque, stable string for ordering
+ * ties only, which holds for any id format.
+ */
 export function generateId(): string {
-  const now = new Date();
-  const pad = (n: number, len = 2) => String(n).padStart(len, "0");
-  const micro = now.getMilliseconds() * 1000 + (idCounter++ % 1000);
-  return [
-    now.getFullYear(),
-    pad(now.getMonth() + 1),
-    pad(now.getDate()),
-    pad(now.getHours()),
-    pad(now.getMinutes()),
-    pad(now.getSeconds()),
-    pad(micro, 6),
-  ].join("-");
+  return generateUlid();
 }
 
 export function createNote(
@@ -183,6 +187,66 @@ export function getNoteByPath(db: Database, path: string, extension?: string): N
   );
 }
 
+/**
+ * Extract a note's "H1 title" — the first line in `content` that starts
+ * with a literal `"# "` (a level-1 Markdown heading; `"## "`+ don't match
+ * because `^#[ \t]` fails at that line's start once a second `#` follows
+ * the first). Returns null when no such line exists, or the text after
+ * the marker is blank once trimmed.
+ */
+export function extractH1Title(content: string): string | null {
+  const match = content.match(/^#[ \t]+(.+)$/m);
+  if (!match) return null;
+  const title = match[1]!.trim();
+  return title.length > 0 ? title : null;
+}
+
+/**
+ * Find every note whose H1 title (see {@link extractH1Title}) equals
+ * `title`, case-insensitively (matches the COLLATE NOCASE policy every
+ * other path/basename lookup in this file uses). Used as the title-fallback
+ * step for wikilink/id/path resolution — a target that misses on
+ * id/path/basename gets one more chance against notes whose displayed
+ * title differs from their path.
+ *
+ * This is a full-content scan: no index exists on "first heading line," so
+ * every note's content is read once and matched in JS. Acceptable because
+ * every call site reaches this ONLY after the cheap indexed lookups (id,
+ * path, basename) have already missed — a rare fallback path, not the hot
+ * resolution route.
+ */
+export function findNotesByTitle(db: Database, title: string): { id: string; path: string | null }[] {
+  const needle = title.trim().toLowerCase();
+  if (!needle) return [];
+  const rows = db.prepare("SELECT id, path, content FROM notes").all() as {
+    id: string;
+    path: string | null;
+    content: string;
+  }[];
+  const matches: { id: string; path: string | null }[] = [];
+  for (const row of rows) {
+    const h1 = extractH1Title(row.content);
+    if (h1 && h1.toLowerCase() === needle) {
+      matches.push({ id: row.id, path: row.path });
+    }
+  }
+  return matches;
+}
+
+/**
+ * Title-fallback lookup: resolve `title` to a note ONLY when exactly one
+ * note in the vault carries that H1 title (see {@link findNotesByTitle}).
+ * Zero matches or 2+ matches (ambiguous) both return null — "don't guess"
+ * mirrors the existing basename-ambiguity policy (vault#328). Callers use
+ * this as the LAST resort after id and path/basename resolution have
+ * already missed; exact id/path matches always win first.
+ */
+export function getNoteByTitle(db: Database, title: string): Note | null {
+  const matches = findNotesByTitle(db, title);
+  if (matches.length !== 1) return null;
+  return getNote(db, matches[0]!.id);
+}
+
 export function getNotes(db: Database, ids: string[]): Note[] {
   if (ids.length === 0) return [];
   // Dedupe before chunking: a duplicate id straddling two chunk boundaries
@@ -769,8 +833,20 @@ function validateIsoDateValue(field: string, value: string): void {
   }
 }
 
-export function queryNotes(db: Database, opts: QueryOpts): Note[] {
-  validateLimitOffset(opts);
+/**
+ * Build the shared WHERE-clause conditions + bound params for the filter
+ * surface both `queryNotes` and `aggregateNotes` apply: tag membership
+ * (include/exclude/presence), link/broken-link presence, id-set scoping,
+ * exact path / path-prefix / extension, write-attribution, metadata
+ * (operator + plain-equality), and date range. Does NOT include the
+ * cursor-keyset predicate, ORDER BY, or LIMIT/OFFSET — those are
+ * query-shape concerns specific to `queryNotes`'s paginated-list contract
+ * and don't apply to an aggregate rollup (which spans every matching row,
+ * not a page of them). Extracted so aggregation reuses the EXACT filter
+ * semantics a normal query applies, rather than risking the two drifting
+ * apart.
+ */
+function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: string[]; params: SQLQueryBindings[] } {
   const conditions: string[] = [];
   const params: SQLQueryBindings[] = [];
 
@@ -851,6 +927,32 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
     );
   }
 
+  // Presence: has_broken_links (vault#555) — a dangling outbound wikilink or
+  // structured `links` target that never resolved. The `unresolved_wikilinks`
+  // table is created lazily (see wikilinks.ts:ensureUnresolvedTable) only when
+  // a link actually goes unresolved — a vault where nothing ever has won't
+  // have the table at all. Check existence first rather than reference it
+  // unconditionally: a read-only query filter shouldn't have the side effect
+  // of creating a table, and a bare `EXISTS`/`NOT EXISTS` against a missing
+  // table would throw "no such table" instead of the correct empty answer.
+  if (opts.hasBrokenLinks !== undefined) {
+    const unresolvedTableExists = db.prepare(
+      "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'unresolved_wikilinks'",
+    ).get() !== null;
+    if (!unresolvedTableExists) {
+      // No table yet means no note has ever had a broken link:
+      // hasBrokenLinks:true matches nothing; hasBrokenLinks:false is a
+      // no-op (every note already qualifies) — add no condition at all.
+      if (opts.hasBrokenLinks) conditions.push("0 = 1");
+    } else {
+      conditions.push(
+        opts.hasBrokenLinks
+          ? `EXISTS (SELECT 1 FROM unresolved_wikilinks ubl WHERE ubl.source_id = n.id)`
+          : `NOT EXISTS (SELECT 1 FROM unresolved_wikilinks ubl WHERE ubl.source_id = n.id)`,
+      );
+    }
+  }
+
   // ID set filter — used by `near` to push neighborhood scoping into SQL so
   // that LIMIT applies to the neighborhood, not the whole notes table.
   if (opts.ids !== undefined) {
@@ -1028,6 +1130,13 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
     }
   }
 
+  return { conditions, params };
+}
+
+export function queryNotes(db: Database, opts: QueryOpts): Note[] {
+  validateLimitOffset(opts);
+  const { conditions, params } = buildFilterConditions(db, opts);
+
   // ---- Cursor predicate (vault#313) ----
   //
   // Cursor mode is keyed on PRESENCE of `opts.cursor`, not truthiness
@@ -1176,6 +1285,133 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
   return fetchNotesByIdsOrdered(db, idRows.map((r) => r.id));
 }
 
+/**
+ * Aggregation / rollup query (top new-feature ask from a UX round — see
+ * `QueryOpts.aggregate` / `AggregateSpec` in types.ts). Applies the SAME
+ * filter surface `queryNotes` does — via the shared `buildFilterConditions`:
+ * tags, exclude-tags, presence filters, `ids`, path/extension, write-
+ * attribution, metadata, date range — BEFORE grouping, so aggregating over a
+ * filtered query rolls up exactly the notes that query would have listed.
+ *
+ * Tag-scope is NOT enforced here — core stays scope-unaware, same division
+ * every other core query surface keeps. A tag-scoped caller narrows via
+ * `opts.ids` to a pre-computed visible-note-id set (see `src/mcp-tools.ts`'s
+ * `aggregateVisibility` wiring for MCP and `src/routes.ts`'s REST aggregate
+ * branch — both filter to visible notes FIRST, then pass the resulting ids
+ * in here, exactly like a normal query's `near` neighborhood scoping does).
+ *
+ * `group_by: "tag"` groups by tag MEMBERSHIP, not partition: implemented as
+ * a JOIN against `note_tags`, so a note carrying N of the tags present in
+ * the filtered result set contributes to N separate group rows (the
+ * existing filter conditions, which reference `n.*`, compose unchanged
+ * against the join). Any other `group_by` value must be a declared
+ * `indexed: true` metadata field — reuses `requireIndexedField`, the same
+ * FIELD_NOT_INDEXED contract `metadata` operator queries and `order_by`
+ * use — and groups by its generated `meta_<field>` column.
+ *
+ * `op: "sum"` requires `field` — another indexed field whose declared
+ * storage type is `INTEGER` (the only indexable numeric shape; a bare
+ * `type: "number"` schema field is never indexed — see
+ * `indexed-fields.ts`'s `TYPE_MAP` — and a `TEXT`-backed field can't be
+ * summed). `op: "count"` ignores `field`.
+ *
+ * A note whose group_by value is absent/null collects into one
+ * `{group: null, ...}` row — standard SQL `GROUP BY` behavior, not silently
+ * dropped.
+ */
+export function aggregateNotes(db: Database, opts: QueryOpts): AggregateRow[] {
+  const spec = opts.aggregate;
+  if (!spec) {
+    throw new QueryError(
+      `aggregateNotes requires opts.aggregate`,
+      "INVALID_QUERY",
+      {
+        error_type: "invalid_query",
+        field: "aggregate",
+        hint: `pass { group_by, op } — group_by is an indexed metadata field or "tag"; op is "count" or "sum"`,
+      },
+    );
+  }
+  if (typeof spec.group_by !== "string" || spec.group_by.length === 0) {
+    throw new QueryError(
+      `aggregate.group_by is required — an indexed metadata field name, or "tag"`,
+      "INVALID_QUERY",
+      {
+        error_type: "invalid_query",
+        field: "aggregate.group_by",
+        got: spec.group_by,
+        hint: `pass an indexed metadata field name, or "tag"`,
+      },
+    );
+  }
+  if (spec.op !== "count" && spec.op !== "sum") {
+    throw new QueryError(
+      `invalid aggregate.op: ${JSON.stringify(spec.op)} — must be "count" or "sum"`,
+      "INVALID_QUERY",
+      { error_type: "invalid_query", field: "aggregate.op", got: spec.op, hint: `pass "count" or "sum"` },
+    );
+  }
+  if (spec.op === "sum" && (typeof spec.field !== "string" || spec.field.length === 0)) {
+    throw new QueryError(
+      `aggregate.field is required when aggregate.op is "sum"`,
+      "INVALID_QUERY",
+      {
+        error_type: "invalid_query",
+        field: "aggregate.field",
+        hint: "pass the indexed numeric metadata field to sum",
+      },
+    );
+  }
+
+  const { conditions, params } = buildFilterConditions(db, opts);
+
+  const groupByTag = spec.group_by === "tag";
+  let groupExpr: string;
+  let fromClause: string;
+  if (groupByTag) {
+    groupExpr = "nt.tag_name";
+    fromClause = "FROM notes n JOIN note_tags nt ON nt.note_id = n.id";
+  } else {
+    // `group_by` came from indexed_fields (validated via FIELD_NAME_RE at
+    // declaration time), so interpolating the column name is safe — same
+    // justification `orderBy`/`buildOperatorClause` use.
+    requireIndexedField(db, spec.group_by);
+    groupExpr = `"meta_${spec.group_by}"`;
+    fromClause = "FROM notes n";
+  }
+
+  let valueExpr: string;
+  if (spec.op === "count") {
+    valueExpr = "COUNT(*)";
+  } else {
+    const fieldInfo = requireIndexedField(db, spec.field!);
+    if (fieldInfo.sqliteType !== "INTEGER") {
+      throw new QueryError(
+        `aggregate.field "${spec.field}" is not numeric (declared sqlite type ${fieldInfo.sqliteType}) — "sum" requires a field declared type: "integer" or "boolean" in its tag schema`,
+        "INVALID_QUERY",
+        {
+          error_type: "invalid_query",
+          field: "aggregate.field",
+          got: spec.field,
+          hint: `sum requires a numeric indexed field (type: "integer"/"boolean"); "${spec.field}" is declared a non-numeric type`,
+        },
+      );
+    }
+    valueExpr = `SUM("meta_${spec.field}")`;
+  }
+
+  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  const sql = `
+    SELECT ${groupExpr} AS group_key, ${valueExpr} AS value
+    ${fromClause}
+    ${whereClause}
+    GROUP BY ${groupExpr}
+    ORDER BY ${groupExpr}
+  `;
+  const rows = db.prepare(sql).all(...params) as { group_key: string | number | null; value: number | null }[];
+  return rows.map((r) => ({ group: r.group_key, value: r.value ?? 0 }));
+}
+
 /**
  * Fetch full note rows for `ids`, preserving the input order, with tags
  * hydrated via ONE batched query per chunk (not one per note). Ids not
@@ -1238,6 +1474,7 @@ function toQueryHashInputs(opts: QueryOpts): QueryHashInputs {
     excludeTags: opts.excludeTags,
     hasTags: opts.hasTags,
     hasLinks: opts.hasLinks,
+    hasBrokenLinks: opts.hasBrokenLinks,
     path: opts.path,
     pathPrefix: opts.pathPrefix,
     extension: opts.extension,
@@ -2332,6 +2569,122 @@ export function getVaultStats(
   };
 }
 
+// ---- Vault map (front-door structural orientation) ----
+
+/** Shared bucket-expression for the top-level path segment: the text before
+ *  the first `/`, or the whole path when it has none. Applied to whichever
+ *  `path` column reference the caller substitutes in (`path` or `n.path`). */
+function pathBucketExpr(pathCol: string): string {
+  return `CASE WHEN instr(${pathCol}, '/') > 0 THEN substr(${pathCol}, 1, instr(${pathCol}, '/') - 1) ELSE ${pathCol} END`;
+}
+
+/**
+ * Compute the compact structural map `vault-info` surfaces for one-call
+ * orientation: total note count, every tag currently carried by at least one
+ * note with its membership count, and every top-level path "bucket" (the
+ * first `/`-delimited segment of `path`) with its note count — plus a count
+ * of notes with no path at all (excluded from `path_buckets`, nothing to
+ * bucket). Counts only, no content; three grouped-COUNT queries, safe on
+ * large vaults.
+ *
+ * `opts.tagFilter`, when the KEY IS PRESENT, restricts every count to notes
+ * reachable through that exact tag-name set (an `IN`-clause join through
+ * `note_tags`) — the scope-aware path a tag-scoped caller's already-expanded
+ * allowlist takes (server layer; core stays scope-unaware, it just filters
+ * by the plain tag-name list it's given). An empty array is a valid,
+ * meaningful filter — "nothing is in scope" — and short-circuits to an
+ * all-zero map WITHOUT falling through to the unfiltered/full-vault query;
+ * only OMITTING `tagFilter` entirely computes the vault-wide map.
+ */
+export function getVaultMap(
+  db: Database,
+  opts?: { tagFilter?: string[] },
+): VaultMap {
+  const hasFilter = opts !== undefined && opts.tagFilter !== undefined;
+  const tagFilter = opts?.tagFilter ?? [];
+
+  if (hasFilter && tagFilter.length === 0) {
+    return { total_notes: 0, tags: [], path_buckets: [], unfiled_notes: 0 };
+  }
+
+  if (hasFilter) {
+    const placeholders = tagFilter.map(() => "?").join(",");
+
+    const totalRow = db
+      .prepare(`SELECT COUNT(DISTINCT note_id) as c FROM note_tags WHERE tag_name IN (${placeholders})`)
+      .get(...tagFilter) as { c: number };
+
+    const tagRows = db
+      .prepare(
+        `SELECT tag_name AS name, COUNT(*) AS count
+         FROM note_tags
+         WHERE tag_name IN (${placeholders})
+         GROUP BY tag_name
+         ORDER BY count DESC, name ASC`,
+      )
+      .all(...tagFilter) as { name: string; count: number }[];
+
+    const bucketRows = db
+      .prepare(
+        `SELECT ${pathBucketExpr("n.path")} AS name, COUNT(DISTINCT n.id) AS count
+         FROM notes n
+         JOIN note_tags nt ON nt.note_id = n.id
+         WHERE nt.tag_name IN (${placeholders}) AND n.path IS NOT NULL
+         GROUP BY name
+         ORDER BY count DESC, name ASC`,
+      )
+      .all(...tagFilter) as { name: string; count: number }[];
+
+    const unfiledRow = db
+      .prepare(
+        `SELECT COUNT(DISTINCT n.id) as c
+         FROM notes n
+         JOIN note_tags nt ON nt.note_id = n.id
+         WHERE nt.tag_name IN (${placeholders}) AND n.path IS NULL`,
+      )
+      .get(...tagFilter) as { c: number };
+
+    return {
+      total_notes: totalRow.c,
+      tags: tagRows,
+      path_buckets: bucketRows,
+      unfiled_notes: unfiledRow.c,
+    };
+  }
+
+  const totalRow = db.prepare("SELECT COUNT(*) as c FROM notes").get() as { c: number };
+
+  const tagRows = db
+    .prepare(
+      `SELECT tag_name AS name, COUNT(*) AS count
+       FROM note_tags
+       GROUP BY tag_name
+       ORDER BY count DESC, name ASC`,
+    )
+    .all() as { name: string; count: number }[];
+
+  const bucketRows = db
+    .prepare(
+      `SELECT ${pathBucketExpr("path")} AS name, COUNT(*) AS count
+       FROM notes
+       WHERE path IS NOT NULL
+       GROUP BY name
+       ORDER BY count DESC, name ASC`,
+    )
+    .all() as { name: string; count: number }[];
+
+  const unfiledRow = db
+    .prepare("SELECT COUNT(*) as c FROM notes WHERE path IS NULL")
+    .get() as { c: number };
+
+  return {
+    total_notes: totalRow.c,
+    tags: tagRows,
+    path_buckets: bucketRows,
+    unfiled_notes: unfiledRow.c,
+  };
+}
+
 // ---- Bulk Operations ----
 
 export interface BulkNoteInput {

package/core/src/query-warnings.ts

@@ -34,6 +34,7 @@ import {
   type TagHierarchy,
 } from "./tag-hierarchy.js";
 import { chunkForInClause } from "./sql-in.js";
+import { escapeFtsToken } from "./search-query.js";
 
 export interface QueryWarning {
   code: string;
@@ -231,13 +232,15 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
  *   - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
  *     created here LAZILY and BEST-EFFORT (not part of the schema/
  *     migration — see the try/catch below) — the terms actually indexed,
- *     POST-tokenization/stemming (porter). A suggestion therefore
- *     sometimes reads as a stemmed form ("propoli" rather than "propolis")
- *     rather than the original dictionary word — an accepted tradeoff
- *     (documented in docs/HTTP_API.md) rather than maintaining a second,
- *     unstemmed index just for spelling suggestions.
+ *     POST-tokenization/stemming (porter). The closest CANDIDATE is found
+ *     against this stemmed vocabulary (small, deduped, cheap to
+ *     range-scan), but a stemmed candidate is never returned verbatim —
+ *     see `resolveSurfaceForm` below, which maps it back to a real word
+ *     before it's handed to the caller (vault#570 — a raw stem like
+ *     "propoli" or "cactu" isn't a word anyone would type).
  *   - tag names, when the caller passes `tagNames` (already cached by the
- *     caller's tag hierarchy — free to include).
+ *     caller's tag hierarchy — free to include, and already real words —
+ *     no de-stemming needed).
  *
  * `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
  * itself (not a separately-enabled module) so it's expected to be
@@ -263,6 +266,8 @@ export function computeSearchDidYouMean(
     const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
     if (tokens.length === 0) return undefined;
 
+    const tagNameSet = tagNames ? new Set(tagNames) : undefined;
+
     db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
     const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
     const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
@@ -284,14 +289,19 @@ export function computeSearchDidYouMean(
       const hi = clean.length + 2;
       const rows = rangeStmt.all(lo, hi) as { term: string }[];
       const candidates = new Set<string>(rows.map((r) => r.term));
-      if (tagNames) for (const t of tagNames) candidates.add(t);
+      if (tagNameSet) for (const t of tagNameSet) candidates.add(t);
 
       const suggestion = suggestSimilarTag(candidates, lower);
-      if (suggestion && suggestion.toLowerCase() !== lower) {
-        changed = true;
-        return suggestion;
-      }
-      return tok;
+      if (!suggestion || suggestion.toLowerCase() === lower) return tok;
+
+      changed = true;
+      // A literal tag name is already a real word — return it verbatim. A
+      // vocabulary hit, though, came out of the PORTER-STEMMED index, so
+      // it's usually a truncated stem ("cactu" for "cactus") rather than a
+      // word a user would ever type (vault#570) — recover the real surface
+      // form before handing it back.
+      if (tagNameSet?.has(suggestion)) return suggestion;
+      return resolveSurfaceForm(db, suggestion) ?? suggestion;
     });
 
     if (!changed) return undefined;
@@ -301,6 +311,44 @@ export function computeSearchDidYouMean(
   }
 }
 
+/**
+ * Map a porter-stemmed vocabulary term ("cactu") back to the real
+ * dictionary word a note actually contains ("cactus") — vault#570 (issue
+ * #570's search polish item). `notes_fts` is porter-tokenized, so MATCHing
+ * the bare stem finds any note carrying a word that stems to it (the stem
+ * came FROM `notes_fts_vocab` in the first place, so at least one match is
+ * guaranteed). From there, tokenize that note's own path/content text —
+ * unstemmed, done here in JS with the same punctuation-stripping rule used
+ * above — and return the first token that literally STARTS WITH the stem.
+ * This works because Porter's rules are almost entirely suffix-stripping:
+ * the stem is near-always a prefix of the original word. Irregular cases
+ * (a stem that isn't a literal prefix) are a known, accepted limitation —
+ * same class as the porter tokenizer's irregular-plural gap documented in
+ * docs/HTTP_API.md — and simply fall back to the raw stem via the caller's
+ * `?? suggestion`, never worse than the pre-fix behavior.
+ *
+ * Bounded to a handful of rows (`LIMIT 5`) and wrapped in try/catch same as
+ * the caller — a nicety, never worth risking the search response itself.
+ */
+function resolveSurfaceForm(db: Database, stem: string): string | undefined {
+  try {
+    const rows = db
+      .prepare("SELECT path, content FROM notes_fts WHERE notes_fts MATCH ? LIMIT 5")
+      .all(escapeFtsToken(stem)) as { path: string | null; content: string | null }[];
+    for (const row of rows) {
+      const text = `${row.path ?? ""} ${row.content ?? ""}`;
+      for (const raw of text.split(/[^\p{L}\p{N}]+/gu)) {
+        if (!raw) continue;
+        const lower = raw.toLowerCase();
+        if (lower.length > stem.length && lower.startsWith(stem)) return lower;
+      }
+    }
+    return undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 /**
  * `search_did_you_mean` warning (vault#551 WS2B) — wraps
  * {@link computeSearchDidYouMean}'s suggestion in the standard warning

package/core/src/schema-defaults.ts

@@ -57,7 +57,7 @@ export interface SchemaField {
    * `type_mismatch` warning that previously fired on every integer-shaped
    * field because the validator had no `"integer"` case. See vault#310.
    */
-  type?: "string" | "number" | "integer" | "boolean" | "array" | "object";
+  type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
   enum?: string[];
   description?: string;
   /**
@@ -369,6 +369,12 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
       return Array.isArray(value);
     case "object":
       return !!value && typeof value === "object" && !Array.isArray(value);
+    // `reference` (vault#typed-reference-field) validates like `string` —
+    // the write path (core/src/store.ts) separately resolves this value to
+    // a note and maintains a graph link; see tag-schemas.ts's
+    // `VALID_FIELD_TYPES` doc comment for the full contract.
+    case "reference":
+      return typeof value === "string";
   }
 }
 

package/core/src/seed-packs.test.ts

@@ -291,6 +291,20 @@ describe("getting-started pack", () => {
     expect(GETTING_STARTED_CONTENT).toContain("over MCP");
   });
 
+  test("Getting Started carries the journaling / agent-memory / search conventions (vault#555)", () => {
+    expect(GETTING_STARTED_CONTENT).toContain("## A few shapes worth reusing");
+    // Journaling: indexed entry_date + mood enum.
+    expect(GETTING_STARTED_CONTENT).toContain("entry_date");
+    expect(GETTING_STARTED_CONTENT).toContain("mood");
+    // Agent-memory pattern: thread + messages + the if_exists:"ignore" crash-replay recipe.
+    expect(GETTING_STARTED_CONTENT).toContain("#thread");
+    expect(GETTING_STARTED_CONTENT).toContain("#message");
+    expect(GETTING_STARTED_CONTENT).toContain('if_exists: "ignore"');
+    // Search one-liner: literal by default, search_mode:"advanced" for FTS syntax.
+    expect(GETTING_STARTED_CONTENT).toContain("literal by default");
+    expect(GETTING_STARTED_CONTENT).toContain('search_mode: "advanced"');
+  });
+
   test("default vault-description constants orient + tell the AI to self-replace them", () => {
     for (const d of [DEFAULT_VAULT_DESCRIPTION, IMPORTED_VAULT_DESCRIPTION]) {
       expect(d.length).toBeGreaterThan(100);

package/core/src/seed-packs.ts

@@ -392,10 +392,14 @@ description) so the next one picks up where you left off.
 ### If this vault already has content
 
 Some vaults arrive full — an import, a restore, months of use. Then your first
-job is to **learn it, not seed it**: \`vault-info { include_stats: true }\`,
-\`list-tags\`, read a sample (\`query-notes { search: "..." }\`), then reflect the
-shape back to the person and ask what's missing or wrong. Propose structure
-only once you can describe what's already there.
+job is to **learn it, not seed it**: \`vault-info\` — a single call now returns
+a compact \`map\` (every tag's note count, every top-level path's note count,
+total notes) alongside the schema catalog, so you get the shape of a strange
+vault in one round trip; add \`include_stats: true\` only if you also want the
+deeper monthly distribution. Then \`list-tags\` and read a sample
+(\`query-notes { search: "..." }\`), and reflect the shape back to the person
+and ask what's missing or wrong. Propose structure only once you can describe
+what's already there.
 
 ### Close the loop: describe the vault
 
@@ -424,7 +428,9 @@ Core moves you already have as MCP tools:
 - \`query-notes\` — by id/path, by tag, full-text \`search\`, or graph \`near\` a note.
 - \`list-tags\` / \`update-tag\` / \`delete-tag\` — manage the tag vocabulary + schemas.
 - \`find-path\` — shortest link path between two notes.
-- \`vault-info\` — refresh the live schema/stats projection any time.
+- \`vault-info\` — refresh the live schema projection any time; the base call
+  includes a compact \`map\` (tag counts, top-level path-bucket counts, total
+  notes) so you can orient in this one call — no flag needed.
 
 \`[[wikilinks]]\` in note content auto-link to the note at that path — use them
 freely; they resolve even if the target is created later.
@@ -481,6 +487,37 @@ support operator queries (\`metadata: { held_on: { gte: "..." } }\`) and
 \`order_by\`; all tags declaring the same field must agree on its \`type\` and
 \`indexed\` flag.
 
+## A few shapes worth reusing
+
+Testers of this vault independently reinvented these — start from them instead:
+
+- **Journaling.** One tag (\`#journal\`) with an indexed \`entry_date\`
+  (\`{ type: "string", indexed: true }\`, ISO date) and a \`mood\` enum
+  (\`{ type: "string", enum: [...] }\`). Indexing \`entry_date\` (see "Only
+  \`indexed: true\` fields support operator queries" above) is what makes
+  date-range queries (\`metadata: { entry_date: { gte: "2026-01-01" } }\`) and
+  \`order_by: "entry_date"\` work.
+- **Agent memory (thread + messages, crash-replay-safe).** Model a
+  conversation as one \`#thread\` note (metadata: \`status\`, \`cursor\`) plus many
+  \`#message\` notes linked to it (\`relationship: "in-thread"\`), each carrying
+  its own \`status\`. If your write loop can crash and replay the same message,
+  give it a stable \`path\` (e.g. \`Threads/<thread-id>/msg-<n>\`) and create it
+  with \`if_exists: "ignore"\` — a retry after a crash safely returns the
+  already-written message instead of creating a duplicate, no
+  query-before-write round trip needed.
+- **Search.** \`query-notes { search: "..." }\` is literal by default — your
+  text is escaped, not parsed as FTS5 syntax, so punctuation like "didn't" or
+  "18.6" just works. Pass \`search_mode: "advanced"\` only when you actually
+  want FTS5 boolean/phrase/prefix syntax.
+- **A front-door "Map" note.** Once a vault grows past a few dozen notes,
+  keep one note (path \`Map\`, or whatever fits) as a human-legible index —
+  the tags and top-level paths that matter and what each means, with
+  wikilinks into anchor notes. It's the "why" companion to \`vault-info\`'s
+  auto-computed \`map\` (tag/path-bucket counts, the "what's there"): the
+  counts tell a fresh AI the shape of the vault in one call, this note tells
+  it what that shape MEANS. Update it when the vault's structure shifts;
+  link to it from the vault description so it's easy to find.
+
 ## Write gotchas
 
 A few behaviors worth knowing before you write at scale: