@staix/vault-mem 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/README.md +147 -23
  3. package/package.json +5 -4
  4. package/src/amem/seam.ts +15 -6
  5. package/src/analysis-commands.ts +1 -1
  6. package/src/analysis-input.ts +24 -3
  7. package/src/analysis-reports.ts +1 -1
  8. package/src/analysis-types.ts +13 -1
  9. package/src/canon-tables.ts +190 -0
  10. package/src/cli-output.ts +15 -1
  11. package/src/cli.ts +3 -1
  12. package/src/commands-table.ts +7 -2
  13. package/src/config.ts +31 -3
  14. package/src/frontmatter.ts +10 -3
  15. package/src/links.ts +13 -1
  16. package/src/mcp-spec.ts +19 -11
  17. package/src/mcp-vault.ts +20 -12
  18. package/src/mcp.ts +4 -2
  19. package/src/org-promote.ts +29 -12
  20. package/src/promote.ts +25 -13
  21. package/src/read-commands.ts +1 -2
  22. package/src/read-output.ts +7 -5
  23. package/src/registry.ts +1 -0
  24. package/src/skills-mine.ts +5 -4
  25. package/src/spec-command-table.ts +10 -1
  26. package/src/spec-commands.ts +7 -3
  27. package/src/spec-gitlinks.ts +106 -0
  28. package/src/spec-output.ts +30 -0
  29. package/src/spec-source.ts +9 -3
  30. package/src/spec-validate.ts +6 -3
  31. package/src/spec-write.ts +469 -0
  32. package/src/ui/inbox.ts +28 -10
  33. package/src/ui/panel.js +26 -5
  34. package/src/ui/plugin.ts +38 -12
  35. package/src/ui/snapshot-cache.ts +123 -0
  36. package/src/vault-brief.ts +90 -0
  37. package/src/vault-index.ts +158 -21
  38. package/src/vault-lint.ts +53 -5
  39. package/src/vault-log.ts +3 -2
  40. package/src/vault-stats.ts +110 -5
  41. package/src/write-commands.ts +4 -3
  42. package/src/write-files.ts +72 -0
  43. package/src/write-operations.ts +175 -25
  44. package/src/write-output.ts +20 -9
  45. package/src/write-types.ts +8 -1
  46. package/src/write-validation.ts +46 -17
  47. package/src/writer.ts +145 -42
  48. package/src/yaml-quoted.ts +12 -2
@@ -1,4 +1,4 @@
1
- import { realPath } from "./platform.ts";
1
+ import { realPath, canonicalPath, isDenied } from "./platform.ts";
2
2
  import { Database } from "bun:sqlite";
3
3
  import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readSync, readdirSync, realpathSync } from "node:fs";
4
4
  import { createHash } from "node:crypto";
@@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve } from "node:path";
6
6
  import type { AnalysisSnapshot } from "./analysis-types.ts";
7
7
  import type { VaultSpec, SpecSourceSpec } from "./config.ts";
8
8
  import { parseFrontmatter, type FrontmatterDocument } from "./frontmatter.ts";
9
- import { normalizeTarget, noteKey, proseLines, resolveLinks, type LinkNote, type ResolvedLink } from "./links.ts";
9
+ import { normalizeTarget, noteKey, proseLines, resolveLinks, type LinkNote, type ResolvedLink, type SuggestedLink, type TitleSuggestion } from "./links.ts";
10
10
 
11
11
  export const MAX_NOTE_BYTES = 2 * 1024 * 1024;
12
12
  export const MAX_RESULTS = 500;
@@ -16,8 +16,12 @@ export const MAX_RESULTS = 500;
16
16
  * share the unversioned `<digest>.sqlite`; the first run of a newer version builds its own file once.
17
17
  */
18
18
  export const LINK_RESOLUTION = 2;
19
- /** Bump when tables or FTS columns change. Named per value like LINK_RESOLUTION. 2 = CJK bigram table; 3 = reference identifiers in FTS. */
20
- export const INDEX_SCHEMA = 3;
19
+ /**
20
+ * Bump when tables, FTS columns or the indexed text change. Named per value like LINK_RESOLUTION. 2 = CJK bigram
21
+ * table; 3 = reference identifiers in FTS; 4 = arXiv ids in the FTS text of reference notes, so a schema-3 cache
22
+ * built before that rebuilds once instead of answering without the id.
23
+ */
24
+ export const INDEX_SCHEMA = 4;
21
25
  const indexName = (root: string) => `${digest(root)}-links${LINK_RESOLUTION}-schema${INDEX_SCHEMA}.sqlite`;
22
26
  /**
23
27
  * unicode61 keeps each Hangul/CJK run as one token, so `사업단` misses `사업단은`. A side table holds the
@@ -51,6 +55,13 @@ export function splitTerms(words: string[]): { tokens: string[]; runs: string[]
51
55
  return { tokens, runs };
52
56
  }
53
57
  export const MAX_READ_BYTES = 64 * 1024;
58
+ /** A hint must beat this: a term that appears in most titles scores at or below zero, and one shared common word stays under it. */
59
+ export const MIN_SUGGESTION_SCORE = 0.5;
60
+ /**
61
+ * A link's target, label, heading and block are reported within this many bytes of their raw text, and a hint is
62
+ * looked up for at most the same bound, so a reported row and the hint for it never judge different text.
63
+ */
64
+ const LINK_TEXT_BYTES = 1024;
54
65
  export interface IndexedNote extends LinkNote { hash: string; description: string; type: string; domain: string }
55
66
  export interface NoteReadResult {
56
67
  path: string;
@@ -67,6 +78,7 @@ export interface NoteReadResult {
67
78
  headings?: string[];
68
79
  headingsTruncated?: boolean;
69
80
  }
81
+ export interface BriefPointer { path: string; title: string; type: string; description: string; bytes: number }
70
82
  export interface SearchOptions { limit?: number; type?: string; domain?: string }
71
83
  export interface SearchResult { path: string; title: string; description: string; type: string; domain: string; score: number; snippet: string }
72
84
  interface StoredNote extends IndexedNote { metadata: string; source: string }
@@ -163,6 +175,14 @@ export interface IndexSourceAdapter {
163
175
  }
164
176
  export class IndexCacheError extends Error {}
165
177
  export interface IndexOptions { memory?: boolean; adapter?: IndexSourceAdapter; reuseCache?: boolean }
178
+ /** Whitespace-separated tokens of a note source, frontmatter included; 0 for an empty source. */
179
+ const wordCount = (source: string) => { const trimmed = source.trim(); return trimmed ? trimmed.split(/\s+/u).length : 0; };
180
+ /** Note text size in bytes and words, with the number of notes that contributed. */
181
+ export interface NoteSize { notes: number; bytes: number; words: number }
182
+ /** One note that declares frontmatter `source`, with the byte size the ratio compares to its sources. */
183
+ export interface SourcedNote { path: string; bytes: number; source: unknown }
184
+ /** The ratio pass reads at most this many notes that declare a source, in path order. */
185
+ export const MAX_RATIO_NOTES = 10000;
166
186
  /** Derived cache only. No vault or application configuration files are written. */
167
187
  export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
168
188
  private readonly db: Database;
@@ -242,10 +262,13 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
242
262
  }
243
263
  close(): void { this.db.close(); }
244
264
  // ponytail: grams stores its bigram text (about 2.3x the CJK bytes); contentless with stable rowids would save it.
245
- /** Keep the stored body and link/analysis input unchanged; identifiers join only the FTS text. */
265
+ /**
266
+ * Keep the stored body and link/analysis input unchanged; identifiers join only the FTS text. The arXiv id goes in
267
+ * without its `vN` version suffix, since a caller holding `2502.04321` cannot tokenize the suffixed form.
268
+ */
246
269
  private searchBody(body: string, data: Record<string, unknown>): string {
247
270
  if (this.adapter || data.type !== "reference") return body;
248
- const identifiers = [text(data.doi), text(data.citekey)].filter(Boolean);
271
+ const identifiers = [text(data.doi), text(data.citekey), text(data.arxiv_id).replace(/v\d+$/, "")].filter(Boolean);
249
272
  return identifiers.length ? body + "\n" + identifiers.join("\n") : body;
250
273
  }
251
274
  private indexText(path: string, title: string, description: string, body: string): void {
@@ -260,7 +283,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
260
283
  return this.db.query<Omit<LinkNote, "aliases"> & { aliases: string }, []>("SELECT path,title,aliases,body FROM notes ORDER BY path").all().map(row => ({ ...row, aliases: JSON.parse(row.aliases) as string[] }));
261
284
  }
262
285
  /** Scan metadata each call; hash only new or metadata-changed files. Atomic cache update. */
263
- scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
286
+ scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean; denyPaths?: string[] } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
287
+ if (rebuild && options.denyPaths?.length) throw new Error("index: rebuild cannot use denyPaths restrictions; use scan(false, { denyPaths }) for a restricted refresh or vmem rebuild <vault> for an unrestricted rebuild");
264
288
  // Discovery/binding Git calls must never hold a SQLite writer lock.
265
289
  let preparedPaths: string[] | undefined;
266
290
  if (this.adapter) { this.adapter.validate(); preparedPaths = this.adapter.paths(); }
@@ -279,6 +303,13 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
279
303
  for (const path of paths) {
280
304
  this.adapter?.validatePath(path);
281
305
  const file = join(this.adapter ? this.root : notesDir, path);
306
+ if (options.denyPaths && (isDenied(file, options.denyPaths) || isDenied(canonicalPath(file), options.denyPaths))) {
307
+ // A restricted reader must not delete another reader's cached rows or invalidate link resolution.
308
+ // Preserve known rows without inspecting their metadata/source; a new denied file stays unindexed.
309
+ present.add(path);
310
+ if (previous.has(path)) live.add(path);
311
+ continue;
312
+ }
282
313
  const before = this.adapter ? specFileIO(path, "inspect", () => lstatSync(file)) : lstatSync(file);
283
314
  if (!before.isFile() || before.isSymbolicLink()) throw new Error(this.adapter ? `spec discovery ${path}: selected path changed; restore a plain file and retry` : `vault ${this.vault.id}: note path changed`);
284
315
  if (!this.adapter && before.size > MAX_NOTE_BYTES) throw new Error(`vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
@@ -419,6 +450,83 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
419
450
  .slice(snapshot ? snapshotOffset : 0, (snapshot ? snapshotOffset : 0) + (snapshot && allSnapshotRows ? 10000 : limit))
420
451
  .map(({ sourceHash: _, ...row }) => ({ ...row, title: boundedText(row.title, 512).content, snippet: boundedText(row.snippet, 1024).content, type: boundedText(row.type, 256).content, domain: boundedText(row.domain, 256).content }));
421
452
  }
453
+ /** Intent words are OR terms; reuse the search tokenizer, BM25 weights and CJK substring/strong-match rules.
454
+ * Materialize scores before grouping: FTS5 auxiliary functions cannot run in an aggregate context. No bodies leave SQL. */
455
+ briefMatches(words: string[]): { path: string; score: number }[] {
456
+ if (!words.length || words.length > 32 || words.some(word => !word || Buffer.byteLength(word) > 4096)) throw new Error("brief: expected 1..32 bounded search words; shorten the intent and path hints");
457
+ const { tokens, runs } = splitTerms(words), parts: string[] = [], params: string[] = [];
458
+ if (tokens.length) {
459
+ parts.push("SELECT path,bm25(fts,0,5,2,1) AS score,1 AS strong FROM fts WHERE fts MATCH ?");
460
+ params.push(tokens.join(" OR "));
461
+ }
462
+ if (runs.length) {
463
+ parts.push(`SELECT path,bm25(grams,0,5,2,1) AS score,
464
+ (path IN (SELECT path FROM fts WHERE fts MATCH ?) OR path IN (SELECT path FROM grams WHERE grams MATCH ?)) AS strong
465
+ FROM grams WHERE grams MATCH ?`);
466
+ params.push(runs.map(run => `"${run}"`).join(" OR "), `{title description} : (${runs.map(gramPhrase).join(" OR ")})`, runs.map(gramPhrase).join(" OR "));
467
+ }
468
+ const rows = this.db.query<{ path: string; score: number }, string[]>(`WITH hits AS MATERIALIZED (${parts.join(" UNION ALL ")})
469
+ SELECT path,min(score) AS score,max(strong) AS strength FROM hits GROUP BY path ORDER BY strength DESC,score,path LIMIT 10001`).all(...params);
470
+ if (rows.length > 10000) throw new Error("brief: more than 10000 matched notes; narrow the intent or use vmem search with --type/--domain filters");
471
+ return rows;
472
+ }
473
+ /** Enforce the remaining metadata read budget inside SQL, before values cross into JavaScript. */
474
+ briefPointer(path: string, maximum: number): BriefPointer | undefined {
475
+ if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 4096) throw new Error("brief: metadata read bound must be 1..4096 bytes; reduce the requested brief");
476
+ const bytes = "length(CAST(path AS BLOB))+length(CAST(title AS BLOB))+length(CAST(type AS BLOB))+length(CAST(description AS BLOB))";
477
+ return this.db.query<BriefPointer, [string, number]>(`SELECT path,title,type,description,${bytes} AS bytes FROM notes WHERE path=? AND (${bytes})<=?`).get(path, maximum) ?? undefined;
478
+ }
479
+ /** Resolved incoming and outgoing paths only, never link context/aliases or note bodies. */
480
+ briefNeighbors(paths: string[]): string[] {
481
+ if (!paths.length) return [];
482
+ if (paths.length > 5) throw new Error("brief: at most five seed notes; use the top ranked hits");
483
+ const slots = paths.map(() => "?").join(",");
484
+ const rows = this.db.query<{ path: string }, string[]>(`SELECT DISTINCT
485
+ CASE WHEN source IN (${slots}) THEN json_extract(data,'$.resolved') ELSE source END AS path
486
+ FROM links WHERE json_extract(data,'$.resolved') IS NOT NULL AND
487
+ (source IN (${slots}) OR json_extract(data,'$.resolved') IN (${slots})) ORDER BY path LIMIT 10001`).all(...paths, ...paths, ...paths);
488
+ if (rows.length > 10000) throw new Error("brief: more than 10000 linked neighbors; narrow the intent or use graph neighbors");
489
+ return rows.map(row => row.path);
490
+ }
491
+ /**
492
+ * The one indexed title closest to a dead link's text, or undefined when no title matches or the best match is at
493
+ * or below `minimum`. Same literal AND terms as `search`, matched in the title column only, because a hint should
494
+ * name a note whose title actually carries the link text. A title must carry every part of the text: a CJK run
495
+ * matches through the bigram table and a Latin term through unicode61, and the two scores are summed, because a part
496
+ * that many titles share scores near zero on its own and would otherwise hide a title that carries the whole text.
497
+ * The score is positive bm25, rounded to three decimals, so a larger value is a closer title. A hint is never an
498
+ * error: raw text past `LINK_TEXT_BYTES`, or text FTS5 cannot parse, returns undefined instead of throwing. The bound
499
+ * is read on the raw text, before normalization and never on the truncated target a row displays, so `links` and
500
+ * `lint` name the same hint for the same dead link. Read-only.
501
+ */
502
+ suggestTitle(text: string, minimum = MIN_SUGGESTION_SCORE): TitleSuggestion | undefined {
503
+ if (Buffer.byteLength(text) > LINK_TEXT_BYTES) return undefined;
504
+ // Link text is note content, so it can carry a control character. unicode61 already indexes one as a separator,
505
+ // and U+0000 kept inside a quoted term ends the string in SQLite and makes FTS5 throw "unterminated string".
506
+ const query = text.replace(/\p{Cc}/gu, " ").trim();
507
+ if (!query) return undefined;
508
+ const { tokens, runs } = splitTerms(query.split(/\s+/));
509
+ if (!tokens.length && !runs.length) return undefined;
510
+ const gramQuery = `{title} : (${runs.map(gramPhrase).join(" AND ")})`, tokenQuery = `{title} : (${tokens.join(" AND ")})`;
511
+ let row: TitleSuggestion | null | undefined;
512
+ try {
513
+ row = runs.length && tokens.length
514
+ ? this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, g.score + f.score AS score
515
+ FROM (SELECT path, -bm25(grams,0,5,2,1) AS score FROM grams WHERE grams MATCH ?) g
516
+ JOIN (SELECT path, -bm25(fts,0,5,2,1) AS score FROM fts WHERE fts MATCH ?) f ON f.path = g.path
517
+ JOIN notes n ON n.path = g.path ORDER BY score DESC, n.path LIMIT 1`).get(gramQuery, tokenQuery)
518
+ : runs.length
519
+ ? this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, -bm25(grams,0,5,2,1) AS score FROM grams JOIN notes n ON n.path=grams.path
520
+ WHERE grams MATCH ? ORDER BY score DESC, n.path LIMIT 1`).get(gramQuery)
521
+ : this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, -bm25(fts,0,5,2,1) AS score FROM fts JOIN notes n ON n.path=fts.path
522
+ WHERE fts MATCH ? ORDER BY score DESC, n.path LIMIT 1`).get(tokenQuery);
523
+ } catch {
524
+ // A hint is advisory: text FTS5 still cannot parse yields no suggestion instead of failing links and lint.
525
+ return undefined;
526
+ }
527
+ if (!row || !(row.score > minimum)) return undefined;
528
+ return { path: row.path, title: boundedText(row.title, 512).content, score: Math.round(row.score * 1000) / 1000 };
529
+ }
422
530
  private find(target: string): string {
423
531
  if (!target || Buffer.byteLength(target) > 1024) throw new Error("note: expected target <=1024 bytes");
424
532
  const key = normalizeTarget(target.replace(/^notes\//i, "").replace(/\.md$/i, ""));
@@ -435,6 +543,14 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
435
543
  if (notes.length !== 1) throw new Error(notes.length ? "note: ambiguous target" : "note: unknown target");
436
544
  return notes[0]!.path;
437
545
  }
546
+ /** Literal indexed-note source, with the byte bound checked before returning/decoding it. */
547
+ noteSource(path: string, maximum = MAX_NOTE_BYTES): string | undefined {
548
+ if (!Number.isInteger(maximum) || maximum < 1 || maximum > MAX_NOTE_BYTES) throw new Error(`note source maximum: expected 1..${MAX_NOTE_BYTES} bytes; choose a bound within the note reader limit`);
549
+ const literal = path.startsWith("notes/") ? path.slice(6) : path;
550
+ const row = this.db.query<{ source: string | null }, [number, string]>("SELECT CASE WHEN length(CAST(source AS BLOB))<=? THEN source ELSE NULL END AS source FROM notes WHERE path=?").get(maximum, literal);
551
+ if (row?.source === null) throw Object.assign(new Error(`note source exceeds the ${maximum}-byte remaining read budget; split the note or reduce configured tables before retrying`), { code: "V09" });
552
+ return row?.source;
553
+ }
438
554
  show(target: string, maxBytes = 16384, section?: string, offset?: number): NoteReadResult {
439
555
  if (!Number.isInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_READ_BYTES) throw new Error(`max-bytes: expected integer 1..${MAX_READ_BYTES}`);
440
556
  if (section !== undefined && (typeof section !== "string" || !section.trim() || /[\r\n]/.test(section) || Buffer.byteLength(section) > 1024)) throw new Error("section: expected one exact H2 heading <=1024 bytes");
@@ -450,7 +566,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
450
566
  if (!page.truncated) return { path, hash: note.hash, ...page, ...metadata };
451
567
  }
452
568
  // Only the derived source is loaded; section discovery never rescans a note file.
453
- const source = this.db.query<{ source: string }, [string]>("SELECT source FROM notes WHERE path=?").get(path)!.source;
569
+ const source = this.noteSource(path);
570
+ if (source === undefined) throw new Error("note: source disappeared or exceeded the read bound during paging; refresh and reread the note");
454
571
  const body = parseFrontmatter(source).body, sections = h2Sections(body);
455
572
  if (section !== undefined) {
456
573
  const found = sections.filter(item => item.name === section);
@@ -480,10 +597,15 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
480
597
  return { notes, total, ...(rows.length > max ? { nextCursor: notes.at(-1)!.path } : {}) };
481
598
  }
482
599
  /**
483
- * Note counts by type, domain and frontmatter `status` ("(none)" when empty), and how many notes have an mtime at or
484
- * after each cutoff (epoch ms). Streams rows; returns no note text.
600
+ * Note counts by type, domain and frontmatter `status` ("(none)" when empty), how many notes have an mtime at or
601
+ * after each cutoff (epoch ms), note text size in bytes and words by type and domain, and the notes that declare a
602
+ * frontmatter `source` (at most `sourcedLimit`, in path order) as the input of the note-to-source ratio, with
603
+ * `truncatedSourced` counting the sourced notes that cap kept out. Streams rows; returns no note text.
485
604
  */
486
- noteStats(cutoffs: readonly number[]): { total: number; byType: Record<string, number>; byDomain: Record<string, number>; byStatus: Record<string, number>; modified: number[] } {
605
+ noteStats(cutoffs: readonly number[], sourcedLimit = MAX_RATIO_NOTES): {
606
+ total: number; byType: Record<string, number>; byDomain: Record<string, number>; byStatus: Record<string, number>; modified: number[];
607
+ size: { total: NoteSize; byType: Record<string, NoteSize>; byDomain: Record<string, NoteSize> }; sourced: SourcedNote[]; truncatedSourced: number;
608
+ } {
487
609
  const add = (map: Map<string, number>, value: string, count = 1) => { const key = value ? boundedText(value, 256).content : "(none)"; map.set(key, (map.get(key) ?? 0) + count); };
488
610
  const sorted = (map: Map<string, number>) => Object.fromEntries([...map].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
489
611
  const grouped = (column: "type" | "domain") => {
@@ -491,21 +613,32 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
491
613
  for (const row of this.db.query<{ value: string; count: number }, []>(`SELECT ${column} AS value, count(*) AS count FROM notes GROUP BY ${column}`).all()) add(map, row.value, row.count);
492
614
  return sorted(map);
493
615
  };
494
- // Status is not a column: stream each stored source once, parsing only its frontmatter.
495
- const byStatus = new Map<string, number>(), modified = cutoffs.map(() => 0);
496
- let total = 0;
497
- const rows = this.db.prepare<{ metadata: string; source: string }, []>("SELECT metadata,source FROM notes");
616
+ const size = { total: { notes: 0, bytes: 0, words: 0 }, byType: new Map<string, NoteSize>(), byDomain: new Map<string, NoteSize>() };
617
+ const grow = (map: Map<string, NoteSize>, value: string, bytes: number, words: number) => {
618
+ const key = value ? boundedText(value, 256).content : "(none)", item = map.get(key) ?? { notes: 0, bytes: 0, words: 0 };
619
+ map.set(key, { notes: item.notes + 1, bytes: item.bytes + bytes, words: item.words + words });
620
+ };
621
+ const sortedSize = (map: Map<string, NoteSize>) => Object.fromEntries([...map].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
622
+ // Status, size and `source` are not columns: stream each stored source once, parsing only its frontmatter.
623
+ const byStatus = new Map<string, number>(), modified = cutoffs.map(() => 0), sourced: SourcedNote[] = [];
624
+ let total = 0, truncatedSourced = 0;
625
+ const rows = this.db.prepare<{ metadata: string; source: string; path: string; type: string; domain: string }, []>("SELECT metadata,source,path,type,domain FROM notes ORDER BY path");
498
626
  try {
499
627
  for (const row of rows.iterate()) {
500
628
  total++;
501
- let status: unknown;
502
- try { status = parseFrontmatter(row.source).data.status; } catch { /* counted as (none) */ }
503
- add(byStatus, text(status));
629
+ const bytes = Buffer.byteLength(row.source), words = wordCount(row.source);
630
+ size.total = { notes: size.total.notes + 1, bytes: size.total.bytes + bytes, words: size.total.words + words };
631
+ grow(size.byType, row.type, bytes, words); grow(size.byDomain, row.domain, bytes, words);
632
+ let data: Record<string, unknown> = {};
633
+ try { data = parseFrontmatter(row.source).data; } catch { /* counted as (none) and skipped by the ratio pass */ }
634
+ add(byStatus, text(data.status));
635
+ if (data.source !== undefined) { if (sourced.length < sourcedLimit) sourced.push({ path: row.path, bytes, source: data.source }); else truncatedSourced++; }
504
636
  const mtime = stampMtime(row.metadata);
505
637
  cutoffs.forEach((cutoff, i) => { if (mtime !== null && mtime >= cutoff) modified[i]!++; });
506
638
  }
507
639
  } finally { rows.finalize(); }
508
- return { total, byType: grouped("type"), byDomain: grouped("domain"), byStatus: sorted(byStatus), modified };
640
+ return { total, byType: grouped("type"), byDomain: grouped("domain"), byStatus: sorted(byStatus), modified,
641
+ size: { total: size.total, byType: sortedSize(size.byType), byDomain: sortedSize(size.byDomain) }, sourced, truncatedSourced };
509
642
  }
510
643
  /** Links that resolve to the target note, ordered by source. */
511
644
  backlinks(target: string, limit = 50): ResolvedLink[] {
@@ -553,8 +686,12 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
553
686
  } finally { rows.finalize(); }
554
687
  return out;
555
688
  }
556
- links(target: string, limit = 50): ResolvedLink[] {
689
+ links(target: string, limit = 50): SuggestedLink[] {
557
690
  const path = this.find(target);
558
- return this.db.query<{ data: string }, [string, number]>("SELECT data FROM links WHERE source=? ORDER BY ordinal LIMIT ?").all(path, resultLimit(limit)).map(row => JSON.parse(row.data) as ResolvedLink).map(link => ({ ...link, target: boundedText(link.target, 1024).content, label: link.label === null ? null : boundedText(link.label, 1024).content, heading: link.heading === null ? null : boundedText(link.heading, 1024).content, block: link.block === null ? null : boundedText(link.block, 1024).content }));
691
+ return this.db.query<{ data: string }, [string, number]>("SELECT data FROM links WHERE source=? ORDER BY ordinal LIMIT ?").all(path, resultLimit(limit)).map(row => JSON.parse(row.data) as ResolvedLink).map(link => ({
692
+ ...link, target: boundedText(link.target, LINK_TEXT_BYTES).content, label: link.label === null ? null : boundedText(link.label, LINK_TEXT_BYTES).content, heading: link.heading === null ? null : boundedText(link.heading, LINK_TEXT_BYTES).content, block: link.block === null ? null : boundedText(link.block, LINK_TEXT_BYTES).content,
693
+ // The hint reads the raw text, never the bounded `target` above: a link past the bound has to miss here and in
694
+ // lint alike, or one surface names a suggestion the other, reading the whole text, cannot.
695
+ ...(link.resolved === null ? { suggested: this.suggestTitle(link.target) ?? null } : {}) }));
559
696
  }
560
697
  }
package/src/vault-lint.ts CHANGED
@@ -9,8 +9,10 @@ const LEGACY = new Set("CREATE UPDATE MIGRATE REFACTOR MERGE MOVE RELOCATE RENAM
9
9
  const CODE_TITLES: Record<LintCode, string> = {
10
10
  L01: "L01 — dead wiki-link", L02: "L02 — 필수 frontmatter / 허용값", L03: "L03 — orphan",
11
11
  L09: "L09 — 로그 포맷", L10: "L10 — project 값", L11: "L11 — graph report staleness", L12: "L12 — island community", L13: "L13 — filename", L14: "L14 — source paths",
12
+ L15: "L15 — proven confidence의 source",
12
13
  };
13
14
  const truth = yamlTruthy;
15
+ const emptySource = (value: unknown) => !truth(value) || typeof value === "string" && !value.trim();
14
16
  /** Python scalar formatting matters to per-code diagnostic parity. */
15
17
  function py(x: unknown): string {
16
18
  const scalar = scalarText(x); if (scalar !== undefined) return scalar;
@@ -37,6 +39,43 @@ function links(value: unknown): string[] {
37
39
  return values.flatMap(v => typeof v === "string" ? [...v.matchAll(/\[\[([^\]|#]+)(?:[|#][^\]]*)?\]\]/g)].map(m => m[1]!.trim()) : []);
38
40
  }
39
41
  const stem = (s: string) => s.trim().replace(/\.md$/, "").split("/").at(-1)!;
42
+ /** Wiki-link texts L01 checks in one note, in the order it reports them: body links, then the link fields. */
43
+ function noteLinkTexts(source: string): string[] {
44
+ const { fm, body } = legacyDocument(source);
45
+ return [...links(body), ...FIELDS.flatMap(field => links(fm?.[field]))];
46
+ }
47
+ /**
48
+ * A snapshot record entry read as data only. Link text and note path reach these lookups unchanged, and a plain
49
+ * object answers `constructor`, `toString` or `__proto__` from `Object.prototype`; a snapshot another caller built
50
+ * can still be a plain record, so inherited members must never resolve as a hint or as warnings.
51
+ */
52
+ function ownValue<T>(record: Record<string, T> | undefined, key: string): T | undefined {
53
+ return record !== undefined && Object.hasOwn(record, key) ? record[key] : undefined;
54
+ }
55
+ /** Unique dead link texts one lint run computes hints for; past it the report says so instead of reading as a miss. */
56
+ export const DEAD_LINK_HINT_CAP = 1000;
57
+ /**
58
+ * The dead link texts a snapshot can be hinted for, deduplicated in note and link order and capped at `maximum`.
59
+ * Captured while the index is open (src/analysis-input.ts), because the lookup needs the title search and
60
+ * `lintDetails` stays a pure transform.
61
+ */
62
+ export function deadLinkTexts(notes: readonly { path: string; source: string }[], maximum = DEAD_LINK_HINT_CAP): string[] {
63
+ const stems = new Set(notes.map(note => note.path.replace(/^notes\//, "").slice(0, -3))), texts = new Set<string>();
64
+ for (const note of notes) {
65
+ for (const text of noteLinkTexts(note.source)) if (!stems.has(stem(text))) texts.add(text);
66
+ if (texts.size >= maximum) break;
67
+ }
68
+ return [...texts].slice(0, maximum);
69
+ }
70
+ /**
71
+ * The texts a hint pass attempts, the bound it worked within and how many unique texts that bound left out. Counting
72
+ * the remainder needs the whole traversal, because a reader that knows only the cap cannot tell an attempted miss from
73
+ * a dead link whose hint was never computed.
74
+ */
75
+ export function deadLinkScan(notes: readonly { path: string; source: string }[], maximum = DEAD_LINK_HINT_CAP): { texts: string[]; cap: number; skipped: number } {
76
+ const all = deadLinkTexts(notes, Number.POSITIVE_INFINITY), texts = all.slice(0, maximum);
77
+ return { texts, cap: maximum, skipped: all.length - texts.length };
78
+ }
40
79
  function graphChecks(snapshot: AnalysisSnapshot): { l11: string[]; l12: string[]; info: string } {
41
80
  const l11: string[] = [], l12: string[] = [], names = Object.keys(snapshot.reports).filter(s => /^graph-report-.*\.md$/.test(s)).sort();
42
81
  const latest = names.at(-1);
@@ -74,19 +113,26 @@ function graphChecks(snapshot: AnalysisSnapshot): { l11: string[]; l12: string[]
74
113
  export function lintVault(snapshot: AnalysisSnapshot, only?: string): LintReport { return lintDetails(snapshot, only).report; }
75
114
  /** The report plus what only its markdown carries: the note count, check titles and per-check info lines, for the CLI text view. */
76
115
  export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report: LintReport; notes: number; titles: Record<LintCode, string>; info: Partial<Record<LintCode, string>> } {
77
- const findings: Record<LintCode,string[]> = {L01:[],L02:[],L03:[],L09:[],L10:[],L11:[],L12:[],L13:[],L14:[]};
116
+ const findings: Record<LintCode,string[]> = {L01:[],L02:[],L03:[],L09:[],L10:[],L11:[],L12:[],L13:[],L14:[],L15:[]};
78
117
  const notes = [...snapshot.notes].sort((a,b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0).map(n => ({...n,...legacyDocument(n.source),stem: n.path.replace(/^notes\//, "").slice(0,-3)}));
79
118
  const stems = new Set(notes.map(n => n.stem)), inlinks = new Map(notes.map(n => [n.stem,0]));
119
+ // A text the hint pass never reached was not looked up, so its line must not read as a title that merely did not match.
120
+ const bound = snapshot.linkHintBound, attempted = bound ? new Set(bound.attempted) : undefined;
80
121
  let superseded = 0;
81
122
  for (const note of notes) {
82
123
  const rel = `notes/${note.stem}.md`, prefix = `\`${rel}\`: `, fm = note.fm;
83
- for(const message of snapshot.sourceWarnings?.[note.path] ?? [])findings.L14.push(prefix+message);
124
+ for (const message of ownValue(snapshot.sourceWarnings, note.path) ?? []) findings.L14.push(prefix+message);
84
125
  const allLinks: [string,string][] = links(note.body).map(t => ["body",t]);
85
126
  if (fm) for (const field of FIELDS) for (const t of links(fm[field])) allLinks.push([field,t]);
86
127
  for (const [field,t] of allLinks) {
87
128
  const target = stem(t);
88
129
  if (stems.has(target)) { if (target !== note.stem) inlinks.set(target,inlinks.get(target)!+1); }
89
- else findings.L01.push(prefix + `\`[[${t}]]\` (${field}) → 대상 노트 없음`);
130
+ else {
131
+ const hint = ownValue(snapshot.linkHints, t);
132
+ const suffix = hint ? ` — suggested \`${hint.path}\` (score ${hint.score.toFixed(3)})`
133
+ : bound && !attempted!.has(t) ? ` — hint not computed (cap ${bound.cap})` : "";
134
+ findings.L01.push(prefix + `\`[[${t}]]\` (${field}) → 대상 노트 없음` + suffix);
135
+ }
90
136
  }
91
137
  if (fm === null) { findings.L02.push(prefix + "frontmatter 없음/파싱 실패"); continue; }
92
138
  if (fm.status === "superseded") superseded++;
@@ -97,6 +143,7 @@ export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report
97
143
  if (fm.type === "moc") { if (!truth(fm.description)) findings.L02.push(prefix + "`type: moc` 노트는 `description` 필수"); }
98
144
  else if (!truth(fm.topics)) findings.L02.push(prefix + "`topics` 없음/빈 배열");
99
145
  for (const [key,allowed] of [["confidence",CONFIDENCE],["status",STATUS]] as const) if (fm[key] !== undefined && fm[key] !== null && !allowed.includes(py(fm[key]))) findings.L02.push(prefix + `\`${key}: ${py(fm[key])}\` 허용값 아님 (${allowed.join(" | ")})${key === "status" ? " (status는 노트 생애주기이지 사업 진행 상태가 아님 — 작업 상태는 본문에)" : ""}`);
146
+ if (py(fm.confidence) === "proven" && emptySource(fm.source)) findings.L15.push(prefix + "`confidence: proven`에 `source` 없음 — 근거(memory id·URL·workspace 경로)를 추가하거나 confidence를 낮추기");
100
147
  const project = fm.project;
101
148
  if (typeof project === "string" && project.trim()) {
102
149
  const targets = links(project);
@@ -120,11 +167,12 @@ export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report
120
167
  }
121
168
  if (data > 20000) findings.L09.push(`log ${data} 라인 > 20000 — 수동 아카이브 권장`);
122
169
  const graph = graphChecks(snapshot); findings.L11 = graph.l11; findings.L12 = graph.l12;
123
- if (only) for (const code of ["L01","L02","L03","L10","L13","L14"] as const) findings[code] = findings[code].filter(s => s.startsWith(`\`${only}\``));
170
+ if (only) for (const code of ["L01","L02","L03","L10","L13","L14","L15"] as const) findings[code] = findings[code].filter(s => s.startsWith(`\`${only}\``));
124
171
  const errors = findings.L01.length + findings.L02.length + findings.L13.length;
125
- const warnings = findings.L03.length + findings.L09.length + findings.L10.length + findings.L11.length + findings.L12.length + findings.L14.length;
172
+ const warnings = findings.L03.length + findings.L09.length + findings.L10.length + findings.L11.length + findings.L12.length + findings.L14.length + findings.L15.length;
126
173
  const summary = `SUMMARY errors=${errors} warnings=${warnings}`;
127
174
  const extra: Partial<Record<LintCode,string>> = {
175
+ L01: bound ? `정보: 힌트 상한 ${bound.cap}개 텍스트 초과 — ${bound.skipped}개 dead link 텍스트는 힌트 미계산 (미탐 아님)` : "",
128
176
  L02: `정보: \`status: superseded\` ${superseded}건${superseded === 0 ? " — supersede 프로토콜 미가동" : ""}`,
129
177
  L03: "hub MOC `notes/index.md`는 설계상 예외",
130
178
  L09: snapshot.log === null ? "정보: log 없음" : `정보: ${legacy ? `legacy TYPE ${legacy}건 (정규화표 등재), 최종 사용 ${last}, 신규 발생 없음` : "legacy TYPE 0건"}`,
package/src/vault-log.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { mutationVault } from "./config.ts";
1
2
  import { compileLogFilter } from "./log-filter.ts";
2
3
  import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, openSync, readSync, realpathSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
3
4
  import { join } from "node:path";
@@ -87,7 +88,7 @@ export function logBinding(vault: VaultSpec, config: VmemConfig): LogBinding {
87
88
  /** True when the registered vault still equals the binding: spec, path policy and root identity. Never throws. */
88
89
  export function matchesBinding(config: VmemConfig, id: string, binding: LogBinding): boolean {
89
90
  const vault = config.vaults.find(v => v.id === id);
90
- if (!vault || JSON.stringify(vault) !== JSON.stringify(binding.vault) || JSON.stringify(config.denyPaths ?? []) !== JSON.stringify(binding.denyPaths) || JSON.stringify(config.piiPatterns ?? []) !== JSON.stringify(binding.piiPatterns)) return false;
91
+ if (!vault || JSON.stringify(mutationVault(vault)) !== JSON.stringify(mutationVault(binding.vault)) || JSON.stringify(config.denyPaths ?? []) !== JSON.stringify(binding.denyPaths) || JSON.stringify(config.piiPatterns ?? []) !== JSON.stringify(binding.piiPatterns)) return false;
91
92
  try { const identity = lstatSync(vault.path); return Number(identity.dev) === binding.dev && Number(identity.ino) === binding.ino; } catch { return false; }
92
93
  }
93
94
  export function appendLog(home: string, request: LogRequest, options: { env?: NodeJS.ProcessEnv; now?: Date; binding?: LogBinding } = {}): LogReceipt {
@@ -108,7 +109,7 @@ export function appendLog(home: string, request: LogRequest, options: { env?: No
108
109
  const rootIdentity = lstatSync(ctx.vault.path);
109
110
  if (options.binding) {
110
111
  const bound = options.binding;
111
- if (JSON.stringify(ctx.vault) !== JSON.stringify(bound.vault) || rootIdentity.dev !== bound.dev || rootIdentity.ino !== bound.ino || JSON.stringify(ctx.config.denyPaths ?? []) !== JSON.stringify(bound.denyPaths) || JSON.stringify(ctx.config.piiPatterns ?? []) !== JSON.stringify(bound.piiPatterns)) refuse("LOG_DRIFT", "vault identity or policy changed after mutation; log not appended");
112
+ if (JSON.stringify(mutationVault(ctx.vault)) !== JSON.stringify(mutationVault(bound.vault)) || rootIdentity.dev !== bound.dev || rootIdentity.ino !== bound.ino || JSON.stringify(ctx.config.denyPaths ?? []) !== JSON.stringify(bound.denyPaths) || JSON.stringify(ctx.config.piiPatterns ?? []) !== JSON.stringify(bound.piiPatterns)) refuse("LOG_DRIFT", "vault identity or policy changed after mutation; log not appended");
112
113
  }
113
114
  const formatted = formatLogEvent(request.event, { ...ctx, env: options.env ?? process.env, now: options.now });
114
115
  if (!formatted.line || formatted.issues.some(issue => issue.severity === "error")) { result.issues.push(...formatted.issues); return result; }
@@ -1,28 +1,129 @@
1
+ import { resolve } from "node:path";
2
+ import { MAX_AUX_BYTES, analysisReadWithSize } from "./analysis-input.ts";
1
3
  import { localDate } from "./local-date.ts";
2
4
  import { LOG_TYPES } from "./log-format.ts";
5
+ import { canonicalPath, isDenied } from "./platform.ts";
3
6
  import { loadRegistry } from "./registry.ts";
4
- import { VaultIndex } from "./vault-index.ts";
7
+ import { sourceReferences, sourceWorkspaceRoot, type SourceReference } from "./source.ts";
8
+ import { VaultIndex, type NoteSize, type SourcedNote } from "./vault-index.ts";
5
9
  import { readLog } from "./vault-log.ts";
6
10
  import type { WriteIssue } from "./write-types.ts";
7
11
  import type { VaultSpec } from "./config.ts";
8
12
 
9
13
  export const STATS_DEFAULT_DAYS = 30;
10
14
  export const STATS_MAX_DAYS = 3650;
15
+ /** Source files the ratio pass reads per call, and how many source bytes it counts in total. */
16
+ export const MAX_RATIO_SOURCES = 10000;
17
+ export const MAX_RATIO_SOURCE_BYTES = 64 * 1024 * 1024;
18
+ /** The caps one ratio call works within; the defaults are the two bounds above, and a test lowers them. */
19
+ export interface RatioLimits { sources: number; bytes: number }
20
+ export const RATIO_LIMITS: RatioLimits = { sources: MAX_RATIO_SOURCES, bytes: MAX_RATIO_SOURCE_BYTES };
21
+ /** One bounded source read; `noteSourceRatio` takes one so a test can count the reads a pass really makes. */
22
+ export type SourceFileRead = (file: string, maximum: number) => { bytes: number } | null;
23
+ /** Notes listed as the largest, and the ratio above which a note is larger than the sources it cites. */
24
+ export const RATIO_LARGEST_SHOWN = 5;
25
+ export const RATIO_OVER_ONE = 1;
26
+ export interface VaultRatio {
27
+ /** Notes whose sources could be read and total more than zero bytes. */
28
+ notes: number;
29
+ /** Source files read within the per-call bounds; a zero-byte file is read and counted but leaves its note out of the ratio. */
30
+ sources: number;
31
+ /** Source references left out of the ratio; `skippedNonFile` of them are not workspace files at all. */
32
+ skipped: number;
33
+ skippedNonFile: number;
34
+ /**
35
+ * Notes left out because their readable sources total zero bytes. Such a note has no ratio, and counting it here
36
+ * keeps `skipped` a count of source references: its sources were read and are already in `sources`.
37
+ */
38
+ emptySourceNotes: number;
39
+ /**
40
+ * Sourced notes the index kept out of the pass (`MAX_RATIO_NOTES`), counted rather than dropped silently. The
41
+ * index stops collecting at the cap, so the median, p90 and the rest describe the notes it returned, not all of them.
42
+ */
43
+ truncatedNotes: number;
44
+ /** Median and nearest-rank p90 of the per-note ratio: note bytes over the bytes of its readable sources. */
45
+ median: number | null;
46
+ p90: number | null;
47
+ /** Notes whose ratio is above `RATIO_OVER_ONE`, so the note is larger than its sources. */
48
+ overOne: number;
49
+ largest: { path: string; ratio: number }[];
50
+ }
11
51
  export interface VaultStats {
12
52
  vault: string;
13
53
  /** Local calendar date the windows end on (today included). */
14
54
  asOf: string;
55
+ /** Note text size in bytes and words, by type and domain; "(none)" groups an empty type or domain. */
56
+ size: { total: NoteSize; byType: Record<string, NoteSize>; byDomain: Record<string, NoteSize> };
57
+ /** How large the notes are next to the workspace files they cite. */
58
+ ratio: VaultRatio;
15
59
  notes: { total: number; byType: Record<string, number>; byDomain: Record<string, number>; byStatus: Record<string, number> };
16
60
  /** Notes whose indexed mtime falls within the last 7 and 30 local calendar days, today included. */
17
61
  modified: { last7Days: number; last30Days: number };
18
62
  /** Log events dated on or after `since` (local date), by type; types outside the known set count as "(other)". */
19
63
  log: { days: number; since: string; total: number | null; byType: Record<string, number> | null; warnings: string[]; issues: WriteIssue[] };
20
64
  }
65
+ const round3 = (value: number) => Math.round(value * 1000) / 1000;
66
+ const middle = (sorted: readonly number[]) => {
67
+ const half = sorted.length >> 1;
68
+ return sorted.length % 2 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2;
69
+ };
70
+ /** Nearest rank: the smallest sorted value that at least `p` percent of the values do not exceed. */
71
+ const percentile = (sorted: readonly number[], p: number) => sorted[Math.min(sorted.length, Math.max(1, Math.ceil((p / 100) * sorted.length))) - 1]!;
72
+ /**
73
+ * Note-to-source byte ratio over notes that declare a frontmatter `source`. Each entry is one note: its own byte size
74
+ * over the summed size of the sources that could be read. The reader is `analysisReadWithSize`, so every file keeps its
75
+ * bound, no link is followed, and the bytes counted are the file's own on-disk size rather than the size of its decoded
76
+ * text, which overstates a file that is not UTF-8. The caps bound the I/O, not just the counted total: a read asks only
77
+ * for the bytes the budget has left, the reader refuses a larger file from its fstat size without reading it, and once
78
+ * the budget is spent no further file is opened. Every read attempt, successful or not, counts against the file cap, so
79
+ * a run of failures or over-budget files cannot outrun it. A memory id, a URL, a denied, missing or oversized path and
80
+ * anything the caps leave out is a skipped source reference, which keeps the pass read-only and bounded; a note whose
81
+ * readable sources total zero bytes has no ratio and is counted in `emptySourceNotes` instead, so `skipped` stays a
82
+ * count of references and never repeats a file that was read. The per-note truncation count belongs to the index, so
83
+ * `vaultStats` adds it to this pass's result.
84
+ */
85
+ export function noteSourceRatio(rows: readonly SourcedNote[], root: string | undefined, denyPaths: string[], limits: RatioLimits = RATIO_LIMITS, readFile: SourceFileRead = analysisReadWithSize): Omit<VaultRatio, "truncatedNotes"> {
86
+ const ratios: { path: string; ratio: number }[] = [];
87
+ let sources = 0, skipped = 0, skippedNonFile = 0, emptySourceNotes = 0, bytesRead = 0, attempts = 0;
88
+ for (const row of rows) {
89
+ let refs: SourceReference[];
90
+ try { refs = sourceReferences(row.source); } catch { skipped++; continue; }
91
+ let size = 0, used = 0;
92
+ for (const ref of refs) {
93
+ if (ref.kind !== "path") { skipped++; skippedNonFile++; continue; }
94
+ const remaining = limits.bytes - bytesRead;
95
+ // Stop before opening anything once a cap is used up, so an exhausted call does no further I/O.
96
+ if (!root || attempts >= limits.sources || remaining <= 0) { skipped++; continue; }
97
+ // Counted before the read, so a failed or over-budget attempt cannot slip past the file cap.
98
+ attempts++;
99
+ const file = resolve(root, ref.value);
100
+ let read: { bytes: number } | null;
101
+ // Ask for no more than the budget has left: the reader refuses a larger file from its fstat size, unread.
102
+ try { read = isDenied(canonicalPath(file), denyPaths) ? null : readFile(file, Math.min(MAX_AUX_BYTES, remaining)); } catch { read = null; }
103
+ // A denied, missing or over-budget file is a skip, and one that would pass the cap is left out whole.
104
+ if (read === null || read.bytes > remaining) { skipped++; continue; }
105
+ size += read.bytes; bytesRead += read.bytes; sources++; used++;
106
+ }
107
+ // A note whose sources are all empty has no ratio: count the note, not a reference, so `skipped` stays reference-only.
108
+ if (!used) continue;
109
+ if (size === 0) { emptySourceNotes++; continue; }
110
+ ratios.push({ path: row.path, ratio: row.bytes / size });
111
+ }
112
+ const values = ratios.map(entry => entry.ratio).sort((a, b) => a - b);
113
+ const largest = [...ratios].sort((a, b) => b.ratio - a.ratio || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)).slice(0, RATIO_LARGEST_SHOWN);
114
+ return {
115
+ notes: ratios.length, sources, skipped, skippedNonFile, emptySourceNotes,
116
+ median: values.length ? round3(middle(values)) : null, p90: values.length ? round3(percentile(values, 90)) : null,
117
+ overOne: ratios.filter(entry => entry.ratio > RATIO_OVER_ONE).length,
118
+ largest: largest.map(entry => ({ path: entry.path, ratio: round3(entry.ratio) })),
119
+ };
120
+ }
21
121
  /**
22
122
  * Read-only vault statistics over the derived index (refreshed like every read tool) and the log streamed by `readLog`:
23
- * no vault file or log is written and no vault or log lock is taken.
123
+ * no vault file or log is written and no vault or log lock is taken. Note sizes come from the same index pass, and the
124
+ * ratio reads only workspace files a note cites, through `analysisReadWithSize` and its bounds.
24
125
  */
25
- export function vaultStats(home: string, id: string, options: { days?: number; now?: Date; expectedVault?: VaultSpec } = {}): VaultStats {
126
+ export function vaultStats(home: string, id: string, options: { days?: number; now?: Date; expectedVault?: VaultSpec; limits?: RatioLimits } = {}): VaultStats {
26
127
  const days = options.days ?? STATS_DEFAULT_DAYS, now = options.now ?? new Date();
27
128
  if (!Number.isInteger(days) || days < 1 || days > STATS_MAX_DAYS) throw new Error(`stats: days must be an integer 1..${STATS_MAX_DAYS}; omit it for the ${STATS_DEFAULT_DAYS}-day default`);
28
129
  const registry = loadRegistry(home), vault = registry.vaults.find(v => v.id === id);
@@ -34,6 +135,8 @@ export function vaultStats(home: string, id: string, options: { days?: number; n
34
135
  const index = new VaultIndex(home, vault, registry.vaults.map(v => v.path));
35
136
  let notes: ReturnType<VaultIndex["noteStats"]>;
36
137
  try { index.scan(); notes = index.noteStats([start(7).getTime(), start(30).getTime()]); } finally { index.close(); }
138
+ const size = notes.size, pass = noteSourceRatio(notes.sourced, sourceWorkspaceRoot(vault), registry.denyPaths ?? [], options.limits);
139
+ const ratio: VaultRatio = { ...pass, truncatedNotes: notes.truncatedSourced };
37
140
  const since = localDate(start(days)), byType: Record<string, number> = Object.fromEntries(LOG_TYPES.map(type => [type, 0]));
38
141
  let total = 0;
39
142
  const log = readLog(home, { vault: vault.id, limit: 1, visit: line => {
@@ -42,9 +145,11 @@ export function vaultStats(home: string, id: string, options: { days?: number; n
42
145
  const type = (LOG_TYPES as readonly string[]).includes(match[2]!) ? match[2]! : "(other)";
43
146
  total++; byType[type] = (byType[type] ?? 0) + 1;
44
147
  } });
45
- const { modified: [last7Days = 0, last30Days = 0], ...counts } = notes;
148
+ const { modified: [last7Days = 0, last30Days = 0] } = notes;
46
149
  return {
47
- vault: vault.id, asOf: localDate(now), notes: counts, modified: { last7Days, last30Days },
150
+ vault: vault.id, asOf: localDate(now), size, ratio,
151
+ notes: { total: notes.total, byType: notes.byType, byDomain: notes.byDomain, byStatus: notes.byStatus },
152
+ modified: { last7Days, last30Days },
48
153
  log: { days, since, total: log.ok ? total : null, byType: log.ok ? byType : null, warnings: log.warnings, issues: log.issues },
49
154
  };
50
155
  }
@@ -12,7 +12,7 @@ import { argumentError } from "./mcp.ts";
12
12
  import { parseTidyMap, tidyVault, TIDY_RULES } from "./tidy.ts";
13
13
  import { PLAIN, type TextOptions } from "./text-output.ts";
14
14
  import { renderTidy, renderWriteReceipt } from "./write-output.ts";
15
- const COMMANDS = new Set(["new", "patch", "append", "frontmatter", "link", "supersede", "move", "delete", "tidy", "report", "check"]);
15
+ const COMMANDS = new Set(["new", "patch", "append", "frontmatter", "link", "supersede", "merge", "move", "delete", "tidy", "report", "check"]);
16
16
  export function isWriteCommand(command: string): boolean { return COMMANDS.has(command); }
17
17
  function object(text: string, label: string): Record<string, unknown> {
18
18
  let value: unknown;
@@ -48,12 +48,12 @@ export function runWriteCommand(command: string, args: string[], home: string, e
48
48
  new: ["--type", "--domain", "--topics", "--description", "--title", "--source", "--confidence", "--status", "--extra", "--body"],
49
49
  patch: ["--old", "--new", "--edits", "--expected-sha"], append: ["--section", "--markdown", "--expected-sha"],
50
50
  frontmatter: ["--set", "--unset", "--expected-sha"], link: ["--relation", "--context"],
51
- move: ["--fingerprint"], delete: ["--fingerprint"], tidy: ["--fingerprint", "--map"], report: ["--body"], check: ["--path"], supersede: [],
51
+ merge: ["--fingerprint"], move: ["--fingerprint"], delete: ["--fingerprint"], tidy: ["--fingerprint", "--map"], report: ["--body"], check: ["--path"], supersede: [],
52
52
  };
53
53
  // For tidy, --log selects the log-redaction rule; tidy always writes its own LINT event. Check never appends an event.
54
54
  const values = valueOptions[command] ? [...valueOptions[command]!, ...(["tidy", "check"].includes(command) ? [] : ["--log"])] : undefined;
55
55
  if (!values) throw new Error("unknown writer command");
56
- const twoStep = ["move", "delete", "tidy"].includes(command);
56
+ const twoStep = ["merge", "move", "delete", "tidy"].includes(command);
57
57
  const booleans = ["--json", "--dry-run", ...(twoStep ? ["--confirm"] : []), ...(command === "link" ? ["--one-way"] : []), ...(command === "tidy" ? TIDY_RULES.map(rule => `--${rule}`).concat("--all") : [])];
58
58
  const flags: Record<string, string | boolean> = {}, positional: string[] = [];
59
59
  for (let i = 0; i < args.length; i++) {
@@ -108,6 +108,7 @@ export function runWriteCommand(command: string, args: string[], home: string, e
108
108
  case "frontmatter": operation = { op: "frontmatter", path: target, set: optional("--set") ? object(required("--set"), "set") : {}, unset: optional("--unset")?.split(","), expectedSha: optional("--expected-sha") }; break;
109
109
  case "link": operation = { op: "link", from: target, to: positional[2]!, relation: required("--relation") as Extract<WriteOp, { op: "link" }>["relation"], context: optional("--context"), bidirectional: !flags["--one-way"] }; break;
110
110
  case "supersede": operation = { op: "supersede", old: target, next: positional[2]! }; break;
111
+ case "merge": operation = { op: "merge", keep: target, absorb: positional[2]! }; break;
111
112
  case "move": operation = { op: "move", from: target, to: positional[2]! }; break;
112
113
  case "delete": operation = { op: "delete", path: target }; break;
113
114
  case "tidy": operation = { op: "tidy" }; break;