@staix/vault-mem 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +147 -23
- package/package.json +5 -4
- package/src/amem/seam.ts +15 -6
- package/src/analysis-commands.ts +1 -1
- package/src/analysis-input.ts +24 -3
- package/src/analysis-reports.ts +1 -1
- package/src/analysis-types.ts +13 -1
- package/src/canon-tables.ts +190 -0
- package/src/cli-output.ts +15 -1
- package/src/cli.ts +3 -1
- package/src/commands-table.ts +7 -2
- package/src/config.ts +31 -3
- package/src/frontmatter.ts +10 -3
- package/src/links.ts +13 -1
- package/src/mcp-spec.ts +19 -11
- package/src/mcp-vault.ts +20 -12
- package/src/mcp.ts +4 -2
- package/src/org-promote.ts +29 -12
- package/src/promote.ts +25 -13
- package/src/read-commands.ts +1 -2
- package/src/read-output.ts +7 -5
- package/src/registry.ts +1 -0
- package/src/skills-mine.ts +5 -4
- package/src/spec-command-table.ts +10 -1
- package/src/spec-commands.ts +7 -3
- package/src/spec-gitlinks.ts +106 -0
- package/src/spec-output.ts +30 -0
- package/src/spec-source.ts +9 -3
- package/src/spec-validate.ts +6 -3
- package/src/spec-write.ts +469 -0
- package/src/ui/inbox.ts +28 -10
- package/src/ui/panel.js +26 -5
- package/src/ui/plugin.ts +38 -12
- package/src/ui/snapshot-cache.ts +123 -0
- package/src/vault-brief.ts +90 -0
- package/src/vault-index.ts +158 -21
- package/src/vault-lint.ts +53 -5
- package/src/vault-log.ts +3 -2
- package/src/vault-stats.ts +110 -5
- package/src/write-commands.ts +4 -3
- package/src/write-files.ts +72 -0
- package/src/write-operations.ts +175 -25
- package/src/write-output.ts +20 -9
- package/src/write-types.ts +8 -1
- package/src/write-validation.ts +46 -17
- package/src/writer.ts +145 -42
- package/src/yaml-quoted.ts +12 -2
package/src/vault-index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { realPath } from "./platform.ts";
|
|
1
|
+
import { realPath, canonicalPath, isDenied } from "./platform.ts";
|
|
2
2
|
import { Database } from "bun:sqlite";
|
|
3
3
|
import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readSync, readdirSync, realpathSync } from "node:fs";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
@@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
|
6
6
|
import type { AnalysisSnapshot } from "./analysis-types.ts";
|
|
7
7
|
import type { VaultSpec, SpecSourceSpec } from "./config.ts";
|
|
8
8
|
import { parseFrontmatter, type FrontmatterDocument } from "./frontmatter.ts";
|
|
9
|
-
import { normalizeTarget, noteKey, proseLines, resolveLinks, type LinkNote, type ResolvedLink } from "./links.ts";
|
|
9
|
+
import { normalizeTarget, noteKey, proseLines, resolveLinks, type LinkNote, type ResolvedLink, type SuggestedLink, type TitleSuggestion } from "./links.ts";
|
|
10
10
|
|
|
11
11
|
export const MAX_NOTE_BYTES = 2 * 1024 * 1024;
|
|
12
12
|
export const MAX_RESULTS = 500;
|
|
@@ -16,8 +16,12 @@ export const MAX_RESULTS = 500;
|
|
|
16
16
|
* share the unversioned `<digest>.sqlite`; the first run of a newer version builds its own file once.
|
|
17
17
|
*/
|
|
18
18
|
export const LINK_RESOLUTION = 2;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Bump when tables, FTS columns or the indexed text change. Named per value like LINK_RESOLUTION. 2 = CJK bigram
|
|
21
|
+
* table; 3 = reference identifiers in FTS; 4 = arXiv ids in the FTS text of reference notes, so a schema-3 cache
|
|
22
|
+
* built before that rebuilds once instead of answering without the id.
|
|
23
|
+
*/
|
|
24
|
+
export const INDEX_SCHEMA = 4;
|
|
21
25
|
const indexName = (root: string) => `${digest(root)}-links${LINK_RESOLUTION}-schema${INDEX_SCHEMA}.sqlite`;
|
|
22
26
|
/**
|
|
23
27
|
* unicode61 keeps each Hangul/CJK run as one token, so `사업단` misses `사업단은`. A side table holds the
|
|
@@ -51,6 +55,13 @@ export function splitTerms(words: string[]): { tokens: string[]; runs: string[]
|
|
|
51
55
|
return { tokens, runs };
|
|
52
56
|
}
|
|
53
57
|
export const MAX_READ_BYTES = 64 * 1024;
|
|
58
|
+
/** A hint must beat this: a term that appears in most titles scores at or below zero, and one shared common word stays under it. */
|
|
59
|
+
export const MIN_SUGGESTION_SCORE = 0.5;
|
|
60
|
+
/**
|
|
61
|
+
* A link's target, label, heading and block are reported within this many bytes of their raw text, and a hint is
|
|
62
|
+
* looked up for at most the same bound, so a reported row and the hint for it never judge different text.
|
|
63
|
+
*/
|
|
64
|
+
const LINK_TEXT_BYTES = 1024;
|
|
54
65
|
export interface IndexedNote extends LinkNote { hash: string; description: string; type: string; domain: string }
|
|
55
66
|
export interface NoteReadResult {
|
|
56
67
|
path: string;
|
|
@@ -67,6 +78,7 @@ export interface NoteReadResult {
|
|
|
67
78
|
headings?: string[];
|
|
68
79
|
headingsTruncated?: boolean;
|
|
69
80
|
}
|
|
81
|
+
export interface BriefPointer { path: string; title: string; type: string; description: string; bytes: number }
|
|
70
82
|
export interface SearchOptions { limit?: number; type?: string; domain?: string }
|
|
71
83
|
export interface SearchResult { path: string; title: string; description: string; type: string; domain: string; score: number; snippet: string }
|
|
72
84
|
interface StoredNote extends IndexedNote { metadata: string; source: string }
|
|
@@ -163,6 +175,14 @@ export interface IndexSourceAdapter {
|
|
|
163
175
|
}
|
|
164
176
|
export class IndexCacheError extends Error {}
|
|
165
177
|
export interface IndexOptions { memory?: boolean; adapter?: IndexSourceAdapter; reuseCache?: boolean }
|
|
178
|
+
/** Whitespace-separated tokens of a note source, frontmatter included; 0 for an empty source. */
|
|
179
|
+
const wordCount = (source: string) => { const trimmed = source.trim(); return trimmed ? trimmed.split(/\s+/u).length : 0; };
|
|
180
|
+
/** Note text size in bytes and words, with the number of notes that contributed. */
|
|
181
|
+
export interface NoteSize { notes: number; bytes: number; words: number }
|
|
182
|
+
/** One note that declares frontmatter `source`, with the byte size the ratio compares to its sources. */
|
|
183
|
+
export interface SourcedNote { path: string; bytes: number; source: unknown }
|
|
184
|
+
/** The ratio pass reads at most this many notes that declare a source, in path order. */
|
|
185
|
+
export const MAX_RATIO_NOTES = 10000;
|
|
166
186
|
/** Derived cache only. No vault or application configuration files are written. */
|
|
167
187
|
export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
168
188
|
private readonly db: Database;
|
|
@@ -242,10 +262,13 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
242
262
|
}
|
|
243
263
|
close(): void { this.db.close(); }
|
|
244
264
|
// ponytail: grams stores its bigram text (about 2.3x the CJK bytes); contentless with stable rowids would save it.
|
|
245
|
-
/**
|
|
265
|
+
/**
|
|
266
|
+
* Keep the stored body and link/analysis input unchanged; identifiers join only the FTS text. The arXiv id goes in
|
|
267
|
+
* without its `vN` version suffix, since a caller holding `2502.04321` cannot tokenize the suffixed form.
|
|
268
|
+
*/
|
|
246
269
|
private searchBody(body: string, data: Record<string, unknown>): string {
|
|
247
270
|
if (this.adapter || data.type !== "reference") return body;
|
|
248
|
-
const identifiers = [text(data.doi), text(data.citekey)].filter(Boolean);
|
|
271
|
+
const identifiers = [text(data.doi), text(data.citekey), text(data.arxiv_id).replace(/v\d+$/, "")].filter(Boolean);
|
|
249
272
|
return identifiers.length ? body + "\n" + identifiers.join("\n") : body;
|
|
250
273
|
}
|
|
251
274
|
private indexText(path: string, title: string, description: string, body: string): void {
|
|
@@ -260,7 +283,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
260
283
|
return this.db.query<Omit<LinkNote, "aliases"> & { aliases: string }, []>("SELECT path,title,aliases,body FROM notes ORDER BY path").all().map(row => ({ ...row, aliases: JSON.parse(row.aliases) as string[] }));
|
|
261
284
|
}
|
|
262
285
|
/** Scan metadata each call; hash only new or metadata-changed files. Atomic cache update. */
|
|
263
|
-
scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
|
|
286
|
+
scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean; denyPaths?: string[] } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
|
|
287
|
+
if (rebuild && options.denyPaths?.length) throw new Error("index: rebuild cannot use denyPaths restrictions; use scan(false, { denyPaths }) for a restricted refresh or vmem rebuild <vault> for an unrestricted rebuild");
|
|
264
288
|
// Discovery/binding Git calls must never hold a SQLite writer lock.
|
|
265
289
|
let preparedPaths: string[] | undefined;
|
|
266
290
|
if (this.adapter) { this.adapter.validate(); preparedPaths = this.adapter.paths(); }
|
|
@@ -279,6 +303,13 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
279
303
|
for (const path of paths) {
|
|
280
304
|
this.adapter?.validatePath(path);
|
|
281
305
|
const file = join(this.adapter ? this.root : notesDir, path);
|
|
306
|
+
if (options.denyPaths && (isDenied(file, options.denyPaths) || isDenied(canonicalPath(file), options.denyPaths))) {
|
|
307
|
+
// A restricted reader must not delete another reader's cached rows or invalidate link resolution.
|
|
308
|
+
// Preserve known rows without inspecting their metadata/source; a new denied file stays unindexed.
|
|
309
|
+
present.add(path);
|
|
310
|
+
if (previous.has(path)) live.add(path);
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
282
313
|
const before = this.adapter ? specFileIO(path, "inspect", () => lstatSync(file)) : lstatSync(file);
|
|
283
314
|
if (!before.isFile() || before.isSymbolicLink()) throw new Error(this.adapter ? `spec discovery ${path}: selected path changed; restore a plain file and retry` : `vault ${this.vault.id}: note path changed`);
|
|
284
315
|
if (!this.adapter && before.size > MAX_NOTE_BYTES) throw new Error(`vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
|
|
@@ -419,6 +450,83 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
419
450
|
.slice(snapshot ? snapshotOffset : 0, (snapshot ? snapshotOffset : 0) + (snapshot && allSnapshotRows ? 10000 : limit))
|
|
420
451
|
.map(({ sourceHash: _, ...row }) => ({ ...row, title: boundedText(row.title, 512).content, snippet: boundedText(row.snippet, 1024).content, type: boundedText(row.type, 256).content, domain: boundedText(row.domain, 256).content }));
|
|
421
452
|
}
|
|
453
|
+
/** Intent words are OR terms; reuse the search tokenizer, BM25 weights and CJK substring/strong-match rules.
|
|
454
|
+
* Materialize scores before grouping: FTS5 auxiliary functions cannot run in an aggregate context. No bodies leave SQL. */
|
|
455
|
+
briefMatches(words: string[]): { path: string; score: number }[] {
|
|
456
|
+
if (!words.length || words.length > 32 || words.some(word => !word || Buffer.byteLength(word) > 4096)) throw new Error("brief: expected 1..32 bounded search words; shorten the intent and path hints");
|
|
457
|
+
const { tokens, runs } = splitTerms(words), parts: string[] = [], params: string[] = [];
|
|
458
|
+
if (tokens.length) {
|
|
459
|
+
parts.push("SELECT path,bm25(fts,0,5,2,1) AS score,1 AS strong FROM fts WHERE fts MATCH ?");
|
|
460
|
+
params.push(tokens.join(" OR "));
|
|
461
|
+
}
|
|
462
|
+
if (runs.length) {
|
|
463
|
+
parts.push(`SELECT path,bm25(grams,0,5,2,1) AS score,
|
|
464
|
+
(path IN (SELECT path FROM fts WHERE fts MATCH ?) OR path IN (SELECT path FROM grams WHERE grams MATCH ?)) AS strong
|
|
465
|
+
FROM grams WHERE grams MATCH ?`);
|
|
466
|
+
params.push(runs.map(run => `"${run}"`).join(" OR "), `{title description} : (${runs.map(gramPhrase).join(" OR ")})`, runs.map(gramPhrase).join(" OR "));
|
|
467
|
+
}
|
|
468
|
+
const rows = this.db.query<{ path: string; score: number }, string[]>(`WITH hits AS MATERIALIZED (${parts.join(" UNION ALL ")})
|
|
469
|
+
SELECT path,min(score) AS score,max(strong) AS strength FROM hits GROUP BY path ORDER BY strength DESC,score,path LIMIT 10001`).all(...params);
|
|
470
|
+
if (rows.length > 10000) throw new Error("brief: more than 10000 matched notes; narrow the intent or use vmem search with --type/--domain filters");
|
|
471
|
+
return rows;
|
|
472
|
+
}
|
|
473
|
+
/** Enforce the remaining metadata read budget inside SQL, before values cross into JavaScript. */
|
|
474
|
+
briefPointer(path: string, maximum: number): BriefPointer | undefined {
|
|
475
|
+
if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 4096) throw new Error("brief: metadata read bound must be 1..4096 bytes; reduce the requested brief");
|
|
476
|
+
const bytes = "length(CAST(path AS BLOB))+length(CAST(title AS BLOB))+length(CAST(type AS BLOB))+length(CAST(description AS BLOB))";
|
|
477
|
+
return this.db.query<BriefPointer, [string, number]>(`SELECT path,title,type,description,${bytes} AS bytes FROM notes WHERE path=? AND (${bytes})<=?`).get(path, maximum) ?? undefined;
|
|
478
|
+
}
|
|
479
|
+
/** Resolved incoming and outgoing paths only, never link context/aliases or note bodies. */
|
|
480
|
+
briefNeighbors(paths: string[]): string[] {
|
|
481
|
+
if (!paths.length) return [];
|
|
482
|
+
if (paths.length > 5) throw new Error("brief: at most five seed notes; use the top ranked hits");
|
|
483
|
+
const slots = paths.map(() => "?").join(",");
|
|
484
|
+
const rows = this.db.query<{ path: string }, string[]>(`SELECT DISTINCT
|
|
485
|
+
CASE WHEN source IN (${slots}) THEN json_extract(data,'$.resolved') ELSE source END AS path
|
|
486
|
+
FROM links WHERE json_extract(data,'$.resolved') IS NOT NULL AND
|
|
487
|
+
(source IN (${slots}) OR json_extract(data,'$.resolved') IN (${slots})) ORDER BY path LIMIT 10001`).all(...paths, ...paths, ...paths);
|
|
488
|
+
if (rows.length > 10000) throw new Error("brief: more than 10000 linked neighbors; narrow the intent or use graph neighbors");
|
|
489
|
+
return rows.map(row => row.path);
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* The one indexed title closest to a dead link's text, or undefined when no title matches or the best match is at
|
|
493
|
+
* or below `minimum`. Same literal AND terms as `search`, matched in the title column only, because a hint should
|
|
494
|
+
* name a note whose title actually carries the link text. A title must carry every part of the text: a CJK run
|
|
495
|
+
* matches through the bigram table and a Latin term through unicode61, and the two scores are summed, because a part
|
|
496
|
+
* that many titles share scores near zero on its own and would otherwise hide a title that carries the whole text.
|
|
497
|
+
* The score is positive bm25, rounded to three decimals, so a larger value is a closer title. A hint is never an
|
|
498
|
+
* error: raw text past `LINK_TEXT_BYTES`, or text FTS5 cannot parse, returns undefined instead of throwing. The bound
|
|
499
|
+
* is read on the raw text, before normalization and never on the truncated target a row displays, so `links` and
|
|
500
|
+
* `lint` name the same hint for the same dead link. Read-only.
|
|
501
|
+
*/
|
|
502
|
+
suggestTitle(text: string, minimum = MIN_SUGGESTION_SCORE): TitleSuggestion | undefined {
|
|
503
|
+
if (Buffer.byteLength(text) > LINK_TEXT_BYTES) return undefined;
|
|
504
|
+
// Link text is note content, so it can carry a control character. unicode61 already indexes one as a separator,
|
|
505
|
+
// and U+0000 kept inside a quoted term ends the string in SQLite and makes FTS5 throw "unterminated string".
|
|
506
|
+
const query = text.replace(/\p{Cc}/gu, " ").trim();
|
|
507
|
+
if (!query) return undefined;
|
|
508
|
+
const { tokens, runs } = splitTerms(query.split(/\s+/));
|
|
509
|
+
if (!tokens.length && !runs.length) return undefined;
|
|
510
|
+
const gramQuery = `{title} : (${runs.map(gramPhrase).join(" AND ")})`, tokenQuery = `{title} : (${tokens.join(" AND ")})`;
|
|
511
|
+
let row: TitleSuggestion | null | undefined;
|
|
512
|
+
try {
|
|
513
|
+
row = runs.length && tokens.length
|
|
514
|
+
? this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, g.score + f.score AS score
|
|
515
|
+
FROM (SELECT path, -bm25(grams,0,5,2,1) AS score FROM grams WHERE grams MATCH ?) g
|
|
516
|
+
JOIN (SELECT path, -bm25(fts,0,5,2,1) AS score FROM fts WHERE fts MATCH ?) f ON f.path = g.path
|
|
517
|
+
JOIN notes n ON n.path = g.path ORDER BY score DESC, n.path LIMIT 1`).get(gramQuery, tokenQuery)
|
|
518
|
+
: runs.length
|
|
519
|
+
? this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, -bm25(grams,0,5,2,1) AS score FROM grams JOIN notes n ON n.path=grams.path
|
|
520
|
+
WHERE grams MATCH ? ORDER BY score DESC, n.path LIMIT 1`).get(gramQuery)
|
|
521
|
+
: this.db.query<TitleSuggestion, string[]>(`SELECT n.path, n.title, -bm25(fts,0,5,2,1) AS score FROM fts JOIN notes n ON n.path=fts.path
|
|
522
|
+
WHERE fts MATCH ? ORDER BY score DESC, n.path LIMIT 1`).get(tokenQuery);
|
|
523
|
+
} catch {
|
|
524
|
+
// A hint is advisory: text FTS5 still cannot parse yields no suggestion instead of failing links and lint.
|
|
525
|
+
return undefined;
|
|
526
|
+
}
|
|
527
|
+
if (!row || !(row.score > minimum)) return undefined;
|
|
528
|
+
return { path: row.path, title: boundedText(row.title, 512).content, score: Math.round(row.score * 1000) / 1000 };
|
|
529
|
+
}
|
|
422
530
|
private find(target: string): string {
|
|
423
531
|
if (!target || Buffer.byteLength(target) > 1024) throw new Error("note: expected target <=1024 bytes");
|
|
424
532
|
const key = normalizeTarget(target.replace(/^notes\//i, "").replace(/\.md$/i, ""));
|
|
@@ -435,6 +543,14 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
435
543
|
if (notes.length !== 1) throw new Error(notes.length ? "note: ambiguous target" : "note: unknown target");
|
|
436
544
|
return notes[0]!.path;
|
|
437
545
|
}
|
|
546
|
+
/** Literal indexed-note source, with the byte bound checked before returning/decoding it. */
|
|
547
|
+
noteSource(path: string, maximum = MAX_NOTE_BYTES): string | undefined {
|
|
548
|
+
if (!Number.isInteger(maximum) || maximum < 1 || maximum > MAX_NOTE_BYTES) throw new Error(`note source maximum: expected 1..${MAX_NOTE_BYTES} bytes; choose a bound within the note reader limit`);
|
|
549
|
+
const literal = path.startsWith("notes/") ? path.slice(6) : path;
|
|
550
|
+
const row = this.db.query<{ source: string | null }, [number, string]>("SELECT CASE WHEN length(CAST(source AS BLOB))<=? THEN source ELSE NULL END AS source FROM notes WHERE path=?").get(maximum, literal);
|
|
551
|
+
if (row?.source === null) throw Object.assign(new Error(`note source exceeds the ${maximum}-byte remaining read budget; split the note or reduce configured tables before retrying`), { code: "V09" });
|
|
552
|
+
return row?.source;
|
|
553
|
+
}
|
|
438
554
|
show(target: string, maxBytes = 16384, section?: string, offset?: number): NoteReadResult {
|
|
439
555
|
if (!Number.isInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_READ_BYTES) throw new Error(`max-bytes: expected integer 1..${MAX_READ_BYTES}`);
|
|
440
556
|
if (section !== undefined && (typeof section !== "string" || !section.trim() || /[\r\n]/.test(section) || Buffer.byteLength(section) > 1024)) throw new Error("section: expected one exact H2 heading <=1024 bytes");
|
|
@@ -450,7 +566,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
450
566
|
if (!page.truncated) return { path, hash: note.hash, ...page, ...metadata };
|
|
451
567
|
}
|
|
452
568
|
// Only the derived source is loaded; section discovery never rescans a note file.
|
|
453
|
-
const source = this.
|
|
569
|
+
const source = this.noteSource(path);
|
|
570
|
+
if (source === undefined) throw new Error("note: source disappeared or exceeded the read bound during paging; refresh and reread the note");
|
|
454
571
|
const body = parseFrontmatter(source).body, sections = h2Sections(body);
|
|
455
572
|
if (section !== undefined) {
|
|
456
573
|
const found = sections.filter(item => item.name === section);
|
|
@@ -480,10 +597,15 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
480
597
|
return { notes, total, ...(rows.length > max ? { nextCursor: notes.at(-1)!.path } : {}) };
|
|
481
598
|
}
|
|
482
599
|
/**
|
|
483
|
-
* Note counts by type, domain and frontmatter `status` ("(none)" when empty),
|
|
484
|
-
* after each cutoff (epoch ms)
|
|
600
|
+
* Note counts by type, domain and frontmatter `status` ("(none)" when empty), how many notes have an mtime at or
|
|
601
|
+
* after each cutoff (epoch ms), note text size in bytes and words by type and domain, and the notes that declare a
|
|
602
|
+
* frontmatter `source` (at most `sourcedLimit`, in path order) as the input of the note-to-source ratio, with
|
|
603
|
+
* `truncatedSourced` counting the sourced notes that cap kept out. Streams rows; returns no note text.
|
|
485
604
|
*/
|
|
486
|
-
noteStats(cutoffs: readonly number[]
|
|
605
|
+
noteStats(cutoffs: readonly number[], sourcedLimit = MAX_RATIO_NOTES): {
|
|
606
|
+
total: number; byType: Record<string, number>; byDomain: Record<string, number>; byStatus: Record<string, number>; modified: number[];
|
|
607
|
+
size: { total: NoteSize; byType: Record<string, NoteSize>; byDomain: Record<string, NoteSize> }; sourced: SourcedNote[]; truncatedSourced: number;
|
|
608
|
+
} {
|
|
487
609
|
const add = (map: Map<string, number>, value: string, count = 1) => { const key = value ? boundedText(value, 256).content : "(none)"; map.set(key, (map.get(key) ?? 0) + count); };
|
|
488
610
|
const sorted = (map: Map<string, number>) => Object.fromEntries([...map].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
|
|
489
611
|
const grouped = (column: "type" | "domain") => {
|
|
@@ -491,21 +613,32 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
491
613
|
for (const row of this.db.query<{ value: string; count: number }, []>(`SELECT ${column} AS value, count(*) AS count FROM notes GROUP BY ${column}`).all()) add(map, row.value, row.count);
|
|
492
614
|
return sorted(map);
|
|
493
615
|
};
|
|
494
|
-
|
|
495
|
-
const
|
|
496
|
-
|
|
497
|
-
|
|
616
|
+
const size = { total: { notes: 0, bytes: 0, words: 0 }, byType: new Map<string, NoteSize>(), byDomain: new Map<string, NoteSize>() };
|
|
617
|
+
const grow = (map: Map<string, NoteSize>, value: string, bytes: number, words: number) => {
|
|
618
|
+
const key = value ? boundedText(value, 256).content : "(none)", item = map.get(key) ?? { notes: 0, bytes: 0, words: 0 };
|
|
619
|
+
map.set(key, { notes: item.notes + 1, bytes: item.bytes + bytes, words: item.words + words });
|
|
620
|
+
};
|
|
621
|
+
const sortedSize = (map: Map<string, NoteSize>) => Object.fromEntries([...map].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
|
|
622
|
+
// Status, size and `source` are not columns: stream each stored source once, parsing only its frontmatter.
|
|
623
|
+
const byStatus = new Map<string, number>(), modified = cutoffs.map(() => 0), sourced: SourcedNote[] = [];
|
|
624
|
+
let total = 0, truncatedSourced = 0;
|
|
625
|
+
const rows = this.db.prepare<{ metadata: string; source: string; path: string; type: string; domain: string }, []>("SELECT metadata,source,path,type,domain FROM notes ORDER BY path");
|
|
498
626
|
try {
|
|
499
627
|
for (const row of rows.iterate()) {
|
|
500
628
|
total++;
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
629
|
+
const bytes = Buffer.byteLength(row.source), words = wordCount(row.source);
|
|
630
|
+
size.total = { notes: size.total.notes + 1, bytes: size.total.bytes + bytes, words: size.total.words + words };
|
|
631
|
+
grow(size.byType, row.type, bytes, words); grow(size.byDomain, row.domain, bytes, words);
|
|
632
|
+
let data: Record<string, unknown> = {};
|
|
633
|
+
try { data = parseFrontmatter(row.source).data; } catch { /* counted as (none) and skipped by the ratio pass */ }
|
|
634
|
+
add(byStatus, text(data.status));
|
|
635
|
+
if (data.source !== undefined) { if (sourced.length < sourcedLimit) sourced.push({ path: row.path, bytes, source: data.source }); else truncatedSourced++; }
|
|
504
636
|
const mtime = stampMtime(row.metadata);
|
|
505
637
|
cutoffs.forEach((cutoff, i) => { if (mtime !== null && mtime >= cutoff) modified[i]!++; });
|
|
506
638
|
}
|
|
507
639
|
} finally { rows.finalize(); }
|
|
508
|
-
return { total, byType: grouped("type"), byDomain: grouped("domain"), byStatus: sorted(byStatus), modified
|
|
640
|
+
return { total, byType: grouped("type"), byDomain: grouped("domain"), byStatus: sorted(byStatus), modified,
|
|
641
|
+
size: { total: size.total, byType: sortedSize(size.byType), byDomain: sortedSize(size.byDomain) }, sourced, truncatedSourced };
|
|
509
642
|
}
|
|
510
643
|
/** Links that resolve to the target note, ordered by source. */
|
|
511
644
|
backlinks(target: string, limit = 50): ResolvedLink[] {
|
|
@@ -553,8 +686,12 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
|
|
|
553
686
|
} finally { rows.finalize(); }
|
|
554
687
|
return out;
|
|
555
688
|
}
|
|
556
|
-
links(target: string, limit = 50):
|
|
689
|
+
links(target: string, limit = 50): SuggestedLink[] {
|
|
557
690
|
const path = this.find(target);
|
|
558
|
-
return this.db.query<{ data: string }, [string, number]>("SELECT data FROM links WHERE source=? ORDER BY ordinal LIMIT ?").all(path, resultLimit(limit)).map(row => JSON.parse(row.data) as ResolvedLink).map(link => ({
|
|
691
|
+
return this.db.query<{ data: string }, [string, number]>("SELECT data FROM links WHERE source=? ORDER BY ordinal LIMIT ?").all(path, resultLimit(limit)).map(row => JSON.parse(row.data) as ResolvedLink).map(link => ({
|
|
692
|
+
...link, target: boundedText(link.target, LINK_TEXT_BYTES).content, label: link.label === null ? null : boundedText(link.label, LINK_TEXT_BYTES).content, heading: link.heading === null ? null : boundedText(link.heading, LINK_TEXT_BYTES).content, block: link.block === null ? null : boundedText(link.block, LINK_TEXT_BYTES).content,
|
|
693
|
+
// The hint reads the raw text, never the bounded `target` above: a link past the bound has to miss here and in
|
|
694
|
+
// lint alike, or one surface names a suggestion the other, reading the whole text, cannot.
|
|
695
|
+
...(link.resolved === null ? { suggested: this.suggestTitle(link.target) ?? null } : {}) }));
|
|
559
696
|
}
|
|
560
697
|
}
|
package/src/vault-lint.ts
CHANGED
|
@@ -9,8 +9,10 @@ const LEGACY = new Set("CREATE UPDATE MIGRATE REFACTOR MERGE MOVE RELOCATE RENAM
|
|
|
9
9
|
const CODE_TITLES: Record<LintCode, string> = {
|
|
10
10
|
L01: "L01 — dead wiki-link", L02: "L02 — 필수 frontmatter / 허용값", L03: "L03 — orphan",
|
|
11
11
|
L09: "L09 — 로그 포맷", L10: "L10 — project 값", L11: "L11 — graph report staleness", L12: "L12 — island community", L13: "L13 — filename", L14: "L14 — source paths",
|
|
12
|
+
L15: "L15 — proven confidence의 source",
|
|
12
13
|
};
|
|
13
14
|
const truth = yamlTruthy;
|
|
15
|
+
const emptySource = (value: unknown) => !truth(value) || typeof value === "string" && !value.trim();
|
|
14
16
|
/** Python scalar formatting matters to per-code diagnostic parity. */
|
|
15
17
|
function py(x: unknown): string {
|
|
16
18
|
const scalar = scalarText(x); if (scalar !== undefined) return scalar;
|
|
@@ -37,6 +39,43 @@ function links(value: unknown): string[] {
|
|
|
37
39
|
return values.flatMap(v => typeof v === "string" ? [...v.matchAll(/\[\[([^\]|#]+)(?:[|#][^\]]*)?\]\]/g)].map(m => m[1]!.trim()) : []);
|
|
38
40
|
}
|
|
39
41
|
const stem = (s: string) => s.trim().replace(/\.md$/, "").split("/").at(-1)!;
|
|
42
|
+
/** Wiki-link texts L01 checks in one note, in the order it reports them: body links, then the link fields. */
|
|
43
|
+
function noteLinkTexts(source: string): string[] {
|
|
44
|
+
const { fm, body } = legacyDocument(source);
|
|
45
|
+
return [...links(body), ...FIELDS.flatMap(field => links(fm?.[field]))];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A snapshot record entry read as data only. Link text and note path reach these lookups unchanged, and a plain
|
|
49
|
+
* object answers `constructor`, `toString` or `__proto__` from `Object.prototype`; a snapshot another caller built
|
|
50
|
+
* can still be a plain record, so inherited members must never resolve as a hint or as warnings.
|
|
51
|
+
*/
|
|
52
|
+
function ownValue<T>(record: Record<string, T> | undefined, key: string): T | undefined {
|
|
53
|
+
return record !== undefined && Object.hasOwn(record, key) ? record[key] : undefined;
|
|
54
|
+
}
|
|
55
|
+
/** Unique dead link texts one lint run computes hints for; past it the report says so instead of reading as a miss. */
|
|
56
|
+
export const DEAD_LINK_HINT_CAP = 1000;
|
|
57
|
+
/**
|
|
58
|
+
* The dead link texts a snapshot can be hinted for, deduplicated in note and link order and capped at `maximum`.
|
|
59
|
+
* Captured while the index is open (src/analysis-input.ts), because the lookup needs the title search and
|
|
60
|
+
* `lintDetails` stays a pure transform.
|
|
61
|
+
*/
|
|
62
|
+
export function deadLinkTexts(notes: readonly { path: string; source: string }[], maximum = DEAD_LINK_HINT_CAP): string[] {
|
|
63
|
+
const stems = new Set(notes.map(note => note.path.replace(/^notes\//, "").slice(0, -3))), texts = new Set<string>();
|
|
64
|
+
for (const note of notes) {
|
|
65
|
+
for (const text of noteLinkTexts(note.source)) if (!stems.has(stem(text))) texts.add(text);
|
|
66
|
+
if (texts.size >= maximum) break;
|
|
67
|
+
}
|
|
68
|
+
return [...texts].slice(0, maximum);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The texts a hint pass attempts, the bound it worked within and how many unique texts that bound left out. Counting
|
|
72
|
+
* the remainder needs the whole traversal, because a reader that knows only the cap cannot tell an attempted miss from
|
|
73
|
+
* a dead link whose hint was never computed.
|
|
74
|
+
*/
|
|
75
|
+
export function deadLinkScan(notes: readonly { path: string; source: string }[], maximum = DEAD_LINK_HINT_CAP): { texts: string[]; cap: number; skipped: number } {
|
|
76
|
+
const all = deadLinkTexts(notes, Number.POSITIVE_INFINITY), texts = all.slice(0, maximum);
|
|
77
|
+
return { texts, cap: maximum, skipped: all.length - texts.length };
|
|
78
|
+
}
|
|
40
79
|
function graphChecks(snapshot: AnalysisSnapshot): { l11: string[]; l12: string[]; info: string } {
|
|
41
80
|
const l11: string[] = [], l12: string[] = [], names = Object.keys(snapshot.reports).filter(s => /^graph-report-.*\.md$/.test(s)).sort();
|
|
42
81
|
const latest = names.at(-1);
|
|
@@ -74,19 +113,26 @@ function graphChecks(snapshot: AnalysisSnapshot): { l11: string[]; l12: string[]
|
|
|
74
113
|
export function lintVault(snapshot: AnalysisSnapshot, only?: string): LintReport { return lintDetails(snapshot, only).report; }
|
|
75
114
|
/** The report plus what only its markdown carries: the note count, check titles and per-check info lines, for the CLI text view. */
|
|
76
115
|
export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report: LintReport; notes: number; titles: Record<LintCode, string>; info: Partial<Record<LintCode, string>> } {
|
|
77
|
-
const findings: Record<LintCode,string[]> = {L01:[],L02:[],L03:[],L09:[],L10:[],L11:[],L12:[],L13:[],L14:[]};
|
|
116
|
+
const findings: Record<LintCode,string[]> = {L01:[],L02:[],L03:[],L09:[],L10:[],L11:[],L12:[],L13:[],L14:[],L15:[]};
|
|
78
117
|
const notes = [...snapshot.notes].sort((a,b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0).map(n => ({...n,...legacyDocument(n.source),stem: n.path.replace(/^notes\//, "").slice(0,-3)}));
|
|
79
118
|
const stems = new Set(notes.map(n => n.stem)), inlinks = new Map(notes.map(n => [n.stem,0]));
|
|
119
|
+
// A text the hint pass never reached was not looked up, so its line must not read as a title that merely did not match.
|
|
120
|
+
const bound = snapshot.linkHintBound, attempted = bound ? new Set(bound.attempted) : undefined;
|
|
80
121
|
let superseded = 0;
|
|
81
122
|
for (const note of notes) {
|
|
82
123
|
const rel = `notes/${note.stem}.md`, prefix = `\`${rel}\`: `, fm = note.fm;
|
|
83
|
-
for(const message of snapshot.sourceWarnings
|
|
124
|
+
for (const message of ownValue(snapshot.sourceWarnings, note.path) ?? []) findings.L14.push(prefix+message);
|
|
84
125
|
const allLinks: [string,string][] = links(note.body).map(t => ["body",t]);
|
|
85
126
|
if (fm) for (const field of FIELDS) for (const t of links(fm[field])) allLinks.push([field,t]);
|
|
86
127
|
for (const [field,t] of allLinks) {
|
|
87
128
|
const target = stem(t);
|
|
88
129
|
if (stems.has(target)) { if (target !== note.stem) inlinks.set(target,inlinks.get(target)!+1); }
|
|
89
|
-
else
|
|
130
|
+
else {
|
|
131
|
+
const hint = ownValue(snapshot.linkHints, t);
|
|
132
|
+
const suffix = hint ? ` — suggested \`${hint.path}\` (score ${hint.score.toFixed(3)})`
|
|
133
|
+
: bound && !attempted!.has(t) ? ` — hint not computed (cap ${bound.cap})` : "";
|
|
134
|
+
findings.L01.push(prefix + `\`[[${t}]]\` (${field}) → 대상 노트 없음` + suffix);
|
|
135
|
+
}
|
|
90
136
|
}
|
|
91
137
|
if (fm === null) { findings.L02.push(prefix + "frontmatter 없음/파싱 실패"); continue; }
|
|
92
138
|
if (fm.status === "superseded") superseded++;
|
|
@@ -97,6 +143,7 @@ export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report
|
|
|
97
143
|
if (fm.type === "moc") { if (!truth(fm.description)) findings.L02.push(prefix + "`type: moc` 노트는 `description` 필수"); }
|
|
98
144
|
else if (!truth(fm.topics)) findings.L02.push(prefix + "`topics` 없음/빈 배열");
|
|
99
145
|
for (const [key,allowed] of [["confidence",CONFIDENCE],["status",STATUS]] as const) if (fm[key] !== undefined && fm[key] !== null && !allowed.includes(py(fm[key]))) findings.L02.push(prefix + `\`${key}: ${py(fm[key])}\` 허용값 아님 (${allowed.join(" | ")})${key === "status" ? " (status는 노트 생애주기이지 사업 진행 상태가 아님 — 작업 상태는 본문에)" : ""}`);
|
|
146
|
+
if (py(fm.confidence) === "proven" && emptySource(fm.source)) findings.L15.push(prefix + "`confidence: proven`에 `source` 없음 — 근거(memory id·URL·workspace 경로)를 추가하거나 confidence를 낮추기");
|
|
100
147
|
const project = fm.project;
|
|
101
148
|
if (typeof project === "string" && project.trim()) {
|
|
102
149
|
const targets = links(project);
|
|
@@ -120,11 +167,12 @@ export function lintDetails(snapshot: AnalysisSnapshot, only?: string): { report
|
|
|
120
167
|
}
|
|
121
168
|
if (data > 20000) findings.L09.push(`log ${data} 라인 > 20000 — 수동 아카이브 권장`);
|
|
122
169
|
const graph = graphChecks(snapshot); findings.L11 = graph.l11; findings.L12 = graph.l12;
|
|
123
|
-
if (only) for (const code of ["L01","L02","L03","L10","L13","L14"] as const) findings[code] = findings[code].filter(s => s.startsWith(`\`${only}\``));
|
|
170
|
+
if (only) for (const code of ["L01","L02","L03","L10","L13","L14","L15"] as const) findings[code] = findings[code].filter(s => s.startsWith(`\`${only}\``));
|
|
124
171
|
const errors = findings.L01.length + findings.L02.length + findings.L13.length;
|
|
125
|
-
const warnings = findings.L03.length + findings.L09.length + findings.L10.length + findings.L11.length + findings.L12.length + findings.L14.length;
|
|
172
|
+
const warnings = findings.L03.length + findings.L09.length + findings.L10.length + findings.L11.length + findings.L12.length + findings.L14.length + findings.L15.length;
|
|
126
173
|
const summary = `SUMMARY errors=${errors} warnings=${warnings}`;
|
|
127
174
|
const extra: Partial<Record<LintCode,string>> = {
|
|
175
|
+
L01: bound ? `정보: 힌트 상한 ${bound.cap}개 텍스트 초과 — ${bound.skipped}개 dead link 텍스트는 힌트 미계산 (미탐 아님)` : "",
|
|
128
176
|
L02: `정보: \`status: superseded\` ${superseded}건${superseded === 0 ? " — supersede 프로토콜 미가동" : ""}`,
|
|
129
177
|
L03: "hub MOC `notes/index.md`는 설계상 예외",
|
|
130
178
|
L09: snapshot.log === null ? "정보: log 없음" : `정보: ${legacy ? `legacy TYPE ${legacy}건 (정규화표 등재), 최종 사용 ${last}, 신규 발생 없음` : "legacy TYPE 0건"}`,
|
package/src/vault-log.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mutationVault } from "./config.ts";
|
|
1
2
|
import { compileLogFilter } from "./log-filter.ts";
|
|
2
3
|
import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, openSync, readSync, realpathSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
3
4
|
import { join } from "node:path";
|
|
@@ -87,7 +88,7 @@ export function logBinding(vault: VaultSpec, config: VmemConfig): LogBinding {
|
|
|
87
88
|
/** True when the registered vault still equals the binding: spec, path policy and root identity. Never throws. */
|
|
88
89
|
export function matchesBinding(config: VmemConfig, id: string, binding: LogBinding): boolean {
|
|
89
90
|
const vault = config.vaults.find(v => v.id === id);
|
|
90
|
-
if (!vault || JSON.stringify(vault) !== JSON.stringify(binding.vault) || JSON.stringify(config.denyPaths ?? []) !== JSON.stringify(binding.denyPaths) || JSON.stringify(config.piiPatterns ?? []) !== JSON.stringify(binding.piiPatterns)) return false;
|
|
91
|
+
if (!vault || JSON.stringify(mutationVault(vault)) !== JSON.stringify(mutationVault(binding.vault)) || JSON.stringify(config.denyPaths ?? []) !== JSON.stringify(binding.denyPaths) || JSON.stringify(config.piiPatterns ?? []) !== JSON.stringify(binding.piiPatterns)) return false;
|
|
91
92
|
try { const identity = lstatSync(vault.path); return Number(identity.dev) === binding.dev && Number(identity.ino) === binding.ino; } catch { return false; }
|
|
92
93
|
}
|
|
93
94
|
export function appendLog(home: string, request: LogRequest, options: { env?: NodeJS.ProcessEnv; now?: Date; binding?: LogBinding } = {}): LogReceipt {
|
|
@@ -108,7 +109,7 @@ export function appendLog(home: string, request: LogRequest, options: { env?: No
|
|
|
108
109
|
const rootIdentity = lstatSync(ctx.vault.path);
|
|
109
110
|
if (options.binding) {
|
|
110
111
|
const bound = options.binding;
|
|
111
|
-
if (JSON.stringify(ctx.vault) !== JSON.stringify(bound.vault) || rootIdentity.dev !== bound.dev || rootIdentity.ino !== bound.ino || JSON.stringify(ctx.config.denyPaths ?? []) !== JSON.stringify(bound.denyPaths) || JSON.stringify(ctx.config.piiPatterns ?? []) !== JSON.stringify(bound.piiPatterns)) refuse("LOG_DRIFT", "vault identity or policy changed after mutation; log not appended");
|
|
112
|
+
if (JSON.stringify(mutationVault(ctx.vault)) !== JSON.stringify(mutationVault(bound.vault)) || rootIdentity.dev !== bound.dev || rootIdentity.ino !== bound.ino || JSON.stringify(ctx.config.denyPaths ?? []) !== JSON.stringify(bound.denyPaths) || JSON.stringify(ctx.config.piiPatterns ?? []) !== JSON.stringify(bound.piiPatterns)) refuse("LOG_DRIFT", "vault identity or policy changed after mutation; log not appended");
|
|
112
113
|
}
|
|
113
114
|
const formatted = formatLogEvent(request.event, { ...ctx, env: options.env ?? process.env, now: options.now });
|
|
114
115
|
if (!formatted.line || formatted.issues.some(issue => issue.severity === "error")) { result.issues.push(...formatted.issues); return result; }
|
package/src/vault-stats.ts
CHANGED
|
@@ -1,28 +1,129 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
|
+
import { MAX_AUX_BYTES, analysisReadWithSize } from "./analysis-input.ts";
|
|
1
3
|
import { localDate } from "./local-date.ts";
|
|
2
4
|
import { LOG_TYPES } from "./log-format.ts";
|
|
5
|
+
import { canonicalPath, isDenied } from "./platform.ts";
|
|
3
6
|
import { loadRegistry } from "./registry.ts";
|
|
4
|
-
import {
|
|
7
|
+
import { sourceReferences, sourceWorkspaceRoot, type SourceReference } from "./source.ts";
|
|
8
|
+
import { VaultIndex, type NoteSize, type SourcedNote } from "./vault-index.ts";
|
|
5
9
|
import { readLog } from "./vault-log.ts";
|
|
6
10
|
import type { WriteIssue } from "./write-types.ts";
|
|
7
11
|
import type { VaultSpec } from "./config.ts";
|
|
8
12
|
|
|
9
13
|
export const STATS_DEFAULT_DAYS = 30;
|
|
10
14
|
export const STATS_MAX_DAYS = 3650;
|
|
15
|
+
/** Source files the ratio pass reads per call, and how many source bytes it counts in total. */
|
|
16
|
+
export const MAX_RATIO_SOURCES = 10000;
|
|
17
|
+
export const MAX_RATIO_SOURCE_BYTES = 64 * 1024 * 1024;
|
|
18
|
+
/** The caps one ratio call works within; the defaults are the two bounds above, and a test lowers them. */
|
|
19
|
+
export interface RatioLimits { sources: number; bytes: number }
|
|
20
|
+
export const RATIO_LIMITS: RatioLimits = { sources: MAX_RATIO_SOURCES, bytes: MAX_RATIO_SOURCE_BYTES };
|
|
21
|
+
/** One bounded source read; `noteSourceRatio` takes one so a test can count the reads a pass really makes. */
|
|
22
|
+
export type SourceFileRead = (file: string, maximum: number) => { bytes: number } | null;
|
|
23
|
+
/** Notes listed as the largest, and the ratio above which a note is larger than the sources it cites. */
|
|
24
|
+
export const RATIO_LARGEST_SHOWN = 5;
|
|
25
|
+
export const RATIO_OVER_ONE = 1;
|
|
26
|
+
export interface VaultRatio {
|
|
27
|
+
/** Notes whose sources could be read and total more than zero bytes. */
|
|
28
|
+
notes: number;
|
|
29
|
+
/** Source files read within the per-call bounds; a zero-byte file is read and counted but leaves its note out of the ratio. */
|
|
30
|
+
sources: number;
|
|
31
|
+
/** Source references left out of the ratio; `skippedNonFile` of them are not workspace files at all. */
|
|
32
|
+
skipped: number;
|
|
33
|
+
skippedNonFile: number;
|
|
34
|
+
/**
|
|
35
|
+
* Notes left out because their readable sources total zero bytes. Such a note has no ratio, and counting it here
|
|
36
|
+
* keeps `skipped` a count of source references: its sources were read and are already in `sources`.
|
|
37
|
+
*/
|
|
38
|
+
emptySourceNotes: number;
|
|
39
|
+
/**
|
|
40
|
+
* Sourced notes the index kept out of the pass (`MAX_RATIO_NOTES`), counted rather than dropped silently. The
|
|
41
|
+
* index stops collecting at the cap, so the median, p90 and the rest describe the notes it returned, not all of them.
|
|
42
|
+
*/
|
|
43
|
+
truncatedNotes: number;
|
|
44
|
+
/** Median and nearest-rank p90 of the per-note ratio: note bytes over the bytes of its readable sources. */
|
|
45
|
+
median: number | null;
|
|
46
|
+
p90: number | null;
|
|
47
|
+
/** Notes whose ratio is above `RATIO_OVER_ONE`, so the note is larger than its sources. */
|
|
48
|
+
overOne: number;
|
|
49
|
+
largest: { path: string; ratio: number }[];
|
|
50
|
+
}
|
|
11
51
|
export interface VaultStats {
|
|
12
52
|
vault: string;
|
|
13
53
|
/** Local calendar date the windows end on (today included). */
|
|
14
54
|
asOf: string;
|
|
55
|
+
/** Note text size in bytes and words, by type and domain; "(none)" groups an empty type or domain. */
|
|
56
|
+
size: { total: NoteSize; byType: Record<string, NoteSize>; byDomain: Record<string, NoteSize> };
|
|
57
|
+
/** How large the notes are next to the workspace files they cite. */
|
|
58
|
+
ratio: VaultRatio;
|
|
15
59
|
notes: { total: number; byType: Record<string, number>; byDomain: Record<string, number>; byStatus: Record<string, number> };
|
|
16
60
|
/** Notes whose indexed mtime falls within the last 7 and 30 local calendar days, today included. */
|
|
17
61
|
modified: { last7Days: number; last30Days: number };
|
|
18
62
|
/** Log events dated on or after `since` (local date), by type; types outside the known set count as "(other)". */
|
|
19
63
|
log: { days: number; since: string; total: number | null; byType: Record<string, number> | null; warnings: string[]; issues: WriteIssue[] };
|
|
20
64
|
}
|
|
65
|
+
const round3 = (value: number) => Math.round(value * 1000) / 1000;
|
|
66
|
+
const middle = (sorted: readonly number[]) => {
|
|
67
|
+
const half = sorted.length >> 1;
|
|
68
|
+
return sorted.length % 2 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2;
|
|
69
|
+
};
|
|
70
|
+
/** Nearest rank: the smallest sorted value that at least `p` percent of the values do not exceed. */
|
|
71
|
+
const percentile = (sorted: readonly number[], p: number) => sorted[Math.min(sorted.length, Math.max(1, Math.ceil((p / 100) * sorted.length))) - 1]!;
|
|
72
|
+
/**
|
|
73
|
+
* Note-to-source byte ratio over notes that declare a frontmatter `source`. Each entry is one note: its own byte size
|
|
74
|
+
* over the summed size of the sources that could be read. The reader is `analysisReadWithSize`, so every file keeps its
|
|
75
|
+
* bound, no link is followed, and the bytes counted are the file's own on-disk size rather than the size of its decoded
|
|
76
|
+
* text, which overstates a file that is not UTF-8. The caps bound the I/O, not just the counted total: a read asks only
|
|
77
|
+
* for the bytes the budget has left, the reader refuses a larger file from its fstat size without reading it, and once
|
|
78
|
+
* the budget is spent no further file is opened. Every read attempt, successful or not, counts against the file cap, so
|
|
79
|
+
* a run of failures or over-budget files cannot outrun it. A memory id, a URL, a denied, missing or oversized path and
|
|
80
|
+
* anything the caps leave out is a skipped source reference, which keeps the pass read-only and bounded; a note whose
|
|
81
|
+
* readable sources total zero bytes has no ratio and is counted in `emptySourceNotes` instead, so `skipped` stays a
|
|
82
|
+
* count of references and never repeats a file that was read. The per-note truncation count belongs to the index, so
|
|
83
|
+
* `vaultStats` adds it to this pass's result.
|
|
84
|
+
*/
|
|
85
|
+
export function noteSourceRatio(rows: readonly SourcedNote[], root: string | undefined, denyPaths: string[], limits: RatioLimits = RATIO_LIMITS, readFile: SourceFileRead = analysisReadWithSize): Omit<VaultRatio, "truncatedNotes"> {
|
|
86
|
+
const ratios: { path: string; ratio: number }[] = [];
|
|
87
|
+
let sources = 0, skipped = 0, skippedNonFile = 0, emptySourceNotes = 0, bytesRead = 0, attempts = 0;
|
|
88
|
+
for (const row of rows) {
|
|
89
|
+
let refs: SourceReference[];
|
|
90
|
+
try { refs = sourceReferences(row.source); } catch { skipped++; continue; }
|
|
91
|
+
let size = 0, used = 0;
|
|
92
|
+
for (const ref of refs) {
|
|
93
|
+
if (ref.kind !== "path") { skipped++; skippedNonFile++; continue; }
|
|
94
|
+
const remaining = limits.bytes - bytesRead;
|
|
95
|
+
// Stop before opening anything once a cap is used up, so an exhausted call does no further I/O.
|
|
96
|
+
if (!root || attempts >= limits.sources || remaining <= 0) { skipped++; continue; }
|
|
97
|
+
// Counted before the read, so a failed or over-budget attempt cannot slip past the file cap.
|
|
98
|
+
attempts++;
|
|
99
|
+
const file = resolve(root, ref.value);
|
|
100
|
+
let read: { bytes: number } | null;
|
|
101
|
+
// Ask for no more than the budget has left: the reader refuses a larger file from its fstat size, unread.
|
|
102
|
+
try { read = isDenied(canonicalPath(file), denyPaths) ? null : readFile(file, Math.min(MAX_AUX_BYTES, remaining)); } catch { read = null; }
|
|
103
|
+
// A denied, missing or over-budget file is a skip, and one that would pass the cap is left out whole.
|
|
104
|
+
if (read === null || read.bytes > remaining) { skipped++; continue; }
|
|
105
|
+
size += read.bytes; bytesRead += read.bytes; sources++; used++;
|
|
106
|
+
}
|
|
107
|
+
// A note whose sources are all empty has no ratio: count the note, not a reference, so `skipped` stays reference-only.
|
|
108
|
+
if (!used) continue;
|
|
109
|
+
if (size === 0) { emptySourceNotes++; continue; }
|
|
110
|
+
ratios.push({ path: row.path, ratio: row.bytes / size });
|
|
111
|
+
}
|
|
112
|
+
const values = ratios.map(entry => entry.ratio).sort((a, b) => a - b);
|
|
113
|
+
const largest = [...ratios].sort((a, b) => b.ratio - a.ratio || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)).slice(0, RATIO_LARGEST_SHOWN);
|
|
114
|
+
return {
|
|
115
|
+
notes: ratios.length, sources, skipped, skippedNonFile, emptySourceNotes,
|
|
116
|
+
median: values.length ? round3(middle(values)) : null, p90: values.length ? round3(percentile(values, 90)) : null,
|
|
117
|
+
overOne: ratios.filter(entry => entry.ratio > RATIO_OVER_ONE).length,
|
|
118
|
+
largest: largest.map(entry => ({ path: entry.path, ratio: round3(entry.ratio) })),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
21
121
|
/**
|
|
22
122
|
* Read-only vault statistics over the derived index (refreshed like every read tool) and the log streamed by `readLog`:
|
|
23
|
-
* no vault file or log is written and no vault or log lock is taken.
|
|
123
|
+
* no vault file or log is written and no vault or log lock is taken. Note sizes come from the same index pass, and the
|
|
124
|
+
* ratio reads only workspace files a note cites, through `analysisReadWithSize` and its bounds.
|
|
24
125
|
*/
|
|
25
|
-
export function vaultStats(home: string, id: string, options: { days?: number; now?: Date; expectedVault?: VaultSpec } = {}): VaultStats {
|
|
126
|
+
export function vaultStats(home: string, id: string, options: { days?: number; now?: Date; expectedVault?: VaultSpec; limits?: RatioLimits } = {}): VaultStats {
|
|
26
127
|
const days = options.days ?? STATS_DEFAULT_DAYS, now = options.now ?? new Date();
|
|
27
128
|
if (!Number.isInteger(days) || days < 1 || days > STATS_MAX_DAYS) throw new Error(`stats: days must be an integer 1..${STATS_MAX_DAYS}; omit it for the ${STATS_DEFAULT_DAYS}-day default`);
|
|
28
129
|
const registry = loadRegistry(home), vault = registry.vaults.find(v => v.id === id);
|
|
@@ -34,6 +135,8 @@ export function vaultStats(home: string, id: string, options: { days?: number; n
|
|
|
34
135
|
const index = new VaultIndex(home, vault, registry.vaults.map(v => v.path));
|
|
35
136
|
let notes: ReturnType<VaultIndex["noteStats"]>;
|
|
36
137
|
try { index.scan(); notes = index.noteStats([start(7).getTime(), start(30).getTime()]); } finally { index.close(); }
|
|
138
|
+
const size = notes.size, pass = noteSourceRatio(notes.sourced, sourceWorkspaceRoot(vault), registry.denyPaths ?? [], options.limits);
|
|
139
|
+
const ratio: VaultRatio = { ...pass, truncatedNotes: notes.truncatedSourced };
|
|
37
140
|
const since = localDate(start(days)), byType: Record<string, number> = Object.fromEntries(LOG_TYPES.map(type => [type, 0]));
|
|
38
141
|
let total = 0;
|
|
39
142
|
const log = readLog(home, { vault: vault.id, limit: 1, visit: line => {
|
|
@@ -42,9 +145,11 @@ export function vaultStats(home: string, id: string, options: { days?: number; n
|
|
|
42
145
|
const type = (LOG_TYPES as readonly string[]).includes(match[2]!) ? match[2]! : "(other)";
|
|
43
146
|
total++; byType[type] = (byType[type] ?? 0) + 1;
|
|
44
147
|
} });
|
|
45
|
-
const { modified: [last7Days = 0, last30Days = 0]
|
|
148
|
+
const { modified: [last7Days = 0, last30Days = 0] } = notes;
|
|
46
149
|
return {
|
|
47
|
-
vault: vault.id, asOf: localDate(now),
|
|
150
|
+
vault: vault.id, asOf: localDate(now), size, ratio,
|
|
151
|
+
notes: { total: notes.total, byType: notes.byType, byDomain: notes.byDomain, byStatus: notes.byStatus },
|
|
152
|
+
modified: { last7Days, last30Days },
|
|
48
153
|
log: { days, since, total: log.ok ? total : null, byType: log.ok ? byType : null, warnings: log.warnings, issues: log.issues },
|
|
49
154
|
};
|
|
50
155
|
}
|
package/src/write-commands.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { argumentError } from "./mcp.ts";
|
|
|
12
12
|
import { parseTidyMap, tidyVault, TIDY_RULES } from "./tidy.ts";
|
|
13
13
|
import { PLAIN, type TextOptions } from "./text-output.ts";
|
|
14
14
|
import { renderTidy, renderWriteReceipt } from "./write-output.ts";
|
|
15
|
-
const COMMANDS = new Set(["new", "patch", "append", "frontmatter", "link", "supersede", "move", "delete", "tidy", "report", "check"]);
|
|
15
|
+
const COMMANDS = new Set(["new", "patch", "append", "frontmatter", "link", "supersede", "merge", "move", "delete", "tidy", "report", "check"]);
|
|
16
16
|
export function isWriteCommand(command: string): boolean { return COMMANDS.has(command); }
|
|
17
17
|
function object(text: string, label: string): Record<string, unknown> {
|
|
18
18
|
let value: unknown;
|
|
@@ -48,12 +48,12 @@ export function runWriteCommand(command: string, args: string[], home: string, e
|
|
|
48
48
|
new: ["--type", "--domain", "--topics", "--description", "--title", "--source", "--confidence", "--status", "--extra", "--body"],
|
|
49
49
|
patch: ["--old", "--new", "--edits", "--expected-sha"], append: ["--section", "--markdown", "--expected-sha"],
|
|
50
50
|
frontmatter: ["--set", "--unset", "--expected-sha"], link: ["--relation", "--context"],
|
|
51
|
-
move: ["--fingerprint"], delete: ["--fingerprint"], tidy: ["--fingerprint", "--map"], report: ["--body"], check: ["--path"], supersede: [],
|
|
51
|
+
merge: ["--fingerprint"], move: ["--fingerprint"], delete: ["--fingerprint"], tidy: ["--fingerprint", "--map"], report: ["--body"], check: ["--path"], supersede: [],
|
|
52
52
|
};
|
|
53
53
|
// For tidy, --log selects the log-redaction rule; tidy always writes its own LINT event. Check never appends an event.
|
|
54
54
|
const values = valueOptions[command] ? [...valueOptions[command]!, ...(["tidy", "check"].includes(command) ? [] : ["--log"])] : undefined;
|
|
55
55
|
if (!values) throw new Error("unknown writer command");
|
|
56
|
-
const twoStep = ["move", "delete", "tidy"].includes(command);
|
|
56
|
+
const twoStep = ["merge", "move", "delete", "tidy"].includes(command);
|
|
57
57
|
const booleans = ["--json", "--dry-run", ...(twoStep ? ["--confirm"] : []), ...(command === "link" ? ["--one-way"] : []), ...(command === "tidy" ? TIDY_RULES.map(rule => `--${rule}`).concat("--all") : [])];
|
|
58
58
|
const flags: Record<string, string | boolean> = {}, positional: string[] = [];
|
|
59
59
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -108,6 +108,7 @@ export function runWriteCommand(command: string, args: string[], home: string, e
|
|
|
108
108
|
case "frontmatter": operation = { op: "frontmatter", path: target, set: optional("--set") ? object(required("--set"), "set") : {}, unset: optional("--unset")?.split(","), expectedSha: optional("--expected-sha") }; break;
|
|
109
109
|
case "link": operation = { op: "link", from: target, to: positional[2]!, relation: required("--relation") as Extract<WriteOp, { op: "link" }>["relation"], context: optional("--context"), bidirectional: !flags["--one-way"] }; break;
|
|
110
110
|
case "supersede": operation = { op: "supersede", old: target, next: positional[2]! }; break;
|
|
111
|
+
case "merge": operation = { op: "merge", keep: target, absorb: positional[2]! }; break;
|
|
111
112
|
case "move": operation = { op: "move", from: target, to: positional[2]! }; break;
|
|
112
113
|
case "delete": operation = { op: "delete", path: target }; break;
|
|
113
114
|
case "tidy": operation = { op: "tidy" }; break;
|