@pablotech/neuro 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +622 -0
- package/benchmarks/staleness-corpus.ts +603 -0
- package/canonical.ts +54 -0
- package/cli.ts +201 -0
- package/compare.ts +41 -0
- package/dag.ts +81 -0
- package/hash-node.ts +8 -0
- package/hash-web.ts +8 -0
- package/index.ts +11 -0
- package/markdown.ts +120 -0
- package/mermaid.ts +37 -0
- package/package.json +32 -0
- package/validate.ts +98 -0
package/cli.ts
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { Dag } from "./dag";
|
|
4
|
+
import { isStamped } from "./dag";
|
|
5
|
+
import type { Slices } from "./canonical";
|
|
6
|
+
import { canonicalFor, driftedKeys } from "./canonical";
|
|
7
|
+
import type { Finding } from "./validate";
|
|
8
|
+
import { validate } from "./validate";
|
|
9
|
+
import { renderMermaid, writeDagBlock } from "./mermaid";
|
|
10
|
+
import { dagFromFiles, parseVaultNode } from "./markdown";
|
|
11
|
+
import { sha256hex12 } from "./hash-node";
|
|
12
|
+
|
|
13
|
+
// The vault-mode front-end over the same Dag type the TypeScript manifest front-end uses
|
|
14
|
+
// (index.ts) — see README.md. Walks a directory tree of plain markdown/YAML vault files rather than
|
|
15
|
+
// a compiled manifest, for a non-code consumer that has no toolchain.
|
|
16
|
+
|
|
17
|
+
export type Command = "lint" | "mermaid" | "stale";
|
|
18
|
+
const COMMANDS: readonly Command[] = ["lint", "mermaid", "stale"];
|
|
19
|
+
|
|
20
|
+
export interface ParsedArgs {
|
|
21
|
+
command: Command;
|
|
22
|
+
dir: string;
|
|
23
|
+
write?: string;
|
|
24
|
+
update?: boolean;
|
|
25
|
+
json?: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Hand-rolled arg parsing — no new dependency for three flags.
|
|
29
|
+
export function parseArgs(argv: string[]): ParsedArgs {
|
|
30
|
+
const [command, dir, ...rest] = argv;
|
|
31
|
+
if (!command || !(COMMANDS as readonly string[]).includes(command)) {
|
|
32
|
+
throw new Error(`unknown subcommand "${command ?? ""}". Expected one of: ${COMMANDS.join(", ")}`);
|
|
33
|
+
}
|
|
34
|
+
if (!dir) throw new Error(`${command} requires a <dir> argument`);
|
|
35
|
+
const out: ParsedArgs = { command: command as Command, dir };
|
|
36
|
+
for (let i = 0; i < rest.length; i++) {
|
|
37
|
+
const a = rest[i];
|
|
38
|
+
if (a === "--write") out.write = rest[++i];
|
|
39
|
+
else if (a === "--update") out.update = true;
|
|
40
|
+
else if (a === "--json") out.json = true;
|
|
41
|
+
else throw new Error(`unknown flag "${a}"`);
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Recursive readdir, collecting *.md (frontmatter nodes) and *.neuro-pil.yml (bare folder
|
|
47
|
+
// manifests) into path -> raw text. Skips .git, node_modules, .neuro-pil (the stale stamp dir),
|
|
48
|
+
// and any other dotfile/dotdir.
|
|
49
|
+
export function walkVault(dir: string): Record<string, string> {
|
|
50
|
+
const out: Record<string, string> = {};
|
|
51
|
+
const walk = (d: string) => {
|
|
52
|
+
for (const entry of readdirSync(d, { withFileTypes: true })) {
|
|
53
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
54
|
+
const p = join(d, entry.name);
|
|
55
|
+
if (entry.isDirectory()) { walk(p); continue; }
|
|
56
|
+
if (entry.name.endsWith(".md") || entry.name.endsWith(".neuro-pil.yml")) out[p] = readFileSync(p, "utf8");
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
walk(dir);
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface LintResult {
|
|
64
|
+
findings: Finding[];
|
|
65
|
+
nodeCount: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// No sliceParity here — that check needs a host's slice map, and vault mode has no analogue for one
|
|
69
|
+
// (see canonical.ts); it's a TS-manifest-only lint, run separately by whichever host wires slices up.
|
|
70
|
+
export function runLint(dir: string): LintResult {
|
|
71
|
+
const dag = dagFromFiles(walkVault(dir));
|
|
72
|
+
return { findings: validate(dag), nodeCount: dag.nodes.length };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function runMermaid(dir: string): string {
|
|
76
|
+
return renderMermaid(dagFromFiles(walkVault(dir)));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// One slice per source node key -> that node's own raw file text, keyed by node key rather than
|
|
80
|
+
// path (a node's key comes from its frontmatter, not its filename).
|
|
81
|
+
function subjectOf(files: Record<string, string>): Record<string, string> {
|
|
82
|
+
const out: Record<string, string> = {};
|
|
83
|
+
for (const text of Object.values(files)) {
|
|
84
|
+
const meta = parseVaultNode(text);
|
|
85
|
+
if (meta) out[meta.node] = text;
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function slicesOf(dag: Dag): Slices<Record<string, string>> {
|
|
91
|
+
const slices: Slices<Record<string, string>> = {};
|
|
92
|
+
for (const n of dag.nodes) if (n.kind === "source") slices[n.key] = (subject) => subject[n.key];
|
|
93
|
+
return slices;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function stampPathOf(dir: string): string {
|
|
97
|
+
return join(dir, ".neuro-pil", "stamp.json");
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface StaleResult {
|
|
101
|
+
baseline: boolean;
|
|
102
|
+
drifted: string[];
|
|
103
|
+
nodeCount: number;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// The same nodeHashesOf/staleNodesOf pattern the TypeScript manifest front-end uses, applied to a
|
|
107
|
+
// vault directory instead of a compiled Dag. Read-only by default — the stamp is written only when
|
|
108
|
+
// opts.update is true, matching this package's "read-only first" doctrine for anything touching a
|
|
109
|
+
// real vault.
|
|
110
|
+
export function runStale(dir: string, opts: { update?: boolean } = {}): StaleResult {
|
|
111
|
+
const files = walkVault(dir);
|
|
112
|
+
const dag = dagFromFiles(files);
|
|
113
|
+
const subject = subjectOf(files);
|
|
114
|
+
const slices = slicesOf(dag);
|
|
115
|
+
const now: Record<string, string> = {};
|
|
116
|
+
for (const n of dag.nodes) if (isStamped(n)) now[n.key] = sha256hex12(canonicalFor(dag, subject, slices, n.key));
|
|
117
|
+
|
|
118
|
+
const stampPath = stampPathOf(dir);
|
|
119
|
+
const stamped: Record<string, string> | null = existsSync(stampPath)
|
|
120
|
+
? JSON.parse(readFileSync(stampPath, "utf8"))
|
|
121
|
+
: null;
|
|
122
|
+
const baseline = stamped === null;
|
|
123
|
+
const drifted = baseline ? [] : driftedKeys(now, stamped);
|
|
124
|
+
|
|
125
|
+
if (opts.update) {
|
|
126
|
+
mkdirSync(join(dir, ".neuro-pil"), { recursive: true });
|
|
127
|
+
writeFileSync(stampPath, JSON.stringify(now, null, 2) + "\n");
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { baseline, drifted, nodeCount: dag.nodes.length };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Drift is reported before the "wrote stamp" line, including under --update: the drift is measured
|
|
134
|
+
// against the *old* stamp (runStale computes it before overwriting), and without this a --update run
|
|
135
|
+
// that exits 1 would print nothing about why.
|
|
136
|
+
export function staleLines(
|
|
137
|
+
result: StaleResult,
|
|
138
|
+
opts: { dir: string; update?: boolean; json?: boolean },
|
|
139
|
+
): string[] {
|
|
140
|
+
if (opts.json) return [JSON.stringify({ ...result, updated: opts.update === true })];
|
|
141
|
+
|
|
142
|
+
const lines: string[] = [];
|
|
143
|
+
if (result.baseline) lines.push("no prior stamp — nothing to compare.");
|
|
144
|
+
else if (result.drifted.length === 0) lines.push(`neuro-pil: no drift across ${result.nodeCount} stamped nodes.`);
|
|
145
|
+
else for (const k of result.drifted) lines.push(`[stale] ${k}`);
|
|
146
|
+
|
|
147
|
+
if (opts.update) lines.push(`Wrote stamp for ${result.nodeCount} nodes into ${stampPathOf(opts.dir)}`);
|
|
148
|
+
return lines;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function requireDir(dir: string): void {
|
|
152
|
+
if (!existsSync(dir) || !statSync(dir).isDirectory()) {
|
|
153
|
+
throw new Error(`no such directory: ${dir}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function main() {
|
|
158
|
+
let parsed: ParsedArgs;
|
|
159
|
+
try {
|
|
160
|
+
parsed = parseArgs(process.argv.slice(2));
|
|
161
|
+
requireDir(parsed.dir);
|
|
162
|
+
} catch (err) {
|
|
163
|
+
process.stderr.write(`${(err as Error).message}\n`);
|
|
164
|
+
process.exitCode = 2;
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (parsed.command === "lint") {
|
|
169
|
+
const result = runLint(parsed.dir);
|
|
170
|
+
if (parsed.json) {
|
|
171
|
+
process.stdout.write(JSON.stringify(result) + "\n");
|
|
172
|
+
} else if (result.findings.length === 0) {
|
|
173
|
+
process.stdout.write(`neuro-pil: no findings across ${result.nodeCount} nodes.\n`);
|
|
174
|
+
} else {
|
|
175
|
+
for (const f of result.findings) process.stdout.write(`[${f.rule}] ${f.node}: ${f.message}\n`);
|
|
176
|
+
}
|
|
177
|
+
if (result.findings.length > 0) process.exitCode = 1;
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (parsed.command === "mermaid") {
|
|
182
|
+
if (parsed.write) {
|
|
183
|
+
const doc = readFileSync(parsed.write, "utf8");
|
|
184
|
+
const dag = dagFromFiles(walkVault(parsed.dir));
|
|
185
|
+
writeFileSync(parsed.write, writeDagBlock(doc, dag));
|
|
186
|
+
process.stdout.write(`Wrote ${dag.nodes.length}-node mermaid into ${parsed.write}\n`);
|
|
187
|
+
} else {
|
|
188
|
+
process.stdout.write(runMermaid(parsed.dir) + "\n");
|
|
189
|
+
}
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// stale
|
|
194
|
+
const result = runStale(parsed.dir, { update: parsed.update });
|
|
195
|
+
for (const line of staleLines(result, { dir: parsed.dir, update: parsed.update, json: parsed.json })) {
|
|
196
|
+
process.stdout.write(line + "\n");
|
|
197
|
+
}
|
|
198
|
+
if (result.drifted.length > 0) process.exitCode = 1;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (import.meta.url === `file://${process.argv[1]}`) main();
|
package/compare.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// A brain-comparison harness: given a case set, a set of candidate versions, a way to run one version
|
|
2
|
+
// against one case, and a way to score the result, report each version's scores. See
|
|
3
|
+
// ./ARCHITECTURE.md and ../akesi-pil/ARCHITECTURE.md for why this exists and what it deliberately
|
|
4
|
+
// doesn't do.
|
|
5
|
+
//
|
|
6
|
+
// `run` and `score` are the entire extension surface, on purpose: this does not call any model
|
|
7
|
+
// provider, does not assume Anthropic or any other vendor, and does not compete with an eval platform
|
|
8
|
+
// (PromptLayer, Langfuse, Portkey, Helicone, MLflow) an adopter may already run. Wire `run` to whichever
|
|
9
|
+
// SDK, self-hosted model, or nothing at all a given brain currently uses — the harness never sees it.
|
|
10
|
+
//
|
|
11
|
+
// `Case`, `Version`, and `Result` are deliberately opaque type parameters, not a shared `BrainEntry`
|
|
12
|
+
// shape: an adopter's own registry entry is passed through untouched as `Version`, whatever shape it
|
|
13
|
+
// happens to be.
|
|
14
|
+
|
|
15
|
+
export interface VersionScore<Version> {
|
|
16
|
+
version: Version;
|
|
17
|
+
scores: number[];
|
|
18
|
+
mean: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface Comparison<Version> {
|
|
22
|
+
perVersion: VersionScore<Version>[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function compareBrains<Case, Version, Result>(
|
|
26
|
+
cases: Case[],
|
|
27
|
+
versions: Version[],
|
|
28
|
+
run: (c: Case, v: Version) => Promise<Result>,
|
|
29
|
+
score: (result: Result, c: Case) => number | Promise<number>,
|
|
30
|
+
): Promise<Comparison<Version>> {
|
|
31
|
+
const perVersion: VersionScore<Version>[] = [];
|
|
32
|
+
for (const version of versions) {
|
|
33
|
+
const scores: number[] = [];
|
|
34
|
+
for (const c of cases) {
|
|
35
|
+
scores.push(await score(await run(c, version), c));
|
|
36
|
+
}
|
|
37
|
+
const mean = scores.length === 0 ? 0 : scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
38
|
+
perVersion.push({ version, scores, mean });
|
|
39
|
+
}
|
|
40
|
+
return { perVersion };
|
|
41
|
+
}
|
package/dag.ts
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// The generalized shape of a source -> derived -> leaf/projection dependency graph.
|
|
2
|
+
//
|
|
3
|
+
// upstreamOf/downstreamOf/sourceClosureOf are the dependency-closure walk any build-graph engine in
|
|
4
|
+
// this lineage needs (see README.md "Lineage") — the set of nodes to recompute is exactly the
|
|
5
|
+
// downstream closure of what changed, never a human-maintained list.
|
|
6
|
+
|
|
7
|
+
export type NodeKind = "source" | "derived" | "leaf" | "projection";
|
|
8
|
+
|
|
9
|
+
export interface DagNode {
|
|
10
|
+
key: string;
|
|
11
|
+
label: string;
|
|
12
|
+
kind: NodeKind;
|
|
13
|
+
inputs: string[]; // keys of upstream nodes
|
|
14
|
+
basis: string; // the human sentence describing what this node is derived from
|
|
15
|
+
// Commentary, not evidence — never allowed to feed a node other than a `noteSink` (validate.ts's
|
|
16
|
+
// note-feeds-non-sink rule). Kept as a flag rather than a fifth NodeKind so it never changes a
|
|
17
|
+
// node's `source`-set membership, which is what canonical hashing keys off — see canonical.ts.
|
|
18
|
+
note?: true;
|
|
19
|
+
// The only kind of node allowed to consume a `note` node in its `inputs`.
|
|
20
|
+
noteSink?: true;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface Dag {
|
|
24
|
+
nodes: DagNode[];
|
|
25
|
+
dagNode(key: string): DagNode | undefined;
|
|
26
|
+
// Transitive upstream (everything this node depends on, directly or indirectly).
|
|
27
|
+
upstreamOf(key: string): Set<string>;
|
|
28
|
+
// Transitive downstream (everything that would be invalidated if this node changed).
|
|
29
|
+
downstreamOf(key: string): Set<string>;
|
|
30
|
+
// The `source` nodes a node ultimately depends on (its transitive upstream, filtered to sources,
|
|
31
|
+
// plus itself if it is itself a source). Every derived node is a deterministic function of the
|
|
32
|
+
// source nodes above it, so hashing over exactly this closure invalidates precisely the nodes whose
|
|
33
|
+
// sources actually changed. Sorted for a stable canonical order.
|
|
34
|
+
sourceClosureOf(key: string): string[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Pins, stars, and other UI-only attention flags are deliberately NOT modeled here — never a node,
|
|
38
|
+
// never an edge. They must not affect any canonical string (see canonical.ts's pinNeutrality helper).
|
|
39
|
+
|
|
40
|
+
export function isStamped(node: DagNode): boolean {
|
|
41
|
+
return node.kind !== "projection";
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function defineDag(nodes: DagNode[]): Dag {
|
|
45
|
+
const byKey = new Map(nodes.map((n) => [n.key, n]));
|
|
46
|
+
const dagNode = (key: string) => byKey.get(key);
|
|
47
|
+
|
|
48
|
+
const upstreamOf = (key: string): Set<string> => {
|
|
49
|
+
const out = new Set<string>();
|
|
50
|
+
const walk = (k: string) => {
|
|
51
|
+
for (const dep of byKey.get(k)?.inputs ?? []) {
|
|
52
|
+
if (!out.has(dep)) { out.add(dep); walk(dep); }
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
walk(key);
|
|
56
|
+
return out;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const downstreamOf = (key: string): Set<string> => {
|
|
60
|
+
const children = (k: string) => nodes.filter((n) => n.inputs.includes(k)).map((n) => n.key);
|
|
61
|
+
const out = new Set<string>();
|
|
62
|
+
const walk = (k: string) => {
|
|
63
|
+
for (const c of children(k)) {
|
|
64
|
+
if (!out.has(c)) { out.add(c); walk(c); }
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
walk(key);
|
|
68
|
+
return out;
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const sourceKeys = new Set(nodes.filter((n) => n.kind === "source").map((n) => n.key));
|
|
72
|
+
|
|
73
|
+
const sourceClosureOf = (key: string): string[] => {
|
|
74
|
+
const closure = new Set<string>();
|
|
75
|
+
if (sourceKeys.has(key)) closure.add(key);
|
|
76
|
+
for (const u of upstreamOf(key)) if (sourceKeys.has(u)) closure.add(u);
|
|
77
|
+
return [...closure].sort();
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
return { nodes, dagNode, upstreamOf, downstreamOf, sourceClosureOf };
|
|
81
|
+
}
|
package/hash-node.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
// node:crypto variant of the truncated-SHA-256 hash. Kept separate from the isomorphic core (dag.ts,
|
|
4
|
+
// canonical.ts) so nothing that runs in a browser or a Cloudflare Pages Function ever pulls in
|
|
5
|
+
// node:crypto — see hash-web.ts for that path.
|
|
6
|
+
export function sha256hex12(str: string): string {
|
|
7
|
+
return createHash("sha256").update(str).digest("hex").slice(0, 12);
|
|
8
|
+
}
|
package/hash-web.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
const enc = new TextEncoder();
|
|
2
|
+
|
|
3
|
+
// SubtleCrypto variant of the truncated-SHA-256 hash, for the browser and Cloudflare Pages Functions
|
|
4
|
+
// (workerd exposes the same Web Crypto API). See hash-node.ts for the node:crypto twin.
|
|
5
|
+
export async function sha256hex12(str: string): Promise<string> {
|
|
6
|
+
const buf = await globalThis.crypto.subtle.digest("SHA-256", enc.encode(str) as BufferSource);
|
|
7
|
+
return [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 12);
|
|
8
|
+
}
|
package/index.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// The isomorphic core: safe to import from a browser bundle, a Cloudflare Pages Function, or a
|
|
2
|
+
// node:crypto-based CLI alike. Runtime-specific hashing lives in ./hash-node and ./hash-web instead —
|
|
3
|
+
// import those directly rather than adding a `node:`/DOM-specific dependency here.
|
|
4
|
+
export type { NodeKind, DagNode, Dag } from "./dag";
|
|
5
|
+
export { defineDag, isStamped } from "./dag";
|
|
6
|
+
export type { Slices } from "./canonical";
|
|
7
|
+
export { stableStringify, canonicalFor, canonicalMap, driftedKeys } from "./canonical";
|
|
8
|
+
export type { MermaidBlockMarkers } from "./mermaid";
|
|
9
|
+
export { renderMermaid, extractDagBlock, writeDagBlock, DEFAULT_MERMAID_MARKERS } from "./mermaid";
|
|
10
|
+
export type { Finding, ValidateOptions } from "./validate";
|
|
11
|
+
export { validate, sliceParity } from "./validate";
|
package/markdown.ts
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import type { Dag, DagNode, NodeKind } from "./dag";
|
|
2
|
+
import { defineDag } from "./dag";
|
|
3
|
+
|
|
4
|
+
// The second front-end for the same Dag type — a plain-markdown vault convention, alongside the
|
|
5
|
+
// TypeScript manifest front-end (defineDag). Built to prove the shape works for a no-toolchain
|
|
6
|
+
// consumer before any concrete vault needs it.
|
|
7
|
+
//
|
|
8
|
+
// Deliberately NOT a general YAML parser — only the flat scalar/flow-list/block-list/boolean subset
|
|
9
|
+
// this schema needs (no nesting beyond one list, no anchors, no block scalars). If a future vault's
|
|
10
|
+
// frontmatter needs more than this, that's the point at which a real YAML dependency earns its keep;
|
|
11
|
+
// not before there's a second real consumer to justify adding one.
|
|
12
|
+
|
|
13
|
+
export interface VaultNodeMeta {
|
|
14
|
+
node: string;
|
|
15
|
+
kind: NodeKind;
|
|
16
|
+
label?: string;
|
|
17
|
+
inputs?: string[];
|
|
18
|
+
basis?: string;
|
|
19
|
+
note?: boolean;
|
|
20
|
+
noteSink?: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---/;
|
|
24
|
+
|
|
25
|
+
// The frontmatter block's raw text (between the `---` delimiters), or null if the file has none —
|
|
26
|
+
// e.g. a `.neuro-pil.yml` folder manifest has no delimiters at all, just the block directly.
|
|
27
|
+
export function extractFrontmatter(text: string): string | null {
|
|
28
|
+
const m = FRONTMATTER.exec(text);
|
|
29
|
+
return m ? m[1] : null;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function unquote(s: string): string {
|
|
33
|
+
const v = s.trim();
|
|
34
|
+
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) return v.slice(1, -1);
|
|
35
|
+
return v;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function parseScalar(raw: string): string | boolean {
|
|
39
|
+
const v = raw.trim();
|
|
40
|
+
if (v === "true") return true;
|
|
41
|
+
if (v === "false") return false;
|
|
42
|
+
return unquote(v);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function parseFlowList(raw: string): string[] {
|
|
46
|
+
const inner = raw.trim().replace(/^\[/, "").replace(/\]$/, "");
|
|
47
|
+
if (!inner.trim()) return [];
|
|
48
|
+
return inner.split(",").map((s) => unquote(s));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// One key per line: `key: value`, `key: [a, b, c]` (flow list), or `key:` followed by ` - item`
|
|
52
|
+
// lines (block list). Not a general YAML parser — see the module comment above.
|
|
53
|
+
export function parseFrontmatterBlock(block: string): Record<string, unknown> {
|
|
54
|
+
const lines = block.split(/\r?\n/);
|
|
55
|
+
const out: Record<string, unknown> = {};
|
|
56
|
+
let currentListKey: string | null = null;
|
|
57
|
+
for (const line of lines) {
|
|
58
|
+
if (!line.trim()) continue;
|
|
59
|
+
const listItem = /^\s+-\s*(.+)$/.exec(line);
|
|
60
|
+
if (listItem && currentListKey) {
|
|
61
|
+
(out[currentListKey] as string[]).push(unquote(listItem[1]));
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
currentListKey = null;
|
|
65
|
+
const kv = /^([A-Za-z_][\w-]*):\s*(.*)$/.exec(line);
|
|
66
|
+
if (!kv) continue;
|
|
67
|
+
const [, key, rest] = kv;
|
|
68
|
+
if (rest.trim() === "") {
|
|
69
|
+
out[key] = [];
|
|
70
|
+
currentListKey = key;
|
|
71
|
+
} else if (rest.trim().startsWith("[")) {
|
|
72
|
+
out[key] = parseFlowList(rest);
|
|
73
|
+
} else {
|
|
74
|
+
out[key] = parseScalar(rest);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Parses one file's frontmatter (or, for a `.neuro-pil.yml` folder manifest, the whole file) into
|
|
81
|
+
// a node descriptor. Returns null for a file with no frontmatter, or missing the two required keys
|
|
82
|
+
// (`node`, `kind`) — silently skipped by dagFromFiles rather than thrown, since a vault will always
|
|
83
|
+
// have plenty of files (a README, a legal PDF's sibling notes) that were never meant to be graph
|
|
84
|
+
// nodes at all.
|
|
85
|
+
export function parseVaultNode(text: string): VaultNodeMeta | null {
|
|
86
|
+
const block = extractFrontmatter(text) ?? (/^[A-Za-z_][\w-]*:/.test(text) ? text : null);
|
|
87
|
+
if (!block) return null;
|
|
88
|
+
const parsed = parseFrontmatterBlock(block);
|
|
89
|
+
if (typeof parsed.node !== "string" || typeof parsed.kind !== "string") return null;
|
|
90
|
+
return {
|
|
91
|
+
node: parsed.node,
|
|
92
|
+
kind: parsed.kind as NodeKind,
|
|
93
|
+
label: typeof parsed.label === "string" ? parsed.label : undefined,
|
|
94
|
+
inputs: Array.isArray(parsed.inputs) ? (parsed.inputs as string[]) : [],
|
|
95
|
+
basis: typeof parsed.basis === "string" ? parsed.basis : "",
|
|
96
|
+
note: parsed.note === true,
|
|
97
|
+
noteSink: parsed.noteSink === true,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Builds a Dag from a set of already-read file contents (path -> text). The host walks the
|
|
102
|
+
// filesystem and reads files; this module only knows how to turn text into a Dag, not how to find
|
|
103
|
+
// it — keeps this library free of any node:fs dependency.
|
|
104
|
+
export function dagFromFiles(files: Record<string, string>): Dag {
|
|
105
|
+
const nodes: DagNode[] = [];
|
|
106
|
+
for (const text of Object.values(files)) {
|
|
107
|
+
const meta = parseVaultNode(text);
|
|
108
|
+
if (!meta) continue;
|
|
109
|
+
nodes.push({
|
|
110
|
+
key: meta.node,
|
|
111
|
+
label: meta.label ?? meta.node,
|
|
112
|
+
kind: meta.kind,
|
|
113
|
+
inputs: meta.inputs ?? [],
|
|
114
|
+
basis: meta.basis ?? "",
|
|
115
|
+
...(meta.note ? { note: true as const } : {}),
|
|
116
|
+
...(meta.noteSink ? { noteSink: true as const } : {}),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return defineDag(nodes);
|
|
120
|
+
}
|
package/mermaid.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Dag } from "./dag";
|
|
2
|
+
|
|
3
|
+
// A DAG diagram generated from the manifest itself, not hand-maintained, so the picture can never
|
|
4
|
+
// drift from the code. Keys off node key + label + inputs only, never `kind`, so it is unaffected by
|
|
5
|
+
// a source/derived rename.
|
|
6
|
+
|
|
7
|
+
export function renderMermaid(dag: Dag): string {
|
|
8
|
+
const lines = ["graph LR"];
|
|
9
|
+
for (const n of dag.nodes) lines.push(` ${n.key}["${n.label}"]`);
|
|
10
|
+
for (const n of dag.nodes) for (const dep of n.inputs) lines.push(` ${dep} --> ${n.key}`);
|
|
11
|
+
return lines.join("\n");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface MermaidBlockMarkers { start: string; end: string }
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_MERMAID_MARKERS: MermaidBlockMarkers = { start: "<!-- DAG:START -->", end: "<!-- DAG:END -->" };
|
|
17
|
+
|
|
18
|
+
// The mermaid body currently checked into a doc (between the markers), or null if the markers/block
|
|
19
|
+
// are missing. Shared by the writer and a drift test so both parse the block identically.
|
|
20
|
+
export function extractDagBlock(docText: string, markers: MermaidBlockMarkers = DEFAULT_MERMAID_MARKERS): string | null {
|
|
21
|
+
const s = docText.indexOf(markers.start);
|
|
22
|
+
const e = docText.indexOf(markers.end);
|
|
23
|
+
if (s < 0 || e < 0 || e < s) return null;
|
|
24
|
+
const between = docText.slice(s + markers.start.length, e);
|
|
25
|
+
const m = /```mermaid\n([\s\S]*?)\n```/.exec(between);
|
|
26
|
+
return m ? m[1] : null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Rewrites the fenced block between the markers to the current renderMermaid(dag) output. Throws if
|
|
30
|
+
// the markers aren't present, rather than silently no-op'ing on a doc that was never wired up.
|
|
31
|
+
export function writeDagBlock(docText: string, dag: Dag, markers: MermaidBlockMarkers = DEFAULT_MERMAID_MARKERS): string {
|
|
32
|
+
const s = docText.indexOf(markers.start);
|
|
33
|
+
const e = docText.indexOf(markers.end);
|
|
34
|
+
if (s < 0 || e < 0 || e < s) throw new Error(`markers ${markers.start} / ${markers.end} not found`);
|
|
35
|
+
const block = `${markers.start}\n\n\`\`\`mermaid\n${renderMermaid(dag)}\n\`\`\`\n\n${markers.end}`;
|
|
36
|
+
return docText.slice(0, s) + block + docText.slice(e + markers.end.length);
|
|
37
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pablotech/neuro",
|
|
3
|
+
"version": "0.1.22",
|
|
4
|
+
"description": "Dependency-graph, staleness and canonical-hashing engine: record what a value was derived from, and know later whether that evidence has moved. Domain-free.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "index.ts",
|
|
7
|
+
"types": "index.ts",
|
|
8
|
+
"engines": {
|
|
9
|
+
"node": ">=22"
|
|
10
|
+
},
|
|
11
|
+
"files": [
|
|
12
|
+
"*.ts",
|
|
13
|
+
"benchmarks/**/*.ts",
|
|
14
|
+
"!tests"
|
|
15
|
+
],
|
|
16
|
+
"exports": {
|
|
17
|
+
".": "./index.ts",
|
|
18
|
+
"./hash-node": "./hash-node.ts",
|
|
19
|
+
"./hash-web": "./hash-web.ts",
|
|
20
|
+
"./markdown": "./markdown.ts",
|
|
21
|
+
"./compare": "./compare.ts"
|
|
22
|
+
},
|
|
23
|
+
"scripts": {
|
|
24
|
+
"test": "vitest run",
|
|
25
|
+
"cli": "tsx cli.ts",
|
|
26
|
+
"bench": "tsx benchmarks/staleness-corpus.ts"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"tsx": "^4.22.3",
|
|
30
|
+
"vitest": "^4.1.8"
|
|
31
|
+
}
|
|
32
|
+
}
|
package/validate.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { Dag } from "./dag";
|
|
2
|
+
import type { Slices } from "./canonical";
|
|
3
|
+
|
|
4
|
+
export interface Finding {
|
|
5
|
+
rule: string;
|
|
6
|
+
node: string;
|
|
7
|
+
message: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface ValidateOptions {
|
|
11
|
+
// source-kind nodes with zero downstream consumers are flagged as `orphan` (dead wiring — data
|
|
12
|
+
// collected that nothing ever reads) unless their key is listed here. derived/leaf/projection nodes
|
|
13
|
+
// are exempt: a "generated together" or "input-local" artifact is routinely also a terminal report
|
|
14
|
+
// section shown directly to the user, not just an intermediate computation step — verified against
|
|
15
|
+
// the live Finding DAG, where several `derived` nodes (e.g. healthProgression) are legitimately
|
|
16
|
+
// terminal and would otherwise false-positive on every run.
|
|
17
|
+
terminalAllowlist?: string[];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Structural checks over the graph shape alone — no host data (a subject, a stamp, a slices map)
|
|
21
|
+
// required. See sliceParity() below for the one check that needs a host's slice map too.
|
|
22
|
+
export function validate(dag: Dag, opts: ValidateOptions = {}): Finding[] {
|
|
23
|
+
const findings: Finding[] = [];
|
|
24
|
+
const keys = new Set(dag.nodes.map((n) => n.key));
|
|
25
|
+
const allowlist = new Set(opts.terminalAllowlist ?? []);
|
|
26
|
+
|
|
27
|
+
const seenKeys = new Set<string>();
|
|
28
|
+
for (const n of dag.nodes) {
|
|
29
|
+
if (seenKeys.has(n.key)) findings.push({ rule: "duplicate-key", node: n.key, message: `duplicate node key "${n.key}"` });
|
|
30
|
+
seenKeys.add(n.key);
|
|
31
|
+
|
|
32
|
+
for (const dep of n.inputs) {
|
|
33
|
+
if (!keys.has(dep)) findings.push({ rule: "unknown-input", node: n.key, message: `"${n.key}" lists unknown input "${dep}"` });
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (n.kind === "source" && n.inputs.length > 0) {
|
|
37
|
+
findings.push({ rule: "source-has-inputs", node: n.key, message: `source node "${n.key}" declares inputs — sources are raw, never derived` });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
for (const n of dag.nodes) {
|
|
42
|
+
const path = cyclePath(dag, n.key);
|
|
43
|
+
if (path) findings.push({ rule: "cycle", node: n.key, message: `cycle: ${path.join(" -> ")}` });
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const downstreamCount = new Map<string, number>(dag.nodes.map((n) => [n.key, 0]));
|
|
47
|
+
for (const n of dag.nodes) for (const dep of n.inputs) downstreamCount.set(dep, (downstreamCount.get(dep) ?? 0) + 1);
|
|
48
|
+
for (const n of dag.nodes) {
|
|
49
|
+
if (n.kind === "source" && downstreamCount.get(n.key) === 0 && !allowlist.has(n.key)) {
|
|
50
|
+
findings.push({ rule: "orphan", node: n.key, message: `"${n.key}" (source) is consumed by nothing — collected but never read by anything downstream` });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
for (const n of dag.nodes) {
|
|
55
|
+
for (const dep of n.inputs) {
|
|
56
|
+
if (dag.dagNode(dep)?.note && !n.noteSink) {
|
|
57
|
+
findings.push({ rule: "note-feeds-non-sink", node: n.key, message: `"${n.key}" consumes note "${dep}" but isn't a noteSink — notes must never become evidence` });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return findings;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// The one node/edge relationship not captured by `Dag` alone: which source keys a host's slice map
|
|
66
|
+
// (the functions that turn a subject into hashable data — see canonicalFor in canonical.ts) actually
|
|
67
|
+
// covers. A source with no matching slice silently contributes nothing to any node's canonical string
|
|
68
|
+
// (canonicalFor's `slices[k]?.(subject)` -> undefined); a slice with no matching source is dead code.
|
|
69
|
+
export function sliceParity<T>(dag: Dag, slices: Slices<T>): Finding[] {
|
|
70
|
+
const findings: Finding[] = [];
|
|
71
|
+
const sourceKeys = new Set(dag.nodes.filter((n) => n.kind === "source").map((n) => n.key));
|
|
72
|
+
for (const k of sourceKeys) {
|
|
73
|
+
if (!(k in slices)) findings.push({ rule: "missing-slice", node: k, message: `source "${k}" has no matching slice function` });
|
|
74
|
+
}
|
|
75
|
+
for (const k of Object.keys(slices)) {
|
|
76
|
+
if (!sourceKeys.has(k)) findings.push({ rule: "missing-slice", node: k, message: `slice "${k}" doesn't match any source node` });
|
|
77
|
+
}
|
|
78
|
+
return findings;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// DFS returning the actual cycle path (not just a boolean) the first time `start` is revisited on its
|
|
82
|
+
// own walk. Re-run per node rather than memoized across the whole graph — the graphs this validates
|
|
83
|
+
// are tens of nodes, not thousands, so the O(n^2) worst case is not worth the complexity to avoid.
|
|
84
|
+
function cyclePath(dag: Dag, start: string): string[] | null {
|
|
85
|
+
const path: string[] = [];
|
|
86
|
+
const visit = (k: string): string[] | null => {
|
|
87
|
+
const idx = path.indexOf(k);
|
|
88
|
+
if (idx !== -1) return [...path.slice(idx), k];
|
|
89
|
+
path.push(k);
|
|
90
|
+
for (const dep of dag.dagNode(k)?.inputs ?? []) {
|
|
91
|
+
const found = visit(dep);
|
|
92
|
+
if (found) return found;
|
|
93
|
+
}
|
|
94
|
+
path.pop();
|
|
95
|
+
return null;
|
|
96
|
+
};
|
|
97
|
+
return visit(start);
|
|
98
|
+
}
|