@monoes/monomindcli 2.7.12 → 2.7.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/mastermind/createorg.md +2 -0
- package/.claude/commands/mastermind/okf-import.md +6 -1
- package/.claude/helpers/handlers/route-handler.cjs +40 -7
- package/.claude/helpers/handlers/session-restore-handler.cjs +24 -6
- package/.claude/skills/mastermind-createorg/SKILL.md +11 -4
- package/README.md +12 -6
- package/dist/src/capabilities/index.d.ts.map +1 -1
- package/dist/src/capabilities/index.js +17 -0
- package/dist/src/capabilities/index.js.map +1 -1
- package/dist/src/capabilities/types.d.ts +0 -11
- package/dist/src/capabilities/types.d.ts.map +1 -1
- package/dist/src/commands/doc.d.ts.map +1 -1
- package/dist/src/commands/doc.js +251 -8
- package/dist/src/commands/doc.js.map +1 -1
- package/dist/src/commands/doctor-project-checks.d.ts +25 -3
- package/dist/src/commands/doctor-project-checks.d.ts.map +1 -1
- package/dist/src/commands/doctor-project-checks.js +111 -8
- package/dist/src/commands/doctor-project-checks.js.map +1 -1
- package/dist/src/commands/doctor.d.ts.map +1 -1
- package/dist/src/commands/doctor.js +18 -2
- package/dist/src/commands/doctor.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +32 -3
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/monograph.d.ts.map +1 -1
- package/dist/src/commands/monograph.js +26 -3
- package/dist/src/commands/monograph.js.map +1 -1
- package/dist/src/commands/org.d.ts +15 -0
- package/dist/src/commands/org.d.ts.map +1 -1
- package/dist/src/commands/org.js +251 -6
- package/dist/src/commands/org.js.map +1 -1
- package/dist/src/init/claudemd-generator.d.ts.map +1 -1
- package/dist/src/init/claudemd-generator.js +4 -1
- package/dist/src/init/claudemd-generator.js.map +1 -1
- package/dist/src/knowledge/document-pipeline.d.ts +77 -4
- package/dist/src/knowledge/document-pipeline.d.ts.map +1 -1
- package/dist/src/knowledge/document-pipeline.js +261 -15
- package/dist/src/knowledge/document-pipeline.js.map +1 -1
- package/dist/src/knowledge/eval/corpus.d.ts +56 -0
- package/dist/src/knowledge/eval/corpus.d.ts.map +1 -0
- package/dist/src/knowledge/eval/corpus.js +126 -0
- package/dist/src/knowledge/eval/corpus.js.map +1 -0
- package/dist/src/knowledge/eval/golden-set.d.ts +62 -0
- package/dist/src/knowledge/eval/golden-set.d.ts.map +1 -0
- package/dist/src/knowledge/eval/golden-set.js +331 -0
- package/dist/src/knowledge/eval/golden-set.js.map +1 -0
- package/dist/src/knowledge/eval/harness.d.ts +221 -0
- package/dist/src/knowledge/eval/harness.d.ts.map +1 -0
- package/dist/src/knowledge/eval/harness.js +610 -0
- package/dist/src/knowledge/eval/harness.js.map +1 -0
- package/dist/src/knowledge/eval/metrics.d.ts +120 -0
- package/dist/src/knowledge/eval/metrics.d.ts.map +1 -0
- package/dist/src/knowledge/eval/metrics.js +243 -0
- package/dist/src/knowledge/eval/metrics.js.map +1 -0
- package/dist/src/knowledge/eval/model-presence.d.ts +48 -0
- package/dist/src/knowledge/eval/model-presence.d.ts.map +1 -0
- package/dist/src/knowledge/eval/model-presence.js +128 -0
- package/dist/src/knowledge/eval/model-presence.js.map +1 -0
- package/dist/src/knowledge/eval/network-guard.d.ts +46 -0
- package/dist/src/knowledge/eval/network-guard.d.ts.map +1 -0
- package/dist/src/knowledge/eval/network-guard.js +112 -0
- package/dist/src/knowledge/eval/network-guard.js.map +1 -0
- package/dist/src/knowledge/eval/retrievers.d.ts +65 -0
- package/dist/src/knowledge/eval/retrievers.d.ts.map +1 -0
- package/dist/src/knowledge/eval/retrievers.js +180 -0
- package/dist/src/knowledge/eval/retrievers.js.map +1 -0
- package/dist/src/knowledge/eval/signals.d.ts +111 -0
- package/dist/src/knowledge/eval/signals.d.ts.map +1 -0
- package/dist/src/knowledge/eval/signals.js +232 -0
- package/dist/src/knowledge/eval/signals.js.map +1 -0
- package/dist/src/mcp-tools/agent-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/agent-tools.js +7 -34
- package/dist/src/mcp-tools/agent-tools.js.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.js +88 -6
- package/dist/src/mcp-tools/knowledge-tools.js.map +1 -1
- package/dist/src/mcp-tools/quality/coverage-analysis/prioritize-gaps.d.ts +44 -144
- package/dist/src/mcp-tools/quality/coverage-analysis/prioritize-gaps.d.ts.map +1 -1
- package/dist/src/mcp-tools/quality/security-compliance/detect-secrets.d.ts +30 -38
- package/dist/src/mcp-tools/quality/security-compliance/detect-secrets.d.ts.map +1 -1
- package/dist/src/mcp-tools/task-tools.d.ts +1 -0
- package/dist/src/mcp-tools/task-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/task-tools.js +76 -111
- package/dist/src/mcp-tools/task-tools.js.map +1 -1
- package/dist/src/mcp-tools/terminal-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/terminal-tools.js +3 -28
- package/dist/src/mcp-tools/terminal-tools.js.map +1 -1
- package/dist/src/memory/bm25-index.d.ts +137 -0
- package/dist/src/memory/bm25-index.d.ts.map +1 -0
- package/dist/src/memory/bm25-index.js +219 -0
- package/dist/src/memory/bm25-index.js.map +1 -0
- package/dist/src/memory/embedding-operations.d.ts.map +1 -1
- package/dist/src/memory/embedding-operations.js +8 -7
- package/dist/src/memory/embedding-operations.js.map +1 -1
- package/dist/src/memory/ewc-consolidation.d.ts.map +1 -1
- package/dist/src/memory/ewc-consolidation.js +2 -1
- package/dist/src/memory/ewc-consolidation.js.map +1 -1
- package/dist/src/memory/hnsw-operations.d.ts.map +1 -1
- package/dist/src/memory/hnsw-operations.js +4 -3
- package/dist/src/memory/hnsw-operations.js.map +1 -1
- package/dist/src/memory/intelligence.d.ts.map +1 -1
- package/dist/src/memory/intelligence.js +2 -1
- package/dist/src/memory/intelligence.js.map +1 -1
- package/dist/src/memory/memory-bridge.d.ts +34 -0
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +184 -26
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/memory/memory-initializer.d.ts.map +1 -1
- package/dist/src/memory/memory-initializer.js +4 -3
- package/dist/src/memory/memory-initializer.js.map +1 -1
- package/dist/src/memory/memory-schema.d.ts +1 -1
- package/dist/src/memory/memory-schema.js +1 -1
- package/dist/src/memory/text-tokens.d.ts +10 -0
- package/dist/src/memory/text-tokens.d.ts.map +1 -0
- package/dist/src/memory/text-tokens.js +37 -0
- package/dist/src/memory/text-tokens.js.map +1 -0
- package/dist/src/orgrt/daemon.d.ts +16 -1
- package/dist/src/orgrt/daemon.d.ts.map +1 -1
- package/dist/src/orgrt/daemon.js +65 -9
- package/dist/src/orgrt/daemon.js.map +1 -1
- package/dist/src/orgrt/scheduler.d.ts +14 -1
- package/dist/src/orgrt/scheduler.d.ts.map +1 -1
- package/dist/src/orgrt/scheduler.js +51 -5
- package/dist/src/orgrt/scheduler.js.map +1 -1
- package/dist/src/orgrt/session.js +12 -0
- package/dist/src/orgrt/session.js.map +1 -1
- package/dist/src/orgrt/types.d.ts +61 -831
- package/dist/src/orgrt/types.d.ts.map +1 -1
- package/dist/src/orgrt/types.js +7 -1
- package/dist/src/orgrt/types.js.map +1 -1
- package/dist/src/services/crash-reporter.d.ts.map +1 -1
- package/dist/src/services/crash-reporter.js +7 -0
- package/dist/src/services/crash-reporter.js.map +1 -1
- package/dist/src/ui/dashboard.html +57 -25
- package/dist/src/ui/orgs.html +30 -13
- package/dist/src/ui/routes-monograph.mjs +929 -0
- package/dist/src/ui/routes-org.mjs +2513 -0
- package/dist/src/ui/server.mjs +726 -4117
- package/dist/src/utils/json-file.d.ts +11 -0
- package/dist/src/utils/json-file.d.ts.map +1 -1
- package/dist/src/utils/json-file.js +27 -0
- package/dist/src/utils/json-file.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +8 -6
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic evaluation corpus.
|
|
3
|
+
*
|
|
4
|
+
* Reproducibility rules this file exists to enforce:
|
|
5
|
+
* - The corpus is derived from git-tracked files only, so a clean checkout at
|
|
6
|
+
* a given commit produces exactly the same document set.
|
|
7
|
+
* - The list is sorted by path, so ordering never depends on filesystem walk
|
|
8
|
+
* order (which differs between macOS and Linux).
|
|
9
|
+
* - Every run records a corpusHash over (path, sha256) pairs. Two scoreboard
|
|
10
|
+
* rows with different corpusHash values are NOT comparable.
|
|
11
|
+
*
|
|
12
|
+
* @module v1/cli/knowledge/eval/corpus
|
|
13
|
+
*/
|
|
14
|
+
export interface CorpusDoc {
|
|
15
|
+
/** Repo-relative POSIX path — the stable document id used in the golden set. */
|
|
16
|
+
id: string;
|
|
17
|
+
absPath: string;
|
|
18
|
+
bytes: number;
|
|
19
|
+
sha256: string;
|
|
20
|
+
}
|
|
21
|
+
export interface Corpus {
|
|
22
|
+
docs: CorpusDoc[];
|
|
23
|
+
corpusHash: string;
|
|
24
|
+
repoRoot: string;
|
|
25
|
+
/** Paths matched by the include rules but dropped, with the reason. */
|
|
26
|
+
excluded: Array<{
|
|
27
|
+
id: string;
|
|
28
|
+
reason: string;
|
|
29
|
+
}>;
|
|
30
|
+
/**
|
|
31
|
+
* Byte-identical documents collapsed to one content unit.
|
|
32
|
+
*
|
|
33
|
+
* This repo ships a verbatim mirror of `.claude/**` inside the CLI package
|
|
34
|
+
* (an npm-packaging requirement), so ~200 documents exist twice. Without
|
|
35
|
+
* this, a retriever that returns the identical twin of the gold document
|
|
36
|
+
* scores a MISS for returning the correct content — which measures path
|
|
37
|
+
* luck, not retrieval. Identical bytes are one document.
|
|
38
|
+
*/
|
|
39
|
+
canonicalOf: Map<string, string>;
|
|
40
|
+
/** Number of distinct content units (<= docs.length). */
|
|
41
|
+
contentUnits: number;
|
|
42
|
+
duplicateGroups: number;
|
|
43
|
+
/** AppleDouble `._` resource forks found. Must be zero. */
|
|
44
|
+
appleDoubleCount: number;
|
|
45
|
+
}
|
|
46
|
+
/** AppleDouble resource forks. Per-SEGMENT anchored, so it catches
|
|
47
|
+
* `docs/concepts/._memory.md` and not merely a leading dot at the repo root. */
|
|
48
|
+
export declare const APPLEDOUBLE_RE: RegExp;
|
|
49
|
+
/** The repository top level. Without this the corpus silently differs
|
|
50
|
+
* depending on which subdirectory you ran from — and two scoreboard rows
|
|
51
|
+
* would be incomparable for a reason nobody would think to check. */
|
|
52
|
+
export declare function resolveRepoRoot(start: string): string;
|
|
53
|
+
export declare function listTrackedFiles(repoRoot: string): string[];
|
|
54
|
+
export declare function buildCorpus(startDir: string): Corpus;
|
|
55
|
+
export declare function readDoc(doc: CorpusDoc): string;
|
|
56
|
+
//# sourceMappingURL=corpus.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../../../src/knowledge/eval/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,MAAM,WAAW,SAAS;IACxB,gFAAgF;IAChF,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD;;;;;;;;OAQG;IACH,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,yDAAyD;IACzD,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAOD;iFACiF;AACjF,eAAO,MAAM,cAAc,QAAc,CAAC;AAc1C;;sEAEsE;AACtE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMrD;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAK3D;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAyDpD;AAED,wBAAgB,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE9C"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic evaluation corpus.
|
|
3
|
+
*
|
|
4
|
+
* Reproducibility rules this file exists to enforce:
|
|
5
|
+
* - The corpus is derived from git-tracked files only, so a clean checkout at
|
|
6
|
+
* a given commit produces exactly the same document set.
|
|
7
|
+
* - The list is sorted by path, so ordering never depends on filesystem walk
|
|
8
|
+
* order (which differs between macOS and Linux).
|
|
9
|
+
* - Every run records a corpusHash over (path, sha256) pairs. Two scoreboard
|
|
10
|
+
* rows with different corpusHash values are NOT comparable.
|
|
11
|
+
*
|
|
12
|
+
* @module v1/cli/knowledge/eval/corpus
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from 'node:fs';
|
|
15
|
+
import * as path from 'node:path';
|
|
16
|
+
import * as crypto from 'node:crypto';
|
|
17
|
+
import { execFileSync } from 'node:child_process';
|
|
18
|
+
/** Extensions the document pipeline can actually extract text from and that
|
|
19
|
+
* carry prose. Source files are deliberately out: the golden set judges
|
|
20
|
+
* document retrieval, and the code graph is a separate surface. */
|
|
21
|
+
const CORPUS_EXTENSIONS = new Set(['.md']);
|
|
22
|
+
/** AppleDouble resource forks. Per-SEGMENT anchored, so it catches
|
|
23
|
+
* `docs/concepts/._memory.md` and not merely a leading dot at the repo root. */
|
|
24
|
+
export const APPLEDOUBLE_RE = /(^|\/)\._/;
|
|
25
|
+
/** Directories whose contents are generated, vendored, or per-machine state. */
|
|
26
|
+
const EXCLUDE_PATTERNS = [
|
|
27
|
+
/(^|\/)node_modules\//,
|
|
28
|
+
/(^|\/)dist\//,
|
|
29
|
+
/(^|\/)\.monomind\//,
|
|
30
|
+
/(^|\/)coverage\//,
|
|
31
|
+
APPLEDOUBLE_RE, // exFAT resource-fork turds
|
|
32
|
+
];
|
|
33
|
+
/** Files too small to contain a retrievable answer. */
|
|
34
|
+
const MIN_BYTES = 400;
|
|
35
|
+
/** The repository top level. Without this the corpus silently differs
|
|
36
|
+
* depending on which subdirectory you ran from — and two scoreboard rows
|
|
37
|
+
* would be incomparable for a reason nobody would think to check. */
|
|
38
|
+
export function resolveRepoRoot(start) {
|
|
39
|
+
try {
|
|
40
|
+
return execFileSync('git', ['-C', start, 'rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim();
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return path.resolve(start);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export function listTrackedFiles(repoRoot) {
|
|
47
|
+
const out = execFileSync('git', ['-C', repoRoot, 'ls-files', '-z'], {
|
|
48
|
+
encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
|
|
49
|
+
});
|
|
50
|
+
return out.split('\0').filter(Boolean);
|
|
51
|
+
}
|
|
52
|
+
export function buildCorpus(startDir) {
|
|
53
|
+
const repoRoot = resolveRepoRoot(startDir);
|
|
54
|
+
const excluded = [];
|
|
55
|
+
const docs = [];
|
|
56
|
+
const tracked = listTrackedFiles(repoRoot).sort();
|
|
57
|
+
// Counted from the RAW tracked list, before any exclusion runs.
|
|
58
|
+
//
|
|
59
|
+
// This was a bug, caught by a deliberate-violation test: the count used to be
|
|
60
|
+
// derived from `docs`, which the exclusion pattern had already emptied of
|
|
61
|
+
// `._` files. It was therefore always 0, and the "hard failure" downstream
|
|
62
|
+
// could never fire — a silent filter wearing an assert's name. A guard with
|
|
63
|
+
// no reachable firing condition is an assumption, not a guard.
|
|
64
|
+
const appleDoubleCount = tracked.filter(rel => APPLEDOUBLE_RE.test(rel) && CORPUS_EXTENSIONS.has(path.extname(rel).toLowerCase())).length;
|
|
65
|
+
for (const rel of tracked) {
|
|
66
|
+
if (!CORPUS_EXTENSIONS.has(path.extname(rel).toLowerCase()))
|
|
67
|
+
continue;
|
|
68
|
+
if (EXCLUDE_PATTERNS.some(re => re.test(rel))) {
|
|
69
|
+
excluded.push({ id: rel, reason: 'excluded path' });
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const abs = path.join(repoRoot, rel);
|
|
73
|
+
let stat;
|
|
74
|
+
try {
|
|
75
|
+
stat = fs.statSync(abs);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
excluded.push({ id: rel, reason: 'unreadable' });
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (!stat.isFile())
|
|
82
|
+
continue;
|
|
83
|
+
if (stat.size < MIN_BYTES) {
|
|
84
|
+
excluded.push({ id: rel, reason: 'under ' + MIN_BYTES + ' bytes' });
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
const content = fs.readFileSync(abs);
|
|
88
|
+
docs.push({
|
|
89
|
+
id: rel,
|
|
90
|
+
absPath: abs,
|
|
91
|
+
bytes: stat.size,
|
|
92
|
+
sha256: crypto.createHash('sha256').update(content).digest('hex'),
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
// Collapse byte-identical documents. Canonical = shortest path, ties broken
|
|
96
|
+
// lexicographically, so the choice is deterministic and never depends on
|
|
97
|
+
// filesystem order.
|
|
98
|
+
const byHash = new Map();
|
|
99
|
+
for (const d of docs) {
|
|
100
|
+
const arr = byHash.get(d.sha256);
|
|
101
|
+
if (arr)
|
|
102
|
+
arr.push(d.id);
|
|
103
|
+
else
|
|
104
|
+
byHash.set(d.sha256, [d.id]);
|
|
105
|
+
}
|
|
106
|
+
const canonicalOf = new Map();
|
|
107
|
+
let duplicateGroups = 0;
|
|
108
|
+
for (const [, ids] of byHash) {
|
|
109
|
+
const canon = [...ids].sort((a, b) => a.length - b.length || a.localeCompare(b))[0];
|
|
110
|
+
for (const id of ids)
|
|
111
|
+
canonicalOf.set(id, canon);
|
|
112
|
+
if (ids.length > 1)
|
|
113
|
+
duplicateGroups++;
|
|
114
|
+
}
|
|
115
|
+
const h = crypto.createHash('sha256');
|
|
116
|
+
for (const d of docs)
|
|
117
|
+
h.update(d.id).update('\0').update(d.sha256).update('\n');
|
|
118
|
+
return {
|
|
119
|
+
docs, corpusHash: h.digest('hex').slice(0, 16), repoRoot, excluded,
|
|
120
|
+
canonicalOf, contentUnits: byHash.size, duplicateGroups, appleDoubleCount,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
export function readDoc(doc) {
|
|
124
|
+
return fs.readFileSync(doc.absPath, 'utf8');
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../../../../src/knowledge/eval/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAiClD;;oEAEoE;AACpE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAE3C;iFACiF;AACjF,MAAM,CAAC,MAAM,cAAc,GAAG,WAAW,CAAC;AAE1C,gFAAgF;AAChF,MAAM,gBAAgB,GAAG;IACvB,sBAAsB;IACtB,cAAc;IACd,oBAAoB;IACpB,kBAAkB;IAClB,cAAc,EAAU,4BAA4B;CACrD,CAAC;AAEF,uDAAuD;AACvD,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB;;sEAEsE;AACtE,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,iBAAiB,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE;QAClE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;KAC9C,CAAC,CAAC;IACH,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAA0C,EAAE,CAAC;IAC3D,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IAElD,gEAAgE;IAChE,EAAE;IACF,8EAA8E;IAC9E,0EAA0E;IAC1E,2EAA2E;IAC3E,4EAA4E;IAC5E,+DAA+D;IAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CACrC,GAAG,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAC1F,CAAC,MAAM,CAAC;IAET,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACtE,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACjH,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAc,CAAC;QACnB,IAAI,CAAC;YAAC,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACtG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,SAAS;QAC7B,IAAI,IAAI,CAAC,IAAI,GAAG,SAAS,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7G,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC;YACR,EAAE,EAAE,GAAG;YACP,OAAO,EAAE,GAAG;YACZ,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAClE,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;;YAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpF,KAAK,MAAM,EAAE,IAAI,GAAG;YAAE,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACjD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,eAAe,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEhF,OAAO;QACL,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ;QAClE,WAAW,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,eAAe,EAAE,gBAAgB;KAC1E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAc;IACpC,OAAO,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local golden set: query -> relevant document id pairs.
|
|
3
|
+
*
|
|
4
|
+
* CONSTRUCTION RULES (enforced by the harness, not just by good intentions):
|
|
5
|
+
* 1. Every `relevant` id is a repo-relative path that must exist in the
|
|
6
|
+
* deterministic corpus. Unknown ids are a hard failure, never a silent skip.
|
|
7
|
+
* 2. Queries are written in DIFFERENT vocabulary from the source document —
|
|
8
|
+
* the wording a person would use months later having forgotten the doc's
|
|
9
|
+
* own terms. Pairs whose query turns out to be near-verbatim in the target
|
|
10
|
+
* are removed automatically by `assessTriviality` and reported.
|
|
11
|
+
* 3. Relevance is pinned to LIVE documents: the eval store is rebuilt from the
|
|
12
|
+
* corpus with exactly one ingest per document, so it contains no superseded
|
|
13
|
+
* versions at all. Deleting dead rows from the user's store (backlog item 4)
|
|
14
|
+
* therefore cannot move these numbers.
|
|
15
|
+
* 4. Ambiguous targets are avoided. Where several near-identical documents
|
|
16
|
+
* could each legitimately answer, either the query is sharpened or all
|
|
17
|
+
* acceptable documents are listed in `relevant`.
|
|
18
|
+
*
|
|
19
|
+
* @module v1/cli/knowledge/eval/golden-set
|
|
20
|
+
*/
|
|
21
|
+
export interface GoldenPair {
|
|
22
|
+
id: string;
|
|
23
|
+
query: string;
|
|
24
|
+
/** Repo-relative paths. A hit on ANY counts for hit-rate; recall uses all. */
|
|
25
|
+
relevant: string[];
|
|
26
|
+
/** Free-form grouping for slice reporting. */
|
|
27
|
+
tags?: string[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* v1 of the set. Authored 2026-07-28 by `benchmark-engineer` against documents
|
|
31
|
+
* read at authoring time. Growing this set is ongoing work; the count is
|
|
32
|
+
* reported honestly by the harness rather than padded with near-duplicates.
|
|
33
|
+
*/
|
|
34
|
+
export declare const GOLDEN_SET: GoldenPair[];
|
|
35
|
+
export type Split = 'dev' | 'test';
|
|
36
|
+
/** Share of pairs assigned to the sealed test split. */
|
|
37
|
+
export declare const TEST_SHARE = 0.3;
|
|
38
|
+
/** Identifies the assignment scheme. Bump this whenever assignment changes:
|
|
39
|
+
* numbers measured under different schemes are not comparable, and the
|
|
40
|
+
* regression suite uses this to say so instead of inventing a regression. */
|
|
41
|
+
export declare const SPLIT_SCHEME = "hash-threshold-v2";
|
|
42
|
+
/**
|
|
43
|
+
* Stratified 70/30 dev/test split, STABLE UNDER GROWTH.
|
|
44
|
+
*
|
|
45
|
+
* The assignment is a pure function of the pair's own id — `pairHash(id) <
|
|
46
|
+
* TEST_SHARE` — so adding pairs never moves an existing pair between splits.
|
|
47
|
+
*
|
|
48
|
+
* The first version ranked pairs within their tag group and took the lowest
|
|
49
|
+
* 30%, which gave an exact 70/30 per tag but was NOT stable: appending one
|
|
50
|
+
* pair to a group re-ranked it and silently reassigned existing pairs. Growing
|
|
51
|
+
* the set would have quietly moved queries out of the sealed half and into the
|
|
52
|
+
* tunable one, which is the seal failing open — and it would have done so with
|
|
53
|
+
* no visible symptom at all.
|
|
54
|
+
*
|
|
55
|
+
* Stratification survives because the hash is uniform and independent of the
|
|
56
|
+
* tag, so each tag group lands near 30% on average rather than exactly. That
|
|
57
|
+
* is the right trade: an approximate stratification that cannot leak beats an
|
|
58
|
+
* exact one that can.
|
|
59
|
+
*/
|
|
60
|
+
export declare function assignSplits(pairs?: GoldenPair[]): Map<string, Split>;
|
|
61
|
+
export declare function pairsForSplit(split: Split | 'all', pairs?: GoldenPair[]): GoldenPair[];
|
|
62
|
+
//# sourceMappingURL=golden-set.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"golden-set.d.ts","sourceRoot":"","sources":["../../../../src/knowledge/eval/golden-set.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;GAIG;AACH,eAAO,MAAM,UAAU,EAAE,UAAU,EAwHlC,CAAC;AAsBF,MAAM,MAAM,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AASnC,wDAAwD;AACxD,eAAO,MAAM,UAAU,MAAM,CAAC;AAE9B;;8EAE8E;AAC9E,eAAO,MAAM,YAAY,sBAAsB,CAAC;AAEhD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,YAAY,CAAC,KAAK,GAAE,UAAU,EAAe,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAIjF;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,GAAE,UAAU,EAAe,GAAG,UAAU,EAAE,CAIlG"}
|