@monoes/monomindcli 2.7.13 → 2.7.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/mastermind/createorg.md +2 -0
- package/.claude/commands/mastermind/okf-import.md +6 -1
- package/.claude/helpers/handlers/route-handler.cjs +40 -7
- package/.claude/helpers/handlers/session-restore-handler.cjs +24 -6
- package/.claude/skills/mastermind-createorg/SKILL.md +11 -4
- package/README.md +12 -6
- package/dist/src/capabilities/index.d.ts.map +1 -1
- package/dist/src/capabilities/index.js +17 -0
- package/dist/src/capabilities/index.js.map +1 -1
- package/dist/src/capabilities/types.d.ts +0 -11
- package/dist/src/capabilities/types.d.ts.map +1 -1
- package/dist/src/commands/doc.d.ts.map +1 -1
- package/dist/src/commands/doc.js +251 -8
- package/dist/src/commands/doc.js.map +1 -1
- package/dist/src/commands/doctor-project-checks.d.ts +25 -3
- package/dist/src/commands/doctor-project-checks.d.ts.map +1 -1
- package/dist/src/commands/doctor-project-checks.js +111 -8
- package/dist/src/commands/doctor-project-checks.js.map +1 -1
- package/dist/src/commands/doctor.d.ts.map +1 -1
- package/dist/src/commands/doctor.js +18 -2
- package/dist/src/commands/doctor.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +32 -3
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/monograph.d.ts.map +1 -1
- package/dist/src/commands/monograph.js +26 -3
- package/dist/src/commands/monograph.js.map +1 -1
- package/dist/src/commands/org.d.ts +15 -0
- package/dist/src/commands/org.d.ts.map +1 -1
- package/dist/src/commands/org.js +251 -6
- package/dist/src/commands/org.js.map +1 -1
- package/dist/src/init/claudemd-generator.d.ts.map +1 -1
- package/dist/src/init/claudemd-generator.js +4 -1
- package/dist/src/init/claudemd-generator.js.map +1 -1
- package/dist/src/knowledge/document-pipeline.d.ts +77 -4
- package/dist/src/knowledge/document-pipeline.d.ts.map +1 -1
- package/dist/src/knowledge/document-pipeline.js +261 -15
- package/dist/src/knowledge/document-pipeline.js.map +1 -1
- package/dist/src/knowledge/eval/corpus.d.ts +56 -0
- package/dist/src/knowledge/eval/corpus.d.ts.map +1 -0
- package/dist/src/knowledge/eval/corpus.js +126 -0
- package/dist/src/knowledge/eval/corpus.js.map +1 -0
- package/dist/src/knowledge/eval/golden-set.d.ts +62 -0
- package/dist/src/knowledge/eval/golden-set.d.ts.map +1 -0
- package/dist/src/knowledge/eval/golden-set.js +331 -0
- package/dist/src/knowledge/eval/golden-set.js.map +1 -0
- package/dist/src/knowledge/eval/harness.d.ts +221 -0
- package/dist/src/knowledge/eval/harness.d.ts.map +1 -0
- package/dist/src/knowledge/eval/harness.js +610 -0
- package/dist/src/knowledge/eval/harness.js.map +1 -0
- package/dist/src/knowledge/eval/metrics.d.ts +120 -0
- package/dist/src/knowledge/eval/metrics.d.ts.map +1 -0
- package/dist/src/knowledge/eval/metrics.js +243 -0
- package/dist/src/knowledge/eval/metrics.js.map +1 -0
- package/dist/src/knowledge/eval/model-presence.d.ts +48 -0
- package/dist/src/knowledge/eval/model-presence.d.ts.map +1 -0
- package/dist/src/knowledge/eval/model-presence.js +128 -0
- package/dist/src/knowledge/eval/model-presence.js.map +1 -0
- package/dist/src/knowledge/eval/network-guard.d.ts +46 -0
- package/dist/src/knowledge/eval/network-guard.d.ts.map +1 -0
- package/dist/src/knowledge/eval/network-guard.js +112 -0
- package/dist/src/knowledge/eval/network-guard.js.map +1 -0
- package/dist/src/knowledge/eval/retrievers.d.ts +65 -0
- package/dist/src/knowledge/eval/retrievers.d.ts.map +1 -0
- package/dist/src/knowledge/eval/retrievers.js +180 -0
- package/dist/src/knowledge/eval/retrievers.js.map +1 -0
- package/dist/src/knowledge/eval/signals.d.ts +111 -0
- package/dist/src/knowledge/eval/signals.d.ts.map +1 -0
- package/dist/src/knowledge/eval/signals.js +232 -0
- package/dist/src/knowledge/eval/signals.js.map +1 -0
- package/dist/src/mcp-tools/agent-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/agent-tools.js +7 -34
- package/dist/src/mcp-tools/agent-tools.js.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.js +88 -6
- package/dist/src/mcp-tools/knowledge-tools.js.map +1 -1
- package/dist/src/mcp-tools/task-tools.d.ts +1 -0
- package/dist/src/mcp-tools/task-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/task-tools.js +76 -111
- package/dist/src/mcp-tools/task-tools.js.map +1 -1
- package/dist/src/mcp-tools/terminal-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/terminal-tools.js +3 -28
- package/dist/src/mcp-tools/terminal-tools.js.map +1 -1
- package/dist/src/memory/bm25-index.d.ts +137 -0
- package/dist/src/memory/bm25-index.d.ts.map +1 -0
- package/dist/src/memory/bm25-index.js +219 -0
- package/dist/src/memory/bm25-index.js.map +1 -0
- package/dist/src/memory/embedding-operations.d.ts.map +1 -1
- package/dist/src/memory/embedding-operations.js +8 -7
- package/dist/src/memory/embedding-operations.js.map +1 -1
- package/dist/src/memory/ewc-consolidation.d.ts.map +1 -1
- package/dist/src/memory/ewc-consolidation.js +2 -1
- package/dist/src/memory/ewc-consolidation.js.map +1 -1
- package/dist/src/memory/hnsw-operations.d.ts.map +1 -1
- package/dist/src/memory/hnsw-operations.js +4 -3
- package/dist/src/memory/hnsw-operations.js.map +1 -1
- package/dist/src/memory/intelligence.d.ts.map +1 -1
- package/dist/src/memory/intelligence.js +2 -1
- package/dist/src/memory/intelligence.js.map +1 -1
- package/dist/src/memory/memory-bridge.d.ts +34 -0
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +184 -26
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/memory/memory-initializer.d.ts.map +1 -1
- package/dist/src/memory/memory-initializer.js +4 -3
- package/dist/src/memory/memory-initializer.js.map +1 -1
- package/dist/src/memory/memory-schema.d.ts +1 -1
- package/dist/src/memory/memory-schema.js +1 -1
- package/dist/src/memory/text-tokens.d.ts +10 -0
- package/dist/src/memory/text-tokens.d.ts.map +1 -0
- package/dist/src/memory/text-tokens.js +37 -0
- package/dist/src/memory/text-tokens.js.map +1 -0
- package/dist/src/orgrt/daemon.d.ts +16 -1
- package/dist/src/orgrt/daemon.d.ts.map +1 -1
- package/dist/src/orgrt/daemon.js +65 -9
- package/dist/src/orgrt/daemon.js.map +1 -1
- package/dist/src/orgrt/scheduler.d.ts +14 -1
- package/dist/src/orgrt/scheduler.d.ts.map +1 -1
- package/dist/src/orgrt/scheduler.js +51 -5
- package/dist/src/orgrt/scheduler.js.map +1 -1
- package/dist/src/orgrt/session.js +12 -0
- package/dist/src/orgrt/session.js.map +1 -1
- package/dist/src/orgrt/types.d.ts +3 -0
- package/dist/src/orgrt/types.d.ts.map +1 -1
- package/dist/src/orgrt/types.js +7 -1
- package/dist/src/orgrt/types.js.map +1 -1
- package/dist/src/services/crash-reporter.d.ts.map +1 -1
- package/dist/src/services/crash-reporter.js +7 -0
- package/dist/src/services/crash-reporter.js.map +1 -1
- package/dist/src/ui/dashboard.html +13 -10
- package/dist/src/ui/orgs.html +30 -13
- package/dist/src/ui/routes-monograph.mjs +929 -0
- package/dist/src/ui/routes-org.mjs +2513 -0
- package/dist/src/ui/server.mjs +726 -4117
- package/dist/src/utils/json-file.d.ts +11 -0
- package/dist/src/utils/json-file.d.ts.map +1 -1
- package/dist/src/utils/json-file.js +27 -0
- package/dist/src/utils/json-file.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Query-time network guard.
|
|
3
|
+
*
|
|
4
|
+
* The stop condition says "zero network calls at query time". This module does
|
|
5
|
+
* not *infer* that from reading code — it BLOCKS the network and records every
|
|
6
|
+
* attempt. If retrieval needs the network, queries fail loudly instead of
|
|
7
|
+
* quietly succeeding and leaving us to assert offline-ness we never tested.
|
|
8
|
+
*
|
|
9
|
+
* Honest limits, stated so nobody over-claims from this evidence:
|
|
10
|
+
* - Coverage is layered, and the layers are not equally strong:
|
|
11
|
+
* * `net.Socket.prototype.connect` / `tls.TLSSocket.prototype.connect` are
|
|
12
|
+
* PROTOTYPE patches, so they bite no matter how the caller imported the
|
|
13
|
+
* module. Every TCP client in Node funnels through them, including
|
|
14
|
+
* undici (and therefore fetch), http, https and any npm HTTP library.
|
|
15
|
+
* This is the layer that actually makes the claim true.
|
|
16
|
+
* * `globalThis.fetch` is a global patch and equally universal.
|
|
17
|
+
* * The module-level patches (http.request, dns.lookup, ...) only bite for
|
|
18
|
+
* `require()` consumers: an ESM `import * as http` snapshots the
|
|
19
|
+
* bindings at load time and cannot be reached afterwards. They are kept
|
|
20
|
+
* as a second net, not relied on.
|
|
21
|
+
* - A native addon opening a socket in C++ would bypass all of it, which is
|
|
22
|
+
* why the baseline report also carries an out-of-process `lsof` check.
|
|
23
|
+
* - It is installed around the query phase only. Ingest may legitimately fetch
|
|
24
|
+
* an embedding model on first run; that is build time, not query time, and
|
|
25
|
+
* is reported separately.
|
|
26
|
+
*
|
|
27
|
+
* @module v1/cli/knowledge/eval/network-guard
|
|
28
|
+
*/
|
|
29
|
+
export interface NetworkAttempt {
|
|
30
|
+
api: string;
|
|
31
|
+
target: string;
|
|
32
|
+
stack: string;
|
|
33
|
+
}
|
|
34
|
+
export interface NetworkGuard {
|
|
35
|
+
attempts: NetworkAttempt[];
|
|
36
|
+
/** APIs that could NOT be patched. A non-empty list downgrades the offline
|
|
37
|
+
* claim from "proven" to "partial" — never silently ignored. */
|
|
38
|
+
unpatched: string[];
|
|
39
|
+
release(): void;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Installs the guard. Every blocked call throws, so any code path that needed
|
|
43
|
+
* the network surfaces as a failed query rather than a silent pass.
|
|
44
|
+
*/
|
|
45
|
+
export declare function installNetworkGuard(): NetworkGuard;
|
|
46
|
+
//# sourceMappingURL=network-guard.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"network-guard.d.ts","sourceRoot":"","sources":["../../../../src/knowledge/eval/network-guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAWH,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B;qEACiE;IACjE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,IAAI,IAAI,CAAC;CACjB;AAcD;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,YAAY,CAsDlD"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Query-time network guard.
|
|
3
|
+
*
|
|
4
|
+
* The stop condition says "zero network calls at query time". This module does
|
|
5
|
+
* not *infer* that from reading code — it BLOCKS the network and records every
|
|
6
|
+
* attempt. If retrieval needs the network, queries fail loudly instead of
|
|
7
|
+
* quietly succeeding and leaving us to assert offline-ness we never tested.
|
|
8
|
+
*
|
|
9
|
+
* Honest limits, stated so nobody over-claims from this evidence:
|
|
10
|
+
* - Coverage is layered, and the layers are not equally strong:
|
|
11
|
+
* * `net.Socket.prototype.connect` / `tls.TLSSocket.prototype.connect` are
|
|
12
|
+
* PROTOTYPE patches, so they bite no matter how the caller imported the
|
|
13
|
+
* module. Every TCP client in Node funnels through them, including
|
|
14
|
+
* undici (and therefore fetch), http, https and any npm HTTP library.
|
|
15
|
+
* This is the layer that actually makes the claim true.
|
|
16
|
+
* * `globalThis.fetch` is a global patch and equally universal.
|
|
17
|
+
* * The module-level patches (http.request, dns.lookup, ...) only bite for
|
|
18
|
+
* `require()` consumers: an ESM `import * as http` snapshots the
|
|
19
|
+
* bindings at load time and cannot be reached afterwards. They are kept
|
|
20
|
+
* as a second net, not relied on.
|
|
21
|
+
* - A native addon opening a socket in C++ would bypass all of it, which is
|
|
22
|
+
* why the baseline report also carries an out-of-process `lsof` check.
|
|
23
|
+
* - It is installed around the query phase only. Ingest may legitimately fetch
|
|
24
|
+
* an embedding model on first run; that is build time, not query time, and
|
|
25
|
+
* is reported separately.
|
|
26
|
+
*
|
|
27
|
+
* @module v1/cli/knowledge/eval/network-guard
|
|
28
|
+
*/
|
|
29
|
+
import { createRequire } from 'node:module';
|
|
30
|
+
// Deliberately require() rather than `import * as http`. ESM namespace objects
|
|
31
|
+
// are non-configurable, so assigning to them throws `Cannot redefine property`
|
|
32
|
+
// — which is exactly how a network guard silently becomes decorative. The CJS
|
|
33
|
+
// module objects for Node builtins ARE writable, which is how every HTTP
|
|
34
|
+
// interception library works. A unit test asserts the guard actually blocks.
|
|
35
|
+
const req = createRequire(import.meta.url);
|
|
36
|
+
function describe(args) {
|
|
37
|
+
try {
|
|
38
|
+
const a = args[0];
|
|
39
|
+
if (typeof a === 'string')
|
|
40
|
+
return a;
|
|
41
|
+
if (a && typeof a === 'object') {
|
|
42
|
+
if (a.href)
|
|
43
|
+
return String(a.href);
|
|
44
|
+
if (a.hostname || a.host)
|
|
45
|
+
return String(a.hostname ?? a.host) + ':' + String(a.port ?? '');
|
|
46
|
+
}
|
|
47
|
+
return String(a);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return '<unprintable>';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Installs the guard. Every blocked call throws, so any code path that needed
|
|
55
|
+
* the network surfaces as a failed query rather than a silent pass.
|
|
56
|
+
*/
|
|
57
|
+
export function installNetworkGuard() {
|
|
58
|
+
const attempts = [];
|
|
59
|
+
const restores = [];
|
|
60
|
+
const block = (api) => (...args) => {
|
|
61
|
+
const err = new Error('[eval] BLOCKED network call at query time: ' + api + ' -> ' + describe(args));
|
|
62
|
+
attempts.push({ api, target: describe(args), stack: (err.stack ?? '').split('\n').slice(1, 8).join('\n') });
|
|
63
|
+
throw err;
|
|
64
|
+
};
|
|
65
|
+
/** Returns false when the property genuinely could not be replaced — the
|
|
66
|
+
* caller turns that into a reported gap rather than a silent pass. */
|
|
67
|
+
const failures = [];
|
|
68
|
+
const patch = (obj, key, api) => {
|
|
69
|
+
if (!obj || typeof obj[key] !== 'function')
|
|
70
|
+
return;
|
|
71
|
+
const orig = obj[key];
|
|
72
|
+
try {
|
|
73
|
+
Object.defineProperty(obj, key, { value: block(api), writable: true, configurable: true });
|
|
74
|
+
restores.push(() => { try {
|
|
75
|
+
Object.defineProperty(obj, key, { value: orig, writable: true, configurable: true });
|
|
76
|
+
}
|
|
77
|
+
catch { /* best effort */ } });
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
failures.push(api);
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
const http = req('node:http');
|
|
84
|
+
const https = req('node:https');
|
|
85
|
+
const net = req('node:net');
|
|
86
|
+
const tls = req('node:tls');
|
|
87
|
+
const dns = req('node:dns');
|
|
88
|
+
const g = globalThis;
|
|
89
|
+
if (typeof g.fetch === 'function') {
|
|
90
|
+
const orig = g.fetch;
|
|
91
|
+
g.fetch = block('fetch');
|
|
92
|
+
restores.push(() => { g.fetch = orig; });
|
|
93
|
+
}
|
|
94
|
+
patch(http, 'request', 'http.request');
|
|
95
|
+
patch(http, 'get', 'http.get');
|
|
96
|
+
patch(https, 'request', 'https.request');
|
|
97
|
+
patch(https, 'get', 'https.get');
|
|
98
|
+
patch(net, 'connect', 'net.connect');
|
|
99
|
+
patch(net, 'createConnection', 'net.createConnection');
|
|
100
|
+
patch(net.Socket.prototype, 'connect', 'net.Socket.connect');
|
|
101
|
+
patch(tls, 'connect', 'tls.connect');
|
|
102
|
+
patch(dns, 'lookup', 'dns.lookup');
|
|
103
|
+
patch(dns, 'resolve', 'dns.resolve');
|
|
104
|
+
patch(dns.promises, 'lookup', 'dns.promises.lookup');
|
|
105
|
+
return {
|
|
106
|
+
attempts,
|
|
107
|
+
unpatched: failures,
|
|
108
|
+
release() { for (const r of restores.reverse())
|
|
109
|
+
r(); },
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=network-guard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"network-guard.js","sourceRoot":"","sources":["../../../../src/knowledge/eval/network-guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,+EAA+E;AAC/E,+EAA+E;AAC/E,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAgB3C,SAAS,QAAQ,CAAC,IAAe;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAQ,CAAC;QACzB,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,IAAI;gBAAE,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI;gBAAE,OAAO,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,eAAe,CAAC;IAAC,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAG,IAAe,EAAS,EAAE;QAC3D,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,6CAA6C,GAAG,GAAG,GAAG,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACrG,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5G,MAAM,GAAG,CAAC;IACZ,CAAC,CAAC;IAEF;2EACuE;IACvE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAQ,EAAE,GAAW,EAAE,GAAW,EAAQ,EAAE;QACzD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,UAAU;YAAE,OAAO;QACnD,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3F,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;gBAAC,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrJ,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,CAAC,GAAG,UAAiB,CAAC;IAC5B,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC;QACrB,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAC/B,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IACjC,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrC,KAAK,CAAC,GAAG,EAAE,kBAAkB,EAAE,sBAAsB,CAAC,CAAC;IACvD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC;IAC7D,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IACnC,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,qBAAqB,CAAC,CAAC;IAErD,OAAO;QACL,QAAQ;QACR,SAAS,EAAE,QAAQ;QACnB,OAAO,KAAK,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE;YAAE,CAAC,EAAE,CAAC,CAAC,CAAC;KACvD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrievers under test, including the deliberately weak baselines.
|
|
3
|
+
*
|
|
4
|
+
* The weak baselines exist to answer one question the headline number cannot:
|
|
5
|
+
* how much of our score is the retrieval stack, and how much is a golden set
|
|
6
|
+
* so easy that keyword matching nearly solves it? A small gap between
|
|
7
|
+
* `bm25-only` and the real stack means the SET is uninformative, not that the
|
|
8
|
+
* stack is good. See Requirement C in the item-0 brief.
|
|
9
|
+
*
|
|
10
|
+
* @module v1/cli/knowledge/eval/retrievers
|
|
11
|
+
*/
|
|
12
|
+
export interface EvalChunk {
|
|
13
|
+
docId: string;
|
|
14
|
+
chunkIndex: number;
|
|
15
|
+
text: string;
|
|
16
|
+
}
|
|
17
|
+
export interface RawHit {
|
|
18
|
+
docId: string;
|
|
19
|
+
chunkIndex: number;
|
|
20
|
+
score: number;
|
|
21
|
+
}
|
|
22
|
+
export interface Retriever {
|
|
23
|
+
name: string;
|
|
24
|
+
description: string;
|
|
25
|
+
/** Must not make network calls. Returns chunk-level hits, best-first. */
|
|
26
|
+
search(query: string, limit: number): Promise<RawHit[]>;
|
|
27
|
+
}
|
|
28
|
+
export declare class Bm25Retriever implements Retriever {
|
|
29
|
+
private chunks;
|
|
30
|
+
name: string;
|
|
31
|
+
description: string;
|
|
32
|
+
private docs;
|
|
33
|
+
private df;
|
|
34
|
+
private avgLen;
|
|
35
|
+
constructor(chunks: EvalChunk[]);
|
|
36
|
+
search(query: string, limit: number): Promise<RawHit[]>;
|
|
37
|
+
}
|
|
38
|
+
export declare class RandomRetriever implements Retriever {
|
|
39
|
+
private seed;
|
|
40
|
+
name: string;
|
|
41
|
+
description: string;
|
|
42
|
+
private docIds;
|
|
43
|
+
constructor(chunks: EvalChunk[], seed?: string);
|
|
44
|
+
search(query: string, limit: number): Promise<RawHit[]>;
|
|
45
|
+
}
|
|
46
|
+
/** Wraps an arbitrary async search function (the real stack) as a Retriever. */
|
|
47
|
+
export declare class FnRetriever implements Retriever {
|
|
48
|
+
name: string;
|
|
49
|
+
description: string;
|
|
50
|
+
private fn;
|
|
51
|
+
constructor(name: string, description: string, fn: (query: string, limit: number) => Promise<RawHit[]>);
|
|
52
|
+
search(query: string, limit: number): Promise<RawHit[]>;
|
|
53
|
+
}
|
|
54
|
+
export declare class RrfRetriever implements Retriever {
|
|
55
|
+
private children;
|
|
56
|
+
/** RRF smoothing constant. Standard values: 10–100. */
|
|
57
|
+
private rrfK;
|
|
58
|
+
name: string;
|
|
59
|
+
description: string;
|
|
60
|
+
constructor(children: Retriever[],
|
|
61
|
+
/** RRF smoothing constant. Standard values: 10–100. */
|
|
62
|
+
rrfK: number, name?: string);
|
|
63
|
+
search(query: string, limit: number): Promise<RawHit[]>;
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=retrievers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retrievers.d.ts","sourceRoot":"","sources":["../../../../src/knowledge/eval/retrievers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACzD;AAQD,qBAAa,aAAc,YAAW,SAAS;IAQjC,OAAO,CAAC,MAAM;IAP1B,IAAI,SAAe;IACnB,WAAW,SAA0E;IAErF,OAAO,CAAC,IAAI,CAAkB;IAC9B,OAAO,CAAC,EAAE,CAA6B;IACvC,OAAO,CAAC,MAAM,CAAK;IAEnB,YAAoB,MAAM,EAAE,SAAS,EAAE,EAStC;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAyB5D;CACF;AAYD,qBAAa,eAAgB,YAAW,SAAS;IAMd,OAAO,CAAC,IAAI;IAL7C,IAAI,SAAY;IAChB,WAAW,SAAkE;IAE7E,OAAO,CAAC,MAAM,CAAW;IAEzB,YAAY,MAAM,EAAE,SAAS,EAAE,EAAU,IAAI,SAAsB,EAElE;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAe5D;CACF;AAED,gFAAgF;AAChF,qBAAa,WAAY,YAAW,SAAS;IAElC,IAAI,EAAE,MAAM;IACZ,WAAW,EAAE,MAAM;IAC1B,OAAO,CAAC,EAAE;IAHZ,YACS,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAClB,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,EAC7D;IACJ,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAkC;CAC1F;AAaD,qBAAa,YAAa,YAAW,SAAS;IAK1C,OAAO,CAAC,QAAQ;IAChB,uDAAuD;IACvD,OAAO,CAAC,IAAI;IANd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IAEpB,YACU,QAAQ,EAAE,SAAS,EAAE;IAC7B,uDAAuD;IAC/C,IAAI,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,MAAM,EAKd;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwC5D;CACF"}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrievers under test, including the deliberately weak baselines.
|
|
3
|
+
*
|
|
4
|
+
* The weak baselines exist to answer one question the headline number cannot:
|
|
5
|
+
* how much of our score is the retrieval stack, and how much is a golden set
|
|
6
|
+
* so easy that keyword matching nearly solves it? A small gap between
|
|
7
|
+
* `bm25-only` and the real stack means the SET is uninformative, not that the
|
|
8
|
+
* stack is good. See Requirement C in the item-0 brief.
|
|
9
|
+
*
|
|
10
|
+
* @module v1/cli/knowledge/eval/retrievers
|
|
11
|
+
*/
|
|
12
|
+
import { contentTokens } from './metrics.js';
|
|
13
|
+
// -- BM25 floor ------------------------------------------------------
|
|
14
|
+
// Okapi BM25 with the standard k1=1.2, b=0.75. This is OUR OWN scorer, run
|
|
15
|
+
// in-process over the same chunk set the dense index holds. It is NOT the
|
|
16
|
+
// SQLite FTS5 implementation that item 1 will add — the two will differ, and
|
|
17
|
+
// scoreboard rows must not treat them as interchangeable.
|
|
18
|
+
export class Bm25Retriever {
|
|
19
|
+
chunks;
|
|
20
|
+
name = 'bm25-only';
|
|
21
|
+
description = 'Okapi BM25 (k1=1.2, b=0.75) over the same chunks, no dense component';
|
|
22
|
+
docs = [];
|
|
23
|
+
df = new Map();
|
|
24
|
+
avgLen = 0;
|
|
25
|
+
constructor(chunks) {
|
|
26
|
+
this.chunks = chunks;
|
|
27
|
+
for (const c of chunks) {
|
|
28
|
+
const toks = contentTokens(c.text);
|
|
29
|
+
this.docs.push(toks);
|
|
30
|
+
for (const t of new Set(toks))
|
|
31
|
+
this.df.set(t, (this.df.get(t) ?? 0) + 1);
|
|
32
|
+
}
|
|
33
|
+
this.avgLen = this.docs.length === 0
|
|
34
|
+
? 0
|
|
35
|
+
: this.docs.reduce((a, d) => a + d.length, 0) / this.docs.length;
|
|
36
|
+
}
|
|
37
|
+
async search(query, limit) {
|
|
38
|
+
const k1 = 1.2, b = 0.75;
|
|
39
|
+
const N = this.docs.length;
|
|
40
|
+
const q = contentTokens(query);
|
|
41
|
+
const scored = [];
|
|
42
|
+
for (let i = 0; i < N; i++) {
|
|
43
|
+
const toks = this.docs[i];
|
|
44
|
+
if (toks.length === 0)
|
|
45
|
+
continue;
|
|
46
|
+
const tf = new Map();
|
|
47
|
+
for (const t of toks)
|
|
48
|
+
tf.set(t, (tf.get(t) ?? 0) + 1);
|
|
49
|
+
let score = 0;
|
|
50
|
+
for (const term of q) {
|
|
51
|
+
const f = tf.get(term);
|
|
52
|
+
if (!f)
|
|
53
|
+
continue;
|
|
54
|
+
const n = this.df.get(term) ?? 0;
|
|
55
|
+
const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));
|
|
56
|
+
score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + b * (toks.length / this.avgLen))));
|
|
57
|
+
}
|
|
58
|
+
if (score > 0) {
|
|
59
|
+
scored.push({ docId: this.chunks[i].docId, chunkIndex: this.chunks[i].chunkIndex, score });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
scored.sort((a, b2) => b2.score - a.score);
|
|
63
|
+
return scored.slice(0, limit);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// -- Random floor ----------------------------------------------------
|
|
67
|
+
// The true floor. Deterministic per (query, corpus) via a seeded PRNG so the
|
|
68
|
+
// number is reproducible on a clean checkout rather than a fresh coin flip.
|
|
69
|
+
function hashSeed(s) {
|
|
70
|
+
let h = 2166136261;
|
|
71
|
+
for (let i = 0; i < s.length; i++) {
|
|
72
|
+
h ^= s.charCodeAt(i);
|
|
73
|
+
h = Math.imul(h, 16777619);
|
|
74
|
+
}
|
|
75
|
+
return h >>> 0;
|
|
76
|
+
}
|
|
77
|
+
export class RandomRetriever {
|
|
78
|
+
seed;
|
|
79
|
+
name = 'random';
|
|
80
|
+
description = 'Deterministic seeded random document sample — the true floor';
|
|
81
|
+
docIds;
|
|
82
|
+
constructor(chunks, seed = 'second-brain-eval') {
|
|
83
|
+
this.seed = seed;
|
|
84
|
+
this.docIds = [...new Set(chunks.map(c => c.docId))].sort();
|
|
85
|
+
}
|
|
86
|
+
async search(query, limit) {
|
|
87
|
+
let state = hashSeed(this.seed + '::' + query);
|
|
88
|
+
const next = () => {
|
|
89
|
+
state ^= state << 13;
|
|
90
|
+
state >>>= 0;
|
|
91
|
+
state ^= state >> 17;
|
|
92
|
+
state ^= state << 5;
|
|
93
|
+
state >>>= 0;
|
|
94
|
+
return state / 0xffffffff;
|
|
95
|
+
};
|
|
96
|
+
const pool = [...this.docIds];
|
|
97
|
+
const out = [];
|
|
98
|
+
for (let i = 0; i < limit && pool.length > 0; i++) {
|
|
99
|
+
const idx = Math.floor(next() * pool.length) % pool.length;
|
|
100
|
+
out.push({ docId: pool.splice(idx, 1)[0], chunkIndex: 0, score: 1 - i / limit });
|
|
101
|
+
}
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/** Wraps an arbitrary async search function (the real stack) as a Retriever. */
|
|
106
|
+
export class FnRetriever {
|
|
107
|
+
name;
|
|
108
|
+
description;
|
|
109
|
+
fn;
|
|
110
|
+
constructor(name, description, fn) {
|
|
111
|
+
this.name = name;
|
|
112
|
+
this.description = description;
|
|
113
|
+
this.fn = fn;
|
|
114
|
+
}
|
|
115
|
+
search(query, limit) { return this.fn(query, limit); }
|
|
116
|
+
}
|
|
117
|
+
// -- RRF fusion -------------------------------------------------------
|
|
118
|
+
// Reciprocal Rank Fusion: fusionScore(d) = Σ 1/(k + rank_i) for each
|
|
119
|
+
// retriever i that returned document d. Equal weight across all sources.
|
|
120
|
+
//
|
|
121
|
+
// `score` on the returned RawHit is the RRF fusion score — a rank-derived
|
|
122
|
+
// quantity, NOT a similarity and NOT comparable to cosine or BM25 scores.
|
|
123
|
+
// Callers that need a per-source score must read the child retrievers
|
|
124
|
+
// directly. This is the null hypothesis row: equal weight, no query
|
|
125
|
+
// adaptation. Expected to fail the low-overlap gate because BM25 scores
|
|
126
|
+
// 0.182 in that tercile and equal-weight fusion drags dense down.
|
|
127
|
+
export class RrfRetriever {
|
|
128
|
+
children;
|
|
129
|
+
rrfK;
|
|
130
|
+
name;
|
|
131
|
+
description;
|
|
132
|
+
constructor(children,
|
|
133
|
+
/** RRF smoothing constant. Standard values: 10–100. */
|
|
134
|
+
rrfK, name) {
|
|
135
|
+
this.children = children;
|
|
136
|
+
this.rrfK = rrfK;
|
|
137
|
+
this.name = name ?? `rrf-k${rrfK}`;
|
|
138
|
+
this.description =
|
|
139
|
+
`Reciprocal Rank Fusion (k=${rrfK}, equal weight) over [${children.map(c => c.name).join(', ')}]`;
|
|
140
|
+
}
|
|
141
|
+
async search(query, limit) {
|
|
142
|
+
// Over-fetch from each child: a document at rank 50 in one child and
|
|
143
|
+
// rank 1 in another should still be fusible, so fetch more than `limit`.
|
|
144
|
+
const childLimit = limit * 5;
|
|
145
|
+
const childResults = await Promise.all(this.children.map(c => c.search(query, childLimit)));
|
|
146
|
+
// Build per-document fusion scores. Track the best chunk per document
|
|
147
|
+
// from whichever child scored it highest (for the chunkIndex field).
|
|
148
|
+
const fusion = new Map();
|
|
149
|
+
for (const hits of childResults) {
|
|
150
|
+
// Dedupe by docId within this child before assigning ranks.
|
|
151
|
+
const seen = new Set();
|
|
152
|
+
let rank = 0;
|
|
153
|
+
for (const h of hits) {
|
|
154
|
+
if (seen.has(h.docId))
|
|
155
|
+
continue;
|
|
156
|
+
seen.add(h.docId);
|
|
157
|
+
rank++;
|
|
158
|
+
const rrfScore = 1 / (this.rrfK + rank);
|
|
159
|
+
const existing = fusion.get(h.docId);
|
|
160
|
+
if (existing) {
|
|
161
|
+
existing.score += rrfScore;
|
|
162
|
+
if (h.score > existing.bestChildScore) {
|
|
163
|
+
existing.chunkIndex = h.chunkIndex;
|
|
164
|
+
existing.bestChildScore = h.score;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
fusion.set(h.docId, { score: rrfScore, chunkIndex: h.chunkIndex, bestChildScore: h.score });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const fused = [];
|
|
173
|
+
for (const [docId, v] of fusion) {
|
|
174
|
+
fused.push({ docId, chunkIndex: v.chunkIndex, score: v.score });
|
|
175
|
+
}
|
|
176
|
+
fused.sort((a, b) => b.score - a.score);
|
|
177
|
+
return fused.slice(0, limit);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=retrievers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retrievers.js","sourceRoot":"","sources":["../../../../src/knowledge/eval/retrievers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAqB7C,uEAAuE;AACvE,2EAA2E;AAC3E,0EAA0E;AAC1E,6EAA6E;AAC7E,0DAA0D;AAE1D,MAAM,OAAO,aAAa;IAQJ,MAAM;IAP1B,IAAI,GAAG,WAAW,CAAC;IACnB,WAAW,GAAG,sEAAsE,CAAC;IAE7E,IAAI,GAAe,EAAE,CAAC;IACtB,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/B,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAoB,MAAmB;sBAAnB,MAAM;QACxB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,KAAK,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAClC,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa;QACvC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAChC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;YACrC,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBACpD,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;YACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;CACF;AAED,uEAAuE;AACvE,6EAA6E;AAC7E,4EAA4E;AAE5E,SAAS,QAAQ,CAAC,CAAS;IACzB,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAAC,CAAC;IACxF,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,eAAe;IAMe,IAAI;IAL7C,IAAI,GAAG,QAAQ,CAAC;IAChB,WAAW,GAAG,8DAA8D,CAAC;IAErE,MAAM,CAAW;IAEzB,YAAY,MAAmB,EAAU,IAAI,GAAG,mBAAmB;oBAA1B,IAAI;QAC3C,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa;QACvC,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,KAAK,IAAI,KAAK,IAAI,EAAE,CAAC;YAAC,KAAK,MAAM,CAAC,CAAC;YACnC,KAAK,IAAI,KAAK,IAAI,EAAE,CAAC;YACrB,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;YAAC,KAAK,MAAM,CAAC,CAAC;YAClC,OAAO,KAAK,GAAG,UAAU,CAAC;QAC5B,CAAC,CAAC;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YAC3D,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED,gFAAgF;AAChF,MAAM,OAAO,WAAW;IAEb,IAAI;IACJ,WAAW;IACV,EAAE;IAHZ,YACS,IAAY,EACZ,WAAmB,EAClB,EAAuD;oBAFxD,IAAI;2BACJ,WAAW;kBACV,EAAE;IACT,CAAC;IACJ,MAAM,CAAC,KAAa,EAAE,KAAa,IAAuB,OAAO,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;CAC1F;AAED,wEAAwE;AACxE,qEAAqE;AACrE,yEAAyE;AACzE,EAAE;AACF,0EAA0E;AAC1E,0EAA0E;AAC1E,sEAAsE;AACtE,oEAAoE;AACpE,wEAAwE;AACxE,kEAAkE;AAElE,MAAM,OAAO,YAAY;IAKb,QAAQ;IAER,IAAI;IANd,IAAI,CAAS;IACb,WAAW,CAAS;IAEpB,YACU,QAAqB;IAC7B,uDAAuD;IAC/C,IAAY,EACpB,IAAa;wBAHL,QAAQ;oBAER,IAAI;QAGZ,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,QAAQ,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,WAAW;YACd,6BAA6B,IAAI,yBAAyB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IACtG,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa;QACvC,qEAAqE;QACrE,yEAAyE;QACzE,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC;QAEF,sEAAsE;QACtE,qEAAqE;QACrE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyE,CAAC;QAEhG,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,4DAA4D;YAC5D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAE,SAAS;gBAChC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,EAAE,CAAC;gBACP,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC;oBAC3B,IAAI,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,cAAc,EAAE,CAAC;wBACtC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;wBACnC,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC;oBACpC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,cAAc,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC9F,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-registered signals and the persistent regression suite.
|
|
3
|
+
*
|
|
4
|
+
* TWO DEFECTS IN THE SCOREBOARD THAT THIS FILE EXISTS TO FIX.
|
|
5
|
+
*
|
|
6
|
+
* 1. The scoreboard could only report NOVELTY, never REGRESSION. Every row was
|
|
7
|
+
* a point measurement of a different item, and nothing was ever re-measured.
|
|
8
|
+
* If item 1 caused item 4's win to evaporate, or an unguarded ingest path
|
|
9
|
+
* quietly refilled the store, no row in an append-only table would ever show
|
|
10
|
+
* it. Here, every row re-runs EVERY prior item's signal, so a win that decays
|
|
11
|
+
* is visible on the next row rather than never.
|
|
12
|
+
*
|
|
13
|
+
* 2. Claims could be fitted after the fact. An item that shipped against
|
|
14
|
+
* Recall@5 could be defended with "but MRR improved" once Recall@5 came in
|
|
15
|
+
* flat. A signal must be declared BEFORE the run, with its direction and
|
|
16
|
+
* magnitude, and `declaredAt` is what makes that checkable.
|
|
17
|
+
*
|
|
18
|
+
* The sealed dev/test split stops us fitting the DATA. This stops us fitting
|
|
19
|
+
* the CLAIM. They are different failure modes and need different machinery.
|
|
20
|
+
*
|
|
21
|
+
* @module v1/cli/knowledge/eval/signals
|
|
22
|
+
*/
|
|
23
|
+
import type { StoreProfile } from './harness.js';
|
|
24
|
+
export interface PreRegisteredSignal {
|
|
25
|
+
/** Stable id, referenced by the scoreboard row that shipped the item. */
|
|
26
|
+
id: string;
|
|
27
|
+
/** Backlog item this signal belongs to. */
|
|
28
|
+
item: string;
|
|
29
|
+
/** ISO date the prediction was recorded. MUST predate the run that tests it —
|
|
30
|
+
* a magnitude written down after seeing the number is not a prediction. */
|
|
31
|
+
declaredAt: string;
|
|
32
|
+
/** One sentence, in words, of what is claimed. */
|
|
33
|
+
claim: string;
|
|
34
|
+
/** Dotted path into the eval report, e.g.
|
|
35
|
+
* 'results.dense-only (gte-modernbert-base).scoreboard.recallAt5'. */
|
|
36
|
+
metric: string;
|
|
37
|
+
direction: 'increase' | 'decrease' | 'no-worse-than';
|
|
38
|
+
/** Declared before measuring. Compared against the CI half-width at scoring
|
|
39
|
+
* time: a prediction smaller than the noise floor is not testable. */
|
|
40
|
+
expectedMagnitude: number;
|
|
41
|
+
/**
|
|
42
|
+
* Store profiles in which this signal is VISIBLE AT ALL. Item 4's superseded
|
|
43
|
+
* rows do not exist in a 'fresh' corpus, so scoring it there yields a flat
|
|
44
|
+
* number that means "cannot see", not "no effect". Getting this wrong is how
|
|
45
|
+
* a working item gets dropped.
|
|
46
|
+
*/
|
|
47
|
+
visibleIn: StoreProfile[];
|
|
48
|
+
/** What would make this win decay. This is what populates the regression
|
|
49
|
+
* suite — declared up front, not reconstructed later from a plan. */
|
|
50
|
+
decayCondition: string;
|
|
51
|
+
/** Value when the item shipped. Written ONCE, never edited. */
|
|
52
|
+
shipValue?: number;
|
|
53
|
+
/** Value before the item shipped. Written ONCE, never edited. */
|
|
54
|
+
baselineValue?: number;
|
|
55
|
+
/**
|
|
56
|
+
* The regime the reference value was measured under: corpus hash, golden-set
|
|
57
|
+
* version and split scheme. If the current run does not match, the reference
|
|
58
|
+
* describes a world that no longer exists and comparing against it is
|
|
59
|
+
* meaningless — the signal reports `stale-baseline`, NOT `DECAYED`.
|
|
60
|
+
*
|
|
61
|
+
* This exists because it fired on its first run: the item-1 gate's 0.500
|
|
62
|
+
* baseline was measured on the rank-based split, and the stability fix
|
|
63
|
+
* changed DEV membership. The suite dutifully reported DECAYED for a number
|
|
64
|
+
* that had not regressed at all — it had been re-measured over a different
|
|
65
|
+
* set. A false alarm is not harmless: it teaches people to ignore the alarm.
|
|
66
|
+
*/
|
|
67
|
+
measuredUnder?: {
|
|
68
|
+
corpusHash?: string;
|
|
69
|
+
goldenSetVersion?: string;
|
|
70
|
+
splitScheme?: string;
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Why a delta came out flat. Three different situations produce an identical
|
|
75
|
+
* null row and only one of them justifies dropping an item, so a null must say
|
|
76
|
+
* which it is. `cannot-see-mechanism` is the DEFAULT: claiming "no effect"
|
|
77
|
+
* requires first showing the harness could have seen an effect.
|
|
78
|
+
*/
|
|
79
|
+
export type NullVerdict = 'no-effect' | 'cannot-see-mechanism' | 'redundant' | 'undetermined';
|
|
80
|
+
export interface SignalResult {
|
|
81
|
+
id: string;
|
|
82
|
+
item: string;
|
|
83
|
+
claim: string;
|
|
84
|
+
metric: string;
|
|
85
|
+
declaredAt: string;
|
|
86
|
+
currentValue: number | null;
|
|
87
|
+
shipValue: number | null;
|
|
88
|
+
baselineValue: number | null;
|
|
89
|
+
/** Movement since ship time. Negative for a signal declared 'increase' is decay. */
|
|
90
|
+
deltaSinceShip: number | null;
|
|
91
|
+
verdict: 'holding' | 'DECAYED' | 'cannot-see' | 'not-yet-shipped' | 'below-noise-floor' | 'stale-baseline';
|
|
92
|
+
nullVerdict?: NullVerdict;
|
|
93
|
+
note: string;
|
|
94
|
+
}
|
|
95
|
+
/** Read a dotted path, tolerating keys that themselves contain dots. */
|
|
96
|
+
export declare function readMetric(report: unknown, dotted: string): number | null;
|
|
97
|
+
/**
|
|
98
|
+
* THE PERSISTENT REGRESSION SUITE.
|
|
99
|
+
*
|
|
100
|
+
* Append one entry per shipped item, with `shipValue` and `baselineValue`
|
|
101
|
+
* frozen at ship time. Never edit an existing entry: an edited prediction is
|
|
102
|
+
* not a prediction. Every future scoreboard row re-scores all of them.
|
|
103
|
+
*/
|
|
104
|
+
export declare const SIGNAL_REGISTRY: PreRegisteredSignal[];
|
|
105
|
+
export interface Regime {
|
|
106
|
+
corpusHash: string;
|
|
107
|
+
goldenSetVersion: string;
|
|
108
|
+
splitScheme: string;
|
|
109
|
+
}
|
|
110
|
+
export declare function scoreSignals(report: unknown, storeProfile: StoreProfile, noiseFloor: number, regime?: Regime): SignalResult[];
|
|
111
|
+
//# sourceMappingURL=signals.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"signals.d.ts","sourceRoot":"","sources":["../../../../src/knowledge/eval/signals.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,WAAW,mBAAmB;IAClC,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;IACX,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb;gFAC4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,KAAK,EAAE,MAAM,CAAC;IACd;2EACuE;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC;IACrD;2EACuE;IACvE,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B;0EACsE;IACtE,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,sBAAsB,GACtB,WAAW,GACX,cAAc,CAAC;AAEnB,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oFAAoF;IACpF,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,SAAS,GAAG,SAAS,GAAG,YAAY,GAAG,iBAAiB,GAAG,mBAAmB,GAAG,gBAAgB,CAAC;IAC3G,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wEAAwE;AACxE,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAwBzE;AAED;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,EAAE,mBAAmB,EA+GhD,CAAC;AAEF,MAAM,WAAW,MAAM;IAAG,UAAU,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE;AAE7F,wBAAgB,YAAY,CAC1B,MAAM,EAAE,OAAO,EACf,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,YAAY,EAAE,CAqEhB"}
|