memkin 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +108 -836
- package/README.md +108 -836
- package/README.zh-CN.md +80 -44
- package/dist/cli-stores.d.ts +12 -0
- package/dist/cli-stores.d.ts.map +1 -1
- package/dist/cli-stores.js +15 -1
- package/dist/cli-stores.js.map +1 -1
- package/dist/cli.js +94 -6
- package/dist/cli.js.map +1 -1
- package/dist/collectors/agent/claude-code.d.ts.map +1 -1
- package/dist/collectors/agent/claude-code.js +10 -29
- package/dist/collectors/agent/claude-code.js.map +1 -1
- package/dist/collectors/agent/codex.d.ts.map +1 -1
- package/dist/collectors/agent/codex.js +24 -10
- package/dist/collectors/agent/codex.js.map +1 -1
- package/dist/collectors/agent/collector.d.ts +38 -0
- package/dist/collectors/agent/collector.d.ts.map +1 -1
- package/dist/collectors/agent/collector.js +91 -20
- package/dist/collectors/agent/collector.js.map +1 -1
- package/dist/collectors/agent/hermes.d.ts.map +1 -1
- package/dist/collectors/agent/hermes.js +5 -1
- package/dist/collectors/agent/hermes.js.map +1 -1
- package/dist/collectors/feishu/sources/calendar.d.ts.map +1 -1
- package/dist/collectors/feishu/sources/calendar.js +18 -7
- package/dist/collectors/feishu/sources/calendar.js.map +1 -1
- package/dist/consolidator/consolidator.d.ts +9 -1
- package/dist/consolidator/consolidator.d.ts.map +1 -1
- package/dist/consolidator/consolidator.js +7 -0
- package/dist/consolidator/consolidator.js.map +1 -1
- package/dist/consolidator/entity-merge.d.ts +33 -0
- package/dist/consolidator/entity-merge.d.ts.map +1 -0
- package/dist/consolidator/entity-merge.js +148 -0
- package/dist/consolidator/entity-merge.js.map +1 -0
- package/dist/core/agent-session-scanner.d.ts +23 -0
- package/dist/core/agent-session-scanner.d.ts.map +1 -0
- package/dist/core/agent-session-scanner.js +69 -0
- package/dist/core/agent-session-scanner.js.map +1 -0
- package/dist/core/canonicalize.d.ts +8 -0
- package/dist/core/canonicalize.d.ts.map +1 -1
- package/dist/core/canonicalize.js +12 -2
- package/dist/core/canonicalize.js.map +1 -1
- package/dist/core/config.d.ts +6 -0
- package/dist/core/config.d.ts.map +1 -1
- package/dist/core/config.js.map +1 -1
- package/dist/core/identity-resolver.d.ts +53 -3
- package/dist/core/identity-resolver.d.ts.map +1 -1
- package/dist/core/identity-resolver.js +112 -17
- package/dist/core/identity-resolver.js.map +1 -1
- package/dist/core/person-identity.d.ts +38 -3
- package/dist/core/person-identity.d.ts.map +1 -1
- package/dist/core/person-identity.js +58 -29
- package/dist/core/person-identity.js.map +1 -1
- package/dist/core/pipeline-factory.d.ts.map +1 -1
- package/dist/core/pipeline-factory.js +2 -1
- package/dist/core/pipeline-factory.js.map +1 -1
- package/dist/core/pipeline.js +3 -3
- package/dist/core/pipeline.js.map +1 -1
- package/dist/daemon/autostart/daemon-state.d.ts +32 -0
- package/dist/daemon/autostart/daemon-state.d.ts.map +1 -1
- package/dist/daemon/autostart/daemon-state.js +27 -1
- package/dist/daemon/autostart/daemon-state.js.map +1 -1
- package/dist/daemon/scheduler.d.ts +10 -0
- package/dist/daemon/scheduler.d.ts.map +1 -1
- package/dist/daemon/scheduler.js +20 -1
- package/dist/daemon/scheduler.js.map +1 -1
- package/dist/demo/seed.d.ts +32 -0
- package/dist/demo/seed.d.ts.map +1 -0
- package/dist/demo/seed.js +185 -0
- package/dist/demo/seed.js.map +1 -0
- package/dist/embedded-assets.generated.d.ts.map +1 -1
- package/dist/embedded-assets.generated.js +2 -2
- package/dist/embedded-assets.generated.js.map +1 -1
- package/dist/eval/golden.d.ts +107 -0
- package/dist/eval/golden.d.ts.map +1 -0
- package/dist/eval/golden.js +74 -0
- package/dist/eval/golden.js.map +1 -0
- package/dist/eval/judge.d.ts +82 -0
- package/dist/eval/judge.d.ts.map +1 -0
- package/dist/eval/judge.js +130 -0
- package/dist/eval/judge.js.map +1 -0
- package/dist/eval/manifest.d.ts +107 -0
- package/dist/eval/manifest.d.ts.map +1 -0
- package/dist/eval/manifest.js +84 -0
- package/dist/eval/manifest.js.map +1 -0
- package/dist/eval/metrics.d.ts +87 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +97 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/extractors/noise-filter.d.ts +7 -3
- package/dist/extractors/noise-filter.d.ts.map +1 -1
- package/dist/extractors/noise-filter.js +16 -3
- package/dist/extractors/noise-filter.js.map +1 -1
- package/dist/extractors/signal-extractor.d.ts.map +1 -1
- package/dist/extractors/signal-extractor.js +13 -2
- package/dist/extractors/signal-extractor.js.map +1 -1
- package/dist/lifecycle/legacy-migration.d.ts +6 -2
- package/dist/lifecycle/legacy-migration.d.ts.map +1 -1
- package/dist/lifecycle/legacy-migration.js +69 -4
- package/dist/lifecycle/legacy-migration.js.map +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js +18 -7
- package/dist/server/api.js.map +1 -1
- package/dist/server/backfill-routes.d.ts.map +1 -1
- package/dist/server/backfill-routes.js +2 -1
- package/dist/server/backfill-routes.js.map +1 -1
- package/dist/server/mcp.d.ts +4 -4
- package/dist/store/agent-sessions.d.ts +59 -0
- package/dist/store/agent-sessions.d.ts.map +1 -0
- package/dist/store/agent-sessions.js +119 -0
- package/dist/store/agent-sessions.js.map +1 -0
- package/dist/store/database.d.ts +20 -1
- package/dist/store/database.d.ts.map +1 -1
- package/dist/store/database.js +44 -6
- package/dist/store/database.js.map +1 -1
- package/dist/store/embedding.d.ts +20 -2
- package/dist/store/embedding.d.ts.map +1 -1
- package/dist/store/embedding.js +20 -5
- package/dist/store/embedding.js.map +1 -1
- package/dist/store/entity-suggestions.d.ts +62 -0
- package/dist/store/entity-suggestions.d.ts.map +1 -0
- package/dist/store/entity-suggestions.js +80 -0
- package/dist/store/entity-suggestions.js.map +1 -0
- package/dist/store/migrations/index.d.ts.map +1 -1
- package/dist/store/migrations/index.js +91 -0
- package/dist/store/migrations/index.js.map +1 -1
- package/dist/store/pages.d.ts.map +1 -1
- package/dist/store/pages.js +12 -3
- package/dist/store/pages.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-as-judge semantic matcher: compares pipeline-extracted signals against a
|
|
3
|
+
* golden annotation for the same session and buckets each signal into
|
|
4
|
+
* matched / missed / extra.
|
|
5
|
+
*
|
|
6
|
+
* Matching rule (spec §10): same `type` AND semantically equivalent `what`.
|
|
7
|
+
* Type is checked programmatically (cheap, deterministic); semantic equivalence
|
|
8
|
+
* within a type is delegated to a JudgeClient, which in production wraps the
|
|
9
|
+
* existing LLMProvider abstraction (see createLLMJudgeClient below) so no new
|
|
10
|
+
* LLM dependency is introduced — the same MiniMax-backed provider used
|
|
11
|
+
* elsewhere in the codebase is reused here.
|
|
12
|
+
*
|
|
13
|
+
* The judge is calibrated against 5 hand-picked human judgment examples before
|
|
14
|
+
* being trusted on golden data — see CALIBRATION_EXAMPLES and calibrateJudge().
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Compare pipeline output signals against a golden annotation.
|
|
18
|
+
*
|
|
19
|
+
* Algorithm: for each pipeline signal (in order), find the first not-yet-matched
|
|
20
|
+
* golden signal of the same `type` that the judge client considers semantically
|
|
21
|
+
* equivalent. Matched pairs are removed from further consideration (one pipeline
|
|
22
|
+
* signal matches at most one golden signal, and vice versa) — this prevents a
|
|
23
|
+
* single golden signal from being "matched" multiple times when the judge client
|
|
24
|
+
* is permissive.
|
|
25
|
+
*/
|
|
26
|
+
export async function judge(output, golden, client) {
|
|
27
|
+
const remainingGolden = [...golden.should_record];
|
|
28
|
+
const matched = [];
|
|
29
|
+
const extra = [];
|
|
30
|
+
for (const signal of output) {
|
|
31
|
+
let matchIndex = -1;
|
|
32
|
+
for (let i = 0; i < remainingGolden.length; i++) {
|
|
33
|
+
const candidate = remainingGolden[i];
|
|
34
|
+
if (candidate.type !== signal.type)
|
|
35
|
+
continue;
|
|
36
|
+
// Sequential await is intentional: preserves output order and avoids
|
|
37
|
+
// re-matching a golden signal against more than one pipeline signal.
|
|
38
|
+
const equivalent = await client.isEquivalent(signal.what, candidate.what);
|
|
39
|
+
if (equivalent) {
|
|
40
|
+
matchIndex = i;
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (matchIndex >= 0) {
|
|
45
|
+
const [goldenSignal] = remainingGolden.splice(matchIndex, 1);
|
|
46
|
+
matched.push({ pipeline: signal, golden: goldenSignal });
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
extra.push(signal);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { matched, missed: remainingGolden, extra };
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Five hand-picked (pipeline text, golden text, expectedEquivalent) examples used
|
|
56
|
+
* to calibrate a JudgeClient before trusting it on real golden data (spec §10:
|
|
57
|
+
* "judge 先用 5 个人工判例校准").
|
|
58
|
+
*/
|
|
59
|
+
export const CALIBRATION_EXAMPLES = [
|
|
60
|
+
{
|
|
61
|
+
a: "Use Bun instead of npm for package management.",
|
|
62
|
+
b: "Decided to switch the project's package manager from npm to Bun.",
|
|
63
|
+
expectedEquivalent: true,
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
a: "Add retry logic with exponential backoff to the HTTP client.",
|
|
67
|
+
b: "The HTTP client should retry failed requests using exponential backoff.",
|
|
68
|
+
expectedEquivalent: true,
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
a: "Always use conventional commits, never add AI co-author trailers.",
|
|
72
|
+
b: "Commit messages must follow the conventional commits format.",
|
|
73
|
+
expectedEquivalent: false, // golden is more specific (also bans AI trailers); not fully equivalent
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
a: "Fixed a bug where the migration ran twice on cold start.",
|
|
77
|
+
b: "Add retry logic with exponential backoff to the HTTP client.",
|
|
78
|
+
expectedEquivalent: false,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
a: "Prefer TypeScript strict mode across the codebase.",
|
|
82
|
+
b: "The team agreed to enable strict mode in tsconfig.",
|
|
83
|
+
expectedEquivalent: true,
|
|
84
|
+
},
|
|
85
|
+
];
|
|
86
|
+
/**
|
|
87
|
+
* Run a JudgeClient against CALIBRATION_EXAMPLES and report how many of the 5
|
|
88
|
+
* hand-labeled examples it got right. Callers should require a high pass rate
|
|
89
|
+
* (e.g. 5/5 or 4/5) before trusting the client for real evaluation runs.
|
|
90
|
+
*/
|
|
91
|
+
export async function calibrateJudge(client) {
|
|
92
|
+
const failures = [];
|
|
93
|
+
let correct = 0;
|
|
94
|
+
for (const example of CALIBRATION_EXAMPLES) {
|
|
95
|
+
// Calibration set is tiny (5 examples); sequential await is simplest and deterministic.
|
|
96
|
+
const actual = await client.isEquivalent(example.a, example.b);
|
|
97
|
+
if (actual === example.expectedEquivalent) {
|
|
98
|
+
correct++;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
failures.push({ a: example.a, b: example.b, expected: example.expectedEquivalent, actual });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return { total: CALIBRATION_EXAMPLES.length, correct, failures };
|
|
105
|
+
}
|
|
106
|
+
const JUDGE_SYSTEM_PROMPT = `You are judging whether two short descriptions of the same *kind* of signal (both already confirmed to be the same type, e.g. both "decision" or both "task") describe the same underlying fact or action.
|
|
107
|
+
|
|
108
|
+
Answer with exactly one word: "yes" if they are semantically equivalent (same underlying decision/task/fact, allowing for paraphrase), or "no" if they describe different things, or if one is meaningfully broader/narrower/more specific than the other in a way that changes what should be recorded.
|
|
109
|
+
|
|
110
|
+
Do not explain your reasoning. Respond with only "yes" or "no".`;
|
|
111
|
+
/**
|
|
112
|
+
* Build a JudgeClient backed by the existing LLMProvider abstraction (the same
|
|
113
|
+
* one used by the extraction pipeline, configured for MiniMax elsewhere in
|
|
114
|
+
* config). No new LLM dependency is introduced.
|
|
115
|
+
*/
|
|
116
|
+
export function createLLMJudgeClient(provider) {
|
|
117
|
+
return {
|
|
118
|
+
async isEquivalent(a, b) {
|
|
119
|
+
const response = await provider.chat([
|
|
120
|
+
{ role: "system", content: JUDGE_SYSTEM_PROMPT },
|
|
121
|
+
{
|
|
122
|
+
role: "user",
|
|
123
|
+
content: `Description A: ${a}\nDescription B: ${b}\n\nAre these semantically equivalent?`,
|
|
124
|
+
},
|
|
125
|
+
], { temperature: 0, responseFormat: "text" });
|
|
126
|
+
return response.trim().toLowerCase().startsWith("yes");
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
//# sourceMappingURL=judge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAgCH;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAwB,EACxB,MAAwB,EACxB,MAAmB;IAEnB,MAAM,eAAe,GAAG,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IAClD,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAqB,EAAE,CAAC;IAEnC,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YACrC,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAC7C,qEAAqE;YACrE,qEAAqE;YACrE,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC1E,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,CAAC,CAAC;gBACf,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,YAAY,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAI5B;IACH;QACE,CAAC,EAAE,gDAAgD;QACnD,CAAC,EAAE,kEAAkE;QACrE,kBAAkB,EAAE,IAAI;KACzB;IACD;QACE,CAAC,EAAE,8DAA8D;QACjE,CAAC,EAAE,yEAAyE;QAC5E,kBAAkB,EAAE,IAAI;KACzB;IACD;QACE,CAAC,EAAE,mEAAmE;QACtE,CAAC,EAAE,8DAA8D;QACjE,kBAAkB,EAAE,KAAK,EAAE,wEAAwE;KACpG;IACD;QACE,CAAC,EAAE,0DAA0D;QAC7D,CAAC,EAAE,8DAA8D;QACjE,kBAAkB,EAAE,KAAK;KAC1B;IACD;QACE,CAAC,EAAE,oDAAoD;QACvD,CAAC,EAAE,oDAAoD;QACvD,kBAAkB,EAAE,IAAI;KACzB;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAmB;IAKtD,MAAM,QAAQ,GAAwE,EAAE,CAAC;IACzF,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,OAAO,IAAI,oBAAoB,EAAE,CAAC;QAC3C,wFAAwF;QACxF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,MAAM,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,oBAAoB,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED,MAAM,mBAAmB,GAAG;;;;gEAIoC,CAAC;AAEjE;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAqB;IACxD,OAAO;QACL,KAAK,CAAC,YAAY,CAAC,CAAS,EAAE,CAAS;YACrC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAClC;gBACE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBAChD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,kBAAkB,CAAC,oBAAoB,CAAC,wCAAwC;iBAC1F;aACF,EACD,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAC3C,CAAC;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval manifest: a versioned, reproducible record of which sessions belong to the
|
|
3
|
+
* golden dataset, which split (tune/holdout) each falls into, and a hash of the
|
|
4
|
+
* annotation content at the time the manifest was created.
|
|
5
|
+
*
|
|
6
|
+
* The manifest itself is safe to commit (it only references sessions by opaque
|
|
7
|
+
* `session_ref` and stores a hash of the annotation, not the annotation content).
|
|
8
|
+
* The actual golden dataset (real session transcripts + human annotations) is
|
|
9
|
+
* gitignored and lives locally — see src/eval/golden.ts and .gitignore.
|
|
10
|
+
*
|
|
11
|
+
* Spec: memoark-2026-07-06-extraction-quality-redesign.md §10 (质量度量).
|
|
12
|
+
* Holdout is locked once created: acceptance is judged only on the holdout split
|
|
13
|
+
* (see src/eval/metrics.ts `report`), never on the full dataset, to avoid
|
|
14
|
+
* training-set leakage into the pass/fail decision.
|
|
15
|
+
*/
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
export declare const EvalSplitSchema: z.ZodEnum<["tune", "holdout"]>;
|
|
18
|
+
export type EvalSplit = z.infer<typeof EvalSplitSchema>;
|
|
19
|
+
export declare const ManifestSessionSchema: z.ZodObject<{
|
|
20
|
+
/** Opaque reference to a session, e.g. "claude-code:sess-1". Resolution to an
|
|
21
|
+
* actual transcript path/DB row is the caller's responsibility. */
|
|
22
|
+
session_ref: z.ZodString;
|
|
23
|
+
split: z.ZodEnum<["tune", "holdout"]>;
|
|
24
|
+
/** Hash (e.g. sha256) of the golden annotation content for this session, so a
|
|
25
|
+
* manifest can detect when annotations have drifted since it was recorded. */
|
|
26
|
+
annotation_hash: z.ZodString;
|
|
27
|
+
}, "strip", z.ZodTypeAny, {
|
|
28
|
+
split: "tune" | "holdout";
|
|
29
|
+
session_ref: string;
|
|
30
|
+
annotation_hash: string;
|
|
31
|
+
}, {
|
|
32
|
+
split: "tune" | "holdout";
|
|
33
|
+
session_ref: string;
|
|
34
|
+
annotation_hash: string;
|
|
35
|
+
}>;
|
|
36
|
+
export type ManifestSession = z.infer<typeof ManifestSessionSchema>;
|
|
37
|
+
export declare const EvalManifestSchema: z.ZodEffects<z.ZodObject<{
|
|
38
|
+
version: z.ZodNumber;
|
|
39
|
+
sessions: z.ZodArray<z.ZodObject<{
|
|
40
|
+
/** Opaque reference to a session, e.g. "claude-code:sess-1". Resolution to an
|
|
41
|
+
* actual transcript path/DB row is the caller's responsibility. */
|
|
42
|
+
session_ref: z.ZodString;
|
|
43
|
+
split: z.ZodEnum<["tune", "holdout"]>;
|
|
44
|
+
/** Hash (e.g. sha256) of the golden annotation content for this session, so a
|
|
45
|
+
* manifest can detect when annotations have drifted since it was recorded. */
|
|
46
|
+
annotation_hash: z.ZodString;
|
|
47
|
+
}, "strip", z.ZodTypeAny, {
|
|
48
|
+
split: "tune" | "holdout";
|
|
49
|
+
session_ref: string;
|
|
50
|
+
annotation_hash: string;
|
|
51
|
+
}, {
|
|
52
|
+
split: "tune" | "holdout";
|
|
53
|
+
session_ref: string;
|
|
54
|
+
annotation_hash: string;
|
|
55
|
+
}>, "many">;
|
|
56
|
+
created_at: z.ZodString;
|
|
57
|
+
}, "strip", z.ZodTypeAny, {
|
|
58
|
+
created_at: string;
|
|
59
|
+
sessions: {
|
|
60
|
+
split: "tune" | "holdout";
|
|
61
|
+
session_ref: string;
|
|
62
|
+
annotation_hash: string;
|
|
63
|
+
}[];
|
|
64
|
+
version: number;
|
|
65
|
+
}, {
|
|
66
|
+
created_at: string;
|
|
67
|
+
sessions: {
|
|
68
|
+
split: "tune" | "holdout";
|
|
69
|
+
session_ref: string;
|
|
70
|
+
annotation_hash: string;
|
|
71
|
+
}[];
|
|
72
|
+
version: number;
|
|
73
|
+
}>, {
|
|
74
|
+
created_at: string;
|
|
75
|
+
sessions: {
|
|
76
|
+
split: "tune" | "holdout";
|
|
77
|
+
session_ref: string;
|
|
78
|
+
annotation_hash: string;
|
|
79
|
+
}[];
|
|
80
|
+
version: number;
|
|
81
|
+
}, {
|
|
82
|
+
created_at: string;
|
|
83
|
+
sessions: {
|
|
84
|
+
split: "tune" | "holdout";
|
|
85
|
+
session_ref: string;
|
|
86
|
+
annotation_hash: string;
|
|
87
|
+
}[];
|
|
88
|
+
version: number;
|
|
89
|
+
}>;
|
|
90
|
+
export type EvalManifest = z.infer<typeof EvalManifestSchema>;
|
|
91
|
+
/**
|
|
92
|
+
* Load and validate an eval manifest from a JSON file on disk.
|
|
93
|
+
* Throws (with a descriptive message) if the file is missing, not valid JSON,
|
|
94
|
+
* or fails schema validation.
|
|
95
|
+
*/
|
|
96
|
+
export declare function loadManifest(path: string): Promise<EvalManifest>;
|
|
97
|
+
/**
|
|
98
|
+
* Partition manifest sessions into tune (~70%) and holdout (~30%) arrays per
|
|
99
|
+
* their recorded `split`. The manifest is the source of truth for the split —
|
|
100
|
+
* this function does not re-randomize; it only groups by the already-assigned
|
|
101
|
+
* `split` field so the holdout stays locked across runs.
|
|
102
|
+
*/
|
|
103
|
+
export declare function splitSessions(manifest: EvalManifest): {
|
|
104
|
+
tune: ManifestSession[];
|
|
105
|
+
holdout: ManifestSession[];
|
|
106
|
+
};
|
|
107
|
+
//# sourceMappingURL=manifest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/eval/manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe,gCAA8B,CAAC;AAC3D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,eAAO,MAAM,qBAAqB;IAChC;uEACmE;;;IAGnE;kFAC8E;;;;;;;;;;EAE9E,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,kBAAkB;;;QAV7B;2EACmE;;;QAGnE;sFAC8E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAuB5E,CAAC;AACL,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAwBtE;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,GAAG;IACrD,IAAI,EAAE,eAAe,EAAE,CAAC;IACxB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B,CAIA"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval manifest: a versioned, reproducible record of which sessions belong to the
|
|
3
|
+
* golden dataset, which split (tune/holdout) each falls into, and a hash of the
|
|
4
|
+
* annotation content at the time the manifest was created.
|
|
5
|
+
*
|
|
6
|
+
* The manifest itself is safe to commit (it only references sessions by opaque
|
|
7
|
+
* `session_ref` and stores a hash of the annotation, not the annotation content).
|
|
8
|
+
* The actual golden dataset (real session transcripts + human annotations) is
|
|
9
|
+
* gitignored and lives locally — see src/eval/golden.ts and .gitignore.
|
|
10
|
+
*
|
|
11
|
+
* Spec: memoark-2026-07-06-extraction-quality-redesign.md §10 (质量度量).
|
|
12
|
+
* Holdout is locked once created: acceptance is judged only on the holdout split
|
|
13
|
+
* (see src/eval/metrics.ts `report`), never on the full dataset, to avoid
|
|
14
|
+
* training-set leakage into the pass/fail decision.
|
|
15
|
+
*/
|
|
16
|
+
import { readFile } from "node:fs/promises";
|
|
17
|
+
import { z } from "zod";
|
|
18
|
+
export const EvalSplitSchema = z.enum(["tune", "holdout"]);
|
|
19
|
+
export const ManifestSessionSchema = z.object({
|
|
20
|
+
/** Opaque reference to a session, e.g. "claude-code:sess-1". Resolution to an
|
|
21
|
+
* actual transcript path/DB row is the caller's responsibility. */
|
|
22
|
+
session_ref: z.string().min(1),
|
|
23
|
+
split: EvalSplitSchema,
|
|
24
|
+
/** Hash (e.g. sha256) of the golden annotation content for this session, so a
|
|
25
|
+
* manifest can detect when annotations have drifted since it was recorded. */
|
|
26
|
+
annotation_hash: z.string().min(1),
|
|
27
|
+
});
|
|
28
|
+
export const EvalManifestSchema = z
|
|
29
|
+
.object({
|
|
30
|
+
version: z.number().int().positive(),
|
|
31
|
+
sessions: z.array(ManifestSessionSchema),
|
|
32
|
+
created_at: z.string().datetime(),
|
|
33
|
+
})
|
|
34
|
+
.superRefine((manifest, ctx) => {
|
|
35
|
+
const seen = new Set();
|
|
36
|
+
for (const [i, session] of manifest.sessions.entries()) {
|
|
37
|
+
if (seen.has(session.session_ref)) {
|
|
38
|
+
ctx.addIssue({
|
|
39
|
+
code: z.ZodIssueCode.custom,
|
|
40
|
+
message: `duplicate session_ref: ${session.session_ref}`,
|
|
41
|
+
path: ["sessions", i, "session_ref"],
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
seen.add(session.session_ref);
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
/**
|
|
48
|
+
* Load and validate an eval manifest from a JSON file on disk.
|
|
49
|
+
* Throws (with a descriptive message) if the file is missing, not valid JSON,
|
|
50
|
+
* or fails schema validation.
|
|
51
|
+
*/
|
|
52
|
+
export async function loadManifest(path) {
|
|
53
|
+
let raw;
|
|
54
|
+
try {
|
|
55
|
+
raw = await readFile(path, "utf-8");
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
throw new Error(`Failed to read eval manifest at ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
59
|
+
}
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(raw);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
throw new Error(`Failed to parse eval manifest at ${path} as JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
66
|
+
}
|
|
67
|
+
const result = EvalManifestSchema.safeParse(parsed);
|
|
68
|
+
if (!result.success) {
|
|
69
|
+
throw new Error(`Invalid eval manifest at ${path}: ${result.error.message}`);
|
|
70
|
+
}
|
|
71
|
+
return result.data;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Partition manifest sessions into tune (~70%) and holdout (~30%) arrays per
|
|
75
|
+
* their recorded `split`. The manifest is the source of truth for the split —
|
|
76
|
+
* this function does not re-randomize; it only groups by the already-assigned
|
|
77
|
+
* `split` field so the holdout stays locked across runs.
|
|
78
|
+
*/
|
|
79
|
+
export function splitSessions(manifest) {
|
|
80
|
+
const tune = manifest.sessions.filter((s) => s.split === "tune");
|
|
81
|
+
const holdout = manifest.sessions.filter((s) => s.split === "holdout");
|
|
82
|
+
return { tune, holdout };
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/eval/manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;AAG3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C;uEACmE;IACnE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,KAAK,EAAE,eAAe;IACtB;kFAC8E;IAC9E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAClC,CAAC;KACD,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAClC,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,OAAO,EAAE,0BAA0B,OAAO,CAAC,WAAW,EAAE;gBACxD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,aAAa,CAAC;aACrC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAChC,CAAC;AACH,CAAC,CAAC,CAAC;AAGL;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,mCAAmC,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/F,CAAC;IACJ,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,aAAa,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,QAAsB;IAIlD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACvE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quality metrics for the extraction pipeline (spec §10):
|
|
3
|
+
*
|
|
4
|
+
* miss rate = missed / golden total (golden total = matched + missed)
|
|
5
|
+
* noise rate = extra / pipeline output total
|
|
6
|
+
* dup rate = Σ(each duplicate group size − 1) / total pages
|
|
7
|
+
* (measured on the final library after apply, not on judge output —
|
|
8
|
+
* spec §10 explicitly corrects the v2 formula)
|
|
9
|
+
*
|
|
10
|
+
* Each evaluation run is repeated 3x and reported as mean ± variance (spec §10),
|
|
11
|
+
* because a single LLM-judge pass is noisy. Acceptance is judged ONLY on the
|
|
12
|
+
* locked holdout split — the tune split is always descriptive-only (spec §10,
|
|
13
|
+
* R3 收紧-2): judging on the full/tune set would be training-set leakage.
|
|
14
|
+
*/
|
|
15
|
+
import type { JudgeResult } from "./judge.js";
|
|
16
|
+
export interface SignalMetrics {
|
|
17
|
+
missRate: number;
|
|
18
|
+
noiseRate: number;
|
|
19
|
+
}
|
|
20
|
+
/** Compute miss rate and noise rate from a single judge() result. */
|
|
21
|
+
export declare function computeMetrics(result: JudgeResult, opts: {
|
|
22
|
+
pipelineOutputTotal: number;
|
|
23
|
+
}): SignalMetrics;
|
|
24
|
+
/**
|
|
25
|
+
* Duplicate rate on the final materialized library: Σ(size−1) over every group of
|
|
26
|
+
* pages judged duplicates of one another, divided by total page count.
|
|
27
|
+
* A group of size 1 (no duplicates) contributes 0.
|
|
28
|
+
*/
|
|
29
|
+
export declare function computeDupRate(opts: {
|
|
30
|
+
groupSizes: number[];
|
|
31
|
+
totalPages: number;
|
|
32
|
+
}): number;
|
|
33
|
+
export interface RunResult {
|
|
34
|
+
result: JudgeResult;
|
|
35
|
+
pipelineOutputTotal: number;
|
|
36
|
+
}
|
|
37
|
+
export interface MetricStats {
|
|
38
|
+
mean: number;
|
|
39
|
+
variance: number;
|
|
40
|
+
}
|
|
41
|
+
export interface EvaluationResult {
|
|
42
|
+
missRate: MetricStats;
|
|
43
|
+
noiseRate: MetricStats;
|
|
44
|
+
runs: SignalMetrics[];
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Run the given judge callback `runs` times (default 3, per spec §10) and return
|
|
48
|
+
* mean ± variance for miss rate and noise rate across the runs.
|
|
49
|
+
*/
|
|
50
|
+
export declare function evaluate(runJudge: () => Promise<RunResult>, opts?: {
|
|
51
|
+
runs?: number;
|
|
52
|
+
}): Promise<EvaluationResult>;
|
|
53
|
+
export interface BaselineMetrics {
|
|
54
|
+
missRate: number;
|
|
55
|
+
noiseRate: number;
|
|
56
|
+
}
|
|
57
|
+
export interface SplitReport {
|
|
58
|
+
missRate: MetricStats;
|
|
59
|
+
noiseRate: MetricStats;
|
|
60
|
+
descriptiveOnly: boolean;
|
|
61
|
+
passed?: boolean;
|
|
62
|
+
}
|
|
63
|
+
export interface AcceptanceReport {
|
|
64
|
+
tune: SplitReport;
|
|
65
|
+
holdout: SplitReport;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Build the acceptance report for an evaluation run (spec §10, R3 收紧-2):
|
|
69
|
+
*
|
|
70
|
+
* - tune split results are ALWAYS descriptive-only (`descriptiveOnly: true`,
|
|
71
|
+
* no `passed` field) — used to iterate prompts/thresholds, never to judge
|
|
72
|
+
* pass/fail.
|
|
73
|
+
* - holdout split results are the only ones that carry a `passed` verdict:
|
|
74
|
+
* noise rate must drop by at least 80% vs. the legacy baseline, AND miss
|
|
75
|
+
* rate must not increase vs. baseline. The conclusion should be reported as
|
|
76
|
+
* directional ("noise rate dropped substantially without a miss rate
|
|
77
|
+
* regression"), not as a claim of statistical significance — holdout sizes
|
|
78
|
+
* in this spec are small (6-9 samples).
|
|
79
|
+
*/
|
|
80
|
+
export declare function report(tune: {
|
|
81
|
+
missRate: MetricStats;
|
|
82
|
+
noiseRate: MetricStats;
|
|
83
|
+
}, holdout: {
|
|
84
|
+
missRate: MetricStats;
|
|
85
|
+
noiseRate: MetricStats;
|
|
86
|
+
}, baseline: BaselineMetrics): AcceptanceReport;
|
|
87
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,qEAAqE;AACrE,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,IAAI,EAAE;IAAE,mBAAmB,EAAE,MAAM,CAAA;CAAE,GACpC,aAAa,CAMf;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAIzF;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,WAAW,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,WAAW,CAAC;IACtB,SAAS,EAAE,WAAW,CAAC;IACvB,IAAI,EAAE,aAAa,EAAE,CAAC;CACvB;AAWD;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,MAAM,OAAO,CAAC,SAAS,CAAC,EAClC,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAqB3B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,WAAW,CAAC;IACtB,SAAS,EAAE,WAAW,CAAC;IACvB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,WAAW,CAAC;CACtB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,MAAM,CACpB,IAAI,EAAE;IAAE,QAAQ,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,WAAW,CAAA;CAAE,EACvD,OAAO,EAAE;IAAE,QAAQ,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,WAAW,CAAA;CAAE,EAC1D,QAAQ,EAAE,eAAe,GACxB,gBAAgB,CAqBlB"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quality metrics for the extraction pipeline (spec §10):
|
|
3
|
+
*
|
|
4
|
+
* miss rate = missed / golden total (golden total = matched + missed)
|
|
5
|
+
* noise rate = extra / pipeline output total
|
|
6
|
+
* dup rate = Σ(each duplicate group size − 1) / total pages
|
|
7
|
+
* (measured on the final library after apply, not on judge output —
|
|
8
|
+
* spec §10 explicitly corrects the v2 formula)
|
|
9
|
+
*
|
|
10
|
+
* Each evaluation run is repeated 3x and reported as mean ± variance (spec §10),
|
|
11
|
+
* because a single LLM-judge pass is noisy. Acceptance is judged ONLY on the
|
|
12
|
+
* locked holdout split — the tune split is always descriptive-only (spec §10,
|
|
13
|
+
* R3 收紧-2): judging on the full/tune set would be training-set leakage.
|
|
14
|
+
*/
|
|
15
|
+
/** Compute miss rate and noise rate from a single judge() result. */
|
|
16
|
+
export function computeMetrics(result, opts) {
|
|
17
|
+
const goldenTotal = result.matched.length + result.missed.length;
|
|
18
|
+
const missRate = goldenTotal === 0 ? 0 : result.missed.length / goldenTotal;
|
|
19
|
+
const noiseRate = opts.pipelineOutputTotal === 0 ? 0 : result.extra.length / opts.pipelineOutputTotal;
|
|
20
|
+
return { missRate, noiseRate };
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Duplicate rate on the final materialized library: Σ(size−1) over every group of
|
|
24
|
+
* pages judged duplicates of one another, divided by total page count.
|
|
25
|
+
* A group of size 1 (no duplicates) contributes 0.
|
|
26
|
+
*/
|
|
27
|
+
export function computeDupRate(opts) {
|
|
28
|
+
if (opts.totalPages === 0)
|
|
29
|
+
return 0;
|
|
30
|
+
const sumExcessCopies = opts.groupSizes.reduce((acc, size) => acc + Math.max(0, size - 1), 0);
|
|
31
|
+
return sumExcessCopies / opts.totalPages;
|
|
32
|
+
}
|
|
33
|
+
function mean(values) {
|
|
34
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
35
|
+
}
|
|
36
|
+
function variance(values, avg) {
|
|
37
|
+
if (values.length <= 1)
|
|
38
|
+
return 0;
|
|
39
|
+
return mean(values.map((v) => (v - avg) ** 2));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Run the given judge callback `runs` times (default 3, per spec §10) and return
|
|
43
|
+
* mean ± variance for miss rate and noise rate across the runs.
|
|
44
|
+
*/
|
|
45
|
+
export async function evaluate(runJudge, opts = {}) {
|
|
46
|
+
const runs = opts.runs ?? 3;
|
|
47
|
+
const perRun = [];
|
|
48
|
+
for (let i = 0; i < runs; i++) {
|
|
49
|
+
// Sequential by design: each run may hit a real LLM judge, and we want
|
|
50
|
+
// deterministic ordering for reproducibility across evaluate() calls.
|
|
51
|
+
const { result, pipelineOutputTotal } = await runJudge();
|
|
52
|
+
perRun.push(computeMetrics(result, { pipelineOutputTotal }));
|
|
53
|
+
}
|
|
54
|
+
const missRates = perRun.map((r) => r.missRate);
|
|
55
|
+
const noiseRates = perRun.map((r) => r.noiseRate);
|
|
56
|
+
const missMean = mean(missRates);
|
|
57
|
+
const noiseMean = mean(noiseRates);
|
|
58
|
+
return {
|
|
59
|
+
missRate: { mean: missMean, variance: variance(missRates, missMean) },
|
|
60
|
+
noiseRate: { mean: noiseMean, variance: variance(noiseRates, noiseMean) },
|
|
61
|
+
runs: perRun,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Build the acceptance report for an evaluation run (spec §10, R3 收紧-2):
|
|
66
|
+
*
|
|
67
|
+
* - tune split results are ALWAYS descriptive-only (`descriptiveOnly: true`,
|
|
68
|
+
* no `passed` field) — used to iterate prompts/thresholds, never to judge
|
|
69
|
+
* pass/fail.
|
|
70
|
+
* - holdout split results are the only ones that carry a `passed` verdict:
|
|
71
|
+
* noise rate must drop by at least 80% vs. the legacy baseline, AND miss
|
|
72
|
+
* rate must not increase vs. baseline. The conclusion should be reported as
|
|
73
|
+
* directional ("noise rate dropped substantially without a miss rate
|
|
74
|
+
* regression"), not as a claim of statistical significance — holdout sizes
|
|
75
|
+
* in this spec are small (6-9 samples).
|
|
76
|
+
*/
|
|
77
|
+
export function report(tune, holdout, baseline) {
|
|
78
|
+
const noiseDrop = baseline.noiseRate === 0
|
|
79
|
+
? 0
|
|
80
|
+
: (baseline.noiseRate - holdout.noiseRate.mean) / baseline.noiseRate;
|
|
81
|
+
const missNotIncreased = holdout.missRate.mean <= baseline.missRate;
|
|
82
|
+
const passed = noiseDrop >= 0.8 && missNotIncreased;
|
|
83
|
+
return {
|
|
84
|
+
tune: {
|
|
85
|
+
missRate: tune.missRate,
|
|
86
|
+
noiseRate: tune.noiseRate,
|
|
87
|
+
descriptiveOnly: true,
|
|
88
|
+
},
|
|
89
|
+
holdout: {
|
|
90
|
+
missRate: holdout.missRate,
|
|
91
|
+
noiseRate: holdout.noiseRate,
|
|
92
|
+
descriptiveOnly: false,
|
|
93
|
+
passed,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AASH,qEAAqE;AACrE,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,IAAqC;IAErC,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;IACjE,MAAM,QAAQ,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;IAC5E,MAAM,SAAS,GACb,IAAI,CAAC,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC;IACtF,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAkD;IAC/E,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9F,OAAO,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC;AAC3C,CAAC;AAkBD,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3D,CAAC;AAED,SAAS,QAAQ,CAAC,MAAgB,EAAE,GAAW;IAC7C,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAkC,EAClC,OAA0B,EAAE;IAE5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,uEAAuE;QACvE,sEAAsE;QACtE,MAAM,EAAE,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,QAAQ,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAEnC,OAAO;QACL,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE;QACrE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE;QACzE,IAAI,EAAE,MAAM;KACb,CAAC;AACJ,CAAC;AAmBD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,MAAM,CACpB,IAAuD,EACvD,OAA0D,EAC1D,QAAyB;IAEzB,MAAM,SAAS,GACb,QAAQ,CAAC,SAAS,KAAK,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC;IACzE,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC;IACpE,MAAM,MAAM,GAAG,SAAS,IAAI,GAAG,IAAI,gBAAgB,CAAC;IAEpD,OAAO;QACL,IAAI,EAAE;YACJ,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,IAAI;SACtB;QACD,OAAO,EAAE;YACP,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,eAAe,EAAE,KAAK;YACtB,MAAM;SACP;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -22,9 +22,13 @@ export declare function filterNoiseL1(block: ConversationBlock): NoiseFilterVerd
|
|
|
22
22
|
*/
|
|
23
23
|
export declare function filterNoise(block: ConversationBlock, provider?: LLMProvider): NoiseFilterVerdict | Promise<NoiseFilterVerdict>;
|
|
24
24
|
/**
|
|
25
|
-
*
|
|
25
|
+
* Resolve a SignalScore decision to a NoiseFilterVerdict.
|
|
26
|
+
*
|
|
27
|
+
* PRD v2 §4.3.2: admit/drop are decided by the cheap 5-dim score alone;
|
|
28
|
+
* the evaluate band (middle scores) gets a final LLM significance judgment.
|
|
29
|
+
* Fail-open: no provider or LLM failure → pass, never lose data silently.
|
|
26
30
|
*/
|
|
27
|
-
export declare function
|
|
31
|
+
export declare function resolveScoreDecision(score: {
|
|
28
32
|
decision: string;
|
|
29
|
-
}): NoiseFilterVerdict
|
|
33
|
+
}, block: ConversationBlock, provider?: LLMProvider): Promise<NoiseFilterVerdict>;
|
|
30
34
|
//# sourceMappingURL=noise-filter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"noise-filter.d.ts","sourceRoot":"","sources":["../../src/extractors/noise-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAmD9D;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,IAAI,CAoDjF;AA4ED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,WAAW,GACrB,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgBlD;AAED
|
|
1
|
+
{"version":3,"file":"noise-filter.d.ts","sourceRoot":"","sources":["../../src/extractors/noise-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAmD9D;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,IAAI,CAoDjF;AA4ED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,WAAW,GACrB,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgBlD;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,EAC3B,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,kBAAkB,CAAC,CAU7B"}
|
|
@@ -175,11 +175,24 @@ export function filterNoise(block, provider) {
|
|
|
175
175
|
return filterL2(block, provider);
|
|
176
176
|
}
|
|
177
177
|
/**
|
|
178
|
-
*
|
|
178
|
+
* Resolve a SignalScore decision to a NoiseFilterVerdict.
|
|
179
|
+
*
|
|
180
|
+
* PRD v2 §4.3.2: admit/drop are decided by the cheap 5-dim score alone;
|
|
181
|
+
* the evaluate band (middle scores) gets a final LLM significance judgment.
|
|
182
|
+
* Fail-open: no provider or LLM failure → pass, never lose data silently.
|
|
179
183
|
*/
|
|
180
|
-
export function
|
|
184
|
+
export async function resolveScoreDecision(score, block, provider) {
|
|
181
185
|
if (score.decision === "drop")
|
|
182
186
|
return "skip";
|
|
183
|
-
|
|
187
|
+
if (score.decision === "admit")
|
|
188
|
+
return "pass";
|
|
189
|
+
if (!provider)
|
|
190
|
+
return "pass";
|
|
191
|
+
try {
|
|
192
|
+
return await filterL2(block, provider);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
return "pass";
|
|
196
|
+
}
|
|
184
197
|
}
|
|
185
198
|
//# sourceMappingURL=noise-filter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"noise-filter.js","sourceRoot":"","sources":["../../src/extractors/noise-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAK1D,oDAAoD;AACpD,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC5B,CAAC,CAAC;AAEH,0BAA0B;AAC1B,MAAM,eAAe,GAAG;IACtB,MAAM;IACN,MAAM;IACN,SAAS;IACT,OAAO;IACP,IAAI;IACJ,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE7D,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAErE,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AAEhF,MAAM,mBAAmB,GAAG;IAC1B,iBAAiB;IACjB,gBAAgB;IAChB,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC;AAEF;;GAEG;AACH,SAAS,WAAW,CAAC,OAAe;IAClC,iEAAiE;IACjE,gFAAgF;IAChF,MAAM,YAAY,GAAG,OAAO;SACzB,OAAO,CACN,wOAAwO,EACxO,EAAE,CACH;SACA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEvB,OAAO,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAwB;IACpD,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IAEpC,sBAAsB;IACtB,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACzE,IAAI,UAAU;YAAE,OAAO,MAAM,CAAC;QAE9B,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,IAAI,mBAAmB;YAAE,OAAO,UAAU,CAAC;QAE3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uBAAuB;IACvB,MAAM,oBAAoB,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7F,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,0BAA0B;IAC1B,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACxF,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sBAAsB;IACtB,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACzE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9F,IAAI,mBAAmB,EAAE,CAAC;QACxB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,gCAAgC;IAChC,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtF,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,8BAA8B;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,QAAQ,CACrB,KAAwB,EACxB,QAAqB;IAErB,MAAM,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAClD,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAEhD,uCAAuC;IACvC,MAAM,YAAY,GAAG;QACnB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,UAAU,EAAE,GAAG,KAAK,CAAC,UAAU,OAAO,KAAK,CAAC,QAAQ,EAAE;QACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC;KACJ,CAAC;IAEF,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CACnC,sBAAsB,EACtB,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CACtC,CAAC;IAEF,WAAW;IACX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAClC;QACE;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,4EAA4E;SACtF;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,EACD,EAAE,cAAc,EAAE,MAAM,EAAE,CAC3B,CAAC;IAEF,8BAA8B;IAC9B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,yBAAyB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,KAAwB,EACxB,QAAsB;IAEtB,2BAA2B;IAC3B,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,yCAAyC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,yBAAyB;IACzB,OAAO,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"noise-filter.js","sourceRoot":"","sources":["../../src/extractors/noise-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAK1D,oDAAoD;AACpD,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC5B,CAAC,CAAC;AAEH,0BAA0B;AAC1B,MAAM,eAAe,GAAG;IACtB,MAAM;IACN,MAAM;IACN,SAAS;IACT,OAAO;IACP,IAAI;IACJ,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE7D,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAErE,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AAEhF,MAAM,mBAAmB,GAAG;IAC1B,iBAAiB;IACjB,gBAAgB;IAChB,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC;AAEF;;GAEG;AACH,SAAS,WAAW,CAAC,OAAe;IAClC,iEAAiE;IACjE,gFAAgF;IAChF,MAAM,YAAY,GAAG,OAAO;SACzB,OAAO,CACN,wOAAwO,EACxO,EAAE,CACH;SACA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEvB,OAAO,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAwB;IACpD,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IAEpC,sBAAsB;IACtB,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACzE,IAAI,UAAU;YAAE,OAAO,MAAM,CAAC;QAE9B,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,IAAI,mBAAmB;YAAE,OAAO,UAAU,CAAC;QAE3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uBAAuB;IACvB,MAAM,oBAAoB,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7F,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,0BAA0B;IAC1B,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACxF,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sBAAsB;IACtB,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACzE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9F,IAAI,mBAAmB,EAAE,CAAC;QACxB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,gCAAgC;IAChC,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtF,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,8BAA8B;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,QAAQ,CACrB,KAAwB,EACxB,QAAqB;IAErB,MAAM,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAClD,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAEhD,uCAAuC;IACvC,MAAM,YAAY,GAAG;QACnB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,UAAU,EAAE,GAAG,KAAK,CAAC,UAAU,OAAO,KAAK,CAAC,QAAQ,EAAE;QACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC;KACJ,CAAC;IAEF,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CACnC,sBAAsB,EACtB,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CACtC,CAAC;IAEF,WAAW;IACX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAClC;QACE;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,4EAA4E;SACtF;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,EACD,EAAE,cAAc,EAAE,MAAM,EAAE,CAC3B,CAAC;IAEF,8BAA8B;IAC9B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,yBAAyB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,KAAwB,EACxB,QAAsB;IAEtB,2BAA2B;IAC3B,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,yCAAyC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,yBAAyB;IACzB,OAAO,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAA2B,EAC3B,KAAwB,EACxB,QAAsB;IAEtB,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IAC7C,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC;IAE9C,IAAI,CAAC,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7B,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"signal-extractor.d.ts","sourceRoot":"","sources":["../../src/extractors/signal-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"signal-extractor.d.ts","sourceRoot":"","sources":["../../src/extractors/signal-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,OAAO,KAAK,EACV,WAAW,EACX,kBAAkB,EAClB,iBAAiB,EAGjB,SAAS,EAEV,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAe,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAsDrE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,KAAK,EAAE,kBAAkB,GAAG,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9E;AAWD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAkB5D;AAgFD,wBAAgB,cAAc,CAAC,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,SAAS,CAwDvF;AA+BD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,WAAW,GAAG,eAAe,CAwH5E"}
|