agent-working-memory 0.13.1 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +200 -238
  2. package/dist/adapters/common.d.ts +6 -0
  3. package/dist/adapters/common.d.ts.map +1 -1
  4. package/dist/adapters/common.js +457 -362
  5. package/dist/adapters/common.js.map +1 -1
  6. package/dist/api/routes.d.ts.map +1 -1
  7. package/dist/api/routes.js +24 -8
  8. package/dist/api/routes.js.map +1 -1
  9. package/dist/core/alias-map.d.ts +16 -0
  10. package/dist/core/alias-map.d.ts.map +1 -0
  11. package/dist/core/alias-map.js +102 -0
  12. package/dist/core/alias-map.js.map +1 -0
  13. package/dist/core/embeddings.d.ts +17 -0
  14. package/dist/core/embeddings.d.ts.map +1 -1
  15. package/dist/core/embeddings.js +50 -1
  16. package/dist/core/embeddings.js.map +1 -1
  17. package/dist/core/recall-config.d.ts +52 -0
  18. package/dist/core/recall-config.d.ts.map +1 -0
  19. package/dist/core/recall-config.js +110 -0
  20. package/dist/core/recall-config.js.map +1 -0
  21. package/dist/core/rerank-window.d.ts +61 -0
  22. package/dist/core/rerank-window.d.ts.map +1 -0
  23. package/dist/core/rerank-window.js +153 -0
  24. package/dist/core/rerank-window.js.map +1 -0
  25. package/dist/core/rerank2.d.ts +62 -0
  26. package/dist/core/rerank2.d.ts.map +1 -0
  27. package/dist/core/rerank2.js +75 -0
  28. package/dist/core/rerank2.js.map +1 -0
  29. package/dist/core/retrieval-text.d.ts +55 -0
  30. package/dist/core/retrieval-text.d.ts.map +1 -0
  31. package/dist/core/retrieval-text.js +87 -0
  32. package/dist/core/retrieval-text.js.map +1 -0
  33. package/dist/core/temporal-query.d.ts +61 -0
  34. package/dist/core/temporal-query.d.ts.map +1 -0
  35. package/dist/core/temporal-query.js +168 -0
  36. package/dist/core/temporal-query.js.map +1 -0
  37. package/dist/core/token-budget.d.ts +75 -0
  38. package/dist/core/token-budget.d.ts.map +1 -0
  39. package/dist/core/token-budget.js +136 -0
  40. package/dist/core/token-budget.js.map +1 -0
  41. package/dist/core/whoami.d.ts +11 -0
  42. package/dist/core/whoami.d.ts.map +1 -1
  43. package/dist/core/whoami.js +10 -0
  44. package/dist/core/whoami.js.map +1 -1
  45. package/dist/core/write-pipeline.d.ts.map +1 -1
  46. package/dist/core/write-pipeline.js +6 -3
  47. package/dist/core/write-pipeline.js.map +1 -1
  48. package/dist/engine/activation.d.ts.map +1 -1
  49. package/dist/engine/activation.js +146 -32
  50. package/dist/engine/activation.js.map +1 -1
  51. package/dist/hooks/prime.d.ts +77 -0
  52. package/dist/hooks/prime.d.ts.map +1 -0
  53. package/dist/hooks/prime.js +92 -0
  54. package/dist/hooks/prime.js.map +1 -0
  55. package/dist/hooks/sidecar.d.ts.map +1 -1
  56. package/dist/hooks/sidecar.js +39 -0
  57. package/dist/hooks/sidecar.js.map +1 -1
  58. package/dist/mcp.js +160 -107
  59. package/dist/mcp.js.map +1 -1
  60. package/dist/storage/pglite.d.ts.map +1 -1
  61. package/dist/storage/pglite.js +10 -2
  62. package/dist/storage/pglite.js.map +1 -1
  63. package/dist/storage/postgres.d.ts.map +1 -1
  64. package/dist/storage/postgres.js +10 -2
  65. package/dist/storage/postgres.js.map +1 -1
  66. package/dist/storage/sqlite.d.ts.map +1 -1
  67. package/dist/storage/sqlite.js +12 -2
  68. package/dist/storage/sqlite.js.map +1 -1
  69. package/dist/types/engram.d.ts +31 -0
  70. package/dist/types/engram.d.ts.map +1 -1
  71. package/package.json +3 -2
  72. package/src/adapters/common.ts +666 -567
  73. package/src/api/routes.ts +1015 -999
  74. package/src/core/alias-map.ts +97 -0
  75. package/src/core/embeddings.ts +172 -115
  76. package/src/core/recall-config.ts +115 -0
  77. package/src/core/rerank-window.ts +158 -0
  78. package/src/core/rerank2.ts +82 -0
  79. package/src/core/retrieval-text.ts +82 -0
  80. package/src/core/temporal-query.ts +193 -0
  81. package/src/core/token-budget.ts +160 -0
  82. package/src/core/whoami.ts +110 -92
  83. package/src/core/write-pipeline.ts +6 -3
  84. package/src/engine/activation.ts +1579 -1468
  85. package/src/hooks/prime.ts +136 -0
  86. package/src/hooks/sidecar.ts +43 -0
  87. package/src/mcp.ts +1443 -1387
  88. package/src/storage/pglite.ts +10 -2
  89. package/src/storage/postgres.ts +10 -2
  90. package/src/storage/sqlite.ts +12 -2
  91. package/src/types/engram.ts +32 -0
@@ -0,0 +1,110 @@
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Effective recall configuration — self-reporting, so a measurement can PROVE
5
+ * which configuration produced it.
6
+ *
7
+ * WHY THIS EXISTS
8
+ * ---------------
9
+ * On 2026-08-23 a benchmark comparison reported "no effect" for a change that
10
+ * demonstrably worked. The cause was not AWM: two arms shared a port, teardown
11
+ * left the first server alive, the second arm health-checked the survivor, and
12
+ * both arms measured the SAME baseline process. Identical inputs, identical
13
+ * outputs, a confident and completely wrong conclusion.
14
+ *
15
+ * Port hygiene fixes that instance. It does not fix the class. The class is:
16
+ * **nothing verified that the system measured was the system configured.**
17
+ * The same gap bit the D11 spreading-activation re-test, where the tracer's arm
18
+ * label omitted `AWM_SPREAD_INHIBIT`, so two materially different arms both
19
+ * printed `arm=spread` and had to be told apart by diffing output by hand.
20
+ *
21
+ * The durable fix is for the running system to state its own effective recall
22
+ * configuration, so a harness can assert it and fail LOUDLY instead of silently
23
+ * measuring the wrong thing. Any new recall flag added to RECALL_FLAGS is
24
+ * automatically covered by every consumer — the eval tracer's arm label, the
25
+ * `/health` payload, and the benchmark driver's assertion.
26
+ *
27
+ * Adding a flag here is the ONLY step required to make it visible everywhere.
28
+ */
29
+ /**
30
+ * Every environment flag that can change what `activate()` returns.
31
+ * Keep this list current — an unlisted flag is an invisible experiment.
32
+ */
33
+ export const RECALL_FLAGS = [
34
+ // Second-stage reorder (phase 9b)
35
+ 'AWM_RERANK2',
36
+ 'AWM_RERANK2_K',
37
+ // Cross-encoder passage selection
38
+ 'AWM_RERANK_WINDOW',
39
+ 'AWM_RERANK_TRUNC',
40
+ 'AWM_RERANK_POOL',
41
+ 'AWM_RERANK_TAGS',
42
+ 'AWM_RERANK_TAGS_LEN',
43
+ // Derived retrieval text (embedding side)
44
+ 'AWM_RETRIEVAL_TEXT',
45
+ // Project-dialect alias expansion
46
+ 'AWM_ALIASES',
47
+ 'AWM_ALIAS_QUERY_CAP',
48
+ // Embedding model — changes every stored vector, so it belongs here
49
+ 'AWM_EMBED_MODEL',
50
+ 'AWM_EMBED_DIMS',
51
+ 'AWM_DISABLE_RERANK_SKIP',
52
+ // Spreading activation (D11 — parked)
53
+ 'AWM_SPREAD',
54
+ 'AWM_SPREAD_INJECT',
55
+ 'AWM_SPREAD_INHIBIT',
56
+ 'AWM_SPREAD_ITERS',
57
+ 'AWM_SPREAD_DAMPING',
58
+ 'AWM_SPREAD_BUDGET',
59
+ 'AWM_SPREAD_BOOST',
60
+ // Candidate pool / retrieval breadth
61
+ 'AWM_TOPN_MULT',
62
+ 'AWM_BROAD_EDGES',
63
+ 'AWM_ENTITY_FETCH',
64
+ 'AWM_ENTITY_INDEX_FETCH',
65
+ 'AWM_QUERY_BRIDGE',
66
+ 'AWM_AUTOTAG',
67
+ // Diagnostic escape hatches that alter ranking
68
+ 'AWM_DISABLE_POOL_FILTER',
69
+ 'AWM_DISABLE_EXPANSION_CACHE',
70
+ 'AWM_DISABLE_SLIM_CACHE',
71
+ 'AWM_ABSTAIN_GATE_K',
72
+ ];
73
+ /** Only the flags actually set, in declaration order. */
74
+ export function activeRecallConfig(env = process.env) {
75
+ const out = {};
76
+ for (const k of RECALL_FLAGS) {
77
+ const v = env[k];
78
+ if (v !== undefined && v !== '')
79
+ out[k] = v;
80
+ }
81
+ return out;
82
+ }
83
+ /**
84
+ * Short, stable, comparable label for the active configuration.
85
+ * `default` when nothing is set. Sorted so it never depends on declaration
86
+ * order or on how the process was launched.
87
+ */
88
+ export function recallConfigFingerprint(env = process.env) {
89
+ const active = activeRecallConfig(env);
90
+ const keys = Object.keys(active).sort();
91
+ if (keys.length === 0)
92
+ return 'default';
93
+ return keys.map(k => `${k.replace(/^AWM_/, '').toLowerCase()}=${active[k]}`).join(',');
94
+ }
95
+ /**
96
+ * Assert the running configuration contains the expected flag values.
97
+ * Returns the mismatches; empty array means the system is configured as
98
+ * intended. Harnesses should treat a non-empty result as fatal — it means the
99
+ * thing being measured is not the thing that was configured.
100
+ */
101
+ export function diffRecallConfig(expected, env = process.env) {
102
+ const active = activeRecallConfig(env);
103
+ const bad = [];
104
+ for (const [flag, want] of Object.entries(expected)) {
105
+ if (active[flag] !== want)
106
+ bad.push({ flag, expected: want, actual: active[flag] });
107
+ }
108
+ return bad;
109
+ }
110
+ //# sourceMappingURL=recall-config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"recall-config.js","sourceRoot":"","sources":["../../src/core/recall-config.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,kCAAkC;IAClC,aAAa;IACb,eAAe;IACf,kCAAkC;IAClC,mBAAmB;IACnB,kBAAkB;IAClB,iBAAiB;IACjB,iBAAiB;IACjB,qBAAqB;IACrB,0CAA0C;IAC1C,oBAAoB;IACpB,kCAAkC;IAClC,aAAa;IACb,qBAAqB;IACrB,oEAAoE;IACpE,iBAAiB;IACjB,gBAAgB;IAChB,yBAAyB;IACzB,sCAAsC;IACtC,YAAY;IACZ,mBAAmB;IACnB,oBAAoB;IACpB,kBAAkB;IAClB,oBAAoB;IACpB,mBAAmB;IACnB,kBAAkB;IAClB,qCAAqC;IACrC,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,wBAAwB;IACxB,kBAAkB;IAClB,aAAa;IACb,+CAA+C;IAC/C,yBAAyB;IACzB,6BAA6B;IAC7B,wBAAwB;IACxB,oBAAoB;CACZ,CAAC;AAIX,yDAAyD;AACzD,MAAM,UAAU,kBAAkB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACrE,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC1E,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAgC,EAChC,MAAyB,OAAO,CAAC,GAAG;IAEpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,GAAG,GAA0E,EAAE,CAAC;IACtF,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,61 @@
1
+ /**
2
+ * Passage selection for cross-encoder reranking.
3
+ *
4
+ * THE PROBLEM
5
+ * -----------
6
+ * Reranking truncates each passage to a fixed prefix (historically the first
7
+ * 400 chars). That truncation exists for a real reason: cross-encoders pad to
8
+ * the longest passage in the batch, so one 5,000-char memory in a 40-item pool
9
+ * drags every passage to ~512 tokens and costs 3-4x. The reranker is already
10
+ * ~90% of warm recall latency, so "just send everything" is not available.
11
+ *
12
+ * But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
13
+ * - canonical memories median 1,965 chars, 98.7% exceed 400
14
+ * - the reranker cannot see 78.8% of each canonical memory's vocabulary
15
+ * - 99.9% of long canonical memories carry identifiers only past char 400
16
+ * And measured on tests/longmem-eval: moving the answer from char 150 to char
17
+ * 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
18
+ * collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
19
+ * retrievable and stops being rankable.
20
+ *
21
+ * THE FIX
22
+ * -------
23
+ * Spend the same character budget on the window that actually contains the
24
+ * query's terms, instead of on whatever happens to be at the top of the memory.
25
+ * Cost is unchanged — same budget, same batch padding, same inference — so this
26
+ * buys ranking quality without buying latency.
27
+ *
28
+ * The concept line is always kept: it is short, it is the memory's title, and
29
+ * it is what a human wrote to summarise the thing.
30
+ */
31
+ /**
32
+ * Choose the `budget`-char window of `content` densest in query terms.
33
+ * Returns the head of the content when nothing matches — the old behaviour,
34
+ * which is the right fallback: with no query signal there is no reason to
35
+ * prefer any other part of the memory.
36
+ */
37
+ export declare function densestWindow(content: string, query: string, budget: number): string;
38
+ /**
39
+ * Build the passage handed to the cross-encoder for one candidate.
40
+ *
41
+ * `mode`:
42
+ * - `'prefix'` (default) — legacy behaviour, the first `budget` chars.
43
+ * - `'query'` — the `budget`-char window densest in query terms.
44
+ */
45
+ export declare function buildRerankPassage(concept: string, content: string, query: string, budget: number, mode: 'prefix' | 'query', tags?: string[]): string;
46
+ /** Whether topical tags are appended to the rerank passage. Default OFF. */
47
+ export declare function rerankTagsEnabled(): boolean;
48
+ /** Character budget for the appended tag text. */
49
+ export declare function rerankTagBudget(): number;
50
+ /**
51
+ * Render topical tags as plain terms for the cross-encoder.
52
+ * Only `topic=` / `proj=` / `project=` carry query vocabulary; date/person/
53
+ * ticket tags are identifiers the body usually already contains, and adding
54
+ * them would spend the budget without adding reachable words.
55
+ */
56
+ export declare function rerankTagText(tags?: string[]): string;
57
+ /** Character budget for a rerank passage. */
58
+ export declare function rerankTruncation(): number;
59
+ /** Passage selection mode. Default `prefix` preserves shipped behaviour. */
60
+ export declare function rerankWindowMode(): 'prefix' | 'query';
61
+ //# sourceMappingURL=rerank-window.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rerank-window.d.ts","sourceRoot":"","sources":["../../src/core/rerank-window.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAaH;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAsCpF;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,QAAQ,GAAG,OAAO,EACxB,IAAI,CAAC,EAAE,MAAM,EAAE,GACd,MAAM,CAWR;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAED,kDAAkD;AAClD,wBAAgB,eAAe,IAAI,MAAM,CAGxC;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAYrD;AAED,6CAA6C;AAC7C,wBAAgB,gBAAgB,IAAI,MAAM,CAGzC;AAED,4EAA4E;AAC5E,wBAAgB,gBAAgB,IAAI,QAAQ,GAAG,OAAO,CAErD"}
@@ -0,0 +1,153 @@
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Passage selection for cross-encoder reranking.
5
+ *
6
+ * THE PROBLEM
7
+ * -----------
8
+ * Reranking truncates each passage to a fixed prefix (historically the first
9
+ * 400 chars). That truncation exists for a real reason: cross-encoders pad to
10
+ * the longest passage in the batch, so one 5,000-char memory in a 40-item pool
11
+ * drags every passage to ~512 tokens and costs 3-4x. The reranker is already
12
+ * ~90% of warm recall latency, so "just send everything" is not available.
13
+ *
14
+ * But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
15
+ * - canonical memories median 1,965 chars, 98.7% exceed 400
16
+ * - the reranker cannot see 78.8% of each canonical memory's vocabulary
17
+ * - 99.9% of long canonical memories carry identifiers only past char 400
18
+ * And measured on tests/longmem-eval: moving the answer from char 150 to char
19
+ * 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
20
+ * collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
21
+ * retrievable and stops being rankable.
22
+ *
23
+ * THE FIX
24
+ * -------
25
+ * Spend the same character budget on the window that actually contains the
26
+ * query's terms, instead of on whatever happens to be at the top of the memory.
27
+ * Cost is unchanged — same budget, same batch padding, same inference — so this
28
+ * buys ranking quality without buying latency.
29
+ *
30
+ * The concept line is always kept: it is short, it is the memory's title, and
31
+ * it is what a human wrote to summarise the thing.
32
+ */
33
+ /** Cheap tokenizer for locating query terms inside a passage. */
34
+ function terms(query) {
35
+ const raw = query.toLowerCase().match(/[a-z0-9_][a-z0-9_.\-]{1,}/g) ?? [];
36
+ const STOP = new Set([
37
+ 'the', 'and', 'for', 'with', 'that', 'this', 'from', 'what', 'where', 'when',
38
+ 'which', 'does', 'did', 'was', 'were', 'are', 'is', 'be', 'to', 'of', 'in',
39
+ 'on', 'at', 'by', 'it', 'as', 'do', 'how', 'why', 'a', 'an', 'or',
40
+ ]);
41
+ return Array.from(new Set(raw.filter(t => t.length >= 3 && !STOP.has(t))));
42
+ }
43
+ /**
44
+ * Choose the `budget`-char window of `content` densest in query terms.
45
+ * Returns the head of the content when nothing matches — the old behaviour,
46
+ * which is the right fallback: with no query signal there is no reason to
47
+ * prefer any other part of the memory.
48
+ */
49
+ export function densestWindow(content, query, budget) {
50
+ if (content.length <= budget)
51
+ return content;
52
+ const toks = terms(query);
53
+ if (toks.length === 0)
54
+ return content.slice(0, budget);
55
+ const lower = content.toLowerCase();
56
+ const hits = [];
57
+ for (const t of toks) {
58
+ let from = 0;
59
+ for (;;) {
60
+ const i = lower.indexOf(t, from);
61
+ if (i < 0)
62
+ break;
63
+ hits.push(i);
64
+ from = i + t.length;
65
+ }
66
+ }
67
+ if (hits.length === 0)
68
+ return content.slice(0, budget);
69
+ hits.sort((a, b) => a - b);
70
+ // Slide a window anchored slightly before each hit; keep the one covering most.
71
+ let bestStart = 0;
72
+ let bestCount = -1;
73
+ for (let i = 0; i < hits.length; i++) {
74
+ const start = Math.max(0, hits[i] - Math.floor(budget / 5));
75
+ let count = 0;
76
+ for (let j = i; j < hits.length && hits[j] - start < budget; j++)
77
+ count++;
78
+ if (count > bestCount) {
79
+ bestCount = count;
80
+ bestStart = start;
81
+ }
82
+ }
83
+ // Snap to a word boundary so the cross-encoder is not handed a split token.
84
+ let start = bestStart;
85
+ if (start > 0) {
86
+ const sp = content.indexOf(' ', start);
87
+ if (sp >= 0 && sp - start < 40)
88
+ start = sp + 1;
89
+ }
90
+ const slice = content.slice(start, start + budget);
91
+ return (start > 0 ? '…' : '') + slice + (start + budget < content.length ? '…' : '');
92
+ }
93
+ /**
94
+ * Build the passage handed to the cross-encoder for one candidate.
95
+ *
96
+ * `mode`:
97
+ * - `'prefix'` (default) — legacy behaviour, the first `budget` chars.
98
+ * - `'query'` — the `budget`-char window densest in query terms.
99
+ */
100
+ export function buildRerankPassage(concept, content, query, budget, mode, tags) {
101
+ const body = mode === 'query'
102
+ ? densestWindow(content, query, budget)
103
+ : (content.length > budget ? content.slice(0, budget) : content);
104
+ // Topical tags, when enabled. The cross-encoder decides final order since
105
+ // phase 9b, and it cannot see tags at all — measured on the live store, 66.2%
106
+ // of topical tag terms never appear in the body, so that vocabulary is
107
+ // invisible to the stage that now decides ranking. Appended (not substituted)
108
+ // and length-capped so it cannot crowd out the content window.
109
+ const extra = rerankTagText(tags);
110
+ return extra ? `${concept}: ${body} ${extra}` : `${concept}: ${body}`;
111
+ }
112
+ /** Whether topical tags are appended to the rerank passage. Default OFF. */
113
+ export function rerankTagsEnabled() {
114
+ return process.env.AWM_RERANK_TAGS === '1';
115
+ }
116
+ /** Character budget for the appended tag text. */
117
+ export function rerankTagBudget() {
118
+ const v = Number(process.env.AWM_RERANK_TAGS_LEN ?? 80);
119
+ return Number.isFinite(v) && v > 0 ? v : 80;
120
+ }
121
+ /**
122
+ * Render topical tags as plain terms for the cross-encoder.
123
+ * Only `topic=` / `proj=` / `project=` carry query vocabulary; date/person/
124
+ * ticket tags are identifiers the body usually already contains, and adding
125
+ * them would spend the budget without adding reachable words.
126
+ */
127
+ export function rerankTagText(tags) {
128
+ if (!rerankTagsEnabled() || !tags || tags.length === 0)
129
+ return '';
130
+ const words = new Set();
131
+ for (const t of tags) {
132
+ const m = /^(?:topic|proj|project)=(.+)$/i.exec(t);
133
+ if (!m)
134
+ continue;
135
+ for (const w of m[1].toLowerCase().split(/[-_\s]+/)) {
136
+ if (w.length > 2)
137
+ words.add(w);
138
+ }
139
+ }
140
+ if (words.size === 0)
141
+ return '';
142
+ return `[${[...words].join(' ').slice(0, rerankTagBudget())}]`;
143
+ }
144
+ /** Character budget for a rerank passage. */
145
+ export function rerankTruncation() {
146
+ const v = Number(process.env.AWM_RERANK_TRUNC ?? 400);
147
+ return Number.isFinite(v) && v > 0 ? v : 400;
148
+ }
149
+ /** Passage selection mode. Default `prefix` preserves shipped behaviour. */
150
+ export function rerankWindowMode() {
151
+ return process.env.AWM_RERANK_WINDOW === 'query' ? 'query' : 'prefix';
152
+ }
153
+ //# sourceMappingURL=rerank-window.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rerank-window.js","sourceRoot":"","sources":["../../src/core/rerank-window.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,iEAAiE;AACjE,SAAS,KAAK,CAAC,KAAa;IAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,4BAA4B,CAAC,IAAI,EAAE,CAAC;IAC1E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC;QACnB,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;QAC5E,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;QAC1E,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI;KAClE,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,KAAa,EAAE,MAAc;IAC1E,IAAI,OAAO,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,OAAO,CAAC;IAE7C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM;YACjB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACtB,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACvD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE3B,gFAAgF;IAChF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,MAAM,EAAE,CAAC,EAAE;YAAE,KAAK,EAAE,CAAC;QAC1E,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YAAC,SAAS,GAAG,KAAK,CAAC;YAAC,SAAS,GAAG,KAAK,CAAC;QAAC,CAAC;IAClE,CAAC;IAED,4EAA4E;IAC5E,IAAI,KAAK,GAAG,SAAS,CAAC;IACtB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,EAAE;YAAE,KAAK,GAAG,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACvF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,OAAe,EACf,KAAa,EACb,MAAc,EACd,IAAwB,EACxB,IAAe;IAEf,MAAM,IAAI,GAAG,IAAI,KAAK,OAAO;QAC3B,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC;QACvC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACnE,0EAA0E;IAC1E,8EAA8E;IAC9E,uEAAuE;IACvE,8EAA8E;IAC9E,+DAA+D;IAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,EAAE,CAAC;AACxE,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,iBAAiB;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,GAAG,CAAC;AAC7C,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,eAAe;IAC7B,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAe;IAC3C,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,gCAAgC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,GAAG,CAAC;AACjE,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,gBAAgB;IAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAC;IACtD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC/C,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,gBAAgB;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Second-stage reordering — "rerank the rerank".
3
+ *
4
+ * THE PROBLEM
5
+ * -----------
6
+ * Phase 7 blends the cross-encoder's judgement with the composite score:
7
+ *
8
+ * score = compositeWeight * composite + rerankWeight * rerankerScore
9
+ *
10
+ * with `rerankWeight` capped at 0.7, so `composite` always keeps at least 30%
11
+ * of the vote on final ordering. Composite carries decay, Hebbian and salience
12
+ * terms — useful for deciding WHICH candidates deserve consideration, but a
13
+ * poor judge of which one actually answers the question.
14
+ *
15
+ * Measured on LoCoMo (616 answerable probes, tests/rerank2-eval): the blend
16
+ * disagrees with the cross-encoder about which item deserves rank 1 on **38.6%**
17
+ * of queries. Where that disagreement is decidable — one of the two is the gold
18
+ * evidence — the **cross-encoder is right 77% of the time** (61 vs 18).
19
+ *
20
+ * THE FIX
21
+ * -------
22
+ * Re-sort only the final returned window by `rerankerScore` alone. Simulated
23
+ * gain: **+8.0pp success@1** (39.1% -> 47.1%), fixing 61 queries and breaking
24
+ * 12, with every category improving.
25
+ *
26
+ * WHY THIS IS SAFE
27
+ * ----------------
28
+ * Applied AFTER the channel-agreement gate, AFTER computeRecallConfidence, and
29
+ * AFTER the requireConfidence check. Those read rerankerScore maxima/margins and
30
+ * the score distribution. Reordering a window afterwards changes neither its
31
+ * membership nor any score, so **adversarial abstention is provably
32
+ * unaffected**. Retuning `rerankWeight` inside phase 7 would NOT have that
33
+ * property — it shifts `item.score`, hence which items clear `minScore`, which
34
+ * is exactly what cost adversarial 73.4->71.0 when the pool was last widened.
35
+ *
36
+ * Margin-guarded variants were simulated and are strictly worse: gating on a
37
+ * >0.15 reranker margin cuts breakage 12->1 but also cuts fixes 61->17.
38
+ */
39
+ /** Minimal shape this needs — anything carrying a cross-encoder score. */
40
+ export interface RerankScored {
41
+ phaseScores: {
42
+ rerankerScore: number;
43
+ };
44
+ }
45
+ /**
46
+ * Re-sort the first `k` entries of `ranked` by descending `rerankerScore`,
47
+ * leaving the tail untouched. Returns a new array; never mutates the input.
48
+ *
49
+ * Returns `ranked` unchanged when reordering would be unsound:
50
+ * - `k <= 1`, or fewer than 2 entries to order
51
+ * - any entry in the window has `rerankerScore <= 0`, which means the
52
+ * cross-encoder did not score it (skipped, failed, or timed out). Sorting
53
+ * on a zero would scramble an otherwise sound composite ordering — the
54
+ * reranker is wrapped in try/catch and a 10s timeout upstream, so this is a
55
+ * real path, not a theoretical one.
56
+ */
57
+ export declare function reorderByReranker<T extends RerankScored>(ranked: T[], k: number): T[];
58
+ /** Window size for the second stage. */
59
+ export declare function rerank2WindowSize(): number;
60
+ /** Whether the second stage is enabled. Default OFF pending the tracer verdict. */
61
+ export declare function rerank2Enabled(): boolean;
62
+ //# sourceMappingURL=rerank2.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rerank2.d.ts","sourceRoot":"","sources":["../../src/core/rerank2.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,0EAA0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE;QAAE,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CACxC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,SAAS,YAAY,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAarF;AAED,wCAAwC;AACxC,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,mFAAmF;AACnF,wBAAgB,cAAc,IAAI,OAAO,CAExC"}
@@ -0,0 +1,75 @@
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Second-stage reordering — "rerank the rerank".
5
+ *
6
+ * THE PROBLEM
7
+ * -----------
8
+ * Phase 7 blends the cross-encoder's judgement with the composite score:
9
+ *
10
+ * score = compositeWeight * composite + rerankWeight * rerankerScore
11
+ *
12
+ * with `rerankWeight` capped at 0.7, so `composite` always keeps at least 30%
13
+ * of the vote on final ordering. Composite carries decay, Hebbian and salience
14
+ * terms — useful for deciding WHICH candidates deserve consideration, but a
15
+ * poor judge of which one actually answers the question.
16
+ *
17
+ * Measured on LoCoMo (616 answerable probes, tests/rerank2-eval): the blend
18
+ * disagrees with the cross-encoder about which item deserves rank 1 on **38.6%**
19
+ * of queries. Where that disagreement is decidable — one of the two is the gold
20
+ * evidence — the **cross-encoder is right 77% of the time** (61 vs 18).
21
+ *
22
+ * THE FIX
23
+ * -------
24
+ * Re-sort only the final returned window by `rerankerScore` alone. Simulated
25
+ * gain: **+8.0pp success@1** (39.1% -> 47.1%), fixing 61 queries and breaking
26
+ * 12, with every category improving.
27
+ *
28
+ * WHY THIS IS SAFE
29
+ * ----------------
30
+ * Applied AFTER the channel-agreement gate, AFTER computeRecallConfidence, and
31
+ * AFTER the requireConfidence check. Those read rerankerScore maxima/margins and
32
+ * the score distribution. Reordering a window afterwards changes neither its
33
+ * membership nor any score, so **adversarial abstention is provably
34
+ * unaffected**. Retuning `rerankWeight` inside phase 7 would NOT have that
35
+ * property — it shifts `item.score`, hence which items clear `minScore`, which
36
+ * is exactly what cost adversarial 73.4->71.0 when the pool was last widened.
37
+ *
38
+ * Margin-guarded variants were simulated and are strictly worse: gating on a
39
+ * >0.15 reranker margin cuts breakage 12->1 but also cuts fixes 61->17.
40
+ */
41
+ /**
42
+ * Re-sort the first `k` entries of `ranked` by descending `rerankerScore`,
43
+ * leaving the tail untouched. Returns a new array; never mutates the input.
44
+ *
45
+ * Returns `ranked` unchanged when reordering would be unsound:
46
+ * - `k <= 1`, or fewer than 2 entries to order
47
+ * - any entry in the window has `rerankerScore <= 0`, which means the
48
+ * cross-encoder did not score it (skipped, failed, or timed out). Sorting
49
+ * on a zero would scramble an otherwise sound composite ordering — the
50
+ * reranker is wrapped in try/catch and a 10s timeout upstream, so this is a
51
+ * real path, not a theoretical one.
52
+ */
53
+ export function reorderByReranker(ranked, k) {
54
+ if (k <= 1 || ranked.length <= 1)
55
+ return ranked;
56
+ const window = Math.min(k, ranked.length);
57
+ if (window <= 1)
58
+ return ranked;
59
+ const head = ranked.slice(0, window);
60
+ for (const r of head) {
61
+ if (!(r.phaseScores.rerankerScore > 0))
62
+ return ranked; // also catches NaN
63
+ }
64
+ const sorted = head.slice().sort((a, b) => b.phaseScores.rerankerScore - a.phaseScores.rerankerScore);
65
+ return sorted.concat(ranked.slice(window));
66
+ }
67
+ /** Window size for the second stage. */
68
+ export function rerank2WindowSize() {
69
+ return Number(process.env.AWM_RERANK2_K ?? 10);
70
+ }
71
+ /** Whether the second stage is enabled. Default OFF pending the tracer verdict. */
72
+ export function rerank2Enabled() {
73
+ return process.env.AWM_RERANK2 === '1';
74
+ }
75
+ //# sourceMappingURL=rerank2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rerank2.js","sourceRoot":"","sources":["../../src/core/rerank2.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAOH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAyB,MAAW,EAAE,CAAS;IAC9E,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAEhD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAE/B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,GAAG,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,CAAE,mBAAmB;IAC7E,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,GAAG,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IACtG,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,iBAAiB;IAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;AACjD,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,GAAG,CAAC;AACzC,CAAC"}
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Derived retrieval text — the machine-facing view of a memory.
3
+ *
4
+ * WHY THIS EXISTS
5
+ * ---------------
6
+ * Measured on the live 11,294-engram store: 66.2% of topical tag terms never
7
+ * appear in the memory body, and 94.3% of tagged memories are missing at least
8
+ * one of their own topical terms. Tags are indexed by BM25 only — the embedding
9
+ * is built from `concept + content` (write-pipeline.ts) and so is the
10
+ * cross-encoder rerank passage (activation.ts). So that vocabulary is invisible
11
+ * to two of the three retrieval channels, including the one that decides final
12
+ * ordering since phase 9b.
13
+ *
14
+ * The consequence, observed on a real memory: "private plan memory peaked 88%,
15
+ * scale P1v3 -> P2v3" was NOT in the top 40 candidates for "azure app service
16
+ * plan capacity increase internal application". Its body contains no "azure",
17
+ * no "capacity", no "app service plan" — only `topic=azure` as a tag.
18
+ *
19
+ * THE SHAPE THAT MATTERS
20
+ * ----------------------
21
+ * This builds a DERIVED text used for embedding and reranking. It does NOT
22
+ * mutate `content`. That distinction is the whole design:
23
+ *
24
+ * - AWM's model slots have always been ADDITIVE — the embedder, expander and
25
+ * reranker score or expand, they never rewrite what was stored. Appending
26
+ * tag terms into the body would break that invariant, and a normaliser that
27
+ * silently edits stored memories is unrecoverable if it is wrong.
28
+ * - A derived view is recomputable. If the rule turns out to be bad, re-derive
29
+ * and re-embed; the source of truth was never touched.
30
+ * - And it can be BACKFILLED over the existing corpus, which body-only fixes
31
+ * (write-time guidance, future writes) cannot reach. 7,350 memories are
32
+ * already wrong.
33
+ *
34
+ * Only `topic=` / `proj=` / `project=` are included. Date, person and ticket
35
+ * tags are identifiers the body usually already carries, so adding them spends
36
+ * budget without adding reachable words.
37
+ */
38
+ /** Whether the derived retrieval text includes tag vocabulary. Default OFF. */
39
+ export declare function retrievalTextEnabled(): boolean;
40
+ /**
41
+ * Extract the topical vocabulary a future question is likely to use.
42
+ * Deduplicated and lowercased; order follows first appearance so the output is
43
+ * deterministic for a given tag list (important: a non-deterministic embedding
44
+ * input would make re-embedding produce different vectors for the same memory).
45
+ */
46
+ export declare function topicalTerms(tags?: string[]): string[];
47
+ /**
48
+ * Build the text used for EMBEDDING a memory.
49
+ *
50
+ * When disabled this returns exactly `concept + ' ' + content`, byte-identical
51
+ * to the historical input — so leaving the flag off cannot change a single
52
+ * stored vector.
53
+ */
54
+ export declare function buildRetrievalText(concept: string, content: string, tags?: string[]): string;
55
+ //# sourceMappingURL=retrieval-text.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieval-text.d.ts","sourceRoot":"","sources":["../../src/core/retrieval-text.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,+EAA+E;AAC/E,wBAAgB,oBAAoB,IAAI,OAAO,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAYtD;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAS5F"}
@@ -0,0 +1,87 @@
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Derived retrieval text — the machine-facing view of a memory.
5
+ *
6
+ * WHY THIS EXISTS
7
+ * ---------------
8
+ * Measured on the live 11,294-engram store: 66.2% of topical tag terms never
9
+ * appear in the memory body, and 94.3% of tagged memories are missing at least
10
+ * one of their own topical terms. Tags are indexed by BM25 only — the embedding
11
+ * is built from `concept + content` (write-pipeline.ts) and so is the
12
+ * cross-encoder rerank passage (activation.ts). So that vocabulary is invisible
13
+ * to two of the three retrieval channels, including the one that decides final
14
+ * ordering since phase 9b.
15
+ *
16
+ * The consequence, observed on a real memory: "private plan memory peaked 88%,
17
+ * scale P1v3 -> P2v3" was NOT in the top 40 candidates for "azure app service
18
+ * plan capacity increase internal application". Its body contains no "azure",
19
+ * no "capacity", no "app service plan" — only `topic=azure` as a tag.
20
+ *
21
+ * THE SHAPE THAT MATTERS
22
+ * ----------------------
23
+ * This builds a DERIVED text used for embedding and reranking. It does NOT
24
+ * mutate `content`. That distinction is the whole design:
25
+ *
26
+ * - AWM's model slots have always been ADDITIVE — the embedder, expander and
27
+ * reranker score or expand, they never rewrite what was stored. Appending
28
+ * tag terms into the body would break that invariant, and a normaliser that
29
+ * silently edits stored memories is unrecoverable if it is wrong.
30
+ * - A derived view is recomputable. If the rule turns out to be bad, re-derive
31
+ * and re-embed; the source of truth was never touched.
32
+ * - And it can be BACKFILLED over the existing corpus, which body-only fixes
33
+ * (write-time guidance, future writes) cannot reach. 7,350 memories are
34
+ * already wrong.
35
+ *
36
+ * Only `topic=` / `proj=` / `project=` are included. Date, person and ticket
37
+ * tags are identifiers the body usually already carries, so adding them spends
38
+ * budget without adding reachable words.
39
+ */
40
+ /** Whether the derived retrieval text includes tag vocabulary. Default OFF. */
41
+ export function retrievalTextEnabled() {
42
+ return process.env.AWM_RETRIEVAL_TEXT === '1';
43
+ }
44
+ /**
45
+ * Extract the topical vocabulary a future question is likely to use.
46
+ * Deduplicated and lowercased; order follows first appearance so the output is
47
+ * deterministic for a given tag list (important: a non-deterministic embedding
48
+ * input would make re-embedding produce different vectors for the same memory).
49
+ */
50
+ export function topicalTerms(tags) {
51
+ if (!tags || tags.length === 0)
52
+ return [];
53
+ const out = [];
54
+ const seen = new Set();
55
+ for (const t of tags) {
56
+ const m = /^(?:topic|proj|project)=(.+)$/i.exec(t);
57
+ if (!m)
58
+ continue;
59
+ for (const w of m[1].toLowerCase().split(/[-_\s]+/)) {
60
+ if (w.length > 2 && !seen.has(w)) {
61
+ seen.add(w);
62
+ out.push(w);
63
+ }
64
+ }
65
+ }
66
+ return out;
67
+ }
68
+ /**
69
+ * Build the text used for EMBEDDING a memory.
70
+ *
71
+ * When disabled this returns exactly `concept + ' ' + content`, byte-identical
72
+ * to the historical input — so leaving the flag off cannot change a single
73
+ * stored vector.
74
+ */
75
+ export function buildRetrievalText(concept, content, tags) {
76
+ const base = `${concept} ${content}`;
77
+ if (!retrievalTextEnabled())
78
+ return base;
79
+ const terms = topicalTerms(tags);
80
+ if (terms.length === 0)
81
+ return base;
82
+ // Appended at the END. The embedding model truncates beyond its context
83
+ // window, and the head of the content is what anchors the topic — putting tag
84
+ // terms first would displace the memory's actual subject in the vector.
85
+ return `${base} ${terms.join(' ')}`;
86
+ }
87
+ //# sourceMappingURL=retrieval-text.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieval-text.js","sourceRoot":"","sources":["../../src/core/retrieval-text.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,+EAA+E;AAC/E,MAAM,UAAU,oBAAoB;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,IAAe;IAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC1C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,gCAAgC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;QACjE,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,OAAe,EAAE,IAAe;IAClF,MAAM,IAAI,GAAG,GAAG,OAAO,IAAI,OAAO,EAAE,CAAC;IACrC,IAAI,CAAC,oBAAoB,EAAE;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,wEAAwE;IACxE,8EAA8E;IAC9E,wEAAwE;IACxE,OAAO,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACtC,CAAC"}