agent-working-memory 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +8 -5
  2. package/dist/adapters/common.d.ts +34 -0
  3. package/dist/adapters/common.d.ts.map +1 -1
  4. package/dist/adapters/common.js +528 -446
  5. package/dist/adapters/common.js.map +1 -1
  6. package/dist/api/routes.d.ts.map +1 -1
  7. package/dist/api/routes.js +19 -0
  8. package/dist/api/routes.js.map +1 -1
  9. package/dist/cli/migrate.d.ts.map +1 -1
  10. package/dist/cli/migrate.js +31 -29
  11. package/dist/cli/migrate.js.map +1 -1
  12. package/dist/core/asked-value.d.ts +37 -0
  13. package/dist/core/asked-value.d.ts.map +1 -0
  14. package/dist/core/asked-value.js +77 -0
  15. package/dist/core/asked-value.js.map +1 -0
  16. package/dist/core/conflicts.d.ts +89 -0
  17. package/dist/core/conflicts.d.ts.map +1 -0
  18. package/dist/core/conflicts.js +267 -0
  19. package/dist/core/conflicts.js.map +1 -0
  20. package/dist/core/embeddings.d.ts +14 -2
  21. package/dist/core/embeddings.d.ts.map +1 -1
  22. package/dist/core/embeddings.js +32 -10
  23. package/dist/core/embeddings.js.map +1 -1
  24. package/dist/core/format-recall.d.ts +21 -1
  25. package/dist/core/format-recall.d.ts.map +1 -1
  26. package/dist/core/format-recall.js +60 -5
  27. package/dist/core/format-recall.js.map +1 -1
  28. package/dist/core/ml-worker.d.ts +24 -1
  29. package/dist/core/ml-worker.d.ts.map +1 -1
  30. package/dist/core/ml-worker.js +165 -4
  31. package/dist/core/ml-worker.js.map +1 -1
  32. package/dist/core/model-daemon-client.d.ts +77 -0
  33. package/dist/core/model-daemon-client.d.ts.map +1 -0
  34. package/dist/core/model-daemon-client.js +383 -0
  35. package/dist/core/model-daemon-client.js.map +1 -0
  36. package/dist/core/model-daemon-server.d.ts +71 -0
  37. package/dist/core/model-daemon-server.d.ts.map +1 -0
  38. package/dist/core/model-daemon-server.js +363 -0
  39. package/dist/core/model-daemon-server.js.map +1 -0
  40. package/dist/core/model-daemon.d.ts +137 -0
  41. package/dist/core/model-daemon.d.ts.map +1 -0
  42. package/dist/core/model-daemon.js +231 -0
  43. package/dist/core/model-daemon.js.map +1 -0
  44. package/dist/core/query-expander.d.ts +5 -0
  45. package/dist/core/query-expander.d.ts.map +1 -1
  46. package/dist/core/query-expander.js +3 -2
  47. package/dist/core/query-expander.js.map +1 -1
  48. package/dist/core/recall-config.d.ts +1 -1
  49. package/dist/core/recall-config.d.ts.map +1 -1
  50. package/dist/core/recall-config.js +5 -0
  51. package/dist/core/recall-config.js.map +1 -1
  52. package/dist/core/rerank-window.d.ts +7 -2
  53. package/dist/core/rerank-window.d.ts.map +1 -1
  54. package/dist/core/rerank-window.js +9 -4
  55. package/dist/core/rerank-window.js.map +1 -1
  56. package/dist/core/rerank2.d.ts +52 -2
  57. package/dist/core/rerank2.d.ts.map +1 -1
  58. package/dist/core/rerank2.js +60 -3
  59. package/dist/core/rerank2.js.map +1 -1
  60. package/dist/core/reranker.d.ts +19 -0
  61. package/dist/core/reranker.d.ts.map +1 -1
  62. package/dist/core/reranker.js +90 -13
  63. package/dist/core/reranker.js.map +1 -1
  64. package/dist/core/trust.d.ts +32 -0
  65. package/dist/core/trust.d.ts.map +1 -0
  66. package/dist/core/trust.js +66 -0
  67. package/dist/core/trust.js.map +1 -0
  68. package/dist/core/validity.d.ts +21 -0
  69. package/dist/core/validity.d.ts.map +1 -0
  70. package/dist/core/validity.js +41 -0
  71. package/dist/core/validity.js.map +1 -0
  72. package/dist/core/whoami.d.ts +3 -0
  73. package/dist/core/whoami.d.ts.map +1 -1
  74. package/dist/core/whoami.js +14 -0
  75. package/dist/core/whoami.js.map +1 -1
  76. package/dist/core/write-pipeline.d.ts +48 -0
  77. package/dist/core/write-pipeline.d.ts.map +1 -1
  78. package/dist/core/write-pipeline.js +80 -3
  79. package/dist/core/write-pipeline.js.map +1 -1
  80. package/dist/engine/activation.d.ts.map +1 -1
  81. package/dist/engine/activation.js +27 -4
  82. package/dist/engine/activation.js.map +1 -1
  83. package/dist/engine/consolidation.d.ts.map +1 -1
  84. package/dist/engine/consolidation.js +53 -4
  85. package/dist/engine/consolidation.js.map +1 -1
  86. package/dist/hooks/prime-query.d.ts +57 -0
  87. package/dist/hooks/prime-query.d.ts.map +1 -0
  88. package/dist/hooks/prime-query.js +388 -0
  89. package/dist/hooks/prime-query.js.map +1 -0
  90. package/dist/hooks/prime.d.ts +8 -0
  91. package/dist/hooks/prime.d.ts.map +1 -1
  92. package/dist/hooks/prime.js +21 -2
  93. package/dist/hooks/prime.js.map +1 -1
  94. package/dist/hooks/sidecar.d.ts +18 -0
  95. package/dist/hooks/sidecar.d.ts.map +1 -1
  96. package/dist/hooks/sidecar.js +58 -2
  97. package/dist/hooks/sidecar.js.map +1 -1
  98. package/dist/index.js +15 -3
  99. package/dist/index.js.map +1 -1
  100. package/dist/mcp.js +157 -104
  101. package/dist/mcp.js.map +1 -1
  102. package/dist/model-daemon.d.ts +8 -0
  103. package/dist/model-daemon.d.ts.map +1 -0
  104. package/dist/model-daemon.js +45 -0
  105. package/dist/model-daemon.js.map +1 -0
  106. package/dist/storage/factory.d.ts.map +1 -1
  107. package/dist/storage/factory.js +5 -0
  108. package/dist/storage/factory.js.map +1 -1
  109. package/dist/storage/pglite-schema.d.ts.map +1 -1
  110. package/dist/storage/pglite-schema.js +176 -170
  111. package/dist/storage/pglite-schema.js.map +1 -1
  112. package/dist/storage/pglite.d.ts +5 -0
  113. package/dist/storage/pglite.d.ts.map +1 -1
  114. package/dist/storage/pglite.js +165 -153
  115. package/dist/storage/pglite.js.map +1 -1
  116. package/dist/storage/postgres.d.ts +5 -0
  117. package/dist/storage/postgres.d.ts.map +1 -1
  118. package/dist/storage/postgres.js +165 -153
  119. package/dist/storage/postgres.js.map +1 -1
  120. package/dist/storage/sqlite.d.ts +22 -0
  121. package/dist/storage/sqlite.d.ts.map +1 -1
  122. package/dist/storage/sqlite.js +415 -363
  123. package/dist/storage/sqlite.js.map +1 -1
  124. package/dist/types/engram.d.ts +33 -0
  125. package/dist/types/engram.d.ts.map +1 -1
  126. package/dist/types/engram.js.map +1 -1
  127. package/package.json +1 -1
  128. package/plugin/.claude-plugin/plugin.json +1 -1
  129. package/plugin/README.md +1 -1
  130. package/plugin/bin/awm-mcp-launcher.cjs +2 -2
  131. package/plugin/skills/awm-memory/SKILL.md +11 -2
  132. package/src/adapters/common.ts +881 -802
  133. package/src/api/routes.ts +20 -0
  134. package/src/cli/migrate.ts +309 -307
  135. package/src/core/asked-value.ts +74 -0
  136. package/src/core/conflicts.ts +280 -0
  137. package/src/core/embeddings.ts +35 -10
  138. package/src/core/format-recall.ts +83 -25
  139. package/src/core/ml-worker.ts +458 -281
  140. package/src/core/model-daemon-client.ts +374 -0
  141. package/src/core/model-daemon-server.ts +344 -0
  142. package/src/core/model-daemon.ts +243 -0
  143. package/src/core/query-expander.ts +127 -126
  144. package/src/core/recall-config.ts +5 -0
  145. package/src/core/rerank-window.ts +164 -159
  146. package/src/core/rerank2.ts +143 -82
  147. package/src/core/reranker.ts +92 -12
  148. package/src/core/trust.ts +65 -0
  149. package/src/core/validity.ts +47 -0
  150. package/src/core/whoami.ts +136 -120
  151. package/src/core/write-pipeline.ts +119 -3
  152. package/src/engine/activation.ts +27 -4
  153. package/src/engine/consolidation.ts +950 -899
  154. package/src/hooks/prime-query.ts +375 -0
  155. package/src/hooks/prime.ts +25 -2
  156. package/src/hooks/sidecar.ts +57 -3
  157. package/src/index.ts +258 -248
  158. package/src/mcp.ts +1529 -1475
  159. package/src/model-daemon.ts +48 -0
  160. package/src/storage/factory.ts +25 -19
  161. package/src/storage/pglite-schema.ts +199 -193
  162. package/src/storage/pglite.ts +1513 -1498
  163. package/src/storage/postgres.ts +1613 -1598
  164. package/src/storage/sqlite.ts +2069 -2019
  165. package/src/types/engram.ts +30 -0
@@ -1,126 +1,127 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * Query Expander - rewrites queries with synonyms and related terms.
5
- *
6
- * Uses Xenova/flan-t5-small — a 141MB encoder plus a 233MB merged decoder, so
7
- * 374MB of ONNX and a 377MB download with its tokenizer. By far the largest of
8
- * the three models AWM fetches. Expands search queries with related terms that
9
- * improve BM25 recall.
10
- *
11
- * AWM 0.8.x: inference dispatches through ml-worker.ts (currently in-process
12
- * — worker_threads reverted because onnxruntime-node bindings cross isolate
13
- * boundaries unsafely; see ml-worker.ts). The dispatch abstraction is
14
- * preserved for a future child_process / HTTP sidecar pool.
15
- *
16
- * The LRU cache + skip heuristic stay on the main thread — they're pure
17
- * filter/lookup logic that shouldn't pay IPC cost.
18
- */
19
-
20
- import { pipeline, type Text2TextGenerationPipeline } from '@huggingface/transformers';
21
- import { dispatchExpand, registerInProcessHandlers } from './ml-worker.js';
22
- import { ensureModelCacheDir } from './model-cache.js';
23
-
24
- const MODEL_ID = 'Xenova/flan-t5-small';
25
-
26
- // --- In-process fallback ---
27
-
28
- let inProcessInstance: Text2TextGenerationPipeline | null = null;
29
- let inProcessInitPromise: Promise<Text2TextGenerationPipeline> | null = null;
30
-
31
- async function loadInProcess(): Promise<Text2TextGenerationPipeline> {
32
- if (inProcessInstance) return inProcessInstance;
33
- if (inProcessInitPromise) return inProcessInitPromise;
34
- ensureModelCacheDir();
35
- inProcessInitPromise = pipeline('text2text-generation', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
36
- inProcessInstance = pipe as Text2TextGenerationPipeline;
37
- console.error(`Query expander loaded in-process: ${MODEL_ID}`);
38
- return inProcessInstance;
39
- });
40
- return inProcessInitPromise;
41
- }
42
-
43
- async function inProcessExpand(args: { prompt: string; maxNewTokens: number; noRepeatNgramSize: number }): Promise<string> {
44
- const expander = await loadInProcess();
45
- const result = await expander(args.prompt, {
46
- max_new_tokens: args.maxNewTokens,
47
- no_repeat_ngram_size: args.noRepeatNgramSize,
48
- });
49
- const text = Array.isArray(result) ? (result[0] as any)?.generated_text ?? '' : '';
50
- return String(text).trim();
51
- }
52
-
53
- registerInProcessHandlers({ expand: inProcessExpand });
54
-
55
- // --- Public API ---
56
-
57
- /** Kept for backwards compat. */
58
- export async function getExpander(): Promise<Text2TextGenerationPipeline> {
59
- return loadInProcess();
60
- }
61
-
62
- /**
63
- * LRU cache of normalized-query → expanded-query mappings.
64
- * Lives on the main thread — cache hits skip the worker IPC entirely.
65
- */
66
- const expansionCache = new Map<string, string>();
67
- const EXPANSION_CACHE_LIMIT = 500;
68
-
69
- /**
70
- * Skip expansion when the query is already specific (long or many tokens).
71
- */
72
- function shouldSkipExpansion(normalized: string): boolean {
73
- if (normalized.length === 0) return true;
74
- if (normalized.length > 50) return true;
75
- const tokens = new Set(normalized.split(/\s+/).filter(t => t.length > 2));
76
- return tokens.size >= 5;
77
- }
78
-
79
- /**
80
- * Expand a query with related terms and synonyms.
81
- * Returns the original query + generated expansion terms.
82
- * Falls back to the original query on any error.
83
- *
84
- * Dispatches inference to the worker pool. Cache + skip heuristic stay
85
- * on the main thread.
86
- */
87
- export async function expandQuery(originalQuery: string): Promise<string> {
88
- const normalized = originalQuery.toLowerCase().trim();
89
- const optimizationsEnabled = process.env.AWM_DISABLE_EXPANSION_CACHE !== '1';
90
-
91
- if (optimizationsEnabled) {
92
- if (shouldSkipExpansion(normalized)) return originalQuery;
93
- const cached = expansionCache.get(normalized);
94
- if (cached !== undefined) {
95
- // LRU touch
96
- expansionCache.delete(normalized);
97
- expansionCache.set(normalized, cached);
98
- return cached;
99
- }
100
- }
101
-
102
- try {
103
- const prompt = `Expand this search query with synonyms and related terms. Only output the additional terms, not the original query. Query: ${originalQuery}. Additional terms:`;
104
- const expansion = await dispatchExpand({ prompt, maxNewTokens: 25, noRepeatNgramSize: 2 });
105
- const finalQuery = expansion && expansion.length > 2
106
- ? `${originalQuery} ${expansion}`
107
- : originalQuery;
108
-
109
- if (optimizationsEnabled) {
110
- if (expansionCache.size >= EXPANSION_CACHE_LIMIT) {
111
- const oldestKey = expansionCache.keys().next().value;
112
- if (oldestKey !== undefined) expansionCache.delete(oldestKey);
113
- }
114
- expansionCache.set(normalized, finalQuery);
115
- }
116
-
117
- return finalQuery;
118
- } catch {
119
- return originalQuery;
120
- }
121
- }
122
-
123
- /** Clear the expansion cache (used by tests + cache invalidation). */
124
- export function clearExpansionCache(): void {
125
- expansionCache.clear();
126
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Query Expander - rewrites queries with synonyms and related terms.
5
+ *
6
+ * Uses Xenova/flan-t5-small — a 141MB encoder plus a 233MB merged decoder, so
7
+ * 374MB of ONNX and a 377MB download with its tokenizer. By far the largest of
8
+ * the three models AWM fetches. Expands search queries with related terms that
9
+ * improve BM25 recall.
10
+ *
11
+ * AWM 0.8.x: inference dispatches through ml-worker.ts (currently in-process
12
+ * — worker_threads reverted because onnxruntime-node bindings cross isolate
13
+ * boundaries unsafely; see ml-worker.ts). The dispatch abstraction is
14
+ * preserved for a future child_process / HTTP sidecar pool.
15
+ *
16
+ * The LRU cache + skip heuristic stay on the main thread — they're pure
17
+ * filter/lookup logic that shouldn't pay IPC cost.
18
+ */
19
+
20
+ import { pipeline, type Text2TextGenerationPipeline } from '@huggingface/transformers';
21
+ import { dispatchExpand, registerInferenceConfig, registerInProcessHandlers } from './ml-worker.js';
22
+ import { ensureModelCacheDir } from './model-cache.js';
23
+
24
+ const MODEL_ID = 'Xenova/flan-t5-small';
25
+
26
+ // --- In-process fallback ---
27
+
28
+ let inProcessInstance: Text2TextGenerationPipeline | null = null;
29
+ let inProcessInitPromise: Promise<Text2TextGenerationPipeline> | null = null;
30
+
31
+ async function loadInProcess(): Promise<Text2TextGenerationPipeline> {
32
+ if (inProcessInstance) return inProcessInstance;
33
+ if (inProcessInitPromise) return inProcessInitPromise;
34
+ ensureModelCacheDir();
35
+ inProcessInitPromise = pipeline('text2text-generation', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
36
+ inProcessInstance = pipe as Text2TextGenerationPipeline;
37
+ console.error(`Query expander loaded in-process: ${MODEL_ID}`);
38
+ return inProcessInstance;
39
+ });
40
+ return inProcessInitPromise;
41
+ }
42
+
43
+ export async function inProcessExpand(args: { prompt: string; maxNewTokens: number; noRepeatNgramSize: number }): Promise<string> {
44
+ const expander = await loadInProcess();
45
+ const result = await expander(args.prompt, {
46
+ max_new_tokens: args.maxNewTokens,
47
+ no_repeat_ngram_size: args.noRepeatNgramSize,
48
+ });
49
+ const text = Array.isArray(result) ? (result[0] as any)?.generated_text ?? '' : '';
50
+ return String(text).trim();
51
+ }
52
+
53
+ registerInProcessHandlers({ expand: inProcessExpand });
54
+ registerInferenceConfig({ expandModel: MODEL_ID });
55
+
56
+ // --- Public API ---
57
+
58
+ /** Kept for backwards compat. */
59
+ export async function getExpander(): Promise<Text2TextGenerationPipeline> {
60
+ return loadInProcess();
61
+ }
62
+
63
+ /**
64
+ * LRU cache of normalized-query → expanded-query mappings.
65
+ * Lives on the main thread — cache hits skip the worker IPC entirely.
66
+ */
67
+ const expansionCache = new Map<string, string>();
68
+ const EXPANSION_CACHE_LIMIT = 500;
69
+
70
+ /**
71
+ * Skip expansion when the query is already specific (long or many tokens).
72
+ */
73
+ function shouldSkipExpansion(normalized: string): boolean {
74
+ if (normalized.length === 0) return true;
75
+ if (normalized.length > 50) return true;
76
+ const tokens = new Set(normalized.split(/\s+/).filter(t => t.length > 2));
77
+ return tokens.size >= 5;
78
+ }
79
+
80
+ /**
81
+ * Expand a query with related terms and synonyms.
82
+ * Returns the original query + generated expansion terms.
83
+ * Falls back to the original query on any error.
84
+ *
85
+ * Dispatches inference to the worker pool. Cache + skip heuristic stay
86
+ * on the main thread.
87
+ */
88
+ export async function expandQuery(originalQuery: string): Promise<string> {
89
+ const normalized = originalQuery.toLowerCase().trim();
90
+ const optimizationsEnabled = process.env.AWM_DISABLE_EXPANSION_CACHE !== '1';
91
+
92
+ if (optimizationsEnabled) {
93
+ if (shouldSkipExpansion(normalized)) return originalQuery;
94
+ const cached = expansionCache.get(normalized);
95
+ if (cached !== undefined) {
96
+ // LRU touch
97
+ expansionCache.delete(normalized);
98
+ expansionCache.set(normalized, cached);
99
+ return cached;
100
+ }
101
+ }
102
+
103
+ try {
104
+ const prompt = `Expand this search query with synonyms and related terms. Only output the additional terms, not the original query. Query: ${originalQuery}. Additional terms:`;
105
+ const expansion = await dispatchExpand({ prompt, maxNewTokens: 25, noRepeatNgramSize: 2 });
106
+ const finalQuery = expansion && expansion.length > 2
107
+ ? `${originalQuery} ${expansion}`
108
+ : originalQuery;
109
+
110
+ if (optimizationsEnabled) {
111
+ if (expansionCache.size >= EXPANSION_CACHE_LIMIT) {
112
+ const oldestKey = expansionCache.keys().next().value;
113
+ if (oldestKey !== undefined) expansionCache.delete(oldestKey);
114
+ }
115
+ expansionCache.set(normalized, finalQuery);
116
+ }
117
+
118
+ return finalQuery;
119
+ } catch {
120
+ return originalQuery;
121
+ }
122
+ }
123
+
124
+ /** Clear the expansion cache (used by tests + cache invalidation). */
125
+ export function clearExpansionCache(): void {
126
+ expansionCache.clear();
127
+ }
@@ -45,6 +45,11 @@ export const RECALL_FLAGS = [
45
45
  // Omitted until 2026-10-09, during which both arms of a dtype sweep printed the
46
46
  // SAME arm= label; the whole point of this list is that that cannot happen.
47
47
  'AWM_RERANKER_DTYPE',
48
+ // ONNX input caps (2026-10-16). The query cap changes scores for queries over the
49
+ // cap; the batch sizes should not, but they are listed so an arm says them.
50
+ 'AWM_RERANK_QUERY_TOKENS',
51
+ 'AWM_RERANK_BATCH',
52
+ 'AWM_EMBED_BATCH',
48
53
  // Derived retrieval text (embedding side)
49
54
  'AWM_RETRIEVAL_TEXT',
50
55
  // Project-dialect alias expansion
@@ -1,159 +1,164 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * Passage selection for cross-encoder reranking.
5
- *
6
- * THE PROBLEM
7
- * -----------
8
- * Reranking truncates each passage to a fixed prefix (historically the first
9
- * 400 chars). That truncation exists for a real reason: cross-encoders pad to
10
- * the longest passage in the batch, so one 5,000-char memory in a 40-item pool
11
- * drags every passage to ~512 tokens and costs 3-4x. The reranker is already
12
- * 83% of warm recall latency (measured, `npm run profile:recall`), so "just send
13
- * everything" is not available.
14
- *
15
- * But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
16
- * - canonical memories median 1,965 chars, 98.7% exceed 400
17
- * - the reranker cannot see 78.8% of each canonical memory's vocabulary
18
- * - 99.9% of long canonical memories carry identifiers only past char 400
19
- * And measured on tests/longmem-eval: moving the answer from char 150 to char
20
- * 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
21
- * collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
22
- * retrievable and stops being rankable.
23
- *
24
- * THE FIX
25
- * -------
26
- * Spend the same character budget on the window that actually contains the
27
- * query's terms, instead of on whatever happens to be at the top of the memory.
28
- * Cost is unchanged — same budget, same batch padding, same inference — so this
29
- * buys ranking quality without buying latency.
30
- *
31
- * The concept line is always kept: it is short, it is the memory's title, and
32
- * it is what a human wrote to summarise the thing.
33
- */
34
-
35
- /** Cheap tokenizer for locating query terms inside a passage. */
36
- function terms(query: string): string[] {
37
- const raw = query.toLowerCase().match(/[a-z0-9_][a-z0-9_.\-]{1,}/g) ?? [];
38
- const STOP = new Set([
39
- 'the', 'and', 'for', 'with', 'that', 'this', 'from', 'what', 'where', 'when',
40
- 'which', 'does', 'did', 'was', 'were', 'are', 'is', 'be', 'to', 'of', 'in',
41
- 'on', 'at', 'by', 'it', 'as', 'do', 'how', 'why', 'a', 'an', 'or',
42
- ]);
43
- return Array.from(new Set(raw.filter(t => t.length >= 3 && !STOP.has(t))));
44
- }
45
-
46
- /**
47
- * Choose the `budget`-char window of `content` densest in query terms.
48
- * Returns the head of the content when nothing matches — the old behavior,
49
- * which is the right fallback: with no query signal there is no reason to
50
- * prefer any other part of the memory.
51
- */
52
- export function densestWindow(content: string, query: string, budget: number): string {
53
- if (content.length <= budget) return content;
54
-
55
- const toks = terms(query);
56
- if (toks.length === 0) return content.slice(0, budget);
57
-
58
- const lower = content.toLowerCase();
59
- const hits: number[] = [];
60
- for (const t of toks) {
61
- let from = 0;
62
- for (;;) {
63
- const i = lower.indexOf(t, from);
64
- if (i < 0) break;
65
- hits.push(i);
66
- from = i + t.length;
67
- }
68
- }
69
- if (hits.length === 0) return content.slice(0, budget);
70
- hits.sort((a, b) => a - b);
71
-
72
- // Slide a window anchored slightly before each hit; keep the one covering most.
73
- let bestStart = 0;
74
- let bestCount = -1;
75
- for (let i = 0; i < hits.length; i++) {
76
- const start = Math.max(0, hits[i] - Math.floor(budget / 5));
77
- let count = 0;
78
- for (let j = i; j < hits.length && hits[j] - start < budget; j++) count++;
79
- if (count > bestCount) { bestCount = count; bestStart = start; }
80
- }
81
-
82
- // Snap to a word boundary so the cross-encoder is not handed a split token.
83
- let start = bestStart;
84
- if (start > 0) {
85
- const sp = content.indexOf(' ', start);
86
- if (sp >= 0 && sp - start < 40) start = sp + 1;
87
- }
88
- const slice = content.slice(start, start + budget);
89
- return (start > 0 ? '…' : '') + slice + (start + budget < content.length ? '…' : '');
90
- }
91
-
92
- /**
93
- * Build the passage handed to the cross-encoder for one candidate.
94
- *
95
- * `mode`:
96
- * - `'prefix'` (default) — legacy behavior, the first `budget` chars.
97
- * - `'query'` — the `budget`-char window densest in query terms.
98
- */
99
- export function buildRerankPassage(
100
- concept: string,
101
- content: string,
102
- query: string,
103
- budget: number,
104
- mode: 'prefix' | 'query',
105
- tags?: string[],
106
- ): string {
107
- const body = mode === 'query'
108
- ? densestWindow(content, query, budget)
109
- : (content.length > budget ? content.slice(0, budget) : content);
110
- // Topical tags, when enabled. The cross-encoder decides final order since
111
- // phase 9b, and it cannot see tags at all — measured on the live store, 66.2%
112
- // of topical tag terms never appear in the body, so that vocabulary is
113
- // invisible to the stage that now decides ranking. Appended (not substituted)
114
- // and length-capped so it cannot crowd out the content window.
115
- const extra = rerankTagText(tags);
116
- return extra ? `${concept}: ${body} ${extra}` : `${concept}: ${body}`;
117
- }
118
-
119
- /** Whether topical tags are appended to the rerank passage. Default OFF. */
120
- export function rerankTagsEnabled(): boolean {
121
- return process.env.AWM_RERANK_TAGS === '1';
122
- }
123
-
124
- /** Character budget for the appended tag text. */
125
- export function rerankTagBudget(): number {
126
- const v = Number(process.env.AWM_RERANK_TAGS_LEN ?? 80);
127
- return Number.isFinite(v) && v > 0 ? v : 80;
128
- }
129
-
130
- /**
131
- * Render topical tags as plain terms for the cross-encoder.
132
- * Only `topic=` / `proj=` / `project=` carry query vocabulary; date/person/
133
- * ticket tags are identifiers the body usually already contains, and adding
134
- * them would spend the budget without adding reachable words.
135
- */
136
- export function rerankTagText(tags?: string[]): string {
137
- if (!rerankTagsEnabled() || !tags || tags.length === 0) return '';
138
- const words = new Set<string>();
139
- for (const t of tags) {
140
- const m = /^(?:topic|proj|project)=(.+)$/i.exec(t);
141
- if (!m) continue;
142
- for (const w of m[1].toLowerCase().split(/[-_\s]+/)) {
143
- if (w.length > 2) words.add(w);
144
- }
145
- }
146
- if (words.size === 0) return '';
147
- return `[${[...words].join(' ').slice(0, rerankTagBudget())}]`;
148
- }
149
-
150
- /** Character budget for a rerank passage. */
151
- export function rerankTruncation(): number {
152
- const v = Number(process.env.AWM_RERANK_TRUNC ?? 400);
153
- return Number.isFinite(v) && v > 0 ? v : 400;
154
- }
155
-
156
- /** Passage selection mode. Default `prefix` preserves shipped behavior. */
157
- export function rerankWindowMode(): 'prefix' | 'query' {
158
- return process.env.AWM_RERANK_WINDOW === 'query' ? 'query' : 'prefix';
159
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Passage selection for cross-encoder reranking.
5
+ *
6
+ * THE PROBLEM
7
+ * -----------
8
+ * Reranking truncates each passage to a fixed prefix (historically the first
9
+ * 400 chars). That truncation exists for a real reason: cross-encoders pad to
10
+ * the longest passage in the batch, so one 5,000-char memory in a 40-item pool
11
+ * drags every passage to ~512 tokens and costs 3-4x. The reranker is already
12
+ * 83% of warm recall latency (measured, `npm run profile:recall`), so "just send
13
+ * everything" is not available.
14
+ *
15
+ * But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
16
+ * - canonical memories median 1,965 chars, 98.7% exceed 400
17
+ * - the reranker cannot see 78.8% of each canonical memory's vocabulary
18
+ * - 99.9% of long canonical memories carry identifiers only past char 400
19
+ * And measured on tests/longmem-eval: moving the answer from char 150 to char
20
+ * 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
21
+ * collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
22
+ * retrievable and stops being rankable.
23
+ *
24
+ * THE FIX
25
+ * -------
26
+ * Spend the same character budget on the window that actually contains the
27
+ * query's terms, instead of on whatever happens to be at the top of the memory.
28
+ * Cost is unchanged — same budget, same batch padding, same inference — so this
29
+ * buys ranking quality without buying latency.
30
+ *
31
+ * The concept line is always kept: it is short, it is the memory's title, and
32
+ * it is what a human wrote to summarise the thing.
33
+ */
34
+
35
+ /** Cheap tokenizer for locating query terms inside a passage. */
36
+ function terms(query: string): string[] {
37
+ const raw = query.toLowerCase().match(/[a-z0-9_][a-z0-9_.\-]{1,}/g) ?? [];
38
+ const STOP = new Set([
39
+ 'the', 'and', 'for', 'with', 'that', 'this', 'from', 'what', 'where', 'when',
40
+ 'which', 'does', 'did', 'was', 'were', 'are', 'is', 'be', 'to', 'of', 'in',
41
+ 'on', 'at', 'by', 'it', 'as', 'do', 'how', 'why', 'a', 'an', 'or',
42
+ ]);
43
+ return Array.from(new Set(raw.filter(t => t.length >= 3 && !STOP.has(t))));
44
+ }
45
+
46
+ /**
47
+ * Choose the `budget`-char window of `content` densest in query terms.
48
+ * Returns the head of the content when nothing matches — the old behavior,
49
+ * which is the right fallback: with no query signal there is no reason to
50
+ * prefer any other part of the memory.
51
+ */
52
+ export function densestWindow(content: string, query: string, budget: number): string {
53
+ if (content.length <= budget) return content;
54
+
55
+ const toks = terms(query);
56
+ if (toks.length === 0) return content.slice(0, budget);
57
+
58
+ const lower = content.toLowerCase();
59
+ const hits: number[] = [];
60
+ for (const t of toks) {
61
+ let from = 0;
62
+ for (;;) {
63
+ const i = lower.indexOf(t, from);
64
+ if (i < 0) break;
65
+ hits.push(i);
66
+ from = i + t.length;
67
+ }
68
+ }
69
+ if (hits.length === 0) return content.slice(0, budget);
70
+ hits.sort((a, b) => a - b);
71
+
72
+ // Slide a window anchored slightly before each hit; keep the one covering most.
73
+ let bestStart = 0;
74
+ let bestCount = -1;
75
+ for (let i = 0; i < hits.length; i++) {
76
+ const start = Math.max(0, hits[i] - Math.floor(budget / 5));
77
+ let count = 0;
78
+ for (let j = i; j < hits.length && hits[j] - start < budget; j++) count++;
79
+ if (count > bestCount) { bestCount = count; bestStart = start; }
80
+ }
81
+
82
+ // Snap to a word boundary so the cross-encoder is not handed a split token.
83
+ let start = bestStart;
84
+ if (start > 0) {
85
+ const sp = content.indexOf(' ', start);
86
+ if (sp >= 0 && sp - start < 40) start = sp + 1;
87
+ }
88
+ const slice = content.slice(start, start + budget);
89
+ return (start > 0 ? '…' : '') + slice + (start + budget < content.length ? '…' : '');
90
+ }
91
+
92
+ /**
93
+ * Build the passage handed to the cross-encoder for one candidate.
94
+ *
95
+ * `mode`:
96
+ * - `'prefix'` (default) — legacy behavior, the first `budget` chars.
97
+ * - `'query'` — the `budget`-char window densest in query terms.
98
+ */
99
+ export function buildRerankPassage(
100
+ concept: string,
101
+ content: string,
102
+ query: string,
103
+ budget: number,
104
+ mode: 'prefix' | 'query',
105
+ tags?: string[],
106
+ ): string {
107
+ const body = mode === 'query'
108
+ ? densestWindow(content, query, budget)
109
+ : (content.length > budget ? content.slice(0, budget) : content);
110
+ // Topical tags, when enabled. The cross-encoder decides final order since
111
+ // phase 9b, and it cannot see tags at all — measured on the live store, 66.2%
112
+ // of topical tag terms never appear in the body, so that vocabulary is
113
+ // invisible to the stage that now decides ranking. Appended (not substituted)
114
+ // and length-capped so it cannot crowd out the content window.
115
+ const extra = rerankTagText(tags);
116
+ return extra ? `${concept}: ${body} ${extra}` : `${concept}: ${body}`;
117
+ }
118
+
119
+ /** Whether topical tags are appended to the rerank passage. Default ON since
120
+ * 2026-10-19 (+7.4pp s@1 measured 2026-08-24); `AWM_RERANK_TAGS=0` disables.
121
+ * See rerank2.ts `rerank2Enabled` for why the measured-good settings became
122
+ * code defaults rather than config strings. */
123
+ export function rerankTagsEnabled(): boolean {
124
+ return process.env.AWM_RERANK_TAGS !== '0';
125
+ }
126
+
127
+ /** Character budget for the appended tag text. */
128
+ export function rerankTagBudget(): number {
129
+ const v = Number(process.env.AWM_RERANK_TAGS_LEN ?? 80);
130
+ return Number.isFinite(v) && v > 0 ? v : 80;
131
+ }
132
+
133
+ /**
134
+ * Render topical tags as plain terms for the cross-encoder.
135
+ * Only `topic=` / `proj=` / `project=` carry query vocabulary; date/person/
136
+ * ticket tags are identifiers the body usually already contains, and adding
137
+ * them would spend the budget without adding reachable words.
138
+ */
139
+ export function rerankTagText(tags?: string[]): string {
140
+ if (!rerankTagsEnabled() || !tags || tags.length === 0) return '';
141
+ const words = new Set<string>();
142
+ for (const t of tags) {
143
+ const m = /^(?:topic|proj|project)=(.+)$/i.exec(t);
144
+ if (!m) continue;
145
+ for (const w of m[1].toLowerCase().split(/[-_\s]+/)) {
146
+ if (w.length > 2) words.add(w);
147
+ }
148
+ }
149
+ if (words.size === 0) return '';
150
+ return `[${[...words].join(' ').slice(0, rerankTagBudget())}]`;
151
+ }
152
+
153
+ /** Character budget for a rerank passage. */
154
+ export function rerankTruncation(): number {
155
+ const v = Number(process.env.AWM_RERANK_TRUNC ?? 400);
156
+ return Number.isFinite(v) && v > 0 ? v : 400;
157
+ }
158
+
159
+ /** Passage selection mode. Default `query` since 2026-10-19 (long-memory s@1
160
+ * 25% → 87.5%, measured 2026-08-24); `AWM_RERANK_WINDOW=prefix` restores the
161
+ * first-N-chars passage. Must ship with rerank2 on, which it now does. */
162
+ export function rerankWindowMode(): 'prefix' | 'query' {
163
+ return process.env.AWM_RERANK_WINDOW === 'prefix' ? 'prefix' : 'query';
164
+ }