agent-working-memory 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -5
- package/dist/adapters/common.d.ts +34 -0
- package/dist/adapters/common.d.ts.map +1 -1
- package/dist/adapters/common.js +528 -446
- package/dist/adapters/common.js.map +1 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +19 -0
- package/dist/api/routes.js.map +1 -1
- package/dist/cli/migrate.d.ts.map +1 -1
- package/dist/cli/migrate.js +31 -29
- package/dist/cli/migrate.js.map +1 -1
- package/dist/core/asked-value.d.ts +37 -0
- package/dist/core/asked-value.d.ts.map +1 -0
- package/dist/core/asked-value.js +77 -0
- package/dist/core/asked-value.js.map +1 -0
- package/dist/core/conflicts.d.ts +89 -0
- package/dist/core/conflicts.d.ts.map +1 -0
- package/dist/core/conflicts.js +267 -0
- package/dist/core/conflicts.js.map +1 -0
- package/dist/core/embeddings.d.ts +14 -2
- package/dist/core/embeddings.d.ts.map +1 -1
- package/dist/core/embeddings.js +32 -10
- package/dist/core/embeddings.js.map +1 -1
- package/dist/core/format-recall.d.ts +21 -1
- package/dist/core/format-recall.d.ts.map +1 -1
- package/dist/core/format-recall.js +60 -5
- package/dist/core/format-recall.js.map +1 -1
- package/dist/core/ml-worker.d.ts +24 -1
- package/dist/core/ml-worker.d.ts.map +1 -1
- package/dist/core/ml-worker.js +165 -4
- package/dist/core/ml-worker.js.map +1 -1
- package/dist/core/model-daemon-client.d.ts +77 -0
- package/dist/core/model-daemon-client.d.ts.map +1 -0
- package/dist/core/model-daemon-client.js +383 -0
- package/dist/core/model-daemon-client.js.map +1 -0
- package/dist/core/model-daemon-server.d.ts +71 -0
- package/dist/core/model-daemon-server.d.ts.map +1 -0
- package/dist/core/model-daemon-server.js +363 -0
- package/dist/core/model-daemon-server.js.map +1 -0
- package/dist/core/model-daemon.d.ts +137 -0
- package/dist/core/model-daemon.d.ts.map +1 -0
- package/dist/core/model-daemon.js +231 -0
- package/dist/core/model-daemon.js.map +1 -0
- package/dist/core/query-expander.d.ts +5 -0
- package/dist/core/query-expander.d.ts.map +1 -1
- package/dist/core/query-expander.js +3 -2
- package/dist/core/query-expander.js.map +1 -1
- package/dist/core/recall-config.d.ts +1 -1
- package/dist/core/recall-config.d.ts.map +1 -1
- package/dist/core/recall-config.js +5 -0
- package/dist/core/recall-config.js.map +1 -1
- package/dist/core/rerank-window.d.ts +7 -2
- package/dist/core/rerank-window.d.ts.map +1 -1
- package/dist/core/rerank-window.js +9 -4
- package/dist/core/rerank-window.js.map +1 -1
- package/dist/core/rerank2.d.ts +52 -2
- package/dist/core/rerank2.d.ts.map +1 -1
- package/dist/core/rerank2.js +60 -3
- package/dist/core/rerank2.js.map +1 -1
- package/dist/core/reranker.d.ts +19 -0
- package/dist/core/reranker.d.ts.map +1 -1
- package/dist/core/reranker.js +90 -13
- package/dist/core/reranker.js.map +1 -1
- package/dist/core/trust.d.ts +32 -0
- package/dist/core/trust.d.ts.map +1 -0
- package/dist/core/trust.js +66 -0
- package/dist/core/trust.js.map +1 -0
- package/dist/core/validity.d.ts +21 -0
- package/dist/core/validity.d.ts.map +1 -0
- package/dist/core/validity.js +41 -0
- package/dist/core/validity.js.map +1 -0
- package/dist/core/whoami.d.ts +3 -0
- package/dist/core/whoami.d.ts.map +1 -1
- package/dist/core/whoami.js +14 -0
- package/dist/core/whoami.js.map +1 -1
- package/dist/core/write-pipeline.d.ts +48 -0
- package/dist/core/write-pipeline.d.ts.map +1 -1
- package/dist/core/write-pipeline.js +80 -3
- package/dist/core/write-pipeline.js.map +1 -1
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +27 -4
- package/dist/engine/activation.js.map +1 -1
- package/dist/engine/consolidation.d.ts.map +1 -1
- package/dist/engine/consolidation.js +53 -4
- package/dist/engine/consolidation.js.map +1 -1
- package/dist/hooks/prime-query.d.ts +57 -0
- package/dist/hooks/prime-query.d.ts.map +1 -0
- package/dist/hooks/prime-query.js +388 -0
- package/dist/hooks/prime-query.js.map +1 -0
- package/dist/hooks/prime.d.ts +8 -0
- package/dist/hooks/prime.d.ts.map +1 -1
- package/dist/hooks/prime.js +21 -2
- package/dist/hooks/prime.js.map +1 -1
- package/dist/hooks/sidecar.d.ts +18 -0
- package/dist/hooks/sidecar.d.ts.map +1 -1
- package/dist/hooks/sidecar.js +58 -2
- package/dist/hooks/sidecar.js.map +1 -1
- package/dist/index.js +15 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +157 -104
- package/dist/mcp.js.map +1 -1
- package/dist/model-daemon.d.ts +8 -0
- package/dist/model-daemon.d.ts.map +1 -0
- package/dist/model-daemon.js +45 -0
- package/dist/model-daemon.js.map +1 -0
- package/dist/storage/factory.d.ts.map +1 -1
- package/dist/storage/factory.js +5 -0
- package/dist/storage/factory.js.map +1 -1
- package/dist/storage/pglite-schema.d.ts.map +1 -1
- package/dist/storage/pglite-schema.js +176 -170
- package/dist/storage/pglite-schema.js.map +1 -1
- package/dist/storage/pglite.d.ts +5 -0
- package/dist/storage/pglite.d.ts.map +1 -1
- package/dist/storage/pglite.js +165 -153
- package/dist/storage/pglite.js.map +1 -1
- package/dist/storage/postgres.d.ts +5 -0
- package/dist/storage/postgres.d.ts.map +1 -1
- package/dist/storage/postgres.js +165 -153
- package/dist/storage/postgres.js.map +1 -1
- package/dist/storage/sqlite.d.ts +22 -0
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +415 -363
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types/engram.d.ts +33 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/dist/types/engram.js.map +1 -1
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/README.md +1 -1
- package/plugin/bin/awm-mcp-launcher.cjs +2 -2
- package/plugin/skills/awm-memory/SKILL.md +11 -2
- package/src/adapters/common.ts +881 -802
- package/src/api/routes.ts +20 -0
- package/src/cli/migrate.ts +309 -307
- package/src/core/asked-value.ts +74 -0
- package/src/core/conflicts.ts +280 -0
- package/src/core/embeddings.ts +35 -10
- package/src/core/format-recall.ts +83 -25
- package/src/core/ml-worker.ts +458 -281
- package/src/core/model-daemon-client.ts +374 -0
- package/src/core/model-daemon-server.ts +344 -0
- package/src/core/model-daemon.ts +243 -0
- package/src/core/query-expander.ts +127 -126
- package/src/core/recall-config.ts +5 -0
- package/src/core/rerank-window.ts +164 -159
- package/src/core/rerank2.ts +143 -82
- package/src/core/reranker.ts +92 -12
- package/src/core/trust.ts +65 -0
- package/src/core/validity.ts +47 -0
- package/src/core/whoami.ts +136 -120
- package/src/core/write-pipeline.ts +119 -3
- package/src/engine/activation.ts +27 -4
- package/src/engine/consolidation.ts +950 -899
- package/src/hooks/prime-query.ts +375 -0
- package/src/hooks/prime.ts +25 -2
- package/src/hooks/sidecar.ts +57 -3
- package/src/index.ts +258 -248
- package/src/mcp.ts +1529 -1475
- package/src/model-daemon.ts +48 -0
- package/src/storage/factory.ts +25 -19
- package/src/storage/pglite-schema.ts +199 -193
- package/src/storage/pglite.ts +1513 -1498
- package/src/storage/postgres.ts +1613 -1598
- package/src/storage/sqlite.ts +2069 -2019
- package/src/types/engram.ts +30 -0
|
@@ -1,126 +1,127 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Query Expander - rewrites queries with synonyms and related terms.
|
|
5
|
-
*
|
|
6
|
-
* Uses Xenova/flan-t5-small — a 141MB encoder plus a 233MB merged decoder, so
|
|
7
|
-
* 374MB of ONNX and a 377MB download with its tokenizer. By far the largest of
|
|
8
|
-
* the three models AWM fetches. Expands search queries with related terms that
|
|
9
|
-
* improve BM25 recall.
|
|
10
|
-
*
|
|
11
|
-
* AWM 0.8.x: inference dispatches through ml-worker.ts (currently in-process
|
|
12
|
-
* — worker_threads reverted because onnxruntime-node bindings cross isolate
|
|
13
|
-
* boundaries unsafely; see ml-worker.ts). The dispatch abstraction is
|
|
14
|
-
* preserved for a future child_process / HTTP sidecar pool.
|
|
15
|
-
*
|
|
16
|
-
* The LRU cache + skip heuristic stay on the main thread — they're pure
|
|
17
|
-
* filter/lookup logic that shouldn't pay IPC cost.
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
import { pipeline, type Text2TextGenerationPipeline } from '@huggingface/transformers';
|
|
21
|
-
import { dispatchExpand, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
-
import { ensureModelCacheDir } from './model-cache.js';
|
|
23
|
-
|
|
24
|
-
const MODEL_ID = 'Xenova/flan-t5-small';
|
|
25
|
-
|
|
26
|
-
// --- In-process fallback ---
|
|
27
|
-
|
|
28
|
-
let inProcessInstance: Text2TextGenerationPipeline | null = null;
|
|
29
|
-
let inProcessInitPromise: Promise<Text2TextGenerationPipeline> | null = null;
|
|
30
|
-
|
|
31
|
-
async function loadInProcess(): Promise<Text2TextGenerationPipeline> {
|
|
32
|
-
if (inProcessInstance) return inProcessInstance;
|
|
33
|
-
if (inProcessInitPromise) return inProcessInitPromise;
|
|
34
|
-
ensureModelCacheDir();
|
|
35
|
-
inProcessInitPromise = pipeline('text2text-generation', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
36
|
-
inProcessInstance = pipe as Text2TextGenerationPipeline;
|
|
37
|
-
console.error(`Query expander loaded in-process: ${MODEL_ID}`);
|
|
38
|
-
return inProcessInstance;
|
|
39
|
-
});
|
|
40
|
-
return inProcessInitPromise;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async function inProcessExpand(args: { prompt: string; maxNewTokens: number; noRepeatNgramSize: number }): Promise<string> {
|
|
44
|
-
const expander = await loadInProcess();
|
|
45
|
-
const result = await expander(args.prompt, {
|
|
46
|
-
max_new_tokens: args.maxNewTokens,
|
|
47
|
-
no_repeat_ngram_size: args.noRepeatNgramSize,
|
|
48
|
-
});
|
|
49
|
-
const text = Array.isArray(result) ? (result[0] as any)?.generated_text ?? '' : '';
|
|
50
|
-
return String(text).trim();
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
registerInProcessHandlers({ expand: inProcessExpand });
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
*
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if (normalized.length
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
expansionCache.
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
const
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Query Expander - rewrites queries with synonyms and related terms.
|
|
5
|
+
*
|
|
6
|
+
* Uses Xenova/flan-t5-small — a 141MB encoder plus a 233MB merged decoder, so
|
|
7
|
+
* 374MB of ONNX and a 377MB download with its tokenizer. By far the largest of
|
|
8
|
+
* the three models AWM fetches. Expands search queries with related terms that
|
|
9
|
+
* improve BM25 recall.
|
|
10
|
+
*
|
|
11
|
+
* AWM 0.8.x: inference dispatches through ml-worker.ts (currently in-process
|
|
12
|
+
* — worker_threads reverted because onnxruntime-node bindings cross isolate
|
|
13
|
+
* boundaries unsafely; see ml-worker.ts). The dispatch abstraction is
|
|
14
|
+
* preserved for a future child_process / HTTP sidecar pool.
|
|
15
|
+
*
|
|
16
|
+
* The LRU cache + skip heuristic stay on the main thread — they're pure
|
|
17
|
+
* filter/lookup logic that shouldn't pay IPC cost.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { pipeline, type Text2TextGenerationPipeline } from '@huggingface/transformers';
|
|
21
|
+
import { dispatchExpand, registerInferenceConfig, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
+
import { ensureModelCacheDir } from './model-cache.js';
|
|
23
|
+
|
|
24
|
+
const MODEL_ID = 'Xenova/flan-t5-small';
|
|
25
|
+
|
|
26
|
+
// --- In-process fallback ---
|
|
27
|
+
|
|
28
|
+
let inProcessInstance: Text2TextGenerationPipeline | null = null;
|
|
29
|
+
let inProcessInitPromise: Promise<Text2TextGenerationPipeline> | null = null;
|
|
30
|
+
|
|
31
|
+
async function loadInProcess(): Promise<Text2TextGenerationPipeline> {
|
|
32
|
+
if (inProcessInstance) return inProcessInstance;
|
|
33
|
+
if (inProcessInitPromise) return inProcessInitPromise;
|
|
34
|
+
ensureModelCacheDir();
|
|
35
|
+
inProcessInitPromise = pipeline('text2text-generation', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
36
|
+
inProcessInstance = pipe as Text2TextGenerationPipeline;
|
|
37
|
+
console.error(`Query expander loaded in-process: ${MODEL_ID}`);
|
|
38
|
+
return inProcessInstance;
|
|
39
|
+
});
|
|
40
|
+
return inProcessInitPromise;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function inProcessExpand(args: { prompt: string; maxNewTokens: number; noRepeatNgramSize: number }): Promise<string> {
|
|
44
|
+
const expander = await loadInProcess();
|
|
45
|
+
const result = await expander(args.prompt, {
|
|
46
|
+
max_new_tokens: args.maxNewTokens,
|
|
47
|
+
no_repeat_ngram_size: args.noRepeatNgramSize,
|
|
48
|
+
});
|
|
49
|
+
const text = Array.isArray(result) ? (result[0] as any)?.generated_text ?? '' : '';
|
|
50
|
+
return String(text).trim();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
registerInProcessHandlers({ expand: inProcessExpand });
|
|
54
|
+
registerInferenceConfig({ expandModel: MODEL_ID });
|
|
55
|
+
|
|
56
|
+
// --- Public API ---
|
|
57
|
+
|
|
58
|
+
/** Kept for backwards compat. */
|
|
59
|
+
export async function getExpander(): Promise<Text2TextGenerationPipeline> {
|
|
60
|
+
return loadInProcess();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* LRU cache of normalized-query → expanded-query mappings.
|
|
65
|
+
* Lives on the main thread — cache hits skip the worker IPC entirely.
|
|
66
|
+
*/
|
|
67
|
+
const expansionCache = new Map<string, string>();
|
|
68
|
+
const EXPANSION_CACHE_LIMIT = 500;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Skip expansion when the query is already specific (long or many tokens).
|
|
72
|
+
*/
|
|
73
|
+
function shouldSkipExpansion(normalized: string): boolean {
|
|
74
|
+
if (normalized.length === 0) return true;
|
|
75
|
+
if (normalized.length > 50) return true;
|
|
76
|
+
const tokens = new Set(normalized.split(/\s+/).filter(t => t.length > 2));
|
|
77
|
+
return tokens.size >= 5;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Expand a query with related terms and synonyms.
|
|
82
|
+
* Returns the original query + generated expansion terms.
|
|
83
|
+
* Falls back to the original query on any error.
|
|
84
|
+
*
|
|
85
|
+
* Dispatches inference to the worker pool. Cache + skip heuristic stay
|
|
86
|
+
* on the main thread.
|
|
87
|
+
*/
|
|
88
|
+
export async function expandQuery(originalQuery: string): Promise<string> {
|
|
89
|
+
const normalized = originalQuery.toLowerCase().trim();
|
|
90
|
+
const optimizationsEnabled = process.env.AWM_DISABLE_EXPANSION_CACHE !== '1';
|
|
91
|
+
|
|
92
|
+
if (optimizationsEnabled) {
|
|
93
|
+
if (shouldSkipExpansion(normalized)) return originalQuery;
|
|
94
|
+
const cached = expansionCache.get(normalized);
|
|
95
|
+
if (cached !== undefined) {
|
|
96
|
+
// LRU touch
|
|
97
|
+
expansionCache.delete(normalized);
|
|
98
|
+
expansionCache.set(normalized, cached);
|
|
99
|
+
return cached;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const prompt = `Expand this search query with synonyms and related terms. Only output the additional terms, not the original query. Query: ${originalQuery}. Additional terms:`;
|
|
105
|
+
const expansion = await dispatchExpand({ prompt, maxNewTokens: 25, noRepeatNgramSize: 2 });
|
|
106
|
+
const finalQuery = expansion && expansion.length > 2
|
|
107
|
+
? `${originalQuery} ${expansion}`
|
|
108
|
+
: originalQuery;
|
|
109
|
+
|
|
110
|
+
if (optimizationsEnabled) {
|
|
111
|
+
if (expansionCache.size >= EXPANSION_CACHE_LIMIT) {
|
|
112
|
+
const oldestKey = expansionCache.keys().next().value;
|
|
113
|
+
if (oldestKey !== undefined) expansionCache.delete(oldestKey);
|
|
114
|
+
}
|
|
115
|
+
expansionCache.set(normalized, finalQuery);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return finalQuery;
|
|
119
|
+
} catch {
|
|
120
|
+
return originalQuery;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Clear the expansion cache (used by tests + cache invalidation). */
|
|
125
|
+
export function clearExpansionCache(): void {
|
|
126
|
+
expansionCache.clear();
|
|
127
|
+
}
|
|
@@ -45,6 +45,11 @@ export const RECALL_FLAGS = [
|
|
|
45
45
|
// Omitted until 2026-10-09, during which both arms of a dtype sweep printed the
|
|
46
46
|
// SAME arm= label; the whole point of this list is that that cannot happen.
|
|
47
47
|
'AWM_RERANKER_DTYPE',
|
|
48
|
+
// ONNX input caps (2026-10-16). The query cap changes scores for queries over the
|
|
49
|
+
// cap; the batch sizes should not, but they are listed so an arm says them.
|
|
50
|
+
'AWM_RERANK_QUERY_TOKENS',
|
|
51
|
+
'AWM_RERANK_BATCH',
|
|
52
|
+
'AWM_EMBED_BATCH',
|
|
48
53
|
// Derived retrieval text (embedding side)
|
|
49
54
|
'AWM_RETRIEVAL_TEXT',
|
|
50
55
|
// Project-dialect alias expansion
|
|
@@ -1,159 +1,164 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Passage selection for cross-encoder reranking.
|
|
5
|
-
*
|
|
6
|
-
* THE PROBLEM
|
|
7
|
-
* -----------
|
|
8
|
-
* Reranking truncates each passage to a fixed prefix (historically the first
|
|
9
|
-
* 400 chars). That truncation exists for a real reason: cross-encoders pad to
|
|
10
|
-
* the longest passage in the batch, so one 5,000-char memory in a 40-item pool
|
|
11
|
-
* drags every passage to ~512 tokens and costs 3-4x. The reranker is already
|
|
12
|
-
* 83% of warm recall latency (measured, `npm run profile:recall`), so "just send
|
|
13
|
-
* everything" is not available.
|
|
14
|
-
*
|
|
15
|
-
* But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
|
|
16
|
-
* - canonical memories median 1,965 chars, 98.7% exceed 400
|
|
17
|
-
* - the reranker cannot see 78.8% of each canonical memory's vocabulary
|
|
18
|
-
* - 99.9% of long canonical memories carry identifiers only past char 400
|
|
19
|
-
* And measured on tests/longmem-eval: moving the answer from char 150 to char
|
|
20
|
-
* 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
|
|
21
|
-
* collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
|
|
22
|
-
* retrievable and stops being rankable.
|
|
23
|
-
*
|
|
24
|
-
* THE FIX
|
|
25
|
-
* -------
|
|
26
|
-
* Spend the same character budget on the window that actually contains the
|
|
27
|
-
* query's terms, instead of on whatever happens to be at the top of the memory.
|
|
28
|
-
* Cost is unchanged — same budget, same batch padding, same inference — so this
|
|
29
|
-
* buys ranking quality without buying latency.
|
|
30
|
-
*
|
|
31
|
-
* The concept line is always kept: it is short, it is the memory's title, and
|
|
32
|
-
* it is what a human wrote to summarise the thing.
|
|
33
|
-
*/
|
|
34
|
-
|
|
35
|
-
/** Cheap tokenizer for locating query terms inside a passage. */
|
|
36
|
-
function terms(query: string): string[] {
|
|
37
|
-
const raw = query.toLowerCase().match(/[a-z0-9_][a-z0-9_.\-]{1,}/g) ?? [];
|
|
38
|
-
const STOP = new Set([
|
|
39
|
-
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'what', 'where', 'when',
|
|
40
|
-
'which', 'does', 'did', 'was', 'were', 'are', 'is', 'be', 'to', 'of', 'in',
|
|
41
|
-
'on', 'at', 'by', 'it', 'as', 'do', 'how', 'why', 'a', 'an', 'or',
|
|
42
|
-
]);
|
|
43
|
-
return Array.from(new Set(raw.filter(t => t.length >= 3 && !STOP.has(t))));
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Choose the `budget`-char window of `content` densest in query terms.
|
|
48
|
-
* Returns the head of the content when nothing matches — the old behavior,
|
|
49
|
-
* which is the right fallback: with no query signal there is no reason to
|
|
50
|
-
* prefer any other part of the memory.
|
|
51
|
-
*/
|
|
52
|
-
export function densestWindow(content: string, query: string, budget: number): string {
|
|
53
|
-
if (content.length <= budget) return content;
|
|
54
|
-
|
|
55
|
-
const toks = terms(query);
|
|
56
|
-
if (toks.length === 0) return content.slice(0, budget);
|
|
57
|
-
|
|
58
|
-
const lower = content.toLowerCase();
|
|
59
|
-
const hits: number[] = [];
|
|
60
|
-
for (const t of toks) {
|
|
61
|
-
let from = 0;
|
|
62
|
-
for (;;) {
|
|
63
|
-
const i = lower.indexOf(t, from);
|
|
64
|
-
if (i < 0) break;
|
|
65
|
-
hits.push(i);
|
|
66
|
-
from = i + t.length;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
if (hits.length === 0) return content.slice(0, budget);
|
|
70
|
-
hits.sort((a, b) => a - b);
|
|
71
|
-
|
|
72
|
-
// Slide a window anchored slightly before each hit; keep the one covering most.
|
|
73
|
-
let bestStart = 0;
|
|
74
|
-
let bestCount = -1;
|
|
75
|
-
for (let i = 0; i < hits.length; i++) {
|
|
76
|
-
const start = Math.max(0, hits[i] - Math.floor(budget / 5));
|
|
77
|
-
let count = 0;
|
|
78
|
-
for (let j = i; j < hits.length && hits[j] - start < budget; j++) count++;
|
|
79
|
-
if (count > bestCount) { bestCount = count; bestStart = start; }
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Snap to a word boundary so the cross-encoder is not handed a split token.
|
|
83
|
-
let start = bestStart;
|
|
84
|
-
if (start > 0) {
|
|
85
|
-
const sp = content.indexOf(' ', start);
|
|
86
|
-
if (sp >= 0 && sp - start < 40) start = sp + 1;
|
|
87
|
-
}
|
|
88
|
-
const slice = content.slice(start, start + budget);
|
|
89
|
-
return (start > 0 ? '…' : '') + slice + (start + budget < content.length ? '…' : '');
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Build the passage handed to the cross-encoder for one candidate.
|
|
94
|
-
*
|
|
95
|
-
* `mode`:
|
|
96
|
-
* - `'prefix'` (default) — legacy behavior, the first `budget` chars.
|
|
97
|
-
* - `'query'` — the `budget`-char window densest in query terms.
|
|
98
|
-
*/
|
|
99
|
-
export function buildRerankPassage(
|
|
100
|
-
concept: string,
|
|
101
|
-
content: string,
|
|
102
|
-
query: string,
|
|
103
|
-
budget: number,
|
|
104
|
-
mode: 'prefix' | 'query',
|
|
105
|
-
tags?: string[],
|
|
106
|
-
): string {
|
|
107
|
-
const body = mode === 'query'
|
|
108
|
-
? densestWindow(content, query, budget)
|
|
109
|
-
: (content.length > budget ? content.slice(0, budget) : content);
|
|
110
|
-
// Topical tags, when enabled. The cross-encoder decides final order since
|
|
111
|
-
// phase 9b, and it cannot see tags at all — measured on the live store, 66.2%
|
|
112
|
-
// of topical tag terms never appear in the body, so that vocabulary is
|
|
113
|
-
// invisible to the stage that now decides ranking. Appended (not substituted)
|
|
114
|
-
// and length-capped so it cannot crowd out the content window.
|
|
115
|
-
const extra = rerankTagText(tags);
|
|
116
|
-
return extra ? `${concept}: ${body} ${extra}` : `${concept}: ${body}`;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/** Whether topical tags are appended to the rerank passage. Default
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Passage selection for cross-encoder reranking.
|
|
5
|
+
*
|
|
6
|
+
* THE PROBLEM
|
|
7
|
+
* -----------
|
|
8
|
+
* Reranking truncates each passage to a fixed prefix (historically the first
|
|
9
|
+
* 400 chars). That truncation exists for a real reason: cross-encoders pad to
|
|
10
|
+
* the longest passage in the batch, so one 5,000-char memory in a 40-item pool
|
|
11
|
+
* drags every passage to ~512 tokens and costs 3-4x. The reranker is already
|
|
12
|
+
* 83% of warm recall latency (measured, `npm run profile:recall`), so "just send
|
|
13
|
+
* everything" is not available.
|
|
14
|
+
*
|
|
15
|
+
* But a PREFIX is the wrong 400 chars. Measured on the live 29.8k store:
|
|
16
|
+
* - canonical memories median 1,965 chars, 98.7% exceed 400
|
|
17
|
+
* - the reranker cannot see 78.8% of each canonical memory's vocabulary
|
|
18
|
+
* - 99.9% of long canonical memories carry identifiers only past char 400
|
|
19
|
+
* And measured on tests/longmem-eval: moving the answer from char 150 to char
|
|
20
|
+
* 700 takes success@1 from 100% to 0%, with the gold's cross-encoder score
|
|
21
|
+
* collapsing 0.986 -> 0.000 while its BM25 score barely moves. The memory stays
|
|
22
|
+
* retrievable and stops being rankable.
|
|
23
|
+
*
|
|
24
|
+
* THE FIX
|
|
25
|
+
* -------
|
|
26
|
+
* Spend the same character budget on the window that actually contains the
|
|
27
|
+
* query's terms, instead of on whatever happens to be at the top of the memory.
|
|
28
|
+
* Cost is unchanged — same budget, same batch padding, same inference — so this
|
|
29
|
+
* buys ranking quality without buying latency.
|
|
30
|
+
*
|
|
31
|
+
* The concept line is always kept: it is short, it is the memory's title, and
|
|
32
|
+
* it is what a human wrote to summarise the thing.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** Cheap tokenizer for locating query terms inside a passage. */
|
|
36
|
+
function terms(query: string): string[] {
|
|
37
|
+
const raw = query.toLowerCase().match(/[a-z0-9_][a-z0-9_.\-]{1,}/g) ?? [];
|
|
38
|
+
const STOP = new Set([
|
|
39
|
+
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'what', 'where', 'when',
|
|
40
|
+
'which', 'does', 'did', 'was', 'were', 'are', 'is', 'be', 'to', 'of', 'in',
|
|
41
|
+
'on', 'at', 'by', 'it', 'as', 'do', 'how', 'why', 'a', 'an', 'or',
|
|
42
|
+
]);
|
|
43
|
+
return Array.from(new Set(raw.filter(t => t.length >= 3 && !STOP.has(t))));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Choose the `budget`-char window of `content` densest in query terms.
|
|
48
|
+
* Returns the head of the content when nothing matches — the old behavior,
|
|
49
|
+
* which is the right fallback: with no query signal there is no reason to
|
|
50
|
+
* prefer any other part of the memory.
|
|
51
|
+
*/
|
|
52
|
+
export function densestWindow(content: string, query: string, budget: number): string {
|
|
53
|
+
if (content.length <= budget) return content;
|
|
54
|
+
|
|
55
|
+
const toks = terms(query);
|
|
56
|
+
if (toks.length === 0) return content.slice(0, budget);
|
|
57
|
+
|
|
58
|
+
const lower = content.toLowerCase();
|
|
59
|
+
const hits: number[] = [];
|
|
60
|
+
for (const t of toks) {
|
|
61
|
+
let from = 0;
|
|
62
|
+
for (;;) {
|
|
63
|
+
const i = lower.indexOf(t, from);
|
|
64
|
+
if (i < 0) break;
|
|
65
|
+
hits.push(i);
|
|
66
|
+
from = i + t.length;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (hits.length === 0) return content.slice(0, budget);
|
|
70
|
+
hits.sort((a, b) => a - b);
|
|
71
|
+
|
|
72
|
+
// Slide a window anchored slightly before each hit; keep the one covering most.
|
|
73
|
+
let bestStart = 0;
|
|
74
|
+
let bestCount = -1;
|
|
75
|
+
for (let i = 0; i < hits.length; i++) {
|
|
76
|
+
const start = Math.max(0, hits[i] - Math.floor(budget / 5));
|
|
77
|
+
let count = 0;
|
|
78
|
+
for (let j = i; j < hits.length && hits[j] - start < budget; j++) count++;
|
|
79
|
+
if (count > bestCount) { bestCount = count; bestStart = start; }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Snap to a word boundary so the cross-encoder is not handed a split token.
|
|
83
|
+
let start = bestStart;
|
|
84
|
+
if (start > 0) {
|
|
85
|
+
const sp = content.indexOf(' ', start);
|
|
86
|
+
if (sp >= 0 && sp - start < 40) start = sp + 1;
|
|
87
|
+
}
|
|
88
|
+
const slice = content.slice(start, start + budget);
|
|
89
|
+
return (start > 0 ? '…' : '') + slice + (start + budget < content.length ? '…' : '');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Build the passage handed to the cross-encoder for one candidate.
|
|
94
|
+
*
|
|
95
|
+
* `mode`:
|
|
96
|
+
* - `'prefix'` (default) — legacy behavior, the first `budget` chars.
|
|
97
|
+
* - `'query'` — the `budget`-char window densest in query terms.
|
|
98
|
+
*/
|
|
99
|
+
export function buildRerankPassage(
|
|
100
|
+
concept: string,
|
|
101
|
+
content: string,
|
|
102
|
+
query: string,
|
|
103
|
+
budget: number,
|
|
104
|
+
mode: 'prefix' | 'query',
|
|
105
|
+
tags?: string[],
|
|
106
|
+
): string {
|
|
107
|
+
const body = mode === 'query'
|
|
108
|
+
? densestWindow(content, query, budget)
|
|
109
|
+
: (content.length > budget ? content.slice(0, budget) : content);
|
|
110
|
+
// Topical tags, when enabled. The cross-encoder decides final order since
|
|
111
|
+
// phase 9b, and it cannot see tags at all — measured on the live store, 66.2%
|
|
112
|
+
// of topical tag terms never appear in the body, so that vocabulary is
|
|
113
|
+
// invisible to the stage that now decides ranking. Appended (not substituted)
|
|
114
|
+
// and length-capped so it cannot crowd out the content window.
|
|
115
|
+
const extra = rerankTagText(tags);
|
|
116
|
+
return extra ? `${concept}: ${body} ${extra}` : `${concept}: ${body}`;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Whether topical tags are appended to the rerank passage. Default ON since
|
|
120
|
+
* 2026-10-19 (+7.4pp s@1 measured 2026-08-24); `AWM_RERANK_TAGS=0` disables.
|
|
121
|
+
* See rerank2.ts `rerank2Enabled` for why the measured-good settings became
|
|
122
|
+
* code defaults rather than config strings. */
|
|
123
|
+
export function rerankTagsEnabled(): boolean {
|
|
124
|
+
return process.env.AWM_RERANK_TAGS !== '0';
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Character budget for the appended tag text. */
|
|
128
|
+
export function rerankTagBudget(): number {
|
|
129
|
+
const v = Number(process.env.AWM_RERANK_TAGS_LEN ?? 80);
|
|
130
|
+
return Number.isFinite(v) && v > 0 ? v : 80;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Render topical tags as plain terms for the cross-encoder.
|
|
135
|
+
* Only `topic=` / `proj=` / `project=` carry query vocabulary; date/person/
|
|
136
|
+
* ticket tags are identifiers the body usually already contains, and adding
|
|
137
|
+
* them would spend the budget without adding reachable words.
|
|
138
|
+
*/
|
|
139
|
+
export function rerankTagText(tags?: string[]): string {
|
|
140
|
+
if (!rerankTagsEnabled() || !tags || tags.length === 0) return '';
|
|
141
|
+
const words = new Set<string>();
|
|
142
|
+
for (const t of tags) {
|
|
143
|
+
const m = /^(?:topic|proj|project)=(.+)$/i.exec(t);
|
|
144
|
+
if (!m) continue;
|
|
145
|
+
for (const w of m[1].toLowerCase().split(/[-_\s]+/)) {
|
|
146
|
+
if (w.length > 2) words.add(w);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (words.size === 0) return '';
|
|
150
|
+
return `[${[...words].join(' ').slice(0, rerankTagBudget())}]`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Character budget for a rerank passage. */
|
|
154
|
+
export function rerankTruncation(): number {
|
|
155
|
+
const v = Number(process.env.AWM_RERANK_TRUNC ?? 400);
|
|
156
|
+
return Number.isFinite(v) && v > 0 ? v : 400;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Passage selection mode. Default `query` since 2026-10-19 (long-memory s@1
|
|
160
|
+
* 25% → 87.5%, measured 2026-08-24); `AWM_RERANK_WINDOW=prefix` restores the
|
|
161
|
+
* first-N-chars passage. Must ship with rerank2 on, which it now does. */
|
|
162
|
+
export function rerankWindowMode(): 'prefix' | 'query' {
|
|
163
|
+
return process.env.AWM_RERANK_WINDOW === 'prefix' ? 'prefix' : 'query';
|
|
164
|
+
}
|