agent-working-memory 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -238
- package/dist/adapters/common.d.ts +6 -0
- package/dist/adapters/common.d.ts.map +1 -1
- package/dist/adapters/common.js +457 -362
- package/dist/adapters/common.js.map +1 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +24 -8
- package/dist/api/routes.js.map +1 -1
- package/dist/core/alias-map.d.ts +16 -0
- package/dist/core/alias-map.d.ts.map +1 -0
- package/dist/core/alias-map.js +102 -0
- package/dist/core/alias-map.js.map +1 -0
- package/dist/core/embeddings.d.ts +17 -0
- package/dist/core/embeddings.d.ts.map +1 -1
- package/dist/core/embeddings.js +52 -1
- package/dist/core/embeddings.js.map +1 -1
- package/dist/core/model-cache.d.ts +28 -0
- package/dist/core/model-cache.d.ts.map +1 -0
- package/dist/core/model-cache.js +50 -0
- package/dist/core/model-cache.js.map +1 -0
- package/dist/core/query-expander.d.ts.map +1 -1
- package/dist/core/query-expander.js +2 -0
- package/dist/core/query-expander.js.map +1 -1
- package/dist/core/recall-config.d.ts +52 -0
- package/dist/core/recall-config.d.ts.map +1 -0
- package/dist/core/recall-config.js +110 -0
- package/dist/core/recall-config.js.map +1 -0
- package/dist/core/rerank-window.d.ts +61 -0
- package/dist/core/rerank-window.d.ts.map +1 -0
- package/dist/core/rerank-window.js +153 -0
- package/dist/core/rerank-window.js.map +1 -0
- package/dist/core/rerank2.d.ts +62 -0
- package/dist/core/rerank2.d.ts.map +1 -0
- package/dist/core/rerank2.js +75 -0
- package/dist/core/rerank2.js.map +1 -0
- package/dist/core/reranker.d.ts.map +1 -1
- package/dist/core/reranker.js +2 -0
- package/dist/core/reranker.js.map +1 -1
- package/dist/core/retrieval-text.d.ts +55 -0
- package/dist/core/retrieval-text.d.ts.map +1 -0
- package/dist/core/retrieval-text.js +87 -0
- package/dist/core/retrieval-text.js.map +1 -0
- package/dist/core/temporal-query.d.ts +61 -0
- package/dist/core/temporal-query.d.ts.map +1 -0
- package/dist/core/temporal-query.js +168 -0
- package/dist/core/temporal-query.js.map +1 -0
- package/dist/core/token-budget.d.ts +75 -0
- package/dist/core/token-budget.d.ts.map +1 -0
- package/dist/core/token-budget.js +136 -0
- package/dist/core/token-budget.js.map +1 -0
- package/dist/core/whoami.d.ts +11 -0
- package/dist/core/whoami.d.ts.map +1 -1
- package/dist/core/whoami.js +10 -0
- package/dist/core/whoami.js.map +1 -1
- package/dist/core/write-pipeline.d.ts.map +1 -1
- package/dist/core/write-pipeline.js +6 -3
- package/dist/core/write-pipeline.js.map +1 -1
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +135 -32
- package/dist/engine/activation.js.map +1 -1
- package/dist/hooks/prime.d.ts +77 -0
- package/dist/hooks/prime.d.ts.map +1 -0
- package/dist/hooks/prime.js +92 -0
- package/dist/hooks/prime.js.map +1 -0
- package/dist/hooks/sidecar.d.ts.map +1 -1
- package/dist/hooks/sidecar.js +39 -0
- package/dist/hooks/sidecar.js.map +1 -1
- package/dist/mcp.js +134 -102
- package/dist/mcp.js.map +1 -1
- package/dist/storage/pglite.d.ts.map +1 -1
- package/dist/storage/pglite.js +10 -2
- package/dist/storage/pglite.js.map +1 -1
- package/dist/storage/postgres.d.ts.map +1 -1
- package/dist/storage/postgres.js +10 -2
- package/dist/storage/postgres.js.map +1 -1
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +12 -2
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types/engram.d.ts +7 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/package.json +3 -2
- package/src/adapters/common.ts +666 -567
- package/src/api/routes.ts +1015 -999
- package/src/core/alias-map.ts +97 -0
- package/src/core/embeddings.ts +172 -113
- package/src/core/model-cache.ts +51 -0
- package/src/core/query-expander.ts +2 -0
- package/src/core/recall-config.ts +115 -0
- package/src/core/rerank-window.ts +158 -0
- package/src/core/rerank2.ts +82 -0
- package/src/core/reranker.ts +2 -0
- package/src/core/retrieval-text.ts +82 -0
- package/src/core/temporal-query.ts +193 -0
- package/src/core/token-budget.ts +160 -0
- package/src/core/whoami.ts +110 -92
- package/src/core/write-pipeline.ts +6 -3
- package/src/engine/activation.ts +1568 -1468
- package/src/hooks/prime.ts +136 -0
- package/src/hooks/sidecar.ts +43 -0
- package/src/mcp.ts +1422 -1387
- package/src/storage/pglite.ts +10 -2
- package/src/storage/postgres.ts +10 -2
- package/src/storage/sqlite.ts +12 -2
- package/src/types/engram.ts +7 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Project-dialect alias expansion.
|
|
5
|
+
*
|
|
6
|
+
* WHAT IT SOLVES
|
|
7
|
+
* --------------
|
|
8
|
+
* A memory tagged `topic=azure` whose body says "private plan… P1v3… app
|
|
9
|
+
* service" but never "azure". Asked for with "azure", it was not in the top 40.
|
|
10
|
+
* No general model knows that connection — it is local to this project — but it
|
|
11
|
+
* is recoverable from the store, because memories tagged `azure` share body
|
|
12
|
+
* vocabulary that the corpus at large does not.
|
|
13
|
+
*
|
|
14
|
+
* The map is mined offline (tests/realstore-eval/mine-aliases.ts) and is a plain
|
|
15
|
+
* JSON artifact: inspectable, cappable, diffable. That mattered immediately —
|
|
16
|
+
* reading the first mined map showed it had learned
|
|
17
|
+
* `agent -> discussed, turns, topics, summary`, i.e. session-summary boilerplate
|
|
18
|
+
* masquerading as dialect, and a hub guardrail was added before any measurement.
|
|
19
|
+
*
|
|
20
|
+
* GUARDRAIL 4 — "require at least one ORIGINAL query term" — is implemented by
|
|
21
|
+
* SCOPE rather than by an extra filter:
|
|
22
|
+
*
|
|
23
|
+
* - alias terms are added ONLY to the BM25 search string, widening what gets
|
|
24
|
+
* retrieved;
|
|
25
|
+
* - `queryTokens`, which drives textMatch scoring, stays ORIGINAL.
|
|
26
|
+
*
|
|
27
|
+
* So a candidate that matches only alias terms enters the pool with a near-zero
|
|
28
|
+
* textMatch and is filtered by the existing minScore gate, while a candidate
|
|
29
|
+
* that matches an original term too is scored normally. The alias terms buy
|
|
30
|
+
* REACH; the original terms still decide RELEVANCE. That is exactly the
|
|
31
|
+
* separation Codex asked for, and it needs no new gate to enforce.
|
|
32
|
+
*/
|
|
33
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
34
|
+
import { fileURLToPath } from 'node:url';
|
|
35
|
+
import { dirname, join } from 'node:path';
|
|
36
|
+
|
|
37
|
+
let cache: Record<string, string[]> | null = null;
|
|
38
|
+
|
|
39
|
+
function mapPath(): string {
|
|
40
|
+
if (process.env.AWM_ALIAS_MAP) return process.env.AWM_ALIAS_MAP;
|
|
41
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
42
|
+
return join(here, '..', '..', 'data', 'alias-map.json');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Load the mined map once. Missing file is not an error — the feature is opt-in. */
|
|
46
|
+
export function aliasMap(): Record<string, string[]> {
|
|
47
|
+
if (cache) return cache;
|
|
48
|
+
const p = mapPath();
|
|
49
|
+
if (!existsSync(p)) { cache = {}; return cache; }
|
|
50
|
+
try {
|
|
51
|
+
const parsed = JSON.parse(readFileSync(p, 'utf8'));
|
|
52
|
+
cache = (parsed.map ?? parsed) as Record<string, string[]>;
|
|
53
|
+
} catch {
|
|
54
|
+
cache = {};
|
|
55
|
+
}
|
|
56
|
+
return cache;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Reset the cache — tests only. */
|
|
60
|
+
export function clearAliasCache(): void { cache = null; }
|
|
61
|
+
|
|
62
|
+
/** Whether alias expansion is enabled. Default OFF. */
|
|
63
|
+
export function aliasEnabled(): boolean {
|
|
64
|
+
return process.env.AWM_ALIASES === '1';
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Max alias terms added to a single query, regardless of how many match. */
|
|
68
|
+
export function aliasQueryCap(): number {
|
|
69
|
+
const v = Number(process.env.AWM_ALIAS_QUERY_CAP ?? 8);
|
|
70
|
+
return Number.isFinite(v) && v > 0 ? v : 8;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Expand a query with dialect terms for any category word it mentions.
|
|
75
|
+
* Returns the added terms (never the rewritten query) so the caller decides
|
|
76
|
+
* which channel sees them — the whole safety property depends on alias terms
|
|
77
|
+
* reaching BM25 only, not the scoring tokens.
|
|
78
|
+
*/
|
|
79
|
+
export function aliasTermsFor(query: string): string[] {
|
|
80
|
+
if (!aliasEnabled()) return [];
|
|
81
|
+
const m = aliasMap();
|
|
82
|
+
if (Object.keys(m).length === 0) return [];
|
|
83
|
+
const toks = new Set((query.toLowerCase().match(/[a-z][a-z0-9_.-]{3,}/g) ?? []));
|
|
84
|
+
const out: string[] = [];
|
|
85
|
+
const seen = new Set<string>(toks);
|
|
86
|
+
for (const t of toks) {
|
|
87
|
+
const aliases = m[t];
|
|
88
|
+
if (!aliases) continue;
|
|
89
|
+
for (const a of aliases) {
|
|
90
|
+
if (seen.has(a)) continue; // never re-add a term the query already has
|
|
91
|
+
seen.add(a);
|
|
92
|
+
out.push(a);
|
|
93
|
+
if (out.length >= aliasQueryCap()) return out;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
package/src/core/embeddings.ts
CHANGED
|
@@ -1,113 +1,172 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Embedding Engine - vector embeddings via the ML worker pool.
|
|
5
|
-
*
|
|
6
|
-
* Default model: bge-small-en-v1.5 (384 dimensions, ~90MB, MTEB retrieval-optimized).
|
|
7
|
-
* Configurable via AWM_EMBED_MODEL env var.
|
|
8
|
-
*
|
|
9
|
-
* AWM 0.8.x: inference dispatches through ml-worker.ts. The worker_threads
|
|
10
|
-
* path was planned but reverted to in-process because onnxruntime-node's
|
|
11
|
-
* native bindings store V8 handles that don't cross isolate boundaries
|
|
12
|
-
* safely — see ml-worker.ts for the full status. The dispatch abstraction
|
|
13
|
-
* is preserved for a future child_process or HTTP sidecar pool.
|
|
14
|
-
* `AWM_ML_INPROCESS=1` is honored as a no-op (in-process is now the default).
|
|
15
|
-
*
|
|
16
|
-
* NOTE: Changing the model invalidates existing embeddings.
|
|
17
|
-
* Set AWM_EMBED_MODEL=Xenova/all-MiniLM-L6-v2 for backward compatibility.
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
21
|
-
import { dispatchEmbed, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
-
import { noteModelLoad } from './write-telemetry.js';
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
let
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
if (
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
*
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Embedding Engine - vector embeddings via the ML worker pool.
|
|
5
|
+
*
|
|
6
|
+
* Default model: bge-small-en-v1.5 (384 dimensions, ~90MB, MTEB retrieval-optimized).
|
|
7
|
+
* Configurable via AWM_EMBED_MODEL env var.
|
|
8
|
+
*
|
|
9
|
+
* AWM 0.8.x: inference dispatches through ml-worker.ts. The worker_threads
|
|
10
|
+
* path was planned but reverted to in-process because onnxruntime-node's
|
|
11
|
+
* native bindings store V8 handles that don't cross isolate boundaries
|
|
12
|
+
* safely — see ml-worker.ts for the full status. The dispatch abstraction
|
|
13
|
+
* is preserved for a future child_process or HTTP sidecar pool.
|
|
14
|
+
* `AWM_ML_INPROCESS=1` is honored as a no-op (in-process is now the default).
|
|
15
|
+
*
|
|
16
|
+
* NOTE: Changing the model invalidates existing embeddings.
|
|
17
|
+
* Set AWM_EMBED_MODEL=Xenova/all-MiniLM-L6-v2 for backward compatibility.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
21
|
+
import { dispatchEmbed, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
+
import { noteModelLoad } from './write-telemetry.js';
|
|
23
|
+
import { ensureModelCacheDir } from './model-cache.js';
|
|
24
|
+
|
|
25
|
+
const MODEL_ID = process.env.AWM_EMBED_MODEL ?? 'Xenova/bge-small-en-v1.5';
|
|
26
|
+
const DIMENSIONS = parseInt(process.env.AWM_EMBED_DIMS ?? '384', 10);
|
|
27
|
+
const POOLING = (process.env.AWM_EMBED_POOLING ?? 'mean') as 'cls' | 'mean';
|
|
28
|
+
|
|
29
|
+
// --- In-process fallback (used by tests and crash recovery) ---
|
|
30
|
+
|
|
31
|
+
let inProcessInstance: FeatureExtractionPipeline | null = null;
|
|
32
|
+
let inProcessInitPromise: Promise<FeatureExtractionPipeline> | null = null;
|
|
33
|
+
|
|
34
|
+
async function loadInProcess(): Promise<FeatureExtractionPipeline> {
|
|
35
|
+
if (inProcessInstance) return inProcessInstance;
|
|
36
|
+
if (inProcessInitPromise) return inProcessInitPromise;
|
|
37
|
+
const tLoadStart = performance.now();
|
|
38
|
+
ensureModelCacheDir();
|
|
39
|
+
inProcessInitPromise = pipeline('feature-extraction', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
40
|
+
inProcessInstance = pipe;
|
|
41
|
+
noteModelLoad(performance.now() - tLoadStart);
|
|
42
|
+
console.error(`Embedding model loaded in-process: ${MODEL_ID} (${DIMENSIONS}d)`);
|
|
43
|
+
return pipe;
|
|
44
|
+
});
|
|
45
|
+
return inProcessInitPromise;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function inProcessEmbed(args: { texts: string[]; pooling: 'cls' | 'mean'; dimensions: number }): Promise<number[][]> {
|
|
49
|
+
const { texts, pooling, dimensions } = args;
|
|
50
|
+
if (texts.length === 0) return [];
|
|
51
|
+
const embedder = await loadInProcess();
|
|
52
|
+
const result = await embedder(texts, { pooling, normalize: true });
|
|
53
|
+
const data = result.data as Float32Array;
|
|
54
|
+
const vectors: number[][] = [];
|
|
55
|
+
for (let i = 0; i < texts.length; i++) {
|
|
56
|
+
vectors.push(Array.from(data.slice(i * dimensions, (i + 1) * dimensions)));
|
|
57
|
+
}
|
|
58
|
+
return vectors;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Register the in-process handler with the pool (used in test mode and as fallback)
|
|
62
|
+
registerInProcessHandlers({ embed: inProcessEmbed });
|
|
63
|
+
|
|
64
|
+
// --- Public API ---
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Get or initialize the embedding pipeline (singleton).
|
|
68
|
+
* Kept for backwards compat — returns the in-process pipeline only.
|
|
69
|
+
* Most consumers should use embed() / embedBatch() which dispatch
|
|
70
|
+
* to the worker pool by default.
|
|
71
|
+
*/
|
|
72
|
+
export async function getEmbedder(): Promise<FeatureExtractionPipeline> {
|
|
73
|
+
return loadInProcess();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Generate an embedding vector for a text string.
|
|
78
|
+
* Dispatches to the worker pool (or in-process fallback).
|
|
79
|
+
*/
|
|
80
|
+
export async function embed(text: string): Promise<number[]> {
|
|
81
|
+
const vectors = await dispatchEmbed({ texts: [text], pooling: POOLING, dimensions: DIMENSIONS });
|
|
82
|
+
return vectors[0] ?? new Array(DIMENSIONS).fill(0);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Generate embeddings for multiple texts in a batch.
|
|
87
|
+
* More efficient than calling embed() in a loop — the worker batches the
|
|
88
|
+
* tokenization + forward pass.
|
|
89
|
+
*/
|
|
90
|
+
export async function embedBatch(texts: string[]): Promise<number[][]> {
|
|
91
|
+
if (texts.length === 0) return [];
|
|
92
|
+
return dispatchEmbed({ texts, pooling: POOLING, dimensions: DIMENSIONS });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Get the current embedding model ID (for version tracking in stored embeddings) */
|
|
96
|
+
export function getModelId(): string {
|
|
97
|
+
return MODEL_ID;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Dimension-mismatch telemetry.
|
|
102
|
+
*
|
|
103
|
+
* A mismatch means the vector channel silently scores 0 for the affected memories —
|
|
104
|
+
* recall still returns results (BM25 is unaffected) but they are quietly much worse.
|
|
105
|
+
* That is invisible without this counter, and a half-migrated corpus is the realistic
|
|
106
|
+
* way it happens: change AWM_EMBED_MODEL without re-embedding, or interrupt a
|
|
107
|
+
* migration, and the un-migrated rows drop out of vector scoring with no error.
|
|
108
|
+
*/
|
|
109
|
+
let dimMismatchCount = 0;
|
|
110
|
+
let dimMismatchWarned = false;
|
|
111
|
+
let dimMismatchSample: { expected: number; got: number } | null = null;
|
|
112
|
+
|
|
113
|
+
/** Observed embedding-dimension mismatches. Surfaced by /health and memory_whoami. */
|
|
114
|
+
export function embeddingHealth(): {
|
|
115
|
+
dimensionMismatches: number;
|
|
116
|
+
expectedDimensions: number;
|
|
117
|
+
sample: { expected: number; got: number } | null;
|
|
118
|
+
} {
|
|
119
|
+
return {
|
|
120
|
+
dimensionMismatches: dimMismatchCount,
|
|
121
|
+
expectedDimensions: DIMENSIONS,
|
|
122
|
+
sample: dimMismatchSample,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Test-only: reset the mismatch counter. */
|
|
127
|
+
export function __resetEmbeddingHealth(): void {
|
|
128
|
+
dimMismatchCount = 0;
|
|
129
|
+
dimMismatchWarned = false;
|
|
130
|
+
dimMismatchSample = null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Cosine similarity between two normalized vectors.
|
|
135
|
+
* Since vectors are pre-normalized, this is just the dot product.
|
|
136
|
+
*
|
|
137
|
+
* Returns 0 on a dimension mismatch — the maths is undefined otherwise — but COUNTS
|
|
138
|
+
* it and warns once, because a silent 0 here reads as "these memories are irrelevant"
|
|
139
|
+
* rather than "this corpus is half-migrated". Deliberately does not throw: this runs
|
|
140
|
+
* over every candidate on every recall, so throwing would escalate degraded quality
|
|
141
|
+
* into an outage.
|
|
142
|
+
*/
|
|
143
|
+
export function cosineSimilarity(a: number[], b: number[]): number {
|
|
144
|
+
if (a.length !== b.length) {
|
|
145
|
+
// length 0 is a legitimate not-yet-embedded state, not a corpus mismatch
|
|
146
|
+
if (a.length !== 0 && b.length !== 0) {
|
|
147
|
+
dimMismatchCount++;
|
|
148
|
+
if (!dimMismatchSample) dimMismatchSample = { expected: a.length, got: b.length };
|
|
149
|
+
if (!dimMismatchWarned) {
|
|
150
|
+
dimMismatchWarned = true;
|
|
151
|
+
console.error(
|
|
152
|
+
`[awm] EMBEDDING DIMENSION MISMATCH: ${a.length}d vs ${b.length}d. ` +
|
|
153
|
+
`Affected memories score 0 on the vector channel and will rank far too low. ` +
|
|
154
|
+
`This usually means the corpus is half-migrated — re-embed the whole store ` +
|
|
155
|
+
`or revert AWM_EMBED_MODEL/AWM_EMBED_DIMS. Warning shown once per process; ` +
|
|
156
|
+
`see embeddingHealth() / GET /health for the running count.`
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return 0;
|
|
161
|
+
}
|
|
162
|
+
if (a.length === 0) return 0;
|
|
163
|
+
let dot = 0;
|
|
164
|
+
for (let i = 0; i < a.length; i++) {
|
|
165
|
+
dot += a[i] * b[i];
|
|
166
|
+
}
|
|
167
|
+
// Clamp to [-1, 1] to handle floating point drift
|
|
168
|
+
return Math.max(-1, Math.min(1, dot));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Vector dimensions for this model */
|
|
172
|
+
export const EMBEDDING_DIMENSIONS = DIMENSIONS;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Model cache directory — makes `HF_HOME` (and `AWM_CACHE_DIR`) actually work.
|
|
5
|
+
*
|
|
6
|
+
* ROOT CAUSE (found 2026-08-22 investigating flaky model loads in ephemeral Docker
|
|
7
|
+
* containers): @huggingface/transformers has NO built-in environment-variable support
|
|
8
|
+
* for its cache location — it reads only `env.cacheDir`, set in code, and nothing else.
|
|
9
|
+
* Left unset, it defaults to `<install-dir-of-@huggingface/transformers>/.cache/` —
|
|
10
|
+
* inside node_modules, wiped on every `npm install`/`npm ci`, i.e. every Docker build
|
|
11
|
+
* and every global upgrade re-downloads all three models from scratch.
|
|
12
|
+
*
|
|
13
|
+
* `docs/deployment.md` has instructed `HF_HOME=/data/models` for Railway/Fly/Render
|
|
14
|
+
* persistence since it was written — that variable was a silent no-op the whole time;
|
|
15
|
+
* nothing in AWM ever read it. This module makes it real, so existing deployments start
|
|
16
|
+
* working with ZERO config changes on the user's end. `AWM_CACHE_DIR` (referenced in
|
|
17
|
+
* docs/architecture.md, also previously fictional) is honored as an AWM-specific
|
|
18
|
+
* override, and the default is now genuinely persistent for a bare npm install too
|
|
19
|
+
* (previously it was ephemeral there as well, just less obviously so).
|
|
20
|
+
*
|
|
21
|
+
* Precedence: AWM_CACHE_DIR > HF_HOME > <packageRoot>/data/models.
|
|
22
|
+
*
|
|
23
|
+
* MUST run before the first `pipeline()` / `AutoTokenizer.from_pretrained()` /
|
|
24
|
+
* `AutoModel*.from_pretrained()` call in the process — `env.cacheDir` is read at
|
|
25
|
+
* download/load time, not import time, but there's no reason to risk a race, so every
|
|
26
|
+
* model-loading module in src/core/ calls `ensureModelCacheDir()` before its first
|
|
27
|
+
* library call. Idempotent — safe to call from all three.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { env } from '@huggingface/transformers';
|
|
31
|
+
import { existsSync, mkdirSync } from 'node:fs';
|
|
32
|
+
import { dirname, join } from 'node:path';
|
|
33
|
+
import { fileURLToPath } from 'node:url';
|
|
34
|
+
|
|
35
|
+
let configured = false;
|
|
36
|
+
|
|
37
|
+
function resolvePackageRoot(): string {
|
|
38
|
+
// This file lives at src/core/model-cache.ts (dev, via tsx) or
|
|
39
|
+
// dist/core/model-cache.js (built) — either way, two levels up is the package root.
|
|
40
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
41
|
+
return dirname(dirname(here));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function ensureModelCacheDir(): string {
|
|
45
|
+
if (configured) return env.cacheDir!;
|
|
46
|
+
const dir = process.env.AWM_CACHE_DIR || process.env.HF_HOME || join(resolvePackageRoot(), 'data', 'models');
|
|
47
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
48
|
+
env.cacheDir = dir;
|
|
49
|
+
configured = true;
|
|
50
|
+
return dir;
|
|
51
|
+
}
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import { pipeline, type Text2TextGenerationPipeline } from '@huggingface/transformers';
|
|
19
19
|
import { dispatchExpand, registerInProcessHandlers } from './ml-worker.js';
|
|
20
|
+
import { ensureModelCacheDir } from './model-cache.js';
|
|
20
21
|
|
|
21
22
|
const MODEL_ID = 'Xenova/flan-t5-small';
|
|
22
23
|
|
|
@@ -28,6 +29,7 @@ let inProcessInitPromise: Promise<Text2TextGenerationPipeline> | null = null;
|
|
|
28
29
|
async function loadInProcess(): Promise<Text2TextGenerationPipeline> {
|
|
29
30
|
if (inProcessInstance) return inProcessInstance;
|
|
30
31
|
if (inProcessInitPromise) return inProcessInitPromise;
|
|
32
|
+
ensureModelCacheDir();
|
|
31
33
|
inProcessInitPromise = pipeline('text2text-generation', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
32
34
|
inProcessInstance = pipe as Text2TextGenerationPipeline;
|
|
33
35
|
console.error(`Query expander loaded in-process: ${MODEL_ID}`);
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Effective recall configuration — self-reporting, so a measurement can PROVE
|
|
5
|
+
* which configuration produced it.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS
|
|
8
|
+
* ---------------
|
|
9
|
+
* On 2026-08-23 a benchmark comparison reported "no effect" for a change that
|
|
10
|
+
* demonstrably worked. The cause was not AWM: two arms shared a port, teardown
|
|
11
|
+
* left the first server alive, the second arm health-checked the survivor, and
|
|
12
|
+
* both arms measured the SAME baseline process. Identical inputs, identical
|
|
13
|
+
* outputs, a confident and completely wrong conclusion.
|
|
14
|
+
*
|
|
15
|
+
* Port hygiene fixes that instance. It does not fix the class. The class is:
|
|
16
|
+
* **nothing verified that the system measured was the system configured.**
|
|
17
|
+
* The same gap bit the D11 spreading-activation re-test, where the tracer's arm
|
|
18
|
+
* label omitted `AWM_SPREAD_INHIBIT`, so two materially different arms both
|
|
19
|
+
* printed `arm=spread` and had to be told apart by diffing output by hand.
|
|
20
|
+
*
|
|
21
|
+
* The durable fix is for the running system to state its own effective recall
|
|
22
|
+
* configuration, so a harness can assert it and fail LOUDLY instead of silently
|
|
23
|
+
* measuring the wrong thing. Any new recall flag added to RECALL_FLAGS is
|
|
24
|
+
* automatically covered by every consumer — the eval tracer's arm label, the
|
|
25
|
+
* `/health` payload, and the benchmark driver's assertion.
|
|
26
|
+
*
|
|
27
|
+
* Adding a flag here is the ONLY step required to make it visible everywhere.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Every environment flag that can change what `activate()` returns.
|
|
32
|
+
* Keep this list current — an unlisted flag is an invisible experiment.
|
|
33
|
+
*/
|
|
34
|
+
export const RECALL_FLAGS = [
|
|
35
|
+
// Second-stage reorder (phase 9b)
|
|
36
|
+
'AWM_RERANK2',
|
|
37
|
+
'AWM_RERANK2_K',
|
|
38
|
+
// Cross-encoder passage selection
|
|
39
|
+
'AWM_RERANK_WINDOW',
|
|
40
|
+
'AWM_RERANK_TRUNC',
|
|
41
|
+
'AWM_RERANK_POOL',
|
|
42
|
+
'AWM_RERANK_TAGS',
|
|
43
|
+
'AWM_RERANK_TAGS_LEN',
|
|
44
|
+
// Derived retrieval text (embedding side)
|
|
45
|
+
'AWM_RETRIEVAL_TEXT',
|
|
46
|
+
// Project-dialect alias expansion
|
|
47
|
+
'AWM_ALIASES',
|
|
48
|
+
'AWM_ALIAS_QUERY_CAP',
|
|
49
|
+
// Embedding model — changes every stored vector, so it belongs here
|
|
50
|
+
'AWM_EMBED_MODEL',
|
|
51
|
+
'AWM_EMBED_DIMS',
|
|
52
|
+
'AWM_DISABLE_RERANK_SKIP',
|
|
53
|
+
// Spreading activation (D11 — parked)
|
|
54
|
+
'AWM_SPREAD',
|
|
55
|
+
'AWM_SPREAD_INJECT',
|
|
56
|
+
'AWM_SPREAD_INHIBIT',
|
|
57
|
+
'AWM_SPREAD_ITERS',
|
|
58
|
+
'AWM_SPREAD_DAMPING',
|
|
59
|
+
'AWM_SPREAD_BUDGET',
|
|
60
|
+
'AWM_SPREAD_BOOST',
|
|
61
|
+
// Candidate pool / retrieval breadth
|
|
62
|
+
'AWM_TOPN_MULT',
|
|
63
|
+
'AWM_BROAD_EDGES',
|
|
64
|
+
'AWM_ENTITY_FETCH',
|
|
65
|
+
'AWM_ENTITY_INDEX_FETCH',
|
|
66
|
+
'AWM_QUERY_BRIDGE',
|
|
67
|
+
'AWM_AUTOTAG',
|
|
68
|
+
// Diagnostic escape hatches that alter ranking
|
|
69
|
+
'AWM_DISABLE_POOL_FILTER',
|
|
70
|
+
'AWM_DISABLE_EXPANSION_CACHE',
|
|
71
|
+
'AWM_DISABLE_SLIM_CACHE',
|
|
72
|
+
'AWM_ABSTAIN_GATE_K',
|
|
73
|
+
] as const;
|
|
74
|
+
|
|
75
|
+
export type RecallFlag = (typeof RECALL_FLAGS)[number];
|
|
76
|
+
|
|
77
|
+
/** Only the flags actually set, in declaration order. */
|
|
78
|
+
export function activeRecallConfig(env: NodeJS.ProcessEnv = process.env): Record<string, string> {
|
|
79
|
+
const out: Record<string, string> = {};
|
|
80
|
+
for (const k of RECALL_FLAGS) {
|
|
81
|
+
const v = env[k];
|
|
82
|
+
if (v !== undefined && v !== '') out[k] = v;
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Short, stable, comparable label for the active configuration.
|
|
89
|
+
* `default` when nothing is set. Sorted so it never depends on declaration
|
|
90
|
+
* order or on how the process was launched.
|
|
91
|
+
*/
|
|
92
|
+
export function recallConfigFingerprint(env: NodeJS.ProcessEnv = process.env): string {
|
|
93
|
+
const active = activeRecallConfig(env);
|
|
94
|
+
const keys = Object.keys(active).sort();
|
|
95
|
+
if (keys.length === 0) return 'default';
|
|
96
|
+
return keys.map(k => `${k.replace(/^AWM_/, '').toLowerCase()}=${active[k]}`).join(',');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Assert the running configuration contains the expected flag values.
|
|
101
|
+
* Returns the mismatches; empty array means the system is configured as
|
|
102
|
+
* intended. Harnesses should treat a non-empty result as fatal — it means the
|
|
103
|
+
* thing being measured is not the thing that was configured.
|
|
104
|
+
*/
|
|
105
|
+
export function diffRecallConfig(
|
|
106
|
+
expected: Record<string, string>,
|
|
107
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
108
|
+
): Array<{ flag: string; expected: string; actual: string | undefined }> {
|
|
109
|
+
const active = activeRecallConfig(env);
|
|
110
|
+
const bad: Array<{ flag: string; expected: string; actual: string | undefined }> = [];
|
|
111
|
+
for (const [flag, want] of Object.entries(expected)) {
|
|
112
|
+
if (active[flag] !== want) bad.push({ flag, expected: want, actual: active[flag] });
|
|
113
|
+
}
|
|
114
|
+
return bad;
|
|
115
|
+
}
|