agent-working-memory 0.8.7 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +207 -46
- package/dist/api/routes.js +1 -1
- package/dist/cli/migrate.js +29 -29
- package/dist/cli.js +1 -1
- package/dist/coordination/circuit-breaker.js +23 -23
- package/dist/core/write-pipeline.d.ts.map +1 -1
- package/dist/core/write-pipeline.js +17 -0
- package/dist/core/write-pipeline.js.map +1 -1
- package/dist/engine/activation.d.ts +28 -0
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +341 -11
- package/dist/engine/activation.js.map +1 -1
- package/dist/engine/connections.d.ts +12 -0
- package/dist/engine/connections.d.ts.map +1 -1
- package/dist/engine/connections.js +95 -0
- package/dist/engine/connections.js.map +1 -1
- package/dist/mcp.js +2 -2
- package/dist/storage/pglite-schema.js +143 -143
- package/dist/storage/pglite.js +138 -138
- package/dist/types/engram.d.ts +1 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/api/index.ts +3 -3
- package/src/api/routes.ts +1 -1
- package/src/cli/migrate.ts +307 -307
- package/src/cli.ts +1 -1
- package/src/coordination/circuit-breaker.ts +83 -83
- package/src/coordination/failure-modes.ts +50 -50
- package/src/core/decay.ts +63 -63
- package/src/core/embeddings.ts +110 -110
- package/src/core/index.ts +5 -5
- package/src/core/logger.ts +36 -36
- package/src/core/ml-worker-entry.ts +194 -194
- package/src/core/ml-worker.ts +281 -281
- package/src/core/query-expander.ts +122 -122
- package/src/core/reranker.ts +119 -119
- package/src/core/write-pipeline.ts +15 -0
- package/src/engine/activation.ts +328 -11
- package/src/engine/confidence.ts +120 -120
- package/src/engine/connections.ts +94 -0
- package/src/engine/consolidation-scheduler.ts +242 -242
- package/src/engine/eval.ts +102 -102
- package/src/engine/eviction.ts +101 -101
- package/src/engine/index.ts +8 -8
- package/src/engine/retraction.ts +366 -366
- package/src/engine/staging.ts +74 -74
- package/src/mcp.ts +2 -2
- package/src/storage/factory.ts +147 -147
- package/src/storage/index.ts +3 -3
- package/src/storage/pglite-schema.ts +166 -166
- package/src/storage/pglite.ts +1363 -1363
- package/src/storage/store.ts +80 -80
- package/src/types/agent.ts +67 -67
- package/src/types/checkpoint.ts +46 -46
- package/src/types/engram.ts +1 -0
- package/src/types/eval.ts +100 -100
- package/src/types/index.ts +6 -6
|
@@ -1,83 +1,83 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Per-worker CircuitBreaker for the coordination control layer.
|
|
5
|
-
* Prevents chronically-stale workers from poisoning the assignment queue.
|
|
6
|
-
* Part of AWM 0.8.1 — additive, no breaking changes.
|
|
7
|
-
*
|
|
8
|
-
* States:
|
|
9
|
-
* closed — normal operation
|
|
10
|
-
* open — worker blocked after FAILURE_THRESHOLD consecutive failures
|
|
11
|
-
* half_open — probe window (30s after open), allows one assignment attempt
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import type Database from 'better-sqlite3';
|
|
15
|
-
|
|
16
|
-
export type CircuitState = 'closed' | 'open' | 'half_open';
|
|
17
|
-
|
|
18
|
-
const FAILURE_THRESHOLD = 5;
|
|
19
|
-
const HALF_OPEN_DELAY_MS = 30_000;
|
|
20
|
-
|
|
21
|
-
/** Record a worker failure. Opens the circuit when consecutive failures hit the threshold. */
|
|
22
|
-
export function recordFailure(db: Database.Database, agentId: string): void {
|
|
23
|
-
db.prepare(`
|
|
24
|
-
INSERT INTO coord_circuit_state (agent_id, consecutive_failures, last_transition_at)
|
|
25
|
-
VALUES (?, 1, datetime('now'))
|
|
26
|
-
ON CONFLICT(agent_id) DO UPDATE SET
|
|
27
|
-
consecutive_failures = consecutive_failures + 1,
|
|
28
|
-
state = CASE
|
|
29
|
-
WHEN consecutive_failures + 1 >= ${FAILURE_THRESHOLD} THEN 'open'
|
|
30
|
-
ELSE state
|
|
31
|
-
END,
|
|
32
|
-
opened_at = CASE
|
|
33
|
-
WHEN consecutive_failures + 1 >= ${FAILURE_THRESHOLD} AND (state != 'open' OR opened_at IS NULL)
|
|
34
|
-
THEN datetime('now')
|
|
35
|
-
ELSE opened_at
|
|
36
|
-
END,
|
|
37
|
-
last_transition_at = datetime('now')
|
|
38
|
-
`).run(agentId);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/** Record a worker success. Resets to closed regardless of prior state. */
|
|
42
|
-
export function recordSuccess(db: Database.Database, agentId: string): void {
|
|
43
|
-
db.prepare(`
|
|
44
|
-
INSERT INTO coord_circuit_state (agent_id, state, consecutive_failures, last_transition_at)
|
|
45
|
-
VALUES (?, 'closed', 0, datetime('now'))
|
|
46
|
-
ON CONFLICT(agent_id) DO UPDATE SET
|
|
47
|
-
state = 'closed',
|
|
48
|
-
consecutive_failures = 0,
|
|
49
|
-
opened_at = NULL,
|
|
50
|
-
last_transition_at = datetime('now')
|
|
51
|
-
`).run(agentId);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Get current circuit state for a worker.
|
|
56
|
-
* If the circuit has been open for >30s, auto-transitions to half_open.
|
|
57
|
-
*/
|
|
58
|
-
export function getState(db: Database.Database, agentId: string): CircuitState {
|
|
59
|
-
const row = db.prepare(
|
|
60
|
-
`SELECT state, opened_at FROM coord_circuit_state WHERE agent_id = ?`
|
|
61
|
-
).get(agentId) as { state: string; opened_at: string | null } | undefined;
|
|
62
|
-
|
|
63
|
-
if (!row || row.state === 'closed') return 'closed';
|
|
64
|
-
if (row.state === 'half_open') return 'half_open';
|
|
65
|
-
|
|
66
|
-
// open — check if half-open window has elapsed
|
|
67
|
-
if (row.state === 'open' && row.opened_at) {
|
|
68
|
-
const openedAt = new Date(row.opened_at.endsWith('Z') ? row.opened_at : row.opened_at + 'Z').getTime();
|
|
69
|
-
if (Date.now() - openedAt > HALF_OPEN_DELAY_MS) {
|
|
70
|
-
db.prepare(
|
|
71
|
-
`UPDATE coord_circuit_state SET state = 'half_open', last_transition_at = datetime('now') WHERE agent_id = ?`
|
|
72
|
-
).run(agentId);
|
|
73
|
-
return 'half_open';
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
return 'open';
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/** Returns true when the worker is eligible to receive an assignment. */
|
|
81
|
-
export function isAvailable(db: Database.Database, agentId: string): boolean {
|
|
82
|
-
return getState(db, agentId) !== 'open';
|
|
83
|
-
}
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Per-worker CircuitBreaker for the coordination control layer.
|
|
5
|
+
* Prevents chronically-stale workers from poisoning the assignment queue.
|
|
6
|
+
* Part of AWM 0.8.1 — additive, no breaking changes.
|
|
7
|
+
*
|
|
8
|
+
* States:
|
|
9
|
+
* closed — normal operation
|
|
10
|
+
* open — worker blocked after FAILURE_THRESHOLD consecutive failures
|
|
11
|
+
* half_open — probe window (30s after open), allows one assignment attempt
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type Database from 'better-sqlite3';
|
|
15
|
+
|
|
16
|
+
export type CircuitState = 'closed' | 'open' | 'half_open';
|
|
17
|
+
|
|
18
|
+
const FAILURE_THRESHOLD = 5;
|
|
19
|
+
const HALF_OPEN_DELAY_MS = 30_000;
|
|
20
|
+
|
|
21
|
+
/** Record a worker failure. Opens the circuit when consecutive failures hit the threshold. */
|
|
22
|
+
export function recordFailure(db: Database.Database, agentId: string): void {
|
|
23
|
+
db.prepare(`
|
|
24
|
+
INSERT INTO coord_circuit_state (agent_id, consecutive_failures, last_transition_at)
|
|
25
|
+
VALUES (?, 1, datetime('now'))
|
|
26
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
27
|
+
consecutive_failures = consecutive_failures + 1,
|
|
28
|
+
state = CASE
|
|
29
|
+
WHEN consecutive_failures + 1 >= ${FAILURE_THRESHOLD} THEN 'open'
|
|
30
|
+
ELSE state
|
|
31
|
+
END,
|
|
32
|
+
opened_at = CASE
|
|
33
|
+
WHEN consecutive_failures + 1 >= ${FAILURE_THRESHOLD} AND (state != 'open' OR opened_at IS NULL)
|
|
34
|
+
THEN datetime('now')
|
|
35
|
+
ELSE opened_at
|
|
36
|
+
END,
|
|
37
|
+
last_transition_at = datetime('now')
|
|
38
|
+
`).run(agentId);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Record a worker success. Resets to closed regardless of prior state. */
|
|
42
|
+
export function recordSuccess(db: Database.Database, agentId: string): void {
|
|
43
|
+
db.prepare(`
|
|
44
|
+
INSERT INTO coord_circuit_state (agent_id, state, consecutive_failures, last_transition_at)
|
|
45
|
+
VALUES (?, 'closed', 0, datetime('now'))
|
|
46
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
47
|
+
state = 'closed',
|
|
48
|
+
consecutive_failures = 0,
|
|
49
|
+
opened_at = NULL,
|
|
50
|
+
last_transition_at = datetime('now')
|
|
51
|
+
`).run(agentId);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Get current circuit state for a worker.
|
|
56
|
+
* If the circuit has been open for >30s, auto-transitions to half_open.
|
|
57
|
+
*/
|
|
58
|
+
export function getState(db: Database.Database, agentId: string): CircuitState {
|
|
59
|
+
const row = db.prepare(
|
|
60
|
+
`SELECT state, opened_at FROM coord_circuit_state WHERE agent_id = ?`
|
|
61
|
+
).get(agentId) as { state: string; opened_at: string | null } | undefined;
|
|
62
|
+
|
|
63
|
+
if (!row || row.state === 'closed') return 'closed';
|
|
64
|
+
if (row.state === 'half_open') return 'half_open';
|
|
65
|
+
|
|
66
|
+
// open — check if half-open window has elapsed
|
|
67
|
+
if (row.state === 'open' && row.opened_at) {
|
|
68
|
+
const openedAt = new Date(row.opened_at.endsWith('Z') ? row.opened_at : row.opened_at + 'Z').getTime();
|
|
69
|
+
if (Date.now() - openedAt > HALF_OPEN_DELAY_MS) {
|
|
70
|
+
db.prepare(
|
|
71
|
+
`UPDATE coord_circuit_state SET state = 'half_open', last_transition_at = datetime('now') WHERE agent_id = ?`
|
|
72
|
+
).run(agentId);
|
|
73
|
+
return 'half_open';
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return 'open';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Returns true when the worker is eligible to receive an assignment. */
|
|
81
|
+
export function isAvailable(db: Database.Database, agentId: string): boolean {
|
|
82
|
+
return getState(db, agentId) !== 'open';
|
|
83
|
+
}
|
|
@@ -1,50 +1,50 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* FailureMode taxonomy and mutation-hint map for the coordination control layer.
|
|
5
|
-
* Part of AWM 0.8.1 — additive, no breaking changes.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export enum FailureMode {
|
|
9
|
-
AGENT_STALE = 'agent_stale',
|
|
10
|
-
TIMEOUT = 'timeout',
|
|
11
|
-
OUTPUT_INVALID = 'output_invalid',
|
|
12
|
-
TEST_FAIL = 'test_fail',
|
|
13
|
-
LINT_FAIL = 'lint_fail',
|
|
14
|
-
MERGE_CONFLICT = 'merge_conflict',
|
|
15
|
-
UNKNOWN = 'unknown',
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/** Classify a failure result string into one of the known modes. */
|
|
19
|
-
export function classifyFailure(result: string | null): FailureMode {
|
|
20
|
-
if (!result) return FailureMode.UNKNOWN;
|
|
21
|
-
const r = result.toLowerCase();
|
|
22
|
-
if (r.includes('stale') || r.includes('disconnected')) return FailureMode.AGENT_STALE;
|
|
23
|
-
if (r.includes('timeout') || r.includes('timed out')) return FailureMode.TIMEOUT;
|
|
24
|
-
if (r.includes('json') || r.includes('schema') || r.includes('parse')) return FailureMode.OUTPUT_INVALID;
|
|
25
|
-
if (r.includes('test fail') || r.includes('vitest') || r.includes('jest')) return FailureMode.TEST_FAIL;
|
|
26
|
-
if (r.includes('lint') || r.includes('eslint') || r.includes('typecheck')) return FailureMode.LINT_FAIL;
|
|
27
|
-
if (r.includes('conflict')) return FailureMode.MERGE_CONFLICT;
|
|
28
|
-
return FailureMode.UNKNOWN;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Corrective guidance injected into the task description on retry.
|
|
33
|
-
* Each hint is written in the vocabulary the next worker will read.
|
|
34
|
-
*/
|
|
35
|
-
export const MUTATION_HINTS: Record<FailureMode, string> = {
|
|
36
|
-
[FailureMode.AGENT_STALE]:
|
|
37
|
-
'Previous worker disconnected before completion. Resume from last known state; check git status before re-running destructive commands.',
|
|
38
|
-
[FailureMode.TIMEOUT]:
|
|
39
|
-
'Previous attempt timed out. Break work into smaller commits; report progress every 5 minutes.',
|
|
40
|
-
[FailureMode.OUTPUT_INVALID]:
|
|
41
|
-
'Previous output failed validation. Return a single fenced code block; verify JSON parses before submitting.',
|
|
42
|
-
[FailureMode.TEST_FAIL]:
|
|
43
|
-
'Previous attempt left tests failing. Run vitest before completion; do NOT mark complete if any test fails.',
|
|
44
|
-
[FailureMode.LINT_FAIL]:
|
|
45
|
-
'Previous attempt had lint/typecheck errors. Run pnpm typecheck and pnpm lint before completion.',
|
|
46
|
-
[FailureMode.MERGE_CONFLICT]:
|
|
47
|
-
'Previous attempt left merge conflicts unresolved. git pull --rebase, resolve, then re-attempt.',
|
|
48
|
-
[FailureMode.UNKNOWN]:
|
|
49
|
-
'Previous attempt failed for an unclassified reason. Investigate the prior result before re-running.',
|
|
50
|
-
};
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* FailureMode taxonomy and mutation-hint map for the coordination control layer.
|
|
5
|
+
* Part of AWM 0.8.1 — additive, no breaking changes.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export enum FailureMode {
|
|
9
|
+
AGENT_STALE = 'agent_stale',
|
|
10
|
+
TIMEOUT = 'timeout',
|
|
11
|
+
OUTPUT_INVALID = 'output_invalid',
|
|
12
|
+
TEST_FAIL = 'test_fail',
|
|
13
|
+
LINT_FAIL = 'lint_fail',
|
|
14
|
+
MERGE_CONFLICT = 'merge_conflict',
|
|
15
|
+
UNKNOWN = 'unknown',
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Classify a failure result string into one of the known modes. */
|
|
19
|
+
export function classifyFailure(result: string | null): FailureMode {
|
|
20
|
+
if (!result) return FailureMode.UNKNOWN;
|
|
21
|
+
const r = result.toLowerCase();
|
|
22
|
+
if (r.includes('stale') || r.includes('disconnected')) return FailureMode.AGENT_STALE;
|
|
23
|
+
if (r.includes('timeout') || r.includes('timed out')) return FailureMode.TIMEOUT;
|
|
24
|
+
if (r.includes('json') || r.includes('schema') || r.includes('parse')) return FailureMode.OUTPUT_INVALID;
|
|
25
|
+
if (r.includes('test fail') || r.includes('vitest') || r.includes('jest')) return FailureMode.TEST_FAIL;
|
|
26
|
+
if (r.includes('lint') || r.includes('eslint') || r.includes('typecheck')) return FailureMode.LINT_FAIL;
|
|
27
|
+
if (r.includes('conflict')) return FailureMode.MERGE_CONFLICT;
|
|
28
|
+
return FailureMode.UNKNOWN;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Corrective guidance injected into the task description on retry.
|
|
33
|
+
* Each hint is written in the vocabulary the next worker will read.
|
|
34
|
+
*/
|
|
35
|
+
export const MUTATION_HINTS: Record<FailureMode, string> = {
|
|
36
|
+
[FailureMode.AGENT_STALE]:
|
|
37
|
+
'Previous worker disconnected before completion. Resume from last known state; check git status before re-running destructive commands.',
|
|
38
|
+
[FailureMode.TIMEOUT]:
|
|
39
|
+
'Previous attempt timed out. Break work into smaller commits; report progress every 5 minutes.',
|
|
40
|
+
[FailureMode.OUTPUT_INVALID]:
|
|
41
|
+
'Previous output failed validation. Return a single fenced code block; verify JSON parses before submitting.',
|
|
42
|
+
[FailureMode.TEST_FAIL]:
|
|
43
|
+
'Previous attempt left tests failing. Run vitest before completion; do NOT mark complete if any test fails.',
|
|
44
|
+
[FailureMode.LINT_FAIL]:
|
|
45
|
+
'Previous attempt had lint/typecheck errors. Run pnpm typecheck and pnpm lint before completion.',
|
|
46
|
+
[FailureMode.MERGE_CONFLICT]:
|
|
47
|
+
'Previous attempt left merge conflicts unresolved. git pull --rebase, resolve, then re-attempt.',
|
|
48
|
+
[FailureMode.UNKNOWN]:
|
|
49
|
+
'Previous attempt failed for an unclassified reason. Investigate the prior result before re-running.',
|
|
50
|
+
};
|
package/src/core/decay.ts
CHANGED
|
@@ -1,63 +1,63 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* ACT-R Base-Level Activation
|
|
5
|
-
*
|
|
6
|
-
* Based on Anderson's ACT-R cognitive architecture (1993).
|
|
7
|
-
* Memories that are accessed more recently and more frequently
|
|
8
|
-
* have higher activation — a well-established model of human memory.
|
|
9
|
-
*
|
|
10
|
-
* Formula: B(M) = ln(n + 1) - d * ln(ageDays / (n + 1))
|
|
11
|
-
*
|
|
12
|
-
* Where:
|
|
13
|
-
* n = access count
|
|
14
|
-
* d = decay exponent (default 0.5)
|
|
15
|
-
* ageDays = age of memory in days
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
export function baseLevelActivation(
|
|
19
|
-
accessCount: number,
|
|
20
|
-
ageDays: number,
|
|
21
|
-
decayExponent: number = 0.5
|
|
22
|
-
): number {
|
|
23
|
-
const n = Math.max(accessCount, 0);
|
|
24
|
-
const age = Math.max(ageDays, 0.001); // Avoid log(0)
|
|
25
|
-
return Math.log(n + 1) - decayExponent * Math.log(age / (n + 1));
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Softplus — smooth approximation of ReLU.
|
|
30
|
-
* Used to keep activation scores positive without hard clipping.
|
|
31
|
-
*/
|
|
32
|
-
export function softplus(x: number): number {
|
|
33
|
-
return Math.log(1 + Math.exp(x));
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Composite activation score combining content match, temporal decay,
|
|
38
|
-
* Hebbian boost, and confidence.
|
|
39
|
-
*
|
|
40
|
-
* Score = contentMatch * softplus(B(M) + scale * hebbianBoost) * confidence
|
|
41
|
-
*/
|
|
42
|
-
export function compositeScore(params: {
|
|
43
|
-
contentMatch: number;
|
|
44
|
-
accessCount: number;
|
|
45
|
-
ageDays: number;
|
|
46
|
-
hebbianBoost: number;
|
|
47
|
-
confidence: number;
|
|
48
|
-
decayExponent?: number;
|
|
49
|
-
hebbianScale?: number;
|
|
50
|
-
}): number {
|
|
51
|
-
const {
|
|
52
|
-
contentMatch,
|
|
53
|
-
accessCount,
|
|
54
|
-
ageDays,
|
|
55
|
-
hebbianBoost,
|
|
56
|
-
confidence,
|
|
57
|
-
decayExponent = 0.5,
|
|
58
|
-
hebbianScale = 1.0,
|
|
59
|
-
} = params;
|
|
60
|
-
|
|
61
|
-
const bm = baseLevelActivation(accessCount, ageDays, decayExponent);
|
|
62
|
-
return contentMatch * softplus(bm + hebbianScale * hebbianBoost) * confidence;
|
|
63
|
-
}
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* ACT-R Base-Level Activation
|
|
5
|
+
*
|
|
6
|
+
* Based on Anderson's ACT-R cognitive architecture (1993).
|
|
7
|
+
* Memories that are accessed more recently and more frequently
|
|
8
|
+
* have higher activation — a well-established model of human memory.
|
|
9
|
+
*
|
|
10
|
+
* Formula: B(M) = ln(n + 1) - d * ln(ageDays / (n + 1))
|
|
11
|
+
*
|
|
12
|
+
* Where:
|
|
13
|
+
* n = access count
|
|
14
|
+
* d = decay exponent (default 0.5)
|
|
15
|
+
* ageDays = age of memory in days
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export function baseLevelActivation(
|
|
19
|
+
accessCount: number,
|
|
20
|
+
ageDays: number,
|
|
21
|
+
decayExponent: number = 0.5
|
|
22
|
+
): number {
|
|
23
|
+
const n = Math.max(accessCount, 0);
|
|
24
|
+
const age = Math.max(ageDays, 0.001); // Avoid log(0)
|
|
25
|
+
return Math.log(n + 1) - decayExponent * Math.log(age / (n + 1));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Softplus — smooth approximation of ReLU.
|
|
30
|
+
* Used to keep activation scores positive without hard clipping.
|
|
31
|
+
*/
|
|
32
|
+
export function softplus(x: number): number {
|
|
33
|
+
return Math.log(1 + Math.exp(x));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Composite activation score combining content match, temporal decay,
|
|
38
|
+
* Hebbian boost, and confidence.
|
|
39
|
+
*
|
|
40
|
+
* Score = contentMatch * softplus(B(M) + scale * hebbianBoost) * confidence
|
|
41
|
+
*/
|
|
42
|
+
export function compositeScore(params: {
|
|
43
|
+
contentMatch: number;
|
|
44
|
+
accessCount: number;
|
|
45
|
+
ageDays: number;
|
|
46
|
+
hebbianBoost: number;
|
|
47
|
+
confidence: number;
|
|
48
|
+
decayExponent?: number;
|
|
49
|
+
hebbianScale?: number;
|
|
50
|
+
}): number {
|
|
51
|
+
const {
|
|
52
|
+
contentMatch,
|
|
53
|
+
accessCount,
|
|
54
|
+
ageDays,
|
|
55
|
+
hebbianBoost,
|
|
56
|
+
confidence,
|
|
57
|
+
decayExponent = 0.5,
|
|
58
|
+
hebbianScale = 1.0,
|
|
59
|
+
} = params;
|
|
60
|
+
|
|
61
|
+
const bm = baseLevelActivation(accessCount, ageDays, decayExponent);
|
|
62
|
+
return contentMatch * softplus(bm + hebbianScale * hebbianBoost) * confidence;
|
|
63
|
+
}
|
package/src/core/embeddings.ts
CHANGED
|
@@ -1,110 +1,110 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Embedding Engine - vector embeddings via the ML worker pool.
|
|
5
|
-
*
|
|
6
|
-
* Default model: bge-small-en-v1.5 (384 dimensions, ~90MB, MTEB retrieval-optimized).
|
|
7
|
-
* Configurable via AWM_EMBED_MODEL env var.
|
|
8
|
-
*
|
|
9
|
-
* AWM 0.8.x: inference dispatches through ml-worker.ts. The worker_threads
|
|
10
|
-
* path was planned but reverted to in-process because onnxruntime-node's
|
|
11
|
-
* native bindings store V8 handles that don't cross isolate boundaries
|
|
12
|
-
* safely — see ml-worker.ts for the full status. The dispatch abstraction
|
|
13
|
-
* is preserved for a future child_process or HTTP sidecar pool.
|
|
14
|
-
* `AWM_ML_INPROCESS=1` is honored as a no-op (in-process is now the default).
|
|
15
|
-
*
|
|
16
|
-
* NOTE: Changing the model invalidates existing embeddings.
|
|
17
|
-
* Set AWM_EMBED_MODEL=Xenova/all-MiniLM-L6-v2 for backward compatibility.
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
21
|
-
import { dispatchEmbed, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
-
|
|
23
|
-
const MODEL_ID = process.env.AWM_EMBED_MODEL ?? 'Xenova/bge-small-en-v1.5';
|
|
24
|
-
const DIMENSIONS = parseInt(process.env.AWM_EMBED_DIMS ?? '384', 10);
|
|
25
|
-
const POOLING = (process.env.AWM_EMBED_POOLING ?? 'mean') as 'cls' | 'mean';
|
|
26
|
-
|
|
27
|
-
// --- In-process fallback (used by tests and crash recovery) ---
|
|
28
|
-
|
|
29
|
-
let inProcessInstance: FeatureExtractionPipeline | null = null;
|
|
30
|
-
let inProcessInitPromise: Promise<FeatureExtractionPipeline> | null = null;
|
|
31
|
-
|
|
32
|
-
async function loadInProcess(): Promise<FeatureExtractionPipeline> {
|
|
33
|
-
if (inProcessInstance) return inProcessInstance;
|
|
34
|
-
if (inProcessInitPromise) return inProcessInitPromise;
|
|
35
|
-
inProcessInitPromise = pipeline('feature-extraction', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
36
|
-
inProcessInstance = pipe;
|
|
37
|
-
console.log(`Embedding model loaded in-process: ${MODEL_ID} (${DIMENSIONS}d)`);
|
|
38
|
-
return pipe;
|
|
39
|
-
});
|
|
40
|
-
return inProcessInitPromise;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async function inProcessEmbed(args: { texts: string[]; pooling: 'cls' | 'mean'; dimensions: number }): Promise<number[][]> {
|
|
44
|
-
const { texts, pooling, dimensions } = args;
|
|
45
|
-
if (texts.length === 0) return [];
|
|
46
|
-
const embedder = await loadInProcess();
|
|
47
|
-
const result = await embedder(texts, { pooling, normalize: true });
|
|
48
|
-
const data = result.data as Float32Array;
|
|
49
|
-
const vectors: number[][] = [];
|
|
50
|
-
for (let i = 0; i < texts.length; i++) {
|
|
51
|
-
vectors.push(Array.from(data.slice(i * dimensions, (i + 1) * dimensions)));
|
|
52
|
-
}
|
|
53
|
-
return vectors;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Register the in-process handler with the pool (used in test mode and as fallback)
|
|
57
|
-
registerInProcessHandlers({ embed: inProcessEmbed });
|
|
58
|
-
|
|
59
|
-
// --- Public API ---
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Get or initialize the embedding pipeline (singleton).
|
|
63
|
-
* Kept for backwards compat — returns the in-process pipeline only.
|
|
64
|
-
* Most consumers should use embed() / embedBatch() which dispatch
|
|
65
|
-
* to the worker pool by default.
|
|
66
|
-
*/
|
|
67
|
-
export async function getEmbedder(): Promise<FeatureExtractionPipeline> {
|
|
68
|
-
return loadInProcess();
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Generate an embedding vector for a text string.
|
|
73
|
-
* Dispatches to the worker pool (or in-process fallback).
|
|
74
|
-
*/
|
|
75
|
-
export async function embed(text: string): Promise<number[]> {
|
|
76
|
-
const vectors = await dispatchEmbed({ texts: [text], pooling: POOLING, dimensions: DIMENSIONS });
|
|
77
|
-
return vectors[0] ?? new Array(DIMENSIONS).fill(0);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Generate embeddings for multiple texts in a batch.
|
|
82
|
-
* More efficient than calling embed() in a loop — the worker batches the
|
|
83
|
-
* tokenization + forward pass.
|
|
84
|
-
*/
|
|
85
|
-
export async function embedBatch(texts: string[]): Promise<number[][]> {
|
|
86
|
-
if (texts.length === 0) return [];
|
|
87
|
-
return dispatchEmbed({ texts, pooling: POOLING, dimensions: DIMENSIONS });
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/** Get the current embedding model ID (for version tracking in stored embeddings) */
|
|
91
|
-
export function getModelId(): string {
|
|
92
|
-
return MODEL_ID;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Cosine similarity between two normalized vectors.
|
|
97
|
-
* Since vectors are pre-normalized, this is just the dot product.
|
|
98
|
-
*/
|
|
99
|
-
export function cosineSimilarity(a: number[], b: number[]): number {
|
|
100
|
-
if (a.length !== b.length || a.length === 0) return 0;
|
|
101
|
-
let dot = 0;
|
|
102
|
-
for (let i = 0; i < a.length; i++) {
|
|
103
|
-
dot += a[i] * b[i];
|
|
104
|
-
}
|
|
105
|
-
// Clamp to [-1, 1] to handle floating point drift
|
|
106
|
-
return Math.max(-1, Math.min(1, dot));
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/** Vector dimensions for this model */
|
|
110
|
-
export const EMBEDDING_DIMENSIONS = DIMENSIONS;
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Embedding Engine - vector embeddings via the ML worker pool.
|
|
5
|
+
*
|
|
6
|
+
* Default model: bge-small-en-v1.5 (384 dimensions, ~90MB, MTEB retrieval-optimized).
|
|
7
|
+
* Configurable via AWM_EMBED_MODEL env var.
|
|
8
|
+
*
|
|
9
|
+
* AWM 0.8.x: inference dispatches through ml-worker.ts. The worker_threads
|
|
10
|
+
* path was planned but reverted to in-process because onnxruntime-node's
|
|
11
|
+
* native bindings store V8 handles that don't cross isolate boundaries
|
|
12
|
+
* safely — see ml-worker.ts for the full status. The dispatch abstraction
|
|
13
|
+
* is preserved for a future child_process or HTTP sidecar pool.
|
|
14
|
+
* `AWM_ML_INPROCESS=1` is honored as a no-op (in-process is now the default).
|
|
15
|
+
*
|
|
16
|
+
* NOTE: Changing the model invalidates existing embeddings.
|
|
17
|
+
* Set AWM_EMBED_MODEL=Xenova/all-MiniLM-L6-v2 for backward compatibility.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
21
|
+
import { dispatchEmbed, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
+
|
|
23
|
+
const MODEL_ID = process.env.AWM_EMBED_MODEL ?? 'Xenova/bge-small-en-v1.5';
|
|
24
|
+
const DIMENSIONS = parseInt(process.env.AWM_EMBED_DIMS ?? '384', 10);
|
|
25
|
+
const POOLING = (process.env.AWM_EMBED_POOLING ?? 'mean') as 'cls' | 'mean';
|
|
26
|
+
|
|
27
|
+
// --- In-process fallback (used by tests and crash recovery) ---
|
|
28
|
+
|
|
29
|
+
let inProcessInstance: FeatureExtractionPipeline | null = null;
|
|
30
|
+
let inProcessInitPromise: Promise<FeatureExtractionPipeline> | null = null;
|
|
31
|
+
|
|
32
|
+
async function loadInProcess(): Promise<FeatureExtractionPipeline> {
|
|
33
|
+
if (inProcessInstance) return inProcessInstance;
|
|
34
|
+
if (inProcessInitPromise) return inProcessInitPromise;
|
|
35
|
+
inProcessInitPromise = pipeline('feature-extraction', MODEL_ID, { dtype: 'fp32' }).then(pipe => {
|
|
36
|
+
inProcessInstance = pipe;
|
|
37
|
+
console.log(`Embedding model loaded in-process: ${MODEL_ID} (${DIMENSIONS}d)`);
|
|
38
|
+
return pipe;
|
|
39
|
+
});
|
|
40
|
+
return inProcessInitPromise;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function inProcessEmbed(args: { texts: string[]; pooling: 'cls' | 'mean'; dimensions: number }): Promise<number[][]> {
|
|
44
|
+
const { texts, pooling, dimensions } = args;
|
|
45
|
+
if (texts.length === 0) return [];
|
|
46
|
+
const embedder = await loadInProcess();
|
|
47
|
+
const result = await embedder(texts, { pooling, normalize: true });
|
|
48
|
+
const data = result.data as Float32Array;
|
|
49
|
+
const vectors: number[][] = [];
|
|
50
|
+
for (let i = 0; i < texts.length; i++) {
|
|
51
|
+
vectors.push(Array.from(data.slice(i * dimensions, (i + 1) * dimensions)));
|
|
52
|
+
}
|
|
53
|
+
return vectors;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Register the in-process handler with the pool (used in test mode and as fallback)
|
|
57
|
+
registerInProcessHandlers({ embed: inProcessEmbed });
|
|
58
|
+
|
|
59
|
+
// --- Public API ---
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Get or initialize the embedding pipeline (singleton).
|
|
63
|
+
* Kept for backwards compat — returns the in-process pipeline only.
|
|
64
|
+
* Most consumers should use embed() / embedBatch() which dispatch
|
|
65
|
+
* to the worker pool by default.
|
|
66
|
+
*/
|
|
67
|
+
export async function getEmbedder(): Promise<FeatureExtractionPipeline> {
|
|
68
|
+
return loadInProcess();
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Generate an embedding vector for a text string.
|
|
73
|
+
* Dispatches to the worker pool (or in-process fallback).
|
|
74
|
+
*/
|
|
75
|
+
export async function embed(text: string): Promise<number[]> {
|
|
76
|
+
const vectors = await dispatchEmbed({ texts: [text], pooling: POOLING, dimensions: DIMENSIONS });
|
|
77
|
+
return vectors[0] ?? new Array(DIMENSIONS).fill(0);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Generate embeddings for multiple texts in a batch.
|
|
82
|
+
* More efficient than calling embed() in a loop — the worker batches the
|
|
83
|
+
* tokenization + forward pass.
|
|
84
|
+
*/
|
|
85
|
+
export async function embedBatch(texts: string[]): Promise<number[][]> {
|
|
86
|
+
if (texts.length === 0) return [];
|
|
87
|
+
return dispatchEmbed({ texts, pooling: POOLING, dimensions: DIMENSIONS });
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Get the current embedding model ID (for version tracking in stored embeddings) */
|
|
91
|
+
export function getModelId(): string {
|
|
92
|
+
return MODEL_ID;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Cosine similarity between two normalized vectors.
|
|
97
|
+
* Since vectors are pre-normalized, this is just the dot product.
|
|
98
|
+
*/
|
|
99
|
+
export function cosineSimilarity(a: number[], b: number[]): number {
|
|
100
|
+
if (a.length !== b.length || a.length === 0) return 0;
|
|
101
|
+
let dot = 0;
|
|
102
|
+
for (let i = 0; i < a.length; i++) {
|
|
103
|
+
dot += a[i] * b[i];
|
|
104
|
+
}
|
|
105
|
+
// Clamp to [-1, 1] to handle floating point drift
|
|
106
|
+
return Math.max(-1, Math.min(1, dot));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Vector dimensions for this model */
|
|
110
|
+
export const EMBEDDING_DIMENSIONS = DIMENSIONS;
|
package/src/core/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
export * from './decay.js';
|
|
4
|
-
export * from './hebbian.js';
|
|
5
|
-
export * from './salience.js';
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
export * from './decay.js';
|
|
4
|
+
export * from './hebbian.js';
|
|
5
|
+
export * from './salience.js';
|