@onenomad/engram-mcp 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -32
- package/dist/auth/login.d.ts +107 -68
- package/dist/auth/login.js +227 -216
- package/dist/auth/login.js.map +1 -1
- package/dist/consolidator.js +519 -519
- package/dist/context-pressure.js +91 -91
- package/dist/handoff.d.ts +53 -53
- package/dist/handoff.js +156 -156
- package/dist/server.js +204 -49
- package/dist/server.js.map +1 -1
- package/dist/source-dedup.d.ts +86 -86
- package/dist/source-dedup.js +147 -147
- package/dist/update-metadata.d.ts +29 -29
- package/dist/update-metadata.js +51 -51
- package/dist/wal.d.ts +95 -95
- package/dist/wal.js +295 -295
- package/package.json +1 -1
package/dist/source-dedup.js
CHANGED
|
@@ -1,148 +1,148 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Session-scoped same-source ingest dedup.
|
|
3
|
-
*
|
|
4
|
-
* Agents in long sessions repeatedly re-read stable files, re-poll
|
|
5
|
-
* unchanged endpoints, and re-list the same directories. Each re-ingest
|
|
6
|
-
* goes through the full chunk → embed → save pipeline even though the
|
|
7
|
-
* content hasn't moved. On CPU embeddings (Engram's default backend),
|
|
8
|
-
* a 20K-token re-read can cost 5–15 seconds; multiplied across a
|
|
9
|
-
* 50-step agent run that's significant wall-clock burn.
|
|
10
|
-
*
|
|
11
|
-
* The existing 0.75-similarity dedup (in `server.ts`'s
|
|
12
|
-
* tool handler) catches semantic duplicates, but does so against the
|
|
13
|
-
* ENTIRE memory store — and at write-time it actually trips on
|
|
14
|
-
* incidentally-similar memories (a fact about Pyre at 0.78 similarity
|
|
15
|
-
* to a fact about Engram). It also requires the new content to be
|
|
16
|
-
* embedded first, so it doesn't save the embedding cost.
|
|
17
|
-
*
|
|
18
|
-
* This module is the cheaper, more conservative path:
|
|
19
|
-
* - Scoped to a single source identifier (file path, URL, etc.).
|
|
20
|
-
* - Hash-based equality (SHA-256 of trimmed content) — exact match
|
|
21
|
-
* only, no false positives.
|
|
22
|
-
* - In-memory LRU keyed by `source` → list of recent content hashes.
|
|
23
|
-
* - Bounded: max 64 sources tracked, max 8 hashes per source.
|
|
24
|
-
*
|
|
25
|
-
* When an ingest hits the dedup cache, the caller can skip embedding
|
|
26
|
-
* AND skip the disk write — return the cached chunk id. Agent's
|
|
27
|
-
* conversation history stays internally consistent (same id for same
|
|
28
|
-
* content), and the wall-clock cost drops from "embed + save" to a
|
|
29
|
-
* map lookup.
|
|
30
|
-
*
|
|
31
|
-
* Process-scoped intentionally: the persistence layer doesn't need
|
|
32
|
-
* to know about this. Engram restart resets the cache — first ingest
|
|
33
|
-
* after restart goes through the full pipeline, which is fine.
|
|
34
|
-
*/
|
|
35
|
-
import { createHash } from 'node:crypto';
|
|
36
|
-
const MAX_SOURCES = 64;
|
|
37
|
-
const MAX_PER_SOURCE = 8;
|
|
38
|
-
export class SourceDedupCache {
|
|
39
|
-
/** sourceKey → list of recent (hash, chunkId) entries, MRU first. */
|
|
40
|
-
bySource = new Map();
|
|
41
|
-
/** Cache hit count since boot. Useful for telemetry. */
|
|
42
|
-
hits = 0;
|
|
43
|
-
/** Cache miss count since boot. */
|
|
44
|
-
misses = 0;
|
|
45
|
-
/**
|
|
46
|
-
* Hash trimmed content. Stable across ingest calls for the same
|
|
47
|
-
* payload — that's the whole point.
|
|
48
|
-
*/
|
|
49
|
-
static hashContent(content) {
|
|
50
|
-
return createHash('sha256').update(content.trim()).digest('hex');
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Look up a (source, content) pair. Returns the cached entry on hit
|
|
54
|
-
* or null on miss. Does NOT promote the entry on read — promote on
|
|
55
|
-
* write only, so a hit doesn't reset its LRU position.
|
|
56
|
-
*/
|
|
57
|
-
lookup(source, content) {
|
|
58
|
-
if (!source) {
|
|
59
|
-
// No source key → no scoping → don't dedup. The caller's
|
|
60
|
-
// existing semantic-similarity dedup is the right tool for
|
|
61
|
-
// unscoped ingests.
|
|
62
|
-
this.misses++;
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
const list = this.bySource.get(source);
|
|
66
|
-
if (!list || list.length === 0) {
|
|
67
|
-
this.misses++;
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
const hash = SourceDedupCache.hashContent(content);
|
|
71
|
-
const found = list.find((e) => e.hash === hash);
|
|
72
|
-
if (found) {
|
|
73
|
-
this.hits++;
|
|
74
|
-
return found;
|
|
75
|
-
}
|
|
76
|
-
this.misses++;
|
|
77
|
-
return null;
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Record a new (source, content, chunkId) entry after a successful
|
|
81
|
-
* ingest. LRU-evicts the oldest entry per source when the per-source
|
|
82
|
-
* cap is hit, and the oldest source when the overall cap is hit.
|
|
83
|
-
*/
|
|
84
|
-
remember(source, content, chunkId) {
|
|
85
|
-
if (!source)
|
|
86
|
-
return;
|
|
87
|
-
const hash = SourceDedupCache.hashContent(content);
|
|
88
|
-
const entry = { hash, chunkId, ts: Date.now() };
|
|
89
|
-
let list = this.bySource.get(source);
|
|
90
|
-
if (!list) {
|
|
91
|
-
list = [];
|
|
92
|
-
// Evict the oldest source if we're at the global cap.
|
|
93
|
-
if (this.bySource.size >= MAX_SOURCES) {
|
|
94
|
-
let oldestKey = null;
|
|
95
|
-
let oldestTs = Infinity;
|
|
96
|
-
for (const [k, entries] of this.bySource.entries()) {
|
|
97
|
-
const recent = entries[0]?.ts ?? 0;
|
|
98
|
-
if (recent < oldestTs) {
|
|
99
|
-
oldestTs = recent;
|
|
100
|
-
oldestKey = k;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
if (oldestKey)
|
|
104
|
-
this.bySource.delete(oldestKey);
|
|
105
|
-
}
|
|
106
|
-
this.bySource.set(source, list);
|
|
107
|
-
}
|
|
108
|
-
// De-dupe by hash within the per-source list — if the same hash
|
|
109
|
-
// is already there, replace it (fresh chunkId), otherwise prepend.
|
|
110
|
-
const existing = list.findIndex((e) => e.hash === hash);
|
|
111
|
-
if (existing >= 0) {
|
|
112
|
-
list[existing] = entry;
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
list.unshift(entry);
|
|
116
|
-
if (list.length > MAX_PER_SOURCE)
|
|
117
|
-
list.length = MAX_PER_SOURCE;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
/** Drop the entire cache. Useful for tests and explicit resets. */
|
|
121
|
-
clear() {
|
|
122
|
-
this.bySource.clear();
|
|
123
|
-
this.hits = 0;
|
|
124
|
-
this.misses = 0;
|
|
125
|
-
}
|
|
126
|
-
/** Snapshot stats for telemetry / Settings UI. */
|
|
127
|
-
stats() {
|
|
128
|
-
let entries = 0;
|
|
129
|
-
for (const list of this.bySource.values())
|
|
130
|
-
entries += list.length;
|
|
131
|
-
const total = this.hits + this.misses;
|
|
132
|
-
return {
|
|
133
|
-
sources: this.bySource.size,
|
|
134
|
-
entries,
|
|
135
|
-
hits: this.hits,
|
|
136
|
-
misses: this.misses,
|
|
137
|
-
hitRate: total === 0 ? 0 : this.hits / total,
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Module-level singleton. Engram is process-singleton anyway (one
|
|
143
|
-
* server instance per data dir), so a single cache covers the whole
|
|
144
|
-
* lifetime. Tests construct fresh `SourceDedupCache` instances; prod
|
|
145
|
-
* uses this default.
|
|
146
|
-
*/
|
|
147
|
-
export const sourceDedup = new SourceDedupCache();
|
|
1
|
+
/**
|
|
2
|
+
* Session-scoped same-source ingest dedup.
|
|
3
|
+
*
|
|
4
|
+
* Agents in long sessions repeatedly re-read stable files, re-poll
|
|
5
|
+
* unchanged endpoints, and re-list the same directories. Each re-ingest
|
|
6
|
+
* goes through the full chunk → embed → save pipeline even though the
|
|
7
|
+
* content hasn't moved. On CPU embeddings (Engram's default backend),
|
|
8
|
+
* a 20K-token re-read can cost 5–15 seconds; multiplied across a
|
|
9
|
+
* 50-step agent run that's significant wall-clock burn.
|
|
10
|
+
*
|
|
11
|
+
* The existing 0.75-similarity dedup (in `server.ts`'s engram-ingest
|
|
12
|
+
* tool handler) catches semantic duplicates, but does so against the
|
|
13
|
+
* ENTIRE memory store — and at write-time it actually trips on
|
|
14
|
+
* incidentally-similar memories (a fact about Pyre at 0.78 similarity
|
|
15
|
+
* to a fact about Engram). It also requires the new content to be
|
|
16
|
+
* embedded first, so it doesn't save the embedding cost.
|
|
17
|
+
*
|
|
18
|
+
* This module is the cheaper, more conservative path:
|
|
19
|
+
* - Scoped to a single source identifier (file path, URL, etc.).
|
|
20
|
+
* - Hash-based equality (SHA-256 of trimmed content) — exact match
|
|
21
|
+
* only, no false positives.
|
|
22
|
+
* - In-memory LRU keyed by `source` → list of recent content hashes.
|
|
23
|
+
* - Bounded: max 64 sources tracked, max 8 hashes per source.
|
|
24
|
+
*
|
|
25
|
+
* When an ingest hits the dedup cache, the caller can skip embedding
|
|
26
|
+
* AND skip the disk write — return the cached chunk id. Agent's
|
|
27
|
+
* conversation history stays internally consistent (same id for same
|
|
28
|
+
* content), and the wall-clock cost drops from "embed + save" to a
|
|
29
|
+
* map lookup.
|
|
30
|
+
*
|
|
31
|
+
* Process-scoped intentionally: the persistence layer doesn't need
|
|
32
|
+
* to know about this. Engram restart resets the cache — first ingest
|
|
33
|
+
* after restart goes through the full pipeline, which is fine.
|
|
34
|
+
*/
|
|
35
|
+
import { createHash } from 'node:crypto';
|
|
36
|
+
const MAX_SOURCES = 64;
|
|
37
|
+
const MAX_PER_SOURCE = 8;
|
|
38
|
+
export class SourceDedupCache {
|
|
39
|
+
/** sourceKey → list of recent (hash, chunkId) entries, MRU first. */
|
|
40
|
+
bySource = new Map();
|
|
41
|
+
/** Cache hit count since boot. Useful for telemetry. */
|
|
42
|
+
hits = 0;
|
|
43
|
+
/** Cache miss count since boot. */
|
|
44
|
+
misses = 0;
|
|
45
|
+
/**
|
|
46
|
+
* Hash trimmed content. Stable across ingest calls for the same
|
|
47
|
+
* payload — that's the whole point.
|
|
48
|
+
*/
|
|
49
|
+
static hashContent(content) {
|
|
50
|
+
return createHash('sha256').update(content.trim()).digest('hex');
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Look up a (source, content) pair. Returns the cached entry on hit
|
|
54
|
+
* or null on miss. Does NOT promote the entry on read — promote on
|
|
55
|
+
* write only, so a hit doesn't reset its LRU position.
|
|
56
|
+
*/
|
|
57
|
+
lookup(source, content) {
|
|
58
|
+
if (!source) {
|
|
59
|
+
// No source key → no scoping → don't dedup. The caller's
|
|
60
|
+
// existing semantic-similarity dedup is the right tool for
|
|
61
|
+
// unscoped ingests.
|
|
62
|
+
this.misses++;
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
const list = this.bySource.get(source);
|
|
66
|
+
if (!list || list.length === 0) {
|
|
67
|
+
this.misses++;
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
const hash = SourceDedupCache.hashContent(content);
|
|
71
|
+
const found = list.find((e) => e.hash === hash);
|
|
72
|
+
if (found) {
|
|
73
|
+
this.hits++;
|
|
74
|
+
return found;
|
|
75
|
+
}
|
|
76
|
+
this.misses++;
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Record a new (source, content, chunkId) entry after a successful
|
|
81
|
+
* ingest. LRU-evicts the oldest entry per source when the per-source
|
|
82
|
+
* cap is hit, and the oldest source when the overall cap is hit.
|
|
83
|
+
*/
|
|
84
|
+
remember(source, content, chunkId) {
|
|
85
|
+
if (!source)
|
|
86
|
+
return;
|
|
87
|
+
const hash = SourceDedupCache.hashContent(content);
|
|
88
|
+
const entry = { hash, chunkId, ts: Date.now() };
|
|
89
|
+
let list = this.bySource.get(source);
|
|
90
|
+
if (!list) {
|
|
91
|
+
list = [];
|
|
92
|
+
// Evict the oldest source if we're at the global cap.
|
|
93
|
+
if (this.bySource.size >= MAX_SOURCES) {
|
|
94
|
+
let oldestKey = null;
|
|
95
|
+
let oldestTs = Infinity;
|
|
96
|
+
for (const [k, entries] of this.bySource.entries()) {
|
|
97
|
+
const recent = entries[0]?.ts ?? 0;
|
|
98
|
+
if (recent < oldestTs) {
|
|
99
|
+
oldestTs = recent;
|
|
100
|
+
oldestKey = k;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (oldestKey)
|
|
104
|
+
this.bySource.delete(oldestKey);
|
|
105
|
+
}
|
|
106
|
+
this.bySource.set(source, list);
|
|
107
|
+
}
|
|
108
|
+
// De-dupe by hash within the per-source list — if the same hash
|
|
109
|
+
// is already there, replace it (fresh chunkId), otherwise prepend.
|
|
110
|
+
const existing = list.findIndex((e) => e.hash === hash);
|
|
111
|
+
if (existing >= 0) {
|
|
112
|
+
list[existing] = entry;
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
list.unshift(entry);
|
|
116
|
+
if (list.length > MAX_PER_SOURCE)
|
|
117
|
+
list.length = MAX_PER_SOURCE;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/** Drop the entire cache. Useful for tests and explicit resets. */
|
|
121
|
+
clear() {
|
|
122
|
+
this.bySource.clear();
|
|
123
|
+
this.hits = 0;
|
|
124
|
+
this.misses = 0;
|
|
125
|
+
}
|
|
126
|
+
/** Snapshot stats for telemetry / Settings UI. */
|
|
127
|
+
stats() {
|
|
128
|
+
let entries = 0;
|
|
129
|
+
for (const list of this.bySource.values())
|
|
130
|
+
entries += list.length;
|
|
131
|
+
const total = this.hits + this.misses;
|
|
132
|
+
return {
|
|
133
|
+
sources: this.bySource.size,
|
|
134
|
+
entries,
|
|
135
|
+
hits: this.hits,
|
|
136
|
+
misses: this.misses,
|
|
137
|
+
hitRate: total === 0 ? 0 : this.hits / total,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Module-level singleton. Engram is process-singleton anyway (one
|
|
143
|
+
* server instance per data dir), so a single cache covers the whole
|
|
144
|
+
* lifetime. Tests construct fresh `SourceDedupCache` instances; prod
|
|
145
|
+
* uses this default.
|
|
146
|
+
*/
|
|
147
|
+
export const sourceDedup = new SourceDedupCache();
|
|
148
148
|
//# sourceMappingURL=source-dedup.js.map
|
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
import type { CognitiveLayer, MemoryChunk, MemoryType, Sentiment } from './types.js';
|
|
2
|
-
export interface UpdateMetadataInput {
|
|
3
|
-
tags?: string[];
|
|
4
|
-
source?: string;
|
|
5
|
-
domain?: string;
|
|
6
|
-
topic?: string;
|
|
7
|
-
type?: MemoryType;
|
|
8
|
-
sentiment?: Sentiment;
|
|
9
|
-
importance?: number;
|
|
10
|
-
cognitiveLayer?: CognitiveLayer;
|
|
11
|
-
}
|
|
12
|
-
export type UpdateMetadataMode = 'merge' | 'replace';
|
|
13
|
-
/**
|
|
14
|
-
* Pure helper: build the storage patch for a
|
|
15
|
-
* call. Separated from server.ts so importing it (e.g. from tests)
|
|
16
|
-
* doesn't pull in the MCP stdio server bootstrap.
|
|
17
|
-
*
|
|
18
|
-
* - `merge`: only fields the caller specified land in the patch.
|
|
19
|
-
* Untouched fields are absent → Storage.updateChunk leaves them alone.
|
|
20
|
-
* - `replace`: every metadata-shape field is set, with caller values
|
|
21
|
-
* where present and engram defaults otherwise. Existing untouched
|
|
22
|
-
* fields get overwritten with the default. Footgun-y; the tool
|
|
23
|
-
* layer logs a warning when this mode fires.
|
|
24
|
-
*
|
|
25
|
-
* Immutable fields (id, createdAt, embedding, embeddingVersion) are
|
|
26
|
-
* never produced by this helper; the tool layer doesn't accept them
|
|
27
|
-
* in its input schema either.
|
|
28
|
-
*/
|
|
29
|
-
export declare function buildUpdateMetadataPatch(metadata: UpdateMetadataInput, mode: UpdateMetadataMode): Partial<MemoryChunk>;
|
|
1
|
+
import type { CognitiveLayer, MemoryChunk, MemoryType, Sentiment } from './types.js';
|
|
2
|
+
export interface UpdateMetadataInput {
|
|
3
|
+
tags?: string[];
|
|
4
|
+
source?: string;
|
|
5
|
+
domain?: string;
|
|
6
|
+
topic?: string;
|
|
7
|
+
type?: MemoryType;
|
|
8
|
+
sentiment?: Sentiment;
|
|
9
|
+
importance?: number;
|
|
10
|
+
cognitiveLayer?: CognitiveLayer;
|
|
11
|
+
}
|
|
12
|
+
export type UpdateMetadataMode = 'merge' | 'replace';
|
|
13
|
+
/**
|
|
14
|
+
* Pure helper: build the storage patch for a engram-update-metadata
|
|
15
|
+
* call. Separated from server.ts so importing it (e.g. from tests)
|
|
16
|
+
* doesn't pull in the MCP stdio server bootstrap.
|
|
17
|
+
*
|
|
18
|
+
* - `merge`: only fields the caller specified land in the patch.
|
|
19
|
+
* Untouched fields are absent → Storage.updateChunk leaves them alone.
|
|
20
|
+
* - `replace`: every metadata-shape field is set, with caller values
|
|
21
|
+
* where present and engram defaults otherwise. Existing untouched
|
|
22
|
+
* fields get overwritten with the default. Footgun-y; the tool
|
|
23
|
+
* layer logs a warning when this mode fires.
|
|
24
|
+
*
|
|
25
|
+
* Immutable fields (id, createdAt, embedding, embeddingVersion) are
|
|
26
|
+
* never produced by this helper; the tool layer doesn't accept them
|
|
27
|
+
* in its input schema either.
|
|
28
|
+
*/
|
|
29
|
+
export declare function buildUpdateMetadataPatch(metadata: UpdateMetadataInput, mode: UpdateMetadataMode): Partial<MemoryChunk>;
|
package/dist/update-metadata.js
CHANGED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure helper: build the storage patch for a
|
|
3
|
-
* call. Separated from server.ts so importing it (e.g. from tests)
|
|
4
|
-
* doesn't pull in the MCP stdio server bootstrap.
|
|
5
|
-
*
|
|
6
|
-
* - `merge`: only fields the caller specified land in the patch.
|
|
7
|
-
* Untouched fields are absent → Storage.updateChunk leaves them alone.
|
|
8
|
-
* - `replace`: every metadata-shape field is set, with caller values
|
|
9
|
-
* where present and engram defaults otherwise. Existing untouched
|
|
10
|
-
* fields get overwritten with the default. Footgun-y; the tool
|
|
11
|
-
* layer logs a warning when this mode fires.
|
|
12
|
-
*
|
|
13
|
-
* Immutable fields (id, createdAt, embedding, embeddingVersion) are
|
|
14
|
-
* never produced by this helper; the tool layer doesn't accept them
|
|
15
|
-
* in its input schema either.
|
|
16
|
-
*/
|
|
17
|
-
export function buildUpdateMetadataPatch(metadata, mode) {
|
|
18
|
-
const patch = {};
|
|
19
|
-
if (mode === 'replace') {
|
|
20
|
-
patch.tags = metadata.tags ?? [];
|
|
21
|
-
patch.source = metadata.source ?? '';
|
|
22
|
-
patch.domain = metadata.domain ?? '';
|
|
23
|
-
patch.topic = metadata.topic ?? '';
|
|
24
|
-
patch.type = metadata.type ?? 'context';
|
|
25
|
-
patch.sentiment = metadata.sentiment ?? 'neutral';
|
|
26
|
-
patch.importance = metadata.importance ?? 0.5;
|
|
27
|
-
if (metadata.cognitiveLayer !== undefined) {
|
|
28
|
-
patch.cognitiveLayer = metadata.cognitiveLayer;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
if (metadata.tags !== undefined)
|
|
33
|
-
patch.tags = metadata.tags;
|
|
34
|
-
if (metadata.source !== undefined)
|
|
35
|
-
patch.source = metadata.source;
|
|
36
|
-
if (metadata.domain !== undefined)
|
|
37
|
-
patch.domain = metadata.domain;
|
|
38
|
-
if (metadata.topic !== undefined)
|
|
39
|
-
patch.topic = metadata.topic;
|
|
40
|
-
if (metadata.type !== undefined)
|
|
41
|
-
patch.type = metadata.type;
|
|
42
|
-
if (metadata.sentiment !== undefined)
|
|
43
|
-
patch.sentiment = metadata.sentiment;
|
|
44
|
-
if (metadata.importance !== undefined)
|
|
45
|
-
patch.importance = metadata.importance;
|
|
46
|
-
if (metadata.cognitiveLayer !== undefined) {
|
|
47
|
-
patch.cognitiveLayer = metadata.cognitiveLayer;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
return patch;
|
|
51
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Pure helper: build the storage patch for a engram-update-metadata
|
|
3
|
+
* call. Separated from server.ts so importing it (e.g. from tests)
|
|
4
|
+
* doesn't pull in the MCP stdio server bootstrap.
|
|
5
|
+
*
|
|
6
|
+
* - `merge`: only fields the caller specified land in the patch.
|
|
7
|
+
* Untouched fields are absent → Storage.updateChunk leaves them alone.
|
|
8
|
+
* - `replace`: every metadata-shape field is set, with caller values
|
|
9
|
+
* where present and engram defaults otherwise. Existing untouched
|
|
10
|
+
* fields get overwritten with the default. Footgun-y; the tool
|
|
11
|
+
* layer logs a warning when this mode fires.
|
|
12
|
+
*
|
|
13
|
+
* Immutable fields (id, createdAt, embedding, embeddingVersion) are
|
|
14
|
+
* never produced by this helper; the tool layer doesn't accept them
|
|
15
|
+
* in its input schema either.
|
|
16
|
+
*/
|
|
17
|
+
export function buildUpdateMetadataPatch(metadata, mode) {
|
|
18
|
+
const patch = {};
|
|
19
|
+
if (mode === 'replace') {
|
|
20
|
+
patch.tags = metadata.tags ?? [];
|
|
21
|
+
patch.source = metadata.source ?? '';
|
|
22
|
+
patch.domain = metadata.domain ?? '';
|
|
23
|
+
patch.topic = metadata.topic ?? '';
|
|
24
|
+
patch.type = metadata.type ?? 'context';
|
|
25
|
+
patch.sentiment = metadata.sentiment ?? 'neutral';
|
|
26
|
+
patch.importance = metadata.importance ?? 0.5;
|
|
27
|
+
if (metadata.cognitiveLayer !== undefined) {
|
|
28
|
+
patch.cognitiveLayer = metadata.cognitiveLayer;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
if (metadata.tags !== undefined)
|
|
33
|
+
patch.tags = metadata.tags;
|
|
34
|
+
if (metadata.source !== undefined)
|
|
35
|
+
patch.source = metadata.source;
|
|
36
|
+
if (metadata.domain !== undefined)
|
|
37
|
+
patch.domain = metadata.domain;
|
|
38
|
+
if (metadata.topic !== undefined)
|
|
39
|
+
patch.topic = metadata.topic;
|
|
40
|
+
if (metadata.type !== undefined)
|
|
41
|
+
patch.type = metadata.type;
|
|
42
|
+
if (metadata.sentiment !== undefined)
|
|
43
|
+
patch.sentiment = metadata.sentiment;
|
|
44
|
+
if (metadata.importance !== undefined)
|
|
45
|
+
patch.importance = metadata.importance;
|
|
46
|
+
if (metadata.cognitiveLayer !== undefined) {
|
|
47
|
+
patch.cognitiveLayer = metadata.cognitiveLayer;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return patch;
|
|
51
|
+
}
|
|
52
52
|
//# sourceMappingURL=update-metadata.js.map
|
package/dist/wal.d.ts
CHANGED
|
@@ -1,95 +1,95 @@
|
|
|
1
|
-
import type { SmartMemoryConfig, MemoryType, CognitiveLayer, Sentiment, MemoryOrigin, MemoryTier } from './types.js';
|
|
2
|
-
import type { StoredChunk } from './storage.js';
|
|
3
|
-
import { Storage } from './storage.js';
|
|
4
|
-
/**
|
|
5
|
-
* Write-Ahead Log (WAL) — real-time memory capture during conversations.
|
|
6
|
-
*
|
|
7
|
-
* The WAL principle: write state BEFORE responding, not after.
|
|
8
|
-
* This ensures no memory is lost if the agent crashes, compacts, or restarts.
|
|
9
|
-
*
|
|
10
|
-
* Use `ingest` for immediate capture of facts/decisions/preferences
|
|
11
|
-
* as they happen, rather than waiting for post-conversation extraction.
|
|
12
|
-
*/
|
|
13
|
-
export interface IngestEntry {
|
|
14
|
-
content: string;
|
|
15
|
-
type?: MemoryType;
|
|
16
|
-
layer?: CognitiveLayer;
|
|
17
|
-
importance?: number;
|
|
18
|
-
tags?: string[];
|
|
19
|
-
source?: string;
|
|
20
|
-
domain?: string;
|
|
21
|
-
topic?: string;
|
|
22
|
-
sentiment?: Sentiment;
|
|
23
|
-
emotionalValence?: number;
|
|
24
|
-
emotionalArousal?: number;
|
|
25
|
-
origin?: MemoryOrigin;
|
|
26
|
-
tier?: MemoryTier;
|
|
27
|
-
/**
|
|
28
|
-
* ISO 8601 timestamp override. Default: ingest time (Date.now()).
|
|
29
|
-
*
|
|
30
|
-
* Critical when the content represents an event that originally
|
|
31
|
-
* happened at a different time — meeting notes from yesterday,
|
|
32
|
-
* dated documents, imported chat history, benchmark fixtures.
|
|
33
|
-
*
|
|
34
|
-
* The createdAt timestamp flows into `buildContextPrefix()` which
|
|
35
|
-
* is included in the embedded text. The retrieval pipeline uses
|
|
36
|
-
* this as a temporal signal — both via similarity match against
|
|
37
|
-
* the prefix in queries, and via downstream temporal-boost logic
|
|
38
|
-
* in `search.ts`.
|
|
39
|
-
*
|
|
40
|
-
* Without an override, every ingested memory shares the ingest-
|
|
41
|
-
* time prefix (which is the same for everything ingested in the
|
|
42
|
-
* same hour), losing all temporal differentiation.
|
|
43
|
-
*/
|
|
44
|
-
createdAt?: string;
|
|
45
|
-
/**
|
|
46
|
-
* When true, skip the per-chunk KG triple extraction. The standalone
|
|
47
|
-
* locomo bench bypasses this (calls saveChunk directly, never enters
|
|
48
|
-
* wal.ts), which is why its wall-clock is ~50× faster than Pyre's
|
|
49
|
-
* MCP-boundary bench on the same dataset.
|
|
50
|
-
*
|
|
51
|
-
* Real users keep KG extraction (it powers
|
|
52
|
-
*
|
|
53
|
-
* apples-to-apples vs the standalone bench should pass this flag
|
|
54
|
-
* so they're measuring the same code path.
|
|
55
|
-
*/
|
|
56
|
-
skipKgExtraction?: boolean;
|
|
57
|
-
/**
|
|
58
|
-
* When true, skip the post-batch appendDailyEntry write. Same
|
|
59
|
-
* rationale as skipKgExtraction — the standalone bench doesn't
|
|
60
|
-
* touch the daily-entries store; bench harnesses matching it
|
|
61
|
-
* should skip the write to compare on equal footing.
|
|
62
|
-
*/
|
|
63
|
-
skipDailyEntry?: boolean;
|
|
64
|
-
/**
|
|
65
|
-
* When false, KG extraction + daily-entry append run in the
|
|
66
|
-
* BACKGROUND after ingest() returns. The caller gets its chunks
|
|
67
|
-
* back as soon as the saveChunk loop finishes; the side effects
|
|
68
|
-
* complete on their own pace.
|
|
69
|
-
*
|
|
70
|
-
* Default true (backwards compatible — caller awaits everything).
|
|
71
|
-
* Production callers where the agent doesn't immediately query
|
|
72
|
-
* the just-written content (chat WAL, tool-vault bridge) should
|
|
73
|
-
* pass false for ~5-30× faster perceived ingest latency.
|
|
74
|
-
*
|
|
75
|
-
* To wait for background work to drain (tests, shutdown), call
|
|
76
|
-
* `flushPendingSideEffects()` from this module.
|
|
77
|
-
*/
|
|
78
|
-
awaitSideEffects?: boolean;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Wait for all in-flight background side-effects (KG extraction +
|
|
82
|
-
* daily-entry append fired with `awaitSideEffects: false`) to
|
|
83
|
-
* complete. No-op when nothing is pending.
|
|
84
|
-
*
|
|
85
|
-
* Tests should call this between ingest and assert; shutdown code
|
|
86
|
-
* should call before process exit to avoid losing KG writes.
|
|
87
|
-
*/
|
|
88
|
-
export declare function flushPendingSideEffects(): Promise<void>;
|
|
89
|
-
/** Pending count — for tests + telemetry. */
|
|
90
|
-
export declare function pendingSideEffectCount(): number;
|
|
91
|
-
/**
|
|
92
|
-
* Immediately persist one or more memory entries.
|
|
93
|
-
* Designed to be called mid-conversation, before the agent responds.
|
|
94
|
-
*/
|
|
95
|
-
export declare function ingest(config: SmartMemoryConfig, storage: Storage, entries: IngestEntry[]): Promise<StoredChunk[]>;
|
|
1
|
+
import type { SmartMemoryConfig, MemoryType, CognitiveLayer, Sentiment, MemoryOrigin, MemoryTier } from './types.js';
|
|
2
|
+
import type { StoredChunk } from './storage.js';
|
|
3
|
+
import { Storage } from './storage.js';
|
|
4
|
+
/**
|
|
5
|
+
* Write-Ahead Log (WAL) — real-time memory capture during conversations.
|
|
6
|
+
*
|
|
7
|
+
* The WAL principle: write state BEFORE responding, not after.
|
|
8
|
+
* This ensures no memory is lost if the agent crashes, compacts, or restarts.
|
|
9
|
+
*
|
|
10
|
+
* Use `ingest` for immediate capture of facts/decisions/preferences
|
|
11
|
+
* as they happen, rather than waiting for post-conversation extraction.
|
|
12
|
+
*/
|
|
13
|
+
export interface IngestEntry {
|
|
14
|
+
content: string;
|
|
15
|
+
type?: MemoryType;
|
|
16
|
+
layer?: CognitiveLayer;
|
|
17
|
+
importance?: number;
|
|
18
|
+
tags?: string[];
|
|
19
|
+
source?: string;
|
|
20
|
+
domain?: string;
|
|
21
|
+
topic?: string;
|
|
22
|
+
sentiment?: Sentiment;
|
|
23
|
+
emotionalValence?: number;
|
|
24
|
+
emotionalArousal?: number;
|
|
25
|
+
origin?: MemoryOrigin;
|
|
26
|
+
tier?: MemoryTier;
|
|
27
|
+
/**
|
|
28
|
+
* ISO 8601 timestamp override. Default: ingest time (Date.now()).
|
|
29
|
+
*
|
|
30
|
+
* Critical when the content represents an event that originally
|
|
31
|
+
* happened at a different time — meeting notes from yesterday,
|
|
32
|
+
* dated documents, imported chat history, benchmark fixtures.
|
|
33
|
+
*
|
|
34
|
+
* The createdAt timestamp flows into `buildContextPrefix()` which
|
|
35
|
+
* is included in the embedded text. The retrieval pipeline uses
|
|
36
|
+
* this as a temporal signal — both via similarity match against
|
|
37
|
+
* the prefix in queries, and via downstream temporal-boost logic
|
|
38
|
+
* in `search.ts`.
|
|
39
|
+
*
|
|
40
|
+
* Without an override, every ingested memory shares the ingest-
|
|
41
|
+
* time prefix (which is the same for everything ingested in the
|
|
42
|
+
* same hour), losing all temporal differentiation.
|
|
43
|
+
*/
|
|
44
|
+
createdAt?: string;
|
|
45
|
+
/**
|
|
46
|
+
* When true, skip the per-chunk KG triple extraction. The standalone
|
|
47
|
+
* locomo bench bypasses this (calls saveChunk directly, never enters
|
|
48
|
+
* wal.ts), which is why its wall-clock is ~50× faster than Pyre's
|
|
49
|
+
* MCP-boundary bench on the same dataset.
|
|
50
|
+
*
|
|
51
|
+
* Real users keep KG extraction (it powers engram-dossier,
|
|
52
|
+
* engram-kg-query, graph rerank). Benchmark harnesses comparing
|
|
53
|
+
* apples-to-apples vs the standalone bench should pass this flag
|
|
54
|
+
* so they're measuring the same code path.
|
|
55
|
+
*/
|
|
56
|
+
skipKgExtraction?: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* When true, skip the post-batch appendDailyEntry write. Same
|
|
59
|
+
* rationale as skipKgExtraction — the standalone bench doesn't
|
|
60
|
+
* touch the daily-entries store; bench harnesses matching it
|
|
61
|
+
* should skip the write to compare on equal footing.
|
|
62
|
+
*/
|
|
63
|
+
skipDailyEntry?: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* When false, KG extraction + daily-entry append run in the
|
|
66
|
+
* BACKGROUND after ingest() returns. The caller gets its chunks
|
|
67
|
+
* back as soon as the saveChunk loop finishes; the side effects
|
|
68
|
+
* complete on their own pace.
|
|
69
|
+
*
|
|
70
|
+
* Default true (backwards compatible — caller awaits everything).
|
|
71
|
+
* Production callers where the agent doesn't immediately query
|
|
72
|
+
* the just-written content (chat WAL, tool-vault bridge) should
|
|
73
|
+
* pass false for ~5-30× faster perceived ingest latency.
|
|
74
|
+
*
|
|
75
|
+
* To wait for background work to drain (tests, shutdown), call
|
|
76
|
+
* `flushPendingSideEffects()` from this module.
|
|
77
|
+
*/
|
|
78
|
+
awaitSideEffects?: boolean;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Wait for all in-flight background side-effects (KG extraction +
|
|
82
|
+
* daily-entry append fired with `awaitSideEffects: false`) to
|
|
83
|
+
* complete. No-op when nothing is pending.
|
|
84
|
+
*
|
|
85
|
+
* Tests should call this between ingest and assert; shutdown code
|
|
86
|
+
* should call before process exit to avoid losing KG writes.
|
|
87
|
+
*/
|
|
88
|
+
export declare function flushPendingSideEffects(): Promise<void>;
|
|
89
|
+
/** Pending count — for tests + telemetry. */
|
|
90
|
+
export declare function pendingSideEffectCount(): number;
|
|
91
|
+
/**
|
|
92
|
+
* Immediately persist one or more memory entries.
|
|
93
|
+
* Designed to be called mid-conversation, before the agent responds.
|
|
94
|
+
*/
|
|
95
|
+
export declare function ingest(config: SmartMemoryConfig, storage: Storage, entries: IngestEntry[]): Promise<StoredChunk[]>;
|