@gnldev/rag 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/semantic-memory.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// vector search. Durable/replayable: the recall result falls into runDurable's input journaling
|
|
4
4
|
// (frozen on resume); embeddings persist in the journal. (Most semantic-recall implementations don't come with these guarantees.)
|
|
5
5
|
import { cosineSimilarity } from 'ai';
|
|
6
|
-
import { matchFilter } from '@gnldev/durable';
|
|
6
|
+
import { matchFilter, memKey, assertThreadId } from '@gnldev/durable';
|
|
7
7
|
function normRange(r) {
|
|
8
8
|
if (r == null)
|
|
9
9
|
return { before: 0, after: 0 };
|
|
@@ -31,6 +31,10 @@ export class SemanticMemory {
|
|
|
31
31
|
this.topK = opts.topK ?? 3;
|
|
32
32
|
}
|
|
33
33
|
key(threadId) {
|
|
34
|
+
// Same hazard, this package's own namespace: `sem:model:log` parses as a run called 'sem', and a
|
|
35
|
+
// sweep then purges every thread's recall log. The prefix differs; the rule does not, because the
|
|
36
|
+
// rule belongs to the ':model:'/':tool:' shape rather than to any one keyspace.
|
|
37
|
+
assertThreadId(threadId);
|
|
34
38
|
return `sem:${threadId}:log`;
|
|
35
39
|
}
|
|
36
40
|
async load(threadId) {
|
|
@@ -80,11 +84,17 @@ export class SemanticMemory {
|
|
|
80
84
|
}
|
|
81
85
|
await this.journal.put(this.key(threadId), log);
|
|
82
86
|
}
|
|
87
|
+
// Built by @gnldev/durable's own memKey rather than by hand. Writing the same string here looked
|
|
88
|
+
// harmless — it IS the same string — but the function is also where the thread id is checked, and
|
|
89
|
+
// spelling the key out skipped the check. Measured: setWorkingMemory('model', …) wrote
|
|
90
|
+
// `mem:model:working`, which parseJournalKey reads as a run called 'mem', and the next sweepRuns
|
|
91
|
+
// purged that "run" by prefix — two unrelated users' threads went from present to gone in one
|
|
92
|
+
// call. BasicMemory refused the same id.
|
|
83
93
|
async getWorkingMemory(threadId) {
|
|
84
|
-
return this.journal.get(
|
|
94
|
+
return this.journal.get(memKey(threadId, 'working'));
|
|
85
95
|
}
|
|
86
96
|
async setWorkingMemory(threadId, value) {
|
|
87
|
-
await this.journal.put(
|
|
97
|
+
await this.journal.put(memKey(threadId, 'working'), value);
|
|
88
98
|
}
|
|
89
99
|
}
|
|
90
100
|
//# sourceMappingURL=semantic-memory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-memory.js","sourceRoot":"","sources":["../src/semantic-memory.ts"],"names":[],"mappings":"AAAA,qGAAqG;AACrG,mGAAmG;AACnG,gGAAgG;AAChG,kIAAkI;AAClI,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAoC9C,SAAS,SAAS,CAAC,CAA8C;IAC/D,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC9C,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,WAAW,CAAC,CAAM;IACzB,IAAI,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC;IAClE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,SAAS,CAAC;IACxB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,CAAY,EAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAE/E,MAAM,OAAO,cAAc;IACjB,OAAO,CAAU;IACjB,KAAK,CAAQ;IACb,OAAO,CAAS;IAChB,IAAI,CAAS;IAErB,YAAY,IAA2B;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEO,GAAG,CAAC,QAAgB;QAC1B,OAAO,OAAO,QAAQ,MAAM,CAAC;IAC/B,CAAC;IACO,KAAK,CAAC,IAAI,CAAC,QAAgB;QACjC,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAa,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,IAAiC;QACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;QAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAEtC,IAAI,CAAC,IAAI,EAAE,KAAK;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAErD,sEAAsE;QACtE,8FAA8F;QAC9F,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACtC,IAAI,MAAM,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,SAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC/F,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,2EAA2E;aAC5G,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAO,CAAC,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAExC,mGAAmG;QACnG,6FAA6F;QAC7F,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YACvD,KAAK,IAAI,GAAG,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,GAAG,GAAG,WAAW,EAAE,GAAG,EAAE;gBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAE7F,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,QAAe;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,OAAO,QAAQ,UAAU,CAAC,CAAC;IAC7D,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,QAAgB,EAAE,KAAa;QACpD,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,UAAU,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;CACF","sourcesContent":["// SemanticMemory — the semantic-recall version of @gnldev/durable's Memory. Embeds past messages and\n// stores them in the journal; when a new turn (query) arrives, retrieves relevant OLD messages via\n// vector search. Durable/replayable: the recall result falls into runDurable's input journaling\n// (frozen on resume); embeddings persist in the journal. (Most semantic-recall implementations don't come with these guarantees.)\nimport { cosineSimilarity } from 'ai';\nimport { matchFilter } from '@gnldev/durable';\nimport type { Memory, Journal } from '@gnldev/durable';\nimport type { Embed } from './vector-store.js';\n\nexport interface SemanticMemoryOptions {\n journal: Journal;\n embed: Embed;\n /** The most recent N messages always included (default 6). */\n recentN?: number;\n /** Number of relevant old messages recalled based on the query (default 3). */\n topK?: number;\n}\n\n/**\n * P1.5 the same recall knobs @gnldev/durable's `RecallOptions` exposes, wired\n * through SemanticMemory's local (journal-backed) cosine recall — `Memory.getMessages`'s `opts` is\n * structurally wider here than the base interface (bivariant method params), so this is call-compatible\n * with plain `{query}` callers.\n */\nexport interface SemanticGetMessagesOptions {\n query?: string;\n /** Minimum cosine similarity to be a candidate (default 0 — any positive score). */\n threshold?: number;\n /** Expand each hit with its before/after neighbors BY LOG POSITION; `n` is sugar for `{before:n,after:n}`. */\n messageRange?: number | { before: number; after: number };\n /** Metadata filter (same operator subset as `@gnldev/durable`'s `matchFilter`), applied BEFORE topK selection. */\n filter?: Record<string, unknown>;\n}\n\ninterface LogEntry {\n message: any;\n text?: string;\n embedding?: number[];\n metadata?: Record<string, unknown>;\n}\n\nfunction normRange(r?: number | { before: number; after: number }): { before: number; after: number } {\n if (r == null) return { before: 0, after: 0 };\n return typeof r === 'number' ? { before: r, after: r } : r;\n}\n\nfunction messageText(m: any): string | undefined {\n if (typeof m?.content === 'string') return m.content || undefined;\n if (Array.isArray(m?.content)) {\n const t = m.content.filter((p: any) => typeof p?.text === 'string').map((p: any) => p.text).join(' ');\n return t || undefined;\n }\n return undefined;\n}\n\nconst hasNorm = (v?: number[]): v is number[] => !!v && v.some((x) => x !== 0);\n\nexport class SemanticMemory implements Memory {\n private journal: Journal;\n private embed: Embed;\n private recentN: number;\n private topK: number;\n\n constructor(opts: SemanticMemoryOptions) {\n this.journal = opts.journal;\n this.embed = opts.embed;\n this.recentN = opts.recentN ?? 6;\n this.topK = opts.topK ?? 3;\n }\n\n private key(threadId: string) {\n return `sem:${threadId}:log`;\n }\n private async load(threadId: string): Promise<LogEntry[]> {\n return (await this.journal.get<LogEntry[]>(this.key(threadId))) ?? [];\n }\n\n async getMessages(threadId: string, opts?: SemanticGetMessagesOptions): Promise<any[]> {\n const log = await this.load(threadId);\n if (log.length <= this.recentN) return log.map((e) => e.message);\n const recentStart = log.length - this.recentN;\n const recent = log.slice(recentStart);\n\n if (!opts?.query) return recent.map((e) => e.message);\n const q = await this.embed(opts.query);\n if (!hasNorm(q)) return recent.map((e) => e.message);\n\n // P1.5 threshold + filter apply BEFORE topK slicing (a filtered-out /\n // below-threshold candidate must not consume a topK slot) — same order as MemoryStore.recall.\n const threshold = opts.threshold ?? 0;\n let scored = log\n .map((e, i) => ({ i, e, score: hasNorm(e.embedding) ? cosineSimilarity(q, e.embedding!) : -1 }))\n .filter((s) => s.i < recentStart) // candidates come from the OLDER portion only (unchanged from before P1.5)\n .filter((s) => s.score > 0 && s.score >= threshold);\n if (opts.filter) scored = scored.filter((s) => matchFilter(s.e.metadata, opts.filter!));\n scored.sort((a, b) => b.score - a.score);\n const hits = scored.slice(0, this.topK);\n\n // messageRange: expand each hit with its before/after neighbors BY LOG POSITION, dedup overlapping\n // windows, exclude anything already covered by `recent` (avoid duplicating a message twice).\n const range = normRange(opts.messageRange);\n const picked = new Map<number, LogEntry>();\n for (const h of hits) {\n const lo = Math.max(0, h.i - range.before);\n const hi = Math.min(log.length - 1, h.i + range.after);\n for (let idx = lo; idx <= hi && idx < recentStart; idx++) picked.set(idx, log[idx]!);\n }\n const recalled = [...picked.entries()].sort((a, b) => a[0] - b[0]).map(([, e]) => e.message);\n\n return [...recalled, ...recent.map((e) => e.message)];\n }\n\n async append(threadId: string, messages: any[]): Promise<void> {\n const log = await this.load(threadId);\n for (const message of messages) {\n const text = messageText(message);\n const embedding = text ? await this.embed(text) : undefined;\n log.push({ message, text, embedding, metadata: message?.metadata });\n }\n await this.journal.put(this.key(threadId), log);\n }\n\n async getWorkingMemory(threadId: string): Promise<string | undefined> {\n return this.journal.get<string>(`mem:${threadId}:working`);\n }\n async setWorkingMemory(threadId: string, value: string): Promise<void> {\n await this.journal.put(`mem:${threadId}:working`, value);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"semantic-memory.js","sourceRoot":"","sources":["../src/semantic-memory.ts"],"names":[],"mappings":"AAAA,qGAAqG;AACrG,mGAAmG;AACnG,gGAAgG;AAChG,kIAAkI;AAClI,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAoCtE,SAAS,SAAS,CAAC,CAA8C;IAC/D,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC9C,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,WAAW,CAAC,CAAM;IACzB,IAAI,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC;IAClE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtG,OAAO,CAAC,IAAI,SAAS,CAAC;IACxB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,CAAY,EAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAE/E,MAAM,OAAO,cAAc;IACjB,OAAO,CAAU;IACjB,KAAK,CAAQ;IACb,OAAO,CAAS;IAChB,IAAI,CAAS;IAErB,YAAY,IAA2B;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEO,GAAG,CAAC,QAAgB;QAC1B,iGAAiG;QACjG,kGAAkG;QAClG,gFAAgF;QAChF,cAAc,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,OAAO,QAAQ,MAAM,CAAC;IAC/B,CAAC;IACO,KAAK,CAAC,IAAI,CAAC,QAAgB;QACjC,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAa,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,IAAiC;QACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;QAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAEtC,IAAI,CAAC,IAAI,EAAE,KAAK;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAErD,sEAAsE;QACtE,8FAA8F;QAC9F,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACtC,IAAI,MAAM,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,SAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC/F,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,2EAA2E;aAC5G,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAO,CAAC,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAExC,mGAAmG;QACnG,6FAA6F;QAC7F,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YACvD,KAAK,IAAI,GAAG,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,GAAG,GAAG,WAAW,EAAE,GAAG,EAAE;gBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAE7F,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,QAAe;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,iGAAiG;IACjG,kGAAkG;IAClG,uFAAuF;IACvF,iGAAiG;IACjG,8FAA8F;IAC9F,yCAAyC;IACzC,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,QAAgB,EAAE,KAAa;QACpD,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;CACF","sourcesContent":["// SemanticMemory — the semantic-recall version of @gnldev/durable's Memory. Embeds past messages and\n// stores them in the journal; when a new turn (query) arrives, retrieves relevant OLD messages via\n// vector search. Durable/replayable: the recall result falls into runDurable's input journaling\n// (frozen on resume); embeddings persist in the journal. (Most semantic-recall implementations don't come with these guarantees.)\nimport { cosineSimilarity } from 'ai';\nimport { matchFilter, memKey, assertThreadId } from '@gnldev/durable';\nimport type { Memory, Journal } from '@gnldev/durable';\nimport type { Embed } from './vector-store.js';\n\nexport interface SemanticMemoryOptions {\n journal: Journal;\n embed: Embed;\n /** The most recent N messages always included (default 6). */\n recentN?: number;\n /** Number of relevant old messages recalled based on the query (default 3). */\n topK?: number;\n}\n\n/**\n * P1.5 the same recall knobs @gnldev/durable's `RecallOptions` exposes, wired\n * through SemanticMemory's local (journal-backed) cosine recall — `Memory.getMessages`'s `opts` is\n * structurally wider here than the base interface (bivariant method params), so this is call-compatible\n * with plain `{query}` callers.\n */\nexport interface SemanticGetMessagesOptions {\n query?: string;\n /** Minimum cosine similarity to be a candidate (default 0 — any positive score). */\n threshold?: number;\n /** Expand each hit with its before/after neighbors BY LOG POSITION; `n` is sugar for `{before:n,after:n}`. */\n messageRange?: number | { before: number; after: number };\n /** Metadata filter (same operator subset as `@gnldev/durable`'s `matchFilter`), applied BEFORE topK selection. */\n filter?: Record<string, unknown>;\n}\n\ninterface LogEntry {\n message: any;\n text?: string;\n embedding?: number[];\n metadata?: Record<string, unknown>;\n}\n\nfunction normRange(r?: number | { before: number; after: number }): { before: number; after: number } {\n if (r == null) return { before: 0, after: 0 };\n return typeof r === 'number' ? { before: r, after: r } : r;\n}\n\nfunction messageText(m: any): string | undefined {\n if (typeof m?.content === 'string') return m.content || undefined;\n if (Array.isArray(m?.content)) {\n const t = m.content.filter((p: any) => typeof p?.text === 'string').map((p: any) => p.text).join(' ');\n return t || undefined;\n }\n return undefined;\n}\n\nconst hasNorm = (v?: number[]): v is number[] => !!v && v.some((x) => x !== 0);\n\nexport class SemanticMemory implements Memory {\n private journal: Journal;\n private embed: Embed;\n private recentN: number;\n private topK: number;\n\n constructor(opts: SemanticMemoryOptions) {\n this.journal = opts.journal;\n this.embed = opts.embed;\n this.recentN = opts.recentN ?? 6;\n this.topK = opts.topK ?? 3;\n }\n\n private key(threadId: string) {\n // Same hazard, this package's own namespace: `sem:model:log` parses as a run called 'sem', and a\n // sweep then purges every thread's recall log. The prefix differs; the rule does not, because the\n // rule belongs to the ':model:'/':tool:' shape rather than to any one keyspace.\n assertThreadId(threadId);\n return `sem:${threadId}:log`;\n }\n private async load(threadId: string): Promise<LogEntry[]> {\n return (await this.journal.get<LogEntry[]>(this.key(threadId))) ?? [];\n }\n\n async getMessages(threadId: string, opts?: SemanticGetMessagesOptions): Promise<any[]> {\n const log = await this.load(threadId);\n if (log.length <= this.recentN) return log.map((e) => e.message);\n const recentStart = log.length - this.recentN;\n const recent = log.slice(recentStart);\n\n if (!opts?.query) return recent.map((e) => e.message);\n const q = await this.embed(opts.query);\n if (!hasNorm(q)) return recent.map((e) => e.message);\n\n // P1.5 threshold + filter apply BEFORE topK slicing (a filtered-out /\n // below-threshold candidate must not consume a topK slot) — same order as MemoryStore.recall.\n const threshold = opts.threshold ?? 0;\n let scored = log\n .map((e, i) => ({ i, e, score: hasNorm(e.embedding) ? cosineSimilarity(q, e.embedding!) : -1 }))\n .filter((s) => s.i < recentStart) // candidates come from the OLDER portion only (unchanged from before P1.5)\n .filter((s) => s.score > 0 && s.score >= threshold);\n if (opts.filter) scored = scored.filter((s) => matchFilter(s.e.metadata, opts.filter!));\n scored.sort((a, b) => b.score - a.score);\n const hits = scored.slice(0, this.topK);\n\n // messageRange: expand each hit with its before/after neighbors BY LOG POSITION, dedup overlapping\n // windows, exclude anything already covered by `recent` (avoid duplicating a message twice).\n const range = normRange(opts.messageRange);\n const picked = new Map<number, LogEntry>();\n for (const h of hits) {\n const lo = Math.max(0, h.i - range.before);\n const hi = Math.min(log.length - 1, h.i + range.after);\n for (let idx = lo; idx <= hi && idx < recentStart; idx++) picked.set(idx, log[idx]!);\n }\n const recalled = [...picked.entries()].sort((a, b) => a[0] - b[0]).map(([, e]) => e.message);\n\n return [...recalled, ...recent.map((e) => e.message)];\n }\n\n async append(threadId: string, messages: any[]): Promise<void> {\n const log = await this.load(threadId);\n for (const message of messages) {\n const text = messageText(message);\n const embedding = text ? await this.embed(text) : undefined;\n log.push({ message, text, embedding, metadata: message?.metadata });\n }\n await this.journal.put(this.key(threadId), log);\n }\n\n // Built by @gnldev/durable's own memKey rather than by hand. Writing the same string here looked\n // harmless — it IS the same string — but the function is also where the thread id is checked, and\n // spelling the key out skipped the check. Measured: setWorkingMemory('model', …) wrote\n // `mem:model:working`, which parseJournalKey reads as a run called 'mem', and the next sweepRuns\n // purged that \"run\" by prefix — two unrelated users' threads went from present to gone in one\n // call. BasicMemory refused the same id.\n async getWorkingMemory(threadId: string): Promise<string | undefined> {\n return this.journal.get<string>(memKey(threadId, 'working'));\n }\n async setWorkingMemory(threadId: string, value: string): Promise<void> {\n await this.journal.put(memKey(threadId, 'working'), value);\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gnldev/rag",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"ai": "^7.0.0",
|
|
40
40
|
"pg": "^8.0.0",
|
|
41
41
|
"zod": "^3.25.76 || ^4.1.8",
|
|
42
|
-
"@gnldev/durable": "^0.
|
|
42
|
+
"@gnldev/durable": "^0.3.0"
|
|
43
43
|
},
|
|
44
44
|
"peerDependenciesMeta": {
|
|
45
45
|
"pg": {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"ai": "^7.0.0",
|
|
51
51
|
"pg": "^8.0.0",
|
|
52
52
|
"zod": "^3.25.0",
|
|
53
|
-
"@gnldev/durable": "0.
|
|
53
|
+
"@gnldev/durable": "0.3.0"
|
|
54
54
|
},
|
|
55
55
|
"author": "Karaca Yılmaz (https://gnl.dev)",
|
|
56
56
|
"homepage": "https://gnl.dev",
|