@dzhechkov/memory 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/backend.d.ts +45 -0
- package/dist/backend.d.ts.map +1 -0
- package/dist/backend.js +7 -0
- package/dist/backend.js.map +1 -0
- package/dist/bridge.d.ts +32 -0
- package/dist/bridge.d.ts.map +1 -0
- package/dist/bridge.js +79 -0
- package/dist/bridge.js.map +1 -0
- package/dist/cascade.d.ts +32 -0
- package/dist/cascade.d.ts.map +1 -0
- package/dist/cascade.js +30 -0
- package/dist/cascade.js.map +1 -0
- package/dist/dreaming.d.ts +37 -0
- package/dist/dreaming.d.ts.map +1 -0
- package/dist/dreaming.js +102 -0
- package/dist/dreaming.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/json-backend.d.ts +33 -0
- package/dist/json-backend.d.ts.map +1 -0
- package/dist/json-backend.js +88 -0
- package/dist/json-backend.js.map +1 -0
- package/dist/reflexion.d.ts +37 -0
- package/dist/reflexion.d.ts.map +1 -0
- package/dist/reflexion.js +62 -0
- package/dist/reflexion.js.map +1 -0
- package/dist/sqlite-backend.d.ts +44 -0
- package/dist/sqlite-backend.d.ts.map +1 -0
- package/dist/sqlite-backend.js +213 -0
- package/dist/sqlite-backend.js.map +1 -0
- package/dist/sqlite-probe.d.ts +27 -0
- package/dist/sqlite-probe.d.ts.map +1 -0
- package/dist/sqlite-probe.js +34 -0
- package/dist/sqlite-probe.js.map +1 -0
- package/package.json +55 -0
- package/src/backend.ts +47 -0
- package/src/bridge.ts +99 -0
- package/src/cascade.ts +52 -0
- package/src/dreaming.ts +127 -0
- package/src/index.ts +24 -0
- package/src/json-backend.ts +109 -0
- package/src/reflexion.ts +79 -0
- package/src/sqlite-backend.ts +263 -0
- package/src/sqlite-probe.ts +44 -0
package/src/cascade.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The backend cascade — probe optional backends, fall back gracefully.
|
|
3
|
+
*
|
|
4
|
+
* Heavier backends (a vector / embedding store, `agentdb`, `sql.js`) can be
|
|
5
|
+
* registered as probes. If none initialises, a guaranteed fallback is used.
|
|
6
|
+
* This keeps such backends *optional* — never a hard dependency.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { MemoryBackend } from './backend.js';
|
|
12
|
+
|
|
13
|
+
/** A candidate backend the cascade may select. */
|
|
14
|
+
export interface BackendProbe {
|
|
15
|
+
/** Probe name, for the `tried` log. */
|
|
16
|
+
readonly name: string;
|
|
17
|
+
/** Try to create the backend; resolve `undefined` (or throw) if unavailable. */
|
|
18
|
+
create(): Promise<MemoryBackend | undefined>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** The outcome of {@link selectBackend}. */
|
|
22
|
+
export interface CascadeResult {
|
|
23
|
+
/** The selected backend. */
|
|
24
|
+
readonly backend: MemoryBackend;
|
|
25
|
+
/** Name of the selected backend (probe name, or the fallback's name). */
|
|
26
|
+
readonly selected: string;
|
|
27
|
+
/** Probe names attempted, in order. */
|
|
28
|
+
readonly tried: string[];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Walk `probes` in order; return the first backend that initialises. If every
|
|
33
|
+
* probe is unavailable (returns `undefined` or throws), return `fallback`.
|
|
34
|
+
*/
|
|
35
|
+
export async function selectBackend(
|
|
36
|
+
probes: readonly BackendProbe[],
|
|
37
|
+
fallback: MemoryBackend,
|
|
38
|
+
): Promise<CascadeResult> {
|
|
39
|
+
const tried: string[] = [];
|
|
40
|
+
for (const probe of probes) {
|
|
41
|
+
tried.push(probe.name);
|
|
42
|
+
try {
|
|
43
|
+
const backend = await probe.create();
|
|
44
|
+
if (backend !== undefined) {
|
|
45
|
+
return { backend, selected: probe.name, tried };
|
|
46
|
+
}
|
|
47
|
+
} catch {
|
|
48
|
+
// probe unavailable — fall through to the next
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return { backend: fallback, selected: fallback.name, tried };
|
|
52
|
+
}
|
package/src/dreaming.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent SDK Dreaming integration — bridges Opus 4.8 Dreaming with Reflexion.
|
|
3
|
+
*
|
|
4
|
+
* Per ADR-005: this is orchestration-layer code. It reads session JSONL files
|
|
5
|
+
* (produced by Agent SDK), extracts patterns, and feeds them into the Reflexion
|
|
6
|
+
* system. The MemoryBackend interface is unchanged.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
|
|
14
|
+
import type { MemoryRecord } from './backend.js';
|
|
15
|
+
|
|
16
|
+
/** A pattern extracted from an Agent SDK session. */
|
|
17
|
+
export interface DreamPattern {
|
|
18
|
+
readonly skillId: string;
|
|
19
|
+
readonly outcome: 'excellent' | 'good' | 'needs_work' | 'failed';
|
|
20
|
+
readonly score: number;
|
|
21
|
+
readonly insight: string;
|
|
22
|
+
readonly sessionFile: string;
|
|
23
|
+
readonly timestamp: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Options for the dream harvester. */
|
|
27
|
+
export interface DreamOptions {
|
|
28
|
+
/** Directory containing Agent SDK session JSONL files. */
|
|
29
|
+
readonly sessionsDir: string;
|
|
30
|
+
/** Only process sessions newer than this ISO timestamp. */
|
|
31
|
+
readonly since?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Harvest patterns from Agent SDK session files.
|
|
36
|
+
*
|
|
37
|
+
* Scans `.jsonl` files in `sessionsDir`, extracts tool-use outcomes
|
|
38
|
+
* and skill invocations, and returns them as `DreamPattern`s ready
|
|
39
|
+
* to be fed into Reflexion via `reflexion.record()`.
|
|
40
|
+
*/
|
|
41
|
+
export function harvestDreamPatterns(options: DreamOptions): DreamPattern[] {
|
|
42
|
+
const { sessionsDir, since } = options;
|
|
43
|
+
if (!existsSync(sessionsDir)) return [];
|
|
44
|
+
|
|
45
|
+
const patterns: DreamPattern[] = [];
|
|
46
|
+
const files = readdirSync(sessionsDir)
|
|
47
|
+
.filter((f) => f.endsWith('.jsonl'))
|
|
48
|
+
.sort();
|
|
49
|
+
|
|
50
|
+
for (const file of files) {
|
|
51
|
+
const filePath = join(sessionsDir, file);
|
|
52
|
+
const lines = readFileSync(filePath, 'utf-8').split('\n').filter((l) => l.trim().length > 0);
|
|
53
|
+
|
|
54
|
+
for (const line of lines) {
|
|
55
|
+
try {
|
|
56
|
+
const entry = JSON.parse(line);
|
|
57
|
+
|
|
58
|
+
// Skip entries older than `since`
|
|
59
|
+
if (since !== undefined && entry.timestamp && entry.timestamp < since) continue;
|
|
60
|
+
|
|
61
|
+
// Extract skill invocations from tool_use messages
|
|
62
|
+
if (entry.type === 'assistant' && entry.message?.content) {
|
|
63
|
+
const content = Array.isArray(entry.message.content) ? entry.message.content : [entry.message.content];
|
|
64
|
+
for (const block of content) {
|
|
65
|
+
if (block.type === 'tool_use' && block.name?.startsWith?.('mcp__')) {
|
|
66
|
+
// MCP tool invocation — extract skill context
|
|
67
|
+
const skillId = block.input?.targetPath ?? block.input?.scope ?? 'unknown';
|
|
68
|
+
patterns.push({
|
|
69
|
+
skillId: typeof skillId === 'string' ? skillId : 'unknown',
|
|
70
|
+
outcome: 'good',
|
|
71
|
+
score: 0.7,
|
|
72
|
+
insight: `Tool ${block.name} invoked during session`,
|
|
73
|
+
sessionFile: file,
|
|
74
|
+
timestamp: entry.timestamp ?? new Date().toISOString(),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Extract checkpoint responses (reward signals)
|
|
81
|
+
if (entry.type === 'user' && typeof entry.message?.content === 'string') {
|
|
82
|
+
const text = entry.message.content.toLowerCase();
|
|
83
|
+
let outcome: DreamPattern['outcome'] = 'good';
|
|
84
|
+
let score = 0.7;
|
|
85
|
+
if (text === 'ок' || text === 'ok' || text === 'next' || text === 'продолжай') {
|
|
86
|
+
outcome = 'excellent';
|
|
87
|
+
score = 1.0;
|
|
88
|
+
} else if (text.includes('переделай') || text.includes('rework') || text.includes('заново')) {
|
|
89
|
+
outcome = 'needs_work';
|
|
90
|
+
score = 0.3;
|
|
91
|
+
} else if (text.includes('стоп') || text.includes('stop') || text.includes('wrong')) {
|
|
92
|
+
outcome = 'failed';
|
|
93
|
+
score = 0.0;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (outcome !== 'good') {
|
|
97
|
+
patterns.push({
|
|
98
|
+
skillId: 'checkpoint-response',
|
|
99
|
+
outcome,
|
|
100
|
+
score,
|
|
101
|
+
insight: `User responded: "${entry.message.content.slice(0, 100)}"`,
|
|
102
|
+
sessionFile: file,
|
|
103
|
+
timestamp: entry.timestamp ?? new Date().toISOString(),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
} catch {
|
|
108
|
+
// Skip malformed JSONL lines
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return patterns;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Convert a DreamPattern to a MemoryRecord for storage via any MemoryBackend. */
|
|
117
|
+
export function dreamPatternToRecord(pattern: DreamPattern): MemoryRecord {
|
|
118
|
+
return {
|
|
119
|
+
id: `dream:${pattern.sessionFile}:${pattern.skillId}:${Date.now()}`,
|
|
120
|
+
skillId: pattern.skillId,
|
|
121
|
+
text: pattern.insight,
|
|
122
|
+
score: pattern.score,
|
|
123
|
+
outcome: pattern.outcome,
|
|
124
|
+
timestamp: pattern.timestamp,
|
|
125
|
+
metadata: { sessionFile: pattern.sessionFile },
|
|
126
|
+
};
|
|
127
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dzhechkov/memory` — the harness memory layer.
|
|
3
|
+
*
|
|
4
|
+
* @packageDocumentation
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Package version. Kept in sync with `package.json`. */
|
|
8
|
+
export const MEMORY_VERSION = '0.1.0';
|
|
9
|
+
|
|
10
|
+
export type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
|
|
11
|
+
export { JsonFileBackend } from './json-backend.js';
|
|
12
|
+
export type { JsonFileBackendOptions } from './json-backend.js';
|
|
13
|
+
export { selectBackend } from './cascade.js';
|
|
14
|
+
export type { BackendProbe, CascadeResult } from './cascade.js';
|
|
15
|
+
export { SqliteBackend } from './sqlite-backend.js';
|
|
16
|
+
export type { SqliteBackendOptions } from './sqlite-backend.js';
|
|
17
|
+
export { SqliteProbe } from './sqlite-probe.js';
|
|
18
|
+
export type { SqliteProbeOptions } from './sqlite-probe.js';
|
|
19
|
+
export { Reflexion } from './reflexion.js';
|
|
20
|
+
export type { ReflexionInput } from './reflexion.js';
|
|
21
|
+
export { importMemoryMarkdown, MemoryBridge } from './bridge.js';
|
|
22
|
+
export type { BridgeOptions } from './bridge.js';
|
|
23
|
+
export { harvestDreamPatterns, dreamPatternToRecord } from './dreaming.js';
|
|
24
|
+
export type { DreamPattern, DreamOptions } from './dreaming.js';
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `JsonFileBackend` — the default memory backend.
|
|
3
|
+
*
|
|
4
|
+
* Pure JavaScript, zero runtime dependencies: records live in an in-memory map
|
|
5
|
+
* and persist to a JSON file. Retrieval is scored keyword overlap. No native
|
|
6
|
+
* build, no WASM, no model download — it works everywhere.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { dirname } from 'node:path';
|
|
13
|
+
|
|
14
|
+
import type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
|
|
15
|
+
|
|
16
|
+
const DEFAULT_LIMIT = 20;
|
|
17
|
+
|
|
18
|
+
/** Split text into lowercase word tokens of length > 1. */
|
|
19
|
+
function tokenize(text: string): string[] {
|
|
20
|
+
return text
|
|
21
|
+
.toLowerCase()
|
|
22
|
+
.split(/[^a-z0-9]+/)
|
|
23
|
+
.filter((token) => token.length > 1);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Count how many query terms appear in a record's text/skillId. */
|
|
27
|
+
function relevanceOf(record: MemoryRecord, terms: readonly string[]): number {
|
|
28
|
+
if (terms.length === 0) return 0;
|
|
29
|
+
const haystack = new Set(tokenize(`${record.text} ${record.skillId}`));
|
|
30
|
+
let hits = 0;
|
|
31
|
+
for (const term of terms) {
|
|
32
|
+
if (haystack.has(term)) hits += 1;
|
|
33
|
+
}
|
|
34
|
+
return hits;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Options for {@link JsonFileBackend}. */
|
|
38
|
+
export interface JsonFileBackendOptions {
|
|
39
|
+
/** File the records persist to. Omit for an in-memory-only backend. */
|
|
40
|
+
readonly filePath?: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The default memory backend — in-memory map with optional JSON-file persistence. */
|
|
44
|
+
export class JsonFileBackend implements MemoryBackend {
|
|
45
|
+
readonly name = 'json-file';
|
|
46
|
+
|
|
47
|
+
private readonly records = new Map<string, MemoryRecord>();
|
|
48
|
+
private readonly filePath: string | undefined;
|
|
49
|
+
|
|
50
|
+
constructor(options: JsonFileBackendOptions = {}) {
|
|
51
|
+
this.filePath = options.filePath;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Create a backend and load any records already persisted at `filePath`. */
|
|
55
|
+
static async open(filePath: string): Promise<JsonFileBackend> {
|
|
56
|
+
const backend = new JsonFileBackend({ filePath });
|
|
57
|
+
await backend.load();
|
|
58
|
+
return backend;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
put(record: MemoryRecord): Promise<void> {
|
|
62
|
+
this.records.set(record.id, record);
|
|
63
|
+
return Promise.resolve();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
query(query: MemoryQuery): Promise<MemoryRecord[]> {
|
|
67
|
+
const limit = query.limit ?? DEFAULT_LIMIT;
|
|
68
|
+
const terms = query.text !== undefined ? tokenize(query.text) : [];
|
|
69
|
+
let candidates = [...this.records.values()];
|
|
70
|
+
if (query.skillId !== undefined) {
|
|
71
|
+
candidates = candidates.filter((record) => record.skillId === query.skillId);
|
|
72
|
+
}
|
|
73
|
+
const ranked = candidates
|
|
74
|
+
.map((record) => ({ record, relevance: relevanceOf(record, terms) }))
|
|
75
|
+
.sort(
|
|
76
|
+
(a, b) =>
|
|
77
|
+
b.relevance - a.relevance ||
|
|
78
|
+
b.record.score - a.record.score ||
|
|
79
|
+
b.record.timestamp.localeCompare(a.record.timestamp),
|
|
80
|
+
);
|
|
81
|
+
return Promise.resolve(ranked.slice(0, limit).map((entry) => entry.record));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
all(): Promise<MemoryRecord[]> {
|
|
85
|
+
return Promise.resolve([...this.records.values()]);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
count(): Promise<number> {
|
|
89
|
+
return Promise.resolve(this.records.size);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Persist every record to `filePath`. No-op when no path is configured. */
|
|
93
|
+
save(): Promise<void> {
|
|
94
|
+
if (this.filePath !== undefined) {
|
|
95
|
+
mkdirSync(dirname(this.filePath), { recursive: true });
|
|
96
|
+
writeFileSync(this.filePath, JSON.stringify([...this.records.values()], null, 2));
|
|
97
|
+
}
|
|
98
|
+
return Promise.resolve();
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Load records from `filePath`. No-op when no path is set or the file is absent. */
|
|
102
|
+
load(): Promise<void> {
|
|
103
|
+
if (this.filePath !== undefined && existsSync(this.filePath)) {
|
|
104
|
+
const data = JSON.parse(readFileSync(this.filePath, 'utf-8')) as MemoryRecord[];
|
|
105
|
+
for (const record of data) this.records.set(record.id, record);
|
|
106
|
+
}
|
|
107
|
+
return Promise.resolve();
|
|
108
|
+
}
|
|
109
|
+
}
|
package/src/reflexion.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Reflexion` — the skill-outcome feedback loop.
|
|
3
|
+
*
|
|
4
|
+
* Records the outcome of using a skill (`record(skillId, outcome, score)`) and
|
|
5
|
+
* ranks skills by their most recent score. Reads are **monotonic**: the latest
|
|
6
|
+
* record for a skill supersedes older ones.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { MemoryBackend, MemoryRecord } from './backend.js';
|
|
12
|
+
|
|
13
|
+
/** Monotonic counter — keeps record ids unique within a process. */
|
|
14
|
+
let sequence = 0;
|
|
15
|
+
|
|
16
|
+
/** Optional extras for {@link Reflexion.record}. */
|
|
17
|
+
export interface ReflexionInput {
|
|
18
|
+
/** Free text to store (defaults to `"<skillId> <outcome>"`). */
|
|
19
|
+
readonly text?: string;
|
|
20
|
+
/** String-keyed metadata. */
|
|
21
|
+
readonly metadata?: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Records skill outcomes against a {@link MemoryBackend} and ranks skills. */
|
|
25
|
+
export class Reflexion {
|
|
26
|
+
constructor(private readonly backend: MemoryBackend) {}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Record the outcome of using a skill.
|
|
30
|
+
*
|
|
31
|
+
* @param score reward score, must be within `[0, 1]`.
|
|
32
|
+
* @throws if `score` is outside `[0, 1]`.
|
|
33
|
+
*/
|
|
34
|
+
async record(
|
|
35
|
+
skillId: string,
|
|
36
|
+
outcome: string,
|
|
37
|
+
score: number,
|
|
38
|
+
input: ReflexionInput = {},
|
|
39
|
+
): Promise<MemoryRecord> {
|
|
40
|
+
if (!Number.isFinite(score) || score < 0 || score > 1) {
|
|
41
|
+
throw new Error(`reflexion: score must be within [0, 1], got ${score}`);
|
|
42
|
+
}
|
|
43
|
+
sequence += 1;
|
|
44
|
+
const record: MemoryRecord = {
|
|
45
|
+
id: `reflexion:${skillId}:${Date.now()}:${sequence}`,
|
|
46
|
+
skillId,
|
|
47
|
+
text: input.text ?? `${skillId} ${outcome}`,
|
|
48
|
+
score,
|
|
49
|
+
outcome,
|
|
50
|
+
timestamp: new Date().toISOString(),
|
|
51
|
+
...(input.metadata !== undefined ? { metadata: input.metadata } : {}),
|
|
52
|
+
};
|
|
53
|
+
await this.backend.put(record);
|
|
54
|
+
return record;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** The most recently recorded score for a skill, or `undefined` if none. */
|
|
58
|
+
async scoreOf(skillId: string): Promise<number | undefined> {
|
|
59
|
+
const records = await this.backend.query({ skillId, limit: Number.MAX_SAFE_INTEGER });
|
|
60
|
+
if (records.length === 0) return undefined;
|
|
61
|
+
return records.reduce((latest, record) =>
|
|
62
|
+
record.timestamp >= latest.timestamp ? record : latest,
|
|
63
|
+
).score;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Every skill with a recorded outcome, ranked by most-recent score, descending. */
|
|
67
|
+
async ranking(): Promise<{ skillId: string; score: number }[]> {
|
|
68
|
+
const latest = new Map<string, MemoryRecord>();
|
|
69
|
+
for (const record of await this.backend.all()) {
|
|
70
|
+
const current = latest.get(record.skillId);
|
|
71
|
+
if (current === undefined || record.timestamp > current.timestamp) {
|
|
72
|
+
latest.set(record.skillId, record);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return [...latest.values()]
|
|
76
|
+
.map((record) => ({ skillId: record.skillId, score: record.score }))
|
|
77
|
+
.sort((a, b) => b.score - a.score);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `SqliteBackend` — production-scale memory backend using better-sqlite3.
|
|
3
|
+
*
|
|
4
|
+
* Write-through persistence (every `put` is durable), WAL mode for concurrency,
|
|
5
|
+
* and indexed columns for efficient queries. Handles 100k+ records where
|
|
6
|
+
* JsonFileBackend degrades.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { createRequire } from 'node:module';
|
|
12
|
+
import { mkdirSync } from 'node:fs';
|
|
13
|
+
import { dirname } from 'node:path';
|
|
14
|
+
|
|
15
|
+
import type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
|
|
16
|
+
|
|
17
|
+
const require = createRequire(import.meta.url);
|
|
18
|
+
|
|
19
|
+
const DEFAULT_LIMIT = 20;
|
|
20
|
+
|
|
21
|
+
/** Split text into lowercase word tokens of length > 1. */
|
|
22
|
+
function tokenize(text: string): string[] {
|
|
23
|
+
return text
|
|
24
|
+
.toLowerCase()
|
|
25
|
+
.split(/[^a-z0-9]+/)
|
|
26
|
+
.filter((token) => token.length > 1);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Count how many query terms appear in a record's text/skillId. */
|
|
30
|
+
function relevanceOf(record: MemoryRecord, terms: readonly string[]): number {
|
|
31
|
+
if (terms.length === 0) return 0;
|
|
32
|
+
const haystack = new Set(tokenize(`${record.text} ${record.skillId}`));
|
|
33
|
+
let hits = 0;
|
|
34
|
+
for (const term of terms) {
|
|
35
|
+
if (haystack.has(term)) hits += 1;
|
|
36
|
+
}
|
|
37
|
+
return hits;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Schema version for future migrations. */
|
|
41
|
+
const SCHEMA_VERSION = 2;
|
|
42
|
+
|
|
43
|
+
const INIT_SQL = `
|
|
44
|
+
CREATE TABLE IF NOT EXISTS memory_records (
|
|
45
|
+
id TEXT PRIMARY KEY,
|
|
46
|
+
skill_id TEXT NOT NULL,
|
|
47
|
+
text TEXT NOT NULL,
|
|
48
|
+
score REAL NOT NULL,
|
|
49
|
+
outcome TEXT NOT NULL,
|
|
50
|
+
timestamp TEXT NOT NULL,
|
|
51
|
+
metadata TEXT
|
|
52
|
+
);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_skill ON memory_records(skill_id);
|
|
54
|
+
CREATE INDEX IF NOT EXISTS idx_timestamp ON memory_records(timestamp);
|
|
55
|
+
PRAGMA user_version = ${SCHEMA_VERSION};
|
|
56
|
+
`;
|
|
57
|
+
|
|
58
|
+
/** FTS5 virtual table + triggers for automatic sync. */
|
|
59
|
+
const FTS5_SQL = `
|
|
60
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
|
|
61
|
+
text, skill_id, content=memory_records, content_rowid=rowid
|
|
62
|
+
);
|
|
63
|
+
CREATE TRIGGER IF NOT EXISTS memory_fts_insert AFTER INSERT ON memory_records BEGIN
|
|
64
|
+
INSERT INTO memory_fts(rowid, text, skill_id) VALUES (new.rowid, new.text, new.skill_id);
|
|
65
|
+
END;
|
|
66
|
+
CREATE TRIGGER IF NOT EXISTS memory_fts_delete AFTER DELETE ON memory_records BEGIN
|
|
67
|
+
INSERT INTO memory_fts(memory_fts, rowid, text, skill_id) VALUES ('delete', old.rowid, old.text, old.skill_id);
|
|
68
|
+
END;
|
|
69
|
+
CREATE TRIGGER IF NOT EXISTS memory_fts_update AFTER UPDATE ON memory_records BEGIN
|
|
70
|
+
INSERT INTO memory_fts(memory_fts, rowid, text, skill_id) VALUES ('delete', old.rowid, old.text, old.skill_id);
|
|
71
|
+
INSERT INTO memory_fts(rowid, text, skill_id) VALUES (new.rowid, new.text, new.skill_id);
|
|
72
|
+
END;
|
|
73
|
+
`;
|
|
74
|
+
|
|
75
|
+
/** FTS5 query — returns matching record ids ranked by relevance. */
|
|
76
|
+
const FTS5_SEARCH_SQL = `
|
|
77
|
+
SELECT mr.* FROM memory_fts fts
|
|
78
|
+
JOIN memory_records mr ON mr.rowid = fts.rowid
|
|
79
|
+
WHERE memory_fts MATCH ?
|
|
80
|
+
ORDER BY fts.rank
|
|
81
|
+
`;
|
|
82
|
+
|
|
83
|
+
const FTS5_SEARCH_SKILL_SQL = `
|
|
84
|
+
SELECT mr.* FROM memory_fts fts
|
|
85
|
+
JOIN memory_records mr ON mr.rowid = fts.rowid
|
|
86
|
+
WHERE memory_fts MATCH ? AND mr.skill_id = ?
|
|
87
|
+
ORDER BY fts.rank
|
|
88
|
+
`;
|
|
89
|
+
|
|
90
|
+
const UPSERT_SQL = `
|
|
91
|
+
INSERT OR REPLACE INTO memory_records (id, skill_id, text, score, outcome, timestamp, metadata)
|
|
92
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
93
|
+
`;
|
|
94
|
+
|
|
95
|
+
const ALL_SQL = 'SELECT * FROM memory_records';
|
|
96
|
+
const COUNT_SQL = 'SELECT COUNT(*) as cnt FROM memory_records';
|
|
97
|
+
const BY_SKILL_SQL = 'SELECT * FROM memory_records WHERE skill_id = ?';
|
|
98
|
+
|
|
99
|
+
/** Options for SqliteBackend. */
|
|
100
|
+
export interface SqliteBackendOptions {
|
|
101
|
+
/** Path to the SQLite database file. */
|
|
102
|
+
readonly filePath: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* SQLite-backed memory store. Write-through, WAL mode, indexed.
|
|
107
|
+
*
|
|
108
|
+
* Requires `better-sqlite3` at runtime — use via {@link SqliteProbe} in the
|
|
109
|
+
* cascade to gracefully fall back when the native module is unavailable.
|
|
110
|
+
*/
|
|
111
|
+
export class SqliteBackend implements MemoryBackend {
|
|
112
|
+
readonly name = 'sqlite';
|
|
113
|
+
|
|
114
|
+
private readonly db: any; // better-sqlite3 Database instance
|
|
115
|
+
private readonly upsertStmt: any;
|
|
116
|
+
private readonly allStmt: any;
|
|
117
|
+
private readonly countStmt: any;
|
|
118
|
+
private readonly bySkillStmt: any;
|
|
119
|
+
private readonly ftsSearchStmt: any | undefined;
|
|
120
|
+
private readonly ftsSearchSkillStmt: any | undefined;
|
|
121
|
+
private readonly hasFts5: boolean;
|
|
122
|
+
|
|
123
|
+
constructor(db: any) {
|
|
124
|
+
this.db = db;
|
|
125
|
+
db.exec(INIT_SQL);
|
|
126
|
+
|
|
127
|
+
// Try to enable FTS5 — gracefully degrade if unavailable
|
|
128
|
+
let ftsOk = false;
|
|
129
|
+
try {
|
|
130
|
+
db.exec(FTS5_SQL);
|
|
131
|
+
// Rebuild FTS index from existing data (idempotent)
|
|
132
|
+
db.exec(`INSERT INTO memory_fts(memory_fts) VALUES ('rebuild')`);
|
|
133
|
+
this.ftsSearchStmt = db.prepare(FTS5_SEARCH_SQL);
|
|
134
|
+
this.ftsSearchSkillStmt = db.prepare(FTS5_SEARCH_SKILL_SQL);
|
|
135
|
+
ftsOk = true;
|
|
136
|
+
} catch {
|
|
137
|
+
// FTS5 not compiled in — fall back to keyword overlap
|
|
138
|
+
}
|
|
139
|
+
this.hasFts5 = ftsOk;
|
|
140
|
+
|
|
141
|
+
this.upsertStmt = db.prepare(UPSERT_SQL);
|
|
142
|
+
this.allStmt = db.prepare(ALL_SQL);
|
|
143
|
+
this.countStmt = db.prepare(COUNT_SQL);
|
|
144
|
+
this.bySkillStmt = db.prepare(BY_SKILL_SQL);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Open (or create) a SQLite database at the given path. */
|
|
148
|
+
static open(filePath: string): SqliteBackend {
|
|
149
|
+
mkdirSync(dirname(filePath), { recursive: true });
|
|
150
|
+
// Dynamic require — better-sqlite3 must be available at runtime
|
|
151
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
152
|
+
const Database = require('better-sqlite3');
|
|
153
|
+
const db = new Database(filePath);
|
|
154
|
+
db.pragma('journal_mode = WAL');
|
|
155
|
+
db.pragma('synchronous = NORMAL');
|
|
156
|
+
return new SqliteBackend(db);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
put(record: MemoryRecord): Promise<void> {
|
|
160
|
+
this.upsertStmt.run(
|
|
161
|
+
record.id,
|
|
162
|
+
record.skillId,
|
|
163
|
+
record.text,
|
|
164
|
+
record.score,
|
|
165
|
+
record.outcome,
|
|
166
|
+
record.timestamp,
|
|
167
|
+
record.metadata ? JSON.stringify(record.metadata) : null,
|
|
168
|
+
);
|
|
169
|
+
return Promise.resolve();
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
query(query: MemoryQuery): Promise<MemoryRecord[]> {
|
|
173
|
+
const limit = query.limit ?? DEFAULT_LIMIT;
|
|
174
|
+
|
|
175
|
+
// FTS5 path — use SQLite full-text search when available and text query provided
|
|
176
|
+
if (this.hasFts5 && query.text !== undefined && query.text.trim().length > 0) {
|
|
177
|
+
try {
|
|
178
|
+
// FTS5 query syntax: simple terms joined by spaces (implicit AND → OR with ranking)
|
|
179
|
+
const ftsQuery = tokenize(query.text).join(' OR ');
|
|
180
|
+
if (ftsQuery.length > 0) {
|
|
181
|
+
let rows: any[];
|
|
182
|
+
if (query.skillId !== undefined) {
|
|
183
|
+
rows = this.ftsSearchSkillStmt!.all(ftsQuery, query.skillId);
|
|
184
|
+
} else {
|
|
185
|
+
rows = this.ftsSearchStmt!.all(ftsQuery);
|
|
186
|
+
}
|
|
187
|
+
const records = rows.map(rowToRecord);
|
|
188
|
+
// FTS5 rank is already applied; tiebreak by score DESC, timestamp DESC
|
|
189
|
+
records.sort(
|
|
190
|
+
(a, b) => b.score - a.score || b.timestamp.localeCompare(a.timestamp),
|
|
191
|
+
);
|
|
192
|
+
return Promise.resolve(records.slice(0, limit));
|
|
193
|
+
}
|
|
194
|
+
} catch {
|
|
195
|
+
// FTS5 query failed (e.g., special chars) — fall through to keyword approach
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Keyword overlap fallback
|
|
200
|
+
const terms = query.text !== undefined ? tokenize(query.text) : [];
|
|
201
|
+
let rows: any[];
|
|
202
|
+
if (query.skillId !== undefined) {
|
|
203
|
+
rows = this.bySkillStmt.all(query.skillId);
|
|
204
|
+
} else {
|
|
205
|
+
rows = this.allStmt.all();
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const records = rows.map(rowToRecord);
|
|
209
|
+
const ranked = records
|
|
210
|
+
.map((record) => ({ record, relevance: relevanceOf(record, terms) }))
|
|
211
|
+
.sort(
|
|
212
|
+
(a, b) =>
|
|
213
|
+
b.relevance - a.relevance ||
|
|
214
|
+
b.record.score - a.record.score ||
|
|
215
|
+
b.record.timestamp.localeCompare(a.record.timestamp),
|
|
216
|
+
);
|
|
217
|
+
return Promise.resolve(ranked.slice(0, limit).map((entry) => entry.record));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
all(): Promise<MemoryRecord[]> {
|
|
221
|
+
return Promise.resolve(this.allStmt.all().map(rowToRecord));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
count(): Promise<number> {
|
|
225
|
+
return Promise.resolve(this.countStmt.get().cnt);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Batch insert records within a transaction (for bulk loading). */
|
|
229
|
+
putMany(records: readonly MemoryRecord[]): void {
|
|
230
|
+
const insertMany = this.db.transaction((items: readonly MemoryRecord[]) => {
|
|
231
|
+
for (const record of items) {
|
|
232
|
+
this.upsertStmt.run(
|
|
233
|
+
record.id,
|
|
234
|
+
record.skillId,
|
|
235
|
+
record.text,
|
|
236
|
+
record.score,
|
|
237
|
+
record.outcome,
|
|
238
|
+
record.timestamp,
|
|
239
|
+
record.metadata ? JSON.stringify(record.metadata) : null,
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
insertMany(records);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Close the database connection. */
|
|
247
|
+
close(): void {
|
|
248
|
+
this.db.close();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Convert a raw SQLite row to a MemoryRecord. */
|
|
253
|
+
function rowToRecord(row: any): MemoryRecord {
|
|
254
|
+
return {
|
|
255
|
+
id: row.id,
|
|
256
|
+
skillId: row.skill_id,
|
|
257
|
+
text: row.text,
|
|
258
|
+
score: row.score,
|
|
259
|
+
outcome: row.outcome,
|
|
260
|
+
timestamp: row.timestamp,
|
|
261
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `SqliteProbe` — cascade probe for the SQLite backend.
|
|
3
|
+
*
|
|
4
|
+
* Attempts to load `better-sqlite3` and open the database. Returns `undefined`
|
|
5
|
+
* if the native module is unavailable (not installed, build failed, etc.).
|
|
6
|
+
*
|
|
7
|
+
* @packageDocumentation
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { BackendProbe } from './cascade.js';
|
|
11
|
+
import type { MemoryBackend } from './backend.js';
|
|
12
|
+
|
|
13
|
+
/** Options for SqliteProbe. */
|
|
14
|
+
export interface SqliteProbeOptions {
|
|
15
|
+
/** Path to the SQLite database file. Default: `.dz/memory.sqlite` */
|
|
16
|
+
readonly filePath?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A {@link BackendProbe} that tries to initialise a SQLite backend.
|
|
21
|
+
* Safe to construct unconditionally — if `better-sqlite3` is not installed,
|
|
22
|
+
* `create()` returns `undefined` and the cascade moves on.
|
|
23
|
+
*/
|
|
24
|
+
export class SqliteProbe implements BackendProbe {
|
|
25
|
+
readonly name = 'sqlite';
|
|
26
|
+
private readonly filePath: string;
|
|
27
|
+
|
|
28
|
+
constructor(options: SqliteProbeOptions = {}) {
|
|
29
|
+
this.filePath = options.filePath ?? '.dz/memory.sqlite';
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async create(): Promise<MemoryBackend | undefined> {
|
|
33
|
+
try {
|
|
34
|
+
// Attempt to load better-sqlite3 via the SqliteBackend.
|
|
35
|
+
// If the native module is not installed, this throws and we return undefined.
|
|
36
|
+
const { SqliteBackend } = await import('./sqlite-backend.js');
|
|
37
|
+
const backend = SqliteBackend.open(this.filePath);
|
|
38
|
+
return backend;
|
|
39
|
+
} catch {
|
|
40
|
+
// Module not available or build failed — fall through
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|