@psiclawops/hypermem 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +4 -3
- package/README.md +457 -174
- package/package.json +15 -5
- package/dist/background-indexer.d.ts +0 -117
- package/dist/background-indexer.d.ts.map +0 -1
- package/dist/background-indexer.js +0 -732
- package/dist/compaction-fence.d.ts +0 -89
- package/dist/compaction-fence.d.ts.map +0 -1
- package/dist/compaction-fence.js +0 -153
- package/dist/compositor.d.ts +0 -139
- package/dist/compositor.d.ts.map +0 -1
- package/dist/compositor.js +0 -1109
- package/dist/cross-agent.d.ts +0 -57
- package/dist/cross-agent.d.ts.map +0 -1
- package/dist/cross-agent.js +0 -254
- package/dist/db.d.ts +0 -131
- package/dist/db.d.ts.map +0 -1
- package/dist/db.js +0 -398
- package/dist/desired-state-store.d.ts +0 -100
- package/dist/desired-state-store.d.ts.map +0 -1
- package/dist/desired-state-store.js +0 -212
- package/dist/doc-chunk-store.d.ts +0 -115
- package/dist/doc-chunk-store.d.ts.map +0 -1
- package/dist/doc-chunk-store.js +0 -278
- package/dist/doc-chunker.d.ts +0 -99
- package/dist/doc-chunker.d.ts.map +0 -1
- package/dist/doc-chunker.js +0 -324
- package/dist/episode-store.d.ts +0 -48
- package/dist/episode-store.d.ts.map +0 -1
- package/dist/episode-store.js +0 -135
- package/dist/fact-store.d.ts +0 -57
- package/dist/fact-store.d.ts.map +0 -1
- package/dist/fact-store.js +0 -175
- package/dist/fleet-store.d.ts +0 -144
- package/dist/fleet-store.d.ts.map +0 -1
- package/dist/fleet-store.js +0 -276
- package/dist/hybrid-retrieval.d.ts +0 -60
- package/dist/hybrid-retrieval.d.ts.map +0 -1
- package/dist/hybrid-retrieval.js +0 -340
- package/dist/index.d.ts +0 -611
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -1042
- package/dist/knowledge-graph.d.ts +0 -110
- package/dist/knowledge-graph.d.ts.map +0 -1
- package/dist/knowledge-graph.js +0 -305
- package/dist/knowledge-store.d.ts +0 -72
- package/dist/knowledge-store.d.ts.map +0 -1
- package/dist/knowledge-store.js +0 -241
- package/dist/library-schema.d.ts +0 -22
- package/dist/library-schema.d.ts.map +0 -1
- package/dist/library-schema.js +0 -717
- package/dist/message-store.d.ts +0 -76
- package/dist/message-store.d.ts.map +0 -1
- package/dist/message-store.js +0 -273
- package/dist/preference-store.d.ts +0 -54
- package/dist/preference-store.d.ts.map +0 -1
- package/dist/preference-store.js +0 -109
- package/dist/preservation-gate.d.ts +0 -82
- package/dist/preservation-gate.d.ts.map +0 -1
- package/dist/preservation-gate.js +0 -150
- package/dist/provider-translator.d.ts +0 -40
- package/dist/provider-translator.d.ts.map +0 -1
- package/dist/provider-translator.js +0 -349
- package/dist/rate-limiter.d.ts +0 -76
- package/dist/rate-limiter.d.ts.map +0 -1
- package/dist/rate-limiter.js +0 -179
- package/dist/redis.d.ts +0 -188
- package/dist/redis.d.ts.map +0 -1
- package/dist/redis.js +0 -534
- package/dist/schema.d.ts +0 -15
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js +0 -203
- package/dist/secret-scanner.d.ts +0 -51
- package/dist/secret-scanner.d.ts.map +0 -1
- package/dist/secret-scanner.js +0 -248
- package/dist/seed.d.ts +0 -108
- package/dist/seed.d.ts.map +0 -1
- package/dist/seed.js +0 -177
- package/dist/system-store.d.ts +0 -73
- package/dist/system-store.d.ts.map +0 -1
- package/dist/system-store.js +0 -182
- package/dist/topic-store.d.ts +0 -45
- package/dist/topic-store.d.ts.map +0 -1
- package/dist/topic-store.js +0 -136
- package/dist/types.d.ts +0 -329
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -9
- package/dist/vector-store.d.ts +0 -132
- package/dist/vector-store.d.ts.map +0 -1
- package/dist/vector-store.js +0 -498
- package/dist/work-store.d.ts +0 -112
- package/dist/work-store.d.ts.map +0 -1
- package/dist/work-store.js +0 -273
|
@@ -1,212 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HyperMem Agent Desired State Store
|
|
3
|
-
*
|
|
4
|
-
* Stores intended configuration for each agent and tracks drift.
|
|
5
|
-
* Enables fleet-wide config visibility and enforcement.
|
|
6
|
-
*
|
|
7
|
-
* Config keys are dot-path strings matching openclaw.json structure:
|
|
8
|
-
* model, thinkingDefault, provider, workspace, tools.exec.host, etc.
|
|
9
|
-
*
|
|
10
|
-
* Drift statuses:
|
|
11
|
-
* - 'ok' — actual matches desired
|
|
12
|
-
* - 'drifted' — actual differs from desired
|
|
13
|
-
* - 'unknown' — not yet checked
|
|
14
|
-
* - 'error' — check failed
|
|
15
|
-
*/
|
|
16
|
-
function nowIso() {
|
|
17
|
-
return new Date().toISOString();
|
|
18
|
-
}
|
|
19
|
-
function parseEntry(row) {
|
|
20
|
-
return {
|
|
21
|
-
agentId: row.agent_id,
|
|
22
|
-
configKey: row.config_key,
|
|
23
|
-
desiredValue: row.desired_value ? JSON.parse(row.desired_value) : null,
|
|
24
|
-
actualValue: row.actual_value ? JSON.parse(row.actual_value) : null,
|
|
25
|
-
source: row.source,
|
|
26
|
-
setBy: row.set_by || null,
|
|
27
|
-
driftStatus: row.drift_status || 'unknown',
|
|
28
|
-
lastChecked: row.last_checked || null,
|
|
29
|
-
createdAt: row.created_at,
|
|
30
|
-
updatedAt: row.updated_at,
|
|
31
|
-
notes: row.notes || null,
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
function parseEvent(row) {
|
|
35
|
-
return {
|
|
36
|
-
id: row.id,
|
|
37
|
-
agentId: row.agent_id,
|
|
38
|
-
configKey: row.config_key,
|
|
39
|
-
eventType: row.event_type,
|
|
40
|
-
oldValue: row.old_value ? JSON.parse(row.old_value) : null,
|
|
41
|
-
newValue: row.new_value ? JSON.parse(row.new_value) : null,
|
|
42
|
-
changedBy: row.changed_by || null,
|
|
43
|
-
createdAt: row.created_at,
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
export class DesiredStateStore {
|
|
47
|
-
db;
|
|
48
|
-
constructor(db) {
|
|
49
|
-
this.db = db;
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Set desired state for a config key on an agent.
|
|
53
|
-
*/
|
|
54
|
-
setDesired(agentId, configKey, desiredValue, opts) {
|
|
55
|
-
const now = nowIso();
|
|
56
|
-
const valueJson = JSON.stringify(desiredValue);
|
|
57
|
-
const existing = this.getEntry(agentId, configKey);
|
|
58
|
-
if (existing) {
|
|
59
|
-
// Record change event
|
|
60
|
-
this.db.prepare(`
|
|
61
|
-
INSERT INTO agent_config_events (agent_id, config_key, event_type, old_value, new_value, changed_by, created_at)
|
|
62
|
-
VALUES (?, ?, 'desired_changed', ?, ?, ?, ?)
|
|
63
|
-
`).run(agentId, configKey, JSON.stringify(existing.desiredValue), valueJson, opts?.setBy || null, now);
|
|
64
|
-
this.db.prepare(`
|
|
65
|
-
UPDATE agent_desired_state SET
|
|
66
|
-
desired_value = ?,
|
|
67
|
-
source = COALESCE(?, source),
|
|
68
|
-
set_by = COALESCE(?, set_by),
|
|
69
|
-
drift_status = 'unknown',
|
|
70
|
-
notes = COALESCE(?, notes),
|
|
71
|
-
updated_at = ?
|
|
72
|
-
WHERE agent_id = ? AND config_key = ?
|
|
73
|
-
`).run(valueJson, opts?.source || null, opts?.setBy || null, opts?.notes || null, now, agentId, configKey);
|
|
74
|
-
}
|
|
75
|
-
else {
|
|
76
|
-
this.db.prepare(`
|
|
77
|
-
INSERT INTO agent_desired_state (agent_id, config_key, desired_value, source, set_by, drift_status, created_at, updated_at, notes)
|
|
78
|
-
VALUES (?, ?, ?, ?, ?, 'unknown', ?, ?, ?)
|
|
79
|
-
`).run(agentId, configKey, valueJson, opts?.source || 'operator', opts?.setBy || null, now, now, opts?.notes || null);
|
|
80
|
-
// Record creation event
|
|
81
|
-
this.db.prepare(`
|
|
82
|
-
INSERT INTO agent_config_events (agent_id, config_key, event_type, new_value, changed_by, created_at)
|
|
83
|
-
VALUES (?, ?, 'desired_set', ?, ?, ?)
|
|
84
|
-
`).run(agentId, configKey, valueJson, opts?.setBy || null, now);
|
|
85
|
-
}
|
|
86
|
-
return this.getEntry(agentId, configKey);
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Report actual state observed at runtime.
|
|
90
|
-
* Compares against desired and updates drift status.
|
|
91
|
-
*/
|
|
92
|
-
reportActual(agentId, configKey, actualValue) {
|
|
93
|
-
const now = nowIso();
|
|
94
|
-
const actualJson = JSON.stringify(actualValue);
|
|
95
|
-
const entry = this.getEntry(agentId, configKey);
|
|
96
|
-
if (!entry)
|
|
97
|
-
return 'unknown';
|
|
98
|
-
const desiredJson = JSON.stringify(entry.desiredValue);
|
|
99
|
-
const driftStatus = actualJson === desiredJson ? 'ok' : 'drifted';
|
|
100
|
-
this.db.prepare(`
|
|
101
|
-
UPDATE agent_desired_state SET
|
|
102
|
-
actual_value = ?,
|
|
103
|
-
drift_status = ?,
|
|
104
|
-
last_checked = ?,
|
|
105
|
-
updated_at = ?
|
|
106
|
-
WHERE agent_id = ? AND config_key = ?
|
|
107
|
-
`).run(actualJson, driftStatus, now, now, agentId, configKey);
|
|
108
|
-
if (driftStatus === 'drifted' && actualJson !== JSON.stringify(entry.actualValue)) {
|
|
109
|
-
this.db.prepare(`
|
|
110
|
-
INSERT INTO agent_config_events (agent_id, config_key, event_type, old_value, new_value, changed_by, created_at)
|
|
111
|
-
VALUES (?, ?, 'drift_detected', ?, ?, 'system', ?)
|
|
112
|
-
`).run(agentId, configKey, JSON.stringify(entry.actualValue), actualJson, now);
|
|
113
|
-
}
|
|
114
|
-
return driftStatus;
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Bulk report actual state for an agent (e.g., on heartbeat).
|
|
118
|
-
*/
|
|
119
|
-
reportActualBulk(agentId, actuals) {
|
|
120
|
-
const results = {};
|
|
121
|
-
for (const [key, value] of Object.entries(actuals)) {
|
|
122
|
-
results[key] = this.reportActual(agentId, key, value);
|
|
123
|
-
}
|
|
124
|
-
return results;
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Get a specific desired state entry.
|
|
128
|
-
*/
|
|
129
|
-
getEntry(agentId, configKey) {
|
|
130
|
-
const row = this.db.prepare('SELECT * FROM agent_desired_state WHERE agent_id = ? AND config_key = ?').get(agentId, configKey);
|
|
131
|
-
return row ? parseEntry(row) : null;
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* Get all desired state for an agent.
|
|
135
|
-
*/
|
|
136
|
-
getAgentState(agentId) {
|
|
137
|
-
const rows = this.db.prepare('SELECT * FROM agent_desired_state WHERE agent_id = ? ORDER BY config_key').all(agentId);
|
|
138
|
-
return rows.map(parseEntry);
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* Get desired state as a flat config object (key → value).
|
|
142
|
-
*/
|
|
143
|
-
getAgentConfig(agentId) {
|
|
144
|
-
const entries = this.getAgentState(agentId);
|
|
145
|
-
const config = {};
|
|
146
|
-
for (const entry of entries) {
|
|
147
|
-
config[entry.configKey] = entry.desiredValue;
|
|
148
|
-
}
|
|
149
|
-
return config;
|
|
150
|
-
}
|
|
151
|
-
/**
|
|
152
|
-
* Get all drifted entries across the fleet.
|
|
153
|
-
*/
|
|
154
|
-
getDrifted() {
|
|
155
|
-
const rows = this.db.prepare("SELECT * FROM agent_desired_state WHERE drift_status = 'drifted' ORDER BY agent_id, config_key").all();
|
|
156
|
-
return rows.map(parseEntry);
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Get fleet-wide view of a specific config key.
|
|
160
|
-
*/
|
|
161
|
-
getFleetConfig(configKey) {
|
|
162
|
-
const rows = this.db.prepare('SELECT * FROM agent_desired_state WHERE config_key = ? ORDER BY agent_id').all(configKey);
|
|
163
|
-
return rows.map(parseEntry);
|
|
164
|
-
}
|
|
165
|
-
/**
|
|
166
|
-
* Get config change history for an agent/key.
|
|
167
|
-
*/
|
|
168
|
-
getHistory(agentId, configKey, limit = 20) {
|
|
169
|
-
let sql = 'SELECT * FROM agent_config_events WHERE agent_id = ?';
|
|
170
|
-
const params = [agentId];
|
|
171
|
-
if (configKey) {
|
|
172
|
-
sql += ' AND config_key = ?';
|
|
173
|
-
params.push(configKey);
|
|
174
|
-
}
|
|
175
|
-
sql += ' ORDER BY created_at DESC LIMIT ?';
|
|
176
|
-
params.push(limit);
|
|
177
|
-
const rows = this.db.prepare(sql).all(...params);
|
|
178
|
-
return rows.map(parseEvent);
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Remove a desired state entry.
|
|
182
|
-
*/
|
|
183
|
-
removeDesired(agentId, configKey, removedBy) {
|
|
184
|
-
const now = nowIso();
|
|
185
|
-
const existing = this.getEntry(agentId, configKey);
|
|
186
|
-
if (existing) {
|
|
187
|
-
this.db.prepare(`
|
|
188
|
-
INSERT INTO agent_config_events (agent_id, config_key, event_type, old_value, changed_by, created_at)
|
|
189
|
-
VALUES (?, ?, 'desired_removed', ?, ?, ?)
|
|
190
|
-
`).run(agentId, configKey, JSON.stringify(existing.desiredValue), removedBy || null, now);
|
|
191
|
-
this.db.prepare('DELETE FROM agent_desired_state WHERE agent_id = ? AND config_key = ?').run(agentId, configKey);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Get fleet drift summary.
|
|
196
|
-
*/
|
|
197
|
-
getDriftSummary() {
|
|
198
|
-
const rows = this.db.prepare(`
|
|
199
|
-
SELECT drift_status, COUNT(*) as count FROM agent_desired_state GROUP BY drift_status
|
|
200
|
-
`).all();
|
|
201
|
-
const summary = { total: 0, ok: 0, drifted: 0, unknown: 0, error: 0 };
|
|
202
|
-
for (const row of rows) {
|
|
203
|
-
const count = row.count;
|
|
204
|
-
summary.total += count;
|
|
205
|
-
if (row.drift_status in summary) {
|
|
206
|
-
summary[row.drift_status] = count;
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
return summary;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
//# sourceMappingURL=desired-state-store.js.map
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HyperMem Document Chunk Store
|
|
3
|
-
*
|
|
4
|
-
* Manages doc_chunks in library.db:
|
|
5
|
-
* - Atomic re-indexing by source hash (no stale/fresh coexistence)
|
|
6
|
-
* - FTS5 keyword search fallback
|
|
7
|
-
* - Collection-scoped queries with agent/tier filtering
|
|
8
|
-
* - Source tracking (what's indexed, when, what hash)
|
|
9
|
-
*/
|
|
10
|
-
import type { DatabaseSync } from 'node:sqlite';
|
|
11
|
-
import type { DocChunk } from './doc-chunker.js';
|
|
12
|
-
export interface DocChunkRow {
|
|
13
|
-
id: string;
|
|
14
|
-
collection: string;
|
|
15
|
-
sectionPath: string;
|
|
16
|
-
depth: number;
|
|
17
|
-
content: string;
|
|
18
|
-
tokenEstimate: number;
|
|
19
|
-
sourceHash: string;
|
|
20
|
-
sourcePath: string;
|
|
21
|
-
scope: string;
|
|
22
|
-
tier: string | null;
|
|
23
|
-
agentId: string | null;
|
|
24
|
-
parentPath: string | null;
|
|
25
|
-
createdAt: string;
|
|
26
|
-
updatedAt: string;
|
|
27
|
-
}
|
|
28
|
-
export interface DocSourceRow {
|
|
29
|
-
sourcePath: string;
|
|
30
|
-
collection: string;
|
|
31
|
-
scope: string;
|
|
32
|
-
agentId: string | null;
|
|
33
|
-
sourceHash: string;
|
|
34
|
-
chunkCount: number;
|
|
35
|
-
indexedAt: string;
|
|
36
|
-
}
|
|
37
|
-
export interface IndexResult {
|
|
38
|
-
/** Number of new chunks inserted */
|
|
39
|
-
inserted: number;
|
|
40
|
-
/** Number of stale chunks deleted (from prior version) */
|
|
41
|
-
deleted: number;
|
|
42
|
-
/** Whether this was a full re-index (hash changed) or a no-op (hash unchanged) */
|
|
43
|
-
reindexed: boolean;
|
|
44
|
-
/** Whether this source was already up-to-date */
|
|
45
|
-
skipped: boolean;
|
|
46
|
-
}
|
|
47
|
-
export interface ChunkQuery {
|
|
48
|
-
/** Collection path to query */
|
|
49
|
-
collection: string;
|
|
50
|
-
/** Filter by scope */
|
|
51
|
-
scope?: string;
|
|
52
|
-
/** Filter by agent ID (for per-agent chunks) */
|
|
53
|
-
agentId?: string;
|
|
54
|
-
/** Filter by tier (for per-tier chunks) */
|
|
55
|
-
tier?: string;
|
|
56
|
-
/** Max number of chunks to return */
|
|
57
|
-
limit?: number;
|
|
58
|
-
/** Keyword search (FTS5) */
|
|
59
|
-
keyword?: string;
|
|
60
|
-
}
|
|
61
|
-
export declare class DocChunkStore {
|
|
62
|
-
private db;
|
|
63
|
-
constructor(db: DatabaseSync);
|
|
64
|
-
/**
|
|
65
|
-
* Index a set of chunks for a source file.
|
|
66
|
-
*
|
|
67
|
-
* Atomic re-indexing:
|
|
68
|
-
* 1. Check if source_hash has changed
|
|
69
|
-
* 2. If unchanged: skip (idempotent)
|
|
70
|
-
* 3. If changed: delete all chunks with old hash, insert new chunks — in one transaction
|
|
71
|
-
*
|
|
72
|
-
* This ensures no window where stale and fresh chunks coexist.
|
|
73
|
-
*/
|
|
74
|
-
indexChunks(chunks: DocChunk[]): IndexResult;
|
|
75
|
-
/**
|
|
76
|
-
* Query chunks by collection with optional filters.
|
|
77
|
-
* Falls back to FTS5 keyword search when keyword is provided.
|
|
78
|
-
*/
|
|
79
|
-
queryChunks(query: ChunkQuery): DocChunkRow[];
|
|
80
|
-
/**
|
|
81
|
-
* FTS5 keyword search across chunks.
|
|
82
|
-
*/
|
|
83
|
-
keywordSearch(keyword: string, query: Omit<ChunkQuery, 'keyword'>): DocChunkRow[];
|
|
84
|
-
/**
|
|
85
|
-
* Get a single chunk by ID.
|
|
86
|
-
*/
|
|
87
|
-
getChunk(id: string): DocChunkRow | null;
|
|
88
|
-
/**
|
|
89
|
-
* Check if a source file needs re-indexing.
|
|
90
|
-
* Returns true if the file has changed or has never been indexed.
|
|
91
|
-
*/
|
|
92
|
-
needsReindex(sourcePath: string, collection: string, currentHash: string): boolean;
|
|
93
|
-
/**
|
|
94
|
-
* List all indexed sources, optionally filtered by agent or collection.
|
|
95
|
-
*/
|
|
96
|
-
listSources(opts?: {
|
|
97
|
-
agentId?: string;
|
|
98
|
-
collection?: string;
|
|
99
|
-
}): DocSourceRow[];
|
|
100
|
-
/**
|
|
101
|
-
* Delete all chunks for a specific source file.
|
|
102
|
-
*/
|
|
103
|
-
deleteSource(sourcePath: string, collection: string): number;
|
|
104
|
-
/**
|
|
105
|
-
* Get chunk stats: count per collection.
|
|
106
|
-
*/
|
|
107
|
-
getStats(): Array<{
|
|
108
|
-
collection: string;
|
|
109
|
-
count: number;
|
|
110
|
-
sources: number;
|
|
111
|
-
totalTokens: number;
|
|
112
|
-
}>;
|
|
113
|
-
private mapRow;
|
|
114
|
-
}
|
|
115
|
-
//# sourceMappingURL=doc-chunk-store.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"doc-chunk-store.d.ts","sourceRoot":"","sources":["../src/doc-chunk-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAiB,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAIjD,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,kFAAkF;IAClF,SAAS,EAAE,OAAO,CAAC;IACnB,iDAAiD;IACjD,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAID,qBAAa,aAAa;IACZ,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,YAAY;IAEpC;;;;;;;;;OASG;IACH,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,WAAW;IAmF5C;;;OAGG;IACH,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,EAAE;IA2C7C;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;IAgDjF;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAYxC;;;OAGG;IACH,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO;IAOlF;;OAEG;IACH,WAAW,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,YAAY,EAAE;IAmC7E;;OAEG;IACH,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;IAgB5D;;OAEG;IACH,QAAQ,IAAI,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAmB9F,OAAO,CAAC,MAAM;CAkBf"}
|
package/dist/doc-chunk-store.js
DELETED
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HyperMem Document Chunk Store
|
|
3
|
-
*
|
|
4
|
-
* Manages doc_chunks in library.db:
|
|
5
|
-
* - Atomic re-indexing by source hash (no stale/fresh coexistence)
|
|
6
|
-
* - FTS5 keyword search fallback
|
|
7
|
-
* - Collection-scoped queries with agent/tier filtering
|
|
8
|
-
* - Source tracking (what's indexed, when, what hash)
|
|
9
|
-
*/
|
|
10
|
-
// ─── Store ──────────────────────────────────────────────────────
|
|
11
|
-
export class DocChunkStore {
|
|
12
|
-
db;
|
|
13
|
-
constructor(db) {
|
|
14
|
-
this.db = db;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Index a set of chunks for a source file.
|
|
18
|
-
*
|
|
19
|
-
* Atomic re-indexing:
|
|
20
|
-
* 1. Check if source_hash has changed
|
|
21
|
-
* 2. If unchanged: skip (idempotent)
|
|
22
|
-
* 3. If changed: delete all chunks with old hash, insert new chunks — in one transaction
|
|
23
|
-
*
|
|
24
|
-
* This ensures no window where stale and fresh chunks coexist.
|
|
25
|
-
*/
|
|
26
|
-
indexChunks(chunks) {
|
|
27
|
-
if (chunks.length === 0) {
|
|
28
|
-
return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
|
|
29
|
-
}
|
|
30
|
-
const first = chunks[0];
|
|
31
|
-
const { sourcePath, collection, sourceHash, scope, agentId } = first;
|
|
32
|
-
const now = new Date().toISOString();
|
|
33
|
-
// Check current indexed state
|
|
34
|
-
const existing = this.db
|
|
35
|
-
.prepare('SELECT source_hash, chunk_count FROM doc_sources WHERE source_path = ? AND collection = ?')
|
|
36
|
-
.get(sourcePath, collection);
|
|
37
|
-
if (existing && existing.source_hash === sourceHash) {
|
|
38
|
-
// Hash unchanged — no-op
|
|
39
|
-
return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
|
|
40
|
-
}
|
|
41
|
-
// Hash changed (or first index) — atomic swap
|
|
42
|
-
let deleted = 0;
|
|
43
|
-
let inserted = 0;
|
|
44
|
-
// Use a transaction for atomicity
|
|
45
|
-
const run = this.db.prepare('SELECT 1').get; // warm
|
|
46
|
-
try {
|
|
47
|
-
// Begin transaction via exec
|
|
48
|
-
this.db.exec('BEGIN');
|
|
49
|
-
// Delete stale chunks for this source
|
|
50
|
-
if (existing) {
|
|
51
|
-
const result = this.db
|
|
52
|
-
.prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
|
|
53
|
-
.run(sourcePath, collection);
|
|
54
|
-
deleted = result.changes;
|
|
55
|
-
}
|
|
56
|
-
// Insert new chunks
|
|
57
|
-
const insertChunk = this.db.prepare(`
|
|
58
|
-
INSERT OR REPLACE INTO doc_chunks
|
|
59
|
-
(id, collection, section_path, depth, content, token_estimate,
|
|
60
|
-
source_hash, source_path, scope, tier, agent_id, parent_path,
|
|
61
|
-
created_at, updated_at)
|
|
62
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
63
|
-
`);
|
|
64
|
-
for (const chunk of chunks) {
|
|
65
|
-
insertChunk.run(chunk.id, chunk.collection, chunk.sectionPath, chunk.depth, chunk.content, chunk.tokenEstimate, chunk.sourceHash, chunk.sourcePath, chunk.scope, chunk.tier ?? null, chunk.agentId ?? null, chunk.parentPath ?? null, now, now);
|
|
66
|
-
inserted++;
|
|
67
|
-
}
|
|
68
|
-
// Update source tracking
|
|
69
|
-
this.db.prepare(`
|
|
70
|
-
INSERT OR REPLACE INTO doc_sources
|
|
71
|
-
(source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at)
|
|
72
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
73
|
-
`).run(sourcePath, collection, scope, agentId ?? null, sourceHash, inserted, now);
|
|
74
|
-
this.db.exec('COMMIT');
|
|
75
|
-
}
|
|
76
|
-
catch (err) {
|
|
77
|
-
this.db.exec('ROLLBACK');
|
|
78
|
-
throw err;
|
|
79
|
-
}
|
|
80
|
-
return { inserted, deleted, reindexed: !!existing, skipped: false };
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Query chunks by collection with optional filters.
|
|
84
|
-
* Falls back to FTS5 keyword search when keyword is provided.
|
|
85
|
-
*/
|
|
86
|
-
queryChunks(query) {
|
|
87
|
-
const { collection, scope, agentId, tier, limit = 20, keyword } = query;
|
|
88
|
-
if (keyword) {
|
|
89
|
-
return this.keywordSearch(keyword, query);
|
|
90
|
-
}
|
|
91
|
-
// Build WHERE clause
|
|
92
|
-
const conditions = ['collection = ?'];
|
|
93
|
-
const params = [collection];
|
|
94
|
-
if (scope) {
|
|
95
|
-
conditions.push('scope = ?');
|
|
96
|
-
params.push(scope);
|
|
97
|
-
}
|
|
98
|
-
if (agentId) {
|
|
99
|
-
conditions.push('(agent_id = ? OR agent_id IS NULL)');
|
|
100
|
-
params.push(agentId);
|
|
101
|
-
}
|
|
102
|
-
if (tier) {
|
|
103
|
-
conditions.push('(tier = ? OR tier IS NULL OR tier = \'all\')');
|
|
104
|
-
params.push(tier);
|
|
105
|
-
}
|
|
106
|
-
params.push(limit);
|
|
107
|
-
const rows = this.db
|
|
108
|
-
.prepare(`
|
|
109
|
-
SELECT id, collection, section_path, depth, content, token_estimate,
|
|
110
|
-
source_hash, source_path, scope, tier, agent_id, parent_path,
|
|
111
|
-
created_at, updated_at
|
|
112
|
-
FROM doc_chunks
|
|
113
|
-
WHERE ${conditions.join(' AND ')}
|
|
114
|
-
ORDER BY depth ASC, section_path ASC
|
|
115
|
-
LIMIT ?
|
|
116
|
-
`)
|
|
117
|
-
.all(...params);
|
|
118
|
-
return rows.map(this.mapRow);
|
|
119
|
-
}
|
|
120
|
-
/**
|
|
121
|
-
* FTS5 keyword search across chunks.
|
|
122
|
-
*/
|
|
123
|
-
keywordSearch(keyword, query) {
|
|
124
|
-
const { collection, agentId, tier, limit = 20 } = query;
|
|
125
|
-
const hasFilters = !!(agentId || tier);
|
|
126
|
-
const innerLimit = hasFilters ? limit * 4 : limit;
|
|
127
|
-
// Two-phase: FTS in subquery, metadata filter on small result set.
|
|
128
|
-
let sql = `
|
|
129
|
-
SELECT c.id, c.collection, c.section_path, c.depth, c.content, c.token_estimate,
|
|
130
|
-
c.source_hash, c.source_path, c.scope, c.tier, c.agent_id, c.parent_path,
|
|
131
|
-
c.created_at, c.updated_at
|
|
132
|
-
FROM (
|
|
133
|
-
SELECT rowid, rank FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ? ORDER BY rank LIMIT ?
|
|
134
|
-
) sub
|
|
135
|
-
JOIN doc_chunks c ON c.rowid = sub.rowid
|
|
136
|
-
WHERE c.collection = ?
|
|
137
|
-
`;
|
|
138
|
-
const params = [keyword, innerLimit, collection];
|
|
139
|
-
if (agentId) {
|
|
140
|
-
sql += ' AND (c.agent_id = ? OR c.agent_id IS NULL)';
|
|
141
|
-
params.push(agentId);
|
|
142
|
-
}
|
|
143
|
-
if (tier) {
|
|
144
|
-
sql += " AND (c.tier = ? OR c.tier IS NULL OR c.tier = 'all')";
|
|
145
|
-
params.push(tier);
|
|
146
|
-
}
|
|
147
|
-
sql += ' ORDER BY sub.rank LIMIT ?';
|
|
148
|
-
params.push(limit * 3); // over-fetch to allow dedup
|
|
149
|
-
const rows = this.db.prepare(sql).all(...params);
|
|
150
|
-
// Deduplicate by source_hash to avoid returning identical content
|
|
151
|
-
// from multiple agent-specific copies of shared-fleet docs.
|
|
152
|
-
const seenHashes = new Set();
|
|
153
|
-
const deduped = rows.filter(r => {
|
|
154
|
-
const hash = r['source_hash'];
|
|
155
|
-
if (!hash)
|
|
156
|
-
return true;
|
|
157
|
-
if (seenHashes.has(hash))
|
|
158
|
-
return false;
|
|
159
|
-
seenHashes.add(hash);
|
|
160
|
-
return true;
|
|
161
|
-
});
|
|
162
|
-
return deduped.slice(0, limit).map(this.mapRow);
|
|
163
|
-
}
|
|
164
|
-
/**
|
|
165
|
-
* Get a single chunk by ID.
|
|
166
|
-
*/
|
|
167
|
-
getChunk(id) {
|
|
168
|
-
const row = this.db
|
|
169
|
-
.prepare(`
|
|
170
|
-
SELECT id, collection, section_path, depth, content, token_estimate,
|
|
171
|
-
source_hash, source_path, scope, tier, agent_id, parent_path,
|
|
172
|
-
created_at, updated_at
|
|
173
|
-
FROM doc_chunks WHERE id = ?
|
|
174
|
-
`)
|
|
175
|
-
.get(id);
|
|
176
|
-
return row ? this.mapRow(row) : null;
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Check if a source file needs re-indexing.
|
|
180
|
-
* Returns true if the file has changed or has never been indexed.
|
|
181
|
-
*/
|
|
182
|
-
needsReindex(sourcePath, collection, currentHash) {
|
|
183
|
-
const row = this.db
|
|
184
|
-
.prepare('SELECT source_hash FROM doc_sources WHERE source_path = ? AND collection = ?')
|
|
185
|
-
.get(sourcePath, collection);
|
|
186
|
-
return !row || row.source_hash !== currentHash;
|
|
187
|
-
}
|
|
188
|
-
/**
|
|
189
|
-
* List all indexed sources, optionally filtered by agent or collection.
|
|
190
|
-
*/
|
|
191
|
-
listSources(opts) {
|
|
192
|
-
const conditions = [];
|
|
193
|
-
const params = [];
|
|
194
|
-
if (opts?.agentId) {
|
|
195
|
-
conditions.push('agent_id = ?');
|
|
196
|
-
params.push(opts.agentId);
|
|
197
|
-
}
|
|
198
|
-
if (opts?.collection) {
|
|
199
|
-
conditions.push('collection = ?');
|
|
200
|
-
params.push(opts.collection);
|
|
201
|
-
}
|
|
202
|
-
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
203
|
-
const rows = this.db
|
|
204
|
-
.prepare(`
|
|
205
|
-
SELECT source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at
|
|
206
|
-
FROM doc_sources ${where}
|
|
207
|
-
ORDER BY indexed_at DESC
|
|
208
|
-
`)
|
|
209
|
-
.all(...params);
|
|
210
|
-
return rows.map(r => ({
|
|
211
|
-
sourcePath: r['source_path'],
|
|
212
|
-
collection: r['collection'],
|
|
213
|
-
scope: r['scope'],
|
|
214
|
-
agentId: r['agent_id'],
|
|
215
|
-
sourceHash: r['source_hash'],
|
|
216
|
-
chunkCount: r['chunk_count'],
|
|
217
|
-
indexedAt: r['indexed_at'],
|
|
218
|
-
}));
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Delete all chunks for a specific source file.
|
|
222
|
-
*/
|
|
223
|
-
deleteSource(sourcePath, collection) {
|
|
224
|
-
this.db.exec('BEGIN');
|
|
225
|
-
try {
|
|
226
|
-
const result = this.db
|
|
227
|
-
.prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
|
|
228
|
-
.run(sourcePath, collection);
|
|
229
|
-
this.db.prepare('DELETE FROM doc_sources WHERE source_path = ? AND collection = ?')
|
|
230
|
-
.run(sourcePath, collection);
|
|
231
|
-
this.db.exec('COMMIT');
|
|
232
|
-
return result.changes;
|
|
233
|
-
}
|
|
234
|
-
catch (err) {
|
|
235
|
-
this.db.exec('ROLLBACK');
|
|
236
|
-
throw err;
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
/**
|
|
240
|
-
* Get chunk stats: count per collection.
|
|
241
|
-
*/
|
|
242
|
-
getStats() {
|
|
243
|
-
const rows = this.db.prepare(`
|
|
244
|
-
SELECT collection,
|
|
245
|
-
COUNT(*) as count,
|
|
246
|
-
COUNT(DISTINCT source_path) as sources,
|
|
247
|
-
SUM(token_estimate) as total_tokens
|
|
248
|
-
FROM doc_chunks
|
|
249
|
-
GROUP BY collection
|
|
250
|
-
ORDER BY collection
|
|
251
|
-
`).all();
|
|
252
|
-
return rows.map(r => ({
|
|
253
|
-
collection: r['collection'],
|
|
254
|
-
count: r['count'],
|
|
255
|
-
sources: r['sources'],
|
|
256
|
-
totalTokens: r['total_tokens'] ?? 0,
|
|
257
|
-
}));
|
|
258
|
-
}
|
|
259
|
-
mapRow(r) {
|
|
260
|
-
return {
|
|
261
|
-
id: r['id'],
|
|
262
|
-
collection: r['collection'],
|
|
263
|
-
sectionPath: r['section_path'],
|
|
264
|
-
depth: r['depth'],
|
|
265
|
-
content: r['content'],
|
|
266
|
-
tokenEstimate: r['token_estimate'],
|
|
267
|
-
sourceHash: r['source_hash'],
|
|
268
|
-
sourcePath: r['source_path'],
|
|
269
|
-
scope: r['scope'],
|
|
270
|
-
tier: r['tier'],
|
|
271
|
-
agentId: r['agent_id'],
|
|
272
|
-
parentPath: r['parent_path'],
|
|
273
|
-
createdAt: r['created_at'],
|
|
274
|
-
updatedAt: r['updated_at'],
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
//# sourceMappingURL=doc-chunk-store.js.map
|