audrey 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +310 -643
- package/benchmarks/baselines.js +169 -0
- package/benchmarks/cases.js +421 -0
- package/benchmarks/reference-results.js +70 -0
- package/benchmarks/report.js +255 -0
- package/benchmarks/run.js +514 -0
- package/docs/assets/benchmarks/local-benchmark.svg +45 -0
- package/docs/assets/benchmarks/operations-benchmark.svg +45 -0
- package/docs/assets/benchmarks/published-memory-standards.svg +50 -0
- package/docs/benchmarking.md +151 -0
- package/docs/production-readiness.md +96 -0
- package/examples/fintech-ops-demo.js +67 -0
- package/examples/healthcare-ops-demo.js +67 -0
- package/examples/stripe-demo.js +105 -0
- package/mcp-server/config.js +81 -24
- package/mcp-server/index.js +611 -75
- package/mcp-server/serve.js +482 -0
- package/package.json +24 -5
- package/src/audrey.js +51 -13
- package/src/consolidate.js +70 -54
- package/src/db.js +22 -1
- package/src/embedding.js +16 -12
- package/src/encode.js +8 -2
- package/src/fts.js +134 -0
- package/src/import.js +28 -0
- package/src/llm.js +6 -3
- package/src/migrate.js +2 -2
- package/src/recall.js +253 -32
- package/src/utils.js +25 -0
- package/types/index.d.ts +434 -0
package/src/audrey.js
CHANGED
|
@@ -44,6 +44,7 @@ import { detectResonance } from './affect.js';
|
|
|
44
44
|
* @property {string} [before]
|
|
45
45
|
* @property {Record<string, string>} [context]
|
|
46
46
|
* @property {{ valence?: number, arousal?: number }} [mood]
|
|
47
|
+
* @property {'hybrid' | 'vector' | 'keyword'} [retrieval]
|
|
47
48
|
*
|
|
48
49
|
* @typedef {Object} RecallResult
|
|
49
50
|
* @property {string} id
|
|
@@ -118,6 +119,7 @@ export class Audrey extends EventEmitter {
|
|
|
118
119
|
const { db, migrated } = createDatabase(dataDir, { dimensions: this.embeddingProvider.dimensions });
|
|
119
120
|
this.db = db;
|
|
120
121
|
this._migrationPending = migrated;
|
|
122
|
+
this._pending = new Set();
|
|
121
123
|
this.llmProvider = llm ? createLLMProvider(llm) : null;
|
|
122
124
|
this.confidenceConfig = {
|
|
123
125
|
weights: confidence.weights,
|
|
@@ -128,10 +130,11 @@ export class Audrey extends EventEmitter {
|
|
|
128
130
|
affectWeight: affect.weight ?? 0.2,
|
|
129
131
|
};
|
|
130
132
|
this.consolidationConfig = {
|
|
131
|
-
minEpisodes: consolidation.minEpisodes
|
|
133
|
+
minEpisodes: consolidation.minEpisodes ?? 3,
|
|
132
134
|
};
|
|
133
|
-
this.decayConfig = { dormantThreshold: decay.dormantThreshold
|
|
135
|
+
this.decayConfig = { dormantThreshold: decay.dormantThreshold ?? 0.1 };
|
|
134
136
|
this._autoConsolidateTimer = null;
|
|
137
|
+
this._closed = false;
|
|
135
138
|
this.interferenceConfig = {
|
|
136
139
|
enabled: interference.enabled ?? true,
|
|
137
140
|
k: interference.k ?? 5,
|
|
@@ -163,8 +166,19 @@ export class Audrey extends EventEmitter {
|
|
|
163
166
|
this.emit('migration', counts);
|
|
164
167
|
}
|
|
165
168
|
|
|
169
|
+
_trackAsync(promise) {
|
|
170
|
+
this._pending.add(promise);
|
|
171
|
+
promise.finally(() => this._pending.delete(promise));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async waitForIdle() {
|
|
175
|
+
while (this._pending.size > 0) {
|
|
176
|
+
await Promise.allSettled([...this._pending]);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
166
180
|
_emitValidation(id, params) {
|
|
167
|
-
validateMemory(this.db, this.embeddingProvider, { id, ...params }, {
|
|
181
|
+
const p = validateMemory(this.db, this.embeddingProvider, { id, ...params }, {
|
|
168
182
|
llmProvider: this.llmProvider,
|
|
169
183
|
})
|
|
170
184
|
.then(validation => {
|
|
@@ -184,7 +198,8 @@ export class Audrey extends EventEmitter {
|
|
|
184
198
|
});
|
|
185
199
|
}
|
|
186
200
|
})
|
|
187
|
-
.catch(err => this.emit('error', err));
|
|
201
|
+
.catch(err => { if (!this._closed) this.emit('error', err); });
|
|
202
|
+
this._trackAsync(p);
|
|
188
203
|
}
|
|
189
204
|
|
|
190
205
|
/**
|
|
@@ -193,26 +208,28 @@ export class Audrey extends EventEmitter {
|
|
|
193
208
|
*/
|
|
194
209
|
async encode(params) {
|
|
195
210
|
await this._ensureMigrated();
|
|
196
|
-
const encodeParams = { ...params, arousalWeight: this.affectConfig.arousalWeight };
|
|
211
|
+
const encodeParams = { agent: this.agent, ...params, arousalWeight: this.affectConfig.arousalWeight };
|
|
197
212
|
const id = await encodeEpisode(this.db, this.embeddingProvider, encodeParams);
|
|
198
213
|
this.emit('encode', { id, ...params });
|
|
199
214
|
if (this.interferenceConfig.enabled) {
|
|
200
|
-
applyInterference(this.db, this.embeddingProvider, id, params, this.interferenceConfig)
|
|
215
|
+
const p = applyInterference(this.db, this.embeddingProvider, id, params, this.interferenceConfig)
|
|
201
216
|
.then(affected => {
|
|
202
217
|
if (affected.length > 0) {
|
|
203
218
|
this.emit('interference', { episodeId: id, affected });
|
|
204
219
|
}
|
|
205
220
|
})
|
|
206
|
-
.catch(err => this.emit('error', err));
|
|
221
|
+
.catch(err => { if (!this._closed) this.emit('error', err); });
|
|
222
|
+
this._trackAsync(p);
|
|
207
223
|
}
|
|
208
224
|
if (this.affectConfig.enabled && this.affectConfig.resonance.enabled && params.affect?.valence !== undefined) {
|
|
209
|
-
detectResonance(this.db, this.embeddingProvider, id, params, this.affectConfig.resonance)
|
|
225
|
+
const p = detectResonance(this.db, this.embeddingProvider, id, params, this.affectConfig.resonance)
|
|
210
226
|
.then(echoes => {
|
|
211
227
|
if (echoes.length > 0) {
|
|
212
228
|
this.emit('resonance', { episodeId: id, affect: params.affect, echoes });
|
|
213
229
|
}
|
|
214
230
|
})
|
|
215
|
-
.catch(err => this.emit('error', err));
|
|
231
|
+
.catch(err => { if (!this._closed) this.emit('error', err); });
|
|
232
|
+
this._trackAsync(p);
|
|
216
233
|
}
|
|
217
234
|
this._emitValidation(id, params);
|
|
218
235
|
return id;
|
|
@@ -288,6 +305,7 @@ export class Audrey extends EventEmitter {
|
|
|
288
305
|
async recall(query, options = {}) {
|
|
289
306
|
await this._ensureMigrated();
|
|
290
307
|
return recallFn(this.db, this.embeddingProvider, query, {
|
|
308
|
+
agent: this.agent,
|
|
291
309
|
...options,
|
|
292
310
|
confidenceConfig: this._recallConfig(options),
|
|
293
311
|
});
|
|
@@ -301,6 +319,7 @@ export class Audrey extends EventEmitter {
|
|
|
301
319
|
async *recallStream(query, options = {}) {
|
|
302
320
|
await this._ensureMigrated();
|
|
303
321
|
yield* recallStreamFn(this.db, this.embeddingProvider, query, {
|
|
322
|
+
agent: this.agent,
|
|
304
323
|
...options,
|
|
305
324
|
confidenceConfig: this._recallConfig(options),
|
|
306
325
|
});
|
|
@@ -324,8 +343,8 @@ export class Audrey extends EventEmitter {
|
|
|
324
343
|
async consolidate(options = {}) {
|
|
325
344
|
await this._ensureMigrated();
|
|
326
345
|
const result = await runConsolidation(this.db, this.embeddingProvider, {
|
|
327
|
-
minClusterSize: options.minClusterSize
|
|
328
|
-
similarityThreshold: options.similarityThreshold
|
|
346
|
+
minClusterSize: options.minClusterSize ?? this.consolidationConfig.minEpisodes,
|
|
347
|
+
similarityThreshold: options.similarityThreshold ?? 0.80,
|
|
329
348
|
extractPrinciple: options.extractPrinciple,
|
|
330
349
|
llmProvider: options.llmProvider || this.llmProvider,
|
|
331
350
|
});
|
|
@@ -341,7 +360,7 @@ export class Audrey extends EventEmitter {
|
|
|
341
360
|
*/
|
|
342
361
|
decay(options = {}) {
|
|
343
362
|
const result = applyDecay(this.db, {
|
|
344
|
-
dormantThreshold: options.dormantThreshold
|
|
363
|
+
dormantThreshold: options.dormantThreshold ?? this.decayConfig.dormantThreshold,
|
|
345
364
|
halfLives: options.halfLives ?? this.confidenceConfig.halfLives,
|
|
346
365
|
});
|
|
347
366
|
this.emit('decay', result);
|
|
@@ -564,6 +583,9 @@ export class Audrey extends EventEmitter {
|
|
|
564
583
|
this._autoConsolidateTimer = setInterval(() => {
|
|
565
584
|
this.consolidate(options).catch(err => this.emit('error', err));
|
|
566
585
|
}, intervalMs);
|
|
586
|
+
if (typeof this._autoConsolidateTimer.unref === 'function') {
|
|
587
|
+
this._autoConsolidateTimer.unref();
|
|
588
|
+
}
|
|
567
589
|
}
|
|
568
590
|
|
|
569
591
|
stopAutoConsolidate() {
|
|
@@ -583,6 +605,20 @@ export class Audrey extends EventEmitter {
|
|
|
583
605
|
return result;
|
|
584
606
|
}
|
|
585
607
|
|
|
608
|
+
markUsed(id) {
|
|
609
|
+
const now = new Date().toISOString();
|
|
610
|
+
const tables = ['episodes', 'semantics', 'procedures'];
|
|
611
|
+
for (const table of tables) {
|
|
612
|
+
const result = this.db.prepare(
|
|
613
|
+
`UPDATE ${table} SET usage_count = usage_count + 1, last_used_at = ? WHERE id = ?`
|
|
614
|
+
).run(now, id);
|
|
615
|
+
if (result.changes > 0) {
|
|
616
|
+
this.emit('used', { id, table, usageCount: result.changes });
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
586
622
|
async forgetByQuery(query, options = {}) {
|
|
587
623
|
await this._ensureMigrated();
|
|
588
624
|
const result = await forgetByQueryFn(this.db, this.embeddingProvider, query, options);
|
|
@@ -596,9 +632,11 @@ export class Audrey extends EventEmitter {
|
|
|
596
632
|
return result;
|
|
597
633
|
}
|
|
598
634
|
|
|
599
|
-
/** @returns {void} */
|
|
600
635
|
close() {
|
|
636
|
+
if (this._closed) return;
|
|
637
|
+
this._closed = true;
|
|
601
638
|
this.stopAutoConsolidate();
|
|
639
|
+
this._pending.clear();
|
|
602
640
|
closeDatabase(this.db);
|
|
603
641
|
}
|
|
604
642
|
}
|
package/src/consolidate.js
CHANGED
|
@@ -97,6 +97,10 @@ async function llmExtractPrinciple(llmProvider, episodes) {
|
|
|
97
97
|
return llmProvider.json(messages);
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
function inClause(ids) {
|
|
101
|
+
return ids.map(() => '?').join(',');
|
|
102
|
+
}
|
|
103
|
+
|
|
100
104
|
/**
|
|
101
105
|
* @param {import('better-sqlite3').Database} db
|
|
102
106
|
* @param {import('./embedding.js').EmbeddingProvider} embeddingProvider
|
|
@@ -132,6 +136,7 @@ export async function runConsolidation(db, embeddingProvider, options = {}) {
|
|
|
132
136
|
const allOutputIds = [];
|
|
133
137
|
let principlesExtracted = 0;
|
|
134
138
|
let proceduresExtracted = 0;
|
|
139
|
+
const preparedClusters = [];
|
|
135
140
|
const insertProcedure = db.prepare(`
|
|
136
141
|
INSERT INTO procedures (
|
|
137
142
|
id, content, embedding, state, trigger_conditions,
|
|
@@ -149,8 +154,6 @@ export async function runConsolidation(db, embeddingProvider, options = {}) {
|
|
|
149
154
|
) VALUES (?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
150
155
|
`);
|
|
151
156
|
const insertVecSemantic = db.prepare('INSERT INTO vec_semantics(id, embedding, state) VALUES (?, ?, ?)');
|
|
152
|
-
const markEpisode = db.prepare('UPDATE episodes SET consolidated = 1 WHERE id = ?');
|
|
153
|
-
const markVecEpisode = db.prepare('UPDATE vec_episodes SET consolidated = ? WHERE id = ?');
|
|
154
157
|
const updateRunCompleted = db.prepare(`
|
|
155
158
|
UPDATE consolidation_runs
|
|
156
159
|
SET status = 'completed',
|
|
@@ -165,70 +168,88 @@ export async function runConsolidation(db, embeddingProvider, options = {}) {
|
|
|
165
168
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
166
169
|
`);
|
|
167
170
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
principle = defaultExtractPrinciple(cluster);
|
|
178
|
-
}
|
|
171
|
+
for (const cluster of clusters) {
|
|
172
|
+
let principle;
|
|
173
|
+
if (extractPrinciple) {
|
|
174
|
+
principle = await extractPrinciple(cluster);
|
|
175
|
+
} else if (llmProvider) {
|
|
176
|
+
principle = await llmExtractPrinciple(llmProvider, cluster);
|
|
177
|
+
} else {
|
|
178
|
+
principle = defaultExtractPrinciple(cluster);
|
|
179
|
+
}
|
|
179
180
|
|
|
180
|
-
|
|
181
|
+
if (!principle || !principle.content) continue;
|
|
181
182
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
183
|
+
const vector = await embeddingProvider.embed(principle.content);
|
|
184
|
+
const clusterIds = cluster.map(ep => ep.id);
|
|
185
|
+
preparedClusters.push({
|
|
186
|
+
principle,
|
|
187
|
+
clusterIds,
|
|
188
|
+
sourceTypeDiversity: new Set(cluster.map(ep => ep.source)).size,
|
|
189
|
+
embeddingBuffer: embeddingProvider.vectorToBuffer(vector),
|
|
190
|
+
memoryId: generateId(),
|
|
191
|
+
createdAt: new Date().toISOString(),
|
|
192
|
+
maxSalience: Math.max(...cluster.map(ep => ep.salience ?? 0.5)),
|
|
193
|
+
});
|
|
194
|
+
}
|
|
189
195
|
|
|
190
|
-
|
|
196
|
+
const writeConsolidation = db.transaction(() => {
|
|
197
|
+
for (const prepared of preparedClusters) {
|
|
198
|
+
const placeholders = inClause(prepared.clusterIds);
|
|
199
|
+
const eligibleCount = db.prepare(`
|
|
200
|
+
SELECT COUNT(*) AS count
|
|
201
|
+
FROM episodes
|
|
202
|
+
WHERE id IN (${placeholders})
|
|
203
|
+
AND consolidated = 0
|
|
204
|
+
AND superseded_by IS NULL
|
|
205
|
+
`).get(...prepared.clusterIds).count;
|
|
191
206
|
|
|
192
|
-
if (
|
|
207
|
+
if (eligibleCount !== prepared.clusterIds.length) {
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (prepared.principle.type === 'procedural') {
|
|
193
212
|
insertProcedure.run(
|
|
194
|
-
memoryId,
|
|
195
|
-
principle.content,
|
|
196
|
-
embeddingBuffer,
|
|
197
|
-
principle.conditions ? JSON.stringify(principle.conditions) : null,
|
|
198
|
-
JSON.stringify(clusterIds),
|
|
213
|
+
prepared.memoryId,
|
|
214
|
+
prepared.principle.content,
|
|
215
|
+
prepared.embeddingBuffer,
|
|
216
|
+
prepared.principle.conditions ? JSON.stringify(prepared.principle.conditions) : null,
|
|
217
|
+
JSON.stringify(prepared.clusterIds),
|
|
199
218
|
embeddingProvider.modelName,
|
|
200
219
|
embeddingProvider.modelVersion,
|
|
201
|
-
createdAt,
|
|
202
|
-
maxSalience,
|
|
220
|
+
prepared.createdAt,
|
|
221
|
+
prepared.maxSalience,
|
|
203
222
|
);
|
|
204
|
-
insertVecProcedure.run(memoryId, embeddingBuffer, 'active');
|
|
223
|
+
insertVecProcedure.run(prepared.memoryId, prepared.embeddingBuffer, 'active');
|
|
205
224
|
proceduresExtracted++;
|
|
206
225
|
} else {
|
|
207
226
|
insertSemantic.run(
|
|
208
|
-
memoryId,
|
|
209
|
-
principle.content,
|
|
210
|
-
embeddingBuffer,
|
|
211
|
-
JSON.stringify(clusterIds),
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
sourceTypeDiversity,
|
|
227
|
+
prepared.memoryId,
|
|
228
|
+
prepared.principle.content,
|
|
229
|
+
prepared.embeddingBuffer,
|
|
230
|
+
JSON.stringify(prepared.clusterIds),
|
|
231
|
+
prepared.clusterIds.length,
|
|
232
|
+
prepared.clusterIds.length,
|
|
233
|
+
prepared.sourceTypeDiversity,
|
|
215
234
|
runId,
|
|
216
235
|
embeddingProvider.modelName,
|
|
217
236
|
embeddingProvider.modelVersion,
|
|
218
237
|
llmProvider?.modelName || null,
|
|
219
|
-
createdAt,
|
|
220
|
-
maxSalience,
|
|
238
|
+
prepared.createdAt,
|
|
239
|
+
prepared.maxSalience,
|
|
221
240
|
);
|
|
222
|
-
insertVecSemantic.run(memoryId, embeddingBuffer, 'active');
|
|
241
|
+
insertVecSemantic.run(prepared.memoryId, prepared.embeddingBuffer, 'active');
|
|
223
242
|
}
|
|
224
243
|
|
|
225
|
-
|
|
226
|
-
|
|
244
|
+
db.prepare(`UPDATE episodes SET consolidated = 1 WHERE id IN (${placeholders})`).run(...prepared.clusterIds);
|
|
245
|
+
db.prepare(`UPDATE vec_episodes SET consolidated = ? WHERE id IN (${placeholders})`).run(
|
|
246
|
+
BigInt(1),
|
|
247
|
+
...prepared.clusterIds,
|
|
248
|
+
);
|
|
227
249
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
}
|
|
250
|
+
allInputIds.push(...prepared.clusterIds);
|
|
251
|
+
allOutputIds.push(prepared.memoryId);
|
|
252
|
+
principlesExtracted++;
|
|
232
253
|
}
|
|
233
254
|
|
|
234
255
|
const completedAt = new Date().toISOString();
|
|
@@ -237,13 +258,8 @@ export async function runConsolidation(db, embeddingProvider, options = {}) {
|
|
|
237
258
|
generateId(), runId, minClusterSize, similarityThreshold,
|
|
238
259
|
episodesEvaluated, clusters.length, principlesExtracted, completedAt,
|
|
239
260
|
);
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
if (db.inTransaction) {
|
|
243
|
-
db.exec('ROLLBACK');
|
|
244
|
-
}
|
|
245
|
-
throw err;
|
|
246
|
-
}
|
|
261
|
+
});
|
|
262
|
+
writeConsolidation.immediate();
|
|
247
263
|
|
|
248
264
|
return {
|
|
249
265
|
runId,
|
package/src/db.js
CHANGED
|
@@ -2,6 +2,7 @@ import Database from 'better-sqlite3';
|
|
|
2
2
|
import * as sqliteVec from 'sqlite-vec';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { mkdirSync, existsSync } from 'node:fs';
|
|
5
|
+
import { createFTSTables, backfillFTS } from './fts.js';
|
|
5
6
|
|
|
6
7
|
const SCHEMA = `
|
|
7
8
|
CREATE TABLE IF NOT EXISTS episodes (
|
|
@@ -248,7 +249,7 @@ function addColumnIfMissing(db, table, column, definition) {
|
|
|
248
249
|
}
|
|
249
250
|
}
|
|
250
251
|
|
|
251
|
-
const SCHEMA_VERSION =
|
|
252
|
+
const SCHEMA_VERSION = 10;
|
|
252
253
|
|
|
253
254
|
const MIGRATIONS = [
|
|
254
255
|
{ version: 1, up(db) { addColumnIfMissing(db, 'episodes', 'context', "TEXT DEFAULT '{}'"); } },
|
|
@@ -258,6 +259,26 @@ const MIGRATIONS = [
|
|
|
258
259
|
{ version: 5, up(db) { addColumnIfMissing(db, 'procedures', 'interference_count', 'INTEGER DEFAULT 0'); } },
|
|
259
260
|
{ version: 6, up(db) { addColumnIfMissing(db, 'procedures', 'salience', 'REAL DEFAULT 0.5'); } },
|
|
260
261
|
{ version: 7, up(db) { addColumnIfMissing(db, 'episodes', 'private', 'INTEGER DEFAULT 0'); } },
|
|
262
|
+
{ version: 8, up(db) {
|
|
263
|
+
addColumnIfMissing(db, 'episodes', 'agent', "TEXT DEFAULT 'default'");
|
|
264
|
+
addColumnIfMissing(db, 'semantics', 'agent', "TEXT DEFAULT 'default'");
|
|
265
|
+
addColumnIfMissing(db, 'procedures', 'agent', "TEXT DEFAULT 'default'");
|
|
266
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent)");
|
|
267
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_semantics_agent ON semantics(agent)");
|
|
268
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_procedures_agent ON procedures(agent)");
|
|
269
|
+
}},
|
|
270
|
+
{ version: 9, up(db) {
|
|
271
|
+
createFTSTables(db);
|
|
272
|
+
backfillFTS(db);
|
|
273
|
+
}},
|
|
274
|
+
{ version: 10, up(db) {
|
|
275
|
+
addColumnIfMissing(db, 'episodes', 'usage_count', 'INTEGER DEFAULT 0');
|
|
276
|
+
addColumnIfMissing(db, 'episodes', 'last_used_at', 'TEXT');
|
|
277
|
+
addColumnIfMissing(db, 'semantics', 'usage_count', 'INTEGER DEFAULT 0');
|
|
278
|
+
addColumnIfMissing(db, 'semantics', 'last_used_at', 'TEXT');
|
|
279
|
+
addColumnIfMissing(db, 'procedures', 'usage_count', 'INTEGER DEFAULT 0');
|
|
280
|
+
addColumnIfMissing(db, 'procedures', 'last_used_at', 'TEXT');
|
|
281
|
+
}},
|
|
261
282
|
];
|
|
262
283
|
|
|
263
284
|
function runMigrations(db) {
|
package/src/embedding.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { describeHttpError, requireApiKey } from './utils.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* @typedef {Object} EmbeddingProvider
|
|
@@ -54,6 +55,7 @@ export class OpenAIEmbeddingProvider {
|
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
async embed(text) {
|
|
58
|
+
requireApiKey(this.apiKey, 'OpenAI embedding', 'OPENAI_API_KEY');
|
|
57
59
|
const controller = new AbortController();
|
|
58
60
|
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
59
61
|
try {
|
|
@@ -66,7 +68,7 @@ export class OpenAIEmbeddingProvider {
|
|
|
66
68
|
body: JSON.stringify({ input: text, model: this.model, dimensions: this.dimensions }),
|
|
67
69
|
signal: controller.signal,
|
|
68
70
|
});
|
|
69
|
-
if (!response.ok) throw new Error(`OpenAI embedding failed: ${response
|
|
71
|
+
if (!response.ok) throw new Error(`OpenAI embedding failed: ${await describeHttpError(response)}`);
|
|
70
72
|
const data = await response.json();
|
|
71
73
|
return data.data[0].embedding;
|
|
72
74
|
} finally {
|
|
@@ -75,6 +77,7 @@ export class OpenAIEmbeddingProvider {
|
|
|
75
77
|
}
|
|
76
78
|
|
|
77
79
|
async embedBatch(texts) {
|
|
80
|
+
requireApiKey(this.apiKey, 'OpenAI embedding', 'OPENAI_API_KEY');
|
|
78
81
|
const controller = new AbortController();
|
|
79
82
|
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
80
83
|
try {
|
|
@@ -87,7 +90,7 @@ export class OpenAIEmbeddingProvider {
|
|
|
87
90
|
body: JSON.stringify({ input: texts, model: this.model, dimensions: this.dimensions }),
|
|
88
91
|
signal: controller.signal,
|
|
89
92
|
});
|
|
90
|
-
if (!response.ok) throw new Error(`OpenAI embedding failed: ${response
|
|
93
|
+
if (!response.ok) throw new Error(`OpenAI embedding failed: ${await describeHttpError(response)}`);
|
|
91
94
|
const data = await response.json();
|
|
92
95
|
return data.data.map(d => d.embedding);
|
|
93
96
|
} finally {
|
|
@@ -106,13 +109,14 @@ export class OpenAIEmbeddingProvider {
|
|
|
106
109
|
|
|
107
110
|
/** @implements {EmbeddingProvider} */
|
|
108
111
|
export class LocalEmbeddingProvider {
|
|
109
|
-
constructor({ model = 'Xenova/all-MiniLM-L6-v2', device = 'gpu', batchSize = 64 } = {}) {
|
|
112
|
+
constructor({ model = 'Xenova/all-MiniLM-L6-v2', device = 'gpu', batchSize = 64, pipelineFactory = null } = {}) {
|
|
110
113
|
this.model = model;
|
|
111
114
|
this.dimensions = 384;
|
|
112
115
|
this.modelName = model;
|
|
113
116
|
this.modelVersion = '1.0.0';
|
|
114
117
|
this.device = device;
|
|
115
118
|
this.batchSize = batchSize;
|
|
119
|
+
this.pipelineFactory = pipelineFactory;
|
|
116
120
|
this._pipeline = null;
|
|
117
121
|
this._readyPromise = null;
|
|
118
122
|
this._actualDevice = null;
|
|
@@ -121,7 +125,7 @@ export class LocalEmbeddingProvider {
|
|
|
121
125
|
ready() {
|
|
122
126
|
if (!this._readyPromise) {
|
|
123
127
|
this._readyPromise = (async () => {
|
|
124
|
-
const
|
|
128
|
+
const pipeline = this.pipelineFactory || (await import('@huggingface/transformers')).pipeline;
|
|
125
129
|
try {
|
|
126
130
|
this._pipeline = await pipeline('feature-extraction', this.model, {
|
|
127
131
|
dtype: 'fp32', device: this.device,
|
|
@@ -177,20 +181,20 @@ export class GeminiEmbeddingProvider {
|
|
|
177
181
|
}
|
|
178
182
|
|
|
179
183
|
async embed(text) {
|
|
180
|
-
|
|
184
|
+
requireApiKey(this.apiKey, 'Gemini embedding', 'GOOGLE_API_KEY');
|
|
181
185
|
const controller = new AbortController();
|
|
182
186
|
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
183
187
|
try {
|
|
184
188
|
const response = await fetch(
|
|
185
|
-
`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent
|
|
189
|
+
`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent`,
|
|
186
190
|
{
|
|
187
191
|
method: 'POST',
|
|
188
|
-
headers: { 'Content-Type': 'application/json' },
|
|
192
|
+
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': this.apiKey },
|
|
189
193
|
body: JSON.stringify({ model: `models/${this.model}`, content: { parts: [{ text }] } }),
|
|
190
194
|
signal: controller.signal,
|
|
191
195
|
}
|
|
192
196
|
);
|
|
193
|
-
if (!response.ok) throw new Error(`Gemini embedding failed: ${response
|
|
197
|
+
if (!response.ok) throw new Error(`Gemini embedding failed: ${await describeHttpError(response)}`);
|
|
194
198
|
const data = await response.json();
|
|
195
199
|
return data.embedding.values;
|
|
196
200
|
} finally {
|
|
@@ -200,7 +204,7 @@ export class GeminiEmbeddingProvider {
|
|
|
200
204
|
|
|
201
205
|
async embedBatch(texts) {
|
|
202
206
|
if (texts.length === 0) return [];
|
|
203
|
-
|
|
207
|
+
requireApiKey(this.apiKey, 'Gemini embedding', 'GOOGLE_API_KEY');
|
|
204
208
|
const results = [];
|
|
205
209
|
for (let i = 0; i < texts.length; i += 100) {
|
|
206
210
|
const chunk = texts.slice(i, i + 100);
|
|
@@ -208,10 +212,10 @@ export class GeminiEmbeddingProvider {
|
|
|
208
212
|
const timer = setTimeout(() => controller.abort(), this.timeout);
|
|
209
213
|
try {
|
|
210
214
|
const response = await fetch(
|
|
211
|
-
`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents
|
|
215
|
+
`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents`,
|
|
212
216
|
{
|
|
213
217
|
method: 'POST',
|
|
214
|
-
headers: { 'Content-Type': 'application/json' },
|
|
218
|
+
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': this.apiKey },
|
|
215
219
|
body: JSON.stringify({
|
|
216
220
|
requests: chunk.map(text => ({
|
|
217
221
|
model: `models/${this.model}`,
|
|
@@ -221,7 +225,7 @@ export class GeminiEmbeddingProvider {
|
|
|
221
225
|
signal: controller.signal,
|
|
222
226
|
}
|
|
223
227
|
);
|
|
224
|
-
if (!response.ok) throw new Error(`Gemini batch embedding failed: ${response
|
|
228
|
+
if (!response.ok) throw new Error(`Gemini batch embedding failed: ${await describeHttpError(response)}`);
|
|
225
229
|
const data = await response.json();
|
|
226
230
|
results.push(...data.embeddings.map(e => e.values));
|
|
227
231
|
} finally {
|
package/src/encode.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { generateId } from './ulid.js';
|
|
2
2
|
import { sourceReliability } from './confidence.js';
|
|
3
3
|
import { arousalSalienceBoost } from './affect.js';
|
|
4
|
+
import { hasFTSTables, insertFTSEpisode } from './fts.js';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* @param {import('better-sqlite3').Database} db
|
|
@@ -19,6 +20,7 @@ export async function encodeEpisode(db, embeddingProvider, {
|
|
|
19
20
|
affect = {},
|
|
20
21
|
arousalWeight = 0.3,
|
|
21
22
|
private: isPrivate = false,
|
|
23
|
+
agent = 'default',
|
|
22
24
|
}) {
|
|
23
25
|
if (!content || typeof content !== 'string') throw new Error('content must be a non-empty string');
|
|
24
26
|
if (salience < 0 || salience > 1) throw new Error('salience must be between 0 and 1');
|
|
@@ -38,8 +40,8 @@ export async function encodeEpisode(db, embeddingProvider, {
|
|
|
38
40
|
INSERT INTO episodes (
|
|
39
41
|
id, content, embedding, source, source_reliability, salience, context, affect,
|
|
40
42
|
tags, causal_trigger, causal_consequence, created_at,
|
|
41
|
-
embedding_model, embedding_version, supersedes, "private"
|
|
42
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
43
|
+
embedding_model, embedding_version, supersedes, "private", agent
|
|
44
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
43
45
|
`).run(
|
|
44
46
|
id, content, embeddingBuffer, source, reliability, effectiveSalience,
|
|
45
47
|
JSON.stringify(context),
|
|
@@ -49,6 +51,7 @@ export async function encodeEpisode(db, embeddingProvider, {
|
|
|
49
51
|
now, embeddingProvider.modelName, embeddingProvider.modelVersion,
|
|
50
52
|
supersedes || null,
|
|
51
53
|
isPrivate ? 1 : 0,
|
|
54
|
+
agent,
|
|
52
55
|
);
|
|
53
56
|
db.prepare(
|
|
54
57
|
'INSERT INTO vec_episodes(id, embedding, source, consolidated) VALUES (?, ?, ?, ?)'
|
|
@@ -56,6 +59,9 @@ export async function encodeEpisode(db, embeddingProvider, {
|
|
|
56
59
|
if (supersedes) {
|
|
57
60
|
db.prepare('UPDATE episodes SET superseded_by = ? WHERE id = ?').run(id, supersedes);
|
|
58
61
|
}
|
|
62
|
+
if (hasFTSTables(db)) {
|
|
63
|
+
insertFTSEpisode(db, id, content, tags);
|
|
64
|
+
}
|
|
59
65
|
});
|
|
60
66
|
|
|
61
67
|
insertAndLink();
|