kiro-memory 1.8.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/plugin/dist/cli/contextkit.js +428 -205
- package/plugin/dist/hooks/agentSpawn.js +311 -180
- package/plugin/dist/hooks/kiro-hooks.js +299 -168
- package/plugin/dist/hooks/postToolUse.js +303 -172
- package/plugin/dist/hooks/stop.js +308 -177
- package/plugin/dist/hooks/userPromptSubmit.js +303 -172
- package/plugin/dist/index.js +303 -299
- package/plugin/dist/sdk/index.js +299 -172
- package/plugin/dist/services/search/EmbeddingService.js +88 -23
- package/plugin/dist/services/search/HybridSearch.js +190 -84
- package/plugin/dist/services/search/VectorSearch.js +128 -45
- package/plugin/dist/services/search/index.js +192 -223
- package/plugin/dist/services/sqlite/Database.js +55 -153
- package/plugin/dist/services/sqlite/Observations.js +23 -12
- package/plugin/dist/services/sqlite/Search.js +31 -19
- package/plugin/dist/services/sqlite/Sessions.js +5 -0
- package/plugin/dist/services/sqlite/index.js +113 -183
- package/plugin/dist/viewer.css +1 -0
- package/plugin/dist/viewer.html +2 -100
- package/plugin/dist/viewer.js +15 -24896
- package/plugin/dist/viewer.js.map +7 -0
- package/plugin/dist/worker-service.js +158 -5551
- package/plugin/dist/worker-service.js.map +7 -0
- package/scripts/postinstall.cjs +42 -0
|
@@ -226,8 +226,8 @@ var EmbeddingService = class {
|
|
|
226
226
|
initialized = false;
|
|
227
227
|
initializing = null;
|
|
228
228
|
/**
|
|
229
|
-
*
|
|
230
|
-
*
|
|
229
|
+
* Initialize the embedding service.
|
|
230
|
+
* Tries fastembed, then @huggingface/transformers, then fallback to null.
|
|
231
231
|
*/
|
|
232
232
|
async initialize() {
|
|
233
233
|
if (this.initialized) return this.provider !== null;
|
|
@@ -248,11 +248,11 @@ var EmbeddingService = class {
|
|
|
248
248
|
});
|
|
249
249
|
this.provider = "fastembed";
|
|
250
250
|
this.initialized = true;
|
|
251
|
-
logger.info("EMBEDDING", "
|
|
251
|
+
logger.info("EMBEDDING", "Initialized with fastembed (BGE-small-en-v1.5)");
|
|
252
252
|
return true;
|
|
253
253
|
}
|
|
254
254
|
} catch (error) {
|
|
255
|
-
logger.debug("EMBEDDING", `fastembed
|
|
255
|
+
logger.debug("EMBEDDING", `fastembed not available: ${error}`);
|
|
256
256
|
}
|
|
257
257
|
try {
|
|
258
258
|
const transformers = await import("@huggingface/transformers");
|
|
@@ -263,20 +263,20 @@ var EmbeddingService = class {
|
|
|
263
263
|
});
|
|
264
264
|
this.provider = "transformers";
|
|
265
265
|
this.initialized = true;
|
|
266
|
-
logger.info("EMBEDDING", "
|
|
266
|
+
logger.info("EMBEDDING", "Initialized with @huggingface/transformers (all-MiniLM-L6-v2)");
|
|
267
267
|
return true;
|
|
268
268
|
}
|
|
269
269
|
} catch (error) {
|
|
270
|
-
logger.debug("EMBEDDING", `@huggingface/transformers
|
|
270
|
+
logger.debug("EMBEDDING", `@huggingface/transformers not available: ${error}`);
|
|
271
271
|
}
|
|
272
272
|
this.provider = null;
|
|
273
273
|
this.initialized = true;
|
|
274
|
-
logger.warn("EMBEDDING", "
|
|
274
|
+
logger.warn("EMBEDDING", "No embedding provider available, semantic search disabled");
|
|
275
275
|
return false;
|
|
276
276
|
}
|
|
277
277
|
/**
|
|
278
|
-
*
|
|
279
|
-
*
|
|
278
|
+
* Generate embedding for a single text.
|
|
279
|
+
* Returns Float32Array with 384 dimensions, or null if not available.
|
|
280
280
|
*/
|
|
281
281
|
async embed(text) {
|
|
282
282
|
if (!this.initialized) await this.initialize();
|
|
@@ -289,46 +289,111 @@ var EmbeddingService = class {
|
|
|
289
289
|
return await this._embedTransformers(truncated);
|
|
290
290
|
}
|
|
291
291
|
} catch (error) {
|
|
292
|
-
logger.error("EMBEDDING", `
|
|
292
|
+
logger.error("EMBEDDING", `Error generating embedding: ${error}`);
|
|
293
293
|
}
|
|
294
294
|
return null;
|
|
295
295
|
}
|
|
296
296
|
/**
|
|
297
|
-
*
|
|
297
|
+
* Generate embeddings in batch.
|
|
298
|
+
* Uses native batch support when available (fastembed, transformers),
|
|
299
|
+
* falls back to serial processing on batch failure.
|
|
298
300
|
*/
|
|
299
301
|
async embedBatch(texts) {
|
|
300
302
|
if (!this.initialized) await this.initialize();
|
|
301
303
|
if (!this.provider || !this.model) return texts.map(() => null);
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
}
|
|
308
|
-
|
|
304
|
+
if (texts.length === 0) return [];
|
|
305
|
+
const truncated = texts.map((t) => t.substring(0, 2e3));
|
|
306
|
+
try {
|
|
307
|
+
if (this.provider === "fastembed") {
|
|
308
|
+
return await this._embedBatchFastembed(truncated);
|
|
309
|
+
} else if (this.provider === "transformers") {
|
|
310
|
+
return await this._embedBatchTransformers(truncated);
|
|
309
311
|
}
|
|
312
|
+
} catch (error) {
|
|
313
|
+
logger.warn("EMBEDDING", `Batch embedding failed, falling back to serial: ${error}`);
|
|
310
314
|
}
|
|
311
|
-
return
|
|
315
|
+
return this._embedBatchSerial(truncated);
|
|
312
316
|
}
|
|
313
317
|
/**
|
|
314
|
-
*
|
|
318
|
+
* Check if the service is available.
|
|
315
319
|
*/
|
|
316
320
|
isAvailable() {
|
|
317
321
|
return this.initialized && this.provider !== null;
|
|
318
322
|
}
|
|
319
323
|
/**
|
|
320
|
-
*
|
|
324
|
+
* Name of the active provider.
|
|
321
325
|
*/
|
|
322
326
|
getProvider() {
|
|
323
327
|
return this.provider;
|
|
324
328
|
}
|
|
325
329
|
/**
|
|
326
|
-
*
|
|
330
|
+
* Embedding vector dimensions.
|
|
327
331
|
*/
|
|
328
332
|
getDimensions() {
|
|
329
333
|
return 384;
|
|
330
334
|
}
|
|
331
|
-
// ---
|
|
335
|
+
// --- Batch implementations ---
|
|
336
|
+
/**
|
|
337
|
+
* Native batch embedding with fastembed.
|
|
338
|
+
* FlagEmbedding.embed() accepts string[] and returns an async iterable of batches.
|
|
339
|
+
*/
|
|
340
|
+
async _embedBatchFastembed(texts) {
|
|
341
|
+
const results = [];
|
|
342
|
+
const embeddings = this.model.embed(texts, texts.length);
|
|
343
|
+
for await (const batch of embeddings) {
|
|
344
|
+
if (batch) {
|
|
345
|
+
for (const vec of batch) {
|
|
346
|
+
results.push(vec instanceof Float32Array ? vec : new Float32Array(vec));
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
while (results.length < texts.length) {
|
|
351
|
+
results.push(null);
|
|
352
|
+
}
|
|
353
|
+
return results;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Batch embedding with @huggingface/transformers pipeline.
|
|
357
|
+
* The pipeline accepts string[] and returns a Tensor with shape [N, dims].
|
|
358
|
+
*/
|
|
359
|
+
async _embedBatchTransformers(texts) {
|
|
360
|
+
const output = await this.model(texts, {
|
|
361
|
+
pooling: "mean",
|
|
362
|
+
normalize: true
|
|
363
|
+
});
|
|
364
|
+
if (!output?.data) {
|
|
365
|
+
return texts.map(() => null);
|
|
366
|
+
}
|
|
367
|
+
const dims = this.getDimensions();
|
|
368
|
+
const data = output.data instanceof Float32Array ? output.data : new Float32Array(output.data);
|
|
369
|
+
const results = [];
|
|
370
|
+
for (let i = 0; i < texts.length; i++) {
|
|
371
|
+
const offset = i * dims;
|
|
372
|
+
if (offset + dims <= data.length) {
|
|
373
|
+
results.push(data.slice(offset, offset + dims));
|
|
374
|
+
} else {
|
|
375
|
+
results.push(null);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return results;
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Serial fallback: embed texts one at a time.
|
|
382
|
+
* Used when native batch fails.
|
|
383
|
+
*/
|
|
384
|
+
async _embedBatchSerial(texts) {
|
|
385
|
+
const results = [];
|
|
386
|
+
for (const text of texts) {
|
|
387
|
+
try {
|
|
388
|
+
const embedding = await this.embed(text);
|
|
389
|
+
results.push(embedding);
|
|
390
|
+
} catch {
|
|
391
|
+
results.push(null);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
return results;
|
|
395
|
+
}
|
|
396
|
+
// --- Single-text provider implementations ---
|
|
332
397
|
async _embedFastembed(text) {
|
|
333
398
|
const embeddings = this.model.embed([text], 1);
|
|
334
399
|
for await (const batch of embeddings) {
|
|
@@ -28,7 +28,7 @@ function escapeLikePattern(input) {
|
|
|
28
28
|
}
|
|
29
29
|
function sanitizeFTS5Query(query) {
|
|
30
30
|
const trimmed = query.length > 1e4 ? query.substring(0, 1e4) : query;
|
|
31
|
-
const terms = trimmed.replace(/[""]/g, "").split(/\s+/).filter((t) => t.length > 0).slice(0, 100).map((t) => `"${t}"`);
|
|
31
|
+
const terms = trimmed.replace(/[""\u0022]/g, "").split(/\s+/).filter((t) => t.length > 0).slice(0, 100).map((t) => `"${t}"`);
|
|
32
32
|
return terms.join(" ");
|
|
33
33
|
}
|
|
34
34
|
function searchObservationsFTS(db, query, filters = {}) {
|
|
@@ -193,26 +193,38 @@ function getTimeline(db, anchorId, depthBefore = 5, depthAfter = 5) {
|
|
|
193
193
|
return [...before, ...self, ...after];
|
|
194
194
|
}
|
|
195
195
|
function getProjectStats(db, project) {
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
196
|
+
const sql = `
|
|
197
|
+
WITH
|
|
198
|
+
obs_stats AS (
|
|
199
|
+
SELECT
|
|
200
|
+
COUNT(*) as count,
|
|
201
|
+
COALESCE(SUM(discovery_tokens), 0) as discovery_tokens,
|
|
202
|
+
COALESCE(SUM(
|
|
203
|
+
CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
|
|
204
|
+
), 0) as read_tokens
|
|
205
|
+
FROM observations WHERE project = ?
|
|
206
|
+
),
|
|
207
|
+
sum_count AS (SELECT COUNT(*) as count FROM summaries WHERE project = ?),
|
|
208
|
+
ses_count AS (SELECT COUNT(*) as count FROM sessions WHERE project = ?),
|
|
209
|
+
prm_count AS (SELECT COUNT(*) as count FROM prompts WHERE project = ?)
|
|
210
|
+
SELECT
|
|
211
|
+
obs_stats.count as observations,
|
|
212
|
+
obs_stats.discovery_tokens,
|
|
213
|
+
obs_stats.read_tokens,
|
|
214
|
+
sum_count.count as summaries,
|
|
215
|
+
ses_count.count as sessions,
|
|
216
|
+
prm_count.count as prompts
|
|
217
|
+
FROM obs_stats, sum_count, ses_count, prm_count
|
|
218
|
+
`;
|
|
219
|
+
const row = db.query(sql).get(project, project, project, project);
|
|
220
|
+
const discoveryTokens = row?.discovery_tokens || 0;
|
|
221
|
+
const readTokens = row?.read_tokens || 0;
|
|
210
222
|
const savings = Math.max(0, discoveryTokens - readTokens);
|
|
211
223
|
return {
|
|
212
|
-
observations:
|
|
213
|
-
summaries:
|
|
214
|
-
sessions:
|
|
215
|
-
prompts:
|
|
224
|
+
observations: row?.observations || 0,
|
|
225
|
+
summaries: row?.summaries || 0,
|
|
226
|
+
sessions: row?.sessions || 0,
|
|
227
|
+
prompts: row?.prompts || 0,
|
|
216
228
|
tokenEconomics: { discoveryTokens, readTokens, savings }
|
|
217
229
|
};
|
|
218
230
|
}
|
|
@@ -346,9 +358,25 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
346
358
|
ORDER BY cnt DESC
|
|
347
359
|
`).all(project, minGroupSize);
|
|
348
360
|
if (groups.length === 0) return { merged: 0, removed: 0 };
|
|
349
|
-
|
|
350
|
-
|
|
361
|
+
if (options.dryRun) {
|
|
362
|
+
let totalMerged = 0;
|
|
363
|
+
let totalRemoved = 0;
|
|
364
|
+
for (const group of groups) {
|
|
365
|
+
const obsIds = group.ids.split(",").map(Number);
|
|
366
|
+
const placeholders = obsIds.map(() => "?").join(",");
|
|
367
|
+
const count = db.query(
|
|
368
|
+
`SELECT COUNT(*) as cnt FROM observations WHERE id IN (${placeholders})`
|
|
369
|
+
).get(...obsIds)?.cnt || 0;
|
|
370
|
+
if (count >= minGroupSize) {
|
|
371
|
+
totalMerged += 1;
|
|
372
|
+
totalRemoved += count - 1;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return { merged: totalMerged, removed: totalRemoved };
|
|
376
|
+
}
|
|
351
377
|
const runConsolidation = db.transaction(() => {
|
|
378
|
+
let merged = 0;
|
|
379
|
+
let removed = 0;
|
|
352
380
|
for (const group of groups) {
|
|
353
381
|
const obsIds = group.ids.split(",").map(Number);
|
|
354
382
|
const placeholders = obsIds.map(() => "?").join(",");
|
|
@@ -356,11 +384,6 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
356
384
|
`SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
|
|
357
385
|
).all(...obsIds);
|
|
358
386
|
if (observations.length < minGroupSize) continue;
|
|
359
|
-
if (options.dryRun) {
|
|
360
|
-
totalMerged += 1;
|
|
361
|
-
totalRemoved += observations.length - 1;
|
|
362
|
-
continue;
|
|
363
|
-
}
|
|
364
387
|
const keeper = observations[0];
|
|
365
388
|
const others = observations.slice(1);
|
|
366
389
|
const uniqueTexts = /* @__PURE__ */ new Set();
|
|
@@ -373,18 +396,18 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
373
396
|
const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
|
|
374
397
|
db.run(
|
|
375
398
|
"UPDATE observations SET text = ?, title = ? WHERE id = ?",
|
|
376
|
-
[consolidatedText, `[
|
|
399
|
+
[consolidatedText, `[consolidated x${observations.length}] ${keeper.title}`, keeper.id]
|
|
377
400
|
);
|
|
378
401
|
const removeIds = others.map((o) => o.id);
|
|
379
402
|
const removePlaceholders = removeIds.map(() => "?").join(",");
|
|
380
403
|
db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
|
|
381
404
|
db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
|
|
382
|
-
|
|
383
|
-
|
|
405
|
+
merged += 1;
|
|
406
|
+
removed += removeIds.length;
|
|
384
407
|
}
|
|
408
|
+
return { merged, removed };
|
|
385
409
|
});
|
|
386
|
-
runConsolidation();
|
|
387
|
-
return { merged: totalMerged, removed: totalRemoved };
|
|
410
|
+
return runConsolidation();
|
|
388
411
|
}
|
|
389
412
|
var init_Observations = __esm({
|
|
390
413
|
"src/services/sqlite/Observations.ts"() {
|
|
@@ -618,8 +641,8 @@ var EmbeddingService = class {
|
|
|
618
641
|
initialized = false;
|
|
619
642
|
initializing = null;
|
|
620
643
|
/**
|
|
621
|
-
*
|
|
622
|
-
*
|
|
644
|
+
* Initialize the embedding service.
|
|
645
|
+
* Tries fastembed, then @huggingface/transformers, then fallback to null.
|
|
623
646
|
*/
|
|
624
647
|
async initialize() {
|
|
625
648
|
if (this.initialized) return this.provider !== null;
|
|
@@ -640,11 +663,11 @@ var EmbeddingService = class {
|
|
|
640
663
|
});
|
|
641
664
|
this.provider = "fastembed";
|
|
642
665
|
this.initialized = true;
|
|
643
|
-
logger.info("EMBEDDING", "
|
|
666
|
+
logger.info("EMBEDDING", "Initialized with fastembed (BGE-small-en-v1.5)");
|
|
644
667
|
return true;
|
|
645
668
|
}
|
|
646
669
|
} catch (error) {
|
|
647
|
-
logger.debug("EMBEDDING", `fastembed
|
|
670
|
+
logger.debug("EMBEDDING", `fastembed not available: ${error}`);
|
|
648
671
|
}
|
|
649
672
|
try {
|
|
650
673
|
const transformers = await import("@huggingface/transformers");
|
|
@@ -655,20 +678,20 @@ var EmbeddingService = class {
|
|
|
655
678
|
});
|
|
656
679
|
this.provider = "transformers";
|
|
657
680
|
this.initialized = true;
|
|
658
|
-
logger.info("EMBEDDING", "
|
|
681
|
+
logger.info("EMBEDDING", "Initialized with @huggingface/transformers (all-MiniLM-L6-v2)");
|
|
659
682
|
return true;
|
|
660
683
|
}
|
|
661
684
|
} catch (error) {
|
|
662
|
-
logger.debug("EMBEDDING", `@huggingface/transformers
|
|
685
|
+
logger.debug("EMBEDDING", `@huggingface/transformers not available: ${error}`);
|
|
663
686
|
}
|
|
664
687
|
this.provider = null;
|
|
665
688
|
this.initialized = true;
|
|
666
|
-
logger.warn("EMBEDDING", "
|
|
689
|
+
logger.warn("EMBEDDING", "No embedding provider available, semantic search disabled");
|
|
667
690
|
return false;
|
|
668
691
|
}
|
|
669
692
|
/**
|
|
670
|
-
*
|
|
671
|
-
*
|
|
693
|
+
* Generate embedding for a single text.
|
|
694
|
+
* Returns Float32Array with 384 dimensions, or null if not available.
|
|
672
695
|
*/
|
|
673
696
|
async embed(text) {
|
|
674
697
|
if (!this.initialized) await this.initialize();
|
|
@@ -681,46 +704,111 @@ var EmbeddingService = class {
|
|
|
681
704
|
return await this._embedTransformers(truncated);
|
|
682
705
|
}
|
|
683
706
|
} catch (error) {
|
|
684
|
-
logger.error("EMBEDDING", `
|
|
707
|
+
logger.error("EMBEDDING", `Error generating embedding: ${error}`);
|
|
685
708
|
}
|
|
686
709
|
return null;
|
|
687
710
|
}
|
|
688
711
|
/**
|
|
689
|
-
*
|
|
712
|
+
* Generate embeddings in batch.
|
|
713
|
+
* Uses native batch support when available (fastembed, transformers),
|
|
714
|
+
* falls back to serial processing on batch failure.
|
|
690
715
|
*/
|
|
691
716
|
async embedBatch(texts) {
|
|
692
717
|
if (!this.initialized) await this.initialize();
|
|
693
718
|
if (!this.provider || !this.model) return texts.map(() => null);
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
}
|
|
700
|
-
|
|
719
|
+
if (texts.length === 0) return [];
|
|
720
|
+
const truncated = texts.map((t) => t.substring(0, 2e3));
|
|
721
|
+
try {
|
|
722
|
+
if (this.provider === "fastembed") {
|
|
723
|
+
return await this._embedBatchFastembed(truncated);
|
|
724
|
+
} else if (this.provider === "transformers") {
|
|
725
|
+
return await this._embedBatchTransformers(truncated);
|
|
701
726
|
}
|
|
727
|
+
} catch (error) {
|
|
728
|
+
logger.warn("EMBEDDING", `Batch embedding failed, falling back to serial: ${error}`);
|
|
702
729
|
}
|
|
703
|
-
return
|
|
730
|
+
return this._embedBatchSerial(truncated);
|
|
704
731
|
}
|
|
705
732
|
/**
|
|
706
|
-
*
|
|
733
|
+
* Check if the service is available.
|
|
707
734
|
*/
|
|
708
735
|
isAvailable() {
|
|
709
736
|
return this.initialized && this.provider !== null;
|
|
710
737
|
}
|
|
711
738
|
/**
|
|
712
|
-
*
|
|
739
|
+
* Name of the active provider.
|
|
713
740
|
*/
|
|
714
741
|
getProvider() {
|
|
715
742
|
return this.provider;
|
|
716
743
|
}
|
|
717
744
|
/**
|
|
718
|
-
*
|
|
745
|
+
* Embedding vector dimensions.
|
|
719
746
|
*/
|
|
720
747
|
getDimensions() {
|
|
721
748
|
return 384;
|
|
722
749
|
}
|
|
723
|
-
// ---
|
|
750
|
+
// --- Batch implementations ---
|
|
751
|
+
/**
|
|
752
|
+
* Native batch embedding with fastembed.
|
|
753
|
+
* FlagEmbedding.embed() accepts string[] and returns an async iterable of batches.
|
|
754
|
+
*/
|
|
755
|
+
async _embedBatchFastembed(texts) {
|
|
756
|
+
const results = [];
|
|
757
|
+
const embeddings = this.model.embed(texts, texts.length);
|
|
758
|
+
for await (const batch of embeddings) {
|
|
759
|
+
if (batch) {
|
|
760
|
+
for (const vec of batch) {
|
|
761
|
+
results.push(vec instanceof Float32Array ? vec : new Float32Array(vec));
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
while (results.length < texts.length) {
|
|
766
|
+
results.push(null);
|
|
767
|
+
}
|
|
768
|
+
return results;
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* Batch embedding with @huggingface/transformers pipeline.
|
|
772
|
+
* The pipeline accepts string[] and returns a Tensor with shape [N, dims].
|
|
773
|
+
*/
|
|
774
|
+
async _embedBatchTransformers(texts) {
|
|
775
|
+
const output = await this.model(texts, {
|
|
776
|
+
pooling: "mean",
|
|
777
|
+
normalize: true
|
|
778
|
+
});
|
|
779
|
+
if (!output?.data) {
|
|
780
|
+
return texts.map(() => null);
|
|
781
|
+
}
|
|
782
|
+
const dims = this.getDimensions();
|
|
783
|
+
const data = output.data instanceof Float32Array ? output.data : new Float32Array(output.data);
|
|
784
|
+
const results = [];
|
|
785
|
+
for (let i = 0; i < texts.length; i++) {
|
|
786
|
+
const offset = i * dims;
|
|
787
|
+
if (offset + dims <= data.length) {
|
|
788
|
+
results.push(data.slice(offset, offset + dims));
|
|
789
|
+
} else {
|
|
790
|
+
results.push(null);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
return results;
|
|
794
|
+
}
|
|
795
|
+
/**
|
|
796
|
+
* Serial fallback: embed texts one at a time.
|
|
797
|
+
* Used when native batch fails.
|
|
798
|
+
*/
|
|
799
|
+
async _embedBatchSerial(texts) {
|
|
800
|
+
const results = [];
|
|
801
|
+
for (const text of texts) {
|
|
802
|
+
try {
|
|
803
|
+
const embedding = await this.embed(text);
|
|
804
|
+
results.push(embedding);
|
|
805
|
+
} catch {
|
|
806
|
+
results.push(null);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
return results;
|
|
810
|
+
}
|
|
811
|
+
// --- Single-text provider implementations ---
|
|
724
812
|
async _embedFastembed(text) {
|
|
725
813
|
const embeddings = this.model.embed([text], 1);
|
|
726
814
|
for await (const batch of embeddings) {
|
|
@@ -751,17 +839,21 @@ function getEmbeddingService() {
|
|
|
751
839
|
}
|
|
752
840
|
|
|
753
841
|
// src/services/search/VectorSearch.ts
|
|
842
|
+
var DEFAULT_MAX_CANDIDATES = 2e3;
|
|
754
843
|
function cosineSimilarity(a, b) {
|
|
755
|
-
|
|
844
|
+
const len = a.length;
|
|
845
|
+
if (len !== b.length) return 0;
|
|
756
846
|
let dotProduct = 0;
|
|
757
847
|
let normA = 0;
|
|
758
848
|
let normB = 0;
|
|
759
|
-
for (let i = 0; i <
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
849
|
+
for (let i = 0; i < len; i++) {
|
|
850
|
+
const ai = a[i];
|
|
851
|
+
const bi = b[i];
|
|
852
|
+
dotProduct += ai * bi;
|
|
853
|
+
normA += ai * ai;
|
|
854
|
+
normB += bi * bi;
|
|
763
855
|
}
|
|
764
|
-
const denominator = Math.sqrt(normA
|
|
856
|
+
const denominator = Math.sqrt(normA * normB);
|
|
765
857
|
if (denominator === 0) return 0;
|
|
766
858
|
return dotProduct / denominator;
|
|
767
859
|
}
|
|
@@ -774,23 +866,36 @@ function bufferToFloat32(buf) {
|
|
|
774
866
|
}
|
|
775
867
|
var VectorSearch = class {
|
|
776
868
|
/**
|
|
777
|
-
*
|
|
869
|
+
* Semantic search with SQL pre-filtering for scalability.
|
|
870
|
+
*
|
|
871
|
+
* 2-phase strategy:
|
|
872
|
+
* 1. SQL pre-filters by project + sorts by recency (loads max N candidates)
|
|
873
|
+
* 2. JS computes cosine similarity only on filtered candidates
|
|
874
|
+
*
|
|
875
|
+
* With 50k observations and maxCandidates=2000, loads only ~4% of data.
|
|
778
876
|
*/
|
|
779
877
|
async search(db, queryEmbedding, options = {}) {
|
|
780
878
|
const limit = options.limit || 10;
|
|
781
879
|
const threshold = options.threshold || 0.3;
|
|
880
|
+
const maxCandidates = options.maxCandidates || DEFAULT_MAX_CANDIDATES;
|
|
782
881
|
try {
|
|
783
|
-
|
|
882
|
+
const conditions = [];
|
|
883
|
+
const params = [];
|
|
884
|
+
if (options.project) {
|
|
885
|
+
conditions.push("o.project = ?");
|
|
886
|
+
params.push(options.project);
|
|
887
|
+
}
|
|
888
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
889
|
+
const sql = `
|
|
784
890
|
SELECT e.observation_id, e.embedding,
|
|
785
891
|
o.title, o.text, o.type, o.project, o.created_at, o.created_at_epoch
|
|
786
892
|
FROM observation_embeddings e
|
|
787
893
|
JOIN observations o ON o.id = e.observation_id
|
|
894
|
+
${whereClause}
|
|
895
|
+
ORDER BY o.created_at_epoch DESC
|
|
896
|
+
LIMIT ?
|
|
788
897
|
`;
|
|
789
|
-
|
|
790
|
-
if (options.project) {
|
|
791
|
-
sql += " WHERE o.project = ?";
|
|
792
|
-
params.push(options.project);
|
|
793
|
-
}
|
|
898
|
+
params.push(maxCandidates);
|
|
794
899
|
const rows = db.query(sql).all(...params);
|
|
795
900
|
const scored = [];
|
|
796
901
|
for (const row of rows) {
|
|
@@ -811,14 +916,15 @@ var VectorSearch = class {
|
|
|
811
916
|
}
|
|
812
917
|
}
|
|
813
918
|
scored.sort((a, b) => b.similarity - a.similarity);
|
|
919
|
+
logger.debug("VECTOR", `Search: ${rows.length} candidates \u2192 ${scored.length} above threshold \u2192 ${Math.min(scored.length, limit)} results`);
|
|
814
920
|
return scored.slice(0, limit);
|
|
815
921
|
} catch (error) {
|
|
816
|
-
logger.error("VECTOR", `
|
|
922
|
+
logger.error("VECTOR", `Vector search error: ${error}`);
|
|
817
923
|
return [];
|
|
818
924
|
}
|
|
819
925
|
}
|
|
820
926
|
/**
|
|
821
|
-
*
|
|
927
|
+
* Store embedding for an observation.
|
|
822
928
|
*/
|
|
823
929
|
async storeEmbedding(db, observationId, embedding, model) {
|
|
824
930
|
try {
|
|
@@ -834,18 +940,18 @@ var VectorSearch = class {
|
|
|
834
940
|
embedding.length,
|
|
835
941
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
836
942
|
);
|
|
837
|
-
logger.debug("VECTOR", `Embedding
|
|
943
|
+
logger.debug("VECTOR", `Embedding saved for observation ${observationId}`);
|
|
838
944
|
} catch (error) {
|
|
839
|
-
logger.error("VECTOR", `
|
|
945
|
+
logger.error("VECTOR", `Error saving embedding: ${error}`);
|
|
840
946
|
}
|
|
841
947
|
}
|
|
842
948
|
/**
|
|
843
|
-
*
|
|
949
|
+
* Generate embeddings for observations that don't have them yet.
|
|
844
950
|
*/
|
|
845
951
|
async backfillEmbeddings(db, batchSize = 50) {
|
|
846
952
|
const embeddingService2 = getEmbeddingService();
|
|
847
953
|
if (!await embeddingService2.initialize()) {
|
|
848
|
-
logger.warn("VECTOR", "Embedding service
|
|
954
|
+
logger.warn("VECTOR", "Embedding service not available, backfill skipped");
|
|
849
955
|
return 0;
|
|
850
956
|
}
|
|
851
957
|
const rows = db.query(`
|
|
@@ -871,11 +977,11 @@ var VectorSearch = class {
|
|
|
871
977
|
count++;
|
|
872
978
|
}
|
|
873
979
|
}
|
|
874
|
-
logger.info("VECTOR", `Backfill
|
|
980
|
+
logger.info("VECTOR", `Backfill completed: ${count}/${rows.length} embeddings generated`);
|
|
875
981
|
return count;
|
|
876
982
|
}
|
|
877
983
|
/**
|
|
878
|
-
*
|
|
984
|
+
* Embedding statistics.
|
|
879
985
|
*/
|
|
880
986
|
getStats(db) {
|
|
881
987
|
try {
|
|
@@ -942,21 +1048,21 @@ function knowledgeTypeBoost(type) {
|
|
|
942
1048
|
var HybridSearch = class {
|
|
943
1049
|
embeddingInitialized = false;
|
|
944
1050
|
/**
|
|
945
|
-
*
|
|
1051
|
+
* Initialize the embedding service (lazy, non-blocking)
|
|
946
1052
|
*/
|
|
947
1053
|
async initialize() {
|
|
948
1054
|
try {
|
|
949
1055
|
const embeddingService2 = getEmbeddingService();
|
|
950
1056
|
await embeddingService2.initialize();
|
|
951
1057
|
this.embeddingInitialized = embeddingService2.isAvailable();
|
|
952
|
-
logger.info("SEARCH", `HybridSearch
|
|
1058
|
+
logger.info("SEARCH", `HybridSearch initialized (embedding: ${this.embeddingInitialized ? "active" : "disabled"})`);
|
|
953
1059
|
} catch (error) {
|
|
954
|
-
logger.warn("SEARCH", "
|
|
1060
|
+
logger.warn("SEARCH", "Embedding initialization failed, using only FTS5", {}, error);
|
|
955
1061
|
this.embeddingInitialized = false;
|
|
956
1062
|
}
|
|
957
1063
|
}
|
|
958
1064
|
/**
|
|
959
|
-
*
|
|
1065
|
+
* Hybrid search with 4-signal scoring
|
|
960
1066
|
*/
|
|
961
1067
|
async search(db, query, options = {}) {
|
|
962
1068
|
const limit = options.limit || 10;
|
|
@@ -972,7 +1078,7 @@ var HybridSearch = class {
|
|
|
972
1078
|
const vectorResults = await vectorSearch2.search(db, queryEmbedding, {
|
|
973
1079
|
project: options.project,
|
|
974
1080
|
limit: limit * 2,
|
|
975
|
-
//
|
|
1081
|
+
// Fetch more results for ranking
|
|
976
1082
|
threshold: 0.3
|
|
977
1083
|
});
|
|
978
1084
|
for (const hit of vectorResults) {
|
|
@@ -989,10 +1095,10 @@ var HybridSearch = class {
|
|
|
989
1095
|
source: "vector"
|
|
990
1096
|
});
|
|
991
1097
|
}
|
|
992
|
-
logger.debug("SEARCH", `Vector search: ${vectorResults.length}
|
|
1098
|
+
logger.debug("SEARCH", `Vector search: ${vectorResults.length} results`);
|
|
993
1099
|
}
|
|
994
1100
|
} catch (error) {
|
|
995
|
-
logger.warn("SEARCH", "
|
|
1101
|
+
logger.warn("SEARCH", "Vector search failed, using only keyword", {}, error);
|
|
996
1102
|
}
|
|
997
1103
|
}
|
|
998
1104
|
try {
|
|
@@ -1022,9 +1128,9 @@ var HybridSearch = class {
|
|
|
1022
1128
|
});
|
|
1023
1129
|
}
|
|
1024
1130
|
}
|
|
1025
|
-
logger.debug("SEARCH", `Keyword search: ${keywordResults.length}
|
|
1131
|
+
logger.debug("SEARCH", `Keyword search: ${keywordResults.length} results`);
|
|
1026
1132
|
} catch (error) {
|
|
1027
|
-
logger.error("SEARCH", "
|
|
1133
|
+
logger.error("SEARCH", "Keyword search failed", {}, error);
|
|
1028
1134
|
}
|
|
1029
1135
|
if (rawItems.size === 0) return [];
|
|
1030
1136
|
const allFTS5Ranks = Array.from(rawItems.values()).filter((item) => item.fts5Rank !== null).map((item) => item.fts5Rank);
|