kiro-memory 1.9.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/plugin/dist/cli/contextkit.js +305 -174
- package/plugin/dist/hooks/agentSpawn.js +311 -180
- package/plugin/dist/hooks/kiro-hooks.js +299 -168
- package/plugin/dist/hooks/postToolUse.js +303 -172
- package/plugin/dist/hooks/stop.js +308 -177
- package/plugin/dist/hooks/userPromptSubmit.js +303 -172
- package/plugin/dist/index.js +303 -299
- package/plugin/dist/sdk/index.js +299 -172
- package/plugin/dist/services/search/EmbeddingService.js +88 -23
- package/plugin/dist/services/search/HybridSearch.js +190 -84
- package/plugin/dist/services/search/VectorSearch.js +128 -45
- package/plugin/dist/services/search/index.js +192 -223
- package/plugin/dist/services/sqlite/Database.js +55 -153
- package/plugin/dist/services/sqlite/Observations.js +23 -12
- package/plugin/dist/services/sqlite/Search.js +31 -19
- package/plugin/dist/services/sqlite/index.js +108 -183
- package/plugin/dist/viewer.css +1 -0
- package/plugin/dist/viewer.html +2 -179
- package/plugin/dist/viewer.js +15 -24942
- package/plugin/dist/viewer.js.map +7 -0
- package/plugin/dist/worker-service.js +158 -5551
- package/plugin/dist/worker-service.js.map +7 -0
|
@@ -22,13 +22,13 @@ __export(Search_exports, {
|
|
|
22
22
|
searchObservationsLIKE: () => searchObservationsLIKE,
|
|
23
23
|
searchSummariesFiltered: () => searchSummariesFiltered
|
|
24
24
|
});
|
|
25
|
-
import { existsSync as
|
|
25
|
+
import { existsSync as existsSync2, statSync } from "fs";
|
|
26
26
|
function escapeLikePattern(input) {
|
|
27
27
|
return input.replace(/[%_\\]/g, "\\$&");
|
|
28
28
|
}
|
|
29
29
|
function sanitizeFTS5Query(query) {
|
|
30
30
|
const trimmed = query.length > 1e4 ? query.substring(0, 1e4) : query;
|
|
31
|
-
const terms = trimmed.replace(/[""]/g, "").split(/\s+/).filter((t) => t.length > 0).slice(0, 100).map((t) => `"${t}"`);
|
|
31
|
+
const terms = trimmed.replace(/[""\u0022]/g, "").split(/\s+/).filter((t) => t.length > 0).slice(0, 100).map((t) => `"${t}"`);
|
|
32
32
|
return terms.join(" ");
|
|
33
33
|
}
|
|
34
34
|
function searchObservationsFTS(db, query, filters = {}) {
|
|
@@ -193,26 +193,38 @@ function getTimeline(db, anchorId, depthBefore = 5, depthAfter = 5) {
|
|
|
193
193
|
return [...before, ...self, ...after];
|
|
194
194
|
}
|
|
195
195
|
function getProjectStats(db, project) {
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
196
|
+
const sql = `
|
|
197
|
+
WITH
|
|
198
|
+
obs_stats AS (
|
|
199
|
+
SELECT
|
|
200
|
+
COUNT(*) as count,
|
|
201
|
+
COALESCE(SUM(discovery_tokens), 0) as discovery_tokens,
|
|
202
|
+
COALESCE(SUM(
|
|
203
|
+
CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
|
|
204
|
+
), 0) as read_tokens
|
|
205
|
+
FROM observations WHERE project = ?
|
|
206
|
+
),
|
|
207
|
+
sum_count AS (SELECT COUNT(*) as count FROM summaries WHERE project = ?),
|
|
208
|
+
ses_count AS (SELECT COUNT(*) as count FROM sessions WHERE project = ?),
|
|
209
|
+
prm_count AS (SELECT COUNT(*) as count FROM prompts WHERE project = ?)
|
|
210
|
+
SELECT
|
|
211
|
+
obs_stats.count as observations,
|
|
212
|
+
obs_stats.discovery_tokens,
|
|
213
|
+
obs_stats.read_tokens,
|
|
214
|
+
sum_count.count as summaries,
|
|
215
|
+
ses_count.count as sessions,
|
|
216
|
+
prm_count.count as prompts
|
|
217
|
+
FROM obs_stats, sum_count, ses_count, prm_count
|
|
218
|
+
`;
|
|
219
|
+
const row = db.query(sql).get(project, project, project, project);
|
|
220
|
+
const discoveryTokens = row?.discovery_tokens || 0;
|
|
221
|
+
const readTokens = row?.read_tokens || 0;
|
|
210
222
|
const savings = Math.max(0, discoveryTokens - readTokens);
|
|
211
223
|
return {
|
|
212
|
-
observations:
|
|
213
|
-
summaries:
|
|
214
|
-
sessions:
|
|
215
|
-
prompts:
|
|
224
|
+
observations: row?.observations || 0,
|
|
225
|
+
summaries: row?.summaries || 0,
|
|
226
|
+
sessions: row?.sessions || 0,
|
|
227
|
+
prompts: row?.prompts || 0,
|
|
216
228
|
tokenEconomics: { discoveryTokens, readTokens, savings }
|
|
217
229
|
};
|
|
218
230
|
}
|
|
@@ -230,7 +242,7 @@ function getStaleObservations(db, project) {
|
|
|
230
242
|
let isStale = false;
|
|
231
243
|
for (const filepath of files) {
|
|
232
244
|
try {
|
|
233
|
-
if (!
|
|
245
|
+
if (!existsSync2(filepath)) continue;
|
|
234
246
|
const stat = statSync(filepath);
|
|
235
247
|
if (stat.mtimeMs > obs.created_at_epoch) {
|
|
236
248
|
isStale = true;
|
|
@@ -346,9 +358,25 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
346
358
|
ORDER BY cnt DESC
|
|
347
359
|
`).all(project, minGroupSize);
|
|
348
360
|
if (groups.length === 0) return { merged: 0, removed: 0 };
|
|
349
|
-
|
|
350
|
-
|
|
361
|
+
if (options.dryRun) {
|
|
362
|
+
let totalMerged = 0;
|
|
363
|
+
let totalRemoved = 0;
|
|
364
|
+
for (const group of groups) {
|
|
365
|
+
const obsIds = group.ids.split(",").map(Number);
|
|
366
|
+
const placeholders = obsIds.map(() => "?").join(",");
|
|
367
|
+
const count = db.query(
|
|
368
|
+
`SELECT COUNT(*) as cnt FROM observations WHERE id IN (${placeholders})`
|
|
369
|
+
).get(...obsIds)?.cnt || 0;
|
|
370
|
+
if (count >= minGroupSize) {
|
|
371
|
+
totalMerged += 1;
|
|
372
|
+
totalRemoved += count - 1;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return { merged: totalMerged, removed: totalRemoved };
|
|
376
|
+
}
|
|
351
377
|
const runConsolidation = db.transaction(() => {
|
|
378
|
+
let merged = 0;
|
|
379
|
+
let removed = 0;
|
|
352
380
|
for (const group of groups) {
|
|
353
381
|
const obsIds = group.ids.split(",").map(Number);
|
|
354
382
|
const placeholders = obsIds.map(() => "?").join(",");
|
|
@@ -356,11 +384,6 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
356
384
|
`SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
|
|
357
385
|
).all(...obsIds);
|
|
358
386
|
if (observations.length < minGroupSize) continue;
|
|
359
|
-
if (options.dryRun) {
|
|
360
|
-
totalMerged += 1;
|
|
361
|
-
totalRemoved += observations.length - 1;
|
|
362
|
-
continue;
|
|
363
|
-
}
|
|
364
387
|
const keeper = observations[0];
|
|
365
388
|
const others = observations.slice(1);
|
|
366
389
|
const uniqueTexts = /* @__PURE__ */ new Set();
|
|
@@ -373,18 +396,18 @@ function consolidateObservations(db, project, options = {}) {
|
|
|
373
396
|
const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
|
|
374
397
|
db.run(
|
|
375
398
|
"UPDATE observations SET text = ?, title = ? WHERE id = ?",
|
|
376
|
-
[consolidatedText, `[
|
|
399
|
+
[consolidatedText, `[consolidated x${observations.length}] ${keeper.title}`, keeper.id]
|
|
377
400
|
);
|
|
378
401
|
const removeIds = others.map((o) => o.id);
|
|
379
402
|
const removePlaceholders = removeIds.map(() => "?").join(",");
|
|
380
403
|
db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
|
|
381
404
|
db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
|
|
382
|
-
|
|
383
|
-
|
|
405
|
+
merged += 1;
|
|
406
|
+
removed += removeIds.length;
|
|
384
407
|
}
|
|
408
|
+
return { merged, removed };
|
|
385
409
|
});
|
|
386
|
-
runConsolidation();
|
|
387
|
-
return { merged: totalMerged, removed: totalRemoved };
|
|
410
|
+
return runConsolidation();
|
|
388
411
|
}
|
|
389
412
|
var init_Observations = __esm({
|
|
390
413
|
"src/services/sqlite/Observations.ts"() {
|
|
@@ -392,12 +415,6 @@ var init_Observations = __esm({
|
|
|
392
415
|
}
|
|
393
416
|
});
|
|
394
417
|
|
|
395
|
-
// src/services/search/ChromaManager.ts
|
|
396
|
-
import { ChromaClient } from "chromadb";
|
|
397
|
-
import { join as join2 } from "path";
|
|
398
|
-
import { homedir as homedir2 } from "os";
|
|
399
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync2 } from "fs";
|
|
400
|
-
|
|
401
418
|
// src/utils/logger.ts
|
|
402
419
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "fs";
|
|
403
420
|
import { join } from "path";
|
|
@@ -617,135 +634,6 @@ ${data.stack}` : ` ${data.message}`;
|
|
|
617
634
|
};
|
|
618
635
|
var logger = new Logger();
|
|
619
636
|
|
|
620
|
-
// src/services/search/ChromaManager.ts
|
|
621
|
-
var VECTOR_DB_DIR = join2(homedir2(), ".contextkit", "vector-db");
|
|
622
|
-
var ChromaManager = class {
|
|
623
|
-
client;
|
|
624
|
-
collection = null;
|
|
625
|
-
isAvailable = false;
|
|
626
|
-
constructor() {
|
|
627
|
-
if (!existsSync2(VECTOR_DB_DIR)) {
|
|
628
|
-
mkdirSync2(VECTOR_DB_DIR, { recursive: true });
|
|
629
|
-
}
|
|
630
|
-
this.client = new ChromaClient({
|
|
631
|
-
path: process.env.CHROMADB_URL || "http://localhost:8000"
|
|
632
|
-
});
|
|
633
|
-
}
|
|
634
|
-
/**
|
|
635
|
-
* Initialize ChromaDB connection and collection
|
|
636
|
-
*/
|
|
637
|
-
async initialize() {
|
|
638
|
-
try {
|
|
639
|
-
await this.client.heartbeat();
|
|
640
|
-
this.collection = await this.client.getOrCreateCollection({
|
|
641
|
-
name: "kiro-memory-observations",
|
|
642
|
-
metadata: { description: "Kiro Memory observation embeddings" }
|
|
643
|
-
});
|
|
644
|
-
this.isAvailable = true;
|
|
645
|
-
logger.info("CHROMA", "ChromaDB initialized successfully");
|
|
646
|
-
return true;
|
|
647
|
-
} catch (error) {
|
|
648
|
-
logger.warn("CHROMA", "ChromaDB not available, falling back to SQLite search", {}, error);
|
|
649
|
-
this.isAvailable = false;
|
|
650
|
-
return false;
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
/**
|
|
654
|
-
* Add observation embedding to ChromaDB
|
|
655
|
-
*/
|
|
656
|
-
async addObservation(id, content, metadata) {
|
|
657
|
-
if (!this.isAvailable || !this.collection) {
|
|
658
|
-
logger.debug("CHROMA", "ChromaDB not available, skipping embedding");
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
try {
|
|
662
|
-
await this.collection.add({
|
|
663
|
-
ids: [id],
|
|
664
|
-
documents: [content],
|
|
665
|
-
metadatas: [metadata]
|
|
666
|
-
});
|
|
667
|
-
logger.debug("CHROMA", `Added observation ${id} to vector DB`);
|
|
668
|
-
} catch (error) {
|
|
669
|
-
logger.error("CHROMA", `Failed to add observation ${id}`, {}, error);
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
/**
|
|
673
|
-
* Search observations by semantic similarity
|
|
674
|
-
*/
|
|
675
|
-
async search(query, options = {}) {
|
|
676
|
-
if (!this.isAvailable || !this.collection) {
|
|
677
|
-
logger.debug("CHROMA", "ChromaDB not available, returning empty results");
|
|
678
|
-
return [];
|
|
679
|
-
}
|
|
680
|
-
try {
|
|
681
|
-
const where = options.project ? { project: options.project } : void 0;
|
|
682
|
-
const results = await this.collection.query({
|
|
683
|
-
queryTexts: [query],
|
|
684
|
-
nResults: options.limit || 10,
|
|
685
|
-
where
|
|
686
|
-
});
|
|
687
|
-
const hits = [];
|
|
688
|
-
if (results.ids && results.ids[0]) {
|
|
689
|
-
for (let i = 0; i < results.ids[0].length; i++) {
|
|
690
|
-
hits.push({
|
|
691
|
-
id: results.ids[0][i],
|
|
692
|
-
content: results.documents?.[0]?.[i] || "",
|
|
693
|
-
metadata: results.metadatas?.[0]?.[i] || {},
|
|
694
|
-
distance: results.distances?.[0]?.[i] || 0
|
|
695
|
-
});
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
logger.debug("CHROMA", `Search returned ${hits.length} results`);
|
|
699
|
-
return hits;
|
|
700
|
-
} catch (error) {
|
|
701
|
-
logger.error("CHROMA", "Search failed", {}, error);
|
|
702
|
-
return [];
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
/**
|
|
706
|
-
* Delete observation from ChromaDB
|
|
707
|
-
*/
|
|
708
|
-
async deleteObservation(id) {
|
|
709
|
-
if (!this.isAvailable || !this.collection) {
|
|
710
|
-
return;
|
|
711
|
-
}
|
|
712
|
-
try {
|
|
713
|
-
await this.collection.delete({ ids: [id] });
|
|
714
|
-
logger.debug("CHROMA", `Deleted observation ${id}`);
|
|
715
|
-
} catch (error) {
|
|
716
|
-
logger.error("CHROMA", `Failed to delete observation ${id}`, {}, error);
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
/**
|
|
720
|
-
* Check if ChromaDB is available
|
|
721
|
-
*/
|
|
722
|
-
isChromaAvailable() {
|
|
723
|
-
return this.isAvailable;
|
|
724
|
-
}
|
|
725
|
-
/**
|
|
726
|
-
* Get collection stats
|
|
727
|
-
*/
|
|
728
|
-
async getStats() {
|
|
729
|
-
if (!this.isAvailable || !this.collection) {
|
|
730
|
-
return { count: 0 };
|
|
731
|
-
}
|
|
732
|
-
try {
|
|
733
|
-
const count = await this.collection.count();
|
|
734
|
-
return { count };
|
|
735
|
-
} catch (error) {
|
|
736
|
-
logger.error("CHROMA", "Failed to get stats", {}, error);
|
|
737
|
-
return { count: 0 };
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
};
|
|
741
|
-
var chromaManager = null;
|
|
742
|
-
function getChromaManager() {
|
|
743
|
-
if (!chromaManager) {
|
|
744
|
-
chromaManager = new ChromaManager();
|
|
745
|
-
}
|
|
746
|
-
return chromaManager;
|
|
747
|
-
}
|
|
748
|
-
|
|
749
637
|
// src/services/search/EmbeddingService.ts
|
|
750
638
|
var EmbeddingService = class {
|
|
751
639
|
provider = null;
|
|
@@ -753,8 +641,8 @@ var EmbeddingService = class {
|
|
|
753
641
|
initialized = false;
|
|
754
642
|
initializing = null;
|
|
755
643
|
/**
|
|
756
|
-
*
|
|
757
|
-
*
|
|
644
|
+
* Initialize the embedding service.
|
|
645
|
+
* Tries fastembed, then @huggingface/transformers, then fallback to null.
|
|
758
646
|
*/
|
|
759
647
|
async initialize() {
|
|
760
648
|
if (this.initialized) return this.provider !== null;
|
|
@@ -775,11 +663,11 @@ var EmbeddingService = class {
|
|
|
775
663
|
});
|
|
776
664
|
this.provider = "fastembed";
|
|
777
665
|
this.initialized = true;
|
|
778
|
-
logger.info("EMBEDDING", "
|
|
666
|
+
logger.info("EMBEDDING", "Initialized with fastembed (BGE-small-en-v1.5)");
|
|
779
667
|
return true;
|
|
780
668
|
}
|
|
781
669
|
} catch (error) {
|
|
782
|
-
logger.debug("EMBEDDING", `fastembed
|
|
670
|
+
logger.debug("EMBEDDING", `fastembed not available: ${error}`);
|
|
783
671
|
}
|
|
784
672
|
try {
|
|
785
673
|
const transformers = await import("@huggingface/transformers");
|
|
@@ -790,20 +678,20 @@ var EmbeddingService = class {
|
|
|
790
678
|
});
|
|
791
679
|
this.provider = "transformers";
|
|
792
680
|
this.initialized = true;
|
|
793
|
-
logger.info("EMBEDDING", "
|
|
681
|
+
logger.info("EMBEDDING", "Initialized with @huggingface/transformers (all-MiniLM-L6-v2)");
|
|
794
682
|
return true;
|
|
795
683
|
}
|
|
796
684
|
} catch (error) {
|
|
797
|
-
logger.debug("EMBEDDING", `@huggingface/transformers
|
|
685
|
+
logger.debug("EMBEDDING", `@huggingface/transformers not available: ${error}`);
|
|
798
686
|
}
|
|
799
687
|
this.provider = null;
|
|
800
688
|
this.initialized = true;
|
|
801
|
-
logger.warn("EMBEDDING", "
|
|
689
|
+
logger.warn("EMBEDDING", "No embedding provider available, semantic search disabled");
|
|
802
690
|
return false;
|
|
803
691
|
}
|
|
804
692
|
/**
|
|
805
|
-
*
|
|
806
|
-
*
|
|
693
|
+
* Generate embedding for a single text.
|
|
694
|
+
* Returns Float32Array with 384 dimensions, or null if not available.
|
|
807
695
|
*/
|
|
808
696
|
async embed(text) {
|
|
809
697
|
if (!this.initialized) await this.initialize();
|
|
@@ -816,46 +704,111 @@ var EmbeddingService = class {
|
|
|
816
704
|
return await this._embedTransformers(truncated);
|
|
817
705
|
}
|
|
818
706
|
} catch (error) {
|
|
819
|
-
logger.error("EMBEDDING", `
|
|
707
|
+
logger.error("EMBEDDING", `Error generating embedding: ${error}`);
|
|
820
708
|
}
|
|
821
709
|
return null;
|
|
822
710
|
}
|
|
823
711
|
/**
|
|
824
|
-
*
|
|
712
|
+
* Generate embeddings in batch.
|
|
713
|
+
* Uses native batch support when available (fastembed, transformers),
|
|
714
|
+
* falls back to serial processing on batch failure.
|
|
825
715
|
*/
|
|
826
716
|
async embedBatch(texts) {
|
|
827
717
|
if (!this.initialized) await this.initialize();
|
|
828
718
|
if (!this.provider || !this.model) return texts.map(() => null);
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
}
|
|
835
|
-
|
|
719
|
+
if (texts.length === 0) return [];
|
|
720
|
+
const truncated = texts.map((t) => t.substring(0, 2e3));
|
|
721
|
+
try {
|
|
722
|
+
if (this.provider === "fastembed") {
|
|
723
|
+
return await this._embedBatchFastembed(truncated);
|
|
724
|
+
} else if (this.provider === "transformers") {
|
|
725
|
+
return await this._embedBatchTransformers(truncated);
|
|
836
726
|
}
|
|
727
|
+
} catch (error) {
|
|
728
|
+
logger.warn("EMBEDDING", `Batch embedding failed, falling back to serial: ${error}`);
|
|
837
729
|
}
|
|
838
|
-
return
|
|
730
|
+
return this._embedBatchSerial(truncated);
|
|
839
731
|
}
|
|
840
732
|
/**
|
|
841
|
-
*
|
|
733
|
+
* Check if the service is available.
|
|
842
734
|
*/
|
|
843
735
|
isAvailable() {
|
|
844
736
|
return this.initialized && this.provider !== null;
|
|
845
737
|
}
|
|
846
738
|
/**
|
|
847
|
-
*
|
|
739
|
+
* Name of the active provider.
|
|
848
740
|
*/
|
|
849
741
|
getProvider() {
|
|
850
742
|
return this.provider;
|
|
851
743
|
}
|
|
852
744
|
/**
|
|
853
|
-
*
|
|
745
|
+
* Embedding vector dimensions.
|
|
854
746
|
*/
|
|
855
747
|
getDimensions() {
|
|
856
748
|
return 384;
|
|
857
749
|
}
|
|
858
|
-
// ---
|
|
750
|
+
// --- Batch implementations ---
|
|
751
|
+
/**
|
|
752
|
+
* Native batch embedding with fastembed.
|
|
753
|
+
* FlagEmbedding.embed() accepts string[] and returns an async iterable of batches.
|
|
754
|
+
*/
|
|
755
|
+
async _embedBatchFastembed(texts) {
|
|
756
|
+
const results = [];
|
|
757
|
+
const embeddings = this.model.embed(texts, texts.length);
|
|
758
|
+
for await (const batch of embeddings) {
|
|
759
|
+
if (batch) {
|
|
760
|
+
for (const vec of batch) {
|
|
761
|
+
results.push(vec instanceof Float32Array ? vec : new Float32Array(vec));
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
while (results.length < texts.length) {
|
|
766
|
+
results.push(null);
|
|
767
|
+
}
|
|
768
|
+
return results;
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* Batch embedding with @huggingface/transformers pipeline.
|
|
772
|
+
* The pipeline accepts string[] and returns a Tensor with shape [N, dims].
|
|
773
|
+
*/
|
|
774
|
+
async _embedBatchTransformers(texts) {
|
|
775
|
+
const output = await this.model(texts, {
|
|
776
|
+
pooling: "mean",
|
|
777
|
+
normalize: true
|
|
778
|
+
});
|
|
779
|
+
if (!output?.data) {
|
|
780
|
+
return texts.map(() => null);
|
|
781
|
+
}
|
|
782
|
+
const dims = this.getDimensions();
|
|
783
|
+
const data = output.data instanceof Float32Array ? output.data : new Float32Array(output.data);
|
|
784
|
+
const results = [];
|
|
785
|
+
for (let i = 0; i < texts.length; i++) {
|
|
786
|
+
const offset = i * dims;
|
|
787
|
+
if (offset + dims <= data.length) {
|
|
788
|
+
results.push(data.slice(offset, offset + dims));
|
|
789
|
+
} else {
|
|
790
|
+
results.push(null);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
return results;
|
|
794
|
+
}
|
|
795
|
+
/**
|
|
796
|
+
* Serial fallback: embed texts one at a time.
|
|
797
|
+
* Used when native batch fails.
|
|
798
|
+
*/
|
|
799
|
+
async _embedBatchSerial(texts) {
|
|
800
|
+
const results = [];
|
|
801
|
+
for (const text of texts) {
|
|
802
|
+
try {
|
|
803
|
+
const embedding = await this.embed(text);
|
|
804
|
+
results.push(embedding);
|
|
805
|
+
} catch {
|
|
806
|
+
results.push(null);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
return results;
|
|
810
|
+
}
|
|
811
|
+
// --- Single-text provider implementations ---
|
|
859
812
|
async _embedFastembed(text) {
|
|
860
813
|
const embeddings = this.model.embed([text], 1);
|
|
861
814
|
for await (const batch of embeddings) {
|
|
@@ -886,17 +839,21 @@ function getEmbeddingService() {
|
|
|
886
839
|
}
|
|
887
840
|
|
|
888
841
|
// src/services/search/VectorSearch.ts
|
|
842
|
+
var DEFAULT_MAX_CANDIDATES = 2e3;
|
|
889
843
|
function cosineSimilarity(a, b) {
|
|
890
|
-
|
|
844
|
+
const len = a.length;
|
|
845
|
+
if (len !== b.length) return 0;
|
|
891
846
|
let dotProduct = 0;
|
|
892
847
|
let normA = 0;
|
|
893
848
|
let normB = 0;
|
|
894
|
-
for (let i = 0; i <
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
849
|
+
for (let i = 0; i < len; i++) {
|
|
850
|
+
const ai = a[i];
|
|
851
|
+
const bi = b[i];
|
|
852
|
+
dotProduct += ai * bi;
|
|
853
|
+
normA += ai * ai;
|
|
854
|
+
normB += bi * bi;
|
|
898
855
|
}
|
|
899
|
-
const denominator = Math.sqrt(normA
|
|
856
|
+
const denominator = Math.sqrt(normA * normB);
|
|
900
857
|
if (denominator === 0) return 0;
|
|
901
858
|
return dotProduct / denominator;
|
|
902
859
|
}
|
|
@@ -909,23 +866,36 @@ function bufferToFloat32(buf) {
|
|
|
909
866
|
}
|
|
910
867
|
var VectorSearch = class {
|
|
911
868
|
/**
|
|
912
|
-
*
|
|
869
|
+
* Semantic search with SQL pre-filtering for scalability.
|
|
870
|
+
*
|
|
871
|
+
* 2-phase strategy:
|
|
872
|
+
* 1. SQL pre-filters by project + sorts by recency (loads max N candidates)
|
|
873
|
+
* 2. JS computes cosine similarity only on filtered candidates
|
|
874
|
+
*
|
|
875
|
+
* With 50k observations and maxCandidates=2000, loads only ~4% of data.
|
|
913
876
|
*/
|
|
914
877
|
async search(db, queryEmbedding, options = {}) {
|
|
915
878
|
const limit = options.limit || 10;
|
|
916
879
|
const threshold = options.threshold || 0.3;
|
|
880
|
+
const maxCandidates = options.maxCandidates || DEFAULT_MAX_CANDIDATES;
|
|
917
881
|
try {
|
|
918
|
-
|
|
882
|
+
const conditions = [];
|
|
883
|
+
const params = [];
|
|
884
|
+
if (options.project) {
|
|
885
|
+
conditions.push("o.project = ?");
|
|
886
|
+
params.push(options.project);
|
|
887
|
+
}
|
|
888
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
889
|
+
const sql = `
|
|
919
890
|
SELECT e.observation_id, e.embedding,
|
|
920
891
|
o.title, o.text, o.type, o.project, o.created_at, o.created_at_epoch
|
|
921
892
|
FROM observation_embeddings e
|
|
922
893
|
JOIN observations o ON o.id = e.observation_id
|
|
894
|
+
${whereClause}
|
|
895
|
+
ORDER BY o.created_at_epoch DESC
|
|
896
|
+
LIMIT ?
|
|
923
897
|
`;
|
|
924
|
-
|
|
925
|
-
if (options.project) {
|
|
926
|
-
sql += " WHERE o.project = ?";
|
|
927
|
-
params.push(options.project);
|
|
928
|
-
}
|
|
898
|
+
params.push(maxCandidates);
|
|
929
899
|
const rows = db.query(sql).all(...params);
|
|
930
900
|
const scored = [];
|
|
931
901
|
for (const row of rows) {
|
|
@@ -946,14 +916,15 @@ var VectorSearch = class {
|
|
|
946
916
|
}
|
|
947
917
|
}
|
|
948
918
|
scored.sort((a, b) => b.similarity - a.similarity);
|
|
919
|
+
logger.debug("VECTOR", `Search: ${rows.length} candidates \u2192 ${scored.length} above threshold \u2192 ${Math.min(scored.length, limit)} results`);
|
|
949
920
|
return scored.slice(0, limit);
|
|
950
921
|
} catch (error) {
|
|
951
|
-
logger.error("VECTOR", `
|
|
922
|
+
logger.error("VECTOR", `Vector search error: ${error}`);
|
|
952
923
|
return [];
|
|
953
924
|
}
|
|
954
925
|
}
|
|
955
926
|
/**
|
|
956
|
-
*
|
|
927
|
+
* Store embedding for an observation.
|
|
957
928
|
*/
|
|
958
929
|
async storeEmbedding(db, observationId, embedding, model) {
|
|
959
930
|
try {
|
|
@@ -969,18 +940,18 @@ var VectorSearch = class {
|
|
|
969
940
|
embedding.length,
|
|
970
941
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
971
942
|
);
|
|
972
|
-
logger.debug("VECTOR", `Embedding
|
|
943
|
+
logger.debug("VECTOR", `Embedding saved for observation ${observationId}`);
|
|
973
944
|
} catch (error) {
|
|
974
|
-
logger.error("VECTOR", `
|
|
945
|
+
logger.error("VECTOR", `Error saving embedding: ${error}`);
|
|
975
946
|
}
|
|
976
947
|
}
|
|
977
948
|
/**
|
|
978
|
-
*
|
|
949
|
+
* Generate embeddings for observations that don't have them yet.
|
|
979
950
|
*/
|
|
980
951
|
async backfillEmbeddings(db, batchSize = 50) {
|
|
981
952
|
const embeddingService2 = getEmbeddingService();
|
|
982
953
|
if (!await embeddingService2.initialize()) {
|
|
983
|
-
logger.warn("VECTOR", "Embedding service
|
|
954
|
+
logger.warn("VECTOR", "Embedding service not available, backfill skipped");
|
|
984
955
|
return 0;
|
|
985
956
|
}
|
|
986
957
|
const rows = db.query(`
|
|
@@ -1006,11 +977,11 @@ var VectorSearch = class {
|
|
|
1006
977
|
count++;
|
|
1007
978
|
}
|
|
1008
979
|
}
|
|
1009
|
-
logger.info("VECTOR", `Backfill
|
|
980
|
+
logger.info("VECTOR", `Backfill completed: ${count}/${rows.length} embeddings generated`);
|
|
1010
981
|
return count;
|
|
1011
982
|
}
|
|
1012
983
|
/**
|
|
1013
|
-
*
|
|
984
|
+
* Embedding statistics.
|
|
1014
985
|
*/
|
|
1015
986
|
getStats(db) {
|
|
1016
987
|
try {
|
|
@@ -1098,21 +1069,21 @@ function estimateTokens(text) {
|
|
|
1098
1069
|
var HybridSearch = class {
|
|
1099
1070
|
embeddingInitialized = false;
|
|
1100
1071
|
/**
|
|
1101
|
-
*
|
|
1072
|
+
* Initialize the embedding service (lazy, non-blocking)
|
|
1102
1073
|
*/
|
|
1103
1074
|
async initialize() {
|
|
1104
1075
|
try {
|
|
1105
1076
|
const embeddingService2 = getEmbeddingService();
|
|
1106
1077
|
await embeddingService2.initialize();
|
|
1107
1078
|
this.embeddingInitialized = embeddingService2.isAvailable();
|
|
1108
|
-
logger.info("SEARCH", `HybridSearch
|
|
1079
|
+
logger.info("SEARCH", `HybridSearch initialized (embedding: ${this.embeddingInitialized ? "active" : "disabled"})`);
|
|
1109
1080
|
} catch (error) {
|
|
1110
|
-
logger.warn("SEARCH", "
|
|
1081
|
+
logger.warn("SEARCH", "Embedding initialization failed, using only FTS5", {}, error);
|
|
1111
1082
|
this.embeddingInitialized = false;
|
|
1112
1083
|
}
|
|
1113
1084
|
}
|
|
1114
1085
|
/**
|
|
1115
|
-
*
|
|
1086
|
+
* Hybrid search with 4-signal scoring
|
|
1116
1087
|
*/
|
|
1117
1088
|
async search(db, query, options = {}) {
|
|
1118
1089
|
const limit = options.limit || 10;
|
|
@@ -1128,7 +1099,7 @@ var HybridSearch = class {
|
|
|
1128
1099
|
const vectorResults = await vectorSearch2.search(db, queryEmbedding, {
|
|
1129
1100
|
project: options.project,
|
|
1130
1101
|
limit: limit * 2,
|
|
1131
|
-
//
|
|
1102
|
+
// Fetch more results for ranking
|
|
1132
1103
|
threshold: 0.3
|
|
1133
1104
|
});
|
|
1134
1105
|
for (const hit of vectorResults) {
|
|
@@ -1145,10 +1116,10 @@ var HybridSearch = class {
|
|
|
1145
1116
|
source: "vector"
|
|
1146
1117
|
});
|
|
1147
1118
|
}
|
|
1148
|
-
logger.debug("SEARCH", `Vector search: ${vectorResults.length}
|
|
1119
|
+
logger.debug("SEARCH", `Vector search: ${vectorResults.length} results`);
|
|
1149
1120
|
}
|
|
1150
1121
|
} catch (error) {
|
|
1151
|
-
logger.warn("SEARCH", "
|
|
1122
|
+
logger.warn("SEARCH", "Vector search failed, using only keyword", {}, error);
|
|
1152
1123
|
}
|
|
1153
1124
|
}
|
|
1154
1125
|
try {
|
|
@@ -1178,9 +1149,9 @@ var HybridSearch = class {
|
|
|
1178
1149
|
});
|
|
1179
1150
|
}
|
|
1180
1151
|
}
|
|
1181
|
-
logger.debug("SEARCH", `Keyword search: ${keywordResults.length}
|
|
1152
|
+
logger.debug("SEARCH", `Keyword search: ${keywordResults.length} results`);
|
|
1182
1153
|
} catch (error) {
|
|
1183
|
-
logger.error("SEARCH", "
|
|
1154
|
+
logger.error("SEARCH", "Keyword search failed", {}, error);
|
|
1184
1155
|
}
|
|
1185
1156
|
if (rawItems.size === 0) return [];
|
|
1186
1157
|
const allFTS5Ranks = Array.from(rawItems.values()).filter((item) => item.fts5Rank !== null).map((item) => item.fts5Rank);
|
|
@@ -1233,14 +1204,12 @@ function getHybridSearch() {
|
|
|
1233
1204
|
}
|
|
1234
1205
|
export {
|
|
1235
1206
|
CONTEXT_WEIGHTS,
|
|
1236
|
-
ChromaManager,
|
|
1237
1207
|
HybridSearch,
|
|
1238
1208
|
KNOWLEDGE_TYPE_BOOST,
|
|
1239
1209
|
SEARCH_WEIGHTS,
|
|
1240
1210
|
accessRecencyScore,
|
|
1241
1211
|
computeCompositeScore,
|
|
1242
1212
|
estimateTokens,
|
|
1243
|
-
getChromaManager,
|
|
1244
1213
|
getHybridSearch,
|
|
1245
1214
|
knowledgeTypeBoost,
|
|
1246
1215
|
normalizeFTS5Rank,
|