kirograph 0.20.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/bin/commands/data.js +52 -0
- package/dist/bin/commands/data.js.map +2 -2
- package/dist/bin/commands/docs.js +2 -1
- package/dist/bin/commands/docs.js.map +2 -2
- package/dist/bin/commands/gain.js +21 -1
- package/dist/bin/commands/gain.js.map +2 -2
- package/dist/bin/commands/status.js +23 -1
- package/dist/bin/commands/status.js.map +2 -2
- package/dist/bin/installer/config-prompt.js +30 -1
- package/dist/bin/installer/config-prompt.js.map +2 -2
- package/dist/bin/installer/index.js +108 -0
- package/dist/bin/installer/index.js.map +2 -2
- package/dist/bin/kirograph.js +1 -1
- package/dist/config.js +19 -2
- package/dist/config.js.map +2 -2
- package/dist/core/pipeline.js +3 -1
- package/dist/core/pipeline.js.map +2 -2
- package/dist/data/indexer.js +4 -4
- package/dist/data/indexer.js.map +2 -2
- package/dist/data/linker.js +9 -5
- package/dist/data/linker.js.map +2 -2
- package/dist/data/linker.test.js +60 -0
- package/dist/data/linker.test.js.map +7 -0
- package/dist/data/parsers/__fixtures__/pdf-inspector-stub.js +41 -0
- package/dist/data/parsers/__fixtures__/pdf-inspector-stub.js.map +7 -0
- package/dist/data/parsers/index.js +3 -1
- package/dist/data/parsers/index.js.map +2 -2
- package/dist/data/parsers/pdf.js +95 -0
- package/dist/data/parsers/pdf.js.map +7 -0
- package/dist/data/parsers/pdf.test.js +125 -0
- package/dist/data/parsers/pdf.test.js.map +7 -0
- package/dist/data/queries.js +39 -2
- package/dist/data/queries.js.map +2 -2
- package/dist/data/types.js.map +1 -1
- package/dist/db/data-schema.sql +2 -1
- package/dist/db/database.js +7 -0
- package/dist/db/database.js.map +2 -2
- package/dist/docs/indexer.js +2 -1
- package/dist/docs/indexer.js.map +2 -2
- package/dist/docs/queries.js +2 -1
- package/dist/docs/queries.js.map +2 -2
- package/dist/docs/vectors.js +46 -5
- package/dist/docs/vectors.js.map +2 -2
- package/dist/index.js +1 -0
- package/dist/index.js.map +2 -2
- package/dist/mcp/tools.js +1 -1
- package/dist/mcp/tools.js.map +2 -2
- package/dist/memory/index.js +2 -1
- package/dist/memory/index.js.map +2 -2
- package/dist/memory/vectors.js +52 -8
- package/dist/memory/vectors.js.map +3 -3
- package/dist/types.js.map +2 -2
- package/dist/vectors/index.js +66 -3
- package/dist/vectors/index.js.map +2 -2
- package/dist/vectors/turbovec-index.js +189 -0
- package/dist/vectors/turbovec-index.js.map +7 -0
- package/package.json +7 -1
package/dist/memory/index.js
CHANGED
|
@@ -44,7 +44,8 @@ class MemoryManager {
|
|
|
44
44
|
this.db = db;
|
|
45
45
|
this.projectRoot = projectRoot;
|
|
46
46
|
this.memDb = new import_database.MemoryDatabase(db);
|
|
47
|
-
|
|
47
|
+
const kirographDir = require("path").join(projectRoot, ".kirograph");
|
|
48
|
+
this.vectorMgr = new import_vectors.MemoryVectorManager(config, this.memDb, kirographDir);
|
|
48
49
|
this.cavemanMode = config.cavemanMode ?? "off";
|
|
49
50
|
this.excludePatterns = config.memoryExcludePatterns ?? [];
|
|
50
51
|
this.watchmenChecker = config.enableWatchmen ? new import_watchmen.WatchmenChecker(config.watchmenThreshold) : null;
|
package/dist/memory/index.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/memory/index.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * KiroGraph Memory \u2014 Public API\n *\n * MemoryManager is the single entry point for all memory operations.\n * Coordinates compression, storage, symbol detection, and embedding.\n */\n\nimport type { KiroGraphConfig } from '../config';\nimport type {\n MemObservation,\n MemObservationInput,\n MemSession,\n ScoredObservation,\n MemSearchOptions,\n MemTimelineOptions,\n MemStats,\n CompressResult,\n WatchmenReadyResult,\n} from './types';\nimport { MemoryDatabase } from './database';\nimport { compressObservation, type CavemanMode } from './compress';\nimport { detectSymbols } from './symbols';\nimport { MemoryVectorManager } from './vectors';\nimport { WatchmenChecker } from '../watchmen';\nimport { logDebug } from '../errors';\n\nexport { MemoryDatabase } from './database';\nexport { compressObservation, extractIdentifiers, type CavemanMode } from './compress';\nexport { detectSymbols } from './symbols';\nexport { MemoryVectorManager } from './vectors';\nexport * from './types';\n\n// \u2500\u2500 MemoryManager \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport class MemoryManager {\n private memDb: MemoryDatabase;\n private vectorMgr: MemoryVectorManager;\n private config: KiroGraphConfig;\n private db: any; // raw sqlite handle for symbol detection\n private cavemanMode: CavemanMode;\n private excludePatterns: string[];\n private watchmenChecker: WatchmenChecker | null;\n private projectRoot: string;\n\n constructor(config: KiroGraphConfig, db: any, projectRoot = process.cwd()) {\n this.config = config;\n this.db = db;\n this.projectRoot = projectRoot;\n this.memDb = new MemoryDatabase(db);\n this.vectorMgr = new MemoryVectorManager(config, this.memDb);\n this.cavemanMode = (config as any).cavemanMode ?? 'off';\n this.excludePatterns = (config as any).memoryExcludePatterns ?? [];\n this.watchmenChecker = config.enableWatchmen\n ? new WatchmenChecker(config.watchmenThreshold)\n : null;\n }\n\n /**\n * Initialize memory tables. Must be called before any operations.\n */\n initialize(): void {\n this.memDb.initialize();\n }\n\n /**\n * Check if memory is enabled in config.\n */\n static isEnabled(config: KiroGraphConfig): boolean {\n return !!(config as any).enableMemory;\n }\n\n // \u2500\u2500 Store \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * Store an observation. Handles:\n * 1. Privacy stripping (<private> blocks)\n * 2. Exclude pattern checking\n * 3. Caveman compression (if enabled)\n * 4. Deduplication (content hash)\n * 5. Symbol detection and linking\n * 6. Embedding (if enabled)\n * 7. Watchmen threshold check (if enabled) \u2014 returns WatchmenReadyResult when synthesis should run\n *\n * Returns the observation ID, a WatchmenReadyResult, or null if skipped (duplicate or excluded).\n */\n async store(input: MemObservationInput, ide = 'kiro'): Promise<string | WatchmenReadyResult | null> {\n let text = input.content;\n\n // 1. Strip <private> blocks\n text = MemoryDatabase.stripPrivate(text);\n if (!text.trim()) return null;\n\n // 2. Check exclude patterns\n if (this.shouldExclude(text)) return null;\n\n // 3. Compress (if caveman is on)\n let contentRaw: string | undefined;\n if (this.cavemanMode !== 'off') {\n const result = compressObservation(text, this.cavemanMode);\n contentRaw = text;\n text = result.compressed;\n }\n\n // 4. Get or create session\n const cwd = process.cwd();\n const sessionTimeout = (this.config as any).memorySessionTimeout ?? 7200;\n const sessionId = this.memDb.getOrCreateSession(ide, cwd, sessionTimeout);\n\n // 5. Insert observation (deduplicates via content_hash)\n const id = this.memDb.insertObservation(text, {\n contentRaw: this.cavemanMode !== 'off' ? contentRaw : undefined,\n kind: input.kind ?? 'note',\n source: input.source ?? 'manual',\n tags: input.tags,\n sessionId,\n });\n\n if (!id) {\n logDebug('Memory: duplicate observation skipped');\n return null;\n }\n\n // 6. Detect and link symbols\n const symbols = detectSymbols(text, this.db);\n if (symbols.length > 0) {\n this.memDb.linkToSymbols(id, symbols.map(s => s.qualifiedName));\n }\n\n // Also detect in raw text if different\n if (contentRaw) {\n const rawSymbols = detectSymbols(contentRaw, this.db);\n const extraSymbols = rawSymbols.filter(\n s => !symbols.some(existing => existing.qualifiedName === s.qualifiedName)\n );\n if (extraSymbols.length > 0) {\n this.memDb.linkToSymbols(id, extraSymbols.map(s => s.qualifiedName));\n }\n }\n\n // 7. Embed (async, non-blocking for the store operation)\n const obs = this.memDb.getObservation(id);\n if (obs) {\n await this.vectorMgr.embedObservation(obs);\n }\n\n // 8. Watchmen threshold check\n if (this.watchmenChecker && input.kind !== 'summary') {\n const check = this.watchmenChecker.shouldSynthesize(this.memDb);\n if (check.ready) {\n return this.watchmenChecker.buildReadyResponse(id, check.pendingCount, this.projectRoot);\n }\n }\n\n return id;\n }\n\n // \u2500\u2500 Search \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * Hybrid search: combines FTS5 and vector search with configurable alpha.\n */\n async search(query: string, opts: MemSearchOptions = {}): Promise<ScoredObservation[]> {\n const limit = opts.limit ?? 10;\n const alpha = opts.alpha ?? (this.config as any).memorySearchAlpha ?? 0.5;\n\n // FTS search\n const ftsResults = this.memDb.searchFTS(query, { ...opts, limit: limit * 2 });\n\n // Vector search (if enabled and no model mismatch)\n let vectorResults: ScoredObservation[] = [];\n if (this.vectorMgr.isEnabled() && !this.vectorMgr.hasModelMismatch()) {\n vectorResults = await this.vectorMgr.search(query, limit * 2);\n }\n\n // If only one source has results, return it directly\n if (vectorResults.length === 0) return ftsResults.slice(0, limit);\n if (ftsResults.length === 0) return vectorResults.slice(0, limit);\n\n // Hybrid merge: normalize scores and blend\n const merged = this.mergeResults(ftsResults, vectorResults, alpha);\n return merged.slice(0, limit);\n }\n\n /**\n * Get observations linked to specific symbols.\n * Used by kirograph_context and kirograph_impact.\n */\n getLinkedObservations(qualifiedNames: string[], limit = 3, threshold = 0.3): ScoredObservation[] {\n const results = this.memDb.getLinkedObservationsForSymbols(qualifiedNames, limit * 2);\n return results.filter(r => r.score >= threshold).slice(0, limit);\n }\n\n // \u2500\u2500 Timeline \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * List sessions with their observation counts.\n */\n timeline(opts: MemTimelineOptions = {}): { sessions: MemSession[]; observations: Map<string, MemObservation[]> } {\n const limit = opts.limit ?? 5;\n\n if (opts.sessionId) {\n const session = this.memDb.getSession(opts.sessionId);\n if (!session) return { sessions: [], observations: new Map() };\n const obs = this.memDb.getObservationsBySession(opts.sessionId);\n const map = new Map<string, MemObservation[]>();\n map.set(opts.sessionId, obs);\n return { sessions: [session], observations: map };\n }\n\n const sessions = this.memDb.listSessions(limit);\n const observations = new Map<string, MemObservation[]>();\n for (const session of sessions) {\n const obs = this.memDb.getObservationsBySession(session.id, 10);\n observations.set(session.id, obs);\n }\n return { sessions, observations };\n }\n\n // \u2500\u2500 Status \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n getStats(): MemStats {\n const modelName = (this.config as any).embeddingModel ?? 'nomic-ai/nomic-embed-text-v1.5';\n return this.memDb.getStats(modelName);\n }\n\n // \u2500\u2500 Session management \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n endCurrentSession(ide = 'kiro'): void {\n const cwd = process.cwd();\n const sessionTimeout = (this.config as any).memorySessionTimeout ?? 7200;\n // Find active session and close it\n const sessionId = this.memDb.getOrCreateSession(ide, cwd, sessionTimeout);\n this.memDb.endSession(sessionId);\n }\n\n // \u2500\u2500 Maintenance \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n prune(olderThanMs: number): number {\n const cutoff = Date.now() - olderThanMs;\n return this.memDb.prune(cutoff);\n }\n\n async reembed(batchSize = 32): Promise<number> {\n return this.vectorMgr.reembed(batchSize);\n }\n\n lint(): { staleLinks: number; modelMismatch: boolean; staleSessions: number } {\n const staleLinks = this.memDb.findStaleLinks().length;\n const modelMismatch = this.vectorMgr.hasModelMismatch();\n const sessionTimeout = ((this.config as any).memorySessionTimeout ?? 7200) * 1000;\n const staleSessions = this.memDb.closeStaleSessionsOlderThan(sessionTimeout);\n return { staleLinks, modelMismatch, staleSessions };\n }\n\n removeStaleLinks(): number {\n return this.memDb.removeStaleLinks();\n }\n\n // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n private shouldExclude(text: string): boolean {\n if (this.excludePatterns.length === 0) return false;\n\n // Simple check: if any exclude pattern appears as a substring in the text\n // This is a basic implementation; could use picomatch for glob matching\n for (const pattern of this.excludePatterns) {\n const normalized = pattern.replace(/\\*/g, '');\n if (normalized && text.includes(normalized)) return true;\n }\n return false;\n }\n\n private mergeResults(\n ftsResults: ScoredObservation[],\n vectorResults: ScoredObservation[],\n alpha: number\n ): ScoredObservation[] {\n // Normalize FTS scores (rank is negative in FTS5, lower = better)\n const maxFts = Math.max(...ftsResults.map(r => r.score), 1);\n const normalizedFts = new Map<string, number>();\n for (const r of ftsResults) {\n normalizedFts.set(r.observation.id, r.score / maxFts);\n }\n\n // Vector scores are already 0-1 (cosine similarity)\n const normalizedVec = new Map<string, number>();\n for (const r of vectorResults) {\n normalizedVec.set(r.observation.id, r.score);\n }\n\n // Merge: collect all unique observation IDs\n const allIds = new Set([...normalizedFts.keys(), ...normalizedVec.keys()]);\n const merged: ScoredObservation[] = [];\n\n // Build a lookup for observations\n const obsMap = new Map<string, MemObservation>();\n for (const r of ftsResults) obsMap.set(r.observation.id, r.observation);\n for (const r of vectorResults) obsMap.set(r.observation.id, r.observation);\n\n for (const id of allIds) {\n const ftsScore = normalizedFts.get(id) ?? 0;\n const vecScore = normalizedVec.get(id) ?? 0;\n const hybridScore = (1 - alpha) * ftsScore + alpha * vecScore;\n const obs = obsMap.get(id)!;\n merged.push({ observation: obs, score: hybridScore, scoreSource: 'hybrid' });\n }\n\n merged.sort((a, b) => b.score - a.score);\n return merged;\n }\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBA,sBAA+B;AAC/B,sBAAsD;AACtD,qBAA8B;AAC9B,qBAAoC;AACpC,sBAAgC;AAChC,oBAAyB;AAEzB,IAAAA,mBAA+B;AAC/B,IAAAC,mBAA0E;AAC1E,IAAAC,kBAA8B;AAC9B,IAAAC,kBAAoC;AACpC,2BAAc,oBA9Bd;AAkCO,MAAM,cAAc;AAAA,EAUzB,YAAY,QAAyB,IAAS,cAAc,QAAQ,IAAI,GAAG;AACzE,SAAK,SAAS;AACd,SAAK,KAAK;AACV,SAAK,cAAc;AACnB,SAAK,QAAQ,IAAI,+BAAe,EAAE;AAClC,SAAK,YAAY,IAAI,mCAAoB,QAAQ,KAAK,
|
|
4
|
+
"sourcesContent": ["/**\n * KiroGraph Memory \u2014 Public API\n *\n * MemoryManager is the single entry point for all memory operations.\n * Coordinates compression, storage, symbol detection, and embedding.\n */\n\nimport type { KiroGraphConfig } from '../config';\nimport type {\n MemObservation,\n MemObservationInput,\n MemSession,\n ScoredObservation,\n MemSearchOptions,\n MemTimelineOptions,\n MemStats,\n CompressResult,\n WatchmenReadyResult,\n} from './types';\nimport { MemoryDatabase } from './database';\nimport { compressObservation, type CavemanMode } from './compress';\nimport { detectSymbols } from './symbols';\nimport { MemoryVectorManager } from './vectors';\nimport { WatchmenChecker } from '../watchmen';\nimport { logDebug } from '../errors';\n\nexport { MemoryDatabase } from './database';\nexport { compressObservation, extractIdentifiers, type CavemanMode } from './compress';\nexport { detectSymbols } from './symbols';\nexport { MemoryVectorManager } from './vectors';\nexport * from './types';\n\n// \u2500\u2500 MemoryManager \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport class MemoryManager {\n private memDb: MemoryDatabase;\n private vectorMgr: MemoryVectorManager;\n private config: KiroGraphConfig;\n private db: any; // raw sqlite handle for symbol detection\n private cavemanMode: CavemanMode;\n private excludePatterns: string[];\n private watchmenChecker: WatchmenChecker | null;\n private projectRoot: string;\n\n constructor(config: KiroGraphConfig, db: any, projectRoot = process.cwd()) {\n this.config = config;\n this.db = db;\n this.projectRoot = projectRoot;\n this.memDb = new MemoryDatabase(db);\n const kirographDir = require('path').join(projectRoot, '.kirograph');\n this.vectorMgr = new MemoryVectorManager(config, this.memDb, kirographDir);\n this.cavemanMode = (config as any).cavemanMode ?? 'off';\n this.excludePatterns = (config as any).memoryExcludePatterns ?? [];\n this.watchmenChecker = config.enableWatchmen\n ? new WatchmenChecker(config.watchmenThreshold)\n : null;\n }\n\n /**\n * Initialize memory tables. Must be called before any operations.\n */\n initialize(): void {\n this.memDb.initialize();\n }\n\n /**\n * Check if memory is enabled in config.\n */\n static isEnabled(config: KiroGraphConfig): boolean {\n return !!(config as any).enableMemory;\n }\n\n // \u2500\u2500 Store \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * Store an observation. Handles:\n * 1. Privacy stripping (<private> blocks)\n * 2. Exclude pattern checking\n * 3. Caveman compression (if enabled)\n * 4. Deduplication (content hash)\n * 5. Symbol detection and linking\n * 6. Embedding (if enabled)\n * 7. Watchmen threshold check (if enabled) \u2014 returns WatchmenReadyResult when synthesis should run\n *\n * Returns the observation ID, a WatchmenReadyResult, or null if skipped (duplicate or excluded).\n */\n async store(input: MemObservationInput, ide = 'kiro'): Promise<string | WatchmenReadyResult | null> {\n let text = input.content;\n\n // 1. Strip <private> blocks\n text = MemoryDatabase.stripPrivate(text);\n if (!text.trim()) return null;\n\n // 2. Check exclude patterns\n if (this.shouldExclude(text)) return null;\n\n // 3. Compress (if caveman is on)\n let contentRaw: string | undefined;\n if (this.cavemanMode !== 'off') {\n const result = compressObservation(text, this.cavemanMode);\n contentRaw = text;\n text = result.compressed;\n }\n\n // 4. Get or create session\n const cwd = process.cwd();\n const sessionTimeout = (this.config as any).memorySessionTimeout ?? 7200;\n const sessionId = this.memDb.getOrCreateSession(ide, cwd, sessionTimeout);\n\n // 5. Insert observation (deduplicates via content_hash)\n const id = this.memDb.insertObservation(text, {\n contentRaw: this.cavemanMode !== 'off' ? contentRaw : undefined,\n kind: input.kind ?? 'note',\n source: input.source ?? 'manual',\n tags: input.tags,\n sessionId,\n });\n\n if (!id) {\n logDebug('Memory: duplicate observation skipped');\n return null;\n }\n\n // 6. Detect and link symbols\n const symbols = detectSymbols(text, this.db);\n if (symbols.length > 0) {\n this.memDb.linkToSymbols(id, symbols.map(s => s.qualifiedName));\n }\n\n // Also detect in raw text if different\n if (contentRaw) {\n const rawSymbols = detectSymbols(contentRaw, this.db);\n const extraSymbols = rawSymbols.filter(\n s => !symbols.some(existing => existing.qualifiedName === s.qualifiedName)\n );\n if (extraSymbols.length > 0) {\n this.memDb.linkToSymbols(id, extraSymbols.map(s => s.qualifiedName));\n }\n }\n\n // 7. Embed (async, non-blocking for the store operation)\n const obs = this.memDb.getObservation(id);\n if (obs) {\n await this.vectorMgr.embedObservation(obs);\n }\n\n // 8. Watchmen threshold check\n if (this.watchmenChecker && input.kind !== 'summary') {\n const check = this.watchmenChecker.shouldSynthesize(this.memDb);\n if (check.ready) {\n return this.watchmenChecker.buildReadyResponse(id, check.pendingCount, this.projectRoot);\n }\n }\n\n return id;\n }\n\n // \u2500\u2500 Search \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * Hybrid search: combines FTS5 and vector search with configurable alpha.\n */\n async search(query: string, opts: MemSearchOptions = {}): Promise<ScoredObservation[]> {\n const limit = opts.limit ?? 10;\n const alpha = opts.alpha ?? (this.config as any).memorySearchAlpha ?? 0.5;\n\n // FTS search\n const ftsResults = this.memDb.searchFTS(query, { ...opts, limit: limit * 2 });\n\n // Vector search (if enabled and no model mismatch)\n let vectorResults: ScoredObservation[] = [];\n if (this.vectorMgr.isEnabled() && !this.vectorMgr.hasModelMismatch()) {\n vectorResults = await this.vectorMgr.search(query, limit * 2);\n }\n\n // If only one source has results, return it directly\n if (vectorResults.length === 0) return ftsResults.slice(0, limit);\n if (ftsResults.length === 0) return vectorResults.slice(0, limit);\n\n // Hybrid merge: normalize scores and blend\n const merged = this.mergeResults(ftsResults, vectorResults, alpha);\n return merged.slice(0, limit);\n }\n\n /**\n * Get observations linked to specific symbols.\n * Used by kirograph_context and kirograph_impact.\n */\n getLinkedObservations(qualifiedNames: string[], limit = 3, threshold = 0.3): ScoredObservation[] {\n const results = this.memDb.getLinkedObservationsForSymbols(qualifiedNames, limit * 2);\n return results.filter(r => r.score >= threshold).slice(0, limit);\n }\n\n // \u2500\u2500 Timeline \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n /**\n * List sessions with their observation counts.\n */\n timeline(opts: MemTimelineOptions = {}): { sessions: MemSession[]; observations: Map<string, MemObservation[]> } {\n const limit = opts.limit ?? 5;\n\n if (opts.sessionId) {\n const session = this.memDb.getSession(opts.sessionId);\n if (!session) return { sessions: [], observations: new Map() };\n const obs = this.memDb.getObservationsBySession(opts.sessionId);\n const map = new Map<string, MemObservation[]>();\n map.set(opts.sessionId, obs);\n return { sessions: [session], observations: map };\n }\n\n const sessions = this.memDb.listSessions(limit);\n const observations = new Map<string, MemObservation[]>();\n for (const session of sessions) {\n const obs = this.memDb.getObservationsBySession(session.id, 10);\n observations.set(session.id, obs);\n }\n return { sessions, observations };\n }\n\n // \u2500\u2500 Status \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n getStats(): MemStats {\n const modelName = (this.config as any).embeddingModel ?? 'nomic-ai/nomic-embed-text-v1.5';\n return this.memDb.getStats(modelName);\n }\n\n // \u2500\u2500 Session management \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n endCurrentSession(ide = 'kiro'): void {\n const cwd = process.cwd();\n const sessionTimeout = (this.config as any).memorySessionTimeout ?? 7200;\n // Find active session and close it\n const sessionId = this.memDb.getOrCreateSession(ide, cwd, sessionTimeout);\n this.memDb.endSession(sessionId);\n }\n\n // \u2500\u2500 Maintenance \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n prune(olderThanMs: number): number {\n const cutoff = Date.now() - olderThanMs;\n return this.memDb.prune(cutoff);\n }\n\n async reembed(batchSize = 32): Promise<number> {\n return this.vectorMgr.reembed(batchSize);\n }\n\n lint(): { staleLinks: number; modelMismatch: boolean; staleSessions: number } {\n const staleLinks = this.memDb.findStaleLinks().length;\n const modelMismatch = this.vectorMgr.hasModelMismatch();\n const sessionTimeout = ((this.config as any).memorySessionTimeout ?? 7200) * 1000;\n const staleSessions = this.memDb.closeStaleSessionsOlderThan(sessionTimeout);\n return { staleLinks, modelMismatch, staleSessions };\n }\n\n removeStaleLinks(): number {\n return this.memDb.removeStaleLinks();\n }\n\n // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n private shouldExclude(text: string): boolean {\n if (this.excludePatterns.length === 0) return false;\n\n // Simple check: if any exclude pattern appears as a substring in the text\n // This is a basic implementation; could use picomatch for glob matching\n for (const pattern of this.excludePatterns) {\n const normalized = pattern.replace(/\\*/g, '');\n if (normalized && text.includes(normalized)) return true;\n }\n return false;\n }\n\n private mergeResults(\n ftsResults: ScoredObservation[],\n vectorResults: ScoredObservation[],\n alpha: number\n ): ScoredObservation[] {\n // Normalize FTS scores (rank is negative in FTS5, lower = better)\n const maxFts = Math.max(...ftsResults.map(r => r.score), 1);\n const normalizedFts = new Map<string, number>();\n for (const r of ftsResults) {\n normalizedFts.set(r.observation.id, r.score / maxFts);\n }\n\n // Vector scores are already 0-1 (cosine similarity)\n const normalizedVec = new Map<string, number>();\n for (const r of vectorResults) {\n normalizedVec.set(r.observation.id, r.score);\n }\n\n // Merge: collect all unique observation IDs\n const allIds = new Set([...normalizedFts.keys(), ...normalizedVec.keys()]);\n const merged: ScoredObservation[] = [];\n\n // Build a lookup for observations\n const obsMap = new Map<string, MemObservation>();\n for (const r of ftsResults) obsMap.set(r.observation.id, r.observation);\n for (const r of vectorResults) obsMap.set(r.observation.id, r.observation);\n\n for (const id of allIds) {\n const ftsScore = normalizedFts.get(id) ?? 0;\n const vecScore = normalizedVec.get(id) ?? 0;\n const hybridScore = (1 - alpha) * ftsScore + alpha * vecScore;\n const obs = obsMap.get(id)!;\n merged.push({ observation: obs, score: hybridScore, scoreSource: 'hybrid' });\n }\n\n merged.sort((a, b) => b.score - a.score);\n return merged;\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBA,sBAA+B;AAC/B,sBAAsD;AACtD,qBAA8B;AAC9B,qBAAoC;AACpC,sBAAgC;AAChC,oBAAyB;AAEzB,IAAAA,mBAA+B;AAC/B,IAAAC,mBAA0E;AAC1E,IAAAC,kBAA8B;AAC9B,IAAAC,kBAAoC;AACpC,2BAAc,oBA9Bd;AAkCO,MAAM,cAAc;AAAA,EAUzB,YAAY,QAAyB,IAAS,cAAc,QAAQ,IAAI,GAAG;AACzE,SAAK,SAAS;AACd,SAAK,KAAK;AACV,SAAK,cAAc;AACnB,SAAK,QAAQ,IAAI,+BAAe,EAAE;AAClC,UAAM,eAAe,QAAQ,MAAM,EAAE,KAAK,aAAa,YAAY;AACnE,SAAK,YAAY,IAAI,mCAAoB,QAAQ,KAAK,OAAO,YAAY;AACzE,SAAK,cAAe,OAAe,eAAe;AAClD,SAAK,kBAAmB,OAAe,yBAAyB,CAAC;AACjE,SAAK,kBAAkB,OAAO,iBAC1B,IAAI,gCAAgB,OAAO,iBAAiB,IAC5C;AAAA,EACN;AAAA;AAAA;AAAA;AAAA,EAKA,aAAmB;AACjB,SAAK,MAAM,WAAW;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,UAAU,QAAkC;AACjD,WAAO,CAAC,CAAE,OAAe;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBA,MAAM,MAAM,OAA4B,MAAM,QAAsD;AAClG,QAAI,OAAO,MAAM;AAGjB,WAAO,+BAAe,aAAa,IAAI;AACvC,QAAI,CAAC,KAAK,KAAK,EAAG,QAAO;AAGzB,QAAI,KAAK,cAAc,IAAI,EAAG,QAAO;AAGrC,QAAI;AACJ,QAAI,KAAK,gBAAgB,OAAO;AAC9B,YAAM,aAAS,qCAAoB,MAAM,KAAK,WAAW;AACzD,mBAAa;AACb,aAAO,OAAO;AAAA,IAChB;AAGA,UAAM,MAAM,QAAQ,IAAI;AACxB,UAAM,iBAAkB,KAAK,OAAe,wBAAwB;AACpE,UAAM,YAAY,KAAK,MAAM,mBAAmB,KAAK,KAAK,cAAc;AAGxE,UAAM,KAAK,KAAK,MAAM,kBAAkB,MAAM;AAAA,MAC5C,YAAY,KAAK,gBAAgB,QAAQ,aAAa;AAAA,MACtD,MAAM,MAAM,QAAQ;AAAA,MACpB,QAAQ,MAAM,UAAU;AAAA,MACxB,MAAM,MAAM;AAAA,MACZ;AAAA,IACF,CAAC;AAED,QAAI,CAAC,IAAI;AACP,kCAAS,uCAAuC;AAChD,aAAO;AAAA,IACT;AAGA,UAAM,cAAU,8BAAc,MAAM,KAAK,EAAE;AAC3C,QAAI,QAAQ,SAAS,GAAG;AACtB,WAAK,MAAM,cAAc,IAAI,QAAQ,IAAI,OAAK,EAAE,aAAa,CAAC;AAAA,IAChE;AAGA,QAAI,YAAY;AACd,YAAM,iBAAa,8BAAc,YAAY,KAAK,EAAE;AACpD,YAAM,eAAe,WAAW;AAAA,QAC9B,OAAK,CAAC,QAAQ,KAAK,cAAY,SAAS,kBAAkB,EAAE,aAAa;AAAA,MAC3E;AACA,UAAI,aAAa,SAAS,GAAG;AAC3B,aAAK,MAAM,cAAc,IAAI,aAAa,IAAI,OAAK,EAAE,aAAa,CAAC;AAAA,MACrE;AAAA,IACF;AAGA,UAAM,MAAM,KAAK,MAAM,eAAe,EAAE;AACxC,QAAI,KAAK;AACP,YAAM,KAAK,UAAU,iBAAiB,GAAG;AAAA,IAC3C;AAGA,QAAI,KAAK,mBAAmB,MAAM,SAAS,WAAW;AACpD,YAAM,QAAQ,KAAK,gBAAgB,iBAAiB,KAAK,KAAK;AAC9D,UAAI,MAAM,OAAO;AACf,eAAO,KAAK,gBAAgB,mBAAmB,IAAI,MAAM,cAAc,KAAK,WAAW;AAAA,MACzF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,OAAO,OAAe,OAAyB,CAAC,GAAiC;AACrF,UAAM,QAAQ,KAAK,SAAS;AAC5B,UAAM,QAAQ,KAAK,SAAU,KAAK,OAAe,qBAAqB;AAGtE,UAAM,aAAa,KAAK,MAAM,UAAU,OAAO,EAAE,GAAG,MAAM,OAAO,QAAQ,EAAE,CAAC;AAG5E,QAAI,gBAAqC,CAAC;AAC1C,QAAI,KAAK,UAAU,UAAU,KAAK,CAAC,KAAK,UAAU,iBAAiB,GAAG;AACpE,sBAAgB,MAAM,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC;AAAA,IAC9D;AAGA,QAAI,cAAc,WAAW,EAAG,QAAO,WAAW,MAAM,GAAG,KAAK;AAChE,QAAI,WAAW,WAAW,EAAG,QAAO,cAAc,MAAM,GAAG,KAAK;AAGhE,UAAM,SAAS,KAAK,aAAa,YAAY,eAAe,KAAK;AACjE,WAAO,OAAO,MAAM,GAAG,KAAK;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,sBAAsB,gBAA0B,QAAQ,GAAG,YAAY,KAA0B;AAC/F,UAAM,UAAU,KAAK,MAAM,gCAAgC,gBAAgB,QAAQ,CAAC;AACpF,WAAO,QAAQ,OAAO,OAAK,EAAE,SAAS,SAAS,EAAE,MAAM,GAAG,KAAK;AAAA,EACjE;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,OAA2B,CAAC,GAA4E;AAC/G,UAAM,QAAQ,KAAK,SAAS;AAE5B,QAAI,KAAK,WAAW;AAClB,YAAM,UAAU,KAAK,MAAM,WAAW,KAAK,SAAS;AACpD,UAAI,CAAC,QAAS,QAAO,EAAE,UAAU,CAAC,GAAG,cAAc,oBAAI,IAAI,EAAE;AAC7D,YAAM,MAAM,KAAK,MAAM,yBAAyB,KAAK,SAAS;AAC9D,YAAM,MAAM,oBAAI,IAA8B;AAC9C,UAAI,IAAI,KAAK,WAAW,GAAG;AAC3B,aAAO,EAAE,UAAU,CAAC,OAAO,GAAG,cAAc,IAAI;AAAA,IAClD;AAEA,UAAM,WAAW,KAAK,MAAM,aAAa,KAAK;AAC9C,UAAM,eAAe,oBAAI,IAA8B;AACvD,eAAW,WAAW,UAAU;AAC9B,YAAM,MAAM,KAAK,MAAM,yBAAyB,QAAQ,IAAI,EAAE;AAC9D,mBAAa,IAAI,QAAQ,IAAI,GAAG;AAAA,IAClC;AACA,WAAO,EAAE,UAAU,aAAa;AAAA,EAClC;AAAA;AAAA,EAIA,WAAqB;AACnB,UAAM,YAAa,KAAK,OAAe,kBAAkB;AACzD,WAAO,KAAK,MAAM,SAAS,SAAS;AAAA,EACtC;AAAA;AAAA,EAIA,kBAAkB,MAAM,QAAc;AACpC,UAAM,MAAM,QAAQ,IAAI;AACxB,UAAM,iBAAkB,KAAK,OAAe,wBAAwB;AAEpE,UAAM,YAAY,KAAK,MAAM,mBAAmB,KAAK,KAAK,cAAc;AACxE,SAAK,MAAM,WAAW,SAAS;AAAA,EACjC;AAAA;AAAA,EAIA,MAAM,aAA6B;AACjC,UAAM,SAAS,KAAK,IAAI,IAAI;AAC5B,WAAO,KAAK,MAAM,MAAM,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,QAAQ,YAAY,IAAqB;AAC7C,WAAO,KAAK,UAAU,QAAQ,SAAS;AAAA,EACzC;AAAA,EAEA,OAA8E;AAC5E,UAAM,aAAa,KAAK,MAAM,eAAe,EAAE;AAC/C,UAAM,gBAAgB,KAAK,UAAU,iBAAiB;AACtD,UAAM,kBAAmB,KAAK,OAAe,wBAAwB,QAAQ;AAC7E,UAAM,gBAAgB,KAAK,MAAM,4BAA4B,cAAc;AAC3E,WAAO,EAAE,YAAY,eAAe,cAAc;AAAA,EACpD;AAAA,EAEA,mBAA2B;AACzB,WAAO,KAAK,MAAM,iBAAiB;AAAA,EACrC;AAAA;AAAA,EAIQ,cAAc,MAAuB;AAC3C,QAAI,KAAK,gBAAgB,WAAW,EAAG,QAAO;AAI9C,eAAW,WAAW,KAAK,iBAAiB;AAC1C,YAAM,aAAa,QAAQ,QAAQ,OAAO,EAAE;AAC5C,UAAI,cAAc,KAAK,SAAS,UAAU,EAAG,QAAO;AAAA,IACtD;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aACN,YACA,eACA,OACqB;AAErB,UAAM,SAAS,KAAK,IAAI,GAAG,WAAW,IAAI,OAAK,EAAE,KAAK,GAAG,CAAC;AAC1D,UAAM,gBAAgB,oBAAI,IAAoB;AAC9C,eAAW,KAAK,YAAY;AAC1B,oBAAc,IAAI,EAAE,YAAY,IAAI,EAAE,QAAQ,MAAM;AAAA,IACtD;AAGA,UAAM,gBAAgB,oBAAI,IAAoB;AAC9C,eAAW,KAAK,eAAe;AAC7B,oBAAc,IAAI,EAAE,YAAY,IAAI,EAAE,KAAK;AAAA,IAC7C;AAGA,UAAM,SAAS,oBAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,cAAc,KAAK,CAAC,CAAC;AACzE,UAAM,SAA8B,CAAC;AAGrC,UAAM,SAAS,oBAAI,IAA4B;AAC/C,eAAW,KAAK,WAAY,QAAO,IAAI,EAAE,YAAY,IAAI,EAAE,WAAW;AACtE,eAAW,KAAK,cAAe,QAAO,IAAI,EAAE,YAAY,IAAI,EAAE,WAAW;AAEzE,eAAW,MAAM,QAAQ;AACvB,YAAM,WAAW,cAAc,IAAI,EAAE,KAAK;AAC1C,YAAM,WAAW,cAAc,IAAI,EAAE,KAAK;AAC1C,YAAM,eAAe,IAAI,SAAS,WAAW,QAAQ;AACrD,YAAM,MAAM,OAAO,IAAI,EAAE;AACzB,aAAO,KAAK,EAAE,aAAa,KAAK,OAAO,aAAa,aAAa,SAAS,CAAC;AAAA,IAC7E;AAEA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO;AAAA,EACT;AACF;",
|
|
6
6
|
"names": ["import_database", "import_compress", "import_symbols", "import_vectors"]
|
|
7
7
|
}
|
package/dist/memory/vectors.js
CHANGED
|
@@ -32,6 +32,7 @@ __export(vectors_exports, {
|
|
|
32
32
|
});
|
|
33
33
|
module.exports = __toCommonJS(vectors_exports);
|
|
34
34
|
var import_errors = require("../errors");
|
|
35
|
+
var import_turboquant_index = require("../vectors/turboquant-index");
|
|
35
36
|
const MAX_TOKEN_CHARS = 2e3;
|
|
36
37
|
let transformers = null;
|
|
37
38
|
let pipeline = null;
|
|
@@ -71,13 +72,27 @@ function cosine(a, b) {
|
|
|
71
72
|
return denom === 0 ? 0 : dot / denom;
|
|
72
73
|
}
|
|
73
74
|
class MemoryVectorManager {
|
|
74
|
-
constructor(config, memDb) {
|
|
75
|
+
constructor(config, memDb, kirographDir) {
|
|
76
|
+
this.tqIndex = null;
|
|
77
|
+
this.tqInitialized = false;
|
|
75
78
|
this.config = config;
|
|
76
79
|
this.memDb = memDb;
|
|
77
80
|
this.modelName = config.embeddingModel ?? "nomic-ai/nomic-embed-text-v1.5";
|
|
78
81
|
const { homedir } = require("os");
|
|
79
|
-
const
|
|
80
|
-
this.cacheDir =
|
|
82
|
+
const pathMod = require("path");
|
|
83
|
+
this.cacheDir = pathMod.join(homedir(), ".kirograph", "models");
|
|
84
|
+
if (kirographDir && config.turboquantMemDocs) {
|
|
85
|
+
const bits = config.turboquantBits ?? 3;
|
|
86
|
+
const dim = config.embeddingDim ?? 768;
|
|
87
|
+
this.tqIndex = new import_turboquant_index.TurboQuantIndex(kirographDir, "turboquant-mem.bin", dim, bits);
|
|
88
|
+
this._kirographDir = kirographDir;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Lazily initialize the TurboQuant index on first use. */
|
|
92
|
+
async ensureTQReady() {
|
|
93
|
+
if (this.tqInitialized) return;
|
|
94
|
+
this.tqInitialized = true;
|
|
95
|
+
if (this.tqIndex) await this.tqIndex.initialize();
|
|
81
96
|
}
|
|
82
97
|
/**
|
|
83
98
|
* Check if embeddings are enabled in config.
|
|
@@ -96,6 +111,10 @@ class MemoryVectorManager {
|
|
|
96
111
|
if (embedding) {
|
|
97
112
|
const buffer = Buffer.from(embedding.buffer);
|
|
98
113
|
this.memDb.insertVector(obs.id, buffer, this.modelName);
|
|
114
|
+
if (this.tqIndex) {
|
|
115
|
+
await this.ensureTQReady();
|
|
116
|
+
this.tqIndex.upsert(obs.id, embedding);
|
|
117
|
+
}
|
|
99
118
|
}
|
|
100
119
|
} catch (err) {
|
|
101
120
|
(0, import_errors.logWarn)(`Failed to embed observation ${obs.id}: ${err}`);
|
|
@@ -126,22 +145,31 @@ class MemoryVectorManager {
|
|
|
126
145
|
try {
|
|
127
146
|
const queryEmbedding = await this.embed(query);
|
|
128
147
|
if (!queryEmbedding) return [];
|
|
148
|
+
if (this.tqIndex) {
|
|
149
|
+
await this.ensureTQReady();
|
|
150
|
+
if (this.tqIndex.isAvailable()) {
|
|
151
|
+
const hits = this.tqIndex.searchWithScores(queryEmbedding, limit);
|
|
152
|
+
const results2 = [];
|
|
153
|
+
for (const { id, score } of hits) {
|
|
154
|
+
const obs = this.memDb.getObservation(id);
|
|
155
|
+
if (obs) results2.push({ observation: obs, score, scoreSource: "vector" });
|
|
156
|
+
}
|
|
157
|
+
return results2;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
129
160
|
const allVectors = this.memDb.getAllVectors(this.modelName);
|
|
130
161
|
if (allVectors.length === 0) return [];
|
|
131
162
|
const scored = [];
|
|
132
163
|
for (const { observationId, embedding } of allVectors) {
|
|
133
164
|
const vec = new Float32Array(embedding.buffer, embedding.byteOffset, embedding.byteLength / 4);
|
|
134
|
-
|
|
135
|
-
scored.push({ observationId, score });
|
|
165
|
+
scored.push({ observationId, score: cosine(queryEmbedding, vec) });
|
|
136
166
|
}
|
|
137
167
|
scored.sort((a, b) => b.score - a.score);
|
|
138
168
|
const topN = scored.slice(0, limit);
|
|
139
169
|
const results = [];
|
|
140
170
|
for (const { observationId, score } of topN) {
|
|
141
171
|
const obs = this.memDb.getObservation(observationId);
|
|
142
|
-
if (obs) {
|
|
143
|
-
results.push({ observation: obs, score, scoreSource: "vector" });
|
|
144
|
-
}
|
|
172
|
+
if (obs) results.push({ observation: obs, score, scoreSource: "vector" });
|
|
145
173
|
}
|
|
146
174
|
return results;
|
|
147
175
|
} catch (err) {
|
|
@@ -160,6 +188,10 @@ class MemoryVectorManager {
|
|
|
160
188
|
*/
|
|
161
189
|
async reembed(batchSize = 32) {
|
|
162
190
|
this.memDb.deleteVectors();
|
|
191
|
+
if (this.tqIndex) {
|
|
192
|
+
this.tqIndex.close();
|
|
193
|
+
this.tqInitialized = false;
|
|
194
|
+
}
|
|
163
195
|
const db = this.memDb.db;
|
|
164
196
|
const rows = db.all("SELECT * FROM mem_observations ORDER BY created_at");
|
|
165
197
|
const observations = rows.map((row) => ({
|
|
@@ -178,6 +210,18 @@ class MemoryVectorManager {
|
|
|
178
210
|
const batch = observations.slice(i, i + batchSize);
|
|
179
211
|
embedded += await this.embedBatch(batch);
|
|
180
212
|
}
|
|
213
|
+
if (this.tqIndex?.isAvailable() && this._kirographDir) {
|
|
214
|
+
await this.tqIndex.save();
|
|
215
|
+
const stats = this.tqIndex.memoryStats();
|
|
216
|
+
const dim = this.config.embeddingDim ?? 768;
|
|
217
|
+
const rawBytes = embedded * dim * 4;
|
|
218
|
+
(0, import_turboquant_index.writeTurboQuantStats)(this._kirographDir, {
|
|
219
|
+
memEnabled: true,
|
|
220
|
+
memCount: embedded,
|
|
221
|
+
memActualBytes: stats.actualBytes,
|
|
222
|
+
memRawBytes: rawBytes
|
|
223
|
+
});
|
|
224
|
+
}
|
|
181
225
|
return embedded;
|
|
182
226
|
}
|
|
183
227
|
// ── Private ────────────────────────────────────────────────────────────────
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/memory/vectors.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * KiroGraph Memory \u2014 Vector embedding and search\n *\n * Reuses the same embedding model and semantic engine as the code graph,\n * but operates on mem_vectors table independently.\n */\n\nimport type { KiroGraphConfig } from '../config';\nimport type { MemObservation, ScoredObservation } from './types';\nimport type { MemoryDatabase } from './database';\nimport { logDebug, logWarn, logError } from '../errors';\n\nconst MAX_TOKEN_CHARS = 2000;\n\n// \u2500\u2500 Embedder (reuses the same pipeline as code vectors) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nlet transformers: typeof import('@huggingface/transformers') | null = null;\nlet pipeline: any = null;\nlet pipelineModel: string | null = null;\n\nasync function getTransformers() {\n if (!transformers) {\n transformers = await import('@huggingface/transformers');\n }\n return transformers;\n}\n\nasync function getPipeline(modelName: string, cacheDir: string) {\n if (pipeline && pipelineModel === modelName) return pipeline;\n\n const tf = await getTransformers();\n pipeline = await tf.pipeline('feature-extraction', modelName, {\n cache_dir: cacheDir,\n dtype: 'fp32',\n } as any);\n pipelineModel = modelName;\n return pipeline;\n}\n\n/**\n * Build searchable text from an observation.\n * Truncated to MAX_TOKEN_CHARS.\n */\nfunction observationToText(obs: MemObservation): string {\n const parts = [`[${obs.kind}] ${obs.content}`];\n if (obs.tags && obs.tags.length > 0) {\n parts.push(`tags: ${obs.tags.join(', ')}`);\n }\n const text = parts.join('\\n');\n return text.length > MAX_TOKEN_CHARS ? text.slice(0, MAX_TOKEN_CHARS) : text;\n}\n\n/** Cosine similarity between two Float32Arrays */\nfunction cosine(a: Float32Array, b: Float32Array): number {\n let dot = 0, na = 0, nb = 0;\n for (let i = 0; i < a.length; i++) {\n dot += a[i] * b[i];\n na += a[i] * a[i];\n nb += b[i] * b[i];\n }\n const denom = Math.sqrt(na) * Math.sqrt(nb);\n return denom === 0 ? 0 : dot / denom;\n}\n\n// \u2500\u2500 MemoryVectorManager \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport class MemoryVectorManager {\n private config: KiroGraphConfig;\n private memDb: MemoryDatabase;\n private modelName: string;\n private cacheDir: string;\n\n constructor(config: KiroGraphConfig, memDb: MemoryDatabase) {\n this.config = config;\n this.memDb = memDb;\n this.modelName = (config as any).embeddingModel ?? 'nomic-ai/nomic-embed-text-v1.5';\n\n const { homedir } = require('os');\n const
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAUA,oBAA4C;
|
|
6
|
-
"names": []
|
|
4
|
+
"sourcesContent": ["/**\n * KiroGraph Memory \u2014 Vector embedding and search\n *\n * Reuses the same embedding model and semantic engine as the code graph,\n * but operates on mem_vectors table independently.\n */\n\nimport type { KiroGraphConfig } from '../config';\nimport type { MemObservation, ScoredObservation } from './types';\nimport type { MemoryDatabase } from './database';\nimport { logDebug, logWarn, logError } from '../errors';\nimport { TurboQuantIndex, writeTurboQuantStats } from '../vectors/turboquant-index';\n\nconst MAX_TOKEN_CHARS = 2000;\n\n// \u2500\u2500 Embedder (reuses the same pipeline as code vectors) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nlet transformers: typeof import('@huggingface/transformers') | null = null;\nlet pipeline: any = null;\nlet pipelineModel: string | null = null;\n\nasync function getTransformers() {\n if (!transformers) {\n transformers = await import('@huggingface/transformers');\n }\n return transformers;\n}\n\nasync function getPipeline(modelName: string, cacheDir: string) {\n if (pipeline && pipelineModel === modelName) return pipeline;\n\n const tf = await getTransformers();\n pipeline = await tf.pipeline('feature-extraction', modelName, {\n cache_dir: cacheDir,\n dtype: 'fp32',\n } as any);\n pipelineModel = modelName;\n return pipeline;\n}\n\n/**\n * Build searchable text from an observation.\n * Truncated to MAX_TOKEN_CHARS.\n */\nfunction observationToText(obs: MemObservation): string {\n const parts = [`[${obs.kind}] ${obs.content}`];\n if (obs.tags && obs.tags.length > 0) {\n parts.push(`tags: ${obs.tags.join(', ')}`);\n }\n const text = parts.join('\\n');\n return text.length > MAX_TOKEN_CHARS ? text.slice(0, MAX_TOKEN_CHARS) : text;\n}\n\n/** Cosine similarity between two Float32Arrays */\nfunction cosine(a: Float32Array, b: Float32Array): number {\n let dot = 0, na = 0, nb = 0;\n for (let i = 0; i < a.length; i++) {\n dot += a[i] * b[i];\n na += a[i] * a[i];\n nb += b[i] * b[i];\n }\n const denom = Math.sqrt(na) * Math.sqrt(nb);\n return denom === 0 ? 0 : dot / denom;\n}\n\n// \u2500\u2500 MemoryVectorManager \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport class MemoryVectorManager {\n private config: KiroGraphConfig;\n private memDb: MemoryDatabase;\n private modelName: string;\n private cacheDir: string;\n private tqIndex: TurboQuantIndex | null = null;\n private tqInitialized = false;\n\n constructor(config: KiroGraphConfig, memDb: MemoryDatabase, kirographDir?: string) {\n this.config = config;\n this.memDb = memDb;\n this.modelName = (config as any).embeddingModel ?? 'nomic-ai/nomic-embed-text-v1.5';\n\n const { homedir } = require('os');\n const pathMod = require('path');\n this.cacheDir = pathMod.join(homedir(), '.kirograph', 'models');\n\n if (kirographDir && (config as any).turboquantMemDocs) {\n const bits = (config as any).turboquantBits ?? 3;\n const dim = (config as any).embeddingDim ?? 768;\n this.tqIndex = new TurboQuantIndex(kirographDir, 'turboquant-mem.bin', dim, bits);\n this._kirographDir = kirographDir;\n }\n }\n\n private _kirographDir: string | undefined;\n\n /** Lazily initialize the TurboQuant index on first use. */\n private async ensureTQReady(): Promise<void> {\n if (this.tqInitialized) return;\n this.tqInitialized = true;\n if (this.tqIndex) await this.tqIndex.initialize();\n }\n\n /**\n * Check if embeddings are enabled in config.\n */\n isEnabled(): boolean {\n return !!(this.config as any).enableEmbeddings;\n }\n\n /**\n * Embed a single observation and store in mem_vectors.\n */\n async embedObservation(obs: MemObservation): Promise<void> {\n if (!this.isEnabled()) return;\n\n try {\n const text = observationToText(obs);\n const embedding = await this.embed(text);\n if (embedding) {\n const buffer = Buffer.from(embedding.buffer);\n this.memDb.insertVector(obs.id, buffer, this.modelName);\n if (this.tqIndex) {\n await this.ensureTQReady();\n this.tqIndex.upsert(obs.id, embedding);\n }\n }\n } catch (err) {\n logWarn(`Failed to embed observation ${obs.id}: ${err}`);\n }\n }\n\n /**\n * Embed multiple observations in batch.\n */\n async embedBatch(observations: MemObservation[]): Promise<number> {\n if (!this.isEnabled()) return 0;\n\n let embedded = 0;\n for (const obs of observations) {\n try {\n await this.embedObservation(obs);\n embedded++;\n } catch (err) {\n logWarn(`Failed to embed observation ${obs.id}: ${err}`);\n }\n }\n return embedded;\n }\n\n /**\n * Vector search over memory observations.\n * Falls back to empty results if embeddings are disabled or model mismatches.\n */\n async search(query: string, limit = 10): Promise<ScoredObservation[]> {\n if (!this.isEnabled()) return [];\n\n try {\n const queryEmbedding = await this.embed(query);\n if (!queryEmbedding) return [];\n\n // ANN path: TurboQuant index replaces the O(n) linear scan\n if (this.tqIndex) {\n await this.ensureTQReady();\n if (this.tqIndex.isAvailable()) {\n const hits = this.tqIndex.searchWithScores(queryEmbedding, limit);\n const results: ScoredObservation[] = [];\n for (const { id, score } of hits) {\n const obs = this.memDb.getObservation(id);\n if (obs) results.push({ observation: obs, score, scoreSource: 'vector' });\n }\n return results;\n }\n }\n\n // Fallback: linear cosine scan over all stored embeddings\n const allVectors = this.memDb.getAllVectors(this.modelName);\n if (allVectors.length === 0) return [];\n\n const scored: Array<{ observationId: string; score: number }> = [];\n for (const { observationId, embedding } of allVectors) {\n const vec = new Float32Array(embedding.buffer, embedding.byteOffset, embedding.byteLength / 4);\n scored.push({ observationId, score: cosine(queryEmbedding, vec) });\n }\n\n scored.sort((a, b) => b.score - a.score);\n const topN = scored.slice(0, limit);\n\n const results: ScoredObservation[] = [];\n for (const { observationId, score } of topN) {\n const obs = this.memDb.getObservation(observationId);\n if (obs) results.push({ observation: obs, score, scoreSource: 'vector' });\n }\n return results;\n } catch (err) {\n logError('Memory vector search failed', { error: err });\n return [];\n }\n }\n\n /**\n * Check if there are vectors with a mismatched model.\n */\n hasModelMismatch(): boolean {\n return this.memDb.getVectorModelMismatch(this.modelName) > 0;\n }\n\n /**\n * Re-embed all observations with the current model.\n */\n async reembed(batchSize = 32): Promise<number> {\n // Delete all existing vectors\n this.memDb.deleteVectors();\n\n // Reset TurboQuant index so it rebuilds from scratch\n if (this.tqIndex) {\n this.tqIndex.close();\n this.tqInitialized = false;\n }\n\n // Get all observations\n const db = (this.memDb as any).db;\n const rows = db.all('SELECT * FROM mem_observations ORDER BY created_at');\n const observations: MemObservation[] = rows.map((row: any) => ({\n id: row.id,\n sessionId: row.session_id ?? undefined,\n content: row.content,\n contentRaw: row.content_raw ?? undefined,\n contentHash: row.content_hash,\n kind: row.kind,\n source: row.source,\n tags: row.tags ? JSON.parse(row.tags) : undefined,\n createdAt: row.created_at,\n }));\n\n let embedded = 0;\n for (let i = 0; i < observations.length; i += batchSize) {\n const batch = observations.slice(i, i + batchSize);\n embedded += await this.embedBatch(batch);\n }\n\n if (this.tqIndex?.isAvailable() && this._kirographDir) {\n await this.tqIndex.save();\n const stats = this.tqIndex.memoryStats();\n const dim = (this.config as any).embeddingDim ?? 768;\n const rawBytes = embedded * dim * 4;\n writeTurboQuantStats(this._kirographDir, {\n memEnabled: true,\n memCount: embedded,\n memActualBytes: stats.actualBytes,\n memRawBytes: rawBytes,\n });\n }\n\n return embedded;\n }\n\n // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n private async embed(text: string): Promise<Float32Array | null> {\n try {\n const pipe = await getPipeline(this.modelName, this.cacheDir);\n const output = await pipe(text, { pooling: 'mean', normalize: true });\n return output.data as Float32Array;\n } catch (err) {\n logDebug(`Embedding failed: ${err}`);\n return null;\n }\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAUA,oBAA4C;AAC5C,8BAAsD;AAEtD,MAAM,kBAAkB;AAIxB,IAAI,eAAkE;AACtE,IAAI,WAAgB;AACpB,IAAI,gBAA+B;AAEnC,eAAe,kBAAkB;AAC/B,MAAI,CAAC,cAAc;AACjB,mBAAe,MAAM,OAAO,2BAA2B;AAAA,EACzD;AACA,SAAO;AACT;AAEA,eAAe,YAAY,WAAmB,UAAkB;AAC9D,MAAI,YAAY,kBAAkB,UAAW,QAAO;AAEpD,QAAM,KAAK,MAAM,gBAAgB;AACjC,aAAW,MAAM,GAAG,SAAS,sBAAsB,WAAW;AAAA,IAC5D,WAAW;AAAA,IACX,OAAO;AAAA,EACT,CAAQ;AACR,kBAAgB;AAChB,SAAO;AACT;AAMA,SAAS,kBAAkB,KAA6B;AACtD,QAAM,QAAQ,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,OAAO,EAAE;AAC7C,MAAI,IAAI,QAAQ,IAAI,KAAK,SAAS,GAAG;AACnC,UAAM,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3C;AACA,QAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,SAAO,KAAK,SAAS,kBAAkB,KAAK,MAAM,GAAG,eAAe,IAAI;AAC1E;AAGA,SAAS,OAAO,GAAiB,GAAyB;AACxD,MAAI,MAAM,GAAG,KAAK,GAAG,KAAK;AAC1B,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,UAAM,EAAE,CAAC,IAAI,EAAE,CAAC;AAChB,UAAM,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EAClB;AACA,QAAM,QAAQ,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,EAAE;AAC1C,SAAO,UAAU,IAAI,IAAI,MAAM;AACjC;AAIO,MAAM,oBAAoB;AAAA,EAQ/B,YAAY,QAAyB,OAAuB,cAAuB;AAHnF,SAAQ,UAAkC;AAC1C,SAAQ,gBAAgB;AAGtB,SAAK,SAAS;AACd,SAAK,QAAQ;AACb,SAAK,YAAa,OAAe,kBAAkB;AAEnD,UAAM,EAAE,QAAQ,IAAI,QAAQ,IAAI;AAChC,UAAM,UAAU,QAAQ,MAAM;AAC9B,SAAK,WAAW,QAAQ,KAAK,QAAQ,GAAG,cAAc,QAAQ;AAE9D,QAAI,gBAAiB,OAAe,mBAAmB;AACrD,YAAM,OAAQ,OAAe,kBAAkB;AAC/C,YAAM,MAAO,OAAe,gBAAgB;AAC5C,WAAK,UAAU,IAAI,wCAAgB,cAAc,sBAAsB,KAAK,IAAI;AAChF,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA;AAAA,EAKA,MAAc,gBAA+B;AAC3C,QAAI,KAAK,cAAe;AACxB,SAAK,gBAAgB;AACrB,QAAI,KAAK,QAAS,OAAM,KAAK,QAAQ,WAAW;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA,EAKA,YAAqB;AACnB,WAAO,CAAC,CAAE,KAAK,OAAe;AAAA,EAChC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,iBAAiB,KAAoC;AACzD,QAAI,CAAC,KAAK,UAAU,EAAG;AAEvB,QAAI;AACF,YAAM,OAAO,kBAAkB,GAAG;AAClC,YAAM,YAAY,MAAM,KAAK,MAAM,IAAI;AACvC,UAAI,WAAW;AACb,cAAM,SAAS,OAAO,KAAK,UAAU,MAAM;AAC3C,aAAK,MAAM,aAAa,IAAI,IAAI,QAAQ,KAAK,SAAS;AACtD,YAAI,KAAK,SAAS;AAChB,gBAAM,KAAK,cAAc;AACzB,eAAK,QAAQ,OAAO,IAAI,IAAI,SAAS;AAAA,QACvC;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,iCAAQ,+BAA+B,IAAI,EAAE,KAAK,GAAG,EAAE;AAAA,IACzD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,WAAW,cAAiD;AAChE,QAAI,CAAC,KAAK,UAAU,EAAG,QAAO;AAE9B,QAAI,WAAW;AACf,eAAW,OAAO,cAAc;AAC9B,UAAI;AACF,cAAM,KAAK,iBAAiB,GAAG;AAC/B;AAAA,MACF,SAAS,KAAK;AACZ,mCAAQ,+BAA+B,IAAI,EAAE,KAAK,GAAG,EAAE;AAAA,MACzD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAO,OAAe,QAAQ,IAAkC;AACpE,QAAI,CAAC,KAAK,UAAU,EAAG,QAAO,CAAC;AAE/B,QAAI;AACF,YAAM,iBAAiB,MAAM,KAAK,MAAM,KAAK;AAC7C,UAAI,CAAC,eAAgB,QAAO,CAAC;AAG7B,UAAI,KAAK,SAAS;AAChB,cAAM,KAAK,cAAc;AACzB,YAAI,KAAK,QAAQ,YAAY,GAAG;AAC9B,gBAAM,OAAO,KAAK,QAAQ,iBAAiB,gBAAgB,KAAK;AAChE,gBAAMA,WAA+B,CAAC;AACtC,qBAAW,EAAE,IAAI,MAAM,KAAK,MAAM;AAChC,kBAAM,MAAM,KAAK,MAAM,eAAe,EAAE;AACxC,gBAAI,IAAK,CAAAA,SAAQ,KAAK,EAAE,aAAa,KAAK,OAAO,aAAa,SAAS,CAAC;AAAA,UAC1E;AACA,iBAAOA;AAAA,QACT;AAAA,MACF;AAGA,YAAM,aAAa,KAAK,MAAM,cAAc,KAAK,SAAS;AAC1D,UAAI,WAAW,WAAW,EAAG,QAAO,CAAC;AAErC,YAAM,SAA0D,CAAC;AACjE,iBAAW,EAAE,eAAe,UAAU,KAAK,YAAY;AACrD,cAAM,MAAM,IAAI,aAAa,UAAU,QAAQ,UAAU,YAAY,UAAU,aAAa,CAAC;AAC7F,eAAO,KAAK,EAAE,eAAe,OAAO,OAAO,gBAAgB,GAAG,EAAE,CAAC;AAAA,MACnE;AAEA,aAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,YAAM,OAAO,OAAO,MAAM,GAAG,KAAK;AAElC,YAAM,UAA+B,CAAC;AACtC,iBAAW,EAAE,eAAe,MAAM,KAAK,MAAM;AAC3C,cAAM,MAAM,KAAK,MAAM,eAAe,aAAa;AACnD,YAAI,IAAK,SAAQ,KAAK,EAAE,aAAa,KAAK,OAAO,aAAa,SAAS,CAAC;AAAA,MAC1E;AACA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,kCAAS,+BAA+B,EAAE,OAAO,IAAI,CAAC;AACtD,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,mBAA4B;AAC1B,WAAO,KAAK,MAAM,uBAAuB,KAAK,SAAS,IAAI;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAQ,YAAY,IAAqB;AAE7C,SAAK,MAAM,cAAc;AAGzB,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,MAAM;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAGA,UAAM,KAAM,KAAK,MAAc;AAC/B,UAAM,OAAO,GAAG,IAAI,oDAAoD;AACxE,UAAM,eAAiC,KAAK,IAAI,CAAC,SAAc;AAAA,MAC7D,IAAI,IAAI;AAAA,MACR,WAAW,IAAI,cAAc;AAAA,MAC7B,SAAS,IAAI;AAAA,MACb,YAAY,IAAI,eAAe;AAAA,MAC/B,aAAa,IAAI;AAAA,MACjB,MAAM,IAAI;AAAA,MACV,QAAQ,IAAI;AAAA,MACZ,MAAM,IAAI,OAAO,KAAK,MAAM,IAAI,IAAI,IAAI;AAAA,MACxC,WAAW,IAAI;AAAA,IACjB,EAAE;AAEF,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,aAAa,QAAQ,KAAK,WAAW;AACvD,YAAM,QAAQ,aAAa,MAAM,GAAG,IAAI,SAAS;AACjD,kBAAY,MAAM,KAAK,WAAW,KAAK;AAAA,IACzC;AAEA,QAAI,KAAK,SAAS,YAAY,KAAK,KAAK,eAAe;AACrD,YAAM,KAAK,QAAQ,KAAK;AACxB,YAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,YAAM,MAAO,KAAK,OAAe,gBAAgB;AACjD,YAAM,WAAW,WAAW,MAAM;AAClC,wDAAqB,KAAK,eAAe;AAAA,QACvC,YAAY;AAAA,QACZ,UAAU;AAAA,QACV,gBAAgB,MAAM;AAAA,QACtB,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA;AAAA,EAIA,MAAc,MAAM,MAA4C;AAC9D,QAAI;AACF,YAAM,OAAO,MAAM,YAAY,KAAK,WAAW,KAAK,QAAQ;AAC5D,YAAM,SAAS,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ,WAAW,KAAK,CAAC;AACpE,aAAO,OAAO;AAAA,IAChB,SAAS,KAAK;AACZ,kCAAS,qBAAqB,GAAG,EAAE;AACnC,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
|
|
6
|
+
"names": ["results"]
|
|
7
7
|
}
|
package/dist/types.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/types.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * KiroGraph Type Definitions\n */\n\nexport type NodeKind =\n | 'file' | 'module' | 'class' | 'struct' | 'interface' | 'trait' | 'protocol'\n | 'function' | 'method' | 'property' | 'field' | 'variable' | 'constant'\n | 'enum' | 'enum_member' | 'type_alias' | 'namespace' | 'parameter'\n | 'import' | 'export' | 'route' | 'component'\n | 'dependency' | 'vulnerability';\n\nexport type EdgeKind =\n | 'contains' | 'calls' | 'imports' | 'exports' | 'extends' | 'implements'\n | 'references' | 'type_of' | 'returns' | 'instantiates' | 'overrides' | 'decorates'\n | 'has_vulnerability' | 'depends_on' | 'declared_in';\n\nexport type Language =\n | 'typescript' | 'javascript' | 'tsx' | 'jsx' | 'python' | 'go' | 'rust'\n | 'java' | 'c' | 'cpp' | 'csharp' | 'php' | 'ruby' | 'swift' | 'kotlin'\n | 'dart' | 'svelte' | 'pascal' | 'liquid' | 'elixir'\n | 'scala' | 'lua' | 'zig' | 'bash' | 'ocaml' | 'elm' | 'solidity' | 'vue' | 'objc'\n | 'yaml' | 'hcl' | 'css' | 'scss' | 'html'\n | 'jupyter'\n | 'rescript' | 'sql' | 'r' | 'julia' | 'powershell' | 'perl' | 'astro' | 'gdscript' | 'nix' | 'verilog'\n | 'haskell'\n | 'unknown';\n\nexport interface Node {\n id: string;\n kind: NodeKind;\n name: string;\n qualifiedName: string;\n filePath: string;\n language: Language;\n startLine: number;\n endLine: number;\n startColumn: number;\n endColumn: number;\n docstring?: string;\n signature?: string;\n visibility?: 'public' | 'private' | 'protected' | 'internal';\n isExported?: boolean;\n isAsync?: boolean;\n isStatic?: boolean;\n isAbstract?: boolean;\n decorators?: string[];\n typeParameters?: string[];\n updatedAt: number;\n}\n\nexport interface Edge {\n source: string;\n target: string;\n kind: EdgeKind;\n metadata?: Record<string, unknown>;\n line?: number;\n column?: number;\n confidence?: 'extracted' | 'inferred' | 'ambiguous';\n confidenceScore?: number;\n}\n\nexport interface FileRecord {\n path: string;\n contentHash: string;\n language: Language;\n fileSize: number;\n symbolCount: number;\n indexedAt: number;\n}\n\nexport interface SearchResult {\n node: Node;\n score: number;\n matchType: 'exact' | 'prefix' | 'fuzzy' | 'semantic';\n}\n\nexport interface SearchOptions {\n kinds?: NodeKind[];\n languages?: Language[];\n limit?: number;\n minScore?: number;\n}\n\nexport interface Subgraph {\n nodes: Node[];\n edges: Edge[];\n entryPoints: string[];\n}\n\nexport interface NodeContext {\n node: Node;\n ancestors: Node[];\n children: Node[];\n callers: Node[];\n callees: Node[];\n}\n\nexport interface NodeMetrics {\n incomingEdgeCount: number;\n outgoingEdgeCount: number;\n callCount: number;\n callerCount: number;\n childCount: number;\n}\n\nexport interface GraphStats {\n files: number;\n nodes: number;\n edges: number;\n nodesByKind: Record<string, number>;\n filesByLanguage: Record<string, number>;\n dbSizeBytes: number;\n embeddingCount: number;\n embeddableNodeCount: number;\n embeddingsEnabled: boolean;\n embeddingModel: string;\n useVecIndex: boolean;\n semanticEngine: 'cosine' | 'sqlite-vec' | 'orama' | 'pglite' | 'lancedb' | 'qdrant' | 'typesense';\n vecIndexCount: number;\n engineFallback: string | null;\n frameworks: string[];\n architectureEnabled: boolean;\n architectureStats?: { packages: number; layers: number; packageDeps: number };\n}\n\nexport interface TaskContext {\n task: string;\n entryPoints: Node[];\n relatedNodes: Node[];\n edges: Edge[];\n codeSnippets: Map<string, string>;\n summary: string;\n}\n\nexport interface IndexProgress {\n phase: 'scanning' | 'parsing' | 'storing' | 'resolving' | 'detecting frameworks' | 'embeddings' | 'architecture' | 'security' | 'docs' | 'data' | 'patterns' | 'retrying';\n current: number;\n total: number;\n currentFile?: string;\n meta?: Record<string, unknown>;\n}\n\nexport interface IndexResult {\n success: boolean;\n filesIndexed: number;\n nodesCreated: number;\n edgesCreated: number;\n errors: string[];\n duration: number;\n}\n\nexport interface SyncResult {\n added: string[];\n modified: string[];\n removed: string[];\n nodesCreated: number;\n nodesUpdated: number;\n nodesRemoved: number;\n edgesCreated: number;\n edgesRemoved: number;\n filesScanned: number;\n errors: string[];\n duration: number;\n}\n\n// \u2500\u2500 Error Classes (re-exported from src/errors.ts for backward compatibility) \u2500\u2500\n\nexport {\n KiroGraphError,\n FileError,\n ParseError,\n DatabaseError,\n ConfigError,\n} from './errors';\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;
|
|
4
|
+
"sourcesContent": ["/**\n * KiroGraph Type Definitions\n */\n\nexport type NodeKind =\n | 'file' | 'module' | 'class' | 'struct' | 'interface' | 'trait' | 'protocol'\n | 'function' | 'method' | 'property' | 'field' | 'variable' | 'constant'\n | 'enum' | 'enum_member' | 'type_alias' | 'namespace' | 'parameter'\n | 'import' | 'export' | 'route' | 'component'\n | 'dependency' | 'vulnerability';\n\nexport type EdgeKind =\n | 'contains' | 'calls' | 'imports' | 'exports' | 'extends' | 'implements'\n | 'references' | 'type_of' | 'returns' | 'instantiates' | 'overrides' | 'decorates'\n | 'has_vulnerability' | 'depends_on' | 'declared_in';\n\nexport type Language =\n | 'typescript' | 'javascript' | 'tsx' | 'jsx' | 'python' | 'go' | 'rust'\n | 'java' | 'c' | 'cpp' | 'csharp' | 'php' | 'ruby' | 'swift' | 'kotlin'\n | 'dart' | 'svelte' | 'pascal' | 'liquid' | 'elixir'\n | 'scala' | 'lua' | 'zig' | 'bash' | 'ocaml' | 'elm' | 'solidity' | 'vue' | 'objc'\n | 'yaml' | 'hcl' | 'css' | 'scss' | 'html'\n | 'jupyter'\n | 'rescript' | 'sql' | 'r' | 'julia' | 'powershell' | 'perl' | 'astro' | 'gdscript' | 'nix' | 'verilog'\n | 'haskell'\n | 'unknown';\n\nexport interface Node {\n id: string;\n kind: NodeKind;\n name: string;\n qualifiedName: string;\n filePath: string;\n language: Language;\n startLine: number;\n endLine: number;\n startColumn: number;\n endColumn: number;\n docstring?: string;\n signature?: string;\n visibility?: 'public' | 'private' | 'protected' | 'internal';\n isExported?: boolean;\n isAsync?: boolean;\n isStatic?: boolean;\n isAbstract?: boolean;\n decorators?: string[];\n typeParameters?: string[];\n updatedAt: number;\n}\n\nexport interface Edge {\n source: string;\n target: string;\n kind: EdgeKind;\n metadata?: Record<string, unknown>;\n line?: number;\n column?: number;\n confidence?: 'extracted' | 'inferred' | 'ambiguous';\n confidenceScore?: number;\n}\n\nexport interface FileRecord {\n path: string;\n contentHash: string;\n language: Language;\n fileSize: number;\n symbolCount: number;\n indexedAt: number;\n}\n\nexport interface SearchResult {\n node: Node;\n score: number;\n matchType: 'exact' | 'prefix' | 'fuzzy' | 'semantic';\n}\n\nexport interface SearchOptions {\n kinds?: NodeKind[];\n languages?: Language[];\n limit?: number;\n minScore?: number;\n}\n\nexport interface Subgraph {\n nodes: Node[];\n edges: Edge[];\n entryPoints: string[];\n}\n\nexport interface NodeContext {\n node: Node;\n ancestors: Node[];\n children: Node[];\n callers: Node[];\n callees: Node[];\n}\n\nexport interface NodeMetrics {\n incomingEdgeCount: number;\n outgoingEdgeCount: number;\n callCount: number;\n callerCount: number;\n childCount: number;\n}\n\nexport interface GraphStats {\n files: number;\n nodes: number;\n edges: number;\n nodesByKind: Record<string, number>;\n filesByLanguage: Record<string, number>;\n dbSizeBytes: number;\n embeddingCount: number;\n embeddableNodeCount: number;\n embeddingsEnabled: boolean;\n embeddingModel: string;\n useVecIndex: boolean;\n semanticEngine: 'cosine' | 'turboquant' | 'turbovec' | 'sqlite-vec' | 'orama' | 'pglite' | 'lancedb' | 'qdrant' | 'typesense';\n turbovecBits?: number;\n vecIndexCount: number;\n engineFallback: string | null;\n frameworks: string[];\n architectureEnabled: boolean;\n architectureStats?: { packages: number; layers: number; packageDeps: number };\n}\n\nexport interface TaskContext {\n task: string;\n entryPoints: Node[];\n relatedNodes: Node[];\n edges: Edge[];\n codeSnippets: Map<string, string>;\n summary: string;\n}\n\nexport interface IndexProgress {\n phase: 'scanning' | 'parsing' | 'storing' | 'resolving' | 'detecting frameworks' | 'embeddings' | 'architecture' | 'security' | 'docs' | 'data' | 'patterns' | 'retrying';\n current: number;\n total: number;\n currentFile?: string;\n meta?: Record<string, unknown>;\n}\n\nexport interface IndexResult {\n success: boolean;\n filesIndexed: number;\n nodesCreated: number;\n edgesCreated: number;\n errors: string[];\n duration: number;\n}\n\nexport interface SyncResult {\n added: string[];\n modified: string[];\n removed: string[];\n nodesCreated: number;\n nodesUpdated: number;\n nodesRemoved: number;\n edgesCreated: number;\n edgesRemoved: number;\n filesScanned: number;\n errors: string[];\n duration: number;\n}\n\n// \u2500\u2500 Error Classes (re-exported from src/errors.ts for backward compatibility) \u2500\u2500\n\nexport {\n KiroGraphError,\n FileError,\n ParseError,\n DatabaseError,\n ConfigError,\n} from './errors';\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwKA,oBAMO;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/vectors/index.js
CHANGED
|
@@ -41,6 +41,8 @@ var import_pglite_index = require("./pglite-index");
|
|
|
41
41
|
var import_lancedb_index = require("./lancedb-index");
|
|
42
42
|
var import_qdrant_index = require("./qdrant-index");
|
|
43
43
|
var import_typesense_index = require("./typesense-index");
|
|
44
|
+
var import_turboquant_index = require("./turboquant-index");
|
|
45
|
+
var import_turbovec_index = require("./turbovec-index");
|
|
44
46
|
const DEFAULT_MODEL = "nomic-ai/nomic-embed-text-v1.5";
|
|
45
47
|
const DEFAULT_EMBEDDING_DIM = 768;
|
|
46
48
|
const GLOBAL_MODELS_DIR = path.join((0, import_os.homedir)(), ".kirograph", "models");
|
|
@@ -109,6 +111,8 @@ class VectorManager {
|
|
|
109
111
|
this.lancedbIndex = null;
|
|
110
112
|
this.qdrantIndex = null;
|
|
111
113
|
this.typesenseIndex = null;
|
|
114
|
+
this.turboquantIndex = null;
|
|
115
|
+
this.turbovecIndex = null;
|
|
112
116
|
this._engineFallback = null;
|
|
113
117
|
}
|
|
114
118
|
isInitialized() {
|
|
@@ -240,6 +244,26 @@ class VectorManager {
|
|
|
240
244
|
this._engineFallback = reason ? `typesense initialization failed: ${reason}` : "typesense unavailable \u2014 run: npm install typesense";
|
|
241
245
|
(0, import_errors.logDebug)("VectorManager: Typesense unavailable, falling back to in-process cosine");
|
|
242
246
|
}
|
|
247
|
+
} else if (engine === "turboquant") {
|
|
248
|
+
const bits = this.config.turboquantBits ?? 3;
|
|
249
|
+
this.turboquantIndex = new import_turboquant_index.TurboQuantIndex(kirographDir, "turboquant.bin", embeddingDim, bits);
|
|
250
|
+
await this.turboquantIndex.initialize();
|
|
251
|
+
if (this.turboquantIndex.isAvailable()) {
|
|
252
|
+
(0, import_errors.logDebug)("VectorManager: TurboQuant ANN index ready", { bits });
|
|
253
|
+
} else {
|
|
254
|
+
this._engineFallback = "turboquant unavailable \u2014 run: npm install turboquant-js";
|
|
255
|
+
(0, import_errors.logDebug)("VectorManager: TurboQuant unavailable, falling back to in-process cosine");
|
|
256
|
+
}
|
|
257
|
+
} else if (engine === "turbovec") {
|
|
258
|
+
const bits = this.config.turbovecBits ?? 4;
|
|
259
|
+
this.turbovecIndex = new import_turbovec_index.TurboVecIndex(kirographDir, "turbovec.tvim", embeddingDim, bits);
|
|
260
|
+
await this.turbovecIndex.initialize();
|
|
261
|
+
if (this.turbovecIndex.isAvailable()) {
|
|
262
|
+
(0, import_errors.logDebug)("VectorManager: TurboVec ANN index ready", { bits });
|
|
263
|
+
} else {
|
|
264
|
+
this._engineFallback = "turbovec unavailable \u2014 build it: cd native/turbovec-node && npm install && npm run build";
|
|
265
|
+
(0, import_errors.logDebug)("VectorManager: TurboVec unavailable, falling back to in-process cosine");
|
|
266
|
+
}
|
|
243
267
|
}
|
|
244
268
|
}
|
|
245
269
|
}
|
|
@@ -258,7 +282,7 @@ class VectorManager {
|
|
|
258
282
|
const text = `search_document: ${nodeToText(node)}`;
|
|
259
283
|
const output = await this.pipeline(text, { pooling: "mean", normalize: true });
|
|
260
284
|
const embedding = toFloat32Array(output.data);
|
|
261
|
-
if (!this.vecIndex?.isAvailable() && !this.oramaIndex?.isAvailable() && !this.pgliteIndex?.isAvailable() && !this.lancedbIndex?.isAvailable() && !this.qdrantIndex?.isAvailable() && !this.typesenseIndex?.isAvailable()) {
|
|
285
|
+
if (!this.vecIndex?.isAvailable() && !this.oramaIndex?.isAvailable() && !this.pgliteIndex?.isAvailable() && !this.lancedbIndex?.isAvailable() && !this.qdrantIndex?.isAvailable() && !this.typesenseIndex?.isAvailable() && !this.turboquantIndex?.isAvailable() && !this.turbovecIndex?.isAvailable()) {
|
|
262
286
|
this.db.storeEmbedding(node.id, embedding, this.config.embeddingModel || DEFAULT_MODEL);
|
|
263
287
|
}
|
|
264
288
|
this.vecIndex?.upsert(node.id, embedding);
|
|
@@ -267,12 +291,16 @@ class VectorManager {
|
|
|
267
291
|
if (this.lancedbIndex?.isAvailable()) await this.lancedbIndex.upsert(node, embedding);
|
|
268
292
|
if (this.qdrantIndex?.isAvailable()) await this.qdrantIndex.upsert(node, embedding);
|
|
269
293
|
if (this.typesenseIndex?.isAvailable()) await this.typesenseIndex.upsert(node, embedding);
|
|
294
|
+
this.turboquantIndex?.upsert(node.id, embedding);
|
|
295
|
+
this.turbovecIndex?.upsert(node.id, embedding);
|
|
270
296
|
} catch (err) {
|
|
271
297
|
(0, import_errors.logWarn)("Failed to embed node", { nodeId: node.id, error: String(err) });
|
|
272
298
|
}
|
|
273
299
|
}
|
|
274
300
|
/** Number of entries currently in the active ANN/hybrid index (0 when not in use). */
|
|
275
301
|
async vecIndexCount() {
|
|
302
|
+
if (this.turbovecIndex?.isAvailable()) return this.turbovecIndex.count();
|
|
303
|
+
if (this.turboquantIndex?.isAvailable()) return this.turboquantIndex.count();
|
|
276
304
|
if (this.vecIndex?.isAvailable()) return this.vecIndex.count();
|
|
277
305
|
if (this.oramaIndex?.isAvailable()) return this.oramaIndex.count();
|
|
278
306
|
if (this.pgliteIndex?.isAvailable()) return this.pgliteIndex.count();
|
|
@@ -295,6 +323,8 @@ class VectorManager {
|
|
|
295
323
|
if (this.lancedbIndex?.isAvailable()) await this.lancedbIndex.delete(id);
|
|
296
324
|
if (this.qdrantIndex?.isAvailable()) await this.qdrantIndex.delete(id);
|
|
297
325
|
if (this.typesenseIndex?.isAvailable()) await this.typesenseIndex.delete(id);
|
|
326
|
+
this.turboquantIndex?.delete(id);
|
|
327
|
+
this.turbovecIndex?.delete(id);
|
|
298
328
|
}
|
|
299
329
|
}
|
|
300
330
|
/**
|
|
@@ -319,7 +349,7 @@ class VectorManager {
|
|
|
319
349
|
onProgress?.(-1, totalEmbeddable);
|
|
320
350
|
}
|
|
321
351
|
const existingIds = new Set(
|
|
322
|
-
this.typesenseIndex?.isAvailable() ? await this.typesenseIndex.getEmbeddedNodeIds() : this.qdrantIndex?.isAvailable() ? await this.qdrantIndex.getEmbeddedNodeIds() : this.lancedbIndex?.isAvailable() ? await this.lancedbIndex.getEmbeddedNodeIds() : this.pgliteIndex?.isAvailable() ? await this.pgliteIndex.getEmbeddedNodeIds() : this.oramaIndex?.isAvailable() ? await this.oramaIndex.getEmbeddedNodeIds() : this.vecIndex?.isAvailable() ? this.vecIndex.getEmbeddedNodeIds() : this.db.getEmbeddedNodeIds()
|
|
352
|
+
this.typesenseIndex?.isAvailable() ? await this.typesenseIndex.getEmbeddedNodeIds() : this.qdrantIndex?.isAvailable() ? await this.qdrantIndex.getEmbeddedNodeIds() : this.lancedbIndex?.isAvailable() ? await this.lancedbIndex.getEmbeddedNodeIds() : this.pgliteIndex?.isAvailable() ? await this.pgliteIndex.getEmbeddedNodeIds() : this.oramaIndex?.isAvailable() ? await this.oramaIndex.getEmbeddedNodeIds() : this.turbovecIndex?.isAvailable() ? this.turbovecIndex.getEmbeddedIds() : this.turboquantIndex?.isAvailable() ? this.turboquantIndex.getEmbeddedIds() : this.vecIndex?.isAvailable() ? this.vecIndex.getEmbeddedNodeIds() : this.db.getEmbeddedNodeIds()
|
|
323
353
|
);
|
|
324
354
|
const PAGE_SIZE = 2e3;
|
|
325
355
|
let pageOffset = 0;
|
|
@@ -364,7 +394,7 @@ class VectorManager {
|
|
|
364
394
|
for (let j = 0; j < batch.length; j++) {
|
|
365
395
|
const node = batch[j];
|
|
366
396
|
const embedding = flat.slice(j * dim, (j + 1) * dim);
|
|
367
|
-
if (!this.vecIndex?.isAvailable() && !this.oramaIndex?.isAvailable() && !this.pgliteIndex?.isAvailable() && !this.lancedbIndex?.isAvailable() && !this.qdrantIndex?.isAvailable() && !this.typesenseIndex?.isAvailable()) {
|
|
397
|
+
if (!this.vecIndex?.isAvailable() && !this.oramaIndex?.isAvailable() && !this.pgliteIndex?.isAvailable() && !this.lancedbIndex?.isAvailable() && !this.qdrantIndex?.isAvailable() && !this.typesenseIndex?.isAvailable() && !this.turboquantIndex?.isAvailable() && !this.turbovecIndex?.isAvailable()) {
|
|
368
398
|
this.db.storeEmbedding(node.id, embedding, modelId);
|
|
369
399
|
}
|
|
370
400
|
this.vecIndex?.upsert(node.id, embedding);
|
|
@@ -379,6 +409,8 @@ class VectorManager {
|
|
|
379
409
|
tsNodes.push(node);
|
|
380
410
|
tsEmbeddings.push(embedding);
|
|
381
411
|
}
|
|
412
|
+
this.turboquantIndex?.upsert(node.id, embedding);
|
|
413
|
+
this.turbovecIndex?.upsert(node.id, embedding);
|
|
382
414
|
}
|
|
383
415
|
if (qdNodes.length > 0) await this.qdrantIndex.bulkUpsert(qdNodes, qdEmbeddings);
|
|
384
416
|
if (tsNodes.length > 0) await this.typesenseIndex.bulkUpsert(tsNodes, tsEmbeddings);
|
|
@@ -421,6 +453,32 @@ class VectorManager {
|
|
|
421
453
|
onProgress?.(processed, totalPending);
|
|
422
454
|
}
|
|
423
455
|
if (this.oramaIndex?.isAvailable()) await this.oramaIndex.save();
|
|
456
|
+
if (this.turboquantIndex?.isAvailable()) {
|
|
457
|
+
await this.turboquantIndex.save();
|
|
458
|
+
if (this.projectRoot) {
|
|
459
|
+
const kirographDir = path.join(this.projectRoot, ".kirograph");
|
|
460
|
+
const stats = this.turboquantIndex.memoryStats();
|
|
461
|
+
const count = this.turboquantIndex.count();
|
|
462
|
+
const dim = this.config.embeddingDim ?? DEFAULT_EMBEDDING_DIM;
|
|
463
|
+
const rawBytes = count * dim * 4;
|
|
464
|
+
(0, import_turboquant_index.writeTurboQuantStats)(kirographDir, {
|
|
465
|
+
count,
|
|
466
|
+
dim,
|
|
467
|
+
bits: this.config.turboquantBits ?? 3,
|
|
468
|
+
compressionRatio: stats.compressionRatio,
|
|
469
|
+
actualBytes: stats.actualBytes,
|
|
470
|
+
rawBytes,
|
|
471
|
+
savedBytes: rawBytes - stats.actualBytes
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (this.turbovecIndex?.isAvailable()) {
|
|
476
|
+
await this.turbovecIndex.save();
|
|
477
|
+
(0, import_errors.logDebug)("VectorManager: TurboVec index saved", {
|
|
478
|
+
count: this.turbovecIndex.count(),
|
|
479
|
+
bits: this.config.turbovecBits ?? 4
|
|
480
|
+
});
|
|
481
|
+
}
|
|
424
482
|
(0, import_errors.logDebug)(`VectorManager: embedding complete \u2014 ${processed} processed, final batch size: ${currentBatchSize}`);
|
|
425
483
|
return processed;
|
|
426
484
|
}
|
|
@@ -450,6 +508,10 @@ class VectorManager {
|
|
|
450
508
|
nodeIds = await this.qdrantIndex.search(queryVec, topN);
|
|
451
509
|
} else if (this.typesenseIndex?.isAvailable()) {
|
|
452
510
|
nodeIds = await this.typesenseIndex.search(queryVec, topN);
|
|
511
|
+
} else if (this.turbovecIndex?.isAvailable()) {
|
|
512
|
+
nodeIds = this.turbovecIndex.search(queryVec, topN);
|
|
513
|
+
} else if (this.turboquantIndex?.isAvailable()) {
|
|
514
|
+
nodeIds = this.turboquantIndex.search(queryVec, topN);
|
|
453
515
|
} else if (this.vecIndex?.isAvailable()) {
|
|
454
516
|
nodeIds = this.vecIndex.search(queryVec, topN);
|
|
455
517
|
} else {
|
|
@@ -471,6 +533,7 @@ class VectorManager {
|
|
|
471
533
|
close() {
|
|
472
534
|
this.qdrantIndex?.close();
|
|
473
535
|
this.typesenseIndex?.close();
|
|
536
|
+
this.turbovecIndex?.close();
|
|
474
537
|
}
|
|
475
538
|
}
|
|
476
539
|
// Annotate the CommonJS export names for ESM import in node:
|