causantic 0.9.3 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/README.md +70 -56
  2. package/dist/cli/skill-templates.d.ts.map +1 -1
  3. package/dist/cli/skill-templates.js +23 -18
  4. package/dist/cli/skill-templates.js.map +1 -1
  5. package/dist/clusters/cluster-manager.d.ts +16 -0
  6. package/dist/clusters/cluster-manager.d.ts.map +1 -1
  7. package/dist/clusters/cluster-manager.js +119 -1
  8. package/dist/clusters/cluster-manager.js.map +1 -1
  9. package/dist/config/loader.d.ts +16 -0
  10. package/dist/config/loader.d.ts.map +1 -1
  11. package/dist/config/loader.js +51 -0
  12. package/dist/config/loader.js.map +1 -1
  13. package/dist/config/memory-config.d.ts +26 -0
  14. package/dist/config/memory-config.d.ts.map +1 -1
  15. package/dist/config/memory-config.js +22 -0
  16. package/dist/config/memory-config.js.map +1 -1
  17. package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts +20 -0
  18. package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts.map +1 -0
  19. package/dist/eval/experiments/embedding-model-comparison/run-experiment.js +289 -0
  20. package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -0
  21. package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts +53 -0
  22. package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -0
  23. package/dist/eval/experiments/index-differentiation/alignment-analysis.js +91 -0
  24. package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -0
  25. package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts +24 -0
  26. package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -0
  27. package/dist/eval/experiments/index-differentiation/discrimination-test.js +79 -0
  28. package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -0
  29. package/dist/eval/experiments/index-differentiation/index.d.ts +11 -0
  30. package/dist/eval/experiments/index-differentiation/index.d.ts.map +1 -0
  31. package/dist/eval/experiments/index-differentiation/index.js +8 -0
  32. package/dist/eval/experiments/index-differentiation/index.js.map +1 -0
  33. package/dist/eval/experiments/index-differentiation/refinement-test.d.ts +32 -0
  34. package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -0
  35. package/dist/eval/experiments/index-differentiation/refinement-test.js +203 -0
  36. package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -0
  37. package/dist/eval/experiments/index-differentiation/run-experiment.d.ts +20 -0
  38. package/dist/eval/experiments/index-differentiation/run-experiment.d.ts.map +1 -0
  39. package/dist/eval/experiments/index-differentiation/run-experiment.js +338 -0
  40. package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -0
  41. package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts +31 -0
  42. package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -0
  43. package/dist/eval/experiments/index-differentiation/similarity-analysis.js +60 -0
  44. package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -0
  45. package/dist/eval/experiments/index-differentiation/types.d.ts +114 -0
  46. package/dist/eval/experiments/index-differentiation/types.d.ts.map +1 -0
  47. package/dist/eval/experiments/index-differentiation/types.js +8 -0
  48. package/dist/eval/experiments/index-differentiation/types.js.map +1 -0
  49. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts +19 -0
  50. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts.map +1 -0
  51. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +328 -0
  52. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -0
  53. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts +27 -0
  54. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts.map +1 -0
  55. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js +154 -0
  56. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -0
  57. package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts +23 -0
  58. package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts.map +1 -0
  59. package/dist/eval/experiments/index-vs-chunk/query-generator.js +113 -0
  60. package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -0
  61. package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts +17 -0
  62. package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts.map +1 -0
  63. package/dist/eval/experiments/index-vs-chunk/run-experiment.js +341 -0
  64. package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -0
  65. package/dist/eval/experiments/index-vs-chunk/types.d.ts +71 -0
  66. package/dist/eval/experiments/index-vs-chunk/types.d.ts.map +1 -0
  67. package/dist/eval/experiments/index-vs-chunk/types.js +8 -0
  68. package/dist/eval/experiments/index-vs-chunk/types.js.map +1 -0
  69. package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts +18 -0
  70. package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts.map +1 -0
  71. package/dist/eval/experiments/pipeline-dropout/run-experiment.js +347 -0
  72. package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -0
  73. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts +17 -0
  74. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts.map +1 -0
  75. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js +247 -0
  76. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -0
  77. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts +18 -0
  78. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts.map +1 -0
  79. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +443 -0
  80. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -0
  81. package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts +16 -0
  82. package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts.map +1 -0
  83. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +226 -0
  84. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -0
  85. package/dist/index-entries/index-generator.d.ts +74 -0
  86. package/dist/index-entries/index-generator.d.ts.map +1 -0
  87. package/dist/index-entries/index-generator.js +323 -0
  88. package/dist/index-entries/index-generator.js.map +1 -0
  89. package/dist/index-entries/index-refresher.d.ts +54 -0
  90. package/dist/index-entries/index-refresher.d.ts.map +1 -0
  91. package/dist/index-entries/index-refresher.js +203 -0
  92. package/dist/index-entries/index-refresher.js.map +1 -0
  93. package/dist/index-entries/index.d.ts +6 -0
  94. package/dist/index-entries/index.d.ts.map +1 -0
  95. package/dist/index-entries/index.js +6 -0
  96. package/dist/index-entries/index.js.map +1 -0
  97. package/dist/index.d.ts +4 -0
  98. package/dist/index.d.ts.map +1 -1
  99. package/dist/index.js +5 -0
  100. package/dist/index.js.map +1 -1
  101. package/dist/ingest/index-entry-hook.d.ts +15 -0
  102. package/dist/ingest/index-entry-hook.d.ts.map +1 -0
  103. package/dist/ingest/index-entry-hook.js +84 -0
  104. package/dist/ingest/index-entry-hook.js.map +1 -0
  105. package/dist/ingest/ingest-session.d.ts.map +1 -1
  106. package/dist/ingest/ingest-session.js +72 -18
  107. package/dist/ingest/ingest-session.js.map +1 -1
  108. package/dist/ingest/session-state.d.ts +49 -0
  109. package/dist/ingest/session-state.d.ts.map +1 -0
  110. package/dist/ingest/session-state.js +158 -0
  111. package/dist/ingest/session-state.js.map +1 -0
  112. package/dist/maintenance/scheduler.d.ts.map +1 -1
  113. package/dist/maintenance/scheduler.js +25 -0
  114. package/dist/maintenance/scheduler.js.map +1 -1
  115. package/dist/maintenance/tasks/backfill-index.d.ts +27 -0
  116. package/dist/maintenance/tasks/backfill-index.d.ts.map +1 -0
  117. package/dist/maintenance/tasks/backfill-index.js +44 -0
  118. package/dist/maintenance/tasks/backfill-index.js.map +1 -0
  119. package/dist/mcp/tools.d.ts +4 -0
  120. package/dist/mcp/tools.d.ts.map +1 -1
  121. package/dist/mcp/tools.js +115 -7
  122. package/dist/mcp/tools.js.map +1 -1
  123. package/dist/models/embedder.js +2 -2
  124. package/dist/models/embedder.js.map +1 -1
  125. package/dist/models/model-registry.d.ts +2 -0
  126. package/dist/models/model-registry.d.ts.map +1 -1
  127. package/dist/models/model-registry.js +15 -0
  128. package/dist/models/model-registry.js.map +1 -1
  129. package/dist/repomap/cache.d.ts +58 -0
  130. package/dist/repomap/cache.d.ts.map +1 -0
  131. package/dist/repomap/cache.js +101 -0
  132. package/dist/repomap/cache.js.map +1 -0
  133. package/dist/repomap/graph.d.ts +54 -0
  134. package/dist/repomap/graph.d.ts.map +1 -0
  135. package/dist/repomap/graph.js +113 -0
  136. package/dist/repomap/graph.js.map +1 -0
  137. package/dist/repomap/index.d.ts +83 -0
  138. package/dist/repomap/index.d.ts.map +1 -0
  139. package/dist/repomap/index.js +99 -0
  140. package/dist/repomap/index.js.map +1 -0
  141. package/dist/repomap/parser.d.ts +43 -0
  142. package/dist/repomap/parser.d.ts.map +1 -0
  143. package/dist/repomap/parser.js +994 -0
  144. package/dist/repomap/parser.js.map +1 -0
  145. package/dist/repomap/regex-parser.d.ts +24 -0
  146. package/dist/repomap/regex-parser.d.ts.map +1 -0
  147. package/dist/repomap/regex-parser.js +190 -0
  148. package/dist/repomap/regex-parser.js.map +1 -0
  149. package/dist/repomap/renderer.d.ts +40 -0
  150. package/dist/repomap/renderer.d.ts.map +1 -0
  151. package/dist/repomap/renderer.js +163 -0
  152. package/dist/repomap/renderer.js.map +1 -0
  153. package/dist/repomap/scanner.d.ts +32 -0
  154. package/dist/repomap/scanner.d.ts.map +1 -0
  155. package/dist/repomap/scanner.js +171 -0
  156. package/dist/repomap/scanner.js.map +1 -0
  157. package/dist/retrieval/chain-assembler.d.ts.map +1 -1
  158. package/dist/retrieval/chain-assembler.js +22 -3
  159. package/dist/retrieval/chain-assembler.js.map +1 -1
  160. package/dist/retrieval/index.d.ts +2 -0
  161. package/dist/retrieval/index.d.ts.map +1 -1
  162. package/dist/retrieval/index.js +2 -0
  163. package/dist/retrieval/index.js.map +1 -1
  164. package/dist/retrieval/mmr.d.ts +1 -0
  165. package/dist/retrieval/mmr.d.ts.map +1 -1
  166. package/dist/retrieval/mmr.js +35 -1
  167. package/dist/retrieval/mmr.js.map +1 -1
  168. package/dist/retrieval/search-assembler.d.ts +10 -1
  169. package/dist/retrieval/search-assembler.d.ts.map +1 -1
  170. package/dist/retrieval/search-assembler.js +249 -81
  171. package/dist/retrieval/search-assembler.js.map +1 -1
  172. package/dist/retrieval/session-reconstructor.d.ts +36 -0
  173. package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
  174. package/dist/retrieval/session-reconstructor.js +126 -0
  175. package/dist/retrieval/session-reconstructor.js.map +1 -1
  176. package/dist/storage/db.d.ts.map +1 -1
  177. package/dist/storage/db.js +15 -0
  178. package/dist/storage/db.js.map +1 -1
  179. package/dist/storage/index-entry-store.d.ts +71 -0
  180. package/dist/storage/index-entry-store.d.ts.map +1 -0
  181. package/dist/storage/index-entry-store.js +275 -0
  182. package/dist/storage/index-entry-store.js.map +1 -0
  183. package/dist/storage/index.d.ts +5 -2
  184. package/dist/storage/index.d.ts.map +1 -1
  185. package/dist/storage/index.js +5 -1
  186. package/dist/storage/index.js.map +1 -1
  187. package/dist/storage/migrations.d.ts.map +1 -1
  188. package/dist/storage/migrations.js +102 -0
  189. package/dist/storage/migrations.js.map +1 -1
  190. package/dist/storage/schema.sql +68 -2
  191. package/dist/storage/session-state-store.d.ts +61 -0
  192. package/dist/storage/session-state-store.d.ts.map +1 -0
  193. package/dist/storage/session-state-store.js +119 -0
  194. package/dist/storage/session-state-store.js.map +1 -0
  195. package/dist/storage/types.d.ts +50 -0
  196. package/dist/storage/types.d.ts.map +1 -1
  197. package/dist/storage/vector-store.d.ts +17 -2
  198. package/dist/storage/vector-store.d.ts.map +1 -1
  199. package/dist/storage/vector-store.js +96 -36
  200. package/dist/storage/vector-store.js.map +1 -1
  201. package/package.json +4 -2
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Batch index entry refinement and backfill.
3
+ *
4
+ * Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
5
+ * keychain API key. Used by the backfill-index maintenance task.
6
+ */
7
+ /** Result from a backfill run. */
8
+ export interface BackfillResult {
9
+ /** Number of new index entries created */
10
+ entriesCreated: number;
11
+ /** Number of entries created via LLM */
12
+ llmEntries: number;
13
+ /** Number of entries created via Jeopardy-style generation */
14
+ jeopardyEntries: number;
15
+ /** Number of entries created via heuristic */
16
+ heuristicEntries: number;
17
+ /** Number of chunks that had no content to index */
18
+ skipped: number;
19
+ /** Total duration in milliseconds */
20
+ durationMs: number;
21
+ }
22
+ /**
23
+ * Index entry refresher for batch backfill and refinement.
24
+ */
25
+ export declare class IndexRefresher {
26
+ private client;
27
+ private rateLimiter;
28
+ private config;
29
+ constructor();
30
+ /**
31
+ * Initialize the Anthropic client. Returns null if no API key available.
32
+ */
33
+ private getClient;
34
+ /**
35
+ * Backfill index entries for chunks that don't have them.
36
+ *
37
+ * Processes chunks in session-grouped batches. Uses LLM when available,
38
+ * falls back to heuristic. Also embeds the generated descriptions.
39
+ */
40
+ backfill(options?: {
41
+ limit?: number;
42
+ onProgress?: (current: number, total: number) => void;
43
+ }): Promise<BackfillResult>;
44
+ /**
45
+ * Get backfill status: how many chunks still need index entries.
46
+ */
47
+ getBackfillStatus(): {
48
+ indexed: number;
49
+ total: number;
50
+ remaining: number;
51
+ };
52
+ }
53
+ export declare const indexRefresher: IndexRefresher;
54
+ //# sourceMappingURL=index-refresher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-refresher.d.ts","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,MAAM,CAAe;;IAM7B;;OAEG;YACW,SAAS;IAuBvB;;;;;OAKG;IACG,QAAQ,CAAC,OAAO,CAAC,EAAE;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACvD,GAAG,OAAO,CAAC,cAAc,CAAC;IAkI3B;;OAEG;IACH,iBAAiB,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAU3E;AAGD,eAAO,MAAM,cAAc,gBAAuB,CAAC"}
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Batch index entry refinement and backfill.
3
+ *
4
+ * Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
5
+ * keychain API key. Used by the backfill-index maintenance task.
6
+ */
7
+ import Anthropic from '@anthropic-ai/sdk';
8
+ import { getConfig } from '../config/memory-config.js';
9
+ import { getChunksByIds } from '../storage/chunk-store.js';
10
+ import { getUnindexedChunkIds, insertIndexEntries, getIndexedChunkCount, } from '../storage/index-entry-store.js';
11
+ import { getChunkCount } from '../storage/chunk-store.js';
12
+ import { indexVectorStore } from '../storage/vector-store.js';
13
+ import { Embedder } from '../models/embedder.js';
14
+ import { getModel } from '../models/model-registry.js';
15
+ import { createSecretStore } from '../utils/secret-store.js';
16
+ import { createLogger } from '../utils/logger.js';
17
+ import { generateLLMEntries, generateHeuristicEntry } from './index-generator.js';
18
+ const log = createLogger('index-refresher');
19
+ /** Rate limiter for API calls. */
20
+ class RateLimiter {
21
+ lastCall = 0;
22
+ minIntervalMs;
23
+ constructor(callsPerMinute) {
24
+ this.minIntervalMs = (60 * 1000) / callsPerMinute;
25
+ }
26
+ async wait() {
27
+ const now = Date.now();
28
+ const elapsed = now - this.lastCall;
29
+ if (elapsed < this.minIntervalMs) {
30
+ await new Promise((resolve) => setTimeout(resolve, this.minIntervalMs - elapsed));
31
+ }
32
+ this.lastCall = Date.now();
33
+ }
34
+ }
35
+ /**
36
+ * Index entry refresher for batch backfill and refinement.
37
+ */
38
+ export class IndexRefresher {
39
+ client = null;
40
+ rateLimiter;
41
+ config = getConfig();
42
+ constructor() {
43
+ this.rateLimiter = new RateLimiter(this.config.refreshRateLimitPerMin);
44
+ }
45
+ /**
46
+ * Initialize the Anthropic client. Returns null if no API key available.
47
+ */
48
+ async getClient() {
49
+ if (!this.client) {
50
+ if (!process.env.ANTHROPIC_API_KEY) {
51
+ try {
52
+ const store = createSecretStore();
53
+ const storedKey = await store.get('anthropic-api-key');
54
+ if (storedKey) {
55
+ process.env.ANTHROPIC_API_KEY = storedKey;
56
+ }
57
+ }
58
+ catch {
59
+ // Keychain not available
60
+ }
61
+ }
62
+ if (!process.env.ANTHROPIC_API_KEY) {
63
+ return null;
64
+ }
65
+ this.client = new Anthropic();
66
+ }
67
+ return this.client;
68
+ }
69
+ /**
70
+ * Backfill index entries for chunks that don't have them.
71
+ *
72
+ * Processes chunks in session-grouped batches. Uses LLM when available,
73
+ * falls back to heuristic. Also embeds the generated descriptions.
74
+ */
75
+ async backfill(options) {
76
+ const startTime = Date.now();
77
+ const limit = options?.limit ?? this.config.semanticIndex.batchRefreshLimit;
78
+ // Find chunks without index entries
79
+ const unindexedChunkIds = getUnindexedChunkIds(limit);
80
+ if (unindexedChunkIds.length === 0) {
81
+ return {
82
+ entriesCreated: 0,
83
+ llmEntries: 0,
84
+ jeopardyEntries: 0,
85
+ heuristicEntries: 0,
86
+ skipped: 0,
87
+ durationMs: Date.now() - startTime,
88
+ };
89
+ }
90
+ log.info('Starting index backfill', { chunkCount: unindexedChunkIds.length });
91
+ // Load chunks
92
+ const chunks = getChunksByIds(unindexedChunkIds);
93
+ // Group by session slug for batched LLM calls
94
+ const sessionGroups = new Map();
95
+ for (const chunk of chunks) {
96
+ const group = sessionGroups.get(chunk.sessionSlug) ?? [];
97
+ group.push({
98
+ id: chunk.id,
99
+ sessionSlug: chunk.sessionSlug,
100
+ startTime: chunk.startTime,
101
+ content: chunk.content,
102
+ approxTokens: chunk.approxTokens,
103
+ agentId: chunk.agentId,
104
+ teamName: chunk.teamName,
105
+ });
106
+ sessionGroups.set(chunk.sessionSlug, group);
107
+ }
108
+ let entriesCreated = 0;
109
+ let llmEntries = 0;
110
+ let jeopardyEntries = 0;
111
+ let heuristicEntries = 0;
112
+ let skipped = 0;
113
+ // Set up embedder
114
+ const embedder = new Embedder();
115
+ const embeddingModel = this.config.embeddingModel;
116
+ await embedder.load(getModel(embeddingModel));
117
+ indexVectorStore.setModelId(embeddingModel);
118
+ try {
119
+ const client = await this.getClient();
120
+ let sessionIndex = 0;
121
+ const totalSessions = sessionGroups.size;
122
+ for (const [sessionSlug, sessionChunks] of sessionGroups) {
123
+ sessionIndex++;
124
+ options?.onProgress?.(sessionIndex, totalSessions);
125
+ // Generate entries (rate limiting happens per sub-batch inside generateLLMEntries)
126
+ let entries;
127
+ if (client) {
128
+ entries = await generateLLMEntries(sessionChunks, sessionSlug, {
129
+ onBeforeBatch: () => this.rateLimiter.wait(),
130
+ });
131
+ }
132
+ else {
133
+ entries = sessionChunks.map((chunk) => generateHeuristicEntry(chunk, sessionSlug));
134
+ }
135
+ // Filter out empty descriptions
136
+ const validEntries = entries.filter((e) => e.description.trim().length > 0);
137
+ skipped += entries.length - validEntries.length;
138
+ if (validEntries.length === 0)
139
+ continue;
140
+ // Insert entries
141
+ const entryIds = insertIndexEntries(validEntries);
142
+ // Embed descriptions
143
+ const embeddings = [];
144
+ for (let i = 0; i < validEntries.length; i++) {
145
+ const result = await embedder.embed(validEntries[i].description, false);
146
+ embeddings.push({ id: entryIds[i], embedding: result.embedding });
147
+ }
148
+ await indexVectorStore.insertBatch(embeddings);
149
+ // Track stats
150
+ for (const entry of validEntries) {
151
+ if (entry.generationMethod === 'jeopardy') {
152
+ jeopardyEntries++;
153
+ }
154
+ else if (entry.generationMethod === 'llm') {
155
+ llmEntries++;
156
+ }
157
+ else {
158
+ heuristicEntries++;
159
+ }
160
+ }
161
+ entriesCreated += validEntries.length;
162
+ log.debug('Backfilled session', {
163
+ sessionSlug,
164
+ entries: validEntries.length,
165
+ });
166
+ }
167
+ }
168
+ finally {
169
+ await embedder.dispose();
170
+ }
171
+ log.info('Index backfill complete', {
172
+ entriesCreated,
173
+ llmEntries,
174
+ jeopardyEntries,
175
+ heuristicEntries,
176
+ skipped,
177
+ durationMs: Date.now() - startTime,
178
+ });
179
+ return {
180
+ entriesCreated,
181
+ llmEntries,
182
+ jeopardyEntries,
183
+ heuristicEntries,
184
+ skipped,
185
+ durationMs: Date.now() - startTime,
186
+ };
187
+ }
188
+ /**
189
+ * Get backfill status: how many chunks still need index entries.
190
+ */
191
+ getBackfillStatus() {
192
+ const indexed = getIndexedChunkCount();
193
+ const total = getChunkCount();
194
+ return {
195
+ indexed,
196
+ total,
197
+ remaining: total - indexed,
198
+ };
199
+ }
200
+ }
201
+ // Singleton
202
+ export const indexRefresher = new IndexRefresher();
203
+ //# sourceMappingURL=index-refresher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-refresher.js","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAGlF,MAAM,GAAG,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC;AAE5C,kCAAkC;AAClC,MAAM,WAAW;IACP,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAAS;IAE9B,YAAY,cAAsB;QAChC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,cAAc,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;CACF;AAkBD;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,GAAqB,IAAI,CAAC;IAChC,WAAW,CAAc;IACzB,MAAM,GAAG,SAAS,EAAE,CAAC;IAE7B;QACE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;oBAClC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;oBACvD,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC;oBAC5C,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,OAGd;QACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC;QAE5E,oCAAoC;QACpC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAEtD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO;gBACL,cAAc,EAAE,CAAC;gBACjB,UAAU,EAAE,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,gBAAgB,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,UAAU,EAAE,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9E,cAAc;QACd,MAAM,MAAM,GAAG,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAEjD,8CAA8C;QAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAA8B,CAAC;QAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC;gBACT,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,kBAAkB;QAClB,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAClD,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;QAC9C,gBAAgB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;YAEzC,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,aAAa,EAAE,CAAC;gBACzD,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,UAAU,EAAE,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;gBAEnD,mFAAmF;gBACnF,IAAI,OAAO,CAAC;gBACZ,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE;wBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;qBAC7C,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAC3C,CAAC;gBACJ,CAAC;gBAED,gCAAgC;gBAChC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5E,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;gBAEhD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAExC,iBAAiB;gBACjB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAElD,qBAAqB;gBACrB,MAAM,UAAU,GAA+C,EAAE,CAAC;gBAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAE/C,cAAc;gBACd,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;oBACjC,IAAI,KAAK,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;wBAC1C,eAAe,EAAE,CAAC;oBACpB,CAAC;yBAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;wBAC5C,UAAU,EAAE,CAAC;oBACf,CAAC;yBAAM,CAAC;wBACN,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBACD,cAAc,IAAI,YAAY,CAAC,MAAM,CAAC;gBAEtC,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE;oBAC9B,WAAW;oBACX,OAAO,EAAE,YAAY,CAAC,MAAM;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAClC,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC,CAAC;QAEH,OAAO;YACL,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;QAE9B,OAAO;YACL,OAAO;YACP,KAAK;YACL,SAAS,EAAE,KAAK,GAAG,OAAO;SAC3B,CAAC;IACJ,CAAC;CACF;AAED,YAAY;AACZ,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Semantic index layer exports.
3
+ */
4
+ export { generateLLMEntries, generateHeuristicEntry, type ChunkForIndexing, type GenerateOptions, } from './index-generator.js';
5
+ export { IndexRefresher, indexRefresher, type BackfillResult, } from './index-refresher.js';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,cAAc,EACd,cAAc,EACd,KAAK,cAAc,GACpB,MAAM,sBAAsB,CAAC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Semantic index layer exports.
3
+ */
4
+ export { generateLLMEntries, generateHeuristicEntry, } from './index-generator.js';
5
+ export { IndexRefresher, indexRefresher, } from './index-refresher.js';
6
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,cAAc,EACd,cAAc,GAEf,MAAM,sBAAsB,CAAC"}
package/dist/index.d.ts CHANGED
@@ -11,10 +11,14 @@ export * from './storage/encryption.js';
11
11
  export * from './storage/archive.js';
12
12
  export { ingestSession } from './ingest/ingest-session.js';
13
13
  export type { IngestResult, IngestOptions } from './ingest/ingest-session.js';
14
+ export { extractSessionState } from './ingest/session-state.js';
15
+ export type { SessionState, SessionError, SessionTask } from './ingest/session-state.js';
14
16
  export { batchIngest } from './ingest/batch-ingest.js';
15
17
  export type { BatchIngestOptions, BatchIngestResult } from './ingest/batch-ingest.js';
16
18
  export * from './retrieval/index.js';
17
19
  export * from './clusters/index.js';
20
+ export * from './index-entries/index.js';
21
+ export * from './repomap/index.js';
18
22
  export * from './maintenance/index.js';
19
23
  export * from './hooks/index.js';
20
24
  export { createSecretStore, getApiKey } from './utils/secret-store.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACzF,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
package/dist/index.js CHANGED
@@ -13,11 +13,16 @@ export * from './storage/encryption.js';
13
13
  export * from './storage/archive.js';
14
14
  // Ingestion
15
15
  export { ingestSession } from './ingest/ingest-session.js';
16
+ export { extractSessionState } from './ingest/session-state.js';
16
17
  export { batchIngest } from './ingest/batch-ingest.js';
17
18
  // Retrieval
18
19
  export * from './retrieval/index.js';
19
20
  // Clustering
20
21
  export * from './clusters/index.js';
22
+ // Semantic Index
23
+ export * from './index-entries/index.js';
24
+ // Repo Map
25
+ export * from './repomap/index.js';
21
26
  // Maintenance
22
27
  export * from './maintenance/index.js';
23
28
  // Hooks
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,iBAAiB;AACjB,cAAc,0BAA0B,CAAC;AAEzC,WAAW;AACX,cAAc,oBAAoB,CAAC;AAEnC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Hook for generating semantic index entries during ingestion.
3
+ *
4
+ * Called after chunks are stored and embedded. Generates index entries
5
+ * (LLM or heuristic) and embeds the descriptions into the index vector store.
6
+ */
7
+ import type { ChunkForIndexing } from '../index-entries/index-generator.js';
8
+ /**
9
+ * Generate and store index entries for newly ingested chunks.
10
+ *
11
+ * This is called from processMainSession() after chunks are stored and embedded.
12
+ * It's a best-effort operation — failures are logged but don't block ingestion.
13
+ */
14
+ export declare function generateIndexEntriesForChunks(chunks: ChunkForIndexing[], sessionSlug: string, _chunkEmbeddings: number[][], _chunkIds: string[], embeddingModel: string): Promise<void>;
15
+ //# sourceMappingURL=index-entry-hook.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-entry-hook.d.ts","sourceRoot":"","sources":["../../src/ingest/index-entry-hook.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAqB5E;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE,gBAAgB,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EAAE,EAAE,EAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAmDf"}
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Hook for generating semantic index entries during ingestion.
3
+ *
4
+ * Called after chunks are stored and embedded. Generates index entries
5
+ * (LLM or heuristic) and embeds the descriptions into the index vector store.
6
+ */
7
+ import { getConfig } from '../config/memory-config.js';
8
+ import { insertIndexEntries } from '../storage/index-entry-store.js';
9
+ import { indexVectorStore } from '../storage/vector-store.js';
10
+ import { Embedder } from '../models/embedder.js';
11
+ import { getModel } from '../models/model-registry.js';
12
+ import { generateLLMEntries, generateHeuristicEntry } from '../index-entries/index-generator.js';
13
+ import { createLogger } from '../utils/logger.js';
14
+ const log = createLogger('index-entry-hook');
15
+ /** Shared embedder for index entry descriptions. */
16
+ let sharedIndexEmbedder = null;
17
+ let sharedIndexEmbedderModelId = null;
18
+ async function getIndexEmbedder(modelId) {
19
+ if (!sharedIndexEmbedder || sharedIndexEmbedderModelId !== modelId) {
20
+ if (sharedIndexEmbedder) {
21
+ await sharedIndexEmbedder.dispose();
22
+ }
23
+ sharedIndexEmbedder = new Embedder();
24
+ await sharedIndexEmbedder.load(getModel(modelId));
25
+ sharedIndexEmbedderModelId = modelId;
26
+ }
27
+ return sharedIndexEmbedder;
28
+ }
29
+ /**
30
+ * Generate and store index entries for newly ingested chunks.
31
+ *
32
+ * This is called from processMainSession() after chunks are stored and embedded.
33
+ * It's a best-effort operation — failures are logged but don't block ingestion.
34
+ */
35
+ export async function generateIndexEntriesForChunks(chunks, sessionSlug, _chunkEmbeddings, _chunkIds, embeddingModel) {
36
+ const config = getConfig();
37
+ if (!config.semanticIndex.enabled)
38
+ return;
39
+ if (chunks.length === 0)
40
+ return;
41
+ try {
42
+ // Generate descriptions (LLM primary, heuristic fallback)
43
+ let entries;
44
+ try {
45
+ entries = await generateLLMEntries(chunks, sessionSlug);
46
+ }
47
+ catch {
48
+ log.debug('LLM generation unavailable, using heuristic');
49
+ entries = chunks.map((chunk) => generateHeuristicEntry(chunk, sessionSlug));
50
+ }
51
+ // Filter out empty descriptions
52
+ const validEntries = entries.filter((e) => e.description.trim().length > 0);
53
+ if (validEntries.length === 0)
54
+ return;
55
+ // Insert entries into database
56
+ const entryIds = insertIndexEntries(validEntries);
57
+ // Embed descriptions into index vector store
58
+ indexVectorStore.setModelId(embeddingModel);
59
+ const embedder = await getIndexEmbedder(embeddingModel);
60
+ const embeddings = [];
61
+ for (let i = 0; i < validEntries.length; i++) {
62
+ const result = await embedder.embed(validEntries[i].description, false);
63
+ embeddings.push({ id: entryIds[i], embedding: result.embedding });
64
+ }
65
+ await indexVectorStore.insertBatch(embeddings);
66
+ const methodCounts = {};
67
+ for (const e of validEntries) {
68
+ methodCounts[e.generationMethod] = (methodCounts[e.generationMethod] ?? 0) + 1;
69
+ }
70
+ log.debug('Generated index entries', {
71
+ sessionSlug,
72
+ count: validEntries.length,
73
+ ...methodCounts,
74
+ });
75
+ }
76
+ catch (error) {
77
+ // Non-fatal: index generation failure should not block ingestion
78
+ log.warn('Failed to generate index entries', {
79
+ sessionSlug,
80
+ error: error.message,
81
+ });
82
+ }
83
+ }
84
+ //# sourceMappingURL=index-entry-hook.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-entry-hook.js","sourceRoot":"","sources":["../../src/ingest/index-entry-hook.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAEjG,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,CAAC,kBAAkB,CAAC,CAAC;AAE7C,oDAAoD;AACpD,IAAI,mBAAmB,GAAoB,IAAI,CAAC;AAChD,IAAI,0BAA0B,GAAkB,IAAI,CAAC;AAErD,KAAK,UAAU,gBAAgB,CAAC,OAAe;IAC7C,IAAI,CAAC,mBAAmB,IAAI,0BAA0B,KAAK,OAAO,EAAE,CAAC;QACnE,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACtC,CAAC;QACD,mBAAmB,GAAG,IAAI,QAAQ,EAAE,CAAC;QACrC,MAAM,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAClD,0BAA0B,GAAG,OAAO,CAAC;IACvC,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAA0B,EAC1B,WAAmB,EACnB,gBAA4B,EAC5B,SAAmB,EACnB,cAAsB;IAEtB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO;QAAE,OAAO;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEhC,IAAI,CAAC;QACH,0DAA0D;QAC1D,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACzD,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;QAC9E,CAAC;QAED,gCAAgC;QAChC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5E,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEtC,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAElD,6CAA6C;QAC7C,gBAAgB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,UAAU,GAA+C,EAAE,CAAC;QAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YACxE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7B,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACjF,CAAC;QACD,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACnC,WAAW;YACX,KAAK,EAAE,YAAY,CAAC,MAAM;YAC1B,GAAG,YAAY;SAChB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,iEAAiE;QACjE,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE;YAC3C,WAAW;YACX,KAAK,EAAG,KAAe,CAAC,OAAO;SAChC,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"ingest-session.d.ts","sourceRoot":"","sources":["../../src/ingest/ingest-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAyBjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,KAAK,EAAQ,MAAM,oBAAoB,CAAC;AAStD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,+CAA+C;IAC/C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8CAA8C;IAC9C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,iDAAiD;IACjD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,+EAA+E;IAC/E,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,OAAO,EAAE,OAAO,CAAC;IACjB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,UAAU,CAAC,EAAE,kBAAkB,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACpE,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4CAA4C;IAC5C,aAAa,EAAE,OAAO,CAAC;CACxB;AA4BD;;GAEG;AACH,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,YAAY,CAAC,CAkWvB;AA6TD;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAarD"}
1
+ {"version":3,"file":"ingest-session.d.ts","sourceRoot":"","sources":["../../src/ingest/ingest-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAyBjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,KAAK,EAAQ,MAAM,oBAAoB,CAAC;AAatD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,+CAA+C;IAC/C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8CAA8C;IAC9C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,iDAAiD;IACjD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,+EAA+E;IAC/E,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,OAAO,EAAE,OAAO,CAAC;IACjB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,UAAU,CAAC,EAAE,kBAAkB,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACpE,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4CAA4C;IAC5C,aAAa,EAAE,OAAO,CAAC;CACxB;AA4BD;;GAEG;AACH,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,YAAY,CAAC,CAsXvB;AAkWD;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAarD"}