causantic 0.9.4 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/README.md +70 -56
  2. package/dist/cli/skill-templates.d.ts.map +1 -1
  3. package/dist/cli/skill-templates.js +11 -8
  4. package/dist/cli/skill-templates.js.map +1 -1
  5. package/dist/clusters/cluster-manager.d.ts +16 -0
  6. package/dist/clusters/cluster-manager.d.ts.map +1 -1
  7. package/dist/clusters/cluster-manager.js +119 -1
  8. package/dist/clusters/cluster-manager.js.map +1 -1
  9. package/dist/config/loader.d.ts +16 -0
  10. package/dist/config/loader.d.ts.map +1 -1
  11. package/dist/config/loader.js +51 -0
  12. package/dist/config/loader.js.map +1 -1
  13. package/dist/config/memory-config.d.ts +26 -0
  14. package/dist/config/memory-config.d.ts.map +1 -1
  15. package/dist/config/memory-config.js +40 -0
  16. package/dist/config/memory-config.js.map +1 -1
  17. package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts +20 -0
  18. package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts.map +1 -0
  19. package/dist/eval/experiments/embedding-model-comparison/run-experiment.js +289 -0
  20. package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -0
  21. package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts +53 -0
  22. package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -0
  23. package/dist/eval/experiments/index-differentiation/alignment-analysis.js +85 -0
  24. package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -0
  25. package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts +24 -0
  26. package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -0
  27. package/dist/eval/experiments/index-differentiation/discrimination-test.js +79 -0
  28. package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -0
  29. package/dist/eval/experiments/index-differentiation/index.d.ts +11 -0
  30. package/dist/eval/experiments/index-differentiation/index.d.ts.map +1 -0
  31. package/dist/eval/experiments/index-differentiation/index.js +8 -0
  32. package/dist/eval/experiments/index-differentiation/index.js.map +1 -0
  33. package/dist/eval/experiments/index-differentiation/refinement-test.d.ts +32 -0
  34. package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -0
  35. package/dist/eval/experiments/index-differentiation/refinement-test.js +201 -0
  36. package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -0
  37. package/dist/eval/experiments/index-differentiation/run-experiment.d.ts +20 -0
  38. package/dist/eval/experiments/index-differentiation/run-experiment.d.ts.map +1 -0
  39. package/dist/eval/experiments/index-differentiation/run-experiment.js +336 -0
  40. package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -0
  41. package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts +31 -0
  42. package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -0
  43. package/dist/eval/experiments/index-differentiation/similarity-analysis.js +60 -0
  44. package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -0
  45. package/dist/eval/experiments/index-differentiation/types.d.ts +114 -0
  46. package/dist/eval/experiments/index-differentiation/types.d.ts.map +1 -0
  47. package/dist/eval/experiments/index-differentiation/types.js +8 -0
  48. package/dist/eval/experiments/index-differentiation/types.js.map +1 -0
  49. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts +19 -0
  50. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts.map +1 -0
  51. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +326 -0
  52. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -0
  53. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts +27 -0
  54. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts.map +1 -0
  55. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js +154 -0
  56. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -0
  57. package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts +23 -0
  58. package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts.map +1 -0
  59. package/dist/eval/experiments/index-vs-chunk/query-generator.js +113 -0
  60. package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -0
  61. package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts +17 -0
  62. package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts.map +1 -0
  63. package/dist/eval/experiments/index-vs-chunk/run-experiment.js +331 -0
  64. package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -0
  65. package/dist/eval/experiments/index-vs-chunk/types.d.ts +71 -0
  66. package/dist/eval/experiments/index-vs-chunk/types.d.ts.map +1 -0
  67. package/dist/eval/experiments/index-vs-chunk/types.js +8 -0
  68. package/dist/eval/experiments/index-vs-chunk/types.js.map +1 -0
  69. package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts +18 -0
  70. package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts.map +1 -0
  71. package/dist/eval/experiments/pipeline-dropout/run-experiment.js +355 -0
  72. package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -0
  73. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts +17 -0
  74. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts.map +1 -0
  75. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js +247 -0
  76. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -0
  77. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts +18 -0
  78. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts.map +1 -0
  79. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +457 -0
  80. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -0
  81. package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts +16 -0
  82. package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts.map +1 -0
  83. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +226 -0
  84. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -0
  85. package/dist/index-entries/index-generator.d.ts +74 -0
  86. package/dist/index-entries/index-generator.d.ts.map +1 -0
  87. package/dist/index-entries/index-generator.js +321 -0
  88. package/dist/index-entries/index-generator.js.map +1 -0
  89. package/dist/index-entries/index-refresher.d.ts +54 -0
  90. package/dist/index-entries/index-refresher.d.ts.map +1 -0
  91. package/dist/index-entries/index-refresher.js +203 -0
  92. package/dist/index-entries/index-refresher.js.map +1 -0
  93. package/dist/index-entries/index.d.ts +6 -0
  94. package/dist/index-entries/index.d.ts.map +1 -0
  95. package/dist/index-entries/index.js +6 -0
  96. package/dist/index-entries/index.js.map +1 -0
  97. package/dist/index.d.ts +4 -0
  98. package/dist/index.d.ts.map +1 -1
  99. package/dist/index.js +5 -0
  100. package/dist/index.js.map +1 -1
  101. package/dist/ingest/entity-extractor.d.ts +23 -0
  102. package/dist/ingest/entity-extractor.d.ts.map +1 -0
  103. package/dist/ingest/entity-extractor.js +233 -0
  104. package/dist/ingest/entity-extractor.js.map +1 -0
  105. package/dist/ingest/index-entry-hook.d.ts +15 -0
  106. package/dist/ingest/index-entry-hook.d.ts.map +1 -0
  107. package/dist/ingest/index-entry-hook.js +84 -0
  108. package/dist/ingest/index-entry-hook.js.map +1 -0
  109. package/dist/ingest/ingest-session.d.ts.map +1 -1
  110. package/dist/ingest/ingest-session.js +89 -17
  111. package/dist/ingest/ingest-session.js.map +1 -1
  112. package/dist/ingest/session-state.d.ts +49 -0
  113. package/dist/ingest/session-state.d.ts.map +1 -0
  114. package/dist/ingest/session-state.js +146 -0
  115. package/dist/ingest/session-state.js.map +1 -0
  116. package/dist/maintenance/scheduler.d.ts.map +1 -1
  117. package/dist/maintenance/scheduler.js +25 -0
  118. package/dist/maintenance/scheduler.js.map +1 -1
  119. package/dist/maintenance/tasks/backfill-index.d.ts +27 -0
  120. package/dist/maintenance/tasks/backfill-index.d.ts.map +1 -0
  121. package/dist/maintenance/tasks/backfill-index.js +44 -0
  122. package/dist/maintenance/tasks/backfill-index.js.map +1 -0
  123. package/dist/mcp/services.d.ts.map +1 -1
  124. package/dist/mcp/services.js +9 -0
  125. package/dist/mcp/services.js.map +1 -1
  126. package/dist/mcp/tools.d.ts +4 -0
  127. package/dist/mcp/tools.d.ts.map +1 -1
  128. package/dist/mcp/tools.js +113 -7
  129. package/dist/mcp/tools.js.map +1 -1
  130. package/dist/models/embedder.js +2 -2
  131. package/dist/models/embedder.js.map +1 -1
  132. package/dist/models/model-registry.d.ts +2 -0
  133. package/dist/models/model-registry.d.ts.map +1 -1
  134. package/dist/models/model-registry.js +15 -0
  135. package/dist/models/model-registry.js.map +1 -1
  136. package/dist/repomap/cache.d.ts +58 -0
  137. package/dist/repomap/cache.d.ts.map +1 -0
  138. package/dist/repomap/cache.js +101 -0
  139. package/dist/repomap/cache.js.map +1 -0
  140. package/dist/repomap/graph.d.ts +54 -0
  141. package/dist/repomap/graph.d.ts.map +1 -0
  142. package/dist/repomap/graph.js +113 -0
  143. package/dist/repomap/graph.js.map +1 -0
  144. package/dist/repomap/index.d.ts +83 -0
  145. package/dist/repomap/index.d.ts.map +1 -0
  146. package/dist/repomap/index.js +99 -0
  147. package/dist/repomap/index.js.map +1 -0
  148. package/dist/repomap/parser.d.ts +43 -0
  149. package/dist/repomap/parser.d.ts.map +1 -0
  150. package/dist/repomap/parser.js +1043 -0
  151. package/dist/repomap/parser.js.map +1 -0
  152. package/dist/repomap/regex-parser.d.ts +24 -0
  153. package/dist/repomap/regex-parser.d.ts.map +1 -0
  154. package/dist/repomap/regex-parser.js +214 -0
  155. package/dist/repomap/regex-parser.js.map +1 -0
  156. package/dist/repomap/renderer.d.ts +40 -0
  157. package/dist/repomap/renderer.d.ts.map +1 -0
  158. package/dist/repomap/renderer.js +163 -0
  159. package/dist/repomap/renderer.js.map +1 -0
  160. package/dist/repomap/scanner.d.ts +32 -0
  161. package/dist/repomap/scanner.d.ts.map +1 -0
  162. package/dist/repomap/scanner.js +190 -0
  163. package/dist/repomap/scanner.js.map +1 -0
  164. package/dist/retrieval/chain-assembler.d.ts.map +1 -1
  165. package/dist/retrieval/chain-assembler.js +22 -3
  166. package/dist/retrieval/chain-assembler.js.map +1 -1
  167. package/dist/retrieval/context-assembler.d.ts +1 -1
  168. package/dist/retrieval/context-assembler.d.ts.map +1 -1
  169. package/dist/retrieval/index.d.ts +2 -0
  170. package/dist/retrieval/index.d.ts.map +1 -1
  171. package/dist/retrieval/index.js +2 -0
  172. package/dist/retrieval/index.js.map +1 -1
  173. package/dist/retrieval/mmr.d.ts +1 -0
  174. package/dist/retrieval/mmr.d.ts.map +1 -1
  175. package/dist/retrieval/mmr.js +35 -1
  176. package/dist/retrieval/mmr.js.map +1 -1
  177. package/dist/retrieval/rrf.d.ts +1 -1
  178. package/dist/retrieval/rrf.d.ts.map +1 -1
  179. package/dist/retrieval/rrf.js +1 -1
  180. package/dist/retrieval/rrf.js.map +1 -1
  181. package/dist/retrieval/search-assembler.d.ts +11 -2
  182. package/dist/retrieval/search-assembler.d.ts.map +1 -1
  183. package/dist/retrieval/search-assembler.js +305 -81
  184. package/dist/retrieval/search-assembler.js.map +1 -1
  185. package/dist/retrieval/session-reconstructor.d.ts +36 -0
  186. package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
  187. package/dist/retrieval/session-reconstructor.js +128 -0
  188. package/dist/retrieval/session-reconstructor.js.map +1 -1
  189. package/dist/storage/db.d.ts.map +1 -1
  190. package/dist/storage/db.js +24 -0
  191. package/dist/storage/db.js.map +1 -1
  192. package/dist/storage/entity-store.d.ts +48 -0
  193. package/dist/storage/entity-store.d.ts.map +1 -0
  194. package/dist/storage/entity-store.js +111 -0
  195. package/dist/storage/entity-store.js.map +1 -0
  196. package/dist/storage/index-entry-store.d.ts +71 -0
  197. package/dist/storage/index-entry-store.d.ts.map +1 -0
  198. package/dist/storage/index-entry-store.js +277 -0
  199. package/dist/storage/index-entry-store.js.map +1 -0
  200. package/dist/storage/index.d.ts +5 -2
  201. package/dist/storage/index.d.ts.map +1 -1
  202. package/dist/storage/index.js +5 -1
  203. package/dist/storage/index.js.map +1 -1
  204. package/dist/storage/migrations.d.ts.map +1 -1
  205. package/dist/storage/migrations.js +147 -0
  206. package/dist/storage/migrations.js.map +1 -1
  207. package/dist/storage/schema.sql +104 -2
  208. package/dist/storage/session-state-store.d.ts +61 -0
  209. package/dist/storage/session-state-store.d.ts.map +1 -0
  210. package/dist/storage/session-state-store.js +157 -0
  211. package/dist/storage/session-state-store.js.map +1 -0
  212. package/dist/storage/types.d.ts +50 -0
  213. package/dist/storage/types.d.ts.map +1 -1
  214. package/dist/storage/vector-store.d.ts +17 -2
  215. package/dist/storage/vector-store.d.ts.map +1 -1
  216. package/dist/storage/vector-store.js +104 -36
  217. package/dist/storage/vector-store.js.map +1 -1
  218. package/package.json +4 -2
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Batch index entry refinement and backfill.
3
+ *
4
+ * Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
5
+ * keychain API key. Used by the backfill-index maintenance task.
6
+ */
7
+ /** Result from a backfill run. */
8
+ export interface BackfillResult {
9
+ /** Number of new index entries created */
10
+ entriesCreated: number;
11
+ /** Number of entries created via LLM */
12
+ llmEntries: number;
13
+ /** Number of entries created via Jeopardy-style generation */
14
+ jeopardyEntries: number;
15
+ /** Number of entries created via heuristic */
16
+ heuristicEntries: number;
17
+ /** Number of chunks that had no content to index */
18
+ skipped: number;
19
+ /** Total duration in milliseconds */
20
+ durationMs: number;
21
+ }
22
+ /**
23
+ * Index entry refresher for batch backfill and refinement.
24
+ */
25
+ export declare class IndexRefresher {
26
+ private client;
27
+ private rateLimiter;
28
+ private config;
29
+ constructor();
30
+ /**
31
+ * Initialize the Anthropic client. Returns null if no API key available.
32
+ */
33
+ private getClient;
34
+ /**
35
+ * Backfill index entries for chunks that don't have them.
36
+ *
37
+ * Processes chunks in session-grouped batches. Uses LLM when available,
38
+ * falls back to heuristic. Also embeds the generated descriptions.
39
+ */
40
+ backfill(options?: {
41
+ limit?: number;
42
+ onProgress?: (current: number, total: number) => void;
43
+ }): Promise<BackfillResult>;
44
+ /**
45
+ * Get backfill status: how many chunks still need index entries.
46
+ */
47
+ getBackfillStatus(): {
48
+ indexed: number;
49
+ total: number;
50
+ remaining: number;
51
+ };
52
+ }
53
+ export declare const indexRefresher: IndexRefresher;
54
+ //# sourceMappingURL=index-refresher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-refresher.d.ts","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,MAAM,CAAe;;IAM7B;;OAEG;YACW,SAAS;IAuBvB;;;;;OAKG;IACG,QAAQ,CAAC,OAAO,CAAC,EAAE;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACvD,GAAG,OAAO,CAAC,cAAc,CAAC;IAgI3B;;OAEG;IACH,iBAAiB,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAU3E;AAGD,eAAO,MAAM,cAAc,gBAAuB,CAAC"}
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Batch index entry refinement and backfill.
3
+ *
4
+ * Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
5
+ * keychain API key. Used by the backfill-index maintenance task.
6
+ */
7
+ import Anthropic from '@anthropic-ai/sdk';
8
+ import { getConfig } from '../config/memory-config.js';
9
+ import { getChunksByIds } from '../storage/chunk-store.js';
10
+ import { getUnindexedChunkIds, insertIndexEntries, getIndexedChunkCount, } from '../storage/index-entry-store.js';
11
+ import { getChunkCount } from '../storage/chunk-store.js';
12
+ import { indexVectorStore } from '../storage/vector-store.js';
13
+ import { Embedder } from '../models/embedder.js';
14
+ import { getModel } from '../models/model-registry.js';
15
+ import { createSecretStore } from '../utils/secret-store.js';
16
+ import { createLogger } from '../utils/logger.js';
17
+ import { generateLLMEntries, generateHeuristicEntry } from './index-generator.js';
18
+ const log = createLogger('index-refresher');
19
+ /** Rate limiter for API calls. */
20
+ class RateLimiter {
21
+ lastCall = 0;
22
+ minIntervalMs;
23
+ constructor(callsPerMinute) {
24
+ this.minIntervalMs = (60 * 1000) / callsPerMinute;
25
+ }
26
+ async wait() {
27
+ const now = Date.now();
28
+ const elapsed = now - this.lastCall;
29
+ if (elapsed < this.minIntervalMs) {
30
+ await new Promise((resolve) => setTimeout(resolve, this.minIntervalMs - elapsed));
31
+ }
32
+ this.lastCall = Date.now();
33
+ }
34
+ }
35
+ /**
36
+ * Index entry refresher for batch backfill and refinement.
37
+ */
38
+ export class IndexRefresher {
39
+ client = null;
40
+ rateLimiter;
41
+ config = getConfig();
42
+ constructor() {
43
+ this.rateLimiter = new RateLimiter(this.config.refreshRateLimitPerMin);
44
+ }
45
+ /**
46
+ * Initialize the Anthropic client. Returns null if no API key available.
47
+ */
48
+ async getClient() {
49
+ if (!this.client) {
50
+ if (!process.env.ANTHROPIC_API_KEY) {
51
+ try {
52
+ const store = createSecretStore();
53
+ const storedKey = await store.get('anthropic-api-key');
54
+ if (storedKey) {
55
+ process.env.ANTHROPIC_API_KEY = storedKey;
56
+ }
57
+ }
58
+ catch {
59
+ // Keychain not available
60
+ }
61
+ }
62
+ if (!process.env.ANTHROPIC_API_KEY) {
63
+ return null;
64
+ }
65
+ this.client = new Anthropic();
66
+ }
67
+ return this.client;
68
+ }
69
+ /**
70
+ * Backfill index entries for chunks that don't have them.
71
+ *
72
+ * Processes chunks in session-grouped batches. Uses LLM when available,
73
+ * falls back to heuristic. Also embeds the generated descriptions.
74
+ */
75
+ async backfill(options) {
76
+ const startTime = Date.now();
77
+ const limit = options?.limit ?? this.config.semanticIndex.batchRefreshLimit;
78
+ // Find chunks without index entries
79
+ const unindexedChunkIds = getUnindexedChunkIds(limit);
80
+ if (unindexedChunkIds.length === 0) {
81
+ return {
82
+ entriesCreated: 0,
83
+ llmEntries: 0,
84
+ jeopardyEntries: 0,
85
+ heuristicEntries: 0,
86
+ skipped: 0,
87
+ durationMs: Date.now() - startTime,
88
+ };
89
+ }
90
+ log.info('Starting index backfill', { chunkCount: unindexedChunkIds.length });
91
+ // Load chunks
92
+ const chunks = getChunksByIds(unindexedChunkIds);
93
+ // Group by session slug for batched LLM calls
94
+ const sessionGroups = new Map();
95
+ for (const chunk of chunks) {
96
+ const group = sessionGroups.get(chunk.sessionSlug) ?? [];
97
+ group.push({
98
+ id: chunk.id,
99
+ sessionSlug: chunk.sessionSlug,
100
+ startTime: chunk.startTime,
101
+ content: chunk.content,
102
+ approxTokens: chunk.approxTokens,
103
+ agentId: chunk.agentId,
104
+ teamName: chunk.teamName,
105
+ });
106
+ sessionGroups.set(chunk.sessionSlug, group);
107
+ }
108
+ let entriesCreated = 0;
109
+ let llmEntries = 0;
110
+ let jeopardyEntries = 0;
111
+ let heuristicEntries = 0;
112
+ let skipped = 0;
113
+ // Set up embedder
114
+ const embedder = new Embedder();
115
+ const embeddingModel = this.config.embeddingModel;
116
+ await embedder.load(getModel(embeddingModel));
117
+ indexVectorStore.setModelId(embeddingModel);
118
+ try {
119
+ const client = await this.getClient();
120
+ let sessionIndex = 0;
121
+ const totalSessions = sessionGroups.size;
122
+ for (const [sessionSlug, sessionChunks] of sessionGroups) {
123
+ sessionIndex++;
124
+ options?.onProgress?.(sessionIndex, totalSessions);
125
+ // Generate entries (rate limiting happens per sub-batch inside generateLLMEntries)
126
+ let entries;
127
+ if (client) {
128
+ entries = await generateLLMEntries(sessionChunks, sessionSlug, {
129
+ onBeforeBatch: () => this.rateLimiter.wait(),
130
+ });
131
+ }
132
+ else {
133
+ entries = sessionChunks.map((chunk) => generateHeuristicEntry(chunk, sessionSlug));
134
+ }
135
+ // Filter out empty descriptions
136
+ const validEntries = entries.filter((e) => e.description.trim().length > 0);
137
+ skipped += entries.length - validEntries.length;
138
+ if (validEntries.length === 0)
139
+ continue;
140
+ // Insert entries
141
+ const entryIds = insertIndexEntries(validEntries);
142
+ // Embed descriptions
143
+ const embeddings = [];
144
+ for (let i = 0; i < validEntries.length; i++) {
145
+ const result = await embedder.embed(validEntries[i].description, false);
146
+ embeddings.push({ id: entryIds[i], embedding: result.embedding });
147
+ }
148
+ await indexVectorStore.insertBatch(embeddings);
149
+ // Track stats
150
+ for (const entry of validEntries) {
151
+ if (entry.generationMethod === 'jeopardy') {
152
+ jeopardyEntries++;
153
+ }
154
+ else if (entry.generationMethod === 'llm') {
155
+ llmEntries++;
156
+ }
157
+ else {
158
+ heuristicEntries++;
159
+ }
160
+ }
161
+ entriesCreated += validEntries.length;
162
+ log.debug('Backfilled session', {
163
+ sessionSlug,
164
+ entries: validEntries.length,
165
+ });
166
+ }
167
+ }
168
+ finally {
169
+ await embedder.dispose();
170
+ }
171
+ log.info('Index backfill complete', {
172
+ entriesCreated,
173
+ llmEntries,
174
+ jeopardyEntries,
175
+ heuristicEntries,
176
+ skipped,
177
+ durationMs: Date.now() - startTime,
178
+ });
179
+ return {
180
+ entriesCreated,
181
+ llmEntries,
182
+ jeopardyEntries,
183
+ heuristicEntries,
184
+ skipped,
185
+ durationMs: Date.now() - startTime,
186
+ };
187
+ }
188
+ /**
189
+ * Get backfill status: how many chunks still need index entries.
190
+ */
191
+ getBackfillStatus() {
192
+ const indexed = getIndexedChunkCount();
193
+ const total = getChunkCount();
194
+ return {
195
+ indexed,
196
+ total,
197
+ remaining: total - indexed,
198
+ };
199
+ }
200
+ }
201
+ // Singleton
202
+ export const indexRefresher = new IndexRefresher();
203
+ //# sourceMappingURL=index-refresher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-refresher.js","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAGlF,MAAM,GAAG,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC;AAE5C,kCAAkC;AAClC,MAAM,WAAW;IACP,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAAS;IAE9B,YAAY,cAAsB;QAChC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,cAAc,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;CACF;AAkBD;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,GAAqB,IAAI,CAAC;IAChC,WAAW,CAAc;IACzB,MAAM,GAAG,SAAS,EAAE,CAAC;IAE7B;QACE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;oBAClC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;oBACvD,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC;oBAC5C,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,OAGd;QACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC;QAE5E,oCAAoC;QACpC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAEtD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO;gBACL,cAAc,EAAE,CAAC;gBACjB,UAAU,EAAE,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,gBAAgB,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,UAAU,EAAE,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9E,cAAc;QACd,MAAM,MAAM,GAAG,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAEjD,8CAA8C;QAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAA8B,CAAC;QAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC;gBACT,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,kBAAkB;QAClB,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAClD,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;QAC9C,gBAAgB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;YAEzC,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,aAAa,EAAE,CAAC;gBACzD,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,UAAU,EAAE,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;gBAEnD,mFAAmF;gBACnF,IAAI,OAAO,CAAC;gBACZ,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE;wBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;qBAC7C,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;gBACrF,CAAC;gBAED,gCAAgC;gBAChC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5E,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;gBAEhD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAExC,iBAAiB;gBACjB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAElD,qBAAqB;gBACrB,MAAM,UAAU,GAA+C,EAAE,CAAC;gBAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAE/C,cAAc;gBACd,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;oBACjC,IAAI,KAAK,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;wBAC1C,eAAe,EAAE,CAAC;oBACpB,CAAC;yBAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;wBAC5C,UAAU,EAAE,CAAC;oBACf,CAAC;yBAAM,CAAC;wBACN,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBACD,cAAc,IAAI,YAAY,CAAC,MAAM,CAAC;gBAEtC,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE;oBAC9B,WAAW;oBACX,OAAO,EAAE,YAAY,CAAC,MAAM;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAClC,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC,CAAC;QAEH,OAAO;YACL,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;QAE9B,OAAO;YACL,OAAO;YACP,KAAK;YACL,SAAS,EAAE,KAAK,GAAG,OAAO;SAC3B,CAAC;IACJ,CAAC;CACF;AAED,YAAY;AACZ,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Semantic index layer exports.
3
+ */
4
+ export { generateLLMEntries, generateHeuristicEntry, type ChunkForIndexing, type GenerateOptions, } from './index-generator.js';
5
+ export { IndexRefresher, indexRefresher, type BackfillResult } from './index-refresher.js';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Semantic index layer exports.
3
+ */
4
+ export { generateLLMEntries, generateHeuristicEntry, } from './index-generator.js';
5
+ export { IndexRefresher, indexRefresher } from './index-refresher.js';
6
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,cAAc,EAAE,cAAc,EAAuB,MAAM,sBAAsB,CAAC"}
package/dist/index.d.ts CHANGED
@@ -11,10 +11,14 @@ export * from './storage/encryption.js';
11
11
  export * from './storage/archive.js';
12
12
  export { ingestSession } from './ingest/ingest-session.js';
13
13
  export type { IngestResult, IngestOptions } from './ingest/ingest-session.js';
14
+ export { extractSessionState } from './ingest/session-state.js';
15
+ export type { SessionState, SessionError, SessionTask } from './ingest/session-state.js';
14
16
  export { batchIngest } from './ingest/batch-ingest.js';
15
17
  export type { BatchIngestOptions, BatchIngestResult } from './ingest/batch-ingest.js';
16
18
  export * from './retrieval/index.js';
17
19
  export * from './clusters/index.js';
20
+ export * from './index-entries/index.js';
21
+ export * from './repomap/index.js';
18
22
  export * from './maintenance/index.js';
19
23
  export * from './hooks/index.js';
20
24
  export { createSecretStore, getApiKey } from './utils/secret-store.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACzF,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
package/dist/index.js CHANGED
@@ -13,11 +13,16 @@ export * from './storage/encryption.js';
13
13
  export * from './storage/archive.js';
14
14
  // Ingestion
15
15
  export { ingestSession } from './ingest/ingest-session.js';
16
+ export { extractSessionState } from './ingest/session-state.js';
16
17
  export { batchIngest } from './ingest/batch-ingest.js';
17
18
  // Retrieval
18
19
  export * from './retrieval/index.js';
19
20
  // Clustering
20
21
  export * from './clusters/index.js';
22
+ // Semantic Index
23
+ export * from './index-entries/index.js';
24
+ // Repo Map
25
+ export * from './repomap/index.js';
21
26
  // Maintenance
22
27
  export * from './maintenance/index.js';
23
28
  // Hooks
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,iBAAiB;AACjB,cAAc,0BAA0B,CAAC;AAEzC,WAAW;AACX,cAAc,oBAAoB,CAAC;AAEnC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Deterministic entity extraction from chunk content.
3
+ *
4
+ * Pure function — no LLM, no database access. Extracts mentions of
5
+ * people, channels, meetings, organizations, and URLs using regex patterns.
6
+ *
7
+ * Only extracts from [User] and [Assistant] blocks; skips [Thinking] blocks
8
+ * and code blocks (triple-backtick fencing) to reduce noise.
9
+ */
10
+ export type EntityType = 'person' | 'channel' | 'meeting' | 'organization' | 'url';
11
+ export interface EntityMention {
12
+ entityType: EntityType;
13
+ mentionForm: string;
14
+ normalizedName: string;
15
+ confidence: number;
16
+ }
17
+ /**
18
+ * Extract entities from chunk text.
19
+ *
20
+ * Returns deduplicated entity mentions with types, normalized names, and confidence scores.
21
+ */
22
+ export declare function extractEntities(text: string): EntityMention[];
23
+ //# sourceMappingURL=entity-extractor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"entity-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,cAAc,GAAG,KAAK,CAAC;AAEnF,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAoJD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CA4F7D"}
@@ -0,0 +1,233 @@
1
+ /**
2
+ * Deterministic entity extraction from chunk content.
3
+ *
4
+ * Pure function — no LLM, no database access. Extracts mentions of
5
+ * people, channels, meetings, organizations, and URLs using regex patterns.
6
+ *
7
+ * Only extracts from [User] and [Assistant] blocks; skips [Thinking] blocks
8
+ * and code blocks (triple-backtick fencing) to reduce noise.
9
+ */
10
+ /** Words that look like names in "X said" patterns but aren't. */
11
+ const BLOCKLIST = new Set([
12
+ // Pronouns
13
+ 'i',
14
+ 'he',
15
+ 'she',
16
+ 'we',
17
+ 'they',
18
+ 'you',
19
+ 'it',
20
+ 'me',
21
+ 'him',
22
+ 'her',
23
+ 'us',
24
+ 'them',
25
+ 'my',
26
+ 'his',
27
+ 'our',
28
+ 'their',
29
+ 'your',
30
+ 'its',
31
+ // Articles / determiners
32
+ 'the',
33
+ 'a',
34
+ 'an',
35
+ 'this',
36
+ 'that',
37
+ 'these',
38
+ 'those',
39
+ // Common sentence starters / conjunctions
40
+ 'but',
41
+ 'and',
42
+ 'or',
43
+ 'so',
44
+ 'if',
45
+ 'when',
46
+ 'then',
47
+ 'also',
48
+ 'just',
49
+ 'now',
50
+ 'here',
51
+ 'there',
52
+ 'what',
53
+ 'who',
54
+ 'how',
55
+ 'why',
56
+ 'which',
57
+ 'where',
58
+ 'some',
59
+ 'all',
60
+ 'any',
61
+ 'each',
62
+ 'every',
63
+ 'not',
64
+ 'no',
65
+ 'yes',
66
+ 'maybe',
67
+ 'perhaps',
68
+ // Common words that appear capitalized at start of sentences
69
+ 'however',
70
+ 'therefore',
71
+ 'meanwhile',
72
+ 'otherwise',
73
+ 'furthermore',
74
+ 'additionally',
75
+ 'finally',
76
+ 'first',
77
+ 'second',
78
+ 'third',
79
+ 'next',
80
+ 'last',
81
+ // Tool/system words often capitalized
82
+ 'error',
83
+ 'warning',
84
+ 'note',
85
+ 'todo',
86
+ 'fixme',
87
+ 'hack',
88
+ 'true',
89
+ 'false',
90
+ 'null',
91
+ 'undefined',
92
+ 'none',
93
+ ]);
94
+ /** Meeting-related keywords. */
95
+ const MEETING_KEYWORDS = /\b(standup|stand-up|retro|retrospective|1:1|one-on-one|sync|daily|weekly|sprint\s+review|sprint\s+planning|kick-?off|all-?hands)\b/gi;
96
+ /** Email regex. */
97
+ const EMAIL_PATTERN = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b/g;
98
+ /** URL regex — matches http/https URLs. */
99
+ const URL_PATTERN = /https?:\/\/[^\s<>"')\]]+/g;
100
+ /** @mention pattern (Slack/Discord style). */
101
+ const AT_MENTION_PATTERN = /@([a-zA-Z][a-zA-Z0-9._-]{0,30})\b/g;
102
+ /** #channel pattern. */
103
+ const CHANNEL_PATTERN = /#([a-zA-Z][a-zA-Z0-9_-]{0,50})\b/g;
104
+ /** "X said" / "with X" / "from X" patterns — capitalized proper nouns. */
105
+ const CONTEXTUAL_NAME_PATTERN = /\b([A-Z][a-z]{1,20}(?:\s+[A-Z][a-z]{1,20})?)\s+(?:said|says|mentioned|asked|replied|suggested|noted|reported|confirmed|explained|wrote|responded)\b/g;
106
+ const WITH_FROM_PATTERN = /\b(?:with|from|by|to|cc|cc'd|cced)\s+([A-Z][a-z]{1,20}(?:\s+[A-Z][a-z]{1,20})?)\b/g;
107
+ /**
108
+ * Strip code blocks (triple-backtick fenced) from text.
109
+ */
110
+ function stripCodeBlocks(text) {
111
+ return text.replace(/```[\s\S]*?```/g, '');
112
+ }
113
+ /**
114
+ * Extract only [User] and [Assistant] blocks, skipping [Thinking] blocks.
115
+ * If no block markers found, returns the full text (non-session content).
116
+ */
117
+ function extractRelevantBlocks(text) {
118
+ // Check if text has block markers
119
+ if (!text.includes('[User]') && !text.includes('[Assistant]')) {
120
+ return text;
121
+ }
122
+ const blocks = [];
123
+ // Split on block headers
124
+ const parts = text.split(/(\[(?:User|Assistant|Thinking)\])/);
125
+ let inRelevantBlock = false;
126
+ for (const part of parts) {
127
+ if (part === '[User]' || part === '[Assistant]') {
128
+ inRelevantBlock = true;
129
+ continue;
130
+ }
131
+ if (part === '[Thinking]') {
132
+ inRelevantBlock = false;
133
+ continue;
134
+ }
135
+ if (inRelevantBlock) {
136
+ blocks.push(part);
137
+ }
138
+ }
139
+ return blocks.join('\n');
140
+ }
141
+ /**
142
+ * Extract entities from chunk text.
143
+ *
144
+ * Returns deduplicated entity mentions with types, normalized names, and confidence scores.
145
+ */
146
+ export function extractEntities(text) {
147
+ // Pre-process: skip thinking blocks and code blocks
148
+ const relevant = extractRelevantBlocks(text);
149
+ const cleaned = stripCodeBlocks(relevant);
150
+ const mentions = [];
151
+ const seen = new Set(); // "type:normalized" dedup key
152
+ function addMention(mention) {
153
+ const key = `${mention.entityType}:${mention.normalizedName}`;
154
+ if (seen.has(key))
155
+ return;
156
+ seen.add(key);
157
+ mentions.push(mention);
158
+ }
159
+ // @mentions → person
160
+ for (const match of cleaned.matchAll(AT_MENTION_PATTERN)) {
161
+ const name = match[1];
162
+ addMention({
163
+ entityType: 'person',
164
+ mentionForm: `@${name}`,
165
+ normalizedName: name.toLowerCase(),
166
+ confidence: 0.95,
167
+ });
168
+ }
169
+ // #channels → channel
170
+ for (const match of cleaned.matchAll(CHANNEL_PATTERN)) {
171
+ const name = match[1];
172
+ // Skip common non-channel uses of # (hex colors, markdown headers handled by context)
173
+ if (/^[0-9a-fA-F]{3,8}$/.test(name))
174
+ continue;
175
+ addMention({
176
+ entityType: 'channel',
177
+ mentionForm: `#${name}`,
178
+ normalizedName: name.toLowerCase(),
179
+ confidence: 0.95,
180
+ });
181
+ }
182
+ // Email addresses → person
183
+ for (const match of cleaned.matchAll(EMAIL_PATTERN)) {
184
+ const email = match[0];
185
+ addMention({
186
+ entityType: 'person',
187
+ mentionForm: email,
188
+ normalizedName: email.toLowerCase(),
189
+ confidence: 0.9,
190
+ });
191
+ }
192
+ // URLs → url
193
+ for (const match of cleaned.matchAll(URL_PATTERN)) {
194
+ const url = match[0];
195
+ addMention({
196
+ entityType: 'url',
197
+ mentionForm: url,
198
+ normalizedName: url,
199
+ confidence: 1.0,
200
+ });
201
+ }
202
+ // "X said" / "with X" patterns → person (lower confidence)
203
+ for (const pattern of [CONTEXTUAL_NAME_PATTERN, WITH_FROM_PATTERN]) {
204
+ pattern.lastIndex = 0; // Reset stateful regex
205
+ for (const match of cleaned.matchAll(pattern)) {
206
+ const name = match[1];
207
+ const normalized = name.toLowerCase();
208
+ // Check blocklist for each word in the name
209
+ const words = normalized.split(/\s+/);
210
+ if (words.some((w) => BLOCKLIST.has(w)))
211
+ continue;
212
+ addMention({
213
+ entityType: 'person',
214
+ mentionForm: name,
215
+ normalizedName: normalized,
216
+ confidence: 0.6,
217
+ });
218
+ }
219
+ }
220
+ // Meeting keywords → meeting
221
+ MEETING_KEYWORDS.lastIndex = 0;
222
+ for (const match of cleaned.matchAll(MEETING_KEYWORDS)) {
223
+ const keyword = match[1];
224
+ addMention({
225
+ entityType: 'meeting',
226
+ mentionForm: keyword,
227
+ normalizedName: keyword.toLowerCase().replace(/\s+/g, '-'),
228
+ confidence: 0.7,
229
+ });
230
+ }
231
+ return mentions;
232
+ }
233
+ //# sourceMappingURL=entity-extractor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"entity-extractor.js","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAWH,kEAAkE;AAClE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,WAAW;IACX,GAAG;IACH,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,MAAM;IACN,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,KAAK;IACL,KAAK;IACL,OAAO;IACP,MAAM;IACN,KAAK;IACL,yBAAyB;IACzB,KAAK;IACL,GAAG;IACH,IAAI;IACJ,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,0CAA0C;IAC1C,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,OAAO;IACP,OAAO;IACP,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,IAAI;IACJ,KAAK;IACL,OAAO;IACP,SAAS;IACT,6DAA6D;IAC7D,SAAS;IACT,WAAW;IACX,WAAW;IACX,WAAW;IACX,aAAa;IACb,cAAc;IACd,SAAS;IACT,OAAO;IACP,QAAQ;IACR,OAAO;IACP,MAAM;IACN,MAAM;IACN,sCAAsC;IACtC,OAAO;IACP,SAAS;IACT,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,WAAW;IACX,MAAM;CACP,CAAC,CAAC;AAEH,gCAAgC;AAChC,MAAM,gBAAgB,GACpB,sIAAsI,CAAC;AAEzI,mBAAmB;AACnB,MAAM,aAAa,GAAG,qDAAqD,CAAC;AAE5E,2CAA2C;AAC3C,MAAM,WAAW,GAAG,2BAA2B,CAAC;AAEhD,8CAA8C;AAC9C,MAAM,kBAAkB,GAAG,oCAAoC,CAAC;AAEhE,wBAAwB;AACxB,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAE5D,0EAA0E;AAC1E,MAAM,uBAAuB,GAC3B,sJAAsJ,CAAC;AACzJ,MAAM,iBAAiB,GACrB,oFAAoF,CAAC;AAEvF;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,IAAY;IACzC,kCAAkC;IAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,yBAAyB;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAE9D,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAChD,eAAe,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,eAAe,GAAG,KAAK,CAAC;YACxB,SAAS;QACX,CAAC;QACD,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,oDAAoD;IACpD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,8BAA8B;IAE9D,SAAS,UAAU,CAAC,OAAsB;QACxC,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,qBAAqB;IACrB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,UAAU,CAAC;YACT,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,sFAAsF;QACtF,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9C,UAAU,CAAC;YACT,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,2BAA2B;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,UAAU,CAAC;YACT,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,KAAK;YAClB,cAAc,EAAE,KAAK,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,aAAa;IACb,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,UAAU,CAAC;YACT,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,GAAG;YAChB,cAAc,EAAE,GAAG;YACnB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,KAAK,MAAM,OAAO,IAAI,CAAC,uBAAuB,EAAE,iBAAiB,CAAC,EAAE,CAAC;QACnE,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,uBAAuB;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,4CAA4C;YAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YAClD,UAAU,CAAC;gBACT,UAAU,EAAE,QAAQ;gBACpB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,UAAU;gBAC1B,UAAU,EAAE,GAAG;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,gBAAgB,CAAC,SAAS,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,UAAU,CAAC;YACT,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,OAAO;YACpB,cAAc,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YAC1D,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Hook for generating semantic index entries during ingestion.
3
+ *
4
+ * Called after chunks are stored and embedded. Generates index entries
5
+ * (LLM or heuristic) and embeds the descriptions into the index vector store.
6
+ */
7
+ import type { ChunkForIndexing } from '../index-entries/index-generator.js';
8
+ /**
9
+ * Generate and store index entries for newly ingested chunks.
10
+ *
11
+ * This is called from processMainSession() after chunks are stored and embedded.
12
+ * It's a best-effort operation — failures are logged but don't block ingestion.
13
+ */
14
+ export declare function generateIndexEntriesForChunks(chunks: ChunkForIndexing[], sessionSlug: string, _chunkEmbeddings: number[][], _chunkIds: string[], embeddingModel: string): Promise<void>;
15
+ //# sourceMappingURL=index-entry-hook.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-entry-hook.d.ts","sourceRoot":"","sources":["../../src/ingest/index-entry-hook.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAqB5E;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE,gBAAgB,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EAAE,EAAE,EAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAmDf"}