causantic 0.9.4 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -56
- package/dist/cli/skill-templates.d.ts.map +1 -1
- package/dist/cli/skill-templates.js +11 -8
- package/dist/cli/skill-templates.js.map +1 -1
- package/dist/clusters/cluster-manager.d.ts +16 -0
- package/dist/clusters/cluster-manager.d.ts.map +1 -1
- package/dist/clusters/cluster-manager.js +119 -1
- package/dist/clusters/cluster-manager.js.map +1 -1
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +51 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/memory-config.d.ts +26 -0
- package/dist/config/memory-config.d.ts.map +1 -1
- package/dist/config/memory-config.js +40 -0
- package/dist/config/memory-config.js.map +1 -1
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts +20 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.js +289 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts +53 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js +85 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts +24 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.js +79 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/index.d.ts +11 -0
- package/dist/eval/experiments/index-differentiation/index.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/index.js +8 -0
- package/dist/eval/experiments/index-differentiation/index.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.d.ts +32 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.js +201 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.d.ts +20 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.js +336 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts +31 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.js +60 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/types.d.ts +114 -0
- package/dist/eval/experiments/index-differentiation/types.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/types.js +8 -0
- package/dist/eval/experiments/index-differentiation/types.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts +19 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +326 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts +27 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js +154 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts +23 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.js +113 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts +17 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js +331 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/types.d.ts +71 -0
- package/dist/eval/experiments/index-vs-chunk/types.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/types.js +8 -0
- package/dist/eval/experiments/index-vs-chunk/types.js.map +1 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts +18 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js +355 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts +17 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js +247 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts +18 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +457 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts +16 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +226 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -0
- package/dist/index-entries/index-generator.d.ts +74 -0
- package/dist/index-entries/index-generator.d.ts.map +1 -0
- package/dist/index-entries/index-generator.js +321 -0
- package/dist/index-entries/index-generator.js.map +1 -0
- package/dist/index-entries/index-refresher.d.ts +54 -0
- package/dist/index-entries/index-refresher.d.ts.map +1 -0
- package/dist/index-entries/index-refresher.js +203 -0
- package/dist/index-entries/index-refresher.js.map +1 -0
- package/dist/index-entries/index.d.ts +6 -0
- package/dist/index-entries/index.d.ts.map +1 -0
- package/dist/index-entries/index.js +6 -0
- package/dist/index-entries/index.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/entity-extractor.d.ts +23 -0
- package/dist/ingest/entity-extractor.d.ts.map +1 -0
- package/dist/ingest/entity-extractor.js +233 -0
- package/dist/ingest/entity-extractor.js.map +1 -0
- package/dist/ingest/index-entry-hook.d.ts +15 -0
- package/dist/ingest/index-entry-hook.d.ts.map +1 -0
- package/dist/ingest/index-entry-hook.js +84 -0
- package/dist/ingest/index-entry-hook.js.map +1 -0
- package/dist/ingest/ingest-session.d.ts.map +1 -1
- package/dist/ingest/ingest-session.js +89 -17
- package/dist/ingest/ingest-session.js.map +1 -1
- package/dist/ingest/session-state.d.ts +49 -0
- package/dist/ingest/session-state.d.ts.map +1 -0
- package/dist/ingest/session-state.js +146 -0
- package/dist/ingest/session-state.js.map +1 -0
- package/dist/maintenance/scheduler.d.ts.map +1 -1
- package/dist/maintenance/scheduler.js +25 -0
- package/dist/maintenance/scheduler.js.map +1 -1
- package/dist/maintenance/tasks/backfill-index.d.ts +27 -0
- package/dist/maintenance/tasks/backfill-index.d.ts.map +1 -0
- package/dist/maintenance/tasks/backfill-index.js +44 -0
- package/dist/maintenance/tasks/backfill-index.js.map +1 -0
- package/dist/mcp/services.d.ts.map +1 -1
- package/dist/mcp/services.js +9 -0
- package/dist/mcp/services.js.map +1 -1
- package/dist/mcp/tools.d.ts +4 -0
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +113 -7
- package/dist/mcp/tools.js.map +1 -1
- package/dist/models/embedder.js +2 -2
- package/dist/models/embedder.js.map +1 -1
- package/dist/models/model-registry.d.ts +2 -0
- package/dist/models/model-registry.d.ts.map +1 -1
- package/dist/models/model-registry.js +15 -0
- package/dist/models/model-registry.js.map +1 -1
- package/dist/repomap/cache.d.ts +58 -0
- package/dist/repomap/cache.d.ts.map +1 -0
- package/dist/repomap/cache.js +101 -0
- package/dist/repomap/cache.js.map +1 -0
- package/dist/repomap/graph.d.ts +54 -0
- package/dist/repomap/graph.d.ts.map +1 -0
- package/dist/repomap/graph.js +113 -0
- package/dist/repomap/graph.js.map +1 -0
- package/dist/repomap/index.d.ts +83 -0
- package/dist/repomap/index.d.ts.map +1 -0
- package/dist/repomap/index.js +99 -0
- package/dist/repomap/index.js.map +1 -0
- package/dist/repomap/parser.d.ts +43 -0
- package/dist/repomap/parser.d.ts.map +1 -0
- package/dist/repomap/parser.js +1043 -0
- package/dist/repomap/parser.js.map +1 -0
- package/dist/repomap/regex-parser.d.ts +24 -0
- package/dist/repomap/regex-parser.d.ts.map +1 -0
- package/dist/repomap/regex-parser.js +214 -0
- package/dist/repomap/regex-parser.js.map +1 -0
- package/dist/repomap/renderer.d.ts +40 -0
- package/dist/repomap/renderer.d.ts.map +1 -0
- package/dist/repomap/renderer.js +163 -0
- package/dist/repomap/renderer.js.map +1 -0
- package/dist/repomap/scanner.d.ts +32 -0
- package/dist/repomap/scanner.d.ts.map +1 -0
- package/dist/repomap/scanner.js +190 -0
- package/dist/repomap/scanner.js.map +1 -0
- package/dist/retrieval/chain-assembler.d.ts.map +1 -1
- package/dist/retrieval/chain-assembler.js +22 -3
- package/dist/retrieval/chain-assembler.js.map +1 -1
- package/dist/retrieval/context-assembler.d.ts +1 -1
- package/dist/retrieval/context-assembler.d.ts.map +1 -1
- package/dist/retrieval/index.d.ts +2 -0
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +2 -0
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/mmr.d.ts +1 -0
- package/dist/retrieval/mmr.d.ts.map +1 -1
- package/dist/retrieval/mmr.js +35 -1
- package/dist/retrieval/mmr.js.map +1 -1
- package/dist/retrieval/rrf.d.ts +1 -1
- package/dist/retrieval/rrf.d.ts.map +1 -1
- package/dist/retrieval/rrf.js +1 -1
- package/dist/retrieval/rrf.js.map +1 -1
- package/dist/retrieval/search-assembler.d.ts +11 -2
- package/dist/retrieval/search-assembler.d.ts.map +1 -1
- package/dist/retrieval/search-assembler.js +305 -81
- package/dist/retrieval/search-assembler.js.map +1 -1
- package/dist/retrieval/session-reconstructor.d.ts +36 -0
- package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
- package/dist/retrieval/session-reconstructor.js +128 -0
- package/dist/retrieval/session-reconstructor.js.map +1 -1
- package/dist/storage/db.d.ts.map +1 -1
- package/dist/storage/db.js +24 -0
- package/dist/storage/db.js.map +1 -1
- package/dist/storage/entity-store.d.ts +48 -0
- package/dist/storage/entity-store.d.ts.map +1 -0
- package/dist/storage/entity-store.js +111 -0
- package/dist/storage/entity-store.js.map +1 -0
- package/dist/storage/index-entry-store.d.ts +71 -0
- package/dist/storage/index-entry-store.d.ts.map +1 -0
- package/dist/storage/index-entry-store.js +277 -0
- package/dist/storage/index-entry-store.js.map +1 -0
- package/dist/storage/index.d.ts +5 -2
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +5 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/migrations.d.ts.map +1 -1
- package/dist/storage/migrations.js +147 -0
- package/dist/storage/migrations.js.map +1 -1
- package/dist/storage/schema.sql +104 -2
- package/dist/storage/session-state-store.d.ts +61 -0
- package/dist/storage/session-state-store.d.ts.map +1 -0
- package/dist/storage/session-state-store.js +157 -0
- package/dist/storage/session-state-store.js.map +1 -0
- package/dist/storage/types.d.ts +50 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/vector-store.d.ts +17 -2
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +104 -36
- package/dist/storage/vector-store.js.map +1 -1
- package/package.json +4 -2
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch index entry refinement and backfill.
|
|
3
|
+
*
|
|
4
|
+
* Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
|
|
5
|
+
* keychain API key. Used by the backfill-index maintenance task.
|
|
6
|
+
*/
|
|
7
|
+
/** Result from a backfill run. */
|
|
8
|
+
export interface BackfillResult {
|
|
9
|
+
/** Number of new index entries created */
|
|
10
|
+
entriesCreated: number;
|
|
11
|
+
/** Number of entries created via LLM */
|
|
12
|
+
llmEntries: number;
|
|
13
|
+
/** Number of entries created via Jeopardy-style generation */
|
|
14
|
+
jeopardyEntries: number;
|
|
15
|
+
/** Number of entries created via heuristic */
|
|
16
|
+
heuristicEntries: number;
|
|
17
|
+
/** Number of chunks that had no content to index */
|
|
18
|
+
skipped: number;
|
|
19
|
+
/** Total duration in milliseconds */
|
|
20
|
+
durationMs: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Index entry refresher for batch backfill and refinement.
|
|
24
|
+
*/
|
|
25
|
+
export declare class IndexRefresher {
|
|
26
|
+
private client;
|
|
27
|
+
private rateLimiter;
|
|
28
|
+
private config;
|
|
29
|
+
constructor();
|
|
30
|
+
/**
|
|
31
|
+
* Initialize the Anthropic client. Returns null if no API key available.
|
|
32
|
+
*/
|
|
33
|
+
private getClient;
|
|
34
|
+
/**
|
|
35
|
+
* Backfill index entries for chunks that don't have them.
|
|
36
|
+
*
|
|
37
|
+
* Processes chunks in session-grouped batches. Uses LLM when available,
|
|
38
|
+
* falls back to heuristic. Also embeds the generated descriptions.
|
|
39
|
+
*/
|
|
40
|
+
backfill(options?: {
|
|
41
|
+
limit?: number;
|
|
42
|
+
onProgress?: (current: number, total: number) => void;
|
|
43
|
+
}): Promise<BackfillResult>;
|
|
44
|
+
/**
|
|
45
|
+
* Get backfill status: how many chunks still need index entries.
|
|
46
|
+
*/
|
|
47
|
+
getBackfillStatus(): {
|
|
48
|
+
indexed: number;
|
|
49
|
+
total: number;
|
|
50
|
+
remaining: number;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
export declare const indexRefresher: IndexRefresher;
|
|
54
|
+
//# sourceMappingURL=index-refresher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-refresher.d.ts","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,MAAM,CAAe;;IAM7B;;OAEG;YACW,SAAS;IAuBvB;;;;;OAKG;IACG,QAAQ,CAAC,OAAO,CAAC,EAAE;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACvD,GAAG,OAAO,CAAC,cAAc,CAAC;IAgI3B;;OAEG;IACH,iBAAiB,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAU3E;AAGD,eAAO,MAAM,cAAc,gBAAuB,CAAC"}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch index entry refinement and backfill.
|
|
3
|
+
*
|
|
4
|
+
* Follows the cluster-refresh.ts pattern: Anthropic client, rate limiter,
|
|
5
|
+
* keychain API key. Used by the backfill-index maintenance task.
|
|
6
|
+
*/
|
|
7
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
8
|
+
import { getConfig } from '../config/memory-config.js';
|
|
9
|
+
import { getChunksByIds } from '../storage/chunk-store.js';
|
|
10
|
+
import { getUnindexedChunkIds, insertIndexEntries, getIndexedChunkCount, } from '../storage/index-entry-store.js';
|
|
11
|
+
import { getChunkCount } from '../storage/chunk-store.js';
|
|
12
|
+
import { indexVectorStore } from '../storage/vector-store.js';
|
|
13
|
+
import { Embedder } from '../models/embedder.js';
|
|
14
|
+
import { getModel } from '../models/model-registry.js';
|
|
15
|
+
import { createSecretStore } from '../utils/secret-store.js';
|
|
16
|
+
import { createLogger } from '../utils/logger.js';
|
|
17
|
+
import { generateLLMEntries, generateHeuristicEntry } from './index-generator.js';
|
|
18
|
+
const log = createLogger('index-refresher');
|
|
19
|
+
/** Rate limiter for API calls. */
|
|
20
|
+
class RateLimiter {
|
|
21
|
+
lastCall = 0;
|
|
22
|
+
minIntervalMs;
|
|
23
|
+
constructor(callsPerMinute) {
|
|
24
|
+
this.minIntervalMs = (60 * 1000) / callsPerMinute;
|
|
25
|
+
}
|
|
26
|
+
async wait() {
|
|
27
|
+
const now = Date.now();
|
|
28
|
+
const elapsed = now - this.lastCall;
|
|
29
|
+
if (elapsed < this.minIntervalMs) {
|
|
30
|
+
await new Promise((resolve) => setTimeout(resolve, this.minIntervalMs - elapsed));
|
|
31
|
+
}
|
|
32
|
+
this.lastCall = Date.now();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Index entry refresher for batch backfill and refinement.
|
|
37
|
+
*/
|
|
38
|
+
export class IndexRefresher {
|
|
39
|
+
client = null;
|
|
40
|
+
rateLimiter;
|
|
41
|
+
config = getConfig();
|
|
42
|
+
constructor() {
|
|
43
|
+
this.rateLimiter = new RateLimiter(this.config.refreshRateLimitPerMin);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Initialize the Anthropic client. Returns null if no API key available.
|
|
47
|
+
*/
|
|
48
|
+
async getClient() {
|
|
49
|
+
if (!this.client) {
|
|
50
|
+
if (!process.env.ANTHROPIC_API_KEY) {
|
|
51
|
+
try {
|
|
52
|
+
const store = createSecretStore();
|
|
53
|
+
const storedKey = await store.get('anthropic-api-key');
|
|
54
|
+
if (storedKey) {
|
|
55
|
+
process.env.ANTHROPIC_API_KEY = storedKey;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
// Keychain not available
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (!process.env.ANTHROPIC_API_KEY) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
this.client = new Anthropic();
|
|
66
|
+
}
|
|
67
|
+
return this.client;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Backfill index entries for chunks that don't have them.
|
|
71
|
+
*
|
|
72
|
+
* Processes chunks in session-grouped batches. Uses LLM when available,
|
|
73
|
+
* falls back to heuristic. Also embeds the generated descriptions.
|
|
74
|
+
*/
|
|
75
|
+
async backfill(options) {
|
|
76
|
+
const startTime = Date.now();
|
|
77
|
+
const limit = options?.limit ?? this.config.semanticIndex.batchRefreshLimit;
|
|
78
|
+
// Find chunks without index entries
|
|
79
|
+
const unindexedChunkIds = getUnindexedChunkIds(limit);
|
|
80
|
+
if (unindexedChunkIds.length === 0) {
|
|
81
|
+
return {
|
|
82
|
+
entriesCreated: 0,
|
|
83
|
+
llmEntries: 0,
|
|
84
|
+
jeopardyEntries: 0,
|
|
85
|
+
heuristicEntries: 0,
|
|
86
|
+
skipped: 0,
|
|
87
|
+
durationMs: Date.now() - startTime,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
log.info('Starting index backfill', { chunkCount: unindexedChunkIds.length });
|
|
91
|
+
// Load chunks
|
|
92
|
+
const chunks = getChunksByIds(unindexedChunkIds);
|
|
93
|
+
// Group by session slug for batched LLM calls
|
|
94
|
+
const sessionGroups = new Map();
|
|
95
|
+
for (const chunk of chunks) {
|
|
96
|
+
const group = sessionGroups.get(chunk.sessionSlug) ?? [];
|
|
97
|
+
group.push({
|
|
98
|
+
id: chunk.id,
|
|
99
|
+
sessionSlug: chunk.sessionSlug,
|
|
100
|
+
startTime: chunk.startTime,
|
|
101
|
+
content: chunk.content,
|
|
102
|
+
approxTokens: chunk.approxTokens,
|
|
103
|
+
agentId: chunk.agentId,
|
|
104
|
+
teamName: chunk.teamName,
|
|
105
|
+
});
|
|
106
|
+
sessionGroups.set(chunk.sessionSlug, group);
|
|
107
|
+
}
|
|
108
|
+
let entriesCreated = 0;
|
|
109
|
+
let llmEntries = 0;
|
|
110
|
+
let jeopardyEntries = 0;
|
|
111
|
+
let heuristicEntries = 0;
|
|
112
|
+
let skipped = 0;
|
|
113
|
+
// Set up embedder
|
|
114
|
+
const embedder = new Embedder();
|
|
115
|
+
const embeddingModel = this.config.embeddingModel;
|
|
116
|
+
await embedder.load(getModel(embeddingModel));
|
|
117
|
+
indexVectorStore.setModelId(embeddingModel);
|
|
118
|
+
try {
|
|
119
|
+
const client = await this.getClient();
|
|
120
|
+
let sessionIndex = 0;
|
|
121
|
+
const totalSessions = sessionGroups.size;
|
|
122
|
+
for (const [sessionSlug, sessionChunks] of sessionGroups) {
|
|
123
|
+
sessionIndex++;
|
|
124
|
+
options?.onProgress?.(sessionIndex, totalSessions);
|
|
125
|
+
// Generate entries (rate limiting happens per sub-batch inside generateLLMEntries)
|
|
126
|
+
let entries;
|
|
127
|
+
if (client) {
|
|
128
|
+
entries = await generateLLMEntries(sessionChunks, sessionSlug, {
|
|
129
|
+
onBeforeBatch: () => this.rateLimiter.wait(),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
entries = sessionChunks.map((chunk) => generateHeuristicEntry(chunk, sessionSlug));
|
|
134
|
+
}
|
|
135
|
+
// Filter out empty descriptions
|
|
136
|
+
const validEntries = entries.filter((e) => e.description.trim().length > 0);
|
|
137
|
+
skipped += entries.length - validEntries.length;
|
|
138
|
+
if (validEntries.length === 0)
|
|
139
|
+
continue;
|
|
140
|
+
// Insert entries
|
|
141
|
+
const entryIds = insertIndexEntries(validEntries);
|
|
142
|
+
// Embed descriptions
|
|
143
|
+
const embeddings = [];
|
|
144
|
+
for (let i = 0; i < validEntries.length; i++) {
|
|
145
|
+
const result = await embedder.embed(validEntries[i].description, false);
|
|
146
|
+
embeddings.push({ id: entryIds[i], embedding: result.embedding });
|
|
147
|
+
}
|
|
148
|
+
await indexVectorStore.insertBatch(embeddings);
|
|
149
|
+
// Track stats
|
|
150
|
+
for (const entry of validEntries) {
|
|
151
|
+
if (entry.generationMethod === 'jeopardy') {
|
|
152
|
+
jeopardyEntries++;
|
|
153
|
+
}
|
|
154
|
+
else if (entry.generationMethod === 'llm') {
|
|
155
|
+
llmEntries++;
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
heuristicEntries++;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
entriesCreated += validEntries.length;
|
|
162
|
+
log.debug('Backfilled session', {
|
|
163
|
+
sessionSlug,
|
|
164
|
+
entries: validEntries.length,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
await embedder.dispose();
|
|
170
|
+
}
|
|
171
|
+
log.info('Index backfill complete', {
|
|
172
|
+
entriesCreated,
|
|
173
|
+
llmEntries,
|
|
174
|
+
jeopardyEntries,
|
|
175
|
+
heuristicEntries,
|
|
176
|
+
skipped,
|
|
177
|
+
durationMs: Date.now() - startTime,
|
|
178
|
+
});
|
|
179
|
+
return {
|
|
180
|
+
entriesCreated,
|
|
181
|
+
llmEntries,
|
|
182
|
+
jeopardyEntries,
|
|
183
|
+
heuristicEntries,
|
|
184
|
+
skipped,
|
|
185
|
+
durationMs: Date.now() - startTime,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Get backfill status: how many chunks still need index entries.
|
|
190
|
+
*/
|
|
191
|
+
getBackfillStatus() {
|
|
192
|
+
const indexed = getIndexedChunkCount();
|
|
193
|
+
const total = getChunkCount();
|
|
194
|
+
return {
|
|
195
|
+
indexed,
|
|
196
|
+
total,
|
|
197
|
+
remaining: total - indexed,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
// Singleton
|
|
202
|
+
export const indexRefresher = new IndexRefresher();
|
|
203
|
+
//# sourceMappingURL=index-refresher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-refresher.js","sourceRoot":"","sources":["../../src/index-entries/index-refresher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAGlF,MAAM,GAAG,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC;AAE5C,kCAAkC;AAClC,MAAM,WAAW;IACP,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAAS;IAE9B,YAAY,cAAsB;QAChC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,cAAc,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;CACF;AAkBD;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,GAAqB,IAAI,CAAC;IAChC,WAAW,CAAc;IACzB,MAAM,GAAG,SAAS,EAAE,CAAC;IAE7B;QACE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;oBAClC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;oBACvD,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC;oBAC5C,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBACnC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,OAGd;QACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC;QAE5E,oCAAoC;QACpC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAEtD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO;gBACL,cAAc,EAAE,CAAC;gBACjB,UAAU,EAAE,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,gBAAgB,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,UAAU,EAAE,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9E,cAAc;QACd,MAAM,MAAM,GAAG,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAEjD,8CAA8C;QAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAA8B,CAAC;QAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC;gBACT,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,kBAAkB;QAClB,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAClD,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;QAC9C,gBAAgB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;YAEzC,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,aAAa,EAAE,CAAC;gBACzD,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,UAAU,EAAE,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;gBAEnD,mFAAmF;gBACnF,IAAI,OAAO,CAAC;gBACZ,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE;wBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;qBAC7C,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;gBACrF,CAAC;gBAED,gCAAgC;gBAChC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5E,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;gBAEhD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAExC,iBAAiB;gBACjB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAElD,qBAAqB;gBACrB,MAAM,UAAU,GAA+C,EAAE,CAAC;gBAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAE/C,cAAc;gBACd,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;oBACjC,IAAI,KAAK,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;wBAC1C,eAAe,EAAE,CAAC;oBACpB,CAAC;yBAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;wBAC5C,UAAU,EAAE,CAAC;oBACf,CAAC;yBAAM,CAAC;wBACN,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBACD,cAAc,IAAI,YAAY,CAAC,MAAM,CAAC;gBAEtC,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE;oBAC9B,WAAW;oBACX,OAAO,EAAE,YAAY,CAAC,MAAM;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAClC,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC,CAAC;QAEH,OAAO;YACL,cAAc;YACd,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;QAE9B,OAAO;YACL,OAAO;YACP,KAAK;YACL,SAAS,EAAE,KAAK,GAAG,OAAO;SAC3B,CAAC;IACJ,CAAC;CACF;AAED,YAAY;AACZ,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic index layer exports.
|
|
3
|
+
*/
|
|
4
|
+
export { generateLLMEntries, generateHeuristicEntry, type ChunkForIndexing, type GenerateOptions, } from './index-generator.js';
|
|
5
|
+
export { IndexRefresher, indexRefresher, type BackfillResult } from './index-refresher.js';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index-entries/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,cAAc,EAAE,cAAc,EAAuB,MAAM,sBAAsB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -11,10 +11,14 @@ export * from './storage/encryption.js';
|
|
|
11
11
|
export * from './storage/archive.js';
|
|
12
12
|
export { ingestSession } from './ingest/ingest-session.js';
|
|
13
13
|
export type { IngestResult, IngestOptions } from './ingest/ingest-session.js';
|
|
14
|
+
export { extractSessionState } from './ingest/session-state.js';
|
|
15
|
+
export type { SessionState, SessionError, SessionTask } from './ingest/session-state.js';
|
|
14
16
|
export { batchIngest } from './ingest/batch-ingest.js';
|
|
15
17
|
export type { BatchIngestOptions, BatchIngestResult } from './ingest/batch-ingest.js';
|
|
16
18
|
export * from './retrieval/index.js';
|
|
17
19
|
export * from './clusters/index.js';
|
|
20
|
+
export * from './index-entries/index.js';
|
|
21
|
+
export * from './repomap/index.js';
|
|
18
22
|
export * from './maintenance/index.js';
|
|
19
23
|
export * from './hooks/index.js';
|
|
20
24
|
export { createSecretStore, getApiKey } from './utils/secret-store.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAGrC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACzF,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGtF,cAAc,sBAAsB,CAAC;AAGrC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -13,11 +13,16 @@ export * from './storage/encryption.js';
|
|
|
13
13
|
export * from './storage/archive.js';
|
|
14
14
|
// Ingestion
|
|
15
15
|
export { ingestSession } from './ingest/ingest-session.js';
|
|
16
|
+
export { extractSessionState } from './ingest/session-state.js';
|
|
16
17
|
export { batchIngest } from './ingest/batch-ingest.js';
|
|
17
18
|
// Retrieval
|
|
18
19
|
export * from './retrieval/index.js';
|
|
19
20
|
// Clustering
|
|
20
21
|
export * from './clusters/index.js';
|
|
22
|
+
// Semantic Index
|
|
23
|
+
export * from './index-entries/index.js';
|
|
24
|
+
// Repo Map
|
|
25
|
+
export * from './repomap/index.js';
|
|
21
26
|
// Maintenance
|
|
22
27
|
export * from './maintenance/index.js';
|
|
23
28
|
// Hooks
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,gBAAgB;AAChB,cAAc,mBAAmB,CAAC;AAElC,yCAAyC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,sBAAsB,CAAC;AAErC,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,aAAa;AACb,cAAc,qBAAqB,CAAC;AAEpC,iBAAiB;AACjB,cAAc,0BAA0B,CAAC;AAEzC,WAAW;AACX,cAAc,oBAAoB,CAAC;AAEnC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,QAAQ;AACR,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic entity extraction from chunk content.
|
|
3
|
+
*
|
|
4
|
+
* Pure function — no LLM, no database access. Extracts mentions of
|
|
5
|
+
* people, channels, meetings, organizations, and URLs using regex patterns.
|
|
6
|
+
*
|
|
7
|
+
* Only extracts from [User] and [Assistant] blocks; skips [Thinking] blocks
|
|
8
|
+
* and code blocks (triple-backtick fencing) to reduce noise.
|
|
9
|
+
*/
|
|
10
|
+
export type EntityType = 'person' | 'channel' | 'meeting' | 'organization' | 'url';
|
|
11
|
+
export interface EntityMention {
|
|
12
|
+
entityType: EntityType;
|
|
13
|
+
mentionForm: string;
|
|
14
|
+
normalizedName: string;
|
|
15
|
+
confidence: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Extract entities from chunk text.
|
|
19
|
+
*
|
|
20
|
+
* Returns deduplicated entity mentions with types, normalized names, and confidence scores.
|
|
21
|
+
*/
|
|
22
|
+
export declare function extractEntities(text: string): EntityMention[];
|
|
23
|
+
//# sourceMappingURL=entity-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entity-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,cAAc,GAAG,KAAK,CAAC;AAEnF,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAoJD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CA4F7D"}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic entity extraction from chunk content.
|
|
3
|
+
*
|
|
4
|
+
* Pure function — no LLM, no database access. Extracts mentions of
|
|
5
|
+
* people, channels, meetings, organizations, and URLs using regex patterns.
|
|
6
|
+
*
|
|
7
|
+
* Only extracts from [User] and [Assistant] blocks; skips [Thinking] blocks
|
|
8
|
+
* and code blocks (triple-backtick fencing) to reduce noise.
|
|
9
|
+
*/
|
|
10
|
+
/** Words that look like names in "X said" patterns but aren't. */
|
|
11
|
+
const BLOCKLIST = new Set([
|
|
12
|
+
// Pronouns
|
|
13
|
+
'i',
|
|
14
|
+
'he',
|
|
15
|
+
'she',
|
|
16
|
+
'we',
|
|
17
|
+
'they',
|
|
18
|
+
'you',
|
|
19
|
+
'it',
|
|
20
|
+
'me',
|
|
21
|
+
'him',
|
|
22
|
+
'her',
|
|
23
|
+
'us',
|
|
24
|
+
'them',
|
|
25
|
+
'my',
|
|
26
|
+
'his',
|
|
27
|
+
'our',
|
|
28
|
+
'their',
|
|
29
|
+
'your',
|
|
30
|
+
'its',
|
|
31
|
+
// Articles / determiners
|
|
32
|
+
'the',
|
|
33
|
+
'a',
|
|
34
|
+
'an',
|
|
35
|
+
'this',
|
|
36
|
+
'that',
|
|
37
|
+
'these',
|
|
38
|
+
'those',
|
|
39
|
+
// Common sentence starters / conjunctions
|
|
40
|
+
'but',
|
|
41
|
+
'and',
|
|
42
|
+
'or',
|
|
43
|
+
'so',
|
|
44
|
+
'if',
|
|
45
|
+
'when',
|
|
46
|
+
'then',
|
|
47
|
+
'also',
|
|
48
|
+
'just',
|
|
49
|
+
'now',
|
|
50
|
+
'here',
|
|
51
|
+
'there',
|
|
52
|
+
'what',
|
|
53
|
+
'who',
|
|
54
|
+
'how',
|
|
55
|
+
'why',
|
|
56
|
+
'which',
|
|
57
|
+
'where',
|
|
58
|
+
'some',
|
|
59
|
+
'all',
|
|
60
|
+
'any',
|
|
61
|
+
'each',
|
|
62
|
+
'every',
|
|
63
|
+
'not',
|
|
64
|
+
'no',
|
|
65
|
+
'yes',
|
|
66
|
+
'maybe',
|
|
67
|
+
'perhaps',
|
|
68
|
+
// Common words that appear capitalized at start of sentences
|
|
69
|
+
'however',
|
|
70
|
+
'therefore',
|
|
71
|
+
'meanwhile',
|
|
72
|
+
'otherwise',
|
|
73
|
+
'furthermore',
|
|
74
|
+
'additionally',
|
|
75
|
+
'finally',
|
|
76
|
+
'first',
|
|
77
|
+
'second',
|
|
78
|
+
'third',
|
|
79
|
+
'next',
|
|
80
|
+
'last',
|
|
81
|
+
// Tool/system words often capitalized
|
|
82
|
+
'error',
|
|
83
|
+
'warning',
|
|
84
|
+
'note',
|
|
85
|
+
'todo',
|
|
86
|
+
'fixme',
|
|
87
|
+
'hack',
|
|
88
|
+
'true',
|
|
89
|
+
'false',
|
|
90
|
+
'null',
|
|
91
|
+
'undefined',
|
|
92
|
+
'none',
|
|
93
|
+
]);
|
|
94
|
+
/** Meeting-related keywords. */
|
|
95
|
+
const MEETING_KEYWORDS = /\b(standup|stand-up|retro|retrospective|1:1|one-on-one|sync|daily|weekly|sprint\s+review|sprint\s+planning|kick-?off|all-?hands)\b/gi;
|
|
96
|
+
/** Email regex. */
|
|
97
|
+
const EMAIL_PATTERN = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b/g;
|
|
98
|
+
/** URL regex — matches http/https URLs. */
|
|
99
|
+
const URL_PATTERN = /https?:\/\/[^\s<>"')\]]+/g;
|
|
100
|
+
/** @mention pattern (Slack/Discord style). */
|
|
101
|
+
const AT_MENTION_PATTERN = /@([a-zA-Z][a-zA-Z0-9._-]{0,30})\b/g;
|
|
102
|
+
/** #channel pattern. */
|
|
103
|
+
const CHANNEL_PATTERN = /#([a-zA-Z][a-zA-Z0-9_-]{0,50})\b/g;
|
|
104
|
+
/** "X said" / "with X" / "from X" patterns — capitalized proper nouns. */
|
|
105
|
+
const CONTEXTUAL_NAME_PATTERN = /\b([A-Z][a-z]{1,20}(?:\s+[A-Z][a-z]{1,20})?)\s+(?:said|says|mentioned|asked|replied|suggested|noted|reported|confirmed|explained|wrote|responded)\b/g;
|
|
106
|
+
const WITH_FROM_PATTERN = /\b(?:with|from|by|to|cc|cc'd|cced)\s+([A-Z][a-z]{1,20}(?:\s+[A-Z][a-z]{1,20})?)\b/g;
|
|
107
|
+
/**
|
|
108
|
+
* Strip code blocks (triple-backtick fenced) from text.
|
|
109
|
+
*/
|
|
110
|
+
function stripCodeBlocks(text) {
|
|
111
|
+
return text.replace(/```[\s\S]*?```/g, '');
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Extract only [User] and [Assistant] blocks, skipping [Thinking] blocks.
|
|
115
|
+
* If no block markers found, returns the full text (non-session content).
|
|
116
|
+
*/
|
|
117
|
+
function extractRelevantBlocks(text) {
|
|
118
|
+
// Check if text has block markers
|
|
119
|
+
if (!text.includes('[User]') && !text.includes('[Assistant]')) {
|
|
120
|
+
return text;
|
|
121
|
+
}
|
|
122
|
+
const blocks = [];
|
|
123
|
+
// Split on block headers
|
|
124
|
+
const parts = text.split(/(\[(?:User|Assistant|Thinking)\])/);
|
|
125
|
+
let inRelevantBlock = false;
|
|
126
|
+
for (const part of parts) {
|
|
127
|
+
if (part === '[User]' || part === '[Assistant]') {
|
|
128
|
+
inRelevantBlock = true;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (part === '[Thinking]') {
|
|
132
|
+
inRelevantBlock = false;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (inRelevantBlock) {
|
|
136
|
+
blocks.push(part);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return blocks.join('\n');
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Extract entities from chunk text.
|
|
143
|
+
*
|
|
144
|
+
* Returns deduplicated entity mentions with types, normalized names, and confidence scores.
|
|
145
|
+
*/
|
|
146
|
+
export function extractEntities(text) {
|
|
147
|
+
// Pre-process: skip thinking blocks and code blocks
|
|
148
|
+
const relevant = extractRelevantBlocks(text);
|
|
149
|
+
const cleaned = stripCodeBlocks(relevant);
|
|
150
|
+
const mentions = [];
|
|
151
|
+
const seen = new Set(); // "type:normalized" dedup key
|
|
152
|
+
function addMention(mention) {
|
|
153
|
+
const key = `${mention.entityType}:${mention.normalizedName}`;
|
|
154
|
+
if (seen.has(key))
|
|
155
|
+
return;
|
|
156
|
+
seen.add(key);
|
|
157
|
+
mentions.push(mention);
|
|
158
|
+
}
|
|
159
|
+
// @mentions → person
|
|
160
|
+
for (const match of cleaned.matchAll(AT_MENTION_PATTERN)) {
|
|
161
|
+
const name = match[1];
|
|
162
|
+
addMention({
|
|
163
|
+
entityType: 'person',
|
|
164
|
+
mentionForm: `@${name}`,
|
|
165
|
+
normalizedName: name.toLowerCase(),
|
|
166
|
+
confidence: 0.95,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
// #channels → channel
|
|
170
|
+
for (const match of cleaned.matchAll(CHANNEL_PATTERN)) {
|
|
171
|
+
const name = match[1];
|
|
172
|
+
// Skip common non-channel uses of # (hex colors, markdown headers handled by context)
|
|
173
|
+
if (/^[0-9a-fA-F]{3,8}$/.test(name))
|
|
174
|
+
continue;
|
|
175
|
+
addMention({
|
|
176
|
+
entityType: 'channel',
|
|
177
|
+
mentionForm: `#${name}`,
|
|
178
|
+
normalizedName: name.toLowerCase(),
|
|
179
|
+
confidence: 0.95,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
// Email addresses → person
|
|
183
|
+
for (const match of cleaned.matchAll(EMAIL_PATTERN)) {
|
|
184
|
+
const email = match[0];
|
|
185
|
+
addMention({
|
|
186
|
+
entityType: 'person',
|
|
187
|
+
mentionForm: email,
|
|
188
|
+
normalizedName: email.toLowerCase(),
|
|
189
|
+
confidence: 0.9,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
// URLs → url
|
|
193
|
+
for (const match of cleaned.matchAll(URL_PATTERN)) {
|
|
194
|
+
const url = match[0];
|
|
195
|
+
addMention({
|
|
196
|
+
entityType: 'url',
|
|
197
|
+
mentionForm: url,
|
|
198
|
+
normalizedName: url,
|
|
199
|
+
confidence: 1.0,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
// "X said" / "with X" patterns → person (lower confidence)
|
|
203
|
+
for (const pattern of [CONTEXTUAL_NAME_PATTERN, WITH_FROM_PATTERN]) {
|
|
204
|
+
pattern.lastIndex = 0; // Reset stateful regex
|
|
205
|
+
for (const match of cleaned.matchAll(pattern)) {
|
|
206
|
+
const name = match[1];
|
|
207
|
+
const normalized = name.toLowerCase();
|
|
208
|
+
// Check blocklist for each word in the name
|
|
209
|
+
const words = normalized.split(/\s+/);
|
|
210
|
+
if (words.some((w) => BLOCKLIST.has(w)))
|
|
211
|
+
continue;
|
|
212
|
+
addMention({
|
|
213
|
+
entityType: 'person',
|
|
214
|
+
mentionForm: name,
|
|
215
|
+
normalizedName: normalized,
|
|
216
|
+
confidence: 0.6,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// Meeting keywords → meeting
|
|
221
|
+
MEETING_KEYWORDS.lastIndex = 0;
|
|
222
|
+
for (const match of cleaned.matchAll(MEETING_KEYWORDS)) {
|
|
223
|
+
const keyword = match[1];
|
|
224
|
+
addMention({
|
|
225
|
+
entityType: 'meeting',
|
|
226
|
+
mentionForm: keyword,
|
|
227
|
+
normalizedName: keyword.toLowerCase().replace(/\s+/g, '-'),
|
|
228
|
+
confidence: 0.7,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return mentions;
|
|
232
|
+
}
|
|
233
|
+
//# sourceMappingURL=entity-extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entity-extractor.js","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAWH,kEAAkE;AAClE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,WAAW;IACX,GAAG;IACH,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,MAAM;IACN,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,KAAK;IACL,KAAK;IACL,OAAO;IACP,MAAM;IACN,KAAK;IACL,yBAAyB;IACzB,KAAK;IACL,GAAG;IACH,IAAI;IACJ,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,0CAA0C;IAC1C,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,OAAO;IACP,OAAO;IACP,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,IAAI;IACJ,KAAK;IACL,OAAO;IACP,SAAS;IACT,6DAA6D;IAC7D,SAAS;IACT,WAAW;IACX,WAAW;IACX,WAAW;IACX,aAAa;IACb,cAAc;IACd,SAAS;IACT,OAAO;IACP,QAAQ;IACR,OAAO;IACP,MAAM;IACN,MAAM;IACN,sCAAsC;IACtC,OAAO;IACP,SAAS;IACT,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,WAAW;IACX,MAAM;CACP,CAAC,CAAC;AAEH,gCAAgC;AAChC,MAAM,gBAAgB,GACpB,sIAAsI,CAAC;AAEzI,mBAAmB;AACnB,MAAM,aAAa,GAAG,qDAAqD,CAAC;AAE5E,2CAA2C;AAC3C,MAAM,WAAW,GAAG,2BAA2B,CAAC;AAEhD,8CAA8C;AAC9C,MAAM,kBAAkB,GAAG,oCAAoC,CAAC;AAEhE,wBAAwB;AACxB,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAE5D,0EAA0E;AAC1E,MAAM,uBAAuB,GAC3B,sJAAsJ,CAAC;AACzJ,MAAM,iBAAiB,GACrB,oFAAoF,CAAC;AAEvF;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,IAAY;IACzC,kCAAkC;IAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,yBAAyB;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAE9D,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAChD,eAAe,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,eAAe,GAAG,KAAK,CAAC;YACxB,SAAS;QACX,CAAC;QACD,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,oDAAoD;IACpD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,8BAA8B;IAE9D,SAAS,UAAU,CAAC,OAAsB;QACxC,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,qBAAqB;IACrB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,UAAU,CAAC;YACT,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,sFAAsF;QACtF,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9C,UAAU,CAAC;YACT,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,2BAA2B;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,UAAU,CAAC;YACT,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,KAAK;YAClB,cAAc,EAAE,KAAK,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,aAAa;IACb,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,UAAU,CAAC;YACT,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,GAAG;YAChB,cAAc,EAAE,GAAG;YACnB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,KAAK,MAAM,OAAO,IAAI,CAAC,uBAAuB,EAAE,iBAAiB,CAAC,EAAE,CAAC;QACnE,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,uBAAuB;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,4CAA4C;YAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YAClD,UAAU,CAAC;gBACT,UAAU,EAAE,QAAQ;gBACpB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,UAAU;gBAC1B,UAAU,EAAE,GAAG;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,gBAAgB,CAAC,SAAS,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,UAAU,CAAC;YACT,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,OAAO;YACpB,cAAc,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YAC1D,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hook for generating semantic index entries during ingestion.
|
|
3
|
+
*
|
|
4
|
+
* Called after chunks are stored and embedded. Generates index entries
|
|
5
|
+
* (LLM or heuristic) and embeds the descriptions into the index vector store.
|
|
6
|
+
*/
|
|
7
|
+
import type { ChunkForIndexing } from '../index-entries/index-generator.js';
|
|
8
|
+
/**
|
|
9
|
+
* Generate and store index entries for newly ingested chunks.
|
|
10
|
+
*
|
|
11
|
+
* This is called from processMainSession() after chunks are stored and embedded.
|
|
12
|
+
* It's a best-effort operation — failures are logged but don't block ingestion.
|
|
13
|
+
*/
|
|
14
|
+
export declare function generateIndexEntriesForChunks(chunks: ChunkForIndexing[], sessionSlug: string, _chunkEmbeddings: number[][], _chunkIds: string[], embeddingModel: string): Promise<void>;
|
|
15
|
+
//# sourceMappingURL=index-entry-hook.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-entry-hook.d.ts","sourceRoot":"","sources":["../../src/ingest/index-entry-hook.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAqB5E;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE,gBAAgB,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EAAE,EAAE,EAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAmDf"}
|