@copilotkit/pathfinder 1.1.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -1
- package/README.md +65 -248
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +70 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +58 -5
- package/dist/config.js.map +1 -1
- package/dist/db/client.d.ts.map +1 -1
- package/dist/db/client.js +3 -1
- package/dist/db/client.js.map +1 -1
- package/dist/db/queries.d.ts +21 -4
- package/dist/db/queries.d.ts.map +1 -1
- package/dist/db/queries.js +101 -45
- package/dist/db/queries.js.map +1 -1
- package/dist/db/schema.d.ts +5 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +11 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/faq-txt.d.ts +12 -0
- package/dist/faq-txt.d.ts.map +1 -0
- package/dist/faq-txt.js +37 -0
- package/dist/faq-txt.js.map +1 -0
- package/dist/index.js +2 -362
- package/dist/index.js.map +1 -1
- package/dist/indexing/chunking/html.d.ts +7 -0
- package/dist/indexing/chunking/html.d.ts.map +1 -0
- package/dist/indexing/chunking/html.js +356 -0
- package/dist/indexing/chunking/html.js.map +1 -0
- package/dist/indexing/chunking/index.js +5 -0
- package/dist/indexing/chunking/index.js.map +1 -1
- package/dist/indexing/chunking/qa.d.ts +8 -0
- package/dist/indexing/chunking/qa.d.ts.map +1 -0
- package/dist/indexing/chunking/qa.js +22 -0
- package/dist/indexing/chunking/qa.js.map +1 -0
- package/dist/indexing/distiller.d.ts +29 -0
- package/dist/indexing/distiller.d.ts.map +1 -0
- package/dist/indexing/distiller.js +104 -0
- package/dist/indexing/distiller.js.map +1 -0
- package/dist/indexing/orchestrator.d.ts +9 -3
- package/dist/indexing/orchestrator.d.ts.map +1 -1
- package/dist/indexing/orchestrator.js +113 -83
- package/dist/indexing/orchestrator.js.map +1 -1
- package/dist/indexing/pipeline.d.ts +18 -0
- package/dist/indexing/pipeline.d.ts.map +1 -0
- package/dist/indexing/pipeline.js +68 -0
- package/dist/indexing/pipeline.js.map +1 -0
- package/dist/indexing/providers/discord-api.d.ts +79 -0
- package/dist/indexing/providers/discord-api.d.ts.map +1 -0
- package/dist/indexing/providers/discord-api.js +167 -0
- package/dist/indexing/providers/discord-api.js.map +1 -0
- package/dist/indexing/providers/discord.d.ts +25 -0
- package/dist/indexing/providers/discord.d.ts.map +1 -0
- package/dist/indexing/providers/discord.js +282 -0
- package/dist/indexing/providers/discord.js.map +1 -0
- package/dist/indexing/providers/file.d.ts +18 -0
- package/dist/indexing/providers/file.d.ts.map +1 -0
- package/dist/indexing/providers/file.js +262 -0
- package/dist/indexing/providers/file.js.map +1 -0
- package/dist/indexing/providers/index.d.ts +5 -0
- package/dist/indexing/providers/index.d.ts.map +1 -0
- package/dist/indexing/providers/index.js +22 -0
- package/dist/indexing/providers/index.js.map +1 -0
- package/dist/indexing/providers/slack-api.d.ts +62 -0
- package/dist/indexing/providers/slack-api.d.ts.map +1 -0
- package/dist/indexing/providers/slack-api.js +167 -0
- package/dist/indexing/providers/slack-api.js.map +1 -0
- package/dist/indexing/providers/slack.d.ts +21 -0
- package/dist/indexing/providers/slack.d.ts.map +1 -0
- package/dist/indexing/providers/slack.js +192 -0
- package/dist/indexing/providers/slack.js.map +1 -0
- package/dist/indexing/providers/types.d.ts +56 -0
- package/dist/indexing/providers/types.d.ts.map +1 -0
- package/dist/indexing/providers/types.js +3 -0
- package/dist/indexing/providers/types.js.map +1 -0
- package/dist/indexing/url-derivation.d.ts +2 -2
- package/dist/indexing/url-derivation.d.ts.map +1 -1
- package/dist/indexing/url-derivation.js.map +1 -1
- package/dist/indexing/utils.d.ts +19 -0
- package/dist/indexing/utils.d.ts.map +1 -0
- package/dist/indexing/utils.js +63 -0
- package/dist/indexing/utils.js.map +1 -0
- package/dist/ip-limiter.d.ts +11 -0
- package/dist/ip-limiter.d.ts.map +1 -0
- package/dist/ip-limiter.js +40 -0
- package/dist/ip-limiter.js.map +1 -0
- package/dist/llms-txt.d.ts +11 -0
- package/dist/llms-txt.d.ts.map +1 -0
- package/dist/llms-txt.js +43 -0
- package/dist/llms-txt.js.map +1 -0
- package/dist/mcp/server.d.ts +3 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +9 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/tools/bash-fs.d.ts.map +1 -1
- package/dist/mcp/tools/bash-fs.js +4 -1
- package/dist/mcp/tools/bash-fs.js.map +1 -1
- package/dist/mcp/tools/bash.d.ts +8 -0
- package/dist/mcp/tools/bash.d.ts.map +1 -1
- package/dist/mcp/tools/bash.js +59 -0
- package/dist/mcp/tools/bash.js.map +1 -1
- package/dist/mcp/tools/knowledge.d.ts +13 -0
- package/dist/mcp/tools/knowledge.d.ts.map +1 -0
- package/dist/mcp/tools/knowledge.js +92 -0
- package/dist/mcp/tools/knowledge.js.map +1 -0
- package/dist/mcp/tools/search.d.ts.map +1 -1
- package/dist/mcp/tools/search.js +11 -3
- package/dist/mcp/tools/search.js.map +1 -1
- package/dist/server.d.ts +6 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +595 -0
- package/dist/server.js.map +1 -0
- package/dist/skill-md.d.ts +3 -0
- package/dist/skill-md.d.ts.map +1 -0
- package/dist/skill-md.js +75 -0
- package/dist/skill-md.js.map +1 -0
- package/dist/types.d.ts +844 -38
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +88 -6
- package/dist/types.js.map +1 -1
- package/dist/validate.d.ts +29 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +192 -0
- package/dist/validate.js.map +1 -0
- package/dist/webhooks/discord.d.ts +13 -0
- package/dist/webhooks/discord.d.ts.map +1 -0
- package/dist/webhooks/discord.js +57 -0
- package/dist/webhooks/discord.js.map +1 -0
- package/dist/webhooks/slack.d.ts +13 -0
- package/dist/webhooks/slack.d.ts.map +1 -0
- package/dist/webhooks/slack.js +106 -0
- package/dist/webhooks/slack.js.map +1 -0
- package/dist/workspace.d.ts +13 -0
- package/dist/workspace.d.ts.map +1 -0
- package/dist/workspace.js +118 -0
- package/dist/workspace.js.map +1 -0
- package/package.json +27 -2
- package/pathfinder-docs.yaml +54 -0
- package/pathfinder.example.yaml +48 -0
- package/.superpowers/brainstorm/47098-1775507869/content/homepage-mockup.html +0 -324
- package/.superpowers/brainstorm/47098-1775507869/state/server-stopped +0 -1
- package/.superpowers/brainstorm/47098-1775507869/state/server.log +0 -13
- package/.superpowers/brainstorm/47098-1775507869/state/server.pid +0 -1
- package/.superpowers/brainstorm/82141-1775511032/content/migration-v2.html +0 -340
- package/.superpowers/brainstorm/82141-1775511032/content/migration.html +0 -340
- package/.superpowers/brainstorm/82141-1775511032/state/server-stopped +0 -1
- package/.superpowers/brainstorm/82141-1775511032/state/server.log +0 -4
- package/.superpowers/brainstorm/82141-1775511032/state/server.pid +0 -1
- package/dist/indexing/source-indexer.d.ts +0 -68
- package/dist/indexing/source-indexer.d.ts.map +0 -1
- package/dist/indexing/source-indexer.js +0 -379
- package/dist/indexing/source-indexer.js.map +0 -1
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
// Job queue and coordination for indexing pipelines.
|
|
2
2
|
// Fully config-driven: indexes sources referenced by search tools in mcp-docs.yaml.
|
|
3
|
-
import
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
4
5
|
import { getConfig, getServerConfig, getIndexableSourceNames } from '../config.js';
|
|
5
6
|
import { EmbeddingClient } from './embeddings.js';
|
|
6
|
-
import {
|
|
7
|
+
import { getProvider } from './providers/index.js';
|
|
8
|
+
import { IndexingPipeline } from './pipeline.js';
|
|
7
9
|
import { getIndexState, upsertIndexState, } from '../db/queries.js';
|
|
10
|
+
import { isFileSourceConfig } from '../types.js';
|
|
8
11
|
/**
|
|
9
12
|
* Find all source configs that reference a given repo URL.
|
|
10
13
|
*/
|
|
11
14
|
function getSourcesByRepo(repoUrl) {
|
|
12
|
-
return getServerConfig().sources.filter(s => s.repo === repoUrl);
|
|
15
|
+
return getServerConfig().sources.filter(s => isFileSourceConfig(s) && s.repo === repoUrl);
|
|
13
16
|
}
|
|
14
17
|
function getStaleThresholdMs() {
|
|
15
18
|
const serverCfg = getServerConfig();
|
|
@@ -23,6 +26,8 @@ export class IndexingOrchestrator {
|
|
|
23
26
|
activeSources = new Set();
|
|
24
27
|
// Track last nightly reindex date to prevent drift-based duplicate triggers
|
|
25
28
|
lastReindexDate = null;
|
|
29
|
+
// Callback fired after each reindex job completes with affected source names
|
|
30
|
+
onReindexComplete;
|
|
26
31
|
constructor() {
|
|
27
32
|
// No-op — all setup happens lazily via getConfig()
|
|
28
33
|
}
|
|
@@ -63,81 +68,93 @@ export class IndexingOrchestrator {
|
|
|
63
68
|
return;
|
|
64
69
|
}
|
|
65
70
|
// Queue incremental reindexes for each affected git-backed repo
|
|
66
|
-
const reposToReindex = new Set(
|
|
71
|
+
const reposToReindex = new Set();
|
|
72
|
+
for (const s of sourcesNeedingFullReindex) {
|
|
73
|
+
if (isFileSourceConfig(s) && s.repo)
|
|
74
|
+
reposToReindex.add(s.repo);
|
|
75
|
+
}
|
|
67
76
|
for (const repoUrl of reposToReindex) {
|
|
68
77
|
this.queueIncrementalReindex(repoUrl);
|
|
69
78
|
}
|
|
70
79
|
// Local sources (no repo) get queued as a full reindex of just those sources
|
|
71
|
-
const localSources = sourcesNeedingFullReindex.filter(s => !s.repo);
|
|
80
|
+
const localSources = sourcesNeedingFullReindex.filter(s => isFileSourceConfig(s) && !s.repo);
|
|
72
81
|
if (localSources.length > 0) {
|
|
73
82
|
this.queue.push({ type: 'full-reindex-local', sources: localSources });
|
|
74
83
|
this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
|
|
75
84
|
}
|
|
85
|
+
// Non-file sources (e.g., Slack) that need reindexing
|
|
86
|
+
const nonFileSources = sourcesNeedingFullReindex.filter(s => !isFileSourceConfig(s));
|
|
87
|
+
for (const source of nonFileSources) {
|
|
88
|
+
this.queueSourceReindex(source.name);
|
|
89
|
+
}
|
|
76
90
|
}
|
|
77
91
|
if (sourcesOk.length === 0)
|
|
78
92
|
return;
|
|
79
93
|
// Local sources in sourcesOk have no remote to check — always reindex on startup
|
|
80
|
-
const localSourcesOk = sourcesOk.filter(s => !s.repo);
|
|
94
|
+
const localSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && !s.repo);
|
|
81
95
|
if (localSourcesOk.length > 0) {
|
|
82
96
|
console.log(`[orchestrator] Queuing reindex for ${localSourcesOk.length} local source(s)`);
|
|
83
97
|
this.queue.push({ type: 'full-reindex-local', sources: localSourcesOk });
|
|
84
98
|
this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
|
|
85
99
|
}
|
|
86
100
|
console.log('[orchestrator] Checking remotes for changes on indexed sources...');
|
|
87
|
-
//
|
|
88
|
-
const
|
|
89
|
-
for (const
|
|
101
|
+
// Check each git-backed source for changes
|
|
102
|
+
const gitSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && s.repo);
|
|
103
|
+
for (const source of gitSourcesOk) {
|
|
90
104
|
try {
|
|
91
|
-
const
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
105
|
+
const currentToken = await this.getSourceStateToken(source);
|
|
106
|
+
const state = await getIndexState(source.type, source.name);
|
|
107
|
+
if (currentToken === null || state?.last_commit_sha !== currentToken) {
|
|
108
|
+
const reason = currentToken === null
|
|
109
|
+
? 'source unavailable (clone missing?)'
|
|
110
|
+
: `remote ${currentToken.slice(0, 8)} differs from indexed`;
|
|
111
|
+
console.log(`[orchestrator] ${reason} for ${source.name} — queuing reindex`);
|
|
112
|
+
if (isFileSourceConfig(source) && source.repo) {
|
|
113
|
+
this.queueIncrementalReindex(source.repo);
|
|
99
114
|
}
|
|
100
115
|
}
|
|
101
|
-
if (anyChanged) {
|
|
102
|
-
console.log(`[orchestrator] Remote HEAD ${remoteHead.slice(0, 8)} for ${repoUrl} differs from indexed — queuing incremental reindex`);
|
|
103
|
-
this.queueIncrementalReindex(repoUrl);
|
|
104
|
-
}
|
|
105
116
|
else {
|
|
106
|
-
console.log(`[orchestrator]
|
|
117
|
+
console.log(`[orchestrator] ${source.name} index current at ${currentToken.slice(0, 8)}`);
|
|
107
118
|
}
|
|
108
119
|
}
|
|
109
120
|
catch (err) {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if (this.isStale(firstState)) {
|
|
115
|
-
const thresholdHours = getServerConfig().indexing?.stale_threshold_hours ?? 24;
|
|
116
|
-
console.log(`[orchestrator] Index for ${repoUrl} is stale (>${thresholdHours}h) — queuing full reindex`);
|
|
121
|
+
console.warn(`[orchestrator] Failed to check state for ${source.name}, falling back to age check:`, err);
|
|
122
|
+
const state = await getIndexState(source.type, source.name);
|
|
123
|
+
if (this.isStale(state)) {
|
|
124
|
+
console.log(`[orchestrator] Index for ${source.name} is stale — queuing full reindex`);
|
|
117
125
|
this.queueFullReindex();
|
|
118
126
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
// Ensure git repos are cloned even when index is current.
|
|
130
|
+
// On fresh deploys, the container has no local clones but the DB may have valid state.
|
|
131
|
+
// Bash tools need the clone directories to build their filesystem.
|
|
132
|
+
const cloneDir = getConfig().cloneDir;
|
|
133
|
+
for (const source of gitSourcesOk) {
|
|
134
|
+
if (!isFileSourceConfig(source) || !source.repo)
|
|
135
|
+
continue;
|
|
136
|
+
const repoName = source.repo.replace(/\.git$/, '').split('/').pop();
|
|
137
|
+
const repoDir = path.join(cloneDir, repoName);
|
|
138
|
+
if (!fs.existsSync(repoDir)) {
|
|
139
|
+
console.log(`[orchestrator] Clone directory missing for ${source.name}, queuing reindex to populate`);
|
|
140
|
+
this.queueIncrementalReindex(source.repo);
|
|
122
141
|
}
|
|
123
142
|
}
|
|
124
143
|
}
|
|
125
144
|
/**
|
|
126
|
-
* Get the
|
|
127
|
-
*
|
|
145
|
+
* Get the current state token for a source without acquiring items.
|
|
146
|
+
* Returns null if the source is unavailable.
|
|
128
147
|
*/
|
|
129
|
-
async
|
|
148
|
+
async getSourceStateToken(source) {
|
|
130
149
|
const config = getConfig();
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
throw new Error(`Could not resolve HEAD for ${repoUrl}`);
|
|
140
|
-
return sha;
|
|
150
|
+
const providerOptions = {
|
|
151
|
+
cloneDir: config.cloneDir,
|
|
152
|
+
githubToken: config.githubToken,
|
|
153
|
+
slackBotToken: config.slackBotToken,
|
|
154
|
+
discordBotToken: config.discordBotToken,
|
|
155
|
+
};
|
|
156
|
+
const provider = getProvider(source.type)(source, providerOptions);
|
|
157
|
+
return provider.getCurrentStateToken();
|
|
141
158
|
}
|
|
142
159
|
/**
|
|
143
160
|
* Queue a full re-index of all sources. Returns immediately.
|
|
@@ -159,6 +176,17 @@ export class IndexingOrchestrator {
|
|
|
159
176
|
console.error('[orchestrator] drain() failed:', err);
|
|
160
177
|
});
|
|
161
178
|
}
|
|
179
|
+
/**
|
|
180
|
+
* Queue a reindex for a single named source. Returns immediately.
|
|
181
|
+
* Used by webhook handlers to trigger reindexing of specific sources.
|
|
182
|
+
*/
|
|
183
|
+
queueSourceReindex(sourceName) {
|
|
184
|
+
this.queue.push({ type: 'source-reindex', sourceName });
|
|
185
|
+
console.log(`[orchestrator] Source re-index queued for ${sourceName}`);
|
|
186
|
+
this.drain().catch((err) => {
|
|
187
|
+
console.error('[orchestrator] drain() failed:', err);
|
|
188
|
+
});
|
|
189
|
+
}
|
|
162
190
|
/**
|
|
163
191
|
* Returns true if any indexing job is currently running.
|
|
164
192
|
*/
|
|
@@ -240,8 +268,11 @@ export class IndexingOrchestrator {
|
|
|
240
268
|
throw new Error('embedding config is required for indexing');
|
|
241
269
|
}
|
|
242
270
|
const embeddingClient = new EmbeddingClient(config.openaiApiKey, serverCfg.embedding.model, serverCfg.embedding.dimensions);
|
|
271
|
+
const serverCfg2 = getServerConfig();
|
|
272
|
+
let affectedSourceNames = [];
|
|
243
273
|
if (job.type === 'full-reindex') {
|
|
244
274
|
await this.runFullReindex(embeddingClient, config.cloneDir, config.githubToken);
|
|
275
|
+
affectedSourceNames = serverCfg2.sources.map(s => s.name);
|
|
245
276
|
}
|
|
246
277
|
else if (job.type === 'full-reindex-local') {
|
|
247
278
|
if (!job.sources || job.sources.length === 0) {
|
|
@@ -251,6 +282,7 @@ export class IndexingOrchestrator {
|
|
|
251
282
|
for (const sourceConfig of job.sources) {
|
|
252
283
|
await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
|
|
253
284
|
}
|
|
285
|
+
affectedSourceNames = job.sources.map(s => s.name);
|
|
254
286
|
}
|
|
255
287
|
else if (job.type === 'incremental-reindex') {
|
|
256
288
|
if (!job.repoUrl) {
|
|
@@ -258,6 +290,28 @@ export class IndexingOrchestrator {
|
|
|
258
290
|
return;
|
|
259
291
|
}
|
|
260
292
|
await this.runIncrementalReindex(embeddingClient, config.cloneDir, config.githubToken, job.repoUrl);
|
|
293
|
+
affectedSourceNames = getSourcesByRepo(job.repoUrl).map(s => s.name);
|
|
294
|
+
}
|
|
295
|
+
else if (job.type === 'source-reindex') {
|
|
296
|
+
if (!job.sourceName) {
|
|
297
|
+
console.warn('[orchestrator] source-reindex job has no sourceName, skipping');
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const sourceConfig = serverCfg2.sources.find(s => s.name === job.sourceName);
|
|
301
|
+
if (!sourceConfig) {
|
|
302
|
+
console.warn(`[orchestrator] source-reindex: source "${job.sourceName}" not found in config`);
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
|
|
306
|
+
affectedSourceNames = [job.sourceName];
|
|
307
|
+
}
|
|
308
|
+
if (affectedSourceNames.length > 0 && this.onReindexComplete) {
|
|
309
|
+
try {
|
|
310
|
+
this.onReindexComplete(affectedSourceNames);
|
|
311
|
+
}
|
|
312
|
+
catch (err) {
|
|
313
|
+
console.error('[orchestrator] onReindexComplete callback failed:', err);
|
|
314
|
+
}
|
|
261
315
|
}
|
|
262
316
|
}
|
|
263
317
|
/**
|
|
@@ -280,38 +334,7 @@ export class IndexingOrchestrator {
|
|
|
280
334
|
const indexableNames = getIndexableSourceNames();
|
|
281
335
|
const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
|
|
282
336
|
for (const sourceConfig of sources) {
|
|
283
|
-
|
|
284
|
-
if (state?.last_commit_sha) {
|
|
285
|
-
await this.withSourceLock(`${sourceConfig.type}:${sourceConfig.name}`, async () => {
|
|
286
|
-
await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
|
|
287
|
-
try {
|
|
288
|
-
const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
|
|
289
|
-
await indexer.incrementalIndex(state.last_commit_sha);
|
|
290
|
-
const headSha = await indexer.getHeadSha();
|
|
291
|
-
await upsertIndexState({
|
|
292
|
-
source_type: sourceConfig.type,
|
|
293
|
-
source_key: sourceConfig.name,
|
|
294
|
-
last_commit_sha: headSha,
|
|
295
|
-
last_indexed_at: new Date(),
|
|
296
|
-
status: 'idle',
|
|
297
|
-
});
|
|
298
|
-
}
|
|
299
|
-
catch (err) {
|
|
300
|
-
console.error(`[orchestrator] Incremental reindex failed for ${sourceConfig.name}:`, err);
|
|
301
|
-
try {
|
|
302
|
-
await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'error', err instanceof Error ? err.message : String(err));
|
|
303
|
-
}
|
|
304
|
-
catch (statusErr) {
|
|
305
|
-
console.error('[orchestrator] Failed to update index status:', statusErr);
|
|
306
|
-
}
|
|
307
|
-
// Don't rethrow — continue with remaining sources
|
|
308
|
-
}
|
|
309
|
-
});
|
|
310
|
-
}
|
|
311
|
-
else {
|
|
312
|
-
// No previous state — do a full index for this source
|
|
313
|
-
await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
|
|
314
|
-
}
|
|
337
|
+
await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
|
|
315
338
|
}
|
|
316
339
|
console.log(`[orchestrator] Incremental re-index complete for ${repoUrl}`);
|
|
317
340
|
}
|
|
@@ -321,21 +344,29 @@ export class IndexingOrchestrator {
|
|
|
321
344
|
async indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken) {
|
|
322
345
|
const lockKey = `${sourceConfig.type}:${sourceConfig.name}`;
|
|
323
346
|
await this.withSourceLock(lockKey, async () => {
|
|
324
|
-
const
|
|
347
|
+
const providerOptions = { cloneDir, githubToken, slackBotToken: getConfig().slackBotToken, discordBotToken: getConfig().discordBotToken };
|
|
348
|
+
const provider = getProvider(sourceConfig.type)(sourceConfig, providerOptions);
|
|
349
|
+
const pipeline = new IndexingPipeline(embeddingClient, sourceConfig);
|
|
325
350
|
await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
|
|
326
351
|
try {
|
|
327
352
|
const state = await getIndexState(sourceConfig.type, sourceConfig.name);
|
|
353
|
+
let result;
|
|
328
354
|
if (state?.last_commit_sha) {
|
|
329
|
-
await
|
|
355
|
+
result = await provider.incrementalAcquire(state.last_commit_sha);
|
|
330
356
|
}
|
|
331
357
|
else {
|
|
332
|
-
await
|
|
358
|
+
result = await provider.fullAcquire();
|
|
359
|
+
}
|
|
360
|
+
if (result.removedIds.length > 0) {
|
|
361
|
+
await pipeline.removeItems(result.removedIds);
|
|
362
|
+
}
|
|
363
|
+
if (result.items.length > 0) {
|
|
364
|
+
await pipeline.indexItems(result.items, result.stateToken);
|
|
333
365
|
}
|
|
334
|
-
const headSha = await indexer.getHeadSha();
|
|
335
366
|
await upsertIndexState({
|
|
336
367
|
source_type: sourceConfig.type,
|
|
337
368
|
source_key: sourceConfig.name,
|
|
338
|
-
last_commit_sha:
|
|
369
|
+
last_commit_sha: result.stateToken,
|
|
339
370
|
last_indexed_at: new Date(),
|
|
340
371
|
status: 'idle',
|
|
341
372
|
});
|
|
@@ -349,7 +380,6 @@ export class IndexingOrchestrator {
|
|
|
349
380
|
catch (statusErr) {
|
|
350
381
|
console.error('[orchestrator] Failed to update index status:', statusErr);
|
|
351
382
|
}
|
|
352
|
-
// Don't rethrow — continue with remaining sources
|
|
353
383
|
}
|
|
354
384
|
});
|
|
355
385
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAG1B;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AAQD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,CAC1B,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAClE,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,4CAA4C;QAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAElF,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5D,IAAI,KAAK,EAAE,eAAe,KAAK,UAAU,EAAE,CAAC;wBACxC,UAAU,GAAG,IAAI,CAAC;wBAClB,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,IAAI,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,GAAG,CACP,8BAA8B,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,OAAO,qDAAqD,CAC3H,CAAC;oBACF,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,mCAAmC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7E,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,6DAA6D;gBAC7D,OAAO,CAAC,IAAI,CAAC,kDAAkD,OAAO,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBAC3G,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjF,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3B,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC;oBAC/E,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,eAAe,cAAc,2BAA2B,CAAC,CAAC;oBACzG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,0BAA0B,CAAC,CAAC;gBAC/E,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,OAAe;QACvC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,OAAO,CAAC;QAClB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,0BAA0B,MAAM,CAAC,WAAW,cAAc,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CACP,oDAAoD,OAAO,EAAE,CAChE,CAAC;QAEF,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;oBAC9E,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;oBAC5E,IAAI,CAAC;wBACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;wBACxF,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAgB,CAAC,CAAC;wBACvD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;wBAC3C,MAAM,gBAAgB,CAAC;4BACnB,WAAW,EAAE,YAAY,CAAC,IAAI;4BAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;4BAC7B,eAAe,EAAE,OAAO;4BACxB,eAAe,EAAE,IAAI,IAAI,EAAE;4BAC3B,MAAM,EAAE,MAAM;yBACjB,CAAC,CAAC;oBACP,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,iDAAiD,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC1F,IAAI,CAAC;4BACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;wBACN,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;wBAC9E,CAAC;wBACD,kDAAkD;oBACtD,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,sDAAsD;gBACtD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YAC1F,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YACxF,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACJ,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC9B,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,OAAO;oBACxB,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CACT,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAC1D,GAAG,CACN,CAAC;gBACF,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;gBACD,kDAAkD;YACtD,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AAC9F,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AASD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C,6EAA6E;IAC7E,iBAAiB,CAAmC;IAEpD;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,yBAAyB,EAAE,CAAC;gBACxC,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI;oBAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7F,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;YAED,sDAAsD;YACtD,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,2CAA2C;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAE5D,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,EAAE,eAAe,KAAK,YAAY,EAAE,CAAC;oBACnE,MAAM,MAAM,GAAG,YAAY,KAAK,IAAI;wBAChC,CAAC,CAAC,qCAAqC;wBACvC,CAAC,CAAC,UAAU,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,uBAAuB,CAAC;oBAChE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,QAAQ,MAAM,CAAC,IAAI,oBAAoB,CAAC,CAAC;oBAC7E,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5C,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC9C,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,qBAAqB,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9F,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,4CAA4C,MAAM,CAAC,IAAI,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBACzG,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,IAAI,kCAAkC,CAAC,CAAC;oBACvF,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC;QAED,0DAA0D;QAC1D,uFAAuF;QACvF,mEAAmE;QACnE,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,QAAQ,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,SAAS;YAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC;YACrE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC9C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,8CAA8C,MAAM,CAAC,IAAI,+BAA+B,CAAC,CAAC;gBACtG,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,MAAoB;QAClD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAoB;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,eAAe,EAAE,MAAM,CAAC,eAAe;SAC1C,CAAC;QACF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnE,OAAO,QAAQ,CAAC,oBAAoB,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,UAAkB;QACjC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAa,EAAE,CAAC;QAEvC,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAChF,mBAAmB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;YACD,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;YACF,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzE,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;gBAC9E,OAAO;YACX,CAAC;YACD,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7E,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,0CAA0C,GAAG,CAAC,UAAU,uBAAuB,CAAC,CAAC;gBAC9F,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChF,mBAAmB,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,GAAG,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;QAE3E,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,eAAe,GAAoB,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,eAAe,EAAE,CAAC;YAC3J,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YAC/E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAErE,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,MAAM,CAAC;gBACX,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACtE,CAAC;qBAAM,CAAC;oBACJ,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1C,CAAC;gBAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAClD,CAAC;gBACD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC/D,CAAC;gBAED,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,MAAM,CAAC,UAAU;oBAClC,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC/E,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { EmbeddingClient } from './embeddings.js';
|
|
2
|
+
import type { SourceConfig } from '../types.js';
|
|
3
|
+
import type { ContentItem } from './providers/types.js';
|
|
4
|
+
export declare class IndexingPipeline {
|
|
5
|
+
private sourceConfig;
|
|
6
|
+
private embeddingClient;
|
|
7
|
+
private logPrefix;
|
|
8
|
+
constructor(embeddingClient: EmbeddingClient, sourceConfig: SourceConfig);
|
|
9
|
+
/**
|
|
10
|
+
* Index a batch of content items: chunk → embed → upsert.
|
|
11
|
+
* Each item's existing chunks are deleted first to handle shrinkage.
|
|
12
|
+
*/
|
|
13
|
+
indexItems(items: ContentItem[], stateToken: string): Promise<void>;
|
|
14
|
+
/** Remove items from the index by ID. */
|
|
15
|
+
removeItems(ids: string[]): Promise<void>;
|
|
16
|
+
private indexItem;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,OAAO,KAAK,EAAS,YAAY,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,SAAS,CAAS;gBAEd,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY;IAMxE;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzE,yCAAyC;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAMjC,SAAS;CAmC1B"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// IndexingPipeline — source-agnostic chunk → embed → upsert logic.
|
|
2
|
+
import { getChunker } from './chunking/index.js';
|
|
3
|
+
import { deriveUrl } from './url-derivation.js';
|
|
4
|
+
import { upsertChunks, deleteChunksByFile } from '../db/queries.js';
|
|
5
|
+
import { isFileSourceConfig } from '../types.js';
|
|
6
|
+
export class IndexingPipeline {
|
|
7
|
+
sourceConfig;
|
|
8
|
+
embeddingClient;
|
|
9
|
+
logPrefix;
|
|
10
|
+
constructor(embeddingClient, sourceConfig) {
|
|
11
|
+
this.embeddingClient = embeddingClient;
|
|
12
|
+
this.sourceConfig = sourceConfig;
|
|
13
|
+
this.logPrefix = `[pipeline:${sourceConfig.name}]`;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Index a batch of content items: chunk → embed → upsert.
|
|
17
|
+
* Each item's existing chunks are deleted first to handle shrinkage.
|
|
18
|
+
*/
|
|
19
|
+
async indexItems(items, stateToken) {
|
|
20
|
+
for (const item of items) {
|
|
21
|
+
try {
|
|
22
|
+
await this.indexItem(item, stateToken);
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
26
|
+
console.error(`${this.logPrefix} Failed to index ${item.id}: ${msg}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Remove items from the index by ID. */
|
|
31
|
+
async removeItems(ids) {
|
|
32
|
+
for (const id of ids) {
|
|
33
|
+
await deleteChunksByFile(this.sourceConfig.name, id);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async indexItem(item, stateToken) {
|
|
37
|
+
const chunker = getChunker(this.sourceConfig.type);
|
|
38
|
+
const chunkOutputs = chunker(item.content, item.id, this.sourceConfig);
|
|
39
|
+
if (chunkOutputs.length === 0) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const texts = chunkOutputs.map(c => c.content);
|
|
43
|
+
const embeddings = await this.embeddingClient.embedBatch(texts);
|
|
44
|
+
const sourceUrl = item.sourceUrl ?? (isFileSourceConfig(this.sourceConfig) ? deriveUrl(item.id, this.sourceConfig) : null);
|
|
45
|
+
const chunks = chunkOutputs.map((chunk, i) => ({
|
|
46
|
+
source_name: this.sourceConfig.name,
|
|
47
|
+
source_url: sourceUrl,
|
|
48
|
+
title: chunk.title ?? item.title ?? null,
|
|
49
|
+
content: chunk.content,
|
|
50
|
+
embedding: embeddings[i],
|
|
51
|
+
repo_url: isFileSourceConfig(this.sourceConfig) ? (this.sourceConfig.repo ?? null) : null,
|
|
52
|
+
file_path: item.id,
|
|
53
|
+
start_line: chunk.startLine ?? null,
|
|
54
|
+
end_line: chunk.endLine ?? null,
|
|
55
|
+
language: chunk.language ?? null,
|
|
56
|
+
chunk_index: chunk.chunkIndex,
|
|
57
|
+
metadata: {
|
|
58
|
+
...(chunk.headingPath ? { headingPath: chunk.headingPath } : {}),
|
|
59
|
+
...(item.metadata ?? {}),
|
|
60
|
+
},
|
|
61
|
+
commit_sha: stateToken,
|
|
62
|
+
version: this.sourceConfig.version ?? null,
|
|
63
|
+
}));
|
|
64
|
+
await deleteChunksByFile(this.sourceConfig.name, item.id);
|
|
65
|
+
await upsertChunks(chunks);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAEnE,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD,MAAM,OAAO,gBAAgB;IACjB,YAAY,CAAe;IAC3B,eAAe,CAAkB;IACjC,SAAS,CAAS;IAE1B,YAAY,eAAgC,EAAE,YAA0B;QACpE,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,aAAa,YAAY,CAAC,IAAI,GAAG,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,UAAkB;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,oBAAoB,IAAI,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;IACL,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,WAAW,CAAC,GAAa;QAC3B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,IAAiB,EAAE,UAAkB;QACzD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAEvE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE3H,MAAM,MAAM,GAAY,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;YACnC,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YACxC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACxB,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACzF,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YACnC,QAAQ,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;YAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;YAChC,WAAW,EAAE,KAAK,CAAC,UAAU;YAC7B,QAAQ,EAAE;gBACN,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;aAC3B;YACD,UAAU,EAAE,UAAU;YACtB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,IAAI,IAAI;SAC7C,CAAC,CAAC,CAAC;QAEJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACJ"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export interface DiscordMessage {
|
|
2
|
+
id: string;
|
|
3
|
+
author: {
|
|
4
|
+
id: string;
|
|
5
|
+
username: string;
|
|
6
|
+
global_name?: string;
|
|
7
|
+
};
|
|
8
|
+
content: string;
|
|
9
|
+
timestamp: string;
|
|
10
|
+
thread?: {
|
|
11
|
+
id: string;
|
|
12
|
+
message_count: number;
|
|
13
|
+
};
|
|
14
|
+
reactions?: Array<{
|
|
15
|
+
emoji: {
|
|
16
|
+
name: string;
|
|
17
|
+
};
|
|
18
|
+
count: number;
|
|
19
|
+
}>;
|
|
20
|
+
}
|
|
21
|
+
export interface DiscordThread {
|
|
22
|
+
id: string;
|
|
23
|
+
name: string;
|
|
24
|
+
parent_id: string;
|
|
25
|
+
message_count: number;
|
|
26
|
+
owner_id: string;
|
|
27
|
+
created_timestamp: string;
|
|
28
|
+
last_message_id: string;
|
|
29
|
+
archived: boolean;
|
|
30
|
+
}
|
|
31
|
+
export interface DiscordUser {
|
|
32
|
+
id: string;
|
|
33
|
+
displayName: string;
|
|
34
|
+
}
|
|
35
|
+
export interface DiscordApiClientOptions {
|
|
36
|
+
token: string;
|
|
37
|
+
/** Max retries on rate-limited requests (default: 5) */
|
|
38
|
+
maxRetries?: number;
|
|
39
|
+
}
|
|
40
|
+
export declare class DiscordApiClient {
|
|
41
|
+
private rest;
|
|
42
|
+
private maxRetries;
|
|
43
|
+
private userCache;
|
|
44
|
+
private logPrefix;
|
|
45
|
+
constructor(options: DiscordApiClientOptions);
|
|
46
|
+
/**
|
|
47
|
+
* Fetch messages from a text channel, paginated using `after` parameter.
|
|
48
|
+
* If `after` is provided, only messages after that snowflake ID are returned.
|
|
49
|
+
*/
|
|
50
|
+
fetchChannelMessages(channelId: string, after?: string, limit?: number): Promise<DiscordMessage[]>;
|
|
51
|
+
/**
|
|
52
|
+
* Fetch all threads in a forum channel (active + archived).
|
|
53
|
+
* Active threads: GET /guilds/{guildId}/threads/active, filtered by parent_id.
|
|
54
|
+
* Archived threads: paginated via before parameter.
|
|
55
|
+
*/
|
|
56
|
+
fetchForumThreads(channelId: string, guildId: string): Promise<DiscordThread[]>;
|
|
57
|
+
/**
|
|
58
|
+
* Fetch messages within a thread.
|
|
59
|
+
*/
|
|
60
|
+
fetchThreadMessages(threadId: string, limit?: number): Promise<DiscordMessage[]>;
|
|
61
|
+
/**
|
|
62
|
+
* Fetch user info with caching.
|
|
63
|
+
*/
|
|
64
|
+
fetchUser(userId: string): Promise<DiscordUser>;
|
|
65
|
+
/**
|
|
66
|
+
* Get a message URL (deterministic, no API call).
|
|
67
|
+
*/
|
|
68
|
+
getMessageUrl(guildId: string, channelId: string, messageId: string): string;
|
|
69
|
+
/**
|
|
70
|
+
* Fetch archived threads (public or private) with pagination.
|
|
71
|
+
*/
|
|
72
|
+
private fetchArchivedThreads;
|
|
73
|
+
/**
|
|
74
|
+
* Call a Discord API method with retry on rate limit (429) responses.
|
|
75
|
+
* Respects retry_after with exponential backoff fallback.
|
|
76
|
+
*/
|
|
77
|
+
private callWithRetry;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=discord-api.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discord-api.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/discord-api.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjE;AAED,MAAM,WAAW,aAAa;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,uBAAuB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAaD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAmB;gBAExB,OAAO,EAAE,uBAAuB;IAK5C;;;OAGG;IACG,oBAAoB,CACtB,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC;IAgC5B;;;;OAIG;IACG,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA2BrF;;OAEG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAItF;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAiBrD;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAQ5E;;OAEG;YACW,oBAAoB;IA4ClC;;;OAGG;YACW,aAAa;CA8B9B"}
|