@copilotkit/pathfinder 1.1.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/CHANGELOG.md +39 -1
  2. package/README.md +65 -248
  3. package/dist/cli.d.ts +3 -0
  4. package/dist/cli.d.ts.map +1 -0
  5. package/dist/cli.js +70 -0
  6. package/dist/cli.js.map +1 -0
  7. package/dist/config.d.ts +8 -0
  8. package/dist/config.d.ts.map +1 -1
  9. package/dist/config.js +58 -5
  10. package/dist/config.js.map +1 -1
  11. package/dist/db/client.d.ts.map +1 -1
  12. package/dist/db/client.js +3 -1
  13. package/dist/db/client.js.map +1 -1
  14. package/dist/db/queries.d.ts +21 -4
  15. package/dist/db/queries.d.ts.map +1 -1
  16. package/dist/db/queries.js +101 -45
  17. package/dist/db/queries.js.map +1 -1
  18. package/dist/db/schema.d.ts +5 -0
  19. package/dist/db/schema.d.ts.map +1 -1
  20. package/dist/db/schema.js +11 -0
  21. package/dist/db/schema.js.map +1 -1
  22. package/dist/faq-txt.d.ts +12 -0
  23. package/dist/faq-txt.d.ts.map +1 -0
  24. package/dist/faq-txt.js +37 -0
  25. package/dist/faq-txt.js.map +1 -0
  26. package/dist/index.js +2 -362
  27. package/dist/index.js.map +1 -1
  28. package/dist/indexing/chunking/html.d.ts +7 -0
  29. package/dist/indexing/chunking/html.d.ts.map +1 -0
  30. package/dist/indexing/chunking/html.js +356 -0
  31. package/dist/indexing/chunking/html.js.map +1 -0
  32. package/dist/indexing/chunking/index.js +5 -0
  33. package/dist/indexing/chunking/index.js.map +1 -1
  34. package/dist/indexing/chunking/qa.d.ts +8 -0
  35. package/dist/indexing/chunking/qa.d.ts.map +1 -0
  36. package/dist/indexing/chunking/qa.js +22 -0
  37. package/dist/indexing/chunking/qa.js.map +1 -0
  38. package/dist/indexing/distiller.d.ts +29 -0
  39. package/dist/indexing/distiller.d.ts.map +1 -0
  40. package/dist/indexing/distiller.js +104 -0
  41. package/dist/indexing/distiller.js.map +1 -0
  42. package/dist/indexing/orchestrator.d.ts +9 -3
  43. package/dist/indexing/orchestrator.d.ts.map +1 -1
  44. package/dist/indexing/orchestrator.js +113 -83
  45. package/dist/indexing/orchestrator.js.map +1 -1
  46. package/dist/indexing/pipeline.d.ts +18 -0
  47. package/dist/indexing/pipeline.d.ts.map +1 -0
  48. package/dist/indexing/pipeline.js +68 -0
  49. package/dist/indexing/pipeline.js.map +1 -0
  50. package/dist/indexing/providers/discord-api.d.ts +79 -0
  51. package/dist/indexing/providers/discord-api.d.ts.map +1 -0
  52. package/dist/indexing/providers/discord-api.js +167 -0
  53. package/dist/indexing/providers/discord-api.js.map +1 -0
  54. package/dist/indexing/providers/discord.d.ts +25 -0
  55. package/dist/indexing/providers/discord.d.ts.map +1 -0
  56. package/dist/indexing/providers/discord.js +282 -0
  57. package/dist/indexing/providers/discord.js.map +1 -0
  58. package/dist/indexing/providers/file.d.ts +18 -0
  59. package/dist/indexing/providers/file.d.ts.map +1 -0
  60. package/dist/indexing/providers/file.js +262 -0
  61. package/dist/indexing/providers/file.js.map +1 -0
  62. package/dist/indexing/providers/index.d.ts +5 -0
  63. package/dist/indexing/providers/index.d.ts.map +1 -0
  64. package/dist/indexing/providers/index.js +22 -0
  65. package/dist/indexing/providers/index.js.map +1 -0
  66. package/dist/indexing/providers/slack-api.d.ts +62 -0
  67. package/dist/indexing/providers/slack-api.d.ts.map +1 -0
  68. package/dist/indexing/providers/slack-api.js +167 -0
  69. package/dist/indexing/providers/slack-api.js.map +1 -0
  70. package/dist/indexing/providers/slack.d.ts +21 -0
  71. package/dist/indexing/providers/slack.d.ts.map +1 -0
  72. package/dist/indexing/providers/slack.js +192 -0
  73. package/dist/indexing/providers/slack.js.map +1 -0
  74. package/dist/indexing/providers/types.d.ts +56 -0
  75. package/dist/indexing/providers/types.d.ts.map +1 -0
  76. package/dist/indexing/providers/types.js +3 -0
  77. package/dist/indexing/providers/types.js.map +1 -0
  78. package/dist/indexing/url-derivation.d.ts +2 -2
  79. package/dist/indexing/url-derivation.d.ts.map +1 -1
  80. package/dist/indexing/url-derivation.js.map +1 -1
  81. package/dist/indexing/utils.d.ts +19 -0
  82. package/dist/indexing/utils.d.ts.map +1 -0
  83. package/dist/indexing/utils.js +63 -0
  84. package/dist/indexing/utils.js.map +1 -0
  85. package/dist/ip-limiter.d.ts +11 -0
  86. package/dist/ip-limiter.d.ts.map +1 -0
  87. package/dist/ip-limiter.js +40 -0
  88. package/dist/ip-limiter.js.map +1 -0
  89. package/dist/llms-txt.d.ts +11 -0
  90. package/dist/llms-txt.d.ts.map +1 -0
  91. package/dist/llms-txt.js +43 -0
  92. package/dist/llms-txt.js.map +1 -0
  93. package/dist/mcp/server.d.ts +3 -1
  94. package/dist/mcp/server.d.ts.map +1 -1
  95. package/dist/mcp/server.js +9 -1
  96. package/dist/mcp/server.js.map +1 -1
  97. package/dist/mcp/tools/bash-fs.d.ts.map +1 -1
  98. package/dist/mcp/tools/bash-fs.js +4 -1
  99. package/dist/mcp/tools/bash-fs.js.map +1 -1
  100. package/dist/mcp/tools/bash.d.ts +8 -0
  101. package/dist/mcp/tools/bash.d.ts.map +1 -1
  102. package/dist/mcp/tools/bash.js +59 -0
  103. package/dist/mcp/tools/bash.js.map +1 -1
  104. package/dist/mcp/tools/knowledge.d.ts +13 -0
  105. package/dist/mcp/tools/knowledge.d.ts.map +1 -0
  106. package/dist/mcp/tools/knowledge.js +92 -0
  107. package/dist/mcp/tools/knowledge.js.map +1 -0
  108. package/dist/mcp/tools/search.d.ts.map +1 -1
  109. package/dist/mcp/tools/search.js +11 -3
  110. package/dist/mcp/tools/search.js.map +1 -1
  111. package/dist/server.d.ts +6 -0
  112. package/dist/server.d.ts.map +1 -0
  113. package/dist/server.js +595 -0
  114. package/dist/server.js.map +1 -0
  115. package/dist/skill-md.d.ts +3 -0
  116. package/dist/skill-md.d.ts.map +1 -0
  117. package/dist/skill-md.js +75 -0
  118. package/dist/skill-md.js.map +1 -0
  119. package/dist/types.d.ts +844 -38
  120. package/dist/types.d.ts.map +1 -1
  121. package/dist/types.js +88 -6
  122. package/dist/types.js.map +1 -1
  123. package/dist/validate.d.ts +29 -0
  124. package/dist/validate.d.ts.map +1 -0
  125. package/dist/validate.js +192 -0
  126. package/dist/validate.js.map +1 -0
  127. package/dist/webhooks/discord.d.ts +13 -0
  128. package/dist/webhooks/discord.d.ts.map +1 -0
  129. package/dist/webhooks/discord.js +57 -0
  130. package/dist/webhooks/discord.js.map +1 -0
  131. package/dist/webhooks/slack.d.ts +13 -0
  132. package/dist/webhooks/slack.d.ts.map +1 -0
  133. package/dist/webhooks/slack.js +106 -0
  134. package/dist/webhooks/slack.js.map +1 -0
  135. package/dist/workspace.d.ts +13 -0
  136. package/dist/workspace.d.ts.map +1 -0
  137. package/dist/workspace.js +118 -0
  138. package/dist/workspace.js.map +1 -0
  139. package/package.json +27 -2
  140. package/pathfinder-docs.yaml +54 -0
  141. package/pathfinder.example.yaml +48 -0
  142. package/.superpowers/brainstorm/47098-1775507869/content/homepage-mockup.html +0 -324
  143. package/.superpowers/brainstorm/47098-1775507869/state/server-stopped +0 -1
  144. package/.superpowers/brainstorm/47098-1775507869/state/server.log +0 -13
  145. package/.superpowers/brainstorm/47098-1775507869/state/server.pid +0 -1
  146. package/.superpowers/brainstorm/82141-1775511032/content/migration-v2.html +0 -340
  147. package/.superpowers/brainstorm/82141-1775511032/content/migration.html +0 -340
  148. package/.superpowers/brainstorm/82141-1775511032/state/server-stopped +0 -1
  149. package/.superpowers/brainstorm/82141-1775511032/state/server.log +0 -4
  150. package/.superpowers/brainstorm/82141-1775511032/state/server.pid +0 -1
  151. package/dist/indexing/source-indexer.d.ts +0 -68
  152. package/dist/indexing/source-indexer.d.ts.map +0 -1
  153. package/dist/indexing/source-indexer.js +0 -379
  154. package/dist/indexing/source-indexer.js.map +0 -1
@@ -1,15 +1,18 @@
1
1
  // Job queue and coordination for indexing pipelines.
2
2
  // Fully config-driven: indexes sources referenced by search tools in mcp-docs.yaml.
3
- import { simpleGit } from 'simple-git';
3
+ import fs from 'node:fs';
4
+ import path from 'node:path';
4
5
  import { getConfig, getServerConfig, getIndexableSourceNames } from '../config.js';
5
6
  import { EmbeddingClient } from './embeddings.js';
6
- import { SourceIndexer } from './source-indexer.js';
7
+ import { getProvider } from './providers/index.js';
8
+ import { IndexingPipeline } from './pipeline.js';
7
9
  import { getIndexState, upsertIndexState, } from '../db/queries.js';
10
+ import { isFileSourceConfig } from '../types.js';
8
11
  /**
9
12
  * Find all source configs that reference a given repo URL.
10
13
  */
11
14
  function getSourcesByRepo(repoUrl) {
12
- return getServerConfig().sources.filter(s => s.repo === repoUrl);
15
+ return getServerConfig().sources.filter(s => isFileSourceConfig(s) && s.repo === repoUrl);
13
16
  }
14
17
  function getStaleThresholdMs() {
15
18
  const serverCfg = getServerConfig();
@@ -23,6 +26,8 @@ export class IndexingOrchestrator {
23
26
  activeSources = new Set();
24
27
  // Track last nightly reindex date to prevent drift-based duplicate triggers
25
28
  lastReindexDate = null;
29
+ // Callback fired after each reindex job completes with affected source names
30
+ onReindexComplete;
26
31
  constructor() {
27
32
  // No-op — all setup happens lazily via getConfig()
28
33
  }
@@ -63,81 +68,93 @@ export class IndexingOrchestrator {
63
68
  return;
64
69
  }
65
70
  // Queue incremental reindexes for each affected git-backed repo
66
- const reposToReindex = new Set(sourcesNeedingFullReindex.filter(s => s.repo).map(s => s.repo));
71
+ const reposToReindex = new Set();
72
+ for (const s of sourcesNeedingFullReindex) {
73
+ if (isFileSourceConfig(s) && s.repo)
74
+ reposToReindex.add(s.repo);
75
+ }
67
76
  for (const repoUrl of reposToReindex) {
68
77
  this.queueIncrementalReindex(repoUrl);
69
78
  }
70
79
  // Local sources (no repo) get queued as a full reindex of just those sources
71
- const localSources = sourcesNeedingFullReindex.filter(s => !s.repo);
80
+ const localSources = sourcesNeedingFullReindex.filter(s => isFileSourceConfig(s) && !s.repo);
72
81
  if (localSources.length > 0) {
73
82
  this.queue.push({ type: 'full-reindex-local', sources: localSources });
74
83
  this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
75
84
  }
85
+ // Non-file sources (e.g., Slack) that need reindexing
86
+ const nonFileSources = sourcesNeedingFullReindex.filter(s => !isFileSourceConfig(s));
87
+ for (const source of nonFileSources) {
88
+ this.queueSourceReindex(source.name);
89
+ }
76
90
  }
77
91
  if (sourcesOk.length === 0)
78
92
  return;
79
93
  // Local sources in sourcesOk have no remote to check — always reindex on startup
80
- const localSourcesOk = sourcesOk.filter(s => !s.repo);
94
+ const localSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && !s.repo);
81
95
  if (localSourcesOk.length > 0) {
82
96
  console.log(`[orchestrator] Queuing reindex for ${localSourcesOk.length} local source(s)`);
83
97
  this.queue.push({ type: 'full-reindex-local', sources: localSourcesOk });
84
98
  this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
85
99
  }
86
100
  console.log('[orchestrator] Checking remotes for changes on indexed sources...');
87
- // Only check remotes for git-backed sources
88
- const repos = [...new Set(sourcesOk.filter(s => s.repo).map(s => s.repo))];
89
- for (const repoUrl of repos) {
101
+ // Check each git-backed source for changes
102
+ const gitSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && s.repo);
103
+ for (const source of gitSourcesOk) {
90
104
  try {
91
- const remoteHead = await this.getRemoteHead(repoUrl);
92
- const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
93
- let anyChanged = false;
94
- for (const source of sources) {
95
- const state = await getIndexState(source.type, source.name);
96
- if (state?.last_commit_sha !== remoteHead) {
97
- anyChanged = true;
98
- break;
105
+ const currentToken = await this.getSourceStateToken(source);
106
+ const state = await getIndexState(source.type, source.name);
107
+ if (currentToken === null || state?.last_commit_sha !== currentToken) {
108
+ const reason = currentToken === null
109
+ ? 'source unavailable (clone missing?)'
110
+ : `remote ${currentToken.slice(0, 8)} differs from indexed`;
111
+ console.log(`[orchestrator] ${reason} for ${source.name} — queuing reindex`);
112
+ if (isFileSourceConfig(source) && source.repo) {
113
+ this.queueIncrementalReindex(source.repo);
99
114
  }
100
115
  }
101
- if (anyChanged) {
102
- console.log(`[orchestrator] Remote HEAD ${remoteHead.slice(0, 8)} for ${repoUrl} differs from indexed — queuing incremental reindex`);
103
- this.queueIncrementalReindex(repoUrl);
104
- }
105
116
  else {
106
- console.log(`[orchestrator] Index current at ${remoteHead.slice(0, 8)}`);
117
+ console.log(`[orchestrator] ${source.name} index current at ${currentToken.slice(0, 8)}`);
107
118
  }
108
119
  }
109
120
  catch (err) {
110
- // If we can't check remote, fall back to age-based staleness
111
- console.warn(`[orchestrator] Failed to check remote HEAD for ${repoUrl}, falling back to age check:`, err);
112
- const repoSources = getSourcesByRepo(repoUrl);
113
- const firstState = await getIndexState(repoSources[0].type, repoSources[0].name);
114
- if (this.isStale(firstState)) {
115
- const thresholdHours = getServerConfig().indexing?.stale_threshold_hours ?? 24;
116
- console.log(`[orchestrator] Index for ${repoUrl} is stale (>${thresholdHours}h) — queuing full reindex`);
121
+ console.warn(`[orchestrator] Failed to check state for ${source.name}, falling back to age check:`, err);
122
+ const state = await getIndexState(source.type, source.name);
123
+ if (this.isStale(state)) {
124
+ console.log(`[orchestrator] Index for ${source.name} is stale — queuing full reindex`);
117
125
  this.queueFullReindex();
118
126
  }
119
- else {
120
- console.log(`[orchestrator] Index for ${repoUrl} appears fresh, skipping`);
121
- }
127
+ }
128
+ }
129
+ // Ensure git repos are cloned even when index is current.
130
+ // On fresh deploys, the container has no local clones but the DB may have valid state.
131
+ // Bash tools need the clone directories to build their filesystem.
132
+ const cloneDir = getConfig().cloneDir;
133
+ for (const source of gitSourcesOk) {
134
+ if (!isFileSourceConfig(source) || !source.repo)
135
+ continue;
136
+ const repoName = source.repo.replace(/\.git$/, '').split('/').pop();
137
+ const repoDir = path.join(cloneDir, repoName);
138
+ if (!fs.existsSync(repoDir)) {
139
+ console.log(`[orchestrator] Clone directory missing for ${source.name}, queuing reindex to populate`);
140
+ this.queueIncrementalReindex(source.repo);
122
141
  }
123
142
  }
124
143
  }
125
144
  /**
126
- * Get the HEAD SHA of a remote repo without cloning.
127
- * Uses `git ls-remote` which only fetches refs.
145
+ * Get the current state token for a source without acquiring items.
146
+ * Returns null if the source is unavailable.
128
147
  */
129
- async getRemoteHead(repoUrl) {
148
+ async getSourceStateToken(source) {
130
149
  const config = getConfig();
131
- let url = repoUrl;
132
- if (config.githubToken) {
133
- url = repoUrl.replace('https://github.com/', `https://x-access-token:${config.githubToken}@github.com/`);
134
- }
135
- const git = simpleGit();
136
- const result = await git.listRemote([url, 'HEAD']);
137
- const sha = result.split('\t')[0]?.trim();
138
- if (!sha)
139
- throw new Error(`Could not resolve HEAD for ${repoUrl}`);
140
- return sha;
150
+ const providerOptions = {
151
+ cloneDir: config.cloneDir,
152
+ githubToken: config.githubToken,
153
+ slackBotToken: config.slackBotToken,
154
+ discordBotToken: config.discordBotToken,
155
+ };
156
+ const provider = getProvider(source.type)(source, providerOptions);
157
+ return provider.getCurrentStateToken();
141
158
  }
142
159
  /**
143
160
  * Queue a full re-index of all sources. Returns immediately.
@@ -159,6 +176,17 @@ export class IndexingOrchestrator {
159
176
  console.error('[orchestrator] drain() failed:', err);
160
177
  });
161
178
  }
179
+ /**
180
+ * Queue a reindex for a single named source. Returns immediately.
181
+ * Used by webhook handlers to trigger reindexing of specific sources.
182
+ */
183
+ queueSourceReindex(sourceName) {
184
+ this.queue.push({ type: 'source-reindex', sourceName });
185
+ console.log(`[orchestrator] Source re-index queued for ${sourceName}`);
186
+ this.drain().catch((err) => {
187
+ console.error('[orchestrator] drain() failed:', err);
188
+ });
189
+ }
162
190
  /**
163
191
  * Returns true if any indexing job is currently running.
164
192
  */
@@ -240,8 +268,11 @@ export class IndexingOrchestrator {
240
268
  throw new Error('embedding config is required for indexing');
241
269
  }
242
270
  const embeddingClient = new EmbeddingClient(config.openaiApiKey, serverCfg.embedding.model, serverCfg.embedding.dimensions);
271
+ const serverCfg2 = getServerConfig();
272
+ let affectedSourceNames = [];
243
273
  if (job.type === 'full-reindex') {
244
274
  await this.runFullReindex(embeddingClient, config.cloneDir, config.githubToken);
275
+ affectedSourceNames = serverCfg2.sources.map(s => s.name);
245
276
  }
246
277
  else if (job.type === 'full-reindex-local') {
247
278
  if (!job.sources || job.sources.length === 0) {
@@ -251,6 +282,7 @@ export class IndexingOrchestrator {
251
282
  for (const sourceConfig of job.sources) {
252
283
  await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
253
284
  }
285
+ affectedSourceNames = job.sources.map(s => s.name);
254
286
  }
255
287
  else if (job.type === 'incremental-reindex') {
256
288
  if (!job.repoUrl) {
@@ -258,6 +290,28 @@ export class IndexingOrchestrator {
258
290
  return;
259
291
  }
260
292
  await this.runIncrementalReindex(embeddingClient, config.cloneDir, config.githubToken, job.repoUrl);
293
+ affectedSourceNames = getSourcesByRepo(job.repoUrl).map(s => s.name);
294
+ }
295
+ else if (job.type === 'source-reindex') {
296
+ if (!job.sourceName) {
297
+ console.warn('[orchestrator] source-reindex job has no sourceName, skipping');
298
+ return;
299
+ }
300
+ const sourceConfig = serverCfg2.sources.find(s => s.name === job.sourceName);
301
+ if (!sourceConfig) {
302
+ console.warn(`[orchestrator] source-reindex: source "${job.sourceName}" not found in config`);
303
+ return;
304
+ }
305
+ await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
306
+ affectedSourceNames = [job.sourceName];
307
+ }
308
+ if (affectedSourceNames.length > 0 && this.onReindexComplete) {
309
+ try {
310
+ this.onReindexComplete(affectedSourceNames);
311
+ }
312
+ catch (err) {
313
+ console.error('[orchestrator] onReindexComplete callback failed:', err);
314
+ }
261
315
  }
262
316
  }
263
317
  /**
@@ -280,38 +334,7 @@ export class IndexingOrchestrator {
280
334
  const indexableNames = getIndexableSourceNames();
281
335
  const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
282
336
  for (const sourceConfig of sources) {
283
- const state = await getIndexState(sourceConfig.type, sourceConfig.name);
284
- if (state?.last_commit_sha) {
285
- await this.withSourceLock(`${sourceConfig.type}:${sourceConfig.name}`, async () => {
286
- await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
287
- try {
288
- const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
289
- await indexer.incrementalIndex(state.last_commit_sha);
290
- const headSha = await indexer.getHeadSha();
291
- await upsertIndexState({
292
- source_type: sourceConfig.type,
293
- source_key: sourceConfig.name,
294
- last_commit_sha: headSha,
295
- last_indexed_at: new Date(),
296
- status: 'idle',
297
- });
298
- }
299
- catch (err) {
300
- console.error(`[orchestrator] Incremental reindex failed for ${sourceConfig.name}:`, err);
301
- try {
302
- await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'error', err instanceof Error ? err.message : String(err));
303
- }
304
- catch (statusErr) {
305
- console.error('[orchestrator] Failed to update index status:', statusErr);
306
- }
307
- // Don't rethrow — continue with remaining sources
308
- }
309
- });
310
- }
311
- else {
312
- // No previous state — do a full index for this source
313
- await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
314
- }
337
+ await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
315
338
  }
316
339
  console.log(`[orchestrator] Incremental re-index complete for ${repoUrl}`);
317
340
  }
@@ -321,21 +344,29 @@ export class IndexingOrchestrator {
321
344
  async indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken) {
322
345
  const lockKey = `${sourceConfig.type}:${sourceConfig.name}`;
323
346
  await this.withSourceLock(lockKey, async () => {
324
- const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
347
+ const providerOptions = { cloneDir, githubToken, slackBotToken: getConfig().slackBotToken, discordBotToken: getConfig().discordBotToken };
348
+ const provider = getProvider(sourceConfig.type)(sourceConfig, providerOptions);
349
+ const pipeline = new IndexingPipeline(embeddingClient, sourceConfig);
325
350
  await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
326
351
  try {
327
352
  const state = await getIndexState(sourceConfig.type, sourceConfig.name);
353
+ let result;
328
354
  if (state?.last_commit_sha) {
329
- await indexer.incrementalIndex(state.last_commit_sha);
355
+ result = await provider.incrementalAcquire(state.last_commit_sha);
330
356
  }
331
357
  else {
332
- await indexer.fullIndex();
358
+ result = await provider.fullAcquire();
359
+ }
360
+ if (result.removedIds.length > 0) {
361
+ await pipeline.removeItems(result.removedIds);
362
+ }
363
+ if (result.items.length > 0) {
364
+ await pipeline.indexItems(result.items, result.stateToken);
333
365
  }
334
- const headSha = await indexer.getHeadSha();
335
366
  await upsertIndexState({
336
367
  source_type: sourceConfig.type,
337
368
  source_key: sourceConfig.name,
338
- last_commit_sha: headSha,
369
+ last_commit_sha: result.stateToken,
339
370
  last_indexed_at: new Date(),
340
371
  status: 'idle',
341
372
  });
@@ -349,7 +380,6 @@ export class IndexingOrchestrator {
349
380
  catch (statusErr) {
350
381
  console.error('[orchestrator] Failed to update index status:', statusErr);
351
382
  }
352
- // Don't rethrow — continue with remaining sources
353
383
  }
354
384
  });
355
385
  }
@@ -1 +1 @@
1
- {"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAG1B;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AAQD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,CAC1B,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAClE,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,4CAA4C;QAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAElF,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5D,IAAI,KAAK,EAAE,eAAe,KAAK,UAAU,EAAE,CAAC;wBACxC,UAAU,GAAG,IAAI,CAAC;wBAClB,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,IAAI,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,GAAG,CACP,8BAA8B,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,OAAO,qDAAqD,CAC3H,CAAC;oBACF,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,mCAAmC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7E,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,6DAA6D;gBAC7D,OAAO,CAAC,IAAI,CAAC,kDAAkD,OAAO,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBAC3G,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjF,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3B,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC;oBAC/E,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,eAAe,cAAc,2BAA2B,CAAC,CAAC;oBACzG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,0BAA0B,CAAC,CAAC;gBAC/E,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,OAAe;QACvC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,OAAO,CAAC;QAClB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,0BAA0B,MAAM,CAAC,WAAW,cAAc,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CACP,oDAAoD,OAAO,EAAE,CAChE,CAAC;QAEF,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;oBAC9E,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;oBAC5E,IAAI,CAAC;wBACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;wBACxF,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAgB,CAAC,CAAC;wBACvD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;wBAC3C,MAAM,gBAAgB,CAAC;4BACnB,WAAW,EAAE,YAAY,CAAC,IAAI;4BAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;4BAC7B,eAAe,EAAE,OAAO;4BACxB,eAAe,EAAE,IAAI,IAAI,EAAE;4BAC3B,MAAM,EAAE,MAAM;yBACjB,CAAC,CAAC;oBACP,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,iDAAiD,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC1F,IAAI,CAAC;4BACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;wBACN,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;wBAC9E,CAAC;wBACD,kDAAkD;oBACtD,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,sDAAsD;gBACtD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YAC1F,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YACxF,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACJ,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC9B,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,OAAO;oBACxB,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CACT,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAC1D,GAAG,CACN,CAAC;gBACF,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;gBACD,kDAAkD;YACtD,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}
1
+ {"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AAC9F,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AASD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C,6EAA6E;IAC7E,iBAAiB,CAAmC;IAEpD;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,yBAAyB,EAAE,CAAC;gBACxC,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI;oBAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7F,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;YAED,sDAAsD;YACtD,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,2CAA2C;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAE5D,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,EAAE,eAAe,KAAK,YAAY,EAAE,CAAC;oBACnE,MAAM,MAAM,GAAG,YAAY,KAAK,IAAI;wBAChC,CAAC,CAAC,qCAAqC;wBACvC,CAAC,CAAC,UAAU,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,uBAAuB,CAAC;oBAChE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,QAAQ,MAAM,CAAC,IAAI,oBAAoB,CAAC,CAAC;oBAC7E,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5C,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC9C,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,qBAAqB,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9F,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,4CAA4C,MAAM,CAAC,IAAI,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBACzG,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,IAAI,kCAAkC,CAAC,CAAC;oBACvF,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC;QAED,0DAA0D;QAC1D,uFAAuF;QACvF,mEAAmE;QACnE,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,QAAQ,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,SAAS;YAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC;YACrE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC9C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,8CAA8C,MAAM,CAAC,IAAI,+BAA+B,CAAC,CAAC;gBACtG,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,MAAoB;QAClD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAoB;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,eAAe,EAAE,MAAM,CAAC,eAAe;SAC1C,CAAC;QACF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnE,OAAO,QAAQ,CAAC,oBAAoB,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,UAAkB;QACjC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAa,EAAE,CAAC;QAEvC,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAChF,mBAAmB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;YACD,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;YACF,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzE,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;gBAC9E,OAAO;YACX,CAAC;YACD,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7E,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,0CAA0C,GAAG,CAAC,UAAU,uBAAuB,CAAC,CAAC;gBAC9F,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChF,mBAAmB,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,GAAG,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;QAE3E,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,eAAe,GAAoB,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,eAAe,EAAE,CAAC;YAC3J,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YAC/E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAErE,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,MAAM,CAAC;gBACX,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACtE,CAAC;qBAAM,CAAC;oBACJ,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1C,CAAC;gBAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAClD,CAAC;gBACD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC/D,CAAC;gBAED,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,MAAM,CAAC,UAAU;oBAClC,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC/E,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}
@@ -0,0 +1,18 @@
1
+ import { EmbeddingClient } from './embeddings.js';
2
+ import type { SourceConfig } from '../types.js';
3
+ import type { ContentItem } from './providers/types.js';
4
+ export declare class IndexingPipeline {
5
+ private sourceConfig;
6
+ private embeddingClient;
7
+ private logPrefix;
8
+ constructor(embeddingClient: EmbeddingClient, sourceConfig: SourceConfig);
9
+ /**
10
+ * Index a batch of content items: chunk → embed → upsert.
11
+ * Each item's existing chunks are deleted first to handle shrinkage.
12
+ */
13
+ indexItems(items: ContentItem[], stateToken: string): Promise<void>;
14
+ /** Remove items from the index by ID. */
15
+ removeItems(ids: string[]): Promise<void>;
16
+ private indexItem;
17
+ }
18
+ //# sourceMappingURL=pipeline.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,OAAO,KAAK,EAAS,YAAY,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,SAAS,CAAS;gBAEd,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY;IAMxE;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzE,yCAAyC;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAMjC,SAAS;CAmC1B"}
@@ -0,0 +1,68 @@
1
+ // IndexingPipeline — source-agnostic chunk → embed → upsert logic.
2
+ import { getChunker } from './chunking/index.js';
3
+ import { deriveUrl } from './url-derivation.js';
4
+ import { upsertChunks, deleteChunksByFile } from '../db/queries.js';
5
+ import { isFileSourceConfig } from '../types.js';
6
+ export class IndexingPipeline {
7
+ sourceConfig;
8
+ embeddingClient;
9
+ logPrefix;
10
+ constructor(embeddingClient, sourceConfig) {
11
+ this.embeddingClient = embeddingClient;
12
+ this.sourceConfig = sourceConfig;
13
+ this.logPrefix = `[pipeline:${sourceConfig.name}]`;
14
+ }
15
+ /**
16
+ * Index a batch of content items: chunk → embed → upsert.
17
+ * Each item's existing chunks are deleted first to handle shrinkage.
18
+ */
19
+ async indexItems(items, stateToken) {
20
+ for (const item of items) {
21
+ try {
22
+ await this.indexItem(item, stateToken);
23
+ }
24
+ catch (err) {
25
+ const msg = err instanceof Error ? err.message : String(err);
26
+ console.error(`${this.logPrefix} Failed to index ${item.id}: ${msg}`);
27
+ }
28
+ }
29
+ }
30
+ /** Remove items from the index by ID. */
31
+ async removeItems(ids) {
32
+ for (const id of ids) {
33
+ await deleteChunksByFile(this.sourceConfig.name, id);
34
+ }
35
+ }
36
+ async indexItem(item, stateToken) {
37
+ const chunker = getChunker(this.sourceConfig.type);
38
+ const chunkOutputs = chunker(item.content, item.id, this.sourceConfig);
39
+ if (chunkOutputs.length === 0) {
40
+ return;
41
+ }
42
+ const texts = chunkOutputs.map(c => c.content);
43
+ const embeddings = await this.embeddingClient.embedBatch(texts);
44
+ const sourceUrl = item.sourceUrl ?? (isFileSourceConfig(this.sourceConfig) ? deriveUrl(item.id, this.sourceConfig) : null);
45
+ const chunks = chunkOutputs.map((chunk, i) => ({
46
+ source_name: this.sourceConfig.name,
47
+ source_url: sourceUrl,
48
+ title: chunk.title ?? item.title ?? null,
49
+ content: chunk.content,
50
+ embedding: embeddings[i],
51
+ repo_url: isFileSourceConfig(this.sourceConfig) ? (this.sourceConfig.repo ?? null) : null,
52
+ file_path: item.id,
53
+ start_line: chunk.startLine ?? null,
54
+ end_line: chunk.endLine ?? null,
55
+ language: chunk.language ?? null,
56
+ chunk_index: chunk.chunkIndex,
57
+ metadata: {
58
+ ...(chunk.headingPath ? { headingPath: chunk.headingPath } : {}),
59
+ ...(item.metadata ?? {}),
60
+ },
61
+ commit_sha: stateToken,
62
+ version: this.sourceConfig.version ?? null,
63
+ }));
64
+ await deleteChunksByFile(this.sourceConfig.name, item.id);
65
+ await upsertChunks(chunks);
66
+ }
67
+ }
68
+ //# sourceMappingURL=pipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAEnE,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD,MAAM,OAAO,gBAAgB;IACjB,YAAY,CAAe;IAC3B,eAAe,CAAkB;IACjC,SAAS,CAAS;IAE1B,YAAY,eAAgC,EAAE,YAA0B;QACpE,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,aAAa,YAAY,CAAC,IAAI,GAAG,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,UAAkB;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,oBAAoB,IAAI,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;IACL,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,WAAW,CAAC,GAAa;QAC3B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,IAAiB,EAAE,UAAkB;QACzD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAEvE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE3H,MAAM,MAAM,GAAY,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;YACnC,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YACxC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACxB,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACzF,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YACnC,QAAQ,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;YAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;YAChC,WAAW,EAAE,KAAK,CAAC,UAAU;YAC7B,QAAQ,EAAE;gBACN,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;aAC3B;YACD,UAAU,EAAE,UAAU;YACtB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,IAAI,IAAI;SAC7C,CAAC,CAAC,CAAC;QAEJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACJ"}
@@ -0,0 +1,79 @@
1
+ export interface DiscordMessage {
2
+ id: string;
3
+ author: {
4
+ id: string;
5
+ username: string;
6
+ global_name?: string;
7
+ };
8
+ content: string;
9
+ timestamp: string;
10
+ thread?: {
11
+ id: string;
12
+ message_count: number;
13
+ };
14
+ reactions?: Array<{
15
+ emoji: {
16
+ name: string;
17
+ };
18
+ count: number;
19
+ }>;
20
+ }
21
+ export interface DiscordThread {
22
+ id: string;
23
+ name: string;
24
+ parent_id: string;
25
+ message_count: number;
26
+ owner_id: string;
27
+ created_timestamp: string;
28
+ last_message_id: string;
29
+ archived: boolean;
30
+ }
31
+ export interface DiscordUser {
32
+ id: string;
33
+ displayName: string;
34
+ }
35
+ export interface DiscordApiClientOptions {
36
+ token: string;
37
+ /** Max retries on rate-limited requests (default: 5) */
38
+ maxRetries?: number;
39
+ }
40
+ export declare class DiscordApiClient {
41
+ private rest;
42
+ private maxRetries;
43
+ private userCache;
44
+ private logPrefix;
45
+ constructor(options: DiscordApiClientOptions);
46
+ /**
47
+ * Fetch messages from a text channel, paginated using `after` parameter.
48
+ * If `after` is provided, only messages after that snowflake ID are returned.
49
+ */
50
+ fetchChannelMessages(channelId: string, after?: string, limit?: number): Promise<DiscordMessage[]>;
51
+ /**
52
+ * Fetch all threads in a forum channel (active + archived).
53
+ * Active threads: GET /guilds/{guildId}/threads/active, filtered by parent_id.
54
+ * Archived threads: paginated via before parameter.
55
+ */
56
+ fetchForumThreads(channelId: string, guildId: string): Promise<DiscordThread[]>;
57
+ /**
58
+ * Fetch messages within a thread.
59
+ */
60
+ fetchThreadMessages(threadId: string, limit?: number): Promise<DiscordMessage[]>;
61
+ /**
62
+ * Fetch user info with caching.
63
+ */
64
+ fetchUser(userId: string): Promise<DiscordUser>;
65
+ /**
66
+ * Get a message URL (deterministic, no API call).
67
+ */
68
+ getMessageUrl(guildId: string, channelId: string, messageId: string): string;
69
+ /**
70
+ * Fetch archived threads (public or private) with pagination.
71
+ */
72
+ private fetchArchivedThreads;
73
+ /**
74
+ * Call a Discord API method with retry on rate limit (429) responses.
75
+ * Respects retry_after with exponential backoff fallback.
76
+ */
77
+ private callWithRetry;
78
+ }
79
+ //# sourceMappingURL=discord-api.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"discord-api.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/discord-api.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjE;AAED,MAAM,WAAW,aAAa;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,uBAAuB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAaD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAmB;gBAExB,OAAO,EAAE,uBAAuB;IAK5C;;;OAGG;IACG,oBAAoB,CACtB,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC;IAgC5B;;;;OAIG;IACG,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA2BrF;;OAEG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAItF;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAiBrD;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAQ5E;;OAEG;YACW,oBAAoB;IA4ClC;;;OAGG;YACW,aAAa;CA8B9B"}