@copilotkit/pathfinder 1.1.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -1
- package/README.md +65 -248
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +70 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +58 -5
- package/dist/config.js.map +1 -1
- package/dist/db/client.d.ts.map +1 -1
- package/dist/db/client.js +3 -1
- package/dist/db/client.js.map +1 -1
- package/dist/db/queries.d.ts +21 -4
- package/dist/db/queries.d.ts.map +1 -1
- package/dist/db/queries.js +101 -45
- package/dist/db/queries.js.map +1 -1
- package/dist/db/schema.d.ts +5 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +11 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/faq-txt.d.ts +12 -0
- package/dist/faq-txt.d.ts.map +1 -0
- package/dist/faq-txt.js +37 -0
- package/dist/faq-txt.js.map +1 -0
- package/dist/index.js +2 -362
- package/dist/index.js.map +1 -1
- package/dist/indexing/chunking/html.d.ts +7 -0
- package/dist/indexing/chunking/html.d.ts.map +1 -0
- package/dist/indexing/chunking/html.js +356 -0
- package/dist/indexing/chunking/html.js.map +1 -0
- package/dist/indexing/chunking/index.js +5 -0
- package/dist/indexing/chunking/index.js.map +1 -1
- package/dist/indexing/chunking/qa.d.ts +8 -0
- package/dist/indexing/chunking/qa.d.ts.map +1 -0
- package/dist/indexing/chunking/qa.js +22 -0
- package/dist/indexing/chunking/qa.js.map +1 -0
- package/dist/indexing/distiller.d.ts +29 -0
- package/dist/indexing/distiller.d.ts.map +1 -0
- package/dist/indexing/distiller.js +104 -0
- package/dist/indexing/distiller.js.map +1 -0
- package/dist/indexing/orchestrator.d.ts +9 -3
- package/dist/indexing/orchestrator.d.ts.map +1 -1
- package/dist/indexing/orchestrator.js +113 -83
- package/dist/indexing/orchestrator.js.map +1 -1
- package/dist/indexing/pipeline.d.ts +18 -0
- package/dist/indexing/pipeline.d.ts.map +1 -0
- package/dist/indexing/pipeline.js +68 -0
- package/dist/indexing/pipeline.js.map +1 -0
- package/dist/indexing/providers/discord-api.d.ts +79 -0
- package/dist/indexing/providers/discord-api.d.ts.map +1 -0
- package/dist/indexing/providers/discord-api.js +167 -0
- package/dist/indexing/providers/discord-api.js.map +1 -0
- package/dist/indexing/providers/discord.d.ts +25 -0
- package/dist/indexing/providers/discord.d.ts.map +1 -0
- package/dist/indexing/providers/discord.js +282 -0
- package/dist/indexing/providers/discord.js.map +1 -0
- package/dist/indexing/providers/file.d.ts +18 -0
- package/dist/indexing/providers/file.d.ts.map +1 -0
- package/dist/indexing/providers/file.js +262 -0
- package/dist/indexing/providers/file.js.map +1 -0
- package/dist/indexing/providers/index.d.ts +5 -0
- package/dist/indexing/providers/index.d.ts.map +1 -0
- package/dist/indexing/providers/index.js +22 -0
- package/dist/indexing/providers/index.js.map +1 -0
- package/dist/indexing/providers/slack-api.d.ts +62 -0
- package/dist/indexing/providers/slack-api.d.ts.map +1 -0
- package/dist/indexing/providers/slack-api.js +167 -0
- package/dist/indexing/providers/slack-api.js.map +1 -0
- package/dist/indexing/providers/slack.d.ts +21 -0
- package/dist/indexing/providers/slack.d.ts.map +1 -0
- package/dist/indexing/providers/slack.js +192 -0
- package/dist/indexing/providers/slack.js.map +1 -0
- package/dist/indexing/providers/types.d.ts +56 -0
- package/dist/indexing/providers/types.d.ts.map +1 -0
- package/dist/indexing/providers/types.js +3 -0
- package/dist/indexing/providers/types.js.map +1 -0
- package/dist/indexing/url-derivation.d.ts +2 -2
- package/dist/indexing/url-derivation.d.ts.map +1 -1
- package/dist/indexing/url-derivation.js.map +1 -1
- package/dist/indexing/utils.d.ts +19 -0
- package/dist/indexing/utils.d.ts.map +1 -0
- package/dist/indexing/utils.js +63 -0
- package/dist/indexing/utils.js.map +1 -0
- package/dist/ip-limiter.d.ts +11 -0
- package/dist/ip-limiter.d.ts.map +1 -0
- package/dist/ip-limiter.js +40 -0
- package/dist/ip-limiter.js.map +1 -0
- package/dist/llms-txt.d.ts +11 -0
- package/dist/llms-txt.d.ts.map +1 -0
- package/dist/llms-txt.js +43 -0
- package/dist/llms-txt.js.map +1 -0
- package/dist/mcp/server.d.ts +3 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +9 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/tools/bash-fs.d.ts.map +1 -1
- package/dist/mcp/tools/bash-fs.js +4 -1
- package/dist/mcp/tools/bash-fs.js.map +1 -1
- package/dist/mcp/tools/bash.d.ts +8 -0
- package/dist/mcp/tools/bash.d.ts.map +1 -1
- package/dist/mcp/tools/bash.js +59 -0
- package/dist/mcp/tools/bash.js.map +1 -1
- package/dist/mcp/tools/knowledge.d.ts +13 -0
- package/dist/mcp/tools/knowledge.d.ts.map +1 -0
- package/dist/mcp/tools/knowledge.js +92 -0
- package/dist/mcp/tools/knowledge.js.map +1 -0
- package/dist/mcp/tools/search.d.ts.map +1 -1
- package/dist/mcp/tools/search.js +11 -3
- package/dist/mcp/tools/search.js.map +1 -1
- package/dist/server.d.ts +6 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +595 -0
- package/dist/server.js.map +1 -0
- package/dist/skill-md.d.ts +3 -0
- package/dist/skill-md.d.ts.map +1 -0
- package/dist/skill-md.js +75 -0
- package/dist/skill-md.js.map +1 -0
- package/dist/types.d.ts +844 -38
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +88 -6
- package/dist/types.js.map +1 -1
- package/dist/validate.d.ts +29 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +192 -0
- package/dist/validate.js.map +1 -0
- package/dist/webhooks/discord.d.ts +13 -0
- package/dist/webhooks/discord.d.ts.map +1 -0
- package/dist/webhooks/discord.js +57 -0
- package/dist/webhooks/discord.js.map +1 -0
- package/dist/webhooks/slack.d.ts +13 -0
- package/dist/webhooks/slack.d.ts.map +1 -0
- package/dist/webhooks/slack.js +106 -0
- package/dist/webhooks/slack.js.map +1 -0
- package/dist/workspace.d.ts +13 -0
- package/dist/workspace.d.ts.map +1 -0
- package/dist/workspace.js +118 -0
- package/dist/workspace.js.map +1 -0
- package/package.json +27 -2
- package/pathfinder-docs.yaml +54 -0
- package/pathfinder.example.yaml +48 -0
- package/.superpowers/brainstorm/47098-1775507869/content/homepage-mockup.html +0 -324
- package/.superpowers/brainstorm/47098-1775507869/state/server-stopped +0 -1
- package/.superpowers/brainstorm/47098-1775507869/state/server.log +0 -13
- package/.superpowers/brainstorm/47098-1775507869/state/server.pid +0 -1
- package/.superpowers/brainstorm/82141-1775511032/content/migration-v2.html +0 -340
- package/.superpowers/brainstorm/82141-1775511032/content/migration.html +0 -340
- package/.superpowers/brainstorm/82141-1775511032/state/server-stopped +0 -1
- package/.superpowers/brainstorm/82141-1775511032/state/server.log +0 -4
- package/.superpowers/brainstorm/82141-1775511032/state/server.pid +0 -1
- package/dist/indexing/source-indexer.d.ts +0 -68
- package/dist/indexing/source-indexer.d.ts.map +0 -1
- package/dist/indexing/source-indexer.js +0 -379
- package/dist/indexing/source-indexer.js.map +0 -1
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// SlackDataProvider — Slack thread acquisition via Web API + LLM distillation.
|
|
2
|
+
// Implements DataProvider: fetches threads from configured channels, distills
|
|
3
|
+
// them into Q&A pairs, and returns ContentItems for the indexing pipeline.
|
|
4
|
+
import OpenAI from 'openai';
|
|
5
|
+
import { SlackApiClient } from './slack-api.js';
|
|
6
|
+
import { distillThread } from '../distiller.js';
|
|
7
|
+
import { getConfig } from '../../config.js';
|
|
8
|
+
export class SlackDataProvider {
|
|
9
|
+
config;
|
|
10
|
+
apiClient;
|
|
11
|
+
openaiClient;
|
|
12
|
+
logPrefix;
|
|
13
|
+
constructor(config, options) {
|
|
14
|
+
if (config.type !== 'slack') {
|
|
15
|
+
throw new Error('SlackDataProvider requires a slack source config');
|
|
16
|
+
}
|
|
17
|
+
this.config = config;
|
|
18
|
+
const token = options.slackBotToken;
|
|
19
|
+
if (!token) {
|
|
20
|
+
throw new Error('SlackDataProvider requires a slackBotToken in provider options');
|
|
21
|
+
}
|
|
22
|
+
this.apiClient = new SlackApiClient({ token });
|
|
23
|
+
this.openaiClient = new OpenAI({ apiKey: getConfig().openaiApiKey });
|
|
24
|
+
this.logPrefix = `[slack-provider:${config.name}]`;
|
|
25
|
+
}
|
|
26
|
+
async fullAcquire() {
|
|
27
|
+
console.log(`${this.logPrefix} Starting full acquire for ${this.config.channels.length} channel(s)`);
|
|
28
|
+
const allItems = [];
|
|
29
|
+
let maxTimestamp = '0';
|
|
30
|
+
let failedChannels = 0;
|
|
31
|
+
for (const channelId of this.config.channels) {
|
|
32
|
+
try {
|
|
33
|
+
const { items, latestTs } = await this.processChannel(channelId);
|
|
34
|
+
allItems.push(...items);
|
|
35
|
+
if (latestTs > maxTimestamp)
|
|
36
|
+
maxTimestamp = latestTs;
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
failedChannels++;
|
|
40
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
41
|
+
console.error(`${this.logPrefix} Failed to process channel ${channelId}: ${msg}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (failedChannels === this.config.channels.length) {
|
|
45
|
+
throw new Error(`All ${failedChannels} channel(s) failed during acquire`);
|
|
46
|
+
}
|
|
47
|
+
console.log(`${this.logPrefix} Full acquire complete: ${allItems.length} Q&A pairs from ${this.config.channels.length} channel(s)`);
|
|
48
|
+
return {
|
|
49
|
+
items: allItems,
|
|
50
|
+
removedIds: [],
|
|
51
|
+
stateToken: maxTimestamp,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async incrementalAcquire(lastStateToken) {
|
|
55
|
+
console.log(`${this.logPrefix} Starting incremental acquire since ${lastStateToken}`);
|
|
56
|
+
const allItems = [];
|
|
57
|
+
let maxTimestamp = lastStateToken;
|
|
58
|
+
let failedChannels = 0;
|
|
59
|
+
for (const channelId of this.config.channels) {
|
|
60
|
+
try {
|
|
61
|
+
const { items, latestTs } = await this.processChannel(channelId, lastStateToken);
|
|
62
|
+
allItems.push(...items);
|
|
63
|
+
if (latestTs > maxTimestamp)
|
|
64
|
+
maxTimestamp = latestTs;
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
failedChannels++;
|
|
68
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
69
|
+
console.error(`${this.logPrefix} Failed to process channel ${channelId}: ${msg}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (failedChannels === this.config.channels.length) {
|
|
73
|
+
throw new Error(`All ${failedChannels} channel(s) failed during acquire`);
|
|
74
|
+
}
|
|
75
|
+
console.log(`${this.logPrefix} Incremental acquire complete: ${allItems.length} Q&A pairs`);
|
|
76
|
+
return {
|
|
77
|
+
items: allItems,
|
|
78
|
+
removedIds: [], // Slack API doesn't surface deletions; caught on next full acquire
|
|
79
|
+
stateToken: maxTimestamp,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
async getCurrentStateToken() {
|
|
83
|
+
let maxTimestamp = '0';
|
|
84
|
+
for (const channelId of this.config.channels) {
|
|
85
|
+
try {
|
|
86
|
+
const messages = await this.apiClient.fetchChannelHistory(channelId, undefined, 1);
|
|
87
|
+
if (messages.length > 0) {
|
|
88
|
+
// Messages are returned newest-first by Slack
|
|
89
|
+
const latest = messages[0].ts;
|
|
90
|
+
if (latest > maxTimestamp)
|
|
91
|
+
maxTimestamp = latest;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
96
|
+
console.warn(`${this.logPrefix} Failed to check channel ${channelId}: ${msg}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return maxTimestamp === '0' ? null : maxTimestamp;
|
|
100
|
+
}
|
|
101
|
+
// -----------------------------------------------------------------------
|
|
102
|
+
// Private helpers
|
|
103
|
+
// -----------------------------------------------------------------------
|
|
104
|
+
/**
|
|
105
|
+
* Process a single channel: fetch threads, distill, return items.
|
|
106
|
+
*/
|
|
107
|
+
async processChannel(channelId, oldest) {
|
|
108
|
+
const messages = await this.apiClient.fetchChannelHistory(channelId, oldest);
|
|
109
|
+
const items = [];
|
|
110
|
+
let latestTs = oldest ?? '0';
|
|
111
|
+
// Filter to threaded messages with sufficient replies
|
|
112
|
+
const threads = messages.filter(m => m.thread_ts === m.ts && (m.reply_count ?? 0) >= this.config.min_thread_replies);
|
|
113
|
+
console.log(`${this.logPrefix} Channel ${channelId}: ${messages.length} messages, ${threads.length} qualifying threads`);
|
|
114
|
+
// Process threads sequentially to avoid OpenAI rate limits
|
|
115
|
+
for (const thread of threads) {
|
|
116
|
+
try {
|
|
117
|
+
const threadItems = await this.processThread(channelId, thread);
|
|
118
|
+
items.push(...threadItems);
|
|
119
|
+
if (thread.ts > latestTs)
|
|
120
|
+
latestTs = thread.ts;
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
124
|
+
console.error(`${this.logPrefix} Failed to distill thread ${thread.ts}: ${msg}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return { items, latestTs };
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Process a single thread: fetch replies, resolve users, distill, create items.
|
|
131
|
+
*/
|
|
132
|
+
async processThread(channelId, parentMessage) {
|
|
133
|
+
const replies = await this.apiClient.fetchThreadReplies(channelId, parentMessage.ts);
|
|
134
|
+
// Resolve user display names
|
|
135
|
+
const threadMessages = [];
|
|
136
|
+
for (const reply of replies) {
|
|
137
|
+
let author = reply.user ?? 'unknown';
|
|
138
|
+
try {
|
|
139
|
+
if (reply.user) {
|
|
140
|
+
const user = await this.apiClient.fetchUserInfo(reply.user);
|
|
141
|
+
author = user.displayName;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
146
|
+
console.warn(`${this.logPrefix} Failed to resolve user ${reply.user}: ${msg}`);
|
|
147
|
+
}
|
|
148
|
+
threadMessages.push({
|
|
149
|
+
author,
|
|
150
|
+
content: reply.text ?? '',
|
|
151
|
+
timestamp: reply.ts,
|
|
152
|
+
reactions: reply.reactions,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
// Distill the thread
|
|
156
|
+
const distillerResult = await distillThread(threadMessages, {
|
|
157
|
+
model: this.config.distiller_model,
|
|
158
|
+
client: this.openaiClient,
|
|
159
|
+
});
|
|
160
|
+
// Create ContentItems for ALL pairs — confidence filtering happens at query time
|
|
161
|
+
const items = [];
|
|
162
|
+
let permalink;
|
|
163
|
+
if (distillerResult.pairs.length > 0) {
|
|
164
|
+
try {
|
|
165
|
+
permalink = await this.apiClient.getChannelPermalink(channelId, parentMessage.ts);
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
169
|
+
console.warn(`${this.logPrefix} Failed to get permalink for ${channelId}/${parentMessage.ts}: ${msg}`);
|
|
170
|
+
permalink = undefined;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const participants = [...new Set(threadMessages.map(m => m.author))];
|
|
174
|
+
for (let i = 0; i < distillerResult.pairs.length; i++) {
|
|
175
|
+
const pair = distillerResult.pairs[i];
|
|
176
|
+
items.push({
|
|
177
|
+
id: `${channelId}:${parentMessage.ts}:${i}`,
|
|
178
|
+
content: `Q: ${pair.question}\n\nA: ${pair.answer}`,
|
|
179
|
+
title: pair.question,
|
|
180
|
+
sourceUrl: permalink,
|
|
181
|
+
metadata: {
|
|
182
|
+
channel: channelId,
|
|
183
|
+
participants,
|
|
184
|
+
confidence: pair.confidence,
|
|
185
|
+
emojiTriggered: false,
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
return items;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
//# sourceMappingURL=slack.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slack.js","sourceRoot":"","sources":["../../../src/indexing/providers/slack.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,8EAA8E;AAC9E,2EAA2E;AAE3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAqB,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,MAAM,OAAO,iBAAiB;IAClB,MAAM,CAAoB;IAC1B,SAAS,CAAiB;IAC1B,YAAY,CAAS;IACrB,SAAS,CAAS;IAE1B,YAAY,MAAoB,EAAE,OAAwB;QACtD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,cAAc,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,YAAY,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,SAAS,GAAG,mBAAmB,MAAM,CAAC,IAAI,GAAG,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,WAAW;QACb,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QACrG,MAAM,QAAQ,GAAkB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,IAAI,QAAQ,GAAG,YAAY;oBAAE,YAAY,GAAG,QAAQ,CAAC;YACzD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,cAAc,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,SAAS,KAAK,GAAG,EAAE,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,IAAI,cAAc,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,OAAO,cAAc,mCAAmC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,QAAQ,CAAC,MAAM,mBAAmB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QAEpI,OAAO;YACH,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,YAAY;SAC3B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,cAAsB;QAC3C,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,uCAAuC,cAAc,EAAE,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAkB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,cAAc,CAAC;QAClC,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;gBACjF,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,IAAI,QAAQ,GAAG,YAAY;oBAAE,YAAY,GAAG,QAAQ,CAAC;YACzD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,cAAc,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,SAAS,KAAK,GAAG,EAAE,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,IAAI,cAAc,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,OAAO,cAAc,mCAAmC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,kCAAkC,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;QAE5F,OAAO;YACH,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,EAAE,EAAE,mEAAmE;YACnF,UAAU,EAAE,YAAY;SAC3B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,oBAAoB;QACtB,IAAI,YAAY,GAAG,GAAG,CAAC;QAEvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;gBACnF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,8CAA8C;oBAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9B,IAAI,MAAM,GAAG,YAAY;wBAAE,YAAY,GAAG,MAAM,CAAC;gBACrD,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,4BAA4B,SAAS,KAAK,GAAG,EAAE,CAAC,CAAC;YACnF,CAAC;QACL,CAAC;QAED,OAAO,YAAY,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,MAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC;QAE7B,sDAAsD;QACtD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAC3B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,CACtF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,YAAY,SAAS,KAAK,QAAQ,CAAC,MAAM,cAAc,OAAO,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEzH,2DAA2D;QAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;gBAE3B,IAAI,MAAM,CAAC,EAAE,GAAG,QAAQ;oBAAE,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;YACnD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,6BAA6B,MAAM,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACvB,SAAiB,EACjB,aAA2B;QAE3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC;QAErF,6BAA6B;QAC7B,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC;YACrC,IAAI,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC5D,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;gBAC9B,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,KAAK,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,cAAc,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;gBACzB,SAAS,EAAE,KAAK,CAAC,EAAE;gBACnB,SAAS,EAAE,KAAK,CAAC,SAAS;aAC7B,CAAC,CAAC;QACP,CAAC;QAED,qBAAqB;QACrB,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE;YACxD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;YAClC,MAAM,EAAE,IAAI,CAAC,YAAY;SAC5B,CAAC,CAAC;QAEH,iFAAiF;QACjF,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,SAA6B,CAAC;QAElC,IAAI,eAAe,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC;gBACD,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,gCAAgC,SAAS,IAAI,aAAa,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;gBACvG,SAAS,GAAG,SAAS,CAAC;YAC1B,CAAC;QACL,CAAC;QAED,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtC,KAAK,CAAC,IAAI,CAAC;gBACP,EAAE,EAAE,GAAG,SAAS,IAAI,aAAa,CAAC,EAAE,IAAI,CAAC,EAAE;gBAC3C,OAAO,EAAE,MAAM,IAAI,CAAC,QAAQ,UAAU,IAAI,CAAC,MAAM,EAAE;gBACnD,KAAK,EAAE,IAAI,CAAC,QAAQ;gBACpB,SAAS,EAAE,SAAS;gBACpB,QAAQ,EAAE;oBACN,OAAO,EAAE,SAAS;oBAClB,YAAY;oBACZ,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,cAAc,EAAE,KAAK;iBACxB;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { SourceConfig } from '../../types.js';
|
|
2
|
+
/** A single content item to be indexed. */
|
|
3
|
+
export interface ContentItem {
|
|
4
|
+
/** Unique identifier within the source (file path, thread ID, page ID, etc.) */
|
|
5
|
+
id: string;
|
|
6
|
+
/** Raw content to be chunked */
|
|
7
|
+
content: string;
|
|
8
|
+
/** Human-readable title (optional — chunker may derive one) */
|
|
9
|
+
title?: string;
|
|
10
|
+
/** URL to the original content (optional) */
|
|
11
|
+
sourceUrl?: string;
|
|
12
|
+
/** Additional metadata passed through to chunk records */
|
|
13
|
+
metadata?: Record<string, unknown>;
|
|
14
|
+
}
|
|
15
|
+
/** Result of a data acquisition run. */
|
|
16
|
+
export interface AcquisitionResult {
|
|
17
|
+
/** Items to index (full list for full acquire, changed items for incremental) */
|
|
18
|
+
items: ContentItem[];
|
|
19
|
+
/**
|
|
20
|
+
* Item IDs to remove from the index.
|
|
21
|
+
* Full acquire: always empty — deleted-file detection is not performed during
|
|
22
|
+
* full acquire, so chunks from files no longer in the source persist until the
|
|
23
|
+
* next incremental acquire or manual cleanup.
|
|
24
|
+
* Incremental acquire: IDs of items deleted since lastStateToken.
|
|
25
|
+
*/
|
|
26
|
+
removedIds: string[];
|
|
27
|
+
/** Opaque state token to persist (commit SHA, API cursor, timestamp) */
|
|
28
|
+
stateToken: string;
|
|
29
|
+
}
|
|
30
|
+
/** Interface that all data providers implement. */
|
|
31
|
+
export interface DataProvider {
|
|
32
|
+
/**
|
|
33
|
+
* Full acquisition — return all indexable items.
|
|
34
|
+
* Providers must apply their own content filtering before returning items.
|
|
35
|
+
*/
|
|
36
|
+
fullAcquire(): Promise<AcquisitionResult>;
|
|
37
|
+
/**
|
|
38
|
+
* Incremental acquisition — only items changed since lastStateToken.
|
|
39
|
+
*/
|
|
40
|
+
incrementalAcquire(lastStateToken: string): Promise<AcquisitionResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Get the current state token without acquiring items (for staleness checks).
|
|
43
|
+
* Returns null if the source is unavailable (e.g., clone dir missing).
|
|
44
|
+
*/
|
|
45
|
+
getCurrentStateToken(): Promise<string | null>;
|
|
46
|
+
}
|
|
47
|
+
/** Options passed to provider factories. */
|
|
48
|
+
export interface ProviderOptions {
|
|
49
|
+
cloneDir: string;
|
|
50
|
+
githubToken?: string;
|
|
51
|
+
slackBotToken?: string;
|
|
52
|
+
discordBotToken?: string;
|
|
53
|
+
}
|
|
54
|
+
/** Factory function that creates a DataProvider for a given source config. */
|
|
55
|
+
export type DataProviderFactory = (config: SourceConfig, options: ProviderOptions) => DataProvider;
|
|
56
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,2CAA2C;AAC3C,MAAM,WAAW,WAAW;IACxB,gFAAgF;IAChF,EAAE,EAAE,MAAM,CAAC;IACX,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,iFAAiF;IACjF,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB;;;;;;OAMG;IACH,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,wEAAwE;IACxE,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,mDAAmD;AACnD,MAAM,WAAW,YAAY;IACzB;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE1C;;OAEG;IACH,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAEvE;;;OAGG;IACH,oBAAoB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAClD;AAED,4CAA4C;AAC5C,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,8EAA8E;AAC9E,MAAM,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,YAAY,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/indexing/providers/types.ts"],"names":[],"mappings":"AAAA,iFAAiF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { FileSourceConfig } from '../types.js';
|
|
2
2
|
/**
|
|
3
3
|
* Derive a public URL from a relative file path using the source's URL derivation rules.
|
|
4
4
|
* Returns null if the source has no base_url or url_derivation configured.
|
|
5
5
|
*/
|
|
6
|
-
export declare function deriveUrl(filePath: string, sourceConfig:
|
|
6
|
+
export declare function deriveUrl(filePath: string, sourceConfig: FileSourceConfig): string | null;
|
|
7
7
|
//# sourceMappingURL=url-derivation.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-derivation.d.ts","sourceRoot":"","sources":["../../src/indexing/url-derivation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"url-derivation.d.ts","sourceRoot":"","sources":["../../src/indexing/url-derivation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAsBzF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-derivation.js","sourceRoot":"","sources":["../../src/indexing/url-derivation.ts"],"names":[],"mappings":"AAAA,sEAAsE;AAItE;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"url-derivation.js","sourceRoot":"","sources":["../../src/indexing/url-derivation.ts"],"names":[],"mappings":"AAAA,sEAAsE;AAItE;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,YAA8B;IACtE,IAAI,CAAC,YAAY,CAAC,QAAQ,IAAI,CAAC,YAAY,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC;IAExE,MAAM,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC;IACtC,IAAI,IAAI,GAAG,QAAQ,CAAC;IAEpB,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC;QACzD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,CAAC,kBAAkB,EAAE,CAAC;QACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAChB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACpC,IAAI,IAAI,KAAK,OAAO;YAAE,IAAI,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,OAAO,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { FileSourceConfig } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Check if file content has low semantic value (SVG paths, base64, minified code).
|
|
4
|
+
* Samples the first 8KB and checks the ratio of digits, dots, commas, semicolons,
|
|
5
|
+
* and equals signs. If >30% of characters are these low-value tokens, the file
|
|
6
|
+
* is likely SVG path data, base64, or minified code with no search value.
|
|
7
|
+
*/
|
|
8
|
+
export declare function hasLowSemanticValue(content: string): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* Convert a glob pattern to a RegExp.
|
|
11
|
+
* Supports: ** (any path), * (any segment), ? (any char)
|
|
12
|
+
*/
|
|
13
|
+
export declare function globToRegex(pattern: string): RegExp;
|
|
14
|
+
/**
|
|
15
|
+
* Check if a relative file path matches the source's file_patterns (include)
|
|
16
|
+
* and does not match exclude_patterns.
|
|
17
|
+
*/
|
|
18
|
+
export declare function matchesPatterns(relPath: string, sourceConfig: FileSourceConfig): boolean;
|
|
19
|
+
//# sourceMappingURL=utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/indexing/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAqB5D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAWnD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,GAAG,OAAO,CAmBxF"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Shared indexing utilities — used by providers, bash-fs, and test scripts.
|
|
2
|
+
/**
|
|
3
|
+
* Check if file content has low semantic value (SVG paths, base64, minified code).
|
|
4
|
+
* Samples the first 8KB and checks the ratio of digits, dots, commas, semicolons,
|
|
5
|
+
* and equals signs. If >30% of characters are these low-value tokens, the file
|
|
6
|
+
* is likely SVG path data, base64, or minified code with no search value.
|
|
7
|
+
*/
|
|
8
|
+
export function hasLowSemanticValue(content) {
|
|
9
|
+
if (content.length < 500)
|
|
10
|
+
return false;
|
|
11
|
+
const sample = content.slice(0, 8192);
|
|
12
|
+
let lowValueChars = 0;
|
|
13
|
+
for (let i = 0; i < sample.length; i++) {
|
|
14
|
+
const c = sample.charCodeAt(i);
|
|
15
|
+
if ((c >= 48 && c <= 57) || // 0-9
|
|
16
|
+
c === 46 || // .
|
|
17
|
+
c === 44 || // ,
|
|
18
|
+
c === 59 || // ;
|
|
19
|
+
c === 61 // =
|
|
20
|
+
) {
|
|
21
|
+
lowValueChars++;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const ratio = lowValueChars / sample.length;
|
|
25
|
+
return ratio > 0.3;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Convert a glob pattern to a RegExp.
|
|
29
|
+
* Supports: ** (any path), * (any segment), ? (any char)
|
|
30
|
+
*/
|
|
31
|
+
export function globToRegex(pattern) {
|
|
32
|
+
let re = pattern
|
|
33
|
+
.replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex chars (except * and ?)
|
|
34
|
+
.replace(/\*\*\//g, '{{GLOBSTAR_SLASH}}') // **/ = any path prefix (including empty)
|
|
35
|
+
.replace(/\*\*/g, '{{GLOBSTAR}}') // ** alone = anything
|
|
36
|
+
.replace(/\*/g, '[^/]*') // * = anything except /
|
|
37
|
+
.replace(/\?/g, '[^/]') // ? = single char except /
|
|
38
|
+
.replace(/\{\{GLOBSTAR_SLASH\}\}/g, '(?:.*/)?') // **/ = optional path prefix
|
|
39
|
+
.replace(/\{\{GLOBSTAR\}\}/g, '.*'); // ** = anything including /
|
|
40
|
+
return new RegExp(`^${re}$`);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Check if a relative file path matches the source's file_patterns (include)
|
|
44
|
+
* and does not match exclude_patterns.
|
|
45
|
+
*/
|
|
46
|
+
export function matchesPatterns(relPath, sourceConfig) {
|
|
47
|
+
const normalized = relPath.replace(/\\/g, '/');
|
|
48
|
+
// Check excludes first (takes precedence)
|
|
49
|
+
const excludes = sourceConfig.exclude_patterns ?? [];
|
|
50
|
+
for (const pattern of excludes) {
|
|
51
|
+
if (globToRegex(pattern).test(normalized)) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
// Must match at least one include pattern
|
|
56
|
+
for (const pattern of sourceConfig.file_patterns) {
|
|
57
|
+
if (globToRegex(pattern).test(normalized)) {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/indexing/utils.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAI5E;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IAC/C,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACtC,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/B,IACI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,IAAK,MAAM;YAC/B,CAAC,KAAK,EAAE,IAAkB,IAAI;YAC9B,CAAC,KAAK,EAAE,IAAkB,IAAI;YAC9B,CAAC,KAAK,EAAE,IAAkB,IAAI;YAC9B,CAAC,KAAK,EAAE,CAAkB,IAAI;UAChC,CAAC;YACC,aAAa,EAAE,CAAC;QACpB,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5C,OAAO,KAAK,GAAG,GAAG,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACvC,IAAI,EAAE,GAAG,OAAO;SACX,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,sCAAsC;SAC3E,OAAO,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,0CAA0C;SACnF,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAU,sBAAsB;SAChE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAoB,wBAAwB;SACnE,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAqB,2BAA2B;SACtE,OAAO,CAAC,yBAAyB,EAAE,UAAU,CAAC,CAAC,6BAA6B;SAC5E,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAa,4BAA4B;IAEjF,OAAO,IAAI,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,YAA8B;IAC3E,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAE/C,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,YAAY,CAAC,gBAAgB,IAAI,EAAE,CAAC;IACrD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;QAC/C,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export declare class IpSessionLimiter {
|
|
2
|
+
private maxPerIp;
|
|
3
|
+
private ipToSessions;
|
|
4
|
+
private sessionToIp;
|
|
5
|
+
constructor(maxPerIp: number);
|
|
6
|
+
tryAdd(ip: string, sessionId: string): boolean;
|
|
7
|
+
remove(sessionId: string): void;
|
|
8
|
+
getSessionCount(ip: string): number;
|
|
9
|
+
getMax(): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=ip-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ip-limiter.d.ts","sourceRoot":"","sources":["../src/ip-limiter.ts"],"names":[],"mappings":"AAAA,qBAAa,gBAAgB;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,WAAW,CAA6B;gBAEpC,QAAQ,EAAE,MAAM;IAI5B,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO;IAa9C,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAW/B,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAInC,MAAM,IAAI,MAAM;CAGnB"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export class IpSessionLimiter {
|
|
2
|
+
maxPerIp;
|
|
3
|
+
ipToSessions = new Map();
|
|
4
|
+
sessionToIp = new Map();
|
|
5
|
+
constructor(maxPerIp) {
|
|
6
|
+
this.maxPerIp = maxPerIp;
|
|
7
|
+
}
|
|
8
|
+
tryAdd(ip, sessionId) {
|
|
9
|
+
const sessions = this.ipToSessions.get(ip);
|
|
10
|
+
if (sessions && sessions.size >= this.maxPerIp)
|
|
11
|
+
return false;
|
|
12
|
+
if (!sessions) {
|
|
13
|
+
this.ipToSessions.set(ip, new Set([sessionId]));
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
sessions.add(sessionId);
|
|
17
|
+
}
|
|
18
|
+
this.sessionToIp.set(sessionId, ip);
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
remove(sessionId) {
|
|
22
|
+
const ip = this.sessionToIp.get(sessionId);
|
|
23
|
+
if (!ip)
|
|
24
|
+
return;
|
|
25
|
+
this.sessionToIp.delete(sessionId);
|
|
26
|
+
const sessions = this.ipToSessions.get(ip);
|
|
27
|
+
if (sessions) {
|
|
28
|
+
sessions.delete(sessionId);
|
|
29
|
+
if (sessions.size === 0)
|
|
30
|
+
this.ipToSessions.delete(ip);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
getSessionCount(ip) {
|
|
34
|
+
return this.ipToSessions.get(ip)?.size ?? 0;
|
|
35
|
+
}
|
|
36
|
+
getMax() {
|
|
37
|
+
return this.maxPerIp;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=ip-limiter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ip-limiter.js","sourceRoot":"","sources":["../src/ip-limiter.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,gBAAgB;IACjB,QAAQ,CAAS;IACjB,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEhD,YAAY,QAAgB;QACxB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,EAAU,EAAE,SAAiB;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE7D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,SAAiB;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3B,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;gBAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;IAED,eAAe,CAAC,EAAU;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;IAChD,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;CACJ"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
interface ChunkForLlms {
|
|
2
|
+
source_name: string;
|
|
3
|
+
file_path: string;
|
|
4
|
+
title: string | null;
|
|
5
|
+
content: string;
|
|
6
|
+
chunk_index: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function generateLlmsTxt(chunks: ChunkForLlms[], serverName: string): string;
|
|
9
|
+
export declare function generateLlmsFullTxt(chunks: ChunkForLlms[]): string;
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=llms-txt.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.d.ts","sourceRoot":"","sources":["../src/llms-txt.ts"],"names":[],"mappings":"AAAA,UAAU,YAAY;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CA2BlF;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAgBlE"}
|
package/dist/llms-txt.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export function generateLlmsTxt(chunks, serverName) {
|
|
2
|
+
const lines = [
|
|
3
|
+
`# ${serverName}`,
|
|
4
|
+
'',
|
|
5
|
+
'> Documentation index for AI agents',
|
|
6
|
+
'',
|
|
7
|
+
];
|
|
8
|
+
// Group by source, deduplicate by file_path, use first chunk's title
|
|
9
|
+
const bySource = new Map();
|
|
10
|
+
for (const c of chunks) {
|
|
11
|
+
if (!bySource.has(c.source_name))
|
|
12
|
+
bySource.set(c.source_name, new Map());
|
|
13
|
+
const files = bySource.get(c.source_name);
|
|
14
|
+
if (!files.has(c.file_path)) {
|
|
15
|
+
files.set(c.file_path, c.title || c.file_path);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
for (const [source, files] of bySource) {
|
|
19
|
+
lines.push(`## ${source}`, '');
|
|
20
|
+
for (const [filePath, title] of files) {
|
|
21
|
+
lines.push(`- ${filePath} — ${title}`);
|
|
22
|
+
}
|
|
23
|
+
lines.push('');
|
|
24
|
+
}
|
|
25
|
+
return lines.join('\n');
|
|
26
|
+
}
|
|
27
|
+
export function generateLlmsFullTxt(chunks) {
|
|
28
|
+
// Group chunks by file_path, sort by chunk_index, join content
|
|
29
|
+
const byFile = new Map();
|
|
30
|
+
for (const c of chunks) {
|
|
31
|
+
if (!byFile.has(c.file_path))
|
|
32
|
+
byFile.set(c.file_path, []);
|
|
33
|
+
byFile.get(c.file_path).push(c);
|
|
34
|
+
}
|
|
35
|
+
const sections = [];
|
|
36
|
+
for (const [filePath, fileChunks] of byFile) {
|
|
37
|
+
fileChunks.sort((a, b) => a.chunk_index - b.chunk_index);
|
|
38
|
+
const content = fileChunks.map(c => c.content).join('\n');
|
|
39
|
+
sections.push(`---\nfile: ${filePath}\n---\n\n${content}`);
|
|
40
|
+
}
|
|
41
|
+
return sections.join('\n\n');
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=llms-txt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../src/llms-txt.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,eAAe,CAAC,MAAsB,EAAE,UAAkB;IACtE,MAAM,KAAK,GAAa;QACpB,KAAK,UAAU,EAAE;QACjB,EAAE;QACF,qCAAqC;QACrC,EAAE;KACL,CAAC;IAEF,qEAAqE;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,MAAM,KAAK,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAsB;IACtD,+DAA+D;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,MAAM,EAAE,CAAC;QAC1C,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,QAAQ,CAAC,IAAI,CAAC,cAAc,QAAQ,YAAY,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC"}
|
package/dist/mcp/server.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import type { Bash } from 'just-bash';
|
|
2
2
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
3
|
import { SessionStateManager } from './tools/bash-session.js';
|
|
4
|
+
import type { BashTelemetry } from './tools/bash-telemetry.js';
|
|
5
|
+
import type { WorkspaceManager } from '../workspace.js';
|
|
4
6
|
/**
|
|
5
7
|
* Creates a new McpServer instance with all tools registered.
|
|
6
8
|
* Each MCP session gets its own server instance. Each bash tool gets its own
|
|
7
9
|
* virtual filesystem instance, shared across all MCP sessions for that tool.
|
|
8
10
|
*/
|
|
9
|
-
export declare function createMcpServer(bashInstances?: Map<string, Bash>, sessionStateManager?: SessionStateManager, getSessionId?: () => string | undefined): McpServer;
|
|
11
|
+
export declare function createMcpServer(bashInstances?: Map<string, Bash>, sessionStateManager?: SessionStateManager, getSessionId?: () => string | undefined, telemetry?: BashTelemetry, workspace?: WorkspaceManager): McpServer;
|
|
10
12
|
//# sourceMappingURL=server.d.ts.map
|
package/dist/mcp/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAOpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAExD;;;;GAIG;AACH,wBAAgB,eAAe,CAC3B,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,EACjC,mBAAmB,CAAC,EAAE,mBAAmB,EACzC,YAAY,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,EACvC,SAAS,CAAC,EAAE,aAAa,EACzB,SAAS,CAAC,EAAE,gBAAgB,GAC7B,SAAS,CAqEX"}
|
package/dist/mcp/server.js
CHANGED
|
@@ -3,13 +3,14 @@ import { EmbeddingClient } from '../indexing/embeddings.js';
|
|
|
3
3
|
import { getConfig, getServerConfig } from '../config.js';
|
|
4
4
|
import { registerSearchTool } from './tools/search.js';
|
|
5
5
|
import { registerCollectTool } from './tools/collect.js';
|
|
6
|
+
import { registerKnowledgeTool } from './tools/knowledge.js';
|
|
6
7
|
import { registerBashTool } from './tools/bash.js';
|
|
7
8
|
/**
|
|
8
9
|
* Creates a new McpServer instance with all tools registered.
|
|
9
10
|
* Each MCP session gets its own server instance. Each bash tool gets its own
|
|
10
11
|
* virtual filesystem instance, shared across all MCP sessions for that tool.
|
|
11
12
|
*/
|
|
12
|
-
export function createMcpServer(bashInstances, sessionStateManager, getSessionId) {
|
|
13
|
+
export function createMcpServer(bashInstances, sessionStateManager, getSessionId, telemetry, workspace) {
|
|
13
14
|
const cfg = getConfig();
|
|
14
15
|
const serverCfg = getServerConfig();
|
|
15
16
|
// Lazily created — only when a RAG tool needs it
|
|
@@ -49,13 +50,20 @@ export function createMcpServer(bashInstances, sessionStateManager, getSessionId
|
|
|
49
50
|
const grepStrategy = tool.bash?.grep_strategy;
|
|
50
51
|
const needsEmbedding = grepStrategy === 'vector' || grepStrategy === 'hybrid';
|
|
51
52
|
const searchToolNames = serverCfg.tools.filter(t => t.type === 'search').map(t => t.name);
|
|
53
|
+
const needsWorkspace = tool.bash?.workspace === true;
|
|
52
54
|
registerBashTool(server, tool, bash, {
|
|
53
55
|
getSessionState,
|
|
54
56
|
embeddingClient: needsEmbedding ? getEmbeddingClient() : undefined,
|
|
55
57
|
searchToolNames,
|
|
58
|
+
telemetry,
|
|
59
|
+
workspace: needsWorkspace ? workspace : undefined,
|
|
60
|
+
getSessionId: needsWorkspace ? getSessionId : undefined,
|
|
56
61
|
});
|
|
57
62
|
break;
|
|
58
63
|
}
|
|
64
|
+
case 'knowledge':
|
|
65
|
+
registerKnowledgeTool(server, getEmbeddingClient(), tool);
|
|
66
|
+
break;
|
|
59
67
|
default: {
|
|
60
68
|
const _exhaustive = tool;
|
|
61
69
|
throw new Error(`Unknown tool type: ${_exhaustive.type}`);
|
package/dist/mcp/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAKnD;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC3B,aAAiC,EACjC,mBAAyC,EACzC,YAAuC,EACvC,SAAyB,EACzB,SAA4B;IAE5B,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IAEpC,iDAAiD;IACjD,IAAI,eAAe,GAA2B,IAAI,CAAC;IACnD,SAAS,kBAAkB;QACvB,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YACrE,CAAC;YACD,eAAe,GAAG,IAAI,eAAe,CACjC,GAAG,CAAC,YAAY,EAChB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QACN,CAAC;QACD,OAAO,eAAe,CAAC;IAC3B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI;QAC3B,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO;KACpC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACjC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,KAAK,SAAS;gBACV,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAClC,MAAM;YACV,KAAK,QAAQ;gBACT,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvD,MAAM;YACV,KAAK,MAAM,CAAC,CAAC,CAAC;gBACV,MAAM,IAAI,GAAG,aAAa,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACR,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,IAAI,mDAAmD,CAAC,CAAC;gBAChG,CAAC;gBACD,MAAM,eAAe,GAAG,CAAC,mBAAmB,IAAI,YAAY,CAAC;oBACzD,CAAC,CAAC,GAAG,EAAE;wBACH,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;wBAC3B,OAAO,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;oBAClE,CAAC;oBACD,CAAC,CAAC,SAAS,CAAC;gBAChB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC;gBAC9C,MAAM,cAAc,GAAG,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ,CAAC;gBAC9E,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC1F,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,CAAC;gBACrD,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE;oBACjC,eAAe;oBACf,eAAe,EAAE,cAAc,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,SAAS;oBAClE,eAAe;oBACf,SAAS;oBACT,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACjD,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;iBAC1D,CAAC,CAAC;gBACH,MAAM;YACV,CAAC;YACD,KAAK,WAAW;gBACZ,qBAAqB,CAAC,MAAM,EAAE,kBAAkB,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC1D,MAAM;YACV,OAAO,CAAC,CAAC,CAAC;gBACN,MAAM,WAAW,GAAU,IAAI,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,sBAAuB,WAAgC,CAAC,IAAI,EAAE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bash-fs.d.ts","sourceRoot":"","sources":["../../../src/mcp/tools/bash-fs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"bash-fs.d.ts","sourceRoot":"","sources":["../../../src/mcp/tools/bash-fs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAqCnD,MAAM,WAAW,mBAAmB;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,iBAAiB,CACnC,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,CAAC,EAAE,mBAAmB,GAC9B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CA+CjC;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAU7F;AAED,wBAAgB,YAAY,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,GACvC,MAAM,CA+BR;AAED,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,CAAC,EAAE,mBAAmB,GAC9B,OAAO,CAAC;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAM5C"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Bash } from 'just-bash';
|
|
4
|
-
import { matchesPatterns } from '../../indexing/
|
|
4
|
+
import { matchesPatterns } from '../../indexing/utils.js';
|
|
5
|
+
import { isFileSourceConfig } from '../../types.js';
|
|
5
6
|
import { generateIndexMd, generateSearchTipsMd } from './bash-virtual-files.js';
|
|
6
7
|
const DEFAULT_SKIP_DIRS = new Set(['node_modules', 'dist', 'build', '.git']);
|
|
7
8
|
const DEFAULT_MAX_FILE_SIZE = 102400; // 100KB
|
|
@@ -40,6 +41,8 @@ export async function buildBashFilesMap(sources, options) {
|
|
|
40
41
|
const files = {};
|
|
41
42
|
const multiSource = sources.length > 1;
|
|
42
43
|
for (const source of sources) {
|
|
44
|
+
if (!isFileSourceConfig(source))
|
|
45
|
+
continue; // Slack sources don't have filesystem paths
|
|
43
46
|
let rootDir;
|
|
44
47
|
if (source.repo && options?.cloneDir) {
|
|
45
48
|
// Git-based source: the orchestrator clones into cloneDir/<repoName>/
|