@usewhisper/mcp-server 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -154
- package/dist/autosubscribe-6EDKPBE2.js +4068 -4068
- package/dist/autosubscribe-GHO6YR5A.js +4068 -4068
- package/dist/autosubscribe-ISDETQIB.js +435 -435
- package/dist/chunk-3WGYBAYR.js +8387 -8387
- package/dist/chunk-52VJYCZ7.js +455 -455
- package/dist/chunk-5KBZQHDL.js +189 -189
- package/dist/chunk-5KIJNY6Z.js +370 -370
- package/dist/chunk-7SN3CKDK.js +1076 -1076
- package/dist/chunk-B3VWOHUA.js +271 -271
- package/dist/chunk-C57DHKTL.js +459 -459
- package/dist/chunk-EI5CE3EY.js +616 -616
- package/dist/chunk-FTWUJBAH.js +386 -386
- package/dist/chunk-H3HSKH2P.js +4841 -4841
- package/dist/chunk-JO3ORBZD.js +616 -616
- package/dist/chunk-L6DXSM2U.js +456 -456
- package/dist/chunk-LMEYV4JD.js +368 -368
- package/dist/chunk-MEFLJ4PV.js +8385 -8385
- package/dist/chunk-OBLI4FE4.js +275 -275
- package/dist/chunk-PPGYJJED.js +271 -271
- package/dist/chunk-QGM4M3NI.js +37 -37
- package/dist/chunk-T7KMSTWP.js +399 -399
- package/dist/chunk-TWEIYHI6.js +399 -399
- package/dist/chunk-UYWE7HSU.js +368 -368
- package/dist/chunk-X2DL2GWT.js +32 -32
- package/dist/chunk-X7HNNNJJ.js +1079 -1079
- package/dist/consolidation-2GCKI4RE.js +220 -220
- package/dist/consolidation-4JOPW6BG.js +220 -220
- package/dist/consolidation-FOVQTWNQ.js +222 -222
- package/dist/consolidation-IFQ52E44.js +209 -209
- package/dist/context-sharing-4ITCNKG4.js +307 -307
- package/dist/context-sharing-6CCFIAKL.js +275 -275
- package/dist/context-sharing-GYKLXHZA.js +307 -307
- package/dist/context-sharing-PH64JTXS.js +308 -308
- package/dist/context-sharing-Y6LTZZOF.js +307 -307
- package/dist/cost-optimization-6OIKRSBV.js +195 -195
- package/dist/cost-optimization-7DVSTL6R.js +307 -307
- package/dist/cost-optimization-BH5NAX33.js +286 -286
- package/dist/cost-optimization-F3L5BS5F.js +303 -303
- package/dist/ingest-2LPTWUUM.js +16 -16
- package/dist/ingest-7T5FAZNC.js +15 -15
- package/dist/ingest-EBNIE7XB.js +15 -15
- package/dist/ingest-FSHT5BCS.js +15 -15
- package/dist/ingest-QE2BTV72.js +14 -14
- package/dist/oracle-3RLQF3DP.js +259 -259
- package/dist/oracle-FKRTQUUG.js +282 -282
- package/dist/oracle-J47QCSEW.js +263 -263
- package/dist/oracle-MDP5MZRC.js +256 -256
- package/dist/search-BLVHWLWC.js +14 -14
- package/dist/search-CZ5NYL5B.js +12 -12
- package/dist/search-EG6TYWWW.js +13 -13
- package/dist/search-I22QQA7T.js +13 -13
- package/dist/search-T7H5G6DW.js +13 -13
- package/dist/server.d.ts +2 -2
- package/dist/server.js +1973 -169
- package/dist/server.js.map +1 -1
- package/package.json +51 -51
|
@@ -1,436 +1,436 @@
|
|
|
1
|
-
import {
|
|
2
|
-
ingestDocument
|
|
3
|
-
} from "./chunk-FTWUJBAH.js";
|
|
4
|
-
import {
|
|
5
|
-
db
|
|
6
|
-
} from "./chunk-X2DL2GWT.js";
|
|
7
|
-
|
|
8
|
-
// src/engine/autosubscribe.ts
|
|
9
|
-
import { Octokit } from "@octokit/rest";
|
|
10
|
-
|
|
11
|
-
// src/connectors/npm_package.ts
|
|
12
|
-
async function syncNpmPackage(sourceId, projectId, config) {
|
|
13
|
-
const { packageName, includeReadme = true } = config;
|
|
14
|
-
let indexed = 0;
|
|
15
|
-
const res = await fetch(`https://registry.npmjs.org/${encodeURIComponent(packageName)}`);
|
|
16
|
-
if (!res.ok) throw new Error(`npm registry error: ${res.status}`);
|
|
17
|
-
const pkg = await res.json();
|
|
18
|
-
const latest = pkg["dist-tags"]?.latest;
|
|
19
|
-
const latestVersion = latest ? pkg.versions?.[latest] : null;
|
|
20
|
-
const overview = [
|
|
21
|
-
`# ${packageName}`,
|
|
22
|
-
pkg.description ? `
|
|
23
|
-
${pkg.description}` : "",
|
|
24
|
-
`
|
|
25
|
-
Latest version: ${latest || "unknown"}`,
|
|
26
|
-
pkg.license ? `License: ${pkg.license}` : "",
|
|
27
|
-
pkg.homepage ? `Homepage: ${pkg.homepage}` : "",
|
|
28
|
-
pkg.repository?.url ? `Repository: ${pkg.repository.url}` : "",
|
|
29
|
-
latestVersion?.keywords?.length ? `
|
|
30
|
-
Keywords: ${latestVersion.keywords.join(", ")}` : ""
|
|
31
|
-
].filter(Boolean).join("\n");
|
|
32
|
-
if (latestVersion?.dependencies) {
|
|
33
|
-
const deps = Object.entries(latestVersion.dependencies).map(([name, ver]) => `- ${name}: ${ver}`).join("\n");
|
|
34
|
-
await ingestDocument({
|
|
35
|
-
sourceId,
|
|
36
|
-
projectId,
|
|
37
|
-
externalId: `npm-${packageName}-deps`,
|
|
38
|
-
title: `${packageName} \u2014 Dependencies`,
|
|
39
|
-
content: `# ${packageName} Dependencies
|
|
40
|
-
|
|
41
|
-
${deps}`,
|
|
42
|
-
metadata: { source: "npm", packageName, section: "dependencies" }
|
|
43
|
-
});
|
|
44
|
-
indexed++;
|
|
45
|
-
}
|
|
46
|
-
if (latestVersion?.main || latestVersion?.types || latestVersion?.exports) {
|
|
47
|
-
const entryPoints = [
|
|
48
|
-
latestVersion.main ? `Main: ${latestVersion.main}` : "",
|
|
49
|
-
latestVersion.types ? `Types: ${latestVersion.types}` : "",
|
|
50
|
-
latestVersion.module ? `Module: ${latestVersion.module}` : ""
|
|
51
|
-
].filter(Boolean).join("\n");
|
|
52
|
-
await ingestDocument({
|
|
53
|
-
sourceId,
|
|
54
|
-
projectId,
|
|
55
|
-
externalId: `npm-${packageName}-overview`,
|
|
56
|
-
title: `${packageName} \u2014 Overview`,
|
|
57
|
-
content: `${overview}
|
|
58
|
-
|
|
59
|
-
## Entry Points
|
|
60
|
-
${entryPoints}`,
|
|
61
|
-
metadata: { source: "npm", packageName, version: latest, section: "overview" }
|
|
62
|
-
});
|
|
63
|
-
indexed++;
|
|
64
|
-
}
|
|
65
|
-
if (includeReadme && pkg.readme) {
|
|
66
|
-
await ingestDocument({
|
|
67
|
-
sourceId,
|
|
68
|
-
projectId,
|
|
69
|
-
externalId: `npm-${packageName}-readme`,
|
|
70
|
-
title: `${packageName} \u2014 README`,
|
|
71
|
-
content: pkg.readme,
|
|
72
|
-
metadata: { source: "npm", packageName, section: "readme" }
|
|
73
|
-
});
|
|
74
|
-
indexed++;
|
|
75
|
-
}
|
|
76
|
-
return { documentsIndexed: indexed };
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// src/connectors/pypi_package.ts
|
|
80
|
-
async function syncPyPIPackage(sourceId, projectId, config) {
|
|
81
|
-
const { packageName, includeDescription = true } = config;
|
|
82
|
-
let indexed = 0;
|
|
83
|
-
const res = await fetch(`https://pypi.org/pypi/${encodeURIComponent(packageName)}/json`);
|
|
84
|
-
if (!res.ok) throw new Error(`PyPI API error: ${res.status}`);
|
|
85
|
-
const data = await res.json();
|
|
86
|
-
const info = data.info || {};
|
|
87
|
-
const overview = [
|
|
88
|
-
`# ${info.name || packageName}`,
|
|
89
|
-
info.summary ? `
|
|
90
|
-
${info.summary}` : "",
|
|
91
|
-
`
|
|
92
|
-
Version: ${info.version || "unknown"}`,
|
|
93
|
-
info.license ? `License: ${info.license}` : "",
|
|
94
|
-
info.author ? `Author: ${info.author}` : "",
|
|
95
|
-
info.home_page ? `Homepage: ${info.home_page}` : "",
|
|
96
|
-
info.project_urls?.Documentation ? `Docs: ${info.project_urls.Documentation}` : "",
|
|
97
|
-
info.project_urls?.Repository || info.project_urls?.Source ? `Repository: ${info.project_urls.Repository || info.project_urls.Source}` : "",
|
|
98
|
-
info.requires_python ? `
|
|
99
|
-
Python: ${info.requires_python}` : "",
|
|
100
|
-
info.keywords ? `Keywords: ${info.keywords}` : ""
|
|
101
|
-
].filter(Boolean).join("\n");
|
|
102
|
-
await ingestDocument({
|
|
103
|
-
sourceId,
|
|
104
|
-
projectId,
|
|
105
|
-
externalId: `pypi-${packageName}-overview`,
|
|
106
|
-
title: `${packageName} \u2014 Overview`,
|
|
107
|
-
content: overview,
|
|
108
|
-
metadata: { source: "pypi", packageName, version: info.version, section: "overview" }
|
|
109
|
-
});
|
|
110
|
-
indexed++;
|
|
111
|
-
if (info.requires_dist?.length) {
|
|
112
|
-
const deps = info.requires_dist.filter((d) => !d.includes("extra ==")).map((d) => `- ${d}`).join("\n");
|
|
113
|
-
if (deps) {
|
|
114
|
-
await ingestDocument({
|
|
115
|
-
sourceId,
|
|
116
|
-
projectId,
|
|
117
|
-
externalId: `pypi-${packageName}-deps`,
|
|
118
|
-
title: `${packageName} \u2014 Dependencies`,
|
|
119
|
-
content: `# ${packageName} Dependencies
|
|
120
|
-
|
|
121
|
-
${deps}`,
|
|
122
|
-
metadata: { source: "pypi", packageName, section: "dependencies" }
|
|
123
|
-
});
|
|
124
|
-
indexed++;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
if (includeDescription && info.description) {
|
|
128
|
-
await ingestDocument({
|
|
129
|
-
sourceId,
|
|
130
|
-
projectId,
|
|
131
|
-
externalId: `pypi-${packageName}-description`,
|
|
132
|
-
title: `${packageName} \u2014 Description`,
|
|
133
|
-
content: info.description,
|
|
134
|
-
metadata: {
|
|
135
|
-
source: "pypi",
|
|
136
|
-
packageName,
|
|
137
|
-
section: "description",
|
|
138
|
-
contentType: info.description_content_type || "text/plain"
|
|
139
|
-
}
|
|
140
|
-
});
|
|
141
|
-
indexed++;
|
|
142
|
-
}
|
|
143
|
-
return { documentsIndexed: indexed };
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// src/engine/autosubscribe.ts
|
|
147
|
-
async function parsePackageJson(content) {
|
|
148
|
-
try {
|
|
149
|
-
const pkg = JSON.parse(content);
|
|
150
|
-
const deps = [];
|
|
151
|
-
if (pkg.dependencies) {
|
|
152
|
-
for (const [name, version] of Object.entries(pkg.dependencies)) {
|
|
153
|
-
deps.push({
|
|
154
|
-
name,
|
|
155
|
-
version: String(version),
|
|
156
|
-
ecosystem: "npm"
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
if (pkg.devDependencies) {
|
|
161
|
-
for (const [name, version] of Object.entries(pkg.devDependencies)) {
|
|
162
|
-
deps.push({
|
|
163
|
-
name,
|
|
164
|
-
version: String(version),
|
|
165
|
-
ecosystem: "npm"
|
|
166
|
-
});
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return deps;
|
|
170
|
-
} catch (error) {
|
|
171
|
-
console.error("Failed to parse package.json:", error);
|
|
172
|
-
return [];
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
function parseRequirementsTxt(content) {
|
|
176
|
-
const deps = [];
|
|
177
|
-
const lines = content.split("\n");
|
|
178
|
-
for (const line of lines) {
|
|
179
|
-
const trimmed = line.trim();
|
|
180
|
-
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
181
|
-
const match = trimmed.match(/^([a-zA-Z0-9_-]+)([><=!~]+)?(.+)?$/);
|
|
182
|
-
if (match) {
|
|
183
|
-
deps.push({
|
|
184
|
-
name: match[1],
|
|
185
|
-
version: match[3] || "latest",
|
|
186
|
-
ecosystem: "pypi"
|
|
187
|
-
});
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
return deps;
|
|
191
|
-
}
|
|
192
|
-
function parseCargoToml(content) {
|
|
193
|
-
const deps = [];
|
|
194
|
-
const depsMatch = content.match(/\[dependencies\]([\s\S]*?)(\[|$)/);
|
|
195
|
-
if (!depsMatch) return deps;
|
|
196
|
-
const depsSection = depsMatch[1];
|
|
197
|
-
const lines = depsSection.split("\n");
|
|
198
|
-
for (const line of lines) {
|
|
199
|
-
const match = line.match(/^([a-zA-Z0-9_-]+)\s*=\s*"([^"]+)"/);
|
|
200
|
-
if (match) {
|
|
201
|
-
deps.push({
|
|
202
|
-
name: match[1],
|
|
203
|
-
version: match[2],
|
|
204
|
-
ecosystem: "cargo"
|
|
205
|
-
});
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
return deps;
|
|
209
|
-
}
|
|
210
|
-
function parseDependencyFile(filename, content) {
|
|
211
|
-
if (filename === "package.json") {
|
|
212
|
-
return parsePackageJson(content);
|
|
213
|
-
} else if (filename === "requirements.txt" || filename.endsWith(".txt")) {
|
|
214
|
-
return parseRequirementsTxt(content);
|
|
215
|
-
} else if (filename === "Cargo.toml") {
|
|
216
|
-
return parseCargoToml(content);
|
|
217
|
-
}
|
|
218
|
-
return [];
|
|
219
|
-
}
|
|
220
|
-
async function fetchDependencyFile(params) {
|
|
221
|
-
const { owner, repo, branch = "main", githubToken } = params;
|
|
222
|
-
const octokit = new Octokit({
|
|
223
|
-
auth: githubToken || process.env.GITHUB_TOKEN
|
|
224
|
-
});
|
|
225
|
-
const files = [
|
|
226
|
-
"package.json",
|
|
227
|
-
"requirements.txt",
|
|
228
|
-
"Cargo.toml",
|
|
229
|
-
"pom.xml",
|
|
230
|
-
// Maven
|
|
231
|
-
"build.gradle"
|
|
232
|
-
// Gradle
|
|
233
|
-
];
|
|
234
|
-
for (const filename of files) {
|
|
235
|
-
try {
|
|
236
|
-
const { data } = await octokit.repos.getContent({
|
|
237
|
-
owner,
|
|
238
|
-
repo,
|
|
239
|
-
path: filename,
|
|
240
|
-
ref: branch
|
|
241
|
-
});
|
|
242
|
-
if ("content" in data && data.content) {
|
|
243
|
-
const content = Buffer.from(data.content, "base64").toString("utf-8");
|
|
244
|
-
return { filename, content };
|
|
245
|
-
}
|
|
246
|
-
} catch (error) {
|
|
247
|
-
continue;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return null;
|
|
251
|
-
}
|
|
252
|
-
async function resolveDocsUrl(dep) {
|
|
253
|
-
switch (dep.ecosystem) {
|
|
254
|
-
case "npm":
|
|
255
|
-
try {
|
|
256
|
-
const response = await fetch(`https://registry.npmjs.org/${dep.name}`);
|
|
257
|
-
const data = await response.json();
|
|
258
|
-
if (data.homepage) return data.homepage;
|
|
259
|
-
if (data.repository?.url) {
|
|
260
|
-
const url = data.repository.url.replace("git+", "").replace(".git", "").replace("git://", "https://");
|
|
261
|
-
return url;
|
|
262
|
-
}
|
|
263
|
-
return `https://www.npmjs.com/package/${dep.name}`;
|
|
264
|
-
} catch (error) {
|
|
265
|
-
return `https://www.npmjs.com/package/${dep.name}`;
|
|
266
|
-
}
|
|
267
|
-
case "pypi":
|
|
268
|
-
try {
|
|
269
|
-
const response = await fetch(`https://pypi.org/pypi/${dep.name}/json`);
|
|
270
|
-
const data = await response.json();
|
|
271
|
-
if (data.info.home_page) return data.info.home_page;
|
|
272
|
-
if (data.info.project_urls?.Documentation) {
|
|
273
|
-
return data.info.project_urls.Documentation;
|
|
274
|
-
}
|
|
275
|
-
return `https://pypi.org/project/${dep.name}/`;
|
|
276
|
-
} catch (error) {
|
|
277
|
-
return `https://pypi.org/project/${dep.name}/`;
|
|
278
|
-
}
|
|
279
|
-
case "cargo":
|
|
280
|
-
return `https://docs.rs/${dep.name}`;
|
|
281
|
-
default:
|
|
282
|
-
return null;
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
async function autosubscribe(params) {
|
|
286
|
-
const { projectId, orgId, source, indexLimit = 50 } = params;
|
|
287
|
-
const result = {
|
|
288
|
-
discovered: 0,
|
|
289
|
-
indexed: 0,
|
|
290
|
-
skipped: 0,
|
|
291
|
-
errors: []
|
|
292
|
-
};
|
|
293
|
-
try {
|
|
294
|
-
let depFile = null;
|
|
295
|
-
if (source.type === "github" && source.owner && source.repo) {
|
|
296
|
-
depFile = await fetchDependencyFile({
|
|
297
|
-
owner: source.owner,
|
|
298
|
-
repo: source.repo,
|
|
299
|
-
branch: source.branch
|
|
300
|
-
});
|
|
301
|
-
} else if (source.type === "local" && source.filePath) {
|
|
302
|
-
const fs = await import("fs");
|
|
303
|
-
const content = fs.readFileSync(source.filePath, "utf-8");
|
|
304
|
-
const filename = source.filePath.split("/").pop() || "package.json";
|
|
305
|
-
depFile = { filename, content };
|
|
306
|
-
}
|
|
307
|
-
if (!depFile) {
|
|
308
|
-
result.errors.push("No dependency file found");
|
|
309
|
-
return result;
|
|
310
|
-
}
|
|
311
|
-
const dependencies = parseDependencyFile(depFile.filename, depFile.content);
|
|
312
|
-
result.discovered = dependencies.length;
|
|
313
|
-
console.log(`\u{1F4E6} Discovered ${dependencies.length} dependencies`);
|
|
314
|
-
const toIndex = dependencies.slice(0, indexLimit);
|
|
315
|
-
for (const dep of toIndex) {
|
|
316
|
-
try {
|
|
317
|
-
const existing = await db.package.findFirst({
|
|
318
|
-
where: {
|
|
319
|
-
orgId,
|
|
320
|
-
ecosystem: dep.ecosystem,
|
|
321
|
-
name: dep.name
|
|
322
|
-
}
|
|
323
|
-
});
|
|
324
|
-
if (existing) {
|
|
325
|
-
console.log(`\u23ED\uFE0F Skipping ${dep.name} (already indexed)`);
|
|
326
|
-
result.skipped++;
|
|
327
|
-
continue;
|
|
328
|
-
}
|
|
329
|
-
const docsUrl = await resolveDocsUrl(dep);
|
|
330
|
-
if (!docsUrl) {
|
|
331
|
-
result.errors.push(`No docs URL for ${dep.name}`);
|
|
332
|
-
continue;
|
|
333
|
-
}
|
|
334
|
-
console.log(`\u{1F4DA} Indexing ${dep.name} from ${docsUrl}`);
|
|
335
|
-
await db.package.create({
|
|
336
|
-
data: {
|
|
337
|
-
orgId,
|
|
338
|
-
name: dep.name,
|
|
339
|
-
ecosystem: dep.ecosystem,
|
|
340
|
-
version: dep.version,
|
|
341
|
-
registryUrl: docsUrl,
|
|
342
|
-
autoSync: true
|
|
343
|
-
}
|
|
344
|
-
});
|
|
345
|
-
if (dep.ecosystem === "npm") {
|
|
346
|
-
await syncNpmPackage({
|
|
347
|
-
packageName: dep.name,
|
|
348
|
-
version: dep.version,
|
|
349
|
-
projectId,
|
|
350
|
-
orgId
|
|
351
|
-
});
|
|
352
|
-
} else if (dep.ecosystem === "pypi") {
|
|
353
|
-
await syncPyPIPackage({
|
|
354
|
-
packageName: dep.name,
|
|
355
|
-
version: dep.version,
|
|
356
|
-
projectId,
|
|
357
|
-
orgId
|
|
358
|
-
});
|
|
359
|
-
}
|
|
360
|
-
result.indexed++;
|
|
361
|
-
console.log(`\u2705 Indexed ${dep.name}`);
|
|
362
|
-
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
363
|
-
} catch (error) {
|
|
364
|
-
result.errors.push(`Failed to index ${dep.name}: ${error}`);
|
|
365
|
-
console.error(`\u274C Failed to index ${dep.name}:`, error);
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
if (dependencies.length > indexLimit) {
|
|
369
|
-
result.errors.push(
|
|
370
|
-
`Only indexed first ${indexLimit} of ${dependencies.length} packages (limit reached)`
|
|
371
|
-
);
|
|
372
|
-
}
|
|
373
|
-
return result;
|
|
374
|
-
} catch (error) {
|
|
375
|
-
result.errors.push(`Autosubscribe failed: ${error}`);
|
|
376
|
-
return result;
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
async function autoSyncPackages(orgId) {
|
|
380
|
-
const packages = await db.package.findMany({
|
|
381
|
-
where: {
|
|
382
|
-
orgId,
|
|
383
|
-
autoSync: true
|
|
384
|
-
}
|
|
385
|
-
});
|
|
386
|
-
console.log(`\u{1F504} Auto-syncing ${packages.length} packages...`);
|
|
387
|
-
for (const pkg of packages) {
|
|
388
|
-
try {
|
|
389
|
-
const shouldUpdate = await checkPackageUpdated(pkg);
|
|
390
|
-
if (shouldUpdate) {
|
|
391
|
-
console.log(`\u{1F4E6} Re-indexing ${pkg.name}...`);
|
|
392
|
-
if (pkg.ecosystem === "npm") {
|
|
393
|
-
await syncNpmPackage({
|
|
394
|
-
packageName: pkg.name,
|
|
395
|
-
version: pkg.version,
|
|
396
|
-
projectId: "",
|
|
397
|
-
// Will need project context
|
|
398
|
-
orgId
|
|
399
|
-
});
|
|
400
|
-
} else if (pkg.ecosystem === "pypi") {
|
|
401
|
-
await syncPyPIPackage({
|
|
402
|
-
packageName: pkg.name,
|
|
403
|
-
version: pkg.version,
|
|
404
|
-
projectId: "",
|
|
405
|
-
orgId
|
|
406
|
-
});
|
|
407
|
-
}
|
|
408
|
-
await db.package.update({
|
|
409
|
-
where: { id: pkg.id },
|
|
410
|
-
data: { lastIndexedAt: /* @__PURE__ */ new Date() }
|
|
411
|
-
});
|
|
412
|
-
}
|
|
413
|
-
} catch (error) {
|
|
414
|
-
console.error(`Failed to sync ${pkg.name}:`, error);
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
console.log("\u2705 Auto-sync complete");
|
|
418
|
-
}
|
|
419
|
-
async function checkPackageUpdated(pkg) {
|
|
420
|
-
if (!pkg.lastIndexedAt) return true;
|
|
421
|
-
const daysSinceIndex = Math.floor(
|
|
422
|
-
(Date.now() - pkg.lastIndexedAt.getTime()) / (1e3 * 60 * 60 * 24)
|
|
423
|
-
);
|
|
424
|
-
return daysSinceIndex > 7;
|
|
425
|
-
}
|
|
426
|
-
export {
|
|
427
|
-
autoSyncPackages,
|
|
428
|
-
autosubscribe,
|
|
429
|
-
fetchDependencyFile,
|
|
430
|
-
parseCargoToml,
|
|
431
|
-
parseDependencyFile,
|
|
432
|
-
parsePackageJson,
|
|
433
|
-
parseRequirementsTxt,
|
|
434
|
-
resolveDocsUrl
|
|
435
|
-
};
|
|
1
|
+
import {
|
|
2
|
+
ingestDocument
|
|
3
|
+
} from "./chunk-FTWUJBAH.js";
|
|
4
|
+
import {
|
|
5
|
+
db
|
|
6
|
+
} from "./chunk-X2DL2GWT.js";
|
|
7
|
+
|
|
8
|
+
// src/engine/autosubscribe.ts
|
|
9
|
+
import { Octokit } from "@octokit/rest";
|
|
10
|
+
|
|
11
|
+
// src/connectors/npm_package.ts
|
|
12
|
+
async function syncNpmPackage(sourceId, projectId, config) {
|
|
13
|
+
const { packageName, includeReadme = true } = config;
|
|
14
|
+
let indexed = 0;
|
|
15
|
+
const res = await fetch(`https://registry.npmjs.org/${encodeURIComponent(packageName)}`);
|
|
16
|
+
if (!res.ok) throw new Error(`npm registry error: ${res.status}`);
|
|
17
|
+
const pkg = await res.json();
|
|
18
|
+
const latest = pkg["dist-tags"]?.latest;
|
|
19
|
+
const latestVersion = latest ? pkg.versions?.[latest] : null;
|
|
20
|
+
const overview = [
|
|
21
|
+
`# ${packageName}`,
|
|
22
|
+
pkg.description ? `
|
|
23
|
+
${pkg.description}` : "",
|
|
24
|
+
`
|
|
25
|
+
Latest version: ${latest || "unknown"}`,
|
|
26
|
+
pkg.license ? `License: ${pkg.license}` : "",
|
|
27
|
+
pkg.homepage ? `Homepage: ${pkg.homepage}` : "",
|
|
28
|
+
pkg.repository?.url ? `Repository: ${pkg.repository.url}` : "",
|
|
29
|
+
latestVersion?.keywords?.length ? `
|
|
30
|
+
Keywords: ${latestVersion.keywords.join(", ")}` : ""
|
|
31
|
+
].filter(Boolean).join("\n");
|
|
32
|
+
if (latestVersion?.dependencies) {
|
|
33
|
+
const deps = Object.entries(latestVersion.dependencies).map(([name, ver]) => `- ${name}: ${ver}`).join("\n");
|
|
34
|
+
await ingestDocument({
|
|
35
|
+
sourceId,
|
|
36
|
+
projectId,
|
|
37
|
+
externalId: `npm-${packageName}-deps`,
|
|
38
|
+
title: `${packageName} \u2014 Dependencies`,
|
|
39
|
+
content: `# ${packageName} Dependencies
|
|
40
|
+
|
|
41
|
+
${deps}`,
|
|
42
|
+
metadata: { source: "npm", packageName, section: "dependencies" }
|
|
43
|
+
});
|
|
44
|
+
indexed++;
|
|
45
|
+
}
|
|
46
|
+
if (latestVersion?.main || latestVersion?.types || latestVersion?.exports) {
|
|
47
|
+
const entryPoints = [
|
|
48
|
+
latestVersion.main ? `Main: ${latestVersion.main}` : "",
|
|
49
|
+
latestVersion.types ? `Types: ${latestVersion.types}` : "",
|
|
50
|
+
latestVersion.module ? `Module: ${latestVersion.module}` : ""
|
|
51
|
+
].filter(Boolean).join("\n");
|
|
52
|
+
await ingestDocument({
|
|
53
|
+
sourceId,
|
|
54
|
+
projectId,
|
|
55
|
+
externalId: `npm-${packageName}-overview`,
|
|
56
|
+
title: `${packageName} \u2014 Overview`,
|
|
57
|
+
content: `${overview}
|
|
58
|
+
|
|
59
|
+
## Entry Points
|
|
60
|
+
${entryPoints}`,
|
|
61
|
+
metadata: { source: "npm", packageName, version: latest, section: "overview" }
|
|
62
|
+
});
|
|
63
|
+
indexed++;
|
|
64
|
+
}
|
|
65
|
+
if (includeReadme && pkg.readme) {
|
|
66
|
+
await ingestDocument({
|
|
67
|
+
sourceId,
|
|
68
|
+
projectId,
|
|
69
|
+
externalId: `npm-${packageName}-readme`,
|
|
70
|
+
title: `${packageName} \u2014 README`,
|
|
71
|
+
content: pkg.readme,
|
|
72
|
+
metadata: { source: "npm", packageName, section: "readme" }
|
|
73
|
+
});
|
|
74
|
+
indexed++;
|
|
75
|
+
}
|
|
76
|
+
return { documentsIndexed: indexed };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// src/connectors/pypi_package.ts
|
|
80
|
+
async function syncPyPIPackage(sourceId, projectId, config) {
|
|
81
|
+
const { packageName, includeDescription = true } = config;
|
|
82
|
+
let indexed = 0;
|
|
83
|
+
const res = await fetch(`https://pypi.org/pypi/${encodeURIComponent(packageName)}/json`);
|
|
84
|
+
if (!res.ok) throw new Error(`PyPI API error: ${res.status}`);
|
|
85
|
+
const data = await res.json();
|
|
86
|
+
const info = data.info || {};
|
|
87
|
+
const overview = [
|
|
88
|
+
`# ${info.name || packageName}`,
|
|
89
|
+
info.summary ? `
|
|
90
|
+
${info.summary}` : "",
|
|
91
|
+
`
|
|
92
|
+
Version: ${info.version || "unknown"}`,
|
|
93
|
+
info.license ? `License: ${info.license}` : "",
|
|
94
|
+
info.author ? `Author: ${info.author}` : "",
|
|
95
|
+
info.home_page ? `Homepage: ${info.home_page}` : "",
|
|
96
|
+
info.project_urls?.Documentation ? `Docs: ${info.project_urls.Documentation}` : "",
|
|
97
|
+
info.project_urls?.Repository || info.project_urls?.Source ? `Repository: ${info.project_urls.Repository || info.project_urls.Source}` : "",
|
|
98
|
+
info.requires_python ? `
|
|
99
|
+
Python: ${info.requires_python}` : "",
|
|
100
|
+
info.keywords ? `Keywords: ${info.keywords}` : ""
|
|
101
|
+
].filter(Boolean).join("\n");
|
|
102
|
+
await ingestDocument({
|
|
103
|
+
sourceId,
|
|
104
|
+
projectId,
|
|
105
|
+
externalId: `pypi-${packageName}-overview`,
|
|
106
|
+
title: `${packageName} \u2014 Overview`,
|
|
107
|
+
content: overview,
|
|
108
|
+
metadata: { source: "pypi", packageName, version: info.version, section: "overview" }
|
|
109
|
+
});
|
|
110
|
+
indexed++;
|
|
111
|
+
if (info.requires_dist?.length) {
|
|
112
|
+
const deps = info.requires_dist.filter((d) => !d.includes("extra ==")).map((d) => `- ${d}`).join("\n");
|
|
113
|
+
if (deps) {
|
|
114
|
+
await ingestDocument({
|
|
115
|
+
sourceId,
|
|
116
|
+
projectId,
|
|
117
|
+
externalId: `pypi-${packageName}-deps`,
|
|
118
|
+
title: `${packageName} \u2014 Dependencies`,
|
|
119
|
+
content: `# ${packageName} Dependencies
|
|
120
|
+
|
|
121
|
+
${deps}`,
|
|
122
|
+
metadata: { source: "pypi", packageName, section: "dependencies" }
|
|
123
|
+
});
|
|
124
|
+
indexed++;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (includeDescription && info.description) {
|
|
128
|
+
await ingestDocument({
|
|
129
|
+
sourceId,
|
|
130
|
+
projectId,
|
|
131
|
+
externalId: `pypi-${packageName}-description`,
|
|
132
|
+
title: `${packageName} \u2014 Description`,
|
|
133
|
+
content: info.description,
|
|
134
|
+
metadata: {
|
|
135
|
+
source: "pypi",
|
|
136
|
+
packageName,
|
|
137
|
+
section: "description",
|
|
138
|
+
contentType: info.description_content_type || "text/plain"
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
indexed++;
|
|
142
|
+
}
|
|
143
|
+
return { documentsIndexed: indexed };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// src/engine/autosubscribe.ts
|
|
147
|
+
async function parsePackageJson(content) {
|
|
148
|
+
try {
|
|
149
|
+
const pkg = JSON.parse(content);
|
|
150
|
+
const deps = [];
|
|
151
|
+
if (pkg.dependencies) {
|
|
152
|
+
for (const [name, version] of Object.entries(pkg.dependencies)) {
|
|
153
|
+
deps.push({
|
|
154
|
+
name,
|
|
155
|
+
version: String(version),
|
|
156
|
+
ecosystem: "npm"
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (pkg.devDependencies) {
|
|
161
|
+
for (const [name, version] of Object.entries(pkg.devDependencies)) {
|
|
162
|
+
deps.push({
|
|
163
|
+
name,
|
|
164
|
+
version: String(version),
|
|
165
|
+
ecosystem: "npm"
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return deps;
|
|
170
|
+
} catch (error) {
|
|
171
|
+
console.error("Failed to parse package.json:", error);
|
|
172
|
+
return [];
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function parseRequirementsTxt(content) {
|
|
176
|
+
const deps = [];
|
|
177
|
+
const lines = content.split("\n");
|
|
178
|
+
for (const line of lines) {
|
|
179
|
+
const trimmed = line.trim();
|
|
180
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
181
|
+
const match = trimmed.match(/^([a-zA-Z0-9_-]+)([><=!~]+)?(.+)?$/);
|
|
182
|
+
if (match) {
|
|
183
|
+
deps.push({
|
|
184
|
+
name: match[1],
|
|
185
|
+
version: match[3] || "latest",
|
|
186
|
+
ecosystem: "pypi"
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return deps;
|
|
191
|
+
}
|
|
192
|
+
function parseCargoToml(content) {
|
|
193
|
+
const deps = [];
|
|
194
|
+
const depsMatch = content.match(/\[dependencies\]([\s\S]*?)(\[|$)/);
|
|
195
|
+
if (!depsMatch) return deps;
|
|
196
|
+
const depsSection = depsMatch[1];
|
|
197
|
+
const lines = depsSection.split("\n");
|
|
198
|
+
for (const line of lines) {
|
|
199
|
+
const match = line.match(/^([a-zA-Z0-9_-]+)\s*=\s*"([^"]+)"/);
|
|
200
|
+
if (match) {
|
|
201
|
+
deps.push({
|
|
202
|
+
name: match[1],
|
|
203
|
+
version: match[2],
|
|
204
|
+
ecosystem: "cargo"
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return deps;
|
|
209
|
+
}
|
|
210
|
+
function parseDependencyFile(filename, content) {
|
|
211
|
+
if (filename === "package.json") {
|
|
212
|
+
return parsePackageJson(content);
|
|
213
|
+
} else if (filename === "requirements.txt" || filename.endsWith(".txt")) {
|
|
214
|
+
return parseRequirementsTxt(content);
|
|
215
|
+
} else if (filename === "Cargo.toml") {
|
|
216
|
+
return parseCargoToml(content);
|
|
217
|
+
}
|
|
218
|
+
return [];
|
|
219
|
+
}
|
|
220
|
+
async function fetchDependencyFile(params) {
|
|
221
|
+
const { owner, repo, branch = "main", githubToken } = params;
|
|
222
|
+
const octokit = new Octokit({
|
|
223
|
+
auth: githubToken || process.env.GITHUB_TOKEN
|
|
224
|
+
});
|
|
225
|
+
const files = [
|
|
226
|
+
"package.json",
|
|
227
|
+
"requirements.txt",
|
|
228
|
+
"Cargo.toml",
|
|
229
|
+
"pom.xml",
|
|
230
|
+
// Maven
|
|
231
|
+
"build.gradle"
|
|
232
|
+
// Gradle
|
|
233
|
+
];
|
|
234
|
+
for (const filename of files) {
|
|
235
|
+
try {
|
|
236
|
+
const { data } = await octokit.repos.getContent({
|
|
237
|
+
owner,
|
|
238
|
+
repo,
|
|
239
|
+
path: filename,
|
|
240
|
+
ref: branch
|
|
241
|
+
});
|
|
242
|
+
if ("content" in data && data.content) {
|
|
243
|
+
const content = Buffer.from(data.content, "base64").toString("utf-8");
|
|
244
|
+
return { filename, content };
|
|
245
|
+
}
|
|
246
|
+
} catch (error) {
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
async function resolveDocsUrl(dep) {
|
|
253
|
+
switch (dep.ecosystem) {
|
|
254
|
+
case "npm":
|
|
255
|
+
try {
|
|
256
|
+
const response = await fetch(`https://registry.npmjs.org/${dep.name}`);
|
|
257
|
+
const data = await response.json();
|
|
258
|
+
if (data.homepage) return data.homepage;
|
|
259
|
+
if (data.repository?.url) {
|
|
260
|
+
const url = data.repository.url.replace("git+", "").replace(".git", "").replace("git://", "https://");
|
|
261
|
+
return url;
|
|
262
|
+
}
|
|
263
|
+
return `https://www.npmjs.com/package/${dep.name}`;
|
|
264
|
+
} catch (error) {
|
|
265
|
+
return `https://www.npmjs.com/package/${dep.name}`;
|
|
266
|
+
}
|
|
267
|
+
case "pypi":
|
|
268
|
+
try {
|
|
269
|
+
const response = await fetch(`https://pypi.org/pypi/${dep.name}/json`);
|
|
270
|
+
const data = await response.json();
|
|
271
|
+
if (data.info.home_page) return data.info.home_page;
|
|
272
|
+
if (data.info.project_urls?.Documentation) {
|
|
273
|
+
return data.info.project_urls.Documentation;
|
|
274
|
+
}
|
|
275
|
+
return `https://pypi.org/project/${dep.name}/`;
|
|
276
|
+
} catch (error) {
|
|
277
|
+
return `https://pypi.org/project/${dep.name}/`;
|
|
278
|
+
}
|
|
279
|
+
case "cargo":
|
|
280
|
+
return `https://docs.rs/${dep.name}`;
|
|
281
|
+
default:
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
async function autosubscribe(params) {
|
|
286
|
+
const { projectId, orgId, source, indexLimit = 50 } = params;
|
|
287
|
+
const result = {
|
|
288
|
+
discovered: 0,
|
|
289
|
+
indexed: 0,
|
|
290
|
+
skipped: 0,
|
|
291
|
+
errors: []
|
|
292
|
+
};
|
|
293
|
+
try {
|
|
294
|
+
let depFile = null;
|
|
295
|
+
if (source.type === "github" && source.owner && source.repo) {
|
|
296
|
+
depFile = await fetchDependencyFile({
|
|
297
|
+
owner: source.owner,
|
|
298
|
+
repo: source.repo,
|
|
299
|
+
branch: source.branch
|
|
300
|
+
});
|
|
301
|
+
} else if (source.type === "local" && source.filePath) {
|
|
302
|
+
const fs = await import("fs");
|
|
303
|
+
const content = fs.readFileSync(source.filePath, "utf-8");
|
|
304
|
+
const filename = source.filePath.split("/").pop() || "package.json";
|
|
305
|
+
depFile = { filename, content };
|
|
306
|
+
}
|
|
307
|
+
if (!depFile) {
|
|
308
|
+
result.errors.push("No dependency file found");
|
|
309
|
+
return result;
|
|
310
|
+
}
|
|
311
|
+
const dependencies = parseDependencyFile(depFile.filename, depFile.content);
|
|
312
|
+
result.discovered = dependencies.length;
|
|
313
|
+
console.log(`\u{1F4E6} Discovered ${dependencies.length} dependencies`);
|
|
314
|
+
const toIndex = dependencies.slice(0, indexLimit);
|
|
315
|
+
for (const dep of toIndex) {
|
|
316
|
+
try {
|
|
317
|
+
const existing = await db.package.findFirst({
|
|
318
|
+
where: {
|
|
319
|
+
orgId,
|
|
320
|
+
ecosystem: dep.ecosystem,
|
|
321
|
+
name: dep.name
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
if (existing) {
|
|
325
|
+
console.log(`\u23ED\uFE0F Skipping ${dep.name} (already indexed)`);
|
|
326
|
+
result.skipped++;
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
const docsUrl = await resolveDocsUrl(dep);
|
|
330
|
+
if (!docsUrl) {
|
|
331
|
+
result.errors.push(`No docs URL for ${dep.name}`);
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
console.log(`\u{1F4DA} Indexing ${dep.name} from ${docsUrl}`);
|
|
335
|
+
await db.package.create({
|
|
336
|
+
data: {
|
|
337
|
+
orgId,
|
|
338
|
+
name: dep.name,
|
|
339
|
+
ecosystem: dep.ecosystem,
|
|
340
|
+
version: dep.version,
|
|
341
|
+
registryUrl: docsUrl,
|
|
342
|
+
autoSync: true
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
if (dep.ecosystem === "npm") {
|
|
346
|
+
await syncNpmPackage({
|
|
347
|
+
packageName: dep.name,
|
|
348
|
+
version: dep.version,
|
|
349
|
+
projectId,
|
|
350
|
+
orgId
|
|
351
|
+
});
|
|
352
|
+
} else if (dep.ecosystem === "pypi") {
|
|
353
|
+
await syncPyPIPackage({
|
|
354
|
+
packageName: dep.name,
|
|
355
|
+
version: dep.version,
|
|
356
|
+
projectId,
|
|
357
|
+
orgId
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
result.indexed++;
|
|
361
|
+
console.log(`\u2705 Indexed ${dep.name}`);
|
|
362
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
363
|
+
} catch (error) {
|
|
364
|
+
result.errors.push(`Failed to index ${dep.name}: ${error}`);
|
|
365
|
+
console.error(`\u274C Failed to index ${dep.name}:`, error);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
if (dependencies.length > indexLimit) {
|
|
369
|
+
result.errors.push(
|
|
370
|
+
`Only indexed first ${indexLimit} of ${dependencies.length} packages (limit reached)`
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
return result;
|
|
374
|
+
} catch (error) {
|
|
375
|
+
result.errors.push(`Autosubscribe failed: ${error}`);
|
|
376
|
+
return result;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
async function autoSyncPackages(orgId) {
|
|
380
|
+
const packages = await db.package.findMany({
|
|
381
|
+
where: {
|
|
382
|
+
orgId,
|
|
383
|
+
autoSync: true
|
|
384
|
+
}
|
|
385
|
+
});
|
|
386
|
+
console.log(`\u{1F504} Auto-syncing ${packages.length} packages...`);
|
|
387
|
+
for (const pkg of packages) {
|
|
388
|
+
try {
|
|
389
|
+
const shouldUpdate = await checkPackageUpdated(pkg);
|
|
390
|
+
if (shouldUpdate) {
|
|
391
|
+
console.log(`\u{1F4E6} Re-indexing ${pkg.name}...`);
|
|
392
|
+
if (pkg.ecosystem === "npm") {
|
|
393
|
+
await syncNpmPackage({
|
|
394
|
+
packageName: pkg.name,
|
|
395
|
+
version: pkg.version,
|
|
396
|
+
projectId: "",
|
|
397
|
+
// Will need project context
|
|
398
|
+
orgId
|
|
399
|
+
});
|
|
400
|
+
} else if (pkg.ecosystem === "pypi") {
|
|
401
|
+
await syncPyPIPackage({
|
|
402
|
+
packageName: pkg.name,
|
|
403
|
+
version: pkg.version,
|
|
404
|
+
projectId: "",
|
|
405
|
+
orgId
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
await db.package.update({
|
|
409
|
+
where: { id: pkg.id },
|
|
410
|
+
data: { lastIndexedAt: /* @__PURE__ */ new Date() }
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
} catch (error) {
|
|
414
|
+
console.error(`Failed to sync ${pkg.name}:`, error);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
console.log("\u2705 Auto-sync complete");
|
|
418
|
+
}
|
|
419
|
+
async function checkPackageUpdated(pkg) {
|
|
420
|
+
if (!pkg.lastIndexedAt) return true;
|
|
421
|
+
const daysSinceIndex = Math.floor(
|
|
422
|
+
(Date.now() - pkg.lastIndexedAt.getTime()) / (1e3 * 60 * 60 * 24)
|
|
423
|
+
);
|
|
424
|
+
return daysSinceIndex > 7;
|
|
425
|
+
}
|
|
426
|
+
export {
|
|
427
|
+
autoSyncPackages,
|
|
428
|
+
autosubscribe,
|
|
429
|
+
fetchDependencyFile,
|
|
430
|
+
parseCargoToml,
|
|
431
|
+
parseDependencyFile,
|
|
432
|
+
parsePackageJson,
|
|
433
|
+
parseRequirementsTxt,
|
|
434
|
+
resolveDocsUrl
|
|
435
|
+
};
|
|
436
436
|
//# sourceMappingURL=autosubscribe-ISDETQIB.js.map
|