@redstone-md/mapr 0.0.1-alpha → 0.0.2-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -28
- package/index.ts +13 -4
- package/lib/ai-analyzer.ts +44 -155
- package/lib/ai-json.ts +134 -0
- package/lib/analysis-schema.ts +135 -0
- package/lib/artifacts.ts +57 -73
- package/lib/cli-args.ts +16 -2
- package/lib/config.ts +95 -37
- package/lib/formatter.ts +1 -4
- package/lib/provider.ts +307 -14
- package/lib/reporter.ts +1 -1
- package/lib/scraper.ts +232 -18
- package/package.json +1 -1
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
import { artifactTypeSchema } from "./artifacts";
|
|
4
|
+
|
|
5
|
+
export const entryPointSchema = z.object({
|
|
6
|
+
symbol: z.string().min(1),
|
|
7
|
+
description: z.string().min(1),
|
|
8
|
+
evidence: z.string().min(1),
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
export const callGraphEdgeSchema = z.object({
|
|
12
|
+
caller: z.string().min(1),
|
|
13
|
+
callee: z.string().min(1),
|
|
14
|
+
rationale: z.string().min(1),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export const renamedSymbolSchema = z.object({
|
|
18
|
+
originalName: z.string().min(1),
|
|
19
|
+
suggestedName: z.string().min(1),
|
|
20
|
+
justification: z.string().min(1),
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
export const agentMemoSchema = z.object({
|
|
24
|
+
role: z.string().min(1),
|
|
25
|
+
summary: z.string().min(1),
|
|
26
|
+
observations: z.array(z.string().min(1)).default([]),
|
|
27
|
+
evidence: z.array(z.string().min(1)).default([]),
|
|
28
|
+
nextQuestions: z.array(z.string().min(1)).default([]),
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
export const chunkAnalysisSchema = z.object({
|
|
32
|
+
entryPoints: z.array(entryPointSchema).default([]),
|
|
33
|
+
initializationFlow: z.array(z.string().min(1)).default([]),
|
|
34
|
+
callGraph: z.array(callGraphEdgeSchema).default([]),
|
|
35
|
+
restoredNames: z.array(renamedSymbolSchema).default([]),
|
|
36
|
+
summary: z.string().min(1),
|
|
37
|
+
notableLibraries: z.array(z.string().min(1)).default([]),
|
|
38
|
+
investigationTips: z.array(z.string().min(1)).default([]),
|
|
39
|
+
risks: z.array(z.string().min(1)).default([]),
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
export const artifactSummarySchema = z.object({
|
|
43
|
+
url: z.string().url(),
|
|
44
|
+
type: artifactTypeSchema,
|
|
45
|
+
chunkCount: z.number().int().nonnegative(),
|
|
46
|
+
summary: z.string().min(1),
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
export const finalAnalysisSchema = z.object({
|
|
50
|
+
overview: z.string().min(1),
|
|
51
|
+
entryPoints: z.array(entryPointSchema).default([]),
|
|
52
|
+
initializationFlow: z.array(z.string().min(1)).default([]),
|
|
53
|
+
callGraph: z.array(callGraphEdgeSchema).default([]),
|
|
54
|
+
restoredNames: z.array(renamedSymbolSchema).default([]),
|
|
55
|
+
notableLibraries: z.array(z.string().min(1)).default([]),
|
|
56
|
+
investigationTips: z.array(z.string().min(1)).default([]),
|
|
57
|
+
risks: z.array(z.string().min(1)).default([]),
|
|
58
|
+
artifactSummaries: z.array(artifactSummarySchema),
|
|
59
|
+
analyzedChunkCount: z.number().int().nonnegative(),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
export type BundleAnalysis = z.infer<typeof finalAnalysisSchema>;
|
|
63
|
+
export type AgentMemo = z.infer<typeof agentMemoSchema>;
|
|
64
|
+
export type ChunkAnalysis = z.infer<typeof chunkAnalysisSchema>;
|
|
65
|
+
export type ArtifactSummary = z.infer<typeof artifactSummarySchema>;
|
|
66
|
+
|
|
67
|
+
function deduplicate<T>(items: T[], keySelector: (item: T) => string): T[] {
|
|
68
|
+
const seen = new Set<string>();
|
|
69
|
+
const deduplicated: T[] = [];
|
|
70
|
+
|
|
71
|
+
for (const item of items) {
|
|
72
|
+
const key = keySelector(item);
|
|
73
|
+
if (seen.has(key)) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
seen.add(key);
|
|
78
|
+
deduplicated.push(item);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return deduplicated;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function buildAnalysisSnapshot(input: {
|
|
85
|
+
overview: string;
|
|
86
|
+
artifactSummaries?: ArtifactSummary[];
|
|
87
|
+
chunkAnalyses?: ChunkAnalysis[];
|
|
88
|
+
}): BundleAnalysis {
|
|
89
|
+
const artifactSummaries = input.artifactSummaries ?? [];
|
|
90
|
+
const chunkAnalyses = input.chunkAnalyses ?? [];
|
|
91
|
+
|
|
92
|
+
return finalAnalysisSchema.parse({
|
|
93
|
+
overview: input.overview,
|
|
94
|
+
entryPoints: deduplicate(
|
|
95
|
+
chunkAnalyses.flatMap((analysis) => analysis.entryPoints),
|
|
96
|
+
(entryPoint) => `${entryPoint.symbol}:${entryPoint.description}`,
|
|
97
|
+
),
|
|
98
|
+
initializationFlow: deduplicate(
|
|
99
|
+
chunkAnalyses.flatMap((analysis) => analysis.initializationFlow),
|
|
100
|
+
(step) => step,
|
|
101
|
+
),
|
|
102
|
+
callGraph: deduplicate(
|
|
103
|
+
chunkAnalyses.flatMap((analysis) => analysis.callGraph),
|
|
104
|
+
(edge) => `${edge.caller}->${edge.callee}`,
|
|
105
|
+
),
|
|
106
|
+
restoredNames: deduplicate(
|
|
107
|
+
chunkAnalyses.flatMap((analysis) => analysis.restoredNames),
|
|
108
|
+
(entry) => `${entry.originalName}:${entry.suggestedName}`,
|
|
109
|
+
),
|
|
110
|
+
notableLibraries: deduplicate(
|
|
111
|
+
chunkAnalyses.flatMap((analysis) => analysis.notableLibraries),
|
|
112
|
+
(library) => library,
|
|
113
|
+
),
|
|
114
|
+
investigationTips: deduplicate(
|
|
115
|
+
chunkAnalyses.flatMap((analysis) => analysis.investigationTips),
|
|
116
|
+
(tip) => tip,
|
|
117
|
+
),
|
|
118
|
+
risks: deduplicate(
|
|
119
|
+
chunkAnalyses.flatMap((analysis) => analysis.risks),
|
|
120
|
+
(risk) => risk,
|
|
121
|
+
),
|
|
122
|
+
artifactSummaries,
|
|
123
|
+
analyzedChunkCount: chunkAnalyses.length,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class PartialAnalysisError extends Error {
|
|
128
|
+
public readonly partialAnalysis: BundleAnalysis;
|
|
129
|
+
|
|
130
|
+
public constructor(message: string, partialAnalysis: BundleAnalysis) {
|
|
131
|
+
super(message);
|
|
132
|
+
this.name = "PartialAnalysisError";
|
|
133
|
+
this.partialAnalysis = partialAnalysis;
|
|
134
|
+
}
|
|
135
|
+
}
|
package/lib/artifacts.ts
CHANGED
|
@@ -6,9 +6,15 @@ export const artifactTypeSchema = z.enum([
|
|
|
6
6
|
"script",
|
|
7
7
|
"service-worker",
|
|
8
8
|
"worker",
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
|
|
9
|
+
"source-map",
|
|
10
|
+
"wasm",
|
|
11
|
+
]);
|
|
12
|
+
|
|
13
|
+
export const analyzableArtifactTypeSchema = z.enum([
|
|
14
|
+
"script",
|
|
15
|
+
"service-worker",
|
|
16
|
+
"worker",
|
|
17
|
+
"source-map",
|
|
12
18
|
"wasm",
|
|
13
19
|
]);
|
|
14
20
|
|
|
@@ -30,6 +36,11 @@ export type ArtifactType = z.infer<typeof artifactTypeSchema>;
|
|
|
30
36
|
export type DiscoveredArtifact = z.infer<typeof discoveredArtifactSchema>;
|
|
31
37
|
export type ArtifactCandidate = z.infer<typeof artifactCandidateSchema>;
|
|
32
38
|
|
|
39
|
+
const binaryOrVisualAssetPattern =
|
|
40
|
+
/\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|mp4|webm|mov|avi|mp3|wav|ogg|flac|aac|m4a|pdf|zip|gz|tar|7z|rar|woff2?|ttf|otf|eot)(?:$|[?#])/i;
|
|
41
|
+
const ignoredContentTypePattern =
|
|
42
|
+
/^(?:image\/|audio\/|video\/|font\/|application\/(?:font|octet-stream|pdf|zip|gzip|x-font|vnd\.ms-fontobject))/i;
|
|
43
|
+
|
|
33
44
|
function makeCandidate(url: string, type: ArtifactType, discoveredFrom: string): ArtifactCandidate | null {
|
|
34
45
|
const parsed = artifactCandidateSchema.safeParse({ url, type, discoveredFrom });
|
|
35
46
|
return parsed.success ? parsed.data : null;
|
|
@@ -61,6 +72,10 @@ function resolveCandidateUrl(reference: string, baseUrl: string): string | null
|
|
|
61
72
|
|
|
62
73
|
try {
|
|
63
74
|
const absoluteUrl = new URL(reference, baseUrl).toString();
|
|
75
|
+
if (binaryOrVisualAssetPattern.test(new URL(absoluteUrl).pathname)) {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
|
|
64
79
|
const parsed = z.string().url().safeParse(absoluteUrl);
|
|
65
80
|
return parsed.success ? parsed.data : null;
|
|
66
81
|
} catch {
|
|
@@ -75,16 +90,8 @@ function inferAssetTypeFromUrl(url: string, fallback: ArtifactType = "script"):
|
|
|
75
90
|
return "wasm";
|
|
76
91
|
}
|
|
77
92
|
|
|
78
|
-
if (pathname.endsWith(".
|
|
79
|
-
return "
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
if (pathname.endsWith(".json")) {
|
|
83
|
-
return "json";
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
if (pathname.endsWith(".webmanifest") || pathname.endsWith("manifest.json")) {
|
|
87
|
-
return "manifest";
|
|
93
|
+
if (pathname.endsWith(".map")) {
|
|
94
|
+
return "source-map";
|
|
88
95
|
}
|
|
89
96
|
|
|
90
97
|
if (pathname.endsWith(".html") || pathname.endsWith(".htm")) {
|
|
@@ -94,14 +101,34 @@ function inferAssetTypeFromUrl(url: string, fallback: ArtifactType = "script"):
|
|
|
94
101
|
return fallback;
|
|
95
102
|
}
|
|
96
103
|
|
|
104
|
+
function extractPageCandidate(reference: string, pageUrl: string, discoveredFrom: string): ArtifactCandidate | null {
|
|
105
|
+
const resolvedUrl = resolveCandidateUrl(reference, pageUrl);
|
|
106
|
+
if (!resolvedUrl) {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const pathname = new URL(resolvedUrl).pathname.toLowerCase();
|
|
111
|
+
const looksLikePage =
|
|
112
|
+
pathname === "" ||
|
|
113
|
+
pathname.endsWith("/") ||
|
|
114
|
+
pathname.endsWith(".html") ||
|
|
115
|
+
pathname.endsWith(".htm") ||
|
|
116
|
+
!/\.[a-z0-9]+$/i.test(pathname);
|
|
117
|
+
|
|
118
|
+
return looksLikePage ? makeCandidate(resolvedUrl, "html", discoveredFrom) : null;
|
|
119
|
+
}
|
|
120
|
+
|
|
97
121
|
function extractFromJavaScript(source: string, baseUrl: string, discoveredFrom: string): ArtifactCandidate[] {
|
|
98
122
|
const candidates = new Map<string, ArtifactCandidate>();
|
|
99
123
|
const regexDefinitions: Array<{ regex: RegExp; type: ArtifactType }> = [
|
|
100
124
|
{ regex: /(?:import|export)\s+(?:[^"'`]+?\s+from\s+)?["'`]([^"'`]+)["'`]/g, type: "script" },
|
|
101
125
|
{ regex: /import\(\s*["'`]([^"'`]+)["'`]\s*\)/g, type: "script" },
|
|
126
|
+
{ regex: /importScripts\(\s*["'`]([^"'`]+)["'`]\s*\)/g, type: "script" },
|
|
102
127
|
{ regex: /navigator\.serviceWorker\.register\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "service-worker" },
|
|
103
128
|
{ regex: /new\s+(?:SharedWorker|Worker)\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "worker" },
|
|
104
|
-
{ regex: /["'`]([^"'`]
|
|
129
|
+
{ regex: /new\s+URL\(\s*["'`]([^"'`]+)["'`]\s*,\s*import\.meta\.url\s*\)/g, type: "script" },
|
|
130
|
+
{ regex: /["'`]([^"'`]+\.(?:m?js|cjs|wasm|map)(?:\?[^"'`]*)?)["'`]/g, type: "script" },
|
|
131
|
+
{ regex: /[@#]\s*sourceMappingURL=([^\s]+)/g, type: "source-map" },
|
|
105
132
|
];
|
|
106
133
|
|
|
107
134
|
for (const definition of regexDefinitions) {
|
|
@@ -124,31 +151,12 @@ function extractFromJavaScript(source: string, baseUrl: string, discoveredFrom:
|
|
|
124
151
|
return [...candidates.values()];
|
|
125
152
|
}
|
|
126
153
|
|
|
127
|
-
function
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
/@import\s+(?:url\()?["']?([^"'()]+)["']?\)?/g,
|
|
131
|
-
/url\(\s*["']?([^"'()]+)["']?\s*\)/g,
|
|
132
|
-
];
|
|
133
|
-
|
|
134
|
-
for (const regex of regexDefinitions) {
|
|
135
|
-
let match: RegExpExecArray | null;
|
|
136
|
-
while ((match = regex.exec(source)) !== null) {
|
|
137
|
-
const resolvedUrl = resolveCandidateUrl(match[1] ?? "", baseUrl);
|
|
138
|
-
if (!resolvedUrl) {
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
addCandidate(
|
|
143
|
-
candidates,
|
|
144
|
-
makeCandidate(resolvedUrl, inferAssetTypeFromUrl(resolvedUrl, "stylesheet"), discoveredFrom),
|
|
145
|
-
false,
|
|
146
|
-
new URL(baseUrl).origin,
|
|
147
|
-
);
|
|
148
|
-
}
|
|
149
|
-
}
|
|
154
|
+
export function isAnalyzableArtifactType(type: ArtifactType): type is z.infer<typeof analyzableArtifactTypeSchema> {
|
|
155
|
+
return analyzableArtifactTypeSchema.safeParse(type).success;
|
|
156
|
+
}
|
|
150
157
|
|
|
151
|
-
|
|
158
|
+
export function isIgnoredContentType(contentType: string): boolean {
|
|
159
|
+
return ignoredContentTypePattern.test(contentType.trim().toLowerCase());
|
|
152
160
|
}
|
|
153
161
|
|
|
154
162
|
export function extractArtifactCandidates(html: string, pageUrl: string): ArtifactCandidate[] {
|
|
@@ -170,40 +178,15 @@ export function extractArtifactCandidates(html: string, pageUrl: string): Artifa
|
|
|
170
178
|
const rel = ($(element).attr("rel") ?? "").toLowerCase();
|
|
171
179
|
const asValue = ($(element).attr("as") ?? "").toLowerCase();
|
|
172
180
|
|
|
173
|
-
if (rel.includes("manifest")) {
|
|
174
|
-
addCandidate(candidates, makeCandidate(href, "manifest", "html:manifest"), false, origin);
|
|
175
|
-
return;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
if (rel.includes("stylesheet")) {
|
|
179
|
-
addCandidate(candidates, makeCandidate(href, "stylesheet", "html:stylesheet"), false, origin);
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
181
|
if (rel.includes("modulepreload") || (rel.includes("preload") && asValue === "script")) {
|
|
184
|
-
addCandidate(candidates, makeCandidate(href, "script", "html:preload"), false, origin);
|
|
182
|
+
addCandidate(candidates, makeCandidate(href, inferAssetTypeFromUrl(href, "script"), "html:preload"), false, origin);
|
|
185
183
|
}
|
|
186
184
|
});
|
|
187
185
|
|
|
188
|
-
$("a[href]").each((_, element) => {
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
const url = new URL(href);
|
|
195
|
-
const isSameOrigin = url.origin === origin;
|
|
196
|
-
const pathname = url.pathname.toLowerCase();
|
|
197
|
-
const looksLikePage =
|
|
198
|
-
pathname === "" ||
|
|
199
|
-
pathname.endsWith("/") ||
|
|
200
|
-
pathname.endsWith(".html") ||
|
|
201
|
-
pathname.endsWith(".htm") ||
|
|
202
|
-
!/\.[a-z0-9]+$/i.test(pathname);
|
|
203
|
-
|
|
204
|
-
if (isSameOrigin && looksLikePage) {
|
|
205
|
-
addCandidate(candidates, makeCandidate(href, "html", "html:anchor"), true, origin);
|
|
206
|
-
}
|
|
186
|
+
$("a[href], iframe[src], form[action]").each((_, element) => {
|
|
187
|
+
const attributeName = element.tagName === "iframe" ? "src" : element.tagName === "form" ? "action" : "href";
|
|
188
|
+
const pageCandidate = extractPageCandidate($(element).attr(attributeName)?.trim() ?? "", pageUrl, `html:${element.tagName}`);
|
|
189
|
+
addCandidate(candidates, pageCandidate, true, origin);
|
|
207
190
|
});
|
|
208
191
|
|
|
209
192
|
$("script:not([src])").each((_, element) => {
|
|
@@ -221,13 +204,14 @@ export function extractNestedCandidates(artifact: DiscoveredArtifact): ArtifactC
|
|
|
221
204
|
return extractArtifactCandidates(artifact.content, artifact.url);
|
|
222
205
|
}
|
|
223
206
|
|
|
224
|
-
if (
|
|
207
|
+
if (
|
|
208
|
+
artifact.type === "script" ||
|
|
209
|
+
artifact.type === "service-worker" ||
|
|
210
|
+
artifact.type === "worker" ||
|
|
211
|
+
artifact.type === "source-map"
|
|
212
|
+
) {
|
|
225
213
|
return extractFromJavaScript(artifact.content, artifact.url, `${artifact.type}:code`);
|
|
226
214
|
}
|
|
227
215
|
|
|
228
|
-
if (artifact.type === "stylesheet") {
|
|
229
|
-
return extractFromCss(artifact.content, artifact.url, "stylesheet:code");
|
|
230
|
-
}
|
|
231
|
-
|
|
232
216
|
return [];
|
|
233
217
|
}
|
package/lib/cli-args.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
import { providerTypeSchema } from "./provider";
|
|
3
|
+
import { getProviderPreset, providerPresetSchema, providerTypeSchema } from "./provider";
|
|
4
4
|
|
|
5
5
|
const rawCliArgsSchema = z.object({
|
|
6
6
|
help: z.boolean().default(false),
|
|
@@ -13,6 +13,7 @@ const rawCliArgsSchema = z.object({
|
|
|
13
13
|
url: z.string().url().optional(),
|
|
14
14
|
output: z.string().min(1).optional(),
|
|
15
15
|
providerType: providerTypeSchema.optional(),
|
|
16
|
+
providerPreset: providerPresetSchema.optional(),
|
|
16
17
|
providerName: z.string().min(1).optional(),
|
|
17
18
|
apiKey: z.string().min(1).optional(),
|
|
18
19
|
baseURL: z.string().url().optional(),
|
|
@@ -20,11 +21,13 @@ const rawCliArgsSchema = z.object({
|
|
|
20
21
|
contextSize: z.number().int().positive().optional(),
|
|
21
22
|
maxPages: z.number().int().positive().optional(),
|
|
22
23
|
maxArtifacts: z.number().int().positive().optional(),
|
|
24
|
+
maxDepth: z.number().int().nonnegative().optional(),
|
|
23
25
|
});
|
|
24
26
|
|
|
25
27
|
const cliConfigOverrideSchema = z
|
|
26
28
|
.object({
|
|
27
29
|
providerType: providerTypeSchema.optional(),
|
|
30
|
+
providerPreset: providerPresetSchema.optional(),
|
|
28
31
|
providerName: z.string().min(1).optional(),
|
|
29
32
|
apiKey: z.string().min(1).optional(),
|
|
30
33
|
baseURL: z.string().url().optional(),
|
|
@@ -49,6 +52,7 @@ const optionMap = new Map<string, keyof CliArgs>([
|
|
|
49
52
|
["-u", "url"],
|
|
50
53
|
["--output", "output"],
|
|
51
54
|
["--provider-type", "providerType"],
|
|
55
|
+
["--provider-preset", "providerPreset"],
|
|
52
56
|
["--provider-name", "providerName"],
|
|
53
57
|
["--api-key", "apiKey"],
|
|
54
58
|
["--base-url", "baseURL"],
|
|
@@ -56,10 +60,11 @@ const optionMap = new Map<string, keyof CliArgs>([
|
|
|
56
60
|
["--context-size", "contextSize"],
|
|
57
61
|
["--max-pages", "maxPages"],
|
|
58
62
|
["--max-artifacts", "maxArtifacts"],
|
|
63
|
+
["--max-depth", "maxDepth"],
|
|
59
64
|
]);
|
|
60
65
|
|
|
61
66
|
const booleanKeys = new Set<keyof CliArgs>(["help", "version", "headless", "reconfigure", "listModels", "localRag", "verboseAgents"]);
|
|
62
|
-
const numberKeys = new Set<keyof CliArgs>(["contextSize", "maxPages", "maxArtifacts"]);
|
|
67
|
+
const numberKeys = new Set<keyof CliArgs>(["contextSize", "maxPages", "maxArtifacts", "maxDepth"]);
|
|
63
68
|
|
|
64
69
|
function normalizeValue(key: keyof CliArgs, value: string): unknown {
|
|
65
70
|
if (numberKeys.has(key)) {
|
|
@@ -110,6 +115,13 @@ export function getConfigOverrides(args: CliArgs) {
|
|
|
110
115
|
const overrides: Record<string, unknown> = {};
|
|
111
116
|
|
|
112
117
|
if (args.providerType !== undefined) overrides.providerType = args.providerType;
|
|
118
|
+
if (args.providerPreset !== undefined) {
|
|
119
|
+
const preset = getProviderPreset(args.providerPreset);
|
|
120
|
+
overrides.providerType = "openai-compatible";
|
|
121
|
+
overrides.providerPreset = args.providerPreset;
|
|
122
|
+
overrides.providerName = args.providerName ?? preset.providerName;
|
|
123
|
+
overrides.baseURL = args.baseURL ?? preset.baseURL;
|
|
124
|
+
}
|
|
113
125
|
if (args.providerName !== undefined) overrides.providerName = args.providerName;
|
|
114
126
|
if (args.apiKey !== undefined) overrides.apiKey = args.apiKey;
|
|
115
127
|
if (args.baseURL !== undefined) overrides.baseURL = args.baseURL;
|
|
@@ -132,9 +144,11 @@ export function renderHelpText(): string {
|
|
|
132
144
|
" --output <path> Write the report to a specific path",
|
|
133
145
|
" --max-pages <number> Limit same-origin HTML pages to crawl",
|
|
134
146
|
" --max-artifacts <number> Limit total downloaded artifacts",
|
|
147
|
+
" --max-depth <number> Limit crawl hop depth from the entry page",
|
|
135
148
|
"",
|
|
136
149
|
"Provider options:",
|
|
137
150
|
" --provider-type <type> openai | openai-compatible",
|
|
151
|
+
" --provider-preset <preset> custom | blackbox | nvidia-nim | onlysq",
|
|
138
152
|
" --provider-name <name> Display name for the provider",
|
|
139
153
|
" --api-key <key> Provider API key",
|
|
140
154
|
" --base-url <url> Base URL for the provider",
|
package/lib/config.ts
CHANGED
|
@@ -10,12 +10,17 @@ import {
|
|
|
10
10
|
DEFAULT_MODEL_CONTEXT_SIZE,
|
|
11
11
|
DEFAULT_OPENAI_BASE_URL,
|
|
12
12
|
aiProviderConfigSchema,
|
|
13
|
+
getOpenAiCompatibleProviderPresets,
|
|
14
|
+
getProviderPreset,
|
|
15
|
+
inferProviderPreset,
|
|
13
16
|
type AiProviderConfig,
|
|
17
|
+
type ProviderModelInfo,
|
|
14
18
|
} from "./provider";
|
|
15
19
|
|
|
16
20
|
const persistedConfigSchema = z
|
|
17
21
|
.object({
|
|
18
22
|
providerType: z.enum(["openai", "openai-compatible"]).optional(),
|
|
23
|
+
providerPreset: z.enum(["custom", "blackbox", "nvidia-nim", "onlysq"]).optional(),
|
|
19
24
|
providerName: z.string().min(1).optional(),
|
|
20
25
|
apiKey: z.string().min(1).optional(),
|
|
21
26
|
openAiApiKey: z.string().min(1).optional(),
|
|
@@ -28,6 +33,7 @@ const persistedConfigSchema = z
|
|
|
28
33
|
const configDraftSchema = aiProviderConfigSchema.partial();
|
|
29
34
|
const modelListingConfigSchema = z.object({
|
|
30
35
|
providerType: z.enum(["openai", "openai-compatible"]).default("openai"),
|
|
36
|
+
providerPreset: z.enum(["custom", "blackbox", "nvidia-nim", "onlysq"]).optional(),
|
|
31
37
|
providerName: z.string().min(1).default("OpenAI"),
|
|
32
38
|
apiKey: z.string().min(1),
|
|
33
39
|
baseURL: z.string().trim().url().default(DEFAULT_OPENAI_BASE_URL),
|
|
@@ -76,21 +82,33 @@ function normalizePersistedConfig(config: PersistedConfig | null): ConfigDraft |
|
|
|
76
82
|
return null;
|
|
77
83
|
}
|
|
78
84
|
|
|
85
|
+
const providerType = config.providerType ?? "openai";
|
|
86
|
+
const baseURL = config.baseURL ?? DEFAULT_OPENAI_BASE_URL;
|
|
87
|
+
|
|
79
88
|
return configDraftSchema.parse({
|
|
80
|
-
providerType
|
|
89
|
+
providerType,
|
|
90
|
+
providerPreset:
|
|
91
|
+
config.providerPreset ??
|
|
92
|
+
(config.providerType === "openai-compatible" || providerType === "openai-compatible"
|
|
93
|
+
? inferProviderPreset(baseURL, providerType)
|
|
94
|
+
: undefined),
|
|
81
95
|
providerName: config.providerName ?? "OpenAI",
|
|
82
96
|
apiKey: config.apiKey ?? config.openAiApiKey,
|
|
83
|
-
baseURL
|
|
97
|
+
baseURL,
|
|
84
98
|
model: config.model ?? DEFAULT_MODEL,
|
|
85
99
|
modelContextSize: config.modelContextSize ?? DEFAULT_MODEL_CONTEXT_SIZE,
|
|
86
100
|
});
|
|
87
101
|
}
|
|
88
102
|
|
|
103
|
+
function formatModelOptionLabel(model: ProviderModelInfo): string {
|
|
104
|
+
return model.contextSize ? `${model.id} (${model.contextSize.toLocaleString()} tokens)` : model.id;
|
|
105
|
+
}
|
|
106
|
+
|
|
89
107
|
async function promptForModel(
|
|
90
108
|
config: Omit<AiProviderConfig, "model" | "modelContextSize">,
|
|
91
109
|
fetcher: FetchLike,
|
|
92
110
|
initialModel?: string,
|
|
93
|
-
): Promise<
|
|
111
|
+
): Promise<ProviderModelInfo> {
|
|
94
112
|
const providerClient = new AiProviderClient({
|
|
95
113
|
...config,
|
|
96
114
|
model: initialModel ?? DEFAULT_MODEL,
|
|
@@ -98,8 +116,8 @@ async function promptForModel(
|
|
|
98
116
|
});
|
|
99
117
|
|
|
100
118
|
try {
|
|
101
|
-
const
|
|
102
|
-
if (
|
|
119
|
+
const catalog = await providerClient.fetchModelCatalog(fetcher);
|
|
120
|
+
if (catalog.length === 0) {
|
|
103
121
|
throw new Error("No models returned by the provider.");
|
|
104
122
|
}
|
|
105
123
|
|
|
@@ -117,8 +135,8 @@ async function promptForModel(
|
|
|
117
135
|
).trim();
|
|
118
136
|
|
|
119
137
|
currentSearch = searchTerm;
|
|
120
|
-
const filteredModels =
|
|
121
|
-
searchTerm.length === 0 ? true : model.toLowerCase().includes(searchTerm.toLowerCase()),
|
|
138
|
+
const filteredModels = catalog.filter((model) =>
|
|
139
|
+
searchTerm.length === 0 ? true : model.id.toLowerCase().includes(searchTerm.toLowerCase()),
|
|
122
140
|
);
|
|
123
141
|
const visibleModels = filteredModels.slice(0, 15);
|
|
124
142
|
|
|
@@ -126,7 +144,7 @@ async function promptForModel(
|
|
|
126
144
|
await select({
|
|
127
145
|
message: "Select model",
|
|
128
146
|
options: [
|
|
129
|
-
...visibleModels.map((model) => ({ value: model, label: model })),
|
|
147
|
+
...visibleModels.map((model) => ({ value: model.id, label: formatModelOptionLabel(model) })),
|
|
130
148
|
{ value: "__search_again__", label: "Search again" },
|
|
131
149
|
{ value: "__manual__", label: "Enter model manually" },
|
|
132
150
|
],
|
|
@@ -141,10 +159,35 @@ async function promptForModel(
|
|
|
141
159
|
break;
|
|
142
160
|
}
|
|
143
161
|
|
|
144
|
-
|
|
162
|
+
const resolvedModel = catalog.find((model) => model.id === selectedModel);
|
|
163
|
+
if (resolvedModel) {
|
|
164
|
+
return resolvedModel;
|
|
165
|
+
}
|
|
145
166
|
}
|
|
146
167
|
} catch {
|
|
147
|
-
return
|
|
168
|
+
return {
|
|
169
|
+
id: z
|
|
170
|
+
.string()
|
|
171
|
+
.trim()
|
|
172
|
+
.min(1, "Model is required.")
|
|
173
|
+
.parse(
|
|
174
|
+
exitIfCancelled(
|
|
175
|
+
await text({
|
|
176
|
+
message: "Enter model ID manually",
|
|
177
|
+
placeholder: DEFAULT_MODEL,
|
|
178
|
+
initialValue: initialModel ?? DEFAULT_MODEL,
|
|
179
|
+
validate(value) {
|
|
180
|
+
const parsed = z.string().trim().min(1).safeParse(value);
|
|
181
|
+
return parsed.success ? undefined : "Model is required.";
|
|
182
|
+
},
|
|
183
|
+
}),
|
|
184
|
+
),
|
|
185
|
+
),
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return {
|
|
190
|
+
id: z
|
|
148
191
|
.string()
|
|
149
192
|
.trim()
|
|
150
193
|
.min(1, "Model is required.")
|
|
@@ -160,26 +203,8 @@ async function promptForModel(
|
|
|
160
203
|
},
|
|
161
204
|
}),
|
|
162
205
|
),
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
return z
|
|
167
|
-
.string()
|
|
168
|
-
.trim()
|
|
169
|
-
.min(1, "Model is required.")
|
|
170
|
-
.parse(
|
|
171
|
-
exitIfCancelled(
|
|
172
|
-
await text({
|
|
173
|
-
message: "Enter model ID manually",
|
|
174
|
-
placeholder: DEFAULT_MODEL,
|
|
175
|
-
initialValue: initialModel ?? DEFAULT_MODEL,
|
|
176
|
-
validate(value) {
|
|
177
|
-
const parsed = z.string().trim().min(1).safeParse(value);
|
|
178
|
-
return parsed.success ? undefined : "Model is required.";
|
|
179
|
-
},
|
|
180
|
-
}),
|
|
181
206
|
),
|
|
182
|
-
|
|
207
|
+
};
|
|
183
208
|
}
|
|
184
209
|
|
|
185
210
|
async function promptForContextSize(defaultValue: number): Promise<number> {
|
|
@@ -285,13 +310,18 @@ export class ConfigManager {
|
|
|
285
310
|
}
|
|
286
311
|
|
|
287
312
|
public async listModels(config: ConfigDraft | null): Promise<string[]> {
|
|
313
|
+
const catalog = await this.listModelCatalog(config);
|
|
314
|
+
return catalog.map((entry) => entry.id);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
public async listModelCatalog(config: ConfigDraft | null): Promise<ProviderModelInfo[]> {
|
|
288
318
|
const resolvedConfig = modelListingConfigSchema.parse(config);
|
|
289
319
|
const providerClient = new AiProviderClient({
|
|
290
320
|
...resolvedConfig,
|
|
291
321
|
model: DEFAULT_MODEL,
|
|
292
322
|
modelContextSize: DEFAULT_MODEL_CONTEXT_SIZE,
|
|
293
323
|
});
|
|
294
|
-
return providerClient.
|
|
324
|
+
return providerClient.fetchModelCatalog(this.fetcher);
|
|
295
325
|
}
|
|
296
326
|
|
|
297
327
|
public async resolveConfigDraft(overrides: ConfigDraft | null): Promise<ConfigDraft | null> {
|
|
@@ -311,6 +341,29 @@ export class ConfigManager {
|
|
|
311
341
|
}),
|
|
312
342
|
) as AiProviderConfig["providerType"];
|
|
313
343
|
|
|
344
|
+
const existingPreset =
|
|
345
|
+
existingConfig?.providerPreset ??
|
|
346
|
+
(existingConfig?.baseURL && existingConfig.providerType === "openai-compatible"
|
|
347
|
+
? inferProviderPreset(existingConfig.baseURL, "openai-compatible")
|
|
348
|
+
: undefined);
|
|
349
|
+
|
|
350
|
+
const providerPreset =
|
|
351
|
+
providerType === "openai"
|
|
352
|
+
? undefined
|
|
353
|
+
: (exitIfCancelled(
|
|
354
|
+
await select({
|
|
355
|
+
message: "Choose provider preset",
|
|
356
|
+
initialValue: existingPreset ?? "custom",
|
|
357
|
+
options: getOpenAiCompatibleProviderPresets().map((preset) => ({
|
|
358
|
+
value: preset.id,
|
|
359
|
+
label: preset.label,
|
|
360
|
+
})),
|
|
361
|
+
}),
|
|
362
|
+
) as NonNullable<AiProviderConfig["providerPreset"]>);
|
|
363
|
+
|
|
364
|
+
const presetDefinition =
|
|
365
|
+
providerType === "openai" ? undefined : getProviderPreset(providerPreset ?? "custom");
|
|
366
|
+
|
|
314
367
|
const providerName =
|
|
315
368
|
providerType === "openai"
|
|
316
369
|
? "OpenAI"
|
|
@@ -319,9 +372,12 @@ export class ConfigManager {
|
|
|
319
372
|
await text({
|
|
320
373
|
message: "Provider display name",
|
|
321
374
|
placeholder: "Local vLLM, LM Studio, Ollama gateway",
|
|
322
|
-
initialValue:
|
|
323
|
-
|
|
324
|
-
|
|
375
|
+
initialValue:
|
|
376
|
+
existingConfig?.providerName &&
|
|
377
|
+
existingConfig.providerType === "openai-compatible" &&
|
|
378
|
+
existingPreset === providerPreset
|
|
379
|
+
? existingConfig.providerName
|
|
380
|
+
: presetDefinition?.providerName ?? "",
|
|
325
381
|
validate(value) {
|
|
326
382
|
const parsed = z.string().trim().min(1).safeParse(value);
|
|
327
383
|
return parsed.success ? undefined : "Provider name is required.";
|
|
@@ -336,9 +392,9 @@ export class ConfigManager {
|
|
|
336
392
|
message: providerType === "openai" ? "OpenAI base URL" : "OpenAI-compatible base URL",
|
|
337
393
|
placeholder: DEFAULT_OPENAI_BASE_URL,
|
|
338
394
|
initialValue:
|
|
339
|
-
existingConfig?.baseURL && existingConfig.providerType === providerType
|
|
395
|
+
existingConfig?.baseURL && existingConfig.providerType === providerType && existingPreset === providerPreset
|
|
340
396
|
? existingConfig.baseURL
|
|
341
|
-
: DEFAULT_OPENAI_BASE_URL,
|
|
397
|
+
: presetDefinition?.baseURL ?? DEFAULT_OPENAI_BASE_URL,
|
|
342
398
|
validate(value) {
|
|
343
399
|
const parsed = z.string().trim().url().safeParse(value);
|
|
344
400
|
return parsed.success ? undefined : "Base URL must be a valid URL.";
|
|
@@ -363,6 +419,7 @@ export class ConfigManager {
|
|
|
363
419
|
const model = await promptForModel(
|
|
364
420
|
{
|
|
365
421
|
providerType,
|
|
422
|
+
...(providerPreset !== undefined ? { providerPreset } : {}),
|
|
366
423
|
providerName,
|
|
367
424
|
baseURL,
|
|
368
425
|
apiKey,
|
|
@@ -371,14 +428,15 @@ export class ConfigManager {
|
|
|
371
428
|
existingConfig?.providerType === providerType ? existingConfig.model : undefined,
|
|
372
429
|
);
|
|
373
430
|
|
|
374
|
-
const modelContextSize = await promptForContextSize(existingConfig?.modelContextSize ?? DEFAULT_MODEL_CONTEXT_SIZE);
|
|
431
|
+
const modelContextSize = model.contextSize ?? (await promptForContextSize(existingConfig?.modelContextSize ?? DEFAULT_MODEL_CONTEXT_SIZE));
|
|
375
432
|
|
|
376
433
|
return aiProviderConfigSchema.parse({
|
|
377
434
|
providerType,
|
|
435
|
+
...(providerPreset !== undefined ? { providerPreset } : {}),
|
|
378
436
|
providerName,
|
|
379
437
|
baseURL,
|
|
380
438
|
apiKey,
|
|
381
|
-
model,
|
|
439
|
+
model: model.id,
|
|
382
440
|
modelContextSize,
|
|
383
441
|
});
|
|
384
442
|
}
|
package/lib/formatter.ts
CHANGED
|
@@ -20,10 +20,7 @@ function resolvePrettierParser(artifactType: FormattedArtifact["type"]): "babel"
|
|
|
20
20
|
switch (artifactType) {
|
|
21
21
|
case "html":
|
|
22
22
|
return "html";
|
|
23
|
-
case "
|
|
24
|
-
return "css";
|
|
25
|
-
case "manifest":
|
|
26
|
-
case "json":
|
|
23
|
+
case "source-map":
|
|
27
24
|
return "json";
|
|
28
25
|
case "script":
|
|
29
26
|
case "service-worker":
|