@pi-unipi/web-api 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -15
- package/package.json +9 -2
- package/skills/web/SKILL.md +54 -11
- package/src/engine/constants.ts +36 -0
- package/src/engine/dependencies.ts +145 -0
- package/src/engine/dom.ts +266 -0
- package/src/engine/extract.ts +642 -0
- package/src/engine/format.ts +306 -0
- package/src/engine/profiles.ts +102 -0
- package/src/engine/types.ts +169 -0
- package/src/index.ts +9 -2
- package/src/providers/base.ts +9 -1
- package/src/settings.ts +70 -4
- package/src/tools.ts +281 -24
- package/src/tui/progress.ts +168 -0
- package/src/tui/result.ts +173 -0
- package/src/tui/settings-dialog.ts +168 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @unipi/web-api — TUI Result Renderer
|
|
3
|
+
*
|
|
4
|
+
* Renders single and batch results for TUI display.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { FetchResult, BatchFetchResult, FetchError } from "../engine/types.js";
|
|
8
|
+
|
|
9
|
+
/** Maximum preview lines */
|
|
10
|
+
const PREVIEW_LINES = 7;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Truncate content for preview.
|
|
14
|
+
*
|
|
15
|
+
* @param content - Content to preview
|
|
16
|
+
* @param maxLines - Maximum lines
|
|
17
|
+
* @returns Preview string
|
|
18
|
+
*/
|
|
19
|
+
function truncatePreview(content: string, maxLines: number = PREVIEW_LINES): string {
|
|
20
|
+
const lines = content.split("\n").slice(0, maxLines);
|
|
21
|
+
return lines.join("\n");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Render a single result for display.
|
|
26
|
+
*
|
|
27
|
+
* @param result - Fetch result
|
|
28
|
+
* @param verbose - Include metadata header
|
|
29
|
+
* @returns Formatted string
|
|
30
|
+
*/
|
|
31
|
+
export function renderSingleResult(
|
|
32
|
+
result: FetchResult,
|
|
33
|
+
verbose: boolean = true
|
|
34
|
+
): string {
|
|
35
|
+
const lines: string[] = [];
|
|
36
|
+
|
|
37
|
+
if (verbose) {
|
|
38
|
+
// Title
|
|
39
|
+
lines.push(`# ${result.title || "Untitled"}`);
|
|
40
|
+
lines.push("");
|
|
41
|
+
|
|
42
|
+
// Metadata
|
|
43
|
+
const meta: string[] = [];
|
|
44
|
+
if (result.author) {
|
|
45
|
+
meta.push(`Author: ${result.author}`);
|
|
46
|
+
}
|
|
47
|
+
if (result.published) {
|
|
48
|
+
meta.push(`Published: ${result.published}`);
|
|
49
|
+
}
|
|
50
|
+
if (result.site) {
|
|
51
|
+
meta.push(`Site: ${result.site}`);
|
|
52
|
+
}
|
|
53
|
+
if (result.language) {
|
|
54
|
+
meta.push(`Language: ${result.language}`);
|
|
55
|
+
}
|
|
56
|
+
if (result.wordCount) {
|
|
57
|
+
meta.push(`Words: ${result.wordCount}`);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (meta.length > 0) {
|
|
61
|
+
lines.push(meta.join(" · "));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// URL
|
|
65
|
+
lines.push(`URL: ${result.url}`);
|
|
66
|
+
if (result.finalUrl !== result.url) {
|
|
67
|
+
lines.push(`Final URL: ${result.finalUrl}`);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
lines.push("");
|
|
71
|
+
lines.push("---");
|
|
72
|
+
lines.push("");
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Content preview
|
|
76
|
+
const preview = truncatePreview(result.content);
|
|
77
|
+
lines.push(preview);
|
|
78
|
+
|
|
79
|
+
// Expand hint
|
|
80
|
+
if (result.content.split("\n").length > PREVIEW_LINES) {
|
|
81
|
+
lines.push("");
|
|
82
|
+
lines.push(`... [${result.wordCount} words total · Ctrl+O to expand]`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return lines.join("\n");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Render a batch result for display.
|
|
90
|
+
*
|
|
91
|
+
* @param result - Batch fetch result
|
|
92
|
+
* @returns Formatted string
|
|
93
|
+
*/
|
|
94
|
+
export function renderBatchResult(result: BatchFetchResult): string {
|
|
95
|
+
const lines: string[] = [];
|
|
96
|
+
|
|
97
|
+
// Summary header
|
|
98
|
+
lines.push(`# Batch Read Results`);
|
|
99
|
+
lines.push("");
|
|
100
|
+
lines.push(
|
|
101
|
+
`Total: ${result.total} · Succeeded: ${result.succeeded} · Failed: ${result.failed}`
|
|
102
|
+
);
|
|
103
|
+
lines.push("");
|
|
104
|
+
|
|
105
|
+
// Per-item results
|
|
106
|
+
for (let i = 0; i < result.items.length; i++) {
|
|
107
|
+
const item = result.items[i];
|
|
108
|
+
const status = item.status === "done" ? "✓" : "✗";
|
|
109
|
+
|
|
110
|
+
lines.push(`## [${i + 1}/${result.total}] ${status}`);
|
|
111
|
+
|
|
112
|
+
if (item.status === "done") {
|
|
113
|
+
lines.push(`**${item.result.title}**`);
|
|
114
|
+
lines.push(`URL: ${item.result.url}`);
|
|
115
|
+
lines.push(`Words: ${item.result.wordCount}`);
|
|
116
|
+
lines.push("");
|
|
117
|
+
|
|
118
|
+
// Content preview
|
|
119
|
+
const preview = truncatePreview(item.result.content);
|
|
120
|
+
lines.push(preview);
|
|
121
|
+
|
|
122
|
+
if (item.result.content.split("\n").length > PREVIEW_LINES) {
|
|
123
|
+
lines.push("...");
|
|
124
|
+
}
|
|
125
|
+
} else {
|
|
126
|
+
lines.push(`URL: ${item.error.url || "unknown"}`);
|
|
127
|
+
lines.push(`Error: ${item.error.error}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
lines.push("");
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return lines.join("\n");
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Render an error result for display.
|
|
138
|
+
*
|
|
139
|
+
* @param error - Fetch error
|
|
140
|
+
* @returns Formatted string
|
|
141
|
+
*/
|
|
142
|
+
export function renderErrorResult(error: FetchError): string {
|
|
143
|
+
const lines: string[] = [];
|
|
144
|
+
|
|
145
|
+
lines.push(`# Fetch Error`);
|
|
146
|
+
lines.push("");
|
|
147
|
+
lines.push(`**${error.error}**`);
|
|
148
|
+
lines.push("");
|
|
149
|
+
lines.push(`Code: \`${error.code}\``);
|
|
150
|
+
lines.push(`Phase: \`${error.phase}\``);
|
|
151
|
+
|
|
152
|
+
if (error.url) {
|
|
153
|
+
lines.push("");
|
|
154
|
+
lines.push(`URL: ${error.url}`);
|
|
155
|
+
if (error.finalUrl && error.finalUrl !== error.url) {
|
|
156
|
+
lines.push(`Final URL: ${error.finalUrl}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (error.statusCode) {
|
|
161
|
+
lines.push("");
|
|
162
|
+
lines.push(
|
|
163
|
+
`HTTP Status: ${error.statusCode}${error.statusText ? ` ${error.statusText}` : ""}`
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (error.retryable) {
|
|
168
|
+
lines.push("");
|
|
169
|
+
lines.push(`*This error may be retried.*`);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return lines.join("\n");
|
|
173
|
+
}
|
|
@@ -14,7 +14,11 @@ import {
|
|
|
14
14
|
isProviderEnabled,
|
|
15
15
|
setProviderEnabled,
|
|
16
16
|
validateApiKeyFormat,
|
|
17
|
+
loadSmartFetchSettings,
|
|
18
|
+
saveSmartFetchSettings,
|
|
19
|
+
resetSmartFetchSettings,
|
|
17
20
|
} from "../settings.js";
|
|
21
|
+
import { BROWSER_PROFILES, OS_PROFILES } from "../engine/profiles.js";
|
|
18
22
|
import { getProviderOptions, getProviderStatuses } from "./provider-selector.js";
|
|
19
23
|
|
|
20
24
|
/**
|
|
@@ -28,6 +32,13 @@ export async function showSettingsDialog(ctx: ExtensionCommandContext): Promise<
|
|
|
28
32
|
while (running) {
|
|
29
33
|
const options = getProviderOptions();
|
|
30
34
|
|
|
35
|
+
// Add smart-fetch defaults option at the top
|
|
36
|
+
options.unshift({
|
|
37
|
+
label: "⚡ Smart Fetch Defaults",
|
|
38
|
+
value: "__smart_fetch__",
|
|
39
|
+
description: "Configure default browser, OS, and fetch settings",
|
|
40
|
+
});
|
|
41
|
+
|
|
31
42
|
// Add exit option
|
|
32
43
|
options.push({
|
|
33
44
|
label: "← Back",
|
|
@@ -64,6 +75,12 @@ export async function showSettingsDialog(ctx: ExtensionCommandContext): Promise<
|
|
|
64
75
|
|
|
65
76
|
lastSelected = selectedValue;
|
|
66
77
|
|
|
78
|
+
// Handle smart-fetch defaults
|
|
79
|
+
if (selectedValue === "__smart_fetch__") {
|
|
80
|
+
await configureSmartFetch(ctx);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
|
|
67
84
|
// Show provider configuration
|
|
68
85
|
await configureProvider(ctx, selectedValue);
|
|
69
86
|
}
|
|
@@ -190,3 +207,154 @@ async function inputApiKey(
|
|
|
190
207
|
);
|
|
191
208
|
}
|
|
192
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Configure smart-fetch defaults.
|
|
213
|
+
*/
|
|
214
|
+
async function configureSmartFetch(
|
|
215
|
+
ctx: ExtensionCommandContext
|
|
216
|
+
): Promise<void> {
|
|
217
|
+
const settings = loadSmartFetchSettings();
|
|
218
|
+
|
|
219
|
+
const options = [
|
|
220
|
+
{
|
|
221
|
+
label: `Browser: ${settings.browser}`,
|
|
222
|
+
value: "browser",
|
|
223
|
+
description: "TLS fingerprint browser profile",
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
label: `OS: ${settings.os}`,
|
|
227
|
+
value: "os",
|
|
228
|
+
description: "OS fingerprint",
|
|
229
|
+
},
|
|
230
|
+
{
|
|
231
|
+
label: `Max Chars: ${settings.maxChars.toLocaleString()}`,
|
|
232
|
+
value: "maxChars",
|
|
233
|
+
description: "Maximum content characters",
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
label: `Timeout: ${settings.timeoutMs}ms`,
|
|
237
|
+
value: "timeoutMs",
|
|
238
|
+
description: "Request timeout",
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
label: `Concurrency: ${settings.batchConcurrency}`,
|
|
242
|
+
value: "batchConcurrency",
|
|
243
|
+
description: "Batch concurrent requests",
|
|
244
|
+
},
|
|
245
|
+
{
|
|
246
|
+
label: `Remove Images: ${settings.removeImages ? "Yes" : "No"}`,
|
|
247
|
+
value: "removeImages",
|
|
248
|
+
description: "Strip image references",
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
label: `Include Replies: ${settings.includeReplies}`,
|
|
252
|
+
value: "includeReplies",
|
|
253
|
+
description: "Include comments/replies",
|
|
254
|
+
},
|
|
255
|
+
{
|
|
256
|
+
label: "Reset to Defaults",
|
|
257
|
+
value: "__reset__",
|
|
258
|
+
description: "Reset all settings to defaults",
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
label: "← Back",
|
|
262
|
+
value: "__back__",
|
|
263
|
+
description: "Return to main menu",
|
|
264
|
+
},
|
|
265
|
+
];
|
|
266
|
+
|
|
267
|
+
const labels = options.map(o => `${o.label} — ${o.description}`);
|
|
268
|
+
const selected = await ctx.ui.select("Smart Fetch Defaults", labels);
|
|
269
|
+
const selectedOpt = options.find(o => `${o.label} — ${o.description}` === selected);
|
|
270
|
+
const selectedValue = selectedOpt?.value;
|
|
271
|
+
|
|
272
|
+
switch (selectedValue) {
|
|
273
|
+
case "browser": {
|
|
274
|
+
const browserOptions = [...BROWSER_PROFILES].reverse(); // Show newest first
|
|
275
|
+
const browser = await ctx.ui.select("Browser Profile", browserOptions);
|
|
276
|
+
if (browser) {
|
|
277
|
+
saveSmartFetchSettings({ browser });
|
|
278
|
+
ctx.ui.notify(`Browser set to ${browser}`, "info");
|
|
279
|
+
}
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
case "os": {
|
|
284
|
+
const os = await ctx.ui.select("OS Fingerprint", [...OS_PROFILES]);
|
|
285
|
+
if (os) {
|
|
286
|
+
saveSmartFetchSettings({ os });
|
|
287
|
+
ctx.ui.notify(`OS set to ${os}`, "info");
|
|
288
|
+
}
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
case "maxChars": {
|
|
293
|
+
const input = await ctx.ui.input("Max Characters", String(settings.maxChars));
|
|
294
|
+
if (input) {
|
|
295
|
+
const maxChars = parseInt(input, 10);
|
|
296
|
+
if (!isNaN(maxChars) && maxChars > 0) {
|
|
297
|
+
saveSmartFetchSettings({ maxChars });
|
|
298
|
+
ctx.ui.notify(`Max chars set to ${maxChars.toLocaleString()}`, "info");
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
case "timeoutMs": {
|
|
305
|
+
const input = await ctx.ui.input("Timeout (ms)", String(settings.timeoutMs));
|
|
306
|
+
if (input) {
|
|
307
|
+
const timeoutMs = parseInt(input, 10);
|
|
308
|
+
if (!isNaN(timeoutMs) && timeoutMs > 0) {
|
|
309
|
+
saveSmartFetchSettings({ timeoutMs });
|
|
310
|
+
ctx.ui.notify(`Timeout set to ${timeoutMs}ms`, "info");
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
case "batchConcurrency": {
|
|
317
|
+
const input = await ctx.ui.input("Batch Concurrency", String(settings.batchConcurrency));
|
|
318
|
+
if (input) {
|
|
319
|
+
const batchConcurrency = parseInt(input, 10);
|
|
320
|
+
if (!isNaN(batchConcurrency) && batchConcurrency > 0) {
|
|
321
|
+
saveSmartFetchSettings({ batchConcurrency });
|
|
322
|
+
ctx.ui.notify(`Concurrency set to ${batchConcurrency}`, "info");
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
break;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
case "removeImages": {
|
|
329
|
+
const removeImages = await ctx.ui.select("Remove Images", ["Yes", "No"]);
|
|
330
|
+
if (removeImages) {
|
|
331
|
+
saveSmartFetchSettings({ removeImages: removeImages === "Yes" });
|
|
332
|
+
ctx.ui.notify(`Remove images set to ${removeImages}`, "info");
|
|
333
|
+
}
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
case "includeReplies": {
|
|
338
|
+
const includeReplies = await ctx.ui.select("Include Replies", [
|
|
339
|
+
"extractors",
|
|
340
|
+
"Yes",
|
|
341
|
+
"No",
|
|
342
|
+
]);
|
|
343
|
+
if (includeReplies) {
|
|
344
|
+
const value = includeReplies === "Yes" ? true : includeReplies === "No" ? false : "extractors";
|
|
345
|
+
saveSmartFetchSettings({ includeReplies: value as boolean | "extractors" });
|
|
346
|
+
ctx.ui.notify(`Include replies set to ${includeReplies}`, "info");
|
|
347
|
+
}
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
case "__reset__":
|
|
352
|
+
resetSmartFetchSettings();
|
|
353
|
+
ctx.ui.notify("Smart-fetch settings reset to defaults", "info");
|
|
354
|
+
break;
|
|
355
|
+
|
|
356
|
+
case "__back__":
|
|
357
|
+
default:
|
|
358
|
+
break;
|
|
359
|
+
}
|
|
360
|
+
}
|