mcp-scraper 0.3.10 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/bin/api-server.cjs +115 -40
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +206 -31
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +3 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +262 -60
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +38 -11
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +21 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-3YGKXXUG.js +133 -0
- package/dist/chunk-3YGKXXUG.js.map +1 -0
- package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} +22 -2
- package/dist/chunk-4OPKIDON.js.map +1 -0
- package/dist/{chunk-CED7X4WB.js → chunk-7R7VBQRV.js} +39 -12
- package/dist/chunk-7R7VBQRV.js.map +1 -0
- package/dist/{chunk-QPFF3V2R.js → chunk-BW3DGFNQ.js} +36 -9
- package/dist/chunk-BW3DGFNQ.js.map +1 -0
- package/dist/chunk-FB5ZJGFN.js +7 -0
- package/dist/chunk-FB5ZJGFN.js.map +1 -0
- package/dist/index.cjs +21 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-QTV2EUKA.js → server-CUPDJLWM.js} +44 -142
- package/dist/server-CUPDJLWM.js.map +1 -0
- package/dist/{worker-56IXWOQU.js → worker-FG7ZWEGA.js} +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +1 -1
- package/docs/mcp-tool-manifest.generated.json +3 -3
- package/package.json +1 -1
- package/dist/chunk-CED7X4WB.js.map +0 -1
- package/dist/chunk-NEGW2ZEJ.js +0 -7
- package/dist/chunk-NEGW2ZEJ.js.map +0 -1
- package/dist/chunk-QPFF3V2R.js.map +0 -1
- package/dist/chunk-UWSG3C5J.js.map +0 -1
- package/dist/server-QTV2EUKA.js.map +0 -1
- /package/dist/{worker-56IXWOQU.js.map → worker-FG7ZWEGA.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -85,7 +85,7 @@ import {
|
|
|
85
85
|
deleteKernelProxyId,
|
|
86
86
|
harvest,
|
|
87
87
|
resolveKernelProxyId
|
|
88
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-4OPKIDON.js";
|
|
89
89
|
import {
|
|
90
90
|
CaptureSerpPageSnapshotsInputSchema,
|
|
91
91
|
CaptureSerpPageSnapshotsOutputSchema,
|
|
@@ -99,12 +99,15 @@ import {
|
|
|
99
99
|
harvestTimeoutBudget,
|
|
100
100
|
liveWebToolAnnotations,
|
|
101
101
|
outputBaseDir
|
|
102
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-7R7VBQRV.js";
|
|
103
103
|
import {
|
|
104
104
|
CaptchaError,
|
|
105
105
|
RECAPTCHA_INSTRUCTIONS,
|
|
106
106
|
sanitizeVendorName
|
|
107
107
|
} from "./chunk-M2S27J6Z.js";
|
|
108
|
+
import {
|
|
109
|
+
exportFanout
|
|
110
|
+
} from "./chunk-3YGKXXUG.js";
|
|
108
111
|
import {
|
|
109
112
|
browserServiceApiKey,
|
|
110
113
|
browserServiceProfileName,
|
|
@@ -132,7 +135,7 @@ import {
|
|
|
132
135
|
RawMapsOverviewSchema,
|
|
133
136
|
RawMapsReviewStatsSchema
|
|
134
137
|
} from "./chunk-LFATOGDF.js";
|
|
135
|
-
import "./chunk-
|
|
138
|
+
import "./chunk-FB5ZJGFN.js";
|
|
136
139
|
|
|
137
140
|
// src/api/outbound-sanitize.ts
|
|
138
141
|
var KEY_RENAMES = {
|
|
@@ -8366,10 +8369,10 @@ function buildTranscriptMarkdown(result) {
|
|
|
8366
8369
|
return lines.join("\n");
|
|
8367
8370
|
}
|
|
8368
8371
|
function buildTranscriptHtml(result) {
|
|
8369
|
-
const
|
|
8372
|
+
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
8370
8373
|
const fmtSecs = (n) => `${Math.floor(n / 60)}:${String(Math.floor(n % 60)).padStart(2, "0")}`;
|
|
8371
8374
|
const chunks = (result.chunks ?? []).map(
|
|
8372
|
-
(ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${
|
|
8375
|
+
(ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${esc(ch.text.trim())}</span></div>`
|
|
8373
8376
|
).join("\n");
|
|
8374
8377
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
8375
8378
|
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
@@ -8380,10 +8383,10 @@ h1{font-size:1.2rem;margin-bottom:.25rem}.meta{color:#666;font-size:.85rem;margi
|
|
|
8380
8383
|
.ts{font-family:monospace;font-size:.75rem;color:#888;white-space:nowrap;flex-shrink:0}
|
|
8381
8384
|
.ct{font-size:.875rem;flex:1;min-width:0;word-break:break-word}
|
|
8382
8385
|
</style></head><body>
|
|
8383
|
-
<h1>Transcript \u2014 ${
|
|
8386
|
+
<h1>Transcript \u2014 ${esc(result.videoId)}</h1>
|
|
8384
8387
|
<div class="meta">Transcribed in ${(result.durationMs / 1e3).toFixed(1)}s</div>
|
|
8385
8388
|
<h2 style="font-size:1.05rem">Full Text</h2>
|
|
8386
|
-
<div class="full-text">${
|
|
8389
|
+
<div class="full-text">${esc(result.text)}</div>
|
|
8387
8390
|
${chunks.length ? `<h2 style="font-size:1.05rem">Timestamped Segments</h2>
|
|
8388
8391
|
${chunks}` : ""}
|
|
8389
8392
|
</body></html>`;
|
|
@@ -9923,8 +9926,8 @@ function ownerFromMetadata(metadata) {
|
|
|
9923
9926
|
return match?.[1]?.trim() || null;
|
|
9924
9927
|
}
|
|
9925
9928
|
async function collectInstagramProfileContentFromPage(page, opts) {
|
|
9926
|
-
const maxItems = Math.min(
|
|
9927
|
-
const maxScrolls = Math.min(
|
|
9929
|
+
const maxItems = Math.min(2e3, Math.max(1, opts.maxItems ?? 50));
|
|
9930
|
+
const maxScrolls = Math.min(250, Math.max(0, opts.maxScrolls ?? 10));
|
|
9928
9931
|
const scrollDelayMs = Math.min(5e3, Math.max(250, opts.scrollDelayMs ?? 1200));
|
|
9929
9932
|
const stableScrollLimit = Math.min(10, Math.max(1, opts.stableScrollLimit ?? 4));
|
|
9930
9933
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -10137,8 +10140,11 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
10137
10140
|
var InstagramProfileContentBodySchema = z14.object({
|
|
10138
10141
|
handle: z14.string().trim().optional(),
|
|
10139
10142
|
url: z14.string().trim().optional(),
|
|
10140
|
-
|
|
10141
|
-
|
|
10143
|
+
profile: z14.string().trim().min(1).optional(),
|
|
10144
|
+
saveProfileChanges: z14.boolean().optional(),
|
|
10145
|
+
save_profile_changes: z14.boolean().optional(),
|
|
10146
|
+
maxItems: z14.number().int().min(1).max(2e3).default(50),
|
|
10147
|
+
maxScrolls: z14.number().int().min(0).max(250).default(10),
|
|
10142
10148
|
scrollDelayMs: z14.number().int().min(250).max(5e3).default(1200),
|
|
10143
10149
|
stableScrollLimit: z14.number().int().min(1).max(10).default(4)
|
|
10144
10150
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
@@ -10147,6 +10153,9 @@ var InstagramProfileContentBodySchema = z14.object({
|
|
|
10147
10153
|
var InstagramMediaTypeSchema = z14.enum(["image", "video", "audio"]);
|
|
10148
10154
|
var InstagramMediaDownloadBodySchema = z14.object({
|
|
10149
10155
|
url: z14.string().trim().min(1, "url is required"),
|
|
10156
|
+
profile: z14.string().trim().min(1).optional(),
|
|
10157
|
+
saveProfileChanges: z14.boolean().optional(),
|
|
10158
|
+
save_profile_changes: z14.boolean().optional(),
|
|
10150
10159
|
mediaTypes: z14.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
10151
10160
|
downloadMedia: z14.boolean().default(true),
|
|
10152
10161
|
downloadAllTracks: z14.boolean().default(false),
|
|
@@ -10157,21 +10166,32 @@ var instagramApp = new Hono5();
|
|
|
10157
10166
|
function invalidRequest2(message) {
|
|
10158
10167
|
return { error_code: "invalid_request", message };
|
|
10159
10168
|
}
|
|
10160
|
-
|
|
10169
|
+
function resolveSaveProfileChanges(body) {
|
|
10170
|
+
return body.saveProfileChanges ?? body.save_profile_changes ?? browserServiceProfileSaveChanges();
|
|
10171
|
+
}
|
|
10172
|
+
function resolveProfileName(body) {
|
|
10173
|
+
const explicit = body.profile?.trim();
|
|
10174
|
+
return explicit || browserServiceProfileName();
|
|
10175
|
+
}
|
|
10176
|
+
async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
|
|
10161
10177
|
return {
|
|
10162
10178
|
headless: true,
|
|
10163
10179
|
kernelApiKey: browserServiceApiKey(),
|
|
10180
|
+
...profileName ? { kernelProfileName: profileName } : {},
|
|
10181
|
+
...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
|
|
10164
10182
|
viewport: { width: 1280, height: 900 },
|
|
10165
10183
|
locale: "en-US"
|
|
10166
10184
|
};
|
|
10167
10185
|
}
|
|
10168
|
-
async function resolveInstagramLaunch() {
|
|
10186
|
+
async function resolveInstagramLaunch(body) {
|
|
10187
|
+
const profileName = resolveProfileName(body);
|
|
10188
|
+
const saveProfileChanges = resolveSaveProfileChanges(body);
|
|
10169
10189
|
return {
|
|
10170
|
-
config: await kernelLaunchOptsDirect(),
|
|
10190
|
+
config: await kernelLaunchOptsDirect(profileName, saveProfileChanges),
|
|
10171
10191
|
browser: {
|
|
10172
10192
|
mode: "hosted",
|
|
10173
10193
|
requestedMode: "hosted",
|
|
10174
|
-
profileName: null,
|
|
10194
|
+
profileName: profileName ?? null,
|
|
10175
10195
|
profileSource: "hosted",
|
|
10176
10196
|
profileDirConfigured: false,
|
|
10177
10197
|
executablePathConfigured: false
|
|
@@ -10262,7 +10282,7 @@ instagramApp.post("/profile-content", createApiKeyAuth(), async (c) => {
|
|
|
10262
10282
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
|
|
10263
10283
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
|
|
10264
10284
|
debited = true;
|
|
10265
|
-
const launch = await resolveInstagramLaunch();
|
|
10285
|
+
const launch = await resolveInstagramLaunch(body);
|
|
10266
10286
|
await driver.launch(launch.config);
|
|
10267
10287
|
await driver.navigateTo(target.profileUrl);
|
|
10268
10288
|
const page = driver.getPage();
|
|
@@ -10309,7 +10329,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
10309
10329
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
|
|
10310
10330
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
|
|
10311
10331
|
mediaDebited = true;
|
|
10312
|
-
const launch = await resolveInstagramLaunch();
|
|
10332
|
+
const launch = await resolveInstagramLaunch(body);
|
|
10313
10333
|
await driver.launch(launch.config);
|
|
10314
10334
|
const page = driver.getPage();
|
|
10315
10335
|
const capturedMediaUrls = [];
|
|
@@ -13595,12 +13615,12 @@ function sliceBalanced(text, openIdx) {
|
|
|
13595
13615
|
const close = open === "[" ? "]" : "}";
|
|
13596
13616
|
let depth = 0;
|
|
13597
13617
|
let inStr = false;
|
|
13598
|
-
let
|
|
13618
|
+
let esc = false;
|
|
13599
13619
|
for (let i = openIdx; i < text.length; i++) {
|
|
13600
13620
|
const ch = text[i];
|
|
13601
13621
|
if (inStr) {
|
|
13602
|
-
if (
|
|
13603
|
-
else if (ch === "\\")
|
|
13622
|
+
if (esc) esc = false;
|
|
13623
|
+
else if (ch === "\\") esc = true;
|
|
13604
13624
|
else if (ch === '"') inStr = false;
|
|
13605
13625
|
continue;
|
|
13606
13626
|
}
|
|
@@ -14098,126 +14118,6 @@ function enrichFanout(result, opts = {}) {
|
|
|
14098
14118
|
};
|
|
14099
14119
|
}
|
|
14100
14120
|
|
|
14101
|
-
// src/services/fanout/export.ts
|
|
14102
|
-
import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
14103
|
-
import { homedir as homedir3 } from "os";
|
|
14104
|
-
import { join as join6 } from "path";
|
|
14105
|
-
import Papa2 from "papaparse";
|
|
14106
|
-
function outputBaseDir3() {
|
|
14107
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
|
|
14108
|
-
}
|
|
14109
|
-
function safe(value) {
|
|
14110
|
-
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
14111
|
-
}
|
|
14112
|
-
function writeTable(path5, rows, delimiter) {
|
|
14113
|
-
writeFileSync2(path5, Papa2.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
14114
|
-
}
|
|
14115
|
-
function exportFanout(enriched) {
|
|
14116
|
-
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
14117
|
-
const dir = join6(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
14118
|
-
mkdirSync3(dir, { recursive: true });
|
|
14119
|
-
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
14120
|
-
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
14121
|
-
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
14122
|
-
return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
|
|
14123
|
-
});
|
|
14124
|
-
const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
14125
|
-
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
14126
|
-
const paths = {
|
|
14127
|
-
dir,
|
|
14128
|
-
json: join6(dir, "fanout.json"),
|
|
14129
|
-
queriesCsv: join6(dir, "queries.csv"),
|
|
14130
|
-
queriesTsv: join6(dir, "queries.tsv"),
|
|
14131
|
-
citationsCsv: join6(dir, "citations.csv"),
|
|
14132
|
-
sourcesCsv: join6(dir, "sources.csv"),
|
|
14133
|
-
domainsCsv: join6(dir, "domains.csv"),
|
|
14134
|
-
report: join6(dir, "report.html")
|
|
14135
|
-
};
|
|
14136
|
-
writeFileSync2(paths.json, JSON.stringify(enriched, null, 2));
|
|
14137
|
-
writeTable(paths.queriesCsv, queryRows, ",");
|
|
14138
|
-
writeTable(paths.queriesTsv, queryRows, " ");
|
|
14139
|
-
writeTable(paths.citationsCsv, citationRows, ",");
|
|
14140
|
-
writeTable(paths.sourcesCsv, sourceRows, ",");
|
|
14141
|
-
writeTable(paths.domainsCsv, domainRows, ",");
|
|
14142
|
-
writeFileSync2(paths.report, renderReportHtml(enriched));
|
|
14143
|
-
return paths;
|
|
14144
|
-
}
|
|
14145
|
-
function esc(s) {
|
|
14146
|
-
return String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
14147
|
-
}
|
|
14148
|
-
function renderReportHtml(enriched) {
|
|
14149
|
-
const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
|
|
14150
|
-
return `<!doctype html>
|
|
14151
|
-
<html lang="en">
|
|
14152
|
-
<head>
|
|
14153
|
-
<meta charset="utf-8" />
|
|
14154
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
14155
|
-
<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
|
|
14156
|
-
<style>
|
|
14157
|
-
:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
|
|
14158
|
-
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
|
|
14159
|
-
.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
|
|
14160
|
-
h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
|
|
14161
|
-
.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
|
|
14162
|
-
.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
|
|
14163
|
-
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
14164
|
-
.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
|
|
14165
|
-
.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
|
|
14166
|
-
.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
|
|
14167
|
-
.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
|
|
14168
|
-
.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
|
|
14169
|
-
.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
|
|
14170
|
-
.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
|
|
14171
|
-
.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
|
|
14172
|
-
.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
|
|
14173
|
-
.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
|
|
14174
|
-
.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
|
|
14175
|
-
.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
|
|
14176
|
-
.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
|
|
14177
|
-
.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
|
|
14178
|
-
.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
|
|
14179
|
-
.hide{display:none}
|
|
14180
|
-
</style>
|
|
14181
|
-
</head>
|
|
14182
|
-
<body><div class="wrap">
|
|
14183
|
-
<h1>AI Search Fan-Out</h1>
|
|
14184
|
-
<p class="sub" id="sub"></p>
|
|
14185
|
-
<div class="stat" id="stat"></div>
|
|
14186
|
-
<div class="cards" id="cards"></div>
|
|
14187
|
-
<div id="charts"></div>
|
|
14188
|
-
<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
|
|
14189
|
-
<div class="chips" id="chips"></div>
|
|
14190
|
-
<div id="list"></div>
|
|
14191
|
-
</div>
|
|
14192
|
-
<script>
|
|
14193
|
-
const D=${data};
|
|
14194
|
-
const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
|
|
14195
|
-
document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
|
|
14196
|
-
document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
|
|
14197
|
-
const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
|
|
14198
|
-
const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
|
|
14199
|
-
function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
|
|
14200
|
-
const charts=document.getElementById('charts');
|
|
14201
|
-
charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
|
|
14202
|
-
let tab='q',filter=null;
|
|
14203
|
-
const chips=document.getElementById('chips'),list=document.getElementById('list');
|
|
14204
|
-
function render(){
|
|
14205
|
-
chips.innerHTML='';list.innerHTML='';
|
|
14206
|
-
if(tab==='q'){
|
|
14207
|
-
D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
|
|
14208
|
-
} else {
|
|
14209
|
-
const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
|
|
14210
|
-
Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
|
|
14211
|
-
D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
|
|
14212
|
-
}
|
|
14213
|
-
}
|
|
14214
|
-
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}
|
|
14215
|
-
document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
|
|
14216
|
-
render();
|
|
14217
|
-
</script>
|
|
14218
|
-
</body></html>`;
|
|
14219
|
-
}
|
|
14220
|
-
|
|
14221
14121
|
// src/services/fanout/run-capture.ts
|
|
14222
14122
|
var COMPOSER_SELECTORS = [
|
|
14223
14123
|
"#prompt-textarea",
|
|
@@ -14255,6 +14155,7 @@ async function runFanoutCapture(page, input) {
|
|
|
14255
14155
|
const cap = new FanoutCdpCapture();
|
|
14256
14156
|
await cap.attach(page);
|
|
14257
14157
|
try {
|
|
14158
|
+
if (input.reset) cap.reset();
|
|
14258
14159
|
const baseline = cap.answerStreamCount();
|
|
14259
14160
|
if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
|
|
14260
14161
|
const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
|
|
@@ -14909,6 +14810,7 @@ function buildBrowserAgentRoutes() {
|
|
|
14909
14810
|
prompt: typeof body.prompt === "string" ? body.prompt : void 0,
|
|
14910
14811
|
wait_ms: typeof body.wait_ms === "number" ? body.wait_ms : void 0,
|
|
14911
14812
|
first_party_domain: typeof body.first_party_domain === "string" ? body.first_party_domain : void 0,
|
|
14813
|
+
reset: body.reset === true,
|
|
14912
14814
|
export: body.export === true
|
|
14913
14815
|
};
|
|
14914
14816
|
const t0 = Date.now();
|
|
@@ -16430,7 +16332,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
16430
16332
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
16431
16333
|
return c.json({ error: "Unauthorized" }, 401);
|
|
16432
16334
|
}
|
|
16433
|
-
const { drainQueue } = await import("./worker-
|
|
16335
|
+
const { drainQueue } = await import("./worker-FG7ZWEGA.js");
|
|
16434
16336
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
16435
16337
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
16436
16338
|
const [results, sweepResult] = await Promise.all([
|
|
@@ -16560,4 +16462,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
16560
16462
|
export {
|
|
16561
16463
|
app
|
|
16562
16464
|
};
|
|
16563
|
-
//# sourceMappingURL=server-
|
|
16465
|
+
//# sourceMappingURL=server-CUPDJLWM.js.map
|