mcp-scraper 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +86 -22
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +198 -27
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +3 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +254 -56
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +38 -11
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +21 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-3YGKXXUG.js +133 -0
- package/dist/chunk-3YGKXXUG.js.map +1 -0
- package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} +22 -2
- package/dist/chunk-4OPKIDON.js.map +1 -0
- package/dist/{chunk-VMH7SRWY.js → chunk-7R7VBQRV.js} +39 -12
- package/dist/chunk-7R7VBQRV.js.map +1 -0
- package/dist/{chunk-L27GJQV7.js → chunk-BW3DGFNQ.js} +28 -5
- package/dist/chunk-BW3DGFNQ.js.map +1 -0
- package/dist/chunk-FB5ZJGFN.js +7 -0
- package/dist/chunk-FB5ZJGFN.js.map +1 -0
- package/dist/index.cjs +21 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-3SA5Q4OF.js → server-CUPDJLWM.js} +42 -151
- package/dist/server-CUPDJLWM.js.map +1 -0
- package/dist/{worker-56IXWOQU.js → worker-FG7ZWEGA.js} +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +1 -1
- package/docs/mcp-tool-manifest.generated.json +1 -1
- package/package.json +1 -1
- package/dist/chunk-L27GJQV7.js.map +0 -1
- package/dist/chunk-RRE7WVHQ.js +0 -7
- package/dist/chunk-RRE7WVHQ.js.map +0 -1
- package/dist/chunk-UWSG3C5J.js.map +0 -1
- package/dist/chunk-VMH7SRWY.js.map +0 -1
- package/dist/server-3SA5Q4OF.js.map +0 -1
- /package/dist/{worker-56IXWOQU.js.map → worker-FG7ZWEGA.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -85,7 +85,7 @@ import {
|
|
|
85
85
|
deleteKernelProxyId,
|
|
86
86
|
harvest,
|
|
87
87
|
resolveKernelProxyId
|
|
88
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-4OPKIDON.js";
|
|
89
89
|
import {
|
|
90
90
|
CaptureSerpPageSnapshotsInputSchema,
|
|
91
91
|
CaptureSerpPageSnapshotsOutputSchema,
|
|
@@ -99,12 +99,15 @@ import {
|
|
|
99
99
|
harvestTimeoutBudget,
|
|
100
100
|
liveWebToolAnnotations,
|
|
101
101
|
outputBaseDir
|
|
102
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-7R7VBQRV.js";
|
|
103
103
|
import {
|
|
104
104
|
CaptchaError,
|
|
105
105
|
RECAPTCHA_INSTRUCTIONS,
|
|
106
106
|
sanitizeVendorName
|
|
107
107
|
} from "./chunk-M2S27J6Z.js";
|
|
108
|
+
import {
|
|
109
|
+
exportFanout
|
|
110
|
+
} from "./chunk-3YGKXXUG.js";
|
|
108
111
|
import {
|
|
109
112
|
browserServiceApiKey,
|
|
110
113
|
browserServiceProfileName,
|
|
@@ -132,7 +135,7 @@ import {
|
|
|
132
135
|
RawMapsOverviewSchema,
|
|
133
136
|
RawMapsReviewStatsSchema
|
|
134
137
|
} from "./chunk-LFATOGDF.js";
|
|
135
|
-
import "./chunk-
|
|
138
|
+
import "./chunk-FB5ZJGFN.js";
|
|
136
139
|
|
|
137
140
|
// src/api/outbound-sanitize.ts
|
|
138
141
|
var KEY_RENAMES = {
|
|
@@ -8366,10 +8369,10 @@ function buildTranscriptMarkdown(result) {
|
|
|
8366
8369
|
return lines.join("\n");
|
|
8367
8370
|
}
|
|
8368
8371
|
function buildTranscriptHtml(result) {
|
|
8369
|
-
const
|
|
8372
|
+
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
8370
8373
|
const fmtSecs = (n) => `${Math.floor(n / 60)}:${String(Math.floor(n % 60)).padStart(2, "0")}`;
|
|
8371
8374
|
const chunks = (result.chunks ?? []).map(
|
|
8372
|
-
(ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${
|
|
8375
|
+
(ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${esc(ch.text.trim())}</span></div>`
|
|
8373
8376
|
).join("\n");
|
|
8374
8377
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
8375
8378
|
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
@@ -8380,10 +8383,10 @@ h1{font-size:1.2rem;margin-bottom:.25rem}.meta{color:#666;font-size:.85rem;margi
|
|
|
8380
8383
|
.ts{font-family:monospace;font-size:.75rem;color:#888;white-space:nowrap;flex-shrink:0}
|
|
8381
8384
|
.ct{font-size:.875rem;flex:1;min-width:0;word-break:break-word}
|
|
8382
8385
|
</style></head><body>
|
|
8383
|
-
<h1>Transcript \u2014 ${
|
|
8386
|
+
<h1>Transcript \u2014 ${esc(result.videoId)}</h1>
|
|
8384
8387
|
<div class="meta">Transcribed in ${(result.durationMs / 1e3).toFixed(1)}s</div>
|
|
8385
8388
|
<h2 style="font-size:1.05rem">Full Text</h2>
|
|
8386
|
-
<div class="full-text">${
|
|
8389
|
+
<div class="full-text">${esc(result.text)}</div>
|
|
8387
8390
|
${chunks.length ? `<h2 style="font-size:1.05rem">Timestamped Segments</h2>
|
|
8388
8391
|
${chunks}` : ""}
|
|
8389
8392
|
</body></html>`;
|
|
@@ -9923,8 +9926,8 @@ function ownerFromMetadata(metadata) {
|
|
|
9923
9926
|
return match?.[1]?.trim() || null;
|
|
9924
9927
|
}
|
|
9925
9928
|
async function collectInstagramProfileContentFromPage(page, opts) {
|
|
9926
|
-
const maxItems = Math.min(
|
|
9927
|
-
const maxScrolls = Math.min(
|
|
9929
|
+
const maxItems = Math.min(2e3, Math.max(1, opts.maxItems ?? 50));
|
|
9930
|
+
const maxScrolls = Math.min(250, Math.max(0, opts.maxScrolls ?? 10));
|
|
9928
9931
|
const scrollDelayMs = Math.min(5e3, Math.max(250, opts.scrollDelayMs ?? 1200));
|
|
9929
9932
|
const stableScrollLimit = Math.min(10, Math.max(1, opts.stableScrollLimit ?? 4));
|
|
9930
9933
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -10137,8 +10140,11 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
10137
10140
|
var InstagramProfileContentBodySchema = z14.object({
|
|
10138
10141
|
handle: z14.string().trim().optional(),
|
|
10139
10142
|
url: z14.string().trim().optional(),
|
|
10140
|
-
|
|
10141
|
-
|
|
10143
|
+
profile: z14.string().trim().min(1).optional(),
|
|
10144
|
+
saveProfileChanges: z14.boolean().optional(),
|
|
10145
|
+
save_profile_changes: z14.boolean().optional(),
|
|
10146
|
+
maxItems: z14.number().int().min(1).max(2e3).default(50),
|
|
10147
|
+
maxScrolls: z14.number().int().min(0).max(250).default(10),
|
|
10142
10148
|
scrollDelayMs: z14.number().int().min(250).max(5e3).default(1200),
|
|
10143
10149
|
stableScrollLimit: z14.number().int().min(1).max(10).default(4)
|
|
10144
10150
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
@@ -10147,6 +10153,9 @@ var InstagramProfileContentBodySchema = z14.object({
|
|
|
10147
10153
|
var InstagramMediaTypeSchema = z14.enum(["image", "video", "audio"]);
|
|
10148
10154
|
var InstagramMediaDownloadBodySchema = z14.object({
|
|
10149
10155
|
url: z14.string().trim().min(1, "url is required"),
|
|
10156
|
+
profile: z14.string().trim().min(1).optional(),
|
|
10157
|
+
saveProfileChanges: z14.boolean().optional(),
|
|
10158
|
+
save_profile_changes: z14.boolean().optional(),
|
|
10150
10159
|
mediaTypes: z14.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
10151
10160
|
downloadMedia: z14.boolean().default(true),
|
|
10152
10161
|
downloadAllTracks: z14.boolean().default(false),
|
|
@@ -10157,21 +10166,32 @@ var instagramApp = new Hono5();
|
|
|
10157
10166
|
function invalidRequest2(message) {
|
|
10158
10167
|
return { error_code: "invalid_request", message };
|
|
10159
10168
|
}
|
|
10160
|
-
|
|
10169
|
+
function resolveSaveProfileChanges(body) {
|
|
10170
|
+
return body.saveProfileChanges ?? body.save_profile_changes ?? browserServiceProfileSaveChanges();
|
|
10171
|
+
}
|
|
10172
|
+
function resolveProfileName(body) {
|
|
10173
|
+
const explicit = body.profile?.trim();
|
|
10174
|
+
return explicit || browserServiceProfileName();
|
|
10175
|
+
}
|
|
10176
|
+
async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
|
|
10161
10177
|
return {
|
|
10162
10178
|
headless: true,
|
|
10163
10179
|
kernelApiKey: browserServiceApiKey(),
|
|
10180
|
+
...profileName ? { kernelProfileName: profileName } : {},
|
|
10181
|
+
...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
|
|
10164
10182
|
viewport: { width: 1280, height: 900 },
|
|
10165
10183
|
locale: "en-US"
|
|
10166
10184
|
};
|
|
10167
10185
|
}
|
|
10168
|
-
async function resolveInstagramLaunch() {
|
|
10186
|
+
async function resolveInstagramLaunch(body) {
|
|
10187
|
+
const profileName = resolveProfileName(body);
|
|
10188
|
+
const saveProfileChanges = resolveSaveProfileChanges(body);
|
|
10169
10189
|
return {
|
|
10170
|
-
config: await kernelLaunchOptsDirect(),
|
|
10190
|
+
config: await kernelLaunchOptsDirect(profileName, saveProfileChanges),
|
|
10171
10191
|
browser: {
|
|
10172
10192
|
mode: "hosted",
|
|
10173
10193
|
requestedMode: "hosted",
|
|
10174
|
-
profileName: null,
|
|
10194
|
+
profileName: profileName ?? null,
|
|
10175
10195
|
profileSource: "hosted",
|
|
10176
10196
|
profileDirConfigured: false,
|
|
10177
10197
|
executablePathConfigured: false
|
|
@@ -10262,7 +10282,7 @@ instagramApp.post("/profile-content", createApiKeyAuth(), async (c) => {
|
|
|
10262
10282
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
|
|
10263
10283
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
|
|
10264
10284
|
debited = true;
|
|
10265
|
-
const launch = await resolveInstagramLaunch();
|
|
10285
|
+
const launch = await resolveInstagramLaunch(body);
|
|
10266
10286
|
await driver.launch(launch.config);
|
|
10267
10287
|
await driver.navigateTo(target.profileUrl);
|
|
10268
10288
|
const page = driver.getPage();
|
|
@@ -10309,7 +10329,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
10309
10329
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
|
|
10310
10330
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
|
|
10311
10331
|
mediaDebited = true;
|
|
10312
|
-
const launch = await resolveInstagramLaunch();
|
|
10332
|
+
const launch = await resolveInstagramLaunch(body);
|
|
10313
10333
|
await driver.launch(launch.config);
|
|
10314
10334
|
const page = driver.getPage();
|
|
10315
10335
|
const capturedMediaUrls = [];
|
|
@@ -13595,12 +13615,12 @@ function sliceBalanced(text, openIdx) {
|
|
|
13595
13615
|
const close = open === "[" ? "]" : "}";
|
|
13596
13616
|
let depth = 0;
|
|
13597
13617
|
let inStr = false;
|
|
13598
|
-
let
|
|
13618
|
+
let esc = false;
|
|
13599
13619
|
for (let i = openIdx; i < text.length; i++) {
|
|
13600
13620
|
const ch = text[i];
|
|
13601
13621
|
if (inStr) {
|
|
13602
|
-
if (
|
|
13603
|
-
else if (ch === "\\")
|
|
13622
|
+
if (esc) esc = false;
|
|
13623
|
+
else if (ch === "\\") esc = true;
|
|
13604
13624
|
else if (ch === '"') inStr = false;
|
|
13605
13625
|
continue;
|
|
13606
13626
|
}
|
|
@@ -14098,135 +14118,6 @@ function enrichFanout(result, opts = {}) {
|
|
|
14098
14118
|
};
|
|
14099
14119
|
}
|
|
14100
14120
|
|
|
14101
|
-
// src/services/fanout/export.ts
|
|
14102
|
-
import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
14103
|
-
import { homedir as homedir3 } from "os";
|
|
14104
|
-
import { join as join6 } from "path";
|
|
14105
|
-
import Papa2 from "papaparse";
|
|
14106
|
-
function outputBaseDir3() {
|
|
14107
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
|
|
14108
|
-
}
|
|
14109
|
-
function safe(value) {
|
|
14110
|
-
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
14111
|
-
}
|
|
14112
|
-
function writeTable(path5, rows, delimiter) {
|
|
14113
|
-
writeFileSync2(path5, Papa2.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
14114
|
-
}
|
|
14115
|
-
function exportFanout(enriched) {
|
|
14116
|
-
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
14117
|
-
const outputDir = outputBaseDir3();
|
|
14118
|
-
const relativeDir = join6("fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
14119
|
-
const dir = join6(outputDir, relativeDir);
|
|
14120
|
-
mkdirSync3(dir, { recursive: true });
|
|
14121
|
-
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
14122
|
-
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
14123
|
-
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
14124
|
-
return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
|
|
14125
|
-
});
|
|
14126
|
-
const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
14127
|
-
const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
14128
|
-
const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
|
|
14129
|
-
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
14130
|
-
const relativePaths = {
|
|
14131
|
-
relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
|
|
14132
|
-
dir: relativeDir,
|
|
14133
|
-
json: join6(relativeDir, "fanout.json"),
|
|
14134
|
-
queriesCsv: join6(relativeDir, "queries.csv"),
|
|
14135
|
-
queriesTsv: join6(relativeDir, "queries.tsv"),
|
|
14136
|
-
citationsCsv: join6(relativeDir, "citations.csv"),
|
|
14137
|
-
sourcesCsv: join6(relativeDir, "sources.csv"),
|
|
14138
|
-
browsedOnlyCsv: join6(relativeDir, "browsed-only.csv"),
|
|
14139
|
-
snippetsCsv: join6(relativeDir, "snippets.csv"),
|
|
14140
|
-
domainsCsv: join6(relativeDir, "domains.csv"),
|
|
14141
|
-
report: join6(relativeDir, "report.html")
|
|
14142
|
-
};
|
|
14143
|
-
writeFileSync2(join6(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
|
|
14144
|
-
writeTable(join6(outputDir, relativePaths.queriesCsv), queryRows, ",");
|
|
14145
|
-
writeTable(join6(outputDir, relativePaths.queriesTsv), queryRows, " ");
|
|
14146
|
-
writeTable(join6(outputDir, relativePaths.citationsCsv), citationRows, ",");
|
|
14147
|
-
writeTable(join6(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
|
|
14148
|
-
writeTable(join6(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
|
|
14149
|
-
writeTable(join6(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
|
|
14150
|
-
writeTable(join6(outputDir, relativePaths.domainsCsv), domainRows, ",");
|
|
14151
|
-
writeFileSync2(join6(outputDir, relativePaths.report), renderReportHtml(enriched));
|
|
14152
|
-
return relativePaths;
|
|
14153
|
-
}
|
|
14154
|
-
function esc(s) {
|
|
14155
|
-
return String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
14156
|
-
}
|
|
14157
|
-
function renderReportHtml(enriched) {
|
|
14158
|
-
const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
|
|
14159
|
-
return `<!doctype html>
|
|
14160
|
-
<html lang="en">
|
|
14161
|
-
<head>
|
|
14162
|
-
<meta charset="utf-8" />
|
|
14163
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
14164
|
-
<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
|
|
14165
|
-
<style>
|
|
14166
|
-
:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
|
|
14167
|
-
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
|
|
14168
|
-
.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
|
|
14169
|
-
h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
|
|
14170
|
-
.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
|
|
14171
|
-
.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
|
|
14172
|
-
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
14173
|
-
.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
|
|
14174
|
-
.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
|
|
14175
|
-
.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
|
|
14176
|
-
.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
|
|
14177
|
-
.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
|
|
14178
|
-
.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
|
|
14179
|
-
.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
|
|
14180
|
-
.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
|
|
14181
|
-
.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
|
|
14182
|
-
.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
|
|
14183
|
-
.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
|
|
14184
|
-
.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
|
|
14185
|
-
.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
|
|
14186
|
-
.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
|
|
14187
|
-
.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
|
|
14188
|
-
.hide{display:none}
|
|
14189
|
-
</style>
|
|
14190
|
-
</head>
|
|
14191
|
-
<body><div class="wrap">
|
|
14192
|
-
<h1>AI Search Fan-Out</h1>
|
|
14193
|
-
<p class="sub" id="sub"></p>
|
|
14194
|
-
<div class="stat" id="stat"></div>
|
|
14195
|
-
<div class="cards" id="cards"></div>
|
|
14196
|
-
<div id="charts"></div>
|
|
14197
|
-
<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
|
|
14198
|
-
<div class="chips" id="chips"></div>
|
|
14199
|
-
<div id="list"></div>
|
|
14200
|
-
</div>
|
|
14201
|
-
<script>
|
|
14202
|
-
const D=${data};
|
|
14203
|
-
const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
|
|
14204
|
-
document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
|
|
14205
|
-
document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
|
|
14206
|
-
const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
|
|
14207
|
-
const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
|
|
14208
|
-
function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
|
|
14209
|
-
const charts=document.getElementById('charts');
|
|
14210
|
-
charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
|
|
14211
|
-
let tab='q',filter=null;
|
|
14212
|
-
const chips=document.getElementById('chips'),list=document.getElementById('list');
|
|
14213
|
-
function render(){
|
|
14214
|
-
chips.innerHTML='';list.innerHTML='';
|
|
14215
|
-
if(tab==='q'){
|
|
14216
|
-
D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
|
|
14217
|
-
} else {
|
|
14218
|
-
const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
|
|
14219
|
-
Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
|
|
14220
|
-
D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
|
|
14221
|
-
}
|
|
14222
|
-
}
|
|
14223
|
-
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}
|
|
14224
|
-
document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
|
|
14225
|
-
render();
|
|
14226
|
-
</script>
|
|
14227
|
-
</body></html>`;
|
|
14228
|
-
}
|
|
14229
|
-
|
|
14230
14121
|
// src/services/fanout/run-capture.ts
|
|
14231
14122
|
var COMPOSER_SELECTORS = [
|
|
14232
14123
|
"#prompt-textarea",
|
|
@@ -16441,7 +16332,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
16441
16332
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
16442
16333
|
return c.json({ error: "Unauthorized" }, 401);
|
|
16443
16334
|
}
|
|
16444
|
-
const { drainQueue } = await import("./worker-
|
|
16335
|
+
const { drainQueue } = await import("./worker-FG7ZWEGA.js");
|
|
16445
16336
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
16446
16337
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
16447
16338
|
const [results, sweepResult] = await Promise.all([
|
|
@@ -16571,4 +16462,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
16571
16462
|
export {
|
|
16572
16463
|
app
|
|
16573
16464
|
};
|
|
16574
|
-
//# sourceMappingURL=server-
|
|
16465
|
+
//# sourceMappingURL=server-CUPDJLWM.js.map
|