mcp-scraper 0.1.9 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -8
- package/dist/bin/api-server.cjs +5615 -3733
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +391 -0
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -0
- package/dist/bin/browser-agent-stdio-server.d.cts +1 -0
- package/dist/bin/browser-agent-stdio-server.d.ts +1 -0
- package/dist/bin/browser-agent-stdio-server.js +390 -0
- package/dist/bin/browser-agent-stdio-server.js.map +1 -0
- package/dist/bin/mcp-stdio-server.cjs +170 -12
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +223 -74
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-ZK456YXN.js → chunk-IQOCZGJJ.js} +58 -4
- package/dist/chunk-IQOCZGJJ.js.map +1 -0
- package/dist/{chunk-ZMOWIBMK.js → chunk-M2S27J6Z.js} +9 -2
- package/dist/{chunk-ZMOWIBMK.js.map → chunk-M2S27J6Z.js.map} +1 -1
- package/dist/{chunk-TM22BLWP.js → chunk-MY3S7EX7.js} +221 -76
- package/dist/chunk-MY3S7EX7.js.map +1 -0
- package/dist/{chunk-JNC32DMS.js → chunk-OR7DLLH2.js} +175 -16
- package/dist/chunk-OR7DLLH2.js.map +1 -0
- package/dist/chunk-XR65SANX.js +7 -0
- package/dist/chunk-XR65SANX.js.map +1 -0
- package/dist/index.cjs +223 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/{server-MTXAJG5J.js → server-CJMX2QUM.js} +1655 -194
- package/dist/server-CJMX2QUM.js.map +1 -0
- package/dist/{worker-AUCXFHEL.js → worker-NAKGTIF5.js} +4 -4
- package/docs/specs/api-forge-spec.md +234 -0
- package/docs/specs/deferred-work-spec.md +74 -0
- package/docs/specs/oauth-mcp-spec.md +213 -0
- package/package.json +3 -2
- package/dist/chunk-JNC32DMS.js.map +0 -1
- package/dist/chunk-TM22BLWP.js.map +0 -1
- package/dist/chunk-ZK456YXN.js.map +0 -1
- package/dist/server-MTXAJG5J.js.map +0 -1
- /package/dist/{worker-AUCXFHEL.js.map → worker-NAKGTIF5.js.map} +0 -0
|
@@ -5,11 +5,14 @@ import {
|
|
|
5
5
|
buildPaaExtractorMcpServer,
|
|
6
6
|
configureReportSaving,
|
|
7
7
|
harvestTimeoutBudget,
|
|
8
|
-
liveWebToolAnnotations
|
|
9
|
-
|
|
8
|
+
liveWebToolAnnotations,
|
|
9
|
+
outputBaseDir
|
|
10
|
+
} from "./chunk-OR7DLLH2.js";
|
|
11
|
+
import "./chunk-XR65SANX.js";
|
|
10
12
|
import {
|
|
11
13
|
BALANCE_PACK_LABELS,
|
|
12
14
|
BALANCE_PRICE_IDS,
|
|
15
|
+
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
13
16
|
CONCURRENCY_PRICE_ID,
|
|
14
17
|
CREDIT_COST_CATALOG,
|
|
15
18
|
FREE_MONTHLY_REFRESH_MC,
|
|
@@ -17,12 +20,13 @@ import {
|
|
|
17
20
|
LedgerOperation,
|
|
18
21
|
MC_COSTS,
|
|
19
22
|
MC_PER_CREDIT,
|
|
23
|
+
browserActiveCostMc,
|
|
20
24
|
classifyHarvestProblem,
|
|
21
25
|
createHarvestAttemptRecorder,
|
|
22
26
|
harvestProblemResponse,
|
|
23
27
|
insufficientBalanceResponse,
|
|
24
28
|
serializeHarvestProblem
|
|
25
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-IQOCZGJJ.js";
|
|
26
30
|
import {
|
|
27
31
|
BrowserDriver,
|
|
28
32
|
MapsPlaceOptionsSchema,
|
|
@@ -35,14 +39,15 @@ import {
|
|
|
35
39
|
browserServiceApiKey,
|
|
36
40
|
browserServiceProxyId,
|
|
37
41
|
buildYouTubeChannelVideosUrl,
|
|
42
|
+
deleteKernelProxyId,
|
|
38
43
|
harvest,
|
|
39
44
|
resolveKernelProxyId
|
|
40
|
-
} from "./chunk-
|
|
45
|
+
} from "./chunk-MY3S7EX7.js";
|
|
41
46
|
import {
|
|
42
47
|
CaptchaError,
|
|
43
48
|
RECAPTCHA_INSTRUCTIONS,
|
|
44
49
|
sanitizeVendorName
|
|
45
|
-
} from "./chunk-
|
|
50
|
+
} from "./chunk-M2S27J6Z.js";
|
|
46
51
|
import {
|
|
47
52
|
SiteAuditJobRowSchema,
|
|
48
53
|
cancelJob,
|
|
@@ -3497,8 +3502,8 @@ import { chromium } from "playwright";
|
|
|
3497
3502
|
async function fetchWithKernel(url) {
|
|
3498
3503
|
const apiKey = browserServiceApiKey();
|
|
3499
3504
|
if (!apiKey) throw new Error("Browser backend API key not set");
|
|
3500
|
-
const
|
|
3501
|
-
const kb = await
|
|
3505
|
+
const client2 = new Kernel({ apiKey });
|
|
3506
|
+
const kb = await client2.browsers.create({ stealth: true, timeout_seconds: 60 });
|
|
3502
3507
|
const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
|
|
3503
3508
|
try {
|
|
3504
3509
|
const context = browser.contexts()[0] ?? await browser.newContext({
|
|
@@ -3511,7 +3516,7 @@ async function fetchWithKernel(url) {
|
|
|
3511
3516
|
} finally {
|
|
3512
3517
|
await browser.close().catch(() => {
|
|
3513
3518
|
});
|
|
3514
|
-
await
|
|
3519
|
+
await client2.browsers.deleteByID(kb.session_id).catch(() => {
|
|
3515
3520
|
});
|
|
3516
3521
|
}
|
|
3517
3522
|
}
|
|
@@ -4843,7 +4848,7 @@ async function extractSite(opts) {
|
|
|
4843
4848
|
}
|
|
4844
4849
|
|
|
4845
4850
|
// src/api/server.ts
|
|
4846
|
-
import { Hono as
|
|
4851
|
+
import { Hono as Hono11 } from "hono";
|
|
4847
4852
|
import { serve as serveInngest } from "inngest/hono";
|
|
4848
4853
|
|
|
4849
4854
|
// src/inngest/client.ts
|
|
@@ -8532,13 +8537,13 @@ var FacebookAdExtractor = class {
|
|
|
8532
8537
|
}
|
|
8533
8538
|
await page.waitForTimeout(1500);
|
|
8534
8539
|
let prevCount = 0;
|
|
8535
|
-
for (let
|
|
8540
|
+
for (let scroll2 = 0; scroll2 < 20; scroll2++) {
|
|
8536
8541
|
const count = await page.evaluate(() => {
|
|
8537
8542
|
const bt = document.body ? document.body.innerText ?? "" : "";
|
|
8538
8543
|
return [...bt.matchAll(/Library ID/g)].length;
|
|
8539
8544
|
});
|
|
8540
8545
|
if (count >= maxAds) break;
|
|
8541
|
-
if (count === prevCount &&
|
|
8546
|
+
if (count === prevCount && scroll2 > 0) break;
|
|
8542
8547
|
prevCount = count;
|
|
8543
8548
|
await page.evaluate(() => {
|
|
8544
8549
|
if (document.body) window.scrollTo(0, document.body.scrollHeight);
|
|
@@ -9184,7 +9189,7 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
|
9184
9189
|
return c.json(searchResult2);
|
|
9185
9190
|
}
|
|
9186
9191
|
await page.waitForTimeout(1500);
|
|
9187
|
-
for (let
|
|
9192
|
+
for (let scroll2 = 0; scroll2 < 3; scroll2++) {
|
|
9188
9193
|
await page.evaluate(() => {
|
|
9189
9194
|
if (document.body) window.scrollTo(0, document.body.scrollHeight);
|
|
9190
9195
|
});
|
|
@@ -9741,8 +9746,11 @@ var MapsSearchExtractor = class {
|
|
|
9741
9746
|
headless: options.headless,
|
|
9742
9747
|
kernelApiKey: options.kernelApiKey,
|
|
9743
9748
|
kernelProxyId: options.kernelProxyId,
|
|
9749
|
+
kernelProxyResolution: options.kernelProxyResolution,
|
|
9750
|
+
proxyMode: options.proxyMode,
|
|
9744
9751
|
viewport: { width: 1280, height: 900 },
|
|
9745
|
-
locale: `${options.hl}-${options.gl.toUpperCase()}
|
|
9752
|
+
locale: `${options.hl}-${options.gl.toUpperCase()}`,
|
|
9753
|
+
debug: options.debug
|
|
9746
9754
|
};
|
|
9747
9755
|
try {
|
|
9748
9756
|
await this.driver.launch(config);
|
|
@@ -9833,6 +9841,9 @@ var MapsSearchExtractor = class {
|
|
|
9833
9841
|
const value = parts.find((part) => pattern.test(part));
|
|
9834
9842
|
return value ?? null;
|
|
9835
9843
|
}
|
|
9844
|
+
function normalizedSet(values) {
|
|
9845
|
+
return new Set(values.filter(Boolean).map((value) => value.toLowerCase()));
|
|
9846
|
+
}
|
|
9836
9847
|
const out = [];
|
|
9837
9848
|
const seen = /* @__PURE__ */ new Set();
|
|
9838
9849
|
const anchors = Array.from(document.querySelectorAll('a[href*="/maps/place/"]'));
|
|
@@ -9848,11 +9859,17 @@ var MapsSearchExtractor = class {
|
|
|
9848
9859
|
const name = aria ?? heading ?? parts[0] ?? stableUrl;
|
|
9849
9860
|
const links = Array.from(card?.querySelectorAll("a[href]") ?? []);
|
|
9850
9861
|
const websiteUrl = links.find((link) => link.href.startsWith("http") && !link.href.includes("google."))?.href ?? null;
|
|
9851
|
-
const directionsUrl = links.find((link) => /google\.[^/]+\/maps\/dir|\/dir\//i.test(link.href))?.href ?? null;
|
|
9852
9862
|
const rating = firstMatching(parts, /^\d(?:\.\d)?$/);
|
|
9853
|
-
const reviewCountRaw = firstMatching(parts, /^\(?[\d,]+\)?$/);
|
|
9854
|
-
const
|
|
9863
|
+
const reviewCountRaw = firstMatching(parts, /^\(?[\d,]+\)?(?:\s+reviews?)?$/i);
|
|
9864
|
+
const phone = firstMatching(parts, /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/);
|
|
9865
|
+
const hoursStatus = parts.find((part) => /^(open|closed|closes|opens)\b|^·\s*(opens|closes)\b/i.test(part)) ?? null;
|
|
9855
9866
|
const address = parts.find((part) => /\b[A-Z]{2}\s+\d{5}\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Drive|Dr)\b/i.test(part)) ?? null;
|
|
9867
|
+
const directionsUrl = links.find((link) => /google\.[^/]+\/maps\/dir|\/dir\//i.test(link.href))?.href ?? `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent([name, address].filter(Boolean).join(", ") || name)}`;
|
|
9868
|
+
const excluded = normalizedSet([name, rating, reviewCountRaw, phone, hoursStatus, address, "Website", "Directions"]);
|
|
9869
|
+
const category = parts.find((part) => {
|
|
9870
|
+
const normalized = part.toLowerCase();
|
|
9871
|
+
return !excluded.has(normalized) && !/^\d(?:\.\d)?$|^\(?[\d,]+\)?(?:\s+reviews?)?$/i.test(part) && !/(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/.test(part) && !/\b[A-Z]{2}\s+\d{5}\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Drive|Dr)\b/i.test(part) && !/^(open|closed|closes|opens)\b|^·\s*(opens|closes)\b|directions|website|book online|sponsored|visit site|financing/i.test(part);
|
|
9872
|
+
}) ?? null;
|
|
9856
9873
|
const { cid, cidDecimal } = cidFromUrl(placeUrl);
|
|
9857
9874
|
out.push({
|
|
9858
9875
|
position: out.length + 1,
|
|
@@ -9864,6 +9881,8 @@ var MapsSearchExtractor = class {
|
|
|
9864
9881
|
reviewCount: reviewCountRaw ? reviewCountRaw.replace(/[()]/g, "") : null,
|
|
9865
9882
|
category,
|
|
9866
9883
|
address,
|
|
9884
|
+
phone,
|
|
9885
|
+
hoursStatus,
|
|
9867
9886
|
websiteUrl,
|
|
9868
9887
|
directionsUrl,
|
|
9869
9888
|
metadata: parts.slice(0, 20)
|
|
@@ -9883,12 +9902,22 @@ function mapsErrorResponse(c, msg, errorCode) {
|
|
|
9883
9902
|
retryable: blocked
|
|
9884
9903
|
}, blocked ? 503 : 500);
|
|
9885
9904
|
}
|
|
9905
|
+
async function cleanupDisposableProxy(kernelApiKey, proxyId) {
|
|
9906
|
+
if (!kernelApiKey || !proxyId) return;
|
|
9907
|
+
await deleteKernelProxyId(kernelApiKey, proxyId).catch((err) => {
|
|
9908
|
+
console.warn(JSON.stringify({
|
|
9909
|
+
event: "maps_search_proxy_delete_failed",
|
|
9910
|
+
proxy_id_suffix: proxyId.slice(-6),
|
|
9911
|
+
message: err instanceof Error ? err.message : String(err)
|
|
9912
|
+
}));
|
|
9913
|
+
});
|
|
9914
|
+
}
|
|
9886
9915
|
var mapsApp = new Hono5();
|
|
9887
9916
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
9888
9917
|
const user = c.get("user");
|
|
9889
9918
|
const body = await c.req.json().catch(() => ({}));
|
|
9890
9919
|
const parsed = MapsSearchOptionsSchema.safeParse({
|
|
9891
|
-
kernelApiKey:
|
|
9920
|
+
kernelApiKey: browserServiceApiKey(),
|
|
9892
9921
|
...body
|
|
9893
9922
|
});
|
|
9894
9923
|
if (!parsed.success) {
|
|
@@ -9903,8 +9932,23 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
|
9903
9932
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
9904
9933
|
const driver = new BrowserDriver();
|
|
9905
9934
|
const extractor = new MapsSearchExtractor(driver);
|
|
9935
|
+
let disposableProxyId;
|
|
9906
9936
|
try {
|
|
9907
|
-
const
|
|
9937
|
+
const resolution = await resolveKernelProxyId({
|
|
9938
|
+
kernelApiKey: parsed.data.kernelApiKey,
|
|
9939
|
+
proxyMode: parsed.data.proxyMode,
|
|
9940
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
9941
|
+
location: parsed.data.location,
|
|
9942
|
+
proxyZip: parsed.data.proxyZip,
|
|
9943
|
+
gl: parsed.data.gl,
|
|
9944
|
+
fresh: parsed.data.proxyMode === "location"
|
|
9945
|
+
});
|
|
9946
|
+
disposableProxyId = resolution.disposableProxyId;
|
|
9947
|
+
const result = await extractor.extract({
|
|
9948
|
+
...parsed.data,
|
|
9949
|
+
kernelProxyId: parsed.data.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
|
|
9950
|
+
kernelProxyResolution: resolution.resolution
|
|
9951
|
+
});
|
|
9908
9952
|
await logRequestEvent({
|
|
9909
9953
|
userId: user.id,
|
|
9910
9954
|
source: "maps_search",
|
|
@@ -9928,6 +9972,7 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
|
9928
9972
|
});
|
|
9929
9973
|
return mapsErrorResponse(c, msg, "maps_search_failed");
|
|
9930
9974
|
} finally {
|
|
9975
|
+
await cleanupDisposableProxy(parsed.data.kernelApiKey, disposableProxyId);
|
|
9931
9976
|
await driver.close();
|
|
9932
9977
|
}
|
|
9933
9978
|
});
|
|
@@ -9935,7 +9980,7 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
9935
9980
|
const user = c.get("user");
|
|
9936
9981
|
const body = await c.req.json().catch(() => ({}));
|
|
9937
9982
|
const parsed = MapsPlaceOptionsSchema.safeParse({
|
|
9938
|
-
kernelApiKey:
|
|
9983
|
+
kernelApiKey: browserServiceApiKey(),
|
|
9939
9984
|
...body
|
|
9940
9985
|
});
|
|
9941
9986
|
if (!parsed.success) {
|
|
@@ -10003,9 +10048,593 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
10003
10048
|
}
|
|
10004
10049
|
});
|
|
10005
10050
|
|
|
10006
|
-
// src/api/
|
|
10051
|
+
// src/api/directory-routes.ts
|
|
10007
10052
|
import { Hono as Hono6 } from "hono";
|
|
10008
10053
|
|
|
10054
|
+
// src/directory/directory-workflow.ts
|
|
10055
|
+
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
10056
|
+
import { join as join4 } from "path";
|
|
10057
|
+
import { z as z15 } from "zod";
|
|
10058
|
+
|
|
10059
|
+
// src/directory/csv.ts
|
|
10060
|
+
function parseCsv(text) {
|
|
10061
|
+
const rows = [];
|
|
10062
|
+
let row = [];
|
|
10063
|
+
let field = "";
|
|
10064
|
+
let quoted = false;
|
|
10065
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
10066
|
+
const ch = text[i];
|
|
10067
|
+
const next = text[i + 1];
|
|
10068
|
+
if (quoted) {
|
|
10069
|
+
if (ch === '"' && next === '"') {
|
|
10070
|
+
field += '"';
|
|
10071
|
+
i += 1;
|
|
10072
|
+
} else if (ch === '"') {
|
|
10073
|
+
quoted = false;
|
|
10074
|
+
} else {
|
|
10075
|
+
field += ch;
|
|
10076
|
+
}
|
|
10077
|
+
continue;
|
|
10078
|
+
}
|
|
10079
|
+
if (ch === '"') {
|
|
10080
|
+
quoted = true;
|
|
10081
|
+
} else if (ch === ",") {
|
|
10082
|
+
row.push(field);
|
|
10083
|
+
field = "";
|
|
10084
|
+
} else if (ch === "\n") {
|
|
10085
|
+
row.push(field);
|
|
10086
|
+
rows.push(row);
|
|
10087
|
+
row = [];
|
|
10088
|
+
field = "";
|
|
10089
|
+
} else if (ch !== "\r") {
|
|
10090
|
+
field += ch;
|
|
10091
|
+
}
|
|
10092
|
+
}
|
|
10093
|
+
if (field.length > 0 || row.length > 0) {
|
|
10094
|
+
row.push(field);
|
|
10095
|
+
rows.push(row);
|
|
10096
|
+
}
|
|
10097
|
+
return rows;
|
|
10098
|
+
}
|
|
10099
|
+
function csvRecords(text) {
|
|
10100
|
+
const rows = parseCsv(text).filter((row) => row.some((cell) => cell.trim() !== ""));
|
|
10101
|
+
const header = rows[0]?.map((cell) => cell.trim()) ?? [];
|
|
10102
|
+
return rows.slice(1).map((row) => {
|
|
10103
|
+
const record = {};
|
|
10104
|
+
for (let i = 0; i < header.length; i += 1) {
|
|
10105
|
+
record[header[i]] = row[i] ?? "";
|
|
10106
|
+
}
|
|
10107
|
+
return record;
|
|
10108
|
+
});
|
|
10109
|
+
}
|
|
10110
|
+
function csvCell(value) {
|
|
10111
|
+
if (value === null || value === void 0) return "";
|
|
10112
|
+
const text = String(value);
|
|
10113
|
+
return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
|
10114
|
+
}
|
|
10115
|
+
function rowsToCsv(headers, rows) {
|
|
10116
|
+
return [
|
|
10117
|
+
headers.join(","),
|
|
10118
|
+
...rows.map((row) => headers.map((header) => csvCell(row[header])).join(","))
|
|
10119
|
+
].join("\n") + "\n";
|
|
10120
|
+
}
|
|
10121
|
+
|
|
10122
|
+
// src/directory/location-db.ts
|
|
10123
|
+
import { access, readFile } from "fs/promises";
|
|
10124
|
+
var POPULATION_YEARS = [2020, 2021, 2022, 2023, 2024, 2025];
|
|
10125
|
+
var STATE_META = {
|
|
10126
|
+
AL: { abbr: "AL", fips: "01", name: "Alabama" },
|
|
10127
|
+
AK: { abbr: "AK", fips: "02", name: "Alaska" },
|
|
10128
|
+
AZ: { abbr: "AZ", fips: "04", name: "Arizona" },
|
|
10129
|
+
AR: { abbr: "AR", fips: "05", name: "Arkansas" },
|
|
10130
|
+
CA: { abbr: "CA", fips: "06", name: "California" },
|
|
10131
|
+
CO: { abbr: "CO", fips: "08", name: "Colorado" },
|
|
10132
|
+
CT: { abbr: "CT", fips: "09", name: "Connecticut" },
|
|
10133
|
+
DE: { abbr: "DE", fips: "10", name: "Delaware" },
|
|
10134
|
+
DC: { abbr: "DC", fips: "11", name: "District of Columbia" },
|
|
10135
|
+
FL: { abbr: "FL", fips: "12", name: "Florida" },
|
|
10136
|
+
GA: { abbr: "GA", fips: "13", name: "Georgia" },
|
|
10137
|
+
HI: { abbr: "HI", fips: "15", name: "Hawaii" },
|
|
10138
|
+
ID: { abbr: "ID", fips: "16", name: "Idaho" },
|
|
10139
|
+
IL: { abbr: "IL", fips: "17", name: "Illinois" },
|
|
10140
|
+
IN: { abbr: "IN", fips: "18", name: "Indiana" },
|
|
10141
|
+
IA: { abbr: "IA", fips: "19", name: "Iowa" },
|
|
10142
|
+
KS: { abbr: "KS", fips: "20", name: "Kansas" },
|
|
10143
|
+
KY: { abbr: "KY", fips: "21", name: "Kentucky" },
|
|
10144
|
+
LA: { abbr: "LA", fips: "22", name: "Louisiana" },
|
|
10145
|
+
ME: { abbr: "ME", fips: "23", name: "Maine" },
|
|
10146
|
+
MD: { abbr: "MD", fips: "24", name: "Maryland" },
|
|
10147
|
+
MA: { abbr: "MA", fips: "25", name: "Massachusetts" },
|
|
10148
|
+
MI: { abbr: "MI", fips: "26", name: "Michigan" },
|
|
10149
|
+
MN: { abbr: "MN", fips: "27", name: "Minnesota" },
|
|
10150
|
+
MS: { abbr: "MS", fips: "28", name: "Mississippi" },
|
|
10151
|
+
MO: { abbr: "MO", fips: "29", name: "Missouri" },
|
|
10152
|
+
MT: { abbr: "MT", fips: "30", name: "Montana" },
|
|
10153
|
+
NE: { abbr: "NE", fips: "31", name: "Nebraska" },
|
|
10154
|
+
NV: { abbr: "NV", fips: "32", name: "Nevada" },
|
|
10155
|
+
NH: { abbr: "NH", fips: "33", name: "New Hampshire" },
|
|
10156
|
+
NJ: { abbr: "NJ", fips: "34", name: "New Jersey" },
|
|
10157
|
+
NM: { abbr: "NM", fips: "35", name: "New Mexico" },
|
|
10158
|
+
NY: { abbr: "NY", fips: "36", name: "New York" },
|
|
10159
|
+
NC: { abbr: "NC", fips: "37", name: "North Carolina" },
|
|
10160
|
+
ND: { abbr: "ND", fips: "38", name: "North Dakota" },
|
|
10161
|
+
OH: { abbr: "OH", fips: "39", name: "Ohio" },
|
|
10162
|
+
OK: { abbr: "OK", fips: "40", name: "Oklahoma" },
|
|
10163
|
+
OR: { abbr: "OR", fips: "41", name: "Oregon" },
|
|
10164
|
+
PA: { abbr: "PA", fips: "42", name: "Pennsylvania" },
|
|
10165
|
+
RI: { abbr: "RI", fips: "44", name: "Rhode Island" },
|
|
10166
|
+
SC: { abbr: "SC", fips: "45", name: "South Carolina" },
|
|
10167
|
+
SD: { abbr: "SD", fips: "46", name: "South Dakota" },
|
|
10168
|
+
TN: { abbr: "TN", fips: "47", name: "Tennessee" },
|
|
10169
|
+
TX: { abbr: "TX", fips: "48", name: "Texas" },
|
|
10170
|
+
UT: { abbr: "UT", fips: "49", name: "Utah" },
|
|
10171
|
+
VT: { abbr: "VT", fips: "50", name: "Vermont" },
|
|
10172
|
+
VA: { abbr: "VA", fips: "51", name: "Virginia" },
|
|
10173
|
+
WA: { abbr: "WA", fips: "53", name: "Washington" },
|
|
10174
|
+
WV: { abbr: "WV", fips: "54", name: "West Virginia" },
|
|
10175
|
+
WI: { abbr: "WI", fips: "55", name: "Wisconsin" },
|
|
10176
|
+
WY: { abbr: "WY", fips: "56", name: "Wyoming" }
|
|
10177
|
+
};
|
|
10178
|
+
var STATE_BY_NAME = new Map(Object.values(STATE_META).map((s) => [s.name.toLowerCase(), s]));
|
|
10179
|
+
function normalizeState(input) {
|
|
10180
|
+
const raw = input.trim();
|
|
10181
|
+
const byAbbr = STATE_META[raw.toUpperCase()];
|
|
10182
|
+
if (byAbbr) return byAbbr;
|
|
10183
|
+
const byName = STATE_BY_NAME.get(raw.toLowerCase());
|
|
10184
|
+
if (byName) return byName;
|
|
10185
|
+
throw new Error(`Unsupported state "${input}". Use a US state abbreviation such as TN.`);
|
|
10186
|
+
}
|
|
10187
|
+
function censusStateUrl(fips) {
|
|
10188
|
+
return `https://www2.census.gov/programs-surveys/popest/datasets/2020-2025/cities/totals/sub-est2025_${fips}.csv`;
|
|
10189
|
+
}
|
|
10190
|
+
function normalizeCityKey(value) {
|
|
10191
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
10192
|
+
}
|
|
10193
|
+
function displayCityFromCensus(name) {
|
|
10194
|
+
if (/^Nashville-Davidson metropolitan government/i.test(name)) return "Nashville";
|
|
10195
|
+
return name.replace(/\s+(city|town|village|municipality|borough)$/i, "").trim();
|
|
10196
|
+
}
|
|
10197
|
+
function numberOrNull(value) {
|
|
10198
|
+
if (value === void 0 || value.trim() === "") return null;
|
|
10199
|
+
const n = Number(value);
|
|
10200
|
+
return Number.isFinite(n) ? n : null;
|
|
10201
|
+
}
|
|
10202
|
+
function localLocationFileAllowed() {
|
|
10203
|
+
if (process.env.MCP_SCRAPER_ALLOW_LOCAL_LOCATION_FILES === "true") return true;
|
|
10204
|
+
if (process.env.VERCEL === "1" || process.env.NODE_ENV === "production") return false;
|
|
10205
|
+
return true;
|
|
10206
|
+
}
|
|
10207
|
+
async function existingPath(value) {
|
|
10208
|
+
const trimmed = value?.trim();
|
|
10209
|
+
if (!trimmed) return null;
|
|
10210
|
+
await access(trimmed);
|
|
10211
|
+
return trimmed;
|
|
10212
|
+
}
|
|
10213
|
+
async function resolveUsZipsPath(requestedPath) {
|
|
10214
|
+
const envPath = process.env.MCP_SCRAPER_USZIPS_CSV_PATH;
|
|
10215
|
+
if (requestedPath && !localLocationFileAllowed()) {
|
|
10216
|
+
throw new Error("usZipsCsvPath is only accepted in local/test mode. Set MCP_SCRAPER_USZIPS_CSV_PATH on the server for deployed use.");
|
|
10217
|
+
}
|
|
10218
|
+
const source = requestedPath ?? envPath;
|
|
10219
|
+
if (!source) return null;
|
|
10220
|
+
return existingPath(source);
|
|
10221
|
+
}
|
|
10222
|
+
async function loadZipGroups(stateAbbr, requestedPath, warnings) {
|
|
10223
|
+
if (!requestedPath && !process.env.MCP_SCRAPER_USZIPS_CSV_PATH) {
|
|
10224
|
+
return { path: null, groups: /* @__PURE__ */ new Map() };
|
|
10225
|
+
}
|
|
10226
|
+
const path5 = await resolveUsZipsPath(requestedPath);
|
|
10227
|
+
if (!path5) return { path: null, groups: /* @__PURE__ */ new Map() };
|
|
10228
|
+
const records = csvRecords(await readFile(path5, "utf8"));
|
|
10229
|
+
const groups = /* @__PURE__ */ new Map();
|
|
10230
|
+
for (const record of records) {
|
|
10231
|
+
const state = (record.state_abbr ?? record.state ?? "").trim().toUpperCase();
|
|
10232
|
+
const zip = (record.zipcode ?? record.zip ?? record.zip_code ?? "").trim();
|
|
10233
|
+
const city = (record.city ?? "").trim();
|
|
10234
|
+
const county = (record.county ?? "").trim();
|
|
10235
|
+
if (state !== stateAbbr || !zip || !city) continue;
|
|
10236
|
+
const key = normalizeCityKey(city);
|
|
10237
|
+
if (!groups.has(key)) groups.set(key, { zips: /* @__PURE__ */ new Set(), counties: /* @__PURE__ */ new Set() });
|
|
10238
|
+
const group = groups.get(key);
|
|
10239
|
+
group?.zips.add(zip);
|
|
10240
|
+
if (county) group?.counties.add(county);
|
|
10241
|
+
}
|
|
10242
|
+
if (!groups.size) warnings.push(`No ${stateAbbr} ZIP groups found in ${path5}`);
|
|
10243
|
+
return { path: path5, groups };
|
|
10244
|
+
}
|
|
10245
|
+
async function resolveDirectoryMarkets(options) {
|
|
10246
|
+
const state = normalizeState(options.state);
|
|
10247
|
+
const sourceUrl = censusStateUrl(state.fips);
|
|
10248
|
+
const warnings = [];
|
|
10249
|
+
const response = await fetch(sourceUrl);
|
|
10250
|
+
if (!response.ok) throw new Error(`Census location dataset request failed: ${response.status} ${response.statusText}`);
|
|
10251
|
+
const records = csvRecords(await response.text());
|
|
10252
|
+
const populationField = `POPESTIMATE${options.populationYear}`;
|
|
10253
|
+
const zipData = options.includeZipGroups ? await loadZipGroups(state.abbr, options.usZipsCsvPath, warnings) : { path: null, groups: /* @__PURE__ */ new Map() };
|
|
10254
|
+
const markets = records.filter((record) => record.SUMLEV === "162").map((record) => {
|
|
10255
|
+
const population = numberOrNull(record[populationField]);
|
|
10256
|
+
if (population === null || population < options.minPopulation) return null;
|
|
10257
|
+
const censusName = record.NAME?.trim() ?? "";
|
|
10258
|
+
if (!censusName) return null;
|
|
10259
|
+
const city = displayCityFromCensus(censusName);
|
|
10260
|
+
const zipGroup = zipData.groups.get(normalizeCityKey(city));
|
|
10261
|
+
return {
|
|
10262
|
+
city,
|
|
10263
|
+
state: state.abbr,
|
|
10264
|
+
location: `${city}, ${state.abbr}`,
|
|
10265
|
+
cityKey: `${city}|${state.abbr}`,
|
|
10266
|
+
censusName,
|
|
10267
|
+
population,
|
|
10268
|
+
populationYear: options.populationYear,
|
|
10269
|
+
estimatesBase2020: numberOrNull(record.ESTIMATESBASE2020),
|
|
10270
|
+
zips: zipGroup ? [...zipGroup.zips].sort() : [],
|
|
10271
|
+
counties: zipGroup ? [...zipGroup.counties].sort() : []
|
|
10272
|
+
};
|
|
10273
|
+
}).filter((market) => market !== null).sort((a, b) => b.population - a.population || a.city.localeCompare(b.city)).slice(0, options.maxCities);
|
|
10274
|
+
if (options.includeZipGroups && zipData.path && markets.some((m) => m.zips.length === 0)) {
|
|
10275
|
+
warnings.push("Some Census places did not match the configured US ZIPS city names.");
|
|
10276
|
+
}
|
|
10277
|
+
return { markets, censusSourceUrl: sourceUrl, usZipsSourcePath: zipData.path, warnings };
|
|
10278
|
+
}
|
|
10279
|
+
|
|
10280
|
+
// src/directory/directory-workflow.ts
|
|
10281
|
+
var DIRECTORY_MAX_ATTEMPTS = 3;
|
|
10282
|
+
var DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS = 5;
|
|
10283
|
+
var DirectoryWorkflowOptionsSchema = z15.object({
|
|
10284
|
+
query: z15.string().min(1),
|
|
10285
|
+
state: z15.string().min(2).default("TN"),
|
|
10286
|
+
minPopulation: z15.number().int().min(0).default(1e5),
|
|
10287
|
+
populationYear: z15.union(POPULATION_YEARS.map((year) => z15.literal(year))).default(2025),
|
|
10288
|
+
maxCities: z15.number().int().min(1).max(100).default(25),
|
|
10289
|
+
maxResultsPerCity: z15.number().int().min(1).max(50).default(50),
|
|
10290
|
+
concurrency: z15.number().int().min(1).max(5).default(5),
|
|
10291
|
+
includeZipGroups: z15.boolean().default(true),
|
|
10292
|
+
usZipsCsvPath: z15.string().optional(),
|
|
10293
|
+
saveCsv: z15.boolean().default(true),
|
|
10294
|
+
gl: z15.string().length(2).default("us"),
|
|
10295
|
+
hl: z15.string().length(2).default("en"),
|
|
10296
|
+
proxyMode: z15.enum(["location", "configured", "none"]).default("location"),
|
|
10297
|
+
proxyZip: z15.string().regex(/^\d{5}$/).optional(),
|
|
10298
|
+
debug: z15.boolean().default(false),
|
|
10299
|
+
headless: z15.boolean().default(true),
|
|
10300
|
+
kernelApiKey: z15.string().optional()
|
|
10301
|
+
});
|
|
10302
|
+
async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
|
|
10303
|
+
if (!kernelApiKey || !proxyId) return;
|
|
10304
|
+
try {
|
|
10305
|
+
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
10306
|
+
} catch (err) {
|
|
10307
|
+
console.warn(JSON.stringify({
|
|
10308
|
+
event: "directory_workflow_proxy_delete_failed",
|
|
10309
|
+
proxy_id_suffix: proxyId.slice(-6),
|
|
10310
|
+
message: err instanceof Error ? err.message : String(err)
|
|
10311
|
+
}));
|
|
10312
|
+
}
|
|
10313
|
+
}
|
|
10314
|
+
function maxAttemptsForProxyMode(proxyMode) {
|
|
10315
|
+
return proxyMode === "location" ? DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS : DIRECTORY_MAX_ATTEMPTS;
|
|
10316
|
+
}
|
|
10317
|
+
function errorMessage(err) {
|
|
10318
|
+
return err instanceof Error ? err.message : String(err);
|
|
10319
|
+
}
|
|
10320
|
+
function looksLikeProxyTunnelFailure(message) {
|
|
10321
|
+
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
10322
|
+
}
|
|
10323
|
+
function looksLikeProxyUnavailable(message) {
|
|
10324
|
+
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
10325
|
+
}
|
|
10326
|
+
function retryableCitySearchError(err, proxyMode) {
|
|
10327
|
+
if (err instanceof CaptchaError) return true;
|
|
10328
|
+
const message = errorMessage(err);
|
|
10329
|
+
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
|
|
10330
|
+
return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
|
|
10331
|
+
}
|
|
10332
|
+
function proxyZipForAttempt(options, market, attemptIndex) {
|
|
10333
|
+
if (options.proxyZip) return options.proxyZip;
|
|
10334
|
+
if (!market.zips.length) return void 0;
|
|
10335
|
+
return market.zips[attemptIndex % market.zips.length];
|
|
10336
|
+
}
|
|
10337
|
+
async function mapLimit(items, limit, fn) {
|
|
10338
|
+
const out = new Array(items.length);
|
|
10339
|
+
let next = 0;
|
|
10340
|
+
async function worker() {
|
|
10341
|
+
while (next < items.length) {
|
|
10342
|
+
const index = next;
|
|
10343
|
+
next += 1;
|
|
10344
|
+
out[index] = await fn(items[index]);
|
|
10345
|
+
}
|
|
10346
|
+
}
|
|
10347
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
10348
|
+
return out;
|
|
10349
|
+
}
|
|
10350
|
+
async function searchCityAttempt(options, market, attemptIndex) {
|
|
10351
|
+
const driver = new BrowserDriver();
|
|
10352
|
+
const extractor = new MapsSearchExtractor(driver);
|
|
10353
|
+
const start = Date.now();
|
|
10354
|
+
let disposableProxyId;
|
|
10355
|
+
try {
|
|
10356
|
+
const proxyZip = proxyZipForAttempt(options, market, attemptIndex);
|
|
10357
|
+
const resolution = await resolveKernelProxyId({
|
|
10358
|
+
kernelApiKey: options.kernelApiKey,
|
|
10359
|
+
proxyMode: options.proxyMode,
|
|
10360
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
10361
|
+
location: market.location,
|
|
10362
|
+
proxyZip,
|
|
10363
|
+
gl: options.gl,
|
|
10364
|
+
attemptIndex,
|
|
10365
|
+
fresh: options.proxyMode === "location"
|
|
10366
|
+
});
|
|
10367
|
+
disposableProxyId = resolution.disposableProxyId;
|
|
10368
|
+
const result = await extractor.extract({
|
|
10369
|
+
query: options.query,
|
|
10370
|
+
location: market.location,
|
|
10371
|
+
gl: options.gl,
|
|
10372
|
+
hl: options.hl,
|
|
10373
|
+
maxResults: options.maxResultsPerCity,
|
|
10374
|
+
headless: options.headless,
|
|
10375
|
+
kernelApiKey: options.kernelApiKey,
|
|
10376
|
+
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
|
|
10377
|
+
kernelProxyResolution: resolution.resolution,
|
|
10378
|
+
proxyMode: options.proxyMode,
|
|
10379
|
+
proxyZip,
|
|
10380
|
+
debug: options.debug
|
|
10381
|
+
});
|
|
10382
|
+
return {
|
|
10383
|
+
city: market.city,
|
|
10384
|
+
state: market.state,
|
|
10385
|
+
location: market.location,
|
|
10386
|
+
cityKey: market.cityKey,
|
|
10387
|
+
censusName: market.censusName,
|
|
10388
|
+
population: market.population,
|
|
10389
|
+
populationYear: market.populationYear,
|
|
10390
|
+
zips: market.zips,
|
|
10391
|
+
counties: market.counties,
|
|
10392
|
+
status: result.results.length ? "ok" : "empty",
|
|
10393
|
+
error: null,
|
|
10394
|
+
resultCount: result.resultCount,
|
|
10395
|
+
durationMs: result.durationMs,
|
|
10396
|
+
results: result.results
|
|
10397
|
+
};
|
|
10398
|
+
} finally {
|
|
10399
|
+
await cleanupDisposableProxy2(options.kernelApiKey, disposableProxyId);
|
|
10400
|
+
}
|
|
10401
|
+
}
|
|
10402
|
+
async function searchCity(options, market) {
|
|
10403
|
+
const started = Date.now();
|
|
10404
|
+
const maxAttempts = maxAttemptsForProxyMode(options.proxyMode);
|
|
10405
|
+
let lastError = null;
|
|
10406
|
+
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
10407
|
+
try {
|
|
10408
|
+
return await searchCityAttempt(options, market, attemptIndex);
|
|
10409
|
+
} catch (err) {
|
|
10410
|
+
lastError = err;
|
|
10411
|
+
const willRetry = attemptIndex < maxAttempts - 1 && retryableCitySearchError(err, options.proxyMode);
|
|
10412
|
+
console.warn(JSON.stringify({
|
|
10413
|
+
event: "directory_workflow_city_attempt_failed",
|
|
10414
|
+
city: market.city,
|
|
10415
|
+
state: market.state,
|
|
10416
|
+
attempt_number: attemptIndex + 1,
|
|
10417
|
+
max_attempts: maxAttempts,
|
|
10418
|
+
will_retry: willRetry,
|
|
10419
|
+
message: errorMessage(err)
|
|
10420
|
+
}));
|
|
10421
|
+
if (!willRetry) break;
|
|
10422
|
+
}
|
|
10423
|
+
}
|
|
10424
|
+
return {
|
|
10425
|
+
city: market.city,
|
|
10426
|
+
state: market.state,
|
|
10427
|
+
location: market.location,
|
|
10428
|
+
cityKey: market.cityKey,
|
|
10429
|
+
censusName: market.censusName,
|
|
10430
|
+
population: market.population,
|
|
10431
|
+
populationYear: market.populationYear,
|
|
10432
|
+
zips: market.zips,
|
|
10433
|
+
counties: market.counties,
|
|
10434
|
+
status: "failed",
|
|
10435
|
+
error: lastError ? errorMessage(lastError) : "City Maps search failed",
|
|
10436
|
+
resultCount: 0,
|
|
10437
|
+
durationMs: Date.now() - started,
|
|
10438
|
+
results: []
|
|
10439
|
+
};
|
|
10440
|
+
}
|
|
10441
|
+
function csvRowsFor(result) {
|
|
10442
|
+
const rows = [];
|
|
10443
|
+
for (const city of result.cities) {
|
|
10444
|
+
if (!city.results.length) {
|
|
10445
|
+
rows.push({
|
|
10446
|
+
source_query: result.query,
|
|
10447
|
+
source_location: city.location,
|
|
10448
|
+
city: city.city,
|
|
10449
|
+
state: city.state,
|
|
10450
|
+
city_key: city.cityKey,
|
|
10451
|
+
census_name: city.censusName,
|
|
10452
|
+
population: city.population,
|
|
10453
|
+
population_year: city.populationYear,
|
|
10454
|
+
zip_count: city.zips.length,
|
|
10455
|
+
zips: city.zips.join(" "),
|
|
10456
|
+
counties: city.counties.join(" | "),
|
|
10457
|
+
result_position: null,
|
|
10458
|
+
business_name: null,
|
|
10459
|
+
review_stars: null,
|
|
10460
|
+
category: null,
|
|
10461
|
+
address: null,
|
|
10462
|
+
phone: null,
|
|
10463
|
+
hours_status: null,
|
|
10464
|
+
website_url: null,
|
|
10465
|
+
directions_url: null,
|
|
10466
|
+
place_url: null,
|
|
10467
|
+
cid: null,
|
|
10468
|
+
cid_decimal: null,
|
|
10469
|
+
metadata: null,
|
|
10470
|
+
result_status: city.status,
|
|
10471
|
+
error: city.error,
|
|
10472
|
+
extracted_at: result.extractedAt,
|
|
10473
|
+
duration_ms: city.durationMs
|
|
10474
|
+
});
|
|
10475
|
+
continue;
|
|
10476
|
+
}
|
|
10477
|
+
for (const business of city.results) {
|
|
10478
|
+
rows.push({
|
|
10479
|
+
source_query: result.query,
|
|
10480
|
+
source_location: city.location,
|
|
10481
|
+
city: city.city,
|
|
10482
|
+
state: city.state,
|
|
10483
|
+
city_key: city.cityKey,
|
|
10484
|
+
census_name: city.censusName,
|
|
10485
|
+
population: city.population,
|
|
10486
|
+
population_year: city.populationYear,
|
|
10487
|
+
zip_count: city.zips.length,
|
|
10488
|
+
zips: city.zips.join(" "),
|
|
10489
|
+
counties: city.counties.join(" | "),
|
|
10490
|
+
result_position: business.position,
|
|
10491
|
+
business_name: business.name,
|
|
10492
|
+
review_stars: business.rating,
|
|
10493
|
+
category: business.category,
|
|
10494
|
+
address: business.address,
|
|
10495
|
+
phone: business.phone,
|
|
10496
|
+
hours_status: business.hoursStatus,
|
|
10497
|
+
website_url: business.websiteUrl,
|
|
10498
|
+
directions_url: business.directionsUrl,
|
|
10499
|
+
place_url: business.placeUrl,
|
|
10500
|
+
cid: business.cid,
|
|
10501
|
+
cid_decimal: business.cidDecimal,
|
|
10502
|
+
metadata: business.metadata.join(" | "),
|
|
10503
|
+
result_status: city.status,
|
|
10504
|
+
error: city.error,
|
|
10505
|
+
extracted_at: result.extractedAt,
|
|
10506
|
+
duration_ms: city.durationMs
|
|
10507
|
+
});
|
|
10508
|
+
}
|
|
10509
|
+
}
|
|
10510
|
+
return rows;
|
|
10511
|
+
}
|
|
10512
|
+
async function saveDirectoryCsv(result) {
|
|
10513
|
+
const outDir = join4(outputBaseDir(), "directory-workflows");
|
|
10514
|
+
await mkdir2(outDir, { recursive: true });
|
|
10515
|
+
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
10516
|
+
const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
10517
|
+
const path5 = join4(outDir, `${stamp}-${slug}-directory-workflow.csv`);
|
|
10518
|
+
const headers = [
|
|
10519
|
+
"source_query",
|
|
10520
|
+
"source_location",
|
|
10521
|
+
"city",
|
|
10522
|
+
"state",
|
|
10523
|
+
"city_key",
|
|
10524
|
+
"census_name",
|
|
10525
|
+
"population",
|
|
10526
|
+
"population_year",
|
|
10527
|
+
"zip_count",
|
|
10528
|
+
"zips",
|
|
10529
|
+
"counties",
|
|
10530
|
+
"result_position",
|
|
10531
|
+
"business_name",
|
|
10532
|
+
"review_stars",
|
|
10533
|
+
"category",
|
|
10534
|
+
"address",
|
|
10535
|
+
"phone",
|
|
10536
|
+
"hours_status",
|
|
10537
|
+
"website_url",
|
|
10538
|
+
"directions_url",
|
|
10539
|
+
"place_url",
|
|
10540
|
+
"cid",
|
|
10541
|
+
"cid_decimal",
|
|
10542
|
+
"metadata",
|
|
10543
|
+
"result_status",
|
|
10544
|
+
"error",
|
|
10545
|
+
"extracted_at",
|
|
10546
|
+
"duration_ms"
|
|
10547
|
+
];
|
|
10548
|
+
await writeFile(path5, rowsToCsv(headers, csvRowsFor(result)), "utf8");
|
|
10549
|
+
return path5;
|
|
10550
|
+
}
|
|
10551
|
+
async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
10552
|
+
const started = Date.now();
|
|
10553
|
+
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10554
|
+
const cities = await mapLimit(plan.markets, options.concurrency, (market) => searchCity(options, market));
|
|
10555
|
+
const base = {
|
|
10556
|
+
query: options.query,
|
|
10557
|
+
state: plan.markets[0]?.state ?? options.state.toUpperCase(),
|
|
10558
|
+
minPopulation: options.minPopulation,
|
|
10559
|
+
populationYear: options.populationYear,
|
|
10560
|
+
maxResultsPerCity: options.maxResultsPerCity,
|
|
10561
|
+
concurrency: options.concurrency,
|
|
10562
|
+
censusSourceUrl: plan.censusSourceUrl,
|
|
10563
|
+
usZipsSourcePath: plan.usZipsSourcePath,
|
|
10564
|
+
warnings: plan.warnings,
|
|
10565
|
+
extractedAt,
|
|
10566
|
+
selectedCityCount: plan.markets.length,
|
|
10567
|
+
totalResultCount: cities.reduce((sum, city) => sum + city.resultCount, 0),
|
|
10568
|
+
cities,
|
|
10569
|
+
durationMs: Date.now() - started
|
|
10570
|
+
};
|
|
10571
|
+
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
10572
|
+
return { ...base, csvPath };
|
|
10573
|
+
}
|
|
10574
|
+
|
|
10575
|
+
// src/api/directory-routes.ts
|
|
10576
|
+
var directoryApp = new Hono6();
|
|
10577
|
+
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
10578
|
+
const user = c.get("user");
|
|
10579
|
+
const body = await c.req.json().catch(() => ({}));
|
|
10580
|
+
const kernelApiKey = browserServiceApiKey();
|
|
10581
|
+
const parsed = DirectoryWorkflowOptionsSchema.safeParse({
|
|
10582
|
+
...body,
|
|
10583
|
+
kernelApiKey
|
|
10584
|
+
});
|
|
10585
|
+
if (!parsed.success) {
|
|
10586
|
+
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
10587
|
+
}
|
|
10588
|
+
if (!kernelApiKey && parsed.data.proxyMode !== "none") {
|
|
10589
|
+
return c.json({ error: "Browser service API key is required for directory workflow Maps searches unless proxyMode is none" }, 503);
|
|
10590
|
+
}
|
|
10591
|
+
const plan = await resolveDirectoryMarkets(parsed.data);
|
|
10592
|
+
const requiredMc = plan.markets.length * MC_COSTS.maps_search;
|
|
10593
|
+
if (requiredMc > 0) {
|
|
10594
|
+
const debit = await debitMc(
|
|
10595
|
+
user.id,
|
|
10596
|
+
requiredMc,
|
|
10597
|
+
LedgerOperation.MAPS_SEARCH,
|
|
10598
|
+
`directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
|
|
10599
|
+
);
|
|
10600
|
+
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
10601
|
+
}
|
|
10602
|
+
try {
|
|
10603
|
+
const result = await runDirectoryWorkflowFromPlan(parsed.data, plan);
|
|
10604
|
+
const failedCities = result.cities.filter((city) => city.status === "failed").length;
|
|
10605
|
+
if (failedCities > 0) {
|
|
10606
|
+
await creditMc(user.id, failedCities * MC_COSTS.maps_search, LedgerOperation.REFUND, "failed directory_workflow city maps searches");
|
|
10607
|
+
}
|
|
10608
|
+
await logRequestEvent({
|
|
10609
|
+
userId: user.id,
|
|
10610
|
+
source: "directory_workflow",
|
|
10611
|
+
status: failedCities === result.cities.length && result.cities.length > 0 ? "failed" : "done",
|
|
10612
|
+
query: result.query,
|
|
10613
|
+
location: result.state,
|
|
10614
|
+
resultCount: result.totalResultCount,
|
|
10615
|
+
result
|
|
10616
|
+
});
|
|
10617
|
+
return c.json(result);
|
|
10618
|
+
} catch (err) {
|
|
10619
|
+
if (requiredMc > 0) {
|
|
10620
|
+
await creditMc(user.id, requiredMc, LedgerOperation.REFUND, "failed directory_workflow call");
|
|
10621
|
+
}
|
|
10622
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
10623
|
+
await logRequestEvent({
|
|
10624
|
+
userId: user.id,
|
|
10625
|
+
source: "directory_workflow",
|
|
10626
|
+
status: "failed",
|
|
10627
|
+
query: parsed.data.query,
|
|
10628
|
+
location: parsed.data.state,
|
|
10629
|
+
error: message
|
|
10630
|
+
});
|
|
10631
|
+
return c.json({ error: message, error_code: "directory_workflow_failed", retryable: true }, 500);
|
|
10632
|
+
}
|
|
10633
|
+
});
|
|
10634
|
+
|
|
10635
|
+
// src/api/serp-intelligence-routes.ts
|
|
10636
|
+
import { Hono as Hono7 } from "hono";
|
|
10637
|
+
|
|
10009
10638
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
10010
10639
|
import { createHash } from "crypto";
|
|
10011
10640
|
import pLimit3 from "p-limit";
|
|
@@ -10316,7 +10945,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
10316
10945
|
}
|
|
10317
10946
|
|
|
10318
10947
|
// src/serp-intelligence/schemas.ts
|
|
10319
|
-
import { z as
|
|
10948
|
+
import { z as z16 } from "zod";
|
|
10320
10949
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
10321
10950
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
10322
10951
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -10328,6 +10957,8 @@ var SerpIntelligenceAttemptOutcomeValues = [
|
|
|
10328
10957
|
"request_aborted",
|
|
10329
10958
|
"timeout",
|
|
10330
10959
|
"location_mismatch",
|
|
10960
|
+
"proxy_tunnel_failed",
|
|
10961
|
+
"proxy_unavailable",
|
|
10331
10962
|
"mcp_unavailable",
|
|
10332
10963
|
"error"
|
|
10333
10964
|
];
|
|
@@ -10376,171 +11007,171 @@ function isPublicHttpUrl(value) {
|
|
|
10376
11007
|
return false;
|
|
10377
11008
|
}
|
|
10378
11009
|
}
|
|
10379
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
10380
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
10381
|
-
query:
|
|
10382
|
-
location:
|
|
10383
|
-
gl:
|
|
10384
|
-
hl:
|
|
10385
|
-
device:
|
|
10386
|
-
proxyMode:
|
|
10387
|
-
proxyZip:
|
|
10388
|
-
pages:
|
|
10389
|
-
debug:
|
|
10390
|
-
includePageSnapshots:
|
|
10391
|
-
pageSnapshotLimit:
|
|
11010
|
+
var SerpIntelligencePublicHttpUrlSchema = z16.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
11011
|
+
var SerpIntelligenceCaptureBodySchema = z16.object({
|
|
11012
|
+
query: z16.string().trim().min(1, "query is required"),
|
|
11013
|
+
location: z16.string().trim().min(1).optional(),
|
|
11014
|
+
gl: z16.string().trim().length(2).default("us"),
|
|
11015
|
+
hl: z16.string().trim().length(2).default("en"),
|
|
11016
|
+
device: z16.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
11017
|
+
proxyMode: z16.enum(SerpIntelligenceProxyModeValues).default("location"),
|
|
11018
|
+
proxyZip: z16.string().regex(/^\d{5}$/).optional(),
|
|
11019
|
+
pages: z16.number().int().min(1).max(2).default(1),
|
|
11020
|
+
debug: z16.boolean().default(false),
|
|
11021
|
+
includePageSnapshots: z16.boolean().default(false),
|
|
11022
|
+
pageSnapshotLimit: z16.number().int().min(0).max(10).default(0)
|
|
10392
11023
|
}).strict();
|
|
10393
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
11024
|
+
var SerpIntelligencePageSnapshotRequestSchema = z16.object({
|
|
10394
11025
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
10395
|
-
sourceKind:
|
|
10396
|
-
sourcePosition:
|
|
11026
|
+
sourceKind: z16.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
11027
|
+
sourcePosition: z16.number().int().min(1).optional()
|
|
10397
11028
|
}).strict();
|
|
10398
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
10399
|
-
urls:
|
|
10400
|
-
targets:
|
|
10401
|
-
maxConcurrency:
|
|
10402
|
-
timeoutMs:
|
|
10403
|
-
debug:
|
|
11029
|
+
var SerpIntelligencePageSnapshotsBodySchema = z16.object({
|
|
11030
|
+
urls: z16.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
11031
|
+
targets: z16.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
11032
|
+
maxConcurrency: z16.number().int().min(1).max(5).default(2),
|
|
11033
|
+
timeoutMs: z16.number().int().min(1e3).max(6e4).default(15e3),
|
|
11034
|
+
debug: z16.boolean().default(false)
|
|
10404
11035
|
}).strict();
|
|
10405
|
-
var SerpIntelligenceAICitationSchema =
|
|
10406
|
-
text:
|
|
10407
|
-
href:
|
|
11036
|
+
var SerpIntelligenceAICitationSchema = z16.object({
|
|
11037
|
+
text: z16.string(),
|
|
11038
|
+
href: z16.string()
|
|
10408
11039
|
}).strict();
|
|
10409
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
10410
|
-
position:
|
|
10411
|
-
title:
|
|
10412
|
-
url:
|
|
10413
|
-
domain:
|
|
10414
|
-
cite:
|
|
10415
|
-
snippet:
|
|
10416
|
-
isRedditStyle:
|
|
10417
|
-
inlineRating:
|
|
10418
|
-
value:
|
|
10419
|
-
count:
|
|
11040
|
+
var SerpIntelligenceOrganicResultSchema = z16.object({
|
|
11041
|
+
position: z16.number().int().min(1),
|
|
11042
|
+
title: z16.string(),
|
|
11043
|
+
url: z16.string(),
|
|
11044
|
+
domain: z16.string(),
|
|
11045
|
+
cite: z16.string().nullable(),
|
|
11046
|
+
snippet: z16.string().nullable(),
|
|
11047
|
+
isRedditStyle: z16.boolean(),
|
|
11048
|
+
inlineRating: z16.object({
|
|
11049
|
+
value: z16.string(),
|
|
11050
|
+
count: z16.string()
|
|
10420
11051
|
}).strict().nullable()
|
|
10421
11052
|
}).strict();
|
|
10422
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
10423
|
-
status:
|
|
10424
|
-
expected:
|
|
10425
|
-
city:
|
|
10426
|
-
regionCode:
|
|
10427
|
-
canonicalLocation:
|
|
11053
|
+
var SerpIntelligenceLocationEvidenceSchema = z16.object({
|
|
11054
|
+
status: z16.enum(SerpIntelligenceLocalizationStatusValues),
|
|
11055
|
+
expected: z16.object({
|
|
11056
|
+
city: z16.string(),
|
|
11057
|
+
regionCode: z16.string().nullable(),
|
|
11058
|
+
canonicalLocation: z16.string()
|
|
10428
11059
|
}).strict().nullable(),
|
|
10429
|
-
candidates:
|
|
10430
|
-
city:
|
|
10431
|
-
regionCode:
|
|
10432
|
-
count:
|
|
10433
|
-
examples:
|
|
11060
|
+
candidates: z16.array(z16.object({
|
|
11061
|
+
city: z16.string(),
|
|
11062
|
+
regionCode: z16.string(),
|
|
11063
|
+
count: z16.number().int().min(0),
|
|
11064
|
+
examples: z16.array(z16.string())
|
|
10434
11065
|
}).strict())
|
|
10435
11066
|
}).strict();
|
|
10436
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
10437
|
-
seed:
|
|
10438
|
-
location:
|
|
10439
|
-
extractedAt:
|
|
10440
|
-
totalQuestions:
|
|
10441
|
-
surface:
|
|
10442
|
-
aiOverview:
|
|
10443
|
-
detected:
|
|
10444
|
-
text:
|
|
10445
|
-
citations:
|
|
10446
|
-
expanded:
|
|
10447
|
-
fullyExpanded:
|
|
10448
|
-
sections:
|
|
11067
|
+
var SerpIntelligenceHarvestResultSchema = z16.object({
|
|
11068
|
+
seed: z16.string(),
|
|
11069
|
+
location: z16.string().nullable(),
|
|
11070
|
+
extractedAt: z16.string(),
|
|
11071
|
+
totalQuestions: z16.number().int().min(0),
|
|
11072
|
+
surface: z16.enum(["web", "aim", "unknown"]),
|
|
11073
|
+
aiOverview: z16.object({
|
|
11074
|
+
detected: z16.boolean(),
|
|
11075
|
+
text: z16.string().nullable(),
|
|
11076
|
+
citations: z16.array(SerpIntelligenceAICitationSchema),
|
|
11077
|
+
expanded: z16.boolean().optional(),
|
|
11078
|
+
fullyExpanded: z16.boolean().optional(),
|
|
11079
|
+
sections: z16.array(z16.string()).optional()
|
|
10449
11080
|
}).strict(),
|
|
10450
|
-
aiMode:
|
|
10451
|
-
detected:
|
|
10452
|
-
text:
|
|
10453
|
-
citations:
|
|
11081
|
+
aiMode: z16.object({
|
|
11082
|
+
detected: z16.boolean(),
|
|
11083
|
+
text: z16.string().nullable(),
|
|
11084
|
+
citations: z16.array(SerpIntelligenceAICitationSchema)
|
|
10454
11085
|
}).strict(),
|
|
10455
|
-
tree:
|
|
10456
|
-
flat:
|
|
10457
|
-
videos:
|
|
10458
|
-
forums:
|
|
10459
|
-
organicResults:
|
|
10460
|
-
localPack:
|
|
10461
|
-
entityIds:
|
|
10462
|
-
entities:
|
|
10463
|
-
name:
|
|
10464
|
-
kgId:
|
|
10465
|
-
cid:
|
|
10466
|
-
gcid:
|
|
11086
|
+
tree: z16.array(z16.unknown()),
|
|
11087
|
+
flat: z16.array(z16.unknown()),
|
|
11088
|
+
videos: z16.array(z16.unknown()),
|
|
11089
|
+
forums: z16.array(z16.unknown()),
|
|
11090
|
+
organicResults: z16.array(SerpIntelligenceOrganicResultSchema),
|
|
11091
|
+
localPack: z16.array(z16.unknown()),
|
|
11092
|
+
entityIds: z16.object({
|
|
11093
|
+
entities: z16.array(z16.object({
|
|
11094
|
+
name: z16.string(),
|
|
11095
|
+
kgId: z16.string().nullable(),
|
|
11096
|
+
cid: z16.string().nullable(),
|
|
11097
|
+
gcid: z16.string().nullable()
|
|
10467
11098
|
}).strict()),
|
|
10468
|
-
kgIds:
|
|
10469
|
-
cids:
|
|
10470
|
-
gcids:
|
|
11099
|
+
kgIds: z16.array(z16.string()),
|
|
11100
|
+
cids: z16.array(z16.string()),
|
|
11101
|
+
gcids: z16.array(z16.string())
|
|
10471
11102
|
}).strict(),
|
|
10472
|
-
stats:
|
|
10473
|
-
seed:
|
|
10474
|
-
totalQuestions:
|
|
10475
|
-
maxDepthReached:
|
|
10476
|
-
durationMs:
|
|
10477
|
-
errorCount:
|
|
11103
|
+
stats: z16.object({
|
|
11104
|
+
seed: z16.string(),
|
|
11105
|
+
totalQuestions: z16.number().int().min(0),
|
|
11106
|
+
maxDepthReached: z16.number().int().min(0),
|
|
11107
|
+
durationMs: z16.number().min(0),
|
|
11108
|
+
errorCount: z16.number().int().min(0)
|
|
10478
11109
|
}).strict(),
|
|
10479
|
-
diagnostics:
|
|
10480
|
-
completionStatus:
|
|
10481
|
-
problem:
|
|
10482
|
-
warnings:
|
|
10483
|
-
debug:
|
|
11110
|
+
diagnostics: z16.object({
|
|
11111
|
+
completionStatus: z16.enum(["paa_found", "no_paa", "serp_only"]),
|
|
11112
|
+
problem: z16.null(),
|
|
11113
|
+
warnings: z16.array(z16.unknown()).optional(),
|
|
11114
|
+
debug: z16.object({
|
|
10484
11115
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
10485
11116
|
}).passthrough().optional()
|
|
10486
11117
|
}).passthrough(),
|
|
10487
|
-
whatPeopleSaying:
|
|
11118
|
+
whatPeopleSaying: z16.array(z16.unknown())
|
|
10488
11119
|
}).strict();
|
|
10489
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
10490
|
-
attemptNumber:
|
|
10491
|
-
outcome:
|
|
10492
|
-
startedAt:
|
|
10493
|
-
completedAt:
|
|
10494
|
-
durationMs:
|
|
10495
|
-
problemCode:
|
|
10496
|
-
message:
|
|
10497
|
-
kernelSessionId:
|
|
10498
|
-
cleanupSucceeded:
|
|
11120
|
+
var SerpIntelligenceCaptureAttemptSchema = z16.object({
|
|
11121
|
+
attemptNumber: z16.number().int().min(1),
|
|
11122
|
+
outcome: z16.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
11123
|
+
startedAt: z16.string().optional(),
|
|
11124
|
+
completedAt: z16.string().optional(),
|
|
11125
|
+
durationMs: z16.number().min(0).optional(),
|
|
11126
|
+
problemCode: z16.string().optional(),
|
|
11127
|
+
message: z16.string().optional(),
|
|
11128
|
+
kernelSessionId: z16.string().nullable().optional(),
|
|
11129
|
+
cleanupSucceeded: z16.boolean().nullable().optional()
|
|
10499
11130
|
}).strict();
|
|
10500
|
-
var SerpPageSnapshotCaptureSchema =
|
|
11131
|
+
var SerpPageSnapshotCaptureSchema = z16.object({
|
|
10501
11132
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
10502
11133
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
10503
11134
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
10504
|
-
sourceKind:
|
|
10505
|
-
sourcePosition:
|
|
10506
|
-
status:
|
|
10507
|
-
fetchedVia:
|
|
10508
|
-
httpStatus:
|
|
10509
|
-
contentType:
|
|
10510
|
-
title:
|
|
11135
|
+
sourceKind: z16.enum(SerpPageSnapshotSourceKindValues),
|
|
11136
|
+
sourcePosition: z16.number().int().min(1).nullable(),
|
|
11137
|
+
status: z16.enum(SerpPageFetchStatusValues),
|
|
11138
|
+
fetchedVia: z16.enum(SerpPageFetchedViaValues).nullable(),
|
|
11139
|
+
httpStatus: z16.number().int().min(100).max(599).nullable(),
|
|
11140
|
+
contentType: z16.string().nullable(),
|
|
11141
|
+
title: z16.string().nullable(),
|
|
10511
11142
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
10512
|
-
metaDescription:
|
|
10513
|
-
headings:
|
|
10514
|
-
level:
|
|
10515
|
-
text:
|
|
11143
|
+
metaDescription: z16.string().nullable(),
|
|
11144
|
+
headings: z16.array(z16.object({
|
|
11145
|
+
level: z16.number().int().min(1).max(6),
|
|
11146
|
+
text: z16.string()
|
|
10516
11147
|
}).strict()).default([]),
|
|
10517
|
-
artifact:
|
|
10518
|
-
htmlBlobUrl:
|
|
10519
|
-
textBlobUrl:
|
|
10520
|
-
markdownBlobUrl:
|
|
10521
|
-
screenshotBlobUrl:
|
|
10522
|
-
contentSha256:
|
|
10523
|
-
capturedAt:
|
|
11148
|
+
artifact: z16.object({
|
|
11149
|
+
htmlBlobUrl: z16.string().url().nullable(),
|
|
11150
|
+
textBlobUrl: z16.string().url().nullable(),
|
|
11151
|
+
markdownBlobUrl: z16.string().url().nullable(),
|
|
11152
|
+
screenshotBlobUrl: z16.string().url().nullable(),
|
|
11153
|
+
contentSha256: z16.string().nullable(),
|
|
11154
|
+
capturedAt: z16.string().nullable()
|
|
10524
11155
|
}).strict(),
|
|
10525
|
-
error:
|
|
10526
|
-
code:
|
|
10527
|
-
message:
|
|
11156
|
+
error: z16.object({
|
|
11157
|
+
code: z16.string(),
|
|
11158
|
+
message: z16.string()
|
|
10528
11159
|
}).strict().nullable()
|
|
10529
11160
|
}).strict();
|
|
10530
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
11161
|
+
var SerpIntelligenceCaptureResponseSchema = z16.object({
|
|
10531
11162
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
10532
|
-
attempts:
|
|
11163
|
+
attempts: z16.array(SerpIntelligenceCaptureAttemptSchema),
|
|
10533
11164
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
10534
|
-
pageSnapshotArtifacts:
|
|
10535
|
-
billing:
|
|
10536
|
-
creditsUsed:
|
|
10537
|
-
requestId:
|
|
10538
|
-
jobId:
|
|
11165
|
+
pageSnapshotArtifacts: z16.array(SerpPageSnapshotCaptureSchema),
|
|
11166
|
+
billing: z16.object({
|
|
11167
|
+
creditsUsed: z16.number().min(0).optional(),
|
|
11168
|
+
requestId: z16.string().optional(),
|
|
11169
|
+
jobId: z16.string().optional()
|
|
10539
11170
|
}).strict().optional()
|
|
10540
11171
|
}).strict();
|
|
10541
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
10542
|
-
pageSnapshotArtifacts:
|
|
10543
|
-
attempts:
|
|
11172
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z16.object({
|
|
11173
|
+
pageSnapshotArtifacts: z16.array(SerpPageSnapshotCaptureSchema),
|
|
11174
|
+
attempts: z16.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
10544
11175
|
}).strict();
|
|
10545
11176
|
|
|
10546
11177
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -10712,7 +11343,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
10712
11343
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
10713
11344
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
10714
11345
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
10715
|
-
var serpIntelligenceApp = new
|
|
11346
|
+
var serpIntelligenceApp = new Hono7();
|
|
10716
11347
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
10717
11348
|
function structuredError(input) {
|
|
10718
11349
|
return {
|
|
@@ -10887,7 +11518,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
10887
11518
|
});
|
|
10888
11519
|
|
|
10889
11520
|
// src/mcp/mcp-routes.ts
|
|
10890
|
-
import { Hono as
|
|
11521
|
+
import { Hono as Hono8 } from "hono";
|
|
10891
11522
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
10892
11523
|
configureReportSaving(false);
|
|
10893
11524
|
function mcpAuthError() {
|
|
@@ -10917,11 +11548,11 @@ async function requireMcpCallerKey(c) {
|
|
|
10917
11548
|
if (!user) return mcpAuthError();
|
|
10918
11549
|
return callerKey;
|
|
10919
11550
|
}
|
|
10920
|
-
var mcpApp = new
|
|
11551
|
+
var mcpApp = new Hono8();
|
|
10921
11552
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
10922
11553
|
server.registerTool("capture_serp_snapshot", {
|
|
10923
11554
|
title: "SERP Intelligence Snapshot",
|
|
10924
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized residential proxy evidence
|
|
11555
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries and rejects wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
10925
11556
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
10926
11557
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
10927
11558
|
}, async (input) => executor.captureSerpSnapshot(input));
|
|
@@ -10952,11 +11583,837 @@ mcpApp.all("/", async (c) => {
|
|
|
10952
11583
|
}
|
|
10953
11584
|
});
|
|
10954
11585
|
|
|
11586
|
+
// src/api/browser-agent-routes.ts
|
|
11587
|
+
import { Hono as Hono9 } from "hono";
|
|
11588
|
+
|
|
11589
|
+
// src/api/browser-agent-db.ts
|
|
11590
|
+
import { randomUUID } from "crypto";
|
|
11591
|
+
var _ready = false;
|
|
11592
|
+
async function migrateBrowserAgent() {
|
|
11593
|
+
if (_ready) return;
|
|
11594
|
+
const db = getDb();
|
|
11595
|
+
await db.execute(`
|
|
11596
|
+
CREATE TABLE IF NOT EXISTS browser_agent_sessions (
|
|
11597
|
+
id TEXT PRIMARY KEY,
|
|
11598
|
+
runtime_session_id TEXT NOT NULL,
|
|
11599
|
+
live_view_url TEXT,
|
|
11600
|
+
cdp_ws_url TEXT NOT NULL,
|
|
11601
|
+
status TEXT NOT NULL DEFAULT 'open',
|
|
11602
|
+
label TEXT,
|
|
11603
|
+
user_id INTEGER,
|
|
11604
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
11605
|
+
closed_at TEXT,
|
|
11606
|
+
last_action_at TEXT,
|
|
11607
|
+
active_ms INTEGER NOT NULL DEFAULT 0,
|
|
11608
|
+
billed_mc INTEGER NOT NULL DEFAULT 0
|
|
11609
|
+
)
|
|
11610
|
+
`);
|
|
11611
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_status ON browser_agent_sessions(status)`);
|
|
11612
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_user ON browser_agent_sessions(user_id)`);
|
|
11613
|
+
await db.execute(`
|
|
11614
|
+
CREATE TABLE IF NOT EXISTS browser_agent_actions (
|
|
11615
|
+
id TEXT PRIMARY KEY,
|
|
11616
|
+
session_id TEXT NOT NULL,
|
|
11617
|
+
type TEXT NOT NULL,
|
|
11618
|
+
params_json TEXT,
|
|
11619
|
+
ok INTEGER NOT NULL DEFAULT 1,
|
|
11620
|
+
error TEXT,
|
|
11621
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
11622
|
+
)
|
|
11623
|
+
`);
|
|
11624
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_actions_session ON browser_agent_actions(session_id)`);
|
|
11625
|
+
await db.execute(`
|
|
11626
|
+
CREATE TABLE IF NOT EXISTS browser_agent_replays (
|
|
11627
|
+
replay_id TEXT PRIMARY KEY,
|
|
11628
|
+
session_id TEXT NOT NULL,
|
|
11629
|
+
view_url TEXT,
|
|
11630
|
+
label TEXT,
|
|
11631
|
+
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
11632
|
+
stopped_at TEXT
|
|
11633
|
+
)
|
|
11634
|
+
`);
|
|
11635
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_replays_session ON browser_agent_replays(session_id)`);
|
|
11636
|
+
_ready = true;
|
|
11637
|
+
}
|
|
11638
|
+
async function createSessionRow(input) {
|
|
11639
|
+
const db = getDb();
|
|
11640
|
+
const id = `bas_${randomUUID().replace(/-/g, "").slice(0, 20)}`;
|
|
11641
|
+
await db.execute({
|
|
11642
|
+
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
11643
|
+
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
11644
|
+
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId]
|
|
11645
|
+
});
|
|
11646
|
+
const row = await getSessionRow(id);
|
|
11647
|
+
if (!row) throw new Error("session insert failed");
|
|
11648
|
+
return row;
|
|
11649
|
+
}
|
|
11650
|
+
async function getSessionRow(id) {
|
|
11651
|
+
const db = getDb();
|
|
11652
|
+
const res = await db.execute({ sql: `SELECT * FROM browser_agent_sessions WHERE id = ?`, args: [id] });
|
|
11653
|
+
return res.rows[0] ?? null;
|
|
11654
|
+
}
|
|
11655
|
+
async function listSessionRows(userId, includeClosed = false) {
|
|
11656
|
+
const db = getDb();
|
|
11657
|
+
const clauses = [];
|
|
11658
|
+
const args = [];
|
|
11659
|
+
if (userId != null) {
|
|
11660
|
+
clauses.push("(user_id = ? OR user_id IS NULL)");
|
|
11661
|
+
args.push(userId);
|
|
11662
|
+
}
|
|
11663
|
+
if (!includeClosed) clauses.push(`status = 'open'`);
|
|
11664
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
11665
|
+
const res = await db.execute({
|
|
11666
|
+
sql: `SELECT * FROM browser_agent_sessions ${where} ORDER BY created_at DESC LIMIT 100`,
|
|
11667
|
+
args
|
|
11668
|
+
});
|
|
11669
|
+
return res.rows;
|
|
11670
|
+
}
|
|
11671
|
+
async function addActiveMs(id, deltaMs) {
|
|
11672
|
+
const db = getDb();
|
|
11673
|
+
await db.execute({
|
|
11674
|
+
sql: `UPDATE browser_agent_sessions SET active_ms = active_ms + ?, last_action_at = datetime('now') WHERE id = ?`,
|
|
11675
|
+
args: [Math.max(0, Math.round(deltaMs)), id]
|
|
11676
|
+
});
|
|
11677
|
+
const res = await db.execute({ sql: `SELECT active_ms, billed_mc FROM browser_agent_sessions WHERE id = ?`, args: [id] });
|
|
11678
|
+
const row = res.rows[0];
|
|
11679
|
+
return { active_ms: Number(row?.active_ms ?? 0), billed_mc: Number(row?.billed_mc ?? 0) };
|
|
11680
|
+
}
|
|
11681
|
+
async function setBilledMc(id, billedMc) {
|
|
11682
|
+
const db = getDb();
|
|
11683
|
+
await db.execute({
|
|
11684
|
+
sql: `UPDATE browser_agent_sessions SET billed_mc = ? WHERE id = ?`,
|
|
11685
|
+
args: [Math.round(billedMc), id]
|
|
11686
|
+
});
|
|
11687
|
+
}
|
|
11688
|
+
async function markSessionClosed(id) {
|
|
11689
|
+
const db = getDb();
|
|
11690
|
+
await db.execute({
|
|
11691
|
+
sql: `UPDATE browser_agent_sessions SET status = 'closed', closed_at = datetime('now') WHERE id = ?`,
|
|
11692
|
+
args: [id]
|
|
11693
|
+
});
|
|
11694
|
+
}
|
|
11695
|
+
async function recordAction(input) {
|
|
11696
|
+
const db = getDb();
|
|
11697
|
+
await db.execute({
|
|
11698
|
+
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
11699
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
11700
|
+
args: [
|
|
11701
|
+
`baa_${randomUUID().replace(/-/g, "").slice(0, 20)}`,
|
|
11702
|
+
input.sessionId,
|
|
11703
|
+
input.type,
|
|
11704
|
+
input.params == null ? null : JSON.stringify(input.params),
|
|
11705
|
+
input.ok ? 1 : 0,
|
|
11706
|
+
input.error ?? null
|
|
11707
|
+
]
|
|
11708
|
+
});
|
|
11709
|
+
}
|
|
11710
|
+
async function recordReplayStart(input) {
|
|
11711
|
+
const db = getDb();
|
|
11712
|
+
await db.execute({
|
|
11713
|
+
sql: `INSERT INTO browser_agent_replays (replay_id, session_id, view_url, label)
|
|
11714
|
+
VALUES (?, ?, ?, ?)`,
|
|
11715
|
+
args: [input.replayId, input.sessionId, input.viewUrl, input.label]
|
|
11716
|
+
});
|
|
11717
|
+
}
|
|
11718
|
+
async function recordReplayStop(replayId, viewUrl) {
|
|
11719
|
+
const db = getDb();
|
|
11720
|
+
await db.execute({
|
|
11721
|
+
sql: `UPDATE browser_agent_replays SET stopped_at = datetime('now'), view_url = COALESCE(?, view_url) WHERE replay_id = ?`,
|
|
11722
|
+
args: [viewUrl, replayId]
|
|
11723
|
+
});
|
|
11724
|
+
}
|
|
11725
|
+
async function listReplayRows(sessionId) {
|
|
11726
|
+
const db = getDb();
|
|
11727
|
+
const res = await db.execute({
|
|
11728
|
+
sql: `SELECT * FROM browser_agent_replays WHERE session_id = ? ORDER BY started_at DESC`,
|
|
11729
|
+
args: [sessionId]
|
|
11730
|
+
});
|
|
11731
|
+
return res.rows;
|
|
11732
|
+
}
|
|
11733
|
+
|
|
11734
|
+
// src/services/browser-agent/browser-agent-service.ts
|
|
11735
|
+
import Kernel3 from "@onkernel/sdk";
|
|
11736
|
+
import { chromium as playwrightChromium } from "playwright";
|
|
11737
|
+
var DEFAULT_TIMEOUT_SECONDS = 600;
|
|
11738
|
+
function client() {
|
|
11739
|
+
const apiKey = browserServiceApiKey();
|
|
11740
|
+
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
11741
|
+
return new Kernel3({ apiKey });
|
|
11742
|
+
}
|
|
11743
|
+
async function createSession(opts = {}) {
|
|
11744
|
+
const k = client();
|
|
11745
|
+
const resolvedProxyId = opts.proxyId ?? browserServiceProxyId();
|
|
11746
|
+
const browser = await k.browsers.create({
|
|
11747
|
+
stealth: opts.stealth ?? true,
|
|
11748
|
+
timeout_seconds: opts.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS,
|
|
11749
|
+
...resolvedProxyId ? { proxy_id: resolvedProxyId } : {},
|
|
11750
|
+
...opts.profileName ? { profile: { name: opts.profileName } } : {}
|
|
11751
|
+
});
|
|
11752
|
+
const runtimeSessionId = browser.session_id;
|
|
11753
|
+
if (opts.disableDefaultProxy) {
|
|
11754
|
+
try {
|
|
11755
|
+
await k.browsers.update(runtimeSessionId, { disable_default_proxy: true });
|
|
11756
|
+
} catch {
|
|
11757
|
+
}
|
|
11758
|
+
}
|
|
11759
|
+
if (opts.viewport) {
|
|
11760
|
+
try {
|
|
11761
|
+
await k.browsers.update(runtimeSessionId, { viewport: opts.viewport });
|
|
11762
|
+
} catch {
|
|
11763
|
+
}
|
|
11764
|
+
}
|
|
11765
|
+
return {
|
|
11766
|
+
runtimeSessionId,
|
|
11767
|
+
liveViewUrl: browser.browser_live_view_url ?? null,
|
|
11768
|
+
cdpWsUrl: browser.cdp_ws_url
|
|
11769
|
+
};
|
|
11770
|
+
}
|
|
11771
|
+
async function closeSession(runtimeSessionId) {
|
|
11772
|
+
const k = client();
|
|
11773
|
+
await k.browsers.deleteByID(runtimeSessionId);
|
|
11774
|
+
}
|
|
11775
|
+
async function screenshot(runtimeSessionId) {
|
|
11776
|
+
const k = client();
|
|
11777
|
+
const res = await k.browsers.computer.captureScreenshot(runtimeSessionId);
|
|
11778
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
11779
|
+
return { base64: buf.toString("base64"), mimeType: "image/png" };
|
|
11780
|
+
}
|
|
11781
|
+
async function click(runtimeSessionId, x, y, opts = {}) {
|
|
11782
|
+
const k = client();
|
|
11783
|
+
await k.browsers.computer.clickMouse(runtimeSessionId, {
|
|
11784
|
+
x,
|
|
11785
|
+
y,
|
|
11786
|
+
button: opts.button ?? "left",
|
|
11787
|
+
...opts.numClicks ? { num_clicks: opts.numClicks } : {}
|
|
11788
|
+
});
|
|
11789
|
+
}
|
|
11790
|
+
async function typeText(runtimeSessionId, text, delayMs) {
|
|
11791
|
+
const k = client();
|
|
11792
|
+
await k.browsers.computer.typeText(runtimeSessionId, {
|
|
11793
|
+
text,
|
|
11794
|
+
...typeof delayMs === "number" ? { delay: delayMs } : {}
|
|
11795
|
+
});
|
|
11796
|
+
}
|
|
11797
|
+
async function scroll(runtimeSessionId, x, y, deltaX, deltaY) {
|
|
11798
|
+
const k = client();
|
|
11799
|
+
await k.browsers.computer.scroll(runtimeSessionId, { x, y, delta_x: deltaX, delta_y: deltaY });
|
|
11800
|
+
}
|
|
11801
|
+
async function pressKeys(runtimeSessionId, keys) {
|
|
11802
|
+
const k = client();
|
|
11803
|
+
await k.browsers.computer.pressKey(runtimeSessionId, { keys });
|
|
11804
|
+
}
|
|
11805
|
+
async function goto(cdpWsUrl, url) {
|
|
11806
|
+
const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
|
|
11807
|
+
try {
|
|
11808
|
+
const context = browser.contexts()[0] ?? await browser.newContext();
|
|
11809
|
+
const page = context.pages()[0] ?? await context.newPage();
|
|
11810
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
11811
|
+
return { url: page.url(), title: await page.title() };
|
|
11812
|
+
} finally {
|
|
11813
|
+
await browser.close().catch(() => {
|
|
11814
|
+
});
|
|
11815
|
+
}
|
|
11816
|
+
}
|
|
11817
|
+
async function readPage(cdpWsUrl) {
|
|
11818
|
+
const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
|
|
11819
|
+
try {
|
|
11820
|
+
const context = browser.contexts()[0] ?? await browser.newContext();
|
|
11821
|
+
const page = context.pages()[0] ?? await context.newPage();
|
|
11822
|
+
const url = page.url();
|
|
11823
|
+
const title = await page.title().catch(() => "");
|
|
11824
|
+
const data = await page.evaluate(() => {
|
|
11825
|
+
const SEL = 'a[href], button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [onclick]';
|
|
11826
|
+
const out = [];
|
|
11827
|
+
const nodes = Array.from(document.querySelectorAll(SEL)).slice(0, 120);
|
|
11828
|
+
for (const el2 of nodes) {
|
|
11829
|
+
const r = el2.getBoundingClientRect();
|
|
11830
|
+
if (r.width < 1 || r.height < 1) continue;
|
|
11831
|
+
if (r.bottom < 0 || r.top > window.innerHeight) continue;
|
|
11832
|
+
const e = el2;
|
|
11833
|
+
const name = (e.getAttribute("aria-label") || e.placeholder || e.innerText || e.getAttribute("value") || e.getAttribute("title") || "").trim().replace(/\s+/g, " ").slice(0, 80);
|
|
11834
|
+
out.push({
|
|
11835
|
+
role: el2.getAttribute("role") || el2.tagName.toLowerCase(),
|
|
11836
|
+
name,
|
|
11837
|
+
x: Math.round(r.left + r.width / 2),
|
|
11838
|
+
y: Math.round(r.top + r.height / 2)
|
|
11839
|
+
});
|
|
11840
|
+
}
|
|
11841
|
+
const text = (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").trim().slice(0, 6e3);
|
|
11842
|
+
return { text, els: out };
|
|
11843
|
+
});
|
|
11844
|
+
return {
|
|
11845
|
+
url,
|
|
11846
|
+
title,
|
|
11847
|
+
text: data.text,
|
|
11848
|
+
elements: data.els.map((e, i) => ({ ref: i + 1, role: e.role, name: e.name, x: e.x, y: e.y }))
|
|
11849
|
+
};
|
|
11850
|
+
} finally {
|
|
11851
|
+
await browser.close().catch(() => {
|
|
11852
|
+
});
|
|
11853
|
+
}
|
|
11854
|
+
}
|
|
11855
|
+
async function replayStart(runtimeSessionId) {
|
|
11856
|
+
const k = client();
|
|
11857
|
+
const res = await k.browsers.replays.start(runtimeSessionId);
|
|
11858
|
+
return { replayId: res.replay_id, viewUrl: res.replay_view_url ?? null };
|
|
11859
|
+
}
|
|
11860
|
+
async function replayStop(runtimeSessionId, replayId) {
|
|
11861
|
+
const k = client();
|
|
11862
|
+
await k.browsers.replays.stop(replayId, { id: runtimeSessionId });
|
|
11863
|
+
}
|
|
11864
|
+
async function replayDownload(runtimeSessionId, replayId) {
|
|
11865
|
+
const k = client();
|
|
11866
|
+
return k.browsers.replays.download(replayId, { id: runtimeSessionId });
|
|
11867
|
+
}
|
|
11868
|
+
async function replayList(runtimeSessionId) {
|
|
11869
|
+
const k = client();
|
|
11870
|
+
const res = await k.browsers.replays.list(runtimeSessionId);
|
|
11871
|
+
return res.map((r) => ({
|
|
11872
|
+
replayId: r.replay_id,
|
|
11873
|
+
viewUrl: r.replay_view_url ?? null,
|
|
11874
|
+
startedAt: r.started_at ?? null,
|
|
11875
|
+
finishedAt: r.finished_at ?? null
|
|
11876
|
+
}));
|
|
11877
|
+
}
|
|
11878
|
+
|
|
11879
|
+
// src/api/browser-agent-routes.ts
|
|
11880
|
+
var auth = createApiKeyAuth();
|
|
11881
|
+
async function charge(sessionId, userId, startedAtMs) {
|
|
11882
|
+
const elapsedMs = Date.now() - startedAtMs;
|
|
11883
|
+
const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
|
|
11884
|
+
const owed = browserActiveCostMc(active_ms);
|
|
11885
|
+
const delta = owed - billed_mc;
|
|
11886
|
+
if (delta > 0) {
|
|
11887
|
+
const res = await debitMc(userId, delta, LedgerOperation.BROWSER_SESSION, sessionId);
|
|
11888
|
+
if (res.ok) await setBilledMc(sessionId, owed);
|
|
11889
|
+
}
|
|
11890
|
+
}
|
|
11891
|
+
function publicSession(row) {
|
|
11892
|
+
return {
|
|
11893
|
+
session_id: row.id,
|
|
11894
|
+
status: row.status,
|
|
11895
|
+
label: row.label,
|
|
11896
|
+
live_view_url: row.live_view_url,
|
|
11897
|
+
created_at: row.created_at,
|
|
11898
|
+
last_action_at: row.last_action_at,
|
|
11899
|
+
closed_at: row.closed_at,
|
|
11900
|
+
active_seconds: Math.round((row.active_ms ?? 0) / 1e3),
|
|
11901
|
+
credits_used: Math.round((row.billed_mc ?? 0) / 10) / 100
|
|
11902
|
+
};
|
|
11903
|
+
}
|
|
11904
|
+
function failure(err) {
|
|
11905
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
11906
|
+
return { error: sanitizeVendorName(msg) };
|
|
11907
|
+
}
|
|
11908
|
+
function replayDownloadUrl(sessionId, replayId) {
|
|
11909
|
+
return `/agent/sessions/${encodeURIComponent(sessionId)}/replays/${encodeURIComponent(replayId)}/download`;
|
|
11910
|
+
}
|
|
11911
|
+
function replayFilename(sessionId, replayId) {
|
|
11912
|
+
const safeSession = sessionId.replace(/[^a-zA-Z0-9_-]/g, "-").slice(0, 80);
|
|
11913
|
+
const safeReplay = replayId.replace(/[^a-zA-Z0-9_-]/g, "-").slice(0, 120);
|
|
11914
|
+
return `${safeSession}-${safeReplay}.mp4`;
|
|
11915
|
+
}
|
|
11916
|
+
async function loadOpenSession(id, userId) {
|
|
11917
|
+
const row = await getSessionRow(id);
|
|
11918
|
+
if (!row) return null;
|
|
11919
|
+
if (row.user_id != null && row.user_id !== userId) return null;
|
|
11920
|
+
return row;
|
|
11921
|
+
}
|
|
11922
|
+
function buildBrowserAgentRoutes() {
|
|
11923
|
+
const app2 = new Hono9();
|
|
11924
|
+
app2.use("*", async (c, next) => {
|
|
11925
|
+
await migrateBrowserAgent();
|
|
11926
|
+
return next();
|
|
11927
|
+
});
|
|
11928
|
+
app2.use("*", auth);
|
|
11929
|
+
app2.post("/sessions", async (c) => {
|
|
11930
|
+
const user = c.get("user");
|
|
11931
|
+
if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
|
|
11932
|
+
return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
|
|
11933
|
+
}
|
|
11934
|
+
const body = await c.req.json().catch(() => ({}));
|
|
11935
|
+
try {
|
|
11936
|
+
const created = await createSession({
|
|
11937
|
+
timeoutSeconds: typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0,
|
|
11938
|
+
proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
|
|
11939
|
+
profileName: typeof body.profile === "string" ? body.profile : void 0,
|
|
11940
|
+
disableDefaultProxy: body.disable_default_proxy === true,
|
|
11941
|
+
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
11942
|
+
});
|
|
11943
|
+
const row = await createSessionRow({
|
|
11944
|
+
runtimeSessionId: created.runtimeSessionId,
|
|
11945
|
+
liveViewUrl: created.liveViewUrl,
|
|
11946
|
+
cdpWsUrl: created.cdpWsUrl,
|
|
11947
|
+
label: typeof body.label === "string" ? body.label : null,
|
|
11948
|
+
userId: user.id
|
|
11949
|
+
});
|
|
11950
|
+
return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
|
|
11951
|
+
} catch (err) {
|
|
11952
|
+
return c.json(failure(err), 502);
|
|
11953
|
+
}
|
|
11954
|
+
});
|
|
11955
|
+
app2.get("/sessions", async (c) => {
|
|
11956
|
+
const user = c.get("user");
|
|
11957
|
+
const includeClosed = c.req.query("all") === "1";
|
|
11958
|
+
const rows = await listSessionRows(user.id, includeClosed);
|
|
11959
|
+
return c.json({ sessions: rows.map(publicSession) });
|
|
11960
|
+
});
|
|
11961
|
+
app2.get("/sessions/:id", async (c) => {
|
|
11962
|
+
const user = c.get("user");
|
|
11963
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
11964
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
11965
|
+
return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
|
|
11966
|
+
});
|
|
11967
|
+
app2.get("/sessions/:id/live-view", async (c) => {
|
|
11968
|
+
const user = c.get("user");
|
|
11969
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
11970
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
11971
|
+
return c.json({ live_view_url: row.live_view_url });
|
|
11972
|
+
});
|
|
11973
|
+
app2.delete("/sessions/:id", async (c) => {
|
|
11974
|
+
const user = c.get("user");
|
|
11975
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
11976
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
11977
|
+
try {
|
|
11978
|
+
await closeSession(row.runtime_session_id);
|
|
11979
|
+
} catch {
|
|
11980
|
+
}
|
|
11981
|
+
await markSessionClosed(row.id);
|
|
11982
|
+
return c.json({ ok: true });
|
|
11983
|
+
});
|
|
11984
|
+
app2.post("/sessions/:id/goto", async (c) => {
|
|
11985
|
+
const user = c.get("user");
|
|
11986
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
11987
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
11988
|
+
const body = await c.req.json().catch(() => ({}));
|
|
11989
|
+
const url = typeof body.url === "string" ? body.url : "";
|
|
11990
|
+
if (!url) return c.json({ error: "url is required" }, 400);
|
|
11991
|
+
const t0 = Date.now();
|
|
11992
|
+
try {
|
|
11993
|
+
const result = await goto(row.cdp_ws_url, url);
|
|
11994
|
+
await charge(row.id, user.id, t0);
|
|
11995
|
+
await recordAction({ sessionId: row.id, type: "goto", params: { url }, ok: true });
|
|
11996
|
+
return c.json(result);
|
|
11997
|
+
} catch (err) {
|
|
11998
|
+
await charge(row.id, user.id, t0);
|
|
11999
|
+
await recordAction({ sessionId: row.id, type: "goto", params: { url }, ok: false, error: String(err) });
|
|
12000
|
+
return c.json(failure(err), 502);
|
|
12001
|
+
}
|
|
12002
|
+
});
|
|
12003
|
+
app2.post("/sessions/:id/screenshot", async (c) => {
|
|
12004
|
+
const user = c.get("user");
|
|
12005
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12006
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12007
|
+
const t0 = Date.now();
|
|
12008
|
+
try {
|
|
12009
|
+
const shot = await screenshot(row.runtime_session_id);
|
|
12010
|
+
let page = null;
|
|
12011
|
+
try {
|
|
12012
|
+
page = await readPage(row.cdp_ws_url);
|
|
12013
|
+
} catch {
|
|
12014
|
+
page = null;
|
|
12015
|
+
}
|
|
12016
|
+
await charge(row.id, user.id, t0);
|
|
12017
|
+
await recordAction({ sessionId: row.id, type: "screenshot", params: null, ok: true });
|
|
12018
|
+
return c.json({
|
|
12019
|
+
image_base64: shot.base64,
|
|
12020
|
+
mime_type: shot.mimeType,
|
|
12021
|
+
url: page?.url ?? null,
|
|
12022
|
+
title: page?.title ?? null,
|
|
12023
|
+
elements: page?.elements ?? [],
|
|
12024
|
+
text: page?.text ?? null
|
|
12025
|
+
});
|
|
12026
|
+
} catch (err) {
|
|
12027
|
+
await charge(row.id, user.id, t0);
|
|
12028
|
+
await recordAction({ sessionId: row.id, type: "screenshot", params: null, ok: false, error: String(err) });
|
|
12029
|
+
return c.json(failure(err), 502);
|
|
12030
|
+
}
|
|
12031
|
+
});
|
|
12032
|
+
app2.post("/sessions/:id/read", async (c) => {
|
|
12033
|
+
const user = c.get("user");
|
|
12034
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12035
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12036
|
+
const t0 = Date.now();
|
|
12037
|
+
try {
|
|
12038
|
+
const page = await readPage(row.cdp_ws_url);
|
|
12039
|
+
await charge(row.id, user.id, t0);
|
|
12040
|
+
await recordAction({ sessionId: row.id, type: "read", params: null, ok: true });
|
|
12041
|
+
return c.json(page);
|
|
12042
|
+
} catch (err) {
|
|
12043
|
+
await charge(row.id, user.id, t0);
|
|
12044
|
+
return c.json(failure(err), 502);
|
|
12045
|
+
}
|
|
12046
|
+
});
|
|
12047
|
+
app2.post("/sessions/:id/click", async (c) => {
|
|
12048
|
+
const user = c.get("user");
|
|
12049
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12050
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12051
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12052
|
+
const x = Number(body.x);
|
|
12053
|
+
const y = Number(body.y);
|
|
12054
|
+
if (!Number.isFinite(x) || !Number.isFinite(y)) return c.json({ error: "x and y are required" }, 400);
|
|
12055
|
+
const t0 = Date.now();
|
|
12056
|
+
try {
|
|
12057
|
+
await click(row.runtime_session_id, x, y, {
|
|
12058
|
+
button: body.button === "right" || body.button === "middle" ? body.button : "left",
|
|
12059
|
+
numClicks: typeof body.num_clicks === "number" ? body.num_clicks : void 0
|
|
12060
|
+
});
|
|
12061
|
+
await charge(row.id, user.id, t0);
|
|
12062
|
+
await recordAction({ sessionId: row.id, type: "click", params: { x, y }, ok: true });
|
|
12063
|
+
return c.json({ ok: true });
|
|
12064
|
+
} catch (err) {
|
|
12065
|
+
await charge(row.id, user.id, t0);
|
|
12066
|
+
await recordAction({ sessionId: row.id, type: "click", params: { x, y }, ok: false, error: String(err) });
|
|
12067
|
+
return c.json(failure(err), 502);
|
|
12068
|
+
}
|
|
12069
|
+
});
|
|
12070
|
+
app2.post("/sessions/:id/type", async (c) => {
|
|
12071
|
+
const user = c.get("user");
|
|
12072
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12073
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12074
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12075
|
+
const text = typeof body.text === "string" ? body.text : "";
|
|
12076
|
+
if (!text) return c.json({ error: "text is required" }, 400);
|
|
12077
|
+
const t0 = Date.now();
|
|
12078
|
+
try {
|
|
12079
|
+
await typeText(row.runtime_session_id, text, typeof body.delay === "number" ? body.delay : void 0);
|
|
12080
|
+
await charge(row.id, user.id, t0);
|
|
12081
|
+
await recordAction({ sessionId: row.id, type: "type", params: { length: text.length }, ok: true });
|
|
12082
|
+
return c.json({ ok: true });
|
|
12083
|
+
} catch (err) {
|
|
12084
|
+
await charge(row.id, user.id, t0);
|
|
12085
|
+
return c.json(failure(err), 502);
|
|
12086
|
+
}
|
|
12087
|
+
});
|
|
12088
|
+
app2.post("/sessions/:id/scroll", async (c) => {
|
|
12089
|
+
const user = c.get("user");
|
|
12090
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12091
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12092
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12093
|
+
const x = typeof body.x === "number" ? body.x : 640;
|
|
12094
|
+
const y = typeof body.y === "number" ? body.y : 400;
|
|
12095
|
+
const deltaX = typeof body.delta_x === "number" ? body.delta_x : 0;
|
|
12096
|
+
const deltaY = typeof body.delta_y === "number" ? body.delta_y : 5;
|
|
12097
|
+
const t0 = Date.now();
|
|
12098
|
+
try {
|
|
12099
|
+
await scroll(row.runtime_session_id, x, y, deltaX, deltaY);
|
|
12100
|
+
await charge(row.id, user.id, t0);
|
|
12101
|
+
await recordAction({ sessionId: row.id, type: "scroll", params: { deltaX, deltaY }, ok: true });
|
|
12102
|
+
return c.json({ ok: true });
|
|
12103
|
+
} catch (err) {
|
|
12104
|
+
await charge(row.id, user.id, t0);
|
|
12105
|
+
return c.json(failure(err), 502);
|
|
12106
|
+
}
|
|
12107
|
+
});
|
|
12108
|
+
app2.post("/sessions/:id/press", async (c) => {
|
|
12109
|
+
const user = c.get("user");
|
|
12110
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12111
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12112
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12113
|
+
const keys = Array.isArray(body.keys) ? body.keys.map(String) : [];
|
|
12114
|
+
if (!keys.length) return c.json({ error: "keys is required" }, 400);
|
|
12115
|
+
const t0 = Date.now();
|
|
12116
|
+
try {
|
|
12117
|
+
await pressKeys(row.runtime_session_id, keys);
|
|
12118
|
+
await charge(row.id, user.id, t0);
|
|
12119
|
+
await recordAction({ sessionId: row.id, type: "press", params: { keys }, ok: true });
|
|
12120
|
+
return c.json({ ok: true });
|
|
12121
|
+
} catch (err) {
|
|
12122
|
+
await charge(row.id, user.id, t0);
|
|
12123
|
+
return c.json(failure(err), 502);
|
|
12124
|
+
}
|
|
12125
|
+
});
|
|
12126
|
+
app2.post("/sessions/:id/replay/start", async (c) => {
|
|
12127
|
+
const user = c.get("user");
|
|
12128
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12129
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12130
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12131
|
+
try {
|
|
12132
|
+
const started = await replayStart(row.runtime_session_id);
|
|
12133
|
+
await recordReplayStart({
|
|
12134
|
+
sessionId: row.id,
|
|
12135
|
+
replayId: started.replayId,
|
|
12136
|
+
viewUrl: started.viewUrl,
|
|
12137
|
+
label: typeof body.label === "string" ? body.label : null
|
|
12138
|
+
});
|
|
12139
|
+
return c.json({
|
|
12140
|
+
replay_id: started.replayId,
|
|
12141
|
+
view_url: started.viewUrl,
|
|
12142
|
+
download_url: replayDownloadUrl(row.id, started.replayId)
|
|
12143
|
+
});
|
|
12144
|
+
} catch (err) {
|
|
12145
|
+
return c.json(failure(err), 502);
|
|
12146
|
+
}
|
|
12147
|
+
});
|
|
12148
|
+
app2.post("/sessions/:id/replay/stop", async (c) => {
|
|
12149
|
+
const user = c.get("user");
|
|
12150
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12151
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12152
|
+
const body = await c.req.json().catch(() => ({}));
|
|
12153
|
+
const replayId = typeof body.replay_id === "string" ? body.replay_id : "";
|
|
12154
|
+
if (!replayId) return c.json({ error: "replay_id is required" }, 400);
|
|
12155
|
+
try {
|
|
12156
|
+
await replayStop(row.runtime_session_id, replayId);
|
|
12157
|
+
let viewUrl = null;
|
|
12158
|
+
try {
|
|
12159
|
+
const all = await replayList(row.runtime_session_id);
|
|
12160
|
+
viewUrl = all.find((r) => r.replayId === replayId)?.viewUrl ?? null;
|
|
12161
|
+
} catch {
|
|
12162
|
+
viewUrl = null;
|
|
12163
|
+
}
|
|
12164
|
+
await recordReplayStop(replayId, viewUrl);
|
|
12165
|
+
return c.json({
|
|
12166
|
+
ok: true,
|
|
12167
|
+
replay_id: replayId,
|
|
12168
|
+
view_url: viewUrl,
|
|
12169
|
+
download_url: replayDownloadUrl(row.id, replayId)
|
|
12170
|
+
});
|
|
12171
|
+
} catch (err) {
|
|
12172
|
+
return c.json(failure(err), 502);
|
|
12173
|
+
}
|
|
12174
|
+
});
|
|
12175
|
+
app2.get("/sessions/:id/replays", async (c) => {
|
|
12176
|
+
const user = c.get("user");
|
|
12177
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12178
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12179
|
+
const rows = await listReplayRows(row.id);
|
|
12180
|
+
return c.json({
|
|
12181
|
+
replays: rows.map((r) => ({
|
|
12182
|
+
replay_id: r.replay_id,
|
|
12183
|
+
view_url: r.view_url,
|
|
12184
|
+
download_url: replayDownloadUrl(row.id, r.replay_id),
|
|
12185
|
+
label: r.label,
|
|
12186
|
+
started_at: r.started_at,
|
|
12187
|
+
stopped_at: r.stopped_at
|
|
12188
|
+
}))
|
|
12189
|
+
});
|
|
12190
|
+
});
|
|
12191
|
+
app2.get("/sessions/:id/replays/:replayId/download", async (c) => {
|
|
12192
|
+
const user = c.get("user");
|
|
12193
|
+
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
12194
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
12195
|
+
const replayId = c.req.param("replayId");
|
|
12196
|
+
const rows = await listReplayRows(row.id);
|
|
12197
|
+
if (!rows.some((r) => r.replay_id === replayId)) return c.json({ error: "replay not found" }, 404);
|
|
12198
|
+
try {
|
|
12199
|
+
const res = await replayDownload(row.runtime_session_id, replayId);
|
|
12200
|
+
if (!res.ok) return c.json({ error: `replay download failed (${res.status})` }, res.status);
|
|
12201
|
+
return new Response(res.body, {
|
|
12202
|
+
status: 200,
|
|
12203
|
+
headers: {
|
|
12204
|
+
"Content-Type": res.headers.get("content-type") ?? "video/mp4",
|
|
12205
|
+
"Content-Disposition": `attachment; filename="${replayFilename(row.id, replayId)}"`,
|
|
12206
|
+
"Cache-Control": "private, max-age=300"
|
|
12207
|
+
}
|
|
12208
|
+
});
|
|
12209
|
+
} catch (err) {
|
|
12210
|
+
return c.json(failure(err), 502);
|
|
12211
|
+
}
|
|
12212
|
+
});
|
|
12213
|
+
return app2;
|
|
12214
|
+
}
|
|
12215
|
+
|
|
12216
|
+
// src/api/browser-agent-console.ts
|
|
12217
|
+
function renderConsoleHtml(initialSessionId) {
|
|
12218
|
+
const initial = JSON.stringify(initialSessionId ?? "");
|
|
12219
|
+
return `<!DOCTYPE html>
|
|
12220
|
+
<html lang="en">
|
|
12221
|
+
<head>
|
|
12222
|
+
<meta charset="UTF-8" />
|
|
12223
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
12224
|
+
<title>Browser Agent Console</title>
|
|
12225
|
+
<style>
|
|
12226
|
+
:root { color-scheme: dark; }
|
|
12227
|
+
:where(*) { box-sizing: border-box; }
|
|
12228
|
+
body { margin: 0; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, sans-serif; background: #0b0e14; color: #d7dce5; }
|
|
12229
|
+
header { display: flex; align-items: center; gap: 12px; padding: 10px 16px; border-bottom: 1px solid #1c2230; background: #0f131c; }
|
|
12230
|
+
header h1 { font-size: 15px; margin: 0; font-weight: 600; color: #fff; letter-spacing: .2px; }
|
|
12231
|
+
header .spacer { flex: 1; }
|
|
12232
|
+
input, button, select { font: inherit; }
|
|
12233
|
+
input[type=text], input[type=password], input[type=url] { background: #141925; border: 1px solid #232b3a; color: #e6eaf2; border-radius: 7px; padding: 7px 10px; }
|
|
12234
|
+
button { background: #2b6cff; border: 0; color: #fff; border-radius: 7px; padding: 7px 12px; cursor: pointer; font-weight: 500; }
|
|
12235
|
+
button.ghost { background: #1a2030; color: #cdd5e4; border: 1px solid #28303f; }
|
|
12236
|
+
button.linkish { background: transparent; color: #7aa2ff; border: 0; padding: 0; font-size: 13px; font-weight: 500; }
|
|
12237
|
+
button:disabled { opacity: .5; cursor: default; }
|
|
12238
|
+
.layout { display: grid; grid-template-columns: 280px 1fr; height: calc(100vh - 53px); }
|
|
12239
|
+
aside { border-right: 1px solid #1c2230; overflow-y: auto; padding: 12px; }
|
|
12240
|
+
aside h2 { font-size: 11px; text-transform: uppercase; letter-spacing: .08em; color: #6b7689; margin: 4px 4px 10px; }
|
|
12241
|
+
.sess { padding: 9px 10px; border-radius: 8px; border: 1px solid #1c2230; margin-bottom: 8px; cursor: pointer; }
|
|
12242
|
+
.sess:hover { border-color: #2b6cff; }
|
|
12243
|
+
.sess.active { border-color: #2b6cff; background: #131b2e; }
|
|
12244
|
+
.sess .id { font-family: ui-monospace, monospace; font-size: 12px; color: #aeb8cc; }
|
|
12245
|
+
.sess .meta { font-size: 11px; color: #6b7689; margin-top: 3px; }
|
|
12246
|
+
.dot { display: inline-block; width: 7px; height: 7px; border-radius: 50%; margin-right: 5px; }
|
|
12247
|
+
.dot.open { background: #36d399; } .dot.closed { background: #5a6677; }
|
|
12248
|
+
main { display: flex; flex-direction: column; overflow: hidden; }
|
|
12249
|
+
.toolbar { display: flex; align-items: center; gap: 10px; padding: 10px 16px; border-bottom: 1px solid #1c2230; }
|
|
12250
|
+
.toolbar label { font-size: 13px; color: #aeb8cc; display: flex; align-items: center; gap: 6px; }
|
|
12251
|
+
.stage { flex: 1; position: relative; background: #05070b; overflow: auto; }
|
|
12252
|
+
.stage iframe { width: 100%; height: 100%; border: 0; display: block; }
|
|
12253
|
+
.empty { display: flex; align-items: center; justify-content: center; height: 100%; color: #5a6677; flex-direction: column; gap: 10px; }
|
|
12254
|
+
.replays { border-top: 1px solid #1c2230; padding: 10px 16px; max-height: 200px; overflow-y: auto; }
|
|
12255
|
+
.replays h3 { font-size: 11px; text-transform: uppercase; letter-spacing: .08em; color: #6b7689; margin: 0 0 8px; }
|
|
12256
|
+
.replay { display: flex; align-items: center; gap: 10px; padding: 6px 0; font-size: 13px; }
|
|
12257
|
+
.replay a { color: #7aa2ff; }
|
|
12258
|
+
.gate { max-width: 380px; margin: 80px auto; padding: 24px; border: 1px solid #1c2230; border-radius: 12px; background: #0f131c; }
|
|
12259
|
+
.gate h2 { margin: 0 0 6px; font-size: 16px; color: #fff; }
|
|
12260
|
+
.gate p { color: #8893a7; margin: 0 0 16px; }
|
|
12261
|
+
.gate input { width: 100%; margin-bottom: 12px; }
|
|
12262
|
+
.gate button { width: 100%; }
|
|
12263
|
+
.muted { color: #6b7689; font-size: 12px; }
|
|
12264
|
+
</style>
|
|
12265
|
+
</head>
|
|
12266
|
+
<body>
|
|
12267
|
+
<div id="app"></div>
|
|
12268
|
+
<script>
|
|
12269
|
+
const INITIAL_SESSION = ${initial};
|
|
12270
|
+
const KEY_STORE = 'browser_agent_api_key';
|
|
12271
|
+
let state = { key: localStorage.getItem(KEY_STORE) || '', sessions: [], current: INITIAL_SESSION || null, readOnly: true, liveUrl: null, replays: [] };
|
|
12272
|
+
|
|
12273
|
+
function api(method, path, body) {
|
|
12274
|
+
return fetch('/agent' + path, {
|
|
12275
|
+
method,
|
|
12276
|
+
headers: { 'Content-Type': 'application/json', 'x-api-key': state.key },
|
|
12277
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
12278
|
+
}).then(async r => ({ ok: r.ok, data: await r.json().catch(() => ({})) }));
|
|
12279
|
+
}
|
|
12280
|
+
|
|
12281
|
+
async function refreshSessions() {
|
|
12282
|
+
const r = await api('GET', '/sessions?all=1');
|
|
12283
|
+
if (r.ok) { state.sessions = r.data.sessions || []; render(); }
|
|
12284
|
+
}
|
|
12285
|
+
|
|
12286
|
+
async function selectSession(id) {
|
|
12287
|
+
state.current = id; state.liveUrl = null; state.replays = [];
|
|
12288
|
+
history.replaceState(null, '', '/console/' + id);
|
|
12289
|
+
render();
|
|
12290
|
+
const live = await api('GET', '/sessions/' + id + '/live-view');
|
|
12291
|
+
state.liveUrl = live.ok ? live.data.live_view_url : null;
|
|
12292
|
+
const reps = await api('GET', '/sessions/' + id + '/replays');
|
|
12293
|
+
state.replays = reps.ok ? (reps.data.replays || []) : [];
|
|
12294
|
+
render();
|
|
12295
|
+
}
|
|
12296
|
+
|
|
12297
|
+
async function openSession() {
|
|
12298
|
+
const r = await api('POST', '/sessions', { label: 'console' });
|
|
12299
|
+
if (r.ok) { await refreshSessions(); selectSession(r.data.session_id); }
|
|
12300
|
+
else alert('Open failed: ' + JSON.stringify(r.data));
|
|
12301
|
+
}
|
|
12302
|
+
|
|
12303
|
+
async function closeCurrent() {
|
|
12304
|
+
if (!state.current) return;
|
|
12305
|
+
await api('DELETE', '/sessions/' + state.current);
|
|
12306
|
+
await refreshSessions();
|
|
12307
|
+
}
|
|
12308
|
+
|
|
12309
|
+
async function downloadReplay(replayId) {
|
|
12310
|
+
if (!state.current || !replayId) return;
|
|
12311
|
+
const res = await fetch('/agent/sessions/' + encodeURIComponent(state.current) + '/replays/' + encodeURIComponent(replayId) + '/download', {
|
|
12312
|
+
headers: { 'x-api-key': state.key },
|
|
12313
|
+
});
|
|
12314
|
+
if (!res.ok) {
|
|
12315
|
+
alert('Replay download failed: ' + await res.text());
|
|
12316
|
+
return;
|
|
12317
|
+
}
|
|
12318
|
+
const blob = await res.blob();
|
|
12319
|
+
const url = URL.createObjectURL(blob);
|
|
12320
|
+
const a = document.createElement('a');
|
|
12321
|
+
a.href = url;
|
|
12322
|
+
a.download = state.current + '-' + replayId + '.mp4';
|
|
12323
|
+
document.body.appendChild(a);
|
|
12324
|
+
a.click();
|
|
12325
|
+
a.remove();
|
|
12326
|
+
URL.revokeObjectURL(url);
|
|
12327
|
+
}
|
|
12328
|
+
|
|
12329
|
+
function frameSrc() {
|
|
12330
|
+
if (!state.liveUrl) return null;
|
|
12331
|
+
const sep = state.liveUrl.includes('?') ? '&' : '?';
|
|
12332
|
+
return state.readOnly ? state.liveUrl + sep + 'readOnly=true' : state.liveUrl;
|
|
12333
|
+
}
|
|
12334
|
+
|
|
12335
|
+
function saveKey(v) { state.key = v.trim(); localStorage.setItem(KEY_STORE, state.key); render(); if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); } }
|
|
12336
|
+
|
|
12337
|
+
function h(html) { const t = document.createElement('template'); t.innerHTML = html.trim(); return t.content.firstChild; }
|
|
12338
|
+
function esc(s) { return String(s == null ? '' : s).replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); }
|
|
12339
|
+
|
|
12340
|
+
function render() {
|
|
12341
|
+
const app = document.getElementById('app');
|
|
12342
|
+
app.innerHTML = '';
|
|
12343
|
+
if (!state.key) {
|
|
12344
|
+
app.appendChild(h('<div class="gate"><h2>Browser Agent Console</h2><p>Paste your API key to watch and control browser sessions.</p><input id="k" type="password" placeholder="API key" /><button id="kb">Continue</button></div>'));
|
|
12345
|
+
document.getElementById('kb').onclick = () => saveKey(document.getElementById('k').value);
|
|
12346
|
+
document.getElementById('k').onkeydown = e => { if (e.key === 'Enter') saveKey(e.target.value); };
|
|
12347
|
+
return;
|
|
12348
|
+
}
|
|
12349
|
+
const header = h('<header><h1>Browser Agent</h1><span class="muted">live control + replays</span><span class="spacer"></span><button id="open">+ New Session</button><button class="ghost" id="logout">Forget key</button></header>');
|
|
12350
|
+
app.appendChild(header);
|
|
12351
|
+
document.getElementById('open').onclick = openSession;
|
|
12352
|
+
document.getElementById('logout').onclick = () => saveKey('');
|
|
12353
|
+
|
|
12354
|
+
const layout = h('<div class="layout"></div>');
|
|
12355
|
+
const aside = h('<aside><h2>Sessions</h2></aside>');
|
|
12356
|
+
if (!state.sessions.length) aside.appendChild(h('<div class="muted" style="padding:4px">No sessions yet.</div>'));
|
|
12357
|
+
for (const s of state.sessions) {
|
|
12358
|
+
const el = h('<div class="sess ' + (s.session_id === state.current ? 'active' : '') + '"><div class="id">' + esc(s.session_id) + '</div><div class="meta"><span class="dot ' + esc(s.status) + '"></span>' + esc(s.status) + (s.label ? ' \xB7 ' + esc(s.label) : '') + '</div></div>');
|
|
12359
|
+
el.onclick = () => selectSession(s.session_id);
|
|
12360
|
+
aside.appendChild(el);
|
|
12361
|
+
}
|
|
12362
|
+
layout.appendChild(aside);
|
|
12363
|
+
|
|
12364
|
+
const main = h('<main></main>');
|
|
12365
|
+
if (!state.current) {
|
|
12366
|
+
main.appendChild(h('<div class="empty"><div>Select or open a session to watch.</div></div>'));
|
|
12367
|
+
} else {
|
|
12368
|
+
const tb = h('<div class="toolbar"><label><input type="checkbox" id="ro" ' + (state.readOnly ? 'checked' : '') + ' /> Read-only (uncheck to take control)</label><span class="spacer"></span><button class="ghost" id="reload">Reload view</button><button class="ghost" id="close">Close session</button></div>');
|
|
12369
|
+
main.appendChild(tb);
|
|
12370
|
+
const stage = h('<div class="stage"></div>');
|
|
12371
|
+
const src = frameSrc();
|
|
12372
|
+
if (src) {
|
|
12373
|
+
const f = h('<iframe allow="autoplay; clipboard-read; clipboard-write" src="' + esc(src) + '"></iframe>');
|
|
12374
|
+
stage.appendChild(f);
|
|
12375
|
+
} else {
|
|
12376
|
+
stage.appendChild(h('<div class="empty"><div>Live view unavailable for this session.</div><div class="muted">It may be closed or still starting.</div></div>'));
|
|
12377
|
+
}
|
|
12378
|
+
main.appendChild(stage);
|
|
12379
|
+
|
|
12380
|
+
const rep = h('<div class="replays"><h3>Replays</h3></div>');
|
|
12381
|
+
if (!state.replays.length) rep.appendChild(h('<div class="muted">No replays recorded.</div>'));
|
|
12382
|
+
for (const r of state.replays) {
|
|
12383
|
+
const status = r.stopped_at ? 'ready' : 'recording...';
|
|
12384
|
+
const link = r.view_url ? '<a href="' + esc(r.view_url) + '" target="_blank" rel="noopener">view mp4</a>' : '<span class="muted">' + status + '</span>';
|
|
12385
|
+
const download = r.stopped_at ? '<button class="linkish replay-download" data-rid="' + esc(r.replay_id) + '">download mp4</button>' : '';
|
|
12386
|
+
rep.appendChild(h('<div class="replay"><span class="muted">' + esc(r.started_at || '') + '</span><span class="spacer"></span>' + link + download + '</div>'));
|
|
12387
|
+
}
|
|
12388
|
+
main.appendChild(rep);
|
|
12389
|
+
|
|
12390
|
+
layout.appendChild(main);
|
|
12391
|
+
}
|
|
12392
|
+
app.appendChild(layout);
|
|
12393
|
+
|
|
12394
|
+
const ro = document.getElementById('ro');
|
|
12395
|
+
if (ro) ro.onchange = e => { state.readOnly = e.target.checked; render(); };
|
|
12396
|
+
const reload = document.getElementById('reload');
|
|
12397
|
+
if (reload) reload.onclick = () => selectSession(state.current);
|
|
12398
|
+
const close = document.getElementById('close');
|
|
12399
|
+
if (close) close.onclick = closeCurrent;
|
|
12400
|
+
document.querySelectorAll('.replay-download').forEach(btn => {
|
|
12401
|
+
btn.onclick = () => downloadReplay(btn.getAttribute('data-rid'));
|
|
12402
|
+
});
|
|
12403
|
+
}
|
|
12404
|
+
|
|
12405
|
+
render();
|
|
12406
|
+
if (state.key) { refreshSessions(); if (state.current) selectSession(state.current); }
|
|
12407
|
+
</script>
|
|
12408
|
+
</body>
|
|
12409
|
+
</html>`;
|
|
12410
|
+
}
|
|
12411
|
+
|
|
10955
12412
|
// src/api/stripe-routes.ts
|
|
10956
12413
|
import Stripe from "stripe";
|
|
10957
|
-
import { Hono as
|
|
12414
|
+
import { Hono as Hono10 } from "hono";
|
|
10958
12415
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
10959
|
-
var stripeApp = new
|
|
12416
|
+
var stripeApp = new Hono10();
|
|
10960
12417
|
stripeApp.post("/webhooks", async (c) => {
|
|
10961
12418
|
const sig = c.req.header("stripe-signature");
|
|
10962
12419
|
const body = await c.req.text();
|
|
@@ -11051,27 +12508,27 @@ import { getCookie, setCookie, deleteCookie } from "hono/cookie";
|
|
|
11051
12508
|
import Stripe2 from "stripe";
|
|
11052
12509
|
|
|
11053
12510
|
// src/api/billing-schemas.ts
|
|
11054
|
-
import { z as
|
|
11055
|
-
var BillingCheckoutBodySchema =
|
|
11056
|
-
priceId:
|
|
12511
|
+
import { z as z17 } from "zod";
|
|
12512
|
+
var BillingCheckoutBodySchema = z17.object({
|
|
12513
|
+
priceId: z17.string().min(1)
|
|
11057
12514
|
});
|
|
11058
|
-
var FreeCreditBreakdownSchema =
|
|
11059
|
-
signup_grant_mc:
|
|
11060
|
-
monthly_refresh_mc:
|
|
11061
|
-
total_free_mc:
|
|
11062
|
-
signup_grant_credits:
|
|
11063
|
-
monthly_refresh_credits:
|
|
11064
|
-
total_free_credits:
|
|
12515
|
+
var FreeCreditBreakdownSchema = z17.object({
|
|
12516
|
+
signup_grant_mc: z17.number().int().nonnegative(),
|
|
12517
|
+
monthly_refresh_mc: z17.number().int().nonnegative(),
|
|
12518
|
+
total_free_mc: z17.number().int().nonnegative(),
|
|
12519
|
+
signup_grant_credits: z17.number().nonnegative(),
|
|
12520
|
+
monthly_refresh_credits: z17.number().nonnegative(),
|
|
12521
|
+
total_free_credits: z17.number().nonnegative()
|
|
11065
12522
|
});
|
|
11066
|
-
var BillingBalanceResponseSchema =
|
|
11067
|
-
balance_mc:
|
|
11068
|
-
balance_credits:
|
|
12523
|
+
var BillingBalanceResponseSchema = z17.object({
|
|
12524
|
+
balance_mc: z17.number().int().nonnegative(),
|
|
12525
|
+
balance_credits: z17.number().nonnegative(),
|
|
11069
12526
|
free_credits: FreeCreditBreakdownSchema,
|
|
11070
|
-
ledger:
|
|
12527
|
+
ledger: z17.array(z17.any())
|
|
11071
12528
|
});
|
|
11072
|
-
var MonthlyRefreshSweepResultSchema =
|
|
11073
|
-
usersRefreshed:
|
|
11074
|
-
totalMcGranted:
|
|
12529
|
+
var MonthlyRefreshSweepResultSchema = z17.object({
|
|
12530
|
+
usersRefreshed: z17.number().int().nonnegative(),
|
|
12531
|
+
totalMcGranted: z17.number().int().nonnegative()
|
|
11075
12532
|
});
|
|
11076
12533
|
|
|
11077
12534
|
// src/api/credit-operations.ts
|
|
@@ -11253,7 +12710,7 @@ var requireAllowedOrigin = createMiddleware3(async (c, next) => {
|
|
|
11253
12710
|
if (!configuredOrigins().has(origin)) return c.json({ error: "Origin not allowed" }, 403);
|
|
11254
12711
|
return next();
|
|
11255
12712
|
});
|
|
11256
|
-
var
|
|
12713
|
+
var auth2 = createMiddleware3(async (c, next) => {
|
|
11257
12714
|
const key = c.req.header("x-api-key");
|
|
11258
12715
|
if (!key) return c.json({ error: "Missing API key" }, 401);
|
|
11259
12716
|
const user = await getUserByApiKey(key);
|
|
@@ -11282,7 +12739,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
11282
12739
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
11283
12740
|
return next();
|
|
11284
12741
|
});
|
|
11285
|
-
var app = new
|
|
12742
|
+
var app = new Hono11();
|
|
11286
12743
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
11287
12744
|
function requireStripeSecret() {
|
|
11288
12745
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -11492,7 +12949,7 @@ async function checkHarvestLimits(userId, email, extraSlots = 0) {
|
|
|
11492
12949
|
if (active >= limit) return { error: `You have ${active} job${active !== 1 ? "s" : ""} running. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}. Wait for one to finish or add a concurrency slot at mcpscraper.dev/billing.` };
|
|
11493
12950
|
return null;
|
|
11494
12951
|
}
|
|
11495
|
-
app.post("/harvest",
|
|
12952
|
+
app.post("/harvest", auth2, async (c) => {
|
|
11496
12953
|
const user = c.get("user");
|
|
11497
12954
|
const raw = await c.req.json().catch(() => ({}));
|
|
11498
12955
|
const bodyResult = HarvestBodySchema.safeParse(raw);
|
|
@@ -11534,7 +12991,7 @@ app.post("/harvest", auth, async (c) => {
|
|
|
11534
12991
|
}
|
|
11535
12992
|
return c.json({ job_id: jobId, status: "pending" }, 202);
|
|
11536
12993
|
});
|
|
11537
|
-
app.post("/harvest/sync",
|
|
12994
|
+
app.post("/harvest/sync", auth2, async (c) => {
|
|
11538
12995
|
const user = c.get("user");
|
|
11539
12996
|
const raw = await c.req.json().catch(() => ({}));
|
|
11540
12997
|
const bodyResult = HarvestBodySchema.safeParse(raw);
|
|
@@ -11599,17 +13056,17 @@ app.post("/harvest/sync", auth, async (c) => {
|
|
|
11599
13056
|
return c.json({ job_id: jobId, status: "failed", ...response, attempts: sanitizeAttempts(attempts) }, problem.httpStatus);
|
|
11600
13057
|
}
|
|
11601
13058
|
});
|
|
11602
|
-
app.get("/jobs/:id",
|
|
13059
|
+
app.get("/jobs/:id", auth2, async (c) => {
|
|
11603
13060
|
const job = await getJob(c.req.param("id"), c.get("user").id);
|
|
11604
13061
|
if (!job) return c.json({ error: "Job not found" }, 404);
|
|
11605
13062
|
const attempts = await listHarvestAttempts(job.id, c.get("user").id);
|
|
11606
13063
|
const safeResult = job.result && typeof job.result === "object" ? sanitizeHarvestResult(job.result) : job.result;
|
|
11607
13064
|
return c.json({ ...job, result: safeResult, attempts: sanitizeAttempts(attempts) });
|
|
11608
13065
|
});
|
|
11609
|
-
app.get("/jobs",
|
|
13066
|
+
app.get("/jobs", auth2, async (c) => {
|
|
11610
13067
|
return c.json(await listJobs(c.get("user").id));
|
|
11611
13068
|
});
|
|
11612
|
-
app.get("/history",
|
|
13069
|
+
app.get("/history", auth2, async (c) => {
|
|
11613
13070
|
const userId = c.get("user").id;
|
|
11614
13071
|
const [jobs, events] = await Promise.all([
|
|
11615
13072
|
listJobs(userId),
|
|
@@ -11639,7 +13096,7 @@ app.get("/history", auth, async (c) => {
|
|
|
11639
13096
|
const rows = [...jobRows, ...eventRows].sort((a, b) => String(b.ts).localeCompare(String(a.ts)));
|
|
11640
13097
|
return c.json(rows.slice(0, 100));
|
|
11641
13098
|
});
|
|
11642
|
-
app.get("/ledger",
|
|
13099
|
+
app.get("/ledger", auth2, async (c) => {
|
|
11643
13100
|
return c.json(await getLedger(c.get("user").id, 100));
|
|
11644
13101
|
});
|
|
11645
13102
|
app.post("/admin/users", adminAuth, async (c) => {
|
|
@@ -11677,11 +13134,11 @@ app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
|
|
|
11677
13134
|
}
|
|
11678
13135
|
return c.json({ processed, credited, skipped, users_credited });
|
|
11679
13136
|
});
|
|
11680
|
-
app.post("/extract-url",
|
|
13137
|
+
app.post("/extract-url", auth2, async (c) => {
|
|
11681
13138
|
const raw = await c.req.json().catch(() => ({}));
|
|
11682
13139
|
const bodyResult = ExtractUrlBodySchema.safeParse(raw);
|
|
11683
13140
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
11684
|
-
const { url, screenshot, screenshotDevice, extractBranding, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
|
|
13141
|
+
const { url, screenshot: screenshot2, screenshotDevice, extractBranding, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
|
|
11685
13142
|
if (!allowLocal) {
|
|
11686
13143
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
11687
13144
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -11707,7 +13164,7 @@ app.post("/extract-url", auth, async (c) => {
|
|
|
11707
13164
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
11708
13165
|
const [result, pageData] = await Promise.all([
|
|
11709
13166
|
extractKpo({ url: canonicalUrl, kernelApiKey }),
|
|
11710
|
-
|
|
13167
|
+
screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch(() => null) : null
|
|
11711
13168
|
]);
|
|
11712
13169
|
const screenshotBuf = pageData?.screenshot ?? null;
|
|
11713
13170
|
const brandingData = pageData?.branding ?? null;
|
|
@@ -11725,7 +13182,7 @@ app.post("/extract-url", auth, async (c) => {
|
|
|
11725
13182
|
return c.json({ error: msg }, 500);
|
|
11726
13183
|
}
|
|
11727
13184
|
});
|
|
11728
|
-
app.post("/map-urls",
|
|
13185
|
+
app.post("/map-urls", auth2, async (c) => {
|
|
11729
13186
|
const raw = await c.req.json().catch(() => ({}));
|
|
11730
13187
|
const bodyResult = MapUrlsBodySchema.safeParse(raw);
|
|
11731
13188
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
@@ -11765,7 +13222,7 @@ app.post("/map-urls", auth, async (c) => {
|
|
|
11765
13222
|
return c.json({ error: msg }, 500);
|
|
11766
13223
|
}
|
|
11767
13224
|
});
|
|
11768
|
-
app.post("/extract-site",
|
|
13225
|
+
app.post("/extract-site", auth2, async (c) => {
|
|
11769
13226
|
const raw = await c.req.json().catch(() => ({}));
|
|
11770
13227
|
const bodyResult = ExtractSiteBodySchema.safeParse(raw);
|
|
11771
13228
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
@@ -11887,7 +13344,7 @@ app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async
|
|
|
11887
13344
|
await setConcurrencySubId(user.id, null);
|
|
11888
13345
|
return c.json({ ok: true, concurrency_limit: user.extra_concurrency_slots });
|
|
11889
13346
|
});
|
|
11890
|
-
app.get("/billing/balance",
|
|
13347
|
+
app.get("/billing/balance", auth2, async (c) => {
|
|
11891
13348
|
const user = c.get("user");
|
|
11892
13349
|
const balanceMc = await reconcileBalanceMc(user.id);
|
|
11893
13350
|
const ledger = await getLedger(user.id, 20);
|
|
@@ -11899,7 +13356,7 @@ app.get("/billing/balance", auth, async (c) => {
|
|
|
11899
13356
|
ledger
|
|
11900
13357
|
});
|
|
11901
13358
|
});
|
|
11902
|
-
app.post("/billing/credits",
|
|
13359
|
+
app.post("/billing/credits", auth2, async (c) => {
|
|
11903
13360
|
const user = c.get("user");
|
|
11904
13361
|
const balanceMc = await reconcileBalanceMc(user.id);
|
|
11905
13362
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -11928,7 +13385,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
11928
13385
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
11929
13386
|
return c.json({ error: "Unauthorized" }, 401);
|
|
11930
13387
|
}
|
|
11931
|
-
const { drainQueue } = await import("./worker-
|
|
13388
|
+
const { drainQueue } = await import("./worker-NAKGTIF5.js");
|
|
11932
13389
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
11933
13390
|
const [results, sweepResult] = await Promise.all([
|
|
11934
13391
|
drainQueue(budget),
|
|
@@ -11942,8 +13399,12 @@ app.route("/youtube", youtubeApp);
|
|
|
11942
13399
|
app.route("/screenshot", screenshotApp);
|
|
11943
13400
|
app.route("/facebook", facebookAdApp);
|
|
11944
13401
|
app.route("/maps", mapsApp);
|
|
13402
|
+
app.route("/directory", directoryApp);
|
|
11945
13403
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
11946
13404
|
app.route("/mcp", mcpApp);
|
|
13405
|
+
app.route("/agent", buildBrowserAgentRoutes());
|
|
13406
|
+
app.get("/console", (c) => c.html(renderConsoleHtml()));
|
|
13407
|
+
app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
|
|
11947
13408
|
app.route("/stripe", stripeApp);
|
|
11948
13409
|
if (!process.env.INNGEST_EVENT_KEY) {
|
|
11949
13410
|
startSiteAuditWorker();
|
|
@@ -12050,4 +13511,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
12050
13511
|
export {
|
|
12051
13512
|
app
|
|
12052
13513
|
};
|
|
12053
|
-
//# sourceMappingURL=server-
|
|
13514
|
+
//# sourceMappingURL=server-CJMX2QUM.js.map
|