mcp-scraper 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +2976 -2023
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +2 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +158 -22
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/bin/paa-harvest.cjs +1 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-6TDUY7CA.js → chunk-4MFYMUIF.js} +25 -3
- package/dist/chunk-4MFYMUIF.js.map +1 -0
- package/dist/{chunk-QUOZ3LZG.js → chunk-BAMVP57P.js} +157 -22
- package/dist/chunk-BAMVP57P.js.map +1 -0
- package/dist/{chunk-NNEIXK5L.js → chunk-FUWZWKGO.js} +2 -2
- package/dist/{chunk-HBC4C75S.js → chunk-NJK5BTUK.js} +2 -2
- package/dist/{chunk-JOEZJL4I.js → chunk-NVUBFCJK.js} +2 -2
- package/dist/chunk-SRMGSCC3.js +7 -0
- package/dist/chunk-SRMGSCC3.js.map +1 -0
- package/dist/{chunk-BP27CZ5Q.js → chunk-XGIPATLV.js} +2 -1
- package/dist/chunk-XGIPATLV.js.map +1 -0
- package/dist/index.cjs +1 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{server-U7V6DUSY.js → server-LBEYRKM2.js} +1134 -400
- package/dist/server-LBEYRKM2.js.map +1 -0
- package/dist/{site-extract-repository-2MQBAOX5.js → site-extract-repository-4NVO5KQA.js} +3 -3
- package/dist/{worker-RVZXIALX.js → worker-7XRZNLKB.js} +4 -4
- package/package.json +2 -1
- package/dist/chunk-2ULBZRQQ.js +0 -7
- package/dist/chunk-2ULBZRQQ.js.map +0 -1
- package/dist/chunk-6TDUY7CA.js.map +0 -1
- package/dist/chunk-BP27CZ5Q.js.map +0 -1
- package/dist/chunk-QUOZ3LZG.js.map +0 -1
- package/dist/server-U7V6DUSY.js.map +0 -1
- /package/dist/{chunk-NNEIXK5L.js.map → chunk-FUWZWKGO.js.map} +0 -0
- /package/dist/{chunk-HBC4C75S.js.map → chunk-NJK5BTUK.js.map} +0 -0
- /package/dist/{chunk-JOEZJL4I.js.map → chunk-NVUBFCJK.js.map} +0 -0
- /package/dist/{site-extract-repository-2MQBAOX5.js.map → site-extract-repository-4NVO5KQA.js.map} +0 -0
- /package/dist/{worker-RVZXIALX.js.map → worker-7XRZNLKB.js.map} +0 -0
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
recordVendorUsage,
|
|
17
17
|
resolveKernelProxyId,
|
|
18
18
|
runWithCostContext
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-NJK5BTUK.js";
|
|
20
20
|
import {
|
|
21
21
|
HttpMcpToolExecutor,
|
|
22
22
|
MemoryMcpToolExecutor,
|
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
registerSerpIntelligenceCaptureTools,
|
|
32
32
|
sanitizeAttempts,
|
|
33
33
|
sanitizeHarvestResult
|
|
34
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-BAMVP57P.js";
|
|
35
35
|
import {
|
|
36
36
|
auditImages,
|
|
37
37
|
getBlobStore
|
|
@@ -53,7 +53,7 @@ import {
|
|
|
53
53
|
workflowOutputBaseDir,
|
|
54
54
|
workflowStepCount,
|
|
55
55
|
workflowSupportsSteps
|
|
56
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-FUWZWKGO.js";
|
|
57
57
|
import "./chunk-K443GQY5.js";
|
|
58
58
|
import {
|
|
59
59
|
DEFAULT_MAPS_PROXY_MODE,
|
|
@@ -64,8 +64,8 @@ import {
|
|
|
64
64
|
RawMapsHoursRowSchema,
|
|
65
65
|
RawMapsOverviewSchema,
|
|
66
66
|
RawMapsReviewStatsSchema
|
|
67
|
-
} from "./chunk-
|
|
68
|
-
import "./chunk-
|
|
67
|
+
} from "./chunk-XGIPATLV.js";
|
|
68
|
+
import "./chunk-SRMGSCC3.js";
|
|
69
69
|
import {
|
|
70
70
|
completeExtractJob,
|
|
71
71
|
countSuccessfulPages,
|
|
@@ -76,7 +76,7 @@ import {
|
|
|
76
76
|
saveExtractPages,
|
|
77
77
|
setExtractJobTotal,
|
|
78
78
|
settleExtractJob
|
|
79
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-NVUBFCJK.js";
|
|
80
80
|
import {
|
|
81
81
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
82
82
|
CONCURRENCY_PRICE_ID,
|
|
@@ -95,7 +95,7 @@ import {
|
|
|
95
95
|
browserActiveCostMc,
|
|
96
96
|
concurrencySlotBillingInfo,
|
|
97
97
|
insufficientBalanceResponse
|
|
98
|
-
} from "./chunk-
|
|
98
|
+
} from "./chunk-4MFYMUIF.js";
|
|
99
99
|
import {
|
|
100
100
|
CaptchaError,
|
|
101
101
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -5274,7 +5274,7 @@ async function extractSite(opts) {
|
|
|
5274
5274
|
}
|
|
5275
5275
|
|
|
5276
5276
|
// src/api/server.ts
|
|
5277
|
-
import { Hono as
|
|
5277
|
+
import { Hono as Hono21 } from "hono";
|
|
5278
5278
|
import { serve as serveInngest } from "inngest/hono";
|
|
5279
5279
|
|
|
5280
5280
|
// src/inngest/client.ts
|
|
@@ -11846,6 +11846,8 @@ function verifySession(token) {
|
|
|
11846
11846
|
}
|
|
11847
11847
|
|
|
11848
11848
|
// src/api/memory.ts
|
|
11849
|
+
import { provisionDefaultsTool } from "mcpscraper-memory-tools/tools/vaults/provision-defaults";
|
|
11850
|
+
import { setScheduleEntitlementTool } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
|
|
11849
11851
|
var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
|
|
11850
11852
|
var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
11851
11853
|
function parseMemoryRpcBody(text) {
|
|
@@ -11933,6 +11935,24 @@ async function provisionMemoryForUser(user) {
|
|
|
11933
11935
|
} catch {
|
|
11934
11936
|
}
|
|
11935
11937
|
}
|
|
11938
|
+
function memoryPlanForUser(user) {
|
|
11939
|
+
switch (user.subscription_tier) {
|
|
11940
|
+
case "starter":
|
|
11941
|
+
return "pro";
|
|
11942
|
+
case "growth":
|
|
11943
|
+
return "team";
|
|
11944
|
+
case "scale":
|
|
11945
|
+
return "enterprise";
|
|
11946
|
+
default:
|
|
11947
|
+
return "free";
|
|
11948
|
+
}
|
|
11949
|
+
}
|
|
11950
|
+
var MEMORY_PLAN_RANK = { free: 0, pro: 1, team: 2, enterprise: 3 };
|
|
11951
|
+
function resolveEffectiveMemoryPlan(user) {
|
|
11952
|
+
const tierPlan = memoryPlanForUser(user);
|
|
11953
|
+
const memoryProPlan = user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
|
|
11954
|
+
return MEMORY_PLAN_RANK[tierPlan] >= MEMORY_PLAN_RANK[memoryProPlan] ? tierPlan : memoryProPlan;
|
|
11955
|
+
}
|
|
11936
11956
|
async function getOrCreateUserMemoryKey(user) {
|
|
11937
11957
|
const creds = await getUserMemoryCreds(user.id);
|
|
11938
11958
|
if (creds.memory_key) {
|
|
@@ -11943,10 +11963,9 @@ async function getOrCreateUserMemoryKey(user) {
|
|
|
11943
11963
|
if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
|
|
11944
11964
|
const identity = memoryIdentity(user);
|
|
11945
11965
|
const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
|
|
11946
|
-
const provisioned = await
|
|
11947
|
-
|
|
11948
|
-
{
|
|
11949
|
-
admin
|
|
11966
|
+
const provisioned = await provisionDefaultsTool.execute(
|
|
11967
|
+
{ apiKey: admin, granteeIdentity: identity, issueKey: true, plan },
|
|
11968
|
+
{}
|
|
11950
11969
|
);
|
|
11951
11970
|
if (!provisioned.ok || !provisioned.secret) {
|
|
11952
11971
|
return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
|
|
@@ -11973,12 +11992,13 @@ async function syncScheduleEntitlement(user, enabled, quotaPerPeriod) {
|
|
|
11973
11992
|
const admin = ADMIN_KEY();
|
|
11974
11993
|
if (!admin) return { ok: false, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
|
|
11975
11994
|
const identity = resolveMemoryIdentity(user);
|
|
11976
|
-
const res = await
|
|
11995
|
+
const res = await setScheduleEntitlementTool.execute({
|
|
11996
|
+
apiKey: admin,
|
|
11977
11997
|
granteeIdentity: identity,
|
|
11978
11998
|
enabled,
|
|
11979
11999
|
quotaPerPeriod,
|
|
11980
12000
|
mcpScraperApiKey: user.api_key
|
|
11981
|
-
},
|
|
12001
|
+
}, {});
|
|
11982
12002
|
return { ok: res.ok, error: res.error };
|
|
11983
12003
|
}
|
|
11984
12004
|
|
|
@@ -12138,6 +12158,33 @@ var MapsNavigator = class {
|
|
|
12138
12158
|
await this.page.waitForSelector('[role="main"] h2', { timeout: 5e3 }).catch(() => {
|
|
12139
12159
|
});
|
|
12140
12160
|
}
|
|
12161
|
+
async navigateToSerpBusinessPanel(businessName, location2) {
|
|
12162
|
+
const query = `${businessName} ${location2}`;
|
|
12163
|
+
const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&udm=1`;
|
|
12164
|
+
await this.page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 45e3 }).catch(() => {
|
|
12165
|
+
});
|
|
12166
|
+
await this.page.waitForTimeout(2e3);
|
|
12167
|
+
const clicked = await this.page.evaluate((name) => {
|
|
12168
|
+
const trimmedName = name.trim();
|
|
12169
|
+
const words = trimmedName.split(/\s+/);
|
|
12170
|
+
const prefix = words.slice(0, Math.min(2, words.length)).join(" ");
|
|
12171
|
+
const headings = Array.from(document.querySelectorAll('h3, div[role="heading"]'));
|
|
12172
|
+
const heading = headings.find((el2) => el2.innerText?.trim() === trimmedName) ?? headings.find((el2) => el2.innerText?.trim().startsWith(prefix));
|
|
12173
|
+
const clickable = heading?.closest('a, div[role="link"], div[jsaction]');
|
|
12174
|
+
if (clickable) {
|
|
12175
|
+
clickable.click();
|
|
12176
|
+
return true;
|
|
12177
|
+
}
|
|
12178
|
+
if (heading) {
|
|
12179
|
+
heading.click();
|
|
12180
|
+
return true;
|
|
12181
|
+
}
|
|
12182
|
+
return false;
|
|
12183
|
+
}, businessName);
|
|
12184
|
+
if (!clicked) return false;
|
|
12185
|
+
await this.page.waitForTimeout(2e3);
|
|
12186
|
+
return true;
|
|
12187
|
+
}
|
|
12141
12188
|
buildReviewsUrl(placeUrl) {
|
|
12142
12189
|
const fid = placeUrl.match(/!1s(0x[0-9a-f]+:0x[0-9a-f]+)/i);
|
|
12143
12190
|
const pin = placeUrl.match(/!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)/);
|
|
@@ -12307,6 +12354,25 @@ var MapsExtractor = class {
|
|
|
12307
12354
|
}
|
|
12308
12355
|
await navigator.navigateToAboutTab();
|
|
12309
12356
|
const aboutAttributes = await this.extractAbout(page);
|
|
12357
|
+
let services = [];
|
|
12358
|
+
let areasServed = [];
|
|
12359
|
+
let servicesStatus = "not_requested";
|
|
12360
|
+
if (options.includeServices) {
|
|
12361
|
+
try {
|
|
12362
|
+
const opened = await navigator.navigateToSerpBusinessPanel(options.businessName, options.location);
|
|
12363
|
+
if (opened) {
|
|
12364
|
+
const result = await this.extractServicesAndAreasServed(page);
|
|
12365
|
+
services = result.services;
|
|
12366
|
+
areasServed = result.areasServed;
|
|
12367
|
+
servicesStatus = services.length > 0 || areasServed.length > 0 ? "collected" : "none_exist";
|
|
12368
|
+
} else {
|
|
12369
|
+
servicesStatus = "unavailable";
|
|
12370
|
+
}
|
|
12371
|
+
} catch (err) {
|
|
12372
|
+
console.warn("[MapsExtractor] services/areas-served extraction failed:", err instanceof Error ? err.message : String(err));
|
|
12373
|
+
servicesStatus = "unavailable";
|
|
12374
|
+
}
|
|
12375
|
+
}
|
|
12310
12376
|
const reviewCountFinal = overview.reviewCount ?? (reviewStats.reviewHistogram.length > 0 ? reviewStats.reviewHistogram.reduce((s, r) => s + parseInt(r.count, 10), 0).toString() : null);
|
|
12311
12377
|
return {
|
|
12312
12378
|
businessName: options.businessName,
|
|
@@ -12322,6 +12388,9 @@ var MapsExtractor = class {
|
|
|
12322
12388
|
aboutAttributes,
|
|
12323
12389
|
reviews,
|
|
12324
12390
|
reviewsStatus,
|
|
12391
|
+
services,
|
|
12392
|
+
areasServed,
|
|
12393
|
+
servicesStatus,
|
|
12325
12394
|
durationMs: Date.now() - startMs
|
|
12326
12395
|
};
|
|
12327
12396
|
} finally {
|
|
@@ -12510,6 +12579,81 @@ var MapsExtractor = class {
|
|
|
12510
12579
|
}
|
|
12511
12580
|
return result.data;
|
|
12512
12581
|
}
|
|
12582
|
+
async extractServicesAndAreasServed(page) {
|
|
12583
|
+
await page.waitForTimeout(1500);
|
|
12584
|
+
let services = [];
|
|
12585
|
+
if (await this.clickRowStartingWithRetry(page, "Services:")) {
|
|
12586
|
+
await page.waitForSelector('[role="dialog"]', { timeout: 5e3 }).catch(() => {
|
|
12587
|
+
});
|
|
12588
|
+
await page.waitForTimeout(800);
|
|
12589
|
+
services = await this.extractOpenDialogItems(page, "Services");
|
|
12590
|
+
await this.closeOpenDialog(page, "Services");
|
|
12591
|
+
await page.waitForTimeout(500);
|
|
12592
|
+
}
|
|
12593
|
+
let areasServed = [];
|
|
12594
|
+
if (await this.clickRowStartingWithRetry(page, "Serves ")) {
|
|
12595
|
+
await page.waitForSelector('[role="dialog"]', { timeout: 5e3 }).catch(() => {
|
|
12596
|
+
});
|
|
12597
|
+
await page.waitForTimeout(800);
|
|
12598
|
+
areasServed = await this.extractOpenDialogItems(page, "Areas Served");
|
|
12599
|
+
await this.closeOpenDialog(page, "Areas Served");
|
|
12600
|
+
}
|
|
12601
|
+
return { services, areasServed };
|
|
12602
|
+
}
|
|
12603
|
+
async clickRowStartingWithRetry(page, prefix) {
|
|
12604
|
+
if (await this.clickRowStartingWith(page, prefix)) return true;
|
|
12605
|
+
await page.waitForTimeout(2e3);
|
|
12606
|
+
return this.clickRowStartingWith(page, prefix);
|
|
12607
|
+
}
|
|
12608
|
+
async clickRowStartingWith(page, prefix) {
|
|
12609
|
+
return page.evaluate((p) => {
|
|
12610
|
+
const el2 = Array.from(document.querySelectorAll('button, a, div[role="button"]')).find(
|
|
12611
|
+
(e) => (e.innerText ?? "").trim().startsWith(p)
|
|
12612
|
+
);
|
|
12613
|
+
if (!el2) return false;
|
|
12614
|
+
el2.click();
|
|
12615
|
+
return true;
|
|
12616
|
+
}, prefix);
|
|
12617
|
+
}
|
|
12618
|
+
async extractOpenDialogItems(page, expectedHeading) {
|
|
12619
|
+
return page.evaluate((expected) => {
|
|
12620
|
+
const dialogs = Array.from(document.querySelectorAll('[role="dialog"]'));
|
|
12621
|
+
const withHeading = dialogs.find((d) => {
|
|
12622
|
+
const h = d.querySelector('h1, h2, [role="heading"]');
|
|
12623
|
+
const headingText = h?.innerText?.trim() ?? "";
|
|
12624
|
+
return headingText.startsWith(expected);
|
|
12625
|
+
});
|
|
12626
|
+
if (!withHeading) return [];
|
|
12627
|
+
const headingEl = withHeading.querySelector('h1, h2, [role="heading"]');
|
|
12628
|
+
const headingLines = new Set(
|
|
12629
|
+
(headingEl?.innerText ?? "").split("\n").map((s) => s.trim()).filter(Boolean)
|
|
12630
|
+
);
|
|
12631
|
+
const seen = /* @__PURE__ */ new Set();
|
|
12632
|
+
const items = [];
|
|
12633
|
+
withHeading.querySelectorAll("*").forEach((el2) => {
|
|
12634
|
+
if (el2.children.length > 0) return;
|
|
12635
|
+
const text = el2.innerText?.trim() ?? el2.textContent?.trim() ?? "";
|
|
12636
|
+
if (!text || headingLines.has(text)) return;
|
|
12637
|
+
if (/^(back|close|done|thanks)$/i.test(text)) return;
|
|
12638
|
+
if (/^serves\s.*following/i.test(text)) return;
|
|
12639
|
+
if (seen.has(text)) return;
|
|
12640
|
+
seen.add(text);
|
|
12641
|
+
items.push(text);
|
|
12642
|
+
});
|
|
12643
|
+
return items;
|
|
12644
|
+
}, expectedHeading);
|
|
12645
|
+
}
|
|
12646
|
+
async closeOpenDialog(page, expectedHeading) {
|
|
12647
|
+
await page.evaluate((expected) => {
|
|
12648
|
+
const dialog = Array.from(document.querySelectorAll('[role="dialog"]')).find((d) => {
|
|
12649
|
+
const h = d.querySelector('h1, h2, [role="heading"]');
|
|
12650
|
+
const headingText = h?.innerText?.trim() ?? "";
|
|
12651
|
+
return headingText.startsWith(expected);
|
|
12652
|
+
});
|
|
12653
|
+
const closeBtn = dialog?.querySelector('[aria-label*="Close" i], button[aria-label*="close" i]');
|
|
12654
|
+
closeBtn?.click();
|
|
12655
|
+
}, expectedHeading);
|
|
12656
|
+
}
|
|
12513
12657
|
};
|
|
12514
12658
|
|
|
12515
12659
|
// src/extractor/MapsSearchExtractor.ts
|
|
@@ -13016,13 +13160,163 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
13016
13160
|
}
|
|
13017
13161
|
});
|
|
13018
13162
|
|
|
13019
|
-
// src/api/
|
|
13163
|
+
// src/api/trustpilot-routes.ts
|
|
13020
13164
|
import { Hono as Hono11 } from "hono";
|
|
13165
|
+
import { z as z19 } from "zod";
|
|
13166
|
+
var TrustpilotReviewsBodySchema = z19.object({
|
|
13167
|
+
domain: z19.string().trim().min(1, "domain is required"),
|
|
13168
|
+
maxPages: z19.number().int().min(1).max(50).default(5)
|
|
13169
|
+
});
|
|
13170
|
+
function invalidRequest6(message) {
|
|
13171
|
+
return { error_code: "invalid_request", message };
|
|
13172
|
+
}
|
|
13173
|
+
function normalizeDomain(input) {
|
|
13174
|
+
return input.trim().replace(/^https?:\/\//i, "").replace(/\/.*$/, "");
|
|
13175
|
+
}
|
|
13176
|
+
function reviewPageUrl(domain, page) {
|
|
13177
|
+
const base = `https://www.trustpilot.com/review/${encodeURIComponent(domain)}`;
|
|
13178
|
+
return page <= 1 ? base : `${base}?page=${page}`;
|
|
13179
|
+
}
|
|
13180
|
+
async function extractReviewCards(page) {
|
|
13181
|
+
return page.evaluate(() => {
|
|
13182
|
+
const cards = Array.from(document.querySelectorAll('article[data-service-review-card-paper="true"]')).filter((a) => a.querySelector('[data-testid="service-review-card-v2"]'));
|
|
13183
|
+
return cards.map((card) => {
|
|
13184
|
+
const profileLink = card.querySelector('a[data-consumer-profile-link="true"]');
|
|
13185
|
+
const profileHref = profileLink?.getAttribute("href") ?? "";
|
|
13186
|
+
const profileId = profileHref.startsWith("/users/") ? profileHref.slice("/users/".length) : null;
|
|
13187
|
+
const name = card.querySelector('[data-consumer-name-typography="true"]')?.textContent?.trim() ?? null;
|
|
13188
|
+
const timeEl = card.querySelector("time[datetime]");
|
|
13189
|
+
const date = timeEl?.getAttribute("datetime") ?? null;
|
|
13190
|
+
const ratingEl = card.querySelector("[data-service-review-rating]");
|
|
13191
|
+
const ratingRaw = ratingEl?.getAttribute("data-service-review-rating");
|
|
13192
|
+
const rating = ratingRaw ? parseInt(ratingRaw, 10) : null;
|
|
13193
|
+
const titleLink = card.querySelector('a[data-review-title-typography="true"]');
|
|
13194
|
+
const titleHref = titleLink?.getAttribute("href") ?? "";
|
|
13195
|
+
const reviewId = titleHref.startsWith("/reviews/") ? titleHref.slice("/reviews/".length) : null;
|
|
13196
|
+
const title = card.querySelector('[data-service-review-title-typography="true"]')?.textContent?.trim() ?? null;
|
|
13197
|
+
const bodyEl = card.querySelector('[data-service-review-text-typography="true"]');
|
|
13198
|
+
const rawBody = bodyEl?.textContent?.trim() ?? "";
|
|
13199
|
+
const seeMoreEl = card.querySelector('[class*="seeMore"]');
|
|
13200
|
+
const truncated = !!seeMoreEl;
|
|
13201
|
+
const body = truncated ? rawBody.replace(/see more\s*$/i, "").trim() : rawBody;
|
|
13202
|
+
const labelEl = card.querySelector('[class*="reviewLabel"]');
|
|
13203
|
+
const labelText = labelEl?.textContent?.trim() ?? null;
|
|
13204
|
+
const origin = labelText && /invit/i.test(labelText) ? "invited" : labelText ? labelText.toLowerCase() : "organic";
|
|
13205
|
+
const companyReplied = !!card.querySelector('[class*="companyReply"]');
|
|
13206
|
+
return {
|
|
13207
|
+
source: "trustpilot",
|
|
13208
|
+
sourceReviewId: reviewId,
|
|
13209
|
+
reviewUrl: reviewId ? `https://www.trustpilot.com/reviews/${reviewId}` : null,
|
|
13210
|
+
reviewer: {
|
|
13211
|
+
name,
|
|
13212
|
+
profileId,
|
|
13213
|
+
title: null,
|
|
13214
|
+
companySegment: null
|
|
13215
|
+
},
|
|
13216
|
+
rating,
|
|
13217
|
+
ratingScale: 5,
|
|
13218
|
+
title,
|
|
13219
|
+
date,
|
|
13220
|
+
body: [{ question: null, answer: body }],
|
|
13221
|
+
truncated,
|
|
13222
|
+
flags: {
|
|
13223
|
+
origin,
|
|
13224
|
+
incentivized: null,
|
|
13225
|
+
validated: null,
|
|
13226
|
+
currentUser: null,
|
|
13227
|
+
companyReplied
|
|
13228
|
+
}
|
|
13229
|
+
};
|
|
13230
|
+
});
|
|
13231
|
+
});
|
|
13232
|
+
}
|
|
13233
|
+
var trustpilotApp = new Hono11();
|
|
13234
|
+
trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13235
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13236
|
+
const parsed = TrustpilotReviewsBodySchema.safeParse(raw);
|
|
13237
|
+
if (!parsed.success) return c.json(invalidRequest6(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13238
|
+
const body = parsed.data;
|
|
13239
|
+
const domain = normalizeDomain(body.domain);
|
|
13240
|
+
const user = c.get("user");
|
|
13241
|
+
const gate = await acquireConcurrencyGate(user, "trustpilot_reviews", {
|
|
13242
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13243
|
+
metadata: { domain, maxPages: body.maxPages }
|
|
13244
|
+
});
|
|
13245
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13246
|
+
let debited = false;
|
|
13247
|
+
let reviewDebitMc = 0;
|
|
13248
|
+
try {
|
|
13249
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS, domain);
|
|
13250
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.trustpilot_reviews), 402);
|
|
13251
|
+
debited = true;
|
|
13252
|
+
const startMs = Date.now();
|
|
13253
|
+
const driver = new BrowserDriver();
|
|
13254
|
+
const allReviews = [];
|
|
13255
|
+
let pagesFetched = 0;
|
|
13256
|
+
try {
|
|
13257
|
+
await driver.launch({
|
|
13258
|
+
headless: true,
|
|
13259
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13260
|
+
viewport: { width: 1280, height: 900 },
|
|
13261
|
+
locale: "en-US"
|
|
13262
|
+
});
|
|
13263
|
+
const page = driver.getPage();
|
|
13264
|
+
for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
|
|
13265
|
+
const url = reviewPageUrl(domain, pageNum);
|
|
13266
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
13267
|
+
await page.waitForTimeout(2e3);
|
|
13268
|
+
const cards = await extractReviewCards(page);
|
|
13269
|
+
pagesFetched = pageNum;
|
|
13270
|
+
if (cards.length === 0) break;
|
|
13271
|
+
allReviews.push(...cards);
|
|
13272
|
+
}
|
|
13273
|
+
} finally {
|
|
13274
|
+
await driver.close();
|
|
13275
|
+
}
|
|
13276
|
+
if (pagesFetched === 0 || allReviews.length === 0) {
|
|
13277
|
+
await creditMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS_REFUND, "no reviews found");
|
|
13278
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "failed", query: domain, error: "no reviews found" });
|
|
13279
|
+
return c.json({ error: "No reviews found for this domain (refunded)", domain }, 404);
|
|
13280
|
+
}
|
|
13281
|
+
reviewDebitMc = allReviews.length * MC_COSTS.trustpilot_review;
|
|
13282
|
+
if (reviewDebitMc > 0) {
|
|
13283
|
+
const reviewDebit = await debitMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW, domain);
|
|
13284
|
+
if (!reviewDebit.ok) {
|
|
13285
|
+
allReviews.length = Math.floor((reviewDebit.balance_mc ?? 0) / MC_COSTS.trustpilot_review);
|
|
13286
|
+
reviewDebitMc = allReviews.length * MC_COSTS.trustpilot_review;
|
|
13287
|
+
if (reviewDebitMc > 0) await debitMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW, domain);
|
|
13288
|
+
}
|
|
13289
|
+
}
|
|
13290
|
+
const result = {
|
|
13291
|
+
domain,
|
|
13292
|
+
reviewUrl: reviewPageUrl(domain, 1),
|
|
13293
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13294
|
+
requestedMaxPages: body.maxPages,
|
|
13295
|
+
pagesFetched,
|
|
13296
|
+
reviewCount: allReviews.length,
|
|
13297
|
+
reviews: allReviews,
|
|
13298
|
+
durationMs: Date.now() - startMs
|
|
13299
|
+
};
|
|
13300
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "done", query: domain, resultCount: allReviews.length, result });
|
|
13301
|
+
return c.json(result);
|
|
13302
|
+
} catch (err) {
|
|
13303
|
+
if (debited) await creditMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS_REFUND, "failed call");
|
|
13304
|
+
if (reviewDebitMc > 0) await creditMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW_REFUND, "failed call");
|
|
13305
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13306
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "failed", query: domain, error: msg });
|
|
13307
|
+
return c.json({ error: msg }, 500);
|
|
13308
|
+
} finally {
|
|
13309
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13310
|
+
}
|
|
13311
|
+
});
|
|
13312
|
+
|
|
13313
|
+
// src/api/directory-routes.ts
|
|
13314
|
+
import { Hono as Hono12 } from "hono";
|
|
13021
13315
|
|
|
13022
13316
|
// src/directory/directory-workflow.ts
|
|
13023
13317
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
13024
13318
|
import { join as join5 } from "path";
|
|
13025
|
-
import { z as
|
|
13319
|
+
import { z as z20 } from "zod";
|
|
13026
13320
|
|
|
13027
13321
|
// src/directory/location-db.ts
|
|
13028
13322
|
import { access, readFile } from "fs/promises";
|
|
@@ -13183,24 +13477,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
13183
13477
|
}
|
|
13184
13478
|
|
|
13185
13479
|
// src/directory/directory-workflow.ts
|
|
13186
|
-
var DirectoryWorkflowOptionsSchema =
|
|
13187
|
-
query:
|
|
13188
|
-
state:
|
|
13189
|
-
minPopulation:
|
|
13190
|
-
populationYear:
|
|
13191
|
-
maxCities:
|
|
13192
|
-
maxResultsPerCity:
|
|
13193
|
-
concurrency:
|
|
13194
|
-
includeZipGroups:
|
|
13195
|
-
usZipsCsvPath:
|
|
13196
|
-
saveCsv:
|
|
13197
|
-
gl:
|
|
13198
|
-
hl:
|
|
13199
|
-
proxyMode:
|
|
13200
|
-
proxyZip:
|
|
13201
|
-
debug:
|
|
13202
|
-
headless:
|
|
13203
|
-
kernelApiKey:
|
|
13480
|
+
var DirectoryWorkflowOptionsSchema = z20.object({
|
|
13481
|
+
query: z20.string().min(1),
|
|
13482
|
+
state: z20.string().min(2).default("TN"),
|
|
13483
|
+
minPopulation: z20.number().int().min(0).default(1e5),
|
|
13484
|
+
populationYear: z20.union(POPULATION_YEARS.map((year) => z20.literal(year))).default(2025),
|
|
13485
|
+
maxCities: z20.number().int().min(1).max(100).default(25),
|
|
13486
|
+
maxResultsPerCity: z20.number().int().min(1).max(50).default(50),
|
|
13487
|
+
concurrency: z20.number().int().min(1).max(5).default(5),
|
|
13488
|
+
includeZipGroups: z20.boolean().default(true),
|
|
13489
|
+
usZipsCsvPath: z20.string().optional(),
|
|
13490
|
+
saveCsv: z20.boolean().default(true),
|
|
13491
|
+
gl: z20.string().length(2).default("us"),
|
|
13492
|
+
hl: z20.string().length(2).default("en"),
|
|
13493
|
+
proxyMode: z20.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
13494
|
+
proxyZip: z20.string().regex(/^\d{5}$/).optional(),
|
|
13495
|
+
debug: z20.boolean().default(false),
|
|
13496
|
+
headless: z20.boolean().default(true),
|
|
13497
|
+
kernelApiKey: z20.string().optional()
|
|
13204
13498
|
});
|
|
13205
13499
|
function errorMessage(err) {
|
|
13206
13500
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -13421,7 +13715,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
13421
13715
|
}
|
|
13422
13716
|
|
|
13423
13717
|
// src/api/directory-routes.ts
|
|
13424
|
-
var directoryApp = new
|
|
13718
|
+
var directoryApp = new Hono12();
|
|
13425
13719
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
13426
13720
|
const user = c.get("user");
|
|
13427
13721
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13494,35 +13788,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
13494
13788
|
// src/api/workflow-routes.ts
|
|
13495
13789
|
import { createHmac as createHmac2 } from "crypto";
|
|
13496
13790
|
import { readFile as readFile2 } from "fs/promises";
|
|
13497
|
-
import { Hono as
|
|
13498
|
-
import { z as
|
|
13499
|
-
var workflowApp = new
|
|
13500
|
-
var WorkflowInputSchema =
|
|
13501
|
-
var WorkflowIdSchema =
|
|
13502
|
-
var CadenceSchema =
|
|
13503
|
-
var ScheduleStatusSchema =
|
|
13504
|
-
var RunBodySchema =
|
|
13791
|
+
import { Hono as Hono13 } from "hono";
|
|
13792
|
+
import { z as z21 } from "zod";
|
|
13793
|
+
var workflowApp = new Hono13();
|
|
13794
|
+
var WorkflowInputSchema = z21.record(z21.unknown()).default({});
|
|
13795
|
+
var WorkflowIdSchema = z21.string().min(1);
|
|
13796
|
+
var CadenceSchema = z21.enum(["daily", "weekly", "monthly"]);
|
|
13797
|
+
var ScheduleStatusSchema = z21.enum(["active", "paused"]);
|
|
13798
|
+
var RunBodySchema = z21.object({
|
|
13505
13799
|
workflowId: WorkflowIdSchema,
|
|
13506
13800
|
input: WorkflowInputSchema,
|
|
13507
|
-
webhookUrl:
|
|
13801
|
+
webhookUrl: z21.string().url().optional()
|
|
13508
13802
|
});
|
|
13509
|
-
var ScheduleCreateSchema =
|
|
13803
|
+
var ScheduleCreateSchema = z21.object({
|
|
13510
13804
|
workflowId: WorkflowIdSchema,
|
|
13511
|
-
name:
|
|
13805
|
+
name: z21.string().min(1).max(120).optional(),
|
|
13512
13806
|
input: WorkflowInputSchema,
|
|
13513
13807
|
cadence: CadenceSchema.default("weekly"),
|
|
13514
|
-
timezone:
|
|
13515
|
-
webhookUrl:
|
|
13516
|
-
nextRunAt:
|
|
13808
|
+
timezone: z21.string().min(1).max(64).default("UTC"),
|
|
13809
|
+
webhookUrl: z21.string().url().optional(),
|
|
13810
|
+
nextRunAt: z21.string().datetime().optional()
|
|
13517
13811
|
});
|
|
13518
|
-
var SchedulePatchSchema =
|
|
13519
|
-
name:
|
|
13812
|
+
var SchedulePatchSchema = z21.object({
|
|
13813
|
+
name: z21.string().min(1).max(120).optional(),
|
|
13520
13814
|
status: ScheduleStatusSchema.optional(),
|
|
13521
13815
|
input: WorkflowInputSchema.optional(),
|
|
13522
13816
|
cadence: CadenceSchema.optional(),
|
|
13523
|
-
timezone:
|
|
13524
|
-
webhookUrl:
|
|
13525
|
-
nextRunAt:
|
|
13817
|
+
timezone: z21.string().min(1).max(64).optional(),
|
|
13818
|
+
webhookUrl: z21.string().url().nullable().optional(),
|
|
13819
|
+
nextRunAt: z21.string().datetime().nullable().optional()
|
|
13526
13820
|
});
|
|
13527
13821
|
function hostedWorkflowOutputDir() {
|
|
13528
13822
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -14026,7 +14320,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
14026
14320
|
});
|
|
14027
14321
|
|
|
14028
14322
|
// src/api/serp-intelligence-routes.ts
|
|
14029
|
-
import { Hono as
|
|
14323
|
+
import { Hono as Hono14 } from "hono";
|
|
14030
14324
|
|
|
14031
14325
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14032
14326
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -14338,7 +14632,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14338
14632
|
}
|
|
14339
14633
|
|
|
14340
14634
|
// src/serp-intelligence/schemas.ts
|
|
14341
|
-
import { z as
|
|
14635
|
+
import { z as z22 } from "zod";
|
|
14342
14636
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
14343
14637
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
14344
14638
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -14400,171 +14694,171 @@ function isPublicHttpUrl(value) {
|
|
|
14400
14694
|
return false;
|
|
14401
14695
|
}
|
|
14402
14696
|
}
|
|
14403
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
14404
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
14405
|
-
query:
|
|
14406
|
-
location:
|
|
14407
|
-
gl:
|
|
14408
|
-
hl:
|
|
14409
|
-
device:
|
|
14410
|
-
proxyMode:
|
|
14411
|
-
proxyZip:
|
|
14412
|
-
pages:
|
|
14413
|
-
debug:
|
|
14414
|
-
includePageSnapshots:
|
|
14415
|
-
pageSnapshotLimit:
|
|
14697
|
+
var SerpIntelligencePublicHttpUrlSchema = z22.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
14698
|
+
var SerpIntelligenceCaptureBodySchema = z22.object({
|
|
14699
|
+
query: z22.string().trim().min(1, "query is required"),
|
|
14700
|
+
location: z22.string().trim().min(1).optional(),
|
|
14701
|
+
gl: z22.string().trim().length(2).default("us"),
|
|
14702
|
+
hl: z22.string().trim().length(2).default("en"),
|
|
14703
|
+
device: z22.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
14704
|
+
proxyMode: z22.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
14705
|
+
proxyZip: z22.string().regex(/^\d{5}$/).optional(),
|
|
14706
|
+
pages: z22.number().int().min(1).max(2).default(1),
|
|
14707
|
+
debug: z22.boolean().default(false),
|
|
14708
|
+
includePageSnapshots: z22.boolean().default(false),
|
|
14709
|
+
pageSnapshotLimit: z22.number().int().min(0).max(10).default(0)
|
|
14416
14710
|
}).strict();
|
|
14417
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
14711
|
+
var SerpIntelligencePageSnapshotRequestSchema = z22.object({
|
|
14418
14712
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14419
|
-
sourceKind:
|
|
14420
|
-
sourcePosition:
|
|
14713
|
+
sourceKind: z22.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
14714
|
+
sourcePosition: z22.number().int().min(1).optional()
|
|
14421
14715
|
}).strict();
|
|
14422
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
14423
|
-
urls:
|
|
14424
|
-
targets:
|
|
14425
|
-
maxConcurrency:
|
|
14426
|
-
timeoutMs:
|
|
14427
|
-
debug:
|
|
14716
|
+
var SerpIntelligencePageSnapshotsBodySchema = z22.object({
|
|
14717
|
+
urls: z22.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
14718
|
+
targets: z22.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
14719
|
+
maxConcurrency: z22.number().int().min(1).max(5).default(2),
|
|
14720
|
+
timeoutMs: z22.number().int().min(1e3).max(6e4).default(15e3),
|
|
14721
|
+
debug: z22.boolean().default(false)
|
|
14428
14722
|
}).strict();
|
|
14429
|
-
var SerpIntelligenceAICitationSchema =
|
|
14430
|
-
text:
|
|
14431
|
-
href:
|
|
14723
|
+
var SerpIntelligenceAICitationSchema = z22.object({
|
|
14724
|
+
text: z22.string(),
|
|
14725
|
+
href: z22.string()
|
|
14432
14726
|
}).strict();
|
|
14433
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
14434
|
-
position:
|
|
14435
|
-
title:
|
|
14436
|
-
url:
|
|
14437
|
-
domain:
|
|
14438
|
-
cite:
|
|
14439
|
-
snippet:
|
|
14440
|
-
isRedditStyle:
|
|
14441
|
-
inlineRating:
|
|
14442
|
-
value:
|
|
14443
|
-
count:
|
|
14727
|
+
var SerpIntelligenceOrganicResultSchema = z22.object({
|
|
14728
|
+
position: z22.number().int().min(1),
|
|
14729
|
+
title: z22.string(),
|
|
14730
|
+
url: z22.string(),
|
|
14731
|
+
domain: z22.string(),
|
|
14732
|
+
cite: z22.string().nullable(),
|
|
14733
|
+
snippet: z22.string().nullable(),
|
|
14734
|
+
isRedditStyle: z22.boolean(),
|
|
14735
|
+
inlineRating: z22.object({
|
|
14736
|
+
value: z22.string(),
|
|
14737
|
+
count: z22.string()
|
|
14444
14738
|
}).strict().nullable()
|
|
14445
14739
|
}).strict();
|
|
14446
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
14447
|
-
status:
|
|
14448
|
-
expected:
|
|
14449
|
-
city:
|
|
14450
|
-
regionCode:
|
|
14451
|
-
canonicalLocation:
|
|
14740
|
+
var SerpIntelligenceLocationEvidenceSchema = z22.object({
|
|
14741
|
+
status: z22.enum(SerpIntelligenceLocalizationStatusValues),
|
|
14742
|
+
expected: z22.object({
|
|
14743
|
+
city: z22.string(),
|
|
14744
|
+
regionCode: z22.string().nullable(),
|
|
14745
|
+
canonicalLocation: z22.string()
|
|
14452
14746
|
}).strict().nullable(),
|
|
14453
|
-
candidates:
|
|
14454
|
-
city:
|
|
14455
|
-
regionCode:
|
|
14456
|
-
count:
|
|
14457
|
-
examples:
|
|
14747
|
+
candidates: z22.array(z22.object({
|
|
14748
|
+
city: z22.string(),
|
|
14749
|
+
regionCode: z22.string(),
|
|
14750
|
+
count: z22.number().int().min(0),
|
|
14751
|
+
examples: z22.array(z22.string())
|
|
14458
14752
|
}).strict())
|
|
14459
14753
|
}).strict();
|
|
14460
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
14461
|
-
seed:
|
|
14462
|
-
location:
|
|
14463
|
-
extractedAt:
|
|
14464
|
-
totalQuestions:
|
|
14465
|
-
surface:
|
|
14466
|
-
aiOverview:
|
|
14467
|
-
detected:
|
|
14468
|
-
text:
|
|
14469
|
-
citations:
|
|
14470
|
-
expanded:
|
|
14471
|
-
fullyExpanded:
|
|
14472
|
-
sections:
|
|
14754
|
+
var SerpIntelligenceHarvestResultSchema = z22.object({
|
|
14755
|
+
seed: z22.string(),
|
|
14756
|
+
location: z22.string().nullable(),
|
|
14757
|
+
extractedAt: z22.string(),
|
|
14758
|
+
totalQuestions: z22.number().int().min(0),
|
|
14759
|
+
surface: z22.enum(["web", "aim", "unknown"]),
|
|
14760
|
+
aiOverview: z22.object({
|
|
14761
|
+
detected: z22.boolean(),
|
|
14762
|
+
text: z22.string().nullable(),
|
|
14763
|
+
citations: z22.array(SerpIntelligenceAICitationSchema),
|
|
14764
|
+
expanded: z22.boolean().optional(),
|
|
14765
|
+
fullyExpanded: z22.boolean().optional(),
|
|
14766
|
+
sections: z22.array(z22.string()).optional()
|
|
14473
14767
|
}).strict(),
|
|
14474
|
-
aiMode:
|
|
14475
|
-
detected:
|
|
14476
|
-
text:
|
|
14477
|
-
citations:
|
|
14768
|
+
aiMode: z22.object({
|
|
14769
|
+
detected: z22.boolean(),
|
|
14770
|
+
text: z22.string().nullable(),
|
|
14771
|
+
citations: z22.array(SerpIntelligenceAICitationSchema)
|
|
14478
14772
|
}).strict(),
|
|
14479
|
-
tree:
|
|
14480
|
-
flat:
|
|
14481
|
-
videos:
|
|
14482
|
-
forums:
|
|
14483
|
-
organicResults:
|
|
14484
|
-
localPack:
|
|
14485
|
-
entityIds:
|
|
14486
|
-
entities:
|
|
14487
|
-
name:
|
|
14488
|
-
kgId:
|
|
14489
|
-
cid:
|
|
14490
|
-
gcid:
|
|
14773
|
+
tree: z22.array(z22.unknown()),
|
|
14774
|
+
flat: z22.array(z22.unknown()),
|
|
14775
|
+
videos: z22.array(z22.unknown()),
|
|
14776
|
+
forums: z22.array(z22.unknown()),
|
|
14777
|
+
organicResults: z22.array(SerpIntelligenceOrganicResultSchema),
|
|
14778
|
+
localPack: z22.array(z22.unknown()),
|
|
14779
|
+
entityIds: z22.object({
|
|
14780
|
+
entities: z22.array(z22.object({
|
|
14781
|
+
name: z22.string(),
|
|
14782
|
+
kgId: z22.string().nullable(),
|
|
14783
|
+
cid: z22.string().nullable(),
|
|
14784
|
+
gcid: z22.string().nullable()
|
|
14491
14785
|
}).strict()),
|
|
14492
|
-
kgIds:
|
|
14493
|
-
cids:
|
|
14494
|
-
gcids:
|
|
14786
|
+
kgIds: z22.array(z22.string()),
|
|
14787
|
+
cids: z22.array(z22.string()),
|
|
14788
|
+
gcids: z22.array(z22.string())
|
|
14495
14789
|
}).strict(),
|
|
14496
|
-
stats:
|
|
14497
|
-
seed:
|
|
14498
|
-
totalQuestions:
|
|
14499
|
-
maxDepthReached:
|
|
14500
|
-
durationMs:
|
|
14501
|
-
errorCount:
|
|
14790
|
+
stats: z22.object({
|
|
14791
|
+
seed: z22.string(),
|
|
14792
|
+
totalQuestions: z22.number().int().min(0),
|
|
14793
|
+
maxDepthReached: z22.number().int().min(0),
|
|
14794
|
+
durationMs: z22.number().min(0),
|
|
14795
|
+
errorCount: z22.number().int().min(0)
|
|
14502
14796
|
}).strict(),
|
|
14503
|
-
diagnostics:
|
|
14504
|
-
completionStatus:
|
|
14505
|
-
problem:
|
|
14506
|
-
warnings:
|
|
14507
|
-
debug:
|
|
14797
|
+
diagnostics: z22.object({
|
|
14798
|
+
completionStatus: z22.enum(["paa_found", "no_paa", "serp_only"]),
|
|
14799
|
+
problem: z22.null(),
|
|
14800
|
+
warnings: z22.array(z22.unknown()).optional(),
|
|
14801
|
+
debug: z22.object({
|
|
14508
14802
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
14509
14803
|
}).passthrough().optional()
|
|
14510
14804
|
}).passthrough(),
|
|
14511
|
-
whatPeopleSaying:
|
|
14805
|
+
whatPeopleSaying: z22.array(z22.unknown())
|
|
14512
14806
|
}).strict();
|
|
14513
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
14514
|
-
attemptNumber:
|
|
14515
|
-
outcome:
|
|
14516
|
-
startedAt:
|
|
14517
|
-
completedAt:
|
|
14518
|
-
durationMs:
|
|
14519
|
-
problemCode:
|
|
14520
|
-
message:
|
|
14521
|
-
kernelSessionId:
|
|
14522
|
-
cleanupSucceeded:
|
|
14807
|
+
var SerpIntelligenceCaptureAttemptSchema = z22.object({
|
|
14808
|
+
attemptNumber: z22.number().int().min(1),
|
|
14809
|
+
outcome: z22.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
14810
|
+
startedAt: z22.string().optional(),
|
|
14811
|
+
completedAt: z22.string().optional(),
|
|
14812
|
+
durationMs: z22.number().min(0).optional(),
|
|
14813
|
+
problemCode: z22.string().optional(),
|
|
14814
|
+
message: z22.string().optional(),
|
|
14815
|
+
kernelSessionId: z22.string().nullable().optional(),
|
|
14816
|
+
cleanupSucceeded: z22.boolean().nullable().optional()
|
|
14523
14817
|
}).strict();
|
|
14524
|
-
var SerpPageSnapshotCaptureSchema =
|
|
14818
|
+
var SerpPageSnapshotCaptureSchema = z22.object({
|
|
14525
14819
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14526
14820
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
14527
14821
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14528
|
-
sourceKind:
|
|
14529
|
-
sourcePosition:
|
|
14530
|
-
status:
|
|
14531
|
-
fetchedVia:
|
|
14532
|
-
httpStatus:
|
|
14533
|
-
contentType:
|
|
14534
|
-
title:
|
|
14822
|
+
sourceKind: z22.enum(SerpPageSnapshotSourceKindValues),
|
|
14823
|
+
sourcePosition: z22.number().int().min(1).nullable(),
|
|
14824
|
+
status: z22.enum(SerpPageFetchStatusValues),
|
|
14825
|
+
fetchedVia: z22.enum(SerpPageFetchedViaValues).nullable(),
|
|
14826
|
+
httpStatus: z22.number().int().min(100).max(599).nullable(),
|
|
14827
|
+
contentType: z22.string().nullable(),
|
|
14828
|
+
title: z22.string().nullable(),
|
|
14535
14829
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14536
|
-
metaDescription:
|
|
14537
|
-
headings:
|
|
14538
|
-
level:
|
|
14539
|
-
text:
|
|
14830
|
+
metaDescription: z22.string().nullable(),
|
|
14831
|
+
headings: z22.array(z22.object({
|
|
14832
|
+
level: z22.number().int().min(1).max(6),
|
|
14833
|
+
text: z22.string()
|
|
14540
14834
|
}).strict()).default([]),
|
|
14541
|
-
artifact:
|
|
14542
|
-
htmlBlobUrl:
|
|
14543
|
-
textBlobUrl:
|
|
14544
|
-
markdownBlobUrl:
|
|
14545
|
-
screenshotBlobUrl:
|
|
14546
|
-
contentSha256:
|
|
14547
|
-
capturedAt:
|
|
14835
|
+
artifact: z22.object({
|
|
14836
|
+
htmlBlobUrl: z22.string().url().nullable(),
|
|
14837
|
+
textBlobUrl: z22.string().url().nullable(),
|
|
14838
|
+
markdownBlobUrl: z22.string().url().nullable(),
|
|
14839
|
+
screenshotBlobUrl: z22.string().url().nullable(),
|
|
14840
|
+
contentSha256: z22.string().nullable(),
|
|
14841
|
+
capturedAt: z22.string().nullable()
|
|
14548
14842
|
}).strict(),
|
|
14549
|
-
error:
|
|
14550
|
-
code:
|
|
14551
|
-
message:
|
|
14843
|
+
error: z22.object({
|
|
14844
|
+
code: z22.string(),
|
|
14845
|
+
message: z22.string()
|
|
14552
14846
|
}).strict().nullable()
|
|
14553
14847
|
}).strict();
|
|
14554
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
14848
|
+
var SerpIntelligenceCaptureResponseSchema = z22.object({
|
|
14555
14849
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
14556
|
-
attempts:
|
|
14850
|
+
attempts: z22.array(SerpIntelligenceCaptureAttemptSchema),
|
|
14557
14851
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
14558
|
-
pageSnapshotArtifacts:
|
|
14559
|
-
billing:
|
|
14560
|
-
creditsUsed:
|
|
14561
|
-
requestId:
|
|
14562
|
-
jobId:
|
|
14852
|
+
pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
|
|
14853
|
+
billing: z22.object({
|
|
14854
|
+
creditsUsed: z22.number().min(0).optional(),
|
|
14855
|
+
requestId: z22.string().optional(),
|
|
14856
|
+
jobId: z22.string().optional()
|
|
14563
14857
|
}).strict().optional()
|
|
14564
14858
|
}).strict();
|
|
14565
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
14566
|
-
pageSnapshotArtifacts:
|
|
14567
|
-
attempts:
|
|
14859
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z22.object({
|
|
14860
|
+
pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
|
|
14861
|
+
attempts: z22.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
14568
14862
|
}).strict();
|
|
14569
14863
|
|
|
14570
14864
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -14736,7 +15030,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
14736
15030
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
14737
15031
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
14738
15032
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
14739
|
-
var serpIntelligenceApp = new
|
|
15033
|
+
var serpIntelligenceApp = new Hono14();
|
|
14740
15034
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
14741
15035
|
function structuredError(input) {
|
|
14742
15036
|
return {
|
|
@@ -14905,9 +15199,447 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
14905
15199
|
});
|
|
14906
15200
|
|
|
14907
15201
|
// src/mcp/mcp-routes.ts
|
|
14908
|
-
import { Hono as
|
|
15202
|
+
import { Hono as Hono15 } from "hono";
|
|
14909
15203
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
14910
15204
|
|
|
15205
|
+
// src/mcp/memory-inprocess-executor.ts
|
|
15206
|
+
var CUTOVER = /* @__PURE__ */ new Map();
|
|
15207
|
+
function registerCutover(upstreamName, handler) {
|
|
15208
|
+
CUTOVER.set(upstreamName, handler);
|
|
15209
|
+
}
|
|
15210
|
+
var HybridMemoryToolExecutor = class {
|
|
15211
|
+
constructor(httpExecutor, user) {
|
|
15212
|
+
this.httpExecutor = httpExecutor;
|
|
15213
|
+
this.user = user;
|
|
15214
|
+
}
|
|
15215
|
+
httpExecutor;
|
|
15216
|
+
user;
|
|
15217
|
+
mkKeyPromise = null;
|
|
15218
|
+
async resolveMkKey() {
|
|
15219
|
+
if (!this.mkKeyPromise) {
|
|
15220
|
+
this.mkKeyPromise = getOrCreateUserMemoryKey(this.user).then((r) => r.key);
|
|
15221
|
+
}
|
|
15222
|
+
return this.mkKeyPromise;
|
|
15223
|
+
}
|
|
15224
|
+
async callMemoryTool(upstreamName, args) {
|
|
15225
|
+
const handler = CUTOVER.get(upstreamName);
|
|
15226
|
+
if (!handler) return this.httpExecutor.callMemoryTool(upstreamName, args);
|
|
15227
|
+
const mkKey = await this.resolveMkKey();
|
|
15228
|
+
if (!mkKey) return { content: [{ type: "text", text: "memory unavailable" }], isError: true };
|
|
15229
|
+
try {
|
|
15230
|
+
return await handler(args, mkKey);
|
|
15231
|
+
} catch (err) {
|
|
15232
|
+
const message = err instanceof Error ? err.message : `memory ${upstreamName} failed in-process`;
|
|
15233
|
+
return { content: [{ type: "text", text: message }], isError: true };
|
|
15234
|
+
}
|
|
15235
|
+
}
|
|
15236
|
+
};
|
|
15237
|
+
|
|
15238
|
+
// src/mcp/memory-cutover/reads.ts
|
|
15239
|
+
import { listApprovedSendersTool } from "mcpscraper-memory-tools/tools/access/list-approved-senders";
|
|
15240
|
+
import { noteInboxTool } from "mcpscraper-memory-tools/tools/access/note-inbox";
|
|
15241
|
+
import { getMessageNoteTool } from "mcpscraper-memory-tools/tools/channels/get-message-note";
|
|
15242
|
+
import { listChannelMembersTool } from "mcpscraper-memory-tools/tools/channels/list-channel-members";
|
|
15243
|
+
import { myMentionsTool } from "mcpscraper-memory-tools/tools/channels/my-mentions";
|
|
15244
|
+
import { factHistoryTool } from "mcpscraper-memory-tools/tools/facts/fact-history";
|
|
15245
|
+
import { exportTool } from "mcpscraper-memory-tools/tools/memory/export";
|
|
15246
|
+
import { getTool } from "mcpscraper-memory-tools/tools/memory/get";
|
|
15247
|
+
import { listTool } from "mcpscraper-memory-tools/tools/memory/list";
|
|
15248
|
+
import { searchTool } from "mcpscraper-memory-tools/tools/memory/search";
|
|
15249
|
+
import { suggestTool } from "mcpscraper-memory-tools/tools/memory/suggest";
|
|
15250
|
+
import { temporalRecallTool } from "mcpscraper-memory-tools/tools/recall/temporal-recall";
|
|
15251
|
+
import { listScheduledActionsTool } from "mcpscraper-memory-tools/tools/schedule/list-scheduled-actions";
|
|
15252
|
+
import { costUsageTool } from "mcpscraper-memory-tools/tools/storage/cost-usage";
|
|
15253
|
+
import { storageUsageTool } from "mcpscraper-memory-tools/tools/storage/storage-usage";
|
|
15254
|
+
import { describeTableTool } from "mcpscraper-memory-tools/tools/tables/table-describe";
|
|
15255
|
+
import { listTablesTool } from "mcpscraper-memory-tools/tools/tables/table-list";
|
|
15256
|
+
import { queryTableTool } from "mcpscraper-memory-tools/tools/tables/table-query";
|
|
15257
|
+
import { listSharedWithMeTool } from "mcpscraper-memory-tools/tools/vaults/list-shared-with-me";
|
|
15258
|
+
import { listVaultsTool } from "mcpscraper-memory-tools/tools/vaults/list-vaults";
|
|
15259
|
+
import { videoAnalyzeStatusTool } from "mcpscraper-memory-tools/tools/video/status";
|
|
15260
|
+
import { listWebhooksTool } from "mcpscraper-memory-tools/tools/webhooks/list-webhooks";
|
|
15261
|
+
var registered = false;
|
|
15262
|
+
function registerReadCutovers() {
|
|
15263
|
+
if (registered) return;
|
|
15264
|
+
registered = true;
|
|
15265
|
+
registerCutover("listApprovedSendersTool", async (input, apiKey) => {
|
|
15266
|
+
const result = await listApprovedSendersTool.execute({ ...input, apiKey }, {});
|
|
15267
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15268
|
+
});
|
|
15269
|
+
registerCutover("noteInboxTool", async (input, apiKey) => {
|
|
15270
|
+
const result = await noteInboxTool.execute({ ...input, apiKey }, {});
|
|
15271
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15272
|
+
});
|
|
15273
|
+
registerCutover("getMessageNoteTool", async (input, apiKey) => {
|
|
15274
|
+
const result = await getMessageNoteTool.execute({ ...input, apiKey }, {});
|
|
15275
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15276
|
+
});
|
|
15277
|
+
registerCutover("listChannelMembersTool", async (input, apiKey) => {
|
|
15278
|
+
const result = await listChannelMembersTool.execute({ ...input, apiKey }, {});
|
|
15279
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15280
|
+
});
|
|
15281
|
+
registerCutover("myMentionsTool", async (input, apiKey) => {
|
|
15282
|
+
const result = await myMentionsTool.execute({ ...input, apiKey }, {});
|
|
15283
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15284
|
+
});
|
|
15285
|
+
registerCutover("factHistoryTool", async (input, apiKey) => {
|
|
15286
|
+
const result = await factHistoryTool.execute({ ...input, apiKey }, {});
|
|
15287
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15288
|
+
});
|
|
15289
|
+
registerCutover("exportTool", async (input, apiKey) => {
|
|
15290
|
+
const result = await exportTool.execute({ ...input, apiKey }, {});
|
|
15291
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15292
|
+
});
|
|
15293
|
+
registerCutover("getTool", async (input, apiKey) => {
|
|
15294
|
+
const result = await getTool.execute({ ...input, apiKey }, {});
|
|
15295
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15296
|
+
});
|
|
15297
|
+
registerCutover("listTool", async (input, apiKey) => {
|
|
15298
|
+
const result = await listTool.execute({ ...input, apiKey }, {});
|
|
15299
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15300
|
+
});
|
|
15301
|
+
registerCutover("searchTool", async (input, apiKey) => {
|
|
15302
|
+
const result = await searchTool.execute({ ...input, apiKey }, {});
|
|
15303
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15304
|
+
});
|
|
15305
|
+
registerCutover("suggestTool", async (input, apiKey) => {
|
|
15306
|
+
const result = await suggestTool.execute({ ...input, apiKey }, {});
|
|
15307
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15308
|
+
});
|
|
15309
|
+
registerCutover("temporalRecallTool", async (input, apiKey) => {
|
|
15310
|
+
const result = await temporalRecallTool.execute({ ...input, apiKey }, {});
|
|
15311
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15312
|
+
});
|
|
15313
|
+
registerCutover("listScheduledActionsTool", async (input, apiKey) => {
|
|
15314
|
+
const result = await listScheduledActionsTool.execute({ ...input, apiKey }, {});
|
|
15315
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15316
|
+
});
|
|
15317
|
+
registerCutover("costUsageTool", async (input, apiKey) => {
|
|
15318
|
+
const result = await costUsageTool.execute({ ...input, apiKey }, {});
|
|
15319
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15320
|
+
});
|
|
15321
|
+
registerCutover("storageUsageTool", async (input, apiKey) => {
|
|
15322
|
+
const result = await storageUsageTool.execute({ ...input, apiKey }, {});
|
|
15323
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15324
|
+
});
|
|
15325
|
+
registerCutover("describeTableTool", async (input, apiKey) => {
|
|
15326
|
+
const result = await describeTableTool.execute({ ...input, apiKey }, {});
|
|
15327
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15328
|
+
});
|
|
15329
|
+
registerCutover("listTablesTool", async (input, apiKey) => {
|
|
15330
|
+
const result = await listTablesTool.execute({ ...input, apiKey }, {});
|
|
15331
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15332
|
+
});
|
|
15333
|
+
registerCutover("queryTableTool", async (input, apiKey) => {
|
|
15334
|
+
const result = await queryTableTool.execute({ ...input, apiKey }, {});
|
|
15335
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15336
|
+
});
|
|
15337
|
+
registerCutover("listSharedWithMeTool", async (input, apiKey) => {
|
|
15338
|
+
const result = await listSharedWithMeTool.execute({ ...input, apiKey }, {});
|
|
15339
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15340
|
+
});
|
|
15341
|
+
registerCutover("listVaultsTool", async (input, apiKey) => {
|
|
15342
|
+
const result = await listVaultsTool.execute({ ...input, apiKey }, {});
|
|
15343
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15344
|
+
});
|
|
15345
|
+
registerCutover("videoAnalyzeStatusTool", async (input, apiKey) => {
|
|
15346
|
+
const result = await videoAnalyzeStatusTool.execute({ ...input, apiKey }, {});
|
|
15347
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15348
|
+
});
|
|
15349
|
+
registerCutover("listWebhooksTool", async (input, apiKey) => {
|
|
15350
|
+
const result = await listWebhooksTool.execute({ ...input, apiKey }, {});
|
|
15351
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15352
|
+
});
|
|
15353
|
+
}
|
|
15354
|
+
|
|
15355
|
+
// src/mcp/memory-cutover/writes.ts
|
|
15356
|
+
import { acceptShareTool } from "mcpscraper-memory-tools/tools/access/accept-share";
|
|
15357
|
+
import { approveSenderTool } from "mcpscraper-memory-tools/tools/access/approve-sender";
|
|
15358
|
+
import { declineShareTool } from "mcpscraper-memory-tools/tools/access/decline-share";
|
|
15359
|
+
import { inboxSettingsTool } from "mcpscraper-memory-tools/tools/access/inbox-settings";
|
|
15360
|
+
import { removeApprovedSenderTool } from "mcpscraper-memory-tools/tools/access/remove-approved-sender";
|
|
15361
|
+
import { revokeShareTool } from "mcpscraper-memory-tools/tools/access/revoke-share";
|
|
15362
|
+
import { setAgentIdentityTool } from "mcpscraper-memory-tools/tools/access/set-agent-identity";
|
|
15363
|
+
import { shareNoteTool } from "mcpscraper-memory-tools/tools/access/share-note";
|
|
15364
|
+
import { shareVaultTool } from "mcpscraper-memory-tools/tools/access/share-vault";
|
|
15365
|
+
import { swapVaultTool } from "mcpscraper-memory-tools/tools/access/swap-vault";
|
|
15366
|
+
import { switchAccountTool } from "mcpscraper-memory-tools/tools/access/switch-account";
|
|
15367
|
+
import { unlinkShareTool } from "mcpscraper-memory-tools/tools/access/unlink-share";
|
|
15368
|
+
import { memoryQuestionsTool } from "mcpscraper-memory-tools/tools/capture/memory-questions";
|
|
15369
|
+
import { createChannelTool } from "mcpscraper-memory-tools/tools/channels/create-channel";
|
|
15370
|
+
import { listChannelMessagesTool } from "mcpscraper-memory-tools/tools/channels/list-channel-messages";
|
|
15371
|
+
import { pollChannelTool } from "mcpscraper-memory-tools/tools/channels/poll-channel";
|
|
15372
|
+
import { postMessageTool } from "mcpscraper-memory-tools/tools/channels/post-message";
|
|
15373
|
+
import { reactMessageTool } from "mcpscraper-memory-tools/tools/channels/react-message";
|
|
15374
|
+
import { removeChannelMemberTool } from "mcpscraper-memory-tools/tools/channels/remove-channel-member";
|
|
15375
|
+
import { replyMessageTool } from "mcpscraper-memory-tools/tools/channels/reply-message";
|
|
15376
|
+
import { recordFactTool } from "mcpscraper-memory-tools/tools/facts/record-fact";
|
|
15377
|
+
import { libraryIngestTool } from "mcpscraper-memory-tools/tools/library/library-ingest";
|
|
15378
|
+
import { deleteNoteTool } from "mcpscraper-memory-tools/tools/memory/delete-note";
|
|
15379
|
+
import { putTool } from "mcpscraper-memory-tools/tools/memory/put";
|
|
15380
|
+
import { uploadTool } from "mcpscraper-memory-tools/tools/memory/upload";
|
|
15381
|
+
import { deleteScheduledActionTool } from "mcpscraper-memory-tools/tools/schedule/delete-scheduled-action";
|
|
15382
|
+
import { pauseScheduledActionTool } from "mcpscraper-memory-tools/tools/schedule/pause-scheduled-action";
|
|
15383
|
+
import { resumeScheduledActionTool } from "mcpscraper-memory-tools/tools/schedule/resume-scheduled-action";
|
|
15384
|
+
import { createTableTool } from "mcpscraper-memory-tools/tools/tables/table-create";
|
|
15385
|
+
import { deleteTableRowsTool } from "mcpscraper-memory-tools/tools/tables/table-delete-rows";
|
|
15386
|
+
import { dropTableTool } from "mcpscraper-memory-tools/tools/tables/table-drop";
|
|
15387
|
+
import { insertTableRowsTool } from "mcpscraper-memory-tools/tools/tables/table-insert-rows";
|
|
15388
|
+
import { addVaultTool } from "mcpscraper-memory-tools/tools/vaults/add-vault";
|
|
15389
|
+
import { deleteVaultTool } from "mcpscraper-memory-tools/tools/vaults/delete-vault";
|
|
15390
|
+
var registered2 = false;
|
|
15391
|
+
function registerWriteCutovers() {
|
|
15392
|
+
if (registered2) return;
|
|
15393
|
+
registered2 = true;
|
|
15394
|
+
registerCutover("acceptShareTool", async (input, apiKey) => {
|
|
15395
|
+
const result = await acceptShareTool.execute({ ...input, apiKey }, {});
|
|
15396
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15397
|
+
});
|
|
15398
|
+
registerCutover("approveSenderTool", async (input, apiKey) => {
|
|
15399
|
+
const result = await approveSenderTool.execute({ ...input, apiKey }, {});
|
|
15400
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15401
|
+
});
|
|
15402
|
+
registerCutover("declineShareTool", async (input, apiKey) => {
|
|
15403
|
+
const result = await declineShareTool.execute({ ...input, apiKey }, {});
|
|
15404
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15405
|
+
});
|
|
15406
|
+
registerCutover("inboxSettingsTool", async (input, apiKey) => {
|
|
15407
|
+
const result = await inboxSettingsTool.execute({ ...input, apiKey }, {});
|
|
15408
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15409
|
+
});
|
|
15410
|
+
registerCutover("removeApprovedSenderTool", async (input, apiKey) => {
|
|
15411
|
+
const result = await removeApprovedSenderTool.execute({ ...input, apiKey }, {});
|
|
15412
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15413
|
+
});
|
|
15414
|
+
registerCutover("revokeShareTool", async (input, apiKey) => {
|
|
15415
|
+
const result = await revokeShareTool.execute({ ...input, apiKey }, {});
|
|
15416
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15417
|
+
});
|
|
15418
|
+
registerCutover("setAgentIdentityTool", async (input, apiKey) => {
|
|
15419
|
+
const result = await setAgentIdentityTool.execute({ ...input, apiKey }, {});
|
|
15420
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15421
|
+
});
|
|
15422
|
+
registerCutover("shareNoteTool", async (input, apiKey) => {
|
|
15423
|
+
const result = await shareNoteTool.execute({ ...input, apiKey }, {});
|
|
15424
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15425
|
+
});
|
|
15426
|
+
registerCutover("shareVaultTool", async (input, apiKey) => {
|
|
15427
|
+
const result = await shareVaultTool.execute({ ...input, apiKey }, {});
|
|
15428
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15429
|
+
});
|
|
15430
|
+
registerCutover("swapVaultTool", async (input, apiKey) => {
|
|
15431
|
+
const result = await swapVaultTool.execute({ ...input, apiKey }, {});
|
|
15432
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15433
|
+
});
|
|
15434
|
+
registerCutover("switchAccountTool", async (input, apiKey) => {
|
|
15435
|
+
const result = await switchAccountTool.execute({ ...input, apiKey }, {});
|
|
15436
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15437
|
+
});
|
|
15438
|
+
registerCutover("unlinkShareTool", async (input, apiKey) => {
|
|
15439
|
+
const result = await unlinkShareTool.execute({ ...input, apiKey }, {});
|
|
15440
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15441
|
+
});
|
|
15442
|
+
registerCutover("memoryQuestionsTool", async (input, apiKey) => {
|
|
15443
|
+
const result = await memoryQuestionsTool.execute({ ...input, apiKey }, {});
|
|
15444
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15445
|
+
});
|
|
15446
|
+
registerCutover("createChannelTool", async (input, apiKey) => {
|
|
15447
|
+
const result = await createChannelTool.execute({ ...input, apiKey }, {});
|
|
15448
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15449
|
+
});
|
|
15450
|
+
registerCutover("listChannelMessagesTool", async (input, apiKey) => {
|
|
15451
|
+
const result = await listChannelMessagesTool.execute({ ...input, apiKey }, {});
|
|
15452
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15453
|
+
});
|
|
15454
|
+
registerCutover("pollChannelTool", async (input, apiKey) => {
|
|
15455
|
+
const result = await pollChannelTool.execute({ ...input, apiKey }, {});
|
|
15456
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15457
|
+
});
|
|
15458
|
+
registerCutover("postMessageTool", async (input, apiKey) => {
|
|
15459
|
+
const result = await postMessageTool.execute({ ...input, apiKey }, {});
|
|
15460
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15461
|
+
});
|
|
15462
|
+
registerCutover("reactMessageTool", async (input, apiKey) => {
|
|
15463
|
+
const result = await reactMessageTool.execute({ ...input, apiKey }, {});
|
|
15464
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15465
|
+
});
|
|
15466
|
+
registerCutover("removeChannelMemberTool", async (input, apiKey) => {
|
|
15467
|
+
const result = await removeChannelMemberTool.execute({ ...input, apiKey }, {});
|
|
15468
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15469
|
+
});
|
|
15470
|
+
registerCutover("replyMessageTool", async (input, apiKey) => {
|
|
15471
|
+
const result = await replyMessageTool.execute({ ...input, apiKey }, {});
|
|
15472
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15473
|
+
});
|
|
15474
|
+
registerCutover("recordFactTool", async (input, apiKey) => {
|
|
15475
|
+
const result = await recordFactTool.execute({ ...input, apiKey }, {});
|
|
15476
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15477
|
+
});
|
|
15478
|
+
registerCutover("libraryIngestTool", async (input, apiKey) => {
|
|
15479
|
+
const result = await libraryIngestTool.execute({ ...input, apiKey }, {});
|
|
15480
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15481
|
+
});
|
|
15482
|
+
registerCutover("deleteNoteTool", async (input, apiKey) => {
|
|
15483
|
+
const result = await deleteNoteTool.execute({ ...input, apiKey }, {});
|
|
15484
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15485
|
+
});
|
|
15486
|
+
registerCutover("putTool", async (input, apiKey) => {
|
|
15487
|
+
const result = await putTool.execute({ ...input, apiKey }, {});
|
|
15488
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15489
|
+
});
|
|
15490
|
+
registerCutover("uploadTool", async (input, apiKey) => {
|
|
15491
|
+
const result = await uploadTool.execute({ ...input, apiKey }, {});
|
|
15492
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15493
|
+
});
|
|
15494
|
+
registerCutover("deleteScheduledActionTool", async (input, apiKey) => {
|
|
15495
|
+
const result = await deleteScheduledActionTool.execute({ ...input, apiKey }, {});
|
|
15496
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15497
|
+
});
|
|
15498
|
+
registerCutover("pauseScheduledActionTool", async (input, apiKey) => {
|
|
15499
|
+
const result = await pauseScheduledActionTool.execute({ ...input, apiKey }, {});
|
|
15500
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15501
|
+
});
|
|
15502
|
+
registerCutover("resumeScheduledActionTool", async (input, apiKey) => {
|
|
15503
|
+
const result = await resumeScheduledActionTool.execute({ ...input, apiKey }, {});
|
|
15504
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15505
|
+
});
|
|
15506
|
+
registerCutover("createTableTool", async (input, apiKey) => {
|
|
15507
|
+
const result = await createTableTool.execute({ ...input, apiKey }, {});
|
|
15508
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15509
|
+
});
|
|
15510
|
+
registerCutover("deleteTableRowsTool", async (input, apiKey) => {
|
|
15511
|
+
const result = await deleteTableRowsTool.execute({ ...input, apiKey }, {});
|
|
15512
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15513
|
+
});
|
|
15514
|
+
registerCutover("dropTableTool", async (input, apiKey) => {
|
|
15515
|
+
const result = await dropTableTool.execute({ ...input, apiKey }, {});
|
|
15516
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15517
|
+
});
|
|
15518
|
+
registerCutover("insertTableRowsTool", async (input, apiKey) => {
|
|
15519
|
+
const result = await insertTableRowsTool.execute({ ...input, apiKey }, {});
|
|
15520
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15521
|
+
});
|
|
15522
|
+
registerCutover("addVaultTool", async (input, apiKey) => {
|
|
15523
|
+
const result = await addVaultTool.execute({ ...input, apiKey }, {});
|
|
15524
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15525
|
+
});
|
|
15526
|
+
registerCutover("deleteVaultTool", async (input, apiKey) => {
|
|
15527
|
+
const result = await deleteVaultTool.execute({ ...input, apiKey }, {});
|
|
15528
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15529
|
+
});
|
|
15530
|
+
}
|
|
15531
|
+
|
|
15532
|
+
// src/mcp/memory-cutover/billing.ts
|
|
15533
|
+
import { issueKeyTool } from "mcpscraper-memory-tools/tools/access/issue-key";
|
|
15534
|
+
import { listKeysTool } from "mcpscraper-memory-tools/tools/access/list-keys";
|
|
15535
|
+
import { revokeKeyTool } from "mcpscraper-memory-tools/tools/access/revoke-key";
|
|
15536
|
+
import { setScopeTool } from "mcpscraper-memory-tools/tools/access/set-scope";
|
|
15537
|
+
import { inviteAccountTool } from "mcpscraper-memory-tools/tools/access/invite-account";
|
|
15538
|
+
import { getChatLinkTool } from "mcpscraper-memory-tools/tools/access/get-chat-link";
|
|
15539
|
+
import { revokeChatLinkTool } from "mcpscraper-memory-tools/tools/access/revoke-chat-link";
|
|
15540
|
+
import { provisionDefaultsTool as provisionDefaultsTool2 } from "mcpscraper-memory-tools/tools/vaults/provision-defaults";
|
|
15541
|
+
import { createSecureVaultTool } from "mcpscraper-memory-tools/tools/vaults/create-secure-vault";
|
|
15542
|
+
import { createScheduledActionTool } from "mcpscraper-memory-tools/tools/schedule/create-scheduled-action";
|
|
15543
|
+
import { getScheduleStatusTool } from "mcpscraper-memory-tools/tools/schedule/get-schedule-status";
|
|
15544
|
+
import { getScheduleLinkTool } from "mcpscraper-memory-tools/tools/schedule/get-schedule-link";
|
|
15545
|
+
import { revokeScheduleLinkTool } from "mcpscraper-memory-tools/tools/schedule/revoke-schedule-link";
|
|
15546
|
+
import { setScheduleEntitlementTool as setScheduleEntitlementTool2 } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
|
|
15547
|
+
import { createWebhookTool } from "mcpscraper-memory-tools/tools/webhooks/create-webhook";
|
|
15548
|
+
import { revokeWebhookTool } from "mcpscraper-memory-tools/tools/webhooks/revoke-webhook";
|
|
15549
|
+
var registered3 = false;
|
|
15550
|
+
function registerBillingCutovers() {
|
|
15551
|
+
if (registered3) return;
|
|
15552
|
+
registered3 = true;
|
|
15553
|
+
registerCutover("issueKeyTool", async (input, apiKey) => {
|
|
15554
|
+
const result = await issueKeyTool.execute({ ...input, apiKey }, {});
|
|
15555
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15556
|
+
});
|
|
15557
|
+
registerCutover("listKeysTool", async (input, apiKey) => {
|
|
15558
|
+
const result = await listKeysTool.execute({ ...input, apiKey }, {});
|
|
15559
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15560
|
+
});
|
|
15561
|
+
registerCutover("revokeKeyTool", async (input, apiKey) => {
|
|
15562
|
+
const result = await revokeKeyTool.execute({ ...input, apiKey }, {});
|
|
15563
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15564
|
+
});
|
|
15565
|
+
registerCutover("setScopeTool", async (input, apiKey) => {
|
|
15566
|
+
const result = await setScopeTool.execute({ ...input, apiKey }, {});
|
|
15567
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15568
|
+
});
|
|
15569
|
+
registerCutover("inviteAccountTool", async (input, apiKey) => {
|
|
15570
|
+
const result = await inviteAccountTool.execute({ ...input, apiKey }, {});
|
|
15571
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15572
|
+
});
|
|
15573
|
+
registerCutover("getChatLinkTool", async (input, apiKey) => {
|
|
15574
|
+
const result = await getChatLinkTool.execute({ ...input, apiKey }, {});
|
|
15575
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15576
|
+
});
|
|
15577
|
+
registerCutover("revokeChatLinkTool", async (input, apiKey) => {
|
|
15578
|
+
const result = await revokeChatLinkTool.execute({ ...input, apiKey }, {});
|
|
15579
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15580
|
+
});
|
|
15581
|
+
registerCutover("provisionDefaultsTool", async (input, apiKey) => {
|
|
15582
|
+
const result = await provisionDefaultsTool2.execute({ ...input, apiKey }, {});
|
|
15583
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15584
|
+
});
|
|
15585
|
+
registerCutover("createSecureVaultTool", async (input, apiKey) => {
|
|
15586
|
+
const result = await createSecureVaultTool.execute({ ...input, apiKey }, {});
|
|
15587
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15588
|
+
});
|
|
15589
|
+
registerCutover("createScheduledActionTool", async (input, apiKey) => {
|
|
15590
|
+
const result = await createScheduledActionTool.execute({ ...input, apiKey }, {});
|
|
15591
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15592
|
+
});
|
|
15593
|
+
registerCutover("getScheduleStatusTool", async (input, apiKey) => {
|
|
15594
|
+
const result = await getScheduleStatusTool.execute({ ...input, apiKey }, {});
|
|
15595
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15596
|
+
});
|
|
15597
|
+
registerCutover("getScheduleLinkTool", async (input, apiKey) => {
|
|
15598
|
+
const result = await getScheduleLinkTool.execute({ ...input, apiKey }, {});
|
|
15599
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15600
|
+
});
|
|
15601
|
+
registerCutover("revokeScheduleLinkTool", async (input, apiKey) => {
|
|
15602
|
+
const result = await revokeScheduleLinkTool.execute({ ...input, apiKey }, {});
|
|
15603
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15604
|
+
});
|
|
15605
|
+
registerCutover("setScheduleEntitlementTool", async (input, apiKey) => {
|
|
15606
|
+
const result = await setScheduleEntitlementTool2.execute({ ...input, apiKey }, {});
|
|
15607
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15608
|
+
});
|
|
15609
|
+
registerCutover("createWebhookTool", async (input, apiKey) => {
|
|
15610
|
+
const result = await createWebhookTool.execute({ ...input, apiKey }, {});
|
|
15611
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15612
|
+
});
|
|
15613
|
+
registerCutover("revokeWebhookTool", async (input, apiKey) => {
|
|
15614
|
+
const result = await revokeWebhookTool.execute({ ...input, apiKey }, {});
|
|
15615
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15616
|
+
});
|
|
15617
|
+
}
|
|
15618
|
+
|
|
15619
|
+
// src/mcp/memory-cutover/cross-service.ts
|
|
15620
|
+
import { proposeScheduledActionTool } from "mcpscraper-memory-tools/tools/schedule/propose-scheduled-action";
|
|
15621
|
+
var registered4 = false;
|
|
15622
|
+
function registerCrossServiceCutovers() {
|
|
15623
|
+
if (registered4) return;
|
|
15624
|
+
registered4 = true;
|
|
15625
|
+
registerCutover("proposeScheduledActionTool", async (input, apiKey) => {
|
|
15626
|
+
const result = await proposeScheduledActionTool.execute({ ...input, apiKey }, {});
|
|
15627
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15628
|
+
});
|
|
15629
|
+
}
|
|
15630
|
+
|
|
15631
|
+
// src/mcp/memory-cutover/video.ts
|
|
15632
|
+
import { videoAnalyzeStartTool } from "mcpscraper-memory-tools/tools/video/start";
|
|
15633
|
+
var registered5 = false;
|
|
15634
|
+
function registerVideoCutovers() {
|
|
15635
|
+
if (registered5) return;
|
|
15636
|
+
registered5 = true;
|
|
15637
|
+
registerCutover("videoAnalyzeStartTool", async (input, apiKey) => {
|
|
15638
|
+
const result = await videoAnalyzeStartTool.execute({ ...input, apiKey }, {});
|
|
15639
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15640
|
+
});
|
|
15641
|
+
}
|
|
15642
|
+
|
|
14911
15643
|
// src/mcp/mcp-oauth.ts
|
|
14912
15644
|
import { createRemoteJWKSet, jwtVerify } from "jose";
|
|
14913
15645
|
function oauthIssuer() {
|
|
@@ -14948,6 +15680,11 @@ async function verifyScraperOAuthJwt(token) {
|
|
|
14948
15680
|
}
|
|
14949
15681
|
|
|
14950
15682
|
// src/mcp/mcp-routes.ts
|
|
15683
|
+
registerReadCutovers();
|
|
15684
|
+
registerWriteCutovers();
|
|
15685
|
+
registerBillingCutovers();
|
|
15686
|
+
registerCrossServiceCutovers();
|
|
15687
|
+
registerVideoCutovers();
|
|
14951
15688
|
configureReportSaving(false);
|
|
14952
15689
|
function mcpAuthError(message = "authentication required") {
|
|
14953
15690
|
const body = JSON.stringify({
|
|
@@ -14981,20 +15718,20 @@ async function requireMcpCallerKey(c) {
|
|
|
14981
15718
|
if (!identity) return mcpAuthError("Bearer token has no subject");
|
|
14982
15719
|
const user2 = await getUserByEmail(identity);
|
|
14983
15720
|
if (!user2 || !user2.api_key || user2.key_active !== 1) return mcpAuthError("No active MCP Scraper account for this identity");
|
|
14984
|
-
return user2.api_key;
|
|
15721
|
+
return { key: user2.api_key, user: user2 };
|
|
14985
15722
|
}
|
|
14986
15723
|
const callerKey = xApiKey ?? bearerToken;
|
|
14987
15724
|
if (!callerKey) return mcpAuthError();
|
|
14988
15725
|
const user = await getUserByApiKey(callerKey);
|
|
14989
15726
|
if (!user) return mcpAuthError();
|
|
14990
|
-
return callerKey;
|
|
15727
|
+
return { key: callerKey, user };
|
|
14991
15728
|
}
|
|
14992
|
-
var mcpApp = new
|
|
15729
|
+
var mcpApp = new Hono15();
|
|
14993
15730
|
mcpApp.all("/", async (c) => {
|
|
14994
15731
|
try {
|
|
14995
15732
|
const keyOrError = await requireMcpCallerKey(c);
|
|
14996
15733
|
if (keyOrError instanceof Response) return keyOrError;
|
|
14997
|
-
const callerKey = keyOrError;
|
|
15734
|
+
const { key: callerKey, user } = keyOrError;
|
|
14998
15735
|
const baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? "https://mcpscraper.dev";
|
|
14999
15736
|
const executor = new HttpMcpToolExecutor(baseUrl, callerKey);
|
|
15000
15737
|
const transport = new WebStandardStreamableHTTPServerTransport({
|
|
@@ -15005,7 +15742,8 @@ mcpApp.all("/", async (c) => {
|
|
|
15005
15742
|
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
|
|
15006
15743
|
registerSerpIntelligenceCaptureTools(server, executor);
|
|
15007
15744
|
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
15008
|
-
|
|
15745
|
+
const memoryExecutor = new HybridMemoryToolExecutor(new MemoryMcpToolExecutor(baseUrl, callerKey), user);
|
|
15746
|
+
registerMemoryMcpTools(server, memoryExecutor);
|
|
15009
15747
|
await server.connect(transport);
|
|
15010
15748
|
return transport.handleRequest(c.req.raw);
|
|
15011
15749
|
} catch {
|
|
@@ -15014,7 +15752,7 @@ mcpApp.all("/", async (c) => {
|
|
|
15014
15752
|
});
|
|
15015
15753
|
|
|
15016
15754
|
// src/api/browser-agent-routes.ts
|
|
15017
|
-
import { Hono as
|
|
15755
|
+
import { Hono as Hono16 } from "hono";
|
|
15018
15756
|
|
|
15019
15757
|
// src/api/browser-agent-db.ts
|
|
15020
15758
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -15093,23 +15831,34 @@ async function migrateBrowserAgent() {
|
|
|
15093
15831
|
await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN updated_at TEXT`);
|
|
15094
15832
|
} catch {
|
|
15095
15833
|
}
|
|
15834
|
+
try {
|
|
15835
|
+
await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN status TEXT NOT NULL DEFAULT 'NEEDS_AUTH'`);
|
|
15836
|
+
} catch {
|
|
15837
|
+
}
|
|
15838
|
+
try {
|
|
15839
|
+
await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN browser_agent_session_id TEXT`);
|
|
15840
|
+
} catch {
|
|
15841
|
+
}
|
|
15096
15842
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
|
|
15097
15843
|
_ready = true;
|
|
15098
15844
|
}
|
|
15099
|
-
async function
|
|
15100
|
-
|
|
15845
|
+
async function createAuthConnectionRow(input) {
|
|
15846
|
+
const db = getDb();
|
|
15847
|
+
const connectionId = `authc_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
|
|
15848
|
+
await db.execute({
|
|
15849
|
+
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
15850
|
+
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
15851
|
+
args: [connectionId, input.domain, input.profile, input.accountEmail, input.note, input.browserAgentSessionId]
|
|
15852
|
+
});
|
|
15853
|
+
const row = await getAuthConnectionRow(connectionId);
|
|
15854
|
+
if (!row) throw new Error("auth connection insert failed");
|
|
15855
|
+
return row;
|
|
15856
|
+
}
|
|
15857
|
+
async function markAuthConnectionAuthenticated(connectionId) {
|
|
15101
15858
|
const db = getDb();
|
|
15102
15859
|
await db.execute({
|
|
15103
|
-
sql: `
|
|
15104
|
-
|
|
15105
|
-
ON CONFLICT(connection_id) DO UPDATE SET
|
|
15106
|
-
hosted_url = COALESCE(excluded.hosted_url, browser_auth_connections.hosted_url),
|
|
15107
|
-
domain = COALESCE(excluded.domain, browser_auth_connections.domain),
|
|
15108
|
-
profile = COALESCE(excluded.profile, browser_auth_connections.profile),
|
|
15109
|
-
account_email = COALESCE(excluded.account_email, browser_auth_connections.account_email),
|
|
15110
|
-
note = COALESCE(excluded.note, browser_auth_connections.note),
|
|
15111
|
-
updated_at = datetime('now')`,
|
|
15112
|
-
args: [input.connectionId, input.hostedUrl, input.domain, input.profile, input.accountEmail ?? null, input.note ?? null]
|
|
15860
|
+
sql: `UPDATE browser_auth_connections SET status = 'AUTHENTICATED', updated_at = datetime('now') WHERE connection_id = ?`,
|
|
15861
|
+
args: [connectionId]
|
|
15113
15862
|
});
|
|
15114
15863
|
}
|
|
15115
15864
|
async function getAuthConnectionRow(connectionId) {
|
|
@@ -15956,79 +16705,10 @@ async function ensureProfile(k, name) {
|
|
|
15956
16705
|
if (!isProfileConflict(err)) throw err;
|
|
15957
16706
|
}
|
|
15958
16707
|
}
|
|
15959
|
-
function profileConnectionMatches(connection, profileName, domain) {
|
|
15960
|
-
if (!connection || typeof connection !== "object") return false;
|
|
15961
|
-
const data = connection;
|
|
15962
|
-
return data.profile_name === profileName && data.domain === domain;
|
|
15963
|
-
}
|
|
15964
|
-
async function findProfileConnection(k, profileName, domain) {
|
|
15965
|
-
for await (const connection of k.auth.connections.list({ profile_name: profileName, domain })) {
|
|
15966
|
-
if (profileConnectionMatches(connection, profileName, domain)) return connection;
|
|
15967
|
-
}
|
|
15968
|
-
return null;
|
|
15969
|
-
}
|
|
15970
|
-
async function listProfileConnections(profileName) {
|
|
15971
|
-
if (!profileName) return [];
|
|
15972
|
-
const k = client();
|
|
15973
|
-
const out = [];
|
|
15974
|
-
for await (const connection of k.auth.connections.list({ profile_name: profileName })) {
|
|
15975
|
-
const data = connection;
|
|
15976
|
-
if (data.profile_name && data.profile_name !== profileName) continue;
|
|
15977
|
-
out.push(profileOnboardResult(connection));
|
|
15978
|
-
}
|
|
15979
|
-
return out;
|
|
15980
|
-
}
|
|
15981
|
-
async function findOrCreateProfileConnection(k, opts) {
|
|
15982
|
-
try {
|
|
15983
|
-
return await k.auth.connections.create({
|
|
15984
|
-
domain: opts.domain,
|
|
15985
|
-
profile_name: opts.profileName,
|
|
15986
|
-
...opts.loginUrl ? { login_url: opts.loginUrl } : {}
|
|
15987
|
-
});
|
|
15988
|
-
} catch (err) {
|
|
15989
|
-
if (!isProfileConflict(err)) throw err;
|
|
15990
|
-
const existing = await findProfileConnection(k, opts.profileName, opts.domain);
|
|
15991
|
-
if (existing) return existing;
|
|
15992
|
-
throw err;
|
|
15993
|
-
}
|
|
15994
|
-
}
|
|
15995
|
-
function profileOnboardResult(connection, hostedUrl = null, liveViewUrl = null) {
|
|
15996
|
-
return {
|
|
15997
|
-
connectionId: String(connection.id),
|
|
15998
|
-
profileName: String(connection.profile_name),
|
|
15999
|
-
domain: String(connection.domain),
|
|
16000
|
-
status: String(connection.status),
|
|
16001
|
-
flowStatus: typeof connection.flow_status === "string" ? connection.flow_status : null,
|
|
16002
|
-
flowStep: typeof connection.flow_step === "string" ? connection.flow_step : null,
|
|
16003
|
-
flowExpiresAt: typeof connection.flow_expires_at === "string" ? connection.flow_expires_at : null,
|
|
16004
|
-
hostedUrl: hostedUrl ?? (typeof connection.hosted_url === "string" ? connection.hosted_url : null),
|
|
16005
|
-
liveViewUrl: liveViewUrl ?? (typeof connection.live_view_url === "string" ? connection.live_view_url : null),
|
|
16006
|
-
browserSessionId: typeof connection.browser_session_id === "string" ? connection.browser_session_id : null,
|
|
16007
|
-
postLoginUrl: typeof connection.post_login_url === "string" ? connection.post_login_url : null
|
|
16008
|
-
};
|
|
16009
|
-
}
|
|
16010
16708
|
function explicitProxyId(value) {
|
|
16011
16709
|
const trimmed = value?.trim();
|
|
16012
16710
|
return trimmed || void 0;
|
|
16013
16711
|
}
|
|
16014
|
-
async function startProfileOnboarding(opts) {
|
|
16015
|
-
const k = client();
|
|
16016
|
-
const connection = await findOrCreateProfileConnection(k, opts);
|
|
16017
|
-
const login = await k.auth.connections.login(connection.id);
|
|
16018
|
-
const state = await k.auth.connections.retrieve(connection.id);
|
|
16019
|
-
return profileOnboardResult(state, login.hosted_url ?? null, login.live_view_url ?? null);
|
|
16020
|
-
}
|
|
16021
|
-
async function getProfileOnboardingStatus(input) {
|
|
16022
|
-
const k = client();
|
|
16023
|
-
let connection = null;
|
|
16024
|
-
if (input.connectionId) {
|
|
16025
|
-
connection = await k.auth.connections.retrieve(input.connectionId);
|
|
16026
|
-
} else if (input.profileName && input.domain) {
|
|
16027
|
-
connection = await findProfileConnection(k, input.profileName, input.domain);
|
|
16028
|
-
}
|
|
16029
|
-
if (!connection) throw new Error("profile auth connection not found");
|
|
16030
|
-
return profileOnboardResult(connection);
|
|
16031
|
-
}
|
|
16032
16712
|
async function createSession(opts = {}) {
|
|
16033
16713
|
const k = client();
|
|
16034
16714
|
const resolvedProxyId = explicitProxyId(opts.proxyId);
|
|
@@ -16370,24 +17050,39 @@ async function charge(sessionId, userId, _startedAtMs) {
|
|
|
16370
17050
|
if (res.ok) await setBilledMc(sessionId, owed);
|
|
16371
17051
|
}
|
|
16372
17052
|
}
|
|
17053
|
+
async function closeAndSettleSession(row) {
|
|
17054
|
+
if (row.user_id != null) {
|
|
17055
|
+
try {
|
|
17056
|
+
await charge(row.id, row.user_id);
|
|
17057
|
+
} catch {
|
|
17058
|
+
}
|
|
17059
|
+
}
|
|
17060
|
+
try {
|
|
17061
|
+
await closeSession(row.runtime_session_id);
|
|
17062
|
+
} catch {
|
|
17063
|
+
}
|
|
17064
|
+
await markSessionClosed(row.id).catch(() => {
|
|
17065
|
+
});
|
|
17066
|
+
await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
|
|
17067
|
+
});
|
|
17068
|
+
}
|
|
17069
|
+
async function completeAuthConnection(connectionId) {
|
|
17070
|
+
const connection = await getAuthConnectionRow(connectionId);
|
|
17071
|
+
if (!connection) return { ok: false, error: "not found" };
|
|
17072
|
+
if (connection.browser_agent_session_id) {
|
|
17073
|
+
const sessionRow = await getSessionRow(connection.browser_agent_session_id);
|
|
17074
|
+
if (sessionRow && sessionRow.status === "open") {
|
|
17075
|
+
await closeAndSettleSession(sessionRow);
|
|
17076
|
+
}
|
|
17077
|
+
}
|
|
17078
|
+
await markAuthConnectionAuthenticated(connectionId);
|
|
17079
|
+
return { ok: true };
|
|
17080
|
+
}
|
|
16373
17081
|
async function reapIdleBrowserSessions(idleSeconds = 120) {
|
|
16374
17082
|
const rows = await listIdleOpenSessionRows(idleSeconds);
|
|
16375
17083
|
let reaped = 0;
|
|
16376
17084
|
for (const row of rows) {
|
|
16377
|
-
|
|
16378
|
-
try {
|
|
16379
|
-
await charge(row.id, row.user_id);
|
|
16380
|
-
} catch {
|
|
16381
|
-
}
|
|
16382
|
-
}
|
|
16383
|
-
try {
|
|
16384
|
-
await closeSession(row.runtime_session_id);
|
|
16385
|
-
} catch {
|
|
16386
|
-
}
|
|
16387
|
-
await markSessionClosed(row.id).catch(() => {
|
|
16388
|
-
});
|
|
16389
|
-
if (row.concurrency_lock_id) await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
|
|
16390
|
-
});
|
|
17085
|
+
await closeAndSettleSession(row);
|
|
16391
17086
|
reaped++;
|
|
16392
17087
|
}
|
|
16393
17088
|
return { reaped };
|
|
@@ -16448,81 +17143,94 @@ async function loadOpenSession(id, userId) {
|
|
|
16448
17143
|
return row;
|
|
16449
17144
|
}
|
|
16450
17145
|
function buildBrowserAgentRoutes() {
|
|
16451
|
-
const app2 = new
|
|
17146
|
+
const app2 = new Hono16();
|
|
16452
17147
|
app2.use("*", async (c, next) => {
|
|
16453
17148
|
await migrateBrowserAgent();
|
|
16454
17149
|
return next();
|
|
16455
17150
|
});
|
|
16456
17151
|
app2.use("*", auth);
|
|
16457
17152
|
app2.post("/profiles/onboard", async (c) => {
|
|
17153
|
+
const user = c.get("user");
|
|
17154
|
+
if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
|
|
17155
|
+
return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
|
|
17156
|
+
}
|
|
16458
17157
|
const body = await c.req.json().catch(() => ({}));
|
|
16459
17158
|
const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
|
|
16460
17159
|
const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : "chatgpt.com";
|
|
16461
17160
|
const loginUrl = typeof body.login_url === "string" ? body.login_url.trim() : void 0;
|
|
16462
17161
|
const accountEmail = typeof body.account_email === "string" ? body.account_email.trim() || null : null;
|
|
16463
17162
|
const note = typeof body.note === "string" ? body.note.trim() || null : null;
|
|
17163
|
+
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
16464
17164
|
if (!profileName) return c.json({ error: "profile is required" }, 400);
|
|
16465
17165
|
if (!domain) return c.json({ error: "domain is required" }, 400);
|
|
17166
|
+
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
17167
|
+
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
17168
|
+
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
17169
|
+
});
|
|
17170
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17171
|
+
let runtimeSessionId = null;
|
|
16466
17172
|
try {
|
|
16467
|
-
const
|
|
16468
|
-
|
|
16469
|
-
|
|
16470
|
-
|
|
16471
|
-
|
|
16472
|
-
|
|
16473
|
-
|
|
16474
|
-
|
|
16475
|
-
|
|
16476
|
-
|
|
16477
|
-
|
|
16478
|
-
|
|
17173
|
+
const created = await createSession({
|
|
17174
|
+
timeoutSeconds,
|
|
17175
|
+
profileName,
|
|
17176
|
+
saveProfileChanges: true,
|
|
17177
|
+
startUrl: loginUrl || `https://${domain}/`,
|
|
17178
|
+
disableDefaultProxy: true
|
|
17179
|
+
});
|
|
17180
|
+
runtimeSessionId = created.runtimeSessionId;
|
|
17181
|
+
const sessionRow = await createSessionRow({
|
|
17182
|
+
runtimeSessionId: created.runtimeSessionId,
|
|
17183
|
+
liveViewUrl: created.liveViewUrl,
|
|
17184
|
+
cdpWsUrl: created.cdpWsUrl,
|
|
17185
|
+
label: typeof body.label === "string" ? body.label : `Login setup: ${domain} \u2192 ${profileName}`,
|
|
17186
|
+
userId: user.id,
|
|
17187
|
+
concurrencyLockId: gate.lockId
|
|
17188
|
+
});
|
|
17189
|
+
const connection = await createAuthConnectionRow({
|
|
17190
|
+
domain,
|
|
17191
|
+
profile: profileName,
|
|
17192
|
+
accountEmail,
|
|
17193
|
+
note,
|
|
17194
|
+
browserAgentSessionId: sessionRow.id
|
|
17195
|
+
});
|
|
16479
17196
|
return c.json({
|
|
16480
|
-
connection_id:
|
|
16481
|
-
profile:
|
|
16482
|
-
domain
|
|
16483
|
-
status:
|
|
16484
|
-
|
|
16485
|
-
flow_step: result.flowStep,
|
|
16486
|
-
flow_expires_at: result.flowExpiresAt,
|
|
16487
|
-
browser_session_id: result.browserSessionId,
|
|
16488
|
-
post_login_url: result.postLoginUrl
|
|
17197
|
+
connection_id: connection.connection_id,
|
|
17198
|
+
profile: profileName,
|
|
17199
|
+
domain,
|
|
17200
|
+
status: connection.status,
|
|
17201
|
+
browser_session_id: sessionRow.id
|
|
16489
17202
|
});
|
|
16490
17203
|
} catch (err) {
|
|
17204
|
+
if (runtimeSessionId) {
|
|
17205
|
+
try {
|
|
17206
|
+
await closeSession(runtimeSessionId);
|
|
17207
|
+
} catch {
|
|
17208
|
+
}
|
|
17209
|
+
}
|
|
17210
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16491
17211
|
return c.json(failure(err), 502);
|
|
16492
17212
|
}
|
|
16493
17213
|
});
|
|
16494
17214
|
app2.post("/profiles/status", async (c) => {
|
|
16495
17215
|
const body = await c.req.json().catch(() => ({}));
|
|
16496
|
-
const connectionId = typeof body.connection_id === "string" ? body.connection_id.trim() :
|
|
16497
|
-
const profileName = typeof body.profile === "string" ? body.profile.trim() :
|
|
17216
|
+
const connectionId = typeof body.connection_id === "string" ? body.connection_id.trim() : "";
|
|
17217
|
+
const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
|
|
16498
17218
|
const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
|
|
16499
|
-
|
|
16500
|
-
|
|
16501
|
-
|
|
16502
|
-
|
|
16503
|
-
|
|
16504
|
-
|
|
16505
|
-
|
|
16506
|
-
|
|
16507
|
-
|
|
16508
|
-
|
|
16509
|
-
|
|
16510
|
-
|
|
16511
|
-
|
|
16512
|
-
|
|
16513
|
-
|
|
16514
|
-
profile: result.profileName,
|
|
16515
|
-
domain: result.domain,
|
|
16516
|
-
status: result.status,
|
|
16517
|
-
flow_status: result.flowStatus,
|
|
16518
|
-
flow_step: result.flowStep,
|
|
16519
|
-
flow_expires_at: result.flowExpiresAt,
|
|
16520
|
-
browser_session_id: result.browserSessionId,
|
|
16521
|
-
post_login_url: result.postLoginUrl
|
|
16522
|
-
});
|
|
16523
|
-
} catch (err) {
|
|
16524
|
-
return c.json(failure(err), 404);
|
|
16525
|
-
}
|
|
17219
|
+
let row = null;
|
|
17220
|
+
if (connectionId) {
|
|
17221
|
+
row = await getAuthConnectionRow(connectionId);
|
|
17222
|
+
} else if (profileName && domain) {
|
|
17223
|
+
const rows = await listAuthConnectionsByProfile(profileName);
|
|
17224
|
+
row = rows.find((r) => r.domain === domain) ?? null;
|
|
17225
|
+
}
|
|
17226
|
+
if (!row) return c.json({ error: "auth connection not found" }, 404);
|
|
17227
|
+
return c.json({
|
|
17228
|
+
connection_id: row.connection_id,
|
|
17229
|
+
profile: row.profile,
|
|
17230
|
+
domain: row.domain,
|
|
17231
|
+
status: row.status,
|
|
17232
|
+
browser_session_id: row.browser_agent_session_id
|
|
17233
|
+
});
|
|
16526
17234
|
});
|
|
16527
17235
|
app2.post("/profiles/list", async (c) => {
|
|
16528
17236
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -16531,23 +17239,15 @@ function buildBrowserAgentRoutes() {
|
|
|
16531
17239
|
const domainFilter = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
|
|
16532
17240
|
const connectionIdFilter = typeof body.connection_id === "string" ? body.connection_id.trim() : void 0;
|
|
16533
17241
|
try {
|
|
16534
|
-
const
|
|
16535
|
-
|
|
16536
|
-
|
|
16537
|
-
|
|
16538
|
-
|
|
16539
|
-
|
|
16540
|
-
|
|
16541
|
-
|
|
16542
|
-
|
|
16543
|
-
connection_id: conn.connectionId,
|
|
16544
|
-
domain: conn.domain,
|
|
16545
|
-
status: conn.status,
|
|
16546
|
-
account_email: record?.account_email ?? null,
|
|
16547
|
-
note: record?.note ?? null,
|
|
16548
|
-
last_connected_at: record?.updated_at ?? record?.created_at ?? null
|
|
16549
|
-
};
|
|
16550
|
-
});
|
|
17242
|
+
const stored = await listAuthConnectionsByProfile(profileName);
|
|
17243
|
+
let connections = stored.map((row) => ({
|
|
17244
|
+
connection_id: row.connection_id,
|
|
17245
|
+
domain: row.domain,
|
|
17246
|
+
status: row.status,
|
|
17247
|
+
account_email: row.account_email,
|
|
17248
|
+
note: row.note,
|
|
17249
|
+
last_connected_at: row.updated_at ?? row.created_at
|
|
17250
|
+
}));
|
|
16551
17251
|
if (connectionIdFilter) connections = connections.filter((c2) => c2.connection_id === connectionIdFilter);
|
|
16552
17252
|
if (domainFilter) connections = connections.filter((c2) => c2.domain === domainFilter);
|
|
16553
17253
|
return c.json({ profile: profileName, connections });
|
|
@@ -16622,13 +17322,7 @@ function buildBrowserAgentRoutes() {
|
|
|
16622
17322
|
const user = c.get("user");
|
|
16623
17323
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
16624
17324
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
16625
|
-
await
|
|
16626
|
-
try {
|
|
16627
|
-
await closeSession(row.runtime_session_id);
|
|
16628
|
-
} catch {
|
|
16629
|
-
}
|
|
16630
|
-
await markSessionClosed(row.id);
|
|
16631
|
-
await releaseConcurrencyGate(row.concurrency_lock_id);
|
|
17325
|
+
await closeAndSettleSession(row);
|
|
16632
17326
|
return c.json({ ok: true });
|
|
16633
17327
|
});
|
|
16634
17328
|
app2.post("/sessions/:id/goto", async (c) => {
|
|
@@ -16912,17 +17606,30 @@ function buildBrowserAgentRoutes() {
|
|
|
16912
17606
|
}
|
|
16913
17607
|
|
|
16914
17608
|
// src/api/browser-agent-console.ts
|
|
16915
|
-
function renderAuthConsoleHtml(connectionId,
|
|
16916
|
-
const
|
|
16917
|
-
const
|
|
16918
|
-
const
|
|
16919
|
-
|
|
16920
|
-
|
|
16921
|
-
|
|
16922
|
-
|
|
17609
|
+
function renderAuthConsoleHtml(connectionId, info) {
|
|
17610
|
+
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ })[c]);
|
|
17611
|
+
const safeId = esc(connectionId);
|
|
17612
|
+
const domainLabel = info.domain ? esc(info.domain) : "this site";
|
|
17613
|
+
let body;
|
|
17614
|
+
if (info.status === "AUTHENTICATED") {
|
|
17615
|
+
body = `<div class="empty">
|
|
17616
|
+
<div class="check">\u2713</div>
|
|
17617
|
+
<div>You're signed in to ${domainLabel}.</div>
|
|
17618
|
+
<div class="muted">This login is saved. You can close this tab.</div>
|
|
17619
|
+
</div>`;
|
|
17620
|
+
} else if (info.sessionOpen && info.liveViewUrl) {
|
|
17621
|
+
body = `<iframe class="stage" allow="clipboard-read; clipboard-write" src="${esc(info.liveViewUrl)}"></iframe>
|
|
17622
|
+
<div class="footer">
|
|
17623
|
+
<span class="muted">Log into ${domainLabel} directly in the window above, then confirm below.</span>
|
|
17624
|
+
<span class="spacer"></span>
|
|
17625
|
+
<button class="btn" id="done">Done \u2014 I'm signed in</button>
|
|
17626
|
+
</div>`;
|
|
17627
|
+
} else {
|
|
17628
|
+
body = `<div class="empty">
|
|
16923
17629
|
<div>This sign-in link is no longer active.</div>
|
|
16924
17630
|
<div class="muted">Ask the assistant to reconnect the profile, then open the new link.</div>
|
|
16925
17631
|
</div>`;
|
|
17632
|
+
}
|
|
16926
17633
|
return `<!DOCTYPE html>
|
|
16927
17634
|
<html lang="en">
|
|
16928
17635
|
<head>
|
|
@@ -16938,9 +17645,11 @@ function renderAuthConsoleHtml(connectionId, hostedUrl) {
|
|
|
16938
17645
|
header .muted { color: #6b7689; font-size: 12px; }
|
|
16939
17646
|
.spacer { flex: 1; }
|
|
16940
17647
|
.stage { flex: 1; width: 100%; border: 0; display: block; background: #05070b; }
|
|
16941
|
-
.
|
|
16942
|
-
.btn { background: #2b6cff; color: #fff; border-radius: 7px; padding:
|
|
17648
|
+
.footer { display: flex; align-items: center; gap: 12px; padding: 10px 18px; border-top: 1px solid #1c2230; background: #0f131c; font-size: 13px; color: #8893a7; }
|
|
17649
|
+
.btn { background: #2b6cff; color: #fff; border: 0; border-radius: 7px; padding: 8px 14px; text-decoration: none; font-weight: 500; font: inherit; cursor: pointer; }
|
|
17650
|
+
.btn:disabled { opacity: .6; cursor: default; }
|
|
16943
17651
|
.empty { flex: 1; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 8px; color: #8893a7; }
|
|
17652
|
+
.check { font-size: 32px; color: #36d399; }
|
|
16944
17653
|
.muted { color: #6b7689; font-size: 12px; }
|
|
16945
17654
|
</style>
|
|
16946
17655
|
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
@@ -16950,9 +17659,23 @@ ${ATTRIBUTION_PIXEL_SCRIPT}
|
|
|
16950
17659
|
<h1>MCP Scraper</h1>
|
|
16951
17660
|
<span class="muted">Secure sign-in \xB7 ${safeId}</span>
|
|
16952
17661
|
<span class="spacer"></span>
|
|
16953
|
-
<span class="muted">
|
|
17662
|
+
<span class="muted">Log in directly to the real site above \u2014 your session is saved to your profile.</span>
|
|
16954
17663
|
</header>
|
|
16955
17664
|
${body}
|
|
17665
|
+
<script>
|
|
17666
|
+
(function(){
|
|
17667
|
+
var doneBtn = document.getElementById('done');
|
|
17668
|
+
if (!doneBtn) return;
|
|
17669
|
+
doneBtn.onclick = function() {
|
|
17670
|
+
doneBtn.disabled = true;
|
|
17671
|
+
doneBtn.textContent = 'Saving\u2026';
|
|
17672
|
+
fetch('/console/auth/${encodeURIComponent(connectionId)}/complete', { method: 'POST' })
|
|
17673
|
+
.then(function(r){ return r.json().catch(function(){ return {}; }); })
|
|
17674
|
+
.then(function(){ location.reload(); })
|
|
17675
|
+
.catch(function(){ doneBtn.disabled = false; doneBtn.textContent = "Done \u2014 I'm signed in"; });
|
|
17676
|
+
};
|
|
17677
|
+
})();
|
|
17678
|
+
</script>
|
|
16956
17679
|
</body>
|
|
16957
17680
|
</html>`;
|
|
16958
17681
|
}
|
|
@@ -17154,9 +17877,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
17154
17877
|
|
|
17155
17878
|
// src/api/stripe-routes.ts
|
|
17156
17879
|
import Stripe from "stripe";
|
|
17157
|
-
import { Hono as
|
|
17880
|
+
import { Hono as Hono17 } from "hono";
|
|
17158
17881
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
17159
|
-
var stripeApp = new
|
|
17882
|
+
var stripeApp = new Hono17();
|
|
17160
17883
|
stripeApp.post("/webhooks", async (c) => {
|
|
17161
17884
|
const sig = c.req.header("stripe-signature");
|
|
17162
17885
|
const body = await c.req.text();
|
|
@@ -17190,8 +17913,9 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17190
17913
|
const user2 = await resolveUser(sub.customer, sub.customer_email);
|
|
17191
17914
|
if (user2) {
|
|
17192
17915
|
const live = sub.status === "active" || sub.status === "trialing";
|
|
17193
|
-
|
|
17194
|
-
await
|
|
17916
|
+
const newMemoryPlan = live ? mem.plan : "free";
|
|
17917
|
+
await setMemoryPlan(user2.id, newMemoryPlan, live ? sub.id : null);
|
|
17918
|
+
await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: newMemoryPlan }));
|
|
17195
17919
|
}
|
|
17196
17920
|
return c.json({ received: true });
|
|
17197
17921
|
}
|
|
@@ -17209,7 +17933,9 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17209
17933
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
17210
17934
|
const tier = SUBSCRIPTION_TIERS[priceId];
|
|
17211
17935
|
const live = sub.status === "active" || sub.status === "trialing";
|
|
17212
|
-
|
|
17936
|
+
const newTier = live ? tier.tier : null;
|
|
17937
|
+
await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
|
|
17938
|
+
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: newTier }));
|
|
17213
17939
|
} else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
|
|
17214
17940
|
await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
|
|
17215
17941
|
await setConcurrencySubId(user.id, sub.id);
|
|
@@ -17222,7 +17948,7 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17222
17948
|
const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
|
|
17223
17949
|
if (user2) {
|
|
17224
17950
|
await setMemoryPlan(user2.id, "free", null);
|
|
17225
|
-
await syncMemoryKeyPlan(user2, "free");
|
|
17951
|
+
await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: "free" }));
|
|
17226
17952
|
}
|
|
17227
17953
|
return c.json({ received: true });
|
|
17228
17954
|
}
|
|
@@ -17235,6 +17961,7 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17235
17961
|
if (!user) return c.json({ received: true });
|
|
17236
17962
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
17237
17963
|
await setSubscriptionTier(user.id, null, 0, null);
|
|
17964
|
+
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: null }));
|
|
17238
17965
|
} else if (priceId === CONCURRENCY_PRICE_ID) {
|
|
17239
17966
|
await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
|
|
17240
17967
|
await setConcurrencySubId(user.id, null);
|
|
@@ -17256,7 +17983,7 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
17256
17983
|
}
|
|
17257
17984
|
|
|
17258
17985
|
// src/api/oauth-routes.ts
|
|
17259
|
-
import { Hono as
|
|
17986
|
+
import { Hono as Hono18 } from "hono";
|
|
17260
17987
|
import { getCookie, setCookie } from "hono/cookie";
|
|
17261
17988
|
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
|
|
17262
17989
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
@@ -17425,7 +18152,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
17425
18152
|
if (state) u.searchParams.set("state", state);
|
|
17426
18153
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
17427
18154
|
}
|
|
17428
|
-
var oauthApp = new
|
|
18155
|
+
var oauthApp = new Hono18();
|
|
17429
18156
|
oauthApp.use("*", async (c, next) => {
|
|
17430
18157
|
c.header("Access-Control-Allow-Origin", "*");
|
|
17431
18158
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -17679,8 +18406,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
17679
18406
|
});
|
|
17680
18407
|
|
|
17681
18408
|
// src/api/chat-routes.ts
|
|
17682
|
-
import { Hono as
|
|
17683
|
-
var chatApp = new
|
|
18409
|
+
import { Hono as Hono19 } from "hono";
|
|
18410
|
+
var chatApp = new Hono19();
|
|
17684
18411
|
function tokenOk(token) {
|
|
17685
18412
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
17686
18413
|
}
|
|
@@ -18350,8 +19077,8 @@ loadChannels();
|
|
|
18350
19077
|
}
|
|
18351
19078
|
|
|
18352
19079
|
// src/api/schedule-routes.ts
|
|
18353
|
-
import { Hono as
|
|
18354
|
-
var scheduleApp = new
|
|
19080
|
+
import { Hono as Hono20 } from "hono";
|
|
19081
|
+
var scheduleApp = new Hono20();
|
|
18355
19082
|
function tokenOk2(token) {
|
|
18356
19083
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18357
19084
|
}
|
|
@@ -19106,7 +19833,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
19106
19833
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
19107
19834
|
return next();
|
|
19108
19835
|
});
|
|
19109
|
-
var app = new
|
|
19836
|
+
var app = new Hono21();
|
|
19110
19837
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
19111
19838
|
function requireStripeSecret() {
|
|
19112
19839
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -20467,7 +21194,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
20467
21194
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
20468
21195
|
return c.json({ error: "Unauthorized" }, 401);
|
|
20469
21196
|
}
|
|
20470
|
-
const { drainQueue } = await import("./worker-
|
|
21197
|
+
const { drainQueue } = await import("./worker-7XRZNLKB.js");
|
|
20471
21198
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
20472
21199
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
20473
21200
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
@@ -20485,7 +21212,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
20485
21212
|
return c.json({ error: "Unauthorized" }, 401);
|
|
20486
21213
|
}
|
|
20487
21214
|
const jobId = c.req.param("id");
|
|
20488
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
21215
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-4NVO5KQA.js");
|
|
20489
21216
|
const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
|
|
20490
21217
|
const job = await getExtractJob2(jobId);
|
|
20491
21218
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
@@ -20493,7 +21220,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
20493
21220
|
await completeExtractJob2(jobId, stored);
|
|
20494
21221
|
let settlement = "already_settled_or_refunded";
|
|
20495
21222
|
if (job.billedMc == null && job.userId != null) {
|
|
20496
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
21223
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-4NVO5KQA.js");
|
|
20497
21224
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
20498
21225
|
const successful = await countSuccessfulPages2(jobId);
|
|
20499
21226
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -20519,6 +21246,7 @@ app.route("/instagram", instagramApp);
|
|
|
20519
21246
|
app.route("/reddit", redditApp);
|
|
20520
21247
|
app.route("/video", videoApp);
|
|
20521
21248
|
app.route("/maps", mapsApp);
|
|
21249
|
+
app.route("/trustpilot", trustpilotApp);
|
|
20522
21250
|
app.route("/directory", directoryApp);
|
|
20523
21251
|
app.route("/workflows", workflowApp);
|
|
20524
21252
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
@@ -20531,14 +21259,20 @@ app.get("/console/auth/:id", async (c) => {
|
|
|
20531
21259
|
const id = c.req.param("id");
|
|
20532
21260
|
await migrateBrowserAgent();
|
|
20533
21261
|
const row = await getAuthConnectionRow(id);
|
|
20534
|
-
|
|
21262
|
+
const sessionRow = row?.browser_agent_session_id ? await getSessionRow(row.browser_agent_session_id) : null;
|
|
21263
|
+
return c.html(renderAuthConsoleHtml(id, {
|
|
21264
|
+
domain: row?.domain ?? null,
|
|
21265
|
+
status: row?.status ?? null,
|
|
21266
|
+
liveViewUrl: sessionRow?.live_view_url ?? null,
|
|
21267
|
+
sessionOpen: sessionRow?.status === "open"
|
|
21268
|
+
}));
|
|
20535
21269
|
});
|
|
20536
|
-
app.
|
|
21270
|
+
app.post("/console/auth/:id/complete", async (c) => {
|
|
20537
21271
|
const id = c.req.param("id");
|
|
20538
21272
|
await migrateBrowserAgent();
|
|
20539
|
-
const
|
|
20540
|
-
if (!
|
|
20541
|
-
return c.
|
|
21273
|
+
const result = await completeAuthConnection(id);
|
|
21274
|
+
if (!result.ok) return c.json({ error: result.error }, 404);
|
|
21275
|
+
return c.json({ ok: true });
|
|
20542
21276
|
});
|
|
20543
21277
|
app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
|
|
20544
21278
|
app.route("/stripe", stripeApp);
|
|
@@ -20649,4 +21383,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
20649
21383
|
export {
|
|
20650
21384
|
app
|
|
20651
21385
|
};
|
|
20652
|
-
//# sourceMappingURL=server-
|
|
21386
|
+
//# sourceMappingURL=server-LBEYRKM2.js.map
|