mcp-scraper 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +579 -40
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +197 -3
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/chunk-5BB6Y3BB.js +7 -0
- package/dist/chunk-5BB6Y3BB.js.map +1 -0
- package/dist/{chunk-NJK5BTUK.js → chunk-5GAD2AN2.js} +2 -2
- package/dist/{chunk-FUVQR6GK.js → chunk-GL4BW4CP.js} +49 -1
- package/dist/chunk-GL4BW4CP.js.map +1 -0
- package/dist/{chunk-3UH3BEIZ.js → chunk-HE2LQPJ2.js} +2 -2
- package/dist/{chunk-Z34NGK64.js → chunk-IFXACD4K.js} +198 -4
- package/dist/chunk-IFXACD4K.js.map +1 -0
- package/dist/{chunk-5LO6EHEH.js → chunk-IVQYNY45.js} +14 -3
- package/dist/chunk-IVQYNY45.js.map +1 -0
- package/dist/{chunk-4ZZWFI6L.js → chunk-NYAWN7CZ.js} +3 -3
- package/dist/{db-H3S3M6KK.js → db-LIOTIWVN.js} +6 -2
- package/dist/{extract-bundle-UKE273TV.js → extract-bundle-COS56ZDO.js} +2 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{server-IKT3QVFB.js → server-R3KVQNOT.js} +316 -35
- package/dist/server-R3KVQNOT.js.map +1 -0
- package/dist/{site-extract-repository-THBVEXMP.js → site-extract-repository-LI2S3S6E.js} +4 -4
- package/dist/{worker-YHKUJR5P.js → worker-VASAJ7FZ.js} +5 -5
- package/package.json +2 -1
- package/dist/chunk-5LO6EHEH.js.map +0 -1
- package/dist/chunk-FUVQR6GK.js.map +0 -1
- package/dist/chunk-TZRPP45I.js +0 -7
- package/dist/chunk-TZRPP45I.js.map +0 -1
- package/dist/chunk-Z34NGK64.js.map +0 -1
- package/dist/server-IKT3QVFB.js.map +0 -1
- /package/dist/{chunk-NJK5BTUK.js.map → chunk-5GAD2AN2.js.map} +0 -0
- /package/dist/{chunk-3UH3BEIZ.js.map → chunk-HE2LQPJ2.js.map} +0 -0
- /package/dist/{chunk-4ZZWFI6L.js.map → chunk-NYAWN7CZ.js.map} +0 -0
- /package/dist/{db-H3S3M6KK.js.map → db-LIOTIWVN.js.map} +0 -0
- /package/dist/{extract-bundle-UKE273TV.js.map → extract-bundle-COS56ZDO.js.map} +0 -0
- /package/dist/{site-extract-repository-THBVEXMP.js.map → site-extract-repository-LI2S3S6E.js.map} +0 -0
- /package/dist/{worker-YHKUJR5P.js.map → worker-VASAJ7FZ.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -3617,6 +3617,7 @@ __export(db_exports, {
|
|
|
3617
3617
|
getLedger: () => getLedger,
|
|
3618
3618
|
getMemoryPlan: () => getMemoryPlan,
|
|
3619
3619
|
getMemoryProvisioned: () => getMemoryProvisioned,
|
|
3620
|
+
getPageSnapshot: () => getPageSnapshot,
|
|
3620
3621
|
getRefresh: () => getRefresh,
|
|
3621
3622
|
getUserByApiKey: () => getUserByApiKey,
|
|
3622
3623
|
getUserByEmail: () => getUserByEmail,
|
|
@@ -3672,6 +3673,7 @@ __export(db_exports, {
|
|
|
3672
3673
|
startHarvestAttempt: () => startHarvestAttempt,
|
|
3673
3674
|
stripeEventAlreadyProcessed: () => stripeEventAlreadyProcessed,
|
|
3674
3675
|
updateKpoJobState: () => updateKpoJobState,
|
|
3676
|
+
upsertPageSnapshot: () => upsertPageSnapshot,
|
|
3675
3677
|
verifyPassword: () => verifyPassword
|
|
3676
3678
|
});
|
|
3677
3679
|
function getDb() {
|
|
@@ -4109,6 +4111,20 @@ async function migrate() {
|
|
|
4109
4111
|
} catch {
|
|
4110
4112
|
}
|
|
4111
4113
|
}
|
|
4114
|
+
await db.execute(`
|
|
4115
|
+
CREATE TABLE IF NOT EXISTS page_snapshots (
|
|
4116
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
4117
|
+
url TEXT NOT NULL,
|
|
4118
|
+
content_hash TEXT NOT NULL,
|
|
4119
|
+
content TEXT NOT NULL,
|
|
4120
|
+
title TEXT,
|
|
4121
|
+
content_bytes INTEGER NOT NULL,
|
|
4122
|
+
truncated INTEGER NOT NULL DEFAULT 0,
|
|
4123
|
+
checked_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
4124
|
+
PRIMARY KEY (user_id, url)
|
|
4125
|
+
)
|
|
4126
|
+
`);
|
|
4127
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS page_snapshots_user_checked_at ON page_snapshots(user_id, checked_at DESC)`);
|
|
4112
4128
|
}
|
|
4113
4129
|
async function registerClient(clientId, redirectUris, name) {
|
|
4114
4130
|
await getDb().execute({
|
|
@@ -4539,6 +4555,38 @@ async function listJobs(userId) {
|
|
|
4539
4555
|
const res = await getDb().execute({ sql: "SELECT * FROM jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50", args: [userId] });
|
|
4540
4556
|
return res.rows.map((r) => deserialize(rowToRawJob(r)));
|
|
4541
4557
|
}
|
|
4558
|
+
async function getPageSnapshot(userId, url) {
|
|
4559
|
+
const res = await getDb().execute({
|
|
4560
|
+
sql: "SELECT content_hash, content, title, content_bytes, truncated, checked_at FROM page_snapshots WHERE user_id = ? AND url = ?",
|
|
4561
|
+
args: [userId, url]
|
|
4562
|
+
});
|
|
4563
|
+
const row = res.rows[0];
|
|
4564
|
+
if (!row) return null;
|
|
4565
|
+
return {
|
|
4566
|
+
contentHash: String(row.content_hash),
|
|
4567
|
+
content: String(row.content),
|
|
4568
|
+
title: row.title == null ? null : String(row.title),
|
|
4569
|
+
contentBytes: Number(row.content_bytes),
|
|
4570
|
+
truncated: Number(row.truncated) === 1,
|
|
4571
|
+
checkedAt: String(row.checked_at)
|
|
4572
|
+
};
|
|
4573
|
+
}
|
|
4574
|
+
async function upsertPageSnapshot(userId, url, snapshot) {
|
|
4575
|
+
await getDb().execute({
|
|
4576
|
+
sql: `
|
|
4577
|
+
INSERT INTO page_snapshots (user_id, url, content_hash, content, title, content_bytes, truncated, checked_at)
|
|
4578
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
|
4579
|
+
ON CONFLICT(user_id, url) DO UPDATE SET
|
|
4580
|
+
content_hash = excluded.content_hash,
|
|
4581
|
+
content = excluded.content,
|
|
4582
|
+
title = excluded.title,
|
|
4583
|
+
content_bytes = excluded.content_bytes,
|
|
4584
|
+
truncated = excluded.truncated,
|
|
4585
|
+
checked_at = excluded.checked_at
|
|
4586
|
+
`,
|
|
4587
|
+
args: [userId, url, snapshot.contentHash, snapshot.content, snapshot.title, snapshot.contentBytes, snapshot.truncated ? 1 : 0]
|
|
4588
|
+
});
|
|
4589
|
+
}
|
|
4542
4590
|
function rowToRequestEvent(row) {
|
|
4543
4591
|
const rawResult = row.result != null ? String(row.result) : null;
|
|
4544
4592
|
return {
|
|
@@ -10046,7 +10094,8 @@ var init_rates = __esm({
|
|
|
10046
10094
|
trustpilot_reviews: 500,
|
|
10047
10095
|
trustpilot_review: 100,
|
|
10048
10096
|
g2_reviews: 500,
|
|
10049
|
-
g2_review: 150
|
|
10097
|
+
g2_review: 150,
|
|
10098
|
+
diff_page: 100
|
|
10050
10099
|
};
|
|
10051
10100
|
MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
|
|
10052
10101
|
BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
|
|
@@ -10249,6 +10298,14 @@ var init_rates = __esm({
|
|
|
10249
10298
|
credits: mcToCredits(MC_COSTS.g2_review),
|
|
10250
10299
|
unit: "per review card",
|
|
10251
10300
|
notes: "Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."
|
|
10301
|
+
},
|
|
10302
|
+
{
|
|
10303
|
+
key: "diff_page",
|
|
10304
|
+
label: "Page change check",
|
|
10305
|
+
aliases: ["diff_page", "page diff", "change detection"],
|
|
10306
|
+
credits: mcToCredits(MC_COSTS.diff_page),
|
|
10307
|
+
unit: "per check",
|
|
10308
|
+
notes: "Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."
|
|
10252
10309
|
}
|
|
10253
10310
|
];
|
|
10254
10311
|
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
@@ -10334,7 +10391,9 @@ var init_rates = __esm({
|
|
|
10334
10391
|
G2_REVIEWS: "g2_reviews",
|
|
10335
10392
|
G2_REVIEWS_REFUND: "g2_reviews_refund",
|
|
10336
10393
|
G2_REVIEW: "g2_review",
|
|
10337
|
-
G2_REVIEW_REFUND: "g2_review_refund"
|
|
10394
|
+
G2_REVIEW_REFUND: "g2_review_refund",
|
|
10395
|
+
DIFF_PAGE: "diff_page",
|
|
10396
|
+
DIFF_PAGE_REFUND: "diff_page_refund"
|
|
10338
10397
|
};
|
|
10339
10398
|
MEMORY_AI_MARGIN_MULTIPLE = 3;
|
|
10340
10399
|
MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
|
|
@@ -13113,7 +13172,7 @@ var init_api_auth = __esm({
|
|
|
13113
13172
|
});
|
|
13114
13173
|
|
|
13115
13174
|
// src/api/server-schemas.ts
|
|
13116
|
-
var import_zod13, HarvestBodySchema, ExtractUrlBodySchema, MapUrlsBodySchema, ExtractSiteBodySchema, YoutubeHarvestBodySchema, YoutubeTranscribeBodySchema;
|
|
13175
|
+
var import_zod13, HarvestBodySchema, ExtractUrlBodySchema, DiffPageBodySchema, MapUrlsBodySchema, ExtractSiteBodySchema, YoutubeHarvestBodySchema, YoutubeTranscribeBodySchema;
|
|
13117
13176
|
var init_server_schemas = __esm({
|
|
13118
13177
|
"src/api/server-schemas.ts"() {
|
|
13119
13178
|
"use strict";
|
|
@@ -13144,6 +13203,11 @@ var init_server_schemas = __esm({
|
|
|
13144
13203
|
depositToVault: import_zod13.z.boolean().optional(),
|
|
13145
13204
|
vaultName: import_zod13.z.string().trim().min(1).max(120).optional()
|
|
13146
13205
|
});
|
|
13206
|
+
DiffPageBodySchema = import_zod13.z.object({
|
|
13207
|
+
url: import_zod13.z.string().min(1, "url is required"),
|
|
13208
|
+
allowLocal: import_zod13.z.boolean().optional(),
|
|
13209
|
+
resetBaseline: import_zod13.z.boolean().optional()
|
|
13210
|
+
});
|
|
13147
13211
|
MapUrlsBodySchema = import_zod13.z.object({
|
|
13148
13212
|
url: import_zod13.z.string().min(1, "url is required"),
|
|
13149
13213
|
maxUrls: import_zod13.z.number().int().min(1).max(2e3).optional(),
|
|
@@ -16934,22 +16998,8 @@ async function provisionMemoryForUser(user) {
|
|
|
16934
16998
|
} catch {
|
|
16935
16999
|
}
|
|
16936
17000
|
}
|
|
16937
|
-
function memoryPlanForUser(user) {
|
|
16938
|
-
switch (user.subscription_tier) {
|
|
16939
|
-
case "starter":
|
|
16940
|
-
return "pro";
|
|
16941
|
-
case "growth":
|
|
16942
|
-
return "team";
|
|
16943
|
-
case "scale":
|
|
16944
|
-
return "enterprise";
|
|
16945
|
-
default:
|
|
16946
|
-
return "free";
|
|
16947
|
-
}
|
|
16948
|
-
}
|
|
16949
17001
|
function resolveEffectiveMemoryPlan(user) {
|
|
16950
|
-
|
|
16951
|
-
const memoryProPlan = user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
|
|
16952
|
-
return MEMORY_PLAN_RANK[tierPlan] >= MEMORY_PLAN_RANK[memoryProPlan] ? tierPlan : memoryProPlan;
|
|
17002
|
+
return user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
|
|
16953
17003
|
}
|
|
16954
17004
|
async function getOrCreateUserMemoryKey(user) {
|
|
16955
17005
|
const creds = await getUserMemoryCreds(user.id);
|
|
@@ -16999,7 +17049,7 @@ async function syncScheduleEntitlement(user, enabled, quotaPerPeriod) {
|
|
|
16999
17049
|
}, {});
|
|
17000
17050
|
return { ok: res.ok, error: res.error };
|
|
17001
17051
|
}
|
|
17002
|
-
var import_node_crypto6, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY
|
|
17052
|
+
var import_node_crypto6, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
|
|
17003
17053
|
var init_memory = __esm({
|
|
17004
17054
|
"src/api/memory.ts"() {
|
|
17005
17055
|
"use strict";
|
|
@@ -17010,7 +17060,6 @@ var init_memory = __esm({
|
|
|
17010
17060
|
import_set_schedule_entitlement = require("mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement");
|
|
17011
17061
|
MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
|
|
17012
17062
|
ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
17013
|
-
MEMORY_PLAN_RANK = { free: 0, pro: 1, team: 2, enterprise: 3 };
|
|
17014
17063
|
}
|
|
17015
17064
|
});
|
|
17016
17065
|
|
|
@@ -19345,6 +19394,55 @@ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${memSection}$
|
|
|
19345
19394
|
}
|
|
19346
19395
|
return { ...textResult, structuredContent };
|
|
19347
19396
|
}
|
|
19397
|
+
function formatDiffPage(raw, input) {
|
|
19398
|
+
const parsed = parseData(raw);
|
|
19399
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
19400
|
+
const d = parsed.data;
|
|
19401
|
+
const url = d.url ?? input.url;
|
|
19402
|
+
const title = d.title ?? "Untitled";
|
|
19403
|
+
const statusLine = d.status === "baseline" ? d.isReset ? "\u{1F195} **Baseline reset** \u2014 prior history discarded, this check is now the new baseline." : "\u{1F195} **Baseline captured** \u2014 first check for this URL, nothing to compare yet." : d.status === "unchanged" ? `\u2705 **No changes** since ${d.previousCheckedAt ?? "the last check"}.` : `\u{1F504} **Changed** since ${d.previousCheckedAt ?? "the last check"} \u2014 +${d.summary.linesAdded}/-${d.summary.linesRemoved} lines (${d.summary.percentChanged ?? 0}% of content).`;
|
|
19404
|
+
const previewHunks = d.hunks.slice(0, DIFF_PAGE_PREVIEW_HUNKS);
|
|
19405
|
+
const diffBlock = previewHunks.length ? [
|
|
19406
|
+
"\n## Diff",
|
|
19407
|
+
"```diff",
|
|
19408
|
+
...previewHunks.flatMap((h) => h.lines.map((l) => `${h.type === "added" ? "+" : "-"} ${l}`)),
|
|
19409
|
+
"```",
|
|
19410
|
+
previewHunks.length < d.hunks.length ? `*(showing ${previewHunks.length} of ${d.hunks.length} returned hunks)*` : ""
|
|
19411
|
+
].filter(Boolean).join("\n") : "";
|
|
19412
|
+
const truncationNotes = [
|
|
19413
|
+
d.contentTruncated ? "\n\u26A0\uFE0F Page content exceeded the storable cap and was truncated before hashing/diffing \u2014 changes past that point are not visible to this comparison." : "",
|
|
19414
|
+
d.hunksTruncatedReason ? `
|
|
19415
|
+
\u26A0\uFE0F ${d.hunksTruncatedReason}` : ""
|
|
19416
|
+
].filter(Boolean).join("\n");
|
|
19417
|
+
const tips = `
|
|
19418
|
+
---
|
|
19419
|
+
\u{1F4A1} **Tips**
|
|
19420
|
+
- Call \`diff_page\` again anytime to re-check \u2014 this tool is on-demand, not automatic.
|
|
19421
|
+
- Want the page's current content with nothing to compare? use \`extract_url\`.`;
|
|
19422
|
+
const full = `# Page Diff: ${url}
|
|
19423
|
+
**${title}**
|
|
19424
|
+
|
|
19425
|
+
${statusLine}${diffBlock}${truncationNotes}${tips}`;
|
|
19426
|
+
return {
|
|
19427
|
+
...oneBlock(full),
|
|
19428
|
+
structuredContent: {
|
|
19429
|
+
url,
|
|
19430
|
+
title: d.title,
|
|
19431
|
+
status: d.status,
|
|
19432
|
+
isReset: d.isReset,
|
|
19433
|
+
previousCheckedAt: d.previousCheckedAt,
|
|
19434
|
+
currentCheckedAt: d.currentCheckedAt,
|
|
19435
|
+
contentHash: d.contentHash,
|
|
19436
|
+
previousContentHash: d.previousContentHash,
|
|
19437
|
+
summary: d.summary,
|
|
19438
|
+
hunks: d.hunks,
|
|
19439
|
+
contentTruncated: d.contentTruncated,
|
|
19440
|
+
hunksTruncated: d.hunksTruncated,
|
|
19441
|
+
hunksTruncatedReason: d.hunksTruncatedReason,
|
|
19442
|
+
totalChangedLineCount: d.totalChangedLineCount
|
|
19443
|
+
}
|
|
19444
|
+
};
|
|
19445
|
+
}
|
|
19348
19446
|
async function formatMapSiteUrls(raw, input, ctx) {
|
|
19349
19447
|
const parsed = parseData(raw);
|
|
19350
19448
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -21069,7 +21167,7 @@ ${rows}` : ""
|
|
|
21069
21167
|
}
|
|
21070
21168
|
};
|
|
21071
21169
|
}
|
|
21072
|
-
var import_node_fs6, import_node_os6, import_node_path8, INLINE_BUDGET_BYTES, STRUCTURED_ARRAY_CAP, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
|
|
21170
|
+
var import_node_fs6, import_node_os6, import_node_path8, INLINE_BUDGET_BYTES, STRUCTURED_ARRAY_CAP, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD, DIFF_PAGE_PREVIEW_HUNKS;
|
|
21073
21171
|
var init_mcp_response_formatter = __esm({
|
|
21074
21172
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
21075
21173
|
"use strict";
|
|
@@ -21088,6 +21186,7 @@ var init_mcp_response_formatter = __esm({
|
|
|
21088
21186
|
reportSavingEnabled = true;
|
|
21089
21187
|
BULK_PAGE_THRESHOLD = 25;
|
|
21090
21188
|
BULK_URL_THRESHOLD = 500;
|
|
21189
|
+
DIFF_PAGE_PREVIEW_HUNKS = 20;
|
|
21091
21190
|
}
|
|
21092
21191
|
});
|
|
21093
21192
|
|
|
@@ -27018,7 +27117,7 @@ var PACKAGE_VERSION;
|
|
|
27018
27117
|
var init_version = __esm({
|
|
27019
27118
|
"src/version.ts"() {
|
|
27020
27119
|
"use strict";
|
|
27021
|
-
PACKAGE_VERSION = "0.4.
|
|
27120
|
+
PACKAGE_VERSION = "0.4.6";
|
|
27022
27121
|
}
|
|
27023
27122
|
});
|
|
27024
27123
|
|
|
@@ -27157,7 +27256,7 @@ var init_output_schema_registry = __esm({
|
|
|
27157
27256
|
});
|
|
27158
27257
|
|
|
27159
27258
|
// src/mcp/mcp-tool-schemas.ts
|
|
27160
|
-
var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema;
|
|
27259
|
+
var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, DiffPageInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema;
|
|
27161
27260
|
var init_mcp_tool_schemas = __esm({
|
|
27162
27261
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
27163
27262
|
"use strict";
|
|
@@ -27185,6 +27284,11 @@ var init_mcp_tool_schemas = __esm({
|
|
|
27185
27284
|
depositToVault: import_zod33.z.boolean().default(false).describe("Save the full page content into the user's MCP Memory vault server-side, embedded for semantic recall \u2014 the full body is NOT returned to chat."),
|
|
27186
27285
|
vaultName: import_zod33.z.string().trim().min(1).max(120).optional().describe("Optional vault to deposit into. Defaults to the user's personal vault.")
|
|
27187
27286
|
};
|
|
27287
|
+
DiffPageInputSchema = {
|
|
27288
|
+
url: import_zod33.z.string().url().describe("Public http/https URL to check for changes since the last diff_page call."),
|
|
27289
|
+
allowLocal: import_zod33.z.boolean().default(false).describe("Allow localhost and private-network URLs. Local development only."),
|
|
27290
|
+
resetBaseline: import_zod33.z.boolean().default(false).describe("Discard any previously stored snapshot for this URL and capture the current content as a fresh baseline instead of diffing against history. Use when you deliberately want to restart change tracking.")
|
|
27291
|
+
};
|
|
27188
27292
|
MapSiteUrlsInputSchema = {
|
|
27189
27293
|
url: import_zod33.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
27190
27294
|
maxUrls: import_zod33.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
|
|
@@ -27578,6 +27682,29 @@ var init_mcp_tool_schemas = __esm({
|
|
|
27578
27682
|
error: import_zod33.z.string().optional()
|
|
27579
27683
|
}).optional()
|
|
27580
27684
|
};
|
|
27685
|
+
DiffPageOutputSchema = {
|
|
27686
|
+
url: import_zod33.z.string(),
|
|
27687
|
+
title: NullableString,
|
|
27688
|
+
status: import_zod33.z.enum(["baseline", "unchanged", "changed"]).describe('"baseline" = first-ever check for this URL, or resetBaseline was used \u2014 nothing to compare against. "unchanged" = content hash matched the stored snapshot. "changed" = a diff was computed.'),
|
|
27689
|
+
isReset: import_zod33.z.boolean().describe("True only when resetBaseline discarded a real prior snapshot \u2014 distinguishes an explicit reset from a URL's true first-ever check."),
|
|
27690
|
+
previousCheckedAt: NullableString.describe("ISO timestamp of the snapshot this was compared against, or null if there was none."),
|
|
27691
|
+
currentCheckedAt: import_zod33.z.string().describe("ISO timestamp of this check, now stored as the new snapshot."),
|
|
27692
|
+
contentHash: import_zod33.z.string().describe("sha256 of the full (untruncated) page content just captured."),
|
|
27693
|
+
previousContentHash: NullableString,
|
|
27694
|
+
summary: import_zod33.z.object({
|
|
27695
|
+
linesAdded: import_zod33.z.number().int().min(0),
|
|
27696
|
+
linesRemoved: import_zod33.z.number().int().min(0),
|
|
27697
|
+
percentChanged: import_zod33.z.number().min(0).max(100).nullable().describe('Proportion of changed lines relative to the larger of the two versions. Null when status is "baseline".')
|
|
27698
|
+
}),
|
|
27699
|
+
hunks: import_zod33.z.array(import_zod33.z.object({
|
|
27700
|
+
type: import_zod33.z.enum(["added", "removed"]).describe("Unchanged context lines are omitted \u2014 only added/removed lines are returned."),
|
|
27701
|
+
lines: import_zod33.z.array(import_zod33.z.string())
|
|
27702
|
+
})).describe("Ordered added/removed line hunks, capped for response size \u2014 see hunksTruncated."),
|
|
27703
|
+
contentTruncated: import_zod33.z.boolean().describe("True if the scraped page exceeded the 250,000-character storable cap and was truncated before hashing/diffing/storing \u2014 changes past that point are invisible to this comparison."),
|
|
27704
|
+
hunksTruncated: import_zod33.z.boolean().describe("True if the hunks list above was capped for response size \u2014 see hunksTruncatedReason."),
|
|
27705
|
+
hunksTruncatedReason: NullableString,
|
|
27706
|
+
totalChangedLineCount: import_zod33.z.number().int().min(0).describe("Total changed lines found before any hunksTruncated capping was applied.")
|
|
27707
|
+
};
|
|
27581
27708
|
ExtractSiteOutputSchema = {
|
|
27582
27709
|
url: import_zod33.z.string(),
|
|
27583
27710
|
pageCount: import_zod33.z.number().int().min(0),
|
|
@@ -28590,6 +28717,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
28590
28717
|
outputSchema: recordOutputSchema("extract_url", ExtractUrlOutputSchema),
|
|
28591
28718
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
28592
28719
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
28720
|
+
server.registerTool("diff_page", {
|
|
28721
|
+
title: "Page Change Check",
|
|
28722
|
+
description: "Check whether a public URL has changed since you last checked it with this tool: scrapes the current page, diffs it against your last stored snapshot for that URL, and returns what was added or removed (or confirms no change). Stores the new snapshot as the baseline for next time \u2014 on-demand only, no automatic recurring checks. Use extract_url instead when you just want the page's current content with nothing to compare against.",
|
|
28723
|
+
inputSchema: DiffPageInputSchema,
|
|
28724
|
+
outputSchema: recordOutputSchema("diff_page", DiffPageOutputSchema),
|
|
28725
|
+
annotations: liveWebToolAnnotations("Page Change Check")
|
|
28726
|
+
}, async (input) => formatDiffPage(await executor.diffPage(input), input));
|
|
28593
28727
|
server.registerTool("map_site_urls", {
|
|
28594
28728
|
title: "Site URL Map",
|
|
28595
28729
|
description: `Map/crawl a public website for a sitemap, URL inventory, or broken-link scan. Returns internal URLs with HTTP status; ${fileBehavior("maps over 500 URLs are written to a local CSV file instead of inlined.", "large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
@@ -29021,6 +29155,9 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
29021
29155
|
extractUrl(input) {
|
|
29022
29156
|
return this.call("/extract-url", input);
|
|
29023
29157
|
}
|
|
29158
|
+
diffPage(input) {
|
|
29159
|
+
return this.call("/diff-page", input);
|
|
29160
|
+
}
|
|
29024
29161
|
mapSiteUrls(input) {
|
|
29025
29162
|
return this.call("/map-urls", input);
|
|
29026
29163
|
}
|
|
@@ -29277,7 +29414,7 @@ var init_export = __esm({
|
|
|
29277
29414
|
});
|
|
29278
29415
|
|
|
29279
29416
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
29280
|
-
var import_zod34, NullableString2, BrowserRawObject, BrowserElementOutput, BrowserBaseOutput, BrowserReplayBaseOutput, BrowserOpenInputSchema, BrowserProfileConnectInputSchema, BrowserProfileListInputSchema, BrowserSessionInputSchema, BrowserLocateTargetSchema, BrowserLocateInputSchema, BrowserGotoInputSchema, BrowserClickInputSchema, BrowserTypeInputSchema, BrowserScrollInputSchema, BrowserPressInputSchema, BrowserReplayStopInputSchema, BrowserReplayDownloadInputSchema, BrowserReplayAnnotationSchema, BrowserReplayMarkInputSchema, BrowserReplayAnnotateInputSchema, BrowserListInputSchema, BrowserCaptureFanoutInputSchema, FanoutSourceOutput, BrowserCaptureFanoutOutputSchema, BrowserOpenOutputSchema, BrowserProfileLogin, BrowserProfileConnectOutputSchema, BrowserProfileListOutputSchema, BrowserScreenshotOutputSchema, BrowserReadOutputSchema, BrowserLocateOutputSchema, BrowserActionOutputSchema, BrowserReplayStartOutputSchema, BrowserReplayStopOutputSchema, BrowserListReplaysOutputSchema, BrowserReplayDownloadOutputSchema, BrowserReplayMarkOutputSchema, BrowserReplayAnnotateOutputSchema, BrowserCloseOutputSchema, BrowserListSessionsOutputSchema;
|
|
29417
|
+
var import_zod34, NullableString2, BrowserRawObject, BrowserElementOutput, BrowserBaseOutput, BrowserReplayBaseOutput, BrowserOpenInputSchema, BrowserProfileConnectInputSchema, BrowserProfileListInputSchema, BrowserSessionInputSchema, BrowserLocateTargetSchema, BrowserLocateInputSchema, BrowserGotoInputSchema, BrowserClickInputSchema, BrowserTypeInputSchema, BrowserScrollInputSchema, BrowserPressInputSchema, BrowserReplayStopInputSchema, BrowserReplayDownloadInputSchema, BrowserReplayAnnotationSchema, BrowserReplayMarkInputSchema, BrowserReplayAnnotateInputSchema, BrowserListInputSchema, BrowserCaptureFanoutInputSchema, FanoutSourceOutput, BrowserCaptureFanoutOutputSchema, BrowserOpenOutputSchema, BrowserProfileLogin, BrowserProfileConnectOutputSchema, BrowserProfileListOutputSchema, BrowserExtensionImportInputSchema, BrowserExtensionImportOutputSchema, BrowserExtensionSummary, BrowserExtensionListInputSchema, BrowserExtensionListOutputSchema, BrowserExtensionDeleteInputSchema, BrowserExtensionDeleteOutputSchema, BrowserScreenshotOutputSchema, BrowserReadOutputSchema, BrowserLocateOutputSchema, BrowserActionOutputSchema, BrowserReplayStartOutputSchema, BrowserReplayStopOutputSchema, BrowserListReplaysOutputSchema, BrowserReplayDownloadOutputSchema, BrowserReplayMarkOutputSchema, BrowserReplayAnnotateOutputSchema, BrowserCloseOutputSchema, BrowserListSessionsOutputSchema;
|
|
29281
29418
|
var init_browser_agent_tool_schemas = __esm({
|
|
29282
29419
|
"src/mcp/browser-agent-tool-schemas.ts"() {
|
|
29283
29420
|
"use strict";
|
|
@@ -29299,7 +29436,10 @@ var init_browser_agent_tool_schemas = __esm({
|
|
|
29299
29436
|
url: import_zod34.z.string().url().optional().describe("Optional URL to navigate to immediately after opening."),
|
|
29300
29437
|
profile: import_zod34.z.string().optional().describe("Optional saved hosted profile name to load a logged-in session for a site."),
|
|
29301
29438
|
save_profile_changes: import_zod34.z.boolean().optional().describe("Persist cookies/storage back to the named profile on close. Avoid parallel sessions writing to the same profile."),
|
|
29302
|
-
timeout_seconds: import_zod34.z.number().int().min(60).max(259200).optional().describe("Session lifetime before auto-termination. Defaults to 600.")
|
|
29439
|
+
timeout_seconds: import_zod34.z.number().int().min(60).max(259200).optional().describe("Session lifetime before auto-termination. Defaults to 600."),
|
|
29440
|
+
extension_names: import_zod34.z.array(import_zod34.z.string()).optional().describe(
|
|
29441
|
+
"Names of extensions previously added with browser_extension_import (see browser_extension_list for what's available) to load into this session. Loading extensions restarts the browser, adding a few seconds to startup."
|
|
29442
|
+
)
|
|
29303
29443
|
};
|
|
29304
29444
|
BrowserProfileConnectInputSchema = {
|
|
29305
29445
|
email: import_zod34.z.string().optional().describe("Account email for the login. Derives a stable profile name and is recorded as a note. Does NOT import existing cookies \u2014 the user signs in fresh."),
|
|
@@ -29526,6 +29666,43 @@ var init_browser_agent_tool_schemas = __esm({
|
|
|
29526
29666
|
connections: import_zod34.z.array(BrowserProfileLogin).describe("All site logins saved in this profile, each with its current auth status and note."),
|
|
29527
29667
|
count: import_zod34.z.number().int().min(0)
|
|
29528
29668
|
};
|
|
29669
|
+
BrowserExtensionImportInputSchema = {
|
|
29670
|
+
store_url: import_zod34.z.string().url().describe("Chrome Web Store URL of the extension to add, e.g. https://chromewebstore.google.com/detail/<slug>/<id>."),
|
|
29671
|
+
name: import_zod34.z.string().min(1).max(64).describe('Short name to save this extension under, e.g. "ani-ai". Reuse it later in extension_names on browser_open.')
|
|
29672
|
+
};
|
|
29673
|
+
BrowserExtensionImportOutputSchema = {
|
|
29674
|
+
ok: import_zod34.z.boolean(),
|
|
29675
|
+
tool: import_zod34.z.literal("browser_extension_import"),
|
|
29676
|
+
session_id: import_zod34.z.null(),
|
|
29677
|
+
name: import_zod34.z.string().describe("The name this extension was saved under."),
|
|
29678
|
+
source_url: import_zod34.z.string().describe("The store URL this extension was imported from."),
|
|
29679
|
+
size_bytes: import_zod34.z.number().nullable().describe("Size of the extension package in bytes.")
|
|
29680
|
+
};
|
|
29681
|
+
BrowserExtensionSummary = import_zod34.z.object({
|
|
29682
|
+
name: import_zod34.z.string(),
|
|
29683
|
+
source: import_zod34.z.string().describe('Always "store" for extensions added via browser_extension_import.'),
|
|
29684
|
+
source_url: NullableString2,
|
|
29685
|
+
size_bytes: import_zod34.z.number().nullable(),
|
|
29686
|
+
created_at: import_zod34.z.string()
|
|
29687
|
+
});
|
|
29688
|
+
BrowserExtensionListInputSchema = {};
|
|
29689
|
+
BrowserExtensionListOutputSchema = {
|
|
29690
|
+
ok: import_zod34.z.boolean(),
|
|
29691
|
+
tool: import_zod34.z.literal("browser_extension_list"),
|
|
29692
|
+
session_id: import_zod34.z.null(),
|
|
29693
|
+
extensions: import_zod34.z.array(BrowserExtensionSummary).describe("Every extension available to load via extension_names on browser_open."),
|
|
29694
|
+
count: import_zod34.z.number().int().min(0)
|
|
29695
|
+
};
|
|
29696
|
+
BrowserExtensionDeleteInputSchema = {
|
|
29697
|
+
name: import_zod34.z.string().min(1).describe("Name of the extension to remove, as returned by browser_extension_list.")
|
|
29698
|
+
};
|
|
29699
|
+
BrowserExtensionDeleteOutputSchema = {
|
|
29700
|
+
ok: import_zod34.z.boolean(),
|
|
29701
|
+
tool: import_zod34.z.literal("browser_extension_delete"),
|
|
29702
|
+
session_id: import_zod34.z.null(),
|
|
29703
|
+
name: import_zod34.z.string(),
|
|
29704
|
+
deleted: import_zod34.z.boolean()
|
|
29705
|
+
};
|
|
29529
29706
|
BrowserScreenshotOutputSchema = {
|
|
29530
29707
|
...BrowserBaseOutput,
|
|
29531
29708
|
tool: import_zod34.z.literal("browser_screenshot"),
|
|
@@ -30149,6 +30326,71 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
30149
30326
|
});
|
|
30150
30327
|
}
|
|
30151
30328
|
);
|
|
30329
|
+
server.registerTool(
|
|
30330
|
+
"browser_extension_import",
|
|
30331
|
+
{
|
|
30332
|
+
title: "Add Browser Extension",
|
|
30333
|
+
description: "Add a Chrome extension from its Chrome Web Store page so it can be loaded into browser_open sessions via extension_names. One-time setup per extension \u2014 check what's already added with browser_extension_list first. The extension starts logged out in any session; sign into it once inside a session, pairing with a saved profile (browser_open's profile + save_profile_changes) to keep it signed in on future opens.",
|
|
30334
|
+
inputSchema: BrowserExtensionImportInputSchema,
|
|
30335
|
+
outputSchema: recordOutputSchema("browser_extension_import", BrowserExtensionImportOutputSchema),
|
|
30336
|
+
annotations: annotations("Add Browser Extension")
|
|
30337
|
+
},
|
|
30338
|
+
async (input) => {
|
|
30339
|
+
const res = await req("POST", "/agent/extensions/import", { store_url: input.store_url, name: input.name });
|
|
30340
|
+
if (!res.ok) return errorResult("browser_extension_import", res.data);
|
|
30341
|
+
return structuredResult({
|
|
30342
|
+
ok: true,
|
|
30343
|
+
tool: "browser_extension_import",
|
|
30344
|
+
session_id: null,
|
|
30345
|
+
name: res.data.name,
|
|
30346
|
+
source_url: res.data.source_url,
|
|
30347
|
+
size_bytes: res.data.size_bytes ?? null
|
|
30348
|
+
});
|
|
30349
|
+
}
|
|
30350
|
+
);
|
|
30351
|
+
server.registerTool(
|
|
30352
|
+
"browser_extension_list",
|
|
30353
|
+
{
|
|
30354
|
+
title: "List Browser Extensions",
|
|
30355
|
+
description: "List extensions added via browser_extension_import, for use as extension_names on browser_open. Read-only, no cost.",
|
|
30356
|
+
inputSchema: BrowserExtensionListInputSchema,
|
|
30357
|
+
outputSchema: recordOutputSchema("browser_extension_list", BrowserExtensionListOutputSchema),
|
|
30358
|
+
annotations: annotations("List Browser Extensions", true)
|
|
30359
|
+
},
|
|
30360
|
+
async () => {
|
|
30361
|
+
const res = await req("GET", "/agent/extensions");
|
|
30362
|
+
if (!res.ok) return errorResult("browser_extension_list", res.data);
|
|
30363
|
+
const extensions = Array.isArray(res.data?.extensions) ? res.data.extensions : [];
|
|
30364
|
+
return structuredResult({
|
|
30365
|
+
ok: true,
|
|
30366
|
+
tool: "browser_extension_list",
|
|
30367
|
+
session_id: null,
|
|
30368
|
+
extensions,
|
|
30369
|
+
count: extensions.length
|
|
30370
|
+
});
|
|
30371
|
+
}
|
|
30372
|
+
);
|
|
30373
|
+
server.registerTool(
|
|
30374
|
+
"browser_extension_delete",
|
|
30375
|
+
{
|
|
30376
|
+
title: "Remove Browser Extension",
|
|
30377
|
+
description: "Remove a previously added extension by name so it can no longer be loaded via extension_names.",
|
|
30378
|
+
inputSchema: BrowserExtensionDeleteInputSchema,
|
|
30379
|
+
outputSchema: recordOutputSchema("browser_extension_delete", BrowserExtensionDeleteOutputSchema),
|
|
30380
|
+
annotations: { title: "Remove Browser Extension", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
30381
|
+
},
|
|
30382
|
+
async (input) => {
|
|
30383
|
+
const res = await req("DELETE", `/agent/extensions/${encodeURIComponent(input.name)}`);
|
|
30384
|
+
if (!res.ok) return errorResult("browser_extension_delete", res.data);
|
|
30385
|
+
return structuredResult({
|
|
30386
|
+
ok: true,
|
|
30387
|
+
tool: "browser_extension_delete",
|
|
30388
|
+
session_id: null,
|
|
30389
|
+
name: input.name,
|
|
30390
|
+
deleted: true
|
|
30391
|
+
});
|
|
30392
|
+
}
|
|
30393
|
+
);
|
|
30152
30394
|
server.registerTool(
|
|
30153
30395
|
"browser_open",
|
|
30154
30396
|
{
|
|
@@ -30167,7 +30409,8 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
30167
30409
|
...profile && typeof saveProfileChanges === "boolean" ? { save_profile_changes: saveProfileChanges } : {},
|
|
30168
30410
|
disable_default_proxy: true,
|
|
30169
30411
|
timeout_seconds: input.timeout_seconds,
|
|
30170
|
-
...input.url ? { url: input.url } : {}
|
|
30412
|
+
...input.url ? { url: input.url } : {},
|
|
30413
|
+
...input.extension_names?.length ? { extension_names: input.extension_names } : {}
|
|
30171
30414
|
});
|
|
30172
30415
|
if (!open.ok) return errorResult("browser_open", open.data);
|
|
30173
30416
|
const session = open.data;
|
|
@@ -33435,8 +33678,58 @@ async function migrateBrowserAgent() {
|
|
|
33435
33678
|
} catch {
|
|
33436
33679
|
}
|
|
33437
33680
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
|
|
33681
|
+
await db.execute(`
|
|
33682
|
+
CREATE TABLE IF NOT EXISTS browser_agent_extensions (
|
|
33683
|
+
id TEXT PRIMARY KEY,
|
|
33684
|
+
user_id INTEGER NOT NULL,
|
|
33685
|
+
name TEXT NOT NULL,
|
|
33686
|
+
backend_id TEXT NOT NULL,
|
|
33687
|
+
backend_name TEXT NOT NULL,
|
|
33688
|
+
source TEXT NOT NULL DEFAULT 'store',
|
|
33689
|
+
source_url TEXT,
|
|
33690
|
+
size_bytes INTEGER,
|
|
33691
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
33692
|
+
)
|
|
33693
|
+
`);
|
|
33694
|
+
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS browser_agent_extensions_user_name ON browser_agent_extensions(user_id, name)`);
|
|
33695
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_extensions_user ON browser_agent_extensions(user_id)`);
|
|
33438
33696
|
_ready2 = true;
|
|
33439
33697
|
}
|
|
33698
|
+
async function createExtensionRow(input) {
|
|
33699
|
+
const db = getDb();
|
|
33700
|
+
const id = `bext_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
33701
|
+
await db.execute({
|
|
33702
|
+
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
33703
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
33704
|
+
args: [id, input.userId, input.name, input.backendId, input.backendName, input.source, input.sourceUrl, input.sizeBytes]
|
|
33705
|
+
});
|
|
33706
|
+
const row = await getExtensionRow(input.userId, input.name);
|
|
33707
|
+
if (!row) throw new Error("extension insert failed");
|
|
33708
|
+
return row;
|
|
33709
|
+
}
|
|
33710
|
+
async function getExtensionRow(userId, name) {
|
|
33711
|
+
const db = getDb();
|
|
33712
|
+
const res = await db.execute({
|
|
33713
|
+
sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
|
|
33714
|
+
args: [userId, name]
|
|
33715
|
+
});
|
|
33716
|
+
return res.rows[0] ?? null;
|
|
33717
|
+
}
|
|
33718
|
+
async function listExtensionRows(userId) {
|
|
33719
|
+
const db = getDb();
|
|
33720
|
+
const res = await db.execute({
|
|
33721
|
+
sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? ORDER BY created_at DESC`,
|
|
33722
|
+
args: [userId]
|
|
33723
|
+
});
|
|
33724
|
+
return res.rows;
|
|
33725
|
+
}
|
|
33726
|
+
async function deleteExtensionRow(userId, name) {
|
|
33727
|
+
const db = getDb();
|
|
33728
|
+
await db.execute({
|
|
33729
|
+
sql: `DELETE FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
|
|
33730
|
+
args: [userId, name]
|
|
33731
|
+
});
|
|
33732
|
+
}
|
|
33440
33733
|
async function createAuthConnectionRow(input) {
|
|
33441
33734
|
const db = getDb();
|
|
33442
33735
|
const connectionId = `authc_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
@@ -34353,6 +34646,28 @@ function isProfileConflict(err) {
|
|
|
34353
34646
|
const message = err instanceof Error ? err.message : String(err);
|
|
34354
34647
|
return /\b409\b|conflict|already exists/i.test(message);
|
|
34355
34648
|
}
|
|
34649
|
+
function isNotFound(err) {
|
|
34650
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
34651
|
+
return /\b404\b|not found/i.test(message);
|
|
34652
|
+
}
|
|
34653
|
+
async function importExtensionFromStore(storeUrl, backendName) {
|
|
34654
|
+
const k = client();
|
|
34655
|
+
const archive = await k.extensions.downloadFromChromeStore({ url: storeUrl });
|
|
34656
|
+
const uploaded = await k.extensions.upload({ file: archive, name: backendName });
|
|
34657
|
+
return {
|
|
34658
|
+
backendId: uploaded.id,
|
|
34659
|
+
backendName: uploaded.name ?? backendName,
|
|
34660
|
+
sizeBytes: typeof uploaded.size_bytes === "number" ? uploaded.size_bytes : null
|
|
34661
|
+
};
|
|
34662
|
+
}
|
|
34663
|
+
async function deleteExtensionBackend(backendName) {
|
|
34664
|
+
const k = client();
|
|
34665
|
+
try {
|
|
34666
|
+
await k.extensions.delete(backendName);
|
|
34667
|
+
} catch (err) {
|
|
34668
|
+
if (!isNotFound(err)) throw err;
|
|
34669
|
+
}
|
|
34670
|
+
}
|
|
34356
34671
|
async function ensureProfile(k, name) {
|
|
34357
34672
|
try {
|
|
34358
34673
|
await k.profiles.create({ name });
|
|
@@ -34371,6 +34686,7 @@ async function createSession(opts = {}) {
|
|
|
34371
34686
|
const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
|
|
34372
34687
|
const disableDefaultProxy = opts.disableDefaultProxy ?? !resolvedProxyId;
|
|
34373
34688
|
const startUrl = opts.startUrl?.trim();
|
|
34689
|
+
const extensionNames = (opts.extensionNames ?? []).map((name) => name.trim()).filter(Boolean);
|
|
34374
34690
|
if (resolvedProfileName && resolvedSaveProfileChanges === true) {
|
|
34375
34691
|
await ensureProfile(k, resolvedProfileName);
|
|
34376
34692
|
}
|
|
@@ -34384,7 +34700,8 @@ async function createSession(opts = {}) {
|
|
|
34384
34700
|
name: resolvedProfileName,
|
|
34385
34701
|
...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
|
|
34386
34702
|
}
|
|
34387
|
-
} : {}
|
|
34703
|
+
} : {},
|
|
34704
|
+
...extensionNames.length ? { extensions: extensionNames.map((name) => ({ name })) } : {}
|
|
34388
34705
|
};
|
|
34389
34706
|
const browser = await k.browsers.create(createPayload);
|
|
34390
34707
|
const runtimeSessionId = browser.session_id;
|
|
@@ -34781,6 +35098,15 @@ function parseAgentDate(value) {
|
|
|
34781
35098
|
const legacyParsed = Date.parse(`${value.replace(" ", "T")}Z`);
|
|
34782
35099
|
return Number.isFinite(legacyParsed) ? legacyParsed : null;
|
|
34783
35100
|
}
|
|
35101
|
+
function publicExtension(row) {
|
|
35102
|
+
return {
|
|
35103
|
+
name: row.name,
|
|
35104
|
+
source: row.source,
|
|
35105
|
+
source_url: row.source_url,
|
|
35106
|
+
size_bytes: row.size_bytes,
|
|
35107
|
+
created_at: row.created_at
|
|
35108
|
+
};
|
|
35109
|
+
}
|
|
34784
35110
|
function replayClock(row) {
|
|
34785
35111
|
if (!row) return null;
|
|
34786
35112
|
const startedAtMs = parseAgentDate(row.started_at);
|
|
@@ -34927,6 +35253,15 @@ function buildBrowserAgentRoutes() {
|
|
|
34927
35253
|
const body = await c.req.json().catch(() => ({}));
|
|
34928
35254
|
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
34929
35255
|
const startUrl = typeof body.url === "string" ? body.url : void 0;
|
|
35256
|
+
const extensionNames = Array.isArray(body.extension_names) ? body.extension_names.filter((v) => typeof v === "string") : void 0;
|
|
35257
|
+
let resolvedExtensionBackendNames;
|
|
35258
|
+
if (extensionNames && extensionNames.length) {
|
|
35259
|
+
const names = extensionNames;
|
|
35260
|
+
const rows = await Promise.all(names.map((name) => getExtensionRow(user.id, name)));
|
|
35261
|
+
const missing = names.filter((_, i) => !rows[i]);
|
|
35262
|
+
if (missing.length) return c.json({ error: `extension(s) not found: ${missing.join(", ")}` }, 400);
|
|
35263
|
+
resolvedExtensionBackendNames = rows.map((row) => row.backend_name);
|
|
35264
|
+
}
|
|
34930
35265
|
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
34931
35266
|
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
34932
35267
|
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
@@ -34941,7 +35276,8 @@ function buildBrowserAgentRoutes() {
|
|
|
34941
35276
|
saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
|
|
34942
35277
|
startUrl,
|
|
34943
35278
|
disableDefaultProxy: typeof body.disable_default_proxy === "boolean" ? body.disable_default_proxy : void 0,
|
|
34944
|
-
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
35279
|
+
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0,
|
|
35280
|
+
extensionNames: resolvedExtensionBackendNames
|
|
34945
35281
|
});
|
|
34946
35282
|
runtimeSessionId = created.runtimeSessionId;
|
|
34947
35283
|
const row = await createSessionRow({
|
|
@@ -35244,6 +35580,52 @@ function buildBrowserAgentRoutes() {
|
|
|
35244
35580
|
}))
|
|
35245
35581
|
});
|
|
35246
35582
|
});
|
|
35583
|
+
app2.post("/extensions/import", async (c) => {
|
|
35584
|
+
const user = c.get("user");
|
|
35585
|
+
const body = await c.req.json().catch(() => ({}));
|
|
35586
|
+
const storeUrl = typeof body.store_url === "string" ? body.store_url.trim() : "";
|
|
35587
|
+
const name = typeof body.name === "string" ? body.name.trim() : "";
|
|
35588
|
+
if (!storeUrl) return c.json({ error: "store_url is required" }, 400);
|
|
35589
|
+
if (!name || !EXTENSION_NAME_RE.test(name)) {
|
|
35590
|
+
return c.json({ error: "name is required (1-64 characters: letters, numbers, spaces, dots, underscores, hyphens)" }, 400);
|
|
35591
|
+
}
|
|
35592
|
+
const existing = await getExtensionRow(user.id, name);
|
|
35593
|
+
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
35594
|
+
const backendName = `u${user.id}_${(0, import_node_crypto12.randomUUID)().replace(/-/g, "")}`;
|
|
35595
|
+
try {
|
|
35596
|
+
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
35597
|
+
const row = await createExtensionRow({
|
|
35598
|
+
userId: user.id,
|
|
35599
|
+
name,
|
|
35600
|
+
backendId: imported.backendId,
|
|
35601
|
+
backendName: imported.backendName,
|
|
35602
|
+
source: "store",
|
|
35603
|
+
sourceUrl: storeUrl,
|
|
35604
|
+
sizeBytes: imported.sizeBytes
|
|
35605
|
+
});
|
|
35606
|
+
return c.json(publicExtension(row));
|
|
35607
|
+
} catch (err) {
|
|
35608
|
+
return c.json(failure(err), 502);
|
|
35609
|
+
}
|
|
35610
|
+
});
|
|
35611
|
+
app2.get("/extensions", async (c) => {
|
|
35612
|
+
const user = c.get("user");
|
|
35613
|
+
const rows = await listExtensionRows(user.id);
|
|
35614
|
+
return c.json({ extensions: rows.map(publicExtension) });
|
|
35615
|
+
});
|
|
35616
|
+
app2.delete("/extensions/:name", async (c) => {
|
|
35617
|
+
const user = c.get("user");
|
|
35618
|
+
const name = c.req.param("name");
|
|
35619
|
+
const row = await getExtensionRow(user.id, name);
|
|
35620
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
35621
|
+
try {
|
|
35622
|
+
await deleteExtensionBackend(row.backend_name);
|
|
35623
|
+
} catch (err) {
|
|
35624
|
+
return c.json(failure(err), 502);
|
|
35625
|
+
}
|
|
35626
|
+
await deleteExtensionRow(user.id, name);
|
|
35627
|
+
return c.json({ ok: true });
|
|
35628
|
+
});
|
|
35247
35629
|
app2.get("/sessions/:id/replays/:replayId/download", async (c) => {
|
|
35248
35630
|
const user = c.get("user");
|
|
35249
35631
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
@@ -35268,10 +35650,11 @@ function buildBrowserAgentRoutes() {
|
|
|
35268
35650
|
});
|
|
35269
35651
|
return app2;
|
|
35270
35652
|
}
|
|
35271
|
-
var import_hono17, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
35653
|
+
var import_node_crypto12, import_hono17, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
|
|
35272
35654
|
var init_browser_agent_routes = __esm({
|
|
35273
35655
|
"src/api/browser-agent-routes.ts"() {
|
|
35274
35656
|
"use strict";
|
|
35657
|
+
import_node_crypto12 = require("crypto");
|
|
35275
35658
|
import_hono17 = require("hono");
|
|
35276
35659
|
init_api_auth();
|
|
35277
35660
|
init_errors();
|
|
@@ -35282,6 +35665,7 @@ var init_browser_agent_routes = __esm({
|
|
|
35282
35665
|
init_concurrency_gates();
|
|
35283
35666
|
auth = createApiKeyAuth();
|
|
35284
35667
|
DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS = 2 * 60 * 60;
|
|
35668
|
+
EXTENSION_NAME_RE = /^[\w .-]{1,64}$/;
|
|
35285
35669
|
}
|
|
35286
35670
|
});
|
|
35287
35671
|
|
|
@@ -35640,7 +36024,6 @@ var init_stripe_routes = __esm({
|
|
|
35640
36024
|
const live = sub.status === "active" || sub.status === "trialing";
|
|
35641
36025
|
const newTier = live ? tier.tier : null;
|
|
35642
36026
|
await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
|
|
35643
|
-
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: newTier }));
|
|
35644
36027
|
} else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
|
|
35645
36028
|
await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
|
|
35646
36029
|
await setConcurrencySubId(user.id, sub.id);
|
|
@@ -35666,7 +36049,6 @@ var init_stripe_routes = __esm({
|
|
|
35666
36049
|
if (!user) return c.json({ received: true });
|
|
35667
36050
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
35668
36051
|
await setSubscriptionTier(user.id, null, 0, null);
|
|
35669
|
-
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: null }));
|
|
35670
36052
|
} else if (priceId === CONCURRENCY_PRICE_ID) {
|
|
35671
36053
|
await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
|
|
35672
36054
|
await setConcurrencySubId(user.id, null);
|
|
@@ -35690,7 +36072,7 @@ async function getKeys() {
|
|
|
35690
36072
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
35691
36073
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
35692
36074
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
35693
|
-
const kid = (0,
|
|
36075
|
+
const kid = (0, import_node_crypto13.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
35694
36076
|
publicJwk.kid = kid;
|
|
35695
36077
|
publicJwk.alg = "RS256";
|
|
35696
36078
|
publicJwk.use = "sig";
|
|
@@ -35869,23 +36251,23 @@ async function validateAuthRequest(p) {
|
|
|
35869
36251
|
}
|
|
35870
36252
|
function pkceMatches(verifier, challenge) {
|
|
35871
36253
|
if (!verifier) return false;
|
|
35872
|
-
const computed = (0,
|
|
36254
|
+
const computed = (0, import_node_crypto13.createHash)("sha256").update(verifier).digest("base64url");
|
|
35873
36255
|
return computed === challenge;
|
|
35874
36256
|
}
|
|
35875
36257
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
35876
36258
|
const { privateKey, kid } = await getKeys();
|
|
35877
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
36259
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto13.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
35878
36260
|
}
|
|
35879
36261
|
function tokenErrorResponse(c, error, description, status) {
|
|
35880
36262
|
return c.json({ error, error_description: description }, status);
|
|
35881
36263
|
}
|
|
35882
|
-
var import_hono19, import_cookie,
|
|
36264
|
+
var import_hono19, import_cookie, import_node_crypto13, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
35883
36265
|
var init_oauth_routes = __esm({
|
|
35884
36266
|
"src/api/oauth-routes.ts"() {
|
|
35885
36267
|
"use strict";
|
|
35886
36268
|
import_hono19 = require("hono");
|
|
35887
36269
|
import_cookie = require("hono/cookie");
|
|
35888
|
-
|
|
36270
|
+
import_node_crypto13 = require("crypto");
|
|
35889
36271
|
import_jose2 = require("jose");
|
|
35890
36272
|
init_session();
|
|
35891
36273
|
init_db();
|
|
@@ -35971,7 +36353,7 @@ var init_oauth_routes = __esm({
|
|
|
35971
36353
|
}
|
|
35972
36354
|
}
|
|
35973
36355
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
35974
|
-
const clientId = `client_${(0,
|
|
36356
|
+
const clientId = `client_${(0, import_node_crypto13.randomBytes)(16).toString("hex")}`;
|
|
35975
36357
|
await registerClient(clientId, redirectUris, clientName);
|
|
35976
36358
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
35977
36359
|
return c.json({
|
|
@@ -36024,7 +36406,7 @@ var init_oauth_routes = __esm({
|
|
|
36024
36406
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
36025
36407
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
36026
36408
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
36027
|
-
const code = `code_${(0,
|
|
36409
|
+
const code = `code_${(0, import_node_crypto13.randomBytes)(32).toString("base64url")}`;
|
|
36028
36410
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
36029
36411
|
await putCode({
|
|
36030
36412
|
code,
|
|
@@ -36063,7 +36445,7 @@ var init_oauth_routes = __esm({
|
|
|
36063
36445
|
const plan = user ? resolvePlan(user) : "free";
|
|
36064
36446
|
const audience = record.resource ?? RESOURCE();
|
|
36065
36447
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
36066
|
-
const refreshToken = `rt_${(0,
|
|
36448
|
+
const refreshToken = `rt_${(0, import_node_crypto13.randomBytes)(40).toString("base64url")}`;
|
|
36067
36449
|
await putRefresh({
|
|
36068
36450
|
refresh_token: refreshToken,
|
|
36069
36451
|
client_id: clientId,
|
|
@@ -36093,7 +36475,7 @@ var init_oauth_routes = __esm({
|
|
|
36093
36475
|
const plan = user ? resolvePlan(user) : "free";
|
|
36094
36476
|
const audience = record.resource ?? RESOURCE();
|
|
36095
36477
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
36096
|
-
const nextRefresh = `rt_${(0,
|
|
36478
|
+
const nextRefresh = `rt_${(0, import_node_crypto13.randomBytes)(40).toString("base64url")}`;
|
|
36097
36479
|
await rotateRefresh(refreshToken, {
|
|
36098
36480
|
refresh_token: nextRefresh,
|
|
36099
36481
|
client_id: record.client_id,
|
|
@@ -37162,6 +37544,77 @@ var init_site_audit_worker = __esm({
|
|
|
37162
37544
|
}
|
|
37163
37545
|
});
|
|
37164
37546
|
|
|
37547
|
+
// src/api/page-diff.ts
|
|
37548
|
+
function sha256Hex(value) {
|
|
37549
|
+
return (0, import_node_crypto14.createHash)("sha256").update(value).digest("hex");
|
|
37550
|
+
}
|
|
37551
|
+
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
37552
|
+
if (value.length <= maxChars) return { value, truncated: false };
|
|
37553
|
+
return { value: value.slice(0, maxChars), truncated: true };
|
|
37554
|
+
}
|
|
37555
|
+
function changeLines(change) {
|
|
37556
|
+
const lines = change.value.split("\n");
|
|
37557
|
+
if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop();
|
|
37558
|
+
return lines;
|
|
37559
|
+
}
|
|
37560
|
+
function diffPageContent(oldContent, newContent) {
|
|
37561
|
+
const changes = (0, import_diff.diffLines)(oldContent, newContent);
|
|
37562
|
+
const rawHunks = [];
|
|
37563
|
+
let linesAdded = 0;
|
|
37564
|
+
let linesRemoved = 0;
|
|
37565
|
+
let totalChangedLineCount = 0;
|
|
37566
|
+
for (const change of changes) {
|
|
37567
|
+
if (!change.added && !change.removed) continue;
|
|
37568
|
+
const lines = changeLines(change);
|
|
37569
|
+
totalChangedLineCount += lines.length;
|
|
37570
|
+
if (change.added) linesAdded += lines.length;
|
|
37571
|
+
if (change.removed) linesRemoved += lines.length;
|
|
37572
|
+
rawHunks.push({ type: change.added ? "added" : "removed", lines });
|
|
37573
|
+
}
|
|
37574
|
+
let hunksTruncated = false;
|
|
37575
|
+
const hunksWithinCount = rawHunks.length > MAX_DIFF_HUNKS ? rawHunks.slice(0, MAX_DIFF_HUNKS) : rawHunks;
|
|
37576
|
+
if (rawHunks.length > MAX_DIFF_HUNKS) hunksTruncated = true;
|
|
37577
|
+
const hunks = [];
|
|
37578
|
+
let emittedLines = 0;
|
|
37579
|
+
for (const hunk of hunksWithinCount) {
|
|
37580
|
+
if (emittedLines >= MAX_DIFF_LINES_PER_RESPONSE) {
|
|
37581
|
+
hunksTruncated = true;
|
|
37582
|
+
break;
|
|
37583
|
+
}
|
|
37584
|
+
const remaining = MAX_DIFF_LINES_PER_RESPONSE - emittedLines;
|
|
37585
|
+
if (hunk.lines.length > remaining) {
|
|
37586
|
+
hunks.push({ type: hunk.type, lines: hunk.lines.slice(0, remaining) });
|
|
37587
|
+
emittedLines += remaining;
|
|
37588
|
+
hunksTruncated = true;
|
|
37589
|
+
break;
|
|
37590
|
+
}
|
|
37591
|
+
hunks.push(hunk);
|
|
37592
|
+
emittedLines += hunk.lines.length;
|
|
37593
|
+
}
|
|
37594
|
+
const denominator = Math.max(oldContent.split("\n").length, newContent.split("\n").length, 1);
|
|
37595
|
+
const percentChanged = totalChangedLineCount > 0 ? Math.min(100, Math.round(totalChangedLineCount / denominator * 1e3) / 10) : 0;
|
|
37596
|
+
return {
|
|
37597
|
+
hunks,
|
|
37598
|
+
linesAdded,
|
|
37599
|
+
linesRemoved,
|
|
37600
|
+
percentChanged,
|
|
37601
|
+
hunksTruncated,
|
|
37602
|
+
hunksTruncatedReason: hunksTruncated ? `Showing ${emittedLines} of ${totalChangedLineCount} changed lines (capped at ${MAX_DIFF_HUNKS} hunks / ${MAX_DIFF_LINES_PER_RESPONSE} lines). Use contentHash/previousContentHash to confirm the full scope of change.` : null,
|
|
37603
|
+
totalChangedLineCount
|
|
37604
|
+
};
|
|
37605
|
+
}
|
|
37606
|
+
var import_node_crypto14, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
37607
|
+
var init_page_diff = __esm({
|
|
37608
|
+
"src/api/page-diff.ts"() {
|
|
37609
|
+
"use strict";
|
|
37610
|
+
import_node_crypto14 = require("crypto");
|
|
37611
|
+
import_diff = require("diff");
|
|
37612
|
+
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
37613
|
+
MAX_DIFF_HUNKS = 200;
|
|
37614
|
+
MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
37615
|
+
}
|
|
37616
|
+
});
|
|
37617
|
+
|
|
37165
37618
|
// src/api/scrape-vault-sink.ts
|
|
37166
37619
|
async function depositScrapeToVault(user, opts) {
|
|
37167
37620
|
try {
|
|
@@ -37859,6 +38312,7 @@ var init_server = __esm({
|
|
|
37859
38312
|
import_stripe2 = __toESM(require("stripe"), 1);
|
|
37860
38313
|
init_rates();
|
|
37861
38314
|
init_server_schemas();
|
|
38315
|
+
init_page_diff();
|
|
37862
38316
|
init_scrape_vault_sink();
|
|
37863
38317
|
init_scrape_blob_cleanup();
|
|
37864
38318
|
init_schemas3();
|
|
@@ -38758,6 +39212,91 @@ var init_server = __esm({
|
|
|
38758
39212
|
await releaseConcurrencyGate(gate.lockId);
|
|
38759
39213
|
}
|
|
38760
39214
|
});
|
|
39215
|
+
app.post("/diff-page", auth2, async (c) => {
|
|
39216
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
39217
|
+
const bodyResult = DiffPageBodySchema.safeParse(raw);
|
|
39218
|
+
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
39219
|
+
const { url, allowLocal, resetBaseline } = bodyResult.data;
|
|
39220
|
+
if (!allowLocal) {
|
|
39221
|
+
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
39222
|
+
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
39223
|
+
} else {
|
|
39224
|
+
try {
|
|
39225
|
+
new URL(url);
|
|
39226
|
+
} catch {
|
|
39227
|
+
return c.json({ error: "Invalid URL" }, 400);
|
|
39228
|
+
}
|
|
39229
|
+
}
|
|
39230
|
+
const canonicalUrl = (() => {
|
|
39231
|
+
try {
|
|
39232
|
+
return new URL(url).href;
|
|
39233
|
+
} catch {
|
|
39234
|
+
return url;
|
|
39235
|
+
}
|
|
39236
|
+
})();
|
|
39237
|
+
const user = c.get("user");
|
|
39238
|
+
const gate = await acquireConcurrencyGate(user, "diff_page", {
|
|
39239
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
39240
|
+
metadata: { url: canonicalUrl }
|
|
39241
|
+
});
|
|
39242
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
39243
|
+
let debited = false;
|
|
39244
|
+
try {
|
|
39245
|
+
const { ok: dpOk, balance_mc: dpBal } = await debitMc(user.id, MC_COSTS.diff_page, LedgerOperation.DIFF_PAGE, new URL(canonicalUrl).hostname);
|
|
39246
|
+
if (!dpOk) return c.json(insufficientBalanceResponse(dpBal, MC_COSTS.diff_page), 402);
|
|
39247
|
+
debited = true;
|
|
39248
|
+
const existingSnapshot = await getPageSnapshot(user.id, canonicalUrl);
|
|
39249
|
+
const previous = resetBaseline ? null : existingSnapshot;
|
|
39250
|
+
const kernelApiKey = browserServiceApiKey();
|
|
39251
|
+
const result = await extractKpo({ url: canonicalUrl, kernelApiKey });
|
|
39252
|
+
const currentContent = result.bodyMarkdown ?? "";
|
|
39253
|
+
const currentHash = sha256Hex(currentContent);
|
|
39254
|
+
const { value: storedContent, truncated: contentTruncated } = truncateForStorage(currentContent);
|
|
39255
|
+
const currentCheckedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
39256
|
+
await upsertPageSnapshot(user.id, canonicalUrl, {
|
|
39257
|
+
contentHash: currentHash,
|
|
39258
|
+
content: storedContent,
|
|
39259
|
+
title: result.title ?? null,
|
|
39260
|
+
contentBytes: Buffer.byteLength(currentContent, "utf8"),
|
|
39261
|
+
truncated: contentTruncated
|
|
39262
|
+
});
|
|
39263
|
+
let status;
|
|
39264
|
+
let diff = { hunks: [], linesAdded: 0, linesRemoved: 0, percentChanged: 0, hunksTruncated: false, hunksTruncatedReason: null, totalChangedLineCount: 0 };
|
|
39265
|
+
if (!previous) {
|
|
39266
|
+
status = "baseline";
|
|
39267
|
+
diff.percentChanged = null;
|
|
39268
|
+
} else if (previous.contentHash === currentHash) {
|
|
39269
|
+
status = "unchanged";
|
|
39270
|
+
} else {
|
|
39271
|
+
status = "changed";
|
|
39272
|
+
diff = diffPageContent(previous.content, storedContent);
|
|
39273
|
+
}
|
|
39274
|
+
await logRequestEvent({ userId: user.id, source: "diff_page", status: "done", query: canonicalUrl, result: { status } });
|
|
39275
|
+
return c.json({
|
|
39276
|
+
url: canonicalUrl,
|
|
39277
|
+
title: result.title ?? null,
|
|
39278
|
+
status,
|
|
39279
|
+
isReset: !!resetBaseline && !!existingSnapshot,
|
|
39280
|
+
previousCheckedAt: previous?.checkedAt ?? null,
|
|
39281
|
+
currentCheckedAt,
|
|
39282
|
+
contentHash: currentHash,
|
|
39283
|
+
previousContentHash: previous?.contentHash ?? null,
|
|
39284
|
+
summary: { linesAdded: diff.linesAdded, linesRemoved: diff.linesRemoved, percentChanged: diff.percentChanged },
|
|
39285
|
+
hunks: diff.hunks,
|
|
39286
|
+
contentTruncated,
|
|
39287
|
+
hunksTruncated: diff.hunksTruncated,
|
|
39288
|
+
hunksTruncatedReason: diff.hunksTruncatedReason,
|
|
39289
|
+
totalChangedLineCount: diff.totalChangedLineCount
|
|
39290
|
+
});
|
|
39291
|
+
} catch (err) {
|
|
39292
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
39293
|
+
if (debited) await creditMc(user.id, MC_COSTS.diff_page, LedgerOperation.DIFF_PAGE, "failed call");
|
|
39294
|
+
await logRequestEvent({ userId: user.id, source: "diff_page", status: "failed", query: canonicalUrl, error: msg });
|
|
39295
|
+
return c.json({ error: msg }, 500);
|
|
39296
|
+
} finally {
|
|
39297
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
39298
|
+
}
|
|
39299
|
+
});
|
|
38761
39300
|
app.post("/map-urls", auth2, async (c) => {
|
|
38762
39301
|
const raw = await c.req.json().catch(() => ({}));
|
|
38763
39302
|
const bodyResult = MapUrlsBodySchema.safeParse(raw);
|