mcp-scraper 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +2068 -1811
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +94 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-GFY6XHZS.js → chunk-BAMVP57P.js} +95 -2
- package/dist/chunk-BAMVP57P.js.map +1 -0
- package/dist/chunk-SRMGSCC3.js +7 -0
- package/dist/chunk-SRMGSCC3.js.map +1 -0
- package/dist/{server-6IYQQK6X.js → server-LBEYRKM2.js} +344 -193
- package/dist/server-LBEYRKM2.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-7HICT4GY.js +0 -7
- package/dist/chunk-7HICT4GY.js.map +0 -1
- package/dist/chunk-GFY6XHZS.js.map +0 -1
- package/dist/server-6IYQQK6X.js.map +0 -1
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
registerSerpIntelligenceCaptureTools,
|
|
32
32
|
sanitizeAttempts,
|
|
33
33
|
sanitizeHarvestResult
|
|
34
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-BAMVP57P.js";
|
|
35
35
|
import {
|
|
36
36
|
auditImages,
|
|
37
37
|
getBlobStore
|
|
@@ -65,7 +65,7 @@ import {
|
|
|
65
65
|
RawMapsOverviewSchema,
|
|
66
66
|
RawMapsReviewStatsSchema
|
|
67
67
|
} from "./chunk-XGIPATLV.js";
|
|
68
|
-
import "./chunk-
|
|
68
|
+
import "./chunk-SRMGSCC3.js";
|
|
69
69
|
import {
|
|
70
70
|
completeExtractJob,
|
|
71
71
|
countSuccessfulPages,
|
|
@@ -5274,7 +5274,7 @@ async function extractSite(opts) {
|
|
|
5274
5274
|
}
|
|
5275
5275
|
|
|
5276
5276
|
// src/api/server.ts
|
|
5277
|
-
import { Hono as
|
|
5277
|
+
import { Hono as Hono21 } from "hono";
|
|
5278
5278
|
import { serve as serveInngest } from "inngest/hono";
|
|
5279
5279
|
|
|
5280
5280
|
// src/inngest/client.ts
|
|
@@ -13160,13 +13160,163 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
13160
13160
|
}
|
|
13161
13161
|
});
|
|
13162
13162
|
|
|
13163
|
-
// src/api/
|
|
13163
|
+
// src/api/trustpilot-routes.ts
|
|
13164
13164
|
import { Hono as Hono11 } from "hono";
|
|
13165
|
+
import { z as z19 } from "zod";
|
|
13166
|
+
var TrustpilotReviewsBodySchema = z19.object({
|
|
13167
|
+
domain: z19.string().trim().min(1, "domain is required"),
|
|
13168
|
+
maxPages: z19.number().int().min(1).max(50).default(5)
|
|
13169
|
+
});
|
|
13170
|
+
function invalidRequest6(message) {
|
|
13171
|
+
return { error_code: "invalid_request", message };
|
|
13172
|
+
}
|
|
13173
|
+
function normalizeDomain(input) {
|
|
13174
|
+
return input.trim().replace(/^https?:\/\//i, "").replace(/\/.*$/, "");
|
|
13175
|
+
}
|
|
13176
|
+
function reviewPageUrl(domain, page) {
|
|
13177
|
+
const base = `https://www.trustpilot.com/review/${encodeURIComponent(domain)}`;
|
|
13178
|
+
return page <= 1 ? base : `${base}?page=${page}`;
|
|
13179
|
+
}
|
|
13180
|
+
async function extractReviewCards(page) {
|
|
13181
|
+
return page.evaluate(() => {
|
|
13182
|
+
const cards = Array.from(document.querySelectorAll('article[data-service-review-card-paper="true"]')).filter((a) => a.querySelector('[data-testid="service-review-card-v2"]'));
|
|
13183
|
+
return cards.map((card) => {
|
|
13184
|
+
const profileLink = card.querySelector('a[data-consumer-profile-link="true"]');
|
|
13185
|
+
const profileHref = profileLink?.getAttribute("href") ?? "";
|
|
13186
|
+
const profileId = profileHref.startsWith("/users/") ? profileHref.slice("/users/".length) : null;
|
|
13187
|
+
const name = card.querySelector('[data-consumer-name-typography="true"]')?.textContent?.trim() ?? null;
|
|
13188
|
+
const timeEl = card.querySelector("time[datetime]");
|
|
13189
|
+
const date = timeEl?.getAttribute("datetime") ?? null;
|
|
13190
|
+
const ratingEl = card.querySelector("[data-service-review-rating]");
|
|
13191
|
+
const ratingRaw = ratingEl?.getAttribute("data-service-review-rating");
|
|
13192
|
+
const rating = ratingRaw ? parseInt(ratingRaw, 10) : null;
|
|
13193
|
+
const titleLink = card.querySelector('a[data-review-title-typography="true"]');
|
|
13194
|
+
const titleHref = titleLink?.getAttribute("href") ?? "";
|
|
13195
|
+
const reviewId = titleHref.startsWith("/reviews/") ? titleHref.slice("/reviews/".length) : null;
|
|
13196
|
+
const title = card.querySelector('[data-service-review-title-typography="true"]')?.textContent?.trim() ?? null;
|
|
13197
|
+
const bodyEl = card.querySelector('[data-service-review-text-typography="true"]');
|
|
13198
|
+
const rawBody = bodyEl?.textContent?.trim() ?? "";
|
|
13199
|
+
const seeMoreEl = card.querySelector('[class*="seeMore"]');
|
|
13200
|
+
const truncated = !!seeMoreEl;
|
|
13201
|
+
const body = truncated ? rawBody.replace(/see more\s*$/i, "").trim() : rawBody;
|
|
13202
|
+
const labelEl = card.querySelector('[class*="reviewLabel"]');
|
|
13203
|
+
const labelText = labelEl?.textContent?.trim() ?? null;
|
|
13204
|
+
const origin = labelText && /invit/i.test(labelText) ? "invited" : labelText ? labelText.toLowerCase() : "organic";
|
|
13205
|
+
const companyReplied = !!card.querySelector('[class*="companyReply"]');
|
|
13206
|
+
return {
|
|
13207
|
+
source: "trustpilot",
|
|
13208
|
+
sourceReviewId: reviewId,
|
|
13209
|
+
reviewUrl: reviewId ? `https://www.trustpilot.com/reviews/${reviewId}` : null,
|
|
13210
|
+
reviewer: {
|
|
13211
|
+
name,
|
|
13212
|
+
profileId,
|
|
13213
|
+
title: null,
|
|
13214
|
+
companySegment: null
|
|
13215
|
+
},
|
|
13216
|
+
rating,
|
|
13217
|
+
ratingScale: 5,
|
|
13218
|
+
title,
|
|
13219
|
+
date,
|
|
13220
|
+
body: [{ question: null, answer: body }],
|
|
13221
|
+
truncated,
|
|
13222
|
+
flags: {
|
|
13223
|
+
origin,
|
|
13224
|
+
incentivized: null,
|
|
13225
|
+
validated: null,
|
|
13226
|
+
currentUser: null,
|
|
13227
|
+
companyReplied
|
|
13228
|
+
}
|
|
13229
|
+
};
|
|
13230
|
+
});
|
|
13231
|
+
});
|
|
13232
|
+
}
|
|
13233
|
+
var trustpilotApp = new Hono11();
|
|
13234
|
+
trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13235
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13236
|
+
const parsed = TrustpilotReviewsBodySchema.safeParse(raw);
|
|
13237
|
+
if (!parsed.success) return c.json(invalidRequest6(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13238
|
+
const body = parsed.data;
|
|
13239
|
+
const domain = normalizeDomain(body.domain);
|
|
13240
|
+
const user = c.get("user");
|
|
13241
|
+
const gate = await acquireConcurrencyGate(user, "trustpilot_reviews", {
|
|
13242
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13243
|
+
metadata: { domain, maxPages: body.maxPages }
|
|
13244
|
+
});
|
|
13245
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13246
|
+
let debited = false;
|
|
13247
|
+
let reviewDebitMc = 0;
|
|
13248
|
+
try {
|
|
13249
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS, domain);
|
|
13250
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.trustpilot_reviews), 402);
|
|
13251
|
+
debited = true;
|
|
13252
|
+
const startMs = Date.now();
|
|
13253
|
+
const driver = new BrowserDriver();
|
|
13254
|
+
const allReviews = [];
|
|
13255
|
+
let pagesFetched = 0;
|
|
13256
|
+
try {
|
|
13257
|
+
await driver.launch({
|
|
13258
|
+
headless: true,
|
|
13259
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13260
|
+
viewport: { width: 1280, height: 900 },
|
|
13261
|
+
locale: "en-US"
|
|
13262
|
+
});
|
|
13263
|
+
const page = driver.getPage();
|
|
13264
|
+
for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
|
|
13265
|
+
const url = reviewPageUrl(domain, pageNum);
|
|
13266
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
13267
|
+
await page.waitForTimeout(2e3);
|
|
13268
|
+
const cards = await extractReviewCards(page);
|
|
13269
|
+
pagesFetched = pageNum;
|
|
13270
|
+
if (cards.length === 0) break;
|
|
13271
|
+
allReviews.push(...cards);
|
|
13272
|
+
}
|
|
13273
|
+
} finally {
|
|
13274
|
+
await driver.close();
|
|
13275
|
+
}
|
|
13276
|
+
if (pagesFetched === 0 || allReviews.length === 0) {
|
|
13277
|
+
await creditMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS_REFUND, "no reviews found");
|
|
13278
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "failed", query: domain, error: "no reviews found" });
|
|
13279
|
+
return c.json({ error: "No reviews found for this domain (refunded)", domain }, 404);
|
|
13280
|
+
}
|
|
13281
|
+
reviewDebitMc = allReviews.length * MC_COSTS.trustpilot_review;
|
|
13282
|
+
if (reviewDebitMc > 0) {
|
|
13283
|
+
const reviewDebit = await debitMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW, domain);
|
|
13284
|
+
if (!reviewDebit.ok) {
|
|
13285
|
+
allReviews.length = Math.floor((reviewDebit.balance_mc ?? 0) / MC_COSTS.trustpilot_review);
|
|
13286
|
+
reviewDebitMc = allReviews.length * MC_COSTS.trustpilot_review;
|
|
13287
|
+
if (reviewDebitMc > 0) await debitMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW, domain);
|
|
13288
|
+
}
|
|
13289
|
+
}
|
|
13290
|
+
const result = {
|
|
13291
|
+
domain,
|
|
13292
|
+
reviewUrl: reviewPageUrl(domain, 1),
|
|
13293
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13294
|
+
requestedMaxPages: body.maxPages,
|
|
13295
|
+
pagesFetched,
|
|
13296
|
+
reviewCount: allReviews.length,
|
|
13297
|
+
reviews: allReviews,
|
|
13298
|
+
durationMs: Date.now() - startMs
|
|
13299
|
+
};
|
|
13300
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "done", query: domain, resultCount: allReviews.length, result });
|
|
13301
|
+
return c.json(result);
|
|
13302
|
+
} catch (err) {
|
|
13303
|
+
if (debited) await creditMc(user.id, MC_COSTS.trustpilot_reviews, LedgerOperation.TRUSTPILOT_REVIEWS_REFUND, "failed call");
|
|
13304
|
+
if (reviewDebitMc > 0) await creditMc(user.id, reviewDebitMc, LedgerOperation.TRUSTPILOT_REVIEW_REFUND, "failed call");
|
|
13305
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13306
|
+
await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "failed", query: domain, error: msg });
|
|
13307
|
+
return c.json({ error: msg }, 500);
|
|
13308
|
+
} finally {
|
|
13309
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13310
|
+
}
|
|
13311
|
+
});
|
|
13312
|
+
|
|
13313
|
+
// src/api/directory-routes.ts
|
|
13314
|
+
import { Hono as Hono12 } from "hono";
|
|
13165
13315
|
|
|
13166
13316
|
// src/directory/directory-workflow.ts
|
|
13167
13317
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
13168
13318
|
import { join as join5 } from "path";
|
|
13169
|
-
import { z as
|
|
13319
|
+
import { z as z20 } from "zod";
|
|
13170
13320
|
|
|
13171
13321
|
// src/directory/location-db.ts
|
|
13172
13322
|
import { access, readFile } from "fs/promises";
|
|
@@ -13327,24 +13477,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
13327
13477
|
}
|
|
13328
13478
|
|
|
13329
13479
|
// src/directory/directory-workflow.ts
|
|
13330
|
-
var DirectoryWorkflowOptionsSchema =
|
|
13331
|
-
query:
|
|
13332
|
-
state:
|
|
13333
|
-
minPopulation:
|
|
13334
|
-
populationYear:
|
|
13335
|
-
maxCities:
|
|
13336
|
-
maxResultsPerCity:
|
|
13337
|
-
concurrency:
|
|
13338
|
-
includeZipGroups:
|
|
13339
|
-
usZipsCsvPath:
|
|
13340
|
-
saveCsv:
|
|
13341
|
-
gl:
|
|
13342
|
-
hl:
|
|
13343
|
-
proxyMode:
|
|
13344
|
-
proxyZip:
|
|
13345
|
-
debug:
|
|
13346
|
-
headless:
|
|
13347
|
-
kernelApiKey:
|
|
13480
|
+
var DirectoryWorkflowOptionsSchema = z20.object({
|
|
13481
|
+
query: z20.string().min(1),
|
|
13482
|
+
state: z20.string().min(2).default("TN"),
|
|
13483
|
+
minPopulation: z20.number().int().min(0).default(1e5),
|
|
13484
|
+
populationYear: z20.union(POPULATION_YEARS.map((year) => z20.literal(year))).default(2025),
|
|
13485
|
+
maxCities: z20.number().int().min(1).max(100).default(25),
|
|
13486
|
+
maxResultsPerCity: z20.number().int().min(1).max(50).default(50),
|
|
13487
|
+
concurrency: z20.number().int().min(1).max(5).default(5),
|
|
13488
|
+
includeZipGroups: z20.boolean().default(true),
|
|
13489
|
+
usZipsCsvPath: z20.string().optional(),
|
|
13490
|
+
saveCsv: z20.boolean().default(true),
|
|
13491
|
+
gl: z20.string().length(2).default("us"),
|
|
13492
|
+
hl: z20.string().length(2).default("en"),
|
|
13493
|
+
proxyMode: z20.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
13494
|
+
proxyZip: z20.string().regex(/^\d{5}$/).optional(),
|
|
13495
|
+
debug: z20.boolean().default(false),
|
|
13496
|
+
headless: z20.boolean().default(true),
|
|
13497
|
+
kernelApiKey: z20.string().optional()
|
|
13348
13498
|
});
|
|
13349
13499
|
function errorMessage(err) {
|
|
13350
13500
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -13565,7 +13715,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
13565
13715
|
}
|
|
13566
13716
|
|
|
13567
13717
|
// src/api/directory-routes.ts
|
|
13568
|
-
var directoryApp = new
|
|
13718
|
+
var directoryApp = new Hono12();
|
|
13569
13719
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
13570
13720
|
const user = c.get("user");
|
|
13571
13721
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13638,35 +13788,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
13638
13788
|
// src/api/workflow-routes.ts
|
|
13639
13789
|
import { createHmac as createHmac2 } from "crypto";
|
|
13640
13790
|
import { readFile as readFile2 } from "fs/promises";
|
|
13641
|
-
import { Hono as
|
|
13642
|
-
import { z as
|
|
13643
|
-
var workflowApp = new
|
|
13644
|
-
var WorkflowInputSchema =
|
|
13645
|
-
var WorkflowIdSchema =
|
|
13646
|
-
var CadenceSchema =
|
|
13647
|
-
var ScheduleStatusSchema =
|
|
13648
|
-
var RunBodySchema =
|
|
13791
|
+
import { Hono as Hono13 } from "hono";
|
|
13792
|
+
import { z as z21 } from "zod";
|
|
13793
|
+
var workflowApp = new Hono13();
|
|
13794
|
+
var WorkflowInputSchema = z21.record(z21.unknown()).default({});
|
|
13795
|
+
var WorkflowIdSchema = z21.string().min(1);
|
|
13796
|
+
var CadenceSchema = z21.enum(["daily", "weekly", "monthly"]);
|
|
13797
|
+
var ScheduleStatusSchema = z21.enum(["active", "paused"]);
|
|
13798
|
+
var RunBodySchema = z21.object({
|
|
13649
13799
|
workflowId: WorkflowIdSchema,
|
|
13650
13800
|
input: WorkflowInputSchema,
|
|
13651
|
-
webhookUrl:
|
|
13801
|
+
webhookUrl: z21.string().url().optional()
|
|
13652
13802
|
});
|
|
13653
|
-
var ScheduleCreateSchema =
|
|
13803
|
+
var ScheduleCreateSchema = z21.object({
|
|
13654
13804
|
workflowId: WorkflowIdSchema,
|
|
13655
|
-
name:
|
|
13805
|
+
name: z21.string().min(1).max(120).optional(),
|
|
13656
13806
|
input: WorkflowInputSchema,
|
|
13657
13807
|
cadence: CadenceSchema.default("weekly"),
|
|
13658
|
-
timezone:
|
|
13659
|
-
webhookUrl:
|
|
13660
|
-
nextRunAt:
|
|
13808
|
+
timezone: z21.string().min(1).max(64).default("UTC"),
|
|
13809
|
+
webhookUrl: z21.string().url().optional(),
|
|
13810
|
+
nextRunAt: z21.string().datetime().optional()
|
|
13661
13811
|
});
|
|
13662
|
-
var SchedulePatchSchema =
|
|
13663
|
-
name:
|
|
13812
|
+
var SchedulePatchSchema = z21.object({
|
|
13813
|
+
name: z21.string().min(1).max(120).optional(),
|
|
13664
13814
|
status: ScheduleStatusSchema.optional(),
|
|
13665
13815
|
input: WorkflowInputSchema.optional(),
|
|
13666
13816
|
cadence: CadenceSchema.optional(),
|
|
13667
|
-
timezone:
|
|
13668
|
-
webhookUrl:
|
|
13669
|
-
nextRunAt:
|
|
13817
|
+
timezone: z21.string().min(1).max(64).optional(),
|
|
13818
|
+
webhookUrl: z21.string().url().nullable().optional(),
|
|
13819
|
+
nextRunAt: z21.string().datetime().nullable().optional()
|
|
13670
13820
|
});
|
|
13671
13821
|
function hostedWorkflowOutputDir() {
|
|
13672
13822
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -14170,7 +14320,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
14170
14320
|
});
|
|
14171
14321
|
|
|
14172
14322
|
// src/api/serp-intelligence-routes.ts
|
|
14173
|
-
import { Hono as
|
|
14323
|
+
import { Hono as Hono14 } from "hono";
|
|
14174
14324
|
|
|
14175
14325
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14176
14326
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -14482,7 +14632,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14482
14632
|
}
|
|
14483
14633
|
|
|
14484
14634
|
// src/serp-intelligence/schemas.ts
|
|
14485
|
-
import { z as
|
|
14635
|
+
import { z as z22 } from "zod";
|
|
14486
14636
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
14487
14637
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
14488
14638
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -14544,171 +14694,171 @@ function isPublicHttpUrl(value) {
|
|
|
14544
14694
|
return false;
|
|
14545
14695
|
}
|
|
14546
14696
|
}
|
|
14547
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
14548
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
14549
|
-
query:
|
|
14550
|
-
location:
|
|
14551
|
-
gl:
|
|
14552
|
-
hl:
|
|
14553
|
-
device:
|
|
14554
|
-
proxyMode:
|
|
14555
|
-
proxyZip:
|
|
14556
|
-
pages:
|
|
14557
|
-
debug:
|
|
14558
|
-
includePageSnapshots:
|
|
14559
|
-
pageSnapshotLimit:
|
|
14697
|
+
var SerpIntelligencePublicHttpUrlSchema = z22.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
14698
|
+
var SerpIntelligenceCaptureBodySchema = z22.object({
|
|
14699
|
+
query: z22.string().trim().min(1, "query is required"),
|
|
14700
|
+
location: z22.string().trim().min(1).optional(),
|
|
14701
|
+
gl: z22.string().trim().length(2).default("us"),
|
|
14702
|
+
hl: z22.string().trim().length(2).default("en"),
|
|
14703
|
+
device: z22.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
14704
|
+
proxyMode: z22.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
14705
|
+
proxyZip: z22.string().regex(/^\d{5}$/).optional(),
|
|
14706
|
+
pages: z22.number().int().min(1).max(2).default(1),
|
|
14707
|
+
debug: z22.boolean().default(false),
|
|
14708
|
+
includePageSnapshots: z22.boolean().default(false),
|
|
14709
|
+
pageSnapshotLimit: z22.number().int().min(0).max(10).default(0)
|
|
14560
14710
|
}).strict();
|
|
14561
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
14711
|
+
var SerpIntelligencePageSnapshotRequestSchema = z22.object({
|
|
14562
14712
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14563
|
-
sourceKind:
|
|
14564
|
-
sourcePosition:
|
|
14713
|
+
sourceKind: z22.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
14714
|
+
sourcePosition: z22.number().int().min(1).optional()
|
|
14565
14715
|
}).strict();
|
|
14566
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
14567
|
-
urls:
|
|
14568
|
-
targets:
|
|
14569
|
-
maxConcurrency:
|
|
14570
|
-
timeoutMs:
|
|
14571
|
-
debug:
|
|
14716
|
+
var SerpIntelligencePageSnapshotsBodySchema = z22.object({
|
|
14717
|
+
urls: z22.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
14718
|
+
targets: z22.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
14719
|
+
maxConcurrency: z22.number().int().min(1).max(5).default(2),
|
|
14720
|
+
timeoutMs: z22.number().int().min(1e3).max(6e4).default(15e3),
|
|
14721
|
+
debug: z22.boolean().default(false)
|
|
14572
14722
|
}).strict();
|
|
14573
|
-
var SerpIntelligenceAICitationSchema =
|
|
14574
|
-
text:
|
|
14575
|
-
href:
|
|
14723
|
+
var SerpIntelligenceAICitationSchema = z22.object({
|
|
14724
|
+
text: z22.string(),
|
|
14725
|
+
href: z22.string()
|
|
14576
14726
|
}).strict();
|
|
14577
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
14578
|
-
position:
|
|
14579
|
-
title:
|
|
14580
|
-
url:
|
|
14581
|
-
domain:
|
|
14582
|
-
cite:
|
|
14583
|
-
snippet:
|
|
14584
|
-
isRedditStyle:
|
|
14585
|
-
inlineRating:
|
|
14586
|
-
value:
|
|
14587
|
-
count:
|
|
14727
|
+
var SerpIntelligenceOrganicResultSchema = z22.object({
|
|
14728
|
+
position: z22.number().int().min(1),
|
|
14729
|
+
title: z22.string(),
|
|
14730
|
+
url: z22.string(),
|
|
14731
|
+
domain: z22.string(),
|
|
14732
|
+
cite: z22.string().nullable(),
|
|
14733
|
+
snippet: z22.string().nullable(),
|
|
14734
|
+
isRedditStyle: z22.boolean(),
|
|
14735
|
+
inlineRating: z22.object({
|
|
14736
|
+
value: z22.string(),
|
|
14737
|
+
count: z22.string()
|
|
14588
14738
|
}).strict().nullable()
|
|
14589
14739
|
}).strict();
|
|
14590
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
14591
|
-
status:
|
|
14592
|
-
expected:
|
|
14593
|
-
city:
|
|
14594
|
-
regionCode:
|
|
14595
|
-
canonicalLocation:
|
|
14740
|
+
var SerpIntelligenceLocationEvidenceSchema = z22.object({
|
|
14741
|
+
status: z22.enum(SerpIntelligenceLocalizationStatusValues),
|
|
14742
|
+
expected: z22.object({
|
|
14743
|
+
city: z22.string(),
|
|
14744
|
+
regionCode: z22.string().nullable(),
|
|
14745
|
+
canonicalLocation: z22.string()
|
|
14596
14746
|
}).strict().nullable(),
|
|
14597
|
-
candidates:
|
|
14598
|
-
city:
|
|
14599
|
-
regionCode:
|
|
14600
|
-
count:
|
|
14601
|
-
examples:
|
|
14747
|
+
candidates: z22.array(z22.object({
|
|
14748
|
+
city: z22.string(),
|
|
14749
|
+
regionCode: z22.string(),
|
|
14750
|
+
count: z22.number().int().min(0),
|
|
14751
|
+
examples: z22.array(z22.string())
|
|
14602
14752
|
}).strict())
|
|
14603
14753
|
}).strict();
|
|
14604
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
14605
|
-
seed:
|
|
14606
|
-
location:
|
|
14607
|
-
extractedAt:
|
|
14608
|
-
totalQuestions:
|
|
14609
|
-
surface:
|
|
14610
|
-
aiOverview:
|
|
14611
|
-
detected:
|
|
14612
|
-
text:
|
|
14613
|
-
citations:
|
|
14614
|
-
expanded:
|
|
14615
|
-
fullyExpanded:
|
|
14616
|
-
sections:
|
|
14754
|
+
var SerpIntelligenceHarvestResultSchema = z22.object({
|
|
14755
|
+
seed: z22.string(),
|
|
14756
|
+
location: z22.string().nullable(),
|
|
14757
|
+
extractedAt: z22.string(),
|
|
14758
|
+
totalQuestions: z22.number().int().min(0),
|
|
14759
|
+
surface: z22.enum(["web", "aim", "unknown"]),
|
|
14760
|
+
aiOverview: z22.object({
|
|
14761
|
+
detected: z22.boolean(),
|
|
14762
|
+
text: z22.string().nullable(),
|
|
14763
|
+
citations: z22.array(SerpIntelligenceAICitationSchema),
|
|
14764
|
+
expanded: z22.boolean().optional(),
|
|
14765
|
+
fullyExpanded: z22.boolean().optional(),
|
|
14766
|
+
sections: z22.array(z22.string()).optional()
|
|
14617
14767
|
}).strict(),
|
|
14618
|
-
aiMode:
|
|
14619
|
-
detected:
|
|
14620
|
-
text:
|
|
14621
|
-
citations:
|
|
14768
|
+
aiMode: z22.object({
|
|
14769
|
+
detected: z22.boolean(),
|
|
14770
|
+
text: z22.string().nullable(),
|
|
14771
|
+
citations: z22.array(SerpIntelligenceAICitationSchema)
|
|
14622
14772
|
}).strict(),
|
|
14623
|
-
tree:
|
|
14624
|
-
flat:
|
|
14625
|
-
videos:
|
|
14626
|
-
forums:
|
|
14627
|
-
organicResults:
|
|
14628
|
-
localPack:
|
|
14629
|
-
entityIds:
|
|
14630
|
-
entities:
|
|
14631
|
-
name:
|
|
14632
|
-
kgId:
|
|
14633
|
-
cid:
|
|
14634
|
-
gcid:
|
|
14773
|
+
tree: z22.array(z22.unknown()),
|
|
14774
|
+
flat: z22.array(z22.unknown()),
|
|
14775
|
+
videos: z22.array(z22.unknown()),
|
|
14776
|
+
forums: z22.array(z22.unknown()),
|
|
14777
|
+
organicResults: z22.array(SerpIntelligenceOrganicResultSchema),
|
|
14778
|
+
localPack: z22.array(z22.unknown()),
|
|
14779
|
+
entityIds: z22.object({
|
|
14780
|
+
entities: z22.array(z22.object({
|
|
14781
|
+
name: z22.string(),
|
|
14782
|
+
kgId: z22.string().nullable(),
|
|
14783
|
+
cid: z22.string().nullable(),
|
|
14784
|
+
gcid: z22.string().nullable()
|
|
14635
14785
|
}).strict()),
|
|
14636
|
-
kgIds:
|
|
14637
|
-
cids:
|
|
14638
|
-
gcids:
|
|
14786
|
+
kgIds: z22.array(z22.string()),
|
|
14787
|
+
cids: z22.array(z22.string()),
|
|
14788
|
+
gcids: z22.array(z22.string())
|
|
14639
14789
|
}).strict(),
|
|
14640
|
-
stats:
|
|
14641
|
-
seed:
|
|
14642
|
-
totalQuestions:
|
|
14643
|
-
maxDepthReached:
|
|
14644
|
-
durationMs:
|
|
14645
|
-
errorCount:
|
|
14790
|
+
stats: z22.object({
|
|
14791
|
+
seed: z22.string(),
|
|
14792
|
+
totalQuestions: z22.number().int().min(0),
|
|
14793
|
+
maxDepthReached: z22.number().int().min(0),
|
|
14794
|
+
durationMs: z22.number().min(0),
|
|
14795
|
+
errorCount: z22.number().int().min(0)
|
|
14646
14796
|
}).strict(),
|
|
14647
|
-
diagnostics:
|
|
14648
|
-
completionStatus:
|
|
14649
|
-
problem:
|
|
14650
|
-
warnings:
|
|
14651
|
-
debug:
|
|
14797
|
+
diagnostics: z22.object({
|
|
14798
|
+
completionStatus: z22.enum(["paa_found", "no_paa", "serp_only"]),
|
|
14799
|
+
problem: z22.null(),
|
|
14800
|
+
warnings: z22.array(z22.unknown()).optional(),
|
|
14801
|
+
debug: z22.object({
|
|
14652
14802
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
14653
14803
|
}).passthrough().optional()
|
|
14654
14804
|
}).passthrough(),
|
|
14655
|
-
whatPeopleSaying:
|
|
14805
|
+
whatPeopleSaying: z22.array(z22.unknown())
|
|
14656
14806
|
}).strict();
|
|
14657
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
14658
|
-
attemptNumber:
|
|
14659
|
-
outcome:
|
|
14660
|
-
startedAt:
|
|
14661
|
-
completedAt:
|
|
14662
|
-
durationMs:
|
|
14663
|
-
problemCode:
|
|
14664
|
-
message:
|
|
14665
|
-
kernelSessionId:
|
|
14666
|
-
cleanupSucceeded:
|
|
14807
|
+
var SerpIntelligenceCaptureAttemptSchema = z22.object({
|
|
14808
|
+
attemptNumber: z22.number().int().min(1),
|
|
14809
|
+
outcome: z22.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
14810
|
+
startedAt: z22.string().optional(),
|
|
14811
|
+
completedAt: z22.string().optional(),
|
|
14812
|
+
durationMs: z22.number().min(0).optional(),
|
|
14813
|
+
problemCode: z22.string().optional(),
|
|
14814
|
+
message: z22.string().optional(),
|
|
14815
|
+
kernelSessionId: z22.string().nullable().optional(),
|
|
14816
|
+
cleanupSucceeded: z22.boolean().nullable().optional()
|
|
14667
14817
|
}).strict();
|
|
14668
|
-
var SerpPageSnapshotCaptureSchema =
|
|
14818
|
+
var SerpPageSnapshotCaptureSchema = z22.object({
|
|
14669
14819
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14670
14820
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
14671
14821
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14672
|
-
sourceKind:
|
|
14673
|
-
sourcePosition:
|
|
14674
|
-
status:
|
|
14675
|
-
fetchedVia:
|
|
14676
|
-
httpStatus:
|
|
14677
|
-
contentType:
|
|
14678
|
-
title:
|
|
14822
|
+
sourceKind: z22.enum(SerpPageSnapshotSourceKindValues),
|
|
14823
|
+
sourcePosition: z22.number().int().min(1).nullable(),
|
|
14824
|
+
status: z22.enum(SerpPageFetchStatusValues),
|
|
14825
|
+
fetchedVia: z22.enum(SerpPageFetchedViaValues).nullable(),
|
|
14826
|
+
httpStatus: z22.number().int().min(100).max(599).nullable(),
|
|
14827
|
+
contentType: z22.string().nullable(),
|
|
14828
|
+
title: z22.string().nullable(),
|
|
14679
14829
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14680
|
-
metaDescription:
|
|
14681
|
-
headings:
|
|
14682
|
-
level:
|
|
14683
|
-
text:
|
|
14830
|
+
metaDescription: z22.string().nullable(),
|
|
14831
|
+
headings: z22.array(z22.object({
|
|
14832
|
+
level: z22.number().int().min(1).max(6),
|
|
14833
|
+
text: z22.string()
|
|
14684
14834
|
}).strict()).default([]),
|
|
14685
|
-
artifact:
|
|
14686
|
-
htmlBlobUrl:
|
|
14687
|
-
textBlobUrl:
|
|
14688
|
-
markdownBlobUrl:
|
|
14689
|
-
screenshotBlobUrl:
|
|
14690
|
-
contentSha256:
|
|
14691
|
-
capturedAt:
|
|
14835
|
+
artifact: z22.object({
|
|
14836
|
+
htmlBlobUrl: z22.string().url().nullable(),
|
|
14837
|
+
textBlobUrl: z22.string().url().nullable(),
|
|
14838
|
+
markdownBlobUrl: z22.string().url().nullable(),
|
|
14839
|
+
screenshotBlobUrl: z22.string().url().nullable(),
|
|
14840
|
+
contentSha256: z22.string().nullable(),
|
|
14841
|
+
capturedAt: z22.string().nullable()
|
|
14692
14842
|
}).strict(),
|
|
14693
|
-
error:
|
|
14694
|
-
code:
|
|
14695
|
-
message:
|
|
14843
|
+
error: z22.object({
|
|
14844
|
+
code: z22.string(),
|
|
14845
|
+
message: z22.string()
|
|
14696
14846
|
}).strict().nullable()
|
|
14697
14847
|
}).strict();
|
|
14698
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
14848
|
+
var SerpIntelligenceCaptureResponseSchema = z22.object({
|
|
14699
14849
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
14700
|
-
attempts:
|
|
14850
|
+
attempts: z22.array(SerpIntelligenceCaptureAttemptSchema),
|
|
14701
14851
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
14702
|
-
pageSnapshotArtifacts:
|
|
14703
|
-
billing:
|
|
14704
|
-
creditsUsed:
|
|
14705
|
-
requestId:
|
|
14706
|
-
jobId:
|
|
14852
|
+
pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
|
|
14853
|
+
billing: z22.object({
|
|
14854
|
+
creditsUsed: z22.number().min(0).optional(),
|
|
14855
|
+
requestId: z22.string().optional(),
|
|
14856
|
+
jobId: z22.string().optional()
|
|
14707
14857
|
}).strict().optional()
|
|
14708
14858
|
}).strict();
|
|
14709
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
14710
|
-
pageSnapshotArtifacts:
|
|
14711
|
-
attempts:
|
|
14859
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z22.object({
|
|
14860
|
+
pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
|
|
14861
|
+
attempts: z22.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
14712
14862
|
}).strict();
|
|
14713
14863
|
|
|
14714
14864
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -14880,7 +15030,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
14880
15030
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
14881
15031
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
14882
15032
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
14883
|
-
var serpIntelligenceApp = new
|
|
15033
|
+
var serpIntelligenceApp = new Hono14();
|
|
14884
15034
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
14885
15035
|
function structuredError(input) {
|
|
14886
15036
|
return {
|
|
@@ -15049,7 +15199,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
15049
15199
|
});
|
|
15050
15200
|
|
|
15051
15201
|
// src/mcp/mcp-routes.ts
|
|
15052
|
-
import { Hono as
|
|
15202
|
+
import { Hono as Hono15 } from "hono";
|
|
15053
15203
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
15054
15204
|
|
|
15055
15205
|
// src/mcp/memory-inprocess-executor.ts
|
|
@@ -15576,7 +15726,7 @@ async function requireMcpCallerKey(c) {
|
|
|
15576
15726
|
if (!user) return mcpAuthError();
|
|
15577
15727
|
return { key: callerKey, user };
|
|
15578
15728
|
}
|
|
15579
|
-
var mcpApp = new
|
|
15729
|
+
var mcpApp = new Hono15();
|
|
15580
15730
|
mcpApp.all("/", async (c) => {
|
|
15581
15731
|
try {
|
|
15582
15732
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -15602,7 +15752,7 @@ mcpApp.all("/", async (c) => {
|
|
|
15602
15752
|
});
|
|
15603
15753
|
|
|
15604
15754
|
// src/api/browser-agent-routes.ts
|
|
15605
|
-
import { Hono as
|
|
15755
|
+
import { Hono as Hono16 } from "hono";
|
|
15606
15756
|
|
|
15607
15757
|
// src/api/browser-agent-db.ts
|
|
15608
15758
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -16993,7 +17143,7 @@ async function loadOpenSession(id, userId) {
|
|
|
16993
17143
|
return row;
|
|
16994
17144
|
}
|
|
16995
17145
|
function buildBrowserAgentRoutes() {
|
|
16996
|
-
const app2 = new
|
|
17146
|
+
const app2 = new Hono16();
|
|
16997
17147
|
app2.use("*", async (c, next) => {
|
|
16998
17148
|
await migrateBrowserAgent();
|
|
16999
17149
|
return next();
|
|
@@ -17727,9 +17877,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
17727
17877
|
|
|
17728
17878
|
// src/api/stripe-routes.ts
|
|
17729
17879
|
import Stripe from "stripe";
|
|
17730
|
-
import { Hono as
|
|
17880
|
+
import { Hono as Hono17 } from "hono";
|
|
17731
17881
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
17732
|
-
var stripeApp = new
|
|
17882
|
+
var stripeApp = new Hono17();
|
|
17733
17883
|
stripeApp.post("/webhooks", async (c) => {
|
|
17734
17884
|
const sig = c.req.header("stripe-signature");
|
|
17735
17885
|
const body = await c.req.text();
|
|
@@ -17833,7 +17983,7 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
17833
17983
|
}
|
|
17834
17984
|
|
|
17835
17985
|
// src/api/oauth-routes.ts
|
|
17836
|
-
import { Hono as
|
|
17986
|
+
import { Hono as Hono18 } from "hono";
|
|
17837
17987
|
import { getCookie, setCookie } from "hono/cookie";
|
|
17838
17988
|
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
|
|
17839
17989
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
@@ -18002,7 +18152,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
18002
18152
|
if (state) u.searchParams.set("state", state);
|
|
18003
18153
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
18004
18154
|
}
|
|
18005
|
-
var oauthApp = new
|
|
18155
|
+
var oauthApp = new Hono18();
|
|
18006
18156
|
oauthApp.use("*", async (c, next) => {
|
|
18007
18157
|
c.header("Access-Control-Allow-Origin", "*");
|
|
18008
18158
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -18256,8 +18406,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
18256
18406
|
});
|
|
18257
18407
|
|
|
18258
18408
|
// src/api/chat-routes.ts
|
|
18259
|
-
import { Hono as
|
|
18260
|
-
var chatApp = new
|
|
18409
|
+
import { Hono as Hono19 } from "hono";
|
|
18410
|
+
var chatApp = new Hono19();
|
|
18261
18411
|
function tokenOk(token) {
|
|
18262
18412
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18263
18413
|
}
|
|
@@ -18927,8 +19077,8 @@ loadChannels();
|
|
|
18927
19077
|
}
|
|
18928
19078
|
|
|
18929
19079
|
// src/api/schedule-routes.ts
|
|
18930
|
-
import { Hono as
|
|
18931
|
-
var scheduleApp = new
|
|
19080
|
+
import { Hono as Hono20 } from "hono";
|
|
19081
|
+
var scheduleApp = new Hono20();
|
|
18932
19082
|
function tokenOk2(token) {
|
|
18933
19083
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18934
19084
|
}
|
|
@@ -19683,7 +19833,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
19683
19833
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
19684
19834
|
return next();
|
|
19685
19835
|
});
|
|
19686
|
-
var app = new
|
|
19836
|
+
var app = new Hono21();
|
|
19687
19837
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
19688
19838
|
function requireStripeSecret() {
|
|
19689
19839
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -21096,6 +21246,7 @@ app.route("/instagram", instagramApp);
|
|
|
21096
21246
|
app.route("/reddit", redditApp);
|
|
21097
21247
|
app.route("/video", videoApp);
|
|
21098
21248
|
app.route("/maps", mapsApp);
|
|
21249
|
+
app.route("/trustpilot", trustpilotApp);
|
|
21099
21250
|
app.route("/directory", directoryApp);
|
|
21100
21251
|
app.route("/workflows", workflowApp);
|
|
21101
21252
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
@@ -21232,4 +21383,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
21232
21383
|
export {
|
|
21233
21384
|
app
|
|
21234
21385
|
};
|
|
21235
|
-
//# sourceMappingURL=server-
|
|
21386
|
+
//# sourceMappingURL=server-LBEYRKM2.js.map
|