mcp-scraper 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +3184 -2649
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +14 -6
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-22VEGGTW.js → chunk-DZBGULWW.js} +41 -1
- package/dist/chunk-DZBGULWW.js.map +1 -0
- package/dist/{chunk-UN7VMHZL.js → chunk-GMZFQRH2.js} +2 -2
- package/dist/{chunk-HL33CGJF.js → chunk-HXGZMMW5.js} +29 -2
- package/dist/chunk-HXGZMMW5.js.map +1 -0
- package/dist/{chunk-ZRKFW5FB.js → chunk-JBME5WJR.js} +2 -2
- package/dist/chunk-L4LMSXLO.js +7 -0
- package/dist/chunk-L4LMSXLO.js.map +1 -0
- package/dist/{chunk-EJK25QOW.js → chunk-TUSLATZY.js} +3 -3
- package/dist/{chunk-KE7KE2Q2.js → chunk-XHJJQRNQ.js} +12 -9
- package/dist/{chunk-KE7KE2Q2.js.map → chunk-XHJJQRNQ.js.map} +1 -1
- package/dist/{chunk-XDFSLSSH.js → chunk-ZJDANZTQ.js} +2 -2
- package/dist/{db-YHZYG7D2.js → db-EONGTNJM.js} +2 -2
- package/dist/{extract-bundle-OSUPAHCE.js → extract-bundle-HMCYUVBJ.js} +2 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-VDWIPV7F.js → server-PH3VJ74X.js} +914 -449
- package/dist/server-PH3VJ74X.js.map +1 -0
- package/dist/{site-extract-repository-GWKGK46Z.js → site-extract-repository-YNPXEZCB.js} +4 -4
- package/dist/{worker-O5PZTTPY.js → worker-45WEWCMQ.js} +6 -6
- package/docs/mcp-tool-manifest.generated.json +17 -7
- package/package.json +2 -2
- package/dist/chunk-22VEGGTW.js.map +0 -1
- package/dist/chunk-HL33CGJF.js.map +0 -1
- package/dist/chunk-PZB3TJWK.js +0 -7
- package/dist/chunk-PZB3TJWK.js.map +0 -1
- package/dist/server-VDWIPV7F.js.map +0 -1
- /package/dist/{chunk-UN7VMHZL.js.map → chunk-GMZFQRH2.js.map} +0 -0
- /package/dist/{chunk-ZRKFW5FB.js.map → chunk-JBME5WJR.js.map} +0 -0
- /package/dist/{chunk-EJK25QOW.js.map → chunk-TUSLATZY.js.map} +0 -0
- /package/dist/{chunk-XDFSLSSH.js.map → chunk-ZJDANZTQ.js.map} +0 -0
- /package/dist/{db-YHZYG7D2.js.map → db-EONGTNJM.js.map} +0 -0
- /package/dist/{extract-bundle-OSUPAHCE.js.map → extract-bundle-HMCYUVBJ.js.map} +0 -0
- /package/dist/{site-extract-repository-GWKGK46Z.js.map → site-extract-repository-YNPXEZCB.js.map} +0 -0
- /package/dist/{worker-O5PZTTPY.js.map → worker-45WEWCMQ.js.map} +0 -0
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
createHarvestAttemptRecorder,
|
|
7
7
|
harvestProblemResponse,
|
|
8
8
|
serializeHarvestProblem
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JBME5WJR.js";
|
|
10
10
|
import {
|
|
11
11
|
BrowserDriver,
|
|
12
12
|
MapsSelectors,
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
deleteKernelProxyId,
|
|
15
15
|
harvest,
|
|
16
16
|
resolveKernelProxyId
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-ZJDANZTQ.js";
|
|
18
18
|
import {
|
|
19
19
|
HttpMcpToolExecutor,
|
|
20
20
|
MemoryMcpToolExecutor,
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
sanitizeAttempts,
|
|
36
36
|
sanitizeHarvestResult,
|
|
37
37
|
transcribeMediaUrl
|
|
38
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-XHJJQRNQ.js";
|
|
39
39
|
import {
|
|
40
40
|
auditImages,
|
|
41
41
|
buildLinkReport,
|
|
@@ -53,7 +53,7 @@ import {
|
|
|
53
53
|
recordKernelSession,
|
|
54
54
|
recordVendorUsage,
|
|
55
55
|
runWithCostContext
|
|
56
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-GMZFQRH2.js";
|
|
57
57
|
import {
|
|
58
58
|
csvRecords,
|
|
59
59
|
listWorkflowDefinitions,
|
|
@@ -77,7 +77,7 @@ import {
|
|
|
77
77
|
RawMapsOverviewSchema,
|
|
78
78
|
RawMapsReviewStatsSchema
|
|
79
79
|
} from "./chunk-XGIPATLV.js";
|
|
80
|
-
import "./chunk-
|
|
80
|
+
import "./chunk-L4LMSXLO.js";
|
|
81
81
|
import {
|
|
82
82
|
completeExtractJob,
|
|
83
83
|
countSuccessfulPages,
|
|
@@ -88,7 +88,7 @@ import {
|
|
|
88
88
|
saveExtractPages,
|
|
89
89
|
setExtractJobTotal,
|
|
90
90
|
settleExtractJob
|
|
91
|
-
} from "./chunk-
|
|
91
|
+
} from "./chunk-TUSLATZY.js";
|
|
92
92
|
import {
|
|
93
93
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
94
94
|
CONCURRENCY_PRICE_ID,
|
|
@@ -103,15 +103,22 @@ import {
|
|
|
103
103
|
MEMORY_AI_MARGIN_MULTIPLE,
|
|
104
104
|
MEMORY_PLANS,
|
|
105
105
|
MEMORY_PLAN_QUOTA,
|
|
106
|
+
SCHEDULED_AGENT_LLM_MARKUP,
|
|
107
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
108
|
+
SCHEDULED_RUN_BASE_CREDITS,
|
|
109
|
+
SCHEDULED_RUN_BASE_MC,
|
|
110
|
+
SCHEDULED_RUN_DEFAULT_MAX_CREDITS,
|
|
106
111
|
SCHEDULING_PLANS,
|
|
107
112
|
SITE_AUDIT_LLM_MARGIN_MULTIPLE,
|
|
108
113
|
SUBSCRIPTION_TIERS,
|
|
109
114
|
SUBSCRIPTION_TIER_BY_KEY,
|
|
115
|
+
USD_NANOS,
|
|
110
116
|
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE,
|
|
111
117
|
browserActiveCostMc,
|
|
112
118
|
concurrencySlotBillingInfo,
|
|
113
|
-
insufficientBalanceResponse
|
|
114
|
-
|
|
119
|
+
insufficientBalanceResponse,
|
|
120
|
+
vendorCostUsdToMc
|
|
121
|
+
} from "./chunk-HXGZMMW5.js";
|
|
115
122
|
import {
|
|
116
123
|
CaptchaError,
|
|
117
124
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -194,7 +201,7 @@ import {
|
|
|
194
201
|
setUserMemoryCreds,
|
|
195
202
|
upsertPageSnapshot,
|
|
196
203
|
verifyPassword
|
|
197
|
-
} from "./chunk-
|
|
204
|
+
} from "./chunk-DZBGULWW.js";
|
|
198
205
|
|
|
199
206
|
// src/blog/registry.ts
|
|
200
207
|
var posts = [
|
|
@@ -5046,7 +5053,7 @@ async function extractSite(opts) {
|
|
|
5046
5053
|
}
|
|
5047
5054
|
|
|
5048
5055
|
// src/api/server.ts
|
|
5049
|
-
import { Hono as
|
|
5056
|
+
import { Hono as Hono24 } from "hono";
|
|
5050
5057
|
import { serve as serveInngest } from "inngest/hono";
|
|
5051
5058
|
|
|
5052
5059
|
// src/inngest/client.ts
|
|
@@ -7352,7 +7359,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
7352
7359
|
return buildSeoAudit(pages, job.startUrl);
|
|
7353
7360
|
}) : null;
|
|
7354
7361
|
const artifacts = await step.run("finalize", async () => {
|
|
7355
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
7362
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-HMCYUVBJ.js");
|
|
7356
7363
|
const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
|
|
7357
7364
|
await completeExtractJob(jobId, stored);
|
|
7358
7365
|
return stored;
|
|
@@ -7972,8 +7979,479 @@ internalMemoryApp.post("/ai-debit", async (c) => {
|
|
|
7972
7979
|
return c.json({ ok, mc, balance_mc, insufficient: !ok }, 200);
|
|
7973
7980
|
});
|
|
7974
7981
|
|
|
7975
|
-
// src/api/
|
|
7982
|
+
// src/api/internal-billing-routes.ts
|
|
7976
7983
|
import { Hono as Hono3 } from "hono";
|
|
7984
|
+
import { z as z10 } from "zod";
|
|
7985
|
+
|
|
7986
|
+
// src/api/unified-billing.ts
|
|
7987
|
+
import { createHash as createHash2, createHmac, randomUUID, timingSafeEqual } from "crypto";
|
|
7988
|
+
var SCHEDULED_RUN_BILLING_CLASS = "scheduled_agent_run";
|
|
7989
|
+
var SCHEDULED_RUN_SOURCE_SURFACE = "mastra_scheduler";
|
|
7990
|
+
var AUTHORIZATION_TTL_MS = 30 * 60 * 1e3;
|
|
7991
|
+
var UnifiedBillingError = class extends Error {
|
|
7992
|
+
constructor(code, message, httpStatus, details = {}) {
|
|
7993
|
+
super(message);
|
|
7994
|
+
this.code = code;
|
|
7995
|
+
this.httpStatus = httpStatus;
|
|
7996
|
+
this.details = details;
|
|
7997
|
+
this.name = "UnifiedBillingError";
|
|
7998
|
+
}
|
|
7999
|
+
code;
|
|
8000
|
+
httpStatus;
|
|
8001
|
+
details;
|
|
8002
|
+
};
|
|
8003
|
+
function billingSecret() {
|
|
8004
|
+
const secret2 = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
8005
|
+
if (!secret2) throw new UnifiedBillingError("unauthorized", "internal billing secret is not configured", 503);
|
|
8006
|
+
return secret2;
|
|
8007
|
+
}
|
|
8008
|
+
function normalizeIdentity(identity) {
|
|
8009
|
+
const normalized = identity.trim().toLowerCase();
|
|
8010
|
+
if (!normalized || normalized.length > 320) {
|
|
8011
|
+
throw new UnifiedBillingError("invalid_request", "identity is required", 400);
|
|
8012
|
+
}
|
|
8013
|
+
return normalized;
|
|
8014
|
+
}
|
|
8015
|
+
function normalizeIdempotencyKey(value) {
|
|
8016
|
+
const key = value.trim();
|
|
8017
|
+
if (!key || key.length > 500) {
|
|
8018
|
+
throw new UnifiedBillingError("invalid_request", "a bounded idempotency key is required", 400);
|
|
8019
|
+
}
|
|
8020
|
+
return key;
|
|
8021
|
+
}
|
|
8022
|
+
function tokenFor(id, idempotencyKey) {
|
|
8023
|
+
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey).digest("base64url");
|
|
8024
|
+
}
|
|
8025
|
+
function tokenHash(token) {
|
|
8026
|
+
return createHash2("sha256").update(token).digest("hex");
|
|
8027
|
+
}
|
|
8028
|
+
function verifyToken(row, token) {
|
|
8029
|
+
const expected = Buffer.from(row.token_hash, "hex");
|
|
8030
|
+
const actual = Buffer.from(tokenHash(token), "hex");
|
|
8031
|
+
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
8032
|
+
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
8033
|
+
}
|
|
8034
|
+
}
|
|
8035
|
+
function safeMetadata(value) {
|
|
8036
|
+
if (!value) return "{}";
|
|
8037
|
+
return JSON.stringify(value).slice(0, 4e3);
|
|
8038
|
+
}
|
|
8039
|
+
function mapAuthorization(row, balanceMc) {
|
|
8040
|
+
return {
|
|
8041
|
+
id: row.id,
|
|
8042
|
+
token: tokenFor(row.id, row.idempotency_key),
|
|
8043
|
+
status: row.status,
|
|
8044
|
+
minimumMc: Number(row.minimum_amount_mc),
|
|
8045
|
+
maximumMc: Number(row.maximum_amount_mc),
|
|
8046
|
+
minimumCredits: Number(row.minimum_amount_mc) / MC_PER_CREDIT,
|
|
8047
|
+
maximumCredits: Number(row.maximum_amount_mc) / MC_PER_CREDIT,
|
|
8048
|
+
balanceMc,
|
|
8049
|
+
expiresAt: row.expires_at,
|
|
8050
|
+
reservationMode: "balance_check"
|
|
8051
|
+
};
|
|
8052
|
+
}
|
|
8053
|
+
async function authorizationById(id) {
|
|
8054
|
+
const result = await getDb().execute({
|
|
8055
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE id = ? LIMIT 1",
|
|
8056
|
+
args: [id]
|
|
8057
|
+
});
|
|
8058
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
8059
|
+
}
|
|
8060
|
+
async function eventByKey(idempotencyKey) {
|
|
8061
|
+
const result = await getDb().execute({
|
|
8062
|
+
sql: "SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",
|
|
8063
|
+
args: [idempotencyKey]
|
|
8064
|
+
});
|
|
8065
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
8066
|
+
}
|
|
8067
|
+
function mapEvent(row, balanceMc, duplicate) {
|
|
8068
|
+
let reason;
|
|
8069
|
+
if (row.metadata) {
|
|
8070
|
+
try {
|
|
8071
|
+
const metadata = JSON.parse(row.metadata);
|
|
8072
|
+
if (typeof metadata.reason === "string") reason = metadata.reason;
|
|
8073
|
+
} catch {
|
|
8074
|
+
}
|
|
8075
|
+
}
|
|
8076
|
+
const rawNanos = row.raw_cost_usd_nanos == null ? null : Number(row.raw_cost_usd_nanos);
|
|
8077
|
+
const modelMc = Math.max(0, Number(row.amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
8078
|
+
return {
|
|
8079
|
+
id: row.id,
|
|
8080
|
+
authorizationId: row.authorization_id,
|
|
8081
|
+
status: row.status === "settled" ? "settled" : "cost_pending",
|
|
8082
|
+
amountMc: Number(row.amount_mc),
|
|
8083
|
+
credits: Number(row.amount_mc) / MC_PER_CREDIT,
|
|
8084
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8085
|
+
modelMc,
|
|
8086
|
+
rawCostUsd: rawNanos == null ? null : rawNanos / USD_NANOS,
|
|
8087
|
+
multiplierBps: Number(row.multiplier_bps ?? SCHEDULED_AGENT_LLM_MARKUP_BPS),
|
|
8088
|
+
balanceMc,
|
|
8089
|
+
...reason ? { reason } : {},
|
|
8090
|
+
duplicate
|
|
8091
|
+
};
|
|
8092
|
+
}
|
|
8093
|
+
async function authorizeScheduledRun(args) {
|
|
8094
|
+
const identity = normalizeIdentity(args.identity);
|
|
8095
|
+
const idempotencyKey = normalizeIdempotencyKey(args.idempotencyKey);
|
|
8096
|
+
const user = await getUserByEmail(identity);
|
|
8097
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8098
|
+
const pending = await getDb().execute({
|
|
8099
|
+
sql: "SELECT id FROM billing_events WHERE user_id = ? AND billing_class = ? AND status = 'cost_pending' LIMIT 1",
|
|
8100
|
+
args: [user.id, SCHEDULED_RUN_BILLING_CLASS]
|
|
8101
|
+
});
|
|
8102
|
+
if (pending.rows.length > 0) {
|
|
8103
|
+
throw new UnifiedBillingError("pending_cost", "a previous scheduled run has an unresolved cost", 409);
|
|
8104
|
+
}
|
|
8105
|
+
const existing = await getDb().execute({
|
|
8106
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
8107
|
+
args: [idempotencyKey]
|
|
8108
|
+
});
|
|
8109
|
+
if (existing.rows[0]) {
|
|
8110
|
+
const row = existing.rows[0];
|
|
8111
|
+
if (Number(row.user_id) !== user.id) {
|
|
8112
|
+
throw new UnifiedBillingError(
|
|
8113
|
+
"invalid_request",
|
|
8114
|
+
"the idempotency key is already in use by another billing identity",
|
|
8115
|
+
409
|
|
8116
|
+
);
|
|
8117
|
+
}
|
|
8118
|
+
const balanceMc2 = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8119
|
+
return mapAuthorization(row, balanceMc2);
|
|
8120
|
+
}
|
|
8121
|
+
const maximumCredits = args.maximumCredits ?? SCHEDULED_RUN_DEFAULT_MAX_CREDITS;
|
|
8122
|
+
if (!Number.isFinite(maximumCredits) || maximumCredits < SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT) {
|
|
8123
|
+
throw new UnifiedBillingError("invalid_request", "maximumCredits must cover the scheduled-run base charge", 400);
|
|
8124
|
+
}
|
|
8125
|
+
const maximumMc = Math.ceil(maximumCredits * MC_PER_CREDIT);
|
|
8126
|
+
const balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8127
|
+
if (balanceMc < SCHEDULED_RUN_BASE_MC) {
|
|
8128
|
+
throw new UnifiedBillingError(
|
|
8129
|
+
"insufficient_credits",
|
|
8130
|
+
"the wallet cannot cover the scheduled-run base charge",
|
|
8131
|
+
402,
|
|
8132
|
+
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
8133
|
+
);
|
|
8134
|
+
}
|
|
8135
|
+
const id = randomUUID();
|
|
8136
|
+
const token = tokenFor(id, idempotencyKey);
|
|
8137
|
+
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
8138
|
+
const inserted = await getDb().execute({
|
|
8139
|
+
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
8140
|
+
args: [
|
|
8141
|
+
id,
|
|
8142
|
+
user.id,
|
|
8143
|
+
idempotencyKey,
|
|
8144
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8145
|
+
SCHEDULED_RUN_BASE_MC,
|
|
8146
|
+
maximumMc,
|
|
8147
|
+
tokenHash(token),
|
|
8148
|
+
safeMetadata(args.metadata),
|
|
8149
|
+
expiresAt
|
|
8150
|
+
]
|
|
8151
|
+
});
|
|
8152
|
+
if (inserted.rowsAffected === 0) {
|
|
8153
|
+
const raced = await getDb().execute({
|
|
8154
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
8155
|
+
args: [idempotencyKey]
|
|
8156
|
+
});
|
|
8157
|
+
if (!raced.rows[0]) throw new Error("billing authorization insert raced without a readable winner");
|
|
8158
|
+
const row = raced.rows[0];
|
|
8159
|
+
if (Number(row.user_id) !== user.id) {
|
|
8160
|
+
throw new UnifiedBillingError(
|
|
8161
|
+
"invalid_request",
|
|
8162
|
+
"the idempotency key is already in use by another billing identity",
|
|
8163
|
+
409
|
|
8164
|
+
);
|
|
8165
|
+
}
|
|
8166
|
+
return mapAuthorization(row, balanceMc);
|
|
8167
|
+
}
|
|
8168
|
+
return {
|
|
8169
|
+
id,
|
|
8170
|
+
token,
|
|
8171
|
+
status: "authorized",
|
|
8172
|
+
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
8173
|
+
maximumMc,
|
|
8174
|
+
minimumCredits: SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT,
|
|
8175
|
+
maximumCredits,
|
|
8176
|
+
balanceMc,
|
|
8177
|
+
expiresAt,
|
|
8178
|
+
reservationMode: "balance_check"
|
|
8179
|
+
};
|
|
8180
|
+
}
|
|
8181
|
+
async function settleScheduledRun(args) {
|
|
8182
|
+
const identity = normalizeIdentity(args.identity);
|
|
8183
|
+
const user = await getUserByEmail(identity);
|
|
8184
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8185
|
+
const authorization = await authorizationById(args.authorizationId);
|
|
8186
|
+
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
8187
|
+
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
8188
|
+
}
|
|
8189
|
+
verifyToken(authorization, args.authorizationToken);
|
|
8190
|
+
const settlementKey = authorization.id + ":settlement";
|
|
8191
|
+
const existing = await eventByKey(settlementKey);
|
|
8192
|
+
if (existing) {
|
|
8193
|
+
const balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8194
|
+
if (existing.status === "settling") {
|
|
8195
|
+
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement is already in progress", 409);
|
|
8196
|
+
}
|
|
8197
|
+
return mapEvent(existing, balanceMc, true);
|
|
8198
|
+
}
|
|
8199
|
+
if (authorization.status !== "authorized") {
|
|
8200
|
+
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer open", 409);
|
|
8201
|
+
}
|
|
8202
|
+
if (Date.parse(authorization.expires_at) <= Date.now()) {
|
|
8203
|
+
await getDb().execute({
|
|
8204
|
+
sql: "UPDATE billing_authorizations SET status = 'expired' WHERE id = ? AND status = 'authorized'",
|
|
8205
|
+
args: [authorization.id]
|
|
8206
|
+
});
|
|
8207
|
+
throw new UnifiedBillingError("authorization_expired", "billing authorization expired before settlement", 409);
|
|
8208
|
+
}
|
|
8209
|
+
const rawCostUsd = args.rawCostUsd ?? null;
|
|
8210
|
+
if (args.modelCostStatus === "reported" && (rawCostUsd == null || !Number.isFinite(rawCostUsd) || rawCostUsd < 0)) {
|
|
8211
|
+
throw new UnifiedBillingError("invalid_request", "reported model cost must be a finite non-negative number", 400);
|
|
8212
|
+
}
|
|
8213
|
+
if (args.modelCostStatus === "not_applicable" && rawCostUsd != null && rawCostUsd !== 0) {
|
|
8214
|
+
throw new UnifiedBillingError("invalid_request", "not_applicable model cost cannot include a non-zero cost", 400);
|
|
8215
|
+
}
|
|
8216
|
+
const eventId = randomUUID();
|
|
8217
|
+
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
8218
|
+
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
8219
|
+
const totalMc = SCHEDULED_RUN_BASE_MC + modelMc;
|
|
8220
|
+
const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : totalMc > Number(authorization.maximum_amount_mc) ? "max_credits_per_run_exceeded" : null;
|
|
8221
|
+
if (pendingReason) {
|
|
8222
|
+
await getDb().execute({
|
|
8223
|
+
sql: "INSERT INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'cost_pending', ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
8224
|
+
args: [
|
|
8225
|
+
eventId,
|
|
8226
|
+
user.id,
|
|
8227
|
+
authorization.id,
|
|
8228
|
+
settlementKey,
|
|
8229
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8230
|
+
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
8231
|
+
totalMc,
|
|
8232
|
+
rawCostNanos,
|
|
8233
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8234
|
+
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
8235
|
+
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
8236
|
+
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
8237
|
+
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
8238
|
+
safeMetadata({ ...args.metadata, reason: pendingReason })
|
|
8239
|
+
]
|
|
8240
|
+
});
|
|
8241
|
+
await getDb().execute({
|
|
8242
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending', settled_at = datetime('now') WHERE id = ?",
|
|
8243
|
+
args: [authorization.id]
|
|
8244
|
+
});
|
|
8245
|
+
return {
|
|
8246
|
+
id: eventId,
|
|
8247
|
+
authorizationId: authorization.id,
|
|
8248
|
+
status: "cost_pending",
|
|
8249
|
+
amountMc: totalMc,
|
|
8250
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8251
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8252
|
+
modelMc,
|
|
8253
|
+
rawCostUsd,
|
|
8254
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8255
|
+
balanceMc: await reconcileBalanceMc(user.id, user.balance_mc),
|
|
8256
|
+
reason: pendingReason,
|
|
8257
|
+
duplicate: false
|
|
8258
|
+
};
|
|
8259
|
+
}
|
|
8260
|
+
const claimed = await getDb().execute({
|
|
8261
|
+
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'settling', ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
8262
|
+
args: [
|
|
8263
|
+
eventId,
|
|
8264
|
+
user.id,
|
|
8265
|
+
authorization.id,
|
|
8266
|
+
settlementKey,
|
|
8267
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8268
|
+
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
8269
|
+
totalMc,
|
|
8270
|
+
rawCostNanos,
|
|
8271
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8272
|
+
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
8273
|
+
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
8274
|
+
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
8275
|
+
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
8276
|
+
safeMetadata(args.metadata)
|
|
8277
|
+
]
|
|
8278
|
+
});
|
|
8279
|
+
if (claimed.rowsAffected === 0) {
|
|
8280
|
+
const raced = await eventByKey(settlementKey);
|
|
8281
|
+
if (!raced) throw new Error("billing settlement raced without a readable winner");
|
|
8282
|
+
if (raced.status === "settling") {
|
|
8283
|
+
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement is already in progress", 409);
|
|
8284
|
+
}
|
|
8285
|
+
return mapEvent(raced, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
8286
|
+
}
|
|
8287
|
+
const debit = await debitMc(
|
|
8288
|
+
user.id,
|
|
8289
|
+
totalMc,
|
|
8290
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
8291
|
+
"scheduled-run:" + authorization.id
|
|
8292
|
+
);
|
|
8293
|
+
if (!debit.ok) {
|
|
8294
|
+
const metadata = safeMetadata({ ...args.metadata, reason: "insufficient_credits_at_settlement" });
|
|
8295
|
+
await getDb().execute({
|
|
8296
|
+
sql: "UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE id = ?",
|
|
8297
|
+
args: [metadata, eventId]
|
|
8298
|
+
});
|
|
8299
|
+
await getDb().execute({
|
|
8300
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending', settled_at = datetime('now') WHERE id = ?",
|
|
8301
|
+
args: [authorization.id]
|
|
8302
|
+
});
|
|
8303
|
+
return {
|
|
8304
|
+
id: eventId,
|
|
8305
|
+
authorizationId: authorization.id,
|
|
8306
|
+
status: "cost_pending",
|
|
8307
|
+
amountMc: totalMc,
|
|
8308
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8309
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8310
|
+
modelMc,
|
|
8311
|
+
rawCostUsd,
|
|
8312
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8313
|
+
balanceMc: debit.balance_mc,
|
|
8314
|
+
reason: "insufficient_credits_at_settlement",
|
|
8315
|
+
duplicate: false
|
|
8316
|
+
};
|
|
8317
|
+
}
|
|
8318
|
+
await getDb().execute({
|
|
8319
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
8320
|
+
args: [eventId]
|
|
8321
|
+
});
|
|
8322
|
+
await getDb().execute({
|
|
8323
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
8324
|
+
args: [authorization.id]
|
|
8325
|
+
});
|
|
8326
|
+
return {
|
|
8327
|
+
id: eventId,
|
|
8328
|
+
authorizationId: authorization.id,
|
|
8329
|
+
status: "settled",
|
|
8330
|
+
amountMc: totalMc,
|
|
8331
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8332
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8333
|
+
modelMc,
|
|
8334
|
+
rawCostUsd,
|
|
8335
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8336
|
+
balanceMc: debit.balance_mc,
|
|
8337
|
+
duplicate: false
|
|
8338
|
+
};
|
|
8339
|
+
}
|
|
8340
|
+
async function voidScheduledRunAuthorization(args) {
|
|
8341
|
+
const identity = normalizeIdentity(args.identity);
|
|
8342
|
+
const user = await getUserByEmail(identity);
|
|
8343
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8344
|
+
const authorization = await authorizationById(args.authorizationId);
|
|
8345
|
+
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
8346
|
+
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
8347
|
+
}
|
|
8348
|
+
verifyToken(authorization, args.authorizationToken);
|
|
8349
|
+
if (authorization.status === "authorized") {
|
|
8350
|
+
await getDb().execute({
|
|
8351
|
+
sql: "UPDATE billing_authorizations SET status = 'void', metadata = ?, settled_at = datetime('now') WHERE id = ? AND status = 'authorized'",
|
|
8352
|
+
args: [safeMetadata({ reason: args.reason ?? "voided_before_execution" }), authorization.id]
|
|
8353
|
+
});
|
|
8354
|
+
return { ok: true, status: "void" };
|
|
8355
|
+
}
|
|
8356
|
+
return { ok: true, status: authorization.status };
|
|
8357
|
+
}
|
|
8358
|
+
|
|
8359
|
+
// src/api/internal-billing-routes.ts
|
|
8360
|
+
var internalBillingApp = new Hono3();
|
|
8361
|
+
var metadataSchema = z10.record(z10.unknown()).optional();
|
|
8362
|
+
var authorizeSchema = z10.object({
|
|
8363
|
+
identity: z10.string().min(1).max(320),
|
|
8364
|
+
idempotencyKey: z10.string().min(1).max(500),
|
|
8365
|
+
maximumCredits: z10.number().finite().positive().optional(),
|
|
8366
|
+
metadata: metadataSchema
|
|
8367
|
+
});
|
|
8368
|
+
var settleSchema = z10.object({
|
|
8369
|
+
identity: z10.string().min(1).max(320),
|
|
8370
|
+
authorizationId: z10.string().uuid(),
|
|
8371
|
+
authorizationToken: z10.string().min(20).max(500),
|
|
8372
|
+
modelCostStatus: z10.enum(["reported", "missing", "not_applicable"]),
|
|
8373
|
+
rawCostUsd: z10.number().finite().nonnegative().nullable().optional(),
|
|
8374
|
+
inputTokens: z10.number().finite().nonnegative().optional(),
|
|
8375
|
+
outputTokens: z10.number().finite().nonnegative().optional(),
|
|
8376
|
+
cachedTokens: z10.number().finite().nonnegative().optional(),
|
|
8377
|
+
modelSteps: z10.number().finite().nonnegative().optional(),
|
|
8378
|
+
metadata: metadataSchema
|
|
8379
|
+
});
|
|
8380
|
+
var voidSchema = z10.object({
|
|
8381
|
+
identity: z10.string().min(1).max(320),
|
|
8382
|
+
authorizationId: z10.string().uuid(),
|
|
8383
|
+
authorizationToken: z10.string().min(20).max(500),
|
|
8384
|
+
reason: z10.string().max(500).optional()
|
|
8385
|
+
});
|
|
8386
|
+
function internalRequestAuthorized(header) {
|
|
8387
|
+
const expected = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
8388
|
+
return Boolean(expected && header && header === expected);
|
|
8389
|
+
}
|
|
8390
|
+
function billingError(error) {
|
|
8391
|
+
if (error instanceof UnifiedBillingError) {
|
|
8392
|
+
return {
|
|
8393
|
+
body: {
|
|
8394
|
+
ok: false,
|
|
8395
|
+
error_code: error.code,
|
|
8396
|
+
error: error.message,
|
|
8397
|
+
...error.details
|
|
8398
|
+
},
|
|
8399
|
+
status: error.httpStatus
|
|
8400
|
+
};
|
|
8401
|
+
}
|
|
8402
|
+
console.error("[internal-billing] unexpected error", error instanceof Error ? error.message : error);
|
|
8403
|
+
return {
|
|
8404
|
+
body: { ok: false, error_code: "internal_error", error: "internal billing operation failed" },
|
|
8405
|
+
status: 500
|
|
8406
|
+
};
|
|
8407
|
+
}
|
|
8408
|
+
internalBillingApp.use("*", async (c, next) => {
|
|
8409
|
+
if (!internalRequestAuthorized(c.req.header("x-internal-secret"))) {
|
|
8410
|
+
return c.json({ ok: false, error_code: "unauthorized", error: "unauthorized" }, 401);
|
|
8411
|
+
}
|
|
8412
|
+
await next();
|
|
8413
|
+
});
|
|
8414
|
+
internalBillingApp.post("/scheduled-run/authorize", async (c) => {
|
|
8415
|
+
const parsed = authorizeSchema.safeParse(await c.req.json().catch(() => null));
|
|
8416
|
+
if (!parsed.success) {
|
|
8417
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid billing authorization request" }, 400);
|
|
8418
|
+
}
|
|
8419
|
+
try {
|
|
8420
|
+
const authorization = await authorizeScheduledRun(parsed.data);
|
|
8421
|
+
return c.json({ ok: true, authorization });
|
|
8422
|
+
} catch (error) {
|
|
8423
|
+
const response = billingError(error);
|
|
8424
|
+
return c.json(response.body, response.status);
|
|
8425
|
+
}
|
|
8426
|
+
});
|
|
8427
|
+
internalBillingApp.post("/scheduled-run/settle", async (c) => {
|
|
8428
|
+
const parsed = settleSchema.safeParse(await c.req.json().catch(() => null));
|
|
8429
|
+
if (!parsed.success) {
|
|
8430
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid scheduled-run settlement request" }, 400);
|
|
8431
|
+
}
|
|
8432
|
+
try {
|
|
8433
|
+
const settlement = await settleScheduledRun(parsed.data);
|
|
8434
|
+
return c.json({ ok: true, settlement });
|
|
8435
|
+
} catch (error) {
|
|
8436
|
+
const response = billingError(error);
|
|
8437
|
+
return c.json(response.body, response.status);
|
|
8438
|
+
}
|
|
8439
|
+
});
|
|
8440
|
+
internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
8441
|
+
const parsed = voidSchema.safeParse(await c.req.json().catch(() => null));
|
|
8442
|
+
if (!parsed.success) {
|
|
8443
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid billing void request" }, 400);
|
|
8444
|
+
}
|
|
8445
|
+
try {
|
|
8446
|
+
return c.json(await voidScheduledRunAuthorization(parsed.data));
|
|
8447
|
+
} catch (error) {
|
|
8448
|
+
const response = billingError(error);
|
|
8449
|
+
return c.json(response.body, response.status);
|
|
8450
|
+
}
|
|
8451
|
+
});
|
|
8452
|
+
|
|
8453
|
+
// src/api/youtube-routes.ts
|
|
8454
|
+
import { Hono as Hono4 } from "hono";
|
|
7977
8455
|
|
|
7978
8456
|
// src/youtube/youtube-harvest.ts
|
|
7979
8457
|
import { promises as fs } from "fs";
|
|
@@ -7981,17 +8459,17 @@ import path4 from "path";
|
|
|
7981
8459
|
import Papa from "papaparse";
|
|
7982
8460
|
|
|
7983
8461
|
// src/youtube/schemas.ts
|
|
7984
|
-
import { z as
|
|
7985
|
-
var YouTubeHarvestOptionsSchema =
|
|
7986
|
-
mode:
|
|
7987
|
-
query:
|
|
7988
|
-
channelHandle:
|
|
7989
|
-
maxVideos:
|
|
7990
|
-
downloadMp3:
|
|
7991
|
-
outputDir:
|
|
7992
|
-
kernelApiKey:
|
|
7993
|
-
kernelProxyId:
|
|
7994
|
-
headless:
|
|
8462
|
+
import { z as z11 } from "zod";
|
|
8463
|
+
var YouTubeHarvestOptionsSchema = z11.object({
|
|
8464
|
+
mode: z11.enum(["search", "channel"]),
|
|
8465
|
+
query: z11.string().optional(),
|
|
8466
|
+
channelHandle: z11.string().optional(),
|
|
8467
|
+
maxVideos: z11.number().int().min(1).max(5e3).default(100),
|
|
8468
|
+
downloadMp3: z11.boolean().default(false),
|
|
8469
|
+
outputDir: z11.string().default("./yt-output"),
|
|
8470
|
+
kernelApiKey: z11.string().optional(),
|
|
8471
|
+
kernelProxyId: z11.string().optional(),
|
|
8472
|
+
headless: z11.boolean().default(false)
|
|
7995
8473
|
}).refine(
|
|
7996
8474
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
7997
8475
|
{ message: "search mode requires query; channel mode requires channelHandle" }
|
|
@@ -8777,68 +9255,68 @@ function createApiKeyAuth() {
|
|
|
8777
9255
|
}
|
|
8778
9256
|
|
|
8779
9257
|
// src/api/server-schemas.ts
|
|
8780
|
-
import { z as
|
|
8781
|
-
var HarvestBodySchema =
|
|
8782
|
-
query:
|
|
8783
|
-
location:
|
|
8784
|
-
depth:
|
|
8785
|
-
maxQuestions:
|
|
8786
|
-
callback_url:
|
|
8787
|
-
gl:
|
|
8788
|
-
hl:
|
|
8789
|
-
device:
|
|
8790
|
-
proxyMode:
|
|
8791
|
-
proxyZip:
|
|
8792
|
-
debug:
|
|
8793
|
-
serpOnly:
|
|
8794
|
-
pages:
|
|
8795
|
-
});
|
|
8796
|
-
var ExtractUrlBodySchema = z11.object({
|
|
8797
|
-
url: z11.string().min(1, "url is required"),
|
|
8798
|
-
screenshot: z11.boolean().optional(),
|
|
8799
|
-
screenshotDevice: z11.enum(["desktop", "mobile"]).optional(),
|
|
8800
|
-
extractBranding: z11.boolean().optional(),
|
|
8801
|
-
downloadMedia: z11.boolean().optional(),
|
|
8802
|
-
mediaTypes: z11.array(z11.enum(["image", "video", "audio"])).optional(),
|
|
8803
|
-
allowLocal: z11.boolean().optional(),
|
|
8804
|
-
depositToVault: z11.boolean().optional(),
|
|
8805
|
-
vaultName: z11.string().trim().min(1).max(120).optional()
|
|
8806
|
-
});
|
|
8807
|
-
var DiffPageBodySchema = z11.object({
|
|
8808
|
-
url: z11.string().min(1, "url is required"),
|
|
8809
|
-
allowLocal: z11.boolean().optional(),
|
|
8810
|
-
resetBaseline: z11.boolean().optional()
|
|
8811
|
-
});
|
|
8812
|
-
var MapUrlsBodySchema = z11.object({
|
|
8813
|
-
url: z11.string().min(1, "url is required"),
|
|
8814
|
-
maxUrls: z11.number().int().min(1).max(2e3).optional(),
|
|
8815
|
-
concurrency: z11.number().int().min(1).max(20).optional(),
|
|
8816
|
-
browserFallback: z11.boolean().optional(),
|
|
8817
|
-
kernelFallback: z11.boolean().optional()
|
|
8818
|
-
});
|
|
8819
|
-
var ExtractSiteBodySchema = z11.object({
|
|
8820
|
-
url: z11.string().min(1, "url is required"),
|
|
8821
|
-
maxPages: z11.number().int().min(1).max(1e4).optional(),
|
|
8822
|
-
browserFallback: z11.boolean().optional(),
|
|
8823
|
-
kernelFallback: z11.boolean().optional(),
|
|
8824
|
-
rotateProxies: z11.boolean().optional(),
|
|
8825
|
-
rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
|
|
8826
|
-
background: z11.boolean().optional(),
|
|
8827
|
-
downloadImages: z11.boolean().optional(),
|
|
8828
|
-
formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
8829
|
-
});
|
|
8830
|
-
var YoutubeHarvestBodySchema = z11.object({
|
|
8831
|
-
mode: z11.enum(["search", "channel"]),
|
|
8832
|
-
query: z11.string().optional(),
|
|
8833
|
-
channelHandle: z11.string().optional(),
|
|
8834
|
-
maxVideos: z11.number().int().min(1).max(500).optional()
|
|
9258
|
+
import { z as z12 } from "zod";
|
|
9259
|
+
var HarvestBodySchema = z12.object({
|
|
9260
|
+
query: z12.string().min(1, "query is required"),
|
|
9261
|
+
location: z12.string().optional(),
|
|
9262
|
+
depth: z12.number().int().min(1).max(4).optional(),
|
|
9263
|
+
maxQuestions: z12.number().int().min(1).optional(),
|
|
9264
|
+
callback_url: z12.string().url().optional(),
|
|
9265
|
+
gl: z12.string().optional(),
|
|
9266
|
+
hl: z12.string().optional(),
|
|
9267
|
+
device: z12.enum(["desktop", "mobile"]).optional(),
|
|
9268
|
+
proxyMode: z12.enum(["location", "configured", "none"]).optional(),
|
|
9269
|
+
proxyZip: z12.string().regex(/^\d{5}$/).optional(),
|
|
9270
|
+
debug: z12.boolean().optional(),
|
|
9271
|
+
serpOnly: z12.boolean().optional(),
|
|
9272
|
+
pages: z12.number().int().min(1).max(2).optional()
|
|
8835
9273
|
});
|
|
8836
|
-
var
|
|
8837
|
-
|
|
9274
|
+
var ExtractUrlBodySchema = z12.object({
|
|
9275
|
+
url: z12.string().min(1, "url is required"),
|
|
9276
|
+
screenshot: z12.boolean().optional(),
|
|
9277
|
+
screenshotDevice: z12.enum(["desktop", "mobile"]).optional(),
|
|
9278
|
+
extractBranding: z12.boolean().optional(),
|
|
9279
|
+
downloadMedia: z12.boolean().optional(),
|
|
9280
|
+
mediaTypes: z12.array(z12.enum(["image", "video", "audio"])).optional(),
|
|
9281
|
+
allowLocal: z12.boolean().optional(),
|
|
9282
|
+
depositToVault: z12.boolean().optional(),
|
|
9283
|
+
vaultName: z12.string().trim().min(1).max(120).optional()
|
|
9284
|
+
});
|
|
9285
|
+
var DiffPageBodySchema = z12.object({
|
|
9286
|
+
url: z12.string().min(1, "url is required"),
|
|
9287
|
+
allowLocal: z12.boolean().optional(),
|
|
9288
|
+
resetBaseline: z12.boolean().optional()
|
|
9289
|
+
});
|
|
9290
|
+
var MapUrlsBodySchema = z12.object({
|
|
9291
|
+
url: z12.string().min(1, "url is required"),
|
|
9292
|
+
maxUrls: z12.number().int().min(1).max(2e3).optional(),
|
|
9293
|
+
concurrency: z12.number().int().min(1).max(20).optional(),
|
|
9294
|
+
browserFallback: z12.boolean().optional(),
|
|
9295
|
+
kernelFallback: z12.boolean().optional()
|
|
9296
|
+
});
|
|
9297
|
+
var ExtractSiteBodySchema = z12.object({
|
|
9298
|
+
url: z12.string().min(1, "url is required"),
|
|
9299
|
+
maxPages: z12.number().int().min(1).max(1e4).optional(),
|
|
9300
|
+
browserFallback: z12.boolean().optional(),
|
|
9301
|
+
kernelFallback: z12.boolean().optional(),
|
|
9302
|
+
rotateProxies: z12.boolean().optional(),
|
|
9303
|
+
rotateProxyEvery: z12.number().int().min(1).max(100).optional(),
|
|
9304
|
+
background: z12.boolean().optional(),
|
|
9305
|
+
downloadImages: z12.boolean().optional(),
|
|
9306
|
+
formats: z12.array(z12.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
9307
|
+
});
|
|
9308
|
+
var YoutubeHarvestBodySchema = z12.object({
|
|
9309
|
+
mode: z12.enum(["search", "channel"]),
|
|
9310
|
+
query: z12.string().optional(),
|
|
9311
|
+
channelHandle: z12.string().optional(),
|
|
9312
|
+
maxVideos: z12.number().int().min(1).max(500).optional()
|
|
9313
|
+
});
|
|
9314
|
+
var YoutubeTranscribeBodySchema = z12.object({
|
|
9315
|
+
videoId: z12.string().min(1, "videoId is required")
|
|
8838
9316
|
});
|
|
8839
9317
|
|
|
8840
9318
|
// src/api/concurrency-gates.ts
|
|
8841
|
-
import { randomUUID } from "crypto";
|
|
9319
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
8842
9320
|
var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
8843
9321
|
var DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
8844
9322
|
var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
@@ -8872,7 +9350,7 @@ function lockTtlModifier(ttlSeconds) {
|
|
|
8872
9350
|
const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
|
|
8873
9351
|
return `+${ttl} seconds`;
|
|
8874
9352
|
}
|
|
8875
|
-
function
|
|
9353
|
+
function safeMetadata2(metadata) {
|
|
8876
9354
|
if (!metadata) return null;
|
|
8877
9355
|
try {
|
|
8878
9356
|
return JSON.stringify(metadata).slice(0, 4e3);
|
|
@@ -8956,7 +9434,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
8956
9434
|
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
8957
9435
|
}
|
|
8958
9436
|
await expireConcurrencyLocksForUser(user.id);
|
|
8959
|
-
const lockId = `cl_${
|
|
9437
|
+
const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
|
|
8960
9438
|
const res = await getDb().execute({
|
|
8961
9439
|
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
8962
9440
|
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
@@ -8978,7 +9456,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
8978
9456
|
user.id,
|
|
8979
9457
|
operation,
|
|
8980
9458
|
lockTtlModifier(options.ttlSeconds),
|
|
8981
|
-
|
|
9459
|
+
safeMetadata2(options.metadata),
|
|
8982
9460
|
user.id,
|
|
8983
9461
|
user.id,
|
|
8984
9462
|
limit
|
|
@@ -9012,7 +9490,7 @@ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECON
|
|
|
9012
9490
|
}
|
|
9013
9491
|
|
|
9014
9492
|
// src/api/youtube-routes.ts
|
|
9015
|
-
var youtubeApp = new
|
|
9493
|
+
var youtubeApp = new Hono4();
|
|
9016
9494
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
9017
9495
|
const raw = await c.req.json().catch(() => ({}));
|
|
9018
9496
|
const parsed = YoutubeHarvestBodySchema.safeParse(raw);
|
|
@@ -9171,14 +9649,14 @@ function fmtTs(secs) {
|
|
|
9171
9649
|
}
|
|
9172
9650
|
|
|
9173
9651
|
// src/api/screenshot-routes.ts
|
|
9174
|
-
import { Hono as
|
|
9175
|
-
import { z as
|
|
9176
|
-
var ScreenshotBodySchema =
|
|
9177
|
-
url:
|
|
9178
|
-
device:
|
|
9179
|
-
allowLocal:
|
|
9652
|
+
import { Hono as Hono5 } from "hono";
|
|
9653
|
+
import { z as z13 } from "zod";
|
|
9654
|
+
var ScreenshotBodySchema = z13.object({
|
|
9655
|
+
url: z13.string().trim().min(1, "url is required"),
|
|
9656
|
+
device: z13.string().trim().optional(),
|
|
9657
|
+
allowLocal: z13.boolean().optional()
|
|
9180
9658
|
});
|
|
9181
|
-
var screenshotApp = new
|
|
9659
|
+
var screenshotApp = new Hono5();
|
|
9182
9660
|
screenshotApp.use("*", createApiKeyAuth());
|
|
9183
9661
|
screenshotApp.post("/", async (c) => {
|
|
9184
9662
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -9235,8 +9713,8 @@ screenshotApp.post("/", async (c) => {
|
|
|
9235
9713
|
});
|
|
9236
9714
|
|
|
9237
9715
|
// src/api/facebook-ad-routes.ts
|
|
9238
|
-
import { Hono as
|
|
9239
|
-
import { z as
|
|
9716
|
+
import { Hono as Hono6 } from "hono";
|
|
9717
|
+
import { z as z14 } from "zod";
|
|
9240
9718
|
|
|
9241
9719
|
// src/extractor/FacebookAdExtractor.ts
|
|
9242
9720
|
var SPRITE_Y_TO_PLATFORM = {
|
|
@@ -10129,35 +10607,35 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
10129
10607
|
|
|
10130
10608
|
// src/api/facebook-ad-routes.ts
|
|
10131
10609
|
import { fal } from "@fal-ai/client";
|
|
10132
|
-
var FacebookAdBodySchema =
|
|
10133
|
-
url:
|
|
10134
|
-
libraryId:
|
|
10135
|
-
openModal:
|
|
10610
|
+
var FacebookAdBodySchema = z14.object({
|
|
10611
|
+
url: z14.string().trim().optional(),
|
|
10612
|
+
libraryId: z14.string().trim().optional(),
|
|
10613
|
+
openModal: z14.boolean().optional()
|
|
10136
10614
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
10137
|
-
var FacebookPageIntelBodySchema =
|
|
10138
|
-
pageId:
|
|
10139
|
-
query:
|
|
10140
|
-
libraryId:
|
|
10141
|
-
maxAds:
|
|
10142
|
-
country:
|
|
10615
|
+
var FacebookPageIntelBodySchema = z14.object({
|
|
10616
|
+
pageId: z14.string().trim().optional(),
|
|
10617
|
+
query: z14.string().trim().optional(),
|
|
10618
|
+
libraryId: z14.string().trim().optional(),
|
|
10619
|
+
maxAds: z14.number().int().min(1).max(200).optional(),
|
|
10620
|
+
country: z14.string().trim().toUpperCase().optional()
|
|
10143
10621
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
10144
10622
|
message: "pageId, libraryId, or query is required"
|
|
10145
10623
|
});
|
|
10146
|
-
var FacebookTranscribeBodySchema =
|
|
10147
|
-
videoUrl:
|
|
10624
|
+
var FacebookTranscribeBodySchema = z14.object({
|
|
10625
|
+
videoUrl: z14.string().trim().min(1, "videoUrl is required")
|
|
10148
10626
|
});
|
|
10149
|
-
var FacebookVideoTranscribeBodySchema =
|
|
10150
|
-
url:
|
|
10151
|
-
quality:
|
|
10627
|
+
var FacebookVideoTranscribeBodySchema = z14.object({
|
|
10628
|
+
url: z14.string().trim().min(1, "url is required"),
|
|
10629
|
+
quality: z14.enum(["best", "hd", "sd"]).default("best")
|
|
10152
10630
|
});
|
|
10153
|
-
var FacebookSearchBodySchema =
|
|
10154
|
-
query:
|
|
10155
|
-
country:
|
|
10156
|
-
maxResults:
|
|
10631
|
+
var FacebookSearchBodySchema = z14.object({
|
|
10632
|
+
query: z14.string().trim().min(1, "query is required"),
|
|
10633
|
+
country: z14.string().trim().toUpperCase().optional(),
|
|
10634
|
+
maxResults: z14.number().int().min(1).max(20).optional()
|
|
10157
10635
|
});
|
|
10158
|
-
var FacebookMediaBodySchema =
|
|
10159
|
-
url:
|
|
10160
|
-
filename:
|
|
10636
|
+
var FacebookMediaBodySchema = z14.object({
|
|
10637
|
+
url: z14.string().trim().min(1, "url is required"),
|
|
10638
|
+
filename: z14.string().trim().optional()
|
|
10161
10639
|
});
|
|
10162
10640
|
function invalidRequest(message) {
|
|
10163
10641
|
return { error_code: "invalid_request", message };
|
|
@@ -10239,7 +10717,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
10239
10717
|
}
|
|
10240
10718
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10241
10719
|
}
|
|
10242
|
-
var facebookAdApp = new
|
|
10720
|
+
var facebookAdApp = new Hono6();
|
|
10243
10721
|
facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
|
|
10244
10722
|
const raw = await c.req.json().catch(() => ({}));
|
|
10245
10723
|
const parsed = FacebookAdBodySchema.safeParse(raw);
|
|
@@ -10583,8 +11061,8 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
|
|
|
10583
11061
|
});
|
|
10584
11062
|
|
|
10585
11063
|
// src/api/google-ads-routes.ts
|
|
10586
|
-
import { Hono as
|
|
10587
|
-
import { z as
|
|
11064
|
+
import { Hono as Hono7 } from "hono";
|
|
11065
|
+
import { z as z15 } from "zod";
|
|
10588
11066
|
|
|
10589
11067
|
// src/extractor/GoogleAdsExtractor.ts
|
|
10590
11068
|
var ORIGIN = "https://adstransparency.google.com";
|
|
@@ -10754,19 +11232,19 @@ var GoogleAdsExtractor = class {
|
|
|
10754
11232
|
|
|
10755
11233
|
// src/api/google-ads-routes.ts
|
|
10756
11234
|
import { fal as fal2 } from "@fal-ai/client";
|
|
10757
|
-
var GoogleAdsSearchBodySchema =
|
|
10758
|
-
query:
|
|
10759
|
-
region:
|
|
10760
|
-
maxResults:
|
|
11235
|
+
var GoogleAdsSearchBodySchema = z15.object({
|
|
11236
|
+
query: z15.string().trim().min(1, "query is required"),
|
|
11237
|
+
region: z15.string().trim().toUpperCase().length(2).optional(),
|
|
11238
|
+
maxResults: z15.number().int().min(1).max(20).optional()
|
|
10761
11239
|
});
|
|
10762
|
-
var GoogleAdsPageIntelBodySchema =
|
|
10763
|
-
advertiserId:
|
|
10764
|
-
domain:
|
|
10765
|
-
region:
|
|
10766
|
-
maxAds:
|
|
11240
|
+
var GoogleAdsPageIntelBodySchema = z15.object({
|
|
11241
|
+
advertiserId: z15.string().trim().optional(),
|
|
11242
|
+
domain: z15.string().trim().optional(),
|
|
11243
|
+
region: z15.string().trim().toUpperCase().length(2).optional(),
|
|
11244
|
+
maxAds: z15.number().int().min(1).max(200).optional()
|
|
10767
11245
|
}).refine((d) => !!d.advertiserId || !!d.domain, { message: "advertiserId or domain is required" });
|
|
10768
|
-
var GoogleAdsTranscribeBodySchema =
|
|
10769
|
-
videoUrl:
|
|
11246
|
+
var GoogleAdsTranscribeBodySchema = z15.object({
|
|
11247
|
+
videoUrl: z15.string().trim().min(1, "videoUrl is required")
|
|
10770
11248
|
});
|
|
10771
11249
|
function invalidRequest2(message) {
|
|
10772
11250
|
return { error_code: "invalid_request", message };
|
|
@@ -10792,7 +11270,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
10792
11270
|
function isBlockedMessage(msg) {
|
|
10793
11271
|
return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
|
|
10794
11272
|
}
|
|
10795
|
-
var googleAdsApp = new
|
|
11273
|
+
var googleAdsApp = new Hono7();
|
|
10796
11274
|
googleAdsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
10797
11275
|
const raw = await c.req.json().catch(() => ({}));
|
|
10798
11276
|
const parsed = GoogleAdsSearchBodySchema.safeParse(raw);
|
|
@@ -10908,8 +11386,8 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10908
11386
|
});
|
|
10909
11387
|
|
|
10910
11388
|
// src/api/instagram-routes.ts
|
|
10911
|
-
import { Hono as
|
|
10912
|
-
import { z as
|
|
11389
|
+
import { Hono as Hono8 } from "hono";
|
|
11390
|
+
import { z as z16 } from "zod";
|
|
10913
11391
|
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
10914
11392
|
import { tmpdir } from "os";
|
|
10915
11393
|
import { join as join3 } from "path";
|
|
@@ -11271,32 +11749,32 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
11271
11749
|
}
|
|
11272
11750
|
|
|
11273
11751
|
// src/api/instagram-routes.ts
|
|
11274
|
-
var InstagramProfileContentBodySchema =
|
|
11275
|
-
handle:
|
|
11276
|
-
url:
|
|
11277
|
-
profile:
|
|
11278
|
-
saveProfileChanges:
|
|
11279
|
-
save_profile_changes:
|
|
11280
|
-
maxItems:
|
|
11281
|
-
maxScrolls:
|
|
11282
|
-
scrollDelayMs:
|
|
11283
|
-
stableScrollLimit:
|
|
11752
|
+
var InstagramProfileContentBodySchema = z16.object({
|
|
11753
|
+
handle: z16.string().trim().optional(),
|
|
11754
|
+
url: z16.string().trim().optional(),
|
|
11755
|
+
profile: z16.string().trim().min(1).optional(),
|
|
11756
|
+
saveProfileChanges: z16.boolean().optional(),
|
|
11757
|
+
save_profile_changes: z16.boolean().optional(),
|
|
11758
|
+
maxItems: z16.number().int().min(1).max(2e3).default(50),
|
|
11759
|
+
maxScrolls: z16.number().int().min(0).max(250).default(10),
|
|
11760
|
+
scrollDelayMs: z16.number().int().min(250).max(5e3).default(1200),
|
|
11761
|
+
stableScrollLimit: z16.number().int().min(1).max(10).default(4)
|
|
11284
11762
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
11285
11763
|
message: "handle or url is required"
|
|
11286
11764
|
});
|
|
11287
|
-
var InstagramMediaTypeSchema =
|
|
11288
|
-
var InstagramMediaDownloadBodySchema =
|
|
11289
|
-
url:
|
|
11290
|
-
profile:
|
|
11291
|
-
saveProfileChanges:
|
|
11292
|
-
save_profile_changes:
|
|
11293
|
-
mediaTypes:
|
|
11294
|
-
downloadMedia:
|
|
11295
|
-
downloadAllTracks:
|
|
11296
|
-
includeTranscript:
|
|
11297
|
-
mux:
|
|
11298
|
-
});
|
|
11299
|
-
var instagramApp = new
|
|
11765
|
+
var InstagramMediaTypeSchema = z16.enum(["image", "video", "audio"]);
|
|
11766
|
+
var InstagramMediaDownloadBodySchema = z16.object({
|
|
11767
|
+
url: z16.string().trim().min(1, "url is required"),
|
|
11768
|
+
profile: z16.string().trim().min(1).optional(),
|
|
11769
|
+
saveProfileChanges: z16.boolean().optional(),
|
|
11770
|
+
save_profile_changes: z16.boolean().optional(),
|
|
11771
|
+
mediaTypes: z16.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
11772
|
+
downloadMedia: z16.boolean().default(true),
|
|
11773
|
+
downloadAllTracks: z16.boolean().default(false),
|
|
11774
|
+
includeTranscript: z16.boolean().default(false),
|
|
11775
|
+
mux: z16.boolean().default(true)
|
|
11776
|
+
});
|
|
11777
|
+
var instagramApp = new Hono8();
|
|
11300
11778
|
function invalidRequest3(message) {
|
|
11301
11779
|
return { error_code: "invalid_request", message };
|
|
11302
11780
|
}
|
|
@@ -11576,11 +12054,11 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11576
12054
|
});
|
|
11577
12055
|
|
|
11578
12056
|
// src/api/reddit-routes.ts
|
|
11579
|
-
import { Hono as
|
|
11580
|
-
import { z as
|
|
11581
|
-
var RedditThreadBodySchema =
|
|
11582
|
-
url:
|
|
11583
|
-
maxComments:
|
|
12057
|
+
import { Hono as Hono9 } from "hono";
|
|
12058
|
+
import { z as z17 } from "zod";
|
|
12059
|
+
var RedditThreadBodySchema = z17.object({
|
|
12060
|
+
url: z17.string().trim().min(1, "url is required"),
|
|
12061
|
+
maxComments: z17.number().int().min(1).max(2e3).optional()
|
|
11584
12062
|
});
|
|
11585
12063
|
function invalidRequest4(message) {
|
|
11586
12064
|
return { error_code: "invalid_request", message };
|
|
@@ -11636,7 +12114,7 @@ var PARSE_REDDIT = `(() => {
|
|
|
11636
12114
|
});
|
|
11637
12115
|
return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
|
|
11638
12116
|
})()`;
|
|
11639
|
-
var redditApp = new
|
|
12117
|
+
var redditApp = new Hono9();
|
|
11640
12118
|
redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
11641
12119
|
const raw = await c.req.json().catch(() => ({}));
|
|
11642
12120
|
const parsed = RedditThreadBodySchema.safeParse(raw);
|
|
@@ -11718,14 +12196,14 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
11718
12196
|
});
|
|
11719
12197
|
|
|
11720
12198
|
// src/api/video-routes.ts
|
|
11721
|
-
import { Hono as
|
|
11722
|
-
import { z as
|
|
12199
|
+
import { Hono as Hono10 } from "hono";
|
|
12200
|
+
import { z as z18 } from "zod";
|
|
11723
12201
|
|
|
11724
12202
|
// src/api/memory.ts
|
|
11725
12203
|
import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
|
|
11726
12204
|
|
|
11727
12205
|
// src/api/session.ts
|
|
11728
|
-
import { createHmac, timingSafeEqual } from "crypto";
|
|
12206
|
+
import { createHmac as createHmac2, timingSafeEqual as timingSafeEqual2 } from "crypto";
|
|
11729
12207
|
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
11730
12208
|
function getSessionSecret() {
|
|
11731
12209
|
const configured = process.env.SESSION_SECRET?.trim();
|
|
@@ -11737,14 +12215,14 @@ var secret = () => getSessionSecret();
|
|
|
11737
12215
|
function safeEqualHex(a, b) {
|
|
11738
12216
|
if (a.length !== b.length) return false;
|
|
11739
12217
|
try {
|
|
11740
|
-
return
|
|
12218
|
+
return timingSafeEqual2(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
11741
12219
|
} catch {
|
|
11742
12220
|
return false;
|
|
11743
12221
|
}
|
|
11744
12222
|
}
|
|
11745
12223
|
function signSession(userId) {
|
|
11746
12224
|
const payload = String(userId);
|
|
11747
|
-
const sig =
|
|
12225
|
+
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
11748
12226
|
return `${payload}.${sig}`;
|
|
11749
12227
|
}
|
|
11750
12228
|
function verifySession(token) {
|
|
@@ -11752,7 +12230,7 @@ function verifySession(token) {
|
|
|
11752
12230
|
if (dot === -1) return null;
|
|
11753
12231
|
const payload = token.slice(0, dot);
|
|
11754
12232
|
const sig = token.slice(dot + 1);
|
|
11755
|
-
const expected =
|
|
12233
|
+
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
11756
12234
|
if (!safeEqualHex(sig, expected)) return null;
|
|
11757
12235
|
const id = parseInt(payload);
|
|
11758
12236
|
return isNaN(id) ? null : id;
|
|
@@ -11898,13 +12376,13 @@ async function syncScheduleEntitlement(user, enabled, quotaPerPeriod) {
|
|
|
11898
12376
|
}
|
|
11899
12377
|
|
|
11900
12378
|
// src/api/video-routes.ts
|
|
11901
|
-
var videoApp = new
|
|
11902
|
-
var AnalyzeBodySchema =
|
|
11903
|
-
sourceUrl:
|
|
11904
|
-
intervalS:
|
|
11905
|
-
maxFrames:
|
|
11906
|
-
detail:
|
|
11907
|
-
vault:
|
|
12379
|
+
var videoApp = new Hono10();
|
|
12380
|
+
var AnalyzeBodySchema = z18.object({
|
|
12381
|
+
sourceUrl: z18.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
12382
|
+
intervalS: z18.number().min(1).max(30).optional(),
|
|
12383
|
+
maxFrames: z18.number().int().min(1).max(480).optional(),
|
|
12384
|
+
detail: z18.enum(["fast", "standard", "deep"]).optional(),
|
|
12385
|
+
vault: z18.string().trim().min(1).optional()
|
|
11908
12386
|
});
|
|
11909
12387
|
function invalidRequest5(message) {
|
|
11910
12388
|
return { error_code: "invalid_request", message };
|
|
@@ -12008,7 +12486,7 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
|
|
|
12008
12486
|
});
|
|
12009
12487
|
|
|
12010
12488
|
// src/api/maps-routes.ts
|
|
12011
|
-
import { Hono as
|
|
12489
|
+
import { Hono as Hono11 } from "hono";
|
|
12012
12490
|
|
|
12013
12491
|
// src/extractor/MapsNavigator.ts
|
|
12014
12492
|
var MapsNavigator = class {
|
|
@@ -12210,7 +12688,7 @@ var MapsReviewCollector = class {
|
|
|
12210
12688
|
};
|
|
12211
12689
|
|
|
12212
12690
|
// src/extractor/MapsExtractor.ts
|
|
12213
|
-
import { z as
|
|
12691
|
+
import { z as z19 } from "zod";
|
|
12214
12692
|
var MapsExtractor = class {
|
|
12215
12693
|
constructor(driver) {
|
|
12216
12694
|
this.driver = driver;
|
|
@@ -12418,7 +12896,7 @@ var MapsExtractor = class {
|
|
|
12418
12896
|
});
|
|
12419
12897
|
return rows;
|
|
12420
12898
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
12421
|
-
const result =
|
|
12899
|
+
const result = z19.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
12422
12900
|
if (!result.success) {
|
|
12423
12901
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
12424
12902
|
return [];
|
|
@@ -12482,7 +12960,7 @@ var MapsExtractor = class {
|
|
|
12482
12960
|
});
|
|
12483
12961
|
return results;
|
|
12484
12962
|
});
|
|
12485
|
-
const result =
|
|
12963
|
+
const result = z19.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
12486
12964
|
if (!result.success) {
|
|
12487
12965
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
12488
12966
|
return [];
|
|
@@ -12933,7 +13411,7 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
12933
13411
|
attempts: rotationError?.attempts ?? void 0
|
|
12934
13412
|
}, retryable ? 503 : 500);
|
|
12935
13413
|
}
|
|
12936
|
-
var mapsApp = new
|
|
13414
|
+
var mapsApp = new Hono11();
|
|
12937
13415
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12938
13416
|
const user = c.get("user");
|
|
12939
13417
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13071,11 +13549,11 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
13071
13549
|
});
|
|
13072
13550
|
|
|
13073
13551
|
// src/api/trustpilot-routes.ts
|
|
13074
|
-
import { Hono as
|
|
13075
|
-
import { z as
|
|
13076
|
-
var TrustpilotReviewsBodySchema =
|
|
13077
|
-
domain:
|
|
13078
|
-
maxPages:
|
|
13552
|
+
import { Hono as Hono12 } from "hono";
|
|
13553
|
+
import { z as z20 } from "zod";
|
|
13554
|
+
var TrustpilotReviewsBodySchema = z20.object({
|
|
13555
|
+
domain: z20.string().trim().min(1, "domain is required"),
|
|
13556
|
+
maxPages: z20.number().int().min(1).max(50).default(5)
|
|
13079
13557
|
});
|
|
13080
13558
|
function invalidRequest6(message) {
|
|
13081
13559
|
return { error_code: "invalid_request", message };
|
|
@@ -13140,7 +13618,7 @@ async function extractReviewCards(page) {
|
|
|
13140
13618
|
});
|
|
13141
13619
|
});
|
|
13142
13620
|
}
|
|
13143
|
-
var trustpilotApp = new
|
|
13621
|
+
var trustpilotApp = new Hono12();
|
|
13144
13622
|
trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13145
13623
|
const raw = await c.req.json().catch(() => ({}));
|
|
13146
13624
|
const parsed = TrustpilotReviewsBodySchema.safeParse(raw);
|
|
@@ -13221,11 +13699,11 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13221
13699
|
});
|
|
13222
13700
|
|
|
13223
13701
|
// src/api/g2-routes.ts
|
|
13224
|
-
import { Hono as
|
|
13225
|
-
import { z as
|
|
13226
|
-
var G2ReviewsBodySchema =
|
|
13227
|
-
product:
|
|
13228
|
-
maxPages:
|
|
13702
|
+
import { Hono as Hono13 } from "hono";
|
|
13703
|
+
import { z as z21 } from "zod";
|
|
13704
|
+
var G2ReviewsBodySchema = z21.object({
|
|
13705
|
+
product: z21.string().trim().min(1, "product is required"),
|
|
13706
|
+
maxPages: z21.number().int().min(1).max(50).default(5)
|
|
13229
13707
|
});
|
|
13230
13708
|
function invalidRequest7(message) {
|
|
13231
13709
|
return { error_code: "invalid_request", message };
|
|
@@ -13330,7 +13808,7 @@ async function extractG2ReviewCards(page) {
|
|
|
13330
13808
|
});
|
|
13331
13809
|
});
|
|
13332
13810
|
}
|
|
13333
|
-
var g2App = new
|
|
13811
|
+
var g2App = new Hono13();
|
|
13334
13812
|
g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13335
13813
|
const raw = await c.req.json().catch(() => ({}));
|
|
13336
13814
|
const parsed = G2ReviewsBodySchema.safeParse(raw);
|
|
@@ -13427,12 +13905,12 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13427
13905
|
});
|
|
13428
13906
|
|
|
13429
13907
|
// src/api/directory-routes.ts
|
|
13430
|
-
import { Hono as
|
|
13908
|
+
import { Hono as Hono14 } from "hono";
|
|
13431
13909
|
|
|
13432
13910
|
// src/directory/directory-workflow.ts
|
|
13433
13911
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
13434
13912
|
import { join as join4 } from "path";
|
|
13435
|
-
import { z as
|
|
13913
|
+
import { z as z22 } from "zod";
|
|
13436
13914
|
|
|
13437
13915
|
// src/directory/location-db.ts
|
|
13438
13916
|
import { access, readFile } from "fs/promises";
|
|
@@ -13593,24 +14071,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
13593
14071
|
}
|
|
13594
14072
|
|
|
13595
14073
|
// src/directory/directory-workflow.ts
|
|
13596
|
-
var DirectoryWorkflowOptionsSchema =
|
|
13597
|
-
query:
|
|
13598
|
-
state:
|
|
13599
|
-
minPopulation:
|
|
13600
|
-
populationYear:
|
|
13601
|
-
maxCities:
|
|
13602
|
-
maxResultsPerCity:
|
|
13603
|
-
concurrency:
|
|
13604
|
-
includeZipGroups:
|
|
13605
|
-
usZipsCsvPath:
|
|
13606
|
-
saveCsv:
|
|
13607
|
-
gl:
|
|
13608
|
-
hl:
|
|
13609
|
-
proxyMode:
|
|
13610
|
-
proxyZip:
|
|
13611
|
-
debug:
|
|
13612
|
-
headless:
|
|
13613
|
-
kernelApiKey:
|
|
14074
|
+
var DirectoryWorkflowOptionsSchema = z22.object({
|
|
14075
|
+
query: z22.string().min(1),
|
|
14076
|
+
state: z22.string().min(2).default("TN"),
|
|
14077
|
+
minPopulation: z22.number().int().min(0).default(1e5),
|
|
14078
|
+
populationYear: z22.union(POPULATION_YEARS.map((year) => z22.literal(year))).default(2025),
|
|
14079
|
+
maxCities: z22.number().int().min(1).max(100).default(25),
|
|
14080
|
+
maxResultsPerCity: z22.number().int().min(1).max(50).default(50),
|
|
14081
|
+
concurrency: z22.number().int().min(1).max(5).default(5),
|
|
14082
|
+
includeZipGroups: z22.boolean().default(true),
|
|
14083
|
+
usZipsCsvPath: z22.string().optional(),
|
|
14084
|
+
saveCsv: z22.boolean().default(true),
|
|
14085
|
+
gl: z22.string().length(2).default("us"),
|
|
14086
|
+
hl: z22.string().length(2).default("en"),
|
|
14087
|
+
proxyMode: z22.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
14088
|
+
proxyZip: z22.string().regex(/^\d{5}$/).optional(),
|
|
14089
|
+
debug: z22.boolean().default(false),
|
|
14090
|
+
headless: z22.boolean().default(true),
|
|
14091
|
+
kernelApiKey: z22.string().optional()
|
|
13614
14092
|
});
|
|
13615
14093
|
function errorMessage(err) {
|
|
13616
14094
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -13831,7 +14309,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
13831
14309
|
}
|
|
13832
14310
|
|
|
13833
14311
|
// src/api/directory-routes.ts
|
|
13834
|
-
var directoryApp = new
|
|
14312
|
+
var directoryApp = new Hono14();
|
|
13835
14313
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
13836
14314
|
const user = c.get("user");
|
|
13837
14315
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13902,37 +14380,37 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
13902
14380
|
});
|
|
13903
14381
|
|
|
13904
14382
|
// src/api/workflow-routes.ts
|
|
13905
|
-
import { createHmac as
|
|
14383
|
+
import { createHmac as createHmac3 } from "crypto";
|
|
13906
14384
|
import { readFile as readFile2 } from "fs/promises";
|
|
13907
|
-
import { Hono as
|
|
13908
|
-
import { z as
|
|
13909
|
-
var workflowApp = new
|
|
13910
|
-
var WorkflowInputSchema =
|
|
13911
|
-
var WorkflowIdSchema =
|
|
13912
|
-
var CadenceSchema =
|
|
13913
|
-
var ScheduleStatusSchema =
|
|
13914
|
-
var RunBodySchema =
|
|
14385
|
+
import { Hono as Hono15 } from "hono";
|
|
14386
|
+
import { z as z23 } from "zod";
|
|
14387
|
+
var workflowApp = new Hono15();
|
|
14388
|
+
var WorkflowInputSchema = z23.record(z23.unknown()).default({});
|
|
14389
|
+
var WorkflowIdSchema = z23.string().min(1);
|
|
14390
|
+
var CadenceSchema = z23.enum(["daily", "weekly", "monthly"]);
|
|
14391
|
+
var ScheduleStatusSchema = z23.enum(["active", "paused"]);
|
|
14392
|
+
var RunBodySchema = z23.object({
|
|
13915
14393
|
workflowId: WorkflowIdSchema,
|
|
13916
14394
|
input: WorkflowInputSchema,
|
|
13917
|
-
webhookUrl:
|
|
14395
|
+
webhookUrl: z23.string().url().optional()
|
|
13918
14396
|
});
|
|
13919
|
-
var ScheduleCreateSchema =
|
|
14397
|
+
var ScheduleCreateSchema = z23.object({
|
|
13920
14398
|
workflowId: WorkflowIdSchema,
|
|
13921
|
-
name:
|
|
14399
|
+
name: z23.string().min(1).max(120).optional(),
|
|
13922
14400
|
input: WorkflowInputSchema,
|
|
13923
14401
|
cadence: CadenceSchema.default("weekly"),
|
|
13924
|
-
timezone:
|
|
13925
|
-
webhookUrl:
|
|
13926
|
-
nextRunAt:
|
|
14402
|
+
timezone: z23.string().min(1).max(64).default("UTC"),
|
|
14403
|
+
webhookUrl: z23.string().url().optional(),
|
|
14404
|
+
nextRunAt: z23.string().datetime().optional()
|
|
13927
14405
|
});
|
|
13928
|
-
var SchedulePatchSchema =
|
|
13929
|
-
name:
|
|
14406
|
+
var SchedulePatchSchema = z23.object({
|
|
14407
|
+
name: z23.string().min(1).max(120).optional(),
|
|
13930
14408
|
status: ScheduleStatusSchema.optional(),
|
|
13931
14409
|
input: WorkflowInputSchema.optional(),
|
|
13932
14410
|
cadence: CadenceSchema.optional(),
|
|
13933
|
-
timezone:
|
|
13934
|
-
webhookUrl:
|
|
13935
|
-
nextRunAt:
|
|
14411
|
+
timezone: z23.string().min(1).max(64).optional(),
|
|
14412
|
+
webhookUrl: z23.string().url().nullable().optional(),
|
|
14413
|
+
nextRunAt: z23.string().datetime().nullable().optional()
|
|
13936
14414
|
});
|
|
13937
14415
|
function hostedWorkflowOutputDir() {
|
|
13938
14416
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -13990,7 +14468,7 @@ async function readManifestFromSummary(summary) {
|
|
|
13990
14468
|
function webhookSignature(body, timestamp) {
|
|
13991
14469
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
13992
14470
|
if (!secret2) return null;
|
|
13993
|
-
return
|
|
14471
|
+
return createHmac3("sha256", secret2).update(`${timestamp}.${body}`).digest("hex");
|
|
13994
14472
|
}
|
|
13995
14473
|
async function deliverWorkflowWebhook(input) {
|
|
13996
14474
|
if (!input.webhookUrl) return;
|
|
@@ -14436,17 +14914,17 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
14436
14914
|
});
|
|
14437
14915
|
|
|
14438
14916
|
// src/api/serp-intelligence-routes.ts
|
|
14439
|
-
import { Hono as
|
|
14917
|
+
import { Hono as Hono16 } from "hono";
|
|
14440
14918
|
|
|
14441
14919
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14442
|
-
import { createHash as
|
|
14920
|
+
import { createHash as createHash3 } from "crypto";
|
|
14443
14921
|
import pLimit3 from "p-limit";
|
|
14444
14922
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
14445
14923
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
14446
14924
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
14447
14925
|
function sha256(value) {
|
|
14448
14926
|
if (!value) return null;
|
|
14449
|
-
return
|
|
14927
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
14450
14928
|
}
|
|
14451
14929
|
function countWords(markdown) {
|
|
14452
14930
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -14748,7 +15226,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14748
15226
|
}
|
|
14749
15227
|
|
|
14750
15228
|
// src/serp-intelligence/schemas.ts
|
|
14751
|
-
import { z as
|
|
15229
|
+
import { z as z24 } from "zod";
|
|
14752
15230
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
14753
15231
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
14754
15232
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -14810,171 +15288,171 @@ function isPublicHttpUrl(value) {
|
|
|
14810
15288
|
return false;
|
|
14811
15289
|
}
|
|
14812
15290
|
}
|
|
14813
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
14814
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
14815
|
-
query:
|
|
14816
|
-
location:
|
|
14817
|
-
gl:
|
|
14818
|
-
hl:
|
|
14819
|
-
device:
|
|
14820
|
-
proxyMode:
|
|
14821
|
-
proxyZip:
|
|
14822
|
-
pages:
|
|
14823
|
-
debug:
|
|
14824
|
-
includePageSnapshots:
|
|
14825
|
-
pageSnapshotLimit:
|
|
15291
|
+
var SerpIntelligencePublicHttpUrlSchema = z24.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
15292
|
+
var SerpIntelligenceCaptureBodySchema = z24.object({
|
|
15293
|
+
query: z24.string().trim().min(1, "query is required"),
|
|
15294
|
+
location: z24.string().trim().min(1).optional(),
|
|
15295
|
+
gl: z24.string().trim().length(2).default("us"),
|
|
15296
|
+
hl: z24.string().trim().length(2).default("en"),
|
|
15297
|
+
device: z24.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
15298
|
+
proxyMode: z24.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
15299
|
+
proxyZip: z24.string().regex(/^\d{5}$/).optional(),
|
|
15300
|
+
pages: z24.number().int().min(1).max(2).default(1),
|
|
15301
|
+
debug: z24.boolean().default(false),
|
|
15302
|
+
includePageSnapshots: z24.boolean().default(false),
|
|
15303
|
+
pageSnapshotLimit: z24.number().int().min(0).max(10).default(0)
|
|
14826
15304
|
}).strict();
|
|
14827
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
15305
|
+
var SerpIntelligencePageSnapshotRequestSchema = z24.object({
|
|
14828
15306
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14829
|
-
sourceKind:
|
|
14830
|
-
sourcePosition:
|
|
15307
|
+
sourceKind: z24.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
15308
|
+
sourcePosition: z24.number().int().min(1).optional()
|
|
14831
15309
|
}).strict();
|
|
14832
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
14833
|
-
urls:
|
|
14834
|
-
targets:
|
|
14835
|
-
maxConcurrency:
|
|
14836
|
-
timeoutMs:
|
|
14837
|
-
debug:
|
|
15310
|
+
var SerpIntelligencePageSnapshotsBodySchema = z24.object({
|
|
15311
|
+
urls: z24.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
15312
|
+
targets: z24.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
15313
|
+
maxConcurrency: z24.number().int().min(1).max(5).default(2),
|
|
15314
|
+
timeoutMs: z24.number().int().min(1e3).max(6e4).default(15e3),
|
|
15315
|
+
debug: z24.boolean().default(false)
|
|
14838
15316
|
}).strict();
|
|
14839
|
-
var SerpIntelligenceAICitationSchema =
|
|
14840
|
-
text:
|
|
14841
|
-
href:
|
|
15317
|
+
var SerpIntelligenceAICitationSchema = z24.object({
|
|
15318
|
+
text: z24.string(),
|
|
15319
|
+
href: z24.string()
|
|
14842
15320
|
}).strict();
|
|
14843
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
14844
|
-
position:
|
|
14845
|
-
title:
|
|
14846
|
-
url:
|
|
14847
|
-
domain:
|
|
14848
|
-
cite:
|
|
14849
|
-
snippet:
|
|
14850
|
-
isRedditStyle:
|
|
14851
|
-
inlineRating:
|
|
14852
|
-
value:
|
|
14853
|
-
count:
|
|
15321
|
+
var SerpIntelligenceOrganicResultSchema = z24.object({
|
|
15322
|
+
position: z24.number().int().min(1),
|
|
15323
|
+
title: z24.string(),
|
|
15324
|
+
url: z24.string(),
|
|
15325
|
+
domain: z24.string(),
|
|
15326
|
+
cite: z24.string().nullable(),
|
|
15327
|
+
snippet: z24.string().nullable(),
|
|
15328
|
+
isRedditStyle: z24.boolean(),
|
|
15329
|
+
inlineRating: z24.object({
|
|
15330
|
+
value: z24.string(),
|
|
15331
|
+
count: z24.string()
|
|
14854
15332
|
}).strict().nullable()
|
|
14855
15333
|
}).strict();
|
|
14856
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
14857
|
-
status:
|
|
14858
|
-
expected:
|
|
14859
|
-
city:
|
|
14860
|
-
regionCode:
|
|
14861
|
-
canonicalLocation:
|
|
15334
|
+
var SerpIntelligenceLocationEvidenceSchema = z24.object({
|
|
15335
|
+
status: z24.enum(SerpIntelligenceLocalizationStatusValues),
|
|
15336
|
+
expected: z24.object({
|
|
15337
|
+
city: z24.string(),
|
|
15338
|
+
regionCode: z24.string().nullable(),
|
|
15339
|
+
canonicalLocation: z24.string()
|
|
14862
15340
|
}).strict().nullable(),
|
|
14863
|
-
candidates:
|
|
14864
|
-
city:
|
|
14865
|
-
regionCode:
|
|
14866
|
-
count:
|
|
14867
|
-
examples:
|
|
15341
|
+
candidates: z24.array(z24.object({
|
|
15342
|
+
city: z24.string(),
|
|
15343
|
+
regionCode: z24.string(),
|
|
15344
|
+
count: z24.number().int().min(0),
|
|
15345
|
+
examples: z24.array(z24.string())
|
|
14868
15346
|
}).strict())
|
|
14869
15347
|
}).strict();
|
|
14870
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
14871
|
-
seed:
|
|
14872
|
-
location:
|
|
14873
|
-
extractedAt:
|
|
14874
|
-
totalQuestions:
|
|
14875
|
-
surface:
|
|
14876
|
-
aiOverview:
|
|
14877
|
-
detected:
|
|
14878
|
-
text:
|
|
14879
|
-
citations:
|
|
14880
|
-
expanded:
|
|
14881
|
-
fullyExpanded:
|
|
14882
|
-
sections:
|
|
15348
|
+
var SerpIntelligenceHarvestResultSchema = z24.object({
|
|
15349
|
+
seed: z24.string(),
|
|
15350
|
+
location: z24.string().nullable(),
|
|
15351
|
+
extractedAt: z24.string(),
|
|
15352
|
+
totalQuestions: z24.number().int().min(0),
|
|
15353
|
+
surface: z24.enum(["web", "aim", "unknown"]),
|
|
15354
|
+
aiOverview: z24.object({
|
|
15355
|
+
detected: z24.boolean(),
|
|
15356
|
+
text: z24.string().nullable(),
|
|
15357
|
+
citations: z24.array(SerpIntelligenceAICitationSchema),
|
|
15358
|
+
expanded: z24.boolean().optional(),
|
|
15359
|
+
fullyExpanded: z24.boolean().optional(),
|
|
15360
|
+
sections: z24.array(z24.string()).optional()
|
|
14883
15361
|
}).strict(),
|
|
14884
|
-
aiMode:
|
|
14885
|
-
detected:
|
|
14886
|
-
text:
|
|
14887
|
-
citations:
|
|
15362
|
+
aiMode: z24.object({
|
|
15363
|
+
detected: z24.boolean(),
|
|
15364
|
+
text: z24.string().nullable(),
|
|
15365
|
+
citations: z24.array(SerpIntelligenceAICitationSchema)
|
|
14888
15366
|
}).strict(),
|
|
14889
|
-
tree:
|
|
14890
|
-
flat:
|
|
14891
|
-
videos:
|
|
14892
|
-
forums:
|
|
14893
|
-
organicResults:
|
|
14894
|
-
localPack:
|
|
14895
|
-
entityIds:
|
|
14896
|
-
entities:
|
|
14897
|
-
name:
|
|
14898
|
-
kgId:
|
|
14899
|
-
cid:
|
|
14900
|
-
gcid:
|
|
15367
|
+
tree: z24.array(z24.unknown()),
|
|
15368
|
+
flat: z24.array(z24.unknown()),
|
|
15369
|
+
videos: z24.array(z24.unknown()),
|
|
15370
|
+
forums: z24.array(z24.unknown()),
|
|
15371
|
+
organicResults: z24.array(SerpIntelligenceOrganicResultSchema),
|
|
15372
|
+
localPack: z24.array(z24.unknown()),
|
|
15373
|
+
entityIds: z24.object({
|
|
15374
|
+
entities: z24.array(z24.object({
|
|
15375
|
+
name: z24.string(),
|
|
15376
|
+
kgId: z24.string().nullable(),
|
|
15377
|
+
cid: z24.string().nullable(),
|
|
15378
|
+
gcid: z24.string().nullable()
|
|
14901
15379
|
}).strict()),
|
|
14902
|
-
kgIds:
|
|
14903
|
-
cids:
|
|
14904
|
-
gcids:
|
|
15380
|
+
kgIds: z24.array(z24.string()),
|
|
15381
|
+
cids: z24.array(z24.string()),
|
|
15382
|
+
gcids: z24.array(z24.string())
|
|
14905
15383
|
}).strict(),
|
|
14906
|
-
stats:
|
|
14907
|
-
seed:
|
|
14908
|
-
totalQuestions:
|
|
14909
|
-
maxDepthReached:
|
|
14910
|
-
durationMs:
|
|
14911
|
-
errorCount:
|
|
15384
|
+
stats: z24.object({
|
|
15385
|
+
seed: z24.string(),
|
|
15386
|
+
totalQuestions: z24.number().int().min(0),
|
|
15387
|
+
maxDepthReached: z24.number().int().min(0),
|
|
15388
|
+
durationMs: z24.number().min(0),
|
|
15389
|
+
errorCount: z24.number().int().min(0)
|
|
14912
15390
|
}).strict(),
|
|
14913
|
-
diagnostics:
|
|
14914
|
-
completionStatus:
|
|
14915
|
-
problem:
|
|
14916
|
-
warnings:
|
|
14917
|
-
debug:
|
|
15391
|
+
diagnostics: z24.object({
|
|
15392
|
+
completionStatus: z24.enum(["paa_found", "no_paa", "serp_only"]),
|
|
15393
|
+
problem: z24.null(),
|
|
15394
|
+
warnings: z24.array(z24.unknown()).optional(),
|
|
15395
|
+
debug: z24.object({
|
|
14918
15396
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
14919
15397
|
}).passthrough().optional()
|
|
14920
15398
|
}).passthrough(),
|
|
14921
|
-
whatPeopleSaying:
|
|
15399
|
+
whatPeopleSaying: z24.array(z24.unknown())
|
|
14922
15400
|
}).strict();
|
|
14923
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
14924
|
-
attemptNumber:
|
|
14925
|
-
outcome:
|
|
14926
|
-
startedAt:
|
|
14927
|
-
completedAt:
|
|
14928
|
-
durationMs:
|
|
14929
|
-
problemCode:
|
|
14930
|
-
message:
|
|
14931
|
-
kernelSessionId:
|
|
14932
|
-
cleanupSucceeded:
|
|
15401
|
+
var SerpIntelligenceCaptureAttemptSchema = z24.object({
|
|
15402
|
+
attemptNumber: z24.number().int().min(1),
|
|
15403
|
+
outcome: z24.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
15404
|
+
startedAt: z24.string().optional(),
|
|
15405
|
+
completedAt: z24.string().optional(),
|
|
15406
|
+
durationMs: z24.number().min(0).optional(),
|
|
15407
|
+
problemCode: z24.string().optional(),
|
|
15408
|
+
message: z24.string().optional(),
|
|
15409
|
+
kernelSessionId: z24.string().nullable().optional(),
|
|
15410
|
+
cleanupSucceeded: z24.boolean().nullable().optional()
|
|
14933
15411
|
}).strict();
|
|
14934
|
-
var SerpPageSnapshotCaptureSchema =
|
|
15412
|
+
var SerpPageSnapshotCaptureSchema = z24.object({
|
|
14935
15413
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14936
15414
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
14937
15415
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14938
|
-
sourceKind:
|
|
14939
|
-
sourcePosition:
|
|
14940
|
-
status:
|
|
14941
|
-
fetchedVia:
|
|
14942
|
-
httpStatus:
|
|
14943
|
-
contentType:
|
|
14944
|
-
title:
|
|
15416
|
+
sourceKind: z24.enum(SerpPageSnapshotSourceKindValues),
|
|
15417
|
+
sourcePosition: z24.number().int().min(1).nullable(),
|
|
15418
|
+
status: z24.enum(SerpPageFetchStatusValues),
|
|
15419
|
+
fetchedVia: z24.enum(SerpPageFetchedViaValues).nullable(),
|
|
15420
|
+
httpStatus: z24.number().int().min(100).max(599).nullable(),
|
|
15421
|
+
contentType: z24.string().nullable(),
|
|
15422
|
+
title: z24.string().nullable(),
|
|
14945
15423
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14946
|
-
metaDescription:
|
|
14947
|
-
headings:
|
|
14948
|
-
level:
|
|
14949
|
-
text:
|
|
15424
|
+
metaDescription: z24.string().nullable(),
|
|
15425
|
+
headings: z24.array(z24.object({
|
|
15426
|
+
level: z24.number().int().min(1).max(6),
|
|
15427
|
+
text: z24.string()
|
|
14950
15428
|
}).strict()).default([]),
|
|
14951
|
-
artifact:
|
|
14952
|
-
htmlBlobUrl:
|
|
14953
|
-
textBlobUrl:
|
|
14954
|
-
markdownBlobUrl:
|
|
14955
|
-
screenshotBlobUrl:
|
|
14956
|
-
contentSha256:
|
|
14957
|
-
capturedAt:
|
|
15429
|
+
artifact: z24.object({
|
|
15430
|
+
htmlBlobUrl: z24.string().url().nullable(),
|
|
15431
|
+
textBlobUrl: z24.string().url().nullable(),
|
|
15432
|
+
markdownBlobUrl: z24.string().url().nullable(),
|
|
15433
|
+
screenshotBlobUrl: z24.string().url().nullable(),
|
|
15434
|
+
contentSha256: z24.string().nullable(),
|
|
15435
|
+
capturedAt: z24.string().nullable()
|
|
14958
15436
|
}).strict(),
|
|
14959
|
-
error:
|
|
14960
|
-
code:
|
|
14961
|
-
message:
|
|
15437
|
+
error: z24.object({
|
|
15438
|
+
code: z24.string(),
|
|
15439
|
+
message: z24.string()
|
|
14962
15440
|
}).strict().nullable()
|
|
14963
15441
|
}).strict();
|
|
14964
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
15442
|
+
var SerpIntelligenceCaptureResponseSchema = z24.object({
|
|
14965
15443
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
14966
|
-
attempts:
|
|
15444
|
+
attempts: z24.array(SerpIntelligenceCaptureAttemptSchema),
|
|
14967
15445
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
14968
|
-
pageSnapshotArtifacts:
|
|
14969
|
-
billing:
|
|
14970
|
-
creditsUsed:
|
|
14971
|
-
requestId:
|
|
14972
|
-
jobId:
|
|
15446
|
+
pageSnapshotArtifacts: z24.array(SerpPageSnapshotCaptureSchema),
|
|
15447
|
+
billing: z24.object({
|
|
15448
|
+
creditsUsed: z24.number().min(0).optional(),
|
|
15449
|
+
requestId: z24.string().optional(),
|
|
15450
|
+
jobId: z24.string().optional()
|
|
14973
15451
|
}).strict().optional()
|
|
14974
15452
|
}).strict();
|
|
14975
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
14976
|
-
pageSnapshotArtifacts:
|
|
14977
|
-
attempts:
|
|
15453
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z24.object({
|
|
15454
|
+
pageSnapshotArtifacts: z24.array(SerpPageSnapshotCaptureSchema),
|
|
15455
|
+
attempts: z24.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
14978
15456
|
}).strict();
|
|
14979
15457
|
|
|
14980
15458
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -15146,7 +15624,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
15146
15624
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
15147
15625
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
15148
15626
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
15149
|
-
var serpIntelligenceApp = new
|
|
15627
|
+
var serpIntelligenceApp = new Hono16();
|
|
15150
15628
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
15151
15629
|
function structuredError(input) {
|
|
15152
15630
|
return {
|
|
@@ -15315,7 +15793,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
15315
15793
|
});
|
|
15316
15794
|
|
|
15317
15795
|
// src/mcp/mcp-routes.ts
|
|
15318
|
-
import { Hono as
|
|
15796
|
+
import { Hono as Hono17 } from "hono";
|
|
15319
15797
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
15320
15798
|
|
|
15321
15799
|
// src/mcp/memory-inprocess-executor.ts
|
|
@@ -15897,7 +16375,7 @@ async function requireMcpCallerKey(c) {
|
|
|
15897
16375
|
if (!user) return mcpAuthError();
|
|
15898
16376
|
return { key: callerKey, user };
|
|
15899
16377
|
}
|
|
15900
|
-
var mcpApp = new
|
|
16378
|
+
var mcpApp = new Hono17();
|
|
15901
16379
|
mcpApp.all("/", async (c) => {
|
|
15902
16380
|
try {
|
|
15903
16381
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -15923,11 +16401,11 @@ mcpApp.all("/", async (c) => {
|
|
|
15923
16401
|
});
|
|
15924
16402
|
|
|
15925
16403
|
// src/api/browser-agent-routes.ts
|
|
15926
|
-
import { randomUUID as
|
|
15927
|
-
import { Hono as
|
|
16404
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
16405
|
+
import { Hono as Hono18 } from "hono";
|
|
15928
16406
|
|
|
15929
16407
|
// src/api/browser-agent-db.ts
|
|
15930
|
-
import { randomUUID as
|
|
16408
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
15931
16409
|
var _ready = false;
|
|
15932
16410
|
async function migrateBrowserAgent() {
|
|
15933
16411
|
if (_ready) return;
|
|
@@ -16031,7 +16509,7 @@ async function migrateBrowserAgent() {
|
|
|
16031
16509
|
}
|
|
16032
16510
|
async function createExtensionRow(input) {
|
|
16033
16511
|
const db = getDb();
|
|
16034
|
-
const id = `bext_${
|
|
16512
|
+
const id = `bext_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16035
16513
|
await db.execute({
|
|
16036
16514
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
16037
16515
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -16066,7 +16544,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
16066
16544
|
}
|
|
16067
16545
|
async function createAuthConnectionRow(input) {
|
|
16068
16546
|
const db = getDb();
|
|
16069
|
-
const connectionId = `authc_${
|
|
16547
|
+
const connectionId = `authc_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16070
16548
|
await db.execute({
|
|
16071
16549
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
16072
16550
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -16100,7 +16578,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
16100
16578
|
}
|
|
16101
16579
|
async function createSessionRow(input) {
|
|
16102
16580
|
const db = getDb();
|
|
16103
|
-
const id = `bas_${
|
|
16581
|
+
const id = `bas_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16104
16582
|
await db.execute({
|
|
16105
16583
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
16106
16584
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -16169,7 +16647,7 @@ async function recordAction(input) {
|
|
|
16169
16647
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
16170
16648
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
16171
16649
|
args: [
|
|
16172
|
-
`baa_${
|
|
16650
|
+
`baa_${randomUUID3().replace(/-/g, "").slice(0, 20)}`,
|
|
16173
16651
|
input.sessionId,
|
|
16174
16652
|
input.type,
|
|
16175
16653
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -17399,7 +17877,7 @@ async function loadOpenSession(id, userId) {
|
|
|
17399
17877
|
return row;
|
|
17400
17878
|
}
|
|
17401
17879
|
function buildBrowserAgentRoutes() {
|
|
17402
|
-
const app2 = new
|
|
17880
|
+
const app2 = new Hono18();
|
|
17403
17881
|
app2.use("*", async (c, next) => {
|
|
17404
17882
|
await migrateBrowserAgent();
|
|
17405
17883
|
return next();
|
|
@@ -17857,7 +18335,7 @@ function buildBrowserAgentRoutes() {
|
|
|
17857
18335
|
}
|
|
17858
18336
|
const existing = await getExtensionRow(user.id, name);
|
|
17859
18337
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
17860
|
-
const backendName = `u${user.id}_${
|
|
18338
|
+
const backendName = `u${user.id}_${randomUUID4().replace(/-/g, "")}`;
|
|
17861
18339
|
try {
|
|
17862
18340
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
17863
18341
|
const row = await createExtensionRow({
|
|
@@ -18189,9 +18667,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
18189
18667
|
|
|
18190
18668
|
// src/api/stripe-routes.ts
|
|
18191
18669
|
import Stripe from "stripe";
|
|
18192
|
-
import { Hono as
|
|
18670
|
+
import { Hono as Hono19 } from "hono";
|
|
18193
18671
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
18194
|
-
var stripeApp = new
|
|
18672
|
+
var stripeApp = new Hono19();
|
|
18195
18673
|
stripeApp.post("/webhooks", async (c) => {
|
|
18196
18674
|
const sig = c.req.header("stripe-signature");
|
|
18197
18675
|
const body = await c.req.text();
|
|
@@ -18293,9 +18771,9 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
18293
18771
|
}
|
|
18294
18772
|
|
|
18295
18773
|
// src/api/oauth-routes.ts
|
|
18296
|
-
import { Hono as
|
|
18774
|
+
import { Hono as Hono20 } from "hono";
|
|
18297
18775
|
import { getCookie, setCookie } from "hono/cookie";
|
|
18298
|
-
import { createHash as
|
|
18776
|
+
import { createHash as createHash4, randomBytes as randomBytes2, randomUUID as randomUUID5 } from "crypto";
|
|
18299
18777
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
18300
18778
|
var ISSUER = "https://mcpscraper.dev";
|
|
18301
18779
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -18328,7 +18806,7 @@ async function getKeys() {
|
|
|
18328
18806
|
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
18329
18807
|
const full = await exportJWK(privateKey);
|
|
18330
18808
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
18331
|
-
const kid =
|
|
18809
|
+
const kid = createHash4("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
18332
18810
|
publicJwk.kid = kid;
|
|
18333
18811
|
publicJwk.alg = "RS256";
|
|
18334
18812
|
publicJwk.use = "sig";
|
|
@@ -18462,7 +18940,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
18462
18940
|
if (state) u.searchParams.set("state", state);
|
|
18463
18941
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
18464
18942
|
}
|
|
18465
|
-
var oauthApp = new
|
|
18943
|
+
var oauthApp = new Hono20();
|
|
18466
18944
|
oauthApp.use("*", async (c, next) => {
|
|
18467
18945
|
c.header("Access-Control-Allow-Origin", "*");
|
|
18468
18946
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -18638,12 +19116,12 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
18638
19116
|
});
|
|
18639
19117
|
function pkceMatches(verifier, challenge) {
|
|
18640
19118
|
if (!verifier) return false;
|
|
18641
|
-
const computed =
|
|
19119
|
+
const computed = createHash4("sha256").update(verifier).digest("base64url");
|
|
18642
19120
|
return computed === challenge;
|
|
18643
19121
|
}
|
|
18644
19122
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
18645
19123
|
const { privateKey, kid } = await getKeys();
|
|
18646
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
19124
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID5()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
18647
19125
|
}
|
|
18648
19126
|
function tokenErrorResponse(c, error, description, status) {
|
|
18649
19127
|
return c.json({ error, error_description: description }, status);
|
|
@@ -18716,8 +19194,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
18716
19194
|
});
|
|
18717
19195
|
|
|
18718
19196
|
// src/api/chat-routes.ts
|
|
18719
|
-
import { Hono as
|
|
18720
|
-
var chatApp = new
|
|
19197
|
+
import { Hono as Hono21 } from "hono";
|
|
19198
|
+
var chatApp = new Hono21();
|
|
18721
19199
|
function tokenOk(token) {
|
|
18722
19200
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18723
19201
|
}
|
|
@@ -19387,8 +19865,8 @@ loadChannels();
|
|
|
19387
19865
|
}
|
|
19388
19866
|
|
|
19389
19867
|
// src/api/schedule-routes.ts
|
|
19390
|
-
import { Hono as
|
|
19391
|
-
var scheduleApp = new
|
|
19868
|
+
import { Hono as Hono22 } from "hono";
|
|
19869
|
+
var scheduleApp = new Hono22();
|
|
19392
19870
|
function tokenOk2(token) {
|
|
19393
19871
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
19394
19872
|
}
|
|
@@ -19579,8 +20057,15 @@ async function loadStatus() {
|
|
|
19579
20057
|
const r = await get('/api/status');
|
|
19580
20058
|
const text = document.getElementById('quota-text');
|
|
19581
20059
|
const fill = document.getElementById('quota-fill');
|
|
19582
|
-
if (!r.ok
|
|
19583
|
-
text.textContent = '
|
|
20060
|
+
if (!r.ok) {
|
|
20061
|
+
text.textContent = 'Scheduled Actions status is temporarily unavailable.';
|
|
20062
|
+
text.classList.remove('warn');
|
|
20063
|
+
fill.style.width = '0%';
|
|
20064
|
+
fill.classList.remove('warn');
|
|
20065
|
+
return;
|
|
20066
|
+
}
|
|
20067
|
+
if (r.billingMode === 'credits') {
|
|
20068
|
+
text.textContent = (r.runBaseCredits || 75) + ' Credits per run + AI at ' + (r.llmCostMultiplier || 1.5) + '\xD7 actual OpenRouter cost';
|
|
19584
20069
|
text.classList.remove('warn');
|
|
19585
20070
|
fill.style.width = '0%';
|
|
19586
20071
|
fill.classList.remove('warn');
|
|
@@ -19712,7 +20197,7 @@ loadActions();
|
|
|
19712
20197
|
}
|
|
19713
20198
|
|
|
19714
20199
|
// src/api/resend-inbound-routes.ts
|
|
19715
|
-
import { Hono as
|
|
20200
|
+
import { Hono as Hono23 } from "hono";
|
|
19716
20201
|
import { Resend } from "resend";
|
|
19717
20202
|
var SUPPORT_ADDRESS = "support@mcpscraper.dev";
|
|
19718
20203
|
var DMARC_ADDRESS = "dmarc@mcpscraper.dev";
|
|
@@ -19888,7 +20373,7 @@ async function forwardReceivedEmail(resend, email, destination, label) {
|
|
|
19888
20373
|
if (sent.error) throw new Error(`forward failed: ${sent.error.message}`);
|
|
19889
20374
|
return sent.data.id;
|
|
19890
20375
|
}
|
|
19891
|
-
var resendInboundApp = new
|
|
20376
|
+
var resendInboundApp = new Hono23();
|
|
19892
20377
|
resendInboundApp.post("/webhooks", async (c) => {
|
|
19893
20378
|
const apiKey = process.env.RESEND_API_KEY?.trim();
|
|
19894
20379
|
const webhookSecret = process.env.RESEND_WEBHOOK_SECRET?.trim();
|
|
@@ -19982,13 +20467,13 @@ import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "
|
|
|
19982
20467
|
import Stripe2 from "stripe";
|
|
19983
20468
|
|
|
19984
20469
|
// src/api/page-diff.ts
|
|
19985
|
-
import { createHash as
|
|
20470
|
+
import { createHash as createHash5 } from "crypto";
|
|
19986
20471
|
import { diffLines } from "diff";
|
|
19987
20472
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
19988
20473
|
var MAX_DIFF_HUNKS = 200;
|
|
19989
20474
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
19990
20475
|
function sha256Hex(value) {
|
|
19991
|
-
return
|
|
20476
|
+
return createHash5("sha256").update(value).digest("hex");
|
|
19992
20477
|
}
|
|
19993
20478
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
19994
20479
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -20095,7 +20580,7 @@ async function persistScrapeBody(user, opts) {
|
|
|
20095
20580
|
}
|
|
20096
20581
|
|
|
20097
20582
|
// src/api/connection-memory-import.ts
|
|
20098
|
-
import { createHash as
|
|
20583
|
+
import { createHash as createHash6 } from "crypto";
|
|
20099
20584
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
20100
20585
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
20101
20586
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -20125,7 +20610,7 @@ function canonicalJson(value) {
|
|
|
20125
20610
|
return JSON.stringify(value);
|
|
20126
20611
|
}
|
|
20127
20612
|
function sha2562(value) {
|
|
20128
|
-
return
|
|
20613
|
+
return createHash6("sha256").update(value).digest("hex");
|
|
20129
20614
|
}
|
|
20130
20615
|
function sensitiveKey(key) {
|
|
20131
20616
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -20394,7 +20879,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
20394
20879
|
}
|
|
20395
20880
|
|
|
20396
20881
|
// src/api/connected-data-export.ts
|
|
20397
|
-
import { randomUUID as
|
|
20882
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
20398
20883
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
20399
20884
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
20400
20885
|
);
|
|
@@ -20519,7 +21004,7 @@ function finitePositive(value, fallback) {
|
|
|
20519
21004
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
20520
21005
|
}
|
|
20521
21006
|
async function collectConnectedDataExport(args) {
|
|
20522
|
-
const exportId =
|
|
21007
|
+
const exportId = randomUUID6();
|
|
20523
21008
|
const now = args.now ?? Date.now;
|
|
20524
21009
|
const startedAt = now();
|
|
20525
21010
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -20848,7 +21333,7 @@ async function dbUsage(identity, plan) {
|
|
|
20848
21333
|
}
|
|
20849
21334
|
|
|
20850
21335
|
// src/api/nango-control.ts
|
|
20851
|
-
import { createHash as
|
|
21336
|
+
import { createHash as createHash7 } from "crypto";
|
|
20852
21337
|
var DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
20853
21338
|
var DISABLED_NANGO_TOOLS = {
|
|
20854
21339
|
"google-mail": /* @__PURE__ */ new Set(["list-filters"]),
|
|
@@ -21447,7 +21932,7 @@ function canonicalJson2(value) {
|
|
|
21447
21932
|
return JSON.stringify(value);
|
|
21448
21933
|
}
|
|
21449
21934
|
function projectedToolSchemaHash(tool) {
|
|
21450
|
-
return
|
|
21935
|
+
return createHash7("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
21451
21936
|
}
|
|
21452
21937
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
21453
21938
|
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
@@ -21978,7 +22463,7 @@ var requireIntegrationsTier = createMiddleware3(async (c, next) => {
|
|
|
21978
22463
|
}
|
|
21979
22464
|
return next();
|
|
21980
22465
|
});
|
|
21981
|
-
var app = new
|
|
22466
|
+
var app = new Hono24();
|
|
21982
22467
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
21983
22468
|
function requireStripeSecret() {
|
|
21984
22469
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -22479,41 +22964,13 @@ app.post("/memory/portal", auth2, async (c) => {
|
|
|
22479
22964
|
}
|
|
22480
22965
|
});
|
|
22481
22966
|
app.post("/schedule-checkout", auth2, async (c) => {
|
|
22482
|
-
|
|
22483
|
-
|
|
22484
|
-
|
|
22485
|
-
|
|
22486
|
-
|
|
22487
|
-
|
|
22488
|
-
|
|
22489
|
-
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
22490
|
-
let customerId = user.stripe_customer_id;
|
|
22491
|
-
if (!customerId) {
|
|
22492
|
-
const customer = await stripeClient.customers.create({ email: user.email });
|
|
22493
|
-
customerId = customer.id;
|
|
22494
|
-
await setStripeCustomerId(user.id, customerId);
|
|
22495
|
-
}
|
|
22496
|
-
const session = await stripeClient.checkout.sessions.create({
|
|
22497
|
-
mode: "subscription",
|
|
22498
|
-
customer: customerId,
|
|
22499
|
-
line_items: [{ price: priceId, quantity: 1 }],
|
|
22500
|
-
client_reference_id: String(user.id),
|
|
22501
|
-
metadata: { product: "mcp-scheduling", userId: String(user.id) },
|
|
22502
|
-
subscription_data: { metadata: { product: "mcp-scheduling", userId: String(user.id) } },
|
|
22503
|
-
automatic_tax: { enabled: true },
|
|
22504
|
-
billing_address_collection: "required",
|
|
22505
|
-
customer_update: { address: "auto", name: "auto" },
|
|
22506
|
-
tax_id_collection: { enabled: true },
|
|
22507
|
-
success_url: `${appOrigin()}/memory?billing=success`,
|
|
22508
|
-
cancel_url: `${appOrigin()}/memory?billing=cancel`
|
|
22509
|
-
});
|
|
22510
|
-
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
22511
|
-
return c.json({ url: session.url });
|
|
22512
|
-
} catch (err) {
|
|
22513
|
-
const message = err instanceof Error ? err.message : "Unable to start scheduling checkout.";
|
|
22514
|
-
console.error("[schedule-checkout]", message);
|
|
22515
|
-
return c.json({ error: message }, 500);
|
|
22516
|
-
}
|
|
22967
|
+
return c.json({
|
|
22968
|
+
error_code: "scheduled_actions_now_credit_metered",
|
|
22969
|
+
error: "Scheduled Actions is included and no longer requires a separate subscription.",
|
|
22970
|
+
billingMode: "credits",
|
|
22971
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
22972
|
+
llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP
|
|
22973
|
+
}, 410);
|
|
22517
22974
|
});
|
|
22518
22975
|
function scheduleConnectionError(c, err, fallback) {
|
|
22519
22976
|
if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
|
|
@@ -22627,7 +23084,14 @@ app.get("/schedule-status", auth2, async (c) => {
|
|
|
22627
23084
|
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
22628
23085
|
if (!key) return c.json({ error: error ?? "memory unavailable" }, 503);
|
|
22629
23086
|
const r = await memoryCall("getScheduleStatusTool", {}, key);
|
|
22630
|
-
return c.json(
|
|
23087
|
+
return c.json({
|
|
23088
|
+
...r,
|
|
23089
|
+
ok: true,
|
|
23090
|
+
enabled: true,
|
|
23091
|
+
billingMode: "credits",
|
|
23092
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
23093
|
+
llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP
|
|
23094
|
+
});
|
|
22631
23095
|
} catch (err) {
|
|
22632
23096
|
const message = err instanceof Error ? err.message : "Unable to get schedule status.";
|
|
22633
23097
|
console.error("[schedule-status]", message);
|
|
@@ -23997,7 +24461,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
23997
24461
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
23998
24462
|
return c.json({ error: "Unauthorized" }, 401);
|
|
23999
24463
|
}
|
|
24000
|
-
const { drainQueue } = await import("./worker-
|
|
24464
|
+
const { drainQueue } = await import("./worker-45WEWCMQ.js");
|
|
24001
24465
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
24002
24466
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
24003
24467
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -24025,15 +24489,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
24025
24489
|
return c.json({ error: "Unauthorized" }, 401);
|
|
24026
24490
|
}
|
|
24027
24491
|
const jobId = c.req.param("id");
|
|
24028
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
24029
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
24492
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-YNPXEZCB.js");
|
|
24493
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-HMCYUVBJ.js");
|
|
24030
24494
|
const job = await getExtractJob2(jobId);
|
|
24031
24495
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
24032
24496
|
const stored = await assembleExtractArtifacts(job, {});
|
|
24033
24497
|
await completeExtractJob2(jobId, stored);
|
|
24034
24498
|
let settlement = "already_settled_or_refunded";
|
|
24035
24499
|
if (job.billedMc == null && job.userId != null) {
|
|
24036
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
24500
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-YNPXEZCB.js");
|
|
24037
24501
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
24038
24502
|
const successful = await countSuccessfulPages2(jobId);
|
|
24039
24503
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -24051,6 +24515,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
24051
24515
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
|
|
24052
24516
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
24053
24517
|
app.route("/api/internal/memory", internalMemoryApp);
|
|
24518
|
+
app.route("/api/internal/billing", internalBillingApp);
|
|
24054
24519
|
app.route("/youtube", youtubeApp);
|
|
24055
24520
|
app.route("/screenshot", screenshotApp);
|
|
24056
24521
|
app.route("/facebook", facebookAdApp);
|
|
@@ -24198,4 +24663,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
24198
24663
|
export {
|
|
24199
24664
|
app
|
|
24200
24665
|
};
|
|
24201
|
-
//# sourceMappingURL=server-
|
|
24666
|
+
//# sourceMappingURL=server-PH3VJ74X.js.map
|