mcp-scraper 0.25.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/bin/api-server.cjs +4980 -2927
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +52 -4
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-PNBYYAEK.js → chunk-3ZZOLQ6A.js} +2 -2
- package/dist/{chunk-JHI373VB.js → chunk-4767BA2O.js} +85 -3
- package/dist/chunk-4767BA2O.js.map +1 -0
- package/dist/chunk-FQQC5OZJ.js +7 -0
- package/dist/chunk-FQQC5OZJ.js.map +1 -0
- package/dist/{chunk-YNHPZU6B.js → chunk-TBRX7VOP.js} +2 -2
- package/dist/{chunk-YNHPZU6B.js.map → chunk-TBRX7VOP.js.map} +1 -1
- package/dist/{chunk-NC37L5RW.js → chunk-YMEPX46N.js} +47 -5
- package/dist/chunk-YMEPX46N.js.map +1 -0
- package/dist/{server-PILR3U5E.js → server-UHDJ5TH5.js} +2484 -580
- package/dist/server-UHDJ5TH5.js.map +1 -0
- package/dist/{site-extract-repository-2QQSL2QC.js → site-extract-repository-O36SIFI2.js} +3 -3
- package/dist/{worker-RV2TYXE3.js → worker-E3KHQH2F.js} +2 -2
- package/docs/mcp-tool-manifest.generated.json +164 -5
- package/docs/specs/main-mcp-integration-ownership-spec.md +1164 -0
- package/package.json +5 -1
- package/dist/chunk-JHI373VB.js.map +0 -1
- package/dist/chunk-NC37L5RW.js.map +0 -1
- package/dist/chunk-UELJZCUC.js +0 -7
- package/dist/chunk-UELJZCUC.js.map +0 -1
- package/dist/server-PILR3U5E.js.map +0 -1
- /package/dist/{chunk-PNBYYAEK.js.map → chunk-3ZZOLQ6A.js.map} +0 -0
- /package/dist/{site-extract-repository-2QQSL2QC.js.map → site-extract-repository-O36SIFI2.js.map} +0 -0
- /package/dist/{worker-RV2TYXE3.js.map → worker-E3KHQH2F.js.map} +0 -0
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
sanitizeAttempts,
|
|
36
36
|
sanitizeHarvestResult,
|
|
37
37
|
transcribeMediaUrl
|
|
38
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-YMEPX46N.js";
|
|
39
39
|
import {
|
|
40
40
|
auditImages,
|
|
41
41
|
buildLinkReport,
|
|
@@ -76,7 +76,7 @@ import {
|
|
|
76
76
|
RawMapsOverviewSchema,
|
|
77
77
|
RawMapsReviewStatsSchema
|
|
78
78
|
} from "./chunk-XGIPATLV.js";
|
|
79
|
-
import "./chunk-
|
|
79
|
+
import "./chunk-FQQC5OZJ.js";
|
|
80
80
|
import {
|
|
81
81
|
completeExtractJob,
|
|
82
82
|
countSuccessfulPages,
|
|
@@ -87,10 +87,17 @@ import {
|
|
|
87
87
|
saveExtractPages,
|
|
88
88
|
setExtractJobTotal,
|
|
89
89
|
settleExtractJob
|
|
90
|
-
} from "./chunk-
|
|
90
|
+
} from "./chunk-3ZZOLQ6A.js";
|
|
91
91
|
import {
|
|
92
92
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
93
93
|
CONCURRENCY_PRICE_ID,
|
|
94
|
+
CONNECTED_ACTIVE_CONNECTION_MONTHLY_USD,
|
|
95
|
+
CONNECTED_COMPUTE_CREDITS_PER_SECOND,
|
|
96
|
+
CONNECTED_FUNCTION_RUN_CREDITS,
|
|
97
|
+
CONNECTED_FUNCTION_RUN_MC,
|
|
98
|
+
CONNECTED_PROXY_REQUEST_CREDITS,
|
|
99
|
+
CONNECTED_USAGE_RATE_POLICY,
|
|
100
|
+
CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
94
101
|
CREDIT_COST_CATALOG,
|
|
95
102
|
LedgerOperation,
|
|
96
103
|
MC_COSTS,
|
|
@@ -112,12 +119,13 @@ import {
|
|
|
112
119
|
USD_NANOS,
|
|
113
120
|
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE,
|
|
114
121
|
browserActiveCostMc,
|
|
122
|
+
calculateConnectedUsageCharge,
|
|
115
123
|
concurrencySlotBillingInfo,
|
|
116
124
|
getPublicRatesPayload,
|
|
117
125
|
insufficientBalanceResponse,
|
|
118
126
|
serpActualCostMc,
|
|
119
127
|
vendorCostUsdToMc
|
|
120
|
-
} from "./chunk-
|
|
128
|
+
} from "./chunk-4767BA2O.js";
|
|
121
129
|
import {
|
|
122
130
|
CaptchaError,
|
|
123
131
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -3463,8 +3471,8 @@ function el(type, style, ...children) {
|
|
|
3463
3471
|
const ch = children.flat().filter((c) => c != null);
|
|
3464
3472
|
return { type, props: { style, children: ch.length === 0 ? void 0 : ch.length === 1 ? ch[0] : ch } };
|
|
3465
3473
|
}
|
|
3466
|
-
function clean(
|
|
3467
|
-
return
|
|
3474
|
+
function clean(text2) {
|
|
3475
|
+
return text2.replace(/[\u{1F000}-\u{1FFFF}\u{2600}-\u{27BF}]/gu, "").replace(/[().,!?]/g, "").trim();
|
|
3468
3476
|
}
|
|
3469
3477
|
function tagline(post) {
|
|
3470
3478
|
if (post.titleDisplayItalic) return clean(post.titleDisplayItalic).slice(0, 38);
|
|
@@ -3618,13 +3626,13 @@ async function extractKpo(opts) {
|
|
|
3618
3626
|
continue;
|
|
3619
3627
|
}
|
|
3620
3628
|
if (!res.ok) return null;
|
|
3621
|
-
const
|
|
3622
|
-
if (
|
|
3623
|
-
const stripped =
|
|
3629
|
+
const text2 = await res.text();
|
|
3630
|
+
if (text2.length < 2e3) return null;
|
|
3631
|
+
const stripped = text2.replace(/<(script|style|noscript|svg|iframe|canvas)[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
3624
3632
|
const contentBlock = stripped.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? stripped.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? "";
|
|
3625
3633
|
const visibleContent = contentBlock.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
3626
3634
|
if (visibleContent.length < 300) return null;
|
|
3627
|
-
return
|
|
3635
|
+
return text2;
|
|
3628
3636
|
}
|
|
3629
3637
|
return null;
|
|
3630
3638
|
} catch {
|
|
@@ -3661,8 +3669,8 @@ async function extractKpo(opts) {
|
|
|
3661
3669
|
const headings = [];
|
|
3662
3670
|
for (const m of html.matchAll(/<(h[1-6])[^>]*>([\s\S]*?)<\/\1>/gi)) {
|
|
3663
3671
|
const level = parseInt(m[1].slice(1));
|
|
3664
|
-
const
|
|
3665
|
-
if (
|
|
3672
|
+
const text2 = stripTags(m[2]);
|
|
3673
|
+
if (text2) headings.push({ level, text: text2 });
|
|
3666
3674
|
}
|
|
3667
3675
|
const sanitizeJsonLd = (s) => s.replace(/[\x00-\x09\x0b\x0c\x0e-\x1f]/g, " ");
|
|
3668
3676
|
const parseJsonLd = (s) => {
|
|
@@ -4092,7 +4100,7 @@ async function extractBrandingFromPage(page) {
|
|
|
4092
4100
|
const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
|
|
4093
4101
|
const primary = navBgUsable ?? svgPrimary ?? navChildBg ?? bodyBgHex;
|
|
4094
4102
|
const accent = rgbToHex(raw.btnBg ?? "");
|
|
4095
|
-
const
|
|
4103
|
+
const text2 = rgbToHex(raw.bodyColor ?? "");
|
|
4096
4104
|
const heading = rgbToHex(raw.h1Color ?? "");
|
|
4097
4105
|
const background = bodyBgHex;
|
|
4098
4106
|
const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
|
|
@@ -4102,7 +4110,7 @@ async function extractBrandingFromPage(page) {
|
|
|
4102
4110
|
primary: primary && !isTransparentOrWhite(primary) ? primary : null,
|
|
4103
4111
|
accent: accent && !isTransparentOrWhite(accent) ? accent : null,
|
|
4104
4112
|
background: background ?? null,
|
|
4105
|
-
text:
|
|
4113
|
+
text: text2 ?? null,
|
|
4106
4114
|
heading: heading ?? null
|
|
4107
4115
|
},
|
|
4108
4116
|
fonts: {
|
|
@@ -4743,8 +4751,8 @@ function parsePageData(url, html, status, via, resp) {
|
|
|
4743
4751
|
const canonicalUrl = html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? html.match(/<link[^>]+href=["']([^"']*)["'][^>]+rel=["']canonical["']/i)?.[1]?.trim() ?? null;
|
|
4744
4752
|
const headings = [];
|
|
4745
4753
|
for (const m of html.matchAll(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi)) {
|
|
4746
|
-
const
|
|
4747
|
-
if (
|
|
4754
|
+
const text2 = m[2].replace(/<[^>]+>/g, "").trim();
|
|
4755
|
+
if (text2) headings.push({ level: Number(m[1]), text: text2.slice(0, 200) });
|
|
4748
4756
|
}
|
|
4749
4757
|
const h1 = headings.find((h) => h.level === 1)?.text ?? null;
|
|
4750
4758
|
const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
@@ -5064,14 +5072,14 @@ var inngest = new Inngest({ id: "mcp-scraper" });
|
|
|
5064
5072
|
// src/lib/llm-parse-with-retry.ts
|
|
5065
5073
|
import Anthropic from "@anthropic-ai/sdk";
|
|
5066
5074
|
function extractJson(raw) {
|
|
5067
|
-
let
|
|
5068
|
-
|
|
5069
|
-
if (
|
|
5070
|
-
const objStart =
|
|
5071
|
-
const arrStart =
|
|
5075
|
+
let text2 = raw.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5076
|
+
text2 = text2.replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/, "").trim();
|
|
5077
|
+
if (text2.startsWith("{") || text2.startsWith("[")) return text2;
|
|
5078
|
+
const objStart = text2.indexOf("{");
|
|
5079
|
+
const arrStart = text2.indexOf("[");
|
|
5072
5080
|
let start = -1;
|
|
5073
5081
|
let open, close;
|
|
5074
|
-
if (objStart === -1 && arrStart === -1) return
|
|
5082
|
+
if (objStart === -1 && arrStart === -1) return text2;
|
|
5075
5083
|
if (objStart === -1) {
|
|
5076
5084
|
start = arrStart;
|
|
5077
5085
|
open = "[";
|
|
@@ -5092,8 +5100,8 @@ function extractJson(raw) {
|
|
|
5092
5100
|
let depth = 0;
|
|
5093
5101
|
let inStr = false;
|
|
5094
5102
|
let escape = false;
|
|
5095
|
-
for (let i = start; i <
|
|
5096
|
-
const ch =
|
|
5103
|
+
for (let i = start; i < text2.length; i++) {
|
|
5104
|
+
const ch = text2[i];
|
|
5097
5105
|
if (escape) {
|
|
5098
5106
|
escape = false;
|
|
5099
5107
|
continue;
|
|
@@ -5109,10 +5117,10 @@ function extractJson(raw) {
|
|
|
5109
5117
|
if (inStr) continue;
|
|
5110
5118
|
if (ch === open) depth++;
|
|
5111
5119
|
else if (ch === close) {
|
|
5112
|
-
if (--depth === 0) return
|
|
5120
|
+
if (--depth === 0) return text2.slice(start, i + 1);
|
|
5113
5121
|
}
|
|
5114
5122
|
}
|
|
5115
|
-
return
|
|
5123
|
+
return text2.slice(start);
|
|
5116
5124
|
}
|
|
5117
5125
|
var DeepInfraLlmClient = class {
|
|
5118
5126
|
apiKey;
|
|
@@ -8513,9 +8521,9 @@ import { provisionDefaultsTool } from "mcpscraper-memory-tools/tools/vaults/prov
|
|
|
8513
8521
|
import { setScheduleEntitlementTool } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
|
|
8514
8522
|
var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
|
|
8515
8523
|
var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
8516
|
-
function parseMemoryRpcBody(
|
|
8517
|
-
const dataLine =
|
|
8518
|
-
const payload = dataLine ? dataLine.slice(5).trim() :
|
|
8524
|
+
function parseMemoryRpcBody(text2) {
|
|
8525
|
+
const dataLine = text2.split("\n").find((line) => line.startsWith("data:"));
|
|
8526
|
+
const payload = dataLine ? dataLine.slice(5).trim() : text2;
|
|
8519
8527
|
return JSON.parse(payload);
|
|
8520
8528
|
}
|
|
8521
8529
|
function isMemoryOperator(email) {
|
|
@@ -8559,9 +8567,9 @@ async function memoryCall(toolName, args, userMemoryKey) {
|
|
|
8559
8567
|
params: { name: toolName, arguments: args }
|
|
8560
8568
|
})
|
|
8561
8569
|
});
|
|
8562
|
-
const
|
|
8570
|
+
const text2 = await res.text();
|
|
8563
8571
|
if (!res.ok) return { ok: false, error: `memory ${toolName} failed (${res.status})` };
|
|
8564
|
-
const body = parseMemoryRpcBody(
|
|
8572
|
+
const body = parseMemoryRpcBody(text2);
|
|
8565
8573
|
if (body.error) return { ok: false, error: body.error.message ?? `memory ${toolName} failed` };
|
|
8566
8574
|
if (body.result?.structuredContent) return body.result.structuredContent;
|
|
8567
8575
|
const textContent = body.result?.content?.find((c) => c.type === "text")?.text;
|
|
@@ -8797,6 +8805,412 @@ internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
|
8797
8805
|
}
|
|
8798
8806
|
});
|
|
8799
8807
|
|
|
8808
|
+
// src/api/connected-usage-billing.ts
|
|
8809
|
+
import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
|
|
8810
|
+
import { z as z11 } from "zod";
|
|
8811
|
+
var CONNECTED_USAGE_BILLING_CLASS = "connected_usage";
|
|
8812
|
+
var CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE = "mastra_scheduler";
|
|
8813
|
+
var ConnectedUsageSafeMetadataSchema = z11.object({
|
|
8814
|
+
toolName: z11.string().trim().min(1).max(200).optional(),
|
|
8815
|
+
operationKind: z11.enum(["read", "action", "sync", "export", "proxy", "other"]).optional(),
|
|
8816
|
+
outcome: z11.enum(["success", "error", "partial"]).optional(),
|
|
8817
|
+
requestId: z11.string().trim().min(1).max(200).optional(),
|
|
8818
|
+
occurrenceId: z11.string().trim().min(1).max(200).optional(),
|
|
8819
|
+
scheduleId: z11.string().trim().min(1).max(200).optional(),
|
|
8820
|
+
nangoOperationId: z11.string().trim().min(1).max(200).optional(),
|
|
8821
|
+
startedAt: z11.string().datetime({ offset: true }).optional(),
|
|
8822
|
+
completedAt: z11.string().datetime({ offset: true }).optional()
|
|
8823
|
+
}).strict();
|
|
8824
|
+
var ConnectedUsageSettlementInputSchema = z11.object({
|
|
8825
|
+
identity: z11.string().trim().min(1).max(320),
|
|
8826
|
+
idempotencyKey: z11.string().trim().min(1).max(500),
|
|
8827
|
+
ratePolicyVersion: z11.string().trim().min(1).max(100),
|
|
8828
|
+
connectionId: z11.string().trim().min(1).max(500),
|
|
8829
|
+
providerConfigKey: z11.string().trim().min(1).max(200),
|
|
8830
|
+
functionRuns: z11.number().int().safe().nonnegative(),
|
|
8831
|
+
proxyRequests: z11.number().int().safe().nonnegative(),
|
|
8832
|
+
computeMilliseconds: z11.number().int().safe().nonnegative(),
|
|
8833
|
+
sourceSurface: z11.enum(["main_mcp", "mastra_scheduler", "nango_reconciliation"]).default(CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE),
|
|
8834
|
+
metadata: ConnectedUsageSafeMetadataSchema.optional()
|
|
8835
|
+
}).strict();
|
|
8836
|
+
var ConnectedUsagePreflightInputSchema = z11.object({
|
|
8837
|
+
identity: z11.string().trim().min(1).max(320),
|
|
8838
|
+
ratePolicyVersion: z11.string().trim().min(1).max(100)
|
|
8839
|
+
}).strict();
|
|
8840
|
+
var ConnectedUsageBillingError = class extends Error {
|
|
8841
|
+
constructor(code, message, httpStatus, details = {}) {
|
|
8842
|
+
super(message);
|
|
8843
|
+
this.code = code;
|
|
8844
|
+
this.httpStatus = httpStatus;
|
|
8845
|
+
this.details = details;
|
|
8846
|
+
this.name = "ConnectedUsageBillingError";
|
|
8847
|
+
}
|
|
8848
|
+
code;
|
|
8849
|
+
httpStatus;
|
|
8850
|
+
details;
|
|
8851
|
+
};
|
|
8852
|
+
function hash(value) {
|
|
8853
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
8854
|
+
}
|
|
8855
|
+
function eventKey(idempotencyKey) {
|
|
8856
|
+
return `connected-usage:${hash(idempotencyKey)}`;
|
|
8857
|
+
}
|
|
8858
|
+
function debitKey(idempotencyKey) {
|
|
8859
|
+
return `connected-usage-debit:${hash(idempotencyKey)}`;
|
|
8860
|
+
}
|
|
8861
|
+
function normalizeIdentity2(identity) {
|
|
8862
|
+
return identity.trim().toLowerCase();
|
|
8863
|
+
}
|
|
8864
|
+
async function resolveUser(identity) {
|
|
8865
|
+
const user = await getUserByEmail(normalizeIdentity2(identity));
|
|
8866
|
+
if (!user) throw new ConnectedUsageBillingError("user_not_found", "billing identity was not found", 404);
|
|
8867
|
+
return user;
|
|
8868
|
+
}
|
|
8869
|
+
function canonicalFingerprint(args) {
|
|
8870
|
+
const metadata = {
|
|
8871
|
+
toolName: args.metadata.toolName ?? null,
|
|
8872
|
+
operationKind: args.metadata.operationKind ?? null,
|
|
8873
|
+
outcome: args.metadata.outcome ?? null,
|
|
8874
|
+
requestId: args.metadata.requestId ?? null,
|
|
8875
|
+
occurrenceId: args.metadata.occurrenceId ?? null,
|
|
8876
|
+
scheduleId: args.metadata.scheduleId ?? null,
|
|
8877
|
+
nangoOperationId: args.metadata.nangoOperationId ?? null,
|
|
8878
|
+
startedAt: args.metadata.startedAt ?? null,
|
|
8879
|
+
completedAt: args.metadata.completedAt ?? null
|
|
8880
|
+
};
|
|
8881
|
+
return hash(JSON.stringify({
|
|
8882
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
8883
|
+
connectionId: args.connectionId,
|
|
8884
|
+
providerConfigKey: args.providerConfigKey,
|
|
8885
|
+
usage: args.usage,
|
|
8886
|
+
sourceSurface: args.sourceSurface,
|
|
8887
|
+
metadata
|
|
8888
|
+
}));
|
|
8889
|
+
}
|
|
8890
|
+
function encodeMetadata(metadata) {
|
|
8891
|
+
const encoded = JSON.stringify(metadata);
|
|
8892
|
+
if (encoded.length > 4e3) throw new ConnectedUsageBillingError("invalid_request", "connected usage metadata is too large", 400);
|
|
8893
|
+
return encoded;
|
|
8894
|
+
}
|
|
8895
|
+
function decodeMetadata(row) {
|
|
8896
|
+
try {
|
|
8897
|
+
const parsed = JSON.parse(row.metadata ?? "");
|
|
8898
|
+
if (parsed.contract !== "connected_usage_receipt" || parsed.ratePolicyVersion !== CONNECTED_USAGE_RATE_POLICY_VERSION || typeof parsed.fingerprint !== "string" || typeof parsed.connectionId !== "string" || typeof parsed.providerConfigKey !== "string" || !parsed.usage || !parsed.charge) {
|
|
8899
|
+
throw new Error("invalid connected usage receipt metadata");
|
|
8900
|
+
}
|
|
8901
|
+
return parsed;
|
|
8902
|
+
} catch {
|
|
8903
|
+
throw new ConnectedUsageBillingError("idempotency_key_conflict", "the idempotency key is already bound to an incompatible billing receipt", 409);
|
|
8904
|
+
}
|
|
8905
|
+
}
|
|
8906
|
+
async function readEvent(key) {
|
|
8907
|
+
const result = await getDb().execute({
|
|
8908
|
+
sql: "SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",
|
|
8909
|
+
args: [key]
|
|
8910
|
+
});
|
|
8911
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
8912
|
+
}
|
|
8913
|
+
async function hasOtherPendingReceipt(userId, excludingKey) {
|
|
8914
|
+
const result = await getDb().execute({
|
|
8915
|
+
sql: `SELECT 1
|
|
8916
|
+
FROM billing_events
|
|
8917
|
+
WHERE user_id = ?
|
|
8918
|
+
AND billing_class = ?
|
|
8919
|
+
AND status = 'cost_pending'
|
|
8920
|
+
${excludingKey ? "AND idempotency_key <> ?" : ""}
|
|
8921
|
+
LIMIT 1`,
|
|
8922
|
+
args: excludingKey ? [userId, CONNECTED_USAGE_BILLING_CLASS, excludingKey] : [userId, CONNECTED_USAGE_BILLING_CLASS]
|
|
8923
|
+
});
|
|
8924
|
+
return result.rows.length > 0;
|
|
8925
|
+
}
|
|
8926
|
+
function debitKeyFromEventKey(key) {
|
|
8927
|
+
const prefix = "connected-usage:";
|
|
8928
|
+
const digest = key.startsWith(prefix) ? key.slice(prefix.length) : "";
|
|
8929
|
+
return /^[a-f0-9]{64}$/.test(digest) ? `connected-usage-debit:${digest}` : null;
|
|
8930
|
+
}
|
|
8931
|
+
async function retryPendingConnectedUsageReceipts(user) {
|
|
8932
|
+
const result = await getDb().execute({
|
|
8933
|
+
sql: `SELECT id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata
|
|
8934
|
+
FROM billing_events
|
|
8935
|
+
WHERE user_id = ? AND billing_class = ? AND status = 'cost_pending'
|
|
8936
|
+
ORDER BY created_at ASC
|
|
8937
|
+
LIMIT 10`,
|
|
8938
|
+
args: [user.id, CONNECTED_USAGE_BILLING_CLASS]
|
|
8939
|
+
});
|
|
8940
|
+
for (const raw of result.rows) {
|
|
8941
|
+
const row = raw;
|
|
8942
|
+
const key = String(row.idempotency_key ?? "");
|
|
8943
|
+
const durableDebitKey = debitKeyFromEventKey(key);
|
|
8944
|
+
let metadata;
|
|
8945
|
+
try {
|
|
8946
|
+
metadata = decodeMetadata(row);
|
|
8947
|
+
} catch {
|
|
8948
|
+
return;
|
|
8949
|
+
}
|
|
8950
|
+
if (!durableDebitKey || Number(row.user_id) !== user.id || row.billing_class !== CONNECTED_USAGE_BILLING_CLASS || Number(row.amount_mc) !== metadata.charge.amountMc) {
|
|
8951
|
+
return;
|
|
8952
|
+
}
|
|
8953
|
+
if (metadata.charge.amountMc > 0) {
|
|
8954
|
+
const debit = await debitMcIdempotent(
|
|
8955
|
+
user.id,
|
|
8956
|
+
metadata.charge.amountMc,
|
|
8957
|
+
LedgerOperation.CONNECTED_USAGE,
|
|
8958
|
+
`connected-usage:${metadata.providerConfigKey}`,
|
|
8959
|
+
durableDebitKey
|
|
8960
|
+
);
|
|
8961
|
+
if (!debit.ok) return;
|
|
8962
|
+
}
|
|
8963
|
+
const settledMetadata = { ...metadata };
|
|
8964
|
+
delete settledMetadata.reason;
|
|
8965
|
+
await getDb().execute({
|
|
8966
|
+
sql: `UPDATE billing_events
|
|
8967
|
+
SET status = 'settled', metadata = ?, settled_at = datetime('now')
|
|
8968
|
+
WHERE idempotency_key = ? AND user_id = ? AND billing_class = ? AND status = 'cost_pending'`,
|
|
8969
|
+
args: [encodeMetadata(settledMetadata), key, user.id, CONNECTED_USAGE_BILLING_CLASS]
|
|
8970
|
+
});
|
|
8971
|
+
}
|
|
8972
|
+
}
|
|
8973
|
+
function assertRatePolicyVersion(ratePolicyVersion) {
|
|
8974
|
+
if (ratePolicyVersion !== CONNECTED_USAGE_RATE_POLICY_VERSION) {
|
|
8975
|
+
throw new ConnectedUsageBillingError(
|
|
8976
|
+
"rate_policy_mismatch",
|
|
8977
|
+
"connected usage receipt rate policy does not match the active policy",
|
|
8978
|
+
409,
|
|
8979
|
+
{ expectedRatePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION }
|
|
8980
|
+
);
|
|
8981
|
+
}
|
|
8982
|
+
}
|
|
8983
|
+
async function preflightConnectedUsage(rawInput) {
|
|
8984
|
+
const parsed = ConnectedUsagePreflightInputSchema.safeParse(rawInput);
|
|
8985
|
+
if (!parsed.success) {
|
|
8986
|
+
throw new ConnectedUsageBillingError("invalid_request", "invalid connected usage preflight request", 400);
|
|
8987
|
+
}
|
|
8988
|
+
assertRatePolicyVersion(parsed.data.ratePolicyVersion);
|
|
8989
|
+
const user = await resolveUser(parsed.data.identity);
|
|
8990
|
+
await retryPendingConnectedUsageReceipts(user);
|
|
8991
|
+
if (await hasOtherPendingReceipt(user.id)) {
|
|
8992
|
+
throw new ConnectedUsageBillingError(
|
|
8993
|
+
"pending_cost",
|
|
8994
|
+
"a previous connected usage receipt has unresolved cost",
|
|
8995
|
+
409
|
|
8996
|
+
);
|
|
8997
|
+
}
|
|
8998
|
+
const balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8999
|
+
if (balanceMc < CONNECTED_FUNCTION_RUN_MC) {
|
|
9000
|
+
throw new ConnectedUsageBillingError(
|
|
9001
|
+
"insufficient_credits",
|
|
9002
|
+
"the wallet cannot cover the minimum connected function charge",
|
|
9003
|
+
402,
|
|
9004
|
+
{ balanceMc, requiredMc: CONNECTED_FUNCTION_RUN_MC }
|
|
9005
|
+
);
|
|
9006
|
+
}
|
|
9007
|
+
return {
|
|
9008
|
+
allowed: true,
|
|
9009
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
9010
|
+
minimumMc: CONNECTED_FUNCTION_RUN_MC,
|
|
9011
|
+
minimumCredits: CONNECTED_FUNCTION_RUN_CREDITS,
|
|
9012
|
+
balanceMc
|
|
9013
|
+
};
|
|
9014
|
+
}
|
|
9015
|
+
function assertMatchingReceipt(row, userId, sourceSurface, fingerprint, amountMc) {
|
|
9016
|
+
const metadata = decodeMetadata(row);
|
|
9017
|
+
if (Number(row.user_id) !== userId || row.billing_class !== CONNECTED_USAGE_BILLING_CLASS || row.source_surface !== sourceSurface || Number(row.amount_mc) !== amountMc || metadata.fingerprint !== fingerprint) {
|
|
9018
|
+
throw new ConnectedUsageBillingError("idempotency_key_conflict", "the idempotency key is already bound to different connected usage", 409);
|
|
9019
|
+
}
|
|
9020
|
+
return metadata;
|
|
9021
|
+
}
|
|
9022
|
+
function costBreakdown(usage, charge2) {
|
|
9023
|
+
return [
|
|
9024
|
+
{
|
|
9025
|
+
key: "function_runs",
|
|
9026
|
+
label: "Connected function runs",
|
|
9027
|
+
quantity: usage.functionRuns,
|
|
9028
|
+
quantityUnit: "function_run",
|
|
9029
|
+
rateCredits: CONNECTED_FUNCTION_RUN_CREDITS,
|
|
9030
|
+
rateUnit: "function_run",
|
|
9031
|
+
amountMc: charge2.functionRunMc,
|
|
9032
|
+
credits: charge2.functionRunMc / MC_PER_CREDIT
|
|
9033
|
+
},
|
|
9034
|
+
{
|
|
9035
|
+
key: "proxy_requests",
|
|
9036
|
+
label: "Connected proxy requests",
|
|
9037
|
+
quantity: usage.proxyRequests,
|
|
9038
|
+
quantityUnit: "proxy_request",
|
|
9039
|
+
rateCredits: CONNECTED_PROXY_REQUEST_CREDITS,
|
|
9040
|
+
rateUnit: "proxy_request",
|
|
9041
|
+
amountMc: charge2.proxyRequestMc,
|
|
9042
|
+
credits: charge2.proxyRequestMc / MC_PER_CREDIT
|
|
9043
|
+
},
|
|
9044
|
+
{
|
|
9045
|
+
key: "compute",
|
|
9046
|
+
label: "Connected function compute",
|
|
9047
|
+
quantity: usage.computeMilliseconds,
|
|
9048
|
+
quantityUnit: "millisecond",
|
|
9049
|
+
rateCredits: CONNECTED_COMPUTE_CREDITS_PER_SECOND,
|
|
9050
|
+
rateUnit: "compute_second",
|
|
9051
|
+
amountMc: charge2.computeMc,
|
|
9052
|
+
credits: charge2.computeMc / MC_PER_CREDIT
|
|
9053
|
+
}
|
|
9054
|
+
];
|
|
9055
|
+
}
|
|
9056
|
+
function mapSettlement(row, metadata, balanceMc, duplicate) {
|
|
9057
|
+
const reason = row.status === "cost_pending" ? metadata.reason : void 0;
|
|
9058
|
+
return {
|
|
9059
|
+
id: row.id,
|
|
9060
|
+
status: row.status === "settled" ? "settled" : "cost_pending",
|
|
9061
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
9062
|
+
connectionId: metadata.connectionId,
|
|
9063
|
+
providerConfigKey: metadata.providerConfigKey,
|
|
9064
|
+
amountMc: metadata.charge.amountMc,
|
|
9065
|
+
credits: metadata.charge.credits,
|
|
9066
|
+
functionRunMc: metadata.charge.functionRunMc,
|
|
9067
|
+
proxyRequestMc: metadata.charge.proxyRequestMc,
|
|
9068
|
+
computeMc: metadata.charge.computeMc,
|
|
9069
|
+
usage: metadata.usage,
|
|
9070
|
+
costBreakdown: costBreakdown(metadata.usage, metadata.charge),
|
|
9071
|
+
balanceMc,
|
|
9072
|
+
...reason ? { reason } : {},
|
|
9073
|
+
duplicate
|
|
9074
|
+
};
|
|
9075
|
+
}
|
|
9076
|
+
async function settleConnectedUsage(rawInput) {
|
|
9077
|
+
const parsed = ConnectedUsageSettlementInputSchema.safeParse(rawInput);
|
|
9078
|
+
if (!parsed.success) {
|
|
9079
|
+
throw new ConnectedUsageBillingError(
|
|
9080
|
+
"invalid_request",
|
|
9081
|
+
"invalid connected usage settlement request",
|
|
9082
|
+
400,
|
|
9083
|
+
{ fields: [...new Set(parsed.error.issues.map((issue) => issue.path.join(".")).filter(Boolean))] }
|
|
9084
|
+
);
|
|
9085
|
+
}
|
|
9086
|
+
const input = parsed.data;
|
|
9087
|
+
assertRatePolicyVersion(input.ratePolicyVersion);
|
|
9088
|
+
const user = await resolveUser(input.identity);
|
|
9089
|
+
const usage = {
|
|
9090
|
+
functionRuns: input.functionRuns,
|
|
9091
|
+
proxyRequests: input.proxyRequests,
|
|
9092
|
+
computeMilliseconds: input.computeMilliseconds
|
|
9093
|
+
};
|
|
9094
|
+
const charge2 = calculateConnectedUsageCharge(usage);
|
|
9095
|
+
const safeMetadata3 = input.metadata ?? {};
|
|
9096
|
+
const fingerprint = canonicalFingerprint({
|
|
9097
|
+
connectionId: input.connectionId,
|
|
9098
|
+
providerConfigKey: input.providerConfigKey,
|
|
9099
|
+
usage,
|
|
9100
|
+
sourceSurface: input.sourceSurface,
|
|
9101
|
+
metadata: safeMetadata3
|
|
9102
|
+
});
|
|
9103
|
+
const storedMetadata = {
|
|
9104
|
+
contract: "connected_usage_receipt",
|
|
9105
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
9106
|
+
fingerprint,
|
|
9107
|
+
connectionId: input.connectionId,
|
|
9108
|
+
providerConfigKey: input.providerConfigKey,
|
|
9109
|
+
usage,
|
|
9110
|
+
charge: charge2,
|
|
9111
|
+
safeMetadata: safeMetadata3
|
|
9112
|
+
};
|
|
9113
|
+
const key = eventKey(input.idempotencyKey);
|
|
9114
|
+
if (await hasOtherPendingReceipt(user.id, key)) {
|
|
9115
|
+
throw new ConnectedUsageBillingError(
|
|
9116
|
+
"pending_cost",
|
|
9117
|
+
"a previous connected usage receipt has unresolved cost",
|
|
9118
|
+
409
|
|
9119
|
+
);
|
|
9120
|
+
}
|
|
9121
|
+
const inserted = await getDb().execute({
|
|
9122
|
+
sql: `INSERT OR IGNORE INTO billing_events
|
|
9123
|
+
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
9124
|
+
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
9125
|
+
args: [randomUUID2(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
|
|
9126
|
+
});
|
|
9127
|
+
let event = await readEvent(key);
|
|
9128
|
+
if (!event) throw new Error("connected usage receipt insert completed without a readable event");
|
|
9129
|
+
let eventMetadata = assertMatchingReceipt(event, user.id, input.sourceSurface, fingerprint, charge2.amountMc);
|
|
9130
|
+
const duplicate = inserted.rowsAffected === 0;
|
|
9131
|
+
if (event.status === "settled") {
|
|
9132
|
+
return mapSettlement(event, eventMetadata, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
9133
|
+
}
|
|
9134
|
+
if (!["settling", "cost_pending"].includes(event.status)) {
|
|
9135
|
+
throw new ConnectedUsageBillingError("idempotency_key_conflict", "the connected usage receipt has an incompatible status", 409);
|
|
9136
|
+
}
|
|
9137
|
+
if (event.status === "cost_pending") {
|
|
9138
|
+
await getDb().execute({
|
|
9139
|
+
sql: "UPDATE billing_events SET status = 'settling', metadata = ? WHERE idempotency_key = ? AND status = 'cost_pending'",
|
|
9140
|
+
args: [encodeMetadata(storedMetadata), key]
|
|
9141
|
+
});
|
|
9142
|
+
}
|
|
9143
|
+
if (charge2.amountMc === 0) {
|
|
9144
|
+
await getDb().execute({
|
|
9145
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE idempotency_key = ? AND status = 'settling'",
|
|
9146
|
+
args: [key]
|
|
9147
|
+
});
|
|
9148
|
+
} else {
|
|
9149
|
+
const debit = await debitMcIdempotent(
|
|
9150
|
+
user.id,
|
|
9151
|
+
charge2.amountMc,
|
|
9152
|
+
LedgerOperation.CONNECTED_USAGE,
|
|
9153
|
+
`connected-usage:${input.providerConfigKey}`,
|
|
9154
|
+
debitKey(input.idempotencyKey)
|
|
9155
|
+
);
|
|
9156
|
+
if (!debit.ok) {
|
|
9157
|
+
const pendingMetadata = { ...storedMetadata, reason: "insufficient_credits" };
|
|
9158
|
+
await getDb().execute({
|
|
9159
|
+
sql: "UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE idempotency_key = ? AND status = 'settling'",
|
|
9160
|
+
args: [encodeMetadata(pendingMetadata), key]
|
|
9161
|
+
});
|
|
9162
|
+
event = await readEvent(key);
|
|
9163
|
+
if (!event) throw new Error("connected usage pending receipt was lost");
|
|
9164
|
+
eventMetadata = assertMatchingReceipt(event, user.id, input.sourceSurface, fingerprint, charge2.amountMc);
|
|
9165
|
+
return mapSettlement(event, eventMetadata, debit.balance_mc, duplicate || debit.duplicate);
|
|
9166
|
+
}
|
|
9167
|
+
await getDb().execute({
|
|
9168
|
+
sql: "UPDATE billing_events SET status = 'settled', metadata = ?, settled_at = datetime('now') WHERE idempotency_key = ? AND status IN ('settling', 'cost_pending')",
|
|
9169
|
+
args: [encodeMetadata(storedMetadata), key]
|
|
9170
|
+
});
|
|
9171
|
+
}
|
|
9172
|
+
event = await readEvent(key);
|
|
9173
|
+
if (!event) throw new Error("connected usage settlement completed without an event receipt");
|
|
9174
|
+
eventMetadata = assertMatchingReceipt(event, user.id, input.sourceSurface, fingerprint, charge2.amountMc);
|
|
9175
|
+
return mapSettlement(
|
|
9176
|
+
event,
|
|
9177
|
+
eventMetadata,
|
|
9178
|
+
await reconcileBalanceMc(user.id, user.balance_mc),
|
|
9179
|
+
duplicate
|
|
9180
|
+
);
|
|
9181
|
+
}
|
|
9182
|
+
async function listConnectedUsageHistory(userId, limit = 100) {
|
|
9183
|
+
const result = await getDb().execute({
|
|
9184
|
+
sql: `SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata, created_at
|
|
9185
|
+
FROM billing_events
|
|
9186
|
+
WHERE user_id = ? AND billing_class = ? AND status IN ('settled', 'cost_pending')
|
|
9187
|
+
ORDER BY created_at DESC
|
|
9188
|
+
LIMIT ?`,
|
|
9189
|
+
args: [userId, CONNECTED_USAGE_BILLING_CLASS, Math.max(1, Math.min(250, Math.round(limit)))]
|
|
9190
|
+
});
|
|
9191
|
+
const history = [];
|
|
9192
|
+
for (const raw of result.rows) {
|
|
9193
|
+
const row = raw;
|
|
9194
|
+
try {
|
|
9195
|
+
const receipt = decodeMetadata(row);
|
|
9196
|
+
history.push({
|
|
9197
|
+
id: row.id,
|
|
9198
|
+
createdAt: String(row.created_at ?? ""),
|
|
9199
|
+
status: row.status === "settled" ? "settled" : "cost_pending",
|
|
9200
|
+
connectionId: receipt.connectionId,
|
|
9201
|
+
providerConfigKey: receipt.providerConfigKey,
|
|
9202
|
+
amountMc: receipt.charge.amountMc,
|
|
9203
|
+
credits: receipt.charge.credits,
|
|
9204
|
+
usage: receipt.usage,
|
|
9205
|
+
costBreakdown: costBreakdown(receipt.usage, receipt.charge),
|
|
9206
|
+
metadata: receipt.safeMetadata
|
|
9207
|
+
});
|
|
9208
|
+
} catch {
|
|
9209
|
+
}
|
|
9210
|
+
}
|
|
9211
|
+
return history;
|
|
9212
|
+
}
|
|
9213
|
+
|
|
8800
9214
|
// src/api/youtube-routes.ts
|
|
8801
9215
|
import { Hono as Hono4 } from "hono";
|
|
8802
9216
|
|
|
@@ -8806,17 +9220,17 @@ import path4 from "path";
|
|
|
8806
9220
|
import Papa from "papaparse";
|
|
8807
9221
|
|
|
8808
9222
|
// src/youtube/schemas.ts
|
|
8809
|
-
import { z as
|
|
8810
|
-
var YouTubeHarvestOptionsSchema =
|
|
8811
|
-
mode:
|
|
8812
|
-
query:
|
|
8813
|
-
channelHandle:
|
|
8814
|
-
maxVideos:
|
|
8815
|
-
downloadMp3:
|
|
8816
|
-
outputDir:
|
|
8817
|
-
kernelApiKey:
|
|
8818
|
-
kernelProxyId:
|
|
8819
|
-
headless:
|
|
9223
|
+
import { z as z12 } from "zod";
|
|
9224
|
+
var YouTubeHarvestOptionsSchema = z12.object({
|
|
9225
|
+
mode: z12.enum(["search", "channel"]),
|
|
9226
|
+
query: z12.string().optional(),
|
|
9227
|
+
channelHandle: z12.string().optional(),
|
|
9228
|
+
maxVideos: z12.number().int().min(1).max(5e3).default(100),
|
|
9229
|
+
downloadMp3: z12.boolean().default(false),
|
|
9230
|
+
outputDir: z12.string().default("./yt-output"),
|
|
9231
|
+
kernelApiKey: z12.string().optional(),
|
|
9232
|
+
kernelProxyId: z12.string().optional(),
|
|
9233
|
+
headless: z12.boolean().default(false)
|
|
8820
9234
|
}).refine(
|
|
8821
9235
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
8822
9236
|
{ message: "search mode requires query; channel mode requires channelHandle" }
|
|
@@ -9357,14 +9771,14 @@ async function fetchViaYoutubeTranscript(videoId) {
|
|
|
9357
9771
|
timestamp: [e.offset / 1e3, (e.offset + e.duration) / 1e3],
|
|
9358
9772
|
text: e.text.trim()
|
|
9359
9773
|
}));
|
|
9360
|
-
const
|
|
9774
|
+
const text2 = chunks.map((c) => c.text).join(" ");
|
|
9361
9775
|
const last = chunks[chunks.length - 1];
|
|
9362
9776
|
const now = Date.now();
|
|
9363
9777
|
await runWithCostContext(
|
|
9364
9778
|
{ ...currentCostContext(), op: "yt_transcription", subOp: "youtube-transcript" },
|
|
9365
9779
|
() => recordKernelSession({ kernelSessionId: null, source: "yt_caption_tier", headlessSent: null, openedAtMs: now, closedAtMs: now })
|
|
9366
9780
|
);
|
|
9367
|
-
return { videoId, text, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
|
|
9781
|
+
return { videoId, text: text2, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
|
|
9368
9782
|
} catch {
|
|
9369
9783
|
return null;
|
|
9370
9784
|
}
|
|
@@ -9379,12 +9793,12 @@ function parseTimedtextJson3(jsonText) {
|
|
|
9379
9793
|
const results = [];
|
|
9380
9794
|
for (const ev of data.events ?? []) {
|
|
9381
9795
|
if (!ev.segs) continue;
|
|
9382
|
-
const
|
|
9383
|
-
if (!
|
|
9796
|
+
const text2 = ev.segs.map((s) => s.utf8 ?? "").join("").replace(/\s+/g, " ").trim();
|
|
9797
|
+
if (!text2) continue;
|
|
9384
9798
|
const start = typeof ev.tStartMs === "number" ? ev.tStartMs : NaN;
|
|
9385
9799
|
const dur = typeof ev.dDurationMs === "number" ? ev.dDurationMs : 0;
|
|
9386
9800
|
if (!Number.isFinite(start)) continue;
|
|
9387
|
-
results.push({ start, dur, text });
|
|
9801
|
+
results.push({ start, dur, text: text2 });
|
|
9388
9802
|
}
|
|
9389
9803
|
return results;
|
|
9390
9804
|
}
|
|
@@ -9396,22 +9810,22 @@ function parseTimedtextXml(xml) {
|
|
|
9396
9810
|
const startMs = parseInt(m[1], 10);
|
|
9397
9811
|
const durMs = parseInt(m[2], 10);
|
|
9398
9812
|
const inner = m[3];
|
|
9399
|
-
let
|
|
9400
|
-
if (!
|
|
9813
|
+
let text2 = inner.replace(/<[^>]+>/g, "").trim();
|
|
9814
|
+
if (!text2) {
|
|
9401
9815
|
const sMatch = /<s[^>]*>([^<]*)<\/s>/g;
|
|
9402
9816
|
let sm;
|
|
9403
9817
|
const parts = [];
|
|
9404
9818
|
while ((sm = sMatch.exec(inner)) !== null) parts.push(sm[1]);
|
|
9405
|
-
|
|
9819
|
+
text2 = parts.join("").trim();
|
|
9406
9820
|
}
|
|
9407
|
-
|
|
9408
|
-
if (
|
|
9821
|
+
text2 = text2.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
9822
|
+
if (text2) results.push({ start: startMs, dur: durMs, text: text2 });
|
|
9409
9823
|
}
|
|
9410
9824
|
if (results.length > 0) return results;
|
|
9411
9825
|
const classicRegex = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
|
|
9412
9826
|
while ((m = classicRegex.exec(xml)) !== null) {
|
|
9413
|
-
const
|
|
9414
|
-
results.push({ start: parseFloat(m[1]) * 1e3, dur: parseFloat(m[2]) * 1e3, text });
|
|
9827
|
+
const text2 = m[3].replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
9828
|
+
results.push({ start: parseFloat(m[1]) * 1e3, dur: parseFloat(m[2]) * 1e3, text: text2 });
|
|
9415
9829
|
}
|
|
9416
9830
|
return results;
|
|
9417
9831
|
}
|
|
@@ -9536,8 +9950,8 @@ async function fetchViaKernelSession(videoId) {
|
|
|
9536
9950
|
timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
|
|
9537
9951
|
text: e.text
|
|
9538
9952
|
}));
|
|
9539
|
-
const
|
|
9540
|
-
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
9953
|
+
const text2 = chunks.map((c) => c.text).join(" ");
|
|
9954
|
+
return { videoId, text: text2, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
9541
9955
|
}
|
|
9542
9956
|
async function fetchViaResolvedAudio(videoId) {
|
|
9543
9957
|
if (!mediaResolverConfigured()) return null;
|
|
@@ -9602,68 +10016,68 @@ function createApiKeyAuth() {
|
|
|
9602
10016
|
}
|
|
9603
10017
|
|
|
9604
10018
|
// src/api/server-schemas.ts
|
|
9605
|
-
import { z as
|
|
9606
|
-
var HarvestBodySchema =
|
|
9607
|
-
query:
|
|
9608
|
-
location:
|
|
9609
|
-
depth:
|
|
9610
|
-
maxQuestions:
|
|
9611
|
-
callback_url:
|
|
9612
|
-
gl:
|
|
9613
|
-
hl:
|
|
9614
|
-
device:
|
|
9615
|
-
proxyMode:
|
|
9616
|
-
proxyZip:
|
|
9617
|
-
debug:
|
|
9618
|
-
serpOnly:
|
|
9619
|
-
pages:
|
|
9620
|
-
});
|
|
9621
|
-
var ExtractUrlBodySchema =
|
|
9622
|
-
url:
|
|
9623
|
-
screenshot:
|
|
9624
|
-
screenshotDevice:
|
|
9625
|
-
extractBranding:
|
|
9626
|
-
downloadMedia:
|
|
9627
|
-
mediaTypes:
|
|
9628
|
-
allowLocal:
|
|
9629
|
-
depositToVault:
|
|
9630
|
-
vaultName:
|
|
9631
|
-
});
|
|
9632
|
-
var DiffPageBodySchema =
|
|
9633
|
-
url:
|
|
9634
|
-
allowLocal:
|
|
9635
|
-
resetBaseline:
|
|
9636
|
-
});
|
|
9637
|
-
var MapUrlsBodySchema =
|
|
9638
|
-
url:
|
|
9639
|
-
maxUrls:
|
|
9640
|
-
concurrency:
|
|
9641
|
-
browserFallback:
|
|
9642
|
-
kernelFallback:
|
|
9643
|
-
});
|
|
9644
|
-
var ExtractSiteBodySchema =
|
|
9645
|
-
url:
|
|
9646
|
-
maxPages:
|
|
9647
|
-
browserFallback:
|
|
9648
|
-
kernelFallback:
|
|
9649
|
-
rotateProxies:
|
|
9650
|
-
rotateProxyEvery:
|
|
9651
|
-
background:
|
|
9652
|
-
downloadImages:
|
|
9653
|
-
formats:
|
|
9654
|
-
});
|
|
9655
|
-
var YoutubeHarvestBodySchema =
|
|
9656
|
-
mode:
|
|
9657
|
-
query:
|
|
9658
|
-
channelHandle:
|
|
9659
|
-
maxVideos:
|
|
9660
|
-
});
|
|
9661
|
-
var YoutubeTranscribeBodySchema =
|
|
9662
|
-
videoId:
|
|
10019
|
+
import { z as z13 } from "zod";
|
|
10020
|
+
var HarvestBodySchema = z13.object({
|
|
10021
|
+
query: z13.string().min(1, "query is required"),
|
|
10022
|
+
location: z13.string().optional(),
|
|
10023
|
+
depth: z13.number().int().min(1).max(4).optional(),
|
|
10024
|
+
maxQuestions: z13.number().int().min(1).optional(),
|
|
10025
|
+
callback_url: z13.string().url().optional(),
|
|
10026
|
+
gl: z13.string().optional(),
|
|
10027
|
+
hl: z13.string().optional(),
|
|
10028
|
+
device: z13.enum(["desktop", "mobile"]).optional(),
|
|
10029
|
+
proxyMode: z13.enum(["location", "configured", "none"]).optional(),
|
|
10030
|
+
proxyZip: z13.string().regex(/^\d{5}$/).optional(),
|
|
10031
|
+
debug: z13.boolean().optional(),
|
|
10032
|
+
serpOnly: z13.boolean().optional(),
|
|
10033
|
+
pages: z13.number().int().min(1).max(2).optional()
|
|
10034
|
+
});
|
|
10035
|
+
var ExtractUrlBodySchema = z13.object({
|
|
10036
|
+
url: z13.string().min(1, "url is required"),
|
|
10037
|
+
screenshot: z13.boolean().optional(),
|
|
10038
|
+
screenshotDevice: z13.enum(["desktop", "mobile"]).optional(),
|
|
10039
|
+
extractBranding: z13.boolean().optional(),
|
|
10040
|
+
downloadMedia: z13.boolean().optional(),
|
|
10041
|
+
mediaTypes: z13.array(z13.enum(["image", "video", "audio"])).optional(),
|
|
10042
|
+
allowLocal: z13.boolean().optional(),
|
|
10043
|
+
depositToVault: z13.boolean().optional(),
|
|
10044
|
+
vaultName: z13.string().trim().min(1).max(120).optional()
|
|
10045
|
+
});
|
|
10046
|
+
var DiffPageBodySchema = z13.object({
|
|
10047
|
+
url: z13.string().min(1, "url is required"),
|
|
10048
|
+
allowLocal: z13.boolean().optional(),
|
|
10049
|
+
resetBaseline: z13.boolean().optional()
|
|
10050
|
+
});
|
|
10051
|
+
var MapUrlsBodySchema = z13.object({
|
|
10052
|
+
url: z13.string().min(1, "url is required"),
|
|
10053
|
+
maxUrls: z13.number().int().min(1).max(2e3).optional(),
|
|
10054
|
+
concurrency: z13.number().int().min(1).max(20).optional(),
|
|
10055
|
+
browserFallback: z13.boolean().optional(),
|
|
10056
|
+
kernelFallback: z13.boolean().optional()
|
|
10057
|
+
});
|
|
10058
|
+
var ExtractSiteBodySchema = z13.object({
|
|
10059
|
+
url: z13.string().min(1, "url is required"),
|
|
10060
|
+
maxPages: z13.number().int().min(1).max(1e4).optional(),
|
|
10061
|
+
browserFallback: z13.boolean().optional(),
|
|
10062
|
+
kernelFallback: z13.boolean().optional(),
|
|
10063
|
+
rotateProxies: z13.boolean().optional(),
|
|
10064
|
+
rotateProxyEvery: z13.number().int().min(1).max(100).optional(),
|
|
10065
|
+
background: z13.boolean().optional(),
|
|
10066
|
+
downloadImages: z13.boolean().optional(),
|
|
10067
|
+
formats: z13.array(z13.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
10068
|
+
});
|
|
10069
|
+
var YoutubeHarvestBodySchema = z13.object({
|
|
10070
|
+
mode: z13.enum(["search", "channel"]),
|
|
10071
|
+
query: z13.string().optional(),
|
|
10072
|
+
channelHandle: z13.string().optional(),
|
|
10073
|
+
maxVideos: z13.number().int().min(1).max(500).optional()
|
|
10074
|
+
});
|
|
10075
|
+
var YoutubeTranscribeBodySchema = z13.object({
|
|
10076
|
+
videoId: z13.string().min(1, "videoId is required")
|
|
9663
10077
|
});
|
|
9664
10078
|
|
|
9665
10079
|
// src/api/concurrency-gates.ts
|
|
9666
|
-
import { randomUUID as
|
|
10080
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
9667
10081
|
var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
9668
10082
|
var DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
9669
10083
|
var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
@@ -9781,7 +10195,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
9781
10195
|
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
9782
10196
|
}
|
|
9783
10197
|
await expireConcurrencyLocksForUser(user.id);
|
|
9784
|
-
const lockId = `cl_${
|
|
10198
|
+
const lockId = `cl_${randomUUID3().replace(/-/g, "").slice(0, 24)}`;
|
|
9785
10199
|
const res = await getDb().execute({
|
|
9786
10200
|
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
9787
10201
|
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
@@ -9997,11 +10411,11 @@ function fmtTs(secs) {
|
|
|
9997
10411
|
|
|
9998
10412
|
// src/api/screenshot-routes.ts
|
|
9999
10413
|
import { Hono as Hono5 } from "hono";
|
|
10000
|
-
import { z as
|
|
10001
|
-
var ScreenshotBodySchema =
|
|
10002
|
-
url:
|
|
10003
|
-
device:
|
|
10004
|
-
allowLocal:
|
|
10414
|
+
import { z as z14 } from "zod";
|
|
10415
|
+
var ScreenshotBodySchema = z14.object({
|
|
10416
|
+
url: z14.string().trim().min(1, "url is required"),
|
|
10417
|
+
device: z14.string().trim().optional(),
|
|
10418
|
+
allowLocal: z14.boolean().optional()
|
|
10005
10419
|
});
|
|
10006
10420
|
var screenshotApp = new Hono5();
|
|
10007
10421
|
screenshotApp.use("*", createApiKeyAuth());
|
|
@@ -10061,7 +10475,7 @@ screenshotApp.post("/", async (c) => {
|
|
|
10061
10475
|
|
|
10062
10476
|
// src/api/facebook-ad-routes.ts
|
|
10063
10477
|
import { Hono as Hono6 } from "hono";
|
|
10064
|
-
import { z as
|
|
10478
|
+
import { z as z15 } from "zod";
|
|
10065
10479
|
|
|
10066
10480
|
// src/extractor/FacebookAdExtractor.ts
|
|
10067
10481
|
var SPRITE_Y_TO_PLATFORM = {
|
|
@@ -10733,9 +11147,9 @@ var FacebookAdExtractor = class {
|
|
|
10733
11147
|
|
|
10734
11148
|
// src/extractor/FacebookAdGraphql.ts
|
|
10735
11149
|
var AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
|
|
10736
|
-
function parseFbGraphqlJson(
|
|
11150
|
+
function parseFbGraphqlJson(text2) {
|
|
10737
11151
|
const out = [];
|
|
10738
|
-
const body =
|
|
11152
|
+
const body = text2.replace(/^for\s*\(;;\);/, "").trim();
|
|
10739
11153
|
try {
|
|
10740
11154
|
out.push(JSON.parse(body));
|
|
10741
11155
|
return out;
|
|
@@ -10783,8 +11197,8 @@ async function collectAdLibraryResults(page, url, maxResults, opts = {}) {
|
|
|
10783
11197
|
if (!resp.url().includes("/api/graphql")) return;
|
|
10784
11198
|
const friendlyName = (resp.request().postData() ?? "").match(/fb_api_req_friendly_name=([^&]+)/)?.[1];
|
|
10785
11199
|
if (friendlyName !== AD_LIBRARY_QUERY) return;
|
|
10786
|
-
void resp.text().then((
|
|
10787
|
-
for (const payload of parseFbGraphqlJson(
|
|
11200
|
+
void resp.text().then((text2) => {
|
|
11201
|
+
for (const payload of parseFbGraphqlJson(text2)) {
|
|
10788
11202
|
for (const result of extractCollatedResults(payload)) {
|
|
10789
11203
|
if (seen.has(result.ad_archive_id)) continue;
|
|
10790
11204
|
seen.add(result.ad_archive_id);
|
|
@@ -10954,35 +11368,35 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
10954
11368
|
|
|
10955
11369
|
// src/api/facebook-ad-routes.ts
|
|
10956
11370
|
import { fal } from "@fal-ai/client";
|
|
10957
|
-
var FacebookAdBodySchema =
|
|
10958
|
-
url:
|
|
10959
|
-
libraryId:
|
|
10960
|
-
openModal:
|
|
11371
|
+
var FacebookAdBodySchema = z15.object({
|
|
11372
|
+
url: z15.string().trim().optional(),
|
|
11373
|
+
libraryId: z15.string().trim().optional(),
|
|
11374
|
+
openModal: z15.boolean().optional()
|
|
10961
11375
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
10962
|
-
var FacebookPageIntelBodySchema =
|
|
10963
|
-
pageId:
|
|
10964
|
-
query:
|
|
10965
|
-
libraryId:
|
|
10966
|
-
maxAds:
|
|
10967
|
-
country:
|
|
11376
|
+
var FacebookPageIntelBodySchema = z15.object({
|
|
11377
|
+
pageId: z15.string().trim().optional(),
|
|
11378
|
+
query: z15.string().trim().optional(),
|
|
11379
|
+
libraryId: z15.string().trim().optional(),
|
|
11380
|
+
maxAds: z15.number().int().min(1).max(200).optional(),
|
|
11381
|
+
country: z15.string().trim().toUpperCase().optional()
|
|
10968
11382
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
10969
11383
|
message: "pageId, libraryId, or query is required"
|
|
10970
11384
|
});
|
|
10971
|
-
var FacebookTranscribeBodySchema =
|
|
10972
|
-
videoUrl:
|
|
11385
|
+
var FacebookTranscribeBodySchema = z15.object({
|
|
11386
|
+
videoUrl: z15.string().trim().min(1, "videoUrl is required")
|
|
10973
11387
|
});
|
|
10974
|
-
var FacebookVideoTranscribeBodySchema =
|
|
10975
|
-
url:
|
|
10976
|
-
quality:
|
|
11388
|
+
var FacebookVideoTranscribeBodySchema = z15.object({
|
|
11389
|
+
url: z15.string().trim().min(1, "url is required"),
|
|
11390
|
+
quality: z15.enum(["best", "hd", "sd"]).default("best")
|
|
10977
11391
|
});
|
|
10978
|
-
var FacebookSearchBodySchema =
|
|
10979
|
-
query:
|
|
10980
|
-
country:
|
|
10981
|
-
maxResults:
|
|
11392
|
+
var FacebookSearchBodySchema = z15.object({
|
|
11393
|
+
query: z15.string().trim().min(1, "query is required"),
|
|
11394
|
+
country: z15.string().trim().toUpperCase().optional(),
|
|
11395
|
+
maxResults: z15.number().int().min(1).max(20).optional()
|
|
10982
11396
|
});
|
|
10983
|
-
var FacebookMediaBodySchema =
|
|
10984
|
-
url:
|
|
10985
|
-
filename:
|
|
11397
|
+
var FacebookMediaBodySchema = z15.object({
|
|
11398
|
+
url: z15.string().trim().min(1, "url is required"),
|
|
11399
|
+
filename: z15.string().trim().optional()
|
|
10986
11400
|
});
|
|
10987
11401
|
function invalidRequest(message) {
|
|
10988
11402
|
return { error_code: "invalid_request", message };
|
|
@@ -11416,7 +11830,7 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
|
|
|
11416
11830
|
|
|
11417
11831
|
// src/api/google-ads-routes.ts
|
|
11418
11832
|
import { Hono as Hono7 } from "hono";
|
|
11419
|
-
import { z as
|
|
11833
|
+
import { z as z16 } from "zod";
|
|
11420
11834
|
|
|
11421
11835
|
// src/extractor/GoogleAdsExtractor.ts
|
|
11422
11836
|
var ORIGIN = "https://adstransparency.google.com";
|
|
@@ -11586,19 +12000,19 @@ var GoogleAdsExtractor = class {
|
|
|
11586
12000
|
|
|
11587
12001
|
// src/api/google-ads-routes.ts
|
|
11588
12002
|
import { fal as fal2 } from "@fal-ai/client";
|
|
11589
|
-
var GoogleAdsSearchBodySchema =
|
|
11590
|
-
query:
|
|
11591
|
-
region:
|
|
11592
|
-
maxResults:
|
|
11593
|
-
});
|
|
11594
|
-
var GoogleAdsPageIntelBodySchema =
|
|
11595
|
-
advertiserId:
|
|
11596
|
-
domain:
|
|
11597
|
-
region:
|
|
11598
|
-
maxAds:
|
|
12003
|
+
var GoogleAdsSearchBodySchema = z16.object({
|
|
12004
|
+
query: z16.string().trim().min(1, "query is required"),
|
|
12005
|
+
region: z16.string().trim().toUpperCase().length(2).optional(),
|
|
12006
|
+
maxResults: z16.number().int().min(1).max(20).optional()
|
|
12007
|
+
});
|
|
12008
|
+
var GoogleAdsPageIntelBodySchema = z16.object({
|
|
12009
|
+
advertiserId: z16.string().trim().optional(),
|
|
12010
|
+
domain: z16.string().trim().optional(),
|
|
12011
|
+
region: z16.string().trim().toUpperCase().length(2).optional(),
|
|
12012
|
+
maxAds: z16.number().int().min(1).max(200).optional()
|
|
11599
12013
|
}).refine((d) => !!d.advertiserId || !!d.domain, { message: "advertiserId or domain is required" });
|
|
11600
|
-
var GoogleAdsTranscribeBodySchema =
|
|
11601
|
-
videoUrl:
|
|
12014
|
+
var GoogleAdsTranscribeBodySchema = z16.object({
|
|
12015
|
+
videoUrl: z16.string().trim().min(1, "videoUrl is required")
|
|
11602
12016
|
});
|
|
11603
12017
|
function invalidRequest2(message) {
|
|
11604
12018
|
return { error_code: "invalid_request", message };
|
|
@@ -11741,7 +12155,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
11741
12155
|
|
|
11742
12156
|
// src/api/instagram-routes.ts
|
|
11743
12157
|
import { Hono as Hono8 } from "hono";
|
|
11744
|
-
import { z as
|
|
12158
|
+
import { z as z17 } from "zod";
|
|
11745
12159
|
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
11746
12160
|
import { tmpdir } from "os";
|
|
11747
12161
|
import { join as join3 } from "path";
|
|
@@ -11832,7 +12246,7 @@ function trackFromUrl(url) {
|
|
|
11832
12246
|
height: typeof parsed?.height === "number" ? parsed.height : null
|
|
11833
12247
|
};
|
|
11834
12248
|
}
|
|
11835
|
-
function extractInstagramMp4UrlsFromText(
|
|
12249
|
+
function extractInstagramMp4UrlsFromText(text2) {
|
|
11836
12250
|
const urls = [];
|
|
11837
12251
|
const seen = /* @__PURE__ */ new Set();
|
|
11838
12252
|
const patterns = [
|
|
@@ -11841,7 +12255,7 @@ function extractInstagramMp4UrlsFromText(text) {
|
|
|
11841
12255
|
];
|
|
11842
12256
|
for (const pattern of patterns) {
|
|
11843
12257
|
let match;
|
|
11844
|
-
while ((match = pattern.exec(
|
|
12258
|
+
while ((match = pattern.exec(text2)) !== null) {
|
|
11845
12259
|
const normalized = normalizeInstagramMediaUrl(match[0].replace(/%5C/gi, "\\"));
|
|
11846
12260
|
if (!normalized || seen.has(normalized)) continue;
|
|
11847
12261
|
seen.add(normalized);
|
|
@@ -11871,9 +12285,9 @@ function selectInstagramTracks(tracks) {
|
|
|
11871
12285
|
const selectedAudioTrack = tracks.filter((track) => track.streamType === "audio").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0] ?? null;
|
|
11872
12286
|
return { selectedVideoTrack, selectedAudioTrack };
|
|
11873
12287
|
}
|
|
11874
|
-
function parseCount(
|
|
11875
|
-
if (!
|
|
11876
|
-
const match =
|
|
12288
|
+
function parseCount(text2) {
|
|
12289
|
+
if (!text2) return null;
|
|
12290
|
+
const match = text2.match(/([\d,.]+)\s*posts?/i);
|
|
11877
12291
|
if (!match) return null;
|
|
11878
12292
|
const value = Number(match[1].replace(/[,.]/g, ""));
|
|
11879
12293
|
return Number.isFinite(value) ? value : null;
|
|
@@ -11932,8 +12346,8 @@ async function collectInstagramProfileContentFromPage(page, opts) {
|
|
|
11932
12346
|
return items.length - before;
|
|
11933
12347
|
};
|
|
11934
12348
|
const readBodyText = async () => page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
11935
|
-
const profileLines = (
|
|
11936
|
-
const profilePostCountText = (
|
|
12349
|
+
const profileLines = (text2) => text2.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
12350
|
+
const profilePostCountText = (text2) => profileLines(text2).find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
11937
12351
|
const scrollMetrics = async () => page.evaluate(() => ({
|
|
11938
12352
|
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
11939
12353
|
scrollHeight: Math.max(
|
|
@@ -12103,30 +12517,30 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
12103
12517
|
}
|
|
12104
12518
|
|
|
12105
12519
|
// src/api/instagram-routes.ts
|
|
12106
|
-
var InstagramProfileContentBodySchema =
|
|
12107
|
-
handle:
|
|
12108
|
-
url:
|
|
12109
|
-
profile:
|
|
12110
|
-
saveProfileChanges:
|
|
12111
|
-
save_profile_changes:
|
|
12112
|
-
maxItems:
|
|
12113
|
-
maxScrolls:
|
|
12114
|
-
scrollDelayMs:
|
|
12115
|
-
stableScrollLimit:
|
|
12520
|
+
var InstagramProfileContentBodySchema = z17.object({
|
|
12521
|
+
handle: z17.string().trim().optional(),
|
|
12522
|
+
url: z17.string().trim().optional(),
|
|
12523
|
+
profile: z17.string().trim().min(1).optional(),
|
|
12524
|
+
saveProfileChanges: z17.boolean().optional(),
|
|
12525
|
+
save_profile_changes: z17.boolean().optional(),
|
|
12526
|
+
maxItems: z17.number().int().min(1).max(2e3).default(50),
|
|
12527
|
+
maxScrolls: z17.number().int().min(0).max(250).default(10),
|
|
12528
|
+
scrollDelayMs: z17.number().int().min(250).max(5e3).default(1200),
|
|
12529
|
+
stableScrollLimit: z17.number().int().min(1).max(10).default(4)
|
|
12116
12530
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
12117
12531
|
message: "handle or url is required"
|
|
12118
12532
|
});
|
|
12119
|
-
var InstagramMediaTypeSchema =
|
|
12120
|
-
var InstagramMediaDownloadBodySchema =
|
|
12121
|
-
url:
|
|
12122
|
-
profile:
|
|
12123
|
-
saveProfileChanges:
|
|
12124
|
-
save_profile_changes:
|
|
12125
|
-
mediaTypes:
|
|
12126
|
-
downloadMedia:
|
|
12127
|
-
downloadAllTracks:
|
|
12128
|
-
includeTranscript:
|
|
12129
|
-
mux:
|
|
12533
|
+
var InstagramMediaTypeSchema = z17.enum(["image", "video", "audio"]);
|
|
12534
|
+
var InstagramMediaDownloadBodySchema = z17.object({
|
|
12535
|
+
url: z17.string().trim().min(1, "url is required"),
|
|
12536
|
+
profile: z17.string().trim().min(1).optional(),
|
|
12537
|
+
saveProfileChanges: z17.boolean().optional(),
|
|
12538
|
+
save_profile_changes: z17.boolean().optional(),
|
|
12539
|
+
mediaTypes: z17.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
12540
|
+
downloadMedia: z17.boolean().default(true),
|
|
12541
|
+
downloadAllTracks: z17.boolean().default(false),
|
|
12542
|
+
includeTranscript: z17.boolean().default(false),
|
|
12543
|
+
mux: z17.boolean().default(true)
|
|
12130
12544
|
});
|
|
12131
12545
|
var instagramApp = new Hono8();
|
|
12132
12546
|
function invalidRequest3(message) {
|
|
@@ -12409,10 +12823,10 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
12409
12823
|
|
|
12410
12824
|
// src/api/reddit-routes.ts
|
|
12411
12825
|
import { Hono as Hono9 } from "hono";
|
|
12412
|
-
import { z as
|
|
12413
|
-
var RedditThreadBodySchema =
|
|
12414
|
-
url:
|
|
12415
|
-
maxComments:
|
|
12826
|
+
import { z as z18 } from "zod";
|
|
12827
|
+
var RedditThreadBodySchema = z18.object({
|
|
12828
|
+
url: z18.string().trim().min(1, "url is required"),
|
|
12829
|
+
maxComments: z18.number().int().min(1).max(2e3).optional()
|
|
12416
12830
|
});
|
|
12417
12831
|
function invalidRequest4(message) {
|
|
12418
12832
|
return { error_code: "invalid_request", message };
|
|
@@ -12551,14 +12965,14 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
12551
12965
|
|
|
12552
12966
|
// src/api/video-routes.ts
|
|
12553
12967
|
import { Hono as Hono10 } from "hono";
|
|
12554
|
-
import { z as
|
|
12968
|
+
import { z as z19 } from "zod";
|
|
12555
12969
|
var videoApp = new Hono10();
|
|
12556
|
-
var AnalyzeBodySchema =
|
|
12557
|
-
sourceUrl:
|
|
12558
|
-
intervalS:
|
|
12559
|
-
maxFrames:
|
|
12560
|
-
detail:
|
|
12561
|
-
vault:
|
|
12970
|
+
var AnalyzeBodySchema = z19.object({
|
|
12971
|
+
sourceUrl: z19.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
12972
|
+
intervalS: z19.number().min(1).max(30).optional(),
|
|
12973
|
+
maxFrames: z19.number().int().min(1).max(480).optional(),
|
|
12974
|
+
detail: z19.enum(["fast", "standard", "deep"]).optional(),
|
|
12975
|
+
vault: z19.string().trim().min(1).optional()
|
|
12562
12976
|
});
|
|
12563
12977
|
function invalidRequest5(message) {
|
|
12564
12978
|
return { error_code: "invalid_request", message };
|
|
@@ -12864,7 +13278,7 @@ var MapsReviewCollector = class {
|
|
|
12864
13278
|
};
|
|
12865
13279
|
|
|
12866
13280
|
// src/extractor/MapsExtractor.ts
|
|
12867
|
-
import { z as
|
|
13281
|
+
import { z as z20 } from "zod";
|
|
12868
13282
|
var MapsExtractor = class {
|
|
12869
13283
|
constructor(driver) {
|
|
12870
13284
|
this.driver = driver;
|
|
@@ -13072,7 +13486,7 @@ var MapsExtractor = class {
|
|
|
13072
13486
|
});
|
|
13073
13487
|
return rows;
|
|
13074
13488
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
13075
|
-
const result =
|
|
13489
|
+
const result = z20.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
13076
13490
|
if (!result.success) {
|
|
13077
13491
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
13078
13492
|
return [];
|
|
@@ -13118,25 +13532,25 @@ var MapsExtractor = class {
|
|
|
13118
13532
|
if (el2.tagName === "H2") {
|
|
13119
13533
|
currentSection = el2.innerText?.trim() || currentSection;
|
|
13120
13534
|
} else if (el2.tagName === "LI") {
|
|
13121
|
-
const
|
|
13122
|
-
const key = `${currentSection}|${
|
|
13123
|
-
if (
|
|
13535
|
+
const text2 = el2.innerText?.replace(/^[\s\S]*?\n/, "").trim();
|
|
13536
|
+
const key = `${currentSection}|${text2}`;
|
|
13537
|
+
if (text2 && !seen.has(key)) {
|
|
13124
13538
|
seen.add(key);
|
|
13125
|
-
results.push({ section: currentSection, attribute:
|
|
13539
|
+
results.push({ section: currentSection, attribute: text2 });
|
|
13126
13540
|
}
|
|
13127
13541
|
}
|
|
13128
13542
|
});
|
|
13129
13543
|
main.querySelectorAll('[href*="businessownership"], [data-item-id^="place-info-links:"]').forEach((el2) => {
|
|
13130
|
-
const
|
|
13131
|
-
const key = `Ownership|${
|
|
13132
|
-
if (
|
|
13544
|
+
const text2 = el2.innerText?.trim();
|
|
13545
|
+
const key = `Ownership|${text2}`;
|
|
13546
|
+
if (text2 && !seen.has(key)) {
|
|
13133
13547
|
seen.add(key);
|
|
13134
|
-
results.push({ section: "Ownership", attribute:
|
|
13548
|
+
results.push({ section: "Ownership", attribute: text2 });
|
|
13135
13549
|
}
|
|
13136
13550
|
});
|
|
13137
13551
|
return results;
|
|
13138
13552
|
});
|
|
13139
|
-
const result =
|
|
13553
|
+
const result = z20.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
13140
13554
|
if (!result.success) {
|
|
13141
13555
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
13142
13556
|
return [];
|
|
@@ -13196,13 +13610,13 @@ var MapsExtractor = class {
|
|
|
13196
13610
|
const items = [];
|
|
13197
13611
|
withHeading.querySelectorAll("*").forEach((el2) => {
|
|
13198
13612
|
if (el2.children.length > 0) return;
|
|
13199
|
-
const
|
|
13200
|
-
if (!
|
|
13201
|
-
if (/^(back|close|done|thanks)$/i.test(
|
|
13202
|
-
if (/^serves\s.*following/i.test(
|
|
13203
|
-
if (seen.has(
|
|
13204
|
-
seen.add(
|
|
13205
|
-
items.push(
|
|
13613
|
+
const text2 = el2.innerText?.trim() ?? el2.textContent?.trim() ?? "";
|
|
13614
|
+
if (!text2 || headingLines.has(text2)) return;
|
|
13615
|
+
if (/^(back|close|done|thanks)$/i.test(text2)) return;
|
|
13616
|
+
if (/^serves\s.*following/i.test(text2)) return;
|
|
13617
|
+
if (seen.has(text2)) return;
|
|
13618
|
+
seen.add(text2);
|
|
13619
|
+
items.push(text2);
|
|
13206
13620
|
});
|
|
13207
13621
|
return items;
|
|
13208
13622
|
}, expectedHeading);
|
|
@@ -13268,8 +13682,8 @@ var MapsSearchExtractor = class {
|
|
|
13268
13682
|
}
|
|
13269
13683
|
async detectBlock(page) {
|
|
13270
13684
|
return page.evaluate(() => {
|
|
13271
|
-
const
|
|
13272
|
-
return /unusual traffic|captcha|recaptcha|about this page/i.test(
|
|
13685
|
+
const text2 = document.body.innerText.slice(0, 2e3);
|
|
13686
|
+
return /unusual traffic|captcha|recaptcha|about this page/i.test(text2) || /\/sorry\//.test(location.href);
|
|
13273
13687
|
});
|
|
13274
13688
|
}
|
|
13275
13689
|
async collectResults(page, maxResults) {
|
|
@@ -13306,8 +13720,8 @@ var MapsSearchExtractor = class {
|
|
|
13306
13720
|
async extractVisibleResults(page) {
|
|
13307
13721
|
return page.evaluate(() => {
|
|
13308
13722
|
function normalizeText(value) {
|
|
13309
|
-
const
|
|
13310
|
-
return
|
|
13723
|
+
const text2 = value?.replace(/\s+/g, " ").trim() ?? "";
|
|
13724
|
+
return text2 || null;
|
|
13311
13725
|
}
|
|
13312
13726
|
function cidFromUrl(url) {
|
|
13313
13727
|
const fid = url.match(/!1s(0x[0-9a-f]+):(0x[0-9a-f]+)/i);
|
|
@@ -13323,8 +13737,8 @@ var MapsSearchExtractor = class {
|
|
|
13323
13737
|
if (!card) return [];
|
|
13324
13738
|
const parts = [];
|
|
13325
13739
|
card.querySelectorAll("div, span").forEach((el2) => {
|
|
13326
|
-
const
|
|
13327
|
-
if (
|
|
13740
|
+
const text2 = Array.from(el2.childNodes).filter((node) => node.nodeType === 3).map((node) => node.textContent?.trim() ?? "").filter((text3) => text3.length > 1 && text3.length < 140).join(" ");
|
|
13741
|
+
if (text2 && !parts.includes(text2)) parts.push(text2);
|
|
13328
13742
|
});
|
|
13329
13743
|
return parts;
|
|
13330
13744
|
}
|
|
@@ -13726,10 +14140,10 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
13726
14140
|
|
|
13727
14141
|
// src/api/trustpilot-routes.ts
|
|
13728
14142
|
import { Hono as Hono12 } from "hono";
|
|
13729
|
-
import { z as
|
|
13730
|
-
var TrustpilotReviewsBodySchema =
|
|
13731
|
-
domain:
|
|
13732
|
-
maxPages:
|
|
14143
|
+
import { z as z21 } from "zod";
|
|
14144
|
+
var TrustpilotReviewsBodySchema = z21.object({
|
|
14145
|
+
domain: z21.string().trim().min(1, "domain is required"),
|
|
14146
|
+
maxPages: z21.number().int().min(1).max(50).default(5)
|
|
13733
14147
|
});
|
|
13734
14148
|
function invalidRequest6(message) {
|
|
13735
14149
|
return { error_code: "invalid_request", message };
|
|
@@ -13876,10 +14290,10 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13876
14290
|
|
|
13877
14291
|
// src/api/g2-routes.ts
|
|
13878
14292
|
import { Hono as Hono13 } from "hono";
|
|
13879
|
-
import { z as
|
|
13880
|
-
var G2ReviewsBodySchema =
|
|
13881
|
-
product:
|
|
13882
|
-
maxPages:
|
|
14293
|
+
import { z as z22 } from "zod";
|
|
14294
|
+
var G2ReviewsBodySchema = z22.object({
|
|
14295
|
+
product: z22.string().trim().min(1, "product is required"),
|
|
14296
|
+
maxPages: z22.number().int().min(1).max(50).default(5)
|
|
13883
14297
|
});
|
|
13884
14298
|
function invalidRequest7(message) {
|
|
13885
14299
|
return { error_code: "invalid_request", message };
|
|
@@ -14086,7 +14500,7 @@ import { Hono as Hono14 } from "hono";
|
|
|
14086
14500
|
// src/directory/directory-workflow.ts
|
|
14087
14501
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
14088
14502
|
import { join as join4 } from "path";
|
|
14089
|
-
import { z as
|
|
14503
|
+
import { z as z23 } from "zod";
|
|
14090
14504
|
|
|
14091
14505
|
// src/directory/location-db.ts
|
|
14092
14506
|
import { access, readFile } from "fs/promises";
|
|
@@ -14247,24 +14661,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
14247
14661
|
}
|
|
14248
14662
|
|
|
14249
14663
|
// src/directory/directory-workflow.ts
|
|
14250
|
-
var DirectoryWorkflowOptionsSchema =
|
|
14251
|
-
query:
|
|
14252
|
-
state:
|
|
14253
|
-
minPopulation:
|
|
14254
|
-
populationYear:
|
|
14255
|
-
maxCities:
|
|
14256
|
-
maxResultsPerCity:
|
|
14257
|
-
concurrency:
|
|
14258
|
-
includeZipGroups:
|
|
14259
|
-
usZipsCsvPath:
|
|
14260
|
-
saveCsv:
|
|
14261
|
-
gl:
|
|
14262
|
-
hl:
|
|
14263
|
-
proxyMode:
|
|
14264
|
-
proxyZip:
|
|
14265
|
-
debug:
|
|
14266
|
-
headless:
|
|
14267
|
-
kernelApiKey:
|
|
14664
|
+
var DirectoryWorkflowOptionsSchema = z23.object({
|
|
14665
|
+
query: z23.string().min(1),
|
|
14666
|
+
state: z23.string().min(2).default("TN"),
|
|
14667
|
+
minPopulation: z23.number().int().min(0).default(1e5),
|
|
14668
|
+
populationYear: z23.union(POPULATION_YEARS.map((year) => z23.literal(year))).default(2025),
|
|
14669
|
+
maxCities: z23.number().int().min(1).max(100).default(25),
|
|
14670
|
+
maxResultsPerCity: z23.number().int().min(1).max(50).default(50),
|
|
14671
|
+
concurrency: z23.number().int().min(1).max(5).default(5),
|
|
14672
|
+
includeZipGroups: z23.boolean().default(true),
|
|
14673
|
+
usZipsCsvPath: z23.string().optional(),
|
|
14674
|
+
saveCsv: z23.boolean().default(true),
|
|
14675
|
+
gl: z23.string().length(2).default("us"),
|
|
14676
|
+
hl: z23.string().length(2).default("en"),
|
|
14677
|
+
proxyMode: z23.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
14678
|
+
proxyZip: z23.string().regex(/^\d{5}$/).optional(),
|
|
14679
|
+
debug: z23.boolean().default(false),
|
|
14680
|
+
headless: z23.boolean().default(true),
|
|
14681
|
+
kernelApiKey: z23.string().optional()
|
|
14268
14682
|
});
|
|
14269
14683
|
function errorMessage(err) {
|
|
14270
14684
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -14559,34 +14973,34 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
14559
14973
|
import { createHmac as createHmac3 } from "crypto";
|
|
14560
14974
|
import { readFile as readFile2 } from "fs/promises";
|
|
14561
14975
|
import { Hono as Hono15 } from "hono";
|
|
14562
|
-
import { z as
|
|
14976
|
+
import { z as z24 } from "zod";
|
|
14563
14977
|
var workflowApp = new Hono15();
|
|
14564
|
-
var WorkflowInputSchema =
|
|
14565
|
-
var WorkflowIdSchema =
|
|
14566
|
-
var CadenceSchema =
|
|
14567
|
-
var ScheduleStatusSchema =
|
|
14568
|
-
var RunBodySchema =
|
|
14978
|
+
var WorkflowInputSchema = z24.record(z24.unknown()).default({});
|
|
14979
|
+
var WorkflowIdSchema = z24.string().min(1);
|
|
14980
|
+
var CadenceSchema = z24.enum(["daily", "weekly", "monthly"]);
|
|
14981
|
+
var ScheduleStatusSchema = z24.enum(["active", "paused"]);
|
|
14982
|
+
var RunBodySchema = z24.object({
|
|
14569
14983
|
workflowId: WorkflowIdSchema,
|
|
14570
14984
|
input: WorkflowInputSchema,
|
|
14571
|
-
webhookUrl:
|
|
14985
|
+
webhookUrl: z24.string().url().optional()
|
|
14572
14986
|
});
|
|
14573
|
-
var ScheduleCreateSchema =
|
|
14987
|
+
var ScheduleCreateSchema = z24.object({
|
|
14574
14988
|
workflowId: WorkflowIdSchema,
|
|
14575
|
-
name:
|
|
14989
|
+
name: z24.string().min(1).max(120).optional(),
|
|
14576
14990
|
input: WorkflowInputSchema,
|
|
14577
14991
|
cadence: CadenceSchema.default("weekly"),
|
|
14578
|
-
timezone:
|
|
14579
|
-
webhookUrl:
|
|
14580
|
-
nextRunAt:
|
|
14992
|
+
timezone: z24.string().min(1).max(64).default("UTC"),
|
|
14993
|
+
webhookUrl: z24.string().url().optional(),
|
|
14994
|
+
nextRunAt: z24.string().datetime().optional()
|
|
14581
14995
|
});
|
|
14582
|
-
var SchedulePatchSchema =
|
|
14583
|
-
name:
|
|
14996
|
+
var SchedulePatchSchema = z24.object({
|
|
14997
|
+
name: z24.string().min(1).max(120).optional(),
|
|
14584
14998
|
status: ScheduleStatusSchema.optional(),
|
|
14585
14999
|
input: WorkflowInputSchema.optional(),
|
|
14586
15000
|
cadence: CadenceSchema.optional(),
|
|
14587
|
-
timezone:
|
|
14588
|
-
webhookUrl:
|
|
14589
|
-
nextRunAt:
|
|
15001
|
+
timezone: z24.string().min(1).max(64).optional(),
|
|
15002
|
+
webhookUrl: z24.string().url().nullable().optional(),
|
|
15003
|
+
nextRunAt: z24.string().datetime().nullable().optional()
|
|
14590
15004
|
});
|
|
14591
15005
|
function hostedWorkflowOutputDir() {
|
|
14592
15006
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -15091,17 +15505,17 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
15091
15505
|
|
|
15092
15506
|
// src/api/serp-intelligence-routes.ts
|
|
15093
15507
|
import { Hono as Hono16 } from "hono";
|
|
15094
|
-
import { createHash as
|
|
15508
|
+
import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
|
|
15095
15509
|
|
|
15096
15510
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
15097
|
-
import { createHash as
|
|
15511
|
+
import { createHash as createHash4 } from "crypto";
|
|
15098
15512
|
import pLimit3 from "p-limit";
|
|
15099
15513
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
15100
15514
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
15101
15515
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
15102
15516
|
function sha256(value) {
|
|
15103
15517
|
if (!value) return null;
|
|
15104
|
-
return
|
|
15518
|
+
return createHash4("sha256").update(value).digest("hex");
|
|
15105
15519
|
}
|
|
15106
15520
|
function countWords(markdown) {
|
|
15107
15521
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -15274,10 +15688,10 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
15274
15688
|
const completedAtDate = /* @__PURE__ */ new Date();
|
|
15275
15689
|
const completedAt = completedAtDate.toISOString();
|
|
15276
15690
|
const durationMs = completedAtDate.getTime() - startedAtDate.getTime();
|
|
15277
|
-
const
|
|
15691
|
+
const text2 = stripMarkdown(extraction.bodyMarkdown);
|
|
15278
15692
|
const htmlContent = truncateContent(extraction.bodyHtml, maxContentChars);
|
|
15279
15693
|
const markdownContent = truncateContent(extraction.bodyMarkdown, maxContentChars);
|
|
15280
|
-
const textContent = truncateContent(
|
|
15694
|
+
const textContent = truncateContent(text2, maxContentChars);
|
|
15281
15695
|
const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
|
|
15282
15696
|
const metrics = countLinks(extraction.bodyHtml, extraction.url);
|
|
15283
15697
|
const wordCount = countWords(extraction.bodyMarkdown);
|
|
@@ -15342,7 +15756,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
15342
15756
|
hashes: {
|
|
15343
15757
|
htmlSha256: sha256(extraction.bodyHtml),
|
|
15344
15758
|
markdownSha256: contentSha256,
|
|
15345
|
-
textSha256: sha256(
|
|
15759
|
+
textSha256: sha256(text2)
|
|
15346
15760
|
},
|
|
15347
15761
|
diagnostics,
|
|
15348
15762
|
...contentIncluded ? {
|
|
@@ -15403,7 +15817,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
15403
15817
|
}
|
|
15404
15818
|
|
|
15405
15819
|
// src/serp-intelligence/schemas.ts
|
|
15406
|
-
import { z as
|
|
15820
|
+
import { z as z25 } from "zod";
|
|
15407
15821
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
15408
15822
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
15409
15823
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -15465,181 +15879,181 @@ function isPublicHttpUrl(value) {
|
|
|
15465
15879
|
return false;
|
|
15466
15880
|
}
|
|
15467
15881
|
}
|
|
15468
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
15469
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
15470
|
-
query:
|
|
15471
|
-
location:
|
|
15472
|
-
gl:
|
|
15473
|
-
hl:
|
|
15474
|
-
device:
|
|
15475
|
-
proxyMode:
|
|
15476
|
-
proxyZip:
|
|
15477
|
-
pages:
|
|
15478
|
-
debug:
|
|
15479
|
-
includePageSnapshots:
|
|
15480
|
-
pageSnapshotLimit:
|
|
15882
|
+
var SerpIntelligencePublicHttpUrlSchema = z25.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
15883
|
+
var SerpIntelligenceCaptureBodySchema = z25.object({
|
|
15884
|
+
query: z25.string().trim().min(1, "query is required"),
|
|
15885
|
+
location: z25.string().trim().min(1).optional(),
|
|
15886
|
+
gl: z25.string().trim().length(2).default("us"),
|
|
15887
|
+
hl: z25.string().trim().length(2).default("en"),
|
|
15888
|
+
device: z25.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
15889
|
+
proxyMode: z25.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
15890
|
+
proxyZip: z25.string().regex(/^\d{5}$/).optional(),
|
|
15891
|
+
pages: z25.number().int().min(1).max(2).default(1),
|
|
15892
|
+
debug: z25.boolean().default(false),
|
|
15893
|
+
includePageSnapshots: z25.boolean().default(false),
|
|
15894
|
+
pageSnapshotLimit: z25.number().int().min(0).max(10).default(0)
|
|
15481
15895
|
}).strict();
|
|
15482
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
15896
|
+
var SerpIntelligencePageSnapshotRequestSchema = z25.object({
|
|
15483
15897
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
15484
|
-
sourceKind:
|
|
15485
|
-
sourcePosition:
|
|
15898
|
+
sourceKind: z25.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
15899
|
+
sourcePosition: z25.number().int().min(1).optional()
|
|
15486
15900
|
}).strict();
|
|
15487
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
15488
|
-
urls:
|
|
15489
|
-
targets:
|
|
15490
|
-
maxConcurrency:
|
|
15491
|
-
timeoutMs:
|
|
15492
|
-
debug:
|
|
15901
|
+
var SerpIntelligencePageSnapshotsBodySchema = z25.object({
|
|
15902
|
+
urls: z25.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
|
|
15903
|
+
targets: z25.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
15904
|
+
maxConcurrency: z25.number().int().min(1).max(5).default(2),
|
|
15905
|
+
timeoutMs: z25.number().int().min(1e3).max(6e4).default(15e3),
|
|
15906
|
+
debug: z25.boolean().default(false)
|
|
15493
15907
|
}).strict().superRefine((body, ctx) => {
|
|
15494
15908
|
const hasUrls = Boolean(body.urls?.length);
|
|
15495
15909
|
const hasTargets = Boolean(body.targets?.length);
|
|
15496
15910
|
if (hasUrls === hasTargets) {
|
|
15497
15911
|
ctx.addIssue({
|
|
15498
|
-
code:
|
|
15912
|
+
code: z25.ZodIssueCode.custom,
|
|
15499
15913
|
message: "provide exactly one of urls or targets",
|
|
15500
15914
|
path: hasUrls ? ["targets"] : ["urls"]
|
|
15501
15915
|
});
|
|
15502
15916
|
}
|
|
15503
15917
|
});
|
|
15504
|
-
var SerpIntelligenceAICitationSchema =
|
|
15505
|
-
text:
|
|
15506
|
-
href:
|
|
15918
|
+
var SerpIntelligenceAICitationSchema = z25.object({
|
|
15919
|
+
text: z25.string(),
|
|
15920
|
+
href: z25.string()
|
|
15507
15921
|
}).strict();
|
|
15508
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
15509
|
-
position:
|
|
15510
|
-
title:
|
|
15511
|
-
url:
|
|
15512
|
-
domain:
|
|
15513
|
-
cite:
|
|
15514
|
-
snippet:
|
|
15515
|
-
isRedditStyle:
|
|
15516
|
-
inlineRating:
|
|
15517
|
-
value:
|
|
15518
|
-
count:
|
|
15922
|
+
var SerpIntelligenceOrganicResultSchema = z25.object({
|
|
15923
|
+
position: z25.number().int().min(1),
|
|
15924
|
+
title: z25.string(),
|
|
15925
|
+
url: z25.string(),
|
|
15926
|
+
domain: z25.string(),
|
|
15927
|
+
cite: z25.string().nullable(),
|
|
15928
|
+
snippet: z25.string().nullable(),
|
|
15929
|
+
isRedditStyle: z25.boolean(),
|
|
15930
|
+
inlineRating: z25.object({
|
|
15931
|
+
value: z25.string(),
|
|
15932
|
+
count: z25.string()
|
|
15519
15933
|
}).strict().nullable()
|
|
15520
15934
|
}).strict();
|
|
15521
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
15522
|
-
status:
|
|
15523
|
-
expected:
|
|
15524
|
-
city:
|
|
15525
|
-
regionCode:
|
|
15526
|
-
canonicalLocation:
|
|
15935
|
+
var SerpIntelligenceLocationEvidenceSchema = z25.object({
|
|
15936
|
+
status: z25.enum(SerpIntelligenceLocalizationStatusValues),
|
|
15937
|
+
expected: z25.object({
|
|
15938
|
+
city: z25.string(),
|
|
15939
|
+
regionCode: z25.string().nullable(),
|
|
15940
|
+
canonicalLocation: z25.string()
|
|
15527
15941
|
}).strict().nullable(),
|
|
15528
|
-
candidates:
|
|
15529
|
-
city:
|
|
15530
|
-
regionCode:
|
|
15531
|
-
count:
|
|
15532
|
-
examples:
|
|
15942
|
+
candidates: z25.array(z25.object({
|
|
15943
|
+
city: z25.string(),
|
|
15944
|
+
regionCode: z25.string(),
|
|
15945
|
+
count: z25.number().int().min(0),
|
|
15946
|
+
examples: z25.array(z25.string())
|
|
15533
15947
|
}).strict())
|
|
15534
15948
|
}).strict();
|
|
15535
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
15536
|
-
seed:
|
|
15537
|
-
location:
|
|
15538
|
-
extractedAt:
|
|
15539
|
-
totalQuestions:
|
|
15540
|
-
surface:
|
|
15541
|
-
aiOverview:
|
|
15542
|
-
detected:
|
|
15543
|
-
text:
|
|
15544
|
-
citations:
|
|
15545
|
-
expanded:
|
|
15546
|
-
fullyExpanded:
|
|
15547
|
-
sections:
|
|
15949
|
+
var SerpIntelligenceHarvestResultSchema = z25.object({
|
|
15950
|
+
seed: z25.string(),
|
|
15951
|
+
location: z25.string().nullable(),
|
|
15952
|
+
extractedAt: z25.string(),
|
|
15953
|
+
totalQuestions: z25.number().int().min(0),
|
|
15954
|
+
surface: z25.enum(["web", "aim", "unknown"]),
|
|
15955
|
+
aiOverview: z25.object({
|
|
15956
|
+
detected: z25.boolean(),
|
|
15957
|
+
text: z25.string().nullable(),
|
|
15958
|
+
citations: z25.array(SerpIntelligenceAICitationSchema),
|
|
15959
|
+
expanded: z25.boolean().optional(),
|
|
15960
|
+
fullyExpanded: z25.boolean().optional(),
|
|
15961
|
+
sections: z25.array(z25.string()).optional()
|
|
15548
15962
|
}).strict(),
|
|
15549
|
-
aiMode:
|
|
15550
|
-
detected:
|
|
15551
|
-
text:
|
|
15552
|
-
citations:
|
|
15963
|
+
aiMode: z25.object({
|
|
15964
|
+
detected: z25.boolean(),
|
|
15965
|
+
text: z25.string().nullable(),
|
|
15966
|
+
citations: z25.array(SerpIntelligenceAICitationSchema)
|
|
15553
15967
|
}).strict(),
|
|
15554
|
-
tree:
|
|
15555
|
-
flat:
|
|
15556
|
-
videos:
|
|
15557
|
-
forums:
|
|
15558
|
-
organicResults:
|
|
15559
|
-
localPack:
|
|
15560
|
-
entityIds:
|
|
15561
|
-
entities:
|
|
15562
|
-
name:
|
|
15563
|
-
kgId:
|
|
15564
|
-
cid:
|
|
15565
|
-
gcid:
|
|
15968
|
+
tree: z25.array(z25.unknown()),
|
|
15969
|
+
flat: z25.array(z25.unknown()),
|
|
15970
|
+
videos: z25.array(z25.unknown()),
|
|
15971
|
+
forums: z25.array(z25.unknown()),
|
|
15972
|
+
organicResults: z25.array(SerpIntelligenceOrganicResultSchema),
|
|
15973
|
+
localPack: z25.array(z25.unknown()),
|
|
15974
|
+
entityIds: z25.object({
|
|
15975
|
+
entities: z25.array(z25.object({
|
|
15976
|
+
name: z25.string(),
|
|
15977
|
+
kgId: z25.string().nullable(),
|
|
15978
|
+
cid: z25.string().nullable(),
|
|
15979
|
+
gcid: z25.string().nullable()
|
|
15566
15980
|
}).strict()),
|
|
15567
|
-
kgIds:
|
|
15568
|
-
cids:
|
|
15569
|
-
gcids:
|
|
15981
|
+
kgIds: z25.array(z25.string()),
|
|
15982
|
+
cids: z25.array(z25.string()),
|
|
15983
|
+
gcids: z25.array(z25.string())
|
|
15570
15984
|
}).strict(),
|
|
15571
|
-
stats:
|
|
15572
|
-
seed:
|
|
15573
|
-
totalQuestions:
|
|
15574
|
-
maxDepthReached:
|
|
15575
|
-
durationMs:
|
|
15576
|
-
errorCount:
|
|
15985
|
+
stats: z25.object({
|
|
15986
|
+
seed: z25.string(),
|
|
15987
|
+
totalQuestions: z25.number().int().min(0),
|
|
15988
|
+
maxDepthReached: z25.number().int().min(0),
|
|
15989
|
+
durationMs: z25.number().min(0),
|
|
15990
|
+
errorCount: z25.number().int().min(0)
|
|
15577
15991
|
}).strict(),
|
|
15578
|
-
diagnostics:
|
|
15579
|
-
completionStatus:
|
|
15580
|
-
problem:
|
|
15581
|
-
warnings:
|
|
15582
|
-
debug:
|
|
15992
|
+
diagnostics: z25.object({
|
|
15993
|
+
completionStatus: z25.enum(["paa_found", "no_paa", "serp_only"]),
|
|
15994
|
+
problem: z25.null(),
|
|
15995
|
+
warnings: z25.array(z25.unknown()).optional(),
|
|
15996
|
+
debug: z25.object({
|
|
15583
15997
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
15584
15998
|
}).passthrough().optional()
|
|
15585
15999
|
}).passthrough(),
|
|
15586
|
-
whatPeopleSaying:
|
|
16000
|
+
whatPeopleSaying: z25.array(z25.unknown())
|
|
15587
16001
|
}).strict();
|
|
15588
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
15589
|
-
attemptNumber:
|
|
15590
|
-
outcome:
|
|
15591
|
-
startedAt:
|
|
15592
|
-
completedAt:
|
|
15593
|
-
durationMs:
|
|
15594
|
-
problemCode:
|
|
15595
|
-
message:
|
|
15596
|
-
kernelSessionId:
|
|
15597
|
-
cleanupSucceeded:
|
|
16002
|
+
var SerpIntelligenceCaptureAttemptSchema = z25.object({
|
|
16003
|
+
attemptNumber: z25.number().int().min(1),
|
|
16004
|
+
outcome: z25.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
16005
|
+
startedAt: z25.string().optional(),
|
|
16006
|
+
completedAt: z25.string().optional(),
|
|
16007
|
+
durationMs: z25.number().min(0).optional(),
|
|
16008
|
+
problemCode: z25.string().optional(),
|
|
16009
|
+
message: z25.string().optional(),
|
|
16010
|
+
kernelSessionId: z25.string().nullable().optional(),
|
|
16011
|
+
cleanupSucceeded: z25.boolean().nullable().optional()
|
|
15598
16012
|
}).strict();
|
|
15599
|
-
var SerpPageSnapshotCaptureSchema =
|
|
16013
|
+
var SerpPageSnapshotCaptureSchema = z25.object({
|
|
15600
16014
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
15601
16015
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
15602
16016
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
15603
|
-
sourceKind:
|
|
15604
|
-
sourcePosition:
|
|
15605
|
-
status:
|
|
15606
|
-
fetchedVia:
|
|
15607
|
-
httpStatus:
|
|
15608
|
-
contentType:
|
|
15609
|
-
title:
|
|
16017
|
+
sourceKind: z25.enum(SerpPageSnapshotSourceKindValues),
|
|
16018
|
+
sourcePosition: z25.number().int().min(1).nullable(),
|
|
16019
|
+
status: z25.enum(SerpPageFetchStatusValues),
|
|
16020
|
+
fetchedVia: z25.enum(SerpPageFetchedViaValues).nullable(),
|
|
16021
|
+
httpStatus: z25.number().int().min(100).max(599).nullable(),
|
|
16022
|
+
contentType: z25.string().nullable(),
|
|
16023
|
+
title: z25.string().nullable(),
|
|
15610
16024
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
15611
|
-
metaDescription:
|
|
15612
|
-
headings:
|
|
15613
|
-
level:
|
|
15614
|
-
text:
|
|
16025
|
+
metaDescription: z25.string().nullable(),
|
|
16026
|
+
headings: z25.array(z25.object({
|
|
16027
|
+
level: z25.number().int().min(1).max(6),
|
|
16028
|
+
text: z25.string()
|
|
15615
16029
|
}).strict()).default([]),
|
|
15616
|
-
artifact:
|
|
15617
|
-
htmlBlobUrl:
|
|
15618
|
-
textBlobUrl:
|
|
15619
|
-
markdownBlobUrl:
|
|
15620
|
-
screenshotBlobUrl:
|
|
15621
|
-
contentSha256:
|
|
15622
|
-
capturedAt:
|
|
16030
|
+
artifact: z25.object({
|
|
16031
|
+
htmlBlobUrl: z25.string().url().nullable(),
|
|
16032
|
+
textBlobUrl: z25.string().url().nullable(),
|
|
16033
|
+
markdownBlobUrl: z25.string().url().nullable(),
|
|
16034
|
+
screenshotBlobUrl: z25.string().url().nullable(),
|
|
16035
|
+
contentSha256: z25.string().nullable(),
|
|
16036
|
+
capturedAt: z25.string().nullable()
|
|
15623
16037
|
}).strict(),
|
|
15624
|
-
error:
|
|
15625
|
-
code:
|
|
15626
|
-
message:
|
|
16038
|
+
error: z25.object({
|
|
16039
|
+
code: z25.string(),
|
|
16040
|
+
message: z25.string()
|
|
15627
16041
|
}).strict().nullable()
|
|
15628
16042
|
}).strict();
|
|
15629
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
16043
|
+
var SerpIntelligenceCaptureResponseSchema = z25.object({
|
|
15630
16044
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
15631
|
-
attempts:
|
|
16045
|
+
attempts: z25.array(SerpIntelligenceCaptureAttemptSchema),
|
|
15632
16046
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
15633
|
-
pageSnapshotArtifacts:
|
|
15634
|
-
billing:
|
|
15635
|
-
creditsUsed:
|
|
15636
|
-
requestId:
|
|
15637
|
-
jobId:
|
|
16047
|
+
pageSnapshotArtifacts: z25.array(SerpPageSnapshotCaptureSchema),
|
|
16048
|
+
billing: z25.object({
|
|
16049
|
+
creditsUsed: z25.number().min(0).optional(),
|
|
16050
|
+
requestId: z25.string().optional(),
|
|
16051
|
+
jobId: z25.string().optional()
|
|
15638
16052
|
}).strict().optional()
|
|
15639
16053
|
}).strict();
|
|
15640
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
15641
|
-
pageSnapshotArtifacts:
|
|
15642
|
-
attempts:
|
|
16054
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z25.object({
|
|
16055
|
+
pageSnapshotArtifacts: z25.array(SerpPageSnapshotCaptureSchema),
|
|
16056
|
+
attempts: z25.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
15643
16057
|
}).strict();
|
|
15644
16058
|
|
|
15645
16059
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -15875,9 +16289,9 @@ async function logRequestEventBestEffort(input) {
|
|
|
15875
16289
|
}
|
|
15876
16290
|
}
|
|
15877
16291
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
15878
|
-
const responseKey = suppliedKey?.trim() ||
|
|
15879
|
-
const requestFingerprint =
|
|
15880
|
-
const keyDigest =
|
|
16292
|
+
const responseKey = suppliedKey?.trim() || randomUUID4();
|
|
16293
|
+
const requestFingerprint = createHash5("sha256").update(JSON.stringify(body)).digest("hex");
|
|
16294
|
+
const keyDigest = createHash5("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
15881
16295
|
return {
|
|
15882
16296
|
responseKey,
|
|
15883
16297
|
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
@@ -16729,11 +17143,11 @@ mcpApp.all("/", async (c) => {
|
|
|
16729
17143
|
});
|
|
16730
17144
|
|
|
16731
17145
|
// src/api/browser-agent-routes.ts
|
|
16732
|
-
import { randomUUID as
|
|
17146
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
16733
17147
|
import { Hono as Hono18 } from "hono";
|
|
16734
17148
|
|
|
16735
17149
|
// src/api/browser-agent-db.ts
|
|
16736
|
-
import { randomUUID as
|
|
17150
|
+
import { randomUUID as randomUUID5 } from "crypto";
|
|
16737
17151
|
var _ready = false;
|
|
16738
17152
|
var _migrationPromise = null;
|
|
16739
17153
|
async function migrateBrowserAgent() {
|
|
@@ -16864,7 +17278,7 @@ async function runBrowserAgentMigration() {
|
|
|
16864
17278
|
}
|
|
16865
17279
|
async function createExtensionRow(input) {
|
|
16866
17280
|
const db = getDb();
|
|
16867
|
-
const id = `bext_${
|
|
17281
|
+
const id = `bext_${randomUUID5().replace(/-/g, "").slice(0, 20)}`;
|
|
16868
17282
|
await db.execute({
|
|
16869
17283
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
16870
17284
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -16899,7 +17313,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
16899
17313
|
}
|
|
16900
17314
|
async function createAuthConnectionRow(input) {
|
|
16901
17315
|
const db = getDb();
|
|
16902
|
-
const connectionId = `authc_${
|
|
17316
|
+
const connectionId = `authc_${randomUUID5().replace(/-/g, "").slice(0, 20)}`;
|
|
16903
17317
|
await db.execute({
|
|
16904
17318
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
16905
17319
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -16933,7 +17347,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
16933
17347
|
}
|
|
16934
17348
|
async function createSessionRow(input) {
|
|
16935
17349
|
const db = getDb();
|
|
16936
|
-
const id = `bas_${
|
|
17350
|
+
const id = `bas_${randomUUID5().replace(/-/g, "").slice(0, 20)}`;
|
|
16937
17351
|
await db.execute({
|
|
16938
17352
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
16939
17353
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -17002,7 +17416,7 @@ async function recordAction(input) {
|
|
|
17002
17416
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
17003
17417
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
17004
17418
|
args: [
|
|
17005
|
-
`baa_${
|
|
17419
|
+
`baa_${randomUUID5().replace(/-/g, "").slice(0, 20)}`,
|
|
17006
17420
|
input.sessionId,
|
|
17007
17421
|
input.type,
|
|
17008
17422
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -17122,9 +17536,9 @@ var FanoutCdpCapture = class {
|
|
|
17122
17536
|
const ct = this.ctById.get(requestId) || "";
|
|
17123
17537
|
const client2 = this.client;
|
|
17124
17538
|
const p = client2.send("Network.getResponseBody", { requestId }).then((r) => {
|
|
17125
|
-
const
|
|
17126
|
-
if (
|
|
17127
|
-
this.bodies.push({ url, ct, text });
|
|
17539
|
+
const text2 = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
|
|
17540
|
+
if (text2) {
|
|
17541
|
+
this.bodies.push({ url, ct, text: text2 });
|
|
17128
17542
|
if (this.isAnswerUrl(url)) this.answerStreams++;
|
|
17129
17543
|
}
|
|
17130
17544
|
}).catch(() => void 0);
|
|
@@ -17168,14 +17582,14 @@ function unesc(s) {
|
|
|
17168
17582
|
function stripCitationMarkers(s) {
|
|
17169
17583
|
return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
|
|
17170
17584
|
}
|
|
17171
|
-
function sliceBalanced(
|
|
17172
|
-
const open =
|
|
17585
|
+
function sliceBalanced(text2, openIdx) {
|
|
17586
|
+
const open = text2[openIdx];
|
|
17173
17587
|
const close = open === "[" ? "]" : "}";
|
|
17174
17588
|
let depth = 0;
|
|
17175
17589
|
let inStr = false;
|
|
17176
17590
|
let esc = false;
|
|
17177
|
-
for (let i = openIdx; i <
|
|
17178
|
-
const ch =
|
|
17591
|
+
for (let i = openIdx; i < text2.length; i++) {
|
|
17592
|
+
const ch = text2[i];
|
|
17179
17593
|
if (inStr) {
|
|
17180
17594
|
if (esc) esc = false;
|
|
17181
17595
|
else if (ch === "\\") esc = true;
|
|
@@ -17189,18 +17603,18 @@ function sliceBalanced(text, openIdx) {
|
|
|
17189
17603
|
if (ch === open) depth++;
|
|
17190
17604
|
else if (ch === close) {
|
|
17191
17605
|
depth--;
|
|
17192
|
-
if (depth === 0) return
|
|
17606
|
+
if (depth === 0) return text2.slice(openIdx, i + 1);
|
|
17193
17607
|
}
|
|
17194
17608
|
}
|
|
17195
17609
|
return null;
|
|
17196
17610
|
}
|
|
17197
|
-
function findJsonValues(
|
|
17611
|
+
function findJsonValues(text2, field) {
|
|
17198
17612
|
const out = [];
|
|
17199
17613
|
const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
|
|
17200
17614
|
let m;
|
|
17201
|
-
while (m = re.exec(
|
|
17615
|
+
while (m = re.exec(text2)) {
|
|
17202
17616
|
const openIdx = m.index + m[0].length - 1;
|
|
17203
|
-
const slice = sliceBalanced(
|
|
17617
|
+
const slice = sliceBalanced(text2, openIdx);
|
|
17204
17618
|
if (slice) {
|
|
17205
17619
|
try {
|
|
17206
17620
|
out.push(JSON.parse(slice));
|
|
@@ -17334,8 +17748,8 @@ var chatgptAdapter = {
|
|
|
17334
17748
|
}
|
|
17335
17749
|
return s;
|
|
17336
17750
|
};
|
|
17337
|
-
const addSnippet = (url, title,
|
|
17338
|
-
const clean2 = stripCitationMarkers(
|
|
17751
|
+
const addSnippet = (url, title, text2) => {
|
|
17752
|
+
const clean2 = stripCitationMarkers(text2);
|
|
17339
17753
|
if (!isExternal(url) || clean2.length < 2) return;
|
|
17340
17754
|
snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean2.slice(0, 400) });
|
|
17341
17755
|
};
|
|
@@ -17685,7 +18099,7 @@ var COMPOSER_SELECTORS = [
|
|
|
17685
18099
|
'div[contenteditable="true"]',
|
|
17686
18100
|
"textarea"
|
|
17687
18101
|
];
|
|
17688
|
-
async function sendPrompt(page,
|
|
18102
|
+
async function sendPrompt(page, text2) {
|
|
17689
18103
|
for (const selector of COMPOSER_SELECTORS) {
|
|
17690
18104
|
const locator = page.locator(selector).first();
|
|
17691
18105
|
try {
|
|
@@ -17695,7 +18109,7 @@ async function sendPrompt(page, text) {
|
|
|
17695
18109
|
}
|
|
17696
18110
|
await locator.click({ timeout: 4e3 }).catch(() => void 0);
|
|
17697
18111
|
await locator.focus().catch(() => void 0);
|
|
17698
|
-
await page.keyboard.type(
|
|
18112
|
+
await page.keyboard.type(text2, { delay: 12 });
|
|
17699
18113
|
await page.waitForTimeout(300).catch(() => void 0);
|
|
17700
18114
|
await page.keyboard.press("Enter");
|
|
17701
18115
|
return true;
|
|
@@ -17729,9 +18143,9 @@ async function runFanoutCapture(page, input) {
|
|
|
17729
18143
|
if (!adapter) {
|
|
17730
18144
|
throw new Error(`query_fanout_workflow supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
|
|
17731
18145
|
}
|
|
17732
|
-
const
|
|
17733
|
-
const parsed = adapter.parse(
|
|
17734
|
-
const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes:
|
|
18146
|
+
const text2 = cap.bestAnswerText();
|
|
18147
|
+
const parsed = adapter.parse(text2, input.prompt || "");
|
|
18148
|
+
const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text2.length });
|
|
17735
18149
|
const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
|
|
17736
18150
|
const exports = input.export ? exportFanout(enriched) : null;
|
|
17737
18151
|
return { result: enriched, exports };
|
|
@@ -17849,10 +18263,10 @@ async function click(runtimeSessionId, x, y, opts = {}) {
|
|
|
17849
18263
|
...opts.numClicks ? { num_clicks: opts.numClicks } : {}
|
|
17850
18264
|
});
|
|
17851
18265
|
}
|
|
17852
|
-
async function typeText(runtimeSessionId,
|
|
18266
|
+
async function typeText(runtimeSessionId, text2, delayMs) {
|
|
17853
18267
|
const k = client();
|
|
17854
18268
|
await k.browsers.computer.typeText(runtimeSessionId, {
|
|
17855
|
-
text,
|
|
18269
|
+
text: text2,
|
|
17856
18270
|
...typeof delayMs === "number" ? { delay: delayMs } : {}
|
|
17857
18271
|
});
|
|
17858
18272
|
}
|
|
@@ -17915,8 +18329,8 @@ async function readPage(cdpWsUrl) {
|
|
|
17915
18329
|
bottom: Math.round(r.bottom)
|
|
17916
18330
|
});
|
|
17917
18331
|
}
|
|
17918
|
-
const
|
|
17919
|
-
return { text, els: out };
|
|
18332
|
+
const text2 = (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").trim().slice(0, 6e3);
|
|
18333
|
+
return { text: text2, els: out };
|
|
17920
18334
|
});
|
|
17921
18335
|
return {
|
|
17922
18336
|
url,
|
|
@@ -18049,13 +18463,13 @@ async function locatePageTargets(cdpWsUrl, targets) {
|
|
|
18049
18463
|
const match = target.match === "exact" ? "exact" : "contains";
|
|
18050
18464
|
try {
|
|
18051
18465
|
const selector = target.selector?.trim();
|
|
18052
|
-
const
|
|
18053
|
-
const candidates = selector ? selectorMatches(selector) :
|
|
18466
|
+
const text2 = target.text?.trim();
|
|
18467
|
+
const candidates = selector ? selectorMatches(selector) : text2 ? textMatches(text2, match) : [];
|
|
18054
18468
|
const selected = candidates[index];
|
|
18055
18469
|
return {
|
|
18056
18470
|
name: target.name ?? null,
|
|
18057
18471
|
selector: selector || null,
|
|
18058
|
-
text:
|
|
18472
|
+
text: text2 || null,
|
|
18059
18473
|
match,
|
|
18060
18474
|
index,
|
|
18061
18475
|
found: Boolean(selected),
|
|
@@ -18563,13 +18977,13 @@ function buildBrowserAgentRoutes() {
|
|
|
18563
18977
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
18564
18978
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
18565
18979
|
const body = await c.req.json().catch(() => ({}));
|
|
18566
|
-
const
|
|
18567
|
-
if (!
|
|
18980
|
+
const text2 = typeof body.text === "string" ? body.text : "";
|
|
18981
|
+
if (!text2) return c.json({ error: "text is required" }, 400);
|
|
18568
18982
|
const t0 = Date.now();
|
|
18569
18983
|
try {
|
|
18570
|
-
await typeText(row.runtime_session_id,
|
|
18984
|
+
await typeText(row.runtime_session_id, text2, typeof body.delay === "number" ? body.delay : void 0);
|
|
18571
18985
|
await charge(row.id, user.id, t0);
|
|
18572
|
-
await recordAction({ sessionId: row.id, type: "type", params: { length:
|
|
18986
|
+
await recordAction({ sessionId: row.id, type: "type", params: { length: text2.length }, ok: true });
|
|
18573
18987
|
return c.json({ ok: true });
|
|
18574
18988
|
} catch (err) {
|
|
18575
18989
|
await charge(row.id, user.id, t0);
|
|
@@ -18690,7 +19104,7 @@ function buildBrowserAgentRoutes() {
|
|
|
18690
19104
|
}
|
|
18691
19105
|
const existing = await getExtensionRow(user.id, name);
|
|
18692
19106
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
18693
|
-
const backendName = `u${user.id}_${
|
|
19107
|
+
const backendName = `u${user.id}_${randomUUID6().replace(/-/g, "")}`;
|
|
18694
19108
|
try {
|
|
18695
19109
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
18696
19110
|
const row = await createExtensionRow({
|
|
@@ -19043,7 +19457,7 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
19043
19457
|
const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
|
|
19044
19458
|
if (lineTierId) {
|
|
19045
19459
|
const tier = SUBSCRIPTION_TIERS[lineTierId];
|
|
19046
|
-
const user = await
|
|
19460
|
+
const user = await resolveUser2(invoice.customer, invoice.customer_email ?? void 0);
|
|
19047
19461
|
if (user && invoice.id && !await ledgerExistsForStripePI(invoice.id)) {
|
|
19048
19462
|
await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id);
|
|
19049
19463
|
await setSubscriptionTier(user.id, tier.tier, tier.concurrency, invoice.subscription ?? user.subscription_id);
|
|
@@ -19057,7 +19471,7 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
19057
19471
|
const priceId = sub.items.data[0]?.price?.id;
|
|
19058
19472
|
if (priceId && priceId in MEMORY_PLANS) {
|
|
19059
19473
|
const mem = MEMORY_PLANS[priceId];
|
|
19060
|
-
const user2 = await
|
|
19474
|
+
const user2 = await resolveUser2(sub.customer, sub.customer_email);
|
|
19061
19475
|
if (user2) {
|
|
19062
19476
|
const live = sub.status === "active" || sub.status === "trialing";
|
|
19063
19477
|
const newMemoryPlan = live ? mem.plan : "free";
|
|
@@ -19108,7 +19522,7 @@ function linePriceId(line) {
|
|
|
19108
19522
|
const l = line;
|
|
19109
19523
|
return l?.price?.id ?? l?.pricing?.price_details?.price ?? l?.plan?.id;
|
|
19110
19524
|
}
|
|
19111
|
-
async function
|
|
19525
|
+
async function resolveUser2(customerId, emailFallback) {
|
|
19112
19526
|
let user = await getUserByStripeCustomerId(customerId);
|
|
19113
19527
|
if (user) return user;
|
|
19114
19528
|
if (!emailFallback) return void 0;
|
|
@@ -19120,7 +19534,7 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
19120
19534
|
// src/api/oauth-routes.ts
|
|
19121
19535
|
import { Hono as Hono20 } from "hono";
|
|
19122
19536
|
import { getCookie, setCookie } from "hono/cookie";
|
|
19123
|
-
import { createHash as
|
|
19537
|
+
import { createHash as createHash6, randomBytes as randomBytes2, randomUUID as randomUUID7 } from "crypto";
|
|
19124
19538
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
19125
19539
|
var ISSUER = "https://mcpscraper.dev";
|
|
19126
19540
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -19153,7 +19567,7 @@ async function getKeys() {
|
|
|
19153
19567
|
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
19154
19568
|
const full = await exportJWK(privateKey);
|
|
19155
19569
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
19156
|
-
const kid =
|
|
19570
|
+
const kid = createHash6("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
19157
19571
|
publicJwk.kid = kid;
|
|
19158
19572
|
publicJwk.alg = "RS256";
|
|
19159
19573
|
publicJwk.use = "sig";
|
|
@@ -19463,12 +19877,12 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
19463
19877
|
});
|
|
19464
19878
|
function pkceMatches(verifier, challenge) {
|
|
19465
19879
|
if (!verifier) return false;
|
|
19466
|
-
const computed =
|
|
19880
|
+
const computed = createHash6("sha256").update(verifier).digest("base64url");
|
|
19467
19881
|
return computed === challenge;
|
|
19468
19882
|
}
|
|
19469
19883
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
19470
19884
|
const { privateKey, kid } = await getKeys();
|
|
19471
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
19885
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID7()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
19472
19886
|
}
|
|
19473
19887
|
function tokenErrorResponse(c, error, description, status) {
|
|
19474
19888
|
return c.json({ error, error_description: description }, status);
|
|
@@ -20811,16 +21225,16 @@ function startSiteAuditWorker() {
|
|
|
20811
21225
|
// src/api/server.ts
|
|
20812
21226
|
import { createMiddleware as createMiddleware3 } from "hono/factory";
|
|
20813
21227
|
import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
|
|
20814
|
-
import
|
|
21228
|
+
import Stripe3 from "stripe";
|
|
20815
21229
|
|
|
20816
21230
|
// src/api/page-diff.ts
|
|
20817
|
-
import { createHash as
|
|
21231
|
+
import { createHash as createHash7 } from "crypto";
|
|
20818
21232
|
import { diffLines } from "diff";
|
|
20819
21233
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
20820
21234
|
var MAX_DIFF_HUNKS = 200;
|
|
20821
21235
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
20822
21236
|
function sha256Hex(value) {
|
|
20823
|
-
return
|
|
21237
|
+
return createHash7("sha256").update(value).digest("hex");
|
|
20824
21238
|
}
|
|
20825
21239
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
20826
21240
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -20927,7 +21341,7 @@ async function persistScrapeBody(user, opts) {
|
|
|
20927
21341
|
}
|
|
20928
21342
|
|
|
20929
21343
|
// src/api/connection-memory-import.ts
|
|
20930
|
-
import { createHash as
|
|
21344
|
+
import { createHash as createHash8 } from "crypto";
|
|
20931
21345
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
20932
21346
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
20933
21347
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -20957,7 +21371,7 @@ function canonicalJson(value) {
|
|
|
20957
21371
|
return JSON.stringify(value);
|
|
20958
21372
|
}
|
|
20959
21373
|
function sha2562(value) {
|
|
20960
|
-
return
|
|
21374
|
+
return createHash8("sha256").update(value).digest("hex");
|
|
20961
21375
|
}
|
|
20962
21376
|
function sensitiveKey(key) {
|
|
20963
21377
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -21226,7 +21640,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
21226
21640
|
}
|
|
21227
21641
|
|
|
21228
21642
|
// src/api/connected-data-export.ts
|
|
21229
|
-
import { randomUUID as
|
|
21643
|
+
import { randomUUID as randomUUID8 } from "crypto";
|
|
21230
21644
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
21231
21645
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
21232
21646
|
);
|
|
@@ -21352,7 +21766,7 @@ function finitePositive(value, fallback) {
|
|
|
21352
21766
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
21353
21767
|
}
|
|
21354
21768
|
async function collectConnectedDataExport(args) {
|
|
21355
|
-
const exportId =
|
|
21769
|
+
const exportId = randomUUID8();
|
|
21356
21770
|
const now = args.now ?? Date.now;
|
|
21357
21771
|
const startedAt = now();
|
|
21358
21772
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -21464,7 +21878,7 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
21464
21878
|
}
|
|
21465
21879
|
|
|
21466
21880
|
// src/api/search-console-table-export.ts
|
|
21467
|
-
import { randomUUID as
|
|
21881
|
+
import { randomUUID as randomUUID9 } from "crypto";
|
|
21468
21882
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
21469
21883
|
var SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
21470
21884
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -21564,7 +21978,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
21564
21978
|
offset += rows.length;
|
|
21565
21979
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
21566
21980
|
}
|
|
21567
|
-
const exportId =
|
|
21981
|
+
const exportId = randomUUID9();
|
|
21568
21982
|
const artifact = await args.writeArtifact({
|
|
21569
21983
|
ownerId: args.ownerId,
|
|
21570
21984
|
exportId,
|
|
@@ -21799,77 +22213,730 @@ async function dbUsage(identity, plan) {
|
|
|
21799
22213
|
}
|
|
21800
22214
|
|
|
21801
22215
|
// src/api/nango-control.ts
|
|
21802
|
-
import { createHash as
|
|
21803
|
-
|
|
21804
|
-
|
|
21805
|
-
|
|
21806
|
-
|
|
21807
|
-
};
|
|
21808
|
-
|
|
21809
|
-
|
|
21810
|
-
|
|
21811
|
-
|
|
21812
|
-
|
|
21813
|
-
|
|
21814
|
-
|
|
21815
|
-
|
|
21816
|
-
|
|
21817
|
-
|
|
21818
|
-
|
|
21819
|
-
|
|
21820
|
-
|
|
21821
|
-
|
|
21822
|
-
|
|
21823
|
-
|
|
21824
|
-
|
|
21825
|
-
|
|
21826
|
-
|
|
21827
|
-
|
|
21828
|
-
|
|
21829
|
-
|
|
21830
|
-
|
|
21831
|
-
|
|
21832
|
-
|
|
21833
|
-
|
|
21834
|
-
|
|
21835
|
-
|
|
21836
|
-
|
|
21837
|
-
|
|
21838
|
-
|
|
21839
|
-
|
|
21840
|
-
|
|
21841
|
-
|
|
21842
|
-
|
|
21843
|
-
|
|
21844
|
-
|
|
21845
|
-
|
|
21846
|
-
|
|
21847
|
-
|
|
21848
|
-
|
|
21849
|
-
|
|
21850
|
-
|
|
21851
|
-
|
|
21852
|
-
|
|
21853
|
-
|
|
21854
|
-
|
|
21855
|
-
|
|
21856
|
-
|
|
21857
|
-
|
|
21858
|
-
|
|
21859
|
-
|
|
21860
|
-
|
|
21861
|
-
|
|
21862
|
-
|
|
21863
|
-
|
|
21864
|
-
|
|
21865
|
-
|
|
21866
|
-
|
|
21867
|
-
|
|
21868
|
-
|
|
21869
|
-
|
|
21870
|
-
|
|
21871
|
-
|
|
21872
|
-
|
|
22216
|
+
import { createHash as createHash10, randomUUID as randomUUID12 } from "crypto";
|
|
22217
|
+
|
|
22218
|
+
// src/api/main-nango-transport.ts
|
|
22219
|
+
import { createHash as createHash9, randomUUID as randomUUID11 } from "crypto";
|
|
22220
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
22221
|
+
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
22222
|
+
|
|
22223
|
+
// src/api/service-connections.ts
|
|
22224
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
22225
|
+
var schemaReady2 = null;
|
|
22226
|
+
var schemaDb = null;
|
|
22227
|
+
function ensureServiceConnectionsSchema() {
|
|
22228
|
+
const currentDb = getDb();
|
|
22229
|
+
if (schemaReady2 && schemaDb === currentDb) return schemaReady2;
|
|
22230
|
+
schemaDb = currentDb;
|
|
22231
|
+
schemaReady2 = (async () => {
|
|
22232
|
+
const db = currentDb;
|
|
22233
|
+
await db.execute(`
|
|
22234
|
+
CREATE TABLE IF NOT EXISTS service_connections (
|
|
22235
|
+
id TEXT PRIMARY KEY,
|
|
22236
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
22237
|
+
provider_config_key TEXT NOT NULL,
|
|
22238
|
+
provider TEXT,
|
|
22239
|
+
transport TEXT NOT NULL,
|
|
22240
|
+
upstream_connection_id TEXT,
|
|
22241
|
+
label TEXT,
|
|
22242
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
22243
|
+
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
22244
|
+
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
22245
|
+
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
22246
|
+
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
22247
|
+
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
22248
|
+
tool_revision TEXT,
|
|
22249
|
+
last_checked_at TEXT,
|
|
22250
|
+
last_successful_call_at TEXT,
|
|
22251
|
+
last_failure_at TEXT,
|
|
22252
|
+
last_failure_code TEXT,
|
|
22253
|
+
last_failure_retryable INTEGER,
|
|
22254
|
+
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
22255
|
+
vault_name TEXT,
|
|
22256
|
+
table_name TEXT,
|
|
22257
|
+
legacy_owner_identity TEXT,
|
|
22258
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
22259
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
22260
|
+
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
22261
|
+
)
|
|
22262
|
+
`);
|
|
22263
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
|
|
22264
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
|
|
22265
|
+
await db.execute(`
|
|
22266
|
+
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
22267
|
+
id TEXT PRIMARY KEY,
|
|
22268
|
+
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
22269
|
+
operational_status TEXT NOT NULL,
|
|
22270
|
+
failure_code TEXT,
|
|
22271
|
+
retryable INTEGER,
|
|
22272
|
+
evidence_source TEXT NOT NULL,
|
|
22273
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
22274
|
+
)
|
|
22275
|
+
`);
|
|
22276
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
|
|
22277
|
+
await db.execute(`
|
|
22278
|
+
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
22279
|
+
id TEXT PRIMARY KEY,
|
|
22280
|
+
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
22281
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
22282
|
+
tool TEXT NOT NULL,
|
|
22283
|
+
request_id TEXT NOT NULL,
|
|
22284
|
+
status TEXT NOT NULL,
|
|
22285
|
+
error_code TEXT,
|
|
22286
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
22287
|
+
UNIQUE(user_id, request_id)
|
|
22288
|
+
)
|
|
22289
|
+
`);
|
|
22290
|
+
})().catch((error) => {
|
|
22291
|
+
schemaReady2 = null;
|
|
22292
|
+
schemaDb = null;
|
|
22293
|
+
throw error;
|
|
22294
|
+
});
|
|
22295
|
+
return schemaReady2;
|
|
22296
|
+
}
|
|
22297
|
+
function parseTools(value) {
|
|
22298
|
+
if (typeof value !== "string") return [];
|
|
22299
|
+
try {
|
|
22300
|
+
const parsed = JSON.parse(value);
|
|
22301
|
+
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
|
|
22302
|
+
} catch {
|
|
22303
|
+
return [];
|
|
22304
|
+
}
|
|
22305
|
+
}
|
|
22306
|
+
function rowToRecord(row) {
|
|
22307
|
+
return {
|
|
22308
|
+
id: String(row.id),
|
|
22309
|
+
userId: Number(row.user_id),
|
|
22310
|
+
providerConfigKey: String(row.provider_config_key),
|
|
22311
|
+
provider: typeof row.provider === "string" ? row.provider : null,
|
|
22312
|
+
transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
|
|
22313
|
+
upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
|
|
22314
|
+
label: typeof row.label === "string" ? row.label : null,
|
|
22315
|
+
lifecycleStatus: String(row.lifecycle_status),
|
|
22316
|
+
operationalStatus: String(row.operational_status),
|
|
22317
|
+
reconnectRequired: Number(row.reconnect_required) === 1,
|
|
22318
|
+
actionsEnabled: Number(row.actions_enabled) === 1,
|
|
22319
|
+
readTools: parseTools(row.read_tools_json),
|
|
22320
|
+
actionTools: parseTools(row.action_tools_json),
|
|
22321
|
+
toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
|
|
22322
|
+
lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
|
|
22323
|
+
lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
|
|
22324
|
+
lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
|
|
22325
|
+
lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
|
|
22326
|
+
lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
|
|
22327
|
+
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
|
22328
|
+
vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
|
|
22329
|
+
tableName: typeof row.table_name === "string" ? row.table_name : null,
|
|
22330
|
+
createdAt: String(row.created_at),
|
|
22331
|
+
updatedAt: String(row.updated_at)
|
|
22332
|
+
};
|
|
22333
|
+
}
|
|
22334
|
+
async function userIdForIdentity(identity) {
|
|
22335
|
+
const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
|
|
22336
|
+
if (!user) throw new Error("service_connection_user_not_found");
|
|
22337
|
+
return Number(user.id);
|
|
22338
|
+
}
|
|
22339
|
+
async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
22340
|
+
await ensureServiceConnectionsSchema();
|
|
22341
|
+
const db = getDb();
|
|
22342
|
+
const userId = await userIdForIdentity(identity);
|
|
22343
|
+
for (const connection of discovered) {
|
|
22344
|
+
await db.execute({
|
|
22345
|
+
sql: `
|
|
22346
|
+
INSERT INTO service_connections (
|
|
22347
|
+
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
22348
|
+
label, lifecycle_status, reconnect_required, legacy_owner_identity, created_at, updated_at
|
|
22349
|
+
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
22350
|
+
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
22351
|
+
provider = excluded.provider,
|
|
22352
|
+
label = COALESCE(excluded.label, service_connections.label),
|
|
22353
|
+
lifecycle_status = excluded.lifecycle_status,
|
|
22354
|
+
reconnect_required = excluded.reconnect_required,
|
|
22355
|
+
updated_at = excluded.updated_at
|
|
22356
|
+
`,
|
|
22357
|
+
args: [
|
|
22358
|
+
randomUUID10(),
|
|
22359
|
+
userId,
|
|
22360
|
+
connection.providerConfigKey,
|
|
22361
|
+
connection.provider,
|
|
22362
|
+
connection.upstreamConnectionId,
|
|
22363
|
+
connection.label,
|
|
22364
|
+
connection.lifecycleStatus,
|
|
22365
|
+
connection.reconnectRequired ? 1 : 0,
|
|
22366
|
+
identity,
|
|
22367
|
+
connection.createdAt,
|
|
22368
|
+
connection.updatedAt
|
|
22369
|
+
]
|
|
22370
|
+
});
|
|
22371
|
+
}
|
|
22372
|
+
return listServiceConnections(identity, "nango");
|
|
22373
|
+
}
|
|
22374
|
+
async function listServiceConnections(identity, transport) {
|
|
22375
|
+
await ensureServiceConnectionsSchema();
|
|
22376
|
+
const userId = await userIdForIdentity(identity);
|
|
22377
|
+
const result = await getDb().execute({
|
|
22378
|
+
sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
|
|
22379
|
+
args: transport ? [userId, transport] : [userId]
|
|
22380
|
+
});
|
|
22381
|
+
return result.rows.map((row) => rowToRecord(row));
|
|
22382
|
+
}
|
|
22383
|
+
async function getOwnedServiceConnection(identity, connectionId) {
|
|
22384
|
+
await ensureServiceConnectionsSchema();
|
|
22385
|
+
const userId = await userIdForIdentity(identity);
|
|
22386
|
+
const result = await getDb().execute({
|
|
22387
|
+
sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
|
|
22388
|
+
args: [connectionId, userId]
|
|
22389
|
+
});
|
|
22390
|
+
const row = result.rows[0];
|
|
22391
|
+
return row ? rowToRecord(row) : null;
|
|
22392
|
+
}
|
|
22393
|
+
async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
|
|
22394
|
+
await ensureServiceConnectionsSchema();
|
|
22395
|
+
await getDb().execute({
|
|
22396
|
+
sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
22397
|
+
args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
|
|
22398
|
+
});
|
|
22399
|
+
}
|
|
22400
|
+
async function recordServiceConnectionHealth(args) {
|
|
22401
|
+
await ensureServiceConnectionsSchema();
|
|
22402
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22403
|
+
await getDb().execute({
|
|
22404
|
+
sql: `
|
|
22405
|
+
UPDATE service_connections SET
|
|
22406
|
+
operational_status = ?, last_checked_at = ?,
|
|
22407
|
+
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
22408
|
+
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
22409
|
+
last_failure_code = ?, last_failure_retryable = ?,
|
|
22410
|
+
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
22411
|
+
updated_at = datetime('now')
|
|
22412
|
+
WHERE id = ?
|
|
22413
|
+
`,
|
|
22414
|
+
args: [
|
|
22415
|
+
args.operationalStatus,
|
|
22416
|
+
now,
|
|
22417
|
+
args.operationalStatus,
|
|
22418
|
+
now,
|
|
22419
|
+
args.operationalStatus,
|
|
22420
|
+
now,
|
|
22421
|
+
args.failureCode ?? null,
|
|
22422
|
+
args.retryable == null ? null : args.retryable ? 1 : 0,
|
|
22423
|
+
args.operationalStatus,
|
|
22424
|
+
args.connectionId
|
|
22425
|
+
]
|
|
22426
|
+
});
|
|
22427
|
+
await getDb().execute({
|
|
22428
|
+
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
22429
|
+
args: [randomUUID10(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
22430
|
+
});
|
|
22431
|
+
}
|
|
22432
|
+
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
22433
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
22434
|
+
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
22435
|
+
await getDb().execute({
|
|
22436
|
+
sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
22437
|
+
args: [enabled ? 1 : 0, connectionId, connection.userId]
|
|
22438
|
+
});
|
|
22439
|
+
return enabled;
|
|
22440
|
+
}
|
|
22441
|
+
async function markServiceConnectionDisconnected(identity, connectionId) {
|
|
22442
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
22443
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
22444
|
+
await getDb().execute({
|
|
22445
|
+
sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
22446
|
+
args: [connectionId, connection.userId]
|
|
22447
|
+
});
|
|
22448
|
+
const remaining = await getDb().execute({
|
|
22449
|
+
sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
|
|
22450
|
+
args: [connection.userId]
|
|
22451
|
+
});
|
|
22452
|
+
return Number(remaining.rows[0]?.count ?? 0);
|
|
22453
|
+
}
|
|
22454
|
+
async function recordServiceConnectionAction(args) {
|
|
22455
|
+
await ensureServiceConnectionsSchema();
|
|
22456
|
+
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
22457
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
22458
|
+
await getDb().execute({
|
|
22459
|
+
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
22460
|
+
args: [args.status, args.errorCode ?? null, connection.userId, args.requestId, connection.id]
|
|
22461
|
+
});
|
|
22462
|
+
}
|
|
22463
|
+
async function claimServiceConnectionAction(args) {
|
|
22464
|
+
await ensureServiceConnectionsSchema();
|
|
22465
|
+
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
22466
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
22467
|
+
const inserted = await getDb().execute({
|
|
22468
|
+
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status) VALUES (?, ?, ?, ?, ?, 'started')`,
|
|
22469
|
+
args: [randomUUID10(), connection.id, connection.userId, args.tool, args.requestId]
|
|
22470
|
+
});
|
|
22471
|
+
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
22472
|
+
const existing = await getDb().execute({
|
|
22473
|
+
sql: `SELECT status, error_code FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
22474
|
+
args: [connection.userId, args.requestId]
|
|
22475
|
+
});
|
|
22476
|
+
return {
|
|
22477
|
+
claimed: false,
|
|
22478
|
+
status: String(existing.rows[0]?.status ?? "unknown"),
|
|
22479
|
+
errorCode: typeof existing.rows[0]?.error_code === "string" ? existing.rows[0].error_code : null
|
|
22480
|
+
};
|
|
22481
|
+
}
|
|
22482
|
+
|
|
22483
|
+
// src/api/main-nango-transport.ts
|
|
22484
|
+
var DEFAULT_NANGO_MCP_URL = "https://api.nango.dev/mcp";
|
|
22485
|
+
var NANGO_TIMEOUT_MS = 2e4;
|
|
22486
|
+
var NANGO_CONNECTION_PAGE_SIZE = 100;
|
|
22487
|
+
var NANGO_CONNECTION_MAX_PAGES = 20;
|
|
22488
|
+
var MAIN_INTEGRATION_CONTRACT_VERSION = "2026-07-16.1";
|
|
22489
|
+
var MAIN_INTEGRATION_CONTRACT_HASH = "f037859ec80c2ea2fac6e4a67c1a59f3200dd22d3642c97789b9bb3236723d3e";
|
|
22490
|
+
var MainNangoTransportError = class extends Error {
|
|
22491
|
+
status;
|
|
22492
|
+
code;
|
|
22493
|
+
retryable;
|
|
22494
|
+
constructor(message, status, code, retryable = status === 429 || status >= 500) {
|
|
22495
|
+
super(message);
|
|
22496
|
+
this.name = "MainNangoTransportError";
|
|
22497
|
+
this.status = status;
|
|
22498
|
+
this.code = code;
|
|
22499
|
+
this.retryable = retryable;
|
|
22500
|
+
}
|
|
22501
|
+
};
|
|
22502
|
+
function asRecord(value) {
|
|
22503
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
22504
|
+
}
|
|
22505
|
+
function text(value) {
|
|
22506
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
22507
|
+
}
|
|
22508
|
+
function arrayAt(payload, key) {
|
|
22509
|
+
if (Array.isArray(payload)) return payload;
|
|
22510
|
+
const root = asRecord(payload);
|
|
22511
|
+
if (Array.isArray(root?.[key])) return root[key];
|
|
22512
|
+
if (Array.isArray(root?.data)) return root.data;
|
|
22513
|
+
const data = asRecord(root?.data);
|
|
22514
|
+
return Array.isArray(data?.[key]) ? data[key] : [];
|
|
22515
|
+
}
|
|
22516
|
+
function nangoSecret() {
|
|
22517
|
+
const secret2 = process.env.NANGO_SECRET_KEY?.trim();
|
|
22518
|
+
if (!secret2) throw new MainNangoTransportError("Nango is not configured.", 503, "nango_not_configured");
|
|
22519
|
+
return secret2;
|
|
22520
|
+
}
|
|
22521
|
+
function nangoMcpUrl() {
|
|
22522
|
+
try {
|
|
22523
|
+
return new URL(process.env.NANGO_MCP_URL?.trim() || DEFAULT_NANGO_MCP_URL);
|
|
22524
|
+
} catch {
|
|
22525
|
+
throw new MainNangoTransportError("Nango MCP URL is invalid.", 503, "nango_invalid_url");
|
|
22526
|
+
}
|
|
22527
|
+
}
|
|
22528
|
+
async function nangoRequest(path5, options = {}) {
|
|
22529
|
+
const url = new URL(path5, `${nangoMcpUrl().origin}/`);
|
|
22530
|
+
for (const [key, value] of Object.entries(options.query ?? {})) {
|
|
22531
|
+
if (value !== void 0) url.searchParams.set(key, String(value));
|
|
22532
|
+
}
|
|
22533
|
+
let response;
|
|
22534
|
+
try {
|
|
22535
|
+
response = await fetch(url, {
|
|
22536
|
+
method: options.method ?? "GET",
|
|
22537
|
+
headers: {
|
|
22538
|
+
authorization: `Bearer ${nangoSecret()}`,
|
|
22539
|
+
accept: "application/json",
|
|
22540
|
+
...options.body ? { "content-type": "application/json" } : {}
|
|
22541
|
+
},
|
|
22542
|
+
body: options.body ? JSON.stringify(options.body) : void 0,
|
|
22543
|
+
signal: AbortSignal.timeout(NANGO_TIMEOUT_MS)
|
|
22544
|
+
});
|
|
22545
|
+
} catch {
|
|
22546
|
+
throw new MainNangoTransportError("Nango is temporarily unavailable.", 503, "connection_transport_unavailable");
|
|
22547
|
+
}
|
|
22548
|
+
const raw = await response.text();
|
|
22549
|
+
let payload = {};
|
|
22550
|
+
try {
|
|
22551
|
+
payload = raw ? JSON.parse(raw) : {};
|
|
22552
|
+
} catch {
|
|
22553
|
+
throw new MainNangoTransportError("Nango returned an invalid response.", 502, "nango_invalid_response");
|
|
22554
|
+
}
|
|
22555
|
+
if (!response.ok) {
|
|
22556
|
+
const status = response.status === 400 || response.status === 403 || response.status === 404 || response.status === 409 || response.status === 429 || response.status === 503 ? response.status : 502;
|
|
22557
|
+
throw new MainNangoTransportError(
|
|
22558
|
+
status === 429 ? "The connected provider is temporarily rate limited." : "Nango request failed.",
|
|
22559
|
+
status,
|
|
22560
|
+
status === 429 ? "upstream_rate_limited" : status === 404 ? "connection_not_found" : "connection_transport_unavailable"
|
|
22561
|
+
);
|
|
22562
|
+
}
|
|
22563
|
+
return payload;
|
|
22564
|
+
}
|
|
22565
|
+
function normalizedIdentity(value) {
|
|
22566
|
+
const valueText = text(value);
|
|
22567
|
+
return valueText ? valueText.normalize("NFKC").toLowerCase() : null;
|
|
22568
|
+
}
|
|
22569
|
+
function connectionIdentityMatches(value, identity) {
|
|
22570
|
+
const row = asRecord(value);
|
|
22571
|
+
const tags = asRecord(row?.tags);
|
|
22572
|
+
const endUser = asRecord(row?.end_user);
|
|
22573
|
+
const expected = normalizedIdentity(identity);
|
|
22574
|
+
if (!expected) return false;
|
|
22575
|
+
const emails = [normalizedIdentity(tags?.end_user_email), normalizedIdentity(endUser?.email)].filter((candidate) => !!candidate);
|
|
22576
|
+
if (emails.some((candidate) => candidate !== expected)) return false;
|
|
22577
|
+
const ids = [normalizedIdentity(tags?.end_user_id), normalizedIdentity(endUser?.id)].filter((candidate) => !!candidate && candidate.includes("@"));
|
|
22578
|
+
if (ids.some((candidate) => candidate !== expected)) return false;
|
|
22579
|
+
return emails.length > 0 ? emails.includes(expected) : ids.includes(expected);
|
|
22580
|
+
}
|
|
22581
|
+
function discoveredConnection(value) {
|
|
22582
|
+
const row = asRecord(value);
|
|
22583
|
+
const upstreamConnectionId = text(row?.connection_id);
|
|
22584
|
+
const providerConfigKey = text(row?.provider_config_key);
|
|
22585
|
+
if (!upstreamConnectionId || !providerConfigKey) return null;
|
|
22586
|
+
const tags = asRecord(row?.tags);
|
|
22587
|
+
const endUser = asRecord(row?.end_user);
|
|
22588
|
+
const errors = Array.isArray(row?.errors) ? row.errors : [];
|
|
22589
|
+
const authError = errors.some((error) => text(asRecord(error)?.type) === "auth");
|
|
22590
|
+
return {
|
|
22591
|
+
upstreamConnectionId,
|
|
22592
|
+
providerConfigKey,
|
|
22593
|
+
provider: text(row?.provider) ?? providerConfigKey,
|
|
22594
|
+
label: text(tags?.end_user_display_name) ?? text(endUser?.display_name) ?? text(tags?.end_user_email) ?? text(endUser?.email),
|
|
22595
|
+
lifecycleStatus: authError ? "needs_reauth" : "connected",
|
|
22596
|
+
reconnectRequired: authError,
|
|
22597
|
+
createdAt: text(row?.created_at) ?? text(row?.created),
|
|
22598
|
+
updatedAt: text(row?.updated_at)
|
|
22599
|
+
};
|
|
22600
|
+
}
|
|
22601
|
+
async function discoverOwnedNangoConnections(identity) {
|
|
22602
|
+
const rows = [];
|
|
22603
|
+
for (let page = 0; page < NANGO_CONNECTION_MAX_PAGES; page += 1) {
|
|
22604
|
+
const payload = await nangoRequest("/connections", {
|
|
22605
|
+
query: { limit: NANGO_CONNECTION_PAGE_SIZE, page }
|
|
22606
|
+
});
|
|
22607
|
+
const pageRows = arrayAt(payload, "connections");
|
|
22608
|
+
rows.push(...pageRows);
|
|
22609
|
+
if (pageRows.length < NANGO_CONNECTION_PAGE_SIZE) break;
|
|
22610
|
+
}
|
|
22611
|
+
const discovered = rows.filter((row) => connectionIdentityMatches(row, identity)).map(discoveredConnection).filter((connection) => connection !== null);
|
|
22612
|
+
return reconcileDiscoveredNangoConnections(identity, discovered);
|
|
22613
|
+
}
|
|
22614
|
+
async function listNangoCatalogDirect() {
|
|
22615
|
+
const [integrationsPayload, providersPayload] = await Promise.all([
|
|
22616
|
+
nangoRequest("/integrations"),
|
|
22617
|
+
nangoRequest("/providers")
|
|
22618
|
+
]);
|
|
22619
|
+
const providers = /* @__PURE__ */ new Map();
|
|
22620
|
+
for (const value of arrayAt(providersPayload, "providers")) {
|
|
22621
|
+
const row = asRecord(value);
|
|
22622
|
+
const name = text(row?.name);
|
|
22623
|
+
if (row && name) providers.set(name, row);
|
|
22624
|
+
}
|
|
22625
|
+
return arrayAt(integrationsPayload, "integrations").flatMap((value) => {
|
|
22626
|
+
const row = asRecord(value);
|
|
22627
|
+
const id = text(row?.unique_key) ?? text(row?.id);
|
|
22628
|
+
const provider = text(row?.provider);
|
|
22629
|
+
if (!id || !provider) return [];
|
|
22630
|
+
const definition = providers.get(provider);
|
|
22631
|
+
return [{
|
|
22632
|
+
id,
|
|
22633
|
+
provider,
|
|
22634
|
+
displayName: text(row?.display_name) ?? text(definition?.display_name) ?? id,
|
|
22635
|
+
authMode: text(definition?.auth_mode),
|
|
22636
|
+
categories: Array.isArray(definition?.categories) ? definition.categories : [],
|
|
22637
|
+
docsUrl: text(definition?.docs),
|
|
22638
|
+
logoUrl: text(row?.logo),
|
|
22639
|
+
safeDefaultAllowedTools: [],
|
|
22640
|
+
actionTools: [],
|
|
22641
|
+
setupStatus: "ready"
|
|
22642
|
+
}];
|
|
22643
|
+
});
|
|
22644
|
+
}
|
|
22645
|
+
function connectDefaults(providerConfigKey) {
|
|
22646
|
+
const scopes = {
|
|
22647
|
+
linkedin: "openid profile email w_member_social",
|
|
22648
|
+
"github-getting-started": "repo workflow read:org read:user user:email",
|
|
22649
|
+
"google-drive": "openid profile email https://www.googleapis.com/auth/drive",
|
|
22650
|
+
"google-search-console": "openid profile email https://www.googleapis.com/auth/webmasters.readonly https://www.googleapis.com/auth/webmasters"
|
|
22651
|
+
};
|
|
22652
|
+
const scope = scopes[providerConfigKey];
|
|
22653
|
+
return scope ? { [providerConfigKey]: { connection_config: { oauth_scopes_override: scope } } } : void 0;
|
|
22654
|
+
}
|
|
22655
|
+
function sessionPayload(value) {
|
|
22656
|
+
const root = asRecord(value);
|
|
22657
|
+
const data = asRecord(root?.data) ?? root;
|
|
22658
|
+
const connectLink = text(data?.connect_link);
|
|
22659
|
+
if (!connectLink) throw new MainNangoTransportError("Nango did not return a connect link.", 502, "nango_missing_connect_link");
|
|
22660
|
+
return { connectLink, expiresAt: text(data?.expires_at) };
|
|
22661
|
+
}
|
|
22662
|
+
async function createNangoSessionDirect(identity, providerConfigKey) {
|
|
22663
|
+
const defaults = connectDefaults(providerConfigKey);
|
|
22664
|
+
return sessionPayload(await nangoRequest("/connect/sessions", {
|
|
22665
|
+
method: "POST",
|
|
22666
|
+
body: {
|
|
22667
|
+
tags: { end_user_id: identity, end_user_email: identity },
|
|
22668
|
+
allowed_integrations: [providerConfigKey],
|
|
22669
|
+
...defaults ? { integrations_config_defaults: defaults } : {}
|
|
22670
|
+
}
|
|
22671
|
+
}));
|
|
22672
|
+
}
|
|
22673
|
+
async function createNangoReconnectSessionDirect(identity, connectionId) {
|
|
22674
|
+
const connection = await requireOwnedActiveConnection(identity, connectionId, true);
|
|
22675
|
+
const defaults = connectDefaults(connection.providerConfigKey);
|
|
22676
|
+
return sessionPayload(await nangoRequest("/connect/sessions/reconnect", {
|
|
22677
|
+
method: "POST",
|
|
22678
|
+
body: {
|
|
22679
|
+
connection_id: connection.upstreamConnectionId,
|
|
22680
|
+
integration_id: connection.providerConfigKey,
|
|
22681
|
+
tags: { end_user_id: identity, end_user_email: identity },
|
|
22682
|
+
...defaults ? { integrations_config_defaults: defaults } : {}
|
|
22683
|
+
}
|
|
22684
|
+
}));
|
|
22685
|
+
}
|
|
22686
|
+
async function deleteNangoConnectionDirect(identity, connectionId) {
|
|
22687
|
+
const connection = await requireOwnedActiveConnection(identity, connectionId, true);
|
|
22688
|
+
try {
|
|
22689
|
+
await nangoRequest(`/connections/${encodeURIComponent(connection.upstreamConnectionId)}`, {
|
|
22690
|
+
method: "DELETE",
|
|
22691
|
+
query: { provider_config_key: connection.providerConfigKey }
|
|
22692
|
+
});
|
|
22693
|
+
} catch (error) {
|
|
22694
|
+
if (!(error instanceof MainNangoTransportError) || error.status !== 404) throw error;
|
|
22695
|
+
}
|
|
22696
|
+
}
|
|
22697
|
+
async function requireOwnedActiveConnection(identity, connectionId, allowNeedsReauth = false) {
|
|
22698
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
22699
|
+
if (!connection || connection.transport !== "nango" || !connection.upstreamConnectionId) {
|
|
22700
|
+
throw new MainNangoTransportError("Service connection not found.", 404, "connection_not_found", false);
|
|
22701
|
+
}
|
|
22702
|
+
if (connection.lifecycleStatus !== "connected" && !(allowNeedsReauth && connection.lifecycleStatus === "needs_reauth")) {
|
|
22703
|
+
throw new MainNangoTransportError("Service connection requires reconnection.", 409, "connection_inactive", false);
|
|
22704
|
+
}
|
|
22705
|
+
return connection;
|
|
22706
|
+
}
|
|
22707
|
+
function classifyTool(tool) {
|
|
22708
|
+
if (tool.annotations?.readOnlyHint === true) return "read";
|
|
22709
|
+
if (tool.annotations?.readOnlyHint === false) return "action";
|
|
22710
|
+
return /^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(tool.name) ? "read" : "action";
|
|
22711
|
+
}
|
|
22712
|
+
async function withNangoClient(connection, run) {
|
|
22713
|
+
const transport = new StreamableHTTPClientTransport(nangoMcpUrl(), {
|
|
22714
|
+
requestInit: {
|
|
22715
|
+
headers: {
|
|
22716
|
+
Authorization: `Bearer ${nangoSecret()}`,
|
|
22717
|
+
"connection-id": connection.upstreamConnectionId,
|
|
22718
|
+
"provider-config-key": connection.providerConfigKey
|
|
22719
|
+
}
|
|
22720
|
+
}
|
|
22721
|
+
});
|
|
22722
|
+
const client2 = new Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
|
|
22723
|
+
try {
|
|
22724
|
+
await client2.connect(transport);
|
|
22725
|
+
const value = await run(client2);
|
|
22726
|
+
await recordServiceConnectionHealth({
|
|
22727
|
+
connectionId: connection.id,
|
|
22728
|
+
operationalStatus: "available",
|
|
22729
|
+
evidenceSource: "nango_mcp"
|
|
22730
|
+
});
|
|
22731
|
+
return value;
|
|
22732
|
+
} catch (error) {
|
|
22733
|
+
await recordServiceConnectionHealth({
|
|
22734
|
+
connectionId: connection.id,
|
|
22735
|
+
operationalStatus: "unavailable",
|
|
22736
|
+
failureCode: "connection_transport_unavailable",
|
|
22737
|
+
retryable: true,
|
|
22738
|
+
evidenceSource: "nango_mcp"
|
|
22739
|
+
}).catch(() => void 0);
|
|
22740
|
+
if (error instanceof MainNangoTransportError) throw error;
|
|
22741
|
+
throw new MainNangoTransportError("The service connection is temporarily unavailable.", 503, "connection_transport_unavailable");
|
|
22742
|
+
} finally {
|
|
22743
|
+
await client2.close().catch(() => void 0);
|
|
22744
|
+
}
|
|
22745
|
+
}
|
|
22746
|
+
async function listNangoToolsDirect(identity, connectionId) {
|
|
22747
|
+
const connection = await requireOwnedActiveConnection(identity, connectionId);
|
|
22748
|
+
const listed = await withNangoClient(connection, (client2) => client2.listTools());
|
|
22749
|
+
const tools = listed.tools.flatMap((raw) => {
|
|
22750
|
+
const inputSchema = asRecord(raw.inputSchema);
|
|
22751
|
+
if (!raw.name || !inputSchema) return [];
|
|
22752
|
+
return [{
|
|
22753
|
+
name: raw.name,
|
|
22754
|
+
...raw.title ? { title: raw.title } : {},
|
|
22755
|
+
...raw.description ? { description: raw.description } : {},
|
|
22756
|
+
inputSchema,
|
|
22757
|
+
...asRecord(raw.outputSchema) ? { outputSchema: asRecord(raw.outputSchema) } : {},
|
|
22758
|
+
...asRecord(raw.annotations) ? { annotations: asRecord(raw.annotations) } : {}
|
|
22759
|
+
}];
|
|
22760
|
+
});
|
|
22761
|
+
const readTools = tools.filter((tool) => classifyTool(tool) === "read").map((tool) => tool.name);
|
|
22762
|
+
const actionTools = tools.filter((tool) => classifyTool(tool) === "action").map((tool) => tool.name);
|
|
22763
|
+
const revision = createHash9("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
22764
|
+
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
22765
|
+
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
22766
|
+
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
22767
|
+
}
|
|
22768
|
+
async function callNangoToolDirect(args) {
|
|
22769
|
+
const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
|
|
22770
|
+
const selected = tools.find((tool) => tool.name === args.tool);
|
|
22771
|
+
if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
22772
|
+
const actualClassification = classifyTool(selected);
|
|
22773
|
+
if (actualClassification !== args.classification) {
|
|
22774
|
+
throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
|
|
22775
|
+
}
|
|
22776
|
+
if (actualClassification === "action" && !connection.actionsEnabled) {
|
|
22777
|
+
throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
|
|
22778
|
+
}
|
|
22779
|
+
await preflightConnectedUsage({
|
|
22780
|
+
identity: args.identity,
|
|
22781
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
22782
|
+
});
|
|
22783
|
+
const requestId = args.requestId?.trim() || randomUUID11();
|
|
22784
|
+
const idempotencyKey = `main-nango:${createHash9("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
22785
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
22786
|
+
const started = performance.now();
|
|
22787
|
+
let result;
|
|
22788
|
+
let providerError;
|
|
22789
|
+
try {
|
|
22790
|
+
result = await withNangoClient(connection, (client2) => client2.callTool({ name: args.tool, arguments: args.input }));
|
|
22791
|
+
} catch (error) {
|
|
22792
|
+
providerError = error;
|
|
22793
|
+
}
|
|
22794
|
+
const completedAt = /* @__PURE__ */ new Date();
|
|
22795
|
+
await settleConnectedUsage({
|
|
22796
|
+
identity: args.identity,
|
|
22797
|
+
idempotencyKey,
|
|
22798
|
+
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
22799
|
+
connectionId: connection.id,
|
|
22800
|
+
providerConfigKey: connection.providerConfigKey,
|
|
22801
|
+
functionRuns: 1,
|
|
22802
|
+
proxyRequests: 0,
|
|
22803
|
+
computeMilliseconds: Math.max(0, Math.round(performance.now() - started)),
|
|
22804
|
+
sourceSurface: "main_mcp",
|
|
22805
|
+
metadata: {
|
|
22806
|
+
toolName: args.tool,
|
|
22807
|
+
operationKind: args.operationKind ?? args.classification,
|
|
22808
|
+
outcome: providerError ? "error" : "partial",
|
|
22809
|
+
requestId: requestId.length <= 200 ? requestId : createHash9("sha256").update(requestId).digest("hex"),
|
|
22810
|
+
startedAt: startedAt.toISOString(),
|
|
22811
|
+
completedAt: completedAt.toISOString()
|
|
22812
|
+
}
|
|
22813
|
+
});
|
|
22814
|
+
if (providerError) throw providerError;
|
|
22815
|
+
return result;
|
|
22816
|
+
}
|
|
22817
|
+
async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
22818
|
+
const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
|
|
22819
|
+
const tool = tools.find((candidate) => candidate.name === toolName);
|
|
22820
|
+
if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
22821
|
+
const classification = classifyTool(tool);
|
|
22822
|
+
const callable = classification === "read" || connection.actionsEnabled;
|
|
22823
|
+
const projected = {
|
|
22824
|
+
name: tool.name,
|
|
22825
|
+
...tool.title ? { title: tool.title } : {},
|
|
22826
|
+
...tool.description ? { description: tool.description } : {},
|
|
22827
|
+
inputSchema: tool.inputSchema,
|
|
22828
|
+
...tool.outputSchema ? { outputSchema: tool.outputSchema } : {},
|
|
22829
|
+
...tool.annotations ? { annotations: tool.annotations } : {}
|
|
22830
|
+
};
|
|
22831
|
+
return {
|
|
22832
|
+
...projected,
|
|
22833
|
+
classification,
|
|
22834
|
+
callable,
|
|
22835
|
+
blockedReason: callable ? null : "actions_disabled",
|
|
22836
|
+
requiredPermissions: [],
|
|
22837
|
+
missingPermissions: [],
|
|
22838
|
+
requiredFeatures: [],
|
|
22839
|
+
missingFeatures: [],
|
|
22840
|
+
transport: "nango",
|
|
22841
|
+
providerConfigKey: connection.providerConfigKey,
|
|
22842
|
+
providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
|
|
22843
|
+
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
22844
|
+
protocolVersion: null,
|
|
22845
|
+
schemaSource: "live_tools_list",
|
|
22846
|
+
schemaHash: createHash9("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
22847
|
+
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22848
|
+
};
|
|
22849
|
+
}
|
|
22850
|
+
|
|
22851
|
+
// src/api/nango-control.ts
|
|
22852
|
+
var DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
22853
|
+
function mainOwnsIntegrations() {
|
|
22854
|
+
const configured = process.env.INTEGRATION_OWNER?.trim().toLowerCase();
|
|
22855
|
+
if (configured) return configured !== "legacy_scheduler";
|
|
22856
|
+
return !process.env.NANGO_CONTROL_URL?.trim();
|
|
22857
|
+
}
|
|
22858
|
+
function asNangoControlError(error) {
|
|
22859
|
+
if (error instanceof NangoControlError) return error;
|
|
22860
|
+
if (error instanceof MainNangoTransportError) {
|
|
22861
|
+
return new NangoControlError(error.message, error.status, error.code, error.retryable);
|
|
22862
|
+
}
|
|
22863
|
+
if (error instanceof Error && error.message === "service_connection_user_not_found") {
|
|
22864
|
+
return new NangoControlError("The connection owner was not found.", 404, "connection_not_found", false);
|
|
22865
|
+
}
|
|
22866
|
+
if (error instanceof Error && error.message === "service_connection_not_found") {
|
|
22867
|
+
return new NangoControlError("The service connection was not found.", 404, "connection_not_found", false);
|
|
22868
|
+
}
|
|
22869
|
+
return new NangoControlError("The service connection request failed.", 502, "connection_control_failed", true);
|
|
22870
|
+
}
|
|
22871
|
+
var DISABLED_NANGO_TOOLS = {
|
|
22872
|
+
"google-mail": /* @__PURE__ */ new Set(["list-filters"]),
|
|
22873
|
+
slack: /* @__PURE__ */ new Set(["search-files", "search-messages"])
|
|
22874
|
+
};
|
|
22875
|
+
var CONNECTION_SYNC_REQUIRED_TOOLS = {
|
|
22876
|
+
"google-mail": ["list-messages", "get-message"],
|
|
22877
|
+
"google-calendar": ["list-events", "get-event"],
|
|
22878
|
+
zoom: [
|
|
22879
|
+
"list-meetings",
|
|
22880
|
+
"get-meeting",
|
|
22881
|
+
"list-recordings",
|
|
22882
|
+
"get-recording",
|
|
22883
|
+
"get-meeting-transcript"
|
|
22884
|
+
],
|
|
22885
|
+
"github-getting-started": [
|
|
22886
|
+
"list-repositories",
|
|
22887
|
+
"list-issues",
|
|
22888
|
+
"list-pull-requests",
|
|
22889
|
+
"list-commits",
|
|
22890
|
+
"list-releases",
|
|
22891
|
+
"list-workflows",
|
|
22892
|
+
"list-workflow-runs"
|
|
22893
|
+
],
|
|
22894
|
+
"google-search-console": [
|
|
22895
|
+
"list-sites",
|
|
22896
|
+
"query-search-analytics"
|
|
22897
|
+
],
|
|
22898
|
+
resend: [
|
|
22899
|
+
"list-emails",
|
|
22900
|
+
"get-email",
|
|
22901
|
+
"list-received-emails",
|
|
22902
|
+
"get-received-email",
|
|
22903
|
+
"list-logs",
|
|
22904
|
+
"get-log",
|
|
22905
|
+
"list-contacts",
|
|
22906
|
+
"get-contact",
|
|
22907
|
+
"list-broadcasts",
|
|
22908
|
+
"get-broadcast",
|
|
22909
|
+
"list-templates",
|
|
22910
|
+
"get-template"
|
|
22911
|
+
],
|
|
22912
|
+
"meta-marketing-api": [
|
|
22913
|
+
"list-ad-accounts",
|
|
22914
|
+
"list-campaigns",
|
|
22915
|
+
"list-ad-sets",
|
|
22916
|
+
"list-ads",
|
|
22917
|
+
"list-ad-creatives",
|
|
22918
|
+
"get-insights"
|
|
22919
|
+
]
|
|
22920
|
+
};
|
|
22921
|
+
var CONNECTION_SYNC_OPTIONAL_TOOLS = {
|
|
22922
|
+
"meta-marketing-api": [
|
|
22923
|
+
"list-catalogs",
|
|
22924
|
+
"list-datasets",
|
|
22925
|
+
"list-custom-audiences",
|
|
22926
|
+
"list-ad-images",
|
|
22927
|
+
"list-ad-videos",
|
|
22928
|
+
"list-ad-rules"
|
|
22929
|
+
]
|
|
22930
|
+
};
|
|
22931
|
+
function connectionSyncPolicyIssues(selections) {
|
|
22932
|
+
const issues = [];
|
|
22933
|
+
for (const selection of selections) {
|
|
22934
|
+
const required = CONNECTION_SYNC_REQUIRED_TOOLS[selection.providerConfigKey];
|
|
22935
|
+
if (!required) {
|
|
22936
|
+
issues.push({ providerConfigKey: selection.providerConfigKey, code: "unsupported_provider" });
|
|
22937
|
+
continue;
|
|
22938
|
+
}
|
|
22939
|
+
const allowed = new Set(selection.allowedTools);
|
|
21873
22940
|
const missingTools = required.filter((tool) => !allowed.has(tool));
|
|
21874
22941
|
if (missingTools.length > 0) {
|
|
21875
22942
|
issues.push({ providerConfigKey: selection.providerConfigKey, code: "missing_required_tools", missingTools });
|
|
@@ -21990,7 +23057,28 @@ var SAFE_CONTROL_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
|
21990
23057
|
"connection_control_failed",
|
|
21991
23058
|
"missing_provider_permission",
|
|
21992
23059
|
"permission_verification_unavailable",
|
|
21993
|
-
"meta_app_feature_not_enabled"
|
|
23060
|
+
"meta_app_feature_not_enabled",
|
|
23061
|
+
"insufficient_credits",
|
|
23062
|
+
"pending_cost",
|
|
23063
|
+
"rate_policy_mismatch",
|
|
23064
|
+
"connected_usage_billing_unavailable",
|
|
23065
|
+
"action_outcome_unknown",
|
|
23066
|
+
"billing_dispatch_incomplete",
|
|
23067
|
+
"billing_dispatch_busy"
|
|
23068
|
+
]);
|
|
23069
|
+
var FIXED_CONTROL_ERROR_MESSAGES = /* @__PURE__ */ new Map([
|
|
23070
|
+
[
|
|
23071
|
+
"action_outcome_unknown",
|
|
23072
|
+
"The previous connected action may have completed, so it was not run again. Verify the provider before retrying with a new Idempotency-Key."
|
|
23073
|
+
],
|
|
23074
|
+
[
|
|
23075
|
+
"billing_dispatch_incomplete",
|
|
23076
|
+
"A previous connected action could not be confirmed safely. Contact support before retrying."
|
|
23077
|
+
],
|
|
23078
|
+
[
|
|
23079
|
+
"billing_dispatch_busy",
|
|
23080
|
+
"Another connected action is currently being finalized. Retry shortly with the same Idempotency-Key."
|
|
23081
|
+
]
|
|
21994
23082
|
]);
|
|
21995
23083
|
var CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
|
|
21996
23084
|
["connection_not_active", "connection_inactive"],
|
|
@@ -22000,13 +23088,14 @@ var CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
|
|
|
22000
23088
|
["tool_not_available", "live_tool_missing"]
|
|
22001
23089
|
]);
|
|
22002
23090
|
function controlErrorStatus(status) {
|
|
22003
|
-
if (status === 400 || status === 403 || status === 404 || status === 409 || status === 429 || status === 502 || status === 503) {
|
|
23091
|
+
if (status === 400 || status === 402 || status === 403 || status === 404 || status === 409 || status === 429 || status === 502 || status === 503) {
|
|
22004
23092
|
return status;
|
|
22005
23093
|
}
|
|
22006
23094
|
return status >= 500 ? 503 : 502;
|
|
22007
23095
|
}
|
|
22008
23096
|
function defaultControlErrorCode(status) {
|
|
22009
23097
|
if (status === 400) return "invalid_request";
|
|
23098
|
+
if (status === 402) return "insufficient_credits";
|
|
22010
23099
|
if (status === 403) return "actions_disabled";
|
|
22011
23100
|
if (status === 404) return "connection_not_found";
|
|
22012
23101
|
if (status === 409) return "connection_inactive";
|
|
@@ -22016,6 +23105,7 @@ function defaultControlErrorCode(status) {
|
|
|
22016
23105
|
}
|
|
22017
23106
|
function defaultControlErrorMessage(status) {
|
|
22018
23107
|
if (status === 400) return "The connection request was invalid.";
|
|
23108
|
+
if (status === 402) return "There are not enough Credits to run this connected operation.";
|
|
22019
23109
|
if (status === 403) return "Actions are disabled for this service connection.";
|
|
22020
23110
|
if (status === 404) return "The service connection was not found.";
|
|
22021
23111
|
if (status === 409) return "The service connection is not currently usable.";
|
|
@@ -22031,7 +23121,8 @@ function safeControlErrorPayload(body, responseStatus) {
|
|
|
22031
23121
|
const normalizedCandidateCode = candidateCode ? CONTROL_ERROR_CODE_ALIASES.get(candidateCode) ?? candidateCode : null;
|
|
22032
23122
|
const code = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) ? normalizedCandidateCode : defaultControlErrorCode(status);
|
|
22033
23123
|
const candidateMessage = record ? firstString(record, ["error", "message"], 500) : null;
|
|
22034
|
-
const
|
|
23124
|
+
const fixedMessage = FIXED_CONTROL_ERROR_MESSAGES.get(code);
|
|
23125
|
+
const message = fixedMessage ?? (normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) && candidateMessage ? candidateMessage.replace(/[\u0000-\u001f\u007f]/g, " ") : defaultControlErrorMessage(status));
|
|
22035
23126
|
const retryable = record && typeof record.retryable === "boolean" ? record.retryable : status === 429 || status >= 500;
|
|
22036
23127
|
return { status, code, message, retryable };
|
|
22037
23128
|
}
|
|
@@ -22054,10 +23145,10 @@ async function controlRequest(path5, init, timeoutMs = 2e4) {
|
|
|
22054
23145
|
true
|
|
22055
23146
|
);
|
|
22056
23147
|
});
|
|
22057
|
-
const
|
|
23148
|
+
const text2 = await response.text();
|
|
22058
23149
|
let body = {};
|
|
22059
23150
|
try {
|
|
22060
|
-
body =
|
|
23151
|
+
body = text2 ? JSON.parse(text2) : {};
|
|
22061
23152
|
} catch {
|
|
22062
23153
|
body = {};
|
|
22063
23154
|
}
|
|
@@ -22088,7 +23179,9 @@ function sanitizeScheduleConnectionSelections(value) {
|
|
|
22088
23179
|
return selections;
|
|
22089
23180
|
}
|
|
22090
23181
|
async function getNangoCatalog() {
|
|
22091
|
-
const body = await
|
|
23182
|
+
const body = mainOwnsIntegrations() ? { providers: await listNangoCatalogDirect().catch((error) => {
|
|
23183
|
+
throw asNangoControlError(error);
|
|
23184
|
+
}) } : await controlRequest("/api/internal/nango/catalog");
|
|
22092
23185
|
const rows = arrayFromPayload(body, ["providers", "catalog", "integrations", "services"]);
|
|
22093
23186
|
const result = [];
|
|
22094
23187
|
for (const row of rows) {
|
|
@@ -22103,9 +23196,10 @@ async function getNangoCatalog() {
|
|
|
22103
23196
|
const authMode = firstString(row, ["authMode", "auth_mode"], 100);
|
|
22104
23197
|
const categories = cleanStringArray(row.categories);
|
|
22105
23198
|
const disabledTools = DISABLED_NANGO_TOOLS[providerConfigKey];
|
|
22106
|
-
const
|
|
23199
|
+
const catalogAllowedTools = cleanTools(
|
|
22107
23200
|
row.safeDefaultAllowedTools ?? row.safe_default_allowed_tools ?? row.defaultAllowedTools ?? row.default_allowed_tools ?? row.allowedTools ?? row.allowed_tools
|
|
22108
23201
|
).filter((tool) => !disabledTools?.has(tool));
|
|
23202
|
+
const safeDefaultAllowedTools = catalogAllowedTools.length > 0 ? catalogAllowedTools : mainOwnsIntegrations() ? [...CONNECTION_SYNC_REQUIRED_TOOLS[providerConfigKey] ?? []] : [];
|
|
22109
23203
|
const actionTools = cleanTools(
|
|
22110
23204
|
row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools
|
|
22111
23205
|
).filter((tool) => !disabledTools?.has(tool));
|
|
@@ -22161,8 +23255,88 @@ async function getNangoCatalog() {
|
|
|
22161
23255
|
}
|
|
22162
23256
|
return result;
|
|
22163
23257
|
}
|
|
22164
|
-
async function getNangoConnections(identity) {
|
|
22165
|
-
|
|
23258
|
+
async function getNangoConnections(identity, options = {}) {
|
|
23259
|
+
if (mainOwnsIntegrations()) {
|
|
23260
|
+
let connections;
|
|
23261
|
+
try {
|
|
23262
|
+
connections = await discoverOwnedNangoConnections(identity);
|
|
23263
|
+
} catch (error) {
|
|
23264
|
+
try {
|
|
23265
|
+
connections = await listServiceConnections(identity, "nango");
|
|
23266
|
+
} catch {
|
|
23267
|
+
throw asNangoControlError(error);
|
|
23268
|
+
}
|
|
23269
|
+
}
|
|
23270
|
+
return Promise.all(connections.map(async (stored) => {
|
|
23271
|
+
let connection = stored;
|
|
23272
|
+
if (!options.summaryOnly && stored.lifecycleStatus === "connected") {
|
|
23273
|
+
try {
|
|
23274
|
+
connection = (await listNangoToolsDirect(identity, stored.id)).connection;
|
|
23275
|
+
} catch {
|
|
23276
|
+
connection = await getOwnedServiceConnection(identity, stored.id) ?? stored;
|
|
23277
|
+
}
|
|
23278
|
+
}
|
|
23279
|
+
return {
|
|
23280
|
+
connectionId: connection.id,
|
|
23281
|
+
providerConfigKey: connection.providerConfigKey,
|
|
23282
|
+
provider: connection.provider,
|
|
23283
|
+
label: connection.label || connection.providerConfigKey,
|
|
23284
|
+
providerAccountId: null,
|
|
23285
|
+
providerAccountEmail: null,
|
|
23286
|
+
providerAccountName: null,
|
|
23287
|
+
providerIdentityStatus: "unavailable",
|
|
23288
|
+
status: connection.reconnectRequired ? "needs_reauth" : "connected",
|
|
23289
|
+
lifecycleStatus: connection.lifecycleStatus,
|
|
23290
|
+
operationalStatus: connection.operationalStatus,
|
|
23291
|
+
billingActive: connection.lifecycleStatus === "connected",
|
|
23292
|
+
reconnectRequired: connection.reconnectRequired,
|
|
23293
|
+
actionsEnabled: connection.actionsEnabled,
|
|
23294
|
+
readTools: connection.readTools,
|
|
23295
|
+
actionTools: connection.actionTools,
|
|
23296
|
+
toolCapabilities: [
|
|
23297
|
+
...connection.readTools.map((name) => ({
|
|
23298
|
+
name,
|
|
23299
|
+
classification: "read",
|
|
23300
|
+
requiredPermissions: [],
|
|
23301
|
+
requiredFeatures: [],
|
|
23302
|
+
available: connection.operationalStatus !== "unavailable",
|
|
23303
|
+
blockedReason: null,
|
|
23304
|
+
missingPermissions: [],
|
|
23305
|
+
missingFeatures: []
|
|
23306
|
+
})),
|
|
23307
|
+
...connection.actionTools.map((name) => ({
|
|
23308
|
+
name,
|
|
23309
|
+
classification: "action",
|
|
23310
|
+
requiredPermissions: [],
|
|
23311
|
+
requiredFeatures: [],
|
|
23312
|
+
available: connection.operationalStatus !== "unavailable",
|
|
23313
|
+
blockedReason: null,
|
|
23314
|
+
missingPermissions: [],
|
|
23315
|
+
missingFeatures: []
|
|
23316
|
+
}))
|
|
23317
|
+
],
|
|
23318
|
+
grantedPermissions: [],
|
|
23319
|
+
enabledFeatures: [],
|
|
23320
|
+
permissionVerification: null,
|
|
23321
|
+
mcpEndpoint: null,
|
|
23322
|
+
schemaDiscovery: "compatibility_describe",
|
|
23323
|
+
toolRevision: connection.toolRevision,
|
|
23324
|
+
vaultName: connection.vaultName,
|
|
23325
|
+
tableName: connection.tableName,
|
|
23326
|
+
createdAt: connection.createdAt,
|
|
23327
|
+
updatedAt: connection.updatedAt,
|
|
23328
|
+
lastCheckedAt: connection.lastCheckedAt,
|
|
23329
|
+
lastSuccessfulCallAt: connection.lastSuccessfulCallAt,
|
|
23330
|
+
lastFailureAt: connection.lastFailureAt,
|
|
23331
|
+
lastFailureCode: connection.lastFailureCode,
|
|
23332
|
+
lastFailureRetryable: connection.lastFailureRetryable
|
|
23333
|
+
};
|
|
23334
|
+
}));
|
|
23335
|
+
}
|
|
23336
|
+
const query = new URLSearchParams({
|
|
23337
|
+
identity,
|
|
23338
|
+
...options.summaryOnly ? { view: "billing_summary" } : {}
|
|
23339
|
+
}).toString();
|
|
22166
23340
|
const body = await controlRequest(`/api/internal/nango/connections?${query}`);
|
|
22167
23341
|
const rows = arrayFromPayload(body, ["connections"]);
|
|
22168
23342
|
const result = [];
|
|
@@ -22172,7 +23346,14 @@ async function getNangoConnections(identity) {
|
|
|
22172
23346
|
const providerConfigKey = firstString(row, ["providerConfigKey", "provider_config_key", "integrationId", "integration_id", "provider"]);
|
|
22173
23347
|
if (!connectionId || !providerConfigKey) continue;
|
|
22174
23348
|
const provider = firstString(row, ["provider"]);
|
|
22175
|
-
const
|
|
23349
|
+
const providerAccountId = firstString(row, ["providerAccountId", "provider_account_id"], 512);
|
|
23350
|
+
const providerAccountEmail = firstString(row, ["providerAccountEmail", "provider_account_email"], 320)?.toLowerCase() ?? null;
|
|
23351
|
+
const providerAccountName = firstString(row, ["providerAccountName", "provider_account_name"], 300);
|
|
23352
|
+
const rawLabel = firstString(row, ["label", "accountLabel", "account_label", "displayName", "display_name"]);
|
|
23353
|
+
const safeLegacyLabel = rawLabel?.toLowerCase() === identity.trim().toLowerCase() ? null : rawLabel;
|
|
23354
|
+
const label = providerAccountEmail || providerAccountName || safeLegacyLabel || providerConfigKey;
|
|
23355
|
+
const rawProviderIdentityStatus = firstString(row, ["providerIdentityStatus", "provider_identity_status"], 32);
|
|
23356
|
+
const providerIdentityStatus = rawProviderIdentityStatus === "verified" || rawProviderIdentityStatus === "unavailable" ? rawProviderIdentityStatus : providerAccountId || providerAccountEmail || providerAccountName ? "verified" : "pending";
|
|
22176
23357
|
const rawStatus = firstString(row, ["status"], 64) || "connected";
|
|
22177
23358
|
const reconnectRequired = row.reconnectRequired === true || row.reconnect_required === true || rawStatus === "needs_reauth" || rawStatus === "reauth_required" || rawStatus === "invalid" || rawStatus === "error" || rawStatus === "revoked" || rawStatus === "disabled";
|
|
22178
23359
|
const toolCapabilities = [];
|
|
@@ -22204,7 +23385,12 @@ async function getNangoConnections(identity) {
|
|
|
22204
23385
|
providerConfigKey,
|
|
22205
23386
|
provider,
|
|
22206
23387
|
label,
|
|
23388
|
+
providerAccountId,
|
|
23389
|
+
providerAccountEmail,
|
|
23390
|
+
providerAccountName,
|
|
23391
|
+
providerIdentityStatus,
|
|
22207
23392
|
status: reconnectRequired ? "needs_reauth" : "connected",
|
|
23393
|
+
billingActive: row.billingActive !== false && row.billing_active !== false && rawStatus !== "revoked",
|
|
22208
23394
|
reconnectRequired,
|
|
22209
23395
|
actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
|
|
22210
23396
|
readTools: cleanTools(row.readTools ?? row.read_tools),
|
|
@@ -22224,6 +23410,30 @@ async function getNangoConnections(identity) {
|
|
|
22224
23410
|
}
|
|
22225
23411
|
return result;
|
|
22226
23412
|
}
|
|
23413
|
+
async function deleteNangoConnection(identity, connectionId) {
|
|
23414
|
+
if (mainOwnsIntegrations()) {
|
|
23415
|
+
try {
|
|
23416
|
+
await deleteNangoConnectionDirect(identity, connectionId);
|
|
23417
|
+
await controlRequest(`/api/internal/integrations/${encodeURIComponent(connectionId)}/purge`, {
|
|
23418
|
+
method: "POST",
|
|
23419
|
+
body: JSON.stringify({ identity })
|
|
23420
|
+
});
|
|
23421
|
+
return await markServiceConnectionDisconnected(identity, connectionId);
|
|
23422
|
+
} catch (error) {
|
|
23423
|
+
throw asNangoControlError(error);
|
|
23424
|
+
}
|
|
23425
|
+
}
|
|
23426
|
+
const body = await controlRequest("/api/internal/nango/connections", {
|
|
23427
|
+
method: "DELETE",
|
|
23428
|
+
body: JSON.stringify({ identity, connectionId })
|
|
23429
|
+
});
|
|
23430
|
+
const data = unwrapData(body);
|
|
23431
|
+
if (!isRecord2(data) || data.ok !== true) {
|
|
23432
|
+
throw new NangoControlError("The connection service returned an invalid deletion response.");
|
|
23433
|
+
}
|
|
23434
|
+
const remaining = Number(data.remainingConnections);
|
|
23435
|
+
return Number.isSafeInteger(remaining) && remaining >= 0 ? remaining : 0;
|
|
23436
|
+
}
|
|
22227
23437
|
function sanitizeConnectSession(body) {
|
|
22228
23438
|
const data = unwrapData(body);
|
|
22229
23439
|
if (!isRecord2(data)) throw new NangoControlError("The connection service returned an invalid connect session.");
|
|
@@ -22236,6 +23446,14 @@ function sanitizeConnectSession(body) {
|
|
|
22236
23446
|
};
|
|
22237
23447
|
}
|
|
22238
23448
|
async function createNangoConnectSession(identity, providerConfigKey) {
|
|
23449
|
+
if (mainOwnsIntegrations()) {
|
|
23450
|
+
try {
|
|
23451
|
+
const session = await createNangoSessionDirect(identity, providerConfigKey);
|
|
23452
|
+
return { ...session, sessionToken: null };
|
|
23453
|
+
} catch (error) {
|
|
23454
|
+
throw asNangoControlError(error);
|
|
23455
|
+
}
|
|
23456
|
+
}
|
|
22239
23457
|
const body = await controlRequest("/api/internal/nango/connect-session", {
|
|
22240
23458
|
method: "POST",
|
|
22241
23459
|
body: JSON.stringify({ identity, email: identity, provider: providerConfigKey })
|
|
@@ -22243,6 +23461,14 @@ async function createNangoConnectSession(identity, providerConfigKey) {
|
|
|
22243
23461
|
return sanitizeConnectSession(body);
|
|
22244
23462
|
}
|
|
22245
23463
|
async function createNangoReconnectSession(identity, connectionId) {
|
|
23464
|
+
if (mainOwnsIntegrations()) {
|
|
23465
|
+
try {
|
|
23466
|
+
const session = await createNangoReconnectSessionDirect(identity, connectionId);
|
|
23467
|
+
return { ...session, sessionToken: null };
|
|
23468
|
+
} catch (error) {
|
|
23469
|
+
throw asNangoControlError(error);
|
|
23470
|
+
}
|
|
23471
|
+
}
|
|
22246
23472
|
const body = await controlRequest("/api/internal/nango/reconnect-session", {
|
|
22247
23473
|
method: "POST",
|
|
22248
23474
|
body: JSON.stringify({ identity, connectionId, email: identity })
|
|
@@ -22296,6 +23522,7 @@ async function putScheduleConnectionBindings(identity, scheduleActionId, connect
|
|
|
22296
23522
|
scheduleId: scheduleActionId,
|
|
22297
23523
|
connections: connections.map((connection) => ({
|
|
22298
23524
|
connectionId: connection.connectionId,
|
|
23525
|
+
providerConfigKey: connection.providerConfigKey,
|
|
22299
23526
|
allowedTools: connection.allowedTools
|
|
22300
23527
|
}))
|
|
22301
23528
|
})
|
|
@@ -22324,6 +23551,13 @@ async function deleteScheduleConnectionBindings(identity, scheduleActionId) {
|
|
|
22324
23551
|
});
|
|
22325
23552
|
}
|
|
22326
23553
|
async function setScheduleConnectionActionsEnabled(identity, connectionId, enabled) {
|
|
23554
|
+
if (mainOwnsIntegrations()) {
|
|
23555
|
+
try {
|
|
23556
|
+
return await setServiceConnectionActions(identity, connectionId, enabled);
|
|
23557
|
+
} catch (error) {
|
|
23558
|
+
throw asNangoControlError(error);
|
|
23559
|
+
}
|
|
23560
|
+
}
|
|
22327
23561
|
const body = await controlRequest("/api/internal/nango/connections/actions/enable", {
|
|
22328
23562
|
method: "POST",
|
|
22329
23563
|
body: JSON.stringify({ identity, connectionId, enabled })
|
|
@@ -22332,15 +23566,71 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
22332
23566
|
if (!isRecord2(data) || !isRecord2(data.connection)) return enabled;
|
|
22333
23567
|
return data.connection.actionsEnabled === true;
|
|
22334
23568
|
}
|
|
22335
|
-
async function callScheduleConnectionAction(identity, connectionId, input, tool) {
|
|
23569
|
+
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey) {
|
|
23570
|
+
const requestId = `main-connected-action:${createHash10("sha256").update(identity).update("\0").update(idempotencyKey?.trim() || randomUUID12()).digest("hex")}`;
|
|
23571
|
+
if (mainOwnsIntegrations()) {
|
|
23572
|
+
const selectedTool = tool?.trim();
|
|
23573
|
+
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
23574
|
+
try {
|
|
23575
|
+
const claim = await claimServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId });
|
|
23576
|
+
if (!claim.claimed) {
|
|
23577
|
+
throw new NangoControlError(
|
|
23578
|
+
"The previous connected action may have completed, so it was not run again.",
|
|
23579
|
+
409,
|
|
23580
|
+
"action_outcome_unknown",
|
|
23581
|
+
false
|
|
23582
|
+
);
|
|
23583
|
+
}
|
|
23584
|
+
const result = await callNangoToolDirect({
|
|
23585
|
+
identity,
|
|
23586
|
+
connectionId,
|
|
23587
|
+
tool: selectedTool,
|
|
23588
|
+
input,
|
|
23589
|
+
classification: "action",
|
|
23590
|
+
requestId,
|
|
23591
|
+
operationKind: "action"
|
|
23592
|
+
});
|
|
23593
|
+
await recordServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, status: "succeeded" });
|
|
23594
|
+
return result;
|
|
23595
|
+
} catch (error) {
|
|
23596
|
+
const normalized = asNangoControlError(error);
|
|
23597
|
+
if (normalized.code !== "action_outcome_unknown") {
|
|
23598
|
+
await recordServiceConnectionAction({
|
|
23599
|
+
identity,
|
|
23600
|
+
connectionId,
|
|
23601
|
+
tool: selectedTool,
|
|
23602
|
+
requestId,
|
|
23603
|
+
status: normalized.retryable ? "unknown" : "failed",
|
|
23604
|
+
errorCode: normalized.code
|
|
23605
|
+
}).catch(() => void 0);
|
|
23606
|
+
}
|
|
23607
|
+
throw normalized;
|
|
23608
|
+
}
|
|
23609
|
+
}
|
|
22336
23610
|
const body = await controlRequest("/api/internal/nango/connections/actions/call", {
|
|
22337
23611
|
method: "POST",
|
|
23612
|
+
headers: { "x-request-id": requestId },
|
|
22338
23613
|
body: JSON.stringify({ identity, connectionId, ...tool ? { tool } : {}, input })
|
|
22339
23614
|
});
|
|
22340
23615
|
const data = unwrapData(body);
|
|
22341
23616
|
return isRecord2(data) ? data.result ?? data : data;
|
|
22342
23617
|
}
|
|
22343
|
-
async function callScheduleConnectionRead(identity, connectionId, tool, args) {
|
|
23618
|
+
async function callScheduleConnectionRead(identity, connectionId, tool, args, idempotencyKey) {
|
|
23619
|
+
if (mainOwnsIntegrations()) {
|
|
23620
|
+
try {
|
|
23621
|
+
return await callNangoToolDirect({
|
|
23622
|
+
identity,
|
|
23623
|
+
connectionId,
|
|
23624
|
+
tool,
|
|
23625
|
+
input: args ?? {},
|
|
23626
|
+
classification: "read",
|
|
23627
|
+
requestId: idempotencyKey,
|
|
23628
|
+
operationKind: "read"
|
|
23629
|
+
});
|
|
23630
|
+
} catch (error) {
|
|
23631
|
+
throw asNangoControlError(error);
|
|
23632
|
+
}
|
|
23633
|
+
}
|
|
22344
23634
|
const body = await controlRequest("/api/internal/nango/connections/actions/read", {
|
|
22345
23635
|
method: "POST",
|
|
22346
23636
|
body: JSON.stringify({ identity, connectionId, tool, args: args ?? {} })
|
|
@@ -22348,6 +23638,22 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args) {
|
|
|
22348
23638
|
const data = unwrapData(body);
|
|
22349
23639
|
return isRecord2(data) ? data.result ?? data : data;
|
|
22350
23640
|
}
|
|
23641
|
+
async function testNangoConnection(identity, connectionId) {
|
|
23642
|
+
if (!mainOwnsIntegrations()) {
|
|
23643
|
+
throw new NangoControlError(
|
|
23644
|
+
"Connection testing requires main-MCP integration ownership.",
|
|
23645
|
+
503,
|
|
23646
|
+
"connection_transport_unavailable",
|
|
23647
|
+
true
|
|
23648
|
+
);
|
|
23649
|
+
}
|
|
23650
|
+
try {
|
|
23651
|
+
const { tools } = await listNangoToolsDirect(identity, connectionId);
|
|
23652
|
+
return { operationalStatus: "available", checkedAt: (/* @__PURE__ */ new Date()).toISOString(), toolCount: tools.length };
|
|
23653
|
+
} catch (error) {
|
|
23654
|
+
throw asNangoControlError(error);
|
|
23655
|
+
}
|
|
23656
|
+
}
|
|
22351
23657
|
function sanitizeToolSchema(value) {
|
|
22352
23658
|
if (!isRecord2(value) || value.type !== "object") return null;
|
|
22353
23659
|
try {
|
|
@@ -22402,9 +23708,18 @@ function canonicalJson2(value) {
|
|
|
22402
23708
|
return JSON.stringify(value);
|
|
22403
23709
|
}
|
|
22404
23710
|
function projectedToolSchemaHash(tool) {
|
|
22405
|
-
return
|
|
23711
|
+
return createHash10("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
22406
23712
|
}
|
|
22407
23713
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
23714
|
+
if (mainOwnsIntegrations()) {
|
|
23715
|
+
let direct;
|
|
23716
|
+
try {
|
|
23717
|
+
direct = await describeNangoToolDirect(identity, connectionId, tool);
|
|
23718
|
+
} catch (error) {
|
|
23719
|
+
throw asNangoControlError(error);
|
|
23720
|
+
}
|
|
23721
|
+
return sanitizeNangoToolDescription(direct, tool);
|
|
23722
|
+
}
|
|
22408
23723
|
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
22409
23724
|
method: "POST",
|
|
22410
23725
|
body: JSON.stringify({
|
|
@@ -22414,6 +23729,9 @@ async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
|
22414
23729
|
...fresh === void 0 ? {} : { fresh }
|
|
22415
23730
|
})
|
|
22416
23731
|
});
|
|
23732
|
+
return sanitizeNangoToolDescription(body, tool);
|
|
23733
|
+
}
|
|
23734
|
+
function sanitizeNangoToolDescription(body, tool) {
|
|
22417
23735
|
const data = unwrapData(body);
|
|
22418
23736
|
const rawTool = isRecord2(data) && isRecord2(data.tool) ? data.tool : data;
|
|
22419
23737
|
if (!isRecord2(rawTool)) {
|
|
@@ -22496,6 +23814,13 @@ async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
|
22496
23814
|
};
|
|
22497
23815
|
}
|
|
22498
23816
|
async function callScheduleConnectionExportPage(identity, input) {
|
|
23817
|
+
if (mainOwnsIntegrations()) {
|
|
23818
|
+
try {
|
|
23819
|
+
return await callMainOwnedExportPage(identity, input);
|
|
23820
|
+
} catch (error) {
|
|
23821
|
+
throw asNangoControlError(error);
|
|
23822
|
+
}
|
|
23823
|
+
}
|
|
22499
23824
|
const body = await controlRequest("/api/internal/nango/connections/export-page", {
|
|
22500
23825
|
method: "POST",
|
|
22501
23826
|
body: JSON.stringify({ identity, ...input })
|
|
@@ -22532,6 +23857,118 @@ async function callScheduleConnectionExportPage(identity, input) {
|
|
|
22532
23857
|
untrustedContent: true
|
|
22533
23858
|
};
|
|
22534
23859
|
}
|
|
23860
|
+
function unwrapMcpToolJson(value) {
|
|
23861
|
+
if (!isRecord2(value)) return value;
|
|
23862
|
+
if (isRecord2(value.structuredContent)) return value.structuredContent;
|
|
23863
|
+
if (Array.isArray(value.content)) {
|
|
23864
|
+
const textPart = value.content.find((part) => isRecord2(part) && part.type === "text" && typeof part.text === "string");
|
|
23865
|
+
if (isRecord2(textPart) && typeof textPart.text === "string") {
|
|
23866
|
+
try {
|
|
23867
|
+
return JSON.parse(textPart.text);
|
|
23868
|
+
} catch {
|
|
23869
|
+
return textPart.text;
|
|
23870
|
+
}
|
|
23871
|
+
}
|
|
23872
|
+
}
|
|
23873
|
+
return value;
|
|
23874
|
+
}
|
|
23875
|
+
function rowsFromToolResult(value, keys) {
|
|
23876
|
+
const unwrapped = unwrapMcpToolJson(value);
|
|
23877
|
+
if (Array.isArray(unwrapped)) return unwrapped;
|
|
23878
|
+
if (!isRecord2(unwrapped)) return [];
|
|
23879
|
+
for (const key of keys) if (Array.isArray(unwrapped[key])) return unwrapped[key];
|
|
23880
|
+
return [];
|
|
23881
|
+
}
|
|
23882
|
+
async function callMainOwnedExportPage(identity, input) {
|
|
23883
|
+
const connection = await getOwnedServiceConnection(identity, input.connectionId);
|
|
23884
|
+
if (!connection) throw new NangoControlError("The service connection was not found.", 404, "connection_not_found", false);
|
|
23885
|
+
if (connection.providerConfigKey !== "google-search-console" || input.dataset !== "search_console_performance") {
|
|
23886
|
+
throw new NangoControlError("This dataset has not moved to main-MCP export execution yet.", 400, "invalid_request", false);
|
|
23887
|
+
}
|
|
23888
|
+
const match = input.cursor?.match(/^gsc1:(\d{1,4}):(\d{1,5})$/);
|
|
23889
|
+
if (input.cursor && !match) throw new NangoControlError("Search Console export cursor is invalid.", 400, "invalid_request", false);
|
|
23890
|
+
const siteIndex = match ? Number(match[1]) : 0;
|
|
23891
|
+
const startRow = match ? Number(match[2]) : 0;
|
|
23892
|
+
const listed = await callNangoToolDirect({
|
|
23893
|
+
identity,
|
|
23894
|
+
connectionId: input.connectionId,
|
|
23895
|
+
tool: "list-sites",
|
|
23896
|
+
input: {},
|
|
23897
|
+
classification: "read",
|
|
23898
|
+
requestId: `export:${input.connectionId}:${input.dataset}:${input.from}:${input.to}:${input.cursor ?? "start"}:list-sites`,
|
|
23899
|
+
operationKind: "export"
|
|
23900
|
+
});
|
|
23901
|
+
const sites = rowsFromToolResult(listed, ["siteEntry", "sites", "items"]).flatMap((value) => {
|
|
23902
|
+
const row = isRecord2(value) ? value : null;
|
|
23903
|
+
const siteUrl2 = row ? cleanString(row.siteUrl ?? row.site_url, 2e3) : null;
|
|
23904
|
+
return siteUrl2 ? [siteUrl2] : [];
|
|
23905
|
+
}).sort().slice(0, 1e4);
|
|
23906
|
+
if (siteIndex >= sites.length) {
|
|
23907
|
+
return {
|
|
23908
|
+
providerConfigKey: connection.providerConfigKey,
|
|
23909
|
+
dataset: "search_console_performance",
|
|
23910
|
+
records: [],
|
|
23911
|
+
nextCursor: null,
|
|
23912
|
+
complete: true,
|
|
23913
|
+
counts: { listed: 0, exported: 0, failed: 0 },
|
|
23914
|
+
warnings: ["search_console_returns_top_rows_not_guaranteed_exhaustive"],
|
|
23915
|
+
untrustedContent: true
|
|
23916
|
+
};
|
|
23917
|
+
}
|
|
23918
|
+
const siteUrl = sites[siteIndex];
|
|
23919
|
+
const endExclusive = new Date(input.to);
|
|
23920
|
+
endExclusive.setUTCDate(endExclusive.getUTCDate() - 1);
|
|
23921
|
+
const performance2 = await callNangoToolDirect({
|
|
23922
|
+
identity,
|
|
23923
|
+
connectionId: input.connectionId,
|
|
23924
|
+
tool: "query-search-analytics",
|
|
23925
|
+
input: {
|
|
23926
|
+
siteUrl,
|
|
23927
|
+
startDate: input.from.slice(0, 10),
|
|
23928
|
+
endDate: endExclusive.toISOString().slice(0, 10),
|
|
23929
|
+
dimensions: ["date", "query", "page", "country", "device"],
|
|
23930
|
+
dataState: "final",
|
|
23931
|
+
rowLimit: input.pageSize,
|
|
23932
|
+
startRow
|
|
23933
|
+
},
|
|
23934
|
+
classification: "read",
|
|
23935
|
+
requestId: `export:${input.connectionId}:${input.dataset}:${input.from}:${input.to}:${input.cursor ?? "start"}:query`,
|
|
23936
|
+
operationKind: "export"
|
|
23937
|
+
});
|
|
23938
|
+
const rows = rowsFromToolResult(performance2, ["rows"]).slice(0, input.pageSize);
|
|
23939
|
+
const records = rows.map((value, index) => {
|
|
23940
|
+
const row = isRecord2(value) ? value : {};
|
|
23941
|
+
const keys = Array.isArray(row.keys) ? row.keys.map((item) => String(item ?? "")) : [];
|
|
23942
|
+
const date = keys[0] || null;
|
|
23943
|
+
return {
|
|
23944
|
+
id: `${siteUrl}:${startRow + index}`,
|
|
23945
|
+
dataset: "search_console_performance",
|
|
23946
|
+
siteUrl,
|
|
23947
|
+
date,
|
|
23948
|
+
occurredAt: date ? `${date}T12:00:00.000Z` : null,
|
|
23949
|
+
query: keys[1] || null,
|
|
23950
|
+
page: keys[2] || null,
|
|
23951
|
+
country: keys[3] || null,
|
|
23952
|
+
device: keys[4] || null,
|
|
23953
|
+
clicks: typeof row.clicks === "number" ? row.clicks : 0,
|
|
23954
|
+
impressions: typeof row.impressions === "number" ? row.impressions : 0,
|
|
23955
|
+
ctr: typeof row.ctr === "number" ? row.ctr : 0,
|
|
23956
|
+
position: typeof row.position === "number" ? row.position : 0
|
|
23957
|
+
};
|
|
23958
|
+
});
|
|
23959
|
+
const nextStartRow = startRow + rows.length;
|
|
23960
|
+
const nextCursor = rows.length === input.pageSize && nextStartRow < 5e4 ? `gsc1:${siteIndex}:${nextStartRow}` : siteIndex + 1 < sites.length ? `gsc1:${siteIndex + 1}:0` : null;
|
|
23961
|
+
return {
|
|
23962
|
+
providerConfigKey: connection.providerConfigKey,
|
|
23963
|
+
dataset: "search_console_performance",
|
|
23964
|
+
records,
|
|
23965
|
+
nextCursor,
|
|
23966
|
+
complete: nextCursor === null,
|
|
23967
|
+
counts: { listed: rows.length, exported: records.length, failed: 0 },
|
|
23968
|
+
warnings: ["search_console_returns_top_rows_not_guaranteed_exhaustive"],
|
|
23969
|
+
untrustedContent: true
|
|
23970
|
+
};
|
|
23971
|
+
}
|
|
22535
23972
|
|
|
22536
23973
|
// src/api/resend-control.ts
|
|
22537
23974
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
@@ -22641,10 +24078,10 @@ async function controlRequest2(path5, init, timeoutMs = 2e4) {
|
|
|
22641
24078
|
}).catch((err) => {
|
|
22642
24079
|
throw new ResendControlError(`Resend connection control is unavailable: ${err instanceof Error ? err.message : "network error"}`);
|
|
22643
24080
|
});
|
|
22644
|
-
const
|
|
24081
|
+
const text2 = await response.text();
|
|
22645
24082
|
let body = {};
|
|
22646
24083
|
try {
|
|
22647
|
-
body =
|
|
24084
|
+
body = text2 ? JSON.parse(text2) : {};
|
|
22648
24085
|
} catch {
|
|
22649
24086
|
body = {};
|
|
22650
24087
|
}
|
|
@@ -22703,11 +24140,22 @@ async function getResendConnections(identity) {
|
|
|
22703
24140
|
const pending = ["pending", "pending_oauth", "authorizing", "authorization_pending"].includes(rawStatus);
|
|
22704
24141
|
const active = ["active", "connected", "ready"].includes(rawStatus);
|
|
22705
24142
|
const reconnectRequired = row.reconnectRequired === true || row.reconnect_required === true || !pending && !active;
|
|
24143
|
+
const providerAccountId = firstString2(row, ["providerAccountId", "provider_account_id"], 512);
|
|
24144
|
+
const providerAccountEmail = firstString2(row, ["providerAccountEmail", "provider_account_email"], 320)?.toLowerCase() ?? null;
|
|
24145
|
+
const providerAccountName = firstString2(row, ["providerAccountName", "provider_account_name"], 300);
|
|
24146
|
+
const rawLabel = firstString2(row, ["label", "accountLabel", "account_label", "displayName", "display_name"]);
|
|
24147
|
+
const safeLegacyLabel = rawLabel?.toLowerCase() === identity.trim().toLowerCase() ? null : rawLabel;
|
|
24148
|
+
const rawProviderIdentityStatus = firstString2(row, ["providerIdentityStatus", "provider_identity_status"], 32);
|
|
24149
|
+
const providerIdentityStatus = rawProviderIdentityStatus === "verified" || rawProviderIdentityStatus === "unavailable" ? rawProviderIdentityStatus : providerAccountId || providerAccountEmail || providerAccountName ? "verified" : pending ? "pending" : "unavailable";
|
|
22706
24150
|
result.push({
|
|
22707
24151
|
connectionId,
|
|
22708
24152
|
providerConfigKey: RESEND_PROVIDER_CONFIG_KEY,
|
|
22709
24153
|
provider: RESEND_PROVIDER_CONFIG_KEY,
|
|
22710
|
-
label:
|
|
24154
|
+
label: providerAccountEmail || providerAccountName || safeLegacyLabel || "Resend account",
|
|
24155
|
+
providerAccountId,
|
|
24156
|
+
providerAccountEmail,
|
|
24157
|
+
providerAccountName,
|
|
24158
|
+
providerIdentityStatus,
|
|
22711
24159
|
status: pending ? "pending_oauth" : reconnectRequired ? "needs_reauth" : "connected",
|
|
22712
24160
|
reconnectRequired,
|
|
22713
24161
|
actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
|
|
@@ -22845,6 +24293,289 @@ async function callResendExportPage(identity, input) {
|
|
|
22845
24293
|
};
|
|
22846
24294
|
}
|
|
22847
24295
|
|
|
24296
|
+
// src/api/scheduler-integration-auth.ts
|
|
24297
|
+
import { createHash as createHash11, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
24298
|
+
var MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
24299
|
+
var SchedulerIntegrationAuthError = class extends Error {
|
|
24300
|
+
code;
|
|
24301
|
+
constructor(code) {
|
|
24302
|
+
super(code);
|
|
24303
|
+
this.name = "SchedulerIntegrationAuthError";
|
|
24304
|
+
this.code = code;
|
|
24305
|
+
}
|
|
24306
|
+
};
|
|
24307
|
+
function signingSecret() {
|
|
24308
|
+
const secret2 = process.env.SCHEDULER_INTEGRATION_SIGNING_SECRET?.trim();
|
|
24309
|
+
if (!secret2) throw new SchedulerIntegrationAuthError("not_configured");
|
|
24310
|
+
return secret2;
|
|
24311
|
+
}
|
|
24312
|
+
function schedulerIntegrationSignature(args) {
|
|
24313
|
+
const bodyHash = createHash11("sha256").update(args.body).digest("hex");
|
|
24314
|
+
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
24315
|
+
${args.path}
|
|
24316
|
+
${args.timestamp}
|
|
24317
|
+
${args.nonce}
|
|
24318
|
+
${bodyHash}`).digest("hex");
|
|
24319
|
+
}
|
|
24320
|
+
async function claimNonce(nonce, timestampMs) {
|
|
24321
|
+
await ensureServiceConnectionsSchema();
|
|
24322
|
+
await getDb().execute(`
|
|
24323
|
+
CREATE TABLE IF NOT EXISTS service_connection_scheduler_nonces (
|
|
24324
|
+
nonce TEXT PRIMARY KEY,
|
|
24325
|
+
expires_at TEXT NOT NULL,
|
|
24326
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
24327
|
+
)
|
|
24328
|
+
`);
|
|
24329
|
+
await getDb().execute(`DELETE FROM service_connection_scheduler_nonces WHERE expires_at < datetime('now')`);
|
|
24330
|
+
try {
|
|
24331
|
+
await getDb().execute({
|
|
24332
|
+
sql: `INSERT INTO service_connection_scheduler_nonces (nonce, expires_at) VALUES (?, ?)`,
|
|
24333
|
+
args: [nonce, new Date(timestampMs + MAX_CLOCK_SKEW_MS).toISOString()]
|
|
24334
|
+
});
|
|
24335
|
+
} catch {
|
|
24336
|
+
throw new SchedulerIntegrationAuthError("replayed_request");
|
|
24337
|
+
}
|
|
24338
|
+
}
|
|
24339
|
+
async function verifySchedulerIntegrationRequest(request, rawBody) {
|
|
24340
|
+
const timestamp = request.headers.get("x-scheduler-timestamp")?.trim() ?? "";
|
|
24341
|
+
const nonce = request.headers.get("x-scheduler-nonce")?.trim() ?? "";
|
|
24342
|
+
const signature = request.headers.get("x-scheduler-signature")?.trim().toLowerCase() ?? "";
|
|
24343
|
+
const requestId = request.headers.get("x-scheduler-request-id")?.trim() ?? "";
|
|
24344
|
+
const timestampMs = Date.parse(timestamp);
|
|
24345
|
+
if (!timestamp || !nonce || !signature || !requestId || nonce.length > 200 || requestId.length > 200 || !Number.isFinite(timestampMs)) {
|
|
24346
|
+
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
24347
|
+
}
|
|
24348
|
+
if (Math.abs(Date.now() - timestampMs) > MAX_CLOCK_SKEW_MS) {
|
|
24349
|
+
throw new SchedulerIntegrationAuthError("expired_request");
|
|
24350
|
+
}
|
|
24351
|
+
const expected = schedulerIntegrationSignature({
|
|
24352
|
+
method: request.method,
|
|
24353
|
+
path: new URL(request.url).pathname,
|
|
24354
|
+
timestamp,
|
|
24355
|
+
nonce,
|
|
24356
|
+
body: rawBody,
|
|
24357
|
+
secret: signingSecret()
|
|
24358
|
+
});
|
|
24359
|
+
const suppliedBytes = Buffer.from(signature, "hex");
|
|
24360
|
+
const expectedBytes = Buffer.from(expected, "hex");
|
|
24361
|
+
if (suppliedBytes.length !== expectedBytes.length || !timingSafeEqual3(suppliedBytes, expectedBytes)) {
|
|
24362
|
+
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
24363
|
+
}
|
|
24364
|
+
await claimNonce(nonce, timestampMs);
|
|
24365
|
+
return { requestId };
|
|
24366
|
+
}
|
|
24367
|
+
|
|
24368
|
+
// src/api/connected-account-billing.ts
|
|
24369
|
+
import Stripe2 from "stripe";
|
|
24370
|
+
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
24371
|
+
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
24372
|
+
async function ensureConnectedAccountBillingSchema() {
|
|
24373
|
+
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
24374
|
+
user_id INTEGER PRIMARY KEY,
|
|
24375
|
+
stripe_subscription_id TEXT,
|
|
24376
|
+
stripe_subscription_item_id TEXT,
|
|
24377
|
+
price_id TEXT,
|
|
24378
|
+
quantity INTEGER NOT NULL DEFAULT 0,
|
|
24379
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
24380
|
+
last_error_code TEXT,
|
|
24381
|
+
synced_at TEXT,
|
|
24382
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
24383
|
+
)`);
|
|
24384
|
+
}
|
|
24385
|
+
async function getConnectedAccountBillingState(userId) {
|
|
24386
|
+
await ensureConnectedAccountBillingSchema();
|
|
24387
|
+
const result = await getDb().execute({
|
|
24388
|
+
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
24389
|
+
args: [Number(userId)]
|
|
24390
|
+
});
|
|
24391
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
24392
|
+
}
|
|
24393
|
+
async function setConnectedAccountBillingState(input) {
|
|
24394
|
+
await ensureConnectedAccountBillingSchema();
|
|
24395
|
+
await getDb().execute({
|
|
24396
|
+
sql: `INSERT INTO connected_account_billing
|
|
24397
|
+
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, synced_at, updated_at)
|
|
24398
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, CASE WHEN ? THEN datetime('now') ELSE NULL END, datetime('now'))
|
|
24399
|
+
ON CONFLICT(user_id) DO UPDATE SET
|
|
24400
|
+
stripe_subscription_id = excluded.stripe_subscription_id,
|
|
24401
|
+
stripe_subscription_item_id = excluded.stripe_subscription_item_id,
|
|
24402
|
+
price_id = excluded.price_id,
|
|
24403
|
+
quantity = excluded.quantity,
|
|
24404
|
+
status = excluded.status,
|
|
24405
|
+
last_error_code = excluded.last_error_code,
|
|
24406
|
+
synced_at = CASE WHEN ? THEN datetime('now') ELSE connected_account_billing.synced_at END,
|
|
24407
|
+
updated_at = datetime('now')`,
|
|
24408
|
+
args: [
|
|
24409
|
+
Number(input.userId),
|
|
24410
|
+
input.stripeSubscriptionId,
|
|
24411
|
+
input.stripeSubscriptionItemId,
|
|
24412
|
+
input.priceId,
|
|
24413
|
+
input.quantity,
|
|
24414
|
+
input.status,
|
|
24415
|
+
input.lastErrorCode,
|
|
24416
|
+
input.synced ? 1 : 0,
|
|
24417
|
+
input.synced ? 1 : 0
|
|
24418
|
+
]
|
|
24419
|
+
});
|
|
24420
|
+
const state = await getConnectedAccountBillingState(input.userId);
|
|
24421
|
+
if (!state) throw new Error("connected account billing state was not persisted");
|
|
24422
|
+
return state;
|
|
24423
|
+
}
|
|
24424
|
+
var ConnectedAccountBillingError = class extends Error {
|
|
24425
|
+
constructor(message, code, status = 409) {
|
|
24426
|
+
super(message);
|
|
24427
|
+
this.code = code;
|
|
24428
|
+
this.status = status;
|
|
24429
|
+
this.name = "ConnectedAccountBillingError";
|
|
24430
|
+
}
|
|
24431
|
+
code;
|
|
24432
|
+
status;
|
|
24433
|
+
};
|
|
24434
|
+
function connectedAccountPriceId() {
|
|
24435
|
+
const configured = process.env.CONNECTED_ACCOUNT_PRICE_ID?.trim();
|
|
24436
|
+
if (configured) return configured;
|
|
24437
|
+
return DEFAULT_CONNECTED_ACCOUNT_PRICE_ID || null;
|
|
24438
|
+
}
|
|
24439
|
+
function connectedAccountBillingView(actualQuantity, state) {
|
|
24440
|
+
const quantity = Math.max(0, Math.round(actualQuantity));
|
|
24441
|
+
return {
|
|
24442
|
+
provider: "nango",
|
|
24443
|
+
billingMode: "flat_recurring_usd_plus_credits",
|
|
24444
|
+
priceId: connectedAccountPriceId(),
|
|
24445
|
+
unitAmountUsd: CONNECTED_ACTIVE_CONNECTION_MONTHLY_USD,
|
|
24446
|
+
interval: "month",
|
|
24447
|
+
quantity,
|
|
24448
|
+
projectedMonthlyUsd: quantity * CONNECTED_ACTIVE_CONNECTION_MONTHLY_USD,
|
|
24449
|
+
status: state?.status ?? "pending",
|
|
24450
|
+
lastErrorCode: state?.last_error_code ?? null,
|
|
24451
|
+
syncedAt: state?.synced_at ?? null,
|
|
24452
|
+
usage: CONNECTED_USAGE_RATE_POLICY.usage
|
|
24453
|
+
};
|
|
24454
|
+
}
|
|
24455
|
+
function findBasePlanItem(subscription) {
|
|
24456
|
+
return subscription.items.data.find((item) => !!item.price?.id && item.price.id in SUBSCRIPTION_TIERS);
|
|
24457
|
+
}
|
|
24458
|
+
function findConnectedAccountItem(subscription, priceId = connectedAccountPriceId()) {
|
|
24459
|
+
return priceId ? subscription.items.data.find((item) => item.price?.id === priceId) : void 0;
|
|
24460
|
+
}
|
|
24461
|
+
function liveSubscription(subscription) {
|
|
24462
|
+
return subscription.status === "active" || subscription.status === "trialing";
|
|
24463
|
+
}
|
|
24464
|
+
function safeStripeErrorCode(error) {
|
|
24465
|
+
if (error instanceof ConnectedAccountBillingError) return error.code;
|
|
24466
|
+
return "stripe_connection_billing_failed";
|
|
24467
|
+
}
|
|
24468
|
+
async function persistFailure(user, actualQuantity, error, subscriptionItemId = null) {
|
|
24469
|
+
await setConnectedAccountBillingState({
|
|
24470
|
+
userId: user.id,
|
|
24471
|
+
stripeSubscriptionId: user.subscription_id,
|
|
24472
|
+
stripeSubscriptionItemId: subscriptionItemId,
|
|
24473
|
+
priceId: connectedAccountPriceId(),
|
|
24474
|
+
quantity: actualQuantity,
|
|
24475
|
+
status: error instanceof ConnectedAccountBillingError && error.code !== "stripe_connection_billing_failed" ? "blocked" : "error",
|
|
24476
|
+
lastErrorCode: safeStripeErrorCode(error),
|
|
24477
|
+
synced: false
|
|
24478
|
+
});
|
|
24479
|
+
}
|
|
24480
|
+
async function reconcileConnectedAccountBilling(user, actualQuantity, stripeClient) {
|
|
24481
|
+
if (!Number.isSafeInteger(actualQuantity) || actualQuantity < 0) {
|
|
24482
|
+
throw new Error("actualQuantity must be a non-negative safe integer");
|
|
24483
|
+
}
|
|
24484
|
+
const priceId = connectedAccountPriceId();
|
|
24485
|
+
if (!priceId) {
|
|
24486
|
+
const error = new ConnectedAccountBillingError(
|
|
24487
|
+
"Connected-account billing is not configured.",
|
|
24488
|
+
"connected_account_price_not_configured",
|
|
24489
|
+
503
|
|
24490
|
+
);
|
|
24491
|
+
await persistFailure(user, actualQuantity, error);
|
|
24492
|
+
throw error;
|
|
24493
|
+
}
|
|
24494
|
+
if (!user.subscription_id) {
|
|
24495
|
+
if (actualQuantity === 0) {
|
|
24496
|
+
const state = await setConnectedAccountBillingState({
|
|
24497
|
+
userId: user.id,
|
|
24498
|
+
stripeSubscriptionId: null,
|
|
24499
|
+
stripeSubscriptionItemId: null,
|
|
24500
|
+
priceId,
|
|
24501
|
+
quantity: 0,
|
|
24502
|
+
status: "synced",
|
|
24503
|
+
lastErrorCode: null,
|
|
24504
|
+
synced: true
|
|
24505
|
+
});
|
|
24506
|
+
return connectedAccountBillingView(0, state);
|
|
24507
|
+
}
|
|
24508
|
+
const error = new ConnectedAccountBillingError(
|
|
24509
|
+
"An active paid plan is required before connected accounts can be billed.",
|
|
24510
|
+
"paid_plan_required",
|
|
24511
|
+
403
|
|
24512
|
+
);
|
|
24513
|
+
await persistFailure(user, actualQuantity, error);
|
|
24514
|
+
throw error;
|
|
24515
|
+
}
|
|
24516
|
+
const client2 = stripeClient ?? (() => {
|
|
24517
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24518
|
+
if (!secret2) {
|
|
24519
|
+
throw new ConnectedAccountBillingError(
|
|
24520
|
+
"Stripe is not configured.",
|
|
24521
|
+
"connected_account_price_not_configured",
|
|
24522
|
+
503
|
|
24523
|
+
);
|
|
24524
|
+
}
|
|
24525
|
+
return new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
24526
|
+
})();
|
|
24527
|
+
let currentItem;
|
|
24528
|
+
try {
|
|
24529
|
+
const subscription = await client2.subscriptions.retrieve(user.subscription_id);
|
|
24530
|
+
currentItem = findConnectedAccountItem(subscription, priceId);
|
|
24531
|
+
if (!liveSubscription(subscription)) {
|
|
24532
|
+
throw new ConnectedAccountBillingError(
|
|
24533
|
+
"The plan subscription is not active.",
|
|
24534
|
+
"subscription_not_active",
|
|
24535
|
+
402
|
|
24536
|
+
);
|
|
24537
|
+
}
|
|
24538
|
+
if (!findBasePlanItem(subscription)) {
|
|
24539
|
+
throw new ConnectedAccountBillingError(
|
|
24540
|
+
"The base plan item could not be found on this subscription.",
|
|
24541
|
+
"subscription_plan_item_missing",
|
|
24542
|
+
409
|
|
24543
|
+
);
|
|
24544
|
+
}
|
|
24545
|
+
const currentQuantity = Math.max(0, currentItem?.quantity ?? 0);
|
|
24546
|
+
let resultingItem = currentItem;
|
|
24547
|
+
if (actualQuantity !== currentQuantity) {
|
|
24548
|
+
const updated = await client2.subscriptions.update(subscription.id, {
|
|
24549
|
+
items: actualQuantity === 0 ? currentItem ? [{ id: currentItem.id, deleted: true }] : [] : currentItem ? [{ id: currentItem.id, quantity: actualQuantity }] : [{ price: priceId, quantity: actualQuantity }],
|
|
24550
|
+
proration_behavior: "create_prorations"
|
|
24551
|
+
});
|
|
24552
|
+
resultingItem = findConnectedAccountItem(updated, priceId);
|
|
24553
|
+
}
|
|
24554
|
+
const state = await setConnectedAccountBillingState({
|
|
24555
|
+
userId: user.id,
|
|
24556
|
+
stripeSubscriptionId: subscription.id,
|
|
24557
|
+
stripeSubscriptionItemId: resultingItem?.id ?? null,
|
|
24558
|
+
priceId,
|
|
24559
|
+
quantity: actualQuantity,
|
|
24560
|
+
status: "synced",
|
|
24561
|
+
lastErrorCode: null,
|
|
24562
|
+
synced: true
|
|
24563
|
+
});
|
|
24564
|
+
return connectedAccountBillingView(actualQuantity, state);
|
|
24565
|
+
} catch (error) {
|
|
24566
|
+
await persistFailure(user, actualQuantity, error, currentItem?.id ?? null);
|
|
24567
|
+
if (error instanceof ConnectedAccountBillingError) throw error;
|
|
24568
|
+
throw new ConnectedAccountBillingError(
|
|
24569
|
+
"Unable to update connected-account billing in Stripe.",
|
|
24570
|
+
"stripe_connection_billing_failed",
|
|
24571
|
+
503
|
|
24572
|
+
);
|
|
24573
|
+
}
|
|
24574
|
+
}
|
|
24575
|
+
async function currentConnectedAccountBillingView(userId, actualQuantity) {
|
|
24576
|
+
return connectedAccountBillingView(actualQuantity, await getConnectedAccountBillingState(userId));
|
|
24577
|
+
}
|
|
24578
|
+
|
|
22848
24579
|
// src/api/server.ts
|
|
22849
24580
|
var secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
22850
24581
|
var isProduction2 = secureCookies2;
|
|
@@ -22939,7 +24670,7 @@ var requirePaidSchedulingTier = createMiddleware3(async (c, next) => {
|
|
|
22939
24670
|
return next();
|
|
22940
24671
|
});
|
|
22941
24672
|
var app = new Hono24();
|
|
22942
|
-
var
|
|
24673
|
+
var STRIPE_API_VERSION2 = "2026-02-25.clover";
|
|
22943
24674
|
function requireStripeSecret() {
|
|
22944
24675
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
22945
24676
|
if (secret2) return secret2;
|
|
@@ -22949,7 +24680,7 @@ function requireStripeSecret() {
|
|
|
22949
24680
|
async function createSignupStripeCustomer(email) {
|
|
22950
24681
|
const secret2 = requireStripeSecret();
|
|
22951
24682
|
if (!secret2) return null;
|
|
22952
|
-
const stripeClient = new
|
|
24683
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
22953
24684
|
const customer = await stripeClient.customers.create({
|
|
22954
24685
|
email,
|
|
22955
24686
|
description: "MCP Scraper free account",
|
|
@@ -23021,7 +24752,7 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
23021
24752
|
const user = await createUser(normalizedEmail, void 0, password, stripeCustomerId ?? void 0);
|
|
23022
24753
|
if (stripeCustomerId) {
|
|
23023
24754
|
try {
|
|
23024
|
-
const stripeClient = new
|
|
24755
|
+
const stripeClient = new Stripe3(process.env.STRIPE_SECRET_KEY, { apiVersion: STRIPE_API_VERSION2 });
|
|
23025
24756
|
await stripeClient.customers.update(stripeCustomerId, {
|
|
23026
24757
|
metadata: {
|
|
23027
24758
|
app: "mcp-scraper",
|
|
@@ -23070,9 +24801,26 @@ app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
|
23070
24801
|
});
|
|
23071
24802
|
app.post("/account/delete", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
23072
24803
|
const user = c.get("sessionUser");
|
|
24804
|
+
if (process.env.SCHEDULE_INTEGRATIONS_SECRET?.trim()) {
|
|
24805
|
+
try {
|
|
24806
|
+
const [nangoConnections, resendConnections] = await Promise.all([
|
|
24807
|
+
getNangoConnections(user.email, { summaryOnly: true }),
|
|
24808
|
+
getResendConnections(user.email)
|
|
24809
|
+
]);
|
|
24810
|
+
for (const connection of nangoConnections) {
|
|
24811
|
+
await deleteNangoConnection(user.email, connection.connectionId);
|
|
24812
|
+
}
|
|
24813
|
+
for (const connection of resendConnections) {
|
|
24814
|
+
await deleteResendConnection(user.email, connection.connectionId);
|
|
24815
|
+
}
|
|
24816
|
+
} catch (err) {
|
|
24817
|
+
console.error("[account/delete] failed to remove provider connections", err instanceof Error ? err.message : String(err));
|
|
24818
|
+
return c.json({ error: "Could not remove every connected account \u2014 please retry before deleting the account." }, 502);
|
|
24819
|
+
}
|
|
24820
|
+
}
|
|
23073
24821
|
const secret2 = requireStripeSecret();
|
|
23074
24822
|
if (secret2 && user.stripe_customer_id) {
|
|
23075
|
-
const stripeClient = new
|
|
24823
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
23076
24824
|
const subs = await stripeClient.subscriptions.list({ customer: user.stripe_customer_id, status: "all", limit: 100 });
|
|
23077
24825
|
for (const sub of subs.data) {
|
|
23078
24826
|
if (sub.status === "active" || sub.status === "trialing" || sub.status === "past_due") {
|
|
@@ -23397,7 +25145,7 @@ app.post("/memory/checkout", auth2, async (c) => {
|
|
|
23397
25145
|
if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
|
|
23398
25146
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
23399
25147
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
23400
|
-
const stripeClient = new
|
|
25148
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
23401
25149
|
let customerId = user.stripe_customer_id;
|
|
23402
25150
|
if (!customerId) {
|
|
23403
25151
|
const customer = await stripeClient.customers.create({ email: user.email });
|
|
@@ -23432,7 +25180,7 @@ app.post("/memory/portal", auth2, async (c) => {
|
|
|
23432
25180
|
if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
|
|
23433
25181
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
23434
25182
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
23435
|
-
const stripeClient = new
|
|
25183
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
23436
25184
|
const session = await stripeClient.billingPortal.sessions.create({
|
|
23437
25185
|
customer: user.stripe_customer_id,
|
|
23438
25186
|
return_url: `${appOrigin()}/memory`
|
|
@@ -23455,6 +25203,14 @@ app.post("/schedule-checkout", auth2, async (c) => {
|
|
|
23455
25203
|
});
|
|
23456
25204
|
function scheduleConnectionError(c, err, fallback) {
|
|
23457
25205
|
if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
|
|
25206
|
+
if (err instanceof ConnectedAccountBillingError) {
|
|
25207
|
+
return c.json({
|
|
25208
|
+
ok: false,
|
|
25209
|
+
error: err.message,
|
|
25210
|
+
code: err.code,
|
|
25211
|
+
errorCode: err.code
|
|
25212
|
+
}, err.status);
|
|
25213
|
+
}
|
|
23458
25214
|
if (err instanceof NangoControlError) {
|
|
23459
25215
|
return c.json({
|
|
23460
25216
|
ok: false,
|
|
@@ -23468,6 +25224,9 @@ function scheduleConnectionError(c, err, fallback) {
|
|
|
23468
25224
|
console.error("[schedule-connections]", err instanceof Error ? err.message : String(err));
|
|
23469
25225
|
return c.json({ ok: false, error: fallback }, 502);
|
|
23470
25226
|
}
|
|
25227
|
+
function connectedActionIdempotencyKey(c) {
|
|
25228
|
+
return c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
25229
|
+
}
|
|
23471
25230
|
function connectionMemoryImportError(c, err) {
|
|
23472
25231
|
return c.json({
|
|
23473
25232
|
ok: false,
|
|
@@ -23559,6 +25318,9 @@ async function combinedConnections(identity) {
|
|
|
23559
25318
|
})) : []
|
|
23560
25319
|
];
|
|
23561
25320
|
}
|
|
25321
|
+
function billableNangoConnectionCount(connections) {
|
|
25322
|
+
return connections.filter((connection) => connection.billingActive).length;
|
|
25323
|
+
}
|
|
23562
25324
|
async function isResendConnection(identity, connectionId, providerHint) {
|
|
23563
25325
|
if (providerHint === "resend") return true;
|
|
23564
25326
|
if (providerHint) return false;
|
|
@@ -23604,17 +25366,43 @@ app.get("/schedule-connection-catalog", auth2, async (c) => {
|
|
|
23604
25366
|
app.get("/schedule-connections", auth2, async (c) => {
|
|
23605
25367
|
try {
|
|
23606
25368
|
const user = c.get("user");
|
|
23607
|
-
|
|
25369
|
+
const connections = await combinedConnections(user.email);
|
|
25370
|
+
const nangoQuantity = connections.filter(
|
|
25371
|
+
(connection) => connection.transport === "nango" && connection.billingActive
|
|
25372
|
+
).length;
|
|
25373
|
+
return c.json({
|
|
25374
|
+
ok: true,
|
|
25375
|
+
connections,
|
|
25376
|
+
billing: await currentConnectedAccountBillingView(user.id, nangoQuantity)
|
|
25377
|
+
});
|
|
23608
25378
|
} catch (err) {
|
|
23609
25379
|
return scheduleConnectionError(c, err, "Unable to load service connections.");
|
|
23610
25380
|
}
|
|
23611
25381
|
});
|
|
25382
|
+
app.post("/schedule-connections/billing/reconcile", auth2, requireIntegrationsTier, async (c) => {
|
|
25383
|
+
const user = c.get("user");
|
|
25384
|
+
try {
|
|
25385
|
+
const quantity = billableNangoConnectionCount(await getNangoConnections(user.email, { summaryOnly: true }));
|
|
25386
|
+
return c.json({
|
|
25387
|
+
ok: true,
|
|
25388
|
+
billing: await reconcileConnectedAccountBilling(user, quantity)
|
|
25389
|
+
});
|
|
25390
|
+
} catch (err) {
|
|
25391
|
+
return scheduleConnectionError(c, err, "Unable to synchronize connected-account billing.");
|
|
25392
|
+
}
|
|
25393
|
+
});
|
|
23612
25394
|
app.post("/schedule-connections/session", auth2, requireIntegrationsTier, async (c) => {
|
|
23613
25395
|
const user = c.get("user");
|
|
23614
25396
|
const body = await c.req.json().catch(() => ({}));
|
|
23615
25397
|
const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
|
|
23616
25398
|
if (!providerConfigKey) return c.json({ ok: false, error: "providerConfigKey is required." }, 400);
|
|
23617
25399
|
try {
|
|
25400
|
+
if (providerConfigKey !== "resend") {
|
|
25401
|
+
await reconcileConnectedAccountBilling(
|
|
25402
|
+
user,
|
|
25403
|
+
billableNangoConnectionCount(await getNangoConnections(user.email, { summaryOnly: true }))
|
|
25404
|
+
);
|
|
25405
|
+
}
|
|
23618
25406
|
const session = providerConfigKey === "resend" ? await createResendConnectSession(user.email, resendOAuthCallbackUrl()) : await createNangoConnectSession(user.email, providerConfigKey);
|
|
23619
25407
|
return c.json({ ok: true, ...session });
|
|
23620
25408
|
} catch (err) {
|
|
@@ -23637,11 +25425,23 @@ app.delete("/schedule-connections/:id", auth2, async (c) => {
|
|
|
23637
25425
|
const body = await c.req.json().catch(() => ({}));
|
|
23638
25426
|
const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
|
|
23639
25427
|
try {
|
|
23640
|
-
if (
|
|
23641
|
-
|
|
25428
|
+
if (await isResendConnection(user.email, c.req.param("id"), providerConfigKey)) {
|
|
25429
|
+
await deleteResendConnection(user.email, c.req.param("id"));
|
|
25430
|
+
return c.json({ ok: true, transport: "remote_mcp" });
|
|
23642
25431
|
}
|
|
23643
|
-
await
|
|
23644
|
-
|
|
25432
|
+
const remainingConnections = await deleteNangoConnection(user.email, c.req.param("id"));
|
|
25433
|
+
let billing;
|
|
25434
|
+
try {
|
|
25435
|
+
billing = await reconcileConnectedAccountBilling(user, remainingConnections);
|
|
25436
|
+
} catch (billingError2) {
|
|
25437
|
+
console.error("[schedule-connections/delete/billing]", billingError2 instanceof Error ? billingError2.message : String(billingError2));
|
|
25438
|
+
}
|
|
25439
|
+
return c.json({
|
|
25440
|
+
ok: true,
|
|
25441
|
+
transport: "nango",
|
|
25442
|
+
remainingConnections,
|
|
25443
|
+
billing: billing ?? await currentConnectedAccountBillingView(user.id, remainingConnections)
|
|
25444
|
+
});
|
|
23645
25445
|
} catch (err) {
|
|
23646
25446
|
return scheduleConnectionError(c, err, "Unable to disconnect this service connection.");
|
|
23647
25447
|
}
|
|
@@ -23658,6 +25458,72 @@ app.post("/schedule-connections/:id/enable-actions", auth2, requireIntegrationsT
|
|
|
23658
25458
|
return scheduleConnectionError(c, err, "Unable to update this connection's action setting.");
|
|
23659
25459
|
}
|
|
23660
25460
|
});
|
|
25461
|
+
app.post("/schedule-connections/:id/test", auth2, requireIntegrationsTier, async (c) => {
|
|
25462
|
+
const user = c.get("user");
|
|
25463
|
+
const body = await c.req.json().catch(() => ({}));
|
|
25464
|
+
const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
|
|
25465
|
+
try {
|
|
25466
|
+
if (await isResendConnection(user.email, c.req.param("id"), providerConfigKey)) {
|
|
25467
|
+
return c.json({
|
|
25468
|
+
ok: false,
|
|
25469
|
+
code: "connection_test_not_supported",
|
|
25470
|
+
error: "Resend connection testing will be enabled after its credential migration to the main MCP."
|
|
25471
|
+
}, 409);
|
|
25472
|
+
}
|
|
25473
|
+
return c.json({ ok: true, ...await testNangoConnection(user.email, c.req.param("id")) });
|
|
25474
|
+
} catch (err) {
|
|
25475
|
+
return scheduleConnectionError(c, err, "Unable to test this service connection.");
|
|
25476
|
+
}
|
|
25477
|
+
});
|
|
25478
|
+
function schedulerIntegrationAuthError(c, error) {
|
|
25479
|
+
if (error instanceof SchedulerIntegrationAuthError) {
|
|
25480
|
+
const status = error.code === "not_configured" ? 503 : error.code === "replayed_request" ? 409 : 401;
|
|
25481
|
+
return c.json({ ok: false, code: error.code, error: "Scheduler integration authorization failed." }, status);
|
|
25482
|
+
}
|
|
25483
|
+
return scheduleConnectionError(c, error, "The scheduler integration request failed.");
|
|
25484
|
+
}
|
|
25485
|
+
app.post("/api/internal/integrations/:id/describe", async (c) => {
|
|
25486
|
+
const rawBody = await c.req.text();
|
|
25487
|
+
try {
|
|
25488
|
+
await verifySchedulerIntegrationRequest(c.req.raw, rawBody);
|
|
25489
|
+
const body = JSON.parse(rawBody);
|
|
25490
|
+
const identity = typeof body.identity === "string" ? body.identity.trim() : "";
|
|
25491
|
+
const tool = providerConfigKeyFrom(body.tool);
|
|
25492
|
+
if (!identity || !tool) return c.json({ ok: false, code: "invalid_request", error: "identity and tool are required." }, 400);
|
|
25493
|
+
return c.json({ ok: true, tool: await describeNangoTool(identity, c.req.param("id"), tool, true) });
|
|
25494
|
+
} catch (error) {
|
|
25495
|
+
return schedulerIntegrationAuthError(c, error);
|
|
25496
|
+
}
|
|
25497
|
+
});
|
|
25498
|
+
app.post("/api/internal/integrations/:id/execute", async (c) => {
|
|
25499
|
+
const rawBody = await c.req.text();
|
|
25500
|
+
try {
|
|
25501
|
+
const authResult = await verifySchedulerIntegrationRequest(c.req.raw, rawBody);
|
|
25502
|
+
const body = JSON.parse(rawBody);
|
|
25503
|
+
const identity = typeof body.identity === "string" ? body.identity.trim() : "";
|
|
25504
|
+
const tool = providerConfigKeyFrom(body.tool);
|
|
25505
|
+
const args = body.args && typeof body.args === "object" && !Array.isArray(body.args) ? body.args : null;
|
|
25506
|
+
if (!identity || !tool || !args || body.classification !== "read" && body.classification !== "action") {
|
|
25507
|
+
return c.json({ ok: false, code: "invalid_request", error: "identity, tool, classification, and args are required." }, 400);
|
|
25508
|
+
}
|
|
25509
|
+
const result = body.classification === "action" ? await callScheduleConnectionAction(identity, c.req.param("id"), args, tool, authResult.requestId) : await callScheduleConnectionRead(identity, c.req.param("id"), tool, args, authResult.requestId);
|
|
25510
|
+
return c.json({ ok: true, result });
|
|
25511
|
+
} catch (error) {
|
|
25512
|
+
return schedulerIntegrationAuthError(c, error);
|
|
25513
|
+
}
|
|
25514
|
+
});
|
|
25515
|
+
app.post("/api/internal/integrations/:id/test", async (c) => {
|
|
25516
|
+
const rawBody = await c.req.text();
|
|
25517
|
+
try {
|
|
25518
|
+
await verifySchedulerIntegrationRequest(c.req.raw, rawBody);
|
|
25519
|
+
const body = JSON.parse(rawBody);
|
|
25520
|
+
const identity = typeof body.identity === "string" ? body.identity.trim() : "";
|
|
25521
|
+
if (!identity) return c.json({ ok: false, code: "invalid_request", error: "identity is required." }, 400);
|
|
25522
|
+
return c.json({ ok: true, ...await testNangoConnection(identity, c.req.param("id")) });
|
|
25523
|
+
} catch (error) {
|
|
25524
|
+
return schedulerIntegrationAuthError(c, error);
|
|
25525
|
+
}
|
|
25526
|
+
});
|
|
23661
25527
|
app.get("/oauth/resend/callback", async (c) => {
|
|
23662
25528
|
const code = c.req.query("code")?.trim();
|
|
23663
25529
|
const state = c.req.query("state")?.trim();
|
|
@@ -23690,7 +25556,13 @@ app.post("/schedule-connections/actions/slack/send-message", auth2, requireInteg
|
|
|
23690
25556
|
return c.json({ ok: false, error: "connectionId, channel, and text are required." }, 400);
|
|
23691
25557
|
}
|
|
23692
25558
|
try {
|
|
23693
|
-
const result = await callScheduleConnectionAction(
|
|
25559
|
+
const result = await callScheduleConnectionAction(
|
|
25560
|
+
user.email,
|
|
25561
|
+
connectionId,
|
|
25562
|
+
{ channel: body.channel, text: body.text },
|
|
25563
|
+
void 0,
|
|
25564
|
+
connectedActionIdempotencyKey(c)
|
|
25565
|
+
);
|
|
23694
25566
|
return c.json({ ok: true, result });
|
|
23695
25567
|
} catch (err) {
|
|
23696
25568
|
return scheduleConnectionError(c, err, "Unable to send the Slack message.");
|
|
@@ -23704,7 +25576,13 @@ app.post("/schedule-connections/actions/gmail/send-message", auth2, requireInteg
|
|
|
23704
25576
|
return c.json({ ok: false, error: "connectionId, to, subject, and body are required." }, 400);
|
|
23705
25577
|
}
|
|
23706
25578
|
try {
|
|
23707
|
-
const result = await callScheduleConnectionAction(
|
|
25579
|
+
const result = await callScheduleConnectionAction(
|
|
25580
|
+
user.email,
|
|
25581
|
+
connectionId,
|
|
25582
|
+
{ to: body.to, subject: body.subject, body: body.body },
|
|
25583
|
+
void 0,
|
|
25584
|
+
connectedActionIdempotencyKey(c)
|
|
25585
|
+
);
|
|
23708
25586
|
return c.json({ ok: true, result });
|
|
23709
25587
|
} catch (err) {
|
|
23710
25588
|
return scheduleConnectionError(c, err, "Unable to send the email.");
|
|
@@ -23728,7 +25606,7 @@ app.post("/schedule-connections/actions/google-calendar/create-event", auth2, re
|
|
|
23728
25606
|
start: { dateTime: body.startDateTime, timeZone },
|
|
23729
25607
|
end: { dateTime: body.endDateTime, timeZone },
|
|
23730
25608
|
attendees
|
|
23731
|
-
});
|
|
25609
|
+
}, void 0, connectedActionIdempotencyKey(c));
|
|
23732
25610
|
return c.json({ ok: true, result });
|
|
23733
25611
|
} catch (err) {
|
|
23734
25612
|
return scheduleConnectionError(c, err, "Unable to create the calendar event.");
|
|
@@ -23748,7 +25626,7 @@ app.post("/schedule-connections/actions/zoom/create-meeting", auth2, requireInte
|
|
|
23748
25626
|
durationMinutes: typeof body.durationMinutes === "number" ? body.durationMinutes : 30,
|
|
23749
25627
|
timezone: typeof body.timezone === "string" ? body.timezone : void 0,
|
|
23750
25628
|
agenda: body.agenda
|
|
23751
|
-
});
|
|
25629
|
+
}, void 0, connectedActionIdempotencyKey(c));
|
|
23752
25630
|
return c.json({ ok: true, result });
|
|
23753
25631
|
} catch (err) {
|
|
23754
25632
|
return scheduleConnectionError(c, err, "Unable to create the Zoom meeting.");
|
|
@@ -23961,13 +25839,31 @@ app.post("/schedule-connections/actions/call", auth2, requireIntegrationsTier, a
|
|
|
23961
25839
|
user.email,
|
|
23962
25840
|
connectionId,
|
|
23963
25841
|
body.args,
|
|
23964
|
-
tool
|
|
25842
|
+
tool,
|
|
25843
|
+
connectedActionIdempotencyKey(c)
|
|
23965
25844
|
);
|
|
23966
25845
|
return c.json({ ok: true, result });
|
|
23967
25846
|
} catch (err) {
|
|
23968
25847
|
return scheduleConnectionError(c, err, "Unable to run this connection action.");
|
|
23969
25848
|
}
|
|
23970
25849
|
});
|
|
25850
|
+
async function forwardIntegrationAlias(c, targetPath) {
|
|
25851
|
+
const url = new URL(c.req.url);
|
|
25852
|
+
url.pathname = targetPath;
|
|
25853
|
+
return await app.fetch(new Request(url, c.req.raw));
|
|
25854
|
+
}
|
|
25855
|
+
app.get("/integrations", (c) => forwardIntegrationAlias(c, "/schedule-connections"));
|
|
25856
|
+
app.get("/integrations/catalog", (c) => forwardIntegrationAlias(c, "/schedule-connection-catalog"));
|
|
25857
|
+
app.post("/integrations/session", (c) => forwardIntegrationAlias(c, "/schedule-connections/session"));
|
|
25858
|
+
app.post("/integrations/billing/reconcile", (c) => forwardIntegrationAlias(c, "/schedule-connections/billing/reconcile"));
|
|
25859
|
+
app.post("/integrations/:id/reconnect-session", (c) => forwardIntegrationAlias(c, `/schedule-connections/${encodeURIComponent(c.req.param("id"))}/reconnect-session`));
|
|
25860
|
+
app.post("/integrations/:id/enable-actions", (c) => forwardIntegrationAlias(c, `/schedule-connections/${encodeURIComponent(c.req.param("id"))}/enable-actions`));
|
|
25861
|
+
app.post("/integrations/:id/test", (c) => forwardIntegrationAlias(c, `/schedule-connections/${encodeURIComponent(c.req.param("id"))}/test`));
|
|
25862
|
+
app.delete("/integrations/:id", (c) => forwardIntegrationAlias(c, `/schedule-connections/${encodeURIComponent(c.req.param("id"))}`));
|
|
25863
|
+
app.post("/integrations/actions/read", (c) => forwardIntegrationAlias(c, "/schedule-connections/actions/read"));
|
|
25864
|
+
app.post("/integrations/actions/describe", (c) => forwardIntegrationAlias(c, "/schedule-connections/actions/describe"));
|
|
25865
|
+
app.post("/integrations/actions/call", (c) => forwardIntegrationAlias(c, "/schedule-connections/actions/call"));
|
|
25866
|
+
app.post("/integrations/actions/export", (c) => forwardIntegrationAlias(c, "/schedule-connections/actions/export"));
|
|
23971
25867
|
app.post("/schedule-link", auth2, requirePaidSchedulingTier, async (c) => {
|
|
23972
25868
|
try {
|
|
23973
25869
|
const user = c.get("user");
|
|
@@ -24733,7 +26629,7 @@ app.post("/billing/concurrency/checkout", requireAllowedOrigin, sessionAuth, asy
|
|
|
24733
26629
|
if (user.concurrency_stripe_sub_id) return c.json({ error: "Already subscribed \u2014 cancel existing slot first to change." }, 409);
|
|
24734
26630
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24735
26631
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24736
|
-
const stripeClient = new
|
|
26632
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
24737
26633
|
let customerId = user.stripe_customer_id;
|
|
24738
26634
|
if (!customerId) {
|
|
24739
26635
|
const customer = await stripeClient.customers.create({ email: user.email });
|
|
@@ -24766,7 +26662,7 @@ app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
24766
26662
|
if (!tier) return c.json({ error: "Invalid tier. Choose starter, growth, or scale." }, 400);
|
|
24767
26663
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24768
26664
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24769
|
-
const stripeClient = new
|
|
26665
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
24770
26666
|
let customerId = user.stripe_customer_id;
|
|
24771
26667
|
if (!customerId) {
|
|
24772
26668
|
const customer = await stripeClient.customers.create({ email: user.email });
|
|
@@ -24775,7 +26671,7 @@ app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
24775
26671
|
}
|
|
24776
26672
|
if (user.subscription_id) {
|
|
24777
26673
|
const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
|
|
24778
|
-
const itemId = sub
|
|
26674
|
+
const itemId = findBasePlanItem(sub)?.id;
|
|
24779
26675
|
if (itemId) {
|
|
24780
26676
|
await stripeClient.subscriptions.update(user.subscription_id, {
|
|
24781
26677
|
items: [{ id: itemId, price: tier.price_id }],
|
|
@@ -24809,7 +26705,7 @@ app.post("/billing/portal", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
24809
26705
|
if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
|
|
24810
26706
|
const secret2 = requireStripeSecret();
|
|
24811
26707
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24812
|
-
const stripeClient = new
|
|
26708
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
24813
26709
|
const session = await stripeClient.billingPortal.sessions.create({
|
|
24814
26710
|
customer: user.stripe_customer_id,
|
|
24815
26711
|
return_url: `${appOrigin()}/billing`
|
|
@@ -24839,7 +26735,7 @@ app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
|
|
|
24839
26735
|
}
|
|
24840
26736
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24841
26737
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24842
|
-
const stripeClient = new
|
|
26738
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
24843
26739
|
let customerId = user.stripe_customer_id;
|
|
24844
26740
|
if (!customerId) {
|
|
24845
26741
|
const customer = await stripeClient.customers.create({ email: user.email });
|
|
@@ -24883,7 +26779,7 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
|
|
|
24883
26779
|
if (!tier) return c.json({ error: "Invalid tier. Choose starter, growth, or scale.", tiers: Object.keys(SUBSCRIPTION_TIER_BY_KEY) }, 400);
|
|
24884
26780
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24885
26781
|
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24886
|
-
const stripeClient = new
|
|
26782
|
+
const stripeClient = new Stripe3(secret2, { apiVersion: STRIPE_API_VERSION2 });
|
|
24887
26783
|
let customerId = user.stripe_customer_id;
|
|
24888
26784
|
if (!customerId) {
|
|
24889
26785
|
const customer = await stripeClient.customers.create({ email: user.email });
|
|
@@ -24892,7 +26788,7 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
|
|
|
24892
26788
|
}
|
|
24893
26789
|
if (user.subscription_id) {
|
|
24894
26790
|
const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
|
|
24895
|
-
const itemId = sub
|
|
26791
|
+
const itemId = findBasePlanItem(sub)?.id;
|
|
24896
26792
|
if (itemId) {
|
|
24897
26793
|
await stripeClient.subscriptions.update(user.subscription_id, {
|
|
24898
26794
|
items: [{ id: itemId, price: tier.price_id }],
|
|
@@ -24935,7 +26831,7 @@ app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async
|
|
|
24935
26831
|
if (!user.concurrency_stripe_sub_id) return c.json({ error: "No active concurrency subscription." }, 404);
|
|
24936
26832
|
const stripeSecret = process.env.STRIPE_SECRET_KEY?.trim();
|
|
24937
26833
|
if (!stripeSecret) return c.json({ error: "Stripe is not configured." }, 503);
|
|
24938
|
-
const stripeClient = new
|
|
26834
|
+
const stripeClient = new Stripe3(stripeSecret, { apiVersion: STRIPE_API_VERSION2 });
|
|
24939
26835
|
await stripeClient.subscriptions.cancel(user.concurrency_stripe_sub_id);
|
|
24940
26836
|
const nextSlots = Math.max(0, user.extra_concurrency_slots - 1);
|
|
24941
26837
|
await setExtraConcurrencySlots(user.id, nextSlots);
|
|
@@ -24954,6 +26850,13 @@ app.get("/billing/balance", auth2, async (c) => {
|
|
|
24954
26850
|
ledger
|
|
24955
26851
|
});
|
|
24956
26852
|
});
|
|
26853
|
+
app.get("/billing/connected-usage/history", auth2, async (c) => {
|
|
26854
|
+
return c.json({
|
|
26855
|
+
ok: true,
|
|
26856
|
+
ratePolicy: CONNECTED_USAGE_RATE_POLICY,
|
|
26857
|
+
receipts: await listConnectedUsageHistory(c.get("user").id, 100)
|
|
26858
|
+
});
|
|
26859
|
+
});
|
|
24957
26860
|
app.post("/billing/credits", auth2, async (c) => {
|
|
24958
26861
|
const user = c.get("user");
|
|
24959
26862
|
const balanceMc = user.balance_mc;
|
|
@@ -24981,6 +26884,7 @@ app.post("/billing/credits", auth2, async (c) => {
|
|
|
24981
26884
|
has_subscription: !!user.concurrency_stripe_sub_id,
|
|
24982
26885
|
upgrade: concurrencySlotBillingInfo()
|
|
24983
26886
|
},
|
|
26887
|
+
connected_accounts: CONNECTED_USAGE_RATE_POLICY,
|
|
24984
26888
|
ledger
|
|
24985
26889
|
});
|
|
24986
26890
|
});
|
|
@@ -24989,7 +26893,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
24989
26893
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
24990
26894
|
return c.json({ error: "Unauthorized" }, 401);
|
|
24991
26895
|
}
|
|
24992
|
-
const { drainQueue } = await import("./worker-
|
|
26896
|
+
const { drainQueue } = await import("./worker-E3KHQH2F.js");
|
|
24993
26897
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
24994
26898
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
24995
26899
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -25017,7 +26921,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
25017
26921
|
return c.json({ error: "Unauthorized" }, 401);
|
|
25018
26922
|
}
|
|
25019
26923
|
const jobId = c.req.param("id");
|
|
25020
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
26924
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-O36SIFI2.js");
|
|
25021
26925
|
const { assembleExtractArtifacts } = await import("./extract-bundle-NFFVMUMT.js");
|
|
25022
26926
|
const job = await getExtractJob2(jobId);
|
|
25023
26927
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
@@ -25025,7 +26929,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
25025
26929
|
await completeExtractJob2(jobId, stored);
|
|
25026
26930
|
let settlement = "already_settled_or_refunded";
|
|
25027
26931
|
if (job.billedMc == null && job.userId != null) {
|
|
25028
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
26932
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-O36SIFI2.js");
|
|
25029
26933
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
25030
26934
|
const successful = await countSuccessfulPages2(jobId);
|
|
25031
26935
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -25191,4 +27095,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
25191
27095
|
export {
|
|
25192
27096
|
app
|
|
25193
27097
|
};
|
|
25194
|
-
//# sourceMappingURL=server-
|
|
27098
|
+
//# sourceMappingURL=server-UHDJ5TH5.js.map
|