mcp-scraper 0.42.2 → 0.43.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +3285 -2121
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2170 -1510
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +51 -23
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +21 -21
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-MJKVDLVR.js → chunk-2K74LVV7.js} +26 -2
- package/dist/{chunk-MJKVDLVR.js.map → chunk-2K74LVV7.js.map} +1 -1
- package/dist/chunk-3CQZ3DAT.js +7 -0
- package/dist/chunk-3CQZ3DAT.js.map +1 -0
- package/dist/{chunk-6Q7V4IQV.js → chunk-6WLNXYRG.js} +19 -2
- package/dist/chunk-6WLNXYRG.js.map +1 -0
- package/dist/{chunk-4MFUKI4X.js → chunk-GUI33Q27.js} +2 -2
- package/dist/{chunk-N2B4AAWG.js → chunk-HO4DOPEG.js} +2194 -1569
- package/dist/chunk-HO4DOPEG.js.map +1 -0
- package/dist/{chunk-DH42LIR3.js → chunk-LXVIQZ7Y.js} +22 -12
- package/dist/chunk-LXVIQZ7Y.js.map +1 -0
- package/dist/{chunk-ENJQ2NSX.js → chunk-TC4GDB2Q.js} +2 -2
- package/dist/{chunk-ENJQ2NSX.js.map → chunk-TC4GDB2Q.js.map} +1 -1
- package/dist/{chunk-SBLGBZZB.js → chunk-X5WDJ7D7.js} +9 -9
- package/dist/chunk-X5WDJ7D7.js.map +1 -0
- package/dist/{extract-bundle-DMZ63HZU.js → extract-bundle-IOM45SFL.js} +5 -5
- package/dist/index.cjs +21 -21
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +21 -57
- package/dist/index.d.ts +21 -57
- package/dist/index.js +1 -1
- package/dist/{server-2S7KQTPC.js → server-WAB6Z47N.js} +798 -333
- package/dist/server-WAB6Z47N.js.map +1 -0
- package/dist/{site-extract-repository-B42SIGAZ.js → site-extract-repository-XNI4ZK2M.js} +3 -3
- package/dist/{worker-4R3UWQZS.js → worker-E6MIVF33.js} +3 -3
- package/package.json +10 -7
- package/dist/chunk-6Q7V4IQV.js.map +0 -1
- package/dist/chunk-DH42LIR3.js.map +0 -1
- package/dist/chunk-N2B4AAWG.js.map +0 -1
- package/dist/chunk-SBLGBZZB.js.map +0 -1
- package/dist/chunk-UQ7XJINF.js +0 -7
- package/dist/chunk-UQ7XJINF.js.map +0 -1
- package/dist/server-2S7KQTPC.js.map +0 -1
- /package/dist/{chunk-4MFUKI4X.js.map → chunk-GUI33Q27.js.map} +0 -0
- /package/dist/{extract-bundle-DMZ63HZU.js.map → extract-bundle-IOM45SFL.js.map} +0 -0
- /package/dist/{site-extract-repository-B42SIGAZ.js.map → site-extract-repository-XNI4ZK2M.js.map} +0 -0
- /package/dist/{worker-4R3UWQZS.js.map → worker-E6MIVF33.js.map} +0 -0
|
@@ -2,8 +2,9 @@ import {
|
|
|
2
2
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
3
3
|
cleanupExpiredSiteExtractArtifacts,
|
|
4
4
|
harvestPageMedia,
|
|
5
|
+
readOwnedSiteExtractArtifactBuffer,
|
|
5
6
|
renewSiteExtractArtifactDownload
|
|
6
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-6WLNXYRG.js";
|
|
7
8
|
import {
|
|
8
9
|
REQUIRED_LOCATION_STATE_CODES,
|
|
9
10
|
importHostedCensusPlacesCsv,
|
|
@@ -40,7 +41,7 @@ import {
|
|
|
40
41
|
resolveBackupKernelProxyId,
|
|
41
42
|
resolveKernelProxyId,
|
|
42
43
|
serpArrivalUrl
|
|
43
|
-
} from "./chunk-
|
|
44
|
+
} from "./chunk-LXVIQZ7Y.js";
|
|
44
45
|
import {
|
|
45
46
|
ApiErrorResponseSchema,
|
|
46
47
|
ArtifactTemplateConfigSchema,
|
|
@@ -69,9 +70,10 @@ import {
|
|
|
69
70
|
createConnectedDataArtifact,
|
|
70
71
|
createDirectoryCsvArtifact,
|
|
71
72
|
exportFanout,
|
|
72
|
-
|
|
73
|
+
handleMcpTasksHttpRequest,
|
|
73
74
|
harvestTimeoutBudget,
|
|
74
75
|
hashOwnerId,
|
|
76
|
+
isLoopbackBaseUrl,
|
|
75
77
|
outputBaseDir,
|
|
76
78
|
registerBrowserAgentMcpTools,
|
|
77
79
|
registerMemoryMcpTools,
|
|
@@ -79,8 +81,9 @@ import {
|
|
|
79
81
|
registerSerpIntelligenceCaptureTools,
|
|
80
82
|
renewConnectedDataArtifactDownload,
|
|
81
83
|
renewDirectoryArtifactDownload,
|
|
84
|
+
resolveDeploymentProfile,
|
|
82
85
|
transcribeMediaUrl
|
|
83
|
-
} from "./chunk-
|
|
86
|
+
} from "./chunk-HO4DOPEG.js";
|
|
84
87
|
import {
|
|
85
88
|
auditImageUrls,
|
|
86
89
|
auditImages,
|
|
@@ -88,10 +91,12 @@ import {
|
|
|
88
91
|
computeIssues,
|
|
89
92
|
createPrivateArtifact,
|
|
90
93
|
isPrivateIpAddress,
|
|
94
|
+
privateArtifactOwnerId,
|
|
95
|
+
readPrivateArtifactBuffer,
|
|
91
96
|
readPrivateArtifactWindow,
|
|
92
97
|
renewPrivateArtifactDownload,
|
|
93
98
|
validatePublicHttpUrl
|
|
94
|
-
} from "./chunk-
|
|
99
|
+
} from "./chunk-2K74LVV7.js";
|
|
95
100
|
import {
|
|
96
101
|
sanitizeAttempts,
|
|
97
102
|
sanitizeHarvestResult,
|
|
@@ -133,7 +138,9 @@ import {
|
|
|
133
138
|
RawMapsOverviewSchema,
|
|
134
139
|
RawMapsReviewStatsSchema
|
|
135
140
|
} from "./chunk-CB5C3BPB.js";
|
|
136
|
-
import
|
|
141
|
+
import {
|
|
142
|
+
PACKAGE_VERSION
|
|
143
|
+
} from "./chunk-3CQZ3DAT.js";
|
|
137
144
|
import {
|
|
138
145
|
abandonExtractSettlement,
|
|
139
146
|
countSuccessfulPages,
|
|
@@ -156,7 +163,7 @@ import {
|
|
|
156
163
|
setExtractJobTotal,
|
|
157
164
|
settleExtractJob,
|
|
158
165
|
terminalExtractJobStatus
|
|
159
|
-
} from "./chunk-
|
|
166
|
+
} from "./chunk-GUI33Q27.js";
|
|
160
167
|
import {
|
|
161
168
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
162
169
|
CONCURRENCY_PRICE_ID,
|
|
@@ -204,7 +211,7 @@ import {
|
|
|
204
211
|
tierChangeLeavesPrivateOffer,
|
|
205
212
|
tierCreditsPerInterval,
|
|
206
213
|
vendorCostUsdToMc
|
|
207
|
-
} from "./chunk-
|
|
214
|
+
} from "./chunk-X5WDJ7D7.js";
|
|
208
215
|
import {
|
|
209
216
|
CaptchaError,
|
|
210
217
|
MapsSearchIncompletePageError,
|
|
@@ -3629,7 +3636,7 @@ async function generateOgImage(post) {
|
|
|
3629
3636
|
|
|
3630
3637
|
// src/api/server.ts
|
|
3631
3638
|
import { Resend as Resend3 } from "resend";
|
|
3632
|
-
import { randomUUID as
|
|
3639
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
3633
3640
|
|
|
3634
3641
|
// src/api/kpo-extractor.ts
|
|
3635
3642
|
import TurndownService from "turndown";
|
|
@@ -8658,7 +8665,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8658
8665
|
});
|
|
8659
8666
|
}) : null;
|
|
8660
8667
|
const finalized = await step.run("finalize", async () => {
|
|
8661
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8668
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
|
|
8662
8669
|
const current = await getExtractJob(jobId2);
|
|
8663
8670
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8664
8671
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10202,11 +10209,11 @@ async function getDirectoryWorkflowJob(id, userId) {
|
|
|
10202
10209
|
const result = userId == null ? await getDb().execute({ sql: "SELECT * FROM directory_workflow_jobs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT * FROM directory_workflow_jobs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
10203
10210
|
return result.rows[0] ? rowToJob(result.rows[0]) : null;
|
|
10204
10211
|
}
|
|
10205
|
-
async function getDirectoryWorkflowJobByIdempotencyKey(userId,
|
|
10212
|
+
async function getDirectoryWorkflowJobByIdempotencyKey(userId, idempotencyKey3) {
|
|
10206
10213
|
await ensureDirectoryWorkflowRepositorySchema();
|
|
10207
10214
|
const result = await getDb().execute({
|
|
10208
10215
|
sql: "SELECT * FROM directory_workflow_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",
|
|
10209
|
-
args: [userId,
|
|
10216
|
+
args: [userId, idempotencyKey3]
|
|
10210
10217
|
});
|
|
10211
10218
|
return result.rows[0] ? rowToJob(result.rows[0]) : null;
|
|
10212
10219
|
}
|
|
@@ -10372,7 +10379,7 @@ async function recordDirectoryWorkflowJobSettlement(jobId2, billedMc) {
|
|
|
10372
10379
|
async function createOrGetDirectoryWorkflowJob(input) {
|
|
10373
10380
|
await ensureDirectoryWorkflowRepositorySchema();
|
|
10374
10381
|
const id = requiredBoundedString(input.id, "job id", 200);
|
|
10375
|
-
const
|
|
10382
|
+
const idempotencyKey3 = requiredBoundedString(input.idempotencyKey, "idempotency key", 500);
|
|
10376
10383
|
const requestFingerprint2 = requiredBoundedString(input.requestFingerprint, "request fingerprint", 200);
|
|
10377
10384
|
const debitKey2 = requiredBoundedString(input.debitKey, "debit key", 500);
|
|
10378
10385
|
const holdMc = nonNegativeSafeInteger(input.holdMc, "holdMc");
|
|
@@ -10387,7 +10394,7 @@ async function createOrGetDirectoryWorkflowJob(input) {
|
|
|
10387
10394
|
args: [
|
|
10388
10395
|
id,
|
|
10389
10396
|
input.userId,
|
|
10390
|
-
|
|
10397
|
+
idempotencyKey3,
|
|
10391
10398
|
requestFingerprint2,
|
|
10392
10399
|
JSON.stringify(safeOptions2),
|
|
10393
10400
|
JSON.stringify(input.plan),
|
|
@@ -10402,10 +10409,10 @@ async function createOrGetDirectoryWorkflowJob(input) {
|
|
|
10402
10409
|
SELECT 'directory:' || id, id, ?, 'held'
|
|
10403
10410
|
FROM directory_workflow_jobs
|
|
10404
10411
|
WHERE user_id = ? AND idempotency_key = ?`,
|
|
10405
|
-
args: [DIRECTORY_EVENT_NAME, input.userId,
|
|
10412
|
+
args: [DIRECTORY_EVENT_NAME, input.userId, idempotencyKey3]
|
|
10406
10413
|
}
|
|
10407
10414
|
], "write");
|
|
10408
|
-
const job = await getDirectoryWorkflowJobByIdempotencyKey(input.userId,
|
|
10415
|
+
const job = await getDirectoryWorkflowJobByIdempotencyKey(input.userId, idempotencyKey3);
|
|
10409
10416
|
if (!job) throw new Error("directory workflow job could not be created");
|
|
10410
10417
|
return {
|
|
10411
10418
|
job,
|
|
@@ -12646,15 +12653,15 @@ function normalizeIdempotencyKey(value) {
|
|
|
12646
12653
|
}
|
|
12647
12654
|
return key;
|
|
12648
12655
|
}
|
|
12649
|
-
function tokenFor(id,
|
|
12650
|
-
return createHmac("sha256", billingSecret()).update(id).update(":").update(
|
|
12656
|
+
function tokenFor(id, idempotencyKey3) {
|
|
12657
|
+
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
|
|
12651
12658
|
}
|
|
12652
|
-
function tokenHash(
|
|
12653
|
-
return createHash3("sha256").update(
|
|
12659
|
+
function tokenHash(token4) {
|
|
12660
|
+
return createHash3("sha256").update(token4).digest("hex");
|
|
12654
12661
|
}
|
|
12655
|
-
function verifyToken(row,
|
|
12662
|
+
function verifyToken(row, token4) {
|
|
12656
12663
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
12657
|
-
const actual = Buffer.from(tokenHash(
|
|
12664
|
+
const actual = Buffer.from(tokenHash(token4), "hex");
|
|
12658
12665
|
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
12659
12666
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
12660
12667
|
}
|
|
@@ -12693,10 +12700,10 @@ async function authorizationById(id) {
|
|
|
12693
12700
|
});
|
|
12694
12701
|
return result.rows[0] ? result.rows[0] : null;
|
|
12695
12702
|
}
|
|
12696
|
-
async function eventByKey(
|
|
12703
|
+
async function eventByKey(idempotencyKey3) {
|
|
12697
12704
|
const result = await getDb().execute({
|
|
12698
12705
|
sql: "SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",
|
|
12699
|
-
args: [
|
|
12706
|
+
args: [idempotencyKey3]
|
|
12700
12707
|
});
|
|
12701
12708
|
return result.rows[0] ? result.rows[0] : null;
|
|
12702
12709
|
}
|
|
@@ -12733,7 +12740,7 @@ function mapSettlementEvent(row, balanceMc, duplicate) {
|
|
|
12733
12740
|
}
|
|
12734
12741
|
async function authorizeScheduledRun(args) {
|
|
12735
12742
|
const identity = normalizeIdentity(args.identity);
|
|
12736
|
-
const
|
|
12743
|
+
const idempotencyKey3 = normalizeIdempotencyKey(args.idempotencyKey);
|
|
12737
12744
|
const user = await resolveSchedulingUser(identity);
|
|
12738
12745
|
if (!hasPaidSchedulingAccess(user)) {
|
|
12739
12746
|
throw new UnifiedBillingError(PAID_PLAN_REQUIRED_CODE, PAID_PLAN_REQUIRED_MESSAGE, 403);
|
|
@@ -12747,7 +12754,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12747
12754
|
}
|
|
12748
12755
|
const existing = await getDb().execute({
|
|
12749
12756
|
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
12750
|
-
args: [
|
|
12757
|
+
args: [idempotencyKey3]
|
|
12751
12758
|
});
|
|
12752
12759
|
if (existing.rows[0]) {
|
|
12753
12760
|
const row = existing.rows[0];
|
|
@@ -12771,16 +12778,16 @@ async function authorizeScheduledRun(args) {
|
|
|
12771
12778
|
);
|
|
12772
12779
|
}
|
|
12773
12780
|
const id = randomUUID3();
|
|
12774
|
-
const
|
|
12781
|
+
const token4 = tokenFor(id, idempotencyKey3);
|
|
12775
12782
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
12776
12783
|
const inserted = await getDb().execute({
|
|
12777
12784
|
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
12778
|
-
args: [id, user.id,
|
|
12785
|
+
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token4), safeMetadata2(args.metadata), expiresAt]
|
|
12779
12786
|
});
|
|
12780
12787
|
if (inserted.rowsAffected === 0) {
|
|
12781
12788
|
const raced = await getDb().execute({
|
|
12782
12789
|
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
12783
|
-
args: [
|
|
12790
|
+
args: [idempotencyKey3]
|
|
12784
12791
|
});
|
|
12785
12792
|
if (!raced.rows[0]) throw new Error("billing authorization insert raced without a readable winner");
|
|
12786
12793
|
const row = raced.rows[0];
|
|
@@ -12791,7 +12798,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12791
12798
|
}
|
|
12792
12799
|
return {
|
|
12793
12800
|
id,
|
|
12794
|
-
token:
|
|
12801
|
+
token: token4,
|
|
12795
12802
|
status: "authorized",
|
|
12796
12803
|
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
12797
12804
|
maximumMc,
|
|
@@ -13061,11 +13068,11 @@ function signSession(userId) {
|
|
|
13061
13068
|
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
13062
13069
|
return `${payload}.${sig}`;
|
|
13063
13070
|
}
|
|
13064
|
-
function verifySession(
|
|
13065
|
-
const dot =
|
|
13071
|
+
function verifySession(token4) {
|
|
13072
|
+
const dot = token4.lastIndexOf(".");
|
|
13066
13073
|
if (dot === -1) return null;
|
|
13067
|
-
const payload =
|
|
13068
|
-
const sig =
|
|
13074
|
+
const payload = token4.slice(0, dot);
|
|
13075
|
+
const sig = token4.slice(dot + 1);
|
|
13069
13076
|
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
13070
13077
|
if (!safeEqualHex(sig, expected)) return null;
|
|
13071
13078
|
const id = parseInt(payload);
|
|
@@ -13293,11 +13300,11 @@ var ConnectedUsageBillingError = class extends Error {
|
|
|
13293
13300
|
function hash(value) {
|
|
13294
13301
|
return createHash4("sha256").update(value).digest("hex");
|
|
13295
13302
|
}
|
|
13296
|
-
function eventKey(
|
|
13297
|
-
return `connected-usage:${hash(
|
|
13303
|
+
function eventKey(idempotencyKey3) {
|
|
13304
|
+
return `connected-usage:${hash(idempotencyKey3)}`;
|
|
13298
13305
|
}
|
|
13299
|
-
function debitKey(
|
|
13300
|
-
return `connected-usage-debit:${hash(
|
|
13306
|
+
function debitKey(idempotencyKey3) {
|
|
13307
|
+
return `connected-usage-debit:${hash(idempotencyKey3)}`;
|
|
13301
13308
|
}
|
|
13302
13309
|
function normalizeIdentity2(identity) {
|
|
13303
13310
|
return identity.trim().toLowerCase();
|
|
@@ -13655,7 +13662,7 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
13655
13662
|
|
|
13656
13663
|
// src/api/internal-billing-routes.ts
|
|
13657
13664
|
var internalBillingApp = new Hono3();
|
|
13658
|
-
var metadataSchema = z12.record(z12.unknown()).optional();
|
|
13665
|
+
var metadataSchema = z12.record(z12.string(), z12.unknown()).optional();
|
|
13659
13666
|
var schedulingAccessSchema = z12.object({
|
|
13660
13667
|
identity: z12.string().min(1).max(320),
|
|
13661
13668
|
provisionCredentials: z12.boolean().optional()
|
|
@@ -14394,9 +14401,9 @@ var YouTubeExtractor = class {
|
|
|
14394
14401
|
}
|
|
14395
14402
|
async fetchContinuation(page, endpoint, contToken, context) {
|
|
14396
14403
|
const result = await page.evaluate(
|
|
14397
|
-
async ({ ep, token:
|
|
14404
|
+
async ({ ep, token: token4, ctx }) => {
|
|
14398
14405
|
const body = {
|
|
14399
|
-
continuation:
|
|
14406
|
+
continuation: token4,
|
|
14400
14407
|
context: {
|
|
14401
14408
|
client: {
|
|
14402
14409
|
clientName: ctx.clientName,
|
|
@@ -15012,7 +15019,7 @@ var HarvestBodySchema = z15.object({
|
|
|
15012
15019
|
pages: z15.number().int().min(1).max(2).optional(),
|
|
15013
15020
|
recency: z15.enum(["day", "week", "month", "year"]).optional()
|
|
15014
15021
|
});
|
|
15015
|
-
var
|
|
15022
|
+
var ExtractUrlBodyShape = {
|
|
15016
15023
|
url: z15.string().min(1, "url is required"),
|
|
15017
15024
|
screenshot: z15.boolean().optional(),
|
|
15018
15025
|
screenshotDevice: z15.enum(["desktop", "mobile"]).optional(),
|
|
@@ -15020,23 +15027,41 @@ var ExtractUrlBodySchema = z15.object({
|
|
|
15020
15027
|
includeFeaturedImage: z15.boolean().optional(),
|
|
15021
15028
|
downloadMedia: z15.boolean().optional(),
|
|
15022
15029
|
mediaTypes: z15.array(z15.enum(["image", "video", "audio"])).optional(),
|
|
15023
|
-
|
|
15030
|
+
delivery: z15.enum(["auto", "inline", "artifact", "memory"]).optional(),
|
|
15031
|
+
preserveMedia: z15.boolean().optional(),
|
|
15024
15032
|
depositToVault: z15.boolean().optional(),
|
|
15025
15033
|
vaultName: z15.string().trim().min(1).max(120).optional()
|
|
15026
|
-
}
|
|
15027
|
-
var
|
|
15034
|
+
};
|
|
15035
|
+
var ExtractUrlBodySchema = z15.object(ExtractUrlBodyShape).strict();
|
|
15036
|
+
var ExtractUrlLocalBodySchema = z15.object({
|
|
15037
|
+
...ExtractUrlBodyShape,
|
|
15038
|
+
allowLocal: z15.boolean().optional()
|
|
15039
|
+
}).strict();
|
|
15040
|
+
var DiffPageBodyShape = {
|
|
15028
15041
|
url: z15.string().min(1, "url is required"),
|
|
15029
|
-
allowLocal: z15.boolean().optional(),
|
|
15030
15042
|
resetBaseline: z15.boolean().optional()
|
|
15031
|
-
}
|
|
15043
|
+
};
|
|
15044
|
+
var DiffPageBodySchema = z15.object(DiffPageBodyShape).strict();
|
|
15045
|
+
var DiffPageLocalBodySchema = z15.object({
|
|
15046
|
+
...DiffPageBodyShape,
|
|
15047
|
+
allowLocal: z15.boolean().optional()
|
|
15048
|
+
}).strict();
|
|
15032
15049
|
var ArchiveReadBodySchema = z15.object({
|
|
15033
|
-
|
|
15050
|
+
artifactId: z15.string().trim().min(1).optional(),
|
|
15051
|
+
url: z15.string().url("url must be a valid HTTPS ZIP URL").optional(),
|
|
15034
15052
|
path: z15.string().trim().min(1).max(2e3).optional(),
|
|
15035
15053
|
offset: z15.number().int().min(0).optional(),
|
|
15036
15054
|
maxBytes: z15.number().int().min(1).max(2e5).optional(),
|
|
15037
15055
|
maxEntries: z15.number().int().min(1).max(1e3).optional(),
|
|
15038
15056
|
depositToLibrary: z15.boolean().optional()
|
|
15039
15057
|
}).superRefine((value, ctx) => {
|
|
15058
|
+
if ((value.artifactId ? 1 : 0) + (value.url ? 1 : 0) !== 1) {
|
|
15059
|
+
ctx.addIssue({
|
|
15060
|
+
code: z15.ZodIssueCode.custom,
|
|
15061
|
+
path: ["artifactId"],
|
|
15062
|
+
message: "Provide exactly one of artifactId or url"
|
|
15063
|
+
});
|
|
15064
|
+
}
|
|
15040
15065
|
if (value.depositToLibrary && !value.path) {
|
|
15041
15066
|
ctx.addIssue({
|
|
15042
15067
|
code: z15.ZodIssueCode.custom,
|
|
@@ -15050,7 +15075,8 @@ var MapUrlsBodySchema = z15.object({
|
|
|
15050
15075
|
maxUrls: z15.number().int().min(1).max(2e3).optional(),
|
|
15051
15076
|
concurrency: z15.number().int().min(1).max(20).optional(),
|
|
15052
15077
|
browserFallback: z15.boolean().optional(),
|
|
15053
|
-
kernelFallback: z15.boolean().optional()
|
|
15078
|
+
kernelFallback: z15.boolean().optional(),
|
|
15079
|
+
delivery: z15.enum(["auto", "inline", "artifact"]).optional()
|
|
15054
15080
|
});
|
|
15055
15081
|
var WaybackInventoryBodySchema = z15.object({
|
|
15056
15082
|
url: z15.string().min(1, "url is required"),
|
|
@@ -15066,6 +15092,8 @@ var ExtractSiteBodySchema = z15.object({
|
|
|
15066
15092
|
rotateProxyEvery: z15.number().int().min(1).max(100).optional(),
|
|
15067
15093
|
background: z15.boolean().optional(),
|
|
15068
15094
|
downloadImages: z15.boolean().optional(),
|
|
15095
|
+
preserveMedia: z15.boolean().optional(),
|
|
15096
|
+
delivery: z15.enum(["auto", "artifact"]).optional(),
|
|
15069
15097
|
formats: z15.array(z15.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
15070
15098
|
});
|
|
15071
15099
|
var YoutubeHarvestBodySchema = z15.object({
|
|
@@ -16275,7 +16303,7 @@ function advertiserTokenOverlap(query, candidate) {
|
|
|
16275
16303
|
const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
|
|
16276
16304
|
if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
|
|
16277
16305
|
let overlap = 0;
|
|
16278
|
-
for (const
|
|
16306
|
+
for (const token4 of queryTokens) if (candidateTokens.has(token4)) overlap++;
|
|
16279
16307
|
return overlap / queryTokens.size;
|
|
16280
16308
|
}
|
|
16281
16309
|
function assessPageIntelMatch(body, result) {
|
|
@@ -17463,11 +17491,85 @@ adminCreditsApp.get("/lookup", async (c) => {
|
|
|
17463
17491
|
// src/api/instagram-routes.ts
|
|
17464
17492
|
import { Hono as Hono10 } from "hono";
|
|
17465
17493
|
import { z as z19 } from "zod";
|
|
17466
|
-
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
17494
|
+
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
17467
17495
|
import { tmpdir } from "os";
|
|
17468
|
-
import { join as
|
|
17496
|
+
import { join as join5 } from "path";
|
|
17469
17497
|
import { spawn } from "child_process";
|
|
17470
17498
|
|
|
17499
|
+
// src/api/blob-store.ts
|
|
17500
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
17501
|
+
import { homedir } from "os";
|
|
17502
|
+
import { join as join4, dirname as dirname2 } from "path";
|
|
17503
|
+
function byteLength(data) {
|
|
17504
|
+
return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
|
|
17505
|
+
}
|
|
17506
|
+
var LocalBlobStore = class {
|
|
17507
|
+
constructor(baseDir) {
|
|
17508
|
+
this.baseDir = baseDir;
|
|
17509
|
+
}
|
|
17510
|
+
baseDir;
|
|
17511
|
+
kind = "local";
|
|
17512
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
17513
|
+
const path5 = join4(this.baseDir, "blobs", key);
|
|
17514
|
+
mkdirSync(dirname2(path5), { recursive: true });
|
|
17515
|
+
writeFileSync(path5, data);
|
|
17516
|
+
return { key, url: `file://${path5}`, bytes: byteLength(data), contentType };
|
|
17517
|
+
}
|
|
17518
|
+
async get(key) {
|
|
17519
|
+
const path5 = join4(this.baseDir, "blobs", key);
|
|
17520
|
+
try {
|
|
17521
|
+
const { readFileSync: readFileSync4 } = await import("fs");
|
|
17522
|
+
return readFileSync4(path5);
|
|
17523
|
+
} catch {
|
|
17524
|
+
return null;
|
|
17525
|
+
}
|
|
17526
|
+
}
|
|
17527
|
+
};
|
|
17528
|
+
function localBaseDir() {
|
|
17529
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir(), "Downloads", "mcp-scraper");
|
|
17530
|
+
}
|
|
17531
|
+
var cached = null;
|
|
17532
|
+
function getBlobStore() {
|
|
17533
|
+
if (cached) return cached;
|
|
17534
|
+
if (process.env.BLOB_READ_WRITE_TOKEN) {
|
|
17535
|
+
cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
|
|
17536
|
+
} else {
|
|
17537
|
+
cached = new LocalBlobStore(localBaseDir());
|
|
17538
|
+
}
|
|
17539
|
+
return cached;
|
|
17540
|
+
}
|
|
17541
|
+
var VercelBlobStore = class {
|
|
17542
|
+
constructor(token4) {
|
|
17543
|
+
this.token = token4;
|
|
17544
|
+
}
|
|
17545
|
+
token;
|
|
17546
|
+
kind = "vercel-blob";
|
|
17547
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
17548
|
+
const { put } = await import("@vercel/blob");
|
|
17549
|
+
const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
17550
|
+
const result = await put(key, body, {
|
|
17551
|
+
access: "public",
|
|
17552
|
+
token: this.token,
|
|
17553
|
+
contentType,
|
|
17554
|
+
addRandomSuffix: true
|
|
17555
|
+
});
|
|
17556
|
+
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
17557
|
+
}
|
|
17558
|
+
async get(key) {
|
|
17559
|
+
try {
|
|
17560
|
+
const { list } = await import("@vercel/blob");
|
|
17561
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
17562
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
17563
|
+
if (!match) return null;
|
|
17564
|
+
const resp = await fetch(match.url);
|
|
17565
|
+
if (!resp.ok) return null;
|
|
17566
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
17567
|
+
} catch {
|
|
17568
|
+
return null;
|
|
17569
|
+
}
|
|
17570
|
+
}
|
|
17571
|
+
};
|
|
17572
|
+
|
|
17471
17573
|
// src/extractor/InstagramContentExtractor.ts
|
|
17472
17574
|
var CONTENT_PATH_RE = /^\/(?:[^/]+\/)?(p|reel|tv)\/([A-Za-z0-9_-]+)\/?/i;
|
|
17473
17575
|
var CDN_HOST_RE = /(^|\.)((cdninstagram\.com)|(scontent[^/]*\.cdninstagram\.com))$/i;
|
|
@@ -18069,9 +18171,9 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
18069
18171
|
downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
18070
18172
|
const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
|
|
18071
18173
|
if (isMuxInput) {
|
|
18072
|
-
if (!muxWorkDir) muxWorkDir = mkdtempSync(
|
|
18073
|
-
const localPath =
|
|
18074
|
-
|
|
18174
|
+
if (!muxWorkDir) muxWorkDir = mkdtempSync(join5(tmpdir(), "ig-mux-"));
|
|
18175
|
+
const localPath = join5(muxWorkDir, trackFilename(track, index, extraction.shortcode));
|
|
18176
|
+
writeFileSync2(localPath, bytes);
|
|
18075
18177
|
muxInputs.set(track.url, localPath);
|
|
18076
18178
|
}
|
|
18077
18179
|
} catch (err) {
|
|
@@ -18079,7 +18181,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
18079
18181
|
}
|
|
18080
18182
|
}
|
|
18081
18183
|
if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
|
|
18082
|
-
const outPath =
|
|
18184
|
+
const outPath = join5(muxWorkDir, "muxed.mp4");
|
|
18083
18185
|
const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
|
|
18084
18186
|
if (muxed.ok) {
|
|
18085
18187
|
const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
|
|
@@ -20522,7 +20624,7 @@ import { readFile as readFile2 } from "fs/promises";
|
|
|
20522
20624
|
import { Hono as Hono18 } from "hono";
|
|
20523
20625
|
import { z as z26 } from "zod";
|
|
20524
20626
|
var workflowApp = new Hono18();
|
|
20525
|
-
var WorkflowInputSchema = z26.record(z26.unknown()).default({});
|
|
20627
|
+
var WorkflowInputSchema = z26.record(z26.string(), z26.unknown()).default({});
|
|
20526
20628
|
var WorkflowIdSchema = z26.string().min(1);
|
|
20527
20629
|
var CadenceSchema = z26.enum(["daily", "weekly", "monthly"]);
|
|
20528
20630
|
var ScheduleStatusSchema = z26.enum(["active", "paused"]);
|
|
@@ -22061,7 +22163,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
22061
22163
|
|
|
22062
22164
|
// src/mcp/mcp-routes.ts
|
|
22063
22165
|
import { Hono as Hono20 } from "hono";
|
|
22064
|
-
import {
|
|
22166
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
22065
22167
|
|
|
22066
22168
|
// src/mcp/memory-mcp-tool-executor.ts
|
|
22067
22169
|
var MemoryMcpToolExecutor = class {
|
|
@@ -22099,6 +22201,58 @@ var MemoryMcpToolExecutor = class {
|
|
|
22099
22201
|
}
|
|
22100
22202
|
};
|
|
22101
22203
|
|
|
22204
|
+
// src/api/image-source-artifacts.ts
|
|
22205
|
+
import { randomUUID as randomUUID9 } from "crypto";
|
|
22206
|
+
var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
22207
|
+
var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
22208
|
+
var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
22209
|
+
var IMAGE_SOURCE_MAX_BYTES = 20 * 1024 * 1024;
|
|
22210
|
+
var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
|
|
22211
|
+
["image/jpeg", "jpg"],
|
|
22212
|
+
["image/png", "png"],
|
|
22213
|
+
["image/webp", "webp"],
|
|
22214
|
+
["image/gif", "gif"]
|
|
22215
|
+
]);
|
|
22216
|
+
function token() {
|
|
22217
|
+
return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
22218
|
+
}
|
|
22219
|
+
function policy() {
|
|
22220
|
+
return {
|
|
22221
|
+
prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
|
|
22222
|
+
artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
|
|
22223
|
+
downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
|
|
22224
|
+
token: token()
|
|
22225
|
+
};
|
|
22226
|
+
}
|
|
22227
|
+
function imageSourceArtifactOwnerId(artifactId) {
|
|
22228
|
+
return privateArtifactOwnerId(artifactId, IMAGE_SOURCE_ARTIFACT_PREFIX);
|
|
22229
|
+
}
|
|
22230
|
+
async function createImageSourceArtifact(args) {
|
|
22231
|
+
const extension = ALLOWED_IMAGE_TYPES.get(args.contentType);
|
|
22232
|
+
if (!extension) throw new Error("image_source_type_unsupported");
|
|
22233
|
+
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
22234
|
+
throw new Error("image_source_size_invalid");
|
|
22235
|
+
}
|
|
22236
|
+
const id = randomUUID9().replaceAll("-", "");
|
|
22237
|
+
return createPrivateArtifact({
|
|
22238
|
+
policy: policy(),
|
|
22239
|
+
ownerId: args.ownerId,
|
|
22240
|
+
artifactKey: `${id}.${extension}`,
|
|
22241
|
+
createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
|
|
22242
|
+
filename: args.filename ?? `mcp-scraper-image-${id.slice(0, 12)}.${extension}`,
|
|
22243
|
+
contentType: args.contentType,
|
|
22244
|
+
content: args.content
|
|
22245
|
+
});
|
|
22246
|
+
}
|
|
22247
|
+
async function readOwnedImageSourceArtifact(args) {
|
|
22248
|
+
if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
22249
|
+
return readPrivateArtifactBuffer({
|
|
22250
|
+
policy: policy(),
|
|
22251
|
+
artifactId: args.artifactId,
|
|
22252
|
+
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
22253
|
+
});
|
|
22254
|
+
}
|
|
22255
|
+
|
|
22102
22256
|
// src/mcp/memory-inprocess-executor.ts
|
|
22103
22257
|
var CUTOVER = /* @__PURE__ */ new Map();
|
|
22104
22258
|
function registerCutover(upstreamName, handler) {
|
|
@@ -22142,6 +22296,32 @@ var HybridMemoryToolExecutor = class {
|
|
|
22142
22296
|
};
|
|
22143
22297
|
}
|
|
22144
22298
|
}
|
|
22299
|
+
if (upstreamName === "image_asset_save" && typeof args.artifactId === "string") {
|
|
22300
|
+
const image = await readOwnedImageSourceArtifact({
|
|
22301
|
+
artifactId: args.artifactId,
|
|
22302
|
+
ownerId: String(this.user.id)
|
|
22303
|
+
});
|
|
22304
|
+
if (!image) {
|
|
22305
|
+
const result = {
|
|
22306
|
+
ok: false,
|
|
22307
|
+
code: "image_artifact_not_found",
|
|
22308
|
+
error: "image artifact not found"
|
|
22309
|
+
};
|
|
22310
|
+
return {
|
|
22311
|
+
content: [{ type: "text", text: JSON.stringify(result) }],
|
|
22312
|
+
structuredContent: result,
|
|
22313
|
+
isError: false
|
|
22314
|
+
};
|
|
22315
|
+
}
|
|
22316
|
+
const sourceRef = typeof args.sourceRef === "object" && args.sourceRef !== null && !Array.isArray(args.sourceRef) ? args.sourceRef : {};
|
|
22317
|
+
args = {
|
|
22318
|
+
...args,
|
|
22319
|
+
artifactId: void 0,
|
|
22320
|
+
imageBase64: image.toString("base64"),
|
|
22321
|
+
sourceRef: { ...sourceRef, artifactId: args.artifactId }
|
|
22322
|
+
};
|
|
22323
|
+
delete args.artifactId;
|
|
22324
|
+
}
|
|
22145
22325
|
const handler = CUTOVER.get(upstreamName);
|
|
22146
22326
|
if (!handler) return this.httpExecutor.callMemoryTool(upstreamName, args);
|
|
22147
22327
|
const mkKey = await this.resolveMkKey();
|
|
@@ -22652,8 +22832,8 @@ function protectedResourceMetadata() {
|
|
|
22652
22832
|
function wwwAuthenticateValue() {
|
|
22653
22833
|
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
22654
22834
|
}
|
|
22655
|
-
function isBearerJwt(
|
|
22656
|
-
return !
|
|
22835
|
+
function isBearerJwt(token4) {
|
|
22836
|
+
return !token4.startsWith("sk_") && token4.split(".").length === 3;
|
|
22657
22837
|
}
|
|
22658
22838
|
var jwksSingleton = null;
|
|
22659
22839
|
function getJwks() {
|
|
@@ -22661,8 +22841,8 @@ function getJwks() {
|
|
|
22661
22841
|
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
22662
22842
|
return jwksSingleton;
|
|
22663
22843
|
}
|
|
22664
|
-
async function verifyScraperOAuthJwt(
|
|
22665
|
-
const { payload } = await jwtVerify(
|
|
22844
|
+
async function verifyScraperOAuthJwt(token4) {
|
|
22845
|
+
const { payload } = await jwtVerify(token4, getJwks(), {
|
|
22666
22846
|
issuer: oauthIssuer(),
|
|
22667
22847
|
audience: oauthResourceUrl()
|
|
22668
22848
|
});
|
|
@@ -22717,37 +22897,91 @@ async function requireMcpCallerKey(c) {
|
|
|
22717
22897
|
return { key: callerKey, user };
|
|
22718
22898
|
}
|
|
22719
22899
|
var mcpApp = new Hono20();
|
|
22900
|
+
function userFromAuthInfo(authInfo) {
|
|
22901
|
+
const user = authInfo?.extra?.user;
|
|
22902
|
+
if (!user || typeof user !== "object" || Array.isArray(user)) {
|
|
22903
|
+
throw new Error("Authenticated MCP user context is missing.");
|
|
22904
|
+
}
|
|
22905
|
+
return user;
|
|
22906
|
+
}
|
|
22907
|
+
var mcpHandler = createMcpHandler(async ({ authInfo }) => {
|
|
22908
|
+
const user = userFromAuthInfo(authInfo);
|
|
22909
|
+
const callerKey = authInfo?.token ?? "";
|
|
22910
|
+
if (!callerKey) throw new Error("Authenticated MCP token is missing.");
|
|
22911
|
+
const baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? "https://mcpscraper.dev";
|
|
22912
|
+
const executor = new HttpMcpToolExecutor(baseUrl, callerKey);
|
|
22913
|
+
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
22914
|
+
const server = buildPaaExtractorMcpServer(executor, {
|
|
22915
|
+
savesReportsLocally: false,
|
|
22916
|
+
ownerId: hashOwnerId(callerKey),
|
|
22917
|
+
deploymentProfile: "production",
|
|
22918
|
+
transportProfile: "hosted_http",
|
|
22919
|
+
baseUrl,
|
|
22920
|
+
taskHandleSecret: callerKey
|
|
22921
|
+
});
|
|
22922
|
+
registerSerpIntelligenceCaptureTools(server, executor, { exposeDevelopmentDiagnostics: false });
|
|
22923
|
+
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
22924
|
+
registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, callerKey));
|
|
22925
|
+
const memoryExecutor = new HybridMemoryToolExecutor(new MemoryMcpToolExecutor(baseUrl, callerKey), user);
|
|
22926
|
+
registerMemoryMcpTools(server, memoryExecutor);
|
|
22927
|
+
return server;
|
|
22928
|
+
}, {
|
|
22929
|
+
legacy: "stateless",
|
|
22930
|
+
responseMode: "auto",
|
|
22931
|
+
onerror(error) {
|
|
22932
|
+
console.error("[mcp] protocol error", error.message);
|
|
22933
|
+
}
|
|
22934
|
+
});
|
|
22935
|
+
function allowedOrigin(request) {
|
|
22936
|
+
const origin = request.headers.get("origin");
|
|
22937
|
+
if (origin === null) return true;
|
|
22938
|
+
const allowed = /* @__PURE__ */ new Set([
|
|
22939
|
+
"https://mcpscraper.dev",
|
|
22940
|
+
"https://www.mcpscraper.dev",
|
|
22941
|
+
...process.env.APP_ORIGIN?.trim() ? [process.env.APP_ORIGIN.trim()] : []
|
|
22942
|
+
]);
|
|
22943
|
+
return allowed.has(origin);
|
|
22944
|
+
}
|
|
22720
22945
|
mcpApp.all("/", async (c) => {
|
|
22721
22946
|
try {
|
|
22947
|
+
if (!allowedOrigin(c.req.raw)) {
|
|
22948
|
+
return c.json({ jsonrpc: "2.0", id: null, error: { code: -32e3, message: "Forbidden Origin" } }, 403);
|
|
22949
|
+
}
|
|
22722
22950
|
const keyOrError = await requireMcpCallerKey(c);
|
|
22723
22951
|
if (keyOrError instanceof Response) return keyOrError;
|
|
22724
22952
|
const { key: callerKey, user } = keyOrError;
|
|
22725
|
-
const
|
|
22726
|
-
|
|
22727
|
-
|
|
22728
|
-
|
|
22729
|
-
|
|
22953
|
+
const taskResponse = await handleMcpTasksHttpRequest(
|
|
22954
|
+
c.req.raw,
|
|
22955
|
+
new HttpMcpToolExecutor(
|
|
22956
|
+
process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? "https://mcpscraper.dev",
|
|
22957
|
+
callerKey
|
|
22958
|
+
),
|
|
22959
|
+
{
|
|
22960
|
+
ownerId: hashOwnerId(callerKey),
|
|
22961
|
+
secret: callerKey,
|
|
22962
|
+
serverInfo: { name: "mcp-scraper", version: PACKAGE_VERSION }
|
|
22963
|
+
}
|
|
22964
|
+
);
|
|
22965
|
+
if (taskResponse) return taskResponse;
|
|
22966
|
+
return mcpHandler.fetch(c.req.raw, {
|
|
22967
|
+
authInfo: {
|
|
22968
|
+
token: callerKey,
|
|
22969
|
+
clientId: `mcp-scraper-user-${user.id}`,
|
|
22970
|
+
scopes: ["mcp:tools"],
|
|
22971
|
+
extra: { user }
|
|
22972
|
+
}
|
|
22730
22973
|
});
|
|
22731
|
-
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
22732
|
-
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
|
|
22733
|
-
registerSerpIntelligenceCaptureTools(server, executor);
|
|
22734
|
-
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
22735
|
-
registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, callerKey));
|
|
22736
|
-
const memoryExecutor = new HybridMemoryToolExecutor(new MemoryMcpToolExecutor(baseUrl, callerKey), user);
|
|
22737
|
-
registerMemoryMcpTools(server, memoryExecutor);
|
|
22738
|
-
await server.connect(transport);
|
|
22739
|
-
return transport.handleRequest(c.req.raw);
|
|
22740
22974
|
} catch {
|
|
22741
22975
|
return c.json({ error: "Internal Server Error" }, 500);
|
|
22742
22976
|
}
|
|
22743
22977
|
});
|
|
22744
22978
|
|
|
22745
22979
|
// src/api/browser-agent-routes.ts
|
|
22746
|
-
import { randomUUID as
|
|
22980
|
+
import { randomUUID as randomUUID11 } from "crypto";
|
|
22747
22981
|
import { Hono as Hono21 } from "hono";
|
|
22748
22982
|
|
|
22749
22983
|
// src/api/browser-agent-db.ts
|
|
22750
|
-
import { randomUUID as
|
|
22984
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
22751
22985
|
var _ready = false;
|
|
22752
22986
|
var _migrationPromise = null;
|
|
22753
22987
|
async function migrateBrowserAgent() {
|
|
@@ -22878,7 +23112,7 @@ async function runBrowserAgentMigration() {
|
|
|
22878
23112
|
}
|
|
22879
23113
|
async function createExtensionRow(input) {
|
|
22880
23114
|
const db = getDb();
|
|
22881
|
-
const id = `bext_${
|
|
23115
|
+
const id = `bext_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
22882
23116
|
await db.execute({
|
|
22883
23117
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
22884
23118
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -22913,7 +23147,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
22913
23147
|
}
|
|
22914
23148
|
async function createAuthConnectionRow(input) {
|
|
22915
23149
|
const db = getDb();
|
|
22916
|
-
const connectionId = `authc_${
|
|
23150
|
+
const connectionId = `authc_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
22917
23151
|
await db.execute({
|
|
22918
23152
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
22919
23153
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -22998,7 +23232,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
22998
23232
|
}
|
|
22999
23233
|
async function createSessionRow(input) {
|
|
23000
23234
|
const db = getDb();
|
|
23001
|
-
const id = `bas_${
|
|
23235
|
+
const id = `bas_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
23002
23236
|
await db.execute({
|
|
23003
23237
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
23004
23238
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -23067,7 +23301,7 @@ async function recordAction(input) {
|
|
|
23067
23301
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
23068
23302
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
23069
23303
|
args: [
|
|
23070
|
-
`baa_${
|
|
23304
|
+
`baa_${randomUUID10().replace(/-/g, "").slice(0, 20)}`,
|
|
23071
23305
|
input.sessionId,
|
|
23072
23306
|
input.type,
|
|
23073
23307
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -24798,7 +25032,7 @@ function buildBrowserAgentRoutes() {
|
|
|
24798
25032
|
}
|
|
24799
25033
|
const existing = await getExtensionRow(user.id, name);
|
|
24800
25034
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
24801
|
-
const backendName = `u${user.id}_${
|
|
25035
|
+
const backendName = `u${user.id}_${randomUUID11().replace(/-/g, "")}`;
|
|
24802
25036
|
try {
|
|
24803
25037
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
24804
25038
|
const row = await createExtensionRow({
|
|
@@ -25292,7 +25526,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
25292
25526
|
// src/api/oauth-routes.ts
|
|
25293
25527
|
import { Hono as Hono23 } from "hono";
|
|
25294
25528
|
import { getCookie, setCookie } from "hono/cookie";
|
|
25295
|
-
import { createHash as createHash8, randomBytes as randomBytes2, randomUUID as
|
|
25529
|
+
import { createHash as createHash8, randomBytes as randomBytes2, randomUUID as randomUUID12 } from "crypto";
|
|
25296
25530
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
25297
25531
|
var ISSUER = "https://mcpscraper.dev";
|
|
25298
25532
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -25366,9 +25600,9 @@ function negotiateScope(requested, user, resource) {
|
|
|
25366
25600
|
if (operator && !unique2.includes("memory.admin")) unique2.push("memory.admin");
|
|
25367
25601
|
return unique2.join(" ");
|
|
25368
25602
|
}
|
|
25369
|
-
async function sessionUser(
|
|
25370
|
-
if (!
|
|
25371
|
-
const userId = verifySession(
|
|
25603
|
+
async function sessionUser(token4) {
|
|
25604
|
+
if (!token4) return null;
|
|
25605
|
+
const userId = verifySession(token4);
|
|
25372
25606
|
if (!userId) return null;
|
|
25373
25607
|
const user = await getUserById(userId);
|
|
25374
25608
|
return user ?? null;
|
|
@@ -25640,7 +25874,7 @@ function pkceMatches(verifier, challenge) {
|
|
|
25640
25874
|
}
|
|
25641
25875
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
25642
25876
|
const { privateKey, kid } = await getKeys();
|
|
25643
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
25877
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID12()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
25644
25878
|
}
|
|
25645
25879
|
function tokenErrorResponse(c, error, description, status) {
|
|
25646
25880
|
return c.json({ error, error_description: description }, status);
|
|
@@ -25715,35 +25949,35 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
25715
25949
|
// src/api/chat-routes.ts
|
|
25716
25950
|
import { Hono as Hono24 } from "hono";
|
|
25717
25951
|
var chatApp = new Hono24();
|
|
25718
|
-
function tokenOk(
|
|
25719
|
-
return !!
|
|
25952
|
+
function tokenOk(token4) {
|
|
25953
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 10;
|
|
25720
25954
|
}
|
|
25721
25955
|
function jsonError(message, status = 400) {
|
|
25722
25956
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
25723
25957
|
}
|
|
25724
25958
|
chatApp.get("/api/channels", async (c) => {
|
|
25725
|
-
const
|
|
25726
|
-
if (!tokenOk(
|
|
25727
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
25959
|
+
const token4 = c.req.query("token");
|
|
25960
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25961
|
+
const r = await memoryCall("listVaultsTool", {}, token4);
|
|
25728
25962
|
if (!r.ok) return c.json(r);
|
|
25729
25963
|
const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
|
|
25730
25964
|
return c.json({ ok: true, channels: vaults });
|
|
25731
25965
|
});
|
|
25732
25966
|
chatApp.get("/api/messages", async (c) => {
|
|
25733
|
-
const
|
|
25734
|
-
if (!tokenOk(
|
|
25967
|
+
const token4 = c.req.query("token");
|
|
25968
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25735
25969
|
const vault = c.req.query("vault");
|
|
25736
25970
|
if (!vault) return jsonError("vault is required");
|
|
25737
25971
|
const parentMessageId = c.req.query("parentMessageId") || void 0;
|
|
25738
|
-
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId },
|
|
25972
|
+
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token4);
|
|
25739
25973
|
return c.json(r);
|
|
25740
25974
|
});
|
|
25741
25975
|
chatApp.get("/api/poll", async (c) => {
|
|
25742
|
-
const
|
|
25743
|
-
if (!tokenOk(
|
|
25976
|
+
const token4 = c.req.query("token");
|
|
25977
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25744
25978
|
const vault = c.req.query("vault");
|
|
25745
25979
|
if (!vault) return jsonError("vault is required");
|
|
25746
|
-
const r = await memoryCall("pollChannelTool", { vault },
|
|
25980
|
+
const r = await memoryCall("pollChannelTool", { vault }, token4);
|
|
25747
25981
|
return c.json(r);
|
|
25748
25982
|
});
|
|
25749
25983
|
function parseAttachNote(v) {
|
|
@@ -25768,12 +26002,12 @@ chatApp.post("/api/reply", async (c) => {
|
|
|
25768
26002
|
return c.json(r);
|
|
25769
26003
|
});
|
|
25770
26004
|
chatApp.get("/api/notes-own", async (c) => {
|
|
25771
|
-
const
|
|
25772
|
-
if (!tokenOk(
|
|
25773
|
-
const vaultsRes = await memoryCall("listVaultsTool", {},
|
|
26005
|
+
const token4 = c.req.query("token");
|
|
26006
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
26007
|
+
const vaultsRes = await memoryCall("listVaultsTool", {}, token4);
|
|
25774
26008
|
if (!vaultsRes.ok) return c.json(vaultsRes);
|
|
25775
26009
|
const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
|
|
25776
|
-
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle },
|
|
26010
|
+
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token4)));
|
|
25777
26011
|
const notes = ownVaults.flatMap((v, i) => {
|
|
25778
26012
|
const r = lists[i];
|
|
25779
26013
|
return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
|
|
@@ -25781,12 +26015,12 @@ chatApp.get("/api/notes-own", async (c) => {
|
|
|
25781
26015
|
return c.json({ ok: true, notes });
|
|
25782
26016
|
});
|
|
25783
26017
|
chatApp.get("/api/message-note", async (c) => {
|
|
25784
|
-
const
|
|
25785
|
-
if (!tokenOk(
|
|
26018
|
+
const token4 = c.req.query("token");
|
|
26019
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25786
26020
|
const vault = c.req.query("vault");
|
|
25787
26021
|
const messageId = c.req.query("messageId");
|
|
25788
26022
|
if (!vault || !messageId) return jsonError("vault and messageId are required");
|
|
25789
|
-
const r = await memoryCall("getMessageNoteTool", { vault, messageId },
|
|
26023
|
+
const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token4);
|
|
25790
26024
|
return c.json(r);
|
|
25791
26025
|
});
|
|
25792
26026
|
chatApp.post("/api/react", async (c) => {
|
|
@@ -25797,17 +26031,17 @@ chatApp.post("/api/react", async (c) => {
|
|
|
25797
26031
|
return c.json(r);
|
|
25798
26032
|
});
|
|
25799
26033
|
chatApp.get("/api/mentions", async (c) => {
|
|
25800
|
-
const
|
|
25801
|
-
if (!tokenOk(
|
|
25802
|
-
const r = await memoryCall("myMentionsTool", {},
|
|
26034
|
+
const token4 = c.req.query("token");
|
|
26035
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
26036
|
+
const r = await memoryCall("myMentionsTool", {}, token4);
|
|
25803
26037
|
return c.json(r);
|
|
25804
26038
|
});
|
|
25805
26039
|
chatApp.get("/api/members", async (c) => {
|
|
25806
|
-
const
|
|
25807
|
-
if (!tokenOk(
|
|
26040
|
+
const token4 = c.req.query("token");
|
|
26041
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25808
26042
|
const vault = c.req.query("vault");
|
|
25809
26043
|
if (!vault) return jsonError("vault is required");
|
|
25810
|
-
const r = await memoryCall("listChannelMembersTool", { vault },
|
|
26044
|
+
const r = await memoryCall("listChannelMembersTool", { vault }, token4);
|
|
25811
26045
|
return c.json(r);
|
|
25812
26046
|
});
|
|
25813
26047
|
chatApp.post("/api/channel", async (c) => {
|
|
@@ -25843,13 +26077,13 @@ chatApp.post("/api/member-remove", async (c) => {
|
|
|
25843
26077
|
return c.json(r);
|
|
25844
26078
|
});
|
|
25845
26079
|
chatApp.get("/:token", (c) => {
|
|
25846
|
-
const
|
|
25847
|
-
if (!tokenOk(
|
|
26080
|
+
const token4 = c.req.param("token");
|
|
26081
|
+
if (!tokenOk(token4)) {
|
|
25848
26082
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
|
|
25849
26083
|
}
|
|
25850
|
-
return c.html(renderChatPage(
|
|
26084
|
+
return c.html(renderChatPage(token4));
|
|
25851
26085
|
});
|
|
25852
|
-
function renderChatPage(
|
|
26086
|
+
function renderChatPage(token4) {
|
|
25853
26087
|
return `<!doctype html>
|
|
25854
26088
|
<html>
|
|
25855
26089
|
<head>
|
|
@@ -26034,7 +26268,7 @@ function renderChatPage(token3) {
|
|
|
26034
26268
|
</div>
|
|
26035
26269
|
|
|
26036
26270
|
<script>
|
|
26037
|
-
const TOKEN = ${JSON.stringify(
|
|
26271
|
+
const TOKEN = ${JSON.stringify(token4)};
|
|
26038
26272
|
const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
|
|
26039
26273
|
|
|
26040
26274
|
function api(path, opts) {
|
|
@@ -26404,8 +26638,8 @@ var MAX_VAULT_RECORDS = 200;
|
|
|
26404
26638
|
var MAX_MEMORY_READ_CONCURRENCY = 4;
|
|
26405
26639
|
var MAX_MEMORY_READ_ATTEMPTS = 3;
|
|
26406
26640
|
var vaultApp = new Hono25();
|
|
26407
|
-
function validToken(
|
|
26408
|
-
return !!
|
|
26641
|
+
function validToken(token4) {
|
|
26642
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 20 && token4.length < 256;
|
|
26409
26643
|
}
|
|
26410
26644
|
function isVault(value) {
|
|
26411
26645
|
return typeof value === "string" && VAULTS.includes(value);
|
|
@@ -26531,11 +26765,11 @@ function shouldRetryMemoryRead(error) {
|
|
|
26531
26765
|
function sleep(ms) {
|
|
26532
26766
|
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
26533
26767
|
}
|
|
26534
|
-
async function readMemory(tool, args,
|
|
26535
|
-
let result = await memoryCall(tool, args,
|
|
26768
|
+
async function readMemory(tool, args, token4) {
|
|
26769
|
+
let result = await memoryCall(tool, args, token4);
|
|
26536
26770
|
for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
|
|
26537
26771
|
await sleep(attempt * 150);
|
|
26538
|
-
result = await memoryCall(tool, args,
|
|
26772
|
+
result = await memoryCall(tool, args, token4);
|
|
26539
26773
|
}
|
|
26540
26774
|
return result;
|
|
26541
26775
|
}
|
|
@@ -26552,27 +26786,27 @@ async function mapWithConcurrency3(items, limit, map) {
|
|
|
26552
26786
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
26553
26787
|
return values;
|
|
26554
26788
|
}
|
|
26555
|
-
async function readVault(vault,
|
|
26556
|
-
const list = await readMemory("listTool", { vault },
|
|
26789
|
+
async function readVault(vault, token4) {
|
|
26790
|
+
const list = await readMemory("listTool", { vault }, token4);
|
|
26557
26791
|
if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
|
|
26558
26792
|
const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
|
|
26559
26793
|
const notes = await mapWithConcurrency3(
|
|
26560
26794
|
noteRefs,
|
|
26561
26795
|
MAX_MEMORY_READ_CONCURRENCY,
|
|
26562
|
-
({ path: path5 }) => readMemory("getTool", { vault, path: path5 },
|
|
26796
|
+
({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token4)
|
|
26563
26797
|
);
|
|
26564
26798
|
const failed = notes.find((result) => !result.ok);
|
|
26565
26799
|
if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
|
|
26566
26800
|
return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
|
|
26567
26801
|
}
|
|
26568
26802
|
vaultApp.get("/api/workspace", async (c) => {
|
|
26569
|
-
const
|
|
26570
|
-
if (!validToken(
|
|
26803
|
+
const token4 = c.req.query("token");
|
|
26804
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26571
26805
|
try {
|
|
26572
26806
|
const requested = c.req.query("vault");
|
|
26573
26807
|
const vaults = isVault(requested) ? [requested] : VAULTS;
|
|
26574
26808
|
const results = [];
|
|
26575
|
-
for (const vault of vaults) results.push(await readVault(vault,
|
|
26809
|
+
for (const vault of vaults) results.push(await readVault(vault, token4));
|
|
26576
26810
|
const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
26577
26811
|
return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
|
|
26578
26812
|
} catch (error) {
|
|
@@ -26581,8 +26815,8 @@ vaultApp.get("/api/workspace", async (c) => {
|
|
|
26581
26815
|
});
|
|
26582
26816
|
vaultApp.post("/api/records", async (c) => {
|
|
26583
26817
|
const body = await c.req.json().catch(() => ({}));
|
|
26584
|
-
const
|
|
26585
|
-
if (!validToken(
|
|
26818
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26819
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26586
26820
|
const vault = body.vault;
|
|
26587
26821
|
const path5 = stringValue(body.path, 320);
|
|
26588
26822
|
const title = stringValue(body.title, 320);
|
|
@@ -26593,19 +26827,19 @@ vaultApp.post("/api/records", async (c) => {
|
|
|
26593
26827
|
const validationError = validateNewRecord(vault, normalizedProps);
|
|
26594
26828
|
if (validationError) return c.json({ ok: false, error: validationError }, 400);
|
|
26595
26829
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26596
|
-
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } },
|
|
26830
|
+
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token4);
|
|
26597
26831
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26598
26832
|
});
|
|
26599
26833
|
vaultApp.post("/api/records/transition", async (c) => {
|
|
26600
26834
|
const body = await c.req.json().catch(() => ({}));
|
|
26601
|
-
const
|
|
26602
|
-
if (!validToken(
|
|
26835
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26836
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26603
26837
|
const vault = typeof body.vault === "string" ? body.vault : "";
|
|
26604
26838
|
const path5 = typeof body.path === "string" ? body.path : "";
|
|
26605
26839
|
const status = stringValue(body.status, 80);
|
|
26606
26840
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26607
26841
|
if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
|
|
26608
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26842
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26609
26843
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26610
26844
|
const result = await memoryCall("putTool", {
|
|
26611
26845
|
vault,
|
|
@@ -26614,13 +26848,13 @@ vaultApp.post("/api/records/transition", async (c) => {
|
|
|
26614
26848
|
content: current.note.content,
|
|
26615
26849
|
props: { ...current.note.props ?? {}, status },
|
|
26616
26850
|
baseRevision: baseRevision ?? current.note.revision
|
|
26617
|
-
},
|
|
26851
|
+
}, token4);
|
|
26618
26852
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26619
26853
|
});
|
|
26620
26854
|
vaultApp.post("/api/records/update", async (c) => {
|
|
26621
26855
|
const body = await c.req.json().catch(() => ({}));
|
|
26622
|
-
const
|
|
26623
|
-
if (!validToken(
|
|
26856
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26857
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26624
26858
|
const vault = body.vault;
|
|
26625
26859
|
const path5 = stringValue(body.path, 320);
|
|
26626
26860
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -26630,7 +26864,7 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
26630
26864
|
if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
|
|
26631
26865
|
return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
|
|
26632
26866
|
}
|
|
26633
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26867
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26634
26868
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26635
26869
|
const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
|
|
26636
26870
|
const validationError = validateNewRecord(vault, nextProps);
|
|
@@ -26642,18 +26876,18 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
26642
26876
|
content: content ?? current.note.content,
|
|
26643
26877
|
props: nextProps,
|
|
26644
26878
|
baseRevision: baseRevision ?? current.note.revision
|
|
26645
|
-
},
|
|
26879
|
+
}, token4);
|
|
26646
26880
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26647
26881
|
});
|
|
26648
26882
|
vaultApp.delete("/api/records", async (c) => {
|
|
26649
26883
|
const body = await c.req.json().catch(() => ({}));
|
|
26650
|
-
const
|
|
26651
|
-
if (!validToken(
|
|
26884
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26885
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26652
26886
|
const vault = body.vault;
|
|
26653
26887
|
const path5 = stringValue(body.path, 320);
|
|
26654
26888
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26655
26889
|
if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
|
|
26656
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26890
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26657
26891
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26658
26892
|
if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
|
|
26659
26893
|
return c.json({ ok: false, error: "only task records can be deleted" }, 400);
|
|
@@ -26662,13 +26896,13 @@ vaultApp.delete("/api/records", async (c) => {
|
|
|
26662
26896
|
vault,
|
|
26663
26897
|
path: path5,
|
|
26664
26898
|
baseRevision: baseRevision ?? current.note.revision
|
|
26665
|
-
},
|
|
26899
|
+
}, token4);
|
|
26666
26900
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26667
26901
|
});
|
|
26668
26902
|
vaultApp.post("/api/communications/review", async (c) => {
|
|
26669
26903
|
const body = await c.req.json().catch(() => ({}));
|
|
26670
|
-
const
|
|
26671
|
-
if (!validToken(
|
|
26904
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26905
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26672
26906
|
const path5 = stringValue(body.path, 320);
|
|
26673
26907
|
const action = stringValue(body.action, 24);
|
|
26674
26908
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -26679,7 +26913,7 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
26679
26913
|
if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
|
|
26680
26914
|
return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
|
|
26681
26915
|
}
|
|
26682
|
-
const current = await readMemory("getTool", { vault: "Communications", path: path5 },
|
|
26916
|
+
const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token4);
|
|
26683
26917
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26684
26918
|
if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
|
|
26685
26919
|
const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
|
|
@@ -26703,13 +26937,13 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
26703
26937
|
content: content ?? current.note.content,
|
|
26704
26938
|
props,
|
|
26705
26939
|
baseRevision: baseRevision ?? current.note.revision
|
|
26706
|
-
},
|
|
26940
|
+
}, token4);
|
|
26707
26941
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26708
26942
|
});
|
|
26709
26943
|
vaultApp.post("/api/activity", async (c) => {
|
|
26710
26944
|
const body = await c.req.json().catch(() => ({}));
|
|
26711
|
-
const
|
|
26712
|
-
if (!validToken(
|
|
26945
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26946
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26713
26947
|
const title = stringValue(body.title, 320);
|
|
26714
26948
|
const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
|
|
26715
26949
|
const props = propsValue(body.props) ?? {};
|
|
@@ -26719,7 +26953,7 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
26719
26953
|
return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
|
|
26720
26954
|
}
|
|
26721
26955
|
for (const ref of personRefs) {
|
|
26722
|
-
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") },
|
|
26956
|
+
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token4);
|
|
26723
26957
|
if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
|
|
26724
26958
|
return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
|
|
26725
26959
|
}
|
|
@@ -26747,37 +26981,37 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
26747
26981
|
...props,
|
|
26748
26982
|
person_refs: personRefs
|
|
26749
26983
|
}
|
|
26750
|
-
},
|
|
26984
|
+
}, token4);
|
|
26751
26985
|
return c.json(result, result.ok ? 200 : 502);
|
|
26752
26986
|
});
|
|
26753
26987
|
|
|
26754
26988
|
// src/api/schedule-routes.ts
|
|
26755
26989
|
import { Hono as Hono26 } from "hono";
|
|
26756
26990
|
var scheduleApp = new Hono26();
|
|
26757
|
-
function tokenOk2(
|
|
26758
|
-
return !!
|
|
26991
|
+
function tokenOk2(token4) {
|
|
26992
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 10;
|
|
26759
26993
|
}
|
|
26760
26994
|
function jsonError2(message, status = 400) {
|
|
26761
26995
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
26762
26996
|
}
|
|
26763
26997
|
scheduleApp.get("/api/status", async (c) => {
|
|
26764
|
-
const
|
|
26765
|
-
if (!tokenOk2(
|
|
26766
|
-
const r = await memoryCall("getScheduleStatusTool", {},
|
|
26998
|
+
const token4 = c.req.query("token");
|
|
26999
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
27000
|
+
const r = await memoryCall("getScheduleStatusTool", {}, token4);
|
|
26767
27001
|
return c.json(r);
|
|
26768
27002
|
});
|
|
26769
27003
|
scheduleApp.get("/api/vaults", async (c) => {
|
|
26770
|
-
const
|
|
26771
|
-
if (!tokenOk2(
|
|
26772
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
27004
|
+
const token4 = c.req.query("token");
|
|
27005
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
27006
|
+
const r = await memoryCall("listVaultsTool", {}, token4);
|
|
26773
27007
|
if (!r.ok) return c.json(r);
|
|
26774
27008
|
const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
|
|
26775
27009
|
return c.json({ ok: true, vaults });
|
|
26776
27010
|
});
|
|
26777
27011
|
scheduleApp.get("/api/actions", async (c) => {
|
|
26778
|
-
const
|
|
26779
|
-
if (!tokenOk2(
|
|
26780
|
-
const r = await memoryCall("listScheduledActionsTool", {},
|
|
27012
|
+
const token4 = c.req.query("token");
|
|
27013
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
27014
|
+
const r = await memoryCall("listScheduledActionsTool", {}, token4);
|
|
26781
27015
|
return c.json(r);
|
|
26782
27016
|
});
|
|
26783
27017
|
scheduleApp.post("/api/propose", async (c) => {
|
|
@@ -26813,13 +27047,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
|
|
|
26813
27047
|
return c.json(r);
|
|
26814
27048
|
});
|
|
26815
27049
|
scheduleApp.get("/:token", (c) => {
|
|
26816
|
-
const
|
|
26817
|
-
if (!tokenOk2(
|
|
27050
|
+
const token4 = c.req.param("token");
|
|
27051
|
+
if (!tokenOk2(token4)) {
|
|
26818
27052
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
|
|
26819
27053
|
}
|
|
26820
|
-
return c.html(renderSchedulePage(
|
|
27054
|
+
return c.html(renderSchedulePage(token4));
|
|
26821
27055
|
});
|
|
26822
|
-
function renderSchedulePage(
|
|
27056
|
+
function renderSchedulePage(token4) {
|
|
26823
27057
|
return `<!doctype html>
|
|
26824
27058
|
<html>
|
|
26825
27059
|
<head>
|
|
@@ -26917,7 +27151,7 @@ function renderSchedulePage(token3) {
|
|
|
26917
27151
|
</div>
|
|
26918
27152
|
|
|
26919
27153
|
<script>
|
|
26920
|
-
const TOKEN = ${JSON.stringify(
|
|
27154
|
+
const TOKEN = ${JSON.stringify(token4)};
|
|
26921
27155
|
const state = { actions: [], vaults: [] };
|
|
26922
27156
|
|
|
26923
27157
|
function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
|
|
@@ -27160,8 +27394,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
|
|
|
27160
27394
|
return shortHash(`${customerEmail.trim().toLowerCase()}
|
|
27161
27395
|
${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
|
|
27162
27396
|
}
|
|
27163
|
-
function supportThreadAddress(
|
|
27164
|
-
const safeToken =
|
|
27397
|
+
function supportThreadAddress(token4, receivingDomain) {
|
|
27398
|
+
const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
27165
27399
|
const domain = receivingDomain.trim().toLowerCase();
|
|
27166
27400
|
if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
|
|
27167
27401
|
return `support+${safeToken}@${domain}`;
|
|
@@ -27188,8 +27422,8 @@ function personPath(identity) {
|
|
|
27188
27422
|
const local = identity.email.split("@")[0];
|
|
27189
27423
|
return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
|
|
27190
27424
|
}
|
|
27191
|
-
function issuePath(
|
|
27192
|
-
return `support/threads/${
|
|
27425
|
+
function issuePath(token4) {
|
|
27426
|
+
return `support/threads/${token4}`;
|
|
27193
27427
|
}
|
|
27194
27428
|
function communicationPath(email) {
|
|
27195
27429
|
const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
|
|
@@ -27373,8 +27607,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
|
|
|
27373
27607
|
`- Person: ${noteRef("People", personPathValue)}`
|
|
27374
27608
|
].join("\n");
|
|
27375
27609
|
}
|
|
27376
|
-
async function findOrCreateIssue(memoryKey, email, customer, personPathValue,
|
|
27377
|
-
const path5 = issuePath(
|
|
27610
|
+
async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token4, threadAddressValue) {
|
|
27611
|
+
const path5 = issuePath(token4);
|
|
27378
27612
|
const existing = await getNote(memoryKey, "Issues", path5);
|
|
27379
27613
|
if (existing) return existing;
|
|
27380
27614
|
const subject = normalizeSupportSubject(email.subject);
|
|
@@ -27384,7 +27618,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27384
27618
|
path: path5,
|
|
27385
27619
|
title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
|
|
27386
27620
|
content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
|
|
27387
|
-
source: `resend:thread:${
|
|
27621
|
+
source: `resend:thread:${token4}`,
|
|
27388
27622
|
type: "user_friction",
|
|
27389
27623
|
tagCandidates: ISSUE_TAGS,
|
|
27390
27624
|
props: {
|
|
@@ -27393,7 +27627,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27393
27627
|
summary,
|
|
27394
27628
|
pinned: false,
|
|
27395
27629
|
source_type: "thread",
|
|
27396
|
-
source_ref: `resend:thread:${
|
|
27630
|
+
source_ref: `resend:thread:${token4}`,
|
|
27397
27631
|
related: [],
|
|
27398
27632
|
related_vault_notes: [noteRef("People", personPathValue)],
|
|
27399
27633
|
embed: true,
|
|
@@ -27408,7 +27642,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27408
27642
|
customer_email: customer.email,
|
|
27409
27643
|
customer_name: customer.name,
|
|
27410
27644
|
normalized_subject: subject,
|
|
27411
|
-
thread_token:
|
|
27645
|
+
thread_token: token4,
|
|
27412
27646
|
thread_address: threadAddressValue,
|
|
27413
27647
|
communication_refs: [],
|
|
27414
27648
|
message_count: 0,
|
|
@@ -27451,7 +27685,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
|
|
|
27451
27685
|
...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
|
|
27452
27686
|
].join("\n");
|
|
27453
27687
|
}
|
|
27454
|
-
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue,
|
|
27688
|
+
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token4, parentPath) {
|
|
27455
27689
|
const path5 = communicationPath(email);
|
|
27456
27690
|
const existing = await getNote(memoryKey, "Communication", path5);
|
|
27457
27691
|
if (existing) return existing;
|
|
@@ -27490,8 +27724,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
|
|
|
27490
27724
|
provider: "resend",
|
|
27491
27725
|
provider_connection_ref: "resend:updates.mcpscraper.dev",
|
|
27492
27726
|
provider_message_ref: providerMessageRef,
|
|
27493
|
-
provider_thread_ref:
|
|
27494
|
-
thread_ref:
|
|
27727
|
+
provider_thread_ref: token4,
|
|
27728
|
+
thread_ref: token4,
|
|
27495
27729
|
issue_ref: issuePathValue,
|
|
27496
27730
|
sender_email: supportSender(email)?.email ?? email.from,
|
|
27497
27731
|
recipient_emails: email.to ?? [],
|
|
@@ -27551,8 +27785,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
|
|
|
27551
27785
|
if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
|
|
27552
27786
|
}
|
|
27553
27787
|
}
|
|
27554
|
-
async function getSupportThreadByToken(memoryKey,
|
|
27555
|
-
const safeToken =
|
|
27788
|
+
async function getSupportThreadByToken(memoryKey, token4) {
|
|
27789
|
+
const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
27556
27790
|
if (safeToken.length !== 24) return null;
|
|
27557
27791
|
const path5 = issuePath(safeToken);
|
|
27558
27792
|
const issue = await getNote(memoryKey, "Issues", path5);
|
|
@@ -27580,8 +27814,8 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27580
27814
|
const customer = options.customer ?? supportSender(email);
|
|
27581
27815
|
if (!customer) throw new Error("support sender does not contain a usable email address");
|
|
27582
27816
|
const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
|
|
27583
|
-
const
|
|
27584
|
-
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(
|
|
27817
|
+
const token4 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
|
|
27818
|
+
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token4, options.receivingDomain);
|
|
27585
27819
|
const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
|
|
27586
27820
|
if (!person) throw new Error("support person could not be resolved");
|
|
27587
27821
|
const issue = options.existingThread?.issue ?? await findOrCreateIssue(
|
|
@@ -27589,7 +27823,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27589
27823
|
email,
|
|
27590
27824
|
customer,
|
|
27591
27825
|
person.path,
|
|
27592
|
-
|
|
27826
|
+
token4,
|
|
27593
27827
|
threadAddressValue
|
|
27594
27828
|
);
|
|
27595
27829
|
const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
|
|
@@ -27600,7 +27834,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27600
27834
|
options.direction,
|
|
27601
27835
|
person.path,
|
|
27602
27836
|
issue.path,
|
|
27603
|
-
|
|
27837
|
+
token4,
|
|
27604
27838
|
parentPath
|
|
27605
27839
|
);
|
|
27606
27840
|
await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
|
|
@@ -27609,7 +27843,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27609
27843
|
communicationPath: communication.path,
|
|
27610
27844
|
personPath: person.path,
|
|
27611
27845
|
parentCommunicationPath: parentPath,
|
|
27612
|
-
threadToken:
|
|
27846
|
+
threadToken: token4,
|
|
27613
27847
|
threadAddress: threadAddressValue,
|
|
27614
27848
|
customerEmail: customer.email,
|
|
27615
27849
|
normalizedSubject,
|
|
@@ -27782,8 +28016,8 @@ resendInboundApp.post("/webhooks", async (c) => {
|
|
|
27782
28016
|
threadAddress: captured.threadAddress
|
|
27783
28017
|
});
|
|
27784
28018
|
}
|
|
27785
|
-
const
|
|
27786
|
-
const thread =
|
|
28019
|
+
const token4 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
|
|
28020
|
+
const thread = token4 ? await getSupportThreadByToken(memoryKey, token4) : null;
|
|
27787
28021
|
if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
|
|
27788
28022
|
const operator = parseMailbox(destination);
|
|
27789
28023
|
if (operator && sender.email === operator.email) {
|
|
@@ -28090,7 +28324,7 @@ function renderWorkshopFieldNotesBody() {
|
|
|
28090
28324
|
}
|
|
28091
28325
|
|
|
28092
28326
|
// src/api/workshop-registration-repository.ts
|
|
28093
|
-
import { randomUUID as
|
|
28327
|
+
import { randomUUID as randomUUID13 } from "crypto";
|
|
28094
28328
|
function rowToRegistration(row) {
|
|
28095
28329
|
return {
|
|
28096
28330
|
id: String(row.id),
|
|
@@ -28159,7 +28393,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
28159
28393
|
) < ?
|
|
28160
28394
|
`,
|
|
28161
28395
|
args: [
|
|
28162
|
-
|
|
28396
|
+
randomUUID13(),
|
|
28163
28397
|
input.eventSlug,
|
|
28164
28398
|
input.email,
|
|
28165
28399
|
input.firstName,
|
|
@@ -28176,7 +28410,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
28176
28410
|
id, event_slug, email, first_name, last_name, status
|
|
28177
28411
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
28178
28412
|
`,
|
|
28179
|
-
args: [
|
|
28413
|
+
args: [randomUUID13(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
28180
28414
|
});
|
|
28181
28415
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
28182
28416
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -28191,7 +28425,7 @@ async function registerWorkshopInterest(input) {
|
|
|
28191
28425
|
id, event_slug, email, first_name, last_name, status
|
|
28192
28426
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
28193
28427
|
`,
|
|
28194
|
-
args: [
|
|
28428
|
+
args: [randomUUID13(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
28195
28429
|
});
|
|
28196
28430
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
28197
28431
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -28629,16 +28863,16 @@ import { z as z29 } from "zod";
|
|
|
28629
28863
|
// src/editorial-reading-room/render.ts
|
|
28630
28864
|
import { createHash as createHash10 } from "crypto";
|
|
28631
28865
|
import { readFileSync as readFileSync3 } from "fs";
|
|
28632
|
-
import { dirname as
|
|
28866
|
+
import { dirname as dirname3, join as join6, resolve } from "path";
|
|
28633
28867
|
import { Marked, Renderer } from "marked";
|
|
28634
|
-
var runtimeEntryDir =
|
|
28868
|
+
var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
|
|
28635
28869
|
var assetCache = /* @__PURE__ */ new Map();
|
|
28636
28870
|
function assetPath(name) {
|
|
28637
28871
|
const candidates = [
|
|
28638
|
-
|
|
28639
|
-
|
|
28640
|
-
|
|
28641
|
-
|
|
28872
|
+
join6(runtimeEntryDir, "editorial-reading-room", "assets", name),
|
|
28873
|
+
join6(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
|
|
28874
|
+
join6(process.cwd(), "src", "editorial-reading-room", "assets", name),
|
|
28875
|
+
join6(process.cwd(), "dist", "editorial-reading-room", "assets", name)
|
|
28642
28876
|
];
|
|
28643
28877
|
for (const candidate of candidates) {
|
|
28644
28878
|
try {
|
|
@@ -28650,8 +28884,8 @@ function assetPath(name) {
|
|
|
28650
28884
|
throw new Error(`Editorial reading-room template asset is missing: ${name}`);
|
|
28651
28885
|
}
|
|
28652
28886
|
function readAsset(name) {
|
|
28653
|
-
const
|
|
28654
|
-
if (
|
|
28887
|
+
const cached2 = assetCache.get(name);
|
|
28888
|
+
if (cached2 !== void 0) return cached2;
|
|
28655
28889
|
const value = readFileSync3(assetPath(name), "utf8");
|
|
28656
28890
|
assetCache.set(name, value);
|
|
28657
28891
|
return value;
|
|
@@ -28851,20 +29085,20 @@ ${provenance}`;
|
|
|
28851
29085
|
var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
|
|
28852
29086
|
var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
28853
29087
|
var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
28854
|
-
function
|
|
29088
|
+
function token2() {
|
|
28855
29089
|
return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
28856
29090
|
}
|
|
28857
|
-
function
|
|
29091
|
+
function policy2() {
|
|
28858
29092
|
return {
|
|
28859
29093
|
prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
|
|
28860
29094
|
artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
|
|
28861
29095
|
downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
|
|
28862
|
-
token:
|
|
29096
|
+
token: token2()
|
|
28863
29097
|
};
|
|
28864
29098
|
}
|
|
28865
29099
|
function createEditorialReadingRoomArtifact(args) {
|
|
28866
29100
|
return createPrivateArtifact({
|
|
28867
|
-
policy:
|
|
29101
|
+
policy: policy2(),
|
|
28868
29102
|
ownerId: args.ownerId,
|
|
28869
29103
|
artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
|
|
28870
29104
|
createdAt: args.rendered.generatedAt,
|
|
@@ -28875,7 +29109,7 @@ function createEditorialReadingRoomArtifact(args) {
|
|
|
28875
29109
|
}
|
|
28876
29110
|
function renewEditorialReadingRoomArtifactDownload(args) {
|
|
28877
29111
|
return renewPrivateArtifactDownload({
|
|
28878
|
-
policy:
|
|
29112
|
+
policy: policy2(),
|
|
28879
29113
|
ownerId: args.ownerId,
|
|
28880
29114
|
artifactId: args.artifactId
|
|
28881
29115
|
});
|
|
@@ -28972,7 +29206,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
28972
29206
|
}
|
|
28973
29207
|
|
|
28974
29208
|
// src/api/scheduled-run-view-links.ts
|
|
28975
|
-
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as
|
|
29209
|
+
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID14 } from "crypto";
|
|
28976
29210
|
var schemaReady2 = false;
|
|
28977
29211
|
async function ensureScheduledRunViewLinksSchema() {
|
|
28978
29212
|
if (schemaReady2) return;
|
|
@@ -29002,8 +29236,8 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
29002
29236
|
`);
|
|
29003
29237
|
schemaReady2 = true;
|
|
29004
29238
|
}
|
|
29005
|
-
function tokenHash2(
|
|
29006
|
-
return createHash12("sha256").update(
|
|
29239
|
+
function tokenHash2(token4) {
|
|
29240
|
+
return createHash12("sha256").update(token4).digest("hex");
|
|
29007
29241
|
}
|
|
29008
29242
|
function mapRow(row) {
|
|
29009
29243
|
return {
|
|
@@ -29023,16 +29257,16 @@ function mapRow(row) {
|
|
|
29023
29257
|
async function createScheduledRunViewLink(input) {
|
|
29024
29258
|
await ensureScheduledRunViewLinksSchema();
|
|
29025
29259
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
29026
|
-
const
|
|
29260
|
+
const token4 = randomBytes3(32).toString("base64url");
|
|
29027
29261
|
const record = {
|
|
29028
|
-
shareId:
|
|
29262
|
+
shareId: randomUUID14(),
|
|
29029
29263
|
ownerId: input.ownerId,
|
|
29030
29264
|
runId: input.runId,
|
|
29031
29265
|
artifactId: input.artifactId,
|
|
29032
29266
|
title: input.title.slice(0, 300),
|
|
29033
29267
|
summary: input.summary.slice(0, 2e3),
|
|
29034
29268
|
rendererKey: input.rendererKey,
|
|
29035
|
-
tokenHash: tokenHash2(
|
|
29269
|
+
tokenHash: tokenHash2(token4),
|
|
29036
29270
|
expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
|
|
29037
29271
|
revokedAt: null,
|
|
29038
29272
|
createdAt: now.toISOString()
|
|
@@ -29057,9 +29291,9 @@ async function createScheduledRunViewLink(input) {
|
|
|
29057
29291
|
record.createdAt
|
|
29058
29292
|
]
|
|
29059
29293
|
});
|
|
29060
|
-
return { record, token:
|
|
29294
|
+
return { record, token: token4 };
|
|
29061
29295
|
}
|
|
29062
|
-
async function resolveScheduledRunViewLink(
|
|
29296
|
+
async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Date()) {
|
|
29063
29297
|
await ensureScheduledRunViewLinksSchema();
|
|
29064
29298
|
const result = await getDb().execute({
|
|
29065
29299
|
sql: `
|
|
@@ -29071,7 +29305,7 @@ async function resolveScheduledRunViewLink(token3, now = /* @__PURE__ */ new Dat
|
|
|
29071
29305
|
AND expires_at > ?
|
|
29072
29306
|
LIMIT 1
|
|
29073
29307
|
`,
|
|
29074
|
-
args: [tokenHash2(
|
|
29308
|
+
args: [tokenHash2(token4), now.toISOString()]
|
|
29075
29309
|
});
|
|
29076
29310
|
const row = result.rows[0];
|
|
29077
29311
|
return row ? mapRow(row) : null;
|
|
@@ -29225,7 +29459,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
29225
29459
|
const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
|
|
29226
29460
|
if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
|
|
29227
29461
|
const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
|
|
29228
|
-
const { record, token:
|
|
29462
|
+
const { record, token: token4 } = await createScheduledRunViewLink({
|
|
29229
29463
|
ownerId,
|
|
29230
29464
|
runId: result.data.run.runId,
|
|
29231
29465
|
artifactId: artifact.artifactId,
|
|
@@ -29240,7 +29474,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
29240
29474
|
shareId: record.shareId,
|
|
29241
29475
|
runId: record.runId,
|
|
29242
29476
|
artifactId: record.artifactId,
|
|
29243
|
-
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${
|
|
29477
|
+
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token4}`,
|
|
29244
29478
|
expiresAt: record.expiresAt,
|
|
29245
29479
|
createdAt: record.createdAt
|
|
29246
29480
|
}, 201);
|
|
@@ -29266,15 +29500,15 @@ import { createHash as createHash13 } from "crypto";
|
|
|
29266
29500
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
29267
29501
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
29268
29502
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
29269
|
-
function
|
|
29503
|
+
function token3() {
|
|
29270
29504
|
return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
29271
29505
|
}
|
|
29272
|
-
function
|
|
29506
|
+
function policy3() {
|
|
29273
29507
|
return {
|
|
29274
29508
|
prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
|
|
29275
29509
|
artifactTtlMs: null,
|
|
29276
29510
|
downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
|
|
29277
|
-
token:
|
|
29511
|
+
token: token3()
|
|
29278
29512
|
};
|
|
29279
29513
|
}
|
|
29280
29514
|
function runStorageSegment(runId) {
|
|
@@ -29285,7 +29519,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
29285
29519
|
throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
|
|
29286
29520
|
}
|
|
29287
29521
|
const stored = await createPrivateArtifact({
|
|
29288
|
-
policy:
|
|
29522
|
+
policy: policy3(),
|
|
29289
29523
|
ownerId: args.request.ownerId,
|
|
29290
29524
|
scopeSegments: [runStorageSegment(args.request.runId)],
|
|
29291
29525
|
artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
|
|
@@ -29311,7 +29545,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
29311
29545
|
async function readScheduledRunArtifact(args) {
|
|
29312
29546
|
if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
|
|
29313
29547
|
const window2 = await readPrivateArtifactWindow({
|
|
29314
|
-
policy:
|
|
29548
|
+
policy: policy3(),
|
|
29315
29549
|
artifactId: args.artifactId,
|
|
29316
29550
|
offset: 0,
|
|
29317
29551
|
maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
|
|
@@ -29388,7 +29622,7 @@ import { Hono as Hono32 } from "hono";
|
|
|
29388
29622
|
import { createHash as createHash14, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
29389
29623
|
|
|
29390
29624
|
// src/api/service-connections.ts
|
|
29391
|
-
import { randomUUID as
|
|
29625
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
29392
29626
|
var schemaReady3 = null;
|
|
29393
29627
|
var schemaDb2 = null;
|
|
29394
29628
|
function ensureServiceConnectionsSchema() {
|
|
@@ -29454,6 +29688,14 @@ function ensureServiceConnectionsSchema() {
|
|
|
29454
29688
|
UNIQUE(user_id, request_id)
|
|
29455
29689
|
)
|
|
29456
29690
|
`);
|
|
29691
|
+
try {
|
|
29692
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
|
|
29693
|
+
} catch {
|
|
29694
|
+
}
|
|
29695
|
+
try {
|
|
29696
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
|
|
29697
|
+
} catch {
|
|
29698
|
+
}
|
|
29457
29699
|
})().catch((error) => {
|
|
29458
29700
|
schemaReady3 = null;
|
|
29459
29701
|
schemaDb2 = null;
|
|
@@ -29522,7 +29764,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
29522
29764
|
updated_at = excluded.updated_at
|
|
29523
29765
|
`,
|
|
29524
29766
|
args: [
|
|
29525
|
-
|
|
29767
|
+
randomUUID15(),
|
|
29526
29768
|
userId,
|
|
29527
29769
|
connection.providerConfigKey,
|
|
29528
29770
|
connection.provider,
|
|
@@ -29593,7 +29835,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
29593
29835
|
});
|
|
29594
29836
|
await getDb().execute({
|
|
29595
29837
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
29596
|
-
args: [
|
|
29838
|
+
args: [randomUUID15(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
29597
29839
|
});
|
|
29598
29840
|
}
|
|
29599
29841
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -29623,8 +29865,8 @@ async function recordServiceConnectionAction(args) {
|
|
|
29623
29865
|
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
29624
29866
|
if (!connection) throw new Error("service_connection_not_found");
|
|
29625
29867
|
await getDb().execute({
|
|
29626
|
-
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
29627
|
-
args: [args.status, args.errorCode ?? null, connection.userId, args.requestId, connection.id]
|
|
29868
|
+
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
29869
|
+
args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
|
|
29628
29870
|
});
|
|
29629
29871
|
}
|
|
29630
29872
|
async function claimServiceConnectionAction(args) {
|
|
@@ -29632,18 +29874,30 @@ async function claimServiceConnectionAction(args) {
|
|
|
29632
29874
|
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
29633
29875
|
if (!connection) throw new Error("service_connection_not_found");
|
|
29634
29876
|
const inserted = await getDb().execute({
|
|
29635
|
-
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status) VALUES (?, ?, ?, ?, ?, 'started')`,
|
|
29636
|
-
args: [
|
|
29877
|
+
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
29878
|
+
args: [randomUUID15(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
29637
29879
|
});
|
|
29638
29880
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
29639
29881
|
const existing = await getDb().execute({
|
|
29640
|
-
sql: `SELECT status, error_code FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
29882
|
+
sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
29641
29883
|
args: [connection.userId, args.requestId]
|
|
29642
29884
|
});
|
|
29885
|
+
const row = existing.rows[0];
|
|
29886
|
+
const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
|
|
29887
|
+
let result = void 0;
|
|
29888
|
+
if (!conflict && typeof row?.result_json === "string") {
|
|
29889
|
+
try {
|
|
29890
|
+
result = JSON.parse(row.result_json);
|
|
29891
|
+
} catch {
|
|
29892
|
+
result = void 0;
|
|
29893
|
+
}
|
|
29894
|
+
}
|
|
29643
29895
|
return {
|
|
29644
29896
|
claimed: false,
|
|
29645
|
-
status: String(
|
|
29646
|
-
errorCode: typeof
|
|
29897
|
+
status: String(row?.status ?? "unknown"),
|
|
29898
|
+
errorCode: typeof row?.error_code === "string" ? row.error_code : null,
|
|
29899
|
+
conflict,
|
|
29900
|
+
...result !== void 0 ? { result } : {}
|
|
29647
29901
|
};
|
|
29648
29902
|
}
|
|
29649
29903
|
|
|
@@ -30066,6 +30320,53 @@ async function persistScrapeBody(user, opts) {
|
|
|
30066
30320
|
return { ...deposit, ...fallback };
|
|
30067
30321
|
}
|
|
30068
30322
|
|
|
30323
|
+
// src/api/scrape-image-sink.ts
|
|
30324
|
+
import { createHash as createHash16 } from "crypto";
|
|
30325
|
+
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
30326
|
+
function idempotencyKey2(userId, vault, input) {
|
|
30327
|
+
return `scrape-image-${createHash16("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
30328
|
+
}
|
|
30329
|
+
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
30330
|
+
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
30331
|
+
if (selected.length === 0) return { requested: 0, saved: 0, skipped: 0, assets: [] };
|
|
30332
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
30333
|
+
if (!key) {
|
|
30334
|
+
return { requested: selected.length, saved: 0, skipped: inputs.length - selected.length, assets: [], error: error ?? "memory unavailable" };
|
|
30335
|
+
}
|
|
30336
|
+
const targetVault = vault?.trim() || "Library";
|
|
30337
|
+
const assets = [];
|
|
30338
|
+
for (const input of selected) {
|
|
30339
|
+
const result = await memoryCall(
|
|
30340
|
+
"image_asset_save",
|
|
30341
|
+
{
|
|
30342
|
+
vault: targetVault,
|
|
30343
|
+
...input.imageBase64 ? { imageBase64: input.imageBase64 } : { sourceUrl: input.imageUrl },
|
|
30344
|
+
title: input.title.slice(0, 240),
|
|
30345
|
+
sourceRef: {
|
|
30346
|
+
sourceUrl: input.sourceUrl,
|
|
30347
|
+
sourceKind: input.sourceKind,
|
|
30348
|
+
...input.sourceRef ?? {}
|
|
30349
|
+
},
|
|
30350
|
+
idempotencyKey: idempotencyKey2(user.id, targetVault, input)
|
|
30351
|
+
},
|
|
30352
|
+
key
|
|
30353
|
+
);
|
|
30354
|
+
assets.push({
|
|
30355
|
+
sourceUrl: input.imageUrl ?? input.sourceUrl,
|
|
30356
|
+
sourceKind: input.sourceKind,
|
|
30357
|
+
saved: result.ok,
|
|
30358
|
+
...result.asset?.assetId ? { assetId: result.asset.assetId } : {},
|
|
30359
|
+
...!result.ok ? { error: result.error ?? "image save failed" } : {}
|
|
30360
|
+
});
|
|
30361
|
+
}
|
|
30362
|
+
return {
|
|
30363
|
+
requested: selected.length,
|
|
30364
|
+
saved: assets.filter((asset) => asset.saved).length,
|
|
30365
|
+
skipped: inputs.length - selected.length,
|
|
30366
|
+
assets
|
|
30367
|
+
};
|
|
30368
|
+
}
|
|
30369
|
+
|
|
30069
30370
|
// src/api/archive-reader.ts
|
|
30070
30371
|
import { basename, extname } from "path";
|
|
30071
30372
|
import yauzl from "yauzl";
|
|
@@ -30156,6 +30457,10 @@ function openZip(buffer) {
|
|
|
30156
30457
|
strictFileNames: true
|
|
30157
30458
|
}, (error, zip) => {
|
|
30158
30459
|
if (error || !zip) {
|
|
30460
|
+
if (error && /invalid relative path|absolute path|backslash/i.test(error.message)) {
|
|
30461
|
+
reject(new ArchiveReadError("archive_unsafe_path", "ZIP archive contains an unsafe entry path."));
|
|
30462
|
+
return;
|
|
30463
|
+
}
|
|
30159
30464
|
reject(new ArchiveReadError("archive_invalid_zip", "The downloaded file is not a valid ZIP archive."));
|
|
30160
30465
|
return;
|
|
30161
30466
|
}
|
|
@@ -30163,6 +30468,13 @@ function openZip(buffer) {
|
|
|
30163
30468
|
});
|
|
30164
30469
|
});
|
|
30165
30470
|
}
|
|
30471
|
+
function normalizedZipError(error, fallbackCode, fallbackMessage) {
|
|
30472
|
+
if (error instanceof ArchiveReadError) return error;
|
|
30473
|
+
if (error instanceof Error && /invalid relative path|absolute path|backslash/i.test(error.message)) {
|
|
30474
|
+
return new ArchiveReadError("archive_unsafe_path", "ZIP archive contains an unsafe entry path.");
|
|
30475
|
+
}
|
|
30476
|
+
return new ArchiveReadError(fallbackCode, error instanceof Error ? error.message : fallbackMessage);
|
|
30477
|
+
}
|
|
30166
30478
|
async function scanZip(buffer, visibleEntryLimit) {
|
|
30167
30479
|
const zip = await openZip(buffer);
|
|
30168
30480
|
return new Promise((resolve2, reject) => {
|
|
@@ -30177,7 +30489,7 @@ async function scanZip(buffer, visibleEntryLimit) {
|
|
|
30177
30489
|
zip.close();
|
|
30178
30490
|
} catch {
|
|
30179
30491
|
}
|
|
30180
|
-
reject(error
|
|
30492
|
+
reject(normalizedZipError(error, "archive_invalid_zip", "Failed to read ZIP archive."));
|
|
30181
30493
|
};
|
|
30182
30494
|
zip.on("error", fail2);
|
|
30183
30495
|
zip.on("entry", (entry) => {
|
|
@@ -30271,7 +30583,7 @@ async function extractEntry(buffer, requestedPath) {
|
|
|
30271
30583
|
zip.close();
|
|
30272
30584
|
} catch {
|
|
30273
30585
|
}
|
|
30274
|
-
reject(error
|
|
30586
|
+
reject(normalizedZipError(error, "archive_entry_read_failed", "Failed to read ZIP entry."));
|
|
30275
30587
|
};
|
|
30276
30588
|
zip.on("error", fail2);
|
|
30277
30589
|
zip.on("entry", (entry) => {
|
|
@@ -30386,6 +30698,9 @@ async function downloadPublicZip(rawUrl) {
|
|
|
30386
30698
|
}
|
|
30387
30699
|
async function listZipArchive(rawUrl, maxEntries) {
|
|
30388
30700
|
const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
|
|
30701
|
+
return listZipArchiveBuffer(buffer, archiveUrl, maxEntries);
|
|
30702
|
+
}
|
|
30703
|
+
async function listZipArchiveBuffer(buffer, archiveUrl, maxEntries) {
|
|
30389
30704
|
const scanned = await scanZip(buffer, maxEntries);
|
|
30390
30705
|
return {
|
|
30391
30706
|
archiveUrl,
|
|
@@ -30426,8 +30741,11 @@ function utf8Window(content, path5, offset, maxBytes) {
|
|
|
30426
30741
|
};
|
|
30427
30742
|
}
|
|
30428
30743
|
async function readZipArchiveFile(rawUrl, rawPath, offset, maxBytes) {
|
|
30429
|
-
const requestedPath = safeEntryPath(rawPath);
|
|
30430
30744
|
const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
|
|
30745
|
+
return readZipArchiveBuffer(buffer, archiveUrl, rawPath, offset, maxBytes);
|
|
30746
|
+
}
|
|
30747
|
+
async function readZipArchiveBuffer(buffer, archiveUrl, rawPath, offset, maxBytes) {
|
|
30748
|
+
const requestedPath = safeEntryPath(rawPath);
|
|
30431
30749
|
const scanned = await scanZip(buffer, 0);
|
|
30432
30750
|
const info = scanned.allEntries.find((entry) => entry.path === requestedPath);
|
|
30433
30751
|
if (!info) throw new ArchiveReadError("archive_entry_not_found", `ZIP entry was not found: ${requestedPath}`, 404);
|
|
@@ -30457,6 +30775,9 @@ async function readZipArchiveFile(rawUrl, rawPath, offset, maxBytes) {
|
|
|
30457
30775
|
};
|
|
30458
30776
|
}
|
|
30459
30777
|
function archiveLibrarySource(archiveUrl, path5) {
|
|
30778
|
+
if (!/^https?:\/\//i.test(archiveUrl)) {
|
|
30779
|
+
return `mcp-artifact:${encodeURIComponent(archiveUrl)}#entry=${encodeURIComponent(path5)}`;
|
|
30780
|
+
}
|
|
30460
30781
|
const source = new URL(archiveUrl);
|
|
30461
30782
|
source.username = "";
|
|
30462
30783
|
source.password = "";
|
|
@@ -30475,7 +30796,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
30475
30796
|
}
|
|
30476
30797
|
|
|
30477
30798
|
// src/api/connection-memory-import.ts
|
|
30478
|
-
import { createHash as
|
|
30799
|
+
import { createHash as createHash17 } from "crypto";
|
|
30479
30800
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
30480
30801
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
30481
30802
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -30505,7 +30826,7 @@ function canonicalJson(value) {
|
|
|
30505
30826
|
return JSON.stringify(value);
|
|
30506
30827
|
}
|
|
30507
30828
|
function sha2562(value) {
|
|
30508
|
-
return
|
|
30829
|
+
return createHash17("sha256").update(value).digest("hex");
|
|
30509
30830
|
}
|
|
30510
30831
|
function sensitiveKey(key) {
|
|
30511
30832
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -30723,17 +31044,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
30723
31044
|
|
|
30724
31045
|
// src/api/scrape-blob-cleanup.ts
|
|
30725
31046
|
import { readdir, stat, unlink } from "fs/promises";
|
|
30726
|
-
import { homedir } from "os";
|
|
30727
|
-
import { join as
|
|
30728
|
-
async function cleanupVercel(
|
|
31047
|
+
import { homedir as homedir2 } from "os";
|
|
31048
|
+
import { join as join7 } from "path";
|
|
31049
|
+
async function cleanupVercel(token4, cutoff) {
|
|
30729
31050
|
const { list, del } = await import("@vercel/blob");
|
|
30730
31051
|
let cursor;
|
|
30731
31052
|
let deleted = 0;
|
|
30732
31053
|
do {
|
|
30733
|
-
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token:
|
|
31054
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token4 });
|
|
30734
31055
|
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
30735
31056
|
if (expired.length) {
|
|
30736
|
-
await del(expired.map((b) => b.url), { token:
|
|
31057
|
+
await del(expired.map((b) => b.url), { token: token4 });
|
|
30737
31058
|
deleted += expired.length;
|
|
30738
31059
|
}
|
|
30739
31060
|
cursor = res.hasMore ? res.cursor : void 0;
|
|
@@ -30741,8 +31062,8 @@ async function cleanupVercel(token3, cutoff) {
|
|
|
30741
31062
|
return { deleted, store: "vercel-blob" };
|
|
30742
31063
|
}
|
|
30743
31064
|
async function cleanupLocal(cutoff) {
|
|
30744
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
30745
|
-
const dir =
|
|
31065
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir2(), "Downloads", "mcp-scraper");
|
|
31066
|
+
const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
30746
31067
|
let deleted = 0;
|
|
30747
31068
|
let entries;
|
|
30748
31069
|
try {
|
|
@@ -30751,7 +31072,7 @@ async function cleanupLocal(cutoff) {
|
|
|
30751
31072
|
return { deleted: 0, store: "local" };
|
|
30752
31073
|
}
|
|
30753
31074
|
for (const name of entries) {
|
|
30754
|
-
const path5 =
|
|
31075
|
+
const path5 = join7(dir, name);
|
|
30755
31076
|
try {
|
|
30756
31077
|
const s = await stat(path5);
|
|
30757
31078
|
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
@@ -30766,15 +31087,15 @@ async function cleanupLocal(cutoff) {
|
|
|
30766
31087
|
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
30767
31088
|
const cutoff = Date.now() - maxAgeMs;
|
|
30768
31089
|
try {
|
|
30769
|
-
const
|
|
30770
|
-
return
|
|
31090
|
+
const token4 = process.env.BLOB_READ_WRITE_TOKEN;
|
|
31091
|
+
return token4 ? await cleanupVercel(token4, cutoff) : await cleanupLocal(cutoff);
|
|
30771
31092
|
} catch {
|
|
30772
31093
|
return { deleted: 0, store: "none" };
|
|
30773
31094
|
}
|
|
30774
31095
|
}
|
|
30775
31096
|
|
|
30776
31097
|
// src/api/connected-data-export.ts
|
|
30777
|
-
import { randomUUID as
|
|
31098
|
+
import { randomUUID as randomUUID16 } from "crypto";
|
|
30778
31099
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
30779
31100
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
30780
31101
|
);
|
|
@@ -30945,7 +31266,7 @@ function finitePositive(value, fallback) {
|
|
|
30945
31266
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
30946
31267
|
}
|
|
30947
31268
|
async function collectConnectedDataExport(args) {
|
|
30948
|
-
const exportId =
|
|
31269
|
+
const exportId = randomUUID16();
|
|
30949
31270
|
const now = args.now ?? Date.now;
|
|
30950
31271
|
const startedAt = now();
|
|
30951
31272
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -31061,7 +31382,7 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
31061
31382
|
}
|
|
31062
31383
|
|
|
31063
31384
|
// src/api/search-console-table-export.ts
|
|
31064
|
-
import { randomUUID as
|
|
31385
|
+
import { randomUUID as randomUUID17 } from "crypto";
|
|
31065
31386
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
31066
31387
|
var SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
31067
31388
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -31161,7 +31482,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
31161
31482
|
offset += rows.length;
|
|
31162
31483
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
31163
31484
|
}
|
|
31164
|
-
const exportId =
|
|
31485
|
+
const exportId = randomUUID17();
|
|
31165
31486
|
const artifact = await args.writeArtifact({
|
|
31166
31487
|
ownerId: args.ownerId,
|
|
31167
31488
|
exportId,
|
|
@@ -31516,7 +31837,7 @@ async function dbUsage(identity, plan) {
|
|
|
31516
31837
|
}
|
|
31517
31838
|
|
|
31518
31839
|
// src/api/nango-control.ts
|
|
31519
|
-
import { createHash as
|
|
31840
|
+
import { createHash as createHash19, randomUUID as randomUUID19 } from "crypto";
|
|
31520
31841
|
|
|
31521
31842
|
// src/api/slack-archive-analysis.ts
|
|
31522
31843
|
function isRecord2(value) {
|
|
@@ -31890,9 +32211,55 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
31890
32211
|
}
|
|
31891
32212
|
|
|
31892
32213
|
// src/api/main-nango-transport.ts
|
|
31893
|
-
import { createHash as
|
|
31894
|
-
import { Client } from "@modelcontextprotocol/
|
|
31895
|
-
|
|
32214
|
+
import { createHash as createHash18, randomUUID as randomUUID18 } from "crypto";
|
|
32215
|
+
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
32216
|
+
|
|
32217
|
+
// src/api/connected-tool-policy.ts
|
|
32218
|
+
function classifyConnectedTool(tool) {
|
|
32219
|
+
if (tool.annotations?.readOnlyHint === true) return "read";
|
|
32220
|
+
if (tool.annotations?.readOnlyHint === false) return "action";
|
|
32221
|
+
return /^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(tool.name) ? "read" : "action";
|
|
32222
|
+
}
|
|
32223
|
+
function normalizedPolicyMap(value) {
|
|
32224
|
+
return new Map(Object.entries(value ?? {}).map(([name, entries]) => [
|
|
32225
|
+
name,
|
|
32226
|
+
[...new Set(entries.map((entry) => entry.trim()).filter(Boolean))].sort()
|
|
32227
|
+
]));
|
|
32228
|
+
}
|
|
32229
|
+
function buildConnectedToolPolicyRegistry(args) {
|
|
32230
|
+
const permissionPolicies = normalizedPolicyMap(args.requiredPermissionsByTool);
|
|
32231
|
+
const featurePolicies = normalizedPolicyMap(args.requiredFeaturesByTool);
|
|
32232
|
+
const granted = new Set(args.grantedPermissions ?? []);
|
|
32233
|
+
const enabled = new Set(args.enabledFeatures ?? []);
|
|
32234
|
+
const adminBlocked = new Set(args.adminBlockedTools ?? []);
|
|
32235
|
+
const policies = /* @__PURE__ */ new Map();
|
|
32236
|
+
for (const tool of args.tools) {
|
|
32237
|
+
const classification = classifyConnectedTool(tool);
|
|
32238
|
+
const requiredPermissions = permissionPolicies.get(tool.name) ?? [];
|
|
32239
|
+
const requiredFeatures = featurePolicies.get(tool.name) ?? [];
|
|
32240
|
+
const missingPermissions = requiredPermissions.filter((permission) => !granted.has(permission));
|
|
32241
|
+
const missingFeatures = requiredFeatures.filter((feature) => !enabled.has(feature));
|
|
32242
|
+
const blockedReason = !args.connectionActive ? "inactive_connection" : adminBlocked.has(tool.name) ? "admin_blocked" : classification === "action" && !args.actionsEnabled ? "actions_disabled" : requiredPermissions.length > 0 && args.permissionVerification === "unavailable" ? "permission_verification_unavailable" : missingPermissions.length > 0 ? "missing_permission" : missingFeatures.length > 0 ? "missing_app_feature" : null;
|
|
32243
|
+
policies.set(tool.name, {
|
|
32244
|
+
name: tool.name,
|
|
32245
|
+
classification,
|
|
32246
|
+
requiredPermissions,
|
|
32247
|
+
requiredFeatures,
|
|
32248
|
+
available: blockedReason === null,
|
|
32249
|
+
blockedReason,
|
|
32250
|
+
missingPermissions,
|
|
32251
|
+
missingFeatures
|
|
32252
|
+
});
|
|
32253
|
+
}
|
|
32254
|
+
return policies;
|
|
32255
|
+
}
|
|
32256
|
+
function connectedToolPolicyOrThrow(registry, toolName) {
|
|
32257
|
+
const policy4 = registry.get(toolName);
|
|
32258
|
+
if (!policy4) throw new Error("connected_tool_policy_missing");
|
|
32259
|
+
return policy4;
|
|
32260
|
+
}
|
|
32261
|
+
|
|
32262
|
+
// src/api/main-nango-transport.ts
|
|
31896
32263
|
var DEFAULT_NANGO_MCP_URL = "https://api.nango.dev/mcp";
|
|
31897
32264
|
var NANGO_TIMEOUT_MS = 2e4;
|
|
31898
32265
|
var NANGO_CONNECTION_PAGE_SIZE = 100;
|
|
@@ -32169,11 +32536,6 @@ async function requireOwnedActiveConnection(identity, connectionId, allowNeedsRe
|
|
|
32169
32536
|
}
|
|
32170
32537
|
return connection;
|
|
32171
32538
|
}
|
|
32172
|
-
function classifyTool(tool) {
|
|
32173
|
-
if (tool.annotations?.readOnlyHint === true) return "read";
|
|
32174
|
-
if (tool.annotations?.readOnlyHint === false) return "action";
|
|
32175
|
-
return /^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(tool.name) ? "read" : "action";
|
|
32176
|
-
}
|
|
32177
32539
|
async function withNangoClient(connection, run) {
|
|
32178
32540
|
const transport = new StreamableHTTPClientTransport(nangoMcpUrl(), {
|
|
32179
32541
|
requestInit: {
|
|
@@ -32235,9 +32597,14 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
32235
32597
|
...asRecord(raw.annotations) ? { annotations: asRecord(raw.annotations) } : {}
|
|
32236
32598
|
}];
|
|
32237
32599
|
});
|
|
32238
|
-
const
|
|
32239
|
-
|
|
32240
|
-
|
|
32600
|
+
const policies = buildConnectedToolPolicyRegistry({
|
|
32601
|
+
tools,
|
|
32602
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32603
|
+
actionsEnabled: connection.actionsEnabled
|
|
32604
|
+
});
|
|
32605
|
+
const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
|
|
32606
|
+
const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
|
|
32607
|
+
const revision = createHash18("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
32241
32608
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
32242
32609
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
32243
32610
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -32246,19 +32613,26 @@ async function callNangoToolDirect(args) {
|
|
|
32246
32613
|
const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
|
|
32247
32614
|
const selected = tools.find((tool) => tool.name === args.tool);
|
|
32248
32615
|
if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
32249
|
-
const
|
|
32250
|
-
|
|
32616
|
+
const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
32617
|
+
tools,
|
|
32618
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32619
|
+
actionsEnabled: connection.actionsEnabled
|
|
32620
|
+
}), selected.name);
|
|
32621
|
+
if (policy4.classification !== args.classification) {
|
|
32251
32622
|
throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
|
|
32252
32623
|
}
|
|
32253
|
-
if (
|
|
32624
|
+
if (!policy4.available && policy4.blockedReason === "actions_disabled") {
|
|
32254
32625
|
throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
|
|
32255
32626
|
}
|
|
32627
|
+
if (!policy4.available) {
|
|
32628
|
+
throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
|
|
32629
|
+
}
|
|
32256
32630
|
await preflightConnectedUsage({
|
|
32257
32631
|
identity: args.identity,
|
|
32258
32632
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
32259
32633
|
});
|
|
32260
|
-
const requestId = args.requestId?.trim() ||
|
|
32261
|
-
const
|
|
32634
|
+
const requestId = args.requestId?.trim() || randomUUID18();
|
|
32635
|
+
const idempotencyKey3 = `main-nango:${createHash18("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
32262
32636
|
const startedAt = /* @__PURE__ */ new Date();
|
|
32263
32637
|
const started = performance.now();
|
|
32264
32638
|
let result;
|
|
@@ -32271,7 +32645,7 @@ async function callNangoToolDirect(args) {
|
|
|
32271
32645
|
const completedAt = /* @__PURE__ */ new Date();
|
|
32272
32646
|
await settleConnectedUsage({
|
|
32273
32647
|
identity: args.identity,
|
|
32274
|
-
idempotencyKey:
|
|
32648
|
+
idempotencyKey: idempotencyKey3,
|
|
32275
32649
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
32276
32650
|
connectionId: connection.id,
|
|
32277
32651
|
providerConfigKey: connection.providerConfigKey,
|
|
@@ -32283,7 +32657,7 @@ async function callNangoToolDirect(args) {
|
|
|
32283
32657
|
toolName: args.tool,
|
|
32284
32658
|
operationKind: args.operationKind ?? args.classification,
|
|
32285
32659
|
outcome: providerError ? "error" : "partial",
|
|
32286
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
32660
|
+
requestId: requestId.length <= 200 ? requestId : createHash18("sha256").update(requestId).digest("hex"),
|
|
32287
32661
|
startedAt: startedAt.toISOString(),
|
|
32288
32662
|
completedAt: completedAt.toISOString()
|
|
32289
32663
|
}
|
|
@@ -32295,8 +32669,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
32295
32669
|
const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
|
|
32296
32670
|
const tool = tools.find((candidate) => candidate.name === toolName);
|
|
32297
32671
|
if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
32298
|
-
const
|
|
32299
|
-
|
|
32672
|
+
const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
32673
|
+
tools,
|
|
32674
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32675
|
+
actionsEnabled: connection.actionsEnabled
|
|
32676
|
+
}), tool.name);
|
|
32677
|
+
const classification = classifyConnectedTool(tool);
|
|
32678
|
+
const callable = policy4.available;
|
|
32300
32679
|
const projected = {
|
|
32301
32680
|
name: tool.name,
|
|
32302
32681
|
...tool.title ? { title: tool.title } : {},
|
|
@@ -32309,18 +32688,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
32309
32688
|
...projected,
|
|
32310
32689
|
classification,
|
|
32311
32690
|
callable,
|
|
32312
|
-
blockedReason:
|
|
32313
|
-
requiredPermissions:
|
|
32314
|
-
missingPermissions:
|
|
32315
|
-
requiredFeatures:
|
|
32316
|
-
missingFeatures:
|
|
32691
|
+
blockedReason: policy4.blockedReason,
|
|
32692
|
+
requiredPermissions: policy4.requiredPermissions,
|
|
32693
|
+
missingPermissions: policy4.missingPermissions,
|
|
32694
|
+
requiredFeatures: policy4.requiredFeatures,
|
|
32695
|
+
missingFeatures: policy4.missingFeatures,
|
|
32317
32696
|
transport: "nango",
|
|
32318
32697
|
providerConfigKey: connection.providerConfigKey,
|
|
32319
32698
|
providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
|
|
32320
32699
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
32321
32700
|
protocolVersion: null,
|
|
32322
32701
|
schemaSource: "live_tools_list",
|
|
32323
|
-
schemaHash:
|
|
32702
|
+
schemaHash: createHash18("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
32324
32703
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
32325
32704
|
};
|
|
32326
32705
|
}
|
|
@@ -32776,8 +33155,8 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32776
33155
|
classification: "read",
|
|
32777
33156
|
requiredPermissions: [],
|
|
32778
33157
|
requiredFeatures: [],
|
|
32779
|
-
available: connection.operationalStatus !== "unavailable",
|
|
32780
|
-
blockedReason: null,
|
|
33158
|
+
available: connection.lifecycleStatus === "connected" && connection.operationalStatus !== "unavailable",
|
|
33159
|
+
blockedReason: connection.lifecycleStatus !== "connected" || connection.operationalStatus === "unavailable" ? "inactive_connection" : null,
|
|
32781
33160
|
missingPermissions: [],
|
|
32782
33161
|
missingFeatures: []
|
|
32783
33162
|
})),
|
|
@@ -32786,8 +33165,8 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32786
33165
|
classification: "action",
|
|
32787
33166
|
requiredPermissions: [],
|
|
32788
33167
|
requiredFeatures: [],
|
|
32789
|
-
available: connection.operationalStatus !== "unavailable",
|
|
32790
|
-
blockedReason: null,
|
|
33168
|
+
available: connection.lifecycleStatus === "connected" && connection.operationalStatus !== "unavailable" && connection.actionsEnabled,
|
|
33169
|
+
blockedReason: connection.lifecycleStatus !== "connected" || connection.operationalStatus === "unavailable" ? "inactive_connection" : connection.actionsEnabled ? null : "actions_disabled",
|
|
32791
33170
|
missingPermissions: [],
|
|
32792
33171
|
missingFeatures: []
|
|
32793
33172
|
}))
|
|
@@ -32841,7 +33220,7 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32841
33220
|
const name = cleanString3(value.name, 200);
|
|
32842
33221
|
const classification = value.classification;
|
|
32843
33222
|
const blockedValue = Object.prototype.hasOwnProperty.call(value, "blockedReason") ? value.blockedReason : value.blocked_reason;
|
|
32844
|
-
const blockedReason = blockedValue === null ? null : blockedValue === "missing_permission" || blockedValue === "permission_policy_missing" || blockedValue === "permission_verification_unavailable" || blockedValue === "missing_app_feature" ? blockedValue : void 0;
|
|
33223
|
+
const blockedReason = blockedValue === null ? null : blockedValue === "missing_permission" || blockedValue === "permission_policy_missing" || blockedValue === "permission_verification_unavailable" || blockedValue === "missing_app_feature" || blockedValue === "inactive_connection" || blockedValue === "actions_disabled" || blockedValue === "admin_blocked" ? blockedValue : void 0;
|
|
32845
33224
|
if (!name || classification !== "read" && classification !== "action" || typeof value.available !== "boolean" || blockedReason === void 0) continue;
|
|
32846
33225
|
toolCapabilities.push({
|
|
32847
33226
|
name,
|
|
@@ -33043,14 +33422,24 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
33043
33422
|
if (!isRecord4(data) || !isRecord4(data.connection)) return enabled;
|
|
33044
33423
|
return data.connection.actionsEnabled === true;
|
|
33045
33424
|
}
|
|
33046
|
-
async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
33047
|
-
const requestId = `main-connected-action:${
|
|
33425
|
+
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
33426
|
+
const requestId = `main-connected-action:${createHash19("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID19()).digest("hex")}`;
|
|
33427
|
+
const requestDigest = createHash19("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
33048
33428
|
if (mainOwnsIntegrations()) {
|
|
33049
33429
|
const selectedTool = tool?.trim();
|
|
33050
33430
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
33051
33431
|
try {
|
|
33052
|
-
const claim = await claimServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId });
|
|
33432
|
+
const claim = await claimServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, requestDigest });
|
|
33053
33433
|
if (!claim.claimed) {
|
|
33434
|
+
if (claim.conflict) {
|
|
33435
|
+
throw new NangoControlError(
|
|
33436
|
+
"This Idempotency-Key was already used for different connected-action arguments.",
|
|
33437
|
+
409,
|
|
33438
|
+
"idempotency_conflict",
|
|
33439
|
+
false
|
|
33440
|
+
);
|
|
33441
|
+
}
|
|
33442
|
+
if (claim.status === "succeeded" && "result" in claim) return claim.result;
|
|
33054
33443
|
throw new NangoControlError(
|
|
33055
33444
|
"The previous connected action may have completed, so it was not run again.",
|
|
33056
33445
|
409,
|
|
@@ -33067,7 +33456,7 @@ async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
|
33067
33456
|
requestId,
|
|
33068
33457
|
operationKind: "action"
|
|
33069
33458
|
});
|
|
33070
|
-
await recordServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, status: "succeeded" });
|
|
33459
|
+
await recordServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, status: "succeeded", result });
|
|
33071
33460
|
return result;
|
|
33072
33461
|
} catch (error) {
|
|
33073
33462
|
const normalized = asNangoControlError(error);
|
|
@@ -33092,7 +33481,7 @@ async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
|
33092
33481
|
const data = unwrapData(body);
|
|
33093
33482
|
return isRecord4(data) ? data.result ?? data : data;
|
|
33094
33483
|
}
|
|
33095
|
-
async function callScheduleConnectionRead(identity, connectionId, tool, args,
|
|
33484
|
+
async function callScheduleConnectionRead(identity, connectionId, tool, args, idempotencyKey3) {
|
|
33096
33485
|
if (mainOwnsIntegrations()) {
|
|
33097
33486
|
try {
|
|
33098
33487
|
return await callNangoToolDirect({
|
|
@@ -33101,7 +33490,7 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args, id
|
|
|
33101
33490
|
tool,
|
|
33102
33491
|
input: args ?? {},
|
|
33103
33492
|
classification: "read",
|
|
33104
|
-
requestId:
|
|
33493
|
+
requestId: idempotencyKey3,
|
|
33105
33494
|
operationKind: "read"
|
|
33106
33495
|
});
|
|
33107
33496
|
} catch (error) {
|
|
@@ -33186,7 +33575,7 @@ function canonicalJson2(value) {
|
|
|
33186
33575
|
return JSON.stringify(value);
|
|
33187
33576
|
}
|
|
33188
33577
|
function projectedToolSchemaHash(tool) {
|
|
33189
|
-
return
|
|
33578
|
+
return createHash19("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
33190
33579
|
}
|
|
33191
33580
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
33192
33581
|
if (mainOwnsIntegrations()) {
|
|
@@ -33465,6 +33854,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
33465
33854
|
}
|
|
33466
33855
|
|
|
33467
33856
|
// src/api/resend-control.ts
|
|
33857
|
+
import { createHash as createHash20 } from "crypto";
|
|
33468
33858
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
33469
33859
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
33470
33860
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -33722,9 +34112,11 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
33722
34112
|
const data = unwrapData2(body);
|
|
33723
34113
|
return isRecord5(data) ? data.result ?? data : data;
|
|
33724
34114
|
}
|
|
33725
|
-
async function callResendAction(identity, connectionId, tool, input) {
|
|
34115
|
+
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
34116
|
+
const requestId = `main-resend-action:${createHash20("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
33726
34117
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
33727
34118
|
method: "POST",
|
|
34119
|
+
headers: { "x-request-id": requestId },
|
|
33728
34120
|
body: JSON.stringify({ identity, connectionId, tool, input })
|
|
33729
34121
|
});
|
|
33730
34122
|
const data = unwrapData2(body);
|
|
@@ -33848,9 +34240,9 @@ var auth2 = createMiddleware4(async (c, next) => {
|
|
|
33848
34240
|
return next();
|
|
33849
34241
|
});
|
|
33850
34242
|
var sessionAuth = createMiddleware4(async (c, next) => {
|
|
33851
|
-
const
|
|
33852
|
-
if (!
|
|
33853
|
-
const userId = verifySession(
|
|
34243
|
+
const token4 = getCookie2(c, "session");
|
|
34244
|
+
if (!token4) return c.json({ error: "Not authenticated" }, 401);
|
|
34245
|
+
const userId = verifySession(token4);
|
|
33854
34246
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
33855
34247
|
const user = await getUserById(userId);
|
|
33856
34248
|
if (!user) return c.json({ error: "User not found" }, 401);
|
|
@@ -33876,7 +34268,11 @@ var requirePaidSchedulingTier = createMiddleware4(async (c, next) => {
|
|
|
33876
34268
|
return next();
|
|
33877
34269
|
});
|
|
33878
34270
|
var app = new Hono33();
|
|
34271
|
+
var deploymentProfile = resolveDeploymentProfile();
|
|
33879
34272
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
34273
|
+
function permitsLocalApiRequest(request) {
|
|
34274
|
+
return (deploymentProfile === "development" || deploymentProfile === "test" || process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1") && request.headers.get("x-mcp-scraper-local-network") === "1" && isLoopbackBaseUrl(request.url);
|
|
34275
|
+
}
|
|
33880
34276
|
function requireStripeSecret() {
|
|
33881
34277
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
33882
34278
|
if (secret2) return secret2;
|
|
@@ -33992,8 +34388,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
33992
34388
|
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
33993
34389
|
}
|
|
33994
34390
|
})();
|
|
33995
|
-
const
|
|
33996
|
-
setCookie2(c, "session",
|
|
34391
|
+
const token4 = signSession(user.id);
|
|
34392
|
+
setCookie2(c, "session", token4, sessionCookieOptions2);
|
|
33997
34393
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
33998
34394
|
} catch (err) {
|
|
33999
34395
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -34011,8 +34407,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
34011
34407
|
if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
|
|
34012
34408
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
34013
34409
|
}
|
|
34014
|
-
const
|
|
34015
|
-
setCookie2(c, "session",
|
|
34410
|
+
const token4 = signSession(user.id);
|
|
34411
|
+
setCookie2(c, "session", token4, sessionCookieOptions2);
|
|
34016
34412
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
34017
34413
|
});
|
|
34018
34414
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
@@ -34067,9 +34463,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
34067
34463
|
if (emailLimited) return emailLimited;
|
|
34068
34464
|
const user = await getUserByEmail(normalizedEmail);
|
|
34069
34465
|
if (!user) return c.json({ ok: true });
|
|
34070
|
-
const
|
|
34466
|
+
const token4 = await createPasswordResetToken(user.id);
|
|
34071
34467
|
const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
|
|
34072
|
-
const resetUrl = `${appUrl}/reset-password?token=${
|
|
34468
|
+
const resetUrl = `${appUrl}/reset-password?token=${token4}`;
|
|
34073
34469
|
if (process.env.RESEND_API_KEY) {
|
|
34074
34470
|
try {
|
|
34075
34471
|
const resend = new Resend3(process.env.RESEND_API_KEY);
|
|
@@ -34089,12 +34485,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
34089
34485
|
return c.json({ ok: true });
|
|
34090
34486
|
});
|
|
34091
34487
|
app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
|
|
34092
|
-
const { token:
|
|
34093
|
-
if (!
|
|
34488
|
+
const { token: token4, password } = await c.req.json();
|
|
34489
|
+
if (!token4 || !password) return c.json({ error: "Token and password required" }, 400);
|
|
34094
34490
|
if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
|
|
34095
34491
|
const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
|
|
34096
34492
|
if (limited) return limited;
|
|
34097
|
-
const userId = await consumePasswordResetToken(
|
|
34493
|
+
const userId = await consumePasswordResetToken(token4);
|
|
34098
34494
|
if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
|
|
34099
34495
|
await setPassword(userId, password);
|
|
34100
34496
|
return c.json({ ok: true });
|
|
@@ -34131,9 +34527,9 @@ function sessionBootstrapPayload(user) {
|
|
|
34131
34527
|
}
|
|
34132
34528
|
app.get("/me", async (c) => {
|
|
34133
34529
|
c.header("Cache-Control", "no-store");
|
|
34134
|
-
const
|
|
34135
|
-
if (!
|
|
34136
|
-
const userId = verifySession(
|
|
34530
|
+
const token4 = getCookie2(c, "session");
|
|
34531
|
+
if (!token4) return c.json({ authenticated: false });
|
|
34532
|
+
const userId = verifySession(token4);
|
|
34137
34533
|
if (!userId) {
|
|
34138
34534
|
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
34139
34535
|
return c.json({ authenticated: false });
|
|
@@ -34470,7 +34866,11 @@ function scheduleConnectionError(c, err, fallback) {
|
|
|
34470
34866
|
return c.json({ ok: false, error: fallback }, 502);
|
|
34471
34867
|
}
|
|
34472
34868
|
function connectedActionIdempotencyKey(c) {
|
|
34473
|
-
|
|
34869
|
+
const key = (c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key"))?.trim();
|
|
34870
|
+
if (!key || key.length < 8 || key.length > 200) {
|
|
34871
|
+
throw new ScheduleConnectionValidationError("Idempotency-Key is required and must contain 8-200 characters for external actions.");
|
|
34872
|
+
}
|
|
34873
|
+
return key;
|
|
34474
34874
|
}
|
|
34475
34875
|
function connectionMemoryImportError(c, err) {
|
|
34476
34876
|
return c.json({
|
|
@@ -35197,12 +35597,13 @@ app.post("/schedule-connections/actions/call", auth2, requireIntegrationsTier, a
|
|
|
35197
35597
|
return c.json({ ok: false, error: "connectionId, tool, and an args object are required." }, 400);
|
|
35198
35598
|
}
|
|
35199
35599
|
try {
|
|
35200
|
-
const
|
|
35600
|
+
const idempotencyKey3 = connectedActionIdempotencyKey(c);
|
|
35601
|
+
const result = await isResendConnection(user.email, connectionId, providerConfigKey) ? await callResendAction(user.email, connectionId, tool, body.args, idempotencyKey3) : await callScheduleConnectionAction(
|
|
35201
35602
|
user.email,
|
|
35202
35603
|
connectionId,
|
|
35203
35604
|
body.args,
|
|
35204
35605
|
tool,
|
|
35205
|
-
|
|
35606
|
+
idempotencyKey3
|
|
35206
35607
|
);
|
|
35207
35608
|
return c.json({ ok: true, result });
|
|
35208
35609
|
} catch (err) {
|
|
@@ -35547,7 +35948,7 @@ app.post("/harvest", auth2, async (c) => {
|
|
|
35547
35948
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
35548
35949
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
35549
35950
|
} else {
|
|
35550
|
-
jobId2 =
|
|
35951
|
+
jobId2 = randomUUID20();
|
|
35551
35952
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
35552
35953
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
35553
35954
|
const hold = await debitMcIdempotent(
|
|
@@ -35740,9 +36141,20 @@ app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
|
|
|
35740
36141
|
});
|
|
35741
36142
|
app.post("/extract-url", auth2, async (c) => {
|
|
35742
36143
|
const raw = await c.req.json().catch(() => ({}));
|
|
35743
|
-
const
|
|
36144
|
+
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
36145
|
+
const bodyResult = (allowLocalRequest ? ExtractUrlLocalBodySchema : ExtractUrlBodySchema).safeParse(raw);
|
|
35744
36146
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
35745
|
-
const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage,
|
|
36147
|
+
const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, mediaTypes } = bodyResult.data;
|
|
36148
|
+
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
36149
|
+
const requestedDelivery = bodyResult.data.delivery ?? "auto";
|
|
36150
|
+
if (bodyResult.data.depositToVault && !["auto", "memory"].includes(requestedDelivery)) {
|
|
36151
|
+
return c.json({ error: 'depositToVault conflicts with the requested delivery; use delivery:"memory" instead.' }, 400);
|
|
36152
|
+
}
|
|
36153
|
+
if (bodyResult.data.preserveMedia !== void 0 && bodyResult.data.downloadMedia !== void 0 && bodyResult.data.preserveMedia !== bodyResult.data.downloadMedia) {
|
|
36154
|
+
return c.json({ error: "preserveMedia conflicts with deprecated downloadMedia." }, 400);
|
|
36155
|
+
}
|
|
36156
|
+
const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
|
|
36157
|
+
const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
|
|
35746
36158
|
if (!allowLocal) {
|
|
35747
36159
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
35748
36160
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -35788,19 +36200,47 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35788
36200
|
if (screenshotBuf) {
|
|
35789
36201
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
35790
36202
|
}
|
|
35791
|
-
const mediaMeta =
|
|
36203
|
+
const mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
35792
36204
|
const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
|
|
35793
36205
|
html: result.bodyHtml,
|
|
35794
36206
|
meta: result.meta,
|
|
35795
36207
|
schema: result.schema,
|
|
35796
36208
|
pageUrl: waybackReplay?.rawReplayUrl ?? canonicalUrl
|
|
35797
36209
|
}) : null;
|
|
35798
|
-
const memory =
|
|
36210
|
+
const memory = effectiveDelivery === "memory" ? await persistScrapeBody(user, {
|
|
35799
36211
|
title: result.title ?? canonicalUrl,
|
|
35800
36212
|
content: result.bodyMarkdown ?? "",
|
|
35801
36213
|
source: canonicalUrl,
|
|
35802
36214
|
vault: bodyResult.data.vaultName
|
|
35803
36215
|
}) : void 0;
|
|
36216
|
+
const imageInputs = [];
|
|
36217
|
+
if (preserveMedia && screenshotMeta?.base64) {
|
|
36218
|
+
imageInputs.push({
|
|
36219
|
+
title: `${result.title ?? canonicalUrl} screenshot`,
|
|
36220
|
+
sourceUrl: canonicalUrl,
|
|
36221
|
+
imageBase64: screenshotMeta.base64,
|
|
36222
|
+
sourceKind: "screenshot",
|
|
36223
|
+
sourceRef: { device }
|
|
36224
|
+
});
|
|
36225
|
+
}
|
|
36226
|
+
if (preserveMedia && mediaMeta) {
|
|
36227
|
+
for (const asset of mediaMeta.assets) {
|
|
36228
|
+
if (asset.type !== "image" || !asset.url.startsWith("https://")) continue;
|
|
36229
|
+
imageInputs.push({
|
|
36230
|
+
title: asset.filename || `${result.title ?? canonicalUrl} image`,
|
|
36231
|
+
sourceUrl: canonicalUrl,
|
|
36232
|
+
imageUrl: asset.url,
|
|
36233
|
+
sourceKind: "page_image"
|
|
36234
|
+
});
|
|
36235
|
+
}
|
|
36236
|
+
}
|
|
36237
|
+
const memoryImages = effectiveDelivery === "memory" && preserveMedia ? await persistScrapeImagesToMemory(user, imageInputs, bodyResult.data.vaultName) : void 0;
|
|
36238
|
+
const screenshotArtifact = preserveMedia && screenshotBuf ? await createImageSourceArtifact({
|
|
36239
|
+
ownerId: String(user.id),
|
|
36240
|
+
content: screenshotBuf,
|
|
36241
|
+
contentType: "image/png",
|
|
36242
|
+
filename: `${new URL(canonicalUrl).hostname}-screenshot.png`
|
|
36243
|
+
}) : void 0;
|
|
35804
36244
|
const archive = waybackReplay ? {
|
|
35805
36245
|
timestamp: waybackReplay.timestamp,
|
|
35806
36246
|
originalUrl: waybackReplay.originalUrl,
|
|
@@ -35816,7 +36256,15 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35816
36256
|
screenshot: screenshotMeta,
|
|
35817
36257
|
branding: brandingData,
|
|
35818
36258
|
media: mediaMeta,
|
|
35819
|
-
memory
|
|
36259
|
+
memory,
|
|
36260
|
+
memoryImages,
|
|
36261
|
+
screenshotArtifact,
|
|
36262
|
+
delivery: {
|
|
36263
|
+
requested: requestedDelivery,
|
|
36264
|
+
effective: effectiveDelivery === "auto" ? "inline" : effectiveDelivery,
|
|
36265
|
+
retained: effectiveDelivery === "memory" ? memory?.deposited === true : effectiveDelivery === "artifact",
|
|
36266
|
+
nextAction: effectiveDelivery === "memory" ? "Use Memory search or read tools with the returned note identifiers." : effectiveDelivery === "artifact" ? "Use report_artifact_read with the returned artifactId." : null
|
|
36267
|
+
}
|
|
35820
36268
|
});
|
|
35821
36269
|
} catch (err) {
|
|
35822
36270
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -35829,9 +36277,11 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35829
36277
|
});
|
|
35830
36278
|
app.post("/diff-page", auth2, async (c) => {
|
|
35831
36279
|
const raw = await c.req.json().catch(() => ({}));
|
|
35832
|
-
const
|
|
36280
|
+
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
36281
|
+
const bodyResult = (allowLocalRequest ? DiffPageLocalBodySchema : DiffPageBodySchema).safeParse(raw);
|
|
35833
36282
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
35834
|
-
const { url,
|
|
36283
|
+
const { url, resetBaseline } = bodyResult.data;
|
|
36284
|
+
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
35835
36285
|
if (!allowLocal) {
|
|
35836
36286
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
35837
36287
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -35923,19 +36373,31 @@ app.post("/archive/read", auth2, async (c) => {
|
|
|
35923
36373
|
message: bodyResult.error.issues[0]?.message ?? "Invalid request"
|
|
35924
36374
|
}, 400);
|
|
35925
36375
|
}
|
|
35926
|
-
const { url, path: path5, depositToLibrary } = bodyResult.data;
|
|
36376
|
+
const { artifactId, url, path: path5, depositToLibrary } = bodyResult.data;
|
|
35927
36377
|
const user = c.get("user");
|
|
35928
36378
|
const gate = await acquireConcurrencyGate(user, "archive_read", {
|
|
35929
36379
|
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
35930
|
-
metadata: { url }
|
|
36380
|
+
metadata: artifactId ? { artifactId } : { url }
|
|
35931
36381
|
});
|
|
35932
36382
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
35933
36383
|
try {
|
|
36384
|
+
let ownedArchive = null;
|
|
36385
|
+
if (artifactId) {
|
|
36386
|
+
const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
36387
|
+
if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
|
|
36388
|
+
ownedArchive = { buffer, ref: artifactId };
|
|
36389
|
+
}
|
|
35934
36390
|
if (!path5) {
|
|
35935
|
-
const inventory = await listZipArchive(url, bodyResult.data.maxEntries ?? 200);
|
|
35936
|
-
return c.json({ mode: "list", ...inventory });
|
|
36391
|
+
const inventory = ownedArchive ? await listZipArchiveBuffer(ownedArchive.buffer, ownedArchive.ref, bodyResult.data.maxEntries ?? 200) : await listZipArchive(url, bodyResult.data.maxEntries ?? 200);
|
|
36392
|
+
return c.json({ mode: "list", ...inventory, ...artifactId ? { artifactId } : {} });
|
|
35937
36393
|
}
|
|
35938
|
-
const file = await
|
|
36394
|
+
const file = ownedArchive ? await readZipArchiveBuffer(
|
|
36395
|
+
ownedArchive.buffer,
|
|
36396
|
+
ownedArchive.ref,
|
|
36397
|
+
path5,
|
|
36398
|
+
bodyResult.data.offset ?? 0,
|
|
36399
|
+
bodyResult.data.maxBytes ?? 5e4
|
|
36400
|
+
) : await readZipArchiveFile(
|
|
35939
36401
|
url,
|
|
35940
36402
|
path5,
|
|
35941
36403
|
bodyResult.data.offset ?? 0,
|
|
@@ -35957,7 +36419,7 @@ app.post("/archive/read", auth2, async (c) => {
|
|
|
35957
36419
|
summary: `Source file \`${file.path}\` preserved from ZIP archive ${librarySource.split("#")[0]}.`
|
|
35958
36420
|
}) : void 0;
|
|
35959
36421
|
const { fullContent: _fullContent, ...responseFile } = file;
|
|
35960
|
-
return c.json({ mode: "read", ...responseFile, memory });
|
|
36422
|
+
return c.json({ mode: "read", ...responseFile, ...artifactId ? { artifactId } : {}, memory });
|
|
35961
36423
|
} catch (error) {
|
|
35962
36424
|
const known = error instanceof ArchiveReadError ? error : new ArchiveReadError(
|
|
35963
36425
|
"archive_read_failed",
|
|
@@ -36099,6 +36561,9 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36099
36561
|
const bodyResult = ExtractSiteBodySchema.safeParse(raw);
|
|
36100
36562
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
36101
36563
|
const body = bodyResult.data;
|
|
36564
|
+
if (body.preserveMedia !== void 0 && body.downloadImages !== void 0 && body.preserveMedia !== body.downloadImages) {
|
|
36565
|
+
return c.json({ error: "preserveMedia conflicts with deprecated downloadImages." }, 400);
|
|
36566
|
+
}
|
|
36102
36567
|
const checked = await validatePublicHttpUrl(body.url, { field: "URL" });
|
|
36103
36568
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
36104
36569
|
const parsed = checked.parsed;
|
|
@@ -36122,7 +36587,7 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36122
36587
|
waybackMonths.length * Math.min(maxPagesPerSnapshot, body.wayback.urls?.length ?? maxPagesPerSnapshot)
|
|
36123
36588
|
) : requestedSiteMaxPages(body.maxPages);
|
|
36124
36589
|
const crawlStartUrl = body.wayback ? waybackRootUrl : waybackReplay?.rawReplayUrl ?? parsed.href;
|
|
36125
|
-
const downloadImages = body.downloadImages
|
|
36590
|
+
const downloadImages = body.preserveMedia ?? body.downloadImages ?? false;
|
|
36126
36591
|
const shouldRunInBackground = Boolean(body.wayback) || shouldRunSiteExtractInBackground(body);
|
|
36127
36592
|
if (shouldRunInBackground) {
|
|
36128
36593
|
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
@@ -36136,9 +36601,9 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36136
36601
|
if (!suppliedIdempotencyKey.trim() || suppliedIdempotencyKey.length > 500) {
|
|
36137
36602
|
return c.json({ error: "Idempotency-Key must contain 1-500 characters.", errorCode: "invalid_idempotency_key", retryable: false }, 400);
|
|
36138
36603
|
}
|
|
36139
|
-
const
|
|
36140
|
-
c.header("Idempotency-Key",
|
|
36141
|
-
const jobId2 = `ext_${sha256Hex(`${user.id}\0${
|
|
36604
|
+
const idempotencyKey3 = suppliedIdempotencyKey.trim();
|
|
36605
|
+
c.header("Idempotency-Key", idempotencyKey3);
|
|
36606
|
+
const jobId2 = `ext_${sha256Hex(`${user.id}\0${idempotencyKey3}`).slice(0, 24)}`;
|
|
36142
36607
|
const requestFingerprint2 = sha256Hex(JSON.stringify({
|
|
36143
36608
|
url: parsed.href,
|
|
36144
36609
|
waybackTimestamp: waybackReplay?.timestamp ?? null,
|
|
@@ -36157,7 +36622,7 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36157
36622
|
balanceMc: user.balance_mc,
|
|
36158
36623
|
startUrl: crawlStartUrl,
|
|
36159
36624
|
requestedMaxPages,
|
|
36160
|
-
idempotencyKey:
|
|
36625
|
+
idempotencyKey: idempotencyKey3,
|
|
36161
36626
|
requestFingerprint: requestFingerprint2,
|
|
36162
36627
|
concurrency: concurrencyLimitForUser(user),
|
|
36163
36628
|
urlsPerBrowser: body.rotateProxyEvery ?? 10,
|
|
@@ -36701,7 +37166,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
36701
37166
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
36702
37167
|
return c.json({ error: "Unauthorized" }, 401);
|
|
36703
37168
|
}
|
|
36704
|
-
const { drainQueue } = await import("./worker-
|
|
37169
|
+
const { drainQueue } = await import("./worker-E6MIVF33.js");
|
|
36705
37170
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
36706
37171
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
36707
37172
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
|
|
@@ -36754,8 +37219,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
36754
37219
|
finishExtractJob: finishExtractJob2,
|
|
36755
37220
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
36756
37221
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
36757
|
-
} = await import("./site-extract-repository-
|
|
36758
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
37222
|
+
} = await import("./site-extract-repository-XNI4ZK2M.js");
|
|
37223
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
|
|
36759
37224
|
const existing = await getExtractJob2(jobId2);
|
|
36760
37225
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
36761
37226
|
if (existing.status !== "failed") {
|
|
@@ -36782,7 +37247,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
36782
37247
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
36783
37248
|
let settlement = "already_settled_or_refunded";
|
|
36784
37249
|
if (job.billedMc == null && job.userId != null) {
|
|
36785
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
37250
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-XNI4ZK2M.js");
|
|
36786
37251
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
36787
37252
|
const successful = await countSuccessfulPages2(jobId2);
|
|
36788
37253
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -36858,7 +37323,7 @@ app.route("/", oauthApp);
|
|
|
36858
37323
|
if (!process.env.INNGEST_EVENT_KEY) {
|
|
36859
37324
|
startSiteAuditWorker();
|
|
36860
37325
|
}
|
|
36861
|
-
app.get("/font-test", (c) => {
|
|
37326
|
+
if (process.env.NODE_ENV !== "production") app.get("/font-test", (c) => {
|
|
36862
37327
|
return c.html(`<!DOCTYPE html>
|
|
36863
37328
|
<html lang="en">
|
|
36864
37329
|
<head>
|
|
@@ -36961,4 +37426,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
36961
37426
|
export {
|
|
36962
37427
|
app
|
|
36963
37428
|
};
|
|
36964
|
-
//# sourceMappingURL=server-
|
|
37429
|
+
//# sourceMappingURL=server-WAB6Z47N.js.map
|