mcp-scraper 0.42.3 → 0.43.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +3284 -2130
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -4
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +3361 -2817
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +51 -24
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +21 -21
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-MJKVDLVR.js → chunk-2K74LVV7.js} +26 -2
- package/dist/{chunk-MJKVDLVR.js.map → chunk-2K74LVV7.js.map} +1 -1
- package/dist/{chunk-CB5C3BPB.js → chunk-3ZUBQQPQ.js} +23 -2
- package/dist/chunk-3ZUBQQPQ.js.map +1 -0
- package/dist/{chunk-6Q7V4IQV.js → chunk-6WLNXYRG.js} +19 -2
- package/dist/chunk-6WLNXYRG.js.map +1 -0
- package/dist/{chunk-G5MHJGKE.js → chunk-GUI33Q27.js} +2 -2
- package/dist/{chunk-DH42LIR3.js → chunk-H7LQ6PTI.js} +24 -16
- package/dist/chunk-H7LQ6PTI.js.map +1 -0
- package/dist/chunk-MD75M2L5.js +7 -0
- package/dist/chunk-MD75M2L5.js.map +1 -0
- package/dist/{chunk-ENJQ2NSX.js → chunk-TC4GDB2Q.js} +2 -2
- package/dist/{chunk-ENJQ2NSX.js.map → chunk-TC4GDB2Q.js.map} +1 -1
- package/dist/{chunk-IMUA3FJB.js → chunk-X5WDJ7D7.js} +6 -6
- package/dist/chunk-X5WDJ7D7.js.map +1 -0
- package/dist/{chunk-QKCDYMFT.js → chunk-YJNGNMDO.js} +2196 -1584
- package/dist/chunk-YJNGNMDO.js.map +1 -0
- package/dist/{chunk-FQI5PFE7.js → chunk-ZSJHRZY5.js} +3 -5
- package/dist/{chunk-FQI5PFE7.js.map → chunk-ZSJHRZY5.js.map} +1 -1
- package/dist/{extract-bundle-FH734F3R.js → extract-bundle-IOM45SFL.js} +5 -5
- package/dist/index.cjs +21 -21
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +21 -57
- package/dist/index.d.ts +21 -57
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-B27DR4YB.js → server-BPEXEKA6.js} +800 -336
- package/dist/server-BPEXEKA6.js.map +1 -0
- package/dist/{site-extract-repository-5CVI3OWT.js → site-extract-repository-XNI4ZK2M.js} +3 -3
- package/dist/{worker-MQJOY4YP.js → worker-2LIOH744.js} +4 -5
- package/dist/{worker-MQJOY4YP.js.map → worker-2LIOH744.js.map} +1 -1
- package/package.json +10 -7
- package/dist/chunk-2YFTFT3V.js +0 -7
- package/dist/chunk-2YFTFT3V.js.map +0 -1
- package/dist/chunk-6Q7V4IQV.js.map +0 -1
- package/dist/chunk-CB5C3BPB.js.map +0 -1
- package/dist/chunk-DH42LIR3.js.map +0 -1
- package/dist/chunk-IMUA3FJB.js.map +0 -1
- package/dist/chunk-K443GQY5.js +0 -24
- package/dist/chunk-K443GQY5.js.map +0 -1
- package/dist/chunk-QKCDYMFT.js.map +0 -1
- package/dist/server-B27DR4YB.js.map +0 -1
- /package/dist/{chunk-G5MHJGKE.js.map → chunk-GUI33Q27.js.map} +0 -0
- /package/dist/{extract-bundle-FH734F3R.js.map → extract-bundle-IOM45SFL.js.map} +0 -0
- /package/dist/{site-extract-repository-5CVI3OWT.js.map → site-extract-repository-XNI4ZK2M.js.map} +0 -0
|
@@ -2,8 +2,9 @@ import {
|
|
|
2
2
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
3
3
|
cleanupExpiredSiteExtractArtifacts,
|
|
4
4
|
harvestPageMedia,
|
|
5
|
+
readOwnedSiteExtractArtifactBuffer,
|
|
5
6
|
renewSiteExtractArtifactDownload
|
|
6
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-6WLNXYRG.js";
|
|
7
8
|
import {
|
|
8
9
|
REQUIRED_LOCATION_STATE_CODES,
|
|
9
10
|
importHostedCensusPlacesCsv,
|
|
@@ -40,7 +41,7 @@ import {
|
|
|
40
41
|
resolveBackupKernelProxyId,
|
|
41
42
|
resolveKernelProxyId,
|
|
42
43
|
serpArrivalUrl
|
|
43
|
-
} from "./chunk-
|
|
44
|
+
} from "./chunk-H7LQ6PTI.js";
|
|
44
45
|
import {
|
|
45
46
|
ApiErrorResponseSchema,
|
|
46
47
|
ArtifactTemplateConfigSchema,
|
|
@@ -69,9 +70,10 @@ import {
|
|
|
69
70
|
createConnectedDataArtifact,
|
|
70
71
|
createDirectoryCsvArtifact,
|
|
71
72
|
exportFanout,
|
|
72
|
-
|
|
73
|
+
handleMcpTasksHttpRequest,
|
|
73
74
|
harvestTimeoutBudget,
|
|
74
75
|
hashOwnerId,
|
|
76
|
+
isLoopbackBaseUrl,
|
|
75
77
|
outputBaseDir,
|
|
76
78
|
registerBrowserAgentMcpTools,
|
|
77
79
|
registerMemoryMcpTools,
|
|
@@ -79,8 +81,9 @@ import {
|
|
|
79
81
|
registerSerpIntelligenceCaptureTools,
|
|
80
82
|
renewConnectedDataArtifactDownload,
|
|
81
83
|
renewDirectoryArtifactDownload,
|
|
84
|
+
resolveDeploymentProfile,
|
|
82
85
|
transcribeMediaUrl
|
|
83
|
-
} from "./chunk-
|
|
86
|
+
} from "./chunk-YJNGNMDO.js";
|
|
84
87
|
import {
|
|
85
88
|
auditImageUrls,
|
|
86
89
|
auditImages,
|
|
@@ -88,10 +91,12 @@ import {
|
|
|
88
91
|
computeIssues,
|
|
89
92
|
createPrivateArtifact,
|
|
90
93
|
isPrivateIpAddress,
|
|
94
|
+
privateArtifactOwnerId,
|
|
95
|
+
readPrivateArtifactBuffer,
|
|
91
96
|
readPrivateArtifactWindow,
|
|
92
97
|
renewPrivateArtifactDownload,
|
|
93
98
|
validatePublicHttpUrl
|
|
94
|
-
} from "./chunk-
|
|
99
|
+
} from "./chunk-2K74LVV7.js";
|
|
95
100
|
import {
|
|
96
101
|
sanitizeAttempts,
|
|
97
102
|
sanitizeHarvestResult,
|
|
@@ -116,13 +121,12 @@ import {
|
|
|
116
121
|
workflowOutputBaseDir,
|
|
117
122
|
workflowStepCount,
|
|
118
123
|
workflowSupportsSteps
|
|
119
|
-
} from "./chunk-
|
|
124
|
+
} from "./chunk-ZSJHRZY5.js";
|
|
120
125
|
import {
|
|
121
126
|
csvRecords,
|
|
122
127
|
parseCsv,
|
|
123
128
|
rowsToCsv
|
|
124
129
|
} from "./chunk-G3P3ZDB4.js";
|
|
125
|
-
import "./chunk-K443GQY5.js";
|
|
126
130
|
import {
|
|
127
131
|
DEFAULT_MAPS_PROXY_MODE,
|
|
128
132
|
DEFAULT_PROXY_MODE,
|
|
@@ -132,8 +136,10 @@ import {
|
|
|
132
136
|
RawMapsHoursRowSchema,
|
|
133
137
|
RawMapsOverviewSchema,
|
|
134
138
|
RawMapsReviewStatsSchema
|
|
135
|
-
} from "./chunk-
|
|
136
|
-
import
|
|
139
|
+
} from "./chunk-3ZUBQQPQ.js";
|
|
140
|
+
import {
|
|
141
|
+
PACKAGE_VERSION
|
|
142
|
+
} from "./chunk-MD75M2L5.js";
|
|
137
143
|
import {
|
|
138
144
|
abandonExtractSettlement,
|
|
139
145
|
countSuccessfulPages,
|
|
@@ -156,7 +162,7 @@ import {
|
|
|
156
162
|
setExtractJobTotal,
|
|
157
163
|
settleExtractJob,
|
|
158
164
|
terminalExtractJobStatus
|
|
159
|
-
} from "./chunk-
|
|
165
|
+
} from "./chunk-GUI33Q27.js";
|
|
160
166
|
import {
|
|
161
167
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
162
168
|
CONCURRENCY_PRICE_ID,
|
|
@@ -204,7 +210,7 @@ import {
|
|
|
204
210
|
tierChangeLeavesPrivateOffer,
|
|
205
211
|
tierCreditsPerInterval,
|
|
206
212
|
vendorCostUsdToMc
|
|
207
|
-
} from "./chunk-
|
|
213
|
+
} from "./chunk-X5WDJ7D7.js";
|
|
208
214
|
import {
|
|
209
215
|
CaptchaError,
|
|
210
216
|
MapsSearchIncompletePageError,
|
|
@@ -3629,7 +3635,7 @@ async function generateOgImage(post) {
|
|
|
3629
3635
|
|
|
3630
3636
|
// src/api/server.ts
|
|
3631
3637
|
import { Resend as Resend3 } from "resend";
|
|
3632
|
-
import { randomUUID as
|
|
3638
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
3633
3639
|
|
|
3634
3640
|
// src/api/kpo-extractor.ts
|
|
3635
3641
|
import TurndownService from "turndown";
|
|
@@ -8658,7 +8664,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8658
8664
|
});
|
|
8659
8665
|
}) : null;
|
|
8660
8666
|
const finalized = await step.run("finalize", async () => {
|
|
8661
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8667
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
|
|
8662
8668
|
const current = await getExtractJob(jobId2);
|
|
8663
8669
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8664
8670
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10202,11 +10208,11 @@ async function getDirectoryWorkflowJob(id, userId) {
|
|
|
10202
10208
|
const result = userId == null ? await getDb().execute({ sql: "SELECT * FROM directory_workflow_jobs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT * FROM directory_workflow_jobs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
10203
10209
|
return result.rows[0] ? rowToJob(result.rows[0]) : null;
|
|
10204
10210
|
}
|
|
10205
|
-
async function getDirectoryWorkflowJobByIdempotencyKey(userId,
|
|
10211
|
+
async function getDirectoryWorkflowJobByIdempotencyKey(userId, idempotencyKey3) {
|
|
10206
10212
|
await ensureDirectoryWorkflowRepositorySchema();
|
|
10207
10213
|
const result = await getDb().execute({
|
|
10208
10214
|
sql: "SELECT * FROM directory_workflow_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",
|
|
10209
|
-
args: [userId,
|
|
10215
|
+
args: [userId, idempotencyKey3]
|
|
10210
10216
|
});
|
|
10211
10217
|
return result.rows[0] ? rowToJob(result.rows[0]) : null;
|
|
10212
10218
|
}
|
|
@@ -10372,7 +10378,7 @@ async function recordDirectoryWorkflowJobSettlement(jobId2, billedMc) {
|
|
|
10372
10378
|
async function createOrGetDirectoryWorkflowJob(input) {
|
|
10373
10379
|
await ensureDirectoryWorkflowRepositorySchema();
|
|
10374
10380
|
const id = requiredBoundedString(input.id, "job id", 200);
|
|
10375
|
-
const
|
|
10381
|
+
const idempotencyKey3 = requiredBoundedString(input.idempotencyKey, "idempotency key", 500);
|
|
10376
10382
|
const requestFingerprint2 = requiredBoundedString(input.requestFingerprint, "request fingerprint", 200);
|
|
10377
10383
|
const debitKey2 = requiredBoundedString(input.debitKey, "debit key", 500);
|
|
10378
10384
|
const holdMc = nonNegativeSafeInteger(input.holdMc, "holdMc");
|
|
@@ -10387,7 +10393,7 @@ async function createOrGetDirectoryWorkflowJob(input) {
|
|
|
10387
10393
|
args: [
|
|
10388
10394
|
id,
|
|
10389
10395
|
input.userId,
|
|
10390
|
-
|
|
10396
|
+
idempotencyKey3,
|
|
10391
10397
|
requestFingerprint2,
|
|
10392
10398
|
JSON.stringify(safeOptions2),
|
|
10393
10399
|
JSON.stringify(input.plan),
|
|
@@ -10402,10 +10408,10 @@ async function createOrGetDirectoryWorkflowJob(input) {
|
|
|
10402
10408
|
SELECT 'directory:' || id, id, ?, 'held'
|
|
10403
10409
|
FROM directory_workflow_jobs
|
|
10404
10410
|
WHERE user_id = ? AND idempotency_key = ?`,
|
|
10405
|
-
args: [DIRECTORY_EVENT_NAME, input.userId,
|
|
10411
|
+
args: [DIRECTORY_EVENT_NAME, input.userId, idempotencyKey3]
|
|
10406
10412
|
}
|
|
10407
10413
|
], "write");
|
|
10408
|
-
const job = await getDirectoryWorkflowJobByIdempotencyKey(input.userId,
|
|
10414
|
+
const job = await getDirectoryWorkflowJobByIdempotencyKey(input.userId, idempotencyKey3);
|
|
10409
10415
|
if (!job) throw new Error("directory workflow job could not be created");
|
|
10410
10416
|
return {
|
|
10411
10417
|
job,
|
|
@@ -12646,15 +12652,15 @@ function normalizeIdempotencyKey(value) {
|
|
|
12646
12652
|
}
|
|
12647
12653
|
return key;
|
|
12648
12654
|
}
|
|
12649
|
-
function tokenFor(id,
|
|
12650
|
-
return createHmac("sha256", billingSecret()).update(id).update(":").update(
|
|
12655
|
+
function tokenFor(id, idempotencyKey3) {
|
|
12656
|
+
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
|
|
12651
12657
|
}
|
|
12652
|
-
function tokenHash(
|
|
12653
|
-
return createHash3("sha256").update(
|
|
12658
|
+
function tokenHash(token4) {
|
|
12659
|
+
return createHash3("sha256").update(token4).digest("hex");
|
|
12654
12660
|
}
|
|
12655
|
-
function verifyToken(row,
|
|
12661
|
+
function verifyToken(row, token4) {
|
|
12656
12662
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
12657
|
-
const actual = Buffer.from(tokenHash(
|
|
12663
|
+
const actual = Buffer.from(tokenHash(token4), "hex");
|
|
12658
12664
|
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
12659
12665
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
12660
12666
|
}
|
|
@@ -12693,10 +12699,10 @@ async function authorizationById(id) {
|
|
|
12693
12699
|
});
|
|
12694
12700
|
return result.rows[0] ? result.rows[0] : null;
|
|
12695
12701
|
}
|
|
12696
|
-
async function eventByKey(
|
|
12702
|
+
async function eventByKey(idempotencyKey3) {
|
|
12697
12703
|
const result = await getDb().execute({
|
|
12698
12704
|
sql: "SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",
|
|
12699
|
-
args: [
|
|
12705
|
+
args: [idempotencyKey3]
|
|
12700
12706
|
});
|
|
12701
12707
|
return result.rows[0] ? result.rows[0] : null;
|
|
12702
12708
|
}
|
|
@@ -12733,7 +12739,7 @@ function mapSettlementEvent(row, balanceMc, duplicate) {
|
|
|
12733
12739
|
}
|
|
12734
12740
|
async function authorizeScheduledRun(args) {
|
|
12735
12741
|
const identity = normalizeIdentity(args.identity);
|
|
12736
|
-
const
|
|
12742
|
+
const idempotencyKey3 = normalizeIdempotencyKey(args.idempotencyKey);
|
|
12737
12743
|
const user = await resolveSchedulingUser(identity);
|
|
12738
12744
|
if (!hasPaidSchedulingAccess(user)) {
|
|
12739
12745
|
throw new UnifiedBillingError(PAID_PLAN_REQUIRED_CODE, PAID_PLAN_REQUIRED_MESSAGE, 403);
|
|
@@ -12747,7 +12753,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12747
12753
|
}
|
|
12748
12754
|
const existing = await getDb().execute({
|
|
12749
12755
|
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
12750
|
-
args: [
|
|
12756
|
+
args: [idempotencyKey3]
|
|
12751
12757
|
});
|
|
12752
12758
|
if (existing.rows[0]) {
|
|
12753
12759
|
const row = existing.rows[0];
|
|
@@ -12771,16 +12777,16 @@ async function authorizeScheduledRun(args) {
|
|
|
12771
12777
|
);
|
|
12772
12778
|
}
|
|
12773
12779
|
const id = randomUUID3();
|
|
12774
|
-
const
|
|
12780
|
+
const token4 = tokenFor(id, idempotencyKey3);
|
|
12775
12781
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
12776
12782
|
const inserted = await getDb().execute({
|
|
12777
12783
|
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
12778
|
-
args: [id, user.id,
|
|
12784
|
+
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token4), safeMetadata2(args.metadata), expiresAt]
|
|
12779
12785
|
});
|
|
12780
12786
|
if (inserted.rowsAffected === 0) {
|
|
12781
12787
|
const raced = await getDb().execute({
|
|
12782
12788
|
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
12783
|
-
args: [
|
|
12789
|
+
args: [idempotencyKey3]
|
|
12784
12790
|
});
|
|
12785
12791
|
if (!raced.rows[0]) throw new Error("billing authorization insert raced without a readable winner");
|
|
12786
12792
|
const row = raced.rows[0];
|
|
@@ -12791,7 +12797,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12791
12797
|
}
|
|
12792
12798
|
return {
|
|
12793
12799
|
id,
|
|
12794
|
-
token:
|
|
12800
|
+
token: token4,
|
|
12795
12801
|
status: "authorized",
|
|
12796
12802
|
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
12797
12803
|
maximumMc,
|
|
@@ -13061,11 +13067,11 @@ function signSession(userId) {
|
|
|
13061
13067
|
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
13062
13068
|
return `${payload}.${sig}`;
|
|
13063
13069
|
}
|
|
13064
|
-
function verifySession(
|
|
13065
|
-
const dot =
|
|
13070
|
+
function verifySession(token4) {
|
|
13071
|
+
const dot = token4.lastIndexOf(".");
|
|
13066
13072
|
if (dot === -1) return null;
|
|
13067
|
-
const payload =
|
|
13068
|
-
const sig =
|
|
13073
|
+
const payload = token4.slice(0, dot);
|
|
13074
|
+
const sig = token4.slice(dot + 1);
|
|
13069
13075
|
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
13070
13076
|
if (!safeEqualHex(sig, expected)) return null;
|
|
13071
13077
|
const id = parseInt(payload);
|
|
@@ -13293,11 +13299,11 @@ var ConnectedUsageBillingError = class extends Error {
|
|
|
13293
13299
|
function hash(value) {
|
|
13294
13300
|
return createHash4("sha256").update(value).digest("hex");
|
|
13295
13301
|
}
|
|
13296
|
-
function eventKey(
|
|
13297
|
-
return `connected-usage:${hash(
|
|
13302
|
+
function eventKey(idempotencyKey3) {
|
|
13303
|
+
return `connected-usage:${hash(idempotencyKey3)}`;
|
|
13298
13304
|
}
|
|
13299
|
-
function debitKey(
|
|
13300
|
-
return `connected-usage-debit:${hash(
|
|
13305
|
+
function debitKey(idempotencyKey3) {
|
|
13306
|
+
return `connected-usage-debit:${hash(idempotencyKey3)}`;
|
|
13301
13307
|
}
|
|
13302
13308
|
function normalizeIdentity2(identity) {
|
|
13303
13309
|
return identity.trim().toLowerCase();
|
|
@@ -13655,7 +13661,7 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
13655
13661
|
|
|
13656
13662
|
// src/api/internal-billing-routes.ts
|
|
13657
13663
|
var internalBillingApp = new Hono3();
|
|
13658
|
-
var metadataSchema = z12.record(z12.unknown()).optional();
|
|
13664
|
+
var metadataSchema = z12.record(z12.string(), z12.unknown()).optional();
|
|
13659
13665
|
var schedulingAccessSchema = z12.object({
|
|
13660
13666
|
identity: z12.string().min(1).max(320),
|
|
13661
13667
|
provisionCredentials: z12.boolean().optional()
|
|
@@ -14394,9 +14400,9 @@ var YouTubeExtractor = class {
|
|
|
14394
14400
|
}
|
|
14395
14401
|
async fetchContinuation(page, endpoint, contToken, context) {
|
|
14396
14402
|
const result = await page.evaluate(
|
|
14397
|
-
async ({ ep, token:
|
|
14403
|
+
async ({ ep, token: token4, ctx }) => {
|
|
14398
14404
|
const body = {
|
|
14399
|
-
continuation:
|
|
14405
|
+
continuation: token4,
|
|
14400
14406
|
context: {
|
|
14401
14407
|
client: {
|
|
14402
14408
|
clientName: ctx.clientName,
|
|
@@ -15012,7 +15018,7 @@ var HarvestBodySchema = z15.object({
|
|
|
15012
15018
|
pages: z15.number().int().min(1).max(2).optional(),
|
|
15013
15019
|
recency: z15.enum(["day", "week", "month", "year"]).optional()
|
|
15014
15020
|
});
|
|
15015
|
-
var
|
|
15021
|
+
var ExtractUrlBodyShape = {
|
|
15016
15022
|
url: z15.string().min(1, "url is required"),
|
|
15017
15023
|
screenshot: z15.boolean().optional(),
|
|
15018
15024
|
screenshotDevice: z15.enum(["desktop", "mobile"]).optional(),
|
|
@@ -15020,23 +15026,41 @@ var ExtractUrlBodySchema = z15.object({
|
|
|
15020
15026
|
includeFeaturedImage: z15.boolean().optional(),
|
|
15021
15027
|
downloadMedia: z15.boolean().optional(),
|
|
15022
15028
|
mediaTypes: z15.array(z15.enum(["image", "video", "audio"])).optional(),
|
|
15023
|
-
|
|
15029
|
+
delivery: z15.enum(["auto", "inline", "artifact", "memory"]).optional(),
|
|
15030
|
+
preserveMedia: z15.boolean().optional(),
|
|
15024
15031
|
depositToVault: z15.boolean().optional(),
|
|
15025
15032
|
vaultName: z15.string().trim().min(1).max(120).optional()
|
|
15026
|
-
}
|
|
15027
|
-
var
|
|
15033
|
+
};
|
|
15034
|
+
var ExtractUrlBodySchema = z15.object(ExtractUrlBodyShape).strict();
|
|
15035
|
+
var ExtractUrlLocalBodySchema = z15.object({
|
|
15036
|
+
...ExtractUrlBodyShape,
|
|
15037
|
+
allowLocal: z15.boolean().optional()
|
|
15038
|
+
}).strict();
|
|
15039
|
+
var DiffPageBodyShape = {
|
|
15028
15040
|
url: z15.string().min(1, "url is required"),
|
|
15029
|
-
allowLocal: z15.boolean().optional(),
|
|
15030
15041
|
resetBaseline: z15.boolean().optional()
|
|
15031
|
-
}
|
|
15042
|
+
};
|
|
15043
|
+
var DiffPageBodySchema = z15.object(DiffPageBodyShape).strict();
|
|
15044
|
+
var DiffPageLocalBodySchema = z15.object({
|
|
15045
|
+
...DiffPageBodyShape,
|
|
15046
|
+
allowLocal: z15.boolean().optional()
|
|
15047
|
+
}).strict();
|
|
15032
15048
|
var ArchiveReadBodySchema = z15.object({
|
|
15033
|
-
|
|
15049
|
+
artifactId: z15.string().trim().min(1).optional(),
|
|
15050
|
+
url: z15.string().url("url must be a valid HTTPS ZIP URL").optional(),
|
|
15034
15051
|
path: z15.string().trim().min(1).max(2e3).optional(),
|
|
15035
15052
|
offset: z15.number().int().min(0).optional(),
|
|
15036
15053
|
maxBytes: z15.number().int().min(1).max(2e5).optional(),
|
|
15037
15054
|
maxEntries: z15.number().int().min(1).max(1e3).optional(),
|
|
15038
15055
|
depositToLibrary: z15.boolean().optional()
|
|
15039
15056
|
}).superRefine((value, ctx) => {
|
|
15057
|
+
if ((value.artifactId ? 1 : 0) + (value.url ? 1 : 0) !== 1) {
|
|
15058
|
+
ctx.addIssue({
|
|
15059
|
+
code: z15.ZodIssueCode.custom,
|
|
15060
|
+
path: ["artifactId"],
|
|
15061
|
+
message: "Provide exactly one of artifactId or url"
|
|
15062
|
+
});
|
|
15063
|
+
}
|
|
15040
15064
|
if (value.depositToLibrary && !value.path) {
|
|
15041
15065
|
ctx.addIssue({
|
|
15042
15066
|
code: z15.ZodIssueCode.custom,
|
|
@@ -15050,7 +15074,8 @@ var MapUrlsBodySchema = z15.object({
|
|
|
15050
15074
|
maxUrls: z15.number().int().min(1).max(2e3).optional(),
|
|
15051
15075
|
concurrency: z15.number().int().min(1).max(20).optional(),
|
|
15052
15076
|
browserFallback: z15.boolean().optional(),
|
|
15053
|
-
kernelFallback: z15.boolean().optional()
|
|
15077
|
+
kernelFallback: z15.boolean().optional(),
|
|
15078
|
+
delivery: z15.enum(["auto", "inline", "artifact"]).optional()
|
|
15054
15079
|
});
|
|
15055
15080
|
var WaybackInventoryBodySchema = z15.object({
|
|
15056
15081
|
url: z15.string().min(1, "url is required"),
|
|
@@ -15066,6 +15091,8 @@ var ExtractSiteBodySchema = z15.object({
|
|
|
15066
15091
|
rotateProxyEvery: z15.number().int().min(1).max(100).optional(),
|
|
15067
15092
|
background: z15.boolean().optional(),
|
|
15068
15093
|
downloadImages: z15.boolean().optional(),
|
|
15094
|
+
preserveMedia: z15.boolean().optional(),
|
|
15095
|
+
delivery: z15.enum(["auto", "artifact"]).optional(),
|
|
15069
15096
|
formats: z15.array(z15.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
15070
15097
|
});
|
|
15071
15098
|
var YoutubeHarvestBodySchema = z15.object({
|
|
@@ -16275,7 +16302,7 @@ function advertiserTokenOverlap(query, candidate) {
|
|
|
16275
16302
|
const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
|
|
16276
16303
|
if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
|
|
16277
16304
|
let overlap = 0;
|
|
16278
|
-
for (const
|
|
16305
|
+
for (const token4 of queryTokens) if (candidateTokens.has(token4)) overlap++;
|
|
16279
16306
|
return overlap / queryTokens.size;
|
|
16280
16307
|
}
|
|
16281
16308
|
function assessPageIntelMatch(body, result) {
|
|
@@ -17463,11 +17490,85 @@ adminCreditsApp.get("/lookup", async (c) => {
|
|
|
17463
17490
|
// src/api/instagram-routes.ts
|
|
17464
17491
|
import { Hono as Hono10 } from "hono";
|
|
17465
17492
|
import { z as z19 } from "zod";
|
|
17466
|
-
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
17493
|
+
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
17467
17494
|
import { tmpdir } from "os";
|
|
17468
|
-
import { join as
|
|
17495
|
+
import { join as join5 } from "path";
|
|
17469
17496
|
import { spawn } from "child_process";
|
|
17470
17497
|
|
|
17498
|
+
// src/api/blob-store.ts
|
|
17499
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
17500
|
+
import { homedir } from "os";
|
|
17501
|
+
import { join as join4, dirname as dirname2 } from "path";
|
|
17502
|
+
function byteLength(data) {
|
|
17503
|
+
return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
|
|
17504
|
+
}
|
|
17505
|
+
var LocalBlobStore = class {
|
|
17506
|
+
constructor(baseDir) {
|
|
17507
|
+
this.baseDir = baseDir;
|
|
17508
|
+
}
|
|
17509
|
+
baseDir;
|
|
17510
|
+
kind = "local";
|
|
17511
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
17512
|
+
const path5 = join4(this.baseDir, "blobs", key);
|
|
17513
|
+
mkdirSync(dirname2(path5), { recursive: true });
|
|
17514
|
+
writeFileSync(path5, data);
|
|
17515
|
+
return { key, url: `file://${path5}`, bytes: byteLength(data), contentType };
|
|
17516
|
+
}
|
|
17517
|
+
async get(key) {
|
|
17518
|
+
const path5 = join4(this.baseDir, "blobs", key);
|
|
17519
|
+
try {
|
|
17520
|
+
const { readFileSync: readFileSync4 } = await import("fs");
|
|
17521
|
+
return readFileSync4(path5);
|
|
17522
|
+
} catch {
|
|
17523
|
+
return null;
|
|
17524
|
+
}
|
|
17525
|
+
}
|
|
17526
|
+
};
|
|
17527
|
+
function localBaseDir() {
|
|
17528
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir(), "Downloads", "mcp-scraper");
|
|
17529
|
+
}
|
|
17530
|
+
var cached = null;
|
|
17531
|
+
function getBlobStore() {
|
|
17532
|
+
if (cached) return cached;
|
|
17533
|
+
if (process.env.BLOB_READ_WRITE_TOKEN) {
|
|
17534
|
+
cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
|
|
17535
|
+
} else {
|
|
17536
|
+
cached = new LocalBlobStore(localBaseDir());
|
|
17537
|
+
}
|
|
17538
|
+
return cached;
|
|
17539
|
+
}
|
|
17540
|
+
var VercelBlobStore = class {
|
|
17541
|
+
constructor(token4) {
|
|
17542
|
+
this.token = token4;
|
|
17543
|
+
}
|
|
17544
|
+
token;
|
|
17545
|
+
kind = "vercel-blob";
|
|
17546
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
17547
|
+
const { put } = await import("@vercel/blob");
|
|
17548
|
+
const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
17549
|
+
const result = await put(key, body, {
|
|
17550
|
+
access: "public",
|
|
17551
|
+
token: this.token,
|
|
17552
|
+
contentType,
|
|
17553
|
+
addRandomSuffix: true
|
|
17554
|
+
});
|
|
17555
|
+
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
17556
|
+
}
|
|
17557
|
+
async get(key) {
|
|
17558
|
+
try {
|
|
17559
|
+
const { list } = await import("@vercel/blob");
|
|
17560
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
17561
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
17562
|
+
if (!match) return null;
|
|
17563
|
+
const resp = await fetch(match.url);
|
|
17564
|
+
if (!resp.ok) return null;
|
|
17565
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
17566
|
+
} catch {
|
|
17567
|
+
return null;
|
|
17568
|
+
}
|
|
17569
|
+
}
|
|
17570
|
+
};
|
|
17571
|
+
|
|
17471
17572
|
// src/extractor/InstagramContentExtractor.ts
|
|
17472
17573
|
var CONTENT_PATH_RE = /^\/(?:[^/]+\/)?(p|reel|tv)\/([A-Za-z0-9_-]+)\/?/i;
|
|
17473
17574
|
var CDN_HOST_RE = /(^|\.)((cdninstagram\.com)|(scontent[^/]*\.cdninstagram\.com))$/i;
|
|
@@ -18069,9 +18170,9 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
18069
18170
|
downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
18070
18171
|
const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
|
|
18071
18172
|
if (isMuxInput) {
|
|
18072
|
-
if (!muxWorkDir) muxWorkDir = mkdtempSync(
|
|
18073
|
-
const localPath =
|
|
18074
|
-
|
|
18173
|
+
if (!muxWorkDir) muxWorkDir = mkdtempSync(join5(tmpdir(), "ig-mux-"));
|
|
18174
|
+
const localPath = join5(muxWorkDir, trackFilename(track, index, extraction.shortcode));
|
|
18175
|
+
writeFileSync2(localPath, bytes);
|
|
18075
18176
|
muxInputs.set(track.url, localPath);
|
|
18076
18177
|
}
|
|
18077
18178
|
} catch (err) {
|
|
@@ -18079,7 +18180,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
18079
18180
|
}
|
|
18080
18181
|
}
|
|
18081
18182
|
if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
|
|
18082
|
-
const outPath =
|
|
18183
|
+
const outPath = join5(muxWorkDir, "muxed.mp4");
|
|
18083
18184
|
const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
|
|
18084
18185
|
if (muxed.ok) {
|
|
18085
18186
|
const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
|
|
@@ -20522,7 +20623,7 @@ import { readFile as readFile2 } from "fs/promises";
|
|
|
20522
20623
|
import { Hono as Hono18 } from "hono";
|
|
20523
20624
|
import { z as z26 } from "zod";
|
|
20524
20625
|
var workflowApp = new Hono18();
|
|
20525
|
-
var WorkflowInputSchema = z26.record(z26.unknown()).default({});
|
|
20626
|
+
var WorkflowInputSchema = z26.record(z26.string(), z26.unknown()).default({});
|
|
20526
20627
|
var WorkflowIdSchema = z26.string().min(1);
|
|
20527
20628
|
var CadenceSchema = z26.enum(["daily", "weekly", "monthly"]);
|
|
20528
20629
|
var ScheduleStatusSchema = z26.enum(["active", "paused"]);
|
|
@@ -22061,7 +22162,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
22061
22162
|
|
|
22062
22163
|
// src/mcp/mcp-routes.ts
|
|
22063
22164
|
import { Hono as Hono20 } from "hono";
|
|
22064
|
-
import {
|
|
22165
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
22065
22166
|
|
|
22066
22167
|
// src/mcp/memory-mcp-tool-executor.ts
|
|
22067
22168
|
var MemoryMcpToolExecutor = class {
|
|
@@ -22099,6 +22200,58 @@ var MemoryMcpToolExecutor = class {
|
|
|
22099
22200
|
}
|
|
22100
22201
|
};
|
|
22101
22202
|
|
|
22203
|
+
// src/api/image-source-artifacts.ts
|
|
22204
|
+
import { randomUUID as randomUUID9 } from "crypto";
|
|
22205
|
+
var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
22206
|
+
var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
22207
|
+
var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
22208
|
+
var IMAGE_SOURCE_MAX_BYTES = 20 * 1024 * 1024;
|
|
22209
|
+
var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
|
|
22210
|
+
["image/jpeg", "jpg"],
|
|
22211
|
+
["image/png", "png"],
|
|
22212
|
+
["image/webp", "webp"],
|
|
22213
|
+
["image/gif", "gif"]
|
|
22214
|
+
]);
|
|
22215
|
+
function token() {
|
|
22216
|
+
return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
22217
|
+
}
|
|
22218
|
+
function policy() {
|
|
22219
|
+
return {
|
|
22220
|
+
prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
|
|
22221
|
+
artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
|
|
22222
|
+
downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
|
|
22223
|
+
token: token()
|
|
22224
|
+
};
|
|
22225
|
+
}
|
|
22226
|
+
function imageSourceArtifactOwnerId(artifactId) {
|
|
22227
|
+
return privateArtifactOwnerId(artifactId, IMAGE_SOURCE_ARTIFACT_PREFIX);
|
|
22228
|
+
}
|
|
22229
|
+
async function createImageSourceArtifact(args) {
|
|
22230
|
+
const extension = ALLOWED_IMAGE_TYPES.get(args.contentType);
|
|
22231
|
+
if (!extension) throw new Error("image_source_type_unsupported");
|
|
22232
|
+
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
22233
|
+
throw new Error("image_source_size_invalid");
|
|
22234
|
+
}
|
|
22235
|
+
const id = randomUUID9().replaceAll("-", "");
|
|
22236
|
+
return createPrivateArtifact({
|
|
22237
|
+
policy: policy(),
|
|
22238
|
+
ownerId: args.ownerId,
|
|
22239
|
+
artifactKey: `${id}.${extension}`,
|
|
22240
|
+
createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
|
|
22241
|
+
filename: args.filename ?? `mcp-scraper-image-${id.slice(0, 12)}.${extension}`,
|
|
22242
|
+
contentType: args.contentType,
|
|
22243
|
+
content: args.content
|
|
22244
|
+
});
|
|
22245
|
+
}
|
|
22246
|
+
async function readOwnedImageSourceArtifact(args) {
|
|
22247
|
+
if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
22248
|
+
return readPrivateArtifactBuffer({
|
|
22249
|
+
policy: policy(),
|
|
22250
|
+
artifactId: args.artifactId,
|
|
22251
|
+
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
22252
|
+
});
|
|
22253
|
+
}
|
|
22254
|
+
|
|
22102
22255
|
// src/mcp/memory-inprocess-executor.ts
|
|
22103
22256
|
var CUTOVER = /* @__PURE__ */ new Map();
|
|
22104
22257
|
function registerCutover(upstreamName, handler) {
|
|
@@ -22142,6 +22295,32 @@ var HybridMemoryToolExecutor = class {
|
|
|
22142
22295
|
};
|
|
22143
22296
|
}
|
|
22144
22297
|
}
|
|
22298
|
+
if (upstreamName === "image_asset_save" && typeof args.artifactId === "string") {
|
|
22299
|
+
const image = await readOwnedImageSourceArtifact({
|
|
22300
|
+
artifactId: args.artifactId,
|
|
22301
|
+
ownerId: String(this.user.id)
|
|
22302
|
+
});
|
|
22303
|
+
if (!image) {
|
|
22304
|
+
const result = {
|
|
22305
|
+
ok: false,
|
|
22306
|
+
code: "image_artifact_not_found",
|
|
22307
|
+
error: "image artifact not found"
|
|
22308
|
+
};
|
|
22309
|
+
return {
|
|
22310
|
+
content: [{ type: "text", text: JSON.stringify(result) }],
|
|
22311
|
+
structuredContent: result,
|
|
22312
|
+
isError: false
|
|
22313
|
+
};
|
|
22314
|
+
}
|
|
22315
|
+
const sourceRef = typeof args.sourceRef === "object" && args.sourceRef !== null && !Array.isArray(args.sourceRef) ? args.sourceRef : {};
|
|
22316
|
+
args = {
|
|
22317
|
+
...args,
|
|
22318
|
+
artifactId: void 0,
|
|
22319
|
+
imageBase64: image.toString("base64"),
|
|
22320
|
+
sourceRef: { ...sourceRef, artifactId: args.artifactId }
|
|
22321
|
+
};
|
|
22322
|
+
delete args.artifactId;
|
|
22323
|
+
}
|
|
22145
22324
|
const handler = CUTOVER.get(upstreamName);
|
|
22146
22325
|
if (!handler) return this.httpExecutor.callMemoryTool(upstreamName, args);
|
|
22147
22326
|
const mkKey = await this.resolveMkKey();
|
|
@@ -22652,8 +22831,8 @@ function protectedResourceMetadata() {
|
|
|
22652
22831
|
function wwwAuthenticateValue() {
|
|
22653
22832
|
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
22654
22833
|
}
|
|
22655
|
-
function isBearerJwt(
|
|
22656
|
-
return !
|
|
22834
|
+
function isBearerJwt(token4) {
|
|
22835
|
+
return !token4.startsWith("sk_") && token4.split(".").length === 3;
|
|
22657
22836
|
}
|
|
22658
22837
|
var jwksSingleton = null;
|
|
22659
22838
|
function getJwks() {
|
|
@@ -22661,8 +22840,8 @@ function getJwks() {
|
|
|
22661
22840
|
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
22662
22841
|
return jwksSingleton;
|
|
22663
22842
|
}
|
|
22664
|
-
async function verifyScraperOAuthJwt(
|
|
22665
|
-
const { payload } = await jwtVerify(
|
|
22843
|
+
async function verifyScraperOAuthJwt(token4) {
|
|
22844
|
+
const { payload } = await jwtVerify(token4, getJwks(), {
|
|
22666
22845
|
issuer: oauthIssuer(),
|
|
22667
22846
|
audience: oauthResourceUrl()
|
|
22668
22847
|
});
|
|
@@ -22717,37 +22896,91 @@ async function requireMcpCallerKey(c) {
|
|
|
22717
22896
|
return { key: callerKey, user };
|
|
22718
22897
|
}
|
|
22719
22898
|
var mcpApp = new Hono20();
|
|
22899
|
+
function userFromAuthInfo(authInfo) {
|
|
22900
|
+
const user = authInfo?.extra?.user;
|
|
22901
|
+
if (!user || typeof user !== "object" || Array.isArray(user)) {
|
|
22902
|
+
throw new Error("Authenticated MCP user context is missing.");
|
|
22903
|
+
}
|
|
22904
|
+
return user;
|
|
22905
|
+
}
|
|
22906
|
+
var mcpHandler = createMcpHandler(async ({ authInfo }) => {
|
|
22907
|
+
const user = userFromAuthInfo(authInfo);
|
|
22908
|
+
const callerKey = authInfo?.token ?? "";
|
|
22909
|
+
if (!callerKey) throw new Error("Authenticated MCP token is missing.");
|
|
22910
|
+
const baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? "https://mcpscraper.dev";
|
|
22911
|
+
const executor = new HttpMcpToolExecutor(baseUrl, callerKey);
|
|
22912
|
+
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
22913
|
+
const server = buildPaaExtractorMcpServer(executor, {
|
|
22914
|
+
savesReportsLocally: false,
|
|
22915
|
+
ownerId: hashOwnerId(callerKey),
|
|
22916
|
+
deploymentProfile: "production",
|
|
22917
|
+
transportProfile: "hosted_http",
|
|
22918
|
+
baseUrl,
|
|
22919
|
+
taskHandleSecret: callerKey
|
|
22920
|
+
});
|
|
22921
|
+
registerSerpIntelligenceCaptureTools(server, executor, { exposeDevelopmentDiagnostics: false });
|
|
22922
|
+
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
22923
|
+
registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, callerKey));
|
|
22924
|
+
const memoryExecutor = new HybridMemoryToolExecutor(new MemoryMcpToolExecutor(baseUrl, callerKey), user);
|
|
22925
|
+
registerMemoryMcpTools(server, memoryExecutor);
|
|
22926
|
+
return server;
|
|
22927
|
+
}, {
|
|
22928
|
+
legacy: "stateless",
|
|
22929
|
+
responseMode: "auto",
|
|
22930
|
+
onerror(error) {
|
|
22931
|
+
console.error("[mcp] protocol error", error.message);
|
|
22932
|
+
}
|
|
22933
|
+
});
|
|
22934
|
+
function allowedOrigin(request) {
|
|
22935
|
+
const origin = request.headers.get("origin");
|
|
22936
|
+
if (origin === null) return true;
|
|
22937
|
+
const allowed = /* @__PURE__ */ new Set([
|
|
22938
|
+
"https://mcpscraper.dev",
|
|
22939
|
+
"https://www.mcpscraper.dev",
|
|
22940
|
+
...process.env.APP_ORIGIN?.trim() ? [process.env.APP_ORIGIN.trim()] : []
|
|
22941
|
+
]);
|
|
22942
|
+
return allowed.has(origin);
|
|
22943
|
+
}
|
|
22720
22944
|
mcpApp.all("/", async (c) => {
|
|
22721
22945
|
try {
|
|
22946
|
+
if (!allowedOrigin(c.req.raw)) {
|
|
22947
|
+
return c.json({ jsonrpc: "2.0", id: null, error: { code: -32e3, message: "Forbidden Origin" } }, 403);
|
|
22948
|
+
}
|
|
22722
22949
|
const keyOrError = await requireMcpCallerKey(c);
|
|
22723
22950
|
if (keyOrError instanceof Response) return keyOrError;
|
|
22724
22951
|
const { key: callerKey, user } = keyOrError;
|
|
22725
|
-
const
|
|
22726
|
-
|
|
22727
|
-
|
|
22728
|
-
|
|
22729
|
-
|
|
22952
|
+
const taskResponse = await handleMcpTasksHttpRequest(
|
|
22953
|
+
c.req.raw,
|
|
22954
|
+
new HttpMcpToolExecutor(
|
|
22955
|
+
process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? "https://mcpscraper.dev",
|
|
22956
|
+
callerKey
|
|
22957
|
+
),
|
|
22958
|
+
{
|
|
22959
|
+
ownerId: hashOwnerId(callerKey),
|
|
22960
|
+
secret: callerKey,
|
|
22961
|
+
serverInfo: { name: "mcp-scraper", version: PACKAGE_VERSION }
|
|
22962
|
+
}
|
|
22963
|
+
);
|
|
22964
|
+
if (taskResponse) return taskResponse;
|
|
22965
|
+
return mcpHandler.fetch(c.req.raw, {
|
|
22966
|
+
authInfo: {
|
|
22967
|
+
token: callerKey,
|
|
22968
|
+
clientId: `mcp-scraper-user-${user.id}`,
|
|
22969
|
+
scopes: ["mcp:tools"],
|
|
22970
|
+
extra: { user }
|
|
22971
|
+
}
|
|
22730
22972
|
});
|
|
22731
|
-
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
22732
|
-
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
|
|
22733
|
-
registerSerpIntelligenceCaptureTools(server, executor);
|
|
22734
|
-
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
22735
|
-
registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, callerKey));
|
|
22736
|
-
const memoryExecutor = new HybridMemoryToolExecutor(new MemoryMcpToolExecutor(baseUrl, callerKey), user);
|
|
22737
|
-
registerMemoryMcpTools(server, memoryExecutor);
|
|
22738
|
-
await server.connect(transport);
|
|
22739
|
-
return transport.handleRequest(c.req.raw);
|
|
22740
22973
|
} catch {
|
|
22741
22974
|
return c.json({ error: "Internal Server Error" }, 500);
|
|
22742
22975
|
}
|
|
22743
22976
|
});
|
|
22744
22977
|
|
|
22745
22978
|
// src/api/browser-agent-routes.ts
|
|
22746
|
-
import { randomUUID as
|
|
22979
|
+
import { randomUUID as randomUUID11 } from "crypto";
|
|
22747
22980
|
import { Hono as Hono21 } from "hono";
|
|
22748
22981
|
|
|
22749
22982
|
// src/api/browser-agent-db.ts
|
|
22750
|
-
import { randomUUID as
|
|
22983
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
22751
22984
|
var _ready = false;
|
|
22752
22985
|
var _migrationPromise = null;
|
|
22753
22986
|
async function migrateBrowserAgent() {
|
|
@@ -22878,7 +23111,7 @@ async function runBrowserAgentMigration() {
|
|
|
22878
23111
|
}
|
|
22879
23112
|
async function createExtensionRow(input) {
|
|
22880
23113
|
const db = getDb();
|
|
22881
|
-
const id = `bext_${
|
|
23114
|
+
const id = `bext_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
22882
23115
|
await db.execute({
|
|
22883
23116
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
22884
23117
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -22913,7 +23146,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
22913
23146
|
}
|
|
22914
23147
|
async function createAuthConnectionRow(input) {
|
|
22915
23148
|
const db = getDb();
|
|
22916
|
-
const connectionId = `authc_${
|
|
23149
|
+
const connectionId = `authc_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
22917
23150
|
await db.execute({
|
|
22918
23151
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
22919
23152
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -22998,7 +23231,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
22998
23231
|
}
|
|
22999
23232
|
async function createSessionRow(input) {
|
|
23000
23233
|
const db = getDb();
|
|
23001
|
-
const id = `bas_${
|
|
23234
|
+
const id = `bas_${randomUUID10().replace(/-/g, "").slice(0, 20)}`;
|
|
23002
23235
|
await db.execute({
|
|
23003
23236
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
23004
23237
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -23067,7 +23300,7 @@ async function recordAction(input) {
|
|
|
23067
23300
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
23068
23301
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
23069
23302
|
args: [
|
|
23070
|
-
`baa_${
|
|
23303
|
+
`baa_${randomUUID10().replace(/-/g, "").slice(0, 20)}`,
|
|
23071
23304
|
input.sessionId,
|
|
23072
23305
|
input.type,
|
|
23073
23306
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -24798,7 +25031,7 @@ function buildBrowserAgentRoutes() {
|
|
|
24798
25031
|
}
|
|
24799
25032
|
const existing = await getExtensionRow(user.id, name);
|
|
24800
25033
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
24801
|
-
const backendName = `u${user.id}_${
|
|
25034
|
+
const backendName = `u${user.id}_${randomUUID11().replace(/-/g, "")}`;
|
|
24802
25035
|
try {
|
|
24803
25036
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
24804
25037
|
const row = await createExtensionRow({
|
|
@@ -25292,7 +25525,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
25292
25525
|
// src/api/oauth-routes.ts
|
|
25293
25526
|
import { Hono as Hono23 } from "hono";
|
|
25294
25527
|
import { getCookie, setCookie } from "hono/cookie";
|
|
25295
|
-
import { createHash as createHash8, randomBytes as randomBytes2, randomUUID as
|
|
25528
|
+
import { createHash as createHash8, randomBytes as randomBytes2, randomUUID as randomUUID12 } from "crypto";
|
|
25296
25529
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
25297
25530
|
var ISSUER = "https://mcpscraper.dev";
|
|
25298
25531
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -25366,9 +25599,9 @@ function negotiateScope(requested, user, resource) {
|
|
|
25366
25599
|
if (operator && !unique2.includes("memory.admin")) unique2.push("memory.admin");
|
|
25367
25600
|
return unique2.join(" ");
|
|
25368
25601
|
}
|
|
25369
|
-
async function sessionUser(
|
|
25370
|
-
if (!
|
|
25371
|
-
const userId = verifySession(
|
|
25602
|
+
async function sessionUser(token4) {
|
|
25603
|
+
if (!token4) return null;
|
|
25604
|
+
const userId = verifySession(token4);
|
|
25372
25605
|
if (!userId) return null;
|
|
25373
25606
|
const user = await getUserById(userId);
|
|
25374
25607
|
return user ?? null;
|
|
@@ -25640,7 +25873,7 @@ function pkceMatches(verifier, challenge) {
|
|
|
25640
25873
|
}
|
|
25641
25874
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
25642
25875
|
const { privateKey, kid } = await getKeys();
|
|
25643
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
25876
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID12()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
25644
25877
|
}
|
|
25645
25878
|
function tokenErrorResponse(c, error, description, status) {
|
|
25646
25879
|
return c.json({ error, error_description: description }, status);
|
|
@@ -25715,35 +25948,35 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
25715
25948
|
// src/api/chat-routes.ts
|
|
25716
25949
|
import { Hono as Hono24 } from "hono";
|
|
25717
25950
|
var chatApp = new Hono24();
|
|
25718
|
-
function tokenOk(
|
|
25719
|
-
return !!
|
|
25951
|
+
function tokenOk(token4) {
|
|
25952
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 10;
|
|
25720
25953
|
}
|
|
25721
25954
|
function jsonError(message, status = 400) {
|
|
25722
25955
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
25723
25956
|
}
|
|
25724
25957
|
chatApp.get("/api/channels", async (c) => {
|
|
25725
|
-
const
|
|
25726
|
-
if (!tokenOk(
|
|
25727
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
25958
|
+
const token4 = c.req.query("token");
|
|
25959
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25960
|
+
const r = await memoryCall("listVaultsTool", {}, token4);
|
|
25728
25961
|
if (!r.ok) return c.json(r);
|
|
25729
25962
|
const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
|
|
25730
25963
|
return c.json({ ok: true, channels: vaults });
|
|
25731
25964
|
});
|
|
25732
25965
|
chatApp.get("/api/messages", async (c) => {
|
|
25733
|
-
const
|
|
25734
|
-
if (!tokenOk(
|
|
25966
|
+
const token4 = c.req.query("token");
|
|
25967
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25735
25968
|
const vault = c.req.query("vault");
|
|
25736
25969
|
if (!vault) return jsonError("vault is required");
|
|
25737
25970
|
const parentMessageId = c.req.query("parentMessageId") || void 0;
|
|
25738
|
-
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId },
|
|
25971
|
+
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token4);
|
|
25739
25972
|
return c.json(r);
|
|
25740
25973
|
});
|
|
25741
25974
|
chatApp.get("/api/poll", async (c) => {
|
|
25742
|
-
const
|
|
25743
|
-
if (!tokenOk(
|
|
25975
|
+
const token4 = c.req.query("token");
|
|
25976
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25744
25977
|
const vault = c.req.query("vault");
|
|
25745
25978
|
if (!vault) return jsonError("vault is required");
|
|
25746
|
-
const r = await memoryCall("pollChannelTool", { vault },
|
|
25979
|
+
const r = await memoryCall("pollChannelTool", { vault }, token4);
|
|
25747
25980
|
return c.json(r);
|
|
25748
25981
|
});
|
|
25749
25982
|
function parseAttachNote(v) {
|
|
@@ -25768,12 +26001,12 @@ chatApp.post("/api/reply", async (c) => {
|
|
|
25768
26001
|
return c.json(r);
|
|
25769
26002
|
});
|
|
25770
26003
|
chatApp.get("/api/notes-own", async (c) => {
|
|
25771
|
-
const
|
|
25772
|
-
if (!tokenOk(
|
|
25773
|
-
const vaultsRes = await memoryCall("listVaultsTool", {},
|
|
26004
|
+
const token4 = c.req.query("token");
|
|
26005
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
26006
|
+
const vaultsRes = await memoryCall("listVaultsTool", {}, token4);
|
|
25774
26007
|
if (!vaultsRes.ok) return c.json(vaultsRes);
|
|
25775
26008
|
const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
|
|
25776
|
-
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle },
|
|
26009
|
+
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token4)));
|
|
25777
26010
|
const notes = ownVaults.flatMap((v, i) => {
|
|
25778
26011
|
const r = lists[i];
|
|
25779
26012
|
return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
|
|
@@ -25781,12 +26014,12 @@ chatApp.get("/api/notes-own", async (c) => {
|
|
|
25781
26014
|
return c.json({ ok: true, notes });
|
|
25782
26015
|
});
|
|
25783
26016
|
chatApp.get("/api/message-note", async (c) => {
|
|
25784
|
-
const
|
|
25785
|
-
if (!tokenOk(
|
|
26017
|
+
const token4 = c.req.query("token");
|
|
26018
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25786
26019
|
const vault = c.req.query("vault");
|
|
25787
26020
|
const messageId = c.req.query("messageId");
|
|
25788
26021
|
if (!vault || !messageId) return jsonError("vault and messageId are required");
|
|
25789
|
-
const r = await memoryCall("getMessageNoteTool", { vault, messageId },
|
|
26022
|
+
const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token4);
|
|
25790
26023
|
return c.json(r);
|
|
25791
26024
|
});
|
|
25792
26025
|
chatApp.post("/api/react", async (c) => {
|
|
@@ -25797,17 +26030,17 @@ chatApp.post("/api/react", async (c) => {
|
|
|
25797
26030
|
return c.json(r);
|
|
25798
26031
|
});
|
|
25799
26032
|
chatApp.get("/api/mentions", async (c) => {
|
|
25800
|
-
const
|
|
25801
|
-
if (!tokenOk(
|
|
25802
|
-
const r = await memoryCall("myMentionsTool", {},
|
|
26033
|
+
const token4 = c.req.query("token");
|
|
26034
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
26035
|
+
const r = await memoryCall("myMentionsTool", {}, token4);
|
|
25803
26036
|
return c.json(r);
|
|
25804
26037
|
});
|
|
25805
26038
|
chatApp.get("/api/members", async (c) => {
|
|
25806
|
-
const
|
|
25807
|
-
if (!tokenOk(
|
|
26039
|
+
const token4 = c.req.query("token");
|
|
26040
|
+
if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
|
|
25808
26041
|
const vault = c.req.query("vault");
|
|
25809
26042
|
if (!vault) return jsonError("vault is required");
|
|
25810
|
-
const r = await memoryCall("listChannelMembersTool", { vault },
|
|
26043
|
+
const r = await memoryCall("listChannelMembersTool", { vault }, token4);
|
|
25811
26044
|
return c.json(r);
|
|
25812
26045
|
});
|
|
25813
26046
|
chatApp.post("/api/channel", async (c) => {
|
|
@@ -25843,13 +26076,13 @@ chatApp.post("/api/member-remove", async (c) => {
|
|
|
25843
26076
|
return c.json(r);
|
|
25844
26077
|
});
|
|
25845
26078
|
chatApp.get("/:token", (c) => {
|
|
25846
|
-
const
|
|
25847
|
-
if (!tokenOk(
|
|
26079
|
+
const token4 = c.req.param("token");
|
|
26080
|
+
if (!tokenOk(token4)) {
|
|
25848
26081
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
|
|
25849
26082
|
}
|
|
25850
|
-
return c.html(renderChatPage(
|
|
26083
|
+
return c.html(renderChatPage(token4));
|
|
25851
26084
|
});
|
|
25852
|
-
function renderChatPage(
|
|
26085
|
+
function renderChatPage(token4) {
|
|
25853
26086
|
return `<!doctype html>
|
|
25854
26087
|
<html>
|
|
25855
26088
|
<head>
|
|
@@ -26034,7 +26267,7 @@ function renderChatPage(token3) {
|
|
|
26034
26267
|
</div>
|
|
26035
26268
|
|
|
26036
26269
|
<script>
|
|
26037
|
-
const TOKEN = ${JSON.stringify(
|
|
26270
|
+
const TOKEN = ${JSON.stringify(token4)};
|
|
26038
26271
|
const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
|
|
26039
26272
|
|
|
26040
26273
|
function api(path, opts) {
|
|
@@ -26404,8 +26637,8 @@ var MAX_VAULT_RECORDS = 200;
|
|
|
26404
26637
|
var MAX_MEMORY_READ_CONCURRENCY = 4;
|
|
26405
26638
|
var MAX_MEMORY_READ_ATTEMPTS = 3;
|
|
26406
26639
|
var vaultApp = new Hono25();
|
|
26407
|
-
function validToken(
|
|
26408
|
-
return !!
|
|
26640
|
+
function validToken(token4) {
|
|
26641
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 20 && token4.length < 256;
|
|
26409
26642
|
}
|
|
26410
26643
|
function isVault(value) {
|
|
26411
26644
|
return typeof value === "string" && VAULTS.includes(value);
|
|
@@ -26531,11 +26764,11 @@ function shouldRetryMemoryRead(error) {
|
|
|
26531
26764
|
function sleep(ms) {
|
|
26532
26765
|
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
26533
26766
|
}
|
|
26534
|
-
async function readMemory(tool, args,
|
|
26535
|
-
let result = await memoryCall(tool, args,
|
|
26767
|
+
async function readMemory(tool, args, token4) {
|
|
26768
|
+
let result = await memoryCall(tool, args, token4);
|
|
26536
26769
|
for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
|
|
26537
26770
|
await sleep(attempt * 150);
|
|
26538
|
-
result = await memoryCall(tool, args,
|
|
26771
|
+
result = await memoryCall(tool, args, token4);
|
|
26539
26772
|
}
|
|
26540
26773
|
return result;
|
|
26541
26774
|
}
|
|
@@ -26552,27 +26785,27 @@ async function mapWithConcurrency3(items, limit, map) {
|
|
|
26552
26785
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
26553
26786
|
return values;
|
|
26554
26787
|
}
|
|
26555
|
-
async function readVault(vault,
|
|
26556
|
-
const list = await readMemory("listTool", { vault },
|
|
26788
|
+
async function readVault(vault, token4) {
|
|
26789
|
+
const list = await readMemory("listTool", { vault }, token4);
|
|
26557
26790
|
if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
|
|
26558
26791
|
const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
|
|
26559
26792
|
const notes = await mapWithConcurrency3(
|
|
26560
26793
|
noteRefs,
|
|
26561
26794
|
MAX_MEMORY_READ_CONCURRENCY,
|
|
26562
|
-
({ path: path5 }) => readMemory("getTool", { vault, path: path5 },
|
|
26795
|
+
({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token4)
|
|
26563
26796
|
);
|
|
26564
26797
|
const failed = notes.find((result) => !result.ok);
|
|
26565
26798
|
if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
|
|
26566
26799
|
return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
|
|
26567
26800
|
}
|
|
26568
26801
|
vaultApp.get("/api/workspace", async (c) => {
|
|
26569
|
-
const
|
|
26570
|
-
if (!validToken(
|
|
26802
|
+
const token4 = c.req.query("token");
|
|
26803
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26571
26804
|
try {
|
|
26572
26805
|
const requested = c.req.query("vault");
|
|
26573
26806
|
const vaults = isVault(requested) ? [requested] : VAULTS;
|
|
26574
26807
|
const results = [];
|
|
26575
|
-
for (const vault of vaults) results.push(await readVault(vault,
|
|
26808
|
+
for (const vault of vaults) results.push(await readVault(vault, token4));
|
|
26576
26809
|
const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
26577
26810
|
return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
|
|
26578
26811
|
} catch (error) {
|
|
@@ -26581,8 +26814,8 @@ vaultApp.get("/api/workspace", async (c) => {
|
|
|
26581
26814
|
});
|
|
26582
26815
|
vaultApp.post("/api/records", async (c) => {
|
|
26583
26816
|
const body = await c.req.json().catch(() => ({}));
|
|
26584
|
-
const
|
|
26585
|
-
if (!validToken(
|
|
26817
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26818
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26586
26819
|
const vault = body.vault;
|
|
26587
26820
|
const path5 = stringValue(body.path, 320);
|
|
26588
26821
|
const title = stringValue(body.title, 320);
|
|
@@ -26593,19 +26826,19 @@ vaultApp.post("/api/records", async (c) => {
|
|
|
26593
26826
|
const validationError = validateNewRecord(vault, normalizedProps);
|
|
26594
26827
|
if (validationError) return c.json({ ok: false, error: validationError }, 400);
|
|
26595
26828
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26596
|
-
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } },
|
|
26829
|
+
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token4);
|
|
26597
26830
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26598
26831
|
});
|
|
26599
26832
|
vaultApp.post("/api/records/transition", async (c) => {
|
|
26600
26833
|
const body = await c.req.json().catch(() => ({}));
|
|
26601
|
-
const
|
|
26602
|
-
if (!validToken(
|
|
26834
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26835
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26603
26836
|
const vault = typeof body.vault === "string" ? body.vault : "";
|
|
26604
26837
|
const path5 = typeof body.path === "string" ? body.path : "";
|
|
26605
26838
|
const status = stringValue(body.status, 80);
|
|
26606
26839
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26607
26840
|
if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
|
|
26608
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26841
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26609
26842
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26610
26843
|
const result = await memoryCall("putTool", {
|
|
26611
26844
|
vault,
|
|
@@ -26614,13 +26847,13 @@ vaultApp.post("/api/records/transition", async (c) => {
|
|
|
26614
26847
|
content: current.note.content,
|
|
26615
26848
|
props: { ...current.note.props ?? {}, status },
|
|
26616
26849
|
baseRevision: baseRevision ?? current.note.revision
|
|
26617
|
-
},
|
|
26850
|
+
}, token4);
|
|
26618
26851
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26619
26852
|
});
|
|
26620
26853
|
vaultApp.post("/api/records/update", async (c) => {
|
|
26621
26854
|
const body = await c.req.json().catch(() => ({}));
|
|
26622
|
-
const
|
|
26623
|
-
if (!validToken(
|
|
26855
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26856
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26624
26857
|
const vault = body.vault;
|
|
26625
26858
|
const path5 = stringValue(body.path, 320);
|
|
26626
26859
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -26630,7 +26863,7 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
26630
26863
|
if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
|
|
26631
26864
|
return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
|
|
26632
26865
|
}
|
|
26633
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26866
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26634
26867
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26635
26868
|
const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
|
|
26636
26869
|
const validationError = validateNewRecord(vault, nextProps);
|
|
@@ -26642,18 +26875,18 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
26642
26875
|
content: content ?? current.note.content,
|
|
26643
26876
|
props: nextProps,
|
|
26644
26877
|
baseRevision: baseRevision ?? current.note.revision
|
|
26645
|
-
},
|
|
26878
|
+
}, token4);
|
|
26646
26879
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26647
26880
|
});
|
|
26648
26881
|
vaultApp.delete("/api/records", async (c) => {
|
|
26649
26882
|
const body = await c.req.json().catch(() => ({}));
|
|
26650
|
-
const
|
|
26651
|
-
if (!validToken(
|
|
26883
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26884
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26652
26885
|
const vault = body.vault;
|
|
26653
26886
|
const path5 = stringValue(body.path, 320);
|
|
26654
26887
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
26655
26888
|
if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
|
|
26656
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
26889
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token4);
|
|
26657
26890
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26658
26891
|
if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
|
|
26659
26892
|
return c.json({ ok: false, error: "only task records can be deleted" }, 400);
|
|
@@ -26662,13 +26895,13 @@ vaultApp.delete("/api/records", async (c) => {
|
|
|
26662
26895
|
vault,
|
|
26663
26896
|
path: path5,
|
|
26664
26897
|
baseRevision: baseRevision ?? current.note.revision
|
|
26665
|
-
},
|
|
26898
|
+
}, token4);
|
|
26666
26899
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26667
26900
|
});
|
|
26668
26901
|
vaultApp.post("/api/communications/review", async (c) => {
|
|
26669
26902
|
const body = await c.req.json().catch(() => ({}));
|
|
26670
|
-
const
|
|
26671
|
-
if (!validToken(
|
|
26903
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26904
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26672
26905
|
const path5 = stringValue(body.path, 320);
|
|
26673
26906
|
const action = stringValue(body.action, 24);
|
|
26674
26907
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -26679,7 +26912,7 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
26679
26912
|
if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
|
|
26680
26913
|
return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
|
|
26681
26914
|
}
|
|
26682
|
-
const current = await readMemory("getTool", { vault: "Communications", path: path5 },
|
|
26915
|
+
const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token4);
|
|
26683
26916
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
26684
26917
|
if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
|
|
26685
26918
|
const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
|
|
@@ -26703,13 +26936,13 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
26703
26936
|
content: content ?? current.note.content,
|
|
26704
26937
|
props,
|
|
26705
26938
|
baseRevision: baseRevision ?? current.note.revision
|
|
26706
|
-
},
|
|
26939
|
+
}, token4);
|
|
26707
26940
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
26708
26941
|
});
|
|
26709
26942
|
vaultApp.post("/api/activity", async (c) => {
|
|
26710
26943
|
const body = await c.req.json().catch(() => ({}));
|
|
26711
|
-
const
|
|
26712
|
-
if (!validToken(
|
|
26944
|
+
const token4 = typeof body.token === "string" ? body.token : void 0;
|
|
26945
|
+
if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
26713
26946
|
const title = stringValue(body.title, 320);
|
|
26714
26947
|
const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
|
|
26715
26948
|
const props = propsValue(body.props) ?? {};
|
|
@@ -26719,7 +26952,7 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
26719
26952
|
return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
|
|
26720
26953
|
}
|
|
26721
26954
|
for (const ref of personRefs) {
|
|
26722
|
-
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") },
|
|
26955
|
+
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token4);
|
|
26723
26956
|
if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
|
|
26724
26957
|
return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
|
|
26725
26958
|
}
|
|
@@ -26747,37 +26980,37 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
26747
26980
|
...props,
|
|
26748
26981
|
person_refs: personRefs
|
|
26749
26982
|
}
|
|
26750
|
-
},
|
|
26983
|
+
}, token4);
|
|
26751
26984
|
return c.json(result, result.ok ? 200 : 502);
|
|
26752
26985
|
});
|
|
26753
26986
|
|
|
26754
26987
|
// src/api/schedule-routes.ts
|
|
26755
26988
|
import { Hono as Hono26 } from "hono";
|
|
26756
26989
|
var scheduleApp = new Hono26();
|
|
26757
|
-
function tokenOk2(
|
|
26758
|
-
return !!
|
|
26990
|
+
function tokenOk2(token4) {
|
|
26991
|
+
return !!token4 && token4.startsWith("mk_") && token4.length > 10;
|
|
26759
26992
|
}
|
|
26760
26993
|
function jsonError2(message, status = 400) {
|
|
26761
26994
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
26762
26995
|
}
|
|
26763
26996
|
scheduleApp.get("/api/status", async (c) => {
|
|
26764
|
-
const
|
|
26765
|
-
if (!tokenOk2(
|
|
26766
|
-
const r = await memoryCall("getScheduleStatusTool", {},
|
|
26997
|
+
const token4 = c.req.query("token");
|
|
26998
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
26999
|
+
const r = await memoryCall("getScheduleStatusTool", {}, token4);
|
|
26767
27000
|
return c.json(r);
|
|
26768
27001
|
});
|
|
26769
27002
|
scheduleApp.get("/api/vaults", async (c) => {
|
|
26770
|
-
const
|
|
26771
|
-
if (!tokenOk2(
|
|
26772
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
27003
|
+
const token4 = c.req.query("token");
|
|
27004
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
27005
|
+
const r = await memoryCall("listVaultsTool", {}, token4);
|
|
26773
27006
|
if (!r.ok) return c.json(r);
|
|
26774
27007
|
const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
|
|
26775
27008
|
return c.json({ ok: true, vaults });
|
|
26776
27009
|
});
|
|
26777
27010
|
scheduleApp.get("/api/actions", async (c) => {
|
|
26778
|
-
const
|
|
26779
|
-
if (!tokenOk2(
|
|
26780
|
-
const r = await memoryCall("listScheduledActionsTool", {},
|
|
27011
|
+
const token4 = c.req.query("token");
|
|
27012
|
+
if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
|
|
27013
|
+
const r = await memoryCall("listScheduledActionsTool", {}, token4);
|
|
26781
27014
|
return c.json(r);
|
|
26782
27015
|
});
|
|
26783
27016
|
scheduleApp.post("/api/propose", async (c) => {
|
|
@@ -26813,13 +27046,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
|
|
|
26813
27046
|
return c.json(r);
|
|
26814
27047
|
});
|
|
26815
27048
|
scheduleApp.get("/:token", (c) => {
|
|
26816
|
-
const
|
|
26817
|
-
if (!tokenOk2(
|
|
27049
|
+
const token4 = c.req.param("token");
|
|
27050
|
+
if (!tokenOk2(token4)) {
|
|
26818
27051
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
|
|
26819
27052
|
}
|
|
26820
|
-
return c.html(renderSchedulePage(
|
|
27053
|
+
return c.html(renderSchedulePage(token4));
|
|
26821
27054
|
});
|
|
26822
|
-
function renderSchedulePage(
|
|
27055
|
+
function renderSchedulePage(token4) {
|
|
26823
27056
|
return `<!doctype html>
|
|
26824
27057
|
<html>
|
|
26825
27058
|
<head>
|
|
@@ -26917,7 +27150,7 @@ function renderSchedulePage(token3) {
|
|
|
26917
27150
|
</div>
|
|
26918
27151
|
|
|
26919
27152
|
<script>
|
|
26920
|
-
const TOKEN = ${JSON.stringify(
|
|
27153
|
+
const TOKEN = ${JSON.stringify(token4)};
|
|
26921
27154
|
const state = { actions: [], vaults: [] };
|
|
26922
27155
|
|
|
26923
27156
|
function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
|
|
@@ -27160,8 +27393,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
|
|
|
27160
27393
|
return shortHash(`${customerEmail.trim().toLowerCase()}
|
|
27161
27394
|
${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
|
|
27162
27395
|
}
|
|
27163
|
-
function supportThreadAddress(
|
|
27164
|
-
const safeToken =
|
|
27396
|
+
function supportThreadAddress(token4, receivingDomain) {
|
|
27397
|
+
const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
27165
27398
|
const domain = receivingDomain.trim().toLowerCase();
|
|
27166
27399
|
if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
|
|
27167
27400
|
return `support+${safeToken}@${domain}`;
|
|
@@ -27188,8 +27421,8 @@ function personPath(identity) {
|
|
|
27188
27421
|
const local = identity.email.split("@")[0];
|
|
27189
27422
|
return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
|
|
27190
27423
|
}
|
|
27191
|
-
function issuePath(
|
|
27192
|
-
return `support/threads/${
|
|
27424
|
+
function issuePath(token4) {
|
|
27425
|
+
return `support/threads/${token4}`;
|
|
27193
27426
|
}
|
|
27194
27427
|
function communicationPath(email) {
|
|
27195
27428
|
const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
|
|
@@ -27373,8 +27606,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
|
|
|
27373
27606
|
`- Person: ${noteRef("People", personPathValue)}`
|
|
27374
27607
|
].join("\n");
|
|
27375
27608
|
}
|
|
27376
|
-
async function findOrCreateIssue(memoryKey, email, customer, personPathValue,
|
|
27377
|
-
const path5 = issuePath(
|
|
27609
|
+
async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token4, threadAddressValue) {
|
|
27610
|
+
const path5 = issuePath(token4);
|
|
27378
27611
|
const existing = await getNote(memoryKey, "Issues", path5);
|
|
27379
27612
|
if (existing) return existing;
|
|
27380
27613
|
const subject = normalizeSupportSubject(email.subject);
|
|
@@ -27384,7 +27617,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27384
27617
|
path: path5,
|
|
27385
27618
|
title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
|
|
27386
27619
|
content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
|
|
27387
|
-
source: `resend:thread:${
|
|
27620
|
+
source: `resend:thread:${token4}`,
|
|
27388
27621
|
type: "user_friction",
|
|
27389
27622
|
tagCandidates: ISSUE_TAGS,
|
|
27390
27623
|
props: {
|
|
@@ -27393,7 +27626,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27393
27626
|
summary,
|
|
27394
27627
|
pinned: false,
|
|
27395
27628
|
source_type: "thread",
|
|
27396
|
-
source_ref: `resend:thread:${
|
|
27629
|
+
source_ref: `resend:thread:${token4}`,
|
|
27397
27630
|
related: [],
|
|
27398
27631
|
related_vault_notes: [noteRef("People", personPathValue)],
|
|
27399
27632
|
embed: true,
|
|
@@ -27408,7 +27641,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
27408
27641
|
customer_email: customer.email,
|
|
27409
27642
|
customer_name: customer.name,
|
|
27410
27643
|
normalized_subject: subject,
|
|
27411
|
-
thread_token:
|
|
27644
|
+
thread_token: token4,
|
|
27412
27645
|
thread_address: threadAddressValue,
|
|
27413
27646
|
communication_refs: [],
|
|
27414
27647
|
message_count: 0,
|
|
@@ -27451,7 +27684,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
|
|
|
27451
27684
|
...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
|
|
27452
27685
|
].join("\n");
|
|
27453
27686
|
}
|
|
27454
|
-
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue,
|
|
27687
|
+
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token4, parentPath) {
|
|
27455
27688
|
const path5 = communicationPath(email);
|
|
27456
27689
|
const existing = await getNote(memoryKey, "Communication", path5);
|
|
27457
27690
|
if (existing) return existing;
|
|
@@ -27490,8 +27723,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
|
|
|
27490
27723
|
provider: "resend",
|
|
27491
27724
|
provider_connection_ref: "resend:updates.mcpscraper.dev",
|
|
27492
27725
|
provider_message_ref: providerMessageRef,
|
|
27493
|
-
provider_thread_ref:
|
|
27494
|
-
thread_ref:
|
|
27726
|
+
provider_thread_ref: token4,
|
|
27727
|
+
thread_ref: token4,
|
|
27495
27728
|
issue_ref: issuePathValue,
|
|
27496
27729
|
sender_email: supportSender(email)?.email ?? email.from,
|
|
27497
27730
|
recipient_emails: email.to ?? [],
|
|
@@ -27551,8 +27784,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
|
|
|
27551
27784
|
if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
|
|
27552
27785
|
}
|
|
27553
27786
|
}
|
|
27554
|
-
async function getSupportThreadByToken(memoryKey,
|
|
27555
|
-
const safeToken =
|
|
27787
|
+
async function getSupportThreadByToken(memoryKey, token4) {
|
|
27788
|
+
const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
27556
27789
|
if (safeToken.length !== 24) return null;
|
|
27557
27790
|
const path5 = issuePath(safeToken);
|
|
27558
27791
|
const issue = await getNote(memoryKey, "Issues", path5);
|
|
@@ -27580,8 +27813,8 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27580
27813
|
const customer = options.customer ?? supportSender(email);
|
|
27581
27814
|
if (!customer) throw new Error("support sender does not contain a usable email address");
|
|
27582
27815
|
const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
|
|
27583
|
-
const
|
|
27584
|
-
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(
|
|
27816
|
+
const token4 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
|
|
27817
|
+
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token4, options.receivingDomain);
|
|
27585
27818
|
const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
|
|
27586
27819
|
if (!person) throw new Error("support person could not be resolved");
|
|
27587
27820
|
const issue = options.existingThread?.issue ?? await findOrCreateIssue(
|
|
@@ -27589,7 +27822,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27589
27822
|
email,
|
|
27590
27823
|
customer,
|
|
27591
27824
|
person.path,
|
|
27592
|
-
|
|
27825
|
+
token4,
|
|
27593
27826
|
threadAddressValue
|
|
27594
27827
|
);
|
|
27595
27828
|
const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
|
|
@@ -27600,7 +27833,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27600
27833
|
options.direction,
|
|
27601
27834
|
person.path,
|
|
27602
27835
|
issue.path,
|
|
27603
|
-
|
|
27836
|
+
token4,
|
|
27604
27837
|
parentPath
|
|
27605
27838
|
);
|
|
27606
27839
|
await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
|
|
@@ -27609,7 +27842,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
27609
27842
|
communicationPath: communication.path,
|
|
27610
27843
|
personPath: person.path,
|
|
27611
27844
|
parentCommunicationPath: parentPath,
|
|
27612
|
-
threadToken:
|
|
27845
|
+
threadToken: token4,
|
|
27613
27846
|
threadAddress: threadAddressValue,
|
|
27614
27847
|
customerEmail: customer.email,
|
|
27615
27848
|
normalizedSubject,
|
|
@@ -27782,8 +28015,8 @@ resendInboundApp.post("/webhooks", async (c) => {
|
|
|
27782
28015
|
threadAddress: captured.threadAddress
|
|
27783
28016
|
});
|
|
27784
28017
|
}
|
|
27785
|
-
const
|
|
27786
|
-
const thread =
|
|
28018
|
+
const token4 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
|
|
28019
|
+
const thread = token4 ? await getSupportThreadByToken(memoryKey, token4) : null;
|
|
27787
28020
|
if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
|
|
27788
28021
|
const operator = parseMailbox(destination);
|
|
27789
28022
|
if (operator && sender.email === operator.email) {
|
|
@@ -28090,7 +28323,7 @@ function renderWorkshopFieldNotesBody() {
|
|
|
28090
28323
|
}
|
|
28091
28324
|
|
|
28092
28325
|
// src/api/workshop-registration-repository.ts
|
|
28093
|
-
import { randomUUID as
|
|
28326
|
+
import { randomUUID as randomUUID13 } from "crypto";
|
|
28094
28327
|
function rowToRegistration(row) {
|
|
28095
28328
|
return {
|
|
28096
28329
|
id: String(row.id),
|
|
@@ -28159,7 +28392,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
28159
28392
|
) < ?
|
|
28160
28393
|
`,
|
|
28161
28394
|
args: [
|
|
28162
|
-
|
|
28395
|
+
randomUUID13(),
|
|
28163
28396
|
input.eventSlug,
|
|
28164
28397
|
input.email,
|
|
28165
28398
|
input.firstName,
|
|
@@ -28176,7 +28409,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
28176
28409
|
id, event_slug, email, first_name, last_name, status
|
|
28177
28410
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
28178
28411
|
`,
|
|
28179
|
-
args: [
|
|
28412
|
+
args: [randomUUID13(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
28180
28413
|
});
|
|
28181
28414
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
28182
28415
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -28191,7 +28424,7 @@ async function registerWorkshopInterest(input) {
|
|
|
28191
28424
|
id, event_slug, email, first_name, last_name, status
|
|
28192
28425
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
28193
28426
|
`,
|
|
28194
|
-
args: [
|
|
28427
|
+
args: [randomUUID13(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
28195
28428
|
});
|
|
28196
28429
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
28197
28430
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -28629,16 +28862,16 @@ import { z as z29 } from "zod";
|
|
|
28629
28862
|
// src/editorial-reading-room/render.ts
|
|
28630
28863
|
import { createHash as createHash10 } from "crypto";
|
|
28631
28864
|
import { readFileSync as readFileSync3 } from "fs";
|
|
28632
|
-
import { dirname as
|
|
28865
|
+
import { dirname as dirname3, join as join6, resolve } from "path";
|
|
28633
28866
|
import { Marked, Renderer } from "marked";
|
|
28634
|
-
var runtimeEntryDir =
|
|
28867
|
+
var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
|
|
28635
28868
|
var assetCache = /* @__PURE__ */ new Map();
|
|
28636
28869
|
function assetPath(name) {
|
|
28637
28870
|
const candidates = [
|
|
28638
|
-
|
|
28639
|
-
|
|
28640
|
-
|
|
28641
|
-
|
|
28871
|
+
join6(runtimeEntryDir, "editorial-reading-room", "assets", name),
|
|
28872
|
+
join6(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
|
|
28873
|
+
join6(process.cwd(), "src", "editorial-reading-room", "assets", name),
|
|
28874
|
+
join6(process.cwd(), "dist", "editorial-reading-room", "assets", name)
|
|
28642
28875
|
];
|
|
28643
28876
|
for (const candidate of candidates) {
|
|
28644
28877
|
try {
|
|
@@ -28650,8 +28883,8 @@ function assetPath(name) {
|
|
|
28650
28883
|
throw new Error(`Editorial reading-room template asset is missing: ${name}`);
|
|
28651
28884
|
}
|
|
28652
28885
|
function readAsset(name) {
|
|
28653
|
-
const
|
|
28654
|
-
if (
|
|
28886
|
+
const cached2 = assetCache.get(name);
|
|
28887
|
+
if (cached2 !== void 0) return cached2;
|
|
28655
28888
|
const value = readFileSync3(assetPath(name), "utf8");
|
|
28656
28889
|
assetCache.set(name, value);
|
|
28657
28890
|
return value;
|
|
@@ -28851,20 +29084,20 @@ ${provenance}`;
|
|
|
28851
29084
|
var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
|
|
28852
29085
|
var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
28853
29086
|
var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
28854
|
-
function
|
|
29087
|
+
function token2() {
|
|
28855
29088
|
return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
28856
29089
|
}
|
|
28857
|
-
function
|
|
29090
|
+
function policy2() {
|
|
28858
29091
|
return {
|
|
28859
29092
|
prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
|
|
28860
29093
|
artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
|
|
28861
29094
|
downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
|
|
28862
|
-
token:
|
|
29095
|
+
token: token2()
|
|
28863
29096
|
};
|
|
28864
29097
|
}
|
|
28865
29098
|
function createEditorialReadingRoomArtifact(args) {
|
|
28866
29099
|
return createPrivateArtifact({
|
|
28867
|
-
policy:
|
|
29100
|
+
policy: policy2(),
|
|
28868
29101
|
ownerId: args.ownerId,
|
|
28869
29102
|
artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
|
|
28870
29103
|
createdAt: args.rendered.generatedAt,
|
|
@@ -28875,7 +29108,7 @@ function createEditorialReadingRoomArtifact(args) {
|
|
|
28875
29108
|
}
|
|
28876
29109
|
function renewEditorialReadingRoomArtifactDownload(args) {
|
|
28877
29110
|
return renewPrivateArtifactDownload({
|
|
28878
|
-
policy:
|
|
29111
|
+
policy: policy2(),
|
|
28879
29112
|
ownerId: args.ownerId,
|
|
28880
29113
|
artifactId: args.artifactId
|
|
28881
29114
|
});
|
|
@@ -28972,7 +29205,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
28972
29205
|
}
|
|
28973
29206
|
|
|
28974
29207
|
// src/api/scheduled-run-view-links.ts
|
|
28975
|
-
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as
|
|
29208
|
+
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID14 } from "crypto";
|
|
28976
29209
|
var schemaReady2 = false;
|
|
28977
29210
|
async function ensureScheduledRunViewLinksSchema() {
|
|
28978
29211
|
if (schemaReady2) return;
|
|
@@ -29002,8 +29235,8 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
29002
29235
|
`);
|
|
29003
29236
|
schemaReady2 = true;
|
|
29004
29237
|
}
|
|
29005
|
-
function tokenHash2(
|
|
29006
|
-
return createHash12("sha256").update(
|
|
29238
|
+
function tokenHash2(token4) {
|
|
29239
|
+
return createHash12("sha256").update(token4).digest("hex");
|
|
29007
29240
|
}
|
|
29008
29241
|
function mapRow(row) {
|
|
29009
29242
|
return {
|
|
@@ -29023,16 +29256,16 @@ function mapRow(row) {
|
|
|
29023
29256
|
async function createScheduledRunViewLink(input) {
|
|
29024
29257
|
await ensureScheduledRunViewLinksSchema();
|
|
29025
29258
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
29026
|
-
const
|
|
29259
|
+
const token4 = randomBytes3(32).toString("base64url");
|
|
29027
29260
|
const record = {
|
|
29028
|
-
shareId:
|
|
29261
|
+
shareId: randomUUID14(),
|
|
29029
29262
|
ownerId: input.ownerId,
|
|
29030
29263
|
runId: input.runId,
|
|
29031
29264
|
artifactId: input.artifactId,
|
|
29032
29265
|
title: input.title.slice(0, 300),
|
|
29033
29266
|
summary: input.summary.slice(0, 2e3),
|
|
29034
29267
|
rendererKey: input.rendererKey,
|
|
29035
|
-
tokenHash: tokenHash2(
|
|
29268
|
+
tokenHash: tokenHash2(token4),
|
|
29036
29269
|
expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
|
|
29037
29270
|
revokedAt: null,
|
|
29038
29271
|
createdAt: now.toISOString()
|
|
@@ -29057,9 +29290,9 @@ async function createScheduledRunViewLink(input) {
|
|
|
29057
29290
|
record.createdAt
|
|
29058
29291
|
]
|
|
29059
29292
|
});
|
|
29060
|
-
return { record, token:
|
|
29293
|
+
return { record, token: token4 };
|
|
29061
29294
|
}
|
|
29062
|
-
async function resolveScheduledRunViewLink(
|
|
29295
|
+
async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Date()) {
|
|
29063
29296
|
await ensureScheduledRunViewLinksSchema();
|
|
29064
29297
|
const result = await getDb().execute({
|
|
29065
29298
|
sql: `
|
|
@@ -29071,7 +29304,7 @@ async function resolveScheduledRunViewLink(token3, now = /* @__PURE__ */ new Dat
|
|
|
29071
29304
|
AND expires_at > ?
|
|
29072
29305
|
LIMIT 1
|
|
29073
29306
|
`,
|
|
29074
|
-
args: [tokenHash2(
|
|
29307
|
+
args: [tokenHash2(token4), now.toISOString()]
|
|
29075
29308
|
});
|
|
29076
29309
|
const row = result.rows[0];
|
|
29077
29310
|
return row ? mapRow(row) : null;
|
|
@@ -29225,7 +29458,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
29225
29458
|
const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
|
|
29226
29459
|
if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
|
|
29227
29460
|
const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
|
|
29228
|
-
const { record, token:
|
|
29461
|
+
const { record, token: token4 } = await createScheduledRunViewLink({
|
|
29229
29462
|
ownerId,
|
|
29230
29463
|
runId: result.data.run.runId,
|
|
29231
29464
|
artifactId: artifact.artifactId,
|
|
@@ -29240,7 +29473,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
29240
29473
|
shareId: record.shareId,
|
|
29241
29474
|
runId: record.runId,
|
|
29242
29475
|
artifactId: record.artifactId,
|
|
29243
|
-
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${
|
|
29476
|
+
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token4}`,
|
|
29244
29477
|
expiresAt: record.expiresAt,
|
|
29245
29478
|
createdAt: record.createdAt
|
|
29246
29479
|
}, 201);
|
|
@@ -29266,15 +29499,15 @@ import { createHash as createHash13 } from "crypto";
|
|
|
29266
29499
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
29267
29500
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
29268
29501
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
29269
|
-
function
|
|
29502
|
+
function token3() {
|
|
29270
29503
|
return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
29271
29504
|
}
|
|
29272
|
-
function
|
|
29505
|
+
function policy3() {
|
|
29273
29506
|
return {
|
|
29274
29507
|
prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
|
|
29275
29508
|
artifactTtlMs: null,
|
|
29276
29509
|
downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
|
|
29277
|
-
token:
|
|
29510
|
+
token: token3()
|
|
29278
29511
|
};
|
|
29279
29512
|
}
|
|
29280
29513
|
function runStorageSegment(runId) {
|
|
@@ -29285,7 +29518,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
29285
29518
|
throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
|
|
29286
29519
|
}
|
|
29287
29520
|
const stored = await createPrivateArtifact({
|
|
29288
|
-
policy:
|
|
29521
|
+
policy: policy3(),
|
|
29289
29522
|
ownerId: args.request.ownerId,
|
|
29290
29523
|
scopeSegments: [runStorageSegment(args.request.runId)],
|
|
29291
29524
|
artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
|
|
@@ -29311,7 +29544,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
29311
29544
|
async function readScheduledRunArtifact(args) {
|
|
29312
29545
|
if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
|
|
29313
29546
|
const window2 = await readPrivateArtifactWindow({
|
|
29314
|
-
policy:
|
|
29547
|
+
policy: policy3(),
|
|
29315
29548
|
artifactId: args.artifactId,
|
|
29316
29549
|
offset: 0,
|
|
29317
29550
|
maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
|
|
@@ -29388,7 +29621,7 @@ import { Hono as Hono32 } from "hono";
|
|
|
29388
29621
|
import { createHash as createHash14, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
29389
29622
|
|
|
29390
29623
|
// src/api/service-connections.ts
|
|
29391
|
-
import { randomUUID as
|
|
29624
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
29392
29625
|
var schemaReady3 = null;
|
|
29393
29626
|
var schemaDb2 = null;
|
|
29394
29627
|
function ensureServiceConnectionsSchema() {
|
|
@@ -29454,6 +29687,14 @@ function ensureServiceConnectionsSchema() {
|
|
|
29454
29687
|
UNIQUE(user_id, request_id)
|
|
29455
29688
|
)
|
|
29456
29689
|
`);
|
|
29690
|
+
try {
|
|
29691
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
|
|
29692
|
+
} catch {
|
|
29693
|
+
}
|
|
29694
|
+
try {
|
|
29695
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
|
|
29696
|
+
} catch {
|
|
29697
|
+
}
|
|
29457
29698
|
})().catch((error) => {
|
|
29458
29699
|
schemaReady3 = null;
|
|
29459
29700
|
schemaDb2 = null;
|
|
@@ -29522,7 +29763,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
29522
29763
|
updated_at = excluded.updated_at
|
|
29523
29764
|
`,
|
|
29524
29765
|
args: [
|
|
29525
|
-
|
|
29766
|
+
randomUUID15(),
|
|
29526
29767
|
userId,
|
|
29527
29768
|
connection.providerConfigKey,
|
|
29528
29769
|
connection.provider,
|
|
@@ -29593,7 +29834,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
29593
29834
|
});
|
|
29594
29835
|
await getDb().execute({
|
|
29595
29836
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
29596
|
-
args: [
|
|
29837
|
+
args: [randomUUID15(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
29597
29838
|
});
|
|
29598
29839
|
}
|
|
29599
29840
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -29623,8 +29864,8 @@ async function recordServiceConnectionAction(args) {
|
|
|
29623
29864
|
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
29624
29865
|
if (!connection) throw new Error("service_connection_not_found");
|
|
29625
29866
|
await getDb().execute({
|
|
29626
|
-
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
29627
|
-
args: [args.status, args.errorCode ?? null, connection.userId, args.requestId, connection.id]
|
|
29867
|
+
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
29868
|
+
args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
|
|
29628
29869
|
});
|
|
29629
29870
|
}
|
|
29630
29871
|
async function claimServiceConnectionAction(args) {
|
|
@@ -29632,18 +29873,30 @@ async function claimServiceConnectionAction(args) {
|
|
|
29632
29873
|
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
29633
29874
|
if (!connection) throw new Error("service_connection_not_found");
|
|
29634
29875
|
const inserted = await getDb().execute({
|
|
29635
|
-
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status) VALUES (?, ?, ?, ?, ?, 'started')`,
|
|
29636
|
-
args: [
|
|
29876
|
+
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
29877
|
+
args: [randomUUID15(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
29637
29878
|
});
|
|
29638
29879
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
29639
29880
|
const existing = await getDb().execute({
|
|
29640
|
-
sql: `SELECT status, error_code FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
29881
|
+
sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
29641
29882
|
args: [connection.userId, args.requestId]
|
|
29642
29883
|
});
|
|
29884
|
+
const row = existing.rows[0];
|
|
29885
|
+
const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
|
|
29886
|
+
let result = void 0;
|
|
29887
|
+
if (!conflict && typeof row?.result_json === "string") {
|
|
29888
|
+
try {
|
|
29889
|
+
result = JSON.parse(row.result_json);
|
|
29890
|
+
} catch {
|
|
29891
|
+
result = void 0;
|
|
29892
|
+
}
|
|
29893
|
+
}
|
|
29643
29894
|
return {
|
|
29644
29895
|
claimed: false,
|
|
29645
|
-
status: String(
|
|
29646
|
-
errorCode: typeof
|
|
29896
|
+
status: String(row?.status ?? "unknown"),
|
|
29897
|
+
errorCode: typeof row?.error_code === "string" ? row.error_code : null,
|
|
29898
|
+
conflict,
|
|
29899
|
+
...result !== void 0 ? { result } : {}
|
|
29647
29900
|
};
|
|
29648
29901
|
}
|
|
29649
29902
|
|
|
@@ -30066,6 +30319,53 @@ async function persistScrapeBody(user, opts) {
|
|
|
30066
30319
|
return { ...deposit, ...fallback };
|
|
30067
30320
|
}
|
|
30068
30321
|
|
|
30322
|
+
// src/api/scrape-image-sink.ts
|
|
30323
|
+
import { createHash as createHash16 } from "crypto";
|
|
30324
|
+
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
30325
|
+
function idempotencyKey2(userId, vault, input) {
|
|
30326
|
+
return `scrape-image-${createHash16("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
30327
|
+
}
|
|
30328
|
+
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
30329
|
+
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
30330
|
+
if (selected.length === 0) return { requested: 0, saved: 0, skipped: 0, assets: [] };
|
|
30331
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
30332
|
+
if (!key) {
|
|
30333
|
+
return { requested: selected.length, saved: 0, skipped: inputs.length - selected.length, assets: [], error: error ?? "memory unavailable" };
|
|
30334
|
+
}
|
|
30335
|
+
const targetVault = vault?.trim() || "Library";
|
|
30336
|
+
const assets = [];
|
|
30337
|
+
for (const input of selected) {
|
|
30338
|
+
const result = await memoryCall(
|
|
30339
|
+
"image_asset_save",
|
|
30340
|
+
{
|
|
30341
|
+
vault: targetVault,
|
|
30342
|
+
...input.imageBase64 ? { imageBase64: input.imageBase64 } : { sourceUrl: input.imageUrl },
|
|
30343
|
+
title: input.title.slice(0, 240),
|
|
30344
|
+
sourceRef: {
|
|
30345
|
+
sourceUrl: input.sourceUrl,
|
|
30346
|
+
sourceKind: input.sourceKind,
|
|
30347
|
+
...input.sourceRef ?? {}
|
|
30348
|
+
},
|
|
30349
|
+
idempotencyKey: idempotencyKey2(user.id, targetVault, input)
|
|
30350
|
+
},
|
|
30351
|
+
key
|
|
30352
|
+
);
|
|
30353
|
+
assets.push({
|
|
30354
|
+
sourceUrl: input.imageUrl ?? input.sourceUrl,
|
|
30355
|
+
sourceKind: input.sourceKind,
|
|
30356
|
+
saved: result.ok,
|
|
30357
|
+
...result.asset?.assetId ? { assetId: result.asset.assetId } : {},
|
|
30358
|
+
...!result.ok ? { error: result.error ?? "image save failed" } : {}
|
|
30359
|
+
});
|
|
30360
|
+
}
|
|
30361
|
+
return {
|
|
30362
|
+
requested: selected.length,
|
|
30363
|
+
saved: assets.filter((asset) => asset.saved).length,
|
|
30364
|
+
skipped: inputs.length - selected.length,
|
|
30365
|
+
assets
|
|
30366
|
+
};
|
|
30367
|
+
}
|
|
30368
|
+
|
|
30069
30369
|
// src/api/archive-reader.ts
|
|
30070
30370
|
import { basename, extname } from "path";
|
|
30071
30371
|
import yauzl from "yauzl";
|
|
@@ -30156,6 +30456,10 @@ function openZip(buffer) {
|
|
|
30156
30456
|
strictFileNames: true
|
|
30157
30457
|
}, (error, zip) => {
|
|
30158
30458
|
if (error || !zip) {
|
|
30459
|
+
if (error && /invalid relative path|absolute path|backslash/i.test(error.message)) {
|
|
30460
|
+
reject(new ArchiveReadError("archive_unsafe_path", "ZIP archive contains an unsafe entry path."));
|
|
30461
|
+
return;
|
|
30462
|
+
}
|
|
30159
30463
|
reject(new ArchiveReadError("archive_invalid_zip", "The downloaded file is not a valid ZIP archive."));
|
|
30160
30464
|
return;
|
|
30161
30465
|
}
|
|
@@ -30163,6 +30467,13 @@ function openZip(buffer) {
|
|
|
30163
30467
|
});
|
|
30164
30468
|
});
|
|
30165
30469
|
}
|
|
30470
|
+
function normalizedZipError(error, fallbackCode, fallbackMessage) {
|
|
30471
|
+
if (error instanceof ArchiveReadError) return error;
|
|
30472
|
+
if (error instanceof Error && /invalid relative path|absolute path|backslash/i.test(error.message)) {
|
|
30473
|
+
return new ArchiveReadError("archive_unsafe_path", "ZIP archive contains an unsafe entry path.");
|
|
30474
|
+
}
|
|
30475
|
+
return new ArchiveReadError(fallbackCode, error instanceof Error ? error.message : fallbackMessage);
|
|
30476
|
+
}
|
|
30166
30477
|
async function scanZip(buffer, visibleEntryLimit) {
|
|
30167
30478
|
const zip = await openZip(buffer);
|
|
30168
30479
|
return new Promise((resolve2, reject) => {
|
|
@@ -30177,7 +30488,7 @@ async function scanZip(buffer, visibleEntryLimit) {
|
|
|
30177
30488
|
zip.close();
|
|
30178
30489
|
} catch {
|
|
30179
30490
|
}
|
|
30180
|
-
reject(error
|
|
30491
|
+
reject(normalizedZipError(error, "archive_invalid_zip", "Failed to read ZIP archive."));
|
|
30181
30492
|
};
|
|
30182
30493
|
zip.on("error", fail2);
|
|
30183
30494
|
zip.on("entry", (entry) => {
|
|
@@ -30271,7 +30582,7 @@ async function extractEntry(buffer, requestedPath) {
|
|
|
30271
30582
|
zip.close();
|
|
30272
30583
|
} catch {
|
|
30273
30584
|
}
|
|
30274
|
-
reject(error
|
|
30585
|
+
reject(normalizedZipError(error, "archive_entry_read_failed", "Failed to read ZIP entry."));
|
|
30275
30586
|
};
|
|
30276
30587
|
zip.on("error", fail2);
|
|
30277
30588
|
zip.on("entry", (entry) => {
|
|
@@ -30386,6 +30697,9 @@ async function downloadPublicZip(rawUrl) {
|
|
|
30386
30697
|
}
|
|
30387
30698
|
async function listZipArchive(rawUrl, maxEntries) {
|
|
30388
30699
|
const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
|
|
30700
|
+
return listZipArchiveBuffer(buffer, archiveUrl, maxEntries);
|
|
30701
|
+
}
|
|
30702
|
+
async function listZipArchiveBuffer(buffer, archiveUrl, maxEntries) {
|
|
30389
30703
|
const scanned = await scanZip(buffer, maxEntries);
|
|
30390
30704
|
return {
|
|
30391
30705
|
archiveUrl,
|
|
@@ -30426,8 +30740,11 @@ function utf8Window(content, path5, offset, maxBytes) {
|
|
|
30426
30740
|
};
|
|
30427
30741
|
}
|
|
30428
30742
|
async function readZipArchiveFile(rawUrl, rawPath, offset, maxBytes) {
|
|
30429
|
-
const requestedPath = safeEntryPath(rawPath);
|
|
30430
30743
|
const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
|
|
30744
|
+
return readZipArchiveBuffer(buffer, archiveUrl, rawPath, offset, maxBytes);
|
|
30745
|
+
}
|
|
30746
|
+
async function readZipArchiveBuffer(buffer, archiveUrl, rawPath, offset, maxBytes) {
|
|
30747
|
+
const requestedPath = safeEntryPath(rawPath);
|
|
30431
30748
|
const scanned = await scanZip(buffer, 0);
|
|
30432
30749
|
const info = scanned.allEntries.find((entry) => entry.path === requestedPath);
|
|
30433
30750
|
if (!info) throw new ArchiveReadError("archive_entry_not_found", `ZIP entry was not found: ${requestedPath}`, 404);
|
|
@@ -30457,6 +30774,9 @@ async function readZipArchiveFile(rawUrl, rawPath, offset, maxBytes) {
|
|
|
30457
30774
|
};
|
|
30458
30775
|
}
|
|
30459
30776
|
function archiveLibrarySource(archiveUrl, path5) {
|
|
30777
|
+
if (!/^https?:\/\//i.test(archiveUrl)) {
|
|
30778
|
+
return `mcp-artifact:${encodeURIComponent(archiveUrl)}#entry=${encodeURIComponent(path5)}`;
|
|
30779
|
+
}
|
|
30460
30780
|
const source = new URL(archiveUrl);
|
|
30461
30781
|
source.username = "";
|
|
30462
30782
|
source.password = "";
|
|
@@ -30475,7 +30795,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
30475
30795
|
}
|
|
30476
30796
|
|
|
30477
30797
|
// src/api/connection-memory-import.ts
|
|
30478
|
-
import { createHash as
|
|
30798
|
+
import { createHash as createHash17 } from "crypto";
|
|
30479
30799
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
30480
30800
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
30481
30801
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -30505,7 +30825,7 @@ function canonicalJson(value) {
|
|
|
30505
30825
|
return JSON.stringify(value);
|
|
30506
30826
|
}
|
|
30507
30827
|
function sha2562(value) {
|
|
30508
|
-
return
|
|
30828
|
+
return createHash17("sha256").update(value).digest("hex");
|
|
30509
30829
|
}
|
|
30510
30830
|
function sensitiveKey(key) {
|
|
30511
30831
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -30723,17 +31043,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
30723
31043
|
|
|
30724
31044
|
// src/api/scrape-blob-cleanup.ts
|
|
30725
31045
|
import { readdir, stat, unlink } from "fs/promises";
|
|
30726
|
-
import { homedir } from "os";
|
|
30727
|
-
import { join as
|
|
30728
|
-
async function cleanupVercel(
|
|
31046
|
+
import { homedir as homedir2 } from "os";
|
|
31047
|
+
import { join as join7 } from "path";
|
|
31048
|
+
async function cleanupVercel(token4, cutoff) {
|
|
30729
31049
|
const { list, del } = await import("@vercel/blob");
|
|
30730
31050
|
let cursor;
|
|
30731
31051
|
let deleted = 0;
|
|
30732
31052
|
do {
|
|
30733
|
-
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token:
|
|
31053
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token4 });
|
|
30734
31054
|
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
30735
31055
|
if (expired.length) {
|
|
30736
|
-
await del(expired.map((b) => b.url), { token:
|
|
31056
|
+
await del(expired.map((b) => b.url), { token: token4 });
|
|
30737
31057
|
deleted += expired.length;
|
|
30738
31058
|
}
|
|
30739
31059
|
cursor = res.hasMore ? res.cursor : void 0;
|
|
@@ -30741,8 +31061,8 @@ async function cleanupVercel(token3, cutoff) {
|
|
|
30741
31061
|
return { deleted, store: "vercel-blob" };
|
|
30742
31062
|
}
|
|
30743
31063
|
async function cleanupLocal(cutoff) {
|
|
30744
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
30745
|
-
const dir =
|
|
31064
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir2(), "Downloads", "mcp-scraper");
|
|
31065
|
+
const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
30746
31066
|
let deleted = 0;
|
|
30747
31067
|
let entries;
|
|
30748
31068
|
try {
|
|
@@ -30751,7 +31071,7 @@ async function cleanupLocal(cutoff) {
|
|
|
30751
31071
|
return { deleted: 0, store: "local" };
|
|
30752
31072
|
}
|
|
30753
31073
|
for (const name of entries) {
|
|
30754
|
-
const path5 =
|
|
31074
|
+
const path5 = join7(dir, name);
|
|
30755
31075
|
try {
|
|
30756
31076
|
const s = await stat(path5);
|
|
30757
31077
|
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
@@ -30766,15 +31086,15 @@ async function cleanupLocal(cutoff) {
|
|
|
30766
31086
|
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
30767
31087
|
const cutoff = Date.now() - maxAgeMs;
|
|
30768
31088
|
try {
|
|
30769
|
-
const
|
|
30770
|
-
return
|
|
31089
|
+
const token4 = process.env.BLOB_READ_WRITE_TOKEN;
|
|
31090
|
+
return token4 ? await cleanupVercel(token4, cutoff) : await cleanupLocal(cutoff);
|
|
30771
31091
|
} catch {
|
|
30772
31092
|
return { deleted: 0, store: "none" };
|
|
30773
31093
|
}
|
|
30774
31094
|
}
|
|
30775
31095
|
|
|
30776
31096
|
// src/api/connected-data-export.ts
|
|
30777
|
-
import { randomUUID as
|
|
31097
|
+
import { randomUUID as randomUUID16 } from "crypto";
|
|
30778
31098
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
30779
31099
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
30780
31100
|
);
|
|
@@ -30945,7 +31265,7 @@ function finitePositive(value, fallback) {
|
|
|
30945
31265
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
30946
31266
|
}
|
|
30947
31267
|
async function collectConnectedDataExport(args) {
|
|
30948
|
-
const exportId =
|
|
31268
|
+
const exportId = randomUUID16();
|
|
30949
31269
|
const now = args.now ?? Date.now;
|
|
30950
31270
|
const startedAt = now();
|
|
30951
31271
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -31061,7 +31381,7 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
31061
31381
|
}
|
|
31062
31382
|
|
|
31063
31383
|
// src/api/search-console-table-export.ts
|
|
31064
|
-
import { randomUUID as
|
|
31384
|
+
import { randomUUID as randomUUID17 } from "crypto";
|
|
31065
31385
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
31066
31386
|
var SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
31067
31387
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -31161,7 +31481,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
31161
31481
|
offset += rows.length;
|
|
31162
31482
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
31163
31483
|
}
|
|
31164
|
-
const exportId =
|
|
31484
|
+
const exportId = randomUUID17();
|
|
31165
31485
|
const artifact = await args.writeArtifact({
|
|
31166
31486
|
ownerId: args.ownerId,
|
|
31167
31487
|
exportId,
|
|
@@ -31516,7 +31836,7 @@ async function dbUsage(identity, plan) {
|
|
|
31516
31836
|
}
|
|
31517
31837
|
|
|
31518
31838
|
// src/api/nango-control.ts
|
|
31519
|
-
import { createHash as
|
|
31839
|
+
import { createHash as createHash19, randomUUID as randomUUID19 } from "crypto";
|
|
31520
31840
|
|
|
31521
31841
|
// src/api/slack-archive-analysis.ts
|
|
31522
31842
|
function isRecord2(value) {
|
|
@@ -31890,9 +32210,55 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
31890
32210
|
}
|
|
31891
32211
|
|
|
31892
32212
|
// src/api/main-nango-transport.ts
|
|
31893
|
-
import { createHash as
|
|
31894
|
-
import { Client } from "@modelcontextprotocol/
|
|
31895
|
-
|
|
32213
|
+
import { createHash as createHash18, randomUUID as randomUUID18 } from "crypto";
|
|
32214
|
+
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
32215
|
+
|
|
32216
|
+
// src/api/connected-tool-policy.ts
|
|
32217
|
+
function classifyConnectedTool(tool) {
|
|
32218
|
+
if (tool.annotations?.readOnlyHint === true) return "read";
|
|
32219
|
+
if (tool.annotations?.readOnlyHint === false) return "action";
|
|
32220
|
+
return /^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(tool.name) ? "read" : "action";
|
|
32221
|
+
}
|
|
32222
|
+
function normalizedPolicyMap(value) {
|
|
32223
|
+
return new Map(Object.entries(value ?? {}).map(([name, entries]) => [
|
|
32224
|
+
name,
|
|
32225
|
+
[...new Set(entries.map((entry) => entry.trim()).filter(Boolean))].sort()
|
|
32226
|
+
]));
|
|
32227
|
+
}
|
|
32228
|
+
function buildConnectedToolPolicyRegistry(args) {
|
|
32229
|
+
const permissionPolicies = normalizedPolicyMap(args.requiredPermissionsByTool);
|
|
32230
|
+
const featurePolicies = normalizedPolicyMap(args.requiredFeaturesByTool);
|
|
32231
|
+
const granted = new Set(args.grantedPermissions ?? []);
|
|
32232
|
+
const enabled = new Set(args.enabledFeatures ?? []);
|
|
32233
|
+
const adminBlocked = new Set(args.adminBlockedTools ?? []);
|
|
32234
|
+
const policies = /* @__PURE__ */ new Map();
|
|
32235
|
+
for (const tool of args.tools) {
|
|
32236
|
+
const classification = classifyConnectedTool(tool);
|
|
32237
|
+
const requiredPermissions = permissionPolicies.get(tool.name) ?? [];
|
|
32238
|
+
const requiredFeatures = featurePolicies.get(tool.name) ?? [];
|
|
32239
|
+
const missingPermissions = requiredPermissions.filter((permission) => !granted.has(permission));
|
|
32240
|
+
const missingFeatures = requiredFeatures.filter((feature) => !enabled.has(feature));
|
|
32241
|
+
const blockedReason = !args.connectionActive ? "inactive_connection" : adminBlocked.has(tool.name) ? "admin_blocked" : classification === "action" && !args.actionsEnabled ? "actions_disabled" : requiredPermissions.length > 0 && args.permissionVerification === "unavailable" ? "permission_verification_unavailable" : missingPermissions.length > 0 ? "missing_permission" : missingFeatures.length > 0 ? "missing_app_feature" : null;
|
|
32242
|
+
policies.set(tool.name, {
|
|
32243
|
+
name: tool.name,
|
|
32244
|
+
classification,
|
|
32245
|
+
requiredPermissions,
|
|
32246
|
+
requiredFeatures,
|
|
32247
|
+
available: blockedReason === null,
|
|
32248
|
+
blockedReason,
|
|
32249
|
+
missingPermissions,
|
|
32250
|
+
missingFeatures
|
|
32251
|
+
});
|
|
32252
|
+
}
|
|
32253
|
+
return policies;
|
|
32254
|
+
}
|
|
32255
|
+
function connectedToolPolicyOrThrow(registry, toolName) {
|
|
32256
|
+
const policy4 = registry.get(toolName);
|
|
32257
|
+
if (!policy4) throw new Error("connected_tool_policy_missing");
|
|
32258
|
+
return policy4;
|
|
32259
|
+
}
|
|
32260
|
+
|
|
32261
|
+
// src/api/main-nango-transport.ts
|
|
31896
32262
|
var DEFAULT_NANGO_MCP_URL = "https://api.nango.dev/mcp";
|
|
31897
32263
|
var NANGO_TIMEOUT_MS = 2e4;
|
|
31898
32264
|
var NANGO_CONNECTION_PAGE_SIZE = 100;
|
|
@@ -32169,11 +32535,6 @@ async function requireOwnedActiveConnection(identity, connectionId, allowNeedsRe
|
|
|
32169
32535
|
}
|
|
32170
32536
|
return connection;
|
|
32171
32537
|
}
|
|
32172
|
-
function classifyTool(tool) {
|
|
32173
|
-
if (tool.annotations?.readOnlyHint === true) return "read";
|
|
32174
|
-
if (tool.annotations?.readOnlyHint === false) return "action";
|
|
32175
|
-
return /^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(tool.name) ? "read" : "action";
|
|
32176
|
-
}
|
|
32177
32538
|
async function withNangoClient(connection, run) {
|
|
32178
32539
|
const transport = new StreamableHTTPClientTransport(nangoMcpUrl(), {
|
|
32179
32540
|
requestInit: {
|
|
@@ -32235,9 +32596,14 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
32235
32596
|
...asRecord(raw.annotations) ? { annotations: asRecord(raw.annotations) } : {}
|
|
32236
32597
|
}];
|
|
32237
32598
|
});
|
|
32238
|
-
const
|
|
32239
|
-
|
|
32240
|
-
|
|
32599
|
+
const policies = buildConnectedToolPolicyRegistry({
|
|
32600
|
+
tools,
|
|
32601
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32602
|
+
actionsEnabled: connection.actionsEnabled
|
|
32603
|
+
});
|
|
32604
|
+
const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
|
|
32605
|
+
const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
|
|
32606
|
+
const revision = createHash18("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
32241
32607
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
32242
32608
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
32243
32609
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -32246,19 +32612,26 @@ async function callNangoToolDirect(args) {
|
|
|
32246
32612
|
const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
|
|
32247
32613
|
const selected = tools.find((tool) => tool.name === args.tool);
|
|
32248
32614
|
if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
32249
|
-
const
|
|
32250
|
-
|
|
32615
|
+
const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
32616
|
+
tools,
|
|
32617
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32618
|
+
actionsEnabled: connection.actionsEnabled
|
|
32619
|
+
}), selected.name);
|
|
32620
|
+
if (policy4.classification !== args.classification) {
|
|
32251
32621
|
throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
|
|
32252
32622
|
}
|
|
32253
|
-
if (
|
|
32623
|
+
if (!policy4.available && policy4.blockedReason === "actions_disabled") {
|
|
32254
32624
|
throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
|
|
32255
32625
|
}
|
|
32626
|
+
if (!policy4.available) {
|
|
32627
|
+
throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
|
|
32628
|
+
}
|
|
32256
32629
|
await preflightConnectedUsage({
|
|
32257
32630
|
identity: args.identity,
|
|
32258
32631
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
32259
32632
|
});
|
|
32260
|
-
const requestId = args.requestId?.trim() ||
|
|
32261
|
-
const
|
|
32633
|
+
const requestId = args.requestId?.trim() || randomUUID18();
|
|
32634
|
+
const idempotencyKey3 = `main-nango:${createHash18("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
32262
32635
|
const startedAt = /* @__PURE__ */ new Date();
|
|
32263
32636
|
const started = performance.now();
|
|
32264
32637
|
let result;
|
|
@@ -32271,7 +32644,7 @@ async function callNangoToolDirect(args) {
|
|
|
32271
32644
|
const completedAt = /* @__PURE__ */ new Date();
|
|
32272
32645
|
await settleConnectedUsage({
|
|
32273
32646
|
identity: args.identity,
|
|
32274
|
-
idempotencyKey:
|
|
32647
|
+
idempotencyKey: idempotencyKey3,
|
|
32275
32648
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION,
|
|
32276
32649
|
connectionId: connection.id,
|
|
32277
32650
|
providerConfigKey: connection.providerConfigKey,
|
|
@@ -32283,7 +32656,7 @@ async function callNangoToolDirect(args) {
|
|
|
32283
32656
|
toolName: args.tool,
|
|
32284
32657
|
operationKind: args.operationKind ?? args.classification,
|
|
32285
32658
|
outcome: providerError ? "error" : "partial",
|
|
32286
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
32659
|
+
requestId: requestId.length <= 200 ? requestId : createHash18("sha256").update(requestId).digest("hex"),
|
|
32287
32660
|
startedAt: startedAt.toISOString(),
|
|
32288
32661
|
completedAt: completedAt.toISOString()
|
|
32289
32662
|
}
|
|
@@ -32295,8 +32668,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
32295
32668
|
const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
|
|
32296
32669
|
const tool = tools.find((candidate) => candidate.name === toolName);
|
|
32297
32670
|
if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
32298
|
-
const
|
|
32299
|
-
|
|
32671
|
+
const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
32672
|
+
tools,
|
|
32673
|
+
connectionActive: connection.lifecycleStatus === "connected",
|
|
32674
|
+
actionsEnabled: connection.actionsEnabled
|
|
32675
|
+
}), tool.name);
|
|
32676
|
+
const classification = classifyConnectedTool(tool);
|
|
32677
|
+
const callable = policy4.available;
|
|
32300
32678
|
const projected = {
|
|
32301
32679
|
name: tool.name,
|
|
32302
32680
|
...tool.title ? { title: tool.title } : {},
|
|
@@ -32309,18 +32687,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
32309
32687
|
...projected,
|
|
32310
32688
|
classification,
|
|
32311
32689
|
callable,
|
|
32312
|
-
blockedReason:
|
|
32313
|
-
requiredPermissions:
|
|
32314
|
-
missingPermissions:
|
|
32315
|
-
requiredFeatures:
|
|
32316
|
-
missingFeatures:
|
|
32690
|
+
blockedReason: policy4.blockedReason,
|
|
32691
|
+
requiredPermissions: policy4.requiredPermissions,
|
|
32692
|
+
missingPermissions: policy4.missingPermissions,
|
|
32693
|
+
requiredFeatures: policy4.requiredFeatures,
|
|
32694
|
+
missingFeatures: policy4.missingFeatures,
|
|
32317
32695
|
transport: "nango",
|
|
32318
32696
|
providerConfigKey: connection.providerConfigKey,
|
|
32319
32697
|
providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
|
|
32320
32698
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
32321
32699
|
protocolVersion: null,
|
|
32322
32700
|
schemaSource: "live_tools_list",
|
|
32323
|
-
schemaHash:
|
|
32701
|
+
schemaHash: createHash18("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
32324
32702
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
32325
32703
|
};
|
|
32326
32704
|
}
|
|
@@ -32776,8 +33154,8 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32776
33154
|
classification: "read",
|
|
32777
33155
|
requiredPermissions: [],
|
|
32778
33156
|
requiredFeatures: [],
|
|
32779
|
-
available: connection.operationalStatus !== "unavailable",
|
|
32780
|
-
blockedReason: null,
|
|
33157
|
+
available: connection.lifecycleStatus === "connected" && connection.operationalStatus !== "unavailable",
|
|
33158
|
+
blockedReason: connection.lifecycleStatus !== "connected" || connection.operationalStatus === "unavailable" ? "inactive_connection" : null,
|
|
32781
33159
|
missingPermissions: [],
|
|
32782
33160
|
missingFeatures: []
|
|
32783
33161
|
})),
|
|
@@ -32786,8 +33164,8 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32786
33164
|
classification: "action",
|
|
32787
33165
|
requiredPermissions: [],
|
|
32788
33166
|
requiredFeatures: [],
|
|
32789
|
-
available: connection.operationalStatus !== "unavailable",
|
|
32790
|
-
blockedReason: null,
|
|
33167
|
+
available: connection.lifecycleStatus === "connected" && connection.operationalStatus !== "unavailable" && connection.actionsEnabled,
|
|
33168
|
+
blockedReason: connection.lifecycleStatus !== "connected" || connection.operationalStatus === "unavailable" ? "inactive_connection" : connection.actionsEnabled ? null : "actions_disabled",
|
|
32791
33169
|
missingPermissions: [],
|
|
32792
33170
|
missingFeatures: []
|
|
32793
33171
|
}))
|
|
@@ -32841,7 +33219,7 @@ async function getNangoConnections(identity, options = {}) {
|
|
|
32841
33219
|
const name = cleanString3(value.name, 200);
|
|
32842
33220
|
const classification = value.classification;
|
|
32843
33221
|
const blockedValue = Object.prototype.hasOwnProperty.call(value, "blockedReason") ? value.blockedReason : value.blocked_reason;
|
|
32844
|
-
const blockedReason = blockedValue === null ? null : blockedValue === "missing_permission" || blockedValue === "permission_policy_missing" || blockedValue === "permission_verification_unavailable" || blockedValue === "missing_app_feature" ? blockedValue : void 0;
|
|
33222
|
+
const blockedReason = blockedValue === null ? null : blockedValue === "missing_permission" || blockedValue === "permission_policy_missing" || blockedValue === "permission_verification_unavailable" || blockedValue === "missing_app_feature" || blockedValue === "inactive_connection" || blockedValue === "actions_disabled" || blockedValue === "admin_blocked" ? blockedValue : void 0;
|
|
32845
33223
|
if (!name || classification !== "read" && classification !== "action" || typeof value.available !== "boolean" || blockedReason === void 0) continue;
|
|
32846
33224
|
toolCapabilities.push({
|
|
32847
33225
|
name,
|
|
@@ -33043,14 +33421,24 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
33043
33421
|
if (!isRecord4(data) || !isRecord4(data.connection)) return enabled;
|
|
33044
33422
|
return data.connection.actionsEnabled === true;
|
|
33045
33423
|
}
|
|
33046
|
-
async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
33047
|
-
const requestId = `main-connected-action:${
|
|
33424
|
+
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
33425
|
+
const requestId = `main-connected-action:${createHash19("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID19()).digest("hex")}`;
|
|
33426
|
+
const requestDigest = createHash19("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
33048
33427
|
if (mainOwnsIntegrations()) {
|
|
33049
33428
|
const selectedTool = tool?.trim();
|
|
33050
33429
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
33051
33430
|
try {
|
|
33052
|
-
const claim = await claimServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId });
|
|
33431
|
+
const claim = await claimServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, requestDigest });
|
|
33053
33432
|
if (!claim.claimed) {
|
|
33433
|
+
if (claim.conflict) {
|
|
33434
|
+
throw new NangoControlError(
|
|
33435
|
+
"This Idempotency-Key was already used for different connected-action arguments.",
|
|
33436
|
+
409,
|
|
33437
|
+
"idempotency_conflict",
|
|
33438
|
+
false
|
|
33439
|
+
);
|
|
33440
|
+
}
|
|
33441
|
+
if (claim.status === "succeeded" && "result" in claim) return claim.result;
|
|
33054
33442
|
throw new NangoControlError(
|
|
33055
33443
|
"The previous connected action may have completed, so it was not run again.",
|
|
33056
33444
|
409,
|
|
@@ -33067,7 +33455,7 @@ async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
|
33067
33455
|
requestId,
|
|
33068
33456
|
operationKind: "action"
|
|
33069
33457
|
});
|
|
33070
|
-
await recordServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, status: "succeeded" });
|
|
33458
|
+
await recordServiceConnectionAction({ identity, connectionId, tool: selectedTool, requestId, status: "succeeded", result });
|
|
33071
33459
|
return result;
|
|
33072
33460
|
} catch (error) {
|
|
33073
33461
|
const normalized = asNangoControlError(error);
|
|
@@ -33092,7 +33480,7 @@ async function callScheduleConnectionAction(identity, connectionId, input, tool,
|
|
|
33092
33480
|
const data = unwrapData(body);
|
|
33093
33481
|
return isRecord4(data) ? data.result ?? data : data;
|
|
33094
33482
|
}
|
|
33095
|
-
async function callScheduleConnectionRead(identity, connectionId, tool, args,
|
|
33483
|
+
async function callScheduleConnectionRead(identity, connectionId, tool, args, idempotencyKey3) {
|
|
33096
33484
|
if (mainOwnsIntegrations()) {
|
|
33097
33485
|
try {
|
|
33098
33486
|
return await callNangoToolDirect({
|
|
@@ -33101,7 +33489,7 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args, id
|
|
|
33101
33489
|
tool,
|
|
33102
33490
|
input: args ?? {},
|
|
33103
33491
|
classification: "read",
|
|
33104
|
-
requestId:
|
|
33492
|
+
requestId: idempotencyKey3,
|
|
33105
33493
|
operationKind: "read"
|
|
33106
33494
|
});
|
|
33107
33495
|
} catch (error) {
|
|
@@ -33186,7 +33574,7 @@ function canonicalJson2(value) {
|
|
|
33186
33574
|
return JSON.stringify(value);
|
|
33187
33575
|
}
|
|
33188
33576
|
function projectedToolSchemaHash(tool) {
|
|
33189
|
-
return
|
|
33577
|
+
return createHash19("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
33190
33578
|
}
|
|
33191
33579
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
33192
33580
|
if (mainOwnsIntegrations()) {
|
|
@@ -33465,6 +33853,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
33465
33853
|
}
|
|
33466
33854
|
|
|
33467
33855
|
// src/api/resend-control.ts
|
|
33856
|
+
import { createHash as createHash20 } from "crypto";
|
|
33468
33857
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
33469
33858
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
33470
33859
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -33722,9 +34111,11 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
33722
34111
|
const data = unwrapData2(body);
|
|
33723
34112
|
return isRecord5(data) ? data.result ?? data : data;
|
|
33724
34113
|
}
|
|
33725
|
-
async function callResendAction(identity, connectionId, tool, input) {
|
|
34114
|
+
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
34115
|
+
const requestId = `main-resend-action:${createHash20("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
33726
34116
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
33727
34117
|
method: "POST",
|
|
34118
|
+
headers: { "x-request-id": requestId },
|
|
33728
34119
|
body: JSON.stringify({ identity, connectionId, tool, input })
|
|
33729
34120
|
});
|
|
33730
34121
|
const data = unwrapData2(body);
|
|
@@ -33848,9 +34239,9 @@ var auth2 = createMiddleware4(async (c, next) => {
|
|
|
33848
34239
|
return next();
|
|
33849
34240
|
});
|
|
33850
34241
|
var sessionAuth = createMiddleware4(async (c, next) => {
|
|
33851
|
-
const
|
|
33852
|
-
if (!
|
|
33853
|
-
const userId = verifySession(
|
|
34242
|
+
const token4 = getCookie2(c, "session");
|
|
34243
|
+
if (!token4) return c.json({ error: "Not authenticated" }, 401);
|
|
34244
|
+
const userId = verifySession(token4);
|
|
33854
34245
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
33855
34246
|
const user = await getUserById(userId);
|
|
33856
34247
|
if (!user) return c.json({ error: "User not found" }, 401);
|
|
@@ -33876,7 +34267,11 @@ var requirePaidSchedulingTier = createMiddleware4(async (c, next) => {
|
|
|
33876
34267
|
return next();
|
|
33877
34268
|
});
|
|
33878
34269
|
var app = new Hono33();
|
|
34270
|
+
var deploymentProfile = resolveDeploymentProfile();
|
|
33879
34271
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
34272
|
+
function permitsLocalApiRequest(request) {
|
|
34273
|
+
return (deploymentProfile === "development" || deploymentProfile === "test" || process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1") && request.headers.get("x-mcp-scraper-local-network") === "1" && isLoopbackBaseUrl(request.url);
|
|
34274
|
+
}
|
|
33880
34275
|
function requireStripeSecret() {
|
|
33881
34276
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
33882
34277
|
if (secret2) return secret2;
|
|
@@ -33992,8 +34387,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
33992
34387
|
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
33993
34388
|
}
|
|
33994
34389
|
})();
|
|
33995
|
-
const
|
|
33996
|
-
setCookie2(c, "session",
|
|
34390
|
+
const token4 = signSession(user.id);
|
|
34391
|
+
setCookie2(c, "session", token4, sessionCookieOptions2);
|
|
33997
34392
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
33998
34393
|
} catch (err) {
|
|
33999
34394
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -34011,8 +34406,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
34011
34406
|
if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
|
|
34012
34407
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
34013
34408
|
}
|
|
34014
|
-
const
|
|
34015
|
-
setCookie2(c, "session",
|
|
34409
|
+
const token4 = signSession(user.id);
|
|
34410
|
+
setCookie2(c, "session", token4, sessionCookieOptions2);
|
|
34016
34411
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
34017
34412
|
});
|
|
34018
34413
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
@@ -34067,9 +34462,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
34067
34462
|
if (emailLimited) return emailLimited;
|
|
34068
34463
|
const user = await getUserByEmail(normalizedEmail);
|
|
34069
34464
|
if (!user) return c.json({ ok: true });
|
|
34070
|
-
const
|
|
34465
|
+
const token4 = await createPasswordResetToken(user.id);
|
|
34071
34466
|
const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
|
|
34072
|
-
const resetUrl = `${appUrl}/reset-password?token=${
|
|
34467
|
+
const resetUrl = `${appUrl}/reset-password?token=${token4}`;
|
|
34073
34468
|
if (process.env.RESEND_API_KEY) {
|
|
34074
34469
|
try {
|
|
34075
34470
|
const resend = new Resend3(process.env.RESEND_API_KEY);
|
|
@@ -34089,12 +34484,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
34089
34484
|
return c.json({ ok: true });
|
|
34090
34485
|
});
|
|
34091
34486
|
app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
|
|
34092
|
-
const { token:
|
|
34093
|
-
if (!
|
|
34487
|
+
const { token: token4, password } = await c.req.json();
|
|
34488
|
+
if (!token4 || !password) return c.json({ error: "Token and password required" }, 400);
|
|
34094
34489
|
if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
|
|
34095
34490
|
const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
|
|
34096
34491
|
if (limited) return limited;
|
|
34097
|
-
const userId = await consumePasswordResetToken(
|
|
34492
|
+
const userId = await consumePasswordResetToken(token4);
|
|
34098
34493
|
if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
|
|
34099
34494
|
await setPassword(userId, password);
|
|
34100
34495
|
return c.json({ ok: true });
|
|
@@ -34131,9 +34526,9 @@ function sessionBootstrapPayload(user) {
|
|
|
34131
34526
|
}
|
|
34132
34527
|
app.get("/me", async (c) => {
|
|
34133
34528
|
c.header("Cache-Control", "no-store");
|
|
34134
|
-
const
|
|
34135
|
-
if (!
|
|
34136
|
-
const userId = verifySession(
|
|
34529
|
+
const token4 = getCookie2(c, "session");
|
|
34530
|
+
if (!token4) return c.json({ authenticated: false });
|
|
34531
|
+
const userId = verifySession(token4);
|
|
34137
34532
|
if (!userId) {
|
|
34138
34533
|
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
34139
34534
|
return c.json({ authenticated: false });
|
|
@@ -34470,7 +34865,11 @@ function scheduleConnectionError(c, err, fallback) {
|
|
|
34470
34865
|
return c.json({ ok: false, error: fallback }, 502);
|
|
34471
34866
|
}
|
|
34472
34867
|
function connectedActionIdempotencyKey(c) {
|
|
34473
|
-
|
|
34868
|
+
const key = (c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key"))?.trim();
|
|
34869
|
+
if (!key || key.length < 8 || key.length > 200) {
|
|
34870
|
+
throw new ScheduleConnectionValidationError("Idempotency-Key is required and must contain 8-200 characters for external actions.");
|
|
34871
|
+
}
|
|
34872
|
+
return key;
|
|
34474
34873
|
}
|
|
34475
34874
|
function connectionMemoryImportError(c, err) {
|
|
34476
34875
|
return c.json({
|
|
@@ -35197,12 +35596,13 @@ app.post("/schedule-connections/actions/call", auth2, requireIntegrationsTier, a
|
|
|
35197
35596
|
return c.json({ ok: false, error: "connectionId, tool, and an args object are required." }, 400);
|
|
35198
35597
|
}
|
|
35199
35598
|
try {
|
|
35200
|
-
const
|
|
35599
|
+
const idempotencyKey3 = connectedActionIdempotencyKey(c);
|
|
35600
|
+
const result = await isResendConnection(user.email, connectionId, providerConfigKey) ? await callResendAction(user.email, connectionId, tool, body.args, idempotencyKey3) : await callScheduleConnectionAction(
|
|
35201
35601
|
user.email,
|
|
35202
35602
|
connectionId,
|
|
35203
35603
|
body.args,
|
|
35204
35604
|
tool,
|
|
35205
|
-
|
|
35605
|
+
idempotencyKey3
|
|
35206
35606
|
);
|
|
35207
35607
|
return c.json({ ok: true, result });
|
|
35208
35608
|
} catch (err) {
|
|
@@ -35547,7 +35947,7 @@ app.post("/harvest", auth2, async (c) => {
|
|
|
35547
35947
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
35548
35948
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
35549
35949
|
} else {
|
|
35550
|
-
jobId2 =
|
|
35950
|
+
jobId2 = randomUUID20();
|
|
35551
35951
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
35552
35952
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
35553
35953
|
const hold = await debitMcIdempotent(
|
|
@@ -35740,9 +36140,20 @@ app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
|
|
|
35740
36140
|
});
|
|
35741
36141
|
app.post("/extract-url", auth2, async (c) => {
|
|
35742
36142
|
const raw = await c.req.json().catch(() => ({}));
|
|
35743
|
-
const
|
|
36143
|
+
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
36144
|
+
const bodyResult = (allowLocalRequest ? ExtractUrlLocalBodySchema : ExtractUrlBodySchema).safeParse(raw);
|
|
35744
36145
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
35745
|
-
const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage,
|
|
36146
|
+
const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, mediaTypes } = bodyResult.data;
|
|
36147
|
+
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
36148
|
+
const requestedDelivery = bodyResult.data.delivery ?? "auto";
|
|
36149
|
+
if (bodyResult.data.depositToVault && !["auto", "memory"].includes(requestedDelivery)) {
|
|
36150
|
+
return c.json({ error: 'depositToVault conflicts with the requested delivery; use delivery:"memory" instead.' }, 400);
|
|
36151
|
+
}
|
|
36152
|
+
if (bodyResult.data.preserveMedia !== void 0 && bodyResult.data.downloadMedia !== void 0 && bodyResult.data.preserveMedia !== bodyResult.data.downloadMedia) {
|
|
36153
|
+
return c.json({ error: "preserveMedia conflicts with deprecated downloadMedia." }, 400);
|
|
36154
|
+
}
|
|
36155
|
+
const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
|
|
36156
|
+
const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
|
|
35746
36157
|
if (!allowLocal) {
|
|
35747
36158
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
35748
36159
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -35788,19 +36199,47 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35788
36199
|
if (screenshotBuf) {
|
|
35789
36200
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
35790
36201
|
}
|
|
35791
|
-
const mediaMeta =
|
|
36202
|
+
const mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
35792
36203
|
const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
|
|
35793
36204
|
html: result.bodyHtml,
|
|
35794
36205
|
meta: result.meta,
|
|
35795
36206
|
schema: result.schema,
|
|
35796
36207
|
pageUrl: waybackReplay?.rawReplayUrl ?? canonicalUrl
|
|
35797
36208
|
}) : null;
|
|
35798
|
-
const memory =
|
|
36209
|
+
const memory = effectiveDelivery === "memory" ? await persistScrapeBody(user, {
|
|
35799
36210
|
title: result.title ?? canonicalUrl,
|
|
35800
36211
|
content: result.bodyMarkdown ?? "",
|
|
35801
36212
|
source: canonicalUrl,
|
|
35802
36213
|
vault: bodyResult.data.vaultName
|
|
35803
36214
|
}) : void 0;
|
|
36215
|
+
const imageInputs = [];
|
|
36216
|
+
if (preserveMedia && screenshotMeta?.base64) {
|
|
36217
|
+
imageInputs.push({
|
|
36218
|
+
title: `${result.title ?? canonicalUrl} screenshot`,
|
|
36219
|
+
sourceUrl: canonicalUrl,
|
|
36220
|
+
imageBase64: screenshotMeta.base64,
|
|
36221
|
+
sourceKind: "screenshot",
|
|
36222
|
+
sourceRef: { device }
|
|
36223
|
+
});
|
|
36224
|
+
}
|
|
36225
|
+
if (preserveMedia && mediaMeta) {
|
|
36226
|
+
for (const asset of mediaMeta.assets) {
|
|
36227
|
+
if (asset.type !== "image" || !asset.url.startsWith("https://")) continue;
|
|
36228
|
+
imageInputs.push({
|
|
36229
|
+
title: asset.filename || `${result.title ?? canonicalUrl} image`,
|
|
36230
|
+
sourceUrl: canonicalUrl,
|
|
36231
|
+
imageUrl: asset.url,
|
|
36232
|
+
sourceKind: "page_image"
|
|
36233
|
+
});
|
|
36234
|
+
}
|
|
36235
|
+
}
|
|
36236
|
+
const memoryImages = effectiveDelivery === "memory" && preserveMedia ? await persistScrapeImagesToMemory(user, imageInputs, bodyResult.data.vaultName) : void 0;
|
|
36237
|
+
const screenshotArtifact = preserveMedia && screenshotBuf ? await createImageSourceArtifact({
|
|
36238
|
+
ownerId: String(user.id),
|
|
36239
|
+
content: screenshotBuf,
|
|
36240
|
+
contentType: "image/png",
|
|
36241
|
+
filename: `${new URL(canonicalUrl).hostname}-screenshot.png`
|
|
36242
|
+
}) : void 0;
|
|
35804
36243
|
const archive = waybackReplay ? {
|
|
35805
36244
|
timestamp: waybackReplay.timestamp,
|
|
35806
36245
|
originalUrl: waybackReplay.originalUrl,
|
|
@@ -35816,7 +36255,15 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35816
36255
|
screenshot: screenshotMeta,
|
|
35817
36256
|
branding: brandingData,
|
|
35818
36257
|
media: mediaMeta,
|
|
35819
|
-
memory
|
|
36258
|
+
memory,
|
|
36259
|
+
memoryImages,
|
|
36260
|
+
screenshotArtifact,
|
|
36261
|
+
delivery: {
|
|
36262
|
+
requested: requestedDelivery,
|
|
36263
|
+
effective: effectiveDelivery === "auto" ? "inline" : effectiveDelivery,
|
|
36264
|
+
retained: effectiveDelivery === "memory" ? memory?.deposited === true : effectiveDelivery === "artifact",
|
|
36265
|
+
nextAction: effectiveDelivery === "memory" ? "Use Memory search or read tools with the returned note identifiers." : effectiveDelivery === "artifact" ? "Use report_artifact_read with the returned artifactId." : null
|
|
36266
|
+
}
|
|
35820
36267
|
});
|
|
35821
36268
|
} catch (err) {
|
|
35822
36269
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -35829,9 +36276,11 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
35829
36276
|
});
|
|
35830
36277
|
app.post("/diff-page", auth2, async (c) => {
|
|
35831
36278
|
const raw = await c.req.json().catch(() => ({}));
|
|
35832
|
-
const
|
|
36279
|
+
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
36280
|
+
const bodyResult = (allowLocalRequest ? DiffPageLocalBodySchema : DiffPageBodySchema).safeParse(raw);
|
|
35833
36281
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
35834
|
-
const { url,
|
|
36282
|
+
const { url, resetBaseline } = bodyResult.data;
|
|
36283
|
+
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
35835
36284
|
if (!allowLocal) {
|
|
35836
36285
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
35837
36286
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -35923,19 +36372,31 @@ app.post("/archive/read", auth2, async (c) => {
|
|
|
35923
36372
|
message: bodyResult.error.issues[0]?.message ?? "Invalid request"
|
|
35924
36373
|
}, 400);
|
|
35925
36374
|
}
|
|
35926
|
-
const { url, path: path5, depositToLibrary } = bodyResult.data;
|
|
36375
|
+
const { artifactId, url, path: path5, depositToLibrary } = bodyResult.data;
|
|
35927
36376
|
const user = c.get("user");
|
|
35928
36377
|
const gate = await acquireConcurrencyGate(user, "archive_read", {
|
|
35929
36378
|
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
35930
|
-
metadata: { url }
|
|
36379
|
+
metadata: artifactId ? { artifactId } : { url }
|
|
35931
36380
|
});
|
|
35932
36381
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
35933
36382
|
try {
|
|
36383
|
+
let ownedArchive = null;
|
|
36384
|
+
if (artifactId) {
|
|
36385
|
+
const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
36386
|
+
if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
|
|
36387
|
+
ownedArchive = { buffer, ref: artifactId };
|
|
36388
|
+
}
|
|
35934
36389
|
if (!path5) {
|
|
35935
|
-
const inventory = await listZipArchive(url, bodyResult.data.maxEntries ?? 200);
|
|
35936
|
-
return c.json({ mode: "list", ...inventory });
|
|
36390
|
+
const inventory = ownedArchive ? await listZipArchiveBuffer(ownedArchive.buffer, ownedArchive.ref, bodyResult.data.maxEntries ?? 200) : await listZipArchive(url, bodyResult.data.maxEntries ?? 200);
|
|
36391
|
+
return c.json({ mode: "list", ...inventory, ...artifactId ? { artifactId } : {} });
|
|
35937
36392
|
}
|
|
35938
|
-
const file = await
|
|
36393
|
+
const file = ownedArchive ? await readZipArchiveBuffer(
|
|
36394
|
+
ownedArchive.buffer,
|
|
36395
|
+
ownedArchive.ref,
|
|
36396
|
+
path5,
|
|
36397
|
+
bodyResult.data.offset ?? 0,
|
|
36398
|
+
bodyResult.data.maxBytes ?? 5e4
|
|
36399
|
+
) : await readZipArchiveFile(
|
|
35939
36400
|
url,
|
|
35940
36401
|
path5,
|
|
35941
36402
|
bodyResult.data.offset ?? 0,
|
|
@@ -35957,7 +36418,7 @@ app.post("/archive/read", auth2, async (c) => {
|
|
|
35957
36418
|
summary: `Source file \`${file.path}\` preserved from ZIP archive ${librarySource.split("#")[0]}.`
|
|
35958
36419
|
}) : void 0;
|
|
35959
36420
|
const { fullContent: _fullContent, ...responseFile } = file;
|
|
35960
|
-
return c.json({ mode: "read", ...responseFile, memory });
|
|
36421
|
+
return c.json({ mode: "read", ...responseFile, ...artifactId ? { artifactId } : {}, memory });
|
|
35961
36422
|
} catch (error) {
|
|
35962
36423
|
const known = error instanceof ArchiveReadError ? error : new ArchiveReadError(
|
|
35963
36424
|
"archive_read_failed",
|
|
@@ -36099,6 +36560,9 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36099
36560
|
const bodyResult = ExtractSiteBodySchema.safeParse(raw);
|
|
36100
36561
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
36101
36562
|
const body = bodyResult.data;
|
|
36563
|
+
if (body.preserveMedia !== void 0 && body.downloadImages !== void 0 && body.preserveMedia !== body.downloadImages) {
|
|
36564
|
+
return c.json({ error: "preserveMedia conflicts with deprecated downloadImages." }, 400);
|
|
36565
|
+
}
|
|
36102
36566
|
const checked = await validatePublicHttpUrl(body.url, { field: "URL" });
|
|
36103
36567
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
36104
36568
|
const parsed = checked.parsed;
|
|
@@ -36122,7 +36586,7 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36122
36586
|
waybackMonths.length * Math.min(maxPagesPerSnapshot, body.wayback.urls?.length ?? maxPagesPerSnapshot)
|
|
36123
36587
|
) : requestedSiteMaxPages(body.maxPages);
|
|
36124
36588
|
const crawlStartUrl = body.wayback ? waybackRootUrl : waybackReplay?.rawReplayUrl ?? parsed.href;
|
|
36125
|
-
const downloadImages = body.downloadImages
|
|
36589
|
+
const downloadImages = body.preserveMedia ?? body.downloadImages ?? false;
|
|
36126
36590
|
const shouldRunInBackground = Boolean(body.wayback) || shouldRunSiteExtractInBackground(body);
|
|
36127
36591
|
if (shouldRunInBackground) {
|
|
36128
36592
|
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
@@ -36136,9 +36600,9 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36136
36600
|
if (!suppliedIdempotencyKey.trim() || suppliedIdempotencyKey.length > 500) {
|
|
36137
36601
|
return c.json({ error: "Idempotency-Key must contain 1-500 characters.", errorCode: "invalid_idempotency_key", retryable: false }, 400);
|
|
36138
36602
|
}
|
|
36139
|
-
const
|
|
36140
|
-
c.header("Idempotency-Key",
|
|
36141
|
-
const jobId2 = `ext_${sha256Hex(`${user.id}\0${
|
|
36603
|
+
const idempotencyKey3 = suppliedIdempotencyKey.trim();
|
|
36604
|
+
c.header("Idempotency-Key", idempotencyKey3);
|
|
36605
|
+
const jobId2 = `ext_${sha256Hex(`${user.id}\0${idempotencyKey3}`).slice(0, 24)}`;
|
|
36142
36606
|
const requestFingerprint2 = sha256Hex(JSON.stringify({
|
|
36143
36607
|
url: parsed.href,
|
|
36144
36608
|
waybackTimestamp: waybackReplay?.timestamp ?? null,
|
|
@@ -36157,7 +36621,7 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
36157
36621
|
balanceMc: user.balance_mc,
|
|
36158
36622
|
startUrl: crawlStartUrl,
|
|
36159
36623
|
requestedMaxPages,
|
|
36160
|
-
idempotencyKey:
|
|
36624
|
+
idempotencyKey: idempotencyKey3,
|
|
36161
36625
|
requestFingerprint: requestFingerprint2,
|
|
36162
36626
|
concurrency: concurrencyLimitForUser(user),
|
|
36163
36627
|
urlsPerBrowser: body.rotateProxyEvery ?? 10,
|
|
@@ -36701,7 +37165,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
36701
37165
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
36702
37166
|
return c.json({ error: "Unauthorized" }, 401);
|
|
36703
37167
|
}
|
|
36704
|
-
const { drainQueue } = await import("./worker-
|
|
37168
|
+
const { drainQueue } = await import("./worker-2LIOH744.js");
|
|
36705
37169
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
36706
37170
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
36707
37171
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
|
|
@@ -36754,8 +37218,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
36754
37218
|
finishExtractJob: finishExtractJob2,
|
|
36755
37219
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
36756
37220
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
36757
|
-
} = await import("./site-extract-repository-
|
|
36758
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
37221
|
+
} = await import("./site-extract-repository-XNI4ZK2M.js");
|
|
37222
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
|
|
36759
37223
|
const existing = await getExtractJob2(jobId2);
|
|
36760
37224
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
36761
37225
|
if (existing.status !== "failed") {
|
|
@@ -36782,7 +37246,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
36782
37246
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
36783
37247
|
let settlement = "already_settled_or_refunded";
|
|
36784
37248
|
if (job.billedMc == null && job.userId != null) {
|
|
36785
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
37249
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-XNI4ZK2M.js");
|
|
36786
37250
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
36787
37251
|
const successful = await countSuccessfulPages2(jobId2);
|
|
36788
37252
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -36858,7 +37322,7 @@ app.route("/", oauthApp);
|
|
|
36858
37322
|
if (!process.env.INNGEST_EVENT_KEY) {
|
|
36859
37323
|
startSiteAuditWorker();
|
|
36860
37324
|
}
|
|
36861
|
-
app.get("/font-test", (c) => {
|
|
37325
|
+
if (process.env.NODE_ENV !== "production") app.get("/font-test", (c) => {
|
|
36862
37326
|
return c.html(`<!DOCTYPE html>
|
|
36863
37327
|
<html lang="en">
|
|
36864
37328
|
<head>
|
|
@@ -36961,4 +37425,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
36961
37425
|
export {
|
|
36962
37426
|
app
|
|
36963
37427
|
};
|
|
36964
|
-
//# sourceMappingURL=server-
|
|
37428
|
+
//# sourceMappingURL=server-BPEXEKA6.js.map
|