mcp-scraper 0.66.1 → 0.66.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +983 -1791
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +58 -90
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-4OT7BAIE.js → chunk-3EYALCYK.js} +1 -1
- package/dist/{chunk-TNLGHNLX.js → chunk-3TKKJQB4.js} +1 -1
- package/dist/{chunk-GTGSD4YS.js → chunk-55B3T5L2.js} +21 -26
- package/dist/{chunk-TNJIPV74.js → chunk-ABYBPODM.js} +1 -1
- package/dist/{chunk-VRJ7CO6L.js → chunk-ALEBMPOW.js} +1 -1
- package/dist/{chunk-NS754CRS.js → chunk-CETDPNMZ.js} +23 -23
- package/dist/{chunk-75VW7CWE.js → chunk-DBWWIK7O.js} +1 -1
- package/dist/{chunk-4P3PHFPQ.js → chunk-K7Y27ND3.js} +2 -12
- package/dist/{chunk-3PIXKLGZ.js → chunk-KXRWY7DZ.js} +1 -1
- package/dist/{chunk-WID3MFRS.js → chunk-OZDCTNUV.js} +1 -1
- package/dist/{chunk-AQKJOSIB.js → chunk-PMCT2YQM.js} +1 -1
- package/dist/{chunk-E4LMY5OJ.js → chunk-YKL3O7IU.js} +134 -94
- package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js} +9 -9
- package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js} +3 -3
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -2
- package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js} +2 -2
- package/dist/{server-ELEJNRRE.js → server-Q7VYGSGQ.js} +413 -1249
- package/dist/{site-extract-repository-ASYEJO7Z.js → site-extract-repository-NTM4MDQN.js} +2 -2
- package/dist/{worker-MAVYYWM7.js → worker-UTBUWEXQ.js} +5 -5
- package/package.json +2 -1
package/dist/bin/api-server.cjs
CHANGED
|
@@ -3899,7 +3899,7 @@ __export(db_exports, {
|
|
|
3899
3899
|
cancelJob: () => cancelJob,
|
|
3900
3900
|
checkRateLimit: () => checkRateLimit,
|
|
3901
3901
|
checkpointJob: () => checkpointJob,
|
|
3902
|
-
|
|
3902
|
+
claimDurablePaaJob: () => claimDurablePaaJob,
|
|
3903
3903
|
claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
|
|
3904
3904
|
claimPendingJob: () => claimPendingJob,
|
|
3905
3905
|
claimPendingKpoJob: () => claimPendingKpoJob,
|
|
@@ -3929,6 +3929,7 @@ __export(db_exports, {
|
|
|
3929
3929
|
expireOldLots: () => expireOldLots,
|
|
3930
3930
|
failJob: () => failJob,
|
|
3931
3931
|
failKpoJob: () => failKpoJob,
|
|
3932
|
+
failRunningJob: () => failRunningJob,
|
|
3932
3933
|
failStripeEvent: () => failStripeEvent,
|
|
3933
3934
|
failWorkflowRunRecord: () => failWorkflowRunRecord,
|
|
3934
3935
|
finishHarvestAttempt: () => finishHarvestAttempt,
|
|
@@ -3968,10 +3969,9 @@ __export(db_exports, {
|
|
|
3968
3969
|
listInboxMessages: () => listInboxMessages,
|
|
3969
3970
|
listJobs: () => listJobs,
|
|
3970
3971
|
listKpoJobs: () => listKpoJobs,
|
|
3971
|
-
|
|
3972
|
-
listRecentTerminalInngestPaaJobs: () => listRecentTerminalInngestPaaJobs,
|
|
3972
|
+
listRecentTerminalDurablePaaJobs: () => listRecentTerminalDurablePaaJobs,
|
|
3973
3973
|
listRequestEvents: () => listRequestEvents,
|
|
3974
|
-
|
|
3974
|
+
listStaleRunningDurablePaaJobs: () => listStaleRunningDurablePaaJobs,
|
|
3975
3975
|
listUsers: () => listUsers,
|
|
3976
3976
|
listUsersDueForConnectedAccountBillingReconciliation: () => listUsersDueForConnectedAccountBillingReconciliation,
|
|
3977
3977
|
listWorkflowArtifacts: () => listWorkflowArtifacts,
|
|
@@ -5905,7 +5905,7 @@ async function claimPendingJob() {
|
|
|
5905
5905
|
const res = await db.execute(`
|
|
5906
5906
|
SELECT * FROM jobs
|
|
5907
5907
|
WHERE status = 'pending'
|
|
5908
|
-
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron')
|
|
5908
|
+
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'inngest')
|
|
5909
5909
|
ORDER BY created_at
|
|
5910
5910
|
LIMIT 1
|
|
5911
5911
|
`);
|
|
@@ -5917,10 +5917,10 @@ async function claimPendingJob() {
|
|
|
5917
5917
|
});
|
|
5918
5918
|
return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
|
|
5919
5919
|
}
|
|
5920
|
-
async function
|
|
5920
|
+
async function claimDurablePaaJob(id) {
|
|
5921
5921
|
const db = getDb();
|
|
5922
5922
|
const current = await getJob(id);
|
|
5923
|
-
if (!current || current.options.executionOwner !== "
|
|
5923
|
+
if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
|
|
5924
5924
|
if (current.status === "running") return void 0;
|
|
5925
5925
|
if (current.status === "pending") {
|
|
5926
5926
|
const updated = await db.execute({
|
|
@@ -5929,7 +5929,7 @@ async function claimInngestPaaJob(id) {
|
|
|
5929
5929
|
WHERE id = ? AND status = 'pending'`,
|
|
5930
5930
|
args: [id]
|
|
5931
5931
|
});
|
|
5932
|
-
if (updated.rowsAffected === 0) return
|
|
5932
|
+
if (updated.rowsAffected === 0) return void 0;
|
|
5933
5933
|
return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
|
|
5934
5934
|
}
|
|
5935
5935
|
return current;
|
|
@@ -5941,25 +5941,12 @@ async function checkpointJob(id, result) {
|
|
|
5941
5941
|
});
|
|
5942
5942
|
return updated.rowsAffected === 1;
|
|
5943
5943
|
}
|
|
5944
|
-
async function
|
|
5945
|
-
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5946
|
-
const rows = await getDb().execute({
|
|
5947
|
-
sql: `SELECT * FROM jobs
|
|
5948
|
-
WHERE status = 'pending'
|
|
5949
|
-
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
5950
|
-
AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
|
|
5951
|
-
ORDER BY created_at
|
|
5952
|
-
LIMIT ?`,
|
|
5953
|
-
args: [safeLimit]
|
|
5954
|
-
});
|
|
5955
|
-
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5956
|
-
}
|
|
5957
|
-
async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
5944
|
+
async function listRecentTerminalDurablePaaJobs(limit = 25) {
|
|
5958
5945
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5959
5946
|
const rows = await getDb().execute({
|
|
5960
5947
|
sql: `SELECT * FROM jobs
|
|
5961
5948
|
WHERE status IN ('done', 'failed', 'cancelled')
|
|
5962
|
-
AND json_extract(options, '$.executionOwner')
|
|
5949
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
5963
5950
|
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
5964
5951
|
AND json_extract(options, '$.billingSettledAt') IS NULL
|
|
5965
5952
|
ORDER BY completed_at DESC
|
|
@@ -5968,13 +5955,13 @@ async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
|
5968
5955
|
});
|
|
5969
5956
|
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5970
5957
|
}
|
|
5971
|
-
async function
|
|
5958
|
+
async function listStaleRunningDurablePaaJobs(limit = 25) {
|
|
5972
5959
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5973
5960
|
const rows = await getDb().execute({
|
|
5974
5961
|
sql: `SELECT * FROM jobs
|
|
5975
5962
|
WHERE status = 'running'
|
|
5976
|
-
AND started_at <= datetime('now', '-
|
|
5977
|
-
AND json_extract(options, '$.executionOwner')
|
|
5963
|
+
AND started_at <= datetime('now', '-15 minutes')
|
|
5964
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
5978
5965
|
ORDER BY started_at
|
|
5979
5966
|
LIMIT ?`,
|
|
5980
5967
|
args: [safeLimit]
|
|
@@ -6003,6 +5990,14 @@ async function failJob(id, error, publicError2) {
|
|
|
6003
5990
|
args: [error, serializePublicErrorEnvelope(publicError2), id]
|
|
6004
5991
|
});
|
|
6005
5992
|
}
|
|
5993
|
+
async function failRunningJob(id, error, publicError2) {
|
|
5994
|
+
const updated = await getDb().execute({
|
|
5995
|
+
sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
|
|
5996
|
+
WHERE id = ? AND status = 'running'`,
|
|
5997
|
+
args: [error, serializePublicErrorEnvelope(publicError2), id]
|
|
5998
|
+
});
|
|
5999
|
+
return updated.rowsAffected === 1;
|
|
6000
|
+
}
|
|
6006
6001
|
async function cancelJob(id, reason, publicError2) {
|
|
6007
6002
|
await getDb().execute({
|
|
6008
6003
|
sql: `UPDATE jobs SET status = 'cancelled', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
@@ -13808,12 +13803,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
13808
13803
|
`{"error_severity":"blocking","error_code":"dep-001","message":"Required Python package missing: {package}. Install with: pip install networkx pandas lxml","file":null,"remediation":"Run: pip install networkx pandas lxml"}`,
|
|
13809
13804
|
`\`\`\``,
|
|
13810
13805
|
``,
|
|
13811
|
-
`Also
|
|
13806
|
+
`Also check whether the optional uszips.csv classifier dataset is accessible. When unavailable, Phase 3 remains functional and falls back to LLM classification:`,
|
|
13812
13807
|
``,
|
|
13813
13808
|
`\`\`\`bash`,
|
|
13814
13809
|
`python3 -c "`,
|
|
13815
13810
|
`import csv, os`,
|
|
13816
|
-
`path = '/
|
|
13811
|
+
`path = os.environ.get('US_ZIPS_CSV_PATH') or os.environ.get('MCP_SCRAPER_USZIPS_CSV_PATH') or '/tmp/uszips.csv'`,
|
|
13817
13812
|
`if not os.path.exists(path):`,
|
|
13818
13813
|
` print('uszips:missing')`,
|
|
13819
13814
|
`else:`,
|
|
@@ -13827,12 +13822,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
13827
13822
|
`If \`uszips:missing\`, write a **warning** (non-blocking) to \`ingestion_error_log.jsonl\`:`,
|
|
13828
13823
|
``,
|
|
13829
13824
|
`\`\`\`json`,
|
|
13830
|
-
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv not found at
|
|
13825
|
+
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv was not found at the resolved path. The location classifier is disabled and Phase 3 will use LLM classification, which may increase cost.","file":"{resolved_uszips_path}","remediation":"Optionally set US_ZIPS_CSV_PATH or MCP_SCRAPER_USZIPS_CSV_PATH to a readable uszips.csv file with the expected schema."}`,
|
|
13831
13826
|
`\`\`\``,
|
|
13832
13827
|
``,
|
|
13833
13828
|
`If \`uszips:schema-mismatch\`, write the same warning with the mismatch detail in the message. The audit continues without the location classifier \u2014 all URLs fall through to Claude classification.`,
|
|
13834
13829
|
``,
|
|
13835
|
-
`
|
|
13830
|
+
`Continue after recording either warning; missing or mismatched uszips.csv data is not a blocking gate.`,
|
|
13836
13831
|
``,
|
|
13837
13832
|
`---`,
|
|
13838
13833
|
``,
|
|
@@ -14766,6 +14761,9 @@ var init_score_synthesize = __esm({
|
|
|
14766
14761
|
});
|
|
14767
14762
|
|
|
14768
14763
|
// src/services/site-architecture-auditor/site-audit-service.ts
|
|
14764
|
+
function resolveSiteAuditUsZipsCsvPath(env = process.env) {
|
|
14765
|
+
return env["US_ZIPS_CSV_PATH"]?.trim() || env["MCP_SCRAPER_USZIPS_CSV_PATH"]?.trim() || "/tmp/uszips.csv";
|
|
14766
|
+
}
|
|
14769
14767
|
var import_zod10, import_p_limit2, OrphanAnnotationOutputSchema, CompareRecommendOutputSchema, SiteAuditService;
|
|
14770
14768
|
var init_site_audit_service = __esm({
|
|
14771
14769
|
"src/services/site-architecture-auditor/site-audit-service.ts"() {
|
|
@@ -14986,7 +14984,7 @@ var init_site_audit_service = __esm({
|
|
|
14986
14984
|
currentPayloadChars += itemChars;
|
|
14987
14985
|
}
|
|
14988
14986
|
if (currentBatch.length > 0) batches.push(currentBatch);
|
|
14989
|
-
const usZipsCsvPath =
|
|
14987
|
+
const usZipsCsvPath = resolveSiteAuditUsZipsCsvPath();
|
|
14990
14988
|
await this.runLlmBilled(job.user_id, `${jobId2}:phase3-classify`, async () => {
|
|
14991
14989
|
for (const batch of batches) {
|
|
14992
14990
|
let classified = [];
|
|
@@ -31852,1301 +31850,6 @@ var init_lead_list_enrichment2 = __esm({
|
|
|
31852
31850
|
}
|
|
31853
31851
|
});
|
|
31854
31852
|
|
|
31855
|
-
// src/api/paa-capture-artifacts.ts
|
|
31856
|
-
function paaCaptureArtifactPolicy() {
|
|
31857
|
-
return {
|
|
31858
|
-
prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
|
|
31859
|
-
artifactTtlMs: 7 * DAY_MS,
|
|
31860
|
-
downloadTtlMs: 15 * 60 * 1e3,
|
|
31861
|
-
token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
|
|
31862
|
-
};
|
|
31863
|
-
}
|
|
31864
|
-
async function createPaaRawDomArtifact(input) {
|
|
31865
|
-
return createPrivateArtifact({
|
|
31866
|
-
policy: paaCaptureArtifactPolicy(),
|
|
31867
|
-
ownerId: String(input.userId),
|
|
31868
|
-
artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
|
|
31869
|
-
createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
|
|
31870
|
-
filename: `${input.jobId}-serp.html.gz`,
|
|
31871
|
-
contentType: "application/gzip",
|
|
31872
|
-
content: input.gzip
|
|
31873
|
-
});
|
|
31874
|
-
}
|
|
31875
|
-
async function tryCreatePaaRawDomArtifact(input) {
|
|
31876
|
-
try {
|
|
31877
|
-
return {
|
|
31878
|
-
artifact: await createPaaRawDomArtifact(input),
|
|
31879
|
-
status: "stored"
|
|
31880
|
-
};
|
|
31881
|
-
} catch (error) {
|
|
31882
|
-
console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
|
|
31883
|
-
return { artifact: null, status: "unavailable" };
|
|
31884
|
-
}
|
|
31885
|
-
}
|
|
31886
|
-
var DAY_MS, PAA_CAPTURE_ARTIFACT_PREFIX;
|
|
31887
|
-
var init_paa_capture_artifacts = __esm({
|
|
31888
|
-
"src/api/paa-capture-artifacts.ts"() {
|
|
31889
|
-
"use strict";
|
|
31890
|
-
init_private_artifacts();
|
|
31891
|
-
DAY_MS = 24 * 60 * 60 * 1e3;
|
|
31892
|
-
PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
|
|
31893
|
-
}
|
|
31894
|
-
});
|
|
31895
|
-
|
|
31896
|
-
// src/api/paa-harvest-settlement.ts
|
|
31897
|
-
function capturedQuestionCount(result) {
|
|
31898
|
-
if (!result || typeof result !== "object") return 0;
|
|
31899
|
-
const value = result;
|
|
31900
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
31901
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
31902
|
-
}
|
|
31903
|
-
function finalCost(result, heldMc) {
|
|
31904
|
-
const questions = capturedQuestionCount(result);
|
|
31905
|
-
if (questions <= 0) return 0;
|
|
31906
|
-
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
31907
|
-
}
|
|
31908
|
-
async function settlePaaHarvestJob(jobOrId) {
|
|
31909
|
-
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
31910
|
-
if (!job) return false;
|
|
31911
|
-
const options = job.options;
|
|
31912
|
-
const debitKey2 = options.billingDebitKey;
|
|
31913
|
-
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
31914
|
-
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
31915
|
-
const settlement = await settleDebitMcIdempotent(
|
|
31916
|
-
job.user_id,
|
|
31917
|
-
debitKey2,
|
|
31918
|
-
finalCost(job.result, heldMc),
|
|
31919
|
-
LedgerOperation.PAA_REFUND,
|
|
31920
|
-
"durable PAA harvest settlement",
|
|
31921
|
-
"paa_harvest"
|
|
31922
|
-
);
|
|
31923
|
-
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
31924
|
-
return true;
|
|
31925
|
-
}
|
|
31926
|
-
var init_paa_harvest_settlement = __esm({
|
|
31927
|
-
"src/api/paa-harvest-settlement.ts"() {
|
|
31928
|
-
"use strict";
|
|
31929
|
-
init_db();
|
|
31930
|
-
init_rates();
|
|
31931
|
-
}
|
|
31932
|
-
});
|
|
31933
|
-
|
|
31934
|
-
// src/api/serp-identity-db.ts
|
|
31935
|
-
async function createSerpIdentityRow(input) {
|
|
31936
|
-
const id = `serpi_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
31937
|
-
await getDb().execute({
|
|
31938
|
-
sql: `INSERT INTO serp_identities
|
|
31939
|
-
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
31940
|
-
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
31941
|
-
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
31942
|
-
});
|
|
31943
|
-
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
31944
|
-
if (!row) throw new Error("SERP identity insert failed");
|
|
31945
|
-
return row;
|
|
31946
|
-
}
|
|
31947
|
-
async function getSerpIdentityRow(userId, name) {
|
|
31948
|
-
const result = await getDb().execute({
|
|
31949
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
31950
|
-
args: [userId, name]
|
|
31951
|
-
});
|
|
31952
|
-
return result.rows[0] ?? null;
|
|
31953
|
-
}
|
|
31954
|
-
async function listSerpIdentityRows(userId) {
|
|
31955
|
-
const result = await getDb().execute({
|
|
31956
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
31957
|
-
args: [userId]
|
|
31958
|
-
});
|
|
31959
|
-
return result.rows;
|
|
31960
|
-
}
|
|
31961
|
-
async function touchSerpIdentity(userId, name) {
|
|
31962
|
-
await getDb().execute({
|
|
31963
|
-
sql: `UPDATE serp_identities
|
|
31964
|
-
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
31965
|
-
WHERE user_id = ? AND name = ?`,
|
|
31966
|
-
args: [userId, name]
|
|
31967
|
-
});
|
|
31968
|
-
}
|
|
31969
|
-
async function setSerpIdentityStatus(userId, name, status) {
|
|
31970
|
-
await getDb().execute({
|
|
31971
|
-
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
31972
|
-
args: [status, userId, name]
|
|
31973
|
-
});
|
|
31974
|
-
}
|
|
31975
|
-
async function deleteSerpIdentityRow(userId, name) {
|
|
31976
|
-
await getDb().execute({
|
|
31977
|
-
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
31978
|
-
args: [userId, name]
|
|
31979
|
-
});
|
|
31980
|
-
}
|
|
31981
|
-
var import_node_crypto15;
|
|
31982
|
-
var init_serp_identity_db = __esm({
|
|
31983
|
-
"src/api/serp-identity-db.ts"() {
|
|
31984
|
-
"use strict";
|
|
31985
|
-
import_node_crypto15 = require("crypto");
|
|
31986
|
-
init_db();
|
|
31987
|
-
}
|
|
31988
|
-
});
|
|
31989
|
-
|
|
31990
|
-
// src/api/webhook.ts
|
|
31991
|
-
async function deliverWebhook(url, payload, retries = 3) {
|
|
31992
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
31993
|
-
try {
|
|
31994
|
-
const res = await fetch(url, {
|
|
31995
|
-
method: "POST",
|
|
31996
|
-
headers: { "content-type": "application/json" },
|
|
31997
|
-
body: JSON.stringify(payload),
|
|
31998
|
-
signal: AbortSignal.timeout(1e4)
|
|
31999
|
-
});
|
|
32000
|
-
if (res.ok) return;
|
|
32001
|
-
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
32002
|
-
} catch (err) {
|
|
32003
|
-
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
32004
|
-
}
|
|
32005
|
-
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
32006
|
-
}
|
|
32007
|
-
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
32008
|
-
}
|
|
32009
|
-
var init_webhook = __esm({
|
|
32010
|
-
"src/api/webhook.ts"() {
|
|
32011
|
-
"use strict";
|
|
32012
|
-
}
|
|
32013
|
-
});
|
|
32014
|
-
|
|
32015
|
-
// src/api/harvest-problems.ts
|
|
32016
|
-
function errorMessage4(err) {
|
|
32017
|
-
return err instanceof Error ? err.message : String(err);
|
|
32018
|
-
}
|
|
32019
|
-
function looksLikeTimeout(err, message) {
|
|
32020
|
-
if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
|
|
32021
|
-
return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
|
|
32022
|
-
}
|
|
32023
|
-
function looksLikeCaptcha(message) {
|
|
32024
|
-
return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
|
|
32025
|
-
}
|
|
32026
|
-
function looksLikeProxyTunnelFailure3(message) {
|
|
32027
|
-
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
32028
|
-
}
|
|
32029
|
-
function looksLikeProxyUnavailable3(message) {
|
|
32030
|
-
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
32031
|
-
}
|
|
32032
|
-
function looksLikeInternalServiceUnavailable(message) {
|
|
32033
|
-
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
32034
|
-
}
|
|
32035
|
-
function looksLikeBrowserSessionInterrupted(message) {
|
|
32036
|
-
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
32037
|
-
}
|
|
32038
|
-
function classifyHarvestProblem(err) {
|
|
32039
|
-
const message = errorMessage4(err);
|
|
32040
|
-
if (err instanceof RequestAbortedError) {
|
|
32041
|
-
return {
|
|
32042
|
-
error_code: "request_aborted",
|
|
32043
|
-
error_type: "request_aborted",
|
|
32044
|
-
message: publicErrorMessage("request_aborted"),
|
|
32045
|
-
retryable: true,
|
|
32046
|
-
httpStatus: 408,
|
|
32047
|
-
terminalStatus: "cancelled"
|
|
32048
|
-
};
|
|
32049
|
-
}
|
|
32050
|
-
if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
|
|
32051
|
-
return {
|
|
32052
|
-
error_code: "captcha_exhausted",
|
|
32053
|
-
error_type: "captcha",
|
|
32054
|
-
message: publicErrorMessage("captcha_exhausted"),
|
|
32055
|
-
retryable: true,
|
|
32056
|
-
httpStatus: 503,
|
|
32057
|
-
terminalStatus: "failed"
|
|
32058
|
-
};
|
|
32059
|
-
}
|
|
32060
|
-
if (isVendorUnavailableError(err, message)) {
|
|
32061
|
-
return {
|
|
32062
|
-
error_code: "vendor_unavailable",
|
|
32063
|
-
error_type: "service_unavailable",
|
|
32064
|
-
message: publicErrorMessage("vendor_unavailable"),
|
|
32065
|
-
retryable: true,
|
|
32066
|
-
httpStatus: 503,
|
|
32067
|
-
terminalStatus: "failed"
|
|
32068
|
-
};
|
|
32069
|
-
}
|
|
32070
|
-
if (err instanceof LocationMismatchError) {
|
|
32071
|
-
return {
|
|
32072
|
-
error_code: "location_mismatch",
|
|
32073
|
-
error_type: "location_mismatch",
|
|
32074
|
-
message: publicErrorMessage("location_mismatch"),
|
|
32075
|
-
retryable: true,
|
|
32076
|
-
httpStatus: 503,
|
|
32077
|
-
terminalStatus: "failed"
|
|
32078
|
-
};
|
|
32079
|
-
}
|
|
32080
|
-
if (looksLikeProxyTunnelFailure3(message)) {
|
|
32081
|
-
return {
|
|
32082
|
-
error_code: "proxy_tunnel_failed",
|
|
32083
|
-
error_type: "connection",
|
|
32084
|
-
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
32085
|
-
retryable: true,
|
|
32086
|
-
httpStatus: 503,
|
|
32087
|
-
terminalStatus: "failed"
|
|
32088
|
-
};
|
|
32089
|
-
}
|
|
32090
|
-
if (looksLikeProxyUnavailable3(message)) {
|
|
32091
|
-
return {
|
|
32092
|
-
error_code: "proxy_unavailable",
|
|
32093
|
-
error_type: "connection",
|
|
32094
|
-
message: publicErrorMessage("proxy_unavailable"),
|
|
32095
|
-
retryable: true,
|
|
32096
|
-
httpStatus: 503,
|
|
32097
|
-
terminalStatus: "failed"
|
|
32098
|
-
};
|
|
32099
|
-
}
|
|
32100
|
-
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
32101
|
-
return {
|
|
32102
|
-
error_code: "browser_session_interrupted",
|
|
32103
|
-
error_type: "extraction",
|
|
32104
|
-
message: publicErrorMessage("browser_session_interrupted"),
|
|
32105
|
-
retryable: true,
|
|
32106
|
-
httpStatus: 503,
|
|
32107
|
-
terminalStatus: "failed"
|
|
32108
|
-
};
|
|
32109
|
-
}
|
|
32110
|
-
if (looksLikeTimeout(err, message)) {
|
|
32111
|
-
return {
|
|
32112
|
-
error_code: "harvest_timeout",
|
|
32113
|
-
error_type: "timeout",
|
|
32114
|
-
message: publicErrorMessage("harvest_timeout"),
|
|
32115
|
-
retryable: true,
|
|
32116
|
-
httpStatus: 504,
|
|
32117
|
-
terminalStatus: "failed"
|
|
32118
|
-
};
|
|
32119
|
-
}
|
|
32120
|
-
if (looksLikeInternalServiceUnavailable(message)) {
|
|
32121
|
-
return {
|
|
32122
|
-
error_code: "service_unavailable",
|
|
32123
|
-
error_type: "service_unavailable",
|
|
32124
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
32125
|
-
retryable: true,
|
|
32126
|
-
httpStatus: 503,
|
|
32127
|
-
terminalStatus: "failed"
|
|
32128
|
-
};
|
|
32129
|
-
}
|
|
32130
|
-
return {
|
|
32131
|
-
error_code: "extraction_failed",
|
|
32132
|
-
error_type: "extraction",
|
|
32133
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
32134
|
-
retryable: false,
|
|
32135
|
-
httpStatus: 500,
|
|
32136
|
-
terminalStatus: "failed"
|
|
32137
|
-
};
|
|
32138
|
-
}
|
|
32139
|
-
function serializeHarvestProblem(problem) {
|
|
32140
|
-
return JSON.stringify({
|
|
32141
|
-
error_code: problem.error_code,
|
|
32142
|
-
error_type: problem.error_type,
|
|
32143
|
-
message: problem.message,
|
|
32144
|
-
retryable: problem.retryable
|
|
32145
|
-
});
|
|
32146
|
-
}
|
|
32147
|
-
function harvestProblemResponse(problem, context = {}) {
|
|
32148
|
-
const envelope = buildPublicErrorEnvelope({
|
|
32149
|
-
errorCode: problem.error_code,
|
|
32150
|
-
errorType: problem.error_type,
|
|
32151
|
-
retryable: problem.retryable,
|
|
32152
|
-
retryAfterSeconds: context.retryAfterSeconds,
|
|
32153
|
-
chargeStatus: context.chargeStatus,
|
|
32154
|
-
details: context.details
|
|
32155
|
-
});
|
|
32156
|
-
return {
|
|
32157
|
-
error: envelope.message,
|
|
32158
|
-
...envelope
|
|
32159
|
-
};
|
|
32160
|
-
}
|
|
32161
|
-
function harvestProblemEnvelope(problem, context = {}) {
|
|
32162
|
-
const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context);
|
|
32163
|
-
return envelope;
|
|
32164
|
-
}
|
|
32165
|
-
function publicErrorBoundary(err, context = {}) {
|
|
32166
|
-
const problem = classifyHarvestProblem(err);
|
|
32167
|
-
return {
|
|
32168
|
-
body: harvestProblemResponse(problem, context),
|
|
32169
|
-
status: problem.httpStatus
|
|
32170
|
-
};
|
|
32171
|
-
}
|
|
32172
|
-
var init_harvest_problems = __esm({
|
|
32173
|
-
"src/api/harvest-problems.ts"() {
|
|
32174
|
-
"use strict";
|
|
32175
|
-
init_errors();
|
|
32176
|
-
init_vendor_errors();
|
|
32177
|
-
}
|
|
32178
|
-
});
|
|
32179
|
-
|
|
32180
|
-
// src/paa/durable-capture.ts
|
|
32181
|
-
function remainingSeconds(deadlineMs, ceiling) {
|
|
32182
|
-
return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
|
|
32183
|
-
}
|
|
32184
|
-
async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
|
|
32185
|
-
if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
|
|
32186
|
-
const response = await kernel.browsers.playwright.execute(sessionId, {
|
|
32187
|
-
code,
|
|
32188
|
-
timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
|
|
32189
|
-
});
|
|
32190
|
-
if (!response.success || response.result === null || response.result === void 0) {
|
|
32191
|
-
throw new Error(response.error || response.stderr || "co-located browser execution failed");
|
|
32192
|
-
}
|
|
32193
|
-
return response.result;
|
|
32194
|
-
}
|
|
32195
|
-
function navigationCode(searchUrl) {
|
|
32196
|
-
return `
|
|
32197
|
-
const wait = ms => page.waitForTimeout(ms);
|
|
32198
|
-
const block = async () => {
|
|
32199
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
32200
|
-
if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
|
|
32201
|
-
const title = await page.title().catch(() => '');
|
|
32202
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
32203
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
|
|
32204
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
|
|
32205
|
-
return null;
|
|
32206
|
-
};
|
|
32207
|
-
await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
32208
|
-
let captchaObserved = false;
|
|
32209
|
-
let solverCleared = false;
|
|
32210
|
-
let blocked = await block();
|
|
32211
|
-
const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
|
|
32212
|
-
while (blocked === 'captcha' && Date.now() < solverDeadline) {
|
|
32213
|
-
captchaObserved = true;
|
|
32214
|
-
await wait(4000 + Math.floor(Math.random() * 1500));
|
|
32215
|
-
blocked = await block();
|
|
32216
|
-
if (!blocked) solverCleared = true;
|
|
32217
|
-
}
|
|
32218
|
-
let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
32219
|
-
for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
|
|
32220
|
-
await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
|
|
32221
|
-
await wait(350 + Math.floor(Math.random() * 500));
|
|
32222
|
-
count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
32223
|
-
}
|
|
32224
|
-
await page.evaluate((itemSelector) => {
|
|
32225
|
-
const state = window;
|
|
32226
|
-
if (state.__mcpTrustedListenerInstalled) return;
|
|
32227
|
-
document.addEventListener('click', event => {
|
|
32228
|
-
if (event.target instanceof Element && event.target.closest(itemSelector)) {
|
|
32229
|
-
state.__mcpLastPaaClickTrusted = event.isTrusted;
|
|
32230
|
-
}
|
|
32231
|
-
}, { capture: true });
|
|
32232
|
-
state.__mcpTrustedListenerInstalled = true;
|
|
32233
|
-
}, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
|
|
32234
|
-
return {
|
|
32235
|
-
outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
|
|
32236
|
-
finalUrl: page.url(),
|
|
32237
|
-
title: await page.title().catch(() => ''),
|
|
32238
|
-
captchaObserved,
|
|
32239
|
-
solverCleared,
|
|
32240
|
-
environment: await page.evaluate(() => ({
|
|
32241
|
-
userAgent: navigator.userAgent,
|
|
32242
|
-
language: navigator.language,
|
|
32243
|
-
platform: navigator.platform,
|
|
32244
|
-
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
32245
|
-
innerWidth: window.innerWidth,
|
|
32246
|
-
innerHeight: window.innerHeight,
|
|
32247
|
-
outerWidth: window.outerWidth,
|
|
32248
|
-
outerHeight: window.outerHeight,
|
|
32249
|
-
screenWidth: window.screen.width,
|
|
32250
|
-
screenHeight: window.screen.height,
|
|
32251
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
32252
|
-
})).catch(() => null),
|
|
32253
|
-
};
|
|
32254
|
-
`;
|
|
32255
|
-
}
|
|
32256
|
-
function recordReaderSource() {
|
|
32257
|
-
return `
|
|
32258
|
-
const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
|
|
32259
|
-
const externalHref = anchor => {
|
|
32260
|
-
const href = anchor && anchor.href || '';
|
|
32261
|
-
if (!/^https?:\\/\\//i.test(href)) return '';
|
|
32262
|
-
try {
|
|
32263
|
-
const url = new URL(href);
|
|
32264
|
-
if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
|
|
32265
|
-
const redirected = url.searchParams.get('q') || url.searchParams.get('url');
|
|
32266
|
-
return redirected && redirected.startsWith('http') ? redirected : '';
|
|
32267
|
-
}
|
|
32268
|
-
} catch {}
|
|
32269
|
-
return href;
|
|
32270
|
-
};
|
|
32271
|
-
const sourceLabel = value => clean(value)
|
|
32272
|
-
.replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
|
|
32273
|
-
.replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
|
|
32274
|
-
const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
|
|
32275
|
-
const question = clean(item.getAttribute(SEL.itemDataQ)
|
|
32276
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
32277
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText);
|
|
32278
|
-
const expanded = item.classList.contains(SEL.expandedClass)
|
|
32279
|
-
|| item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
|
|
32280
|
-
const container = expanded ? item.querySelector(SEL.answerContainer) : null;
|
|
32281
|
-
const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
|
|
32282
|
-
const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
|
|
32283
|
-
SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
|
|
32284
|
-
) || []) : [];
|
|
32285
|
-
const byUrl = new Map();
|
|
32286
|
-
for (const anchor of anchors) {
|
|
32287
|
-
const url = externalHref(anchor);
|
|
32288
|
-
if (!url || byUrl.has(url)) continue;
|
|
32289
|
-
let site = '';
|
|
32290
|
-
try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
32291
|
-
byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
|
|
32292
|
-
}
|
|
32293
|
-
const sources = Array.from(byUrl.values());
|
|
32294
|
-
let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
|
|
32295
|
-
let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
|
|
32296
|
-
let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
|
|
32297
|
-
const primary = sources.find(source => source.title.length > 2) || sources[0];
|
|
32298
|
-
if (!sourceCite && primary) {
|
|
32299
|
-
sourceTitle ||= primary.title;
|
|
32300
|
-
sourceSite ||= primary.site;
|
|
32301
|
-
sourceCite = primary.url;
|
|
32302
|
-
}
|
|
32303
|
-
return {
|
|
32304
|
-
question,
|
|
32305
|
-
answer: clean(answerElement?.innerText || answerElement?.textContent),
|
|
32306
|
-
sourceTitle,
|
|
32307
|
-
sourceSite,
|
|
32308
|
-
sourceCite,
|
|
32309
|
-
sources,
|
|
32310
|
-
depth: 1,
|
|
32311
|
-
parentQuestion: null,
|
|
32312
|
-
};
|
|
32313
|
-
}).filter(record => record.question);
|
|
32314
|
-
`;
|
|
32315
|
-
}
|
|
32316
|
-
function expansionCode(clickedQuestions) {
|
|
32317
|
-
return `
|
|
32318
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
32319
|
-
${recordReaderSource()}
|
|
32320
|
-
const deliveryBlock = async () => {
|
|
32321
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
32322
|
-
if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
|
|
32323
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
32324
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
|
|
32325
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
|
|
32326
|
-
return null;
|
|
32327
|
-
};
|
|
32328
|
-
const blocked = await deliveryBlock();
|
|
32329
|
-
const records = await page.evaluate(({ SEL }) => {
|
|
32330
|
-
${recordReaderSource()}
|
|
32331
|
-
return readRecords();
|
|
32332
|
-
}, { SEL });
|
|
32333
|
-
if (blocked) return {
|
|
32334
|
-
outcome: blocked, targetQuestion: null, records, newQuestions: [],
|
|
32335
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32336
|
-
interactionTrusted: null, clickError: null,
|
|
32337
|
-
};
|
|
32338
|
-
const clicked = new Set(${JSON.stringify(clickedQuestions)});
|
|
32339
|
-
const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
|
|
32340
|
-
if (!targetQuestion) return {
|
|
32341
|
-
outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
|
|
32342
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32343
|
-
interactionTrusted: null, clickError: null,
|
|
32344
|
-
};
|
|
32345
|
-
const items = page.locator(SEL.item);
|
|
32346
|
-
let targetIndex = -1;
|
|
32347
|
-
for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
|
|
32348
|
-
const question = await items.nth(index).evaluate((item, SEL) =>
|
|
32349
|
-
item.getAttribute(SEL.itemDataQ)
|
|
32350
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
32351
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
|
|
32352
|
-
|| '', SEL).catch(() => '');
|
|
32353
|
-
if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
|
|
32354
|
-
}
|
|
32355
|
-
if (targetIndex < 0) return {
|
|
32356
|
-
outcome: 'exhausted', targetQuestion, records, newQuestions: [],
|
|
32357
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32358
|
-
interactionTrusted: null, clickError: 'question disappeared before click',
|
|
32359
|
-
};
|
|
32360
|
-
const beforeQuestions = new Set(records.map(record => record.question));
|
|
32361
|
-
const item = items.nth(targetIndex);
|
|
32362
|
-
await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
|
|
32363
|
-
await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
|
|
32364
|
-
await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
|
|
32365
|
-
let clickError = null;
|
|
32366
|
-
try {
|
|
32367
|
-
await item.locator(SEL.clickTarget).first().click({
|
|
32368
|
-
timeout: 6000,
|
|
32369
|
-
delay: 55 + Math.floor(Math.random() * 120),
|
|
32370
|
-
});
|
|
32371
|
-
} catch (error) {
|
|
32372
|
-
clickError = String(error && error.message || error).slice(0, 240);
|
|
32373
|
-
}
|
|
32374
|
-
const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
|
|
32375
|
-
await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
|
|
32376
|
-
const afterRecords = await page.evaluate(({ SEL }) => {
|
|
32377
|
-
${recordReaderSource()}
|
|
32378
|
-
return readRecords();
|
|
32379
|
-
}, { SEL });
|
|
32380
|
-
return {
|
|
32381
|
-
outcome: clickError ? 'exhausted' : 'clicked',
|
|
32382
|
-
targetQuestion,
|
|
32383
|
-
records: afterRecords,
|
|
32384
|
-
newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
|
|
32385
|
-
finalUrl: page.url(),
|
|
32386
|
-
title: await page.title().catch(() => ''),
|
|
32387
|
-
interactionTrusted,
|
|
32388
|
-
clickError,
|
|
32389
|
-
};
|
|
32390
|
-
`;
|
|
32391
|
-
}
|
|
32392
|
-
function finalCaptureCode(maxQuestions, parentByQuestion) {
|
|
32393
|
-
const surfaces = {
|
|
32394
|
-
aiOverview: AIOverviewSelectors,
|
|
32395
|
-
aiMode: AIModeSelectors,
|
|
32396
|
-
organic: OrganicSelectors,
|
|
32397
|
-
localPack: LocalPackSelectors,
|
|
32398
|
-
video: VideoSelectors,
|
|
32399
|
-
forum: ForumSelectors,
|
|
32400
|
-
whatPeopleSaying: WhatPeopleSayingSelectors
|
|
32401
|
-
};
|
|
32402
|
-
return `
|
|
32403
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
32404
|
-
const SURF = ${JSON.stringify(surfaces)};
|
|
32405
|
-
const PARENTS = ${JSON.stringify(parentByQuestion)};
|
|
32406
|
-
${recordReaderSource()}
|
|
32407
|
-
const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
|
|
32408
|
-
${recordReaderSource()}
|
|
32409
|
-
const unique = (items, key) => {
|
|
32410
|
-
const seen = new Set();
|
|
32411
|
-
return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
|
|
32412
|
-
};
|
|
32413
|
-
const paa = readRecords().map(record => {
|
|
32414
|
-
const parentQuestion = PARENTS[record.question] || null;
|
|
32415
|
-
let depth = 1, parent = parentQuestion;
|
|
32416
|
-
const seen = new Set();
|
|
32417
|
-
while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
|
|
32418
|
-
return { ...record, depth, parentQuestion };
|
|
32419
|
-
}).slice(0, maxQuestions);
|
|
32420
|
-
const organicResults = [];
|
|
32421
|
-
document.querySelectorAll(SURF.organic.result).forEach(card => {
|
|
32422
|
-
const titleElement = card.querySelector(SURF.organic.title);
|
|
32423
|
-
const title = clean(titleElement?.textContent);
|
|
32424
|
-
const link = titleElement?.closest('a');
|
|
32425
|
-
const url = externalHref(link);
|
|
32426
|
-
if (!title || !url) return;
|
|
32427
|
-
let domain = '';
|
|
32428
|
-
try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
32429
|
-
const rating = card.querySelector(SURF.organic.ratingWrap);
|
|
32430
|
-
organicResults.push({
|
|
32431
|
-
position: organicResults.length + 1, title, url, domain,
|
|
32432
|
-
cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
|
|
32433
|
-
snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
|
|
32434
|
-
isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
|
|
32435
|
-
inlineRating: rating ? {
|
|
32436
|
-
value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
|
|
32437
|
-
count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
|
|
32438
|
-
} : null,
|
|
32439
|
-
});
|
|
32440
|
-
});
|
|
32441
|
-
const localPack = [];
|
|
32442
|
-
document.querySelectorAll(SURF.localPack.card).forEach(card => {
|
|
32443
|
-
const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
|
|
32444
|
-
if (!name) return;
|
|
32445
|
-
const links = Array.from(card.querySelectorAll('a[href]'));
|
|
32446
|
-
let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
|
|
32447
|
-
for (const link of links) {
|
|
32448
|
-
if (cid) break;
|
|
32449
|
-
const match = link.href.match(/[?&]cid=(\\d+)/);
|
|
32450
|
-
if (match) cid = match[1];
|
|
32451
|
-
}
|
|
32452
|
-
localPack.push({
|
|
32453
|
-
position: localPack.length + 1, name, cid,
|
|
32454
|
-
rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
|
|
32455
|
-
reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
|
|
32456
|
-
metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
|
|
32457
|
-
directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
|
|
32458
|
-
});
|
|
32459
|
-
});
|
|
32460
|
-
const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
|
|
32461
|
-
const url = externalHref(link);
|
|
32462
|
-
const raw = clean(link.textContent);
|
|
32463
|
-
const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
|
|
32464
|
-
return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
|
|
32465
|
-
}).filter(item => item.title && item.url), item => item.url);
|
|
32466
|
-
const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
|
|
32467
|
-
title: clean(link.querySelector(SURF.forum.title)?.textContent),
|
|
32468
|
-
source: clean(link.querySelector(SURF.forum.source)?.textContent),
|
|
32469
|
-
url: externalHref(link),
|
|
32470
|
-
})).filter(item => item.title && item.url), item => item.url);
|
|
32471
|
-
const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
|
|
32472
|
-
const whatPeopleSaying = unique(socialCards.map(card => {
|
|
32473
|
-
const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
|
|
32474
|
-
const url = externalHref(link);
|
|
32475
|
-
const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
|
|
32476
|
-
const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
|
|
32477
|
-
const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
|
|
32478
|
-
const identity = (platform + source + url).toLowerCase();
|
|
32479
|
-
const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
|
|
32480
|
-
return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
|
|
32481
|
-
}).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
|
|
32482
|
-
const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
|
|
32483
|
-
const aimRoot = document.querySelector(SURF.aiMode.root);
|
|
32484
|
-
const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
|
|
32485
|
-
text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
|
|
32486
|
-
})).filter(item => item.href), item => item.href);
|
|
32487
|
-
const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
|
|
32488
|
-
const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
|
|
32489
|
-
const html = document.documentElement.outerHTML;
|
|
32490
|
-
const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
|
|
32491
|
-
const cids = unique([
|
|
32492
|
-
...localPack.map(item => item.cid).filter(Boolean),
|
|
32493
|
-
...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
|
|
32494
|
-
], value => value);
|
|
32495
|
-
const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
|
|
32496
|
-
const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
|
|
32497
|
-
const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
|
|
32498
|
-
const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
|
|
32499
|
-
text: clean(link.textContent), url: link.href,
|
|
32500
|
-
})).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
|
|
32501
|
-
const counts = {
|
|
32502
|
-
peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
|
|
32503
|
-
videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
|
|
32504
|
-
aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
|
|
32505
|
-
relatedSearches: relatedSearches.length,
|
|
32506
|
-
};
|
|
32507
|
-
return {
|
|
32508
|
-
paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
|
|
32509
|
-
videos, forums, whatPeopleSaying, aiOverview, aiMode,
|
|
32510
|
-
surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
|
|
32511
|
-
entityIds: { entities, kgIds, cids, gcids },
|
|
32512
|
-
serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
|
|
32513
|
-
relatedSearches,
|
|
32514
|
-
};
|
|
32515
|
-
}, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
|
|
32516
|
-
return capture;
|
|
32517
|
-
`;
|
|
32518
|
-
}
|
|
32519
|
-
function rawChunkCode(offset, length) {
|
|
32520
|
-
return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
|
|
32521
|
-
}
|
|
32522
|
-
function buildPublicResult(input) {
|
|
32523
|
-
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
32524
|
-
const selected = input.records;
|
|
32525
|
-
const flat = selected.map((record) => ({
|
|
32526
|
-
seed_query: input.query,
|
|
32527
|
-
question: record.question,
|
|
32528
|
-
answer: record.answer,
|
|
32529
|
-
source_title: record.sourceTitle,
|
|
32530
|
-
source_site: record.sourceSite,
|
|
32531
|
-
source_cite: record.sourceCite,
|
|
32532
|
-
depth: record.depth,
|
|
32533
|
-
parent_question: record.parentQuestion ?? "",
|
|
32534
|
-
extracted_at: extractedAt
|
|
32535
|
-
}));
|
|
32536
|
-
const tree = selected.map((record) => ({
|
|
32537
|
-
question: record.question,
|
|
32538
|
-
answer: record.answer || null,
|
|
32539
|
-
sourceTitle: record.sourceTitle || null,
|
|
32540
|
-
sourceSite: record.sourceSite || null,
|
|
32541
|
-
sourceCite: record.sourceCite || null,
|
|
32542
|
-
depth: record.depth,
|
|
32543
|
-
parentQuestion: record.parentQuestion,
|
|
32544
|
-
children: []
|
|
32545
|
-
}));
|
|
32546
|
-
return {
|
|
32547
|
-
seed: input.query,
|
|
32548
|
-
location: input.location,
|
|
32549
|
-
extractedAt,
|
|
32550
|
-
diagnostics: {
|
|
32551
|
-
completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
|
|
32552
|
-
problem: null,
|
|
32553
|
-
resultQuality: input.partial ? "partial" : "complete",
|
|
32554
|
-
degradedResult: input.partial,
|
|
32555
|
-
retryRecommended: false,
|
|
32556
|
-
completeness: {
|
|
32557
|
-
paaWithoutAnswer: selected.filter((record) => !record.answer).length,
|
|
32558
|
-
paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
|
|
32559
|
-
paaAnswersRecovered: 0,
|
|
32560
|
-
aioShareCaptured: null
|
|
32561
|
-
}
|
|
32562
|
-
},
|
|
32563
|
-
totalQuestions: selected.length,
|
|
32564
|
-
surface: input.capture.surface,
|
|
32565
|
-
aiOverview: input.capture.aiOverview,
|
|
32566
|
-
aiMode: input.capture.aiMode,
|
|
32567
|
-
whatPeopleSaying: input.capture.whatPeopleSaying,
|
|
32568
|
-
tree,
|
|
32569
|
-
flat,
|
|
32570
|
-
videos: input.capture.videos,
|
|
32571
|
-
forums: input.capture.forums,
|
|
32572
|
-
organicResults: input.capture.organicResults,
|
|
32573
|
-
localPack: input.capture.localPack,
|
|
32574
|
-
entityIds: input.capture.entityIds,
|
|
32575
|
-
serpFeatures: input.capture.serpFeatures,
|
|
32576
|
-
relatedSearches: input.capture.relatedSearches,
|
|
32577
|
-
stats: {
|
|
32578
|
-
seed: input.query,
|
|
32579
|
-
totalQuestions: selected.length,
|
|
32580
|
-
maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
|
|
32581
|
-
durationMs: input.durationMs,
|
|
32582
|
-
errorCount: input.partial ? 1 : 0
|
|
32583
|
-
},
|
|
32584
|
-
capture: {
|
|
32585
|
-
execution: "kernel_colocated_playwright",
|
|
32586
|
-
stealth: true,
|
|
32587
|
-
managedProxy: input.persistentIdentity ? "serp_identity" : "kernel_default",
|
|
32588
|
-
metadataOverrides: false,
|
|
32589
|
-
locationSignal: input.location ? "uule_only" : "none",
|
|
32590
|
-
attempt: input.attempt,
|
|
32591
|
-
finalUrl: input.navigation.finalUrl,
|
|
32592
|
-
captchaObserved: input.navigation.captchaObserved,
|
|
32593
|
-
solverCleared: input.navigation.solverCleared,
|
|
32594
|
-
environment: input.navigation.environment,
|
|
32595
|
-
rawDomSha256: input.rawDomSha256
|
|
32596
|
-
}
|
|
32597
|
-
};
|
|
32598
|
-
}
|
|
32599
|
-
async function runDurablePaaCapture(input) {
|
|
32600
|
-
const startedAt = input.startedAtMs ?? Date.now();
|
|
32601
|
-
const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
|
|
32602
|
-
const location2 = input.location?.trim() || null;
|
|
32603
|
-
const params = new URLSearchParams({
|
|
32604
|
-
q: input.query,
|
|
32605
|
-
gl: input.gl ?? "us",
|
|
32606
|
-
hl: input.hl ?? "en",
|
|
32607
|
-
pws: "0"
|
|
32608
|
-
});
|
|
32609
|
-
if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
|
|
32610
|
-
const searchUrl = `https://www.google.com/search?${params.toString()}`;
|
|
32611
|
-
const kernel = new import_sdk9.default({ apiKey: input.apiKey });
|
|
32612
|
-
let lastCheckpoint = {
|
|
32613
|
-
phase: "navigating",
|
|
32614
|
-
attempt: 1,
|
|
32615
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32616
|
-
records: [],
|
|
32617
|
-
clickedQuestions: [],
|
|
32618
|
-
finalUrl: null,
|
|
32619
|
-
captchaObserved: false,
|
|
32620
|
-
solverCleared: false,
|
|
32621
|
-
error: null
|
|
32622
|
-
};
|
|
32623
|
-
const persist = async (patch) => {
|
|
32624
|
-
lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
32625
|
-
await input.onCheckpoint?.(lastCheckpoint);
|
|
32626
|
-
};
|
|
32627
|
-
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
32628
|
-
if (Date.now() >= workDeadline) break;
|
|
32629
|
-
let sessionId = null;
|
|
32630
|
-
const records = /* @__PURE__ */ new Map();
|
|
32631
|
-
const clicked = /* @__PURE__ */ new Set();
|
|
32632
|
-
const parents = {};
|
|
32633
|
-
let navigation = null;
|
|
32634
|
-
try {
|
|
32635
|
-
await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
|
|
32636
|
-
const browser = await kernel.browsers.create({
|
|
32637
|
-
stealth: true,
|
|
32638
|
-
timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20)),
|
|
32639
|
-
...input.proxyId ? { proxy_id: input.proxyId } : {},
|
|
32640
|
-
...input.profileName ? { profile: { name: input.profileName, save_changes: false } } : {}
|
|
32641
|
-
});
|
|
32642
|
-
sessionId = browser.session_id;
|
|
32643
|
-
navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
|
|
32644
|
-
await persist({
|
|
32645
|
-
phase: navigation.outcome === "ready" ? "expanding" : "blocked",
|
|
32646
|
-
finalUrl: navigation.finalUrl,
|
|
32647
|
-
captchaObserved: navigation.captchaObserved,
|
|
32648
|
-
solverCleared: navigation.solverCleared
|
|
32649
|
-
});
|
|
32650
|
-
if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
|
|
32651
|
-
if (attempt === 1 && records.size === 0) continue;
|
|
32652
|
-
throw new Error(navigation.outcome);
|
|
32653
|
-
}
|
|
32654
|
-
let stagnant = 0;
|
|
32655
|
-
for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
|
|
32656
|
-
const observed = await execute(
|
|
32657
|
-
kernel,
|
|
32658
|
-
sessionId,
|
|
32659
|
-
expansionCode(Array.from(clicked)),
|
|
32660
|
-
workDeadline,
|
|
32661
|
-
25
|
|
32662
|
-
);
|
|
32663
|
-
const beforeSize = records.size;
|
|
32664
|
-
for (const record of observed.records) {
|
|
32665
|
-
const existing = records.get(record.question);
|
|
32666
|
-
records.set(record.question, {
|
|
32667
|
-
...record,
|
|
32668
|
-
answer: record.answer || existing?.answer || "",
|
|
32669
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
32670
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
32671
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
32672
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
32673
|
-
parentQuestion: parents[record.question] ?? null,
|
|
32674
|
-
depth: parents[record.question] ? 2 : 1
|
|
32675
|
-
});
|
|
32676
|
-
}
|
|
32677
|
-
if (observed.outcome === "clicked" && observed.targetQuestion) {
|
|
32678
|
-
clicked.add(observed.targetQuestion);
|
|
32679
|
-
for (const question of observed.newQuestions) {
|
|
32680
|
-
if (!parents[question]) parents[question] = observed.targetQuestion;
|
|
32681
|
-
}
|
|
32682
|
-
}
|
|
32683
|
-
const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
32684
|
-
await persist({
|
|
32685
|
-
phase: "expanding",
|
|
32686
|
-
records: selected2,
|
|
32687
|
-
clickedQuestions: Array.from(clicked),
|
|
32688
|
-
finalUrl: observed.finalUrl,
|
|
32689
|
-
captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
|
|
32690
|
-
});
|
|
32691
|
-
if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
|
|
32692
|
-
if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
|
|
32693
|
-
stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
|
|
32694
|
-
if (stagnant >= 4) break;
|
|
32695
|
-
}
|
|
32696
|
-
const finalCapture = await execute(
|
|
32697
|
-
kernel,
|
|
32698
|
-
sessionId,
|
|
32699
|
-
finalCaptureCode(input.maxQuestions, parents),
|
|
32700
|
-
workDeadline,
|
|
32701
|
-
45
|
|
32702
|
-
);
|
|
32703
|
-
for (const record of finalCapture.paa) {
|
|
32704
|
-
const existing = records.get(record.question);
|
|
32705
|
-
records.set(record.question, {
|
|
32706
|
-
...record,
|
|
32707
|
-
answer: record.answer || existing?.answer || "",
|
|
32708
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
32709
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
32710
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
32711
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
32712
|
-
parentQuestion: parents[record.question] ?? record.parentQuestion,
|
|
32713
|
-
depth: record.depth
|
|
32714
|
-
});
|
|
32715
|
-
}
|
|
32716
|
-
let rawDomGzip = null;
|
|
32717
|
-
let rawDomSha256 = null;
|
|
32718
|
-
if (Date.now() < workDeadline - 12e3) {
|
|
32719
|
-
const prepared = await execute(
|
|
32720
|
-
kernel,
|
|
32721
|
-
sessionId,
|
|
32722
|
-
RAW_PREPARE_CODE,
|
|
32723
|
-
workDeadline,
|
|
32724
|
-
35
|
|
32725
|
-
);
|
|
32726
|
-
const chunks = [];
|
|
32727
|
-
for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
|
|
32728
|
-
const part = await execute(
|
|
32729
|
-
kernel,
|
|
32730
|
-
sessionId,
|
|
32731
|
-
rawChunkCode(offset, 18e4),
|
|
32732
|
-
workDeadline,
|
|
32733
|
-
15
|
|
32734
|
-
);
|
|
32735
|
-
if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
|
|
32736
|
-
chunks.push(part.chunk);
|
|
32737
|
-
}
|
|
32738
|
-
if (chunks.join("").length === prepared.chars) {
|
|
32739
|
-
const transported = Buffer.from(chunks.join(""), "base64");
|
|
32740
|
-
if (prepared.encoding === "gzip") rawDomGzip = transported;
|
|
32741
|
-
else {
|
|
32742
|
-
const { gzipSync: gzipSync2 } = await import("zlib");
|
|
32743
|
-
rawDomGzip = gzipSync2(transported);
|
|
32744
|
-
}
|
|
32745
|
-
rawDomSha256 = (0, import_node_crypto16.createHash)("sha256").update(rawDomGzip).digest("hex");
|
|
32746
|
-
}
|
|
32747
|
-
}
|
|
32748
|
-
const selected = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
32749
|
-
const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
|
|
32750
|
-
await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
|
|
32751
|
-
return {
|
|
32752
|
-
result: buildPublicResult({
|
|
32753
|
-
query: input.query,
|
|
32754
|
-
location: location2,
|
|
32755
|
-
capture: finalCapture,
|
|
32756
|
-
records: selected,
|
|
32757
|
-
navigation,
|
|
32758
|
-
durationMs: Date.now() - startedAt,
|
|
32759
|
-
partial,
|
|
32760
|
-
attempt,
|
|
32761
|
-
rawDomSha256,
|
|
32762
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
32763
|
-
}),
|
|
32764
|
-
rawDomGzip,
|
|
32765
|
-
rawDomSha256,
|
|
32766
|
-
attempt,
|
|
32767
|
-
partial,
|
|
32768
|
-
checkpoint: lastCheckpoint
|
|
32769
|
-
};
|
|
32770
|
-
} catch (error) {
|
|
32771
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
32772
|
-
await persist({
|
|
32773
|
-
phase: records.size > 0 ? "captured" : "failed",
|
|
32774
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
32775
|
-
clickedQuestions: Array.from(clicked),
|
|
32776
|
-
error: message.slice(0, 500)
|
|
32777
|
-
});
|
|
32778
|
-
if (records.size > 0) {
|
|
32779
|
-
const emptyCapture = {
|
|
32780
|
-
paa: [],
|
|
32781
|
-
organicResults: [],
|
|
32782
|
-
localPack: [],
|
|
32783
|
-
videos: [],
|
|
32784
|
-
forums: [],
|
|
32785
|
-
whatPeopleSaying: [],
|
|
32786
|
-
aiOverview: { detected: false, text: null, citations: [] },
|
|
32787
|
-
aiMode: { detected: false, text: null, citations: [] },
|
|
32788
|
-
surface: "unknown",
|
|
32789
|
-
entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
|
|
32790
|
-
serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
|
|
32791
|
-
relatedSearches: []
|
|
32792
|
-
};
|
|
32793
|
-
return {
|
|
32794
|
-
result: buildPublicResult({
|
|
32795
|
-
query: input.query,
|
|
32796
|
-
location: location2,
|
|
32797
|
-
capture: emptyCapture,
|
|
32798
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
32799
|
-
navigation: navigation ?? {
|
|
32800
|
-
outcome: "soft_block",
|
|
32801
|
-
finalUrl: lastCheckpoint.finalUrl ?? "",
|
|
32802
|
-
title: "",
|
|
32803
|
-
captchaObserved: lastCheckpoint.captchaObserved,
|
|
32804
|
-
solverCleared: lastCheckpoint.solverCleared,
|
|
32805
|
-
environment: null
|
|
32806
|
-
},
|
|
32807
|
-
durationMs: Date.now() - startedAt,
|
|
32808
|
-
partial: true,
|
|
32809
|
-
attempt,
|
|
32810
|
-
rawDomSha256: null,
|
|
32811
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
32812
|
-
}),
|
|
32813
|
-
rawDomGzip: null,
|
|
32814
|
-
rawDomSha256: null,
|
|
32815
|
-
attempt,
|
|
32816
|
-
partial: true,
|
|
32817
|
-
checkpoint: lastCheckpoint
|
|
32818
|
-
};
|
|
32819
|
-
}
|
|
32820
|
-
if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
|
|
32821
|
-
} finally {
|
|
32822
|
-
if (sessionId) {
|
|
32823
|
-
await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
|
|
32824
|
-
}
|
|
32825
|
-
}
|
|
32826
|
-
}
|
|
32827
|
-
throw new Error("paa_work_deadline_exhausted_before_capture");
|
|
32828
|
-
}
|
|
32829
|
-
var import_sdk9, import_node_crypto16, PAA_INVOCATION_BUDGET_MS, PAA_BROWSER_WORK_BUDGET_MS, PAA_SOLVER_BUDGET_MS, RAW_PREPARE_CODE;
|
|
32830
|
-
var init_durable_capture = __esm({
|
|
32831
|
-
"src/paa/durable-capture.ts"() {
|
|
32832
|
-
"use strict";
|
|
32833
|
-
import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
|
|
32834
|
-
import_node_crypto16 = require("crypto");
|
|
32835
|
-
init_selectors();
|
|
32836
|
-
init_uule();
|
|
32837
|
-
PAA_INVOCATION_BUDGET_MS = 28e4;
|
|
32838
|
-
PAA_BROWSER_WORK_BUDGET_MS = 25e4;
|
|
32839
|
-
PAA_SOLVER_BUDGET_MS = 6e4;
|
|
32840
|
-
RAW_PREPARE_CODE = `
|
|
32841
|
-
const result = await page.evaluate(async () => {
|
|
32842
|
-
const source = new TextEncoder().encode(document.documentElement.outerHTML);
|
|
32843
|
-
const compressed = typeof CompressionStream === 'undefined'
|
|
32844
|
-
? source
|
|
32845
|
-
: new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
|
|
32846
|
-
const pieces = [];
|
|
32847
|
-
for (let offset = 0; offset < compressed.length; offset += 32768) {
|
|
32848
|
-
pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
|
|
32849
|
-
}
|
|
32850
|
-
window.__mcpPaaRawDom = btoa(pieces.join(''));
|
|
32851
|
-
return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
|
|
32852
|
-
});
|
|
32853
|
-
return result;
|
|
32854
|
-
`;
|
|
32855
|
-
}
|
|
32856
|
-
});
|
|
32857
|
-
|
|
32858
|
-
// src/inngest/functions/paa-harvest.ts
|
|
32859
|
-
function durableRecordsFromResult(result) {
|
|
32860
|
-
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
32861
|
-
return flat.flatMap((value) => {
|
|
32862
|
-
if (!value || typeof value !== "object") return [];
|
|
32863
|
-
const row = value;
|
|
32864
|
-
const question = typeof row.question === "string" ? row.question : "";
|
|
32865
|
-
if (!question) return [];
|
|
32866
|
-
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
32867
|
-
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
32868
|
-
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
32869
|
-
return [{
|
|
32870
|
-
question,
|
|
32871
|
-
answer: typeof row.answer === "string" ? row.answer : "",
|
|
32872
|
-
sourceTitle,
|
|
32873
|
-
sourceSite,
|
|
32874
|
-
sourceCite,
|
|
32875
|
-
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
32876
|
-
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
32877
|
-
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
32878
|
-
}];
|
|
32879
|
-
});
|
|
32880
|
-
}
|
|
32881
|
-
function questionCount(result) {
|
|
32882
|
-
if (!result || typeof result !== "object") return 0;
|
|
32883
|
-
const value = result;
|
|
32884
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
32885
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
32886
|
-
}
|
|
32887
|
-
var BRIGHTDATA_INVOCATION_BUDGET_MS, BRIGHTDATA_WORK_BUDGET_MS, paaHarvestFn;
|
|
32888
|
-
var init_paa_harvest = __esm({
|
|
32889
|
-
"src/inngest/functions/paa-harvest.ts"() {
|
|
32890
|
-
"use strict";
|
|
32891
|
-
init_client();
|
|
32892
|
-
init_browser_service_env();
|
|
32893
|
-
init_BrightDataSerpDriver();
|
|
32894
|
-
init_harvest();
|
|
32895
|
-
init_db();
|
|
32896
|
-
init_paa_capture_artifacts();
|
|
32897
|
-
init_paa_harvest_settlement();
|
|
32898
|
-
init_serp_identity_db();
|
|
32899
|
-
init_webhook();
|
|
32900
|
-
init_harvest_problems();
|
|
32901
|
-
init_durable_capture();
|
|
32902
|
-
BRIGHTDATA_INVOCATION_BUDGET_MS = 76e4;
|
|
32903
|
-
BRIGHTDATA_WORK_BUDGET_MS = 72e4;
|
|
32904
|
-
paaHarvestFn = inngest.createFunction(
|
|
32905
|
-
{
|
|
32906
|
-
id: "paa-harvest",
|
|
32907
|
-
retries: 0,
|
|
32908
|
-
triggers: [{ event: "mcp-scraper/paa.requested" }],
|
|
32909
|
-
onFailure: async ({ event: event2 }) => {
|
|
32910
|
-
const jobId2 = event2?.data?.event?.data?.jobId;
|
|
32911
|
-
if (!jobId2) return;
|
|
32912
|
-
const current = await getJob(jobId2);
|
|
32913
|
-
if (!current) return;
|
|
32914
|
-
if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
|
|
32915
|
-
await settlePaaHarvestJob(jobId2).catch((error) => {
|
|
32916
|
-
console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
|
|
32917
|
-
});
|
|
32918
|
-
return;
|
|
32919
|
-
}
|
|
32920
|
-
const rawError = String(event2?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
|
|
32921
|
-
const problem = classifyHarvestProblem(new Error(rawError));
|
|
32922
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
32923
|
-
await settlePaaHarvestJob(jobId2).then(async (settled) => {
|
|
32924
|
-
if (!settled) return;
|
|
32925
|
-
const settledJob = await getJob(jobId2);
|
|
32926
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
32927
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
32928
|
-
}).catch((error) => {
|
|
32929
|
-
console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
|
|
32930
|
-
});
|
|
32931
|
-
}
|
|
32932
|
-
},
|
|
32933
|
-
async ({ event: event2 }) => {
|
|
32934
|
-
const invocationStartedAt = Date.now();
|
|
32935
|
-
const job = await claimInngestPaaJob(event2.data.jobId);
|
|
32936
|
-
if (!job) return { jobId: event2.data.jobId, status: "missing_or_not_paa" };
|
|
32937
|
-
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
32938
|
-
await settlePaaHarvestJob(job.id);
|
|
32939
|
-
return { jobId: job.id, status: job.status };
|
|
32940
|
-
}
|
|
32941
|
-
const options = job.options;
|
|
32942
|
-
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
32943
|
-
const serpIdentity = options.serpIdentity ? await getSerpIdentityRow(job.user_id, options.serpIdentity) : null;
|
|
32944
|
-
if (options.serpIdentity && !serpIdentity) throw new Error("SERP identity not found");
|
|
32945
|
-
if (serpIdentity && serpIdentity.status !== "ready") throw new Error("SERP identity is not ready");
|
|
32946
|
-
const useBrightData = brightDataSerpEnabled() && !serpIdentity;
|
|
32947
|
-
const apiKey = browserServiceApiKey();
|
|
32948
|
-
if (!useBrightData && !apiKey) throw new Error("browser service is not configured");
|
|
32949
|
-
const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
32950
|
-
await startHarvestAttempt({
|
|
32951
|
-
jobId: job.id,
|
|
32952
|
-
userId: job.user_id,
|
|
32953
|
-
attemptNumber: 1,
|
|
32954
|
-
maxAttempts: 2,
|
|
32955
|
-
query: options.query ?? job.query,
|
|
32956
|
-
location: options.location ?? null,
|
|
32957
|
-
maxQuestions,
|
|
32958
|
-
startedAt: attemptStartedAt
|
|
32959
|
-
});
|
|
32960
|
-
const checkpointState = { value: null };
|
|
32961
|
-
try {
|
|
32962
|
-
const persistCheckpoint = async (checkpoint, brightData = false) => {
|
|
32963
|
-
checkpointState.value = checkpoint;
|
|
32964
|
-
const persisted = await checkpointJob(job.id, {
|
|
32965
|
-
progress: checkpoint,
|
|
32966
|
-
durable: {
|
|
32967
|
-
invocationBudgetMs: brightData ? BRIGHTDATA_INVOCATION_BUDGET_MS : PAA_INVOCATION_BUDGET_MS,
|
|
32968
|
-
workBudgetMs: brightData ? BRIGHTDATA_WORK_BUDGET_MS : 25e4,
|
|
32969
|
-
cleanupReserveMs: brightData ? 4e4 : 3e4
|
|
32970
|
-
}
|
|
32971
|
-
});
|
|
32972
|
-
if (!persisted) throw new Error("job_left_running_state_during_capture");
|
|
32973
|
-
};
|
|
32974
|
-
const query = options.query ?? job.query;
|
|
32975
|
-
const capture = useBrightData ? await (async () => {
|
|
32976
|
-
const navigating = {
|
|
32977
|
-
phase: "navigating",
|
|
32978
|
-
attempt: 1,
|
|
32979
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32980
|
-
records: [],
|
|
32981
|
-
clickedQuestions: [],
|
|
32982
|
-
finalUrl: null,
|
|
32983
|
-
captchaObserved: false,
|
|
32984
|
-
solverCleared: false,
|
|
32985
|
-
error: null
|
|
32986
|
-
};
|
|
32987
|
-
await persistCheckpoint(navigating, true);
|
|
32988
|
-
const result2 = await harvest({
|
|
32989
|
-
query,
|
|
32990
|
-
location: options.location ?? void 0,
|
|
32991
|
-
gl: options.gl ?? "us",
|
|
32992
|
-
hl: options.hl ?? "en",
|
|
32993
|
-
maxQuestions,
|
|
32994
|
-
serpOnly: options.serpOnly ?? false,
|
|
32995
|
-
maxAttempts: 1,
|
|
32996
|
-
headless: true,
|
|
32997
|
-
format: "json",
|
|
32998
|
-
outputDir: "/tmp/paa-output-inngest",
|
|
32999
|
-
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
33000
|
-
includeLocalPack: options.includeLocalPack ?? false,
|
|
33001
|
-
includeForums: options.includeForums ?? false,
|
|
33002
|
-
includeVideos: options.includeVideos ?? false,
|
|
33003
|
-
includeAiOverview: options.includeAiOverview ?? false,
|
|
33004
|
-
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
33005
|
-
softDeadlineMs: invocationStartedAt + BRIGHTDATA_WORK_BUDGET_MS
|
|
33006
|
-
});
|
|
33007
|
-
const records = durableRecordsFromResult(result2);
|
|
33008
|
-
const captured = {
|
|
33009
|
-
...navigating,
|
|
33010
|
-
phase: "captured",
|
|
33011
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33012
|
-
records,
|
|
33013
|
-
clickedQuestions: records.map((record) => record.question)
|
|
33014
|
-
};
|
|
33015
|
-
await persistCheckpoint(captured, true);
|
|
33016
|
-
return {
|
|
33017
|
-
result: {
|
|
33018
|
-
...result2,
|
|
33019
|
-
capture: {
|
|
33020
|
-
execution: "brightdata_browser_api_controller_fulfilled",
|
|
33021
|
-
provider: "bright_data_browser_api",
|
|
33022
|
-
sameQueryClickGraph: true
|
|
33023
|
-
}
|
|
33024
|
-
},
|
|
33025
|
-
rawDomGzip: null,
|
|
33026
|
-
rawDomSha256: null,
|
|
33027
|
-
attempt: 1,
|
|
33028
|
-
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
33029
|
-
checkpoint: captured
|
|
33030
|
-
};
|
|
33031
|
-
})() : await runDurablePaaCapture({
|
|
33032
|
-
apiKey,
|
|
33033
|
-
query,
|
|
33034
|
-
location: options.location ?? null,
|
|
33035
|
-
gl: options.gl ?? "us",
|
|
33036
|
-
hl: options.hl ?? "en",
|
|
33037
|
-
maxQuestions,
|
|
33038
|
-
...serpIdentity ? {
|
|
33039
|
-
proxyId: serpIdentity.kernel_proxy_id,
|
|
33040
|
-
profileName: serpIdentity.kernel_profile_name
|
|
33041
|
-
} : {},
|
|
33042
|
-
startedAtMs: invocationStartedAt,
|
|
33043
|
-
onCheckpoint: (checkpoint) => persistCheckpoint(checkpoint, false)
|
|
33044
|
-
});
|
|
33045
|
-
const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
|
|
33046
|
-
userId: job.user_id,
|
|
33047
|
-
jobId: job.id,
|
|
33048
|
-
attempt: capture.attempt,
|
|
33049
|
-
gzip: capture.rawDomGzip
|
|
33050
|
-
}) : { artifact: null, status: "unavailable" };
|
|
33051
|
-
const artifact = artifactResult.artifact;
|
|
33052
|
-
const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
|
|
33053
|
-
const captureResult = capture.result;
|
|
33054
|
-
const result = {
|
|
33055
|
-
...captureResult,
|
|
33056
|
-
diagnostics: {
|
|
33057
|
-
...captureResult.diagnostics,
|
|
33058
|
-
...artifactMissing ? {
|
|
33059
|
-
resultQuality: "partial",
|
|
33060
|
-
degradedResult: true,
|
|
33061
|
-
retryRecommended: false
|
|
33062
|
-
} : {}
|
|
33063
|
-
},
|
|
33064
|
-
capture: {
|
|
33065
|
-
...captureResult.capture,
|
|
33066
|
-
rawDomArtifact: artifact,
|
|
33067
|
-
rawDomArtifactStatus: artifactResult.status
|
|
33068
|
-
}
|
|
33069
|
-
};
|
|
33070
|
-
await completeJob(job.id, result);
|
|
33071
|
-
if (options.serpIdentity) await touchSerpIdentity(job.user_id, options.serpIdentity).catch(() => void 0);
|
|
33072
|
-
await finishHarvestAttempt({
|
|
33073
|
-
jobId: job.id,
|
|
33074
|
-
attemptNumber: 1,
|
|
33075
|
-
outcome: capture.partial ? "partial" : "paa_found",
|
|
33076
|
-
kernelSessionId: null,
|
|
33077
|
-
questionCount: questionCount(result),
|
|
33078
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
33079
|
-
error: capture.checkpoint.error,
|
|
33080
|
-
willRetry: false,
|
|
33081
|
-
kernelDeleteStarted: true,
|
|
33082
|
-
kernelDeleteSucceeded: true,
|
|
33083
|
-
kernelDeleteError: null,
|
|
33084
|
-
browserCloseSucceeded: true,
|
|
33085
|
-
browserCloseError: null,
|
|
33086
|
-
debug: {
|
|
33087
|
-
execution: useBrightData ? "brightdata_browser_api_controller_fulfilled" : "kernel_colocated_playwright",
|
|
33088
|
-
stealth: true,
|
|
33089
|
-
attemptUsed: capture.attempt,
|
|
33090
|
-
rawDomStored: Boolean(artifact)
|
|
33091
|
-
},
|
|
33092
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
33093
|
-
});
|
|
33094
|
-
await settlePaaHarvestJob(job.id);
|
|
33095
|
-
if (job.callback_url) {
|
|
33096
|
-
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
33097
|
-
}
|
|
33098
|
-
return {
|
|
33099
|
-
jobId: job.id,
|
|
33100
|
-
status: "done",
|
|
33101
|
-
totalQuestions: questionCount(result),
|
|
33102
|
-
partial: capture.partial,
|
|
33103
|
-
artifactStored: Boolean(artifact)
|
|
33104
|
-
};
|
|
33105
|
-
} catch (error) {
|
|
33106
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
33107
|
-
const problem = classifyHarvestProblem(error);
|
|
33108
|
-
const latestCheckpoint = checkpointState.value;
|
|
33109
|
-
await finishHarvestAttempt({
|
|
33110
|
-
jobId: job.id,
|
|
33111
|
-
attemptNumber: 1,
|
|
33112
|
-
outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
|
|
33113
|
-
kernelSessionId: null,
|
|
33114
|
-
questionCount: latestCheckpoint?.records.length ?? 0,
|
|
33115
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
33116
|
-
error: message.slice(0, 1e3),
|
|
33117
|
-
willRetry: false,
|
|
33118
|
-
kernelDeleteStarted: true,
|
|
33119
|
-
kernelDeleteSucceeded: null,
|
|
33120
|
-
kernelDeleteError: null,
|
|
33121
|
-
browserCloseSucceeded: null,
|
|
33122
|
-
browserCloseError: null,
|
|
33123
|
-
debug: latestCheckpoint,
|
|
33124
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
33125
|
-
});
|
|
33126
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
33127
|
-
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
33128
|
-
if (!settled) return;
|
|
33129
|
-
const settledJob = await getJob(job.id);
|
|
33130
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
33131
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
33132
|
-
}).catch((settlementError) => {
|
|
33133
|
-
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
33134
|
-
});
|
|
33135
|
-
if (job.callback_url) {
|
|
33136
|
-
await deliverWebhook(job.callback_url, {
|
|
33137
|
-
job_id: job.id,
|
|
33138
|
-
status: "failed",
|
|
33139
|
-
...harvestProblemResponse(problem),
|
|
33140
|
-
progress: latestCheckpoint
|
|
33141
|
-
});
|
|
33142
|
-
}
|
|
33143
|
-
throw error;
|
|
33144
|
-
}
|
|
33145
|
-
}
|
|
33146
|
-
);
|
|
33147
|
-
}
|
|
33148
|
-
});
|
|
33149
|
-
|
|
33150
31853
|
// src/api/local-sourcebook-repository.ts
|
|
33151
31854
|
function ensureLocalSourcebookSchema() {
|
|
33152
31855
|
const currentDb = getDb();
|
|
@@ -33259,7 +31962,7 @@ function mapSubmission(row) {
|
|
|
33259
31962
|
async function event(submissionId, eventType, actorKind, actorId, metadata = {}) {
|
|
33260
31963
|
await getDb().execute({
|
|
33261
31964
|
sql: `INSERT INTO local_sourcebook_events (id, submission_id, event_type, actor_kind, actor_id, metadata_json) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
33262
|
-
args: [(0,
|
|
31965
|
+
args: [(0, import_node_crypto15.randomUUID)(), submissionId, eventType, actorKind, actorId, JSON.stringify(metadata)]
|
|
33263
31966
|
});
|
|
33264
31967
|
}
|
|
33265
31968
|
async function createLocalSourcebookSubmission(input) {
|
|
@@ -33273,7 +31976,7 @@ async function createLocalSourcebookSubmission(input) {
|
|
|
33273
31976
|
return submission;
|
|
33274
31977
|
}
|
|
33275
31978
|
}
|
|
33276
|
-
const id = `lsb_${(0,
|
|
31979
|
+
const id = `lsb_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "")}`;
|
|
33277
31980
|
const coverage = {
|
|
33278
31981
|
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
33279
31982
|
crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
|
|
@@ -33414,11 +32117,11 @@ async function getPublicLocalSourcebook(category, state, slug4) {
|
|
|
33414
32117
|
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
|
|
33415
32118
|
return result.rows[0] ? parseJson3(result.rows[0].payload_json) : null;
|
|
33416
32119
|
}
|
|
33417
|
-
var
|
|
32120
|
+
var import_node_crypto15, LOCAL_SOURCEBOOK_CATEGORIES, schemaPromise4, schemaDb4;
|
|
33418
32121
|
var init_local_sourcebook_repository = __esm({
|
|
33419
32122
|
"src/api/local-sourcebook-repository.ts"() {
|
|
33420
32123
|
"use strict";
|
|
33421
|
-
|
|
32124
|
+
import_node_crypto15 = require("crypto");
|
|
33422
32125
|
init_db();
|
|
33423
32126
|
LOCAL_SOURCEBOOK_CATEGORIES = ["home", "professional", "restaurants", "financial", "realestate", "auto", "wellness"];
|
|
33424
32127
|
schemaPromise4 = null;
|
|
@@ -34570,7 +33273,7 @@ var init_local_sourcebook_schema = __esm({
|
|
|
34570
33273
|
|
|
34571
33274
|
// src/api/local-sourcebook-compiler.ts
|
|
34572
33275
|
function digest(value) {
|
|
34573
|
-
return (0,
|
|
33276
|
+
return (0, import_node_crypto16.createHash)("sha256").update(value).digest("hex").slice(0, 20);
|
|
34574
33277
|
}
|
|
34575
33278
|
function unique(values, limit = 100) {
|
|
34576
33279
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -35007,11 +33710,11 @@ function compileLocalSourcebookListing(input) {
|
|
|
35007
33710
|
}
|
|
35008
33711
|
};
|
|
35009
33712
|
}
|
|
35010
|
-
var
|
|
33713
|
+
var import_node_crypto16, CATEGORY_LABELS, STATE_NAMES, GENERIC_HEADINGS, SERVICE_PATH, PRODUCT_PATH, TEAM_PATH, WORK_PATH, CREDENTIAL_PATH, BLOG_PATH, LOCATION_PATH, REVIEW_THEME_RULES;
|
|
35011
33714
|
var init_local_sourcebook_compiler = __esm({
|
|
35012
33715
|
"src/api/local-sourcebook-compiler.ts"() {
|
|
35013
33716
|
"use strict";
|
|
35014
|
-
|
|
33717
|
+
import_node_crypto16 = require("crypto");
|
|
35015
33718
|
init_local_sourcebook_public_urls();
|
|
35016
33719
|
init_local_sourcebook_schema();
|
|
35017
33720
|
CATEGORY_LABELS = {
|
|
@@ -35186,7 +33889,7 @@ function createDefaultLocalSourcebookProvider() {
|
|
|
35186
33889
|
}
|
|
35187
33890
|
};
|
|
35188
33891
|
}
|
|
35189
|
-
function
|
|
33892
|
+
function errorMessage4(error) {
|
|
35190
33893
|
return (error instanceof Error ? error.message : String(error)).replace(/\s+/g, " ").trim().slice(0, 1e3);
|
|
35191
33894
|
}
|
|
35192
33895
|
async function runLocalSourcebookAcquisition(submission, provider = createDefaultLocalSourcebookProvider()) {
|
|
@@ -35219,7 +33922,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35219
33922
|
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
35220
33923
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
35221
33924
|
} catch (error) {
|
|
35222
|
-
siteError =
|
|
33925
|
+
siteError = errorMessage4(error);
|
|
35223
33926
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
35224
33927
|
}
|
|
35225
33928
|
try {
|
|
@@ -35227,7 +33930,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35227
33930
|
validateLocalSourcebookMapsIdentity(submission, maps);
|
|
35228
33931
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "done", query: submission.businessName, location: submission.state.toUpperCase(), resultCount: maps.reviews.length, result: { submissionId: submission.id, placeUrl: maps.placeUrl, cid: maps.cidDecimal ?? maps.cid, reviewsStatus: maps.reviewsStatus, reviewsRetained: maps.reviews.length, services: maps.services.length, areasServed: maps.areasServed.length } });
|
|
35229
33932
|
} catch (error) {
|
|
35230
|
-
mapsError =
|
|
33933
|
+
mapsError = errorMessage4(error);
|
|
35231
33934
|
maps = null;
|
|
35232
33935
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "failed", query: submission.businessName, location: submission.state.toUpperCase(), error: mapsError });
|
|
35233
33936
|
}
|
|
@@ -35265,7 +33968,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35265
33968
|
kind,
|
|
35266
33969
|
userId: submission.ownerUserId,
|
|
35267
33970
|
debitKey: debitKey2,
|
|
35268
|
-
reason: `Local Sourcebook acquisition failed: ${
|
|
33971
|
+
reason: `Local Sourcebook acquisition failed: ${errorMessage4(error)}`
|
|
35269
33972
|
}).catch(() => null);
|
|
35270
33973
|
}
|
|
35271
33974
|
throw error;
|
|
@@ -35278,7 +33981,7 @@ async function processLocalSourcebookSubmission(submissionId) {
|
|
|
35278
33981
|
const completed = await runLocalSourcebookAcquisition(submission);
|
|
35279
33982
|
return { claimed: true, submissionId: completed.id, status: completed.status };
|
|
35280
33983
|
} catch (error) {
|
|
35281
|
-
const message =
|
|
33984
|
+
const message = errorMessage4(error);
|
|
35282
33985
|
await failLocalSourcebookAcquisition(submission.id, message);
|
|
35283
33986
|
return { claimed: true, submissionId: submission.id, status: "failed" };
|
|
35284
33987
|
}
|
|
@@ -36117,15 +34820,15 @@ function normalizeIdempotencyKey(value) {
|
|
|
36117
34820
|
return key;
|
|
36118
34821
|
}
|
|
36119
34822
|
function tokenFor(id, idempotencyKey4) {
|
|
36120
|
-
return (0,
|
|
34823
|
+
return (0, import_node_crypto17.createHmac)("sha256", billingSecret()).update(id).update(":").update(idempotencyKey4).digest("base64url");
|
|
36121
34824
|
}
|
|
36122
34825
|
function tokenHash(token6) {
|
|
36123
|
-
return (0,
|
|
34826
|
+
return (0, import_node_crypto17.createHash)("sha256").update(token6).digest("hex");
|
|
36124
34827
|
}
|
|
36125
34828
|
function verifyToken(row, token6) {
|
|
36126
34829
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
36127
34830
|
const actual = Buffer.from(tokenHash(token6), "hex");
|
|
36128
|
-
if (expected.length !== actual.length || !(0,
|
|
34831
|
+
if (expected.length !== actual.length || !(0, import_node_crypto17.timingSafeEqual)(expected, actual)) {
|
|
36129
34832
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
36130
34833
|
}
|
|
36131
34834
|
}
|
|
@@ -36240,7 +34943,7 @@ async function authorizeScheduledRun(args) {
|
|
|
36240
34943
|
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
36241
34944
|
);
|
|
36242
34945
|
}
|
|
36243
|
-
const id = (0,
|
|
34946
|
+
const id = (0, import_node_crypto17.randomUUID)();
|
|
36244
34947
|
const token6 = tokenFor(id, idempotencyKey4);
|
|
36245
34948
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
36246
34949
|
const inserted = await getDb().execute({
|
|
@@ -36308,7 +35011,7 @@ async function startScheduledRun(args) {
|
|
|
36308
35011
|
if (!authorization || !["starting", "started"].includes(authorization.status)) {
|
|
36309
35012
|
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
|
|
36310
35013
|
}
|
|
36311
|
-
const eventId = existing?.id ?? (0,
|
|
35014
|
+
const eventId = existing?.id ?? (0, import_node_crypto17.randomUUID)();
|
|
36312
35015
|
if (!existing) {
|
|
36313
35016
|
await getDb().execute({
|
|
36314
35017
|
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
|
|
@@ -36447,7 +35150,7 @@ async function settleScheduledRun(args) {
|
|
|
36447
35150
|
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
36448
35151
|
...pendingReason ? { reason: pendingReason } : {}
|
|
36449
35152
|
});
|
|
36450
|
-
const eventId = existing?.id ?? (0,
|
|
35153
|
+
const eventId = existing?.id ?? (0, import_node_crypto17.randomUUID)();
|
|
36451
35154
|
const status = pendingReason ? "cost_pending" : "settling";
|
|
36452
35155
|
const eventValues = [
|
|
36453
35156
|
modelMc,
|
|
@@ -36572,11 +35275,11 @@ async function voidScheduledRunAuthorization(args) {
|
|
|
36572
35275
|
}
|
|
36573
35276
|
return { ok: true, status: authorization.status };
|
|
36574
35277
|
}
|
|
36575
|
-
var
|
|
35278
|
+
var import_node_crypto17, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, AUTHORIZATION_TTL_MS, UnifiedBillingError;
|
|
36576
35279
|
var init_unified_billing = __esm({
|
|
36577
35280
|
"src/api/unified-billing.ts"() {
|
|
36578
35281
|
"use strict";
|
|
36579
|
-
|
|
35282
|
+
import_node_crypto17 = require("crypto");
|
|
36580
35283
|
init_db();
|
|
36581
35284
|
init_rates();
|
|
36582
35285
|
init_scheduling_access();
|
|
@@ -36608,14 +35311,14 @@ function getSessionSecret() {
|
|
|
36608
35311
|
function safeEqualHex(a, b) {
|
|
36609
35312
|
if (a.length !== b.length) return false;
|
|
36610
35313
|
try {
|
|
36611
|
-
return (0,
|
|
35314
|
+
return (0, import_node_crypto18.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
36612
35315
|
} catch {
|
|
36613
35316
|
return false;
|
|
36614
35317
|
}
|
|
36615
35318
|
}
|
|
36616
35319
|
function signSession(userId) {
|
|
36617
35320
|
const payload = String(userId);
|
|
36618
|
-
const sig = (0,
|
|
35321
|
+
const sig = (0, import_node_crypto18.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
36619
35322
|
return `${payload}.${sig}`;
|
|
36620
35323
|
}
|
|
36621
35324
|
function verifySession(token6) {
|
|
@@ -36623,16 +35326,16 @@ function verifySession(token6) {
|
|
|
36623
35326
|
if (dot === -1) return null;
|
|
36624
35327
|
const payload = token6.slice(0, dot);
|
|
36625
35328
|
const sig = token6.slice(dot + 1);
|
|
36626
|
-
const expected = (0,
|
|
35329
|
+
const expected = (0, import_node_crypto18.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
36627
35330
|
if (!safeEqualHex(sig, expected)) return null;
|
|
36628
35331
|
const id = parseInt(payload);
|
|
36629
35332
|
return isNaN(id) ? null : id;
|
|
36630
35333
|
}
|
|
36631
|
-
var
|
|
35334
|
+
var import_node_crypto18, isProduction, secret;
|
|
36632
35335
|
var init_session = __esm({
|
|
36633
35336
|
"src/api/session.ts"() {
|
|
36634
35337
|
"use strict";
|
|
36635
|
-
|
|
35338
|
+
import_node_crypto18 = require("crypto");
|
|
36636
35339
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
36637
35340
|
secret = () => getSessionSecret();
|
|
36638
35341
|
}
|
|
@@ -36650,11 +35353,11 @@ function isMemoryOperator(email) {
|
|
|
36650
35353
|
return ops.includes(email.trim().toLowerCase());
|
|
36651
35354
|
}
|
|
36652
35355
|
function encKey() {
|
|
36653
|
-
return (0,
|
|
35356
|
+
return (0, import_node_crypto19.scryptSync)(getSessionSecret(), "mcp-memory-key-v1", 32);
|
|
36654
35357
|
}
|
|
36655
35358
|
function encryptMemoryKey(secret2) {
|
|
36656
|
-
const iv = (0,
|
|
36657
|
-
const cipher = (0,
|
|
35359
|
+
const iv = (0, import_node_crypto19.randomBytes)(12);
|
|
35360
|
+
const cipher = (0, import_node_crypto19.createCipheriv)("aes-256-gcm", encKey(), iv);
|
|
36658
35361
|
const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
|
|
36659
35362
|
const tag = cipher.getAuthTag();
|
|
36660
35363
|
return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
|
|
@@ -36662,7 +35365,7 @@ function encryptMemoryKey(secret2) {
|
|
|
36662
35365
|
function decryptMemoryKey(stored) {
|
|
36663
35366
|
try {
|
|
36664
35367
|
const [ivB, tagB, dataB] = stored.split(":");
|
|
36665
|
-
const decipher = (0,
|
|
35368
|
+
const decipher = (0, import_node_crypto19.createDecipheriv)("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
|
|
36666
35369
|
decipher.setAuthTag(Buffer.from(tagB, "base64"));
|
|
36667
35370
|
return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
|
|
36668
35371
|
} catch {
|
|
@@ -36776,11 +35479,11 @@ async function syncScheduledActionCredentials(user) {
|
|
|
36776
35479
|
}, {});
|
|
36777
35480
|
return { ok: res.ok, error: res.error };
|
|
36778
35481
|
}
|
|
36779
|
-
var
|
|
35482
|
+
var import_node_crypto19, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
|
|
36780
35483
|
var init_memory = __esm({
|
|
36781
35484
|
"src/api/memory.ts"() {
|
|
36782
35485
|
"use strict";
|
|
36783
|
-
|
|
35486
|
+
import_node_crypto19 = require("crypto");
|
|
36784
35487
|
init_session();
|
|
36785
35488
|
init_db();
|
|
36786
35489
|
init_rates();
|
|
@@ -36830,7 +35533,7 @@ var init_connected_cost_telemetry = __esm({
|
|
|
36830
35533
|
|
|
36831
35534
|
// src/api/connected-usage-billing.ts
|
|
36832
35535
|
function hash(value) {
|
|
36833
|
-
return (0,
|
|
35536
|
+
return (0, import_node_crypto20.createHash)("sha256").update(value).digest("hex");
|
|
36834
35537
|
}
|
|
36835
35538
|
function eventKey(idempotencyKey4) {
|
|
36836
35539
|
return `connected-usage:${hash(idempotencyKey4)}`;
|
|
@@ -37102,7 +35805,7 @@ async function settleConnectedUsage(rawInput) {
|
|
|
37102
35805
|
sql: `INSERT OR IGNORE INTO billing_events
|
|
37103
35806
|
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
37104
35807
|
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
37105
|
-
args: [(0,
|
|
35808
|
+
args: [(0, import_node_crypto20.randomUUID)(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
|
|
37106
35809
|
});
|
|
37107
35810
|
let event2 = await readEvent(key);
|
|
37108
35811
|
if (!event2) throw new Error("connected usage receipt insert completed without a readable event");
|
|
@@ -37191,11 +35894,11 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
37191
35894
|
}
|
|
37192
35895
|
return history;
|
|
37193
35896
|
}
|
|
37194
|
-
var
|
|
35897
|
+
var import_node_crypto20, import_zod17, CONNECTED_USAGE_BILLING_CLASS, CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE, ConnectedUsageSafeMetadataSchema, ConnectedUsageSettlementInputSchema, ConnectedUsagePreflightInputSchema, ConnectedUsageBillingError;
|
|
37195
35898
|
var init_connected_usage_billing = __esm({
|
|
37196
35899
|
"src/api/connected-usage-billing.ts"() {
|
|
37197
35900
|
"use strict";
|
|
37198
|
-
|
|
35901
|
+
import_node_crypto20 = require("crypto");
|
|
37199
35902
|
import_zod17 = require("zod");
|
|
37200
35903
|
init_connected_cost_telemetry();
|
|
37201
35904
|
init_db();
|
|
@@ -38894,6 +37597,171 @@ var init_server_schemas = __esm({
|
|
|
38894
37597
|
}
|
|
38895
37598
|
});
|
|
38896
37599
|
|
|
37600
|
+
// src/api/harvest-problems.ts
|
|
37601
|
+
function errorMessage5(err) {
|
|
37602
|
+
return err instanceof Error ? err.message : String(err);
|
|
37603
|
+
}
|
|
37604
|
+
function looksLikeTimeout(err, message) {
|
|
37605
|
+
if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
|
|
37606
|
+
return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
|
|
37607
|
+
}
|
|
37608
|
+
function looksLikeCaptcha(message) {
|
|
37609
|
+
return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
|
|
37610
|
+
}
|
|
37611
|
+
function looksLikeProxyTunnelFailure3(message) {
|
|
37612
|
+
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
37613
|
+
}
|
|
37614
|
+
function looksLikeProxyUnavailable3(message) {
|
|
37615
|
+
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
37616
|
+
}
|
|
37617
|
+
function looksLikeInternalServiceUnavailable(message) {
|
|
37618
|
+
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
37619
|
+
}
|
|
37620
|
+
function looksLikeBrowserSessionInterrupted(message) {
|
|
37621
|
+
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
37622
|
+
}
|
|
37623
|
+
function classifyHarvestProblem(err) {
|
|
37624
|
+
const message = errorMessage5(err);
|
|
37625
|
+
if (err instanceof RequestAbortedError) {
|
|
37626
|
+
return {
|
|
37627
|
+
error_code: "request_aborted",
|
|
37628
|
+
error_type: "request_aborted",
|
|
37629
|
+
message: publicErrorMessage("request_aborted"),
|
|
37630
|
+
retryable: true,
|
|
37631
|
+
httpStatus: 408,
|
|
37632
|
+
terminalStatus: "cancelled"
|
|
37633
|
+
};
|
|
37634
|
+
}
|
|
37635
|
+
if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
|
|
37636
|
+
return {
|
|
37637
|
+
error_code: "captcha_exhausted",
|
|
37638
|
+
error_type: "captcha",
|
|
37639
|
+
message: publicErrorMessage("captcha_exhausted"),
|
|
37640
|
+
retryable: true,
|
|
37641
|
+
httpStatus: 503,
|
|
37642
|
+
terminalStatus: "failed"
|
|
37643
|
+
};
|
|
37644
|
+
}
|
|
37645
|
+
if (isVendorUnavailableError(err, message)) {
|
|
37646
|
+
return {
|
|
37647
|
+
error_code: "vendor_unavailable",
|
|
37648
|
+
error_type: "service_unavailable",
|
|
37649
|
+
message: publicErrorMessage("vendor_unavailable"),
|
|
37650
|
+
retryable: true,
|
|
37651
|
+
httpStatus: 503,
|
|
37652
|
+
terminalStatus: "failed"
|
|
37653
|
+
};
|
|
37654
|
+
}
|
|
37655
|
+
if (err instanceof LocationMismatchError) {
|
|
37656
|
+
return {
|
|
37657
|
+
error_code: "location_mismatch",
|
|
37658
|
+
error_type: "location_mismatch",
|
|
37659
|
+
message: publicErrorMessage("location_mismatch"),
|
|
37660
|
+
retryable: true,
|
|
37661
|
+
httpStatus: 503,
|
|
37662
|
+
terminalStatus: "failed"
|
|
37663
|
+
};
|
|
37664
|
+
}
|
|
37665
|
+
if (looksLikeProxyTunnelFailure3(message)) {
|
|
37666
|
+
return {
|
|
37667
|
+
error_code: "proxy_tunnel_failed",
|
|
37668
|
+
error_type: "connection",
|
|
37669
|
+
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
37670
|
+
retryable: true,
|
|
37671
|
+
httpStatus: 503,
|
|
37672
|
+
terminalStatus: "failed"
|
|
37673
|
+
};
|
|
37674
|
+
}
|
|
37675
|
+
if (looksLikeProxyUnavailable3(message)) {
|
|
37676
|
+
return {
|
|
37677
|
+
error_code: "proxy_unavailable",
|
|
37678
|
+
error_type: "connection",
|
|
37679
|
+
message: publicErrorMessage("proxy_unavailable"),
|
|
37680
|
+
retryable: true,
|
|
37681
|
+
httpStatus: 503,
|
|
37682
|
+
terminalStatus: "failed"
|
|
37683
|
+
};
|
|
37684
|
+
}
|
|
37685
|
+
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
37686
|
+
return {
|
|
37687
|
+
error_code: "browser_session_interrupted",
|
|
37688
|
+
error_type: "extraction",
|
|
37689
|
+
message: publicErrorMessage("browser_session_interrupted"),
|
|
37690
|
+
retryable: true,
|
|
37691
|
+
httpStatus: 503,
|
|
37692
|
+
terminalStatus: "failed"
|
|
37693
|
+
};
|
|
37694
|
+
}
|
|
37695
|
+
if (looksLikeTimeout(err, message)) {
|
|
37696
|
+
return {
|
|
37697
|
+
error_code: "harvest_timeout",
|
|
37698
|
+
error_type: "timeout",
|
|
37699
|
+
message: publicErrorMessage("harvest_timeout"),
|
|
37700
|
+
retryable: true,
|
|
37701
|
+
httpStatus: 504,
|
|
37702
|
+
terminalStatus: "failed"
|
|
37703
|
+
};
|
|
37704
|
+
}
|
|
37705
|
+
if (looksLikeInternalServiceUnavailable(message)) {
|
|
37706
|
+
return {
|
|
37707
|
+
error_code: "service_unavailable",
|
|
37708
|
+
error_type: "service_unavailable",
|
|
37709
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
37710
|
+
retryable: true,
|
|
37711
|
+
httpStatus: 503,
|
|
37712
|
+
terminalStatus: "failed"
|
|
37713
|
+
};
|
|
37714
|
+
}
|
|
37715
|
+
return {
|
|
37716
|
+
error_code: "extraction_failed",
|
|
37717
|
+
error_type: "extraction",
|
|
37718
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
37719
|
+
retryable: false,
|
|
37720
|
+
httpStatus: 500,
|
|
37721
|
+
terminalStatus: "failed"
|
|
37722
|
+
};
|
|
37723
|
+
}
|
|
37724
|
+
function serializeHarvestProblem(problem) {
|
|
37725
|
+
return JSON.stringify({
|
|
37726
|
+
error_code: problem.error_code,
|
|
37727
|
+
error_type: problem.error_type,
|
|
37728
|
+
message: problem.message,
|
|
37729
|
+
retryable: problem.retryable
|
|
37730
|
+
});
|
|
37731
|
+
}
|
|
37732
|
+
function harvestProblemResponse(problem, context = {}) {
|
|
37733
|
+
const envelope = buildPublicErrorEnvelope({
|
|
37734
|
+
errorCode: problem.error_code,
|
|
37735
|
+
errorType: problem.error_type,
|
|
37736
|
+
retryable: problem.retryable,
|
|
37737
|
+
retryAfterSeconds: context.retryAfterSeconds,
|
|
37738
|
+
chargeStatus: context.chargeStatus,
|
|
37739
|
+
details: context.details
|
|
37740
|
+
});
|
|
37741
|
+
return {
|
|
37742
|
+
error: envelope.message,
|
|
37743
|
+
...envelope
|
|
37744
|
+
};
|
|
37745
|
+
}
|
|
37746
|
+
function harvestProblemEnvelope(problem, context = {}) {
|
|
37747
|
+
const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context);
|
|
37748
|
+
return envelope;
|
|
37749
|
+
}
|
|
37750
|
+
function publicErrorBoundary(err, context = {}) {
|
|
37751
|
+
const problem = classifyHarvestProblem(err);
|
|
37752
|
+
return {
|
|
37753
|
+
body: harvestProblemResponse(problem, context),
|
|
37754
|
+
status: problem.httpStatus
|
|
37755
|
+
};
|
|
37756
|
+
}
|
|
37757
|
+
var init_harvest_problems = __esm({
|
|
37758
|
+
"src/api/harvest-problems.ts"() {
|
|
37759
|
+
"use strict";
|
|
37760
|
+
init_errors();
|
|
37761
|
+
init_vendor_errors();
|
|
37762
|
+
}
|
|
37763
|
+
});
|
|
37764
|
+
|
|
38897
37765
|
// src/api/youtube-routes.ts
|
|
38898
37766
|
function buildTranscriptMarkdown(result) {
|
|
38899
37767
|
const lines = [];
|
|
@@ -40177,7 +39045,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
40177
39045
|
}
|
|
40178
39046
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
40179
39047
|
}
|
|
40180
|
-
var import_hono7, import_zod24,
|
|
39048
|
+
var import_hono7, import_zod24, import_client7, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
40181
39049
|
var init_facebook_ad_routes = __esm({
|
|
40182
39050
|
"src/api/facebook-ad-routes.ts"() {
|
|
40183
39051
|
"use strict";
|
|
@@ -40192,7 +39060,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40192
39060
|
init_FacebookOrganicVideoExtractor();
|
|
40193
39061
|
init_kernel_proxy_resolver();
|
|
40194
39062
|
init_schemas2();
|
|
40195
|
-
|
|
39063
|
+
import_client7 = require("@fal-ai/client");
|
|
40196
39064
|
init_api_auth();
|
|
40197
39065
|
init_url_utils();
|
|
40198
39066
|
init_concurrency_gates();
|
|
@@ -40340,7 +39208,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40340
39208
|
metadata: { videoUrl }
|
|
40341
39209
|
});
|
|
40342
39210
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40343
|
-
|
|
39211
|
+
import_client7.fal.config({ credentials: process.env.FAL_KEY });
|
|
40344
39212
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40345
39213
|
let debited = false;
|
|
40346
39214
|
try {
|
|
@@ -40389,7 +39257,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40389
39257
|
metadata: { url: sourceUrl.href }
|
|
40390
39258
|
});
|
|
40391
39259
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40392
|
-
|
|
39260
|
+
import_client7.fal.config({ credentials: process.env.FAL_KEY });
|
|
40393
39261
|
const driver = new BrowserDriver();
|
|
40394
39262
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40395
39263
|
let debited = false;
|
|
@@ -40845,7 +39713,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
40845
39713
|
}
|
|
40846
39714
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
40847
39715
|
}
|
|
40848
|
-
var import_hono8, import_zod25,
|
|
39716
|
+
var import_hono8, import_zod25, import_client8, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
|
|
40849
39717
|
var init_google_ads_routes = __esm({
|
|
40850
39718
|
"src/api/google-ads-routes.ts"() {
|
|
40851
39719
|
"use strict";
|
|
@@ -40858,7 +39726,7 @@ var init_google_ads_routes = __esm({
|
|
|
40858
39726
|
init_GoogleAdsExtractor();
|
|
40859
39727
|
init_kernel_proxy_resolver();
|
|
40860
39728
|
init_schemas2();
|
|
40861
|
-
|
|
39729
|
+
import_client8 = require("@fal-ai/client");
|
|
40862
39730
|
init_api_auth();
|
|
40863
39731
|
init_url_utils();
|
|
40864
39732
|
init_concurrency_gates();
|
|
@@ -40983,7 +39851,7 @@ var init_google_ads_routes = __esm({
|
|
|
40983
39851
|
const user = c.get("user");
|
|
40984
39852
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
40985
39853
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40986
|
-
|
|
39854
|
+
import_client8.fal.config({ credentials: process.env.FAL_KEY });
|
|
40987
39855
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40988
39856
|
let debited = false;
|
|
40989
39857
|
try {
|
|
@@ -41164,7 +40032,7 @@ async function applyAdjustment(input) {
|
|
|
41164
40032
|
const credits = validateCredits(input.credits, input.confirmLarge === true);
|
|
41165
40033
|
const reason = validateReason(input.reason);
|
|
41166
40034
|
const actor = validateActor(input.actor);
|
|
41167
|
-
const reference = input.reference?.trim() || `adj_${(0,
|
|
40035
|
+
const reference = input.reference?.trim() || `adj_${(0, import_node_crypto21.randomUUID)()}`;
|
|
41168
40036
|
if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
|
|
41169
40037
|
const db = getDb();
|
|
41170
40038
|
const existing = await db.execute({
|
|
@@ -41309,11 +40177,11 @@ async function lookupAccount(target, limit = 20) {
|
|
|
41309
40177
|
}))
|
|
41310
40178
|
};
|
|
41311
40179
|
}
|
|
41312
|
-
var
|
|
40180
|
+
var import_node_crypto21, MIN_ADJUSTMENT_CREDITS, LARGE_ADJUSTMENT_CREDITS, MAX_ADJUSTMENT_CREDITS, MIN_REASON_LENGTH, MAX_REASON_LENGTH, MAX_ACTOR_LENGTH, AdminCreditError;
|
|
41313
40181
|
var init_admin_credits = __esm({
|
|
41314
40182
|
"src/api/admin-credits.ts"() {
|
|
41315
40183
|
"use strict";
|
|
41316
|
-
|
|
40184
|
+
import_node_crypto21 = require("crypto");
|
|
41317
40185
|
init_db();
|
|
41318
40186
|
init_rates();
|
|
41319
40187
|
MIN_ADJUSTMENT_CREDITS = 1;
|
|
@@ -43278,7 +42146,7 @@ async function packageMapsMedia(args) {
|
|
|
43278
42146
|
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
43279
42147
|
files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
|
|
43280
42148
|
const archive = await zipBuffer(files);
|
|
43281
|
-
const id = (0,
|
|
42149
|
+
const id = (0, import_node_crypto22.randomBytes)(6).toString("hex");
|
|
43282
42150
|
const pointer = await createPrivateArtifact({
|
|
43283
42151
|
policy: policy3(),
|
|
43284
42152
|
ownerId: args.ownerId,
|
|
@@ -43291,13 +42159,13 @@ async function packageMapsMedia(args) {
|
|
|
43291
42159
|
const localPath = token() ? null : (0, import_node_path13.join)(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path13.join)((0, import_node_os9.homedir)(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
43292
42160
|
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
43293
42161
|
}
|
|
43294
|
-
var import_node_os9, import_node_path13,
|
|
42162
|
+
var import_node_os9, import_node_path13, import_node_crypto22, import_yazl2, import_p_limit5, MAPS_MEDIA_ARTIFACT_PREFIX, MAPS_MEDIA_ARTIFACT_TTL_MS, MAPS_MEDIA_DOWNLOAD_TTL_MS, MAX_IMAGE_BYTES2, MAX_ARCHIVE_IMAGE_BYTES, MAX_INLINE_IMAGE_BYTES, MAX_INLINE_TOTAL_BYTES, DOWNLOAD_CONCURRENCY, MAX_REDIRECTS;
|
|
43295
42163
|
var init_maps_media_artifacts = __esm({
|
|
43296
42164
|
"src/api/maps-media-artifacts.ts"() {
|
|
43297
42165
|
"use strict";
|
|
43298
42166
|
import_node_os9 = require("os");
|
|
43299
42167
|
import_node_path13 = require("path");
|
|
43300
|
-
|
|
42168
|
+
import_node_crypto22 = require("crypto");
|
|
43301
42169
|
import_yazl2 = require("yazl");
|
|
43302
42170
|
import_p_limit5 = __toESM(require("p-limit"), 1);
|
|
43303
42171
|
init_private_artifacts();
|
|
@@ -43921,7 +42789,7 @@ function retryDelaySeconds(attempts) {
|
|
|
43921
42789
|
}
|
|
43922
42790
|
async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
43923
42791
|
const rows = await claimDirectoryWorkflowOutbox({
|
|
43924
|
-
workerId: `directory-dispatch-${process.pid}-${(0,
|
|
42792
|
+
workerId: `directory-dispatch-${process.pid}-${(0, import_node_crypto23.randomUUID)().slice(0, 8)}`,
|
|
43925
42793
|
limit
|
|
43926
42794
|
});
|
|
43927
42795
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
@@ -43943,11 +42811,11 @@ async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
|
43943
42811
|
}
|
|
43944
42812
|
return result;
|
|
43945
42813
|
}
|
|
43946
|
-
var
|
|
42814
|
+
var import_node_crypto23;
|
|
43947
42815
|
var init_directory_workflow_dispatch = __esm({
|
|
43948
42816
|
"src/api/directory-workflow-dispatch.ts"() {
|
|
43949
42817
|
"use strict";
|
|
43950
|
-
|
|
42818
|
+
import_node_crypto23 = require("crypto");
|
|
43951
42819
|
init_client();
|
|
43952
42820
|
init_directory_workflow_repository();
|
|
43953
42821
|
}
|
|
@@ -43959,7 +42827,7 @@ function safeOptions(options) {
|
|
|
43959
42827
|
return safe2;
|
|
43960
42828
|
}
|
|
43961
42829
|
function requestFingerprint(options) {
|
|
43962
|
-
return (0,
|
|
42830
|
+
return (0, import_node_crypto24.createHash)("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
|
|
43963
42831
|
}
|
|
43964
42832
|
function idempotencyKey(raw) {
|
|
43965
42833
|
if (raw !== void 0) {
|
|
@@ -43968,14 +42836,14 @@ function idempotencyKey(raw) {
|
|
|
43968
42836
|
if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
|
|
43969
42837
|
return { ok: true, key: trimmed };
|
|
43970
42838
|
}
|
|
43971
|
-
return { ok: true, key: `directory-${(0,
|
|
42839
|
+
return { ok: true, key: `directory-${(0, import_node_crypto24.randomUUID)()}` };
|
|
43972
42840
|
}
|
|
43973
42841
|
function debitKeyFor(userId, responseKey) {
|
|
43974
|
-
const digest2 = (0,
|
|
42842
|
+
const digest2 = (0, import_node_crypto24.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
43975
42843
|
return `directory-workflow:${userId}:${digest2}`;
|
|
43976
42844
|
}
|
|
43977
42845
|
function jobId() {
|
|
43978
|
-
return `dir_${(0,
|
|
42846
|
+
return `dir_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "")}`;
|
|
43979
42847
|
}
|
|
43980
42848
|
function publicStatus(job, result) {
|
|
43981
42849
|
if (job.status === "failed") return "failed";
|
|
@@ -44084,7 +42952,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
44084
42952
|
const csv = renderDirectoryWorkflowCsv(result);
|
|
44085
42953
|
const artifact = await createDirectoryCsvArtifact({
|
|
44086
42954
|
ownerId: String(user.id),
|
|
44087
|
-
jobId: (0,
|
|
42955
|
+
jobId: (0, import_node_crypto24.createHash)("sha256").update(debitKey2).digest("hex").slice(0, 32),
|
|
44088
42956
|
createdAt: result.extractedAt,
|
|
44089
42957
|
filename: `${options.state}-${options.query}-directory.csv`,
|
|
44090
42958
|
csv,
|
|
@@ -44140,11 +43008,11 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
44140
43008
|
await releaseConcurrencyGate(gate.lockId);
|
|
44141
43009
|
}
|
|
44142
43010
|
}
|
|
44143
|
-
var
|
|
43011
|
+
var import_node_crypto24, import_hono16, directoryApp;
|
|
44144
43012
|
var init_directory_routes = __esm({
|
|
44145
43013
|
"src/api/directory-routes.ts"() {
|
|
44146
43014
|
"use strict";
|
|
44147
|
-
|
|
43015
|
+
import_node_crypto24 = require("crypto");
|
|
44148
43016
|
import_hono16 = require("hono");
|
|
44149
43017
|
init_api_auth();
|
|
44150
43018
|
init_db();
|
|
@@ -44362,10 +43230,10 @@ function bounded(value, field, min, max) {
|
|
|
44362
43230
|
return normalized;
|
|
44363
43231
|
}
|
|
44364
43232
|
function newLeadListUploadId() {
|
|
44365
|
-
return `upl_${(0,
|
|
43233
|
+
return `upl_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
|
|
44366
43234
|
}
|
|
44367
43235
|
function newImportedLeadListId() {
|
|
44368
|
-
return `lst_${(0,
|
|
43236
|
+
return `lst_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
|
|
44369
43237
|
}
|
|
44370
43238
|
async function ensureLeadListImportRepositorySchema() {
|
|
44371
43239
|
const db = getDb();
|
|
@@ -44565,11 +43433,11 @@ async function deleteExpiredLeadListInputRecords(now = /* @__PURE__ */ new Date(
|
|
|
44565
43433
|
], "write");
|
|
44566
43434
|
return { uploads: Number(uploads.rowsAffected ?? 0), lists: Number(lists.rowsAffected ?? 0) };
|
|
44567
43435
|
}
|
|
44568
|
-
var
|
|
43436
|
+
var import_node_crypto25, schemaDb5, schemaPromise5;
|
|
44569
43437
|
var init_lead_list_import_repository = __esm({
|
|
44570
43438
|
"src/api/lead-list-import-repository.ts"() {
|
|
44571
43439
|
"use strict";
|
|
44572
|
-
|
|
43440
|
+
import_node_crypto25 = require("crypto");
|
|
44573
43441
|
init_db();
|
|
44574
43442
|
schemaDb5 = null;
|
|
44575
43443
|
schemaPromise5 = null;
|
|
@@ -44611,7 +43479,7 @@ function safeFilenameHint(value) {
|
|
|
44611
43479
|
return cleaned || null;
|
|
44612
43480
|
}
|
|
44613
43481
|
function requestFingerprint2(filenameHint) {
|
|
44614
|
-
return (0,
|
|
43482
|
+
return (0, import_node_crypto26.createHash)("sha256").update(JSON.stringify({ filenameHint })).digest("hex");
|
|
44615
43483
|
}
|
|
44616
43484
|
function publicBaseUrl() {
|
|
44617
43485
|
const configured = process.env.MCP_SCRAPER_PUBLIC_BASE_URL?.trim() || process.env.PUBLIC_BASE_URL?.trim() || (process.env.VERCEL_PROJECT_PRODUCTION_URL ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}` : "") || "https://mcpscraper.dev";
|
|
@@ -44767,7 +43635,7 @@ async function inspectLeadListUpload(uploadId, ownerId2) {
|
|
|
44767
43635
|
maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
|
|
44768
43636
|
});
|
|
44769
43637
|
if (!buffer) return null;
|
|
44770
|
-
const sha2565 = (0,
|
|
43638
|
+
const sha2565 = (0, import_node_crypto26.createHash)("sha256").update(buffer).digest("hex");
|
|
44771
43639
|
const completed = await completeLeadListUpload({
|
|
44772
43640
|
id: record.id,
|
|
44773
43641
|
ownerId: record.ownerId,
|
|
@@ -44891,11 +43759,11 @@ async function cleanupExpiredLeadListInputs(args = {}) {
|
|
|
44891
43759
|
const records = await deleteExpiredLeadListInputRecords(now);
|
|
44892
43760
|
return { deletedBlobs, deletedUploadRecords: records.uploads, deletedListRecords: records.lists };
|
|
44893
43761
|
}
|
|
44894
|
-
var
|
|
43762
|
+
var import_node_crypto26, import_promises11, import_node_os10, import_node_path14, LEAD_LIST_UPLOAD_PREFIX, LEAD_LIST_NORMALIZED_PREFIX, LEAD_LIST_UPLOAD_MAX_BYTES, LEAD_LIST_UPLOAD_URL_TTL_MS, LEAD_LIST_UPLOAD_SOURCE_TTL_MS, LEAD_LIST_NORMALIZED_TTL_MS, LEAD_LIST_DOWNLOAD_TTL_MS, LEAD_LIST_UPLOAD_MIME_TYPES;
|
|
44895
43763
|
var init_lead_list_input_artifacts = __esm({
|
|
44896
43764
|
"src/api/lead-list-input-artifacts.ts"() {
|
|
44897
43765
|
"use strict";
|
|
44898
|
-
|
|
43766
|
+
import_node_crypto26 = require("crypto");
|
|
44899
43767
|
import_promises11 = require("fs/promises");
|
|
44900
43768
|
import_node_os10 = require("os");
|
|
44901
43769
|
import_node_path14 = require("path");
|
|
@@ -45278,7 +44146,7 @@ function canonicalize(value) {
|
|
|
45278
44146
|
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalize(object[key])}`).join(",")}}`;
|
|
45279
44147
|
}
|
|
45280
44148
|
function fingerprint(value) {
|
|
45281
|
-
return (0,
|
|
44149
|
+
return (0, import_node_crypto27.createHash)("sha256").update(canonicalize(value)).digest("hex");
|
|
45282
44150
|
}
|
|
45283
44151
|
async function safeJson(c) {
|
|
45284
44152
|
try {
|
|
@@ -45355,11 +44223,11 @@ function textDelimiter(raw, text2, observedMime) {
|
|
|
45355
44223
|
if (observedMime === "text/tab-separated-values") return " ";
|
|
45356
44224
|
return detectLeadListTextDelimiter(text2);
|
|
45357
44225
|
}
|
|
45358
|
-
var
|
|
44226
|
+
var import_node_crypto27, import_hono17, leadListInputApp;
|
|
45359
44227
|
var init_lead_list_input_routes = __esm({
|
|
45360
44228
|
"src/api/lead-list-input-routes.ts"() {
|
|
45361
44229
|
"use strict";
|
|
45362
|
-
|
|
44230
|
+
import_node_crypto27 = require("crypto");
|
|
45363
44231
|
import_hono17 = require("hono");
|
|
45364
44232
|
init_api_auth();
|
|
45365
44233
|
init_lead_list_input_artifacts();
|
|
@@ -45551,7 +44419,7 @@ function retryDelaySeconds2(attempts) {
|
|
|
45551
44419
|
return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
45552
44420
|
}
|
|
45553
44421
|
async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
45554
|
-
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${(0,
|
|
44422
|
+
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${(0, import_node_crypto28.randomUUID)().slice(0, 8)}`, limit });
|
|
45555
44423
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
45556
44424
|
for (const row of rows) {
|
|
45557
44425
|
if (!row.claimToken) continue;
|
|
@@ -45570,11 +44438,11 @@ async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
|
45570
44438
|
}
|
|
45571
44439
|
return result;
|
|
45572
44440
|
}
|
|
45573
|
-
var
|
|
44441
|
+
var import_node_crypto28;
|
|
45574
44442
|
var init_lead_list_enrichment_dispatch = __esm({
|
|
45575
44443
|
"src/api/lead-list-enrichment-dispatch.ts"() {
|
|
45576
44444
|
"use strict";
|
|
45577
|
-
|
|
44445
|
+
import_node_crypto28 = require("crypto");
|
|
45578
44446
|
init_client();
|
|
45579
44447
|
init_lead_list_enrichment_repository();
|
|
45580
44448
|
}
|
|
@@ -45590,15 +44458,15 @@ function idempotencyKey2(raw) {
|
|
|
45590
44458
|
return { ok: true, value };
|
|
45591
44459
|
}
|
|
45592
44460
|
function newJobId() {
|
|
45593
|
-
return `lle_${(0,
|
|
44461
|
+
return `lle_${(0, import_node_crypto29.randomUUID)().replace(/-/g, "")}`;
|
|
45594
44462
|
}
|
|
45595
44463
|
function debitKeyFor2(userId, key) {
|
|
45596
|
-
const digest2 = (0,
|
|
44464
|
+
const digest2 = (0, import_node_crypto29.createHash)("sha256").update(String(userId)).update("\0").update(key).digest("hex");
|
|
45597
44465
|
return `lead-list-enrichment:${userId}:${digest2}`;
|
|
45598
44466
|
}
|
|
45599
44467
|
function inlineSourceDigest(headers, rows) {
|
|
45600
44468
|
const values = rows.map((row) => headers.map((header) => row[header] ?? null));
|
|
45601
|
-
return (0,
|
|
44469
|
+
return (0, import_node_crypto29.createHash)("sha256").update(stableCanonicalJson({ headers, values })).digest("hex");
|
|
45602
44470
|
}
|
|
45603
44471
|
function issueMessage(error) {
|
|
45604
44472
|
const issue = error.issues[0];
|
|
@@ -45796,11 +44664,11 @@ async function resolveInput(ownerId2, parsed) {
|
|
|
45796
44664
|
sourceDigest: sourceDigest ?? inlineSourceDigest(normalized.headers, rows)
|
|
45797
44665
|
};
|
|
45798
44666
|
}
|
|
45799
|
-
var
|
|
44667
|
+
var import_node_crypto29, import_hono18, import_zod31, SourceSchema, StartSchema, TERMINAL, leadListEnrichmentApp;
|
|
45800
44668
|
var init_lead_list_enrichment_routes = __esm({
|
|
45801
44669
|
"src/api/lead-list-enrichment-routes.ts"() {
|
|
45802
44670
|
"use strict";
|
|
45803
|
-
|
|
44671
|
+
import_node_crypto29 = require("crypto");
|
|
45804
44672
|
import_hono18 = require("hono");
|
|
45805
44673
|
import_zod31 = require("zod");
|
|
45806
44674
|
init_api_auth();
|
|
@@ -46755,50 +45623,324 @@ var init_location_data_routes = __esm({
|
|
|
46755
45623
|
}
|
|
46756
45624
|
});
|
|
46757
45625
|
|
|
46758
|
-
// src/api/
|
|
46759
|
-
|
|
46760
|
-
|
|
46761
|
-
|
|
46762
|
-
|
|
46763
|
-
|
|
46764
|
-
|
|
45626
|
+
// src/api/harvest-attempt-events.ts
|
|
45627
|
+
function createHarvestAttemptRecorder(jobId2, userId) {
|
|
45628
|
+
return async (event2) => {
|
|
45629
|
+
if (event2.type === "started") {
|
|
45630
|
+
await startHarvestAttempt({
|
|
45631
|
+
jobId: jobId2,
|
|
45632
|
+
userId,
|
|
45633
|
+
attemptNumber: event2.attemptNumber,
|
|
45634
|
+
maxAttempts: event2.maxAttempts,
|
|
45635
|
+
query: event2.query,
|
|
45636
|
+
location: event2.location,
|
|
45637
|
+
maxQuestions: event2.maxQuestions,
|
|
45638
|
+
startedAt: event2.startedAt
|
|
45639
|
+
});
|
|
45640
|
+
return;
|
|
45641
|
+
}
|
|
45642
|
+
await finishHarvestAttempt({
|
|
45643
|
+
jobId: jobId2,
|
|
45644
|
+
attemptNumber: event2.attemptNumber,
|
|
45645
|
+
outcome: event2.outcome,
|
|
45646
|
+
kernelSessionId: event2.kernelSessionId,
|
|
45647
|
+
questionCount: event2.questionCount,
|
|
45648
|
+
durationMs: event2.durationMs,
|
|
45649
|
+
error: event2.error,
|
|
45650
|
+
willRetry: event2.willRetry,
|
|
45651
|
+
kernelDeleteStarted: event2.cleanup.kernelDeleteStarted,
|
|
45652
|
+
kernelDeleteSucceeded: event2.cleanup.kernelDeleteSucceeded,
|
|
45653
|
+
kernelDeleteError: event2.cleanup.kernelDeleteError,
|
|
45654
|
+
browserCloseSucceeded: event2.cleanup.browserCloseSucceeded,
|
|
45655
|
+
browserCloseError: event2.cleanup.browserCloseError,
|
|
45656
|
+
debug: event2.debug,
|
|
45657
|
+
completedAt: event2.completedAt
|
|
45658
|
+
});
|
|
45659
|
+
};
|
|
46765
45660
|
}
|
|
46766
|
-
|
|
46767
|
-
|
|
46768
|
-
|
|
46769
|
-
|
|
46770
|
-
|
|
45661
|
+
var init_harvest_attempt_events = __esm({
|
|
45662
|
+
"src/api/harvest-attempt-events.ts"() {
|
|
45663
|
+
"use strict";
|
|
45664
|
+
init_db();
|
|
45665
|
+
}
|
|
45666
|
+
});
|
|
45667
|
+
|
|
45668
|
+
// src/api/paa-harvest-settlement.ts
|
|
45669
|
+
function capturedQuestionCount(result) {
|
|
45670
|
+
if (!result || typeof result !== "object") return 0;
|
|
45671
|
+
const value = result;
|
|
45672
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
45673
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
45674
|
+
}
|
|
45675
|
+
function finalCost(result, heldMc) {
|
|
45676
|
+
const questions = capturedQuestionCount(result);
|
|
45677
|
+
if (questions <= 0) return 0;
|
|
45678
|
+
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
45679
|
+
}
|
|
45680
|
+
async function settlePaaHarvestJob(jobOrId) {
|
|
45681
|
+
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
45682
|
+
if (!job) return false;
|
|
45683
|
+
const options = job.options;
|
|
45684
|
+
const debitKey2 = options.billingDebitKey;
|
|
45685
|
+
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
45686
|
+
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
45687
|
+
const settlement = await settleDebitMcIdempotent(
|
|
45688
|
+
job.user_id,
|
|
45689
|
+
debitKey2,
|
|
45690
|
+
finalCost(job.result, heldMc),
|
|
45691
|
+
LedgerOperation.PAA_REFUND,
|
|
45692
|
+
"durable PAA harvest settlement",
|
|
45693
|
+
"paa_harvest"
|
|
45694
|
+
);
|
|
45695
|
+
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
45696
|
+
return true;
|
|
45697
|
+
}
|
|
45698
|
+
var init_paa_harvest_settlement = __esm({
|
|
45699
|
+
"src/api/paa-harvest-settlement.ts"() {
|
|
45700
|
+
"use strict";
|
|
45701
|
+
init_db();
|
|
45702
|
+
init_rates();
|
|
45703
|
+
}
|
|
45704
|
+
});
|
|
45705
|
+
|
|
45706
|
+
// src/api/webhook.ts
|
|
45707
|
+
async function deliverWebhook(url, payload, retries = 3) {
|
|
45708
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
46771
45709
|
try {
|
|
46772
|
-
await
|
|
46773
|
-
|
|
46774
|
-
|
|
46775
|
-
|
|
45710
|
+
const res = await fetch(url, {
|
|
45711
|
+
method: "POST",
|
|
45712
|
+
headers: { "content-type": "application/json" },
|
|
45713
|
+
body: JSON.stringify(payload),
|
|
45714
|
+
signal: AbortSignal.timeout(1e4)
|
|
45715
|
+
});
|
|
45716
|
+
if (res.ok) return;
|
|
45717
|
+
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
45718
|
+
} catch (err) {
|
|
45719
|
+
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
45720
|
+
}
|
|
45721
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
45722
|
+
}
|
|
45723
|
+
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
45724
|
+
}
|
|
45725
|
+
var init_webhook = __esm({
|
|
45726
|
+
"src/api/webhook.ts"() {
|
|
45727
|
+
"use strict";
|
|
45728
|
+
}
|
|
45729
|
+
});
|
|
45730
|
+
|
|
45731
|
+
// src/api/paa-harvest-runner.ts
|
|
45732
|
+
function durableRecordsFromResult(result) {
|
|
45733
|
+
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
45734
|
+
return flat.flatMap((value) => {
|
|
45735
|
+
if (!value || typeof value !== "object") return [];
|
|
45736
|
+
const row = value;
|
|
45737
|
+
const question = typeof row.question === "string" ? row.question : "";
|
|
45738
|
+
if (!question) return [];
|
|
45739
|
+
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
45740
|
+
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
45741
|
+
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
45742
|
+
return [{
|
|
45743
|
+
question,
|
|
45744
|
+
answer: typeof row.answer === "string" ? row.answer : "",
|
|
45745
|
+
sourceTitle,
|
|
45746
|
+
sourceSite,
|
|
45747
|
+
sourceCite,
|
|
45748
|
+
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
45749
|
+
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
45750
|
+
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
45751
|
+
}];
|
|
45752
|
+
});
|
|
45753
|
+
}
|
|
45754
|
+
function questionCount(result) {
|
|
45755
|
+
if (!result || typeof result !== "object") return 0;
|
|
45756
|
+
const value = result;
|
|
45757
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
45758
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
45759
|
+
}
|
|
45760
|
+
async function runPaaHarvestJob(jobId2) {
|
|
45761
|
+
const invocationStartedAt = Date.now();
|
|
45762
|
+
const job = await claimDurablePaaJob(jobId2);
|
|
45763
|
+
if (!job) return { jobId: jobId2, status: "missing_or_not_paa" };
|
|
45764
|
+
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
45765
|
+
await settlePaaHarvestJob(job.id);
|
|
45766
|
+
return { jobId: job.id, status: job.status };
|
|
45767
|
+
}
|
|
45768
|
+
const options = job.options;
|
|
45769
|
+
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
45770
|
+
const timeoutBudget = harvestTimeoutBudget(maxQuestions);
|
|
45771
|
+
const workBudgetMs = Math.max(1, timeoutBudget.serverMs - CLEANUP_RESERVE_MS);
|
|
45772
|
+
const checkpointState = { value: null };
|
|
45773
|
+
try {
|
|
45774
|
+
if (!brightDataSerpEnabled()) throw new Error("SERP browser service is not configured");
|
|
45775
|
+
const navigating = {
|
|
45776
|
+
phase: "navigating",
|
|
45777
|
+
attempt: 1,
|
|
45778
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
45779
|
+
records: [],
|
|
45780
|
+
clickedQuestions: [],
|
|
45781
|
+
finalUrl: null,
|
|
45782
|
+
captchaObserved: false,
|
|
45783
|
+
solverCleared: false,
|
|
45784
|
+
error: null
|
|
45785
|
+
};
|
|
45786
|
+
checkpointState.value = navigating;
|
|
45787
|
+
if (!await checkpointJob(job.id, {
|
|
45788
|
+
progress: navigating,
|
|
45789
|
+
durable: {
|
|
45790
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
45791
|
+
workBudgetMs,
|
|
45792
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
45793
|
+
}
|
|
45794
|
+
})) {
|
|
45795
|
+
throw new Error("job_left_running_state_during_capture");
|
|
45796
|
+
}
|
|
45797
|
+
const persistAttempt = createHarvestAttemptRecorder(job.id, job.user_id);
|
|
45798
|
+
let lastAttempt = 1;
|
|
45799
|
+
const onAttemptEvent = async (event2) => {
|
|
45800
|
+
lastAttempt = event2.attemptNumber;
|
|
45801
|
+
await persistAttempt(event2);
|
|
45802
|
+
};
|
|
45803
|
+
const harvestCtx = currentCostContext();
|
|
45804
|
+
const harvested = await runWithCostContext(
|
|
45805
|
+
{ ...harvestCtx, op: "paa", userId: job.user_id },
|
|
45806
|
+
() => harvest({
|
|
45807
|
+
query: options.query ?? job.query,
|
|
45808
|
+
location: options.location ?? void 0,
|
|
45809
|
+
gl: options.gl ?? "us",
|
|
45810
|
+
hl: options.hl ?? "en",
|
|
45811
|
+
maxQuestions,
|
|
45812
|
+
maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
|
|
45813
|
+
proxyMode: "none",
|
|
45814
|
+
forceManagedSerp: true,
|
|
45815
|
+
headless: true,
|
|
45816
|
+
format: "json",
|
|
45817
|
+
outputDir: "/tmp/paa-output-direct",
|
|
45818
|
+
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
45819
|
+
includeLocalPack: options.includeLocalPack ?? false,
|
|
45820
|
+
includeForums: options.includeForums ?? false,
|
|
45821
|
+
includeVideos: options.includeVideos ?? false,
|
|
45822
|
+
includeAiOverview: options.includeAiOverview ?? false,
|
|
45823
|
+
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
45824
|
+
softDeadlineMs: invocationStartedAt + workBudgetMs,
|
|
45825
|
+
onAttemptEvent
|
|
45826
|
+
})
|
|
45827
|
+
);
|
|
45828
|
+
const harvestResult = harvested;
|
|
45829
|
+
const records = durableRecordsFromResult(harvestResult);
|
|
45830
|
+
const captured = {
|
|
45831
|
+
...navigating,
|
|
45832
|
+
phase: "captured",
|
|
45833
|
+
attempt: lastAttempt,
|
|
45834
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
45835
|
+
records,
|
|
45836
|
+
clickedQuestions: records.map((record) => record.question)
|
|
45837
|
+
};
|
|
45838
|
+
checkpointState.value = captured;
|
|
45839
|
+
if (!await checkpointJob(job.id, {
|
|
45840
|
+
progress: captured,
|
|
45841
|
+
durable: {
|
|
45842
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
45843
|
+
workBudgetMs,
|
|
45844
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
45845
|
+
}
|
|
45846
|
+
})) {
|
|
45847
|
+
throw new Error("job_left_running_state_during_capture");
|
|
45848
|
+
}
|
|
45849
|
+
const result = {
|
|
45850
|
+
...harvestResult,
|
|
45851
|
+
capture: {
|
|
45852
|
+
...harvestResult.capture,
|
|
45853
|
+
execution: "brightdata_browser_api_controller_fulfilled",
|
|
45854
|
+
provider: "bright_data_browser_api",
|
|
45855
|
+
sameQueryClickGraph: true,
|
|
45856
|
+
rawDomArtifact: null,
|
|
45857
|
+
rawDomArtifactStatus: "unavailable"
|
|
45858
|
+
}
|
|
45859
|
+
};
|
|
45860
|
+
await completeJob(job.id, result);
|
|
45861
|
+
await settlePaaHarvestJob(job.id);
|
|
45862
|
+
if (job.callback_url) {
|
|
45863
|
+
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
45864
|
+
}
|
|
45865
|
+
return {
|
|
45866
|
+
jobId: job.id,
|
|
45867
|
+
status: "done",
|
|
45868
|
+
totalQuestions: questionCount(result),
|
|
45869
|
+
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
45870
|
+
artifactStored: false
|
|
45871
|
+
};
|
|
45872
|
+
} catch (error) {
|
|
45873
|
+
const problem = classifyHarvestProblem(error);
|
|
45874
|
+
const latestCheckpoint = checkpointState.value;
|
|
45875
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
45876
|
+
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
45877
|
+
if (!settled) return;
|
|
45878
|
+
const settledJob = await getJob(job.id);
|
|
45879
|
+
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
45880
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
45881
|
+
}).catch((settlementError) => {
|
|
45882
|
+
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
45883
|
+
});
|
|
45884
|
+
if (job.callback_url) {
|
|
45885
|
+
await deliverWebhook(job.callback_url, {
|
|
45886
|
+
job_id: job.id,
|
|
45887
|
+
status: "failed",
|
|
45888
|
+
...harvestProblemResponse(problem),
|
|
45889
|
+
progress: latestCheckpoint
|
|
45890
|
+
});
|
|
46776
45891
|
}
|
|
45892
|
+
throw error;
|
|
46777
45893
|
}
|
|
46778
|
-
return { checked: jobs.length, dispatched, failed };
|
|
46779
45894
|
}
|
|
46780
|
-
var
|
|
46781
|
-
|
|
45895
|
+
var CLEANUP_RESERVE_MS, DIRECT_PAA_MAX_ATTEMPTS;
|
|
45896
|
+
var init_paa_harvest_runner = __esm({
|
|
45897
|
+
"src/api/paa-harvest-runner.ts"() {
|
|
46782
45898
|
"use strict";
|
|
46783
|
-
|
|
45899
|
+
init_BrightDataSerpDriver();
|
|
45900
|
+
init_harvest();
|
|
45901
|
+
init_harvest_timeout();
|
|
45902
|
+
init_cost_context();
|
|
46784
45903
|
init_db();
|
|
45904
|
+
init_harvest_attempt_events();
|
|
45905
|
+
init_harvest_problems();
|
|
45906
|
+
init_paa_harvest_settlement();
|
|
45907
|
+
init_webhook();
|
|
45908
|
+
CLEANUP_RESERVE_MS = 4e4;
|
|
45909
|
+
DIRECT_PAA_MAX_ATTEMPTS = 3;
|
|
45910
|
+
}
|
|
45911
|
+
});
|
|
45912
|
+
|
|
45913
|
+
// src/api/paa-harvest-direct.ts
|
|
45914
|
+
function scheduleDirectPaaHarvest(jobId2) {
|
|
45915
|
+
const work = runPaaHarvestJob(jobId2).catch((error) => {
|
|
45916
|
+
console.error("[paa-harvest/direct] run failed:", error instanceof Error ? error.message : String(error));
|
|
45917
|
+
});
|
|
45918
|
+
(0, import_functions.waitUntil)(work);
|
|
45919
|
+
}
|
|
45920
|
+
var import_functions;
|
|
45921
|
+
var init_paa_harvest_direct = __esm({
|
|
45922
|
+
"src/api/paa-harvest-direct.ts"() {
|
|
45923
|
+
"use strict";
|
|
45924
|
+
import_functions = require("@vercel/functions");
|
|
45925
|
+
init_paa_harvest_runner();
|
|
46785
45926
|
}
|
|
46786
45927
|
});
|
|
46787
45928
|
|
|
46788
45929
|
// src/api/paa-harvest-reconciliation.ts
|
|
46789
45930
|
async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
46790
|
-
const stale = await
|
|
45931
|
+
const stale = await listStaleRunningDurablePaaJobs(limit);
|
|
46791
45932
|
let staleFailed = 0;
|
|
46792
45933
|
for (const job of stale) {
|
|
46793
45934
|
try {
|
|
46794
|
-
const privateError = "
|
|
45935
|
+
const privateError = "direct PAA invocation exceeded its platform safety window; preserved progress remains in result";
|
|
46795
45936
|
const problem = classifyHarvestProblem(new Error("harvest timeout"));
|
|
46796
|
-
await
|
|
46797
|
-
|
|
45937
|
+
if (await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
|
|
45938
|
+
staleFailed += 1;
|
|
45939
|
+
}
|
|
46798
45940
|
} catch {
|
|
46799
45941
|
}
|
|
46800
45942
|
}
|
|
46801
|
-
const jobs = await
|
|
45943
|
+
const jobs = await listRecentTerminalDurablePaaJobs(limit);
|
|
46802
45944
|
let settled = 0;
|
|
46803
45945
|
let failed = 0;
|
|
46804
45946
|
for (const job of jobs) {
|
|
@@ -48993,7 +48135,7 @@ async function readManifestFromSummary(summary) {
|
|
|
48993
48135
|
function webhookSignature(body, timestamp2) {
|
|
48994
48136
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
48995
48137
|
if (!secret2) return null;
|
|
48996
|
-
return (0,
|
|
48138
|
+
return (0, import_node_crypto30.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
48997
48139
|
}
|
|
48998
48140
|
async function deliverWorkflowWebhook(input) {
|
|
48999
48141
|
if (!input.webhookUrl) return;
|
|
@@ -49200,11 +48342,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
49200
48342
|
}
|
|
49201
48343
|
return { dispatched: results.length, results };
|
|
49202
48344
|
}
|
|
49203
|
-
var
|
|
48345
|
+
var import_node_crypto30, import_promises14, import_hono21, import_zod41, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
49204
48346
|
var init_workflow_routes = __esm({
|
|
49205
48347
|
"src/api/workflow-routes.ts"() {
|
|
49206
48348
|
"use strict";
|
|
49207
|
-
|
|
48349
|
+
import_node_crypto30 = require("crypto");
|
|
49208
48350
|
import_promises14 = require("fs/promises");
|
|
49209
48351
|
import_hono21 = require("hono");
|
|
49210
48352
|
import_zod41 = require("zod");
|
|
@@ -49485,7 +48627,7 @@ var init_workflow_routes = __esm({
|
|
|
49485
48627
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
49486
48628
|
function sha2562(value) {
|
|
49487
48629
|
if (!value) return null;
|
|
49488
|
-
return (0,
|
|
48630
|
+
return (0, import_node_crypto31.createHash)("sha256").update(value).digest("hex");
|
|
49489
48631
|
}
|
|
49490
48632
|
function countWords(markdown) {
|
|
49491
48633
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -49785,11 +48927,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
49785
48927
|
}
|
|
49786
48928
|
};
|
|
49787
48929
|
}
|
|
49788
|
-
var
|
|
48930
|
+
var import_node_crypto31, import_p_limit6, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
49789
48931
|
var init_page_snapshot_extractor = __esm({
|
|
49790
48932
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
49791
48933
|
"use strict";
|
|
49792
|
-
|
|
48934
|
+
import_node_crypto31 = require("crypto");
|
|
49793
48935
|
import_p_limit6 = __toESM(require("p-limit"), 1);
|
|
49794
48936
|
init_kpo_extractor();
|
|
49795
48937
|
init_url_utils();
|
|
@@ -50286,21 +49428,21 @@ async function logRequestEventBestEffort(input) {
|
|
|
50286
49428
|
}
|
|
50287
49429
|
}
|
|
50288
49430
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
50289
|
-
const responseKey = suppliedKey?.trim() || (0,
|
|
50290
|
-
const requestFingerprint3 = (0,
|
|
50291
|
-
const keyDigest = (0,
|
|
49431
|
+
const responseKey = suppliedKey?.trim() || (0, import_node_crypto32.randomUUID)();
|
|
49432
|
+
const requestFingerprint3 = (0, import_node_crypto32.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
|
|
49433
|
+
const keyDigest = (0, import_node_crypto32.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
50292
49434
|
return {
|
|
50293
49435
|
responseKey,
|
|
50294
49436
|
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
50295
49437
|
debitDescription: `${body.query} [request:${requestFingerprint3}]`
|
|
50296
49438
|
};
|
|
50297
49439
|
}
|
|
50298
|
-
var import_hono22,
|
|
49440
|
+
var import_hono22, import_node_crypto32, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
|
|
50299
49441
|
var init_serp_intelligence_routes = __esm({
|
|
50300
49442
|
"src/api/serp-intelligence-routes.ts"() {
|
|
50301
49443
|
"use strict";
|
|
50302
49444
|
import_hono22 = require("hono");
|
|
50303
|
-
|
|
49445
|
+
import_node_crypto32 = require("crypto");
|
|
50304
49446
|
init_browser_service_env();
|
|
50305
49447
|
init_page_snapshot_extractor();
|
|
50306
49448
|
init_serp_capture_service();
|
|
@@ -50537,7 +49679,7 @@ var PACKAGE_VERSION;
|
|
|
50537
49679
|
var init_version = __esm({
|
|
50538
49680
|
"src/version.ts"() {
|
|
50539
49681
|
"use strict";
|
|
50540
|
-
PACKAGE_VERSION = "0.66.
|
|
49682
|
+
PACKAGE_VERSION = "0.66.2";
|
|
50541
49683
|
}
|
|
50542
49684
|
});
|
|
50543
49685
|
|
|
@@ -50967,64 +50109,14 @@ var init_server_instructions = __esm({
|
|
|
50967
50109
|
});
|
|
50968
50110
|
|
|
50969
50111
|
// src/mcp/output-schema-registry.ts
|
|
50970
|
-
function resolveOutputSchemaMode(env = process.env) {
|
|
50971
|
-
const explicit = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
|
|
50972
|
-
if (explicit === "none" || explicit === "essential" || explicit === "all") return explicit;
|
|
50973
|
-
return env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true" ? "all" : "none";
|
|
50974
|
-
}
|
|
50975
50112
|
function recordOutputSchema(name, schema) {
|
|
50976
50113
|
OUTPUT_SCHEMAS[name] = schema;
|
|
50977
|
-
return
|
|
50114
|
+
return schema;
|
|
50978
50115
|
}
|
|
50979
|
-
var
|
|
50116
|
+
var OUTPUT_SCHEMAS;
|
|
50980
50117
|
var init_output_schema_registry = __esm({
|
|
50981
50118
|
"src/mcp/output-schema-registry.ts"() {
|
|
50982
50119
|
"use strict";
|
|
50983
|
-
ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
|
|
50984
|
-
"extract_url",
|
|
50985
|
-
"extract_site",
|
|
50986
|
-
"analyze_site_similarity",
|
|
50987
|
-
"audit_site",
|
|
50988
|
-
"check_site_export",
|
|
50989
|
-
"site_export_read",
|
|
50990
|
-
"site_export_image",
|
|
50991
|
-
"archive_read",
|
|
50992
|
-
"report_artifact_read",
|
|
50993
|
-
"directory_workflow",
|
|
50994
|
-
"directory_workflow_status",
|
|
50995
|
-
"lead_list_upload_start",
|
|
50996
|
-
"lead_list_import",
|
|
50997
|
-
"lead_list_enrich",
|
|
50998
|
-
"lead_list_enrich_status",
|
|
50999
|
-
"video_frame_analysis",
|
|
51000
|
-
"video_frame_analysis_status",
|
|
51001
|
-
"workflow_run",
|
|
51002
|
-
"workflow_step",
|
|
51003
|
-
"workflow_status",
|
|
51004
|
-
"workflow_artifact_read",
|
|
51005
|
-
"export_connected_service_data",
|
|
51006
|
-
"export_search_console_table_data",
|
|
51007
|
-
"renew_connected_data_download",
|
|
51008
|
-
"browser_open",
|
|
51009
|
-
"browser_screenshot",
|
|
51010
|
-
"browser_replay_start",
|
|
51011
|
-
"browser_replay_stop",
|
|
51012
|
-
"browser_replay_download",
|
|
51013
|
-
"create_scheduled_run_view_link",
|
|
51014
|
-
"get_scheduled_run",
|
|
51015
|
-
"library-ingest",
|
|
51016
|
-
"image_project_create",
|
|
51017
|
-
"image_project_list",
|
|
51018
|
-
"image_folder_create",
|
|
51019
|
-
"image_folder_list",
|
|
51020
|
-
"image_asset_save",
|
|
51021
|
-
"image_asset_get",
|
|
51022
|
-
"image_asset_list",
|
|
51023
|
-
"image_asset_search",
|
|
51024
|
-
"image_asset_move",
|
|
51025
|
-
"image_asset_delete"
|
|
51026
|
-
]);
|
|
51027
|
-
OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
|
|
51028
50120
|
OUTPUT_SCHEMAS = {};
|
|
51029
50121
|
}
|
|
51030
50122
|
});
|
|
@@ -52381,7 +51473,7 @@ var init_contracts = __esm({
|
|
|
52381
51473
|
});
|
|
52382
51474
|
|
|
52383
51475
|
// src/mcp/mcp-tool-schemas.ts
|
|
52384
|
-
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
51476
|
+
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, HarvestPaaHostedInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
52385
51477
|
var init_mcp_tool_schemas = __esm({
|
|
52386
51478
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
52387
51479
|
"use strict";
|
|
@@ -52429,7 +51521,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52429
51521
|
gl: import_zod45.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
52430
51522
|
hl: import_zod45.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
52431
51523
|
device: import_zod45.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
|
|
52432
|
-
idempotencyKey: import_zod45.z.string().trim().min(8).max(200).optional().describe("
|
|
51524
|
+
idempotencyKey: import_zod45.z.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
|
|
52433
51525
|
serpIdentity: import_zod45.z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
|
|
52434
51526
|
includeAllSerpFeatures: import_zod45.z.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
|
|
52435
51527
|
includeLocalPack: import_zod45.z.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
|
|
@@ -52439,6 +51531,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52439
51531
|
includeWhatPeopleSaying: import_zod45.z.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
|
|
52440
51532
|
debug: import_zod45.z.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
|
|
52441
51533
|
};
|
|
51534
|
+
HarvestPaaHostedInputSchema = Object.fromEntries(
|
|
51535
|
+
Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
|
|
51536
|
+
);
|
|
52442
51537
|
ExtractUrlBaseInputSchema = {
|
|
52443
51538
|
url: import_zod45.z.string().url().describe("Public http/https URL to extract."),
|
|
52444
51539
|
screenshot: import_zod45.z.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
|
|
@@ -55691,11 +54786,11 @@ function requireTasksCapability(capabilityValue) {
|
|
|
55691
54786
|
);
|
|
55692
54787
|
}
|
|
55693
54788
|
function taskKey(secret2) {
|
|
55694
|
-
return (0,
|
|
54789
|
+
return (0, import_node_crypto33.createHash)("sha256").update("mcp-scraper-task-handle\0", "utf8").update(secret2, "utf8").digest();
|
|
55695
54790
|
}
|
|
55696
54791
|
function encodeTaskHandle(payload, secret2) {
|
|
55697
|
-
const nonce = (0,
|
|
55698
|
-
const cipher = (0,
|
|
54792
|
+
const nonce = (0, import_node_crypto33.randomBytes)(12);
|
|
54793
|
+
const cipher = (0, import_node_crypto33.createCipheriv)("aes-256-gcm", taskKey(secret2), nonce);
|
|
55699
54794
|
const ciphertext = Buffer.concat([
|
|
55700
54795
|
cipher.update(JSON.stringify(payload), "utf8"),
|
|
55701
54796
|
cipher.final()
|
|
@@ -55712,7 +54807,7 @@ function decodeTaskHandle(taskId, secret2, ownerId2) {
|
|
|
55712
54807
|
const nonce = bytes.subarray(0, 12);
|
|
55713
54808
|
const tag = bytes.subarray(bytes.length - 16);
|
|
55714
54809
|
const ciphertext = bytes.subarray(12, bytes.length - 16);
|
|
55715
|
-
const decipher = (0,
|
|
54810
|
+
const decipher = (0, import_node_crypto33.createDecipheriv)("aes-256-gcm", taskKey(secret2), nonce);
|
|
55716
54811
|
decipher.setAuthTag(tag);
|
|
55717
54812
|
const parsed = JSON.parse(Buffer.concat([
|
|
55718
54813
|
decipher.update(ciphertext),
|
|
@@ -55770,6 +54865,73 @@ function taskSeed(kind, result) {
|
|
|
55770
54865
|
if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
|
|
55771
54866
|
return { kind, operationId, statusMessage: "Video analysis is running." };
|
|
55772
54867
|
}
|
|
54868
|
+
function createTaskResult(kind, result, options) {
|
|
54869
|
+
const seed = taskSeed(kind, result);
|
|
54870
|
+
if (!seed) return result;
|
|
54871
|
+
const createdAtMs2 = Date.now();
|
|
54872
|
+
const payload = {
|
|
54873
|
+
v: 1,
|
|
54874
|
+
ownerId: options.ownerId,
|
|
54875
|
+
kind: seed.kind,
|
|
54876
|
+
operationId: seed.operationId,
|
|
54877
|
+
createdAtMs: createdAtMs2,
|
|
54878
|
+
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
54879
|
+
};
|
|
54880
|
+
return {
|
|
54881
|
+
resultType: "task",
|
|
54882
|
+
taskId: encodeTaskHandle(payload, options.secret),
|
|
54883
|
+
status: "working",
|
|
54884
|
+
statusMessage: seed.statusMessage,
|
|
54885
|
+
createdAt: new Date(createdAtMs2).toISOString(),
|
|
54886
|
+
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
54887
|
+
ttlMs: TASK_TTL_MS,
|
|
54888
|
+
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
54889
|
+
// The registered SDK path validates structuredContent against the tool's
|
|
54890
|
+
// terminal output schema before it can serialize a Task envelope. Retain
|
|
54891
|
+
// the bounded start receipt for that path; the hosted HTTP Task seam runs
|
|
54892
|
+
// before terminal-output serialization.
|
|
54893
|
+
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
54894
|
+
};
|
|
54895
|
+
}
|
|
54896
|
+
async function startTaskTool(executor, toolName, rawArguments) {
|
|
54897
|
+
const args = objectValue(rawArguments) ?? {};
|
|
54898
|
+
if (toolName === "harvest_paa") {
|
|
54899
|
+
const parsed = import_zod46.z.object(HarvestPaaHostedInputSchema).strict().safeParse(args);
|
|
54900
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid harvest_paa arguments");
|
|
54901
|
+
return { kind: "paa_harvest", result: await executor.startHarvestPaa({ ...parsed.data, debug: false }) };
|
|
54902
|
+
}
|
|
54903
|
+
if (toolName === "extract_site") {
|
|
54904
|
+
const parsed = import_zod46.z.object(ExtractSiteInputSchema).safeParse(args);
|
|
54905
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid extract_site arguments");
|
|
54906
|
+
return { kind: "site_export", result: await executor.extractSite(parsed.data) };
|
|
54907
|
+
}
|
|
54908
|
+
if (toolName === "analyze_site_similarity") {
|
|
54909
|
+
const parsed = import_zod46.z.object(AnalyzeSiteSimilarityInputSchema).safeParse(args);
|
|
54910
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid analyze_site_similarity arguments");
|
|
54911
|
+
return { kind: "site_export", result: await executor.analyzeSiteSimilarity(parsed.data) };
|
|
54912
|
+
}
|
|
54913
|
+
if (toolName === "audit_site") {
|
|
54914
|
+
const parsed = import_zod46.z.object(AuditSiteInputSchema).safeParse(args);
|
|
54915
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid audit_site arguments");
|
|
54916
|
+
return { kind: "site_export", result: await executor.auditSite(parsed.data) };
|
|
54917
|
+
}
|
|
54918
|
+
if (toolName === "video_frame_analysis") {
|
|
54919
|
+
const parsed = import_zod46.z.object(VideoFrameAnalysisInputSchema).safeParse(args);
|
|
54920
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid video_frame_analysis arguments");
|
|
54921
|
+
return { kind: "video_frame_analysis", result: await executor.videoFrameAnalysis(parsed.data) };
|
|
54922
|
+
}
|
|
54923
|
+
if (toolName === "directory_workflow") {
|
|
54924
|
+
const parsed = import_zod46.z.object(DirectoryWorkflowInputSchema).safeParse(args);
|
|
54925
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid directory_workflow arguments");
|
|
54926
|
+
return { kind: "directory_workflow", result: await executor.directoryWorkflow(parsed.data) };
|
|
54927
|
+
}
|
|
54928
|
+
if (toolName === "lead_list_enrich") {
|
|
54929
|
+
const parsed = import_zod46.z.object(LeadListEnrichInputSchema).safeParse(args);
|
|
54930
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid lead_list_enrich arguments");
|
|
54931
|
+
return { kind: "lead_list_enrichment", result: await executor.leadListEnrich(parsed.data) };
|
|
54932
|
+
}
|
|
54933
|
+
throw new import_server.ProtocolError(TASK_INVALID_PARAMS, `Unsupported Task-start tool: ${toolName}`);
|
|
54934
|
+
}
|
|
55773
54935
|
function taskBase(payload, taskId) {
|
|
55774
54936
|
return {
|
|
55775
54937
|
taskId,
|
|
@@ -55888,6 +55050,13 @@ function isWorking(kind, status) {
|
|
|
55888
55050
|
return status === "queued" || status === "running" || status === "processing" || status === "pending";
|
|
55889
55051
|
}
|
|
55890
55052
|
function registerMcpTasksExtension(server, executor, options) {
|
|
55053
|
+
const enabled = options.enableTaskStarts !== false;
|
|
55054
|
+
if (!enabled) {
|
|
55055
|
+
return {
|
|
55056
|
+
supportsTasks: () => false,
|
|
55057
|
+
taskify: (_kind, result) => result
|
|
55058
|
+
};
|
|
55059
|
+
}
|
|
55891
55060
|
server.server.registerCapabilities({
|
|
55892
55061
|
extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
|
|
55893
55062
|
});
|
|
@@ -55918,32 +55087,7 @@ function registerMcpTasksExtension(server, executor, options) {
|
|
|
55918
55087
|
},
|
|
55919
55088
|
taskify(kind, result) {
|
|
55920
55089
|
if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
|
|
55921
|
-
|
|
55922
|
-
if (!seed) return result;
|
|
55923
|
-
const createdAtMs2 = Date.now();
|
|
55924
|
-
const payload = {
|
|
55925
|
-
v: 1,
|
|
55926
|
-
ownerId: options.ownerId,
|
|
55927
|
-
kind: seed.kind,
|
|
55928
|
-
operationId: seed.operationId,
|
|
55929
|
-
createdAtMs: createdAtMs2,
|
|
55930
|
-
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
55931
|
-
};
|
|
55932
|
-
const taskId = encodeTaskHandle(payload, options.secret);
|
|
55933
|
-
return {
|
|
55934
|
-
resultType: "task",
|
|
55935
|
-
taskId,
|
|
55936
|
-
status: "working",
|
|
55937
|
-
statusMessage: seed.statusMessage,
|
|
55938
|
-
createdAt: new Date(createdAtMs2).toISOString(),
|
|
55939
|
-
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
55940
|
-
ttlMs: TASK_TTL_MS,
|
|
55941
|
-
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
55942
|
-
// The SDK validates advertised outputSchema before serializing the
|
|
55943
|
-
// Task envelope. Preserve the already-bounded start receipt so typed
|
|
55944
|
-
// tools remain task-capable when essential output schemas are on.
|
|
55945
|
-
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
55946
|
-
};
|
|
55090
|
+
return createTaskResult(kind, result, options);
|
|
55947
55091
|
}
|
|
55948
55092
|
};
|
|
55949
55093
|
}
|
|
@@ -55956,10 +55100,12 @@ function taskHttpError(id, error) {
|
|
|
55956
55100
|
}
|
|
55957
55101
|
async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
55958
55102
|
const routedMethod = request.headers.get("mcp-method");
|
|
55959
|
-
|
|
55103
|
+
const routedToolName = request.headers.get("mcp-name");
|
|
55104
|
+
const taskStartCall = routedMethod === "tools/call" && TASK_START_TOOL_NAMES.has(routedToolName ?? "");
|
|
55105
|
+
if (!taskStartCall && !["tasks/get", "tasks/update", "tasks/cancel"].includes(routedMethod ?? "")) return void 0;
|
|
55960
55106
|
let payload;
|
|
55961
55107
|
try {
|
|
55962
|
-
payload = await request.json();
|
|
55108
|
+
payload = await request.clone().json();
|
|
55963
55109
|
} catch {
|
|
55964
55110
|
return taskHttpError(null, new import_server.ProtocolError(-32700, "Parse error"));
|
|
55965
55111
|
}
|
|
@@ -55972,6 +55118,28 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
|
55972
55118
|
throw new import_server.ProtocolError(-32020, "Mcp-Method header does not match the JSON-RPC method");
|
|
55973
55119
|
}
|
|
55974
55120
|
const params = objectValue(payload.params);
|
|
55121
|
+
if (taskStartCall) {
|
|
55122
|
+
if (params?.name !== routedToolName || typeof routedToolName !== "string") {
|
|
55123
|
+
throw new import_server.ProtocolError(-32020, "Mcp-Name header must match params.name");
|
|
55124
|
+
}
|
|
55125
|
+
const meta2 = objectValue(params?._meta);
|
|
55126
|
+
if (!hasTasksCapability(meta2?.["io.modelcontextprotocol/clientCapabilities"])) return void 0;
|
|
55127
|
+
if (routedToolName === "harvest_paa" && objectValue(params?.arguments)?.serpIdentity !== void 0) {
|
|
55128
|
+
return void 0;
|
|
55129
|
+
}
|
|
55130
|
+
const started = await startTaskTool(executor, routedToolName, params?.arguments);
|
|
55131
|
+
const task = createTaskResult(started.kind, started.result, options);
|
|
55132
|
+
const taskObject = task;
|
|
55133
|
+
return new Response(JSON.stringify({
|
|
55134
|
+
jsonrpc: "2.0",
|
|
55135
|
+
id,
|
|
55136
|
+
result: {
|
|
55137
|
+
...taskObject,
|
|
55138
|
+
resultType: taskObject.resultType === "task" ? "task" : "complete",
|
|
55139
|
+
_meta: { "io.modelcontextprotocol/serverInfo": options.serverInfo }
|
|
55140
|
+
}
|
|
55141
|
+
}), { status: 200, headers: { "content-type": "application/json" } });
|
|
55142
|
+
}
|
|
55975
55143
|
const taskId = params?.taskId;
|
|
55976
55144
|
if (typeof taskId !== "string" || !taskId) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, "Invalid taskId");
|
|
55977
55145
|
if (request.headers.get("mcp-name") !== taskId) {
|
|
@@ -56003,19 +55171,29 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
|
56003
55171
|
return taskHttpError(id, error instanceof import_server.ProtocolError ? error : new import_server.ProtocolError(-32603, error instanceof Error ? error.message : "Internal error"));
|
|
56004
55172
|
}
|
|
56005
55173
|
}
|
|
56006
|
-
var import_server,
|
|
55174
|
+
var import_server, import_node_crypto33, import_zod46, MCP_TASKS_EXTENSION_ID, TASK_HANDLE_VERSION, TASK_TTL_MS, TASK_POLL_INTERVAL_MS, TASK_INVALID_PARAMS, TASK_MISSING_CAPABILITY, TASK_START_TOOL_NAMES;
|
|
56007
55175
|
var init_mcp_tasks_extension = __esm({
|
|
56008
55176
|
"src/mcp/mcp-tasks-extension.ts"() {
|
|
56009
55177
|
"use strict";
|
|
56010
55178
|
import_server = require("@modelcontextprotocol/server");
|
|
56011
|
-
|
|
55179
|
+
import_node_crypto33 = require("crypto");
|
|
56012
55180
|
import_zod46 = require("zod");
|
|
55181
|
+
init_mcp_tool_schemas();
|
|
56013
55182
|
MCP_TASKS_EXTENSION_ID = "io.modelcontextprotocol/tasks";
|
|
56014
55183
|
TASK_HANDLE_VERSION = "mt1";
|
|
56015
55184
|
TASK_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
56016
55185
|
TASK_POLL_INTERVAL_MS = 5e3;
|
|
56017
55186
|
TASK_INVALID_PARAMS = -32602;
|
|
56018
55187
|
TASK_MISSING_CAPABILITY = -32003;
|
|
55188
|
+
TASK_START_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
55189
|
+
"harvest_paa",
|
|
55190
|
+
"extract_site",
|
|
55191
|
+
"analyze_site_similarity",
|
|
55192
|
+
"audit_site",
|
|
55193
|
+
"video_frame_analysis",
|
|
55194
|
+
"directory_workflow",
|
|
55195
|
+
"lead_list_enrich"
|
|
55196
|
+
]);
|
|
56019
55197
|
}
|
|
56020
55198
|
});
|
|
56021
55199
|
|
|
@@ -56306,7 +55484,7 @@ var init_analytics_mcp_tools = __esm({
|
|
|
56306
55484
|
|
|
56307
55485
|
// src/mcp/paa-mcp-server.ts
|
|
56308
55486
|
function hashOwnerId(callerKey) {
|
|
56309
|
-
return (0,
|
|
55487
|
+
return (0, import_node_crypto34.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
56310
55488
|
}
|
|
56311
55489
|
function liveWebToolAnnotations(title) {
|
|
56312
55490
|
return {
|
|
@@ -56446,19 +55624,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
56446
55624
|
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
56447
55625
|
const ownerId2 = options.ownerId ?? "local";
|
|
56448
55626
|
const ctx = { hosted: !savesReports, ownerId: ownerId2 };
|
|
55627
|
+
const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
|
|
56449
55628
|
registerAnalyticsMcpTools(server, executor);
|
|
56450
55629
|
const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
|
|
56451
55630
|
deploymentProfile: options.deploymentProfile ?? "production",
|
|
56452
|
-
transportProfile
|
|
55631
|
+
transportProfile,
|
|
56453
55632
|
baseUrl: options.baseUrl
|
|
56454
55633
|
});
|
|
56455
55634
|
const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
|
|
56456
55635
|
const tasks = registerMcpTasksExtension(server, executor, {
|
|
56457
55636
|
ownerId: ownerId2,
|
|
56458
|
-
secret: options.taskHandleSecret ?? `local:${ownerId2}
|
|
55637
|
+
secret: options.taskHandleSecret ?? `local:${ownerId2}`,
|
|
55638
|
+
enableTaskStarts: transportProfile === "hosted_http"
|
|
56459
55639
|
});
|
|
56460
55640
|
const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
|
|
56461
|
-
const harvestPaaInputSchema =
|
|
55641
|
+
const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
|
|
56462
55642
|
const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
|
|
56463
55643
|
const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
|
|
56464
55644
|
const directoryWorkflowInputSchema = schemaWithoutFields(
|
|
@@ -56473,10 +55653,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
56473
55653
|
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
56474
55654
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
56475
55655
|
}, async (input, requestContext) => {
|
|
56476
|
-
|
|
56477
|
-
|
|
55656
|
+
const executionInput = { ...input, debug: input.debug === true };
|
|
55657
|
+
if (tasks.supportsTasks() && !executionInput.serpIdentity) {
|
|
55658
|
+
return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
|
|
56478
55659
|
}
|
|
56479
|
-
return formatHarvestPaa(await executor.harvestPaa(
|
|
55660
|
+
return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
|
|
56480
55661
|
});
|
|
56481
55662
|
server.registerTool("search_serp", {
|
|
56482
55663
|
title: "Google SERP Lookup",
|
|
@@ -57291,7 +56472,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
57291
56472
|
annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
57292
56473
|
}, async (input) => executor.setScheduledActionConnections(input));
|
|
57293
56474
|
}
|
|
57294
|
-
var import_server2, import_zod48, import_node_fs11, import_node_path17,
|
|
56475
|
+
var import_server2, import_zod48, import_node_fs11, import_node_path17, import_node_crypto34, ACTION_CONFIRMATION_SCHEMA;
|
|
57295
56476
|
var init_paa_mcp_server = __esm({
|
|
57296
56477
|
"src/mcp/paa-mcp-server.ts"() {
|
|
57297
56478
|
"use strict";
|
|
@@ -57299,7 +56480,7 @@ var init_paa_mcp_server = __esm({
|
|
|
57299
56480
|
import_zod48 = require("zod");
|
|
57300
56481
|
import_node_fs11 = require("fs");
|
|
57301
56482
|
import_node_path17 = require("path");
|
|
57302
|
-
|
|
56483
|
+
import_node_crypto34 = require("crypto");
|
|
57303
56484
|
init_version();
|
|
57304
56485
|
init_rates();
|
|
57305
56486
|
init_mcp_response_formatter();
|
|
@@ -57420,11 +56601,11 @@ function analyticsReportPath(input, report) {
|
|
|
57420
56601
|
const suffix2 = query.size ? `?${query.toString()}` : "";
|
|
57421
56602
|
return `/analytics/sites/${encodeURIComponent(input.siteId)}/${report}${suffix2}`;
|
|
57422
56603
|
}
|
|
57423
|
-
var
|
|
56604
|
+
var import_node_crypto35, HttpMcpToolExecutor;
|
|
57424
56605
|
var init_http_mcp_tool_executor = __esm({
|
|
57425
56606
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
57426
56607
|
"use strict";
|
|
57427
|
-
|
|
56608
|
+
import_node_crypto35 = require("crypto");
|
|
57428
56609
|
init_harvest_timeout();
|
|
57429
56610
|
init_browser_service_env();
|
|
57430
56611
|
init_errors();
|
|
@@ -57493,13 +56674,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57493
56674
|
});
|
|
57494
56675
|
}
|
|
57495
56676
|
async callDirectoryWorkflowStart(body, explicitIdempotencyKey) {
|
|
57496
|
-
const idempotencyKey4 = `mcp-directory-${(0,
|
|
56677
|
+
const idempotencyKey4 = `mcp-directory-${(0, import_node_crypto35.createHash)("sha256").update(explicitIdempotencyKey).digest("hex")}`;
|
|
57497
56678
|
return this.call("/directory/run", body, this.timeoutMs, "POST", {
|
|
57498
56679
|
"Idempotency-Key": idempotencyKey4
|
|
57499
56680
|
});
|
|
57500
56681
|
}
|
|
57501
56682
|
async callSiteExtractStart(toolName, body, explicitIdempotencyKey) {
|
|
57502
|
-
const idempotencyKey4 = `mcp-site-${(0,
|
|
56683
|
+
const idempotencyKey4 = `mcp-site-${(0, import_node_crypto35.createHash)("sha256").update(toolName).update("\0").update(explicitIdempotencyKey).digest("hex")}`;
|
|
57503
56684
|
return this.call("/extract-site", body, this.timeoutMs, "POST", {
|
|
57504
56685
|
"Idempotency-Key": idempotencyKey4
|
|
57505
56686
|
});
|
|
@@ -57561,12 +56742,14 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57561
56742
|
}
|
|
57562
56743
|
harvestPaa(input) {
|
|
57563
56744
|
const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
|
|
57564
|
-
const
|
|
57565
|
-
|
|
56745
|
+
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
56746
|
+
const headers = idempotencyKey4 ? { "Idempotency-Key": idempotencyKey4 } : {};
|
|
56747
|
+
return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
|
|
57566
56748
|
}
|
|
57567
56749
|
startHarvestPaa(input) {
|
|
57568
|
-
const
|
|
57569
|
-
|
|
56750
|
+
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
56751
|
+
const headers = idempotencyKey4 ? { "Idempotency-Key": idempotencyKey4 } : {};
|
|
56752
|
+
return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
|
|
57570
56753
|
}
|
|
57571
56754
|
harvestPaaStatus(input) {
|
|
57572
56755
|
return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);
|
|
@@ -57941,7 +57124,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57941
57124
|
report: input.report,
|
|
57942
57125
|
format: input.format
|
|
57943
57126
|
}, this.timeoutMs, "POST", {
|
|
57944
|
-
"Idempotency-Key": `analytics-export-${(0,
|
|
57127
|
+
"Idempotency-Key": `analytics-export-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57945
57128
|
});
|
|
57946
57129
|
}
|
|
57947
57130
|
commonsSearchEntities(input) {
|
|
@@ -57980,7 +57163,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57980
57163
|
commonsSubmitEntity(input) {
|
|
57981
57164
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
57982
57165
|
return this.call("/commons/entities/propose", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
57983
|
-
"Idempotency-Key": `commons-${(0,
|
|
57166
|
+
"Idempotency-Key": `commons-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57984
57167
|
});
|
|
57985
57168
|
}
|
|
57986
57169
|
commonsGetEntityLedger(input) {
|
|
@@ -57995,7 +57178,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57995
57178
|
commonsUpdateEditorialArticle(input) {
|
|
57996
57179
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
57997
57180
|
return this.call("/commons/publications/articles", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
57998
|
-
"Idempotency-Key": `commons-article-${(0,
|
|
57181
|
+
"Idempotency-Key": `commons-article-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57999
57182
|
});
|
|
58000
57183
|
}
|
|
58001
57184
|
commonsSaveFilter(input) {
|
|
@@ -58016,13 +57199,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
58016
57199
|
commonsClaimPublication(input) {
|
|
58017
57200
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
58018
57201
|
return this.call("/commons/publications/claim", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
58019
|
-
"Idempotency-Key": `commons-publication-claim-${(0,
|
|
57202
|
+
"Idempotency-Key": `commons-publication-claim-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
58020
57203
|
});
|
|
58021
57204
|
}
|
|
58022
57205
|
commonsPublishEditorial(input) {
|
|
58023
57206
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
58024
57207
|
return this.call("/commons/publications/publish", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
58025
|
-
"Idempotency-Key": `commons-publication-publish-${(0,
|
|
57208
|
+
"Idempotency-Key": `commons-publication-publish-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
58026
57209
|
});
|
|
58027
57210
|
}
|
|
58028
57211
|
commonsGetPublication(input) {
|
|
@@ -58030,13 +57213,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
58030
57213
|
return this.getJson(input.subdomain ? `/commons/publications/${encodeURIComponent(input.subdomain)}?${query}` : `/commons/publications/me?${query}`);
|
|
58031
57214
|
}
|
|
58032
57215
|
async captureSerpSnapshot(input) {
|
|
58033
|
-
const fingerprint2 = (0,
|
|
57216
|
+
const fingerprint2 = (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
58034
57217
|
const now = Date.now();
|
|
58035
57218
|
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
58036
57219
|
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
58037
57220
|
}
|
|
58038
57221
|
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint2);
|
|
58039
|
-
const idempotencyKey4 = pending && pending.expiresAt > now ? pending.key : (0,
|
|
57222
|
+
const idempotencyKey4 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto35.randomUUID)();
|
|
58040
57223
|
this.pendingSerpCaptureBillingKeys.set(fingerprint2, {
|
|
58041
57224
|
key: idempotencyKey4,
|
|
58042
57225
|
expiresAt: now + 15 * 6e4
|
|
@@ -64046,7 +63229,7 @@ async function createImageSourceArtifact(args) {
|
|
|
64046
63229
|
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
64047
63230
|
throw new Error("image_source_size_invalid");
|
|
64048
63231
|
}
|
|
64049
|
-
const id = (0,
|
|
63232
|
+
const id = (0, import_node_crypto36.randomUUID)().replaceAll("-", "");
|
|
64050
63233
|
return createPrivateArtifact({
|
|
64051
63234
|
policy: policy4(),
|
|
64052
63235
|
ownerId: args.ownerId,
|
|
@@ -64065,11 +63248,11 @@ async function readOwnedImageSourceArtifact(args) {
|
|
|
64065
63248
|
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
64066
63249
|
});
|
|
64067
63250
|
}
|
|
64068
|
-
var
|
|
63251
|
+
var import_node_crypto36, IMAGE_SOURCE_ARTIFACT_PREFIX, IMAGE_SOURCE_ARTIFACT_TTL_MS, IMAGE_SOURCE_DOWNLOAD_TTL_MS, IMAGE_SOURCE_MAX_BYTES, ALLOWED_IMAGE_TYPES;
|
|
64069
63252
|
var init_image_source_artifacts = __esm({
|
|
64070
63253
|
"src/api/image-source-artifacts.ts"() {
|
|
64071
63254
|
"use strict";
|
|
64072
|
-
|
|
63255
|
+
import_node_crypto36 = require("crypto");
|
|
64073
63256
|
init_private_artifacts();
|
|
64074
63257
|
IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
64075
63258
|
IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
@@ -65683,7 +64866,7 @@ async function runBrowserAgentMigration() {
|
|
|
65683
64866
|
}
|
|
65684
64867
|
async function createExtensionRow(input) {
|
|
65685
64868
|
const db = getDb();
|
|
65686
|
-
const id = `bext_${(0,
|
|
64869
|
+
const id = `bext_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65687
64870
|
await db.execute({
|
|
65688
64871
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
65689
64872
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -65718,7 +64901,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
65718
64901
|
}
|
|
65719
64902
|
async function createAuthConnectionRow(input) {
|
|
65720
64903
|
const db = getDb();
|
|
65721
|
-
const connectionId = `authc_${(0,
|
|
64904
|
+
const connectionId = `authc_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65722
64905
|
await db.execute({
|
|
65723
64906
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
65724
64907
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -65803,7 +64986,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
65803
64986
|
}
|
|
65804
64987
|
async function createSessionRow(input) {
|
|
65805
64988
|
const db = getDb();
|
|
65806
|
-
const id = `bas_${(0,
|
|
64989
|
+
const id = `bas_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65807
64990
|
await db.execute({
|
|
65808
64991
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
65809
64992
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -65880,7 +65063,7 @@ async function recordAction(input) {
|
|
|
65880
65063
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
65881
65064
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
65882
65065
|
args: [
|
|
65883
|
-
`baa_${(0,
|
|
65066
|
+
`baa_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
65884
65067
|
input.sessionId,
|
|
65885
65068
|
input.type,
|
|
65886
65069
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -65920,11 +65103,11 @@ async function listReplayRows(sessionId) {
|
|
|
65920
65103
|
});
|
|
65921
65104
|
return res.rows;
|
|
65922
65105
|
}
|
|
65923
|
-
var
|
|
65106
|
+
var import_node_crypto37, _ready2, _migrationPromise2, ORPHANED_SESSION_STATUS;
|
|
65924
65107
|
var init_browser_agent_db = __esm({
|
|
65925
65108
|
"src/api/browser-agent-db.ts"() {
|
|
65926
65109
|
"use strict";
|
|
65927
|
-
|
|
65110
|
+
import_node_crypto37 = require("crypto");
|
|
65928
65111
|
init_db();
|
|
65929
65112
|
_ready2 = false;
|
|
65930
65113
|
_migrationPromise2 = null;
|
|
@@ -66693,7 +65876,7 @@ async function currentOrRuntimeContext(browser) {
|
|
|
66693
65876
|
function client() {
|
|
66694
65877
|
const apiKey = browserServiceApiKey();
|
|
66695
65878
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
66696
|
-
return new
|
|
65879
|
+
return new import_sdk9.default({ apiKey });
|
|
66697
65880
|
}
|
|
66698
65881
|
function isProfileConflict(err) {
|
|
66699
65882
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -67051,11 +66234,11 @@ async function replayList(runtimeSessionId) {
|
|
|
67051
66234
|
finishedAt: r.finished_at ?? null
|
|
67052
66235
|
}));
|
|
67053
66236
|
}
|
|
67054
|
-
var
|
|
66237
|
+
var import_sdk9, import_playwright6, DEFAULT_TIMEOUT_SECONDS;
|
|
67055
66238
|
var init_browser_agent_service = __esm({
|
|
67056
66239
|
"src/services/browser-agent/browser-agent-service.ts"() {
|
|
67057
66240
|
"use strict";
|
|
67058
|
-
|
|
66241
|
+
import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
|
|
67059
66242
|
import_playwright6 = require("playwright");
|
|
67060
66243
|
init_browser_service_env();
|
|
67061
66244
|
init_run_capture();
|
|
@@ -67064,15 +66247,71 @@ var init_browser_agent_service = __esm({
|
|
|
67064
66247
|
}
|
|
67065
66248
|
});
|
|
67066
66249
|
|
|
66250
|
+
// src/api/serp-identity-db.ts
|
|
66251
|
+
async function createSerpIdentityRow(input) {
|
|
66252
|
+
const id = `serpi_${(0, import_node_crypto38.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
66253
|
+
await getDb().execute({
|
|
66254
|
+
sql: `INSERT INTO serp_identities
|
|
66255
|
+
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
66256
|
+
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
66257
|
+
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
66258
|
+
});
|
|
66259
|
+
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
66260
|
+
if (!row) throw new Error("SERP identity insert failed");
|
|
66261
|
+
return row;
|
|
66262
|
+
}
|
|
66263
|
+
async function getSerpIdentityRow(userId, name) {
|
|
66264
|
+
const result = await getDb().execute({
|
|
66265
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
66266
|
+
args: [userId, name]
|
|
66267
|
+
});
|
|
66268
|
+
return result.rows[0] ?? null;
|
|
66269
|
+
}
|
|
66270
|
+
async function listSerpIdentityRows(userId) {
|
|
66271
|
+
const result = await getDb().execute({
|
|
66272
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
66273
|
+
args: [userId]
|
|
66274
|
+
});
|
|
66275
|
+
return result.rows;
|
|
66276
|
+
}
|
|
66277
|
+
async function touchSerpIdentity(userId, name) {
|
|
66278
|
+
await getDb().execute({
|
|
66279
|
+
sql: `UPDATE serp_identities
|
|
66280
|
+
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
66281
|
+
WHERE user_id = ? AND name = ?`,
|
|
66282
|
+
args: [userId, name]
|
|
66283
|
+
});
|
|
66284
|
+
}
|
|
66285
|
+
async function setSerpIdentityStatus(userId, name, status) {
|
|
66286
|
+
await getDb().execute({
|
|
66287
|
+
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
66288
|
+
args: [status, userId, name]
|
|
66289
|
+
});
|
|
66290
|
+
}
|
|
66291
|
+
async function deleteSerpIdentityRow(userId, name) {
|
|
66292
|
+
await getDb().execute({
|
|
66293
|
+
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
66294
|
+
args: [userId, name]
|
|
66295
|
+
});
|
|
66296
|
+
}
|
|
66297
|
+
var import_node_crypto38;
|
|
66298
|
+
var init_serp_identity_db = __esm({
|
|
66299
|
+
"src/api/serp-identity-db.ts"() {
|
|
66300
|
+
"use strict";
|
|
66301
|
+
import_node_crypto38 = require("crypto");
|
|
66302
|
+
init_db();
|
|
66303
|
+
}
|
|
66304
|
+
});
|
|
66305
|
+
|
|
67067
66306
|
// src/api/serp-identity-service.ts
|
|
67068
66307
|
function kernelClient() {
|
|
67069
66308
|
const apiKey = browserServiceApiKey();
|
|
67070
66309
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
67071
|
-
return new
|
|
66310
|
+
return new import_sdk10.default({ apiKey });
|
|
67072
66311
|
}
|
|
67073
66312
|
function backendName(userId, name, resource) {
|
|
67074
|
-
const digest2 = (0,
|
|
67075
|
-
const nonce = (0,
|
|
66313
|
+
const digest2 = (0, import_node_crypto39.createHash)("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
|
|
66314
|
+
const nonce = (0, import_node_crypto39.randomUUID)().replace(/-/g, "").slice(0, 8);
|
|
67076
66315
|
return `mcp-serp-${resource}-${digest2}-${nonce}`;
|
|
67077
66316
|
}
|
|
67078
66317
|
function isNotFound2(error) {
|
|
@@ -67150,12 +66389,12 @@ async function deleteSerpIdentity(userId, name) {
|
|
|
67150
66389
|
throw error;
|
|
67151
66390
|
}
|
|
67152
66391
|
}
|
|
67153
|
-
var
|
|
66392
|
+
var import_node_crypto39, import_sdk10, MAX_SERP_IDENTITIES_PER_USER;
|
|
67154
66393
|
var init_serp_identity_service = __esm({
|
|
67155
66394
|
"src/api/serp-identity-service.ts"() {
|
|
67156
66395
|
"use strict";
|
|
67157
|
-
|
|
67158
|
-
|
|
66396
|
+
import_node_crypto39 = require("crypto");
|
|
66397
|
+
import_sdk10 = __toESM(require("@onkernel/sdk"), 1);
|
|
67159
66398
|
init_browser_service_env();
|
|
67160
66399
|
init_serp_identity_db();
|
|
67161
66400
|
MAX_SERP_IDENTITIES_PER_USER = 3;
|
|
@@ -67865,7 +67104,7 @@ function buildBrowserAgentRoutes() {
|
|
|
67865
67104
|
}
|
|
67866
67105
|
const existing = await getExtensionRow(user.id, name);
|
|
67867
67106
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
67868
|
-
const backendName2 = `u${user.id}_${(0,
|
|
67107
|
+
const backendName2 = `u${user.id}_${(0, import_node_crypto40.randomUUID)().replace(/-/g, "")}`;
|
|
67869
67108
|
try {
|
|
67870
67109
|
const imported = await importExtensionFromStore(storeUrl, backendName2);
|
|
67871
67110
|
const row = await createExtensionRow({
|
|
@@ -67924,11 +67163,11 @@ function buildBrowserAgentRoutes() {
|
|
|
67924
67163
|
});
|
|
67925
67164
|
return app2;
|
|
67926
67165
|
}
|
|
67927
|
-
var
|
|
67166
|
+
var import_node_crypto40, import_hono24, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE, SERP_IDENTITY_NAME_RE;
|
|
67928
67167
|
var init_browser_agent_routes = __esm({
|
|
67929
67168
|
"src/api/browser-agent-routes.ts"() {
|
|
67930
67169
|
"use strict";
|
|
67931
|
-
|
|
67170
|
+
import_node_crypto40 = require("crypto");
|
|
67932
67171
|
import_hono24 = require("hono");
|
|
67933
67172
|
init_api_auth();
|
|
67934
67173
|
init_errors();
|
|
@@ -68486,7 +67725,7 @@ async function getKeys() {
|
|
|
68486
67725
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
68487
67726
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
68488
67727
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
68489
|
-
const kid = (0,
|
|
67728
|
+
const kid = (0, import_node_crypto41.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
68490
67729
|
publicJwk.kid = kid;
|
|
68491
67730
|
publicJwk.alg = "RS256";
|
|
68492
67731
|
publicJwk.use = "sig";
|
|
@@ -68665,23 +67904,23 @@ async function validateAuthRequest(p) {
|
|
|
68665
67904
|
}
|
|
68666
67905
|
function pkceMatches(verifier, challenge) {
|
|
68667
67906
|
if (!verifier) return false;
|
|
68668
|
-
const computed = (0,
|
|
67907
|
+
const computed = (0, import_node_crypto41.createHash)("sha256").update(verifier).digest("base64url");
|
|
68669
67908
|
return computed === challenge;
|
|
68670
67909
|
}
|
|
68671
67910
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
68672
67911
|
const { privateKey, kid } = await getKeys();
|
|
68673
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
67912
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto41.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
68674
67913
|
}
|
|
68675
67914
|
function tokenErrorResponse(c, error, description, status) {
|
|
68676
67915
|
return c.json({ error, error_description: description }, status);
|
|
68677
67916
|
}
|
|
68678
|
-
var import_hono26, import_cookie,
|
|
67917
|
+
var import_hono26, import_cookie, import_node_crypto41, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, OAUTH_DATABASE_TIMEOUT_MS, OAuthDatabaseUnavailableError, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
68679
67918
|
var init_oauth_routes = __esm({
|
|
68680
67919
|
"src/api/oauth-routes.ts"() {
|
|
68681
67920
|
"use strict";
|
|
68682
67921
|
import_hono26 = require("hono");
|
|
68683
67922
|
import_cookie = require("hono/cookie");
|
|
68684
|
-
|
|
67923
|
+
import_node_crypto41 = require("crypto");
|
|
68685
67924
|
import_jose2 = require("jose");
|
|
68686
67925
|
init_session();
|
|
68687
67926
|
init_db();
|
|
@@ -68788,7 +68027,7 @@ var init_oauth_routes = __esm({
|
|
|
68788
68027
|
}
|
|
68789
68028
|
}
|
|
68790
68029
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
68791
|
-
const clientId = `client_${(0,
|
|
68030
|
+
const clientId = `client_${(0, import_node_crypto41.randomBytes)(16).toString("hex")}`;
|
|
68792
68031
|
await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
|
|
68793
68032
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
68794
68033
|
return c.json({
|
|
@@ -68841,7 +68080,7 @@ var init_oauth_routes = __esm({
|
|
|
68841
68080
|
if (action3 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
68842
68081
|
if (action3 !== "approve") return c.text("unsupported action", 400);
|
|
68843
68082
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
68844
|
-
const code = `code_${(0,
|
|
68083
|
+
const code = `code_${(0, import_node_crypto41.randomBytes)(32).toString("base64url")}`;
|
|
68845
68084
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
68846
68085
|
await withOAuthDatabaseDeadline("put-authorization-code", putCode({
|
|
68847
68086
|
code,
|
|
@@ -68880,7 +68119,7 @@ var init_oauth_routes = __esm({
|
|
|
68880
68119
|
const plan = user ? resolvePlan(user) : "free";
|
|
68881
68120
|
const audience = record.resource ?? RESOURCE();
|
|
68882
68121
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
68883
|
-
const refreshToken = `rt_${(0,
|
|
68122
|
+
const refreshToken = `rt_${(0, import_node_crypto41.randomBytes)(40).toString("base64url")}`;
|
|
68884
68123
|
await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
|
|
68885
68124
|
refresh_token: refreshToken,
|
|
68886
68125
|
client_id: clientId,
|
|
@@ -68910,7 +68149,7 @@ var init_oauth_routes = __esm({
|
|
|
68910
68149
|
const plan = user ? resolvePlan(user) : "free";
|
|
68911
68150
|
const audience = record.resource ?? RESOURCE();
|
|
68912
68151
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
68913
|
-
const nextRefresh = `rt_${(0,
|
|
68152
|
+
const nextRefresh = `rt_${(0, import_node_crypto41.randomBytes)(40).toString("base64url")}`;
|
|
68914
68153
|
await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
|
|
68915
68154
|
refresh_token: nextRefresh,
|
|
68916
68155
|
client_id: record.client_id,
|
|
@@ -70330,7 +69569,7 @@ function quoteUntrusted(value) {
|
|
|
70330
69569
|
return clean3.split("\n").map((line) => `> ${line}`).join("\n");
|
|
70331
69570
|
}
|
|
70332
69571
|
function shortHash(value, length = 24) {
|
|
70333
|
-
return (0,
|
|
69572
|
+
return (0, import_node_crypto42.createHash)("sha256").update(value).digest("hex").slice(0, length);
|
|
70334
69573
|
}
|
|
70335
69574
|
function safePathPart(value) {
|
|
70336
69575
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
|
|
@@ -70818,11 +70057,11 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
70818
70057
|
direction: options.direction
|
|
70819
70058
|
};
|
|
70820
70059
|
}
|
|
70821
|
-
var
|
|
70060
|
+
var import_node_crypto42, SUPPORT_TAGS, ISSUE_TAGS;
|
|
70822
70061
|
var init_resend_support_thread = __esm({
|
|
70823
70062
|
"src/api/resend-support-thread.ts"() {
|
|
70824
70063
|
"use strict";
|
|
70825
|
-
|
|
70064
|
+
import_node_crypto42 = require("crypto");
|
|
70826
70065
|
init_memory();
|
|
70827
70066
|
SUPPORT_TAGS = [
|
|
70828
70067
|
{
|
|
@@ -71407,7 +70646,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
71407
70646
|
) < ?
|
|
71408
70647
|
`,
|
|
71409
70648
|
args: [
|
|
71410
|
-
(0,
|
|
70649
|
+
(0, import_node_crypto43.randomUUID)(),
|
|
71411
70650
|
input.eventSlug,
|
|
71412
70651
|
input.email,
|
|
71413
70652
|
input.firstName,
|
|
@@ -71424,7 +70663,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
71424
70663
|
id, event_slug, email, first_name, last_name, status
|
|
71425
70664
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
71426
70665
|
`,
|
|
71427
|
-
args: [(0,
|
|
70666
|
+
args: [(0, import_node_crypto43.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
71428
70667
|
});
|
|
71429
70668
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
71430
70669
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -71439,7 +70678,7 @@ async function registerWorkshopInterest(input) {
|
|
|
71439
70678
|
id, event_slug, email, first_name, last_name, status
|
|
71440
70679
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
71441
70680
|
`,
|
|
71442
|
-
args: [(0,
|
|
70681
|
+
args: [(0, import_node_crypto43.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
71443
70682
|
});
|
|
71444
70683
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
71445
70684
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -71469,11 +70708,11 @@ async function updateWorkshopRegistration(id, patch) {
|
|
|
71469
70708
|
args: [...entries.map(([, value]) => value), id]
|
|
71470
70709
|
});
|
|
71471
70710
|
}
|
|
71472
|
-
var
|
|
70711
|
+
var import_node_crypto43;
|
|
71473
70712
|
var init_workshop_registration_repository = __esm({
|
|
71474
70713
|
"src/api/workshop-registration-repository.ts"() {
|
|
71475
70714
|
"use strict";
|
|
71476
|
-
|
|
70715
|
+
import_node_crypto43 = require("crypto");
|
|
71477
70716
|
init_db();
|
|
71478
70717
|
}
|
|
71479
70718
|
});
|
|
@@ -72046,7 +71285,7 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
|
|
|
72046
71285
|
articleCount: articles.length,
|
|
72047
71286
|
wordCount: totalWordCount,
|
|
72048
71287
|
bytes,
|
|
72049
|
-
sha256: (0,
|
|
71288
|
+
sha256: (0, import_node_crypto44.createHash)("sha256").update(html).digest("hex"),
|
|
72050
71289
|
warnings
|
|
72051
71290
|
};
|
|
72052
71291
|
}
|
|
@@ -72129,14 +71368,14 @@ ${provenance}`;
|
|
|
72129
71368
|
...rendered,
|
|
72130
71369
|
html,
|
|
72131
71370
|
bytes: Buffer.byteLength(html),
|
|
72132
|
-
sha256: (0,
|
|
71371
|
+
sha256: (0, import_node_crypto44.createHash)("sha256").update(html).digest("hex")
|
|
72133
71372
|
};
|
|
72134
71373
|
}
|
|
72135
|
-
var
|
|
71374
|
+
var import_node_crypto44, import_node_fs14, import_node_path21, import_marked, runtimeEntryDir, assetCache;
|
|
72136
71375
|
var init_render = __esm({
|
|
72137
71376
|
"src/editorial-reading-room/render.ts"() {
|
|
72138
71377
|
"use strict";
|
|
72139
|
-
|
|
71378
|
+
import_node_crypto44 = require("crypto");
|
|
72140
71379
|
import_node_fs14 = require("fs");
|
|
72141
71380
|
import_node_path21 = require("path");
|
|
72142
71381
|
import_marked = require("marked");
|
|
@@ -72400,7 +71639,7 @@ async function hostCommonsImage(input) {
|
|
|
72400
71639
|
415
|
|
72401
71640
|
);
|
|
72402
71641
|
}
|
|
72403
|
-
const digest2 = (0,
|
|
71642
|
+
const digest2 = (0, import_node_crypto45.createHash)("sha256").update(bytes).digest("hex");
|
|
72404
71643
|
const existing = await getDb().execute({
|
|
72405
71644
|
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
72406
71645
|
args: [digest2]
|
|
@@ -72488,11 +71727,11 @@ async function hostEntityImages(input) {
|
|
|
72488
71727
|
}
|
|
72489
71728
|
return rewrite;
|
|
72490
71729
|
}
|
|
72491
|
-
var
|
|
71730
|
+
var import_node_crypto45, import_promises16, import_node_net2, COMMONS_IMAGE_MAX_BYTES, COMMONS_IMAGE_MAX_REDIRECTS, COMMONS_IMAGE_FETCH_TIMEOUT_MS, CommonsImageError, imageSchemaReady, hostOverride;
|
|
72492
71731
|
var init_commons_image_store = __esm({
|
|
72493
71732
|
"src/api/commons-image-store.ts"() {
|
|
72494
71733
|
"use strict";
|
|
72495
|
-
|
|
71734
|
+
import_node_crypto45 = require("crypto");
|
|
72496
71735
|
import_promises16 = require("dns/promises");
|
|
72497
71736
|
import_node_net2 = require("net");
|
|
72498
71737
|
init_blob_store();
|
|
@@ -72617,7 +71856,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
72617
71856
|
context[claim.predicate] = targets;
|
|
72618
71857
|
}
|
|
72619
71858
|
const document2 = { linkset: [context] };
|
|
72620
|
-
const etag = `"${(0,
|
|
71859
|
+
const etag = `"${(0, import_node_crypto46.createHash)("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
72621
71860
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
72622
71861
|
}
|
|
72623
71862
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -72685,11 +71924,11 @@ function normalizeHreflang(value) {
|
|
|
72685
71924
|
const languages = [...new Set(value.map((item) => optionalText(item, 80)).filter((item) => Boolean(item)))];
|
|
72686
71925
|
return languages.length ? languages : void 0;
|
|
72687
71926
|
}
|
|
72688
|
-
var
|
|
71927
|
+
var import_node_crypto46, COMMONS_LINKSET_MEDIA_TYPE, COMMONS_RELATIONSHIP_PROFILE, REGISTERED_RELATIONS;
|
|
72689
71928
|
var init_commons_linksets = __esm({
|
|
72690
71929
|
"src/api/commons-linksets.ts"() {
|
|
72691
71930
|
"use strict";
|
|
72692
|
-
|
|
71931
|
+
import_node_crypto46 = require("crypto");
|
|
72693
71932
|
COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
72694
71933
|
COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
72695
71934
|
REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -73329,7 +72568,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
73329
72568
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
73330
72569
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
73331
72570
|
const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
|
|
73332
|
-
const proposalId = `commons-proposal-${(0,
|
|
72571
|
+
const proposalId = `commons-proposal-${(0, import_node_crypto47.randomUUID)()}`;
|
|
73333
72572
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
73334
72573
|
const entityId = existing?.id ?? normalized.entityId ?? allocateEntityId();
|
|
73335
72574
|
const proposalStatus = shouldApply ? "accepted" : "pending_review";
|
|
@@ -73384,7 +72623,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
73384
72623
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
73385
72624
|
`,
|
|
73386
72625
|
args: [
|
|
73387
|
-
`commons-ledger-${(0,
|
|
72626
|
+
`commons-ledger-${(0, import_node_crypto47.randomUUID)()}`,
|
|
73388
72627
|
nextEntity.id,
|
|
73389
72628
|
proposalId,
|
|
73390
72629
|
user.id,
|
|
@@ -73490,7 +72729,7 @@ async function saveCommonsFilter(input, user) {
|
|
|
73490
72729
|
sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
|
|
73491
72730
|
args: [user.id, name]
|
|
73492
72731
|
});
|
|
73493
|
-
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0,
|
|
72732
|
+
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0, import_node_crypto47.randomUUID)()}`;
|
|
73494
72733
|
await getDb().execute({
|
|
73495
72734
|
sql: `
|
|
73496
72735
|
INSERT INTO commons_saved_filters (id, user_id, name, description, filter_json, created_at, updated_at)
|
|
@@ -74268,7 +73507,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
74268
73507
|
return documents;
|
|
74269
73508
|
}
|
|
74270
73509
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
74271
|
-
return `commons-index-${(0,
|
|
73510
|
+
return `commons-index-${(0, import_node_crypto47.createHash)("sha256").update(`${entityId}
|
|
74272
73511
|
${documentType}
|
|
74273
73512
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
74274
73513
|
}
|
|
@@ -74834,11 +74073,11 @@ function jsonLikeValue(value) {
|
|
|
74834
74073
|
function escapeLike(value) {
|
|
74835
74074
|
return value.replace(/[%_]/g, "");
|
|
74836
74075
|
}
|
|
74837
|
-
var
|
|
74076
|
+
var import_node_crypto47, COMMONS_SCHEMA_VERSION, DEFAULT_COMMONS_BASE_URL, DEFAULT_ENTITY_TYPE, COMMONS_ENTITY_PROFILES, schemaReady, COMMONS_SCHEMA_OBJECTS, SEARCH_CANDIDATE_LIMIT, CommonsRepositoryError;
|
|
74838
74077
|
var init_commons_repository = __esm({
|
|
74839
74078
|
"src/api/commons-repository.ts"() {
|
|
74840
74079
|
"use strict";
|
|
74841
|
-
|
|
74080
|
+
import_node_crypto47 = require("crypto");
|
|
74842
74081
|
init_db();
|
|
74843
74082
|
init_commons_image_store();
|
|
74844
74083
|
init_commons_embeddings();
|
|
@@ -75217,7 +74456,7 @@ async function claimCommonsPublication(input, user) {
|
|
|
75217
74456
|
}
|
|
75218
74457
|
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
75219
74458
|
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
75220
|
-
const id = `tcpub_${(0,
|
|
74459
|
+
const id = `tcpub_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75221
74460
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75222
74461
|
try {
|
|
75223
74462
|
await getDb().execute({
|
|
@@ -75269,7 +74508,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
75269
74508
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
75270
74509
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
75271
74510
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
75272
|
-
const editionId = `tced_${(0,
|
|
74511
|
+
const editionId = `tced_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75273
74512
|
const revision = (latest?.revision ?? 0) + 1;
|
|
75274
74513
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75275
74514
|
await getDb().batch([
|
|
@@ -75293,7 +74532,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
75293
74532
|
JSON.stringify(editionInput.articles),
|
|
75294
74533
|
html,
|
|
75295
74534
|
rendered.filename,
|
|
75296
|
-
(0,
|
|
74535
|
+
(0, import_node_crypto48.createHash)("sha256").update(html).digest("hex"),
|
|
75297
74536
|
rendered.articleCount,
|
|
75298
74537
|
rendered.wordCount,
|
|
75299
74538
|
Buffer.byteLength(html),
|
|
@@ -75351,7 +74590,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
75351
74590
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
75352
74591
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
75353
74592
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
75354
|
-
const editionId = `tced_${(0,
|
|
74593
|
+
const editionId = `tced_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75355
74594
|
const revision = latest.revision + 1;
|
|
75356
74595
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75357
74596
|
await getDb().batch([
|
|
@@ -75375,7 +74614,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
75375
74614
|
JSON.stringify(nextArticles),
|
|
75376
74615
|
html,
|
|
75377
74616
|
rendered.filename,
|
|
75378
|
-
(0,
|
|
74617
|
+
(0, import_node_crypto48.createHash)("sha256").update(html).digest("hex"),
|
|
75379
74618
|
rendered.articleCount,
|
|
75380
74619
|
rendered.wordCount,
|
|
75381
74620
|
Buffer.byteLength(html),
|
|
@@ -75580,11 +74819,11 @@ function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
|
75580
74819
|
"</head>"
|
|
75581
74820
|
].join("\n"));
|
|
75582
74821
|
}
|
|
75583
|
-
var
|
|
74822
|
+
var import_node_crypto48, PUBLICATION_ROOT_DOMAIN, RESERVED_SUBDOMAINS, CommonsPublicationError;
|
|
75584
74823
|
var init_commons_publication_repository = __esm({
|
|
75585
74824
|
"src/api/commons-publication-repository.ts"() {
|
|
75586
74825
|
"use strict";
|
|
75587
|
-
|
|
74826
|
+
import_node_crypto48 = require("crypto");
|
|
75588
74827
|
init_db();
|
|
75589
74828
|
init_commons_repository();
|
|
75590
74829
|
init_render();
|
|
@@ -76817,7 +76056,7 @@ async function migrateAnalytics() {
|
|
|
76817
76056
|
}
|
|
76818
76057
|
function normalizeSlug2(value) {
|
|
76819
76058
|
const slug4 = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
76820
|
-
return slug4 || `site-${(0,
|
|
76059
|
+
return slug4 || `site-${(0, import_node_crypto49.randomBytes)(4).toString("hex")}`;
|
|
76821
76060
|
}
|
|
76822
76061
|
function normalizeObservedHostname(origin) {
|
|
76823
76062
|
if (!origin || origin === "null") return null;
|
|
@@ -76830,7 +76069,7 @@ function normalizeObservedHostname(origin) {
|
|
|
76830
76069
|
}
|
|
76831
76070
|
}
|
|
76832
76071
|
function publicPixelId() {
|
|
76833
|
-
return `px_${(0,
|
|
76072
|
+
return `px_${(0, import_node_crypto49.randomBytes)(18).toString("base64url")}`;
|
|
76834
76073
|
}
|
|
76835
76074
|
function mapRows(rows) {
|
|
76836
76075
|
return rows;
|
|
@@ -76862,7 +76101,7 @@ async function requireEditor(client2, siteId, userId) {
|
|
|
76862
76101
|
async function createAnalyticsSite(input) {
|
|
76863
76102
|
const db = getAnalyticsPool();
|
|
76864
76103
|
const client2 = await db.connect();
|
|
76865
|
-
const id = (0,
|
|
76104
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
76866
76105
|
const baseSlug = normalizeSlug2(input.slug || input.name);
|
|
76867
76106
|
try {
|
|
76868
76107
|
await client2.query("BEGIN");
|
|
@@ -76878,7 +76117,7 @@ async function createAnalyticsSite(input) {
|
|
|
76878
76117
|
} catch (error) {
|
|
76879
76118
|
const code = error.code;
|
|
76880
76119
|
if (code !== "23505" || attempt === 2) throw error;
|
|
76881
|
-
slug4 = `${baseSlug}-${(0,
|
|
76120
|
+
slug4 = `${baseSlug}-${(0, import_node_crypto49.randomBytes)(2).toString("hex")}`;
|
|
76882
76121
|
}
|
|
76883
76122
|
}
|
|
76884
76123
|
await client2.query(
|
|
@@ -77093,7 +76332,7 @@ async function createAnalyticsPixel(input) {
|
|
|
77093
76332
|
VALUES ($1, $2, $3, $4, $5)
|
|
77094
76333
|
RETURNING id, site_id, public_id, name, environment, status, created_at::text, NULL::text AS last_event_at`,
|
|
77095
76334
|
[
|
|
77096
|
-
(0,
|
|
76335
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77097
76336
|
input.siteId,
|
|
77098
76337
|
publicPixelId(),
|
|
77099
76338
|
input.name.trim(),
|
|
@@ -77170,7 +76409,7 @@ async function setAnalyticsPixelDomainState(input) {
|
|
|
77170
76409
|
SELECT $1, p.id, $4, $5 FROM analytics_pixels p WHERE p.id = $2 AND p.site_id = $3
|
|
77171
76410
|
ON CONFLICT(pixel_id, hostname) DO UPDATE SET state = EXCLUDED.state, updated_at = now()
|
|
77172
76411
|
RETURNING id`,
|
|
77173
|
-
[(0,
|
|
76412
|
+
[(0, import_node_crypto49.randomUUID)(), input.pixelId, input.siteId, hostname, input.state]
|
|
77174
76413
|
);
|
|
77175
76414
|
if (!result.rowCount)
|
|
77176
76415
|
throw new AnalyticsRepositoryError(
|
|
@@ -77220,7 +76459,7 @@ function sanitizeAnalyticsProperties(value) {
|
|
|
77220
76459
|
}
|
|
77221
76460
|
async function ingestAnalyticsEvents(input) {
|
|
77222
76461
|
const db = getAnalyticsPool();
|
|
77223
|
-
const requestId = `air_${(0,
|
|
76462
|
+
const requestId = `air_${(0, import_node_crypto49.randomUUID)()}`;
|
|
77224
76463
|
const hostname = normalizeObservedHostname(input.origin);
|
|
77225
76464
|
const pixelResult = await db.query(
|
|
77226
76465
|
`SELECT p.id, p.site_id, p.status,
|
|
@@ -77241,7 +76480,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77241
76480
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77242
76481
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77243
76482
|
[
|
|
77244
|
-
(0,
|
|
76483
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77245
76484
|
requestId,
|
|
77246
76485
|
pixel?.site_id ?? null,
|
|
77247
76486
|
pixel?.id ?? null,
|
|
@@ -77266,7 +76505,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77266
76505
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77267
76506
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77268
76507
|
[
|
|
77269
|
-
(0,
|
|
76508
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77270
76509
|
requestId,
|
|
77271
76510
|
pixel.site_id,
|
|
77272
76511
|
pixel.id,
|
|
@@ -77291,7 +76530,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77291
76530
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, rejected_count, reason_codes)
|
|
77292
76531
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`,
|
|
77293
76532
|
[
|
|
77294
|
-
(0,
|
|
76533
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77295
76534
|
requestId,
|
|
77296
76535
|
pixel.site_id,
|
|
77297
76536
|
pixel.id,
|
|
@@ -77315,7 +76554,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77315
76554
|
ON CONFLICT(pixel_id, hostname) DO UPDATE
|
|
77316
76555
|
SET last_seen_at = now(), updated_at = now()
|
|
77317
76556
|
RETURNING state`,
|
|
77318
|
-
[(0,
|
|
76557
|
+
[(0, import_node_crypto49.randomUUID)(), pixel.id, hostname]
|
|
77319
76558
|
);
|
|
77320
76559
|
if (domainResult.rows[0]?.state !== "approved") {
|
|
77321
76560
|
await db.query(
|
|
@@ -77329,7 +76568,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77329
76568
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77330
76569
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77331
76570
|
[
|
|
77332
|
-
(0,
|
|
76571
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77333
76572
|
requestId,
|
|
77334
76573
|
pixel.site_id,
|
|
77335
76574
|
pixel.id,
|
|
@@ -77379,7 +76618,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77379
76618
|
$23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
|
|
77380
76619
|
) ON CONFLICT(site_id, event_id) DO NOTHING`,
|
|
77381
76620
|
[
|
|
77382
|
-
(0,
|
|
76621
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77383
76622
|
event2.eventId,
|
|
77384
76623
|
pixel.site_id,
|
|
77385
76624
|
pixel.id,
|
|
@@ -77437,7 +76676,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77437
76676
|
id, request_id, site_id, pixel_id, hostname, accepted_count, rejected_count, reason_codes
|
|
77438
76677
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
|
77439
76678
|
[
|
|
77440
|
-
(0,
|
|
76679
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77441
76680
|
requestId,
|
|
77442
76681
|
pixel.site_id,
|
|
77443
76682
|
pixel.id,
|
|
@@ -77469,7 +76708,7 @@ function normalizeGeographyCode(value, max) {
|
|
|
77469
76708
|
return normalized && /^[A-Z0-9-]+$/.test(normalized) ? normalized.slice(0, max) : null;
|
|
77470
76709
|
}
|
|
77471
76710
|
function pageFingerprint(scope, siteId, filters) {
|
|
77472
|
-
return (0,
|
|
76711
|
+
return (0, import_node_crypto49.createHash)("sha256").update(JSON.stringify({ scope, siteId, filters })).digest("base64url").slice(0, 18);
|
|
77473
76712
|
}
|
|
77474
76713
|
function decodePageOffset(cursor, fingerprint2) {
|
|
77475
76714
|
if (!cursor) return 0;
|
|
@@ -77519,7 +76758,7 @@ async function createAnalyticsConversion(input) {
|
|
|
77519
76758
|
404
|
|
77520
76759
|
);
|
|
77521
76760
|
}
|
|
77522
|
-
const id = (0,
|
|
76761
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
77523
76762
|
const resolvedPerson = input.sessionId ? await db.query(
|
|
77524
76763
|
`SELECT e.person_id FROM analytics_identity_nodes n JOIN analytics_identity_edges e ON e.identity_node_id = n.id
|
|
77525
76764
|
WHERE n.site_id = $1 AND n.kind = 'session_id' AND n.value_hmac = $2 ORDER BY e.confidence DESC LIMIT 1`,
|
|
@@ -78268,7 +77507,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
78268
77507
|
404
|
|
78269
77508
|
);
|
|
78270
77509
|
}
|
|
78271
|
-
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0,
|
|
77510
|
+
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0, import_node_crypto49.randomBytes)(5).toString("base64url").toLowerCase()).slice(0, 48);
|
|
78272
77511
|
try {
|
|
78273
77512
|
const result = await db.query(
|
|
78274
77513
|
`INSERT INTO analytics_campaign_links(
|
|
@@ -78277,7 +77516,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
78277
77516
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
|
|
78278
77517
|
RETURNING *, 0::int AS click_count`,
|
|
78279
77518
|
[
|
|
78280
|
-
(0,
|
|
77519
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78281
77520
|
input.siteId,
|
|
78282
77521
|
input.pixelId ?? null,
|
|
78283
77522
|
input.name.trim(),
|
|
@@ -78348,7 +77587,7 @@ async function resolveAnalyticsCampaignLink(shortCode, referrer) {
|
|
|
78348
77587
|
if (!row) return null;
|
|
78349
77588
|
await db.query(
|
|
78350
77589
|
`INSERT INTO analytics_campaign_clicks(id, link_id, referrer) VALUES ($1, $2, $3)`,
|
|
78351
|
-
[(0,
|
|
77590
|
+
[(0, import_node_crypto49.randomUUID)(), row.id, normalizeAnalyticsUrl(referrer || void 0)]
|
|
78352
77591
|
);
|
|
78353
77592
|
return buildTaggedCampaignUrl(row);
|
|
78354
77593
|
}
|
|
@@ -78366,14 +77605,14 @@ async function createAnalyticsForm(input) {
|
|
|
78366
77605
|
404
|
|
78367
77606
|
);
|
|
78368
77607
|
const baseSlug = normalizeSlug2(input.name);
|
|
78369
|
-
const slug4 = `${baseSlug}-${(0,
|
|
78370
|
-
const publicId = `form_${(0,
|
|
77608
|
+
const slug4 = `${baseSlug}-${(0, import_node_crypto49.randomBytes)(3).toString("hex")}`;
|
|
77609
|
+
const publicId = `form_${(0, import_node_crypto49.randomBytes)(18).toString("base64url")}`;
|
|
78371
77610
|
const result = await db.query(
|
|
78372
77611
|
`INSERT INTO analytics_forms(id, public_id, site_id, pixel_id, name, slug, fields, brand, submit_label, success_message, redirect_url, consent_text, status, created_by_user_id)
|
|
78373
77612
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10,$11,$12,$13,$14)
|
|
78374
77613
|
RETURNING *, 0::int AS submission_count`,
|
|
78375
77614
|
[
|
|
78376
|
-
(0,
|
|
77615
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78377
77616
|
publicId,
|
|
78378
77617
|
input.siteId,
|
|
78379
77618
|
input.pixelId,
|
|
@@ -78418,7 +77657,7 @@ async function getPublicAnalyticsForm(publicId) {
|
|
|
78418
77657
|
return result.rows[0] ?? null;
|
|
78419
77658
|
}
|
|
78420
77659
|
async function recordAnalyticsFormSubmission(input) {
|
|
78421
|
-
const id = (0,
|
|
77660
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78422
77661
|
await getAnalyticsPool().query(
|
|
78423
77662
|
`INSERT INTO analytics_form_submissions(id, form_id, site_id, pixel_id, visitor_id, session_id, crm_person_ref, source, medium, campaign, crm_delivery_status, click_ids)
|
|
78424
77663
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::jsonb)`,
|
|
@@ -78449,7 +77688,7 @@ function sanitizeClickIds(value) {
|
|
|
78449
77688
|
return output;
|
|
78450
77689
|
}
|
|
78451
77690
|
function identityHmac(kind, value) {
|
|
78452
|
-
return (0,
|
|
77691
|
+
return (0, import_node_crypto49.createHmac)("sha256", getSessionSecret()).update(`${kind}:${value.trim().toLowerCase()}`).digest("hex");
|
|
78453
77692
|
}
|
|
78454
77693
|
async function linkAnalyticsFormIdentity(input) {
|
|
78455
77694
|
const client2 = await getAnalyticsPool().connect();
|
|
@@ -78458,7 +77697,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78458
77697
|
const person = await client2.query(
|
|
78459
77698
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
78460
77699
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
78461
|
-
[(0,
|
|
77700
|
+
[(0, import_node_crypto49.randomUUID)(), input.siteId, input.crmPersonRef]
|
|
78462
77701
|
);
|
|
78463
77702
|
const personId = person.rows[0].id;
|
|
78464
77703
|
const signals = [];
|
|
@@ -78505,7 +77744,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78505
77744
|
`INSERT INTO analytics_identity_nodes(id, site_id, kind, value_hmac) VALUES ($1,$2,$3,$4)
|
|
78506
77745
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
78507
77746
|
[
|
|
78508
|
-
(0,
|
|
77747
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78509
77748
|
input.siteId,
|
|
78510
77749
|
signal.kind,
|
|
78511
77750
|
identityHmac(signal.kind, signal.value)
|
|
@@ -78516,7 +77755,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78516
77755
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
78517
77756
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence)`,
|
|
78518
77757
|
[
|
|
78519
|
-
(0,
|
|
77758
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78520
77759
|
input.siteId,
|
|
78521
77760
|
personId,
|
|
78522
77761
|
node.rows[0].id,
|
|
@@ -78614,7 +77853,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
78614
77853
|
const db = getAnalyticsPool();
|
|
78615
77854
|
await requireEditor(db, input.siteId, input.userId);
|
|
78616
77855
|
const client2 = await db.connect();
|
|
78617
|
-
const id = (0,
|
|
77856
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78618
77857
|
try {
|
|
78619
77858
|
await client2.query("BEGIN");
|
|
78620
77859
|
await client2.query(
|
|
@@ -78634,7 +77873,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
78634
77873
|
await client2.query(
|
|
78635
77874
|
`INSERT INTO analytics_crm_import_rows(id, import_id, crm_person_ref, payload_ciphertext)
|
|
78636
77875
|
VALUES ($1,$2,$3,$4) ON CONFLICT(import_id, crm_person_ref) DO NOTHING`,
|
|
78637
|
-
[(0,
|
|
77876
|
+
[(0, import_node_crypto49.randomUUID)(), id, row.crmPersonRef, row.payloadCiphertext]
|
|
78638
77877
|
);
|
|
78639
77878
|
}
|
|
78640
77879
|
await client2.query("COMMIT");
|
|
@@ -78673,7 +77912,7 @@ async function createAnalyticsActivationDestination(input) {
|
|
|
78673
77912
|
`INSERT INTO analytics_activation_destinations(id, site_id, platform, name, connection_ref, external_dataset_id, event_mapping, created_by_user_id)
|
|
78674
77913
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8) RETURNING *`,
|
|
78675
77914
|
[
|
|
78676
|
-
(0,
|
|
77915
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78677
77916
|
input.siteId,
|
|
78678
77917
|
input.platform,
|
|
78679
77918
|
input.name.trim(),
|
|
@@ -78725,7 +77964,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
78725
77964
|
`INSERT INTO analytics_activation_jobs(id, destination_id, conversion_id, person_id, payload_ciphertext)
|
|
78726
77965
|
VALUES ($1,$2,$3,$4,$5) ON CONFLICT(destination_id, conversion_id) DO NOTHING`,
|
|
78727
77966
|
[
|
|
78728
|
-
(0,
|
|
77967
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78729
77968
|
destination.id,
|
|
78730
77969
|
input.conversionId,
|
|
78731
77970
|
input.personId ?? null,
|
|
@@ -78737,7 +77976,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
78737
77976
|
return queued;
|
|
78738
77977
|
}
|
|
78739
77978
|
async function queueAnalyticsFormDelivery(input) {
|
|
78740
|
-
const id = (0,
|
|
77979
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78741
77980
|
await getAnalyticsPool().query(
|
|
78742
77981
|
`INSERT INTO analytics_form_delivery_jobs(id, submission_id, owner_user_id, payload_ciphertext, last_error_code)
|
|
78743
77982
|
VALUES ($1,$2,$3,$4,$5)`,
|
|
@@ -78864,7 +78103,7 @@ async function analyticsHealth(siteId, userId) {
|
|
|
78864
78103
|
}
|
|
78865
78104
|
async function refreshAnalyticsDailyRollups(input) {
|
|
78866
78105
|
const db = getAnalyticsPool();
|
|
78867
|
-
const runId = (0,
|
|
78106
|
+
const runId = (0, import_node_crypto49.randomUUID)();
|
|
78868
78107
|
await db.query(
|
|
78869
78108
|
`INSERT INTO analytics_rollup_runs(id, window_start, window_end, status) VALUES ($1, $2, $3, 'running')`,
|
|
78870
78109
|
[runId, input.start, input.end]
|
|
@@ -79061,7 +78300,7 @@ async function createAnalyticsExport(input) {
|
|
|
79061
78300
|
`Generated ${(/* @__PURE__ */ new Date()).toISOString()} from the governed ${input.report} report contract.`
|
|
79062
78301
|
].join("\n");
|
|
79063
78302
|
}
|
|
79064
|
-
const id = (0,
|
|
78303
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
79065
78304
|
const inserted = await getAnalyticsPool().query(
|
|
79066
78305
|
`INSERT INTO analytics_exports(id, site_id, requested_by_user_id, idempotency_key, report, format, filters, content)
|
|
79067
78306
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)
|
|
@@ -79094,11 +78333,11 @@ async function createAnalyticsExport(input) {
|
|
|
79094
78333
|
}
|
|
79095
78334
|
return describeArtifact(replay.rows[0]);
|
|
79096
78335
|
}
|
|
79097
|
-
var
|
|
78336
|
+
var import_node_crypto49, import_pg, AnalyticsRepositoryError, pool2, MAX_ENGAGED_MS, ENGAGED_SESSION_MS, blockedPropertyName, inferredFamilySql, ANALYTICS_CONTENT_SORTS, clickIdKeys;
|
|
79098
78337
|
var init_analytics_repository = __esm({
|
|
79099
78338
|
"src/api/analytics-repository.ts"() {
|
|
79100
78339
|
"use strict";
|
|
79101
|
-
|
|
78340
|
+
import_node_crypto49 = require("crypto");
|
|
79102
78341
|
import_pg = require("pg");
|
|
79103
78342
|
init_session();
|
|
79104
78343
|
init_analytics_attribution();
|
|
@@ -79337,8 +78576,8 @@ function dashboardCallbackUrl() {
|
|
|
79337
78576
|
}
|
|
79338
78577
|
async function createThorbitConnectUrl(userId) {
|
|
79339
78578
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
79340
|
-
const state = (0,
|
|
79341
|
-
const stateHash = (0,
|
|
78579
|
+
const state = (0, import_node_crypto50.randomBytes)(32).toString("base64url");
|
|
78580
|
+
const stateHash = (0, import_node_crypto50.createHash)("sha256").update(state).digest("hex");
|
|
79342
78581
|
await getAnalyticsPool().query(
|
|
79343
78582
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
79344
78583
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -79353,9 +78592,9 @@ async function createThorbitConnectUrl(userId) {
|
|
|
79353
78592
|
function verifyThorbitAssertion(token6) {
|
|
79354
78593
|
const [encoded, signature] = token6.split(".");
|
|
79355
78594
|
if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
|
|
79356
|
-
const expected = (0,
|
|
78595
|
+
const expected = (0, import_node_crypto50.createHmac)("sha256", bridgeSecret()).update(encoded).digest();
|
|
79357
78596
|
const actual = Buffer.from(signature, "base64url");
|
|
79358
|
-
if (actual.length !== expected.length || !(0,
|
|
78597
|
+
if (actual.length !== expected.length || !(0, import_node_crypto50.timingSafeEqual)(actual, expected)) {
|
|
79359
78598
|
throw new Error("Invalid Thorbit assertion signature");
|
|
79360
78599
|
}
|
|
79361
78600
|
const parsed = AssertionSchema.safeParse(
|
|
@@ -79367,7 +78606,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
79367
78606
|
return parsed.data.entitlement;
|
|
79368
78607
|
}
|
|
79369
78608
|
async function consumeThorbitConnectCallback(input) {
|
|
79370
|
-
const stateHash = (0,
|
|
78609
|
+
const stateHash = (0, import_node_crypto50.createHash)("sha256").update(input.state).digest("hex");
|
|
79371
78610
|
const client2 = await getAnalyticsPool().connect();
|
|
79372
78611
|
try {
|
|
79373
78612
|
await client2.query("BEGIN");
|
|
@@ -79407,11 +78646,11 @@ async function disconnectAnalyticsEntitlement(userId) {
|
|
|
79407
78646
|
[userId]
|
|
79408
78647
|
);
|
|
79409
78648
|
}
|
|
79410
|
-
var
|
|
78649
|
+
var import_node_crypto50, import_zod55, THORBIT_PRODUCT_URL, REFRESH_INTERVAL_MS, ELIGIBLE_GRACE_MS, TRIAL_LENGTH_MS, ThorbitEntitlementSchema, AssertionSchema;
|
|
79411
78650
|
var init_analytics_entitlement = __esm({
|
|
79412
78651
|
"src/api/analytics-entitlement.ts"() {
|
|
79413
78652
|
"use strict";
|
|
79414
|
-
|
|
78653
|
+
import_node_crypto50 = require("crypto");
|
|
79415
78654
|
import_zod55 = require("zod");
|
|
79416
78655
|
init_analytics_repository();
|
|
79417
78656
|
THORBIT_PRODUCT_URL = "https://thorbit.ai";
|
|
@@ -79551,14 +78790,14 @@ function renderPublicForm(form, placementUrl) {
|
|
|
79551
78790
|
const brand = form.brand;
|
|
79552
78791
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><style>:root{font-family:ui-sans-serif,system-ui;color:${escapeHtml8(brand.textColor)};background:${escapeHtml8(brand.backgroundColor)}}*{box-sizing:border-box}body{margin:0;padding:18px}form{display:grid;gap:14px}label{display:grid;gap:6px;font-size:13px;font-weight:650}.check{grid-template-columns:auto 1fr;align-items:center}.check span{grid-column:2}.check input{grid-column:1;grid-row:1}input,textarea,select{width:100%;min-height:44px;padding:10px 12px;border:1px solid #ccd3df;border-radius:${brand.radius}px;background:white;color:inherit;font:inherit}textarea{min-height:110px;resize:vertical}button{min-height:46px;border:0;border-radius:${brand.radius}px;color:white;background:${escapeHtml8(brand.primaryColor)};font:inherit;font-weight:750;cursor:pointer}.consent{margin:0;color:#667085;font-size:11px;line-height:1.45}.notice{display:none;padding:12px;border-radius:${brand.radius}px;background:#edf8f2;color:#17613c}.hp{position:absolute!important;left:-9999px!important}</style></head><body><form id="mcp-form">${fields}<label class="hp">Website<input name="website" autocomplete="off" tabindex="-1"></label>${form.consent_text ? `<p class="consent">${escapeHtml8(form.consent_text)}</p>` : ""}<button>${escapeHtml8(form.submit_label)}</button><div class="notice" role="status"></div></form><script>(()=>{const form=document.querySelector('#mcp-form'),notice=form.querySelector('.notice'),q=new URLSearchParams(location.search);form.addEventListener('submit',async event=>{event.preventDefault();const button=form.querySelector('button');button.disabled=true;const raw=Object.fromEntries(new FormData(form).entries()),website=String(raw.website||''),clickIds={};delete raw.website;for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}try{const response=await fetch('/analytics/forms/${escapeHtml8(form.public_id)}/submissions',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({data:raw,website,placementUrl:${JSON.stringify(placementUrl)},visitorId:q.get('visitor')||undefined,sessionId:q.get('session')||undefined,source:q.get('source')||undefined,medium:q.get('medium')||undefined,campaign:q.get('campaign')||undefined,clickIds})});if(!response.ok)throw new Error('submit');notice.textContent=${JSON.stringify(form.success_message)};notice.style.display='block';form.reset();${form.redirect_url ? `setTimeout(()=>{top.location.href=${JSON.stringify(form.redirect_url)}},700);` : ""}}catch{notice.textContent='Your response could not be submitted. Please try again.';notice.style.display='block'}finally{button.disabled=false}})})()</script></body></html>`;
|
|
79553
78792
|
}
|
|
79554
|
-
var import_hono34, import_factory6, import_zod56,
|
|
78793
|
+
var import_hono34, import_factory6, import_zod56, import_node_crypto51, import_papaparse6, analyticsApp, auth3, entitlementGuard, ThorbitApiKeySchema, ThorbitCallbackSchema, SiteInputSchema, PixelInputSchema, PixelUpdateSchema, EventSchema, IngestionSchema, ConversionSchema, ExportSchema2, CampaignLinkSchema, FormFieldSchema, FormInputSchema, FormSubmissionSchema, CrmImportSchema, ActivationDestinationSchema, BusinessModelSchema, AdSpendSchema;
|
|
79555
78794
|
var init_analytics_routes = __esm({
|
|
79556
78795
|
"src/api/analytics-routes.ts"() {
|
|
79557
78796
|
"use strict";
|
|
79558
78797
|
import_hono34 = require("hono");
|
|
79559
78798
|
import_factory6 = require("hono/factory");
|
|
79560
78799
|
import_zod56 = require("zod");
|
|
79561
|
-
|
|
78800
|
+
import_node_crypto51 = require("crypto");
|
|
79562
78801
|
import_papaparse6 = __toESM(require("papaparse"), 1);
|
|
79563
78802
|
init_api_auth();
|
|
79564
78803
|
init_db();
|
|
@@ -79889,7 +79128,7 @@ var init_analytics_routes = __esm({
|
|
|
79889
79128
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
79890
79129
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
79891
79130
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
79892
|
-
const suffix2 = (0,
|
|
79131
|
+
const suffix2 = (0, import_node_crypto51.createHash)("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
79893
79132
|
const path6 = `Leads/person-${suffix2}`;
|
|
79894
79133
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
79895
79134
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path6 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -79988,9 +79227,9 @@ Submitted the published form.
|
|
|
79988
79227
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
79989
79228
|
});
|
|
79990
79229
|
const match = {
|
|
79991
|
-
...email ? { emailSha256: (0,
|
|
79230
|
+
...email ? { emailSha256: (0, import_node_crypto51.createHash)("sha256").update(email).digest("hex") } : {},
|
|
79992
79231
|
...phone ? {
|
|
79993
|
-
phoneSha256: (0,
|
|
79232
|
+
phoneSha256: (0, import_node_crypto51.createHash)("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
79994
79233
|
} : {}
|
|
79995
79234
|
};
|
|
79996
79235
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -80418,7 +79657,7 @@ Submitted the published form.
|
|
|
80418
79657
|
rejectedCount += 1;
|
|
80419
79658
|
continue;
|
|
80420
79659
|
}
|
|
80421
|
-
const digest2 = (0,
|
|
79660
|
+
const digest2 = (0, import_node_crypto51.createHash)("sha256").update(
|
|
80422
79661
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
80423
79662
|
).digest("hex").slice(0, 16);
|
|
80424
79663
|
const path6 = `Leads/person-${digest2}`;
|
|
@@ -80876,13 +80115,13 @@ var init_analytics_delivery = __esm({
|
|
|
80876
80115
|
|
|
80877
80116
|
// src/api/scheduled-artifact-owner.ts
|
|
80878
80117
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
80879
|
-
return (0,
|
|
80118
|
+
return (0, import_node_crypto52.createHash)("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
80880
80119
|
}
|
|
80881
|
-
var
|
|
80120
|
+
var import_node_crypto52;
|
|
80882
80121
|
var init_scheduled_artifact_owner = __esm({
|
|
80883
80122
|
"src/api/scheduled-artifact-owner.ts"() {
|
|
80884
80123
|
"use strict";
|
|
80885
|
-
|
|
80124
|
+
import_node_crypto52 = require("crypto");
|
|
80886
80125
|
}
|
|
80887
80126
|
});
|
|
80888
80127
|
|
|
@@ -80939,7 +80178,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
80939
80178
|
schemaReady2 = true;
|
|
80940
80179
|
}
|
|
80941
80180
|
function tokenHash2(token6) {
|
|
80942
|
-
return (0,
|
|
80181
|
+
return (0, import_node_crypto53.createHash)("sha256").update(token6).digest("hex");
|
|
80943
80182
|
}
|
|
80944
80183
|
function mapRow(row) {
|
|
80945
80184
|
return {
|
|
@@ -80959,9 +80198,9 @@ function mapRow(row) {
|
|
|
80959
80198
|
async function createScheduledRunViewLink(input) {
|
|
80960
80199
|
await ensureScheduledRunViewLinksSchema();
|
|
80961
80200
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
80962
|
-
const token6 = (0,
|
|
80201
|
+
const token6 = (0, import_node_crypto53.randomBytes)(32).toString("base64url");
|
|
80963
80202
|
const record = {
|
|
80964
|
-
shareId: (0,
|
|
80203
|
+
shareId: (0, import_node_crypto53.randomUUID)(),
|
|
80965
80204
|
ownerId: input.ownerId,
|
|
80966
80205
|
runId: input.runId,
|
|
80967
80206
|
artifactId: input.artifactId,
|
|
@@ -81024,11 +80263,11 @@ async function revokeScheduledRunViewLink(ownerId2, runId, shareId, now = /* @__
|
|
|
81024
80263
|
});
|
|
81025
80264
|
return result.rowsAffected > 0;
|
|
81026
80265
|
}
|
|
81027
|
-
var
|
|
80266
|
+
var import_node_crypto53, schemaReady2;
|
|
81028
80267
|
var init_scheduled_run_view_links = __esm({
|
|
81029
80268
|
"src/api/scheduled-run-view-links.ts"() {
|
|
81030
80269
|
"use strict";
|
|
81031
|
-
|
|
80270
|
+
import_node_crypto53 = require("crypto");
|
|
81032
80271
|
init_db();
|
|
81033
80272
|
schemaReady2 = false;
|
|
81034
80273
|
}
|
|
@@ -81094,15 +80333,15 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
81094
80333
|
html,
|
|
81095
80334
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
81096
80335
|
bytes,
|
|
81097
|
-
sha256: (0,
|
|
80336
|
+
sha256: (0, import_node_crypto54.createHash)("sha256").update(html).digest("hex"),
|
|
81098
80337
|
generatedAt: generatedAtIso
|
|
81099
80338
|
};
|
|
81100
80339
|
}
|
|
81101
|
-
var
|
|
80340
|
+
var import_node_crypto54;
|
|
81102
80341
|
var init_render2 = __esm({
|
|
81103
80342
|
"src/personal-authority/render.ts"() {
|
|
81104
80343
|
"use strict";
|
|
81105
|
-
|
|
80344
|
+
import_node_crypto54 = require("crypto");
|
|
81106
80345
|
}
|
|
81107
80346
|
});
|
|
81108
80347
|
|
|
@@ -81205,15 +80444,15 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
81205
80444
|
html,
|
|
81206
80445
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
81207
80446
|
bytes,
|
|
81208
|
-
sha256: (0,
|
|
80447
|
+
sha256: (0, import_node_crypto55.createHash)("sha256").update(html).digest("hex"),
|
|
81209
80448
|
generatedAt: generatedAtIso
|
|
81210
80449
|
};
|
|
81211
80450
|
}
|
|
81212
|
-
var
|
|
80451
|
+
var import_node_crypto55, FONT_STACKS;
|
|
81213
80452
|
var init_render_v2 = __esm({
|
|
81214
80453
|
"src/personal-authority/render-v2.ts"() {
|
|
81215
80454
|
"use strict";
|
|
81216
|
-
|
|
80455
|
+
import_node_crypto55 = require("crypto");
|
|
81217
80456
|
init_contracts();
|
|
81218
80457
|
FONT_STACKS = Object.freeze({
|
|
81219
80458
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
@@ -81329,15 +80568,15 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
81329
80568
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
81330
80569
|
</style></head><body data-theme="${escapeHtml11(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
81331
80570
|
const bytes = Buffer.byteLength(html);
|
|
81332
|
-
const sha2565 = (0,
|
|
80571
|
+
const sha2565 = (0, import_node_crypto56.createHash)("sha256").update(html).digest("hex");
|
|
81333
80572
|
const filename2 = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
81334
80573
|
return { html, filename: filename2, bytes, sha256: sha2565, generatedAt: generatedAtIso };
|
|
81335
80574
|
}
|
|
81336
|
-
var
|
|
80575
|
+
var import_node_crypto56;
|
|
81337
80576
|
var init_render3 = __esm({
|
|
81338
80577
|
"src/newsroom-publisher/render.ts"() {
|
|
81339
80578
|
"use strict";
|
|
81340
|
-
|
|
80579
|
+
import_node_crypto56 = require("crypto");
|
|
81341
80580
|
}
|
|
81342
80581
|
});
|
|
81343
80582
|
|
|
@@ -81429,7 +80668,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
81429
80668
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
81430
80669
|
</style></head><body data-theme="${escapeHtml12(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml12(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml12(input.eyebrow)}</span><h1>${escapeHtml12(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml12(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml12(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml12(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml12(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
81431
80670
|
const bytes = Buffer.byteLength(html);
|
|
81432
|
-
const sha2565 = (0,
|
|
80671
|
+
const sha2565 = (0, import_node_crypto57.createHash)("sha256").update(html).digest("hex");
|
|
81433
80672
|
return {
|
|
81434
80673
|
html,
|
|
81435
80674
|
filename: "blog-article.html",
|
|
@@ -81439,11 +80678,11 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
81439
80678
|
generatedAt: generatedAtIso
|
|
81440
80679
|
};
|
|
81441
80680
|
}
|
|
81442
|
-
var
|
|
80681
|
+
var import_node_crypto57;
|
|
81443
80682
|
var init_render4 = __esm({
|
|
81444
80683
|
"src/blog-article/render.ts"() {
|
|
81445
80684
|
"use strict";
|
|
81446
|
-
|
|
80685
|
+
import_node_crypto57 = require("crypto");
|
|
81447
80686
|
}
|
|
81448
80687
|
});
|
|
81449
80688
|
|
|
@@ -81774,7 +81013,7 @@ function policy6() {
|
|
|
81774
81013
|
};
|
|
81775
81014
|
}
|
|
81776
81015
|
function runStorageSegment(runId) {
|
|
81777
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0,
|
|
81016
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0, import_node_crypto58.createHash)("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
81778
81017
|
}
|
|
81779
81018
|
async function createScheduledRunArtifact(args) {
|
|
81780
81019
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -81815,12 +81054,12 @@ async function readScheduledRunArtifact(args) {
|
|
|
81815
81054
|
if (!window2 || window2.nextOffset !== null || window2.totalBytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) return null;
|
|
81816
81055
|
return window2.text;
|
|
81817
81056
|
}
|
|
81818
|
-
var
|
|
81057
|
+
var import_node_crypto58, SCHEDULED_RUN_ARTIFACT_PREFIX, SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS, SCHEDULED_RUN_ARTIFACT_MAX_BYTES;
|
|
81819
81058
|
var init_scheduled_run_artifact_store = __esm({
|
|
81820
81059
|
"src/scheduled-artifacts/scheduled-run-artifact-store.ts"() {
|
|
81821
81060
|
"use strict";
|
|
81822
81061
|
init_private_artifacts();
|
|
81823
|
-
|
|
81062
|
+
import_node_crypto58 = require("crypto");
|
|
81824
81063
|
SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
81825
81064
|
SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
81826
81065
|
SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -82038,7 +81277,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
82038
81277
|
updated_at = excluded.updated_at
|
|
82039
81278
|
`,
|
|
82040
81279
|
args: [
|
|
82041
|
-
(0,
|
|
81280
|
+
(0, import_node_crypto59.randomUUID)(),
|
|
82042
81281
|
userId,
|
|
82043
81282
|
connection.providerConfigKey,
|
|
82044
81283
|
connection.provider,
|
|
@@ -82109,7 +81348,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
82109
81348
|
});
|
|
82110
81349
|
await getDb().execute({
|
|
82111
81350
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
82112
|
-
args: [(0,
|
|
81351
|
+
args: [(0, import_node_crypto59.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
82113
81352
|
});
|
|
82114
81353
|
}
|
|
82115
81354
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -82149,7 +81388,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
82149
81388
|
if (!connection) throw new Error("service_connection_not_found");
|
|
82150
81389
|
const inserted = await getDb().execute({
|
|
82151
81390
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
82152
|
-
args: [(0,
|
|
81391
|
+
args: [(0, import_node_crypto59.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
82153
81392
|
});
|
|
82154
81393
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
82155
81394
|
const existing = await getDb().execute({
|
|
@@ -82174,11 +81413,11 @@ async function claimServiceConnectionAction(args) {
|
|
|
82174
81413
|
...result !== void 0 ? { result } : {}
|
|
82175
81414
|
};
|
|
82176
81415
|
}
|
|
82177
|
-
var
|
|
81416
|
+
var import_node_crypto59, schemaReady3, schemaDb6;
|
|
82178
81417
|
var init_service_connections = __esm({
|
|
82179
81418
|
"src/api/service-connections.ts"() {
|
|
82180
81419
|
"use strict";
|
|
82181
|
-
|
|
81420
|
+
import_node_crypto59 = require("crypto");
|
|
82182
81421
|
init_db();
|
|
82183
81422
|
schemaReady3 = null;
|
|
82184
81423
|
schemaDb6 = null;
|
|
@@ -82192,8 +81431,8 @@ function signingSecret() {
|
|
|
82192
81431
|
return secret2;
|
|
82193
81432
|
}
|
|
82194
81433
|
function schedulerIntegrationSignature(args) {
|
|
82195
|
-
const bodyHash = (0,
|
|
82196
|
-
return (0,
|
|
81434
|
+
const bodyHash = (0, import_node_crypto60.createHash)("sha256").update(args.body).digest("hex");
|
|
81435
|
+
return (0, import_node_crypto60.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
82197
81436
|
${args.path}
|
|
82198
81437
|
${args.timestamp}
|
|
82199
81438
|
${args.nonce}
|
|
@@ -82240,17 +81479,17 @@ async function verifySchedulerIntegrationRequest(request, rawBody) {
|
|
|
82240
81479
|
});
|
|
82241
81480
|
const suppliedBytes = Buffer.from(signature, "hex");
|
|
82242
81481
|
const expectedBytes = Buffer.from(expected, "hex");
|
|
82243
|
-
if (suppliedBytes.length !== expectedBytes.length || !(0,
|
|
81482
|
+
if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto60.timingSafeEqual)(suppliedBytes, expectedBytes)) {
|
|
82244
81483
|
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
82245
81484
|
}
|
|
82246
81485
|
await claimNonce(nonce, timestampMs);
|
|
82247
81486
|
return { requestId };
|
|
82248
81487
|
}
|
|
82249
|
-
var
|
|
81488
|
+
var import_node_crypto60, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
|
|
82250
81489
|
var init_scheduler_integration_auth = __esm({
|
|
82251
81490
|
"src/api/scheduler-integration-auth.ts"() {
|
|
82252
81491
|
"use strict";
|
|
82253
|
-
|
|
81492
|
+
import_node_crypto60 = require("crypto");
|
|
82254
81493
|
init_db();
|
|
82255
81494
|
init_service_connections();
|
|
82256
81495
|
MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
@@ -82558,7 +81797,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
82558
81797
|
finalUrl: checked.parsed.href,
|
|
82559
81798
|
width: dimensions?.width ?? null,
|
|
82560
81799
|
height: dimensions?.height ?? null,
|
|
82561
|
-
sha256: (0,
|
|
81800
|
+
sha256: (0, import_node_crypto61.createHash)("sha256").update(bytes).digest("hex")
|
|
82562
81801
|
};
|
|
82563
81802
|
}
|
|
82564
81803
|
throw new Error("media_redirect_rejected");
|
|
@@ -82672,7 +81911,7 @@ async function packagePageMedia(args) {
|
|
|
82672
81911
|
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
82673
81912
|
files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
|
|
82674
81913
|
const archive = await zipBuffer2(files);
|
|
82675
|
-
const id = (0,
|
|
81914
|
+
const id = (0, import_node_crypto61.randomBytes)(6).toString("hex");
|
|
82676
81915
|
const pointer = await createPrivateArtifact({
|
|
82677
81916
|
policy: policy7(),
|
|
82678
81917
|
ownerId: args.ownerId,
|
|
@@ -82685,11 +81924,11 @@ async function packagePageMedia(args) {
|
|
|
82685
81924
|
const localPath = token5() ? null : (0, import_node_path22.join)(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path22.join)((0, import_node_os15.homedir)(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
82686
81925
|
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
82687
81926
|
}
|
|
82688
|
-
var
|
|
81927
|
+
var import_node_crypto61, import_node_os15, import_node_path22, import_p_limit7, import_yazl3, PAGE_MEDIA_ARTIFACT_PREFIX, PAGE_MEDIA_ARTIFACT_TTL_MS, PAGE_MEDIA_DOWNLOAD_TTL_MS, MAX_FILE_BYTES, MAX_ARCHIVE_MEDIA_BYTES, MAX_INLINE_IMAGE_BYTES2, MAX_INLINE_TOTAL_BYTES2, DOWNLOAD_CONCURRENCY2, MAX_REDIRECTS3;
|
|
82689
81928
|
var init_page_media_artifacts = __esm({
|
|
82690
81929
|
"src/api/page-media-artifacts.ts"() {
|
|
82691
81930
|
"use strict";
|
|
82692
|
-
|
|
81931
|
+
import_node_crypto61 = require("crypto");
|
|
82693
81932
|
import_node_os15 = require("os");
|
|
82694
81933
|
import_node_path22 = require("path");
|
|
82695
81934
|
import_p_limit7 = __toESM(require("p-limit"), 1);
|
|
@@ -82798,7 +82037,7 @@ var init_site_extract_reconciliation = __esm({
|
|
|
82798
82037
|
|
|
82799
82038
|
// src/api/page-diff.ts
|
|
82800
82039
|
function sha256Hex(value) {
|
|
82801
|
-
return (0,
|
|
82040
|
+
return (0, import_node_crypto62.createHash)("sha256").update(value).digest("hex");
|
|
82802
82041
|
}
|
|
82803
82042
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
82804
82043
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -82855,11 +82094,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
82855
82094
|
totalChangedLineCount
|
|
82856
82095
|
};
|
|
82857
82096
|
}
|
|
82858
|
-
var
|
|
82097
|
+
var import_node_crypto62, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
82859
82098
|
var init_page_diff = __esm({
|
|
82860
82099
|
"src/api/page-diff.ts"() {
|
|
82861
82100
|
"use strict";
|
|
82862
|
-
|
|
82101
|
+
import_node_crypto62 = require("crypto");
|
|
82863
82102
|
import_diff = require("diff");
|
|
82864
82103
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
82865
82104
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -82933,7 +82172,7 @@ var init_scrape_vault_sink = __esm({
|
|
|
82933
82172
|
|
|
82934
82173
|
// src/api/scrape-image-sink.ts
|
|
82935
82174
|
function idempotencyKey3(userId, vault, input) {
|
|
82936
|
-
return `scrape-image-${(0,
|
|
82175
|
+
return `scrape-image-${(0, import_node_crypto63.createHash)("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
82937
82176
|
}
|
|
82938
82177
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
82939
82178
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -82975,11 +82214,11 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
|
82975
82214
|
assets
|
|
82976
82215
|
};
|
|
82977
82216
|
}
|
|
82978
|
-
var
|
|
82217
|
+
var import_node_crypto63, MAX_IMAGES_PER_SCRAPE;
|
|
82979
82218
|
var init_scrape_image_sink = __esm({
|
|
82980
82219
|
"src/api/scrape-image-sink.ts"() {
|
|
82981
82220
|
"use strict";
|
|
82982
|
-
|
|
82221
|
+
import_node_crypto63 = require("crypto");
|
|
82983
82222
|
init_memory();
|
|
82984
82223
|
MAX_IMAGES_PER_SCRAPE = 25;
|
|
82985
82224
|
}
|
|
@@ -83432,7 +82671,7 @@ function canonicalJson(value) {
|
|
|
83432
82671
|
return JSON.stringify(value);
|
|
83433
82672
|
}
|
|
83434
82673
|
function sha2563(value) {
|
|
83435
|
-
return (0,
|
|
82674
|
+
return (0, import_node_crypto64.createHash)("sha256").update(value).digest("hex");
|
|
83436
82675
|
}
|
|
83437
82676
|
function sensitiveKey(key) {
|
|
83438
82677
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -83647,11 +82886,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
83647
82886
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
83648
82887
|
};
|
|
83649
82888
|
}
|
|
83650
|
-
var
|
|
82889
|
+
var import_node_crypto64, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
83651
82890
|
var init_connection_memory_import = __esm({
|
|
83652
82891
|
"src/api/connection-memory-import.ts"() {
|
|
83653
82892
|
"use strict";
|
|
83654
|
-
|
|
82893
|
+
import_node_crypto64 = require("crypto");
|
|
83655
82894
|
init_slugify();
|
|
83656
82895
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
83657
82896
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -83735,7 +82974,7 @@ var init_scrape_blob_cleanup = __esm({
|
|
|
83735
82974
|
|
|
83736
82975
|
// src/api/site-export-reader.ts
|
|
83737
82976
|
function sha2564(value) {
|
|
83738
|
-
return (0,
|
|
82977
|
+
return (0, import_node_crypto65.createHash)("sha256").update(value).digest("hex");
|
|
83739
82978
|
}
|
|
83740
82979
|
function publicPageRecord(page) {
|
|
83741
82980
|
const { bodyMarkdown: _body, contentRef: _contentRef, discoveryLinks: _discovery, ...metadata } = page;
|
|
@@ -83843,11 +83082,11 @@ async function readOwnedSiteExportImage(input) {
|
|
|
83843
83082
|
if (!bytes || artifact.sha256 && sha2564(bytes) !== artifact.sha256) return null;
|
|
83844
83083
|
return { bytes, artifact };
|
|
83845
83084
|
}
|
|
83846
|
-
var
|
|
83085
|
+
var import_node_crypto65, SiteExportFormatUnavailableError;
|
|
83847
83086
|
var init_site_export_reader = __esm({
|
|
83848
83087
|
"src/api/site-export-reader.ts"() {
|
|
83849
83088
|
"use strict";
|
|
83850
|
-
|
|
83089
|
+
import_node_crypto65 = require("crypto");
|
|
83851
83090
|
init_site_extract_repository();
|
|
83852
83091
|
init_site_extract_content_store();
|
|
83853
83092
|
init_site_extract_artifacts();
|
|
@@ -83998,7 +83237,7 @@ function finitePositive(value, fallback) {
|
|
|
83998
83237
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
83999
83238
|
}
|
|
84000
83239
|
async function collectConnectedDataExport(args) {
|
|
84001
|
-
const exportId = (0,
|
|
83240
|
+
const exportId = (0, import_node_crypto66.randomUUID)();
|
|
84002
83241
|
const now = args.now ?? Date.now;
|
|
84003
83242
|
const startedAt = now();
|
|
84004
83243
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -84112,11 +83351,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
84112
83351
|
untrustedContent: true
|
|
84113
83352
|
};
|
|
84114
83353
|
}
|
|
84115
|
-
var
|
|
83354
|
+
var import_node_crypto66, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
84116
83355
|
var init_connected_data_export = __esm({
|
|
84117
83356
|
"src/api/connected-data-export.ts"() {
|
|
84118
83357
|
"use strict";
|
|
84119
|
-
|
|
83358
|
+
import_node_crypto66 = require("crypto");
|
|
84120
83359
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
84121
83360
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
84122
83361
|
);
|
|
@@ -84230,7 +83469,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
84230
83469
|
offset += rows.length;
|
|
84231
83470
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
84232
83471
|
}
|
|
84233
|
-
const exportId = (0,
|
|
83472
|
+
const exportId = (0, import_node_crypto67.randomUUID)();
|
|
84234
83473
|
const artifact = await args.writeArtifact({
|
|
84235
83474
|
ownerId: args.ownerId,
|
|
84236
83475
|
exportId,
|
|
@@ -84252,11 +83491,11 @@ async function exportSearchConsoleTableData(args) {
|
|
|
84252
83491
|
warnings
|
|
84253
83492
|
};
|
|
84254
83493
|
}
|
|
84255
|
-
var
|
|
83494
|
+
var import_node_crypto67, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
|
|
84256
83495
|
var init_search_console_table_export = __esm({
|
|
84257
83496
|
"src/api/search-console-table-export.ts"() {
|
|
84258
83497
|
"use strict";
|
|
84259
|
-
|
|
83498
|
+
import_node_crypto67 = require("crypto");
|
|
84260
83499
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
84261
83500
|
SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
84262
83501
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -84329,48 +83568,6 @@ var init_credit_operations = __esm({
|
|
|
84329
83568
|
}
|
|
84330
83569
|
});
|
|
84331
83570
|
|
|
84332
|
-
// src/api/harvest-attempt-events.ts
|
|
84333
|
-
function createHarvestAttemptRecorder(jobId2, userId) {
|
|
84334
|
-
return async (event2) => {
|
|
84335
|
-
if (event2.type === "started") {
|
|
84336
|
-
await startHarvestAttempt({
|
|
84337
|
-
jobId: jobId2,
|
|
84338
|
-
userId,
|
|
84339
|
-
attemptNumber: event2.attemptNumber,
|
|
84340
|
-
maxAttempts: event2.maxAttempts,
|
|
84341
|
-
query: event2.query,
|
|
84342
|
-
location: event2.location,
|
|
84343
|
-
maxQuestions: event2.maxQuestions,
|
|
84344
|
-
startedAt: event2.startedAt
|
|
84345
|
-
});
|
|
84346
|
-
return;
|
|
84347
|
-
}
|
|
84348
|
-
await finishHarvestAttempt({
|
|
84349
|
-
jobId: jobId2,
|
|
84350
|
-
attemptNumber: event2.attemptNumber,
|
|
84351
|
-
outcome: event2.outcome,
|
|
84352
|
-
kernelSessionId: event2.kernelSessionId,
|
|
84353
|
-
questionCount: event2.questionCount,
|
|
84354
|
-
durationMs: event2.durationMs,
|
|
84355
|
-
error: event2.error,
|
|
84356
|
-
willRetry: event2.willRetry,
|
|
84357
|
-
kernelDeleteStarted: event2.cleanup.kernelDeleteStarted,
|
|
84358
|
-
kernelDeleteSucceeded: event2.cleanup.kernelDeleteSucceeded,
|
|
84359
|
-
kernelDeleteError: event2.cleanup.kernelDeleteError,
|
|
84360
|
-
browserCloseSucceeded: event2.cleanup.browserCloseSucceeded,
|
|
84361
|
-
browserCloseError: event2.cleanup.browserCloseError,
|
|
84362
|
-
debug: event2.debug,
|
|
84363
|
-
completedAt: event2.completedAt
|
|
84364
|
-
});
|
|
84365
|
-
};
|
|
84366
|
-
}
|
|
84367
|
-
var init_harvest_attempt_events = __esm({
|
|
84368
|
-
"src/api/harvest-attempt-events.ts"() {
|
|
84369
|
-
"use strict";
|
|
84370
|
-
init_db();
|
|
84371
|
-
}
|
|
84372
|
-
});
|
|
84373
|
-
|
|
84374
83571
|
// src/api/memory-universe.ts
|
|
84375
83572
|
function stripExt(value) {
|
|
84376
83573
|
return value.replace(/\.(md|markdown|txt)$/i, "");
|
|
@@ -85368,7 +84565,7 @@ async function requireOwnedActiveConnection(identity, connectionId, allowNeedsRe
|
|
|
85368
84565
|
return connection;
|
|
85369
84566
|
}
|
|
85370
84567
|
async function withNangoClient(connection, run) {
|
|
85371
|
-
const transport = new
|
|
84568
|
+
const transport = new import_client13.StreamableHTTPClientTransport(nangoMcpUrl(), {
|
|
85372
84569
|
requestInit: {
|
|
85373
84570
|
headers: {
|
|
85374
84571
|
Authorization: `Bearer ${nangoSecret()}`,
|
|
@@ -85377,7 +84574,7 @@ async function withNangoClient(connection, run) {
|
|
|
85377
84574
|
}
|
|
85378
84575
|
}
|
|
85379
84576
|
});
|
|
85380
|
-
const client2 = new
|
|
84577
|
+
const client2 = new import_client13.Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
|
|
85381
84578
|
try {
|
|
85382
84579
|
await client2.connect(transport);
|
|
85383
84580
|
const value = await run(client2);
|
|
@@ -85435,7 +84632,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
85435
84632
|
});
|
|
85436
84633
|
const readTools = [...policies.values()].filter((policy8) => policy8.classification === "read").map((policy8) => policy8.name);
|
|
85437
84634
|
const actionTools = [...policies.values()].filter((policy8) => policy8.classification === "action").map((policy8) => policy8.name);
|
|
85438
|
-
const revision = (0,
|
|
84635
|
+
const revision = (0, import_node_crypto68.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
85439
84636
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
85440
84637
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
85441
84638
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -85462,8 +84659,8 @@ async function callNangoToolDirect(args) {
|
|
|
85462
84659
|
identity: args.identity,
|
|
85463
84660
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
85464
84661
|
});
|
|
85465
|
-
const requestId = args.requestId?.trim() || (0,
|
|
85466
|
-
const idempotencyKey4 = `main-nango:${(0,
|
|
84662
|
+
const requestId = args.requestId?.trim() || (0, import_node_crypto68.randomUUID)();
|
|
84663
|
+
const idempotencyKey4 = `main-nango:${(0, import_node_crypto68.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
85467
84664
|
const startedAt = /* @__PURE__ */ new Date();
|
|
85468
84665
|
const started = performance.now();
|
|
85469
84666
|
let result;
|
|
@@ -85488,7 +84685,7 @@ async function callNangoToolDirect(args) {
|
|
|
85488
84685
|
toolName: args.tool,
|
|
85489
84686
|
operationKind: args.operationKind ?? args.classification,
|
|
85490
84687
|
outcome: providerError ? "error" : "partial",
|
|
85491
|
-
requestId: requestId.length <= 200 ? requestId : (0,
|
|
84688
|
+
requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto68.createHash)("sha256").update(requestId).digest("hex"),
|
|
85492
84689
|
startedAt: startedAt.toISOString(),
|
|
85493
84690
|
completedAt: completedAt.toISOString()
|
|
85494
84691
|
}
|
|
@@ -85530,16 +84727,16 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
85530
84727
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
85531
84728
|
protocolVersion: null,
|
|
85532
84729
|
schemaSource: "live_tools_list",
|
|
85533
|
-
schemaHash: (0,
|
|
84730
|
+
schemaHash: (0, import_node_crypto68.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
85534
84731
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
85535
84732
|
};
|
|
85536
84733
|
}
|
|
85537
|
-
var
|
|
84734
|
+
var import_node_crypto68, import_client13, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
|
|
85538
84735
|
var init_main_nango_transport = __esm({
|
|
85539
84736
|
"src/api/main-nango-transport.ts"() {
|
|
85540
84737
|
"use strict";
|
|
85541
|
-
|
|
85542
|
-
|
|
84738
|
+
import_node_crypto68 = require("crypto");
|
|
84739
|
+
import_client13 = require("@modelcontextprotocol/client");
|
|
85543
84740
|
init_service_connections();
|
|
85544
84741
|
init_connected_usage_billing();
|
|
85545
84742
|
init_rates();
|
|
@@ -86162,8 +85359,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
86162
85359
|
return data.connection.actionsEnabled === true;
|
|
86163
85360
|
}
|
|
86164
85361
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey4) {
|
|
86165
|
-
const requestId = `main-connected-action:${(0,
|
|
86166
|
-
const requestDigest = (0,
|
|
85362
|
+
const requestId = `main-connected-action:${(0, import_node_crypto69.createHash)("sha256").update(identity).update("\0").update(idempotencyKey4?.trim() || (0, import_node_crypto69.randomUUID)()).digest("hex")}`;
|
|
85363
|
+
const requestDigest = (0, import_node_crypto69.createHash)("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
86167
85364
|
if (mainOwnsIntegrations()) {
|
|
86168
85365
|
const selectedTool = tool?.trim();
|
|
86169
85366
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -86314,7 +85511,7 @@ function canonicalJson2(value) {
|
|
|
86314
85511
|
return JSON.stringify(value);
|
|
86315
85512
|
}
|
|
86316
85513
|
function projectedToolSchemaHash(tool) {
|
|
86317
|
-
return (0,
|
|
85514
|
+
return (0, import_node_crypto69.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
86318
85515
|
}
|
|
86319
85516
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
86320
85517
|
if (mainOwnsIntegrations()) {
|
|
@@ -86591,11 +85788,11 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
86591
85788
|
untrustedContent: true
|
|
86592
85789
|
};
|
|
86593
85790
|
}
|
|
86594
|
-
var
|
|
85791
|
+
var import_node_crypto69, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
|
|
86595
85792
|
var init_nango_control = __esm({
|
|
86596
85793
|
"src/api/nango-control.ts"() {
|
|
86597
85794
|
"use strict";
|
|
86598
|
-
|
|
85795
|
+
import_node_crypto69 = require("crypto");
|
|
86599
85796
|
init_connected_data_export();
|
|
86600
85797
|
init_slack_connected_data_export();
|
|
86601
85798
|
init_main_nango_transport();
|
|
@@ -86951,7 +86148,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
86951
86148
|
return isRecord5(data) ? data.result ?? data : data;
|
|
86952
86149
|
}
|
|
86953
86150
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey4) {
|
|
86954
|
-
const requestId = `main-resend-action:${(0,
|
|
86151
|
+
const requestId = `main-resend-action:${(0, import_node_crypto70.createHash)("sha256").update(identity).update("\0").update(idempotencyKey4.trim()).digest("hex")}`;
|
|
86955
86152
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
86956
86153
|
method: "POST",
|
|
86957
86154
|
headers: { "x-request-id": requestId },
|
|
@@ -87016,12 +86213,12 @@ async function callResendExportPage(identity, input) {
|
|
|
87016
86213
|
untrustedContent: true
|
|
87017
86214
|
};
|
|
87018
86215
|
}
|
|
87019
|
-
var
|
|
86216
|
+
var import_node_crypto70, DEFAULT_CONNECTION_CONTROL_URL, RESEND_PROVIDER_CONFIG_KEY, RESEND_LOGO_URL, RESEND_DOCS_URL, RESEND_ADMIN_BLOCKED_TOOLS, RESEND_CONNECTION_SYNC_REQUIRED_TOOLS, ResendControlError;
|
|
87020
86217
|
var init_resend_control = __esm({
|
|
87021
86218
|
"src/api/resend-control.ts"() {
|
|
87022
86219
|
"use strict";
|
|
87023
86220
|
init_connected_data_export();
|
|
87024
|
-
|
|
86221
|
+
import_node_crypto70 = require("crypto");
|
|
87025
86222
|
DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
87026
86223
|
RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
87027
86224
|
RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -87628,7 +86825,7 @@ function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
|
|
|
87628
86825
|
);
|
|
87629
86826
|
});
|
|
87630
86827
|
}
|
|
87631
|
-
var import_resend3,
|
|
86828
|
+
var import_resend3, import_node_crypto71, import_hono38, import_hono39, import_factory8, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth4, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, deploymentProfile, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS, CRON_TICK_BUDGET_MS, CRON_TICK_DRAIN_BUDGET_MS;
|
|
87632
86829
|
var init_server = __esm({
|
|
87633
86830
|
"src/api/server.ts"() {
|
|
87634
86831
|
"use strict";
|
|
@@ -87641,7 +86838,7 @@ var init_server = __esm({
|
|
|
87641
86838
|
init_og();
|
|
87642
86839
|
import_resend3 = require("resend");
|
|
87643
86840
|
init_url_utils();
|
|
87644
|
-
|
|
86841
|
+
import_node_crypto71 = require("crypto");
|
|
87645
86842
|
init_kpo_extractor();
|
|
87646
86843
|
init_screenshot();
|
|
87647
86844
|
init_media_extractor();
|
|
@@ -87654,7 +86851,6 @@ var init_server = __esm({
|
|
|
87654
86851
|
init_site_extract();
|
|
87655
86852
|
init_directory_workflow2();
|
|
87656
86853
|
init_lead_list_enrichment2();
|
|
87657
|
-
init_paa_harvest();
|
|
87658
86854
|
init_local_sourcebook();
|
|
87659
86855
|
init_site_extract_repository();
|
|
87660
86856
|
init_site_extract_start();
|
|
@@ -87683,7 +86879,7 @@ var init_server = __esm({
|
|
|
87683
86879
|
init_local_sourcebook_routes();
|
|
87684
86880
|
init_location_data_routes();
|
|
87685
86881
|
init_directory_workflow_dispatch();
|
|
87686
|
-
|
|
86882
|
+
init_paa_harvest_direct();
|
|
87687
86883
|
init_paa_harvest_reconciliation();
|
|
87688
86884
|
init_unified_billing();
|
|
87689
86885
|
init_retention_sweeps();
|
|
@@ -89220,7 +88416,7 @@ var init_server = __esm({
|
|
|
89220
88416
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
89221
88417
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
89222
88418
|
} else {
|
|
89223
|
-
jobId2 = derivedJobId ?? (0,
|
|
88419
|
+
jobId2 = derivedJobId ?? (0, import_node_crypto71.randomUUID)();
|
|
89224
88420
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
89225
88421
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
89226
88422
|
const hold = await debitMcIdempotent(
|
|
@@ -89234,7 +88430,7 @@ var init_server = __esm({
|
|
|
89234
88430
|
try {
|
|
89235
88431
|
await createJobWithId(jobId2, user.id, options.query, {
|
|
89236
88432
|
...options,
|
|
89237
|
-
executionOwner: "
|
|
88433
|
+
executionOwner: "direct",
|
|
89238
88434
|
billingHoldMc: harvestCost,
|
|
89239
88435
|
billingDebitKey,
|
|
89240
88436
|
...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
|
|
@@ -89254,11 +88450,9 @@ var init_server = __esm({
|
|
|
89254
88450
|
).catch(() => void 0);
|
|
89255
88451
|
throw error;
|
|
89256
88452
|
}
|
|
89257
|
-
|
|
89258
|
-
console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
|
|
89259
|
-
});
|
|
88453
|
+
scheduleDirectPaaHarvest(jobId2);
|
|
89260
88454
|
}
|
|
89261
|
-
if (process.env.CRON_SECRET) {
|
|
88455
|
+
if (options.serpOnly && process.env.CRON_SECRET) {
|
|
89262
88456
|
const url = new URL(c.req.url);
|
|
89263
88457
|
void fetch(`${url.origin}/cron/tick`, {
|
|
89264
88458
|
headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
|
|
@@ -90726,7 +89920,7 @@ var init_server = __esm({
|
|
|
90726
89920
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
90727
89921
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
90728
89922
|
const unfinished = [];
|
|
90729
|
-
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation,
|
|
89923
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestReconciliation, commonsEmbeddingDrain, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
90730
89924
|
settleWithinTickBudget("drainQueue", unfinished, drainQueue2(budget), []),
|
|
90731
89925
|
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
90732
89926
|
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
@@ -90745,7 +89939,6 @@ var init_server = __esm({
|
|
|
90745
89939
|
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
90746
89940
|
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
90747
89941
|
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
90748
|
-
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
90749
89942
|
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
90750
89943
|
settleWithinTickBudget("commonsEmbeddingDrain", unfinished, embedQueuedCommonsDocuments(50), { claimed: 0, embedded: 0, failed: 0, remaining: 0 }),
|
|
90751
89944
|
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
@@ -90780,7 +89973,6 @@ var init_server = __esm({
|
|
|
90780
89973
|
siteExtractArtifactCleanup,
|
|
90781
89974
|
siteExtractRedispatch,
|
|
90782
89975
|
siteExtractReconciliation,
|
|
90783
|
-
paaHarvestRedispatch,
|
|
90784
89976
|
paaHarvestReconciliation,
|
|
90785
89977
|
commonsEmbeddingDrain,
|
|
90786
89978
|
localSourcebookRedispatch,
|
|
@@ -90847,7 +90039,7 @@ var init_server = __esm({
|
|
|
90847
90039
|
});
|
|
90848
90040
|
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono39.serve)({
|
|
90849
90041
|
client: inngest,
|
|
90850
|
-
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn,
|
|
90042
|
+
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn, localSourcebookFn]
|
|
90851
90043
|
}));
|
|
90852
90044
|
app.route("/admin/credits", adminCreditsApp);
|
|
90853
90045
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|