mcp-scraper 0.66.1 → 0.66.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +996 -1788
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +71 -87
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-WID3MFRS.js → chunk-2XVDLF6H.js} +1 -1
- package/dist/{chunk-4OT7BAIE.js → chunk-3EYALCYK.js} +1 -1
- package/dist/{chunk-TNLGHNLX.js → chunk-3TKKJQB4.js} +1 -1
- package/dist/{chunk-GTGSD4YS.js → chunk-55B3T5L2.js} +21 -26
- package/dist/{chunk-TNJIPV74.js → chunk-ABYBPODM.js} +1 -1
- package/dist/{chunk-NS754CRS.js → chunk-CETDPNMZ.js} +23 -23
- package/dist/{chunk-75VW7CWE.js → chunk-DBWWIK7O.js} +1 -1
- package/dist/{chunk-VRJ7CO6L.js → chunk-JBFLOJGM.js} +1 -1
- package/dist/{chunk-4P3PHFPQ.js → chunk-K7Y27ND3.js} +2 -12
- package/dist/{chunk-3PIXKLGZ.js → chunk-KXRWY7DZ.js} +1 -1
- package/dist/{chunk-E4LMY5OJ.js → chunk-LNLMOBP6.js} +147 -91
- package/dist/{chunk-AQKJOSIB.js → chunk-PMCT2YQM.js} +1 -1
- package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js} +9 -9
- package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js} +3 -3
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -2
- package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js} +2 -2
- package/dist/{server-ELEJNRRE.js → server-D47PNEXR.js} +413 -1249
- package/dist/{site-extract-repository-ASYEJO7Z.js → site-extract-repository-NTM4MDQN.js} +2 -2
- package/dist/{worker-MAVYYWM7.js → worker-UTBUWEXQ.js} +5 -5
- package/package.json +5 -2
package/dist/bin/api-server.cjs
CHANGED
|
@@ -3899,7 +3899,7 @@ __export(db_exports, {
|
|
|
3899
3899
|
cancelJob: () => cancelJob,
|
|
3900
3900
|
checkRateLimit: () => checkRateLimit,
|
|
3901
3901
|
checkpointJob: () => checkpointJob,
|
|
3902
|
-
|
|
3902
|
+
claimDurablePaaJob: () => claimDurablePaaJob,
|
|
3903
3903
|
claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
|
|
3904
3904
|
claimPendingJob: () => claimPendingJob,
|
|
3905
3905
|
claimPendingKpoJob: () => claimPendingKpoJob,
|
|
@@ -3929,6 +3929,7 @@ __export(db_exports, {
|
|
|
3929
3929
|
expireOldLots: () => expireOldLots,
|
|
3930
3930
|
failJob: () => failJob,
|
|
3931
3931
|
failKpoJob: () => failKpoJob,
|
|
3932
|
+
failRunningJob: () => failRunningJob,
|
|
3932
3933
|
failStripeEvent: () => failStripeEvent,
|
|
3933
3934
|
failWorkflowRunRecord: () => failWorkflowRunRecord,
|
|
3934
3935
|
finishHarvestAttempt: () => finishHarvestAttempt,
|
|
@@ -3968,10 +3969,9 @@ __export(db_exports, {
|
|
|
3968
3969
|
listInboxMessages: () => listInboxMessages,
|
|
3969
3970
|
listJobs: () => listJobs,
|
|
3970
3971
|
listKpoJobs: () => listKpoJobs,
|
|
3971
|
-
|
|
3972
|
-
listRecentTerminalInngestPaaJobs: () => listRecentTerminalInngestPaaJobs,
|
|
3972
|
+
listRecentTerminalDurablePaaJobs: () => listRecentTerminalDurablePaaJobs,
|
|
3973
3973
|
listRequestEvents: () => listRequestEvents,
|
|
3974
|
-
|
|
3974
|
+
listStaleRunningDurablePaaJobs: () => listStaleRunningDurablePaaJobs,
|
|
3975
3975
|
listUsers: () => listUsers,
|
|
3976
3976
|
listUsersDueForConnectedAccountBillingReconciliation: () => listUsersDueForConnectedAccountBillingReconciliation,
|
|
3977
3977
|
listWorkflowArtifacts: () => listWorkflowArtifacts,
|
|
@@ -5905,7 +5905,7 @@ async function claimPendingJob() {
|
|
|
5905
5905
|
const res = await db.execute(`
|
|
5906
5906
|
SELECT * FROM jobs
|
|
5907
5907
|
WHERE status = 'pending'
|
|
5908
|
-
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron')
|
|
5908
|
+
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'inngest')
|
|
5909
5909
|
ORDER BY created_at
|
|
5910
5910
|
LIMIT 1
|
|
5911
5911
|
`);
|
|
@@ -5917,10 +5917,10 @@ async function claimPendingJob() {
|
|
|
5917
5917
|
});
|
|
5918
5918
|
return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
|
|
5919
5919
|
}
|
|
5920
|
-
async function
|
|
5920
|
+
async function claimDurablePaaJob(id) {
|
|
5921
5921
|
const db = getDb();
|
|
5922
5922
|
const current = await getJob(id);
|
|
5923
|
-
if (!current || current.options.executionOwner !== "
|
|
5923
|
+
if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
|
|
5924
5924
|
if (current.status === "running") return void 0;
|
|
5925
5925
|
if (current.status === "pending") {
|
|
5926
5926
|
const updated = await db.execute({
|
|
@@ -5929,7 +5929,7 @@ async function claimInngestPaaJob(id) {
|
|
|
5929
5929
|
WHERE id = ? AND status = 'pending'`,
|
|
5930
5930
|
args: [id]
|
|
5931
5931
|
});
|
|
5932
|
-
if (updated.rowsAffected === 0) return
|
|
5932
|
+
if (updated.rowsAffected === 0) return void 0;
|
|
5933
5933
|
return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
|
|
5934
5934
|
}
|
|
5935
5935
|
return current;
|
|
@@ -5941,25 +5941,12 @@ async function checkpointJob(id, result) {
|
|
|
5941
5941
|
});
|
|
5942
5942
|
return updated.rowsAffected === 1;
|
|
5943
5943
|
}
|
|
5944
|
-
async function
|
|
5945
|
-
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5946
|
-
const rows = await getDb().execute({
|
|
5947
|
-
sql: `SELECT * FROM jobs
|
|
5948
|
-
WHERE status = 'pending'
|
|
5949
|
-
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
5950
|
-
AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
|
|
5951
|
-
ORDER BY created_at
|
|
5952
|
-
LIMIT ?`,
|
|
5953
|
-
args: [safeLimit]
|
|
5954
|
-
});
|
|
5955
|
-
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5956
|
-
}
|
|
5957
|
-
async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
5944
|
+
async function listRecentTerminalDurablePaaJobs(limit = 25) {
|
|
5958
5945
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5959
5946
|
const rows = await getDb().execute({
|
|
5960
5947
|
sql: `SELECT * FROM jobs
|
|
5961
5948
|
WHERE status IN ('done', 'failed', 'cancelled')
|
|
5962
|
-
AND json_extract(options, '$.executionOwner')
|
|
5949
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
5963
5950
|
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
5964
5951
|
AND json_extract(options, '$.billingSettledAt') IS NULL
|
|
5965
5952
|
ORDER BY completed_at DESC
|
|
@@ -5968,13 +5955,13 @@ async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
|
5968
5955
|
});
|
|
5969
5956
|
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5970
5957
|
}
|
|
5971
|
-
async function
|
|
5958
|
+
async function listStaleRunningDurablePaaJobs(limit = 25) {
|
|
5972
5959
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5973
5960
|
const rows = await getDb().execute({
|
|
5974
5961
|
sql: `SELECT * FROM jobs
|
|
5975
5962
|
WHERE status = 'running'
|
|
5976
|
-
AND started_at <= datetime('now', '-
|
|
5977
|
-
AND json_extract(options, '$.executionOwner')
|
|
5963
|
+
AND started_at <= datetime('now', '-15 minutes')
|
|
5964
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
5978
5965
|
ORDER BY started_at
|
|
5979
5966
|
LIMIT ?`,
|
|
5980
5967
|
args: [safeLimit]
|
|
@@ -6003,6 +5990,14 @@ async function failJob(id, error, publicError2) {
|
|
|
6003
5990
|
args: [error, serializePublicErrorEnvelope(publicError2), id]
|
|
6004
5991
|
});
|
|
6005
5992
|
}
|
|
5993
|
+
async function failRunningJob(id, error, publicError2) {
|
|
5994
|
+
const updated = await getDb().execute({
|
|
5995
|
+
sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
|
|
5996
|
+
WHERE id = ? AND status = 'running'`,
|
|
5997
|
+
args: [error, serializePublicErrorEnvelope(publicError2), id]
|
|
5998
|
+
});
|
|
5999
|
+
return updated.rowsAffected === 1;
|
|
6000
|
+
}
|
|
6006
6001
|
async function cancelJob(id, reason, publicError2) {
|
|
6007
6002
|
await getDb().execute({
|
|
6008
6003
|
sql: `UPDATE jobs SET status = 'cancelled', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
@@ -13808,12 +13803,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
13808
13803
|
`{"error_severity":"blocking","error_code":"dep-001","message":"Required Python package missing: {package}. Install with: pip install networkx pandas lxml","file":null,"remediation":"Run: pip install networkx pandas lxml"}`,
|
|
13809
13804
|
`\`\`\``,
|
|
13810
13805
|
``,
|
|
13811
|
-
`Also
|
|
13806
|
+
`Also check whether the optional uszips.csv classifier dataset is accessible. When unavailable, Phase 3 remains functional and falls back to LLM classification:`,
|
|
13812
13807
|
``,
|
|
13813
13808
|
`\`\`\`bash`,
|
|
13814
13809
|
`python3 -c "`,
|
|
13815
13810
|
`import csv, os`,
|
|
13816
|
-
`path = '/
|
|
13811
|
+
`path = os.environ.get('US_ZIPS_CSV_PATH') or os.environ.get('MCP_SCRAPER_USZIPS_CSV_PATH') or '/tmp/uszips.csv'`,
|
|
13817
13812
|
`if not os.path.exists(path):`,
|
|
13818
13813
|
` print('uszips:missing')`,
|
|
13819
13814
|
`else:`,
|
|
@@ -13827,12 +13822,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
13827
13822
|
`If \`uszips:missing\`, write a **warning** (non-blocking) to \`ingestion_error_log.jsonl\`:`,
|
|
13828
13823
|
``,
|
|
13829
13824
|
`\`\`\`json`,
|
|
13830
|
-
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv not found at
|
|
13825
|
+
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv was not found at the resolved path. The location classifier is disabled and Phase 3 will use LLM classification, which may increase cost.","file":"{resolved_uszips_path}","remediation":"Optionally set US_ZIPS_CSV_PATH or MCP_SCRAPER_USZIPS_CSV_PATH to a readable uszips.csv file with the expected schema."}`,
|
|
13831
13826
|
`\`\`\``,
|
|
13832
13827
|
``,
|
|
13833
13828
|
`If \`uszips:schema-mismatch\`, write the same warning with the mismatch detail in the message. The audit continues without the location classifier \u2014 all URLs fall through to Claude classification.`,
|
|
13834
13829
|
``,
|
|
13835
|
-
`
|
|
13830
|
+
`Continue after recording either warning; missing or mismatched uszips.csv data is not a blocking gate.`,
|
|
13836
13831
|
``,
|
|
13837
13832
|
`---`,
|
|
13838
13833
|
``,
|
|
@@ -14766,6 +14761,9 @@ var init_score_synthesize = __esm({
|
|
|
14766
14761
|
});
|
|
14767
14762
|
|
|
14768
14763
|
// src/services/site-architecture-auditor/site-audit-service.ts
|
|
14764
|
+
function resolveSiteAuditUsZipsCsvPath(env = process.env) {
|
|
14765
|
+
return env["US_ZIPS_CSV_PATH"]?.trim() || env["MCP_SCRAPER_USZIPS_CSV_PATH"]?.trim() || "/tmp/uszips.csv";
|
|
14766
|
+
}
|
|
14769
14767
|
var import_zod10, import_p_limit2, OrphanAnnotationOutputSchema, CompareRecommendOutputSchema, SiteAuditService;
|
|
14770
14768
|
var init_site_audit_service = __esm({
|
|
14771
14769
|
"src/services/site-architecture-auditor/site-audit-service.ts"() {
|
|
@@ -14986,7 +14984,7 @@ var init_site_audit_service = __esm({
|
|
|
14986
14984
|
currentPayloadChars += itemChars;
|
|
14987
14985
|
}
|
|
14988
14986
|
if (currentBatch.length > 0) batches.push(currentBatch);
|
|
14989
|
-
const usZipsCsvPath =
|
|
14987
|
+
const usZipsCsvPath = resolveSiteAuditUsZipsCsvPath();
|
|
14990
14988
|
await this.runLlmBilled(job.user_id, `${jobId2}:phase3-classify`, async () => {
|
|
14991
14989
|
for (const batch of batches) {
|
|
14992
14990
|
let classified = [];
|
|
@@ -31852,1301 +31850,6 @@ var init_lead_list_enrichment2 = __esm({
|
|
|
31852
31850
|
}
|
|
31853
31851
|
});
|
|
31854
31852
|
|
|
31855
|
-
// src/api/paa-capture-artifacts.ts
|
|
31856
|
-
function paaCaptureArtifactPolicy() {
|
|
31857
|
-
return {
|
|
31858
|
-
prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
|
|
31859
|
-
artifactTtlMs: 7 * DAY_MS,
|
|
31860
|
-
downloadTtlMs: 15 * 60 * 1e3,
|
|
31861
|
-
token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
|
|
31862
|
-
};
|
|
31863
|
-
}
|
|
31864
|
-
async function createPaaRawDomArtifact(input) {
|
|
31865
|
-
return createPrivateArtifact({
|
|
31866
|
-
policy: paaCaptureArtifactPolicy(),
|
|
31867
|
-
ownerId: String(input.userId),
|
|
31868
|
-
artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
|
|
31869
|
-
createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
|
|
31870
|
-
filename: `${input.jobId}-serp.html.gz`,
|
|
31871
|
-
contentType: "application/gzip",
|
|
31872
|
-
content: input.gzip
|
|
31873
|
-
});
|
|
31874
|
-
}
|
|
31875
|
-
async function tryCreatePaaRawDomArtifact(input) {
|
|
31876
|
-
try {
|
|
31877
|
-
return {
|
|
31878
|
-
artifact: await createPaaRawDomArtifact(input),
|
|
31879
|
-
status: "stored"
|
|
31880
|
-
};
|
|
31881
|
-
} catch (error) {
|
|
31882
|
-
console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
|
|
31883
|
-
return { artifact: null, status: "unavailable" };
|
|
31884
|
-
}
|
|
31885
|
-
}
|
|
31886
|
-
var DAY_MS, PAA_CAPTURE_ARTIFACT_PREFIX;
|
|
31887
|
-
var init_paa_capture_artifacts = __esm({
|
|
31888
|
-
"src/api/paa-capture-artifacts.ts"() {
|
|
31889
|
-
"use strict";
|
|
31890
|
-
init_private_artifacts();
|
|
31891
|
-
DAY_MS = 24 * 60 * 60 * 1e3;
|
|
31892
|
-
PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
|
|
31893
|
-
}
|
|
31894
|
-
});
|
|
31895
|
-
|
|
31896
|
-
// src/api/paa-harvest-settlement.ts
|
|
31897
|
-
function capturedQuestionCount(result) {
|
|
31898
|
-
if (!result || typeof result !== "object") return 0;
|
|
31899
|
-
const value = result;
|
|
31900
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
31901
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
31902
|
-
}
|
|
31903
|
-
function finalCost(result, heldMc) {
|
|
31904
|
-
const questions = capturedQuestionCount(result);
|
|
31905
|
-
if (questions <= 0) return 0;
|
|
31906
|
-
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
31907
|
-
}
|
|
31908
|
-
async function settlePaaHarvestJob(jobOrId) {
|
|
31909
|
-
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
31910
|
-
if (!job) return false;
|
|
31911
|
-
const options = job.options;
|
|
31912
|
-
const debitKey2 = options.billingDebitKey;
|
|
31913
|
-
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
31914
|
-
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
31915
|
-
const settlement = await settleDebitMcIdempotent(
|
|
31916
|
-
job.user_id,
|
|
31917
|
-
debitKey2,
|
|
31918
|
-
finalCost(job.result, heldMc),
|
|
31919
|
-
LedgerOperation.PAA_REFUND,
|
|
31920
|
-
"durable PAA harvest settlement",
|
|
31921
|
-
"paa_harvest"
|
|
31922
|
-
);
|
|
31923
|
-
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
31924
|
-
return true;
|
|
31925
|
-
}
|
|
31926
|
-
var init_paa_harvest_settlement = __esm({
|
|
31927
|
-
"src/api/paa-harvest-settlement.ts"() {
|
|
31928
|
-
"use strict";
|
|
31929
|
-
init_db();
|
|
31930
|
-
init_rates();
|
|
31931
|
-
}
|
|
31932
|
-
});
|
|
31933
|
-
|
|
31934
|
-
// src/api/serp-identity-db.ts
|
|
31935
|
-
async function createSerpIdentityRow(input) {
|
|
31936
|
-
const id = `serpi_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
31937
|
-
await getDb().execute({
|
|
31938
|
-
sql: `INSERT INTO serp_identities
|
|
31939
|
-
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
31940
|
-
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
31941
|
-
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
31942
|
-
});
|
|
31943
|
-
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
31944
|
-
if (!row) throw new Error("SERP identity insert failed");
|
|
31945
|
-
return row;
|
|
31946
|
-
}
|
|
31947
|
-
async function getSerpIdentityRow(userId, name) {
|
|
31948
|
-
const result = await getDb().execute({
|
|
31949
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
31950
|
-
args: [userId, name]
|
|
31951
|
-
});
|
|
31952
|
-
return result.rows[0] ?? null;
|
|
31953
|
-
}
|
|
31954
|
-
async function listSerpIdentityRows(userId) {
|
|
31955
|
-
const result = await getDb().execute({
|
|
31956
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
31957
|
-
args: [userId]
|
|
31958
|
-
});
|
|
31959
|
-
return result.rows;
|
|
31960
|
-
}
|
|
31961
|
-
async function touchSerpIdentity(userId, name) {
|
|
31962
|
-
await getDb().execute({
|
|
31963
|
-
sql: `UPDATE serp_identities
|
|
31964
|
-
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
31965
|
-
WHERE user_id = ? AND name = ?`,
|
|
31966
|
-
args: [userId, name]
|
|
31967
|
-
});
|
|
31968
|
-
}
|
|
31969
|
-
async function setSerpIdentityStatus(userId, name, status) {
|
|
31970
|
-
await getDb().execute({
|
|
31971
|
-
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
31972
|
-
args: [status, userId, name]
|
|
31973
|
-
});
|
|
31974
|
-
}
|
|
31975
|
-
async function deleteSerpIdentityRow(userId, name) {
|
|
31976
|
-
await getDb().execute({
|
|
31977
|
-
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
31978
|
-
args: [userId, name]
|
|
31979
|
-
});
|
|
31980
|
-
}
|
|
31981
|
-
var import_node_crypto15;
|
|
31982
|
-
var init_serp_identity_db = __esm({
|
|
31983
|
-
"src/api/serp-identity-db.ts"() {
|
|
31984
|
-
"use strict";
|
|
31985
|
-
import_node_crypto15 = require("crypto");
|
|
31986
|
-
init_db();
|
|
31987
|
-
}
|
|
31988
|
-
});
|
|
31989
|
-
|
|
31990
|
-
// src/api/webhook.ts
|
|
31991
|
-
async function deliverWebhook(url, payload, retries = 3) {
|
|
31992
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
31993
|
-
try {
|
|
31994
|
-
const res = await fetch(url, {
|
|
31995
|
-
method: "POST",
|
|
31996
|
-
headers: { "content-type": "application/json" },
|
|
31997
|
-
body: JSON.stringify(payload),
|
|
31998
|
-
signal: AbortSignal.timeout(1e4)
|
|
31999
|
-
});
|
|
32000
|
-
if (res.ok) return;
|
|
32001
|
-
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
32002
|
-
} catch (err) {
|
|
32003
|
-
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
32004
|
-
}
|
|
32005
|
-
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
32006
|
-
}
|
|
32007
|
-
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
32008
|
-
}
|
|
32009
|
-
var init_webhook = __esm({
|
|
32010
|
-
"src/api/webhook.ts"() {
|
|
32011
|
-
"use strict";
|
|
32012
|
-
}
|
|
32013
|
-
});
|
|
32014
|
-
|
|
32015
|
-
// src/api/harvest-problems.ts
|
|
32016
|
-
function errorMessage4(err) {
|
|
32017
|
-
return err instanceof Error ? err.message : String(err);
|
|
32018
|
-
}
|
|
32019
|
-
function looksLikeTimeout(err, message) {
|
|
32020
|
-
if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
|
|
32021
|
-
return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
|
|
32022
|
-
}
|
|
32023
|
-
function looksLikeCaptcha(message) {
|
|
32024
|
-
return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
|
|
32025
|
-
}
|
|
32026
|
-
function looksLikeProxyTunnelFailure3(message) {
|
|
32027
|
-
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
32028
|
-
}
|
|
32029
|
-
function looksLikeProxyUnavailable3(message) {
|
|
32030
|
-
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
32031
|
-
}
|
|
32032
|
-
function looksLikeInternalServiceUnavailable(message) {
|
|
32033
|
-
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
32034
|
-
}
|
|
32035
|
-
function looksLikeBrowserSessionInterrupted(message) {
|
|
32036
|
-
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
32037
|
-
}
|
|
32038
|
-
function classifyHarvestProblem(err) {
|
|
32039
|
-
const message = errorMessage4(err);
|
|
32040
|
-
if (err instanceof RequestAbortedError) {
|
|
32041
|
-
return {
|
|
32042
|
-
error_code: "request_aborted",
|
|
32043
|
-
error_type: "request_aborted",
|
|
32044
|
-
message: publicErrorMessage("request_aborted"),
|
|
32045
|
-
retryable: true,
|
|
32046
|
-
httpStatus: 408,
|
|
32047
|
-
terminalStatus: "cancelled"
|
|
32048
|
-
};
|
|
32049
|
-
}
|
|
32050
|
-
if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
|
|
32051
|
-
return {
|
|
32052
|
-
error_code: "captcha_exhausted",
|
|
32053
|
-
error_type: "captcha",
|
|
32054
|
-
message: publicErrorMessage("captcha_exhausted"),
|
|
32055
|
-
retryable: true,
|
|
32056
|
-
httpStatus: 503,
|
|
32057
|
-
terminalStatus: "failed"
|
|
32058
|
-
};
|
|
32059
|
-
}
|
|
32060
|
-
if (isVendorUnavailableError(err, message)) {
|
|
32061
|
-
return {
|
|
32062
|
-
error_code: "vendor_unavailable",
|
|
32063
|
-
error_type: "service_unavailable",
|
|
32064
|
-
message: publicErrorMessage("vendor_unavailable"),
|
|
32065
|
-
retryable: true,
|
|
32066
|
-
httpStatus: 503,
|
|
32067
|
-
terminalStatus: "failed"
|
|
32068
|
-
};
|
|
32069
|
-
}
|
|
32070
|
-
if (err instanceof LocationMismatchError) {
|
|
32071
|
-
return {
|
|
32072
|
-
error_code: "location_mismatch",
|
|
32073
|
-
error_type: "location_mismatch",
|
|
32074
|
-
message: publicErrorMessage("location_mismatch"),
|
|
32075
|
-
retryable: true,
|
|
32076
|
-
httpStatus: 503,
|
|
32077
|
-
terminalStatus: "failed"
|
|
32078
|
-
};
|
|
32079
|
-
}
|
|
32080
|
-
if (looksLikeProxyTunnelFailure3(message)) {
|
|
32081
|
-
return {
|
|
32082
|
-
error_code: "proxy_tunnel_failed",
|
|
32083
|
-
error_type: "connection",
|
|
32084
|
-
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
32085
|
-
retryable: true,
|
|
32086
|
-
httpStatus: 503,
|
|
32087
|
-
terminalStatus: "failed"
|
|
32088
|
-
};
|
|
32089
|
-
}
|
|
32090
|
-
if (looksLikeProxyUnavailable3(message)) {
|
|
32091
|
-
return {
|
|
32092
|
-
error_code: "proxy_unavailable",
|
|
32093
|
-
error_type: "connection",
|
|
32094
|
-
message: publicErrorMessage("proxy_unavailable"),
|
|
32095
|
-
retryable: true,
|
|
32096
|
-
httpStatus: 503,
|
|
32097
|
-
terminalStatus: "failed"
|
|
32098
|
-
};
|
|
32099
|
-
}
|
|
32100
|
-
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
32101
|
-
return {
|
|
32102
|
-
error_code: "browser_session_interrupted",
|
|
32103
|
-
error_type: "extraction",
|
|
32104
|
-
message: publicErrorMessage("browser_session_interrupted"),
|
|
32105
|
-
retryable: true,
|
|
32106
|
-
httpStatus: 503,
|
|
32107
|
-
terminalStatus: "failed"
|
|
32108
|
-
};
|
|
32109
|
-
}
|
|
32110
|
-
if (looksLikeTimeout(err, message)) {
|
|
32111
|
-
return {
|
|
32112
|
-
error_code: "harvest_timeout",
|
|
32113
|
-
error_type: "timeout",
|
|
32114
|
-
message: publicErrorMessage("harvest_timeout"),
|
|
32115
|
-
retryable: true,
|
|
32116
|
-
httpStatus: 504,
|
|
32117
|
-
terminalStatus: "failed"
|
|
32118
|
-
};
|
|
32119
|
-
}
|
|
32120
|
-
if (looksLikeInternalServiceUnavailable(message)) {
|
|
32121
|
-
return {
|
|
32122
|
-
error_code: "service_unavailable",
|
|
32123
|
-
error_type: "service_unavailable",
|
|
32124
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
32125
|
-
retryable: true,
|
|
32126
|
-
httpStatus: 503,
|
|
32127
|
-
terminalStatus: "failed"
|
|
32128
|
-
};
|
|
32129
|
-
}
|
|
32130
|
-
return {
|
|
32131
|
-
error_code: "extraction_failed",
|
|
32132
|
-
error_type: "extraction",
|
|
32133
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
32134
|
-
retryable: false,
|
|
32135
|
-
httpStatus: 500,
|
|
32136
|
-
terminalStatus: "failed"
|
|
32137
|
-
};
|
|
32138
|
-
}
|
|
32139
|
-
function serializeHarvestProblem(problem) {
|
|
32140
|
-
return JSON.stringify({
|
|
32141
|
-
error_code: problem.error_code,
|
|
32142
|
-
error_type: problem.error_type,
|
|
32143
|
-
message: problem.message,
|
|
32144
|
-
retryable: problem.retryable
|
|
32145
|
-
});
|
|
32146
|
-
}
|
|
32147
|
-
function harvestProblemResponse(problem, context = {}) {
|
|
32148
|
-
const envelope = buildPublicErrorEnvelope({
|
|
32149
|
-
errorCode: problem.error_code,
|
|
32150
|
-
errorType: problem.error_type,
|
|
32151
|
-
retryable: problem.retryable,
|
|
32152
|
-
retryAfterSeconds: context.retryAfterSeconds,
|
|
32153
|
-
chargeStatus: context.chargeStatus,
|
|
32154
|
-
details: context.details
|
|
32155
|
-
});
|
|
32156
|
-
return {
|
|
32157
|
-
error: envelope.message,
|
|
32158
|
-
...envelope
|
|
32159
|
-
};
|
|
32160
|
-
}
|
|
32161
|
-
function harvestProblemEnvelope(problem, context = {}) {
|
|
32162
|
-
const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context);
|
|
32163
|
-
return envelope;
|
|
32164
|
-
}
|
|
32165
|
-
function publicErrorBoundary(err, context = {}) {
|
|
32166
|
-
const problem = classifyHarvestProblem(err);
|
|
32167
|
-
return {
|
|
32168
|
-
body: harvestProblemResponse(problem, context),
|
|
32169
|
-
status: problem.httpStatus
|
|
32170
|
-
};
|
|
32171
|
-
}
|
|
32172
|
-
var init_harvest_problems = __esm({
|
|
32173
|
-
"src/api/harvest-problems.ts"() {
|
|
32174
|
-
"use strict";
|
|
32175
|
-
init_errors();
|
|
32176
|
-
init_vendor_errors();
|
|
32177
|
-
}
|
|
32178
|
-
});
|
|
32179
|
-
|
|
32180
|
-
// src/paa/durable-capture.ts
|
|
32181
|
-
function remainingSeconds(deadlineMs, ceiling) {
|
|
32182
|
-
return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
|
|
32183
|
-
}
|
|
32184
|
-
async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
|
|
32185
|
-
if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
|
|
32186
|
-
const response = await kernel.browsers.playwright.execute(sessionId, {
|
|
32187
|
-
code,
|
|
32188
|
-
timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
|
|
32189
|
-
});
|
|
32190
|
-
if (!response.success || response.result === null || response.result === void 0) {
|
|
32191
|
-
throw new Error(response.error || response.stderr || "co-located browser execution failed");
|
|
32192
|
-
}
|
|
32193
|
-
return response.result;
|
|
32194
|
-
}
|
|
32195
|
-
function navigationCode(searchUrl) {
|
|
32196
|
-
return `
|
|
32197
|
-
const wait = ms => page.waitForTimeout(ms);
|
|
32198
|
-
const block = async () => {
|
|
32199
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
32200
|
-
if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
|
|
32201
|
-
const title = await page.title().catch(() => '');
|
|
32202
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
32203
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
|
|
32204
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
|
|
32205
|
-
return null;
|
|
32206
|
-
};
|
|
32207
|
-
await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
32208
|
-
let captchaObserved = false;
|
|
32209
|
-
let solverCleared = false;
|
|
32210
|
-
let blocked = await block();
|
|
32211
|
-
const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
|
|
32212
|
-
while (blocked === 'captcha' && Date.now() < solverDeadline) {
|
|
32213
|
-
captchaObserved = true;
|
|
32214
|
-
await wait(4000 + Math.floor(Math.random() * 1500));
|
|
32215
|
-
blocked = await block();
|
|
32216
|
-
if (!blocked) solverCleared = true;
|
|
32217
|
-
}
|
|
32218
|
-
let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
32219
|
-
for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
|
|
32220
|
-
await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
|
|
32221
|
-
await wait(350 + Math.floor(Math.random() * 500));
|
|
32222
|
-
count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
32223
|
-
}
|
|
32224
|
-
await page.evaluate((itemSelector) => {
|
|
32225
|
-
const state = window;
|
|
32226
|
-
if (state.__mcpTrustedListenerInstalled) return;
|
|
32227
|
-
document.addEventListener('click', event => {
|
|
32228
|
-
if (event.target instanceof Element && event.target.closest(itemSelector)) {
|
|
32229
|
-
state.__mcpLastPaaClickTrusted = event.isTrusted;
|
|
32230
|
-
}
|
|
32231
|
-
}, { capture: true });
|
|
32232
|
-
state.__mcpTrustedListenerInstalled = true;
|
|
32233
|
-
}, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
|
|
32234
|
-
return {
|
|
32235
|
-
outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
|
|
32236
|
-
finalUrl: page.url(),
|
|
32237
|
-
title: await page.title().catch(() => ''),
|
|
32238
|
-
captchaObserved,
|
|
32239
|
-
solverCleared,
|
|
32240
|
-
environment: await page.evaluate(() => ({
|
|
32241
|
-
userAgent: navigator.userAgent,
|
|
32242
|
-
language: navigator.language,
|
|
32243
|
-
platform: navigator.platform,
|
|
32244
|
-
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
32245
|
-
innerWidth: window.innerWidth,
|
|
32246
|
-
innerHeight: window.innerHeight,
|
|
32247
|
-
outerWidth: window.outerWidth,
|
|
32248
|
-
outerHeight: window.outerHeight,
|
|
32249
|
-
screenWidth: window.screen.width,
|
|
32250
|
-
screenHeight: window.screen.height,
|
|
32251
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
32252
|
-
})).catch(() => null),
|
|
32253
|
-
};
|
|
32254
|
-
`;
|
|
32255
|
-
}
|
|
32256
|
-
function recordReaderSource() {
|
|
32257
|
-
return `
|
|
32258
|
-
const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
|
|
32259
|
-
const externalHref = anchor => {
|
|
32260
|
-
const href = anchor && anchor.href || '';
|
|
32261
|
-
if (!/^https?:\\/\\//i.test(href)) return '';
|
|
32262
|
-
try {
|
|
32263
|
-
const url = new URL(href);
|
|
32264
|
-
if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
|
|
32265
|
-
const redirected = url.searchParams.get('q') || url.searchParams.get('url');
|
|
32266
|
-
return redirected && redirected.startsWith('http') ? redirected : '';
|
|
32267
|
-
}
|
|
32268
|
-
} catch {}
|
|
32269
|
-
return href;
|
|
32270
|
-
};
|
|
32271
|
-
const sourceLabel = value => clean(value)
|
|
32272
|
-
.replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
|
|
32273
|
-
.replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
|
|
32274
|
-
const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
|
|
32275
|
-
const question = clean(item.getAttribute(SEL.itemDataQ)
|
|
32276
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
32277
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText);
|
|
32278
|
-
const expanded = item.classList.contains(SEL.expandedClass)
|
|
32279
|
-
|| item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
|
|
32280
|
-
const container = expanded ? item.querySelector(SEL.answerContainer) : null;
|
|
32281
|
-
const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
|
|
32282
|
-
const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
|
|
32283
|
-
SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
|
|
32284
|
-
) || []) : [];
|
|
32285
|
-
const byUrl = new Map();
|
|
32286
|
-
for (const anchor of anchors) {
|
|
32287
|
-
const url = externalHref(anchor);
|
|
32288
|
-
if (!url || byUrl.has(url)) continue;
|
|
32289
|
-
let site = '';
|
|
32290
|
-
try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
32291
|
-
byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
|
|
32292
|
-
}
|
|
32293
|
-
const sources = Array.from(byUrl.values());
|
|
32294
|
-
let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
|
|
32295
|
-
let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
|
|
32296
|
-
let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
|
|
32297
|
-
const primary = sources.find(source => source.title.length > 2) || sources[0];
|
|
32298
|
-
if (!sourceCite && primary) {
|
|
32299
|
-
sourceTitle ||= primary.title;
|
|
32300
|
-
sourceSite ||= primary.site;
|
|
32301
|
-
sourceCite = primary.url;
|
|
32302
|
-
}
|
|
32303
|
-
return {
|
|
32304
|
-
question,
|
|
32305
|
-
answer: clean(answerElement?.innerText || answerElement?.textContent),
|
|
32306
|
-
sourceTitle,
|
|
32307
|
-
sourceSite,
|
|
32308
|
-
sourceCite,
|
|
32309
|
-
sources,
|
|
32310
|
-
depth: 1,
|
|
32311
|
-
parentQuestion: null,
|
|
32312
|
-
};
|
|
32313
|
-
}).filter(record => record.question);
|
|
32314
|
-
`;
|
|
32315
|
-
}
|
|
32316
|
-
function expansionCode(clickedQuestions) {
|
|
32317
|
-
return `
|
|
32318
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
32319
|
-
${recordReaderSource()}
|
|
32320
|
-
const deliveryBlock = async () => {
|
|
32321
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
32322
|
-
if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
|
|
32323
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
32324
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
|
|
32325
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
|
|
32326
|
-
return null;
|
|
32327
|
-
};
|
|
32328
|
-
const blocked = await deliveryBlock();
|
|
32329
|
-
const records = await page.evaluate(({ SEL }) => {
|
|
32330
|
-
${recordReaderSource()}
|
|
32331
|
-
return readRecords();
|
|
32332
|
-
}, { SEL });
|
|
32333
|
-
if (blocked) return {
|
|
32334
|
-
outcome: blocked, targetQuestion: null, records, newQuestions: [],
|
|
32335
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32336
|
-
interactionTrusted: null, clickError: null,
|
|
32337
|
-
};
|
|
32338
|
-
const clicked = new Set(${JSON.stringify(clickedQuestions)});
|
|
32339
|
-
const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
|
|
32340
|
-
if (!targetQuestion) return {
|
|
32341
|
-
outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
|
|
32342
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32343
|
-
interactionTrusted: null, clickError: null,
|
|
32344
|
-
};
|
|
32345
|
-
const items = page.locator(SEL.item);
|
|
32346
|
-
let targetIndex = -1;
|
|
32347
|
-
for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
|
|
32348
|
-
const question = await items.nth(index).evaluate((item, SEL) =>
|
|
32349
|
-
item.getAttribute(SEL.itemDataQ)
|
|
32350
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
32351
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
|
|
32352
|
-
|| '', SEL).catch(() => '');
|
|
32353
|
-
if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
|
|
32354
|
-
}
|
|
32355
|
-
if (targetIndex < 0) return {
|
|
32356
|
-
outcome: 'exhausted', targetQuestion, records, newQuestions: [],
|
|
32357
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
32358
|
-
interactionTrusted: null, clickError: 'question disappeared before click',
|
|
32359
|
-
};
|
|
32360
|
-
const beforeQuestions = new Set(records.map(record => record.question));
|
|
32361
|
-
const item = items.nth(targetIndex);
|
|
32362
|
-
await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
|
|
32363
|
-
await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
|
|
32364
|
-
await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
|
|
32365
|
-
let clickError = null;
|
|
32366
|
-
try {
|
|
32367
|
-
await item.locator(SEL.clickTarget).first().click({
|
|
32368
|
-
timeout: 6000,
|
|
32369
|
-
delay: 55 + Math.floor(Math.random() * 120),
|
|
32370
|
-
});
|
|
32371
|
-
} catch (error) {
|
|
32372
|
-
clickError = String(error && error.message || error).slice(0, 240);
|
|
32373
|
-
}
|
|
32374
|
-
const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
|
|
32375
|
-
await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
|
|
32376
|
-
const afterRecords = await page.evaluate(({ SEL }) => {
|
|
32377
|
-
${recordReaderSource()}
|
|
32378
|
-
return readRecords();
|
|
32379
|
-
}, { SEL });
|
|
32380
|
-
return {
|
|
32381
|
-
outcome: clickError ? 'exhausted' : 'clicked',
|
|
32382
|
-
targetQuestion,
|
|
32383
|
-
records: afterRecords,
|
|
32384
|
-
newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
|
|
32385
|
-
finalUrl: page.url(),
|
|
32386
|
-
title: await page.title().catch(() => ''),
|
|
32387
|
-
interactionTrusted,
|
|
32388
|
-
clickError,
|
|
32389
|
-
};
|
|
32390
|
-
`;
|
|
32391
|
-
}
|
|
32392
|
-
function finalCaptureCode(maxQuestions, parentByQuestion) {
|
|
32393
|
-
const surfaces = {
|
|
32394
|
-
aiOverview: AIOverviewSelectors,
|
|
32395
|
-
aiMode: AIModeSelectors,
|
|
32396
|
-
organic: OrganicSelectors,
|
|
32397
|
-
localPack: LocalPackSelectors,
|
|
32398
|
-
video: VideoSelectors,
|
|
32399
|
-
forum: ForumSelectors,
|
|
32400
|
-
whatPeopleSaying: WhatPeopleSayingSelectors
|
|
32401
|
-
};
|
|
32402
|
-
return `
|
|
32403
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
32404
|
-
const SURF = ${JSON.stringify(surfaces)};
|
|
32405
|
-
const PARENTS = ${JSON.stringify(parentByQuestion)};
|
|
32406
|
-
${recordReaderSource()}
|
|
32407
|
-
const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
|
|
32408
|
-
${recordReaderSource()}
|
|
32409
|
-
const unique = (items, key) => {
|
|
32410
|
-
const seen = new Set();
|
|
32411
|
-
return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
|
|
32412
|
-
};
|
|
32413
|
-
const paa = readRecords().map(record => {
|
|
32414
|
-
const parentQuestion = PARENTS[record.question] || null;
|
|
32415
|
-
let depth = 1, parent = parentQuestion;
|
|
32416
|
-
const seen = new Set();
|
|
32417
|
-
while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
|
|
32418
|
-
return { ...record, depth, parentQuestion };
|
|
32419
|
-
}).slice(0, maxQuestions);
|
|
32420
|
-
const organicResults = [];
|
|
32421
|
-
document.querySelectorAll(SURF.organic.result).forEach(card => {
|
|
32422
|
-
const titleElement = card.querySelector(SURF.organic.title);
|
|
32423
|
-
const title = clean(titleElement?.textContent);
|
|
32424
|
-
const link = titleElement?.closest('a');
|
|
32425
|
-
const url = externalHref(link);
|
|
32426
|
-
if (!title || !url) return;
|
|
32427
|
-
let domain = '';
|
|
32428
|
-
try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
32429
|
-
const rating = card.querySelector(SURF.organic.ratingWrap);
|
|
32430
|
-
organicResults.push({
|
|
32431
|
-
position: organicResults.length + 1, title, url, domain,
|
|
32432
|
-
cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
|
|
32433
|
-
snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
|
|
32434
|
-
isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
|
|
32435
|
-
inlineRating: rating ? {
|
|
32436
|
-
value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
|
|
32437
|
-
count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
|
|
32438
|
-
} : null,
|
|
32439
|
-
});
|
|
32440
|
-
});
|
|
32441
|
-
const localPack = [];
|
|
32442
|
-
document.querySelectorAll(SURF.localPack.card).forEach(card => {
|
|
32443
|
-
const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
|
|
32444
|
-
if (!name) return;
|
|
32445
|
-
const links = Array.from(card.querySelectorAll('a[href]'));
|
|
32446
|
-
let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
|
|
32447
|
-
for (const link of links) {
|
|
32448
|
-
if (cid) break;
|
|
32449
|
-
const match = link.href.match(/[?&]cid=(\\d+)/);
|
|
32450
|
-
if (match) cid = match[1];
|
|
32451
|
-
}
|
|
32452
|
-
localPack.push({
|
|
32453
|
-
position: localPack.length + 1, name, cid,
|
|
32454
|
-
rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
|
|
32455
|
-
reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
|
|
32456
|
-
metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
|
|
32457
|
-
directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
|
|
32458
|
-
});
|
|
32459
|
-
});
|
|
32460
|
-
const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
|
|
32461
|
-
const url = externalHref(link);
|
|
32462
|
-
const raw = clean(link.textContent);
|
|
32463
|
-
const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
|
|
32464
|
-
return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
|
|
32465
|
-
}).filter(item => item.title && item.url), item => item.url);
|
|
32466
|
-
const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
|
|
32467
|
-
title: clean(link.querySelector(SURF.forum.title)?.textContent),
|
|
32468
|
-
source: clean(link.querySelector(SURF.forum.source)?.textContent),
|
|
32469
|
-
url: externalHref(link),
|
|
32470
|
-
})).filter(item => item.title && item.url), item => item.url);
|
|
32471
|
-
const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
|
|
32472
|
-
const whatPeopleSaying = unique(socialCards.map(card => {
|
|
32473
|
-
const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
|
|
32474
|
-
const url = externalHref(link);
|
|
32475
|
-
const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
|
|
32476
|
-
const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
|
|
32477
|
-
const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
|
|
32478
|
-
const identity = (platform + source + url).toLowerCase();
|
|
32479
|
-
const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
|
|
32480
|
-
return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
|
|
32481
|
-
}).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
|
|
32482
|
-
const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
|
|
32483
|
-
const aimRoot = document.querySelector(SURF.aiMode.root);
|
|
32484
|
-
const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
|
|
32485
|
-
text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
|
|
32486
|
-
})).filter(item => item.href), item => item.href);
|
|
32487
|
-
const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
|
|
32488
|
-
const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
|
|
32489
|
-
const html = document.documentElement.outerHTML;
|
|
32490
|
-
const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
|
|
32491
|
-
const cids = unique([
|
|
32492
|
-
...localPack.map(item => item.cid).filter(Boolean),
|
|
32493
|
-
...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
|
|
32494
|
-
], value => value);
|
|
32495
|
-
const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
|
|
32496
|
-
const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
|
|
32497
|
-
const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
|
|
32498
|
-
const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
|
|
32499
|
-
text: clean(link.textContent), url: link.href,
|
|
32500
|
-
})).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
|
|
32501
|
-
const counts = {
|
|
32502
|
-
peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
|
|
32503
|
-
videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
|
|
32504
|
-
aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
|
|
32505
|
-
relatedSearches: relatedSearches.length,
|
|
32506
|
-
};
|
|
32507
|
-
return {
|
|
32508
|
-
paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
|
|
32509
|
-
videos, forums, whatPeopleSaying, aiOverview, aiMode,
|
|
32510
|
-
surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
|
|
32511
|
-
entityIds: { entities, kgIds, cids, gcids },
|
|
32512
|
-
serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
|
|
32513
|
-
relatedSearches,
|
|
32514
|
-
};
|
|
32515
|
-
}, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
|
|
32516
|
-
return capture;
|
|
32517
|
-
`;
|
|
32518
|
-
}
|
|
32519
|
-
function rawChunkCode(offset, length) {
|
|
32520
|
-
return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
|
|
32521
|
-
}
|
|
32522
|
-
function buildPublicResult(input) {
|
|
32523
|
-
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
32524
|
-
const selected = input.records;
|
|
32525
|
-
const flat = selected.map((record) => ({
|
|
32526
|
-
seed_query: input.query,
|
|
32527
|
-
question: record.question,
|
|
32528
|
-
answer: record.answer,
|
|
32529
|
-
source_title: record.sourceTitle,
|
|
32530
|
-
source_site: record.sourceSite,
|
|
32531
|
-
source_cite: record.sourceCite,
|
|
32532
|
-
depth: record.depth,
|
|
32533
|
-
parent_question: record.parentQuestion ?? "",
|
|
32534
|
-
extracted_at: extractedAt
|
|
32535
|
-
}));
|
|
32536
|
-
const tree = selected.map((record) => ({
|
|
32537
|
-
question: record.question,
|
|
32538
|
-
answer: record.answer || null,
|
|
32539
|
-
sourceTitle: record.sourceTitle || null,
|
|
32540
|
-
sourceSite: record.sourceSite || null,
|
|
32541
|
-
sourceCite: record.sourceCite || null,
|
|
32542
|
-
depth: record.depth,
|
|
32543
|
-
parentQuestion: record.parentQuestion,
|
|
32544
|
-
children: []
|
|
32545
|
-
}));
|
|
32546
|
-
return {
|
|
32547
|
-
seed: input.query,
|
|
32548
|
-
location: input.location,
|
|
32549
|
-
extractedAt,
|
|
32550
|
-
diagnostics: {
|
|
32551
|
-
completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
|
|
32552
|
-
problem: null,
|
|
32553
|
-
resultQuality: input.partial ? "partial" : "complete",
|
|
32554
|
-
degradedResult: input.partial,
|
|
32555
|
-
retryRecommended: false,
|
|
32556
|
-
completeness: {
|
|
32557
|
-
paaWithoutAnswer: selected.filter((record) => !record.answer).length,
|
|
32558
|
-
paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
|
|
32559
|
-
paaAnswersRecovered: 0,
|
|
32560
|
-
aioShareCaptured: null
|
|
32561
|
-
}
|
|
32562
|
-
},
|
|
32563
|
-
totalQuestions: selected.length,
|
|
32564
|
-
surface: input.capture.surface,
|
|
32565
|
-
aiOverview: input.capture.aiOverview,
|
|
32566
|
-
aiMode: input.capture.aiMode,
|
|
32567
|
-
whatPeopleSaying: input.capture.whatPeopleSaying,
|
|
32568
|
-
tree,
|
|
32569
|
-
flat,
|
|
32570
|
-
videos: input.capture.videos,
|
|
32571
|
-
forums: input.capture.forums,
|
|
32572
|
-
organicResults: input.capture.organicResults,
|
|
32573
|
-
localPack: input.capture.localPack,
|
|
32574
|
-
entityIds: input.capture.entityIds,
|
|
32575
|
-
serpFeatures: input.capture.serpFeatures,
|
|
32576
|
-
relatedSearches: input.capture.relatedSearches,
|
|
32577
|
-
stats: {
|
|
32578
|
-
seed: input.query,
|
|
32579
|
-
totalQuestions: selected.length,
|
|
32580
|
-
maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
|
|
32581
|
-
durationMs: input.durationMs,
|
|
32582
|
-
errorCount: input.partial ? 1 : 0
|
|
32583
|
-
},
|
|
32584
|
-
capture: {
|
|
32585
|
-
execution: "kernel_colocated_playwright",
|
|
32586
|
-
stealth: true,
|
|
32587
|
-
managedProxy: input.persistentIdentity ? "serp_identity" : "kernel_default",
|
|
32588
|
-
metadataOverrides: false,
|
|
32589
|
-
locationSignal: input.location ? "uule_only" : "none",
|
|
32590
|
-
attempt: input.attempt,
|
|
32591
|
-
finalUrl: input.navigation.finalUrl,
|
|
32592
|
-
captchaObserved: input.navigation.captchaObserved,
|
|
32593
|
-
solverCleared: input.navigation.solverCleared,
|
|
32594
|
-
environment: input.navigation.environment,
|
|
32595
|
-
rawDomSha256: input.rawDomSha256
|
|
32596
|
-
}
|
|
32597
|
-
};
|
|
32598
|
-
}
|
|
32599
|
-
async function runDurablePaaCapture(input) {
|
|
32600
|
-
const startedAt = input.startedAtMs ?? Date.now();
|
|
32601
|
-
const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
|
|
32602
|
-
const location2 = input.location?.trim() || null;
|
|
32603
|
-
const params = new URLSearchParams({
|
|
32604
|
-
q: input.query,
|
|
32605
|
-
gl: input.gl ?? "us",
|
|
32606
|
-
hl: input.hl ?? "en",
|
|
32607
|
-
pws: "0"
|
|
32608
|
-
});
|
|
32609
|
-
if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
|
|
32610
|
-
const searchUrl = `https://www.google.com/search?${params.toString()}`;
|
|
32611
|
-
const kernel = new import_sdk9.default({ apiKey: input.apiKey });
|
|
32612
|
-
let lastCheckpoint = {
|
|
32613
|
-
phase: "navigating",
|
|
32614
|
-
attempt: 1,
|
|
32615
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32616
|
-
records: [],
|
|
32617
|
-
clickedQuestions: [],
|
|
32618
|
-
finalUrl: null,
|
|
32619
|
-
captchaObserved: false,
|
|
32620
|
-
solverCleared: false,
|
|
32621
|
-
error: null
|
|
32622
|
-
};
|
|
32623
|
-
const persist = async (patch) => {
|
|
32624
|
-
lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
32625
|
-
await input.onCheckpoint?.(lastCheckpoint);
|
|
32626
|
-
};
|
|
32627
|
-
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
32628
|
-
if (Date.now() >= workDeadline) break;
|
|
32629
|
-
let sessionId = null;
|
|
32630
|
-
const records = /* @__PURE__ */ new Map();
|
|
32631
|
-
const clicked = /* @__PURE__ */ new Set();
|
|
32632
|
-
const parents = {};
|
|
32633
|
-
let navigation = null;
|
|
32634
|
-
try {
|
|
32635
|
-
await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
|
|
32636
|
-
const browser = await kernel.browsers.create({
|
|
32637
|
-
stealth: true,
|
|
32638
|
-
timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20)),
|
|
32639
|
-
...input.proxyId ? { proxy_id: input.proxyId } : {},
|
|
32640
|
-
...input.profileName ? { profile: { name: input.profileName, save_changes: false } } : {}
|
|
32641
|
-
});
|
|
32642
|
-
sessionId = browser.session_id;
|
|
32643
|
-
navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
|
|
32644
|
-
await persist({
|
|
32645
|
-
phase: navigation.outcome === "ready" ? "expanding" : "blocked",
|
|
32646
|
-
finalUrl: navigation.finalUrl,
|
|
32647
|
-
captchaObserved: navigation.captchaObserved,
|
|
32648
|
-
solverCleared: navigation.solverCleared
|
|
32649
|
-
});
|
|
32650
|
-
if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
|
|
32651
|
-
if (attempt === 1 && records.size === 0) continue;
|
|
32652
|
-
throw new Error(navigation.outcome);
|
|
32653
|
-
}
|
|
32654
|
-
let stagnant = 0;
|
|
32655
|
-
for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
|
|
32656
|
-
const observed = await execute(
|
|
32657
|
-
kernel,
|
|
32658
|
-
sessionId,
|
|
32659
|
-
expansionCode(Array.from(clicked)),
|
|
32660
|
-
workDeadline,
|
|
32661
|
-
25
|
|
32662
|
-
);
|
|
32663
|
-
const beforeSize = records.size;
|
|
32664
|
-
for (const record of observed.records) {
|
|
32665
|
-
const existing = records.get(record.question);
|
|
32666
|
-
records.set(record.question, {
|
|
32667
|
-
...record,
|
|
32668
|
-
answer: record.answer || existing?.answer || "",
|
|
32669
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
32670
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
32671
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
32672
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
32673
|
-
parentQuestion: parents[record.question] ?? null,
|
|
32674
|
-
depth: parents[record.question] ? 2 : 1
|
|
32675
|
-
});
|
|
32676
|
-
}
|
|
32677
|
-
if (observed.outcome === "clicked" && observed.targetQuestion) {
|
|
32678
|
-
clicked.add(observed.targetQuestion);
|
|
32679
|
-
for (const question of observed.newQuestions) {
|
|
32680
|
-
if (!parents[question]) parents[question] = observed.targetQuestion;
|
|
32681
|
-
}
|
|
32682
|
-
}
|
|
32683
|
-
const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
32684
|
-
await persist({
|
|
32685
|
-
phase: "expanding",
|
|
32686
|
-
records: selected2,
|
|
32687
|
-
clickedQuestions: Array.from(clicked),
|
|
32688
|
-
finalUrl: observed.finalUrl,
|
|
32689
|
-
captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
|
|
32690
|
-
});
|
|
32691
|
-
if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
|
|
32692
|
-
if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
|
|
32693
|
-
stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
|
|
32694
|
-
if (stagnant >= 4) break;
|
|
32695
|
-
}
|
|
32696
|
-
const finalCapture = await execute(
|
|
32697
|
-
kernel,
|
|
32698
|
-
sessionId,
|
|
32699
|
-
finalCaptureCode(input.maxQuestions, parents),
|
|
32700
|
-
workDeadline,
|
|
32701
|
-
45
|
|
32702
|
-
);
|
|
32703
|
-
for (const record of finalCapture.paa) {
|
|
32704
|
-
const existing = records.get(record.question);
|
|
32705
|
-
records.set(record.question, {
|
|
32706
|
-
...record,
|
|
32707
|
-
answer: record.answer || existing?.answer || "",
|
|
32708
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
32709
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
32710
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
32711
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
32712
|
-
parentQuestion: parents[record.question] ?? record.parentQuestion,
|
|
32713
|
-
depth: record.depth
|
|
32714
|
-
});
|
|
32715
|
-
}
|
|
32716
|
-
let rawDomGzip = null;
|
|
32717
|
-
let rawDomSha256 = null;
|
|
32718
|
-
if (Date.now() < workDeadline - 12e3) {
|
|
32719
|
-
const prepared = await execute(
|
|
32720
|
-
kernel,
|
|
32721
|
-
sessionId,
|
|
32722
|
-
RAW_PREPARE_CODE,
|
|
32723
|
-
workDeadline,
|
|
32724
|
-
35
|
|
32725
|
-
);
|
|
32726
|
-
const chunks = [];
|
|
32727
|
-
for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
|
|
32728
|
-
const part = await execute(
|
|
32729
|
-
kernel,
|
|
32730
|
-
sessionId,
|
|
32731
|
-
rawChunkCode(offset, 18e4),
|
|
32732
|
-
workDeadline,
|
|
32733
|
-
15
|
|
32734
|
-
);
|
|
32735
|
-
if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
|
|
32736
|
-
chunks.push(part.chunk);
|
|
32737
|
-
}
|
|
32738
|
-
if (chunks.join("").length === prepared.chars) {
|
|
32739
|
-
const transported = Buffer.from(chunks.join(""), "base64");
|
|
32740
|
-
if (prepared.encoding === "gzip") rawDomGzip = transported;
|
|
32741
|
-
else {
|
|
32742
|
-
const { gzipSync: gzipSync2 } = await import("zlib");
|
|
32743
|
-
rawDomGzip = gzipSync2(transported);
|
|
32744
|
-
}
|
|
32745
|
-
rawDomSha256 = (0, import_node_crypto16.createHash)("sha256").update(rawDomGzip).digest("hex");
|
|
32746
|
-
}
|
|
32747
|
-
}
|
|
32748
|
-
const selected = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
32749
|
-
const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
|
|
32750
|
-
await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
|
|
32751
|
-
return {
|
|
32752
|
-
result: buildPublicResult({
|
|
32753
|
-
query: input.query,
|
|
32754
|
-
location: location2,
|
|
32755
|
-
capture: finalCapture,
|
|
32756
|
-
records: selected,
|
|
32757
|
-
navigation,
|
|
32758
|
-
durationMs: Date.now() - startedAt,
|
|
32759
|
-
partial,
|
|
32760
|
-
attempt,
|
|
32761
|
-
rawDomSha256,
|
|
32762
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
32763
|
-
}),
|
|
32764
|
-
rawDomGzip,
|
|
32765
|
-
rawDomSha256,
|
|
32766
|
-
attempt,
|
|
32767
|
-
partial,
|
|
32768
|
-
checkpoint: lastCheckpoint
|
|
32769
|
-
};
|
|
32770
|
-
} catch (error) {
|
|
32771
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
32772
|
-
await persist({
|
|
32773
|
-
phase: records.size > 0 ? "captured" : "failed",
|
|
32774
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
32775
|
-
clickedQuestions: Array.from(clicked),
|
|
32776
|
-
error: message.slice(0, 500)
|
|
32777
|
-
});
|
|
32778
|
-
if (records.size > 0) {
|
|
32779
|
-
const emptyCapture = {
|
|
32780
|
-
paa: [],
|
|
32781
|
-
organicResults: [],
|
|
32782
|
-
localPack: [],
|
|
32783
|
-
videos: [],
|
|
32784
|
-
forums: [],
|
|
32785
|
-
whatPeopleSaying: [],
|
|
32786
|
-
aiOverview: { detected: false, text: null, citations: [] },
|
|
32787
|
-
aiMode: { detected: false, text: null, citations: [] },
|
|
32788
|
-
surface: "unknown",
|
|
32789
|
-
entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
|
|
32790
|
-
serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
|
|
32791
|
-
relatedSearches: []
|
|
32792
|
-
};
|
|
32793
|
-
return {
|
|
32794
|
-
result: buildPublicResult({
|
|
32795
|
-
query: input.query,
|
|
32796
|
-
location: location2,
|
|
32797
|
-
capture: emptyCapture,
|
|
32798
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
32799
|
-
navigation: navigation ?? {
|
|
32800
|
-
outcome: "soft_block",
|
|
32801
|
-
finalUrl: lastCheckpoint.finalUrl ?? "",
|
|
32802
|
-
title: "",
|
|
32803
|
-
captchaObserved: lastCheckpoint.captchaObserved,
|
|
32804
|
-
solverCleared: lastCheckpoint.solverCleared,
|
|
32805
|
-
environment: null
|
|
32806
|
-
},
|
|
32807
|
-
durationMs: Date.now() - startedAt,
|
|
32808
|
-
partial: true,
|
|
32809
|
-
attempt,
|
|
32810
|
-
rawDomSha256: null,
|
|
32811
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
32812
|
-
}),
|
|
32813
|
-
rawDomGzip: null,
|
|
32814
|
-
rawDomSha256: null,
|
|
32815
|
-
attempt,
|
|
32816
|
-
partial: true,
|
|
32817
|
-
checkpoint: lastCheckpoint
|
|
32818
|
-
};
|
|
32819
|
-
}
|
|
32820
|
-
if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
|
|
32821
|
-
} finally {
|
|
32822
|
-
if (sessionId) {
|
|
32823
|
-
await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
|
|
32824
|
-
}
|
|
32825
|
-
}
|
|
32826
|
-
}
|
|
32827
|
-
throw new Error("paa_work_deadline_exhausted_before_capture");
|
|
32828
|
-
}
|
|
32829
|
-
var import_sdk9, import_node_crypto16, PAA_INVOCATION_BUDGET_MS, PAA_BROWSER_WORK_BUDGET_MS, PAA_SOLVER_BUDGET_MS, RAW_PREPARE_CODE;
|
|
32830
|
-
var init_durable_capture = __esm({
|
|
32831
|
-
"src/paa/durable-capture.ts"() {
|
|
32832
|
-
"use strict";
|
|
32833
|
-
import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
|
|
32834
|
-
import_node_crypto16 = require("crypto");
|
|
32835
|
-
init_selectors();
|
|
32836
|
-
init_uule();
|
|
32837
|
-
PAA_INVOCATION_BUDGET_MS = 28e4;
|
|
32838
|
-
PAA_BROWSER_WORK_BUDGET_MS = 25e4;
|
|
32839
|
-
PAA_SOLVER_BUDGET_MS = 6e4;
|
|
32840
|
-
RAW_PREPARE_CODE = `
|
|
32841
|
-
const result = await page.evaluate(async () => {
|
|
32842
|
-
const source = new TextEncoder().encode(document.documentElement.outerHTML);
|
|
32843
|
-
const compressed = typeof CompressionStream === 'undefined'
|
|
32844
|
-
? source
|
|
32845
|
-
: new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
|
|
32846
|
-
const pieces = [];
|
|
32847
|
-
for (let offset = 0; offset < compressed.length; offset += 32768) {
|
|
32848
|
-
pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
|
|
32849
|
-
}
|
|
32850
|
-
window.__mcpPaaRawDom = btoa(pieces.join(''));
|
|
32851
|
-
return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
|
|
32852
|
-
});
|
|
32853
|
-
return result;
|
|
32854
|
-
`;
|
|
32855
|
-
}
|
|
32856
|
-
});
|
|
32857
|
-
|
|
32858
|
-
// src/inngest/functions/paa-harvest.ts
|
|
32859
|
-
function durableRecordsFromResult(result) {
|
|
32860
|
-
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
32861
|
-
return flat.flatMap((value) => {
|
|
32862
|
-
if (!value || typeof value !== "object") return [];
|
|
32863
|
-
const row = value;
|
|
32864
|
-
const question = typeof row.question === "string" ? row.question : "";
|
|
32865
|
-
if (!question) return [];
|
|
32866
|
-
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
32867
|
-
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
32868
|
-
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
32869
|
-
return [{
|
|
32870
|
-
question,
|
|
32871
|
-
answer: typeof row.answer === "string" ? row.answer : "",
|
|
32872
|
-
sourceTitle,
|
|
32873
|
-
sourceSite,
|
|
32874
|
-
sourceCite,
|
|
32875
|
-
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
32876
|
-
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
32877
|
-
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
32878
|
-
}];
|
|
32879
|
-
});
|
|
32880
|
-
}
|
|
32881
|
-
function questionCount(result) {
|
|
32882
|
-
if (!result || typeof result !== "object") return 0;
|
|
32883
|
-
const value = result;
|
|
32884
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
32885
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
32886
|
-
}
|
|
32887
|
-
var BRIGHTDATA_INVOCATION_BUDGET_MS, BRIGHTDATA_WORK_BUDGET_MS, paaHarvestFn;
|
|
32888
|
-
var init_paa_harvest = __esm({
|
|
32889
|
-
"src/inngest/functions/paa-harvest.ts"() {
|
|
32890
|
-
"use strict";
|
|
32891
|
-
init_client();
|
|
32892
|
-
init_browser_service_env();
|
|
32893
|
-
init_BrightDataSerpDriver();
|
|
32894
|
-
init_harvest();
|
|
32895
|
-
init_db();
|
|
32896
|
-
init_paa_capture_artifacts();
|
|
32897
|
-
init_paa_harvest_settlement();
|
|
32898
|
-
init_serp_identity_db();
|
|
32899
|
-
init_webhook();
|
|
32900
|
-
init_harvest_problems();
|
|
32901
|
-
init_durable_capture();
|
|
32902
|
-
BRIGHTDATA_INVOCATION_BUDGET_MS = 76e4;
|
|
32903
|
-
BRIGHTDATA_WORK_BUDGET_MS = 72e4;
|
|
32904
|
-
paaHarvestFn = inngest.createFunction(
|
|
32905
|
-
{
|
|
32906
|
-
id: "paa-harvest",
|
|
32907
|
-
retries: 0,
|
|
32908
|
-
triggers: [{ event: "mcp-scraper/paa.requested" }],
|
|
32909
|
-
onFailure: async ({ event: event2 }) => {
|
|
32910
|
-
const jobId2 = event2?.data?.event?.data?.jobId;
|
|
32911
|
-
if (!jobId2) return;
|
|
32912
|
-
const current = await getJob(jobId2);
|
|
32913
|
-
if (!current) return;
|
|
32914
|
-
if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
|
|
32915
|
-
await settlePaaHarvestJob(jobId2).catch((error) => {
|
|
32916
|
-
console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
|
|
32917
|
-
});
|
|
32918
|
-
return;
|
|
32919
|
-
}
|
|
32920
|
-
const rawError = String(event2?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
|
|
32921
|
-
const problem = classifyHarvestProblem(new Error(rawError));
|
|
32922
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
32923
|
-
await settlePaaHarvestJob(jobId2).then(async (settled) => {
|
|
32924
|
-
if (!settled) return;
|
|
32925
|
-
const settledJob = await getJob(jobId2);
|
|
32926
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
32927
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
32928
|
-
}).catch((error) => {
|
|
32929
|
-
console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
|
|
32930
|
-
});
|
|
32931
|
-
}
|
|
32932
|
-
},
|
|
32933
|
-
async ({ event: event2 }) => {
|
|
32934
|
-
const invocationStartedAt = Date.now();
|
|
32935
|
-
const job = await claimInngestPaaJob(event2.data.jobId);
|
|
32936
|
-
if (!job) return { jobId: event2.data.jobId, status: "missing_or_not_paa" };
|
|
32937
|
-
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
32938
|
-
await settlePaaHarvestJob(job.id);
|
|
32939
|
-
return { jobId: job.id, status: job.status };
|
|
32940
|
-
}
|
|
32941
|
-
const options = job.options;
|
|
32942
|
-
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
32943
|
-
const serpIdentity = options.serpIdentity ? await getSerpIdentityRow(job.user_id, options.serpIdentity) : null;
|
|
32944
|
-
if (options.serpIdentity && !serpIdentity) throw new Error("SERP identity not found");
|
|
32945
|
-
if (serpIdentity && serpIdentity.status !== "ready") throw new Error("SERP identity is not ready");
|
|
32946
|
-
const useBrightData = brightDataSerpEnabled() && !serpIdentity;
|
|
32947
|
-
const apiKey = browserServiceApiKey();
|
|
32948
|
-
if (!useBrightData && !apiKey) throw new Error("browser service is not configured");
|
|
32949
|
-
const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
32950
|
-
await startHarvestAttempt({
|
|
32951
|
-
jobId: job.id,
|
|
32952
|
-
userId: job.user_id,
|
|
32953
|
-
attemptNumber: 1,
|
|
32954
|
-
maxAttempts: 2,
|
|
32955
|
-
query: options.query ?? job.query,
|
|
32956
|
-
location: options.location ?? null,
|
|
32957
|
-
maxQuestions,
|
|
32958
|
-
startedAt: attemptStartedAt
|
|
32959
|
-
});
|
|
32960
|
-
const checkpointState = { value: null };
|
|
32961
|
-
try {
|
|
32962
|
-
const persistCheckpoint = async (checkpoint, brightData = false) => {
|
|
32963
|
-
checkpointState.value = checkpoint;
|
|
32964
|
-
const persisted = await checkpointJob(job.id, {
|
|
32965
|
-
progress: checkpoint,
|
|
32966
|
-
durable: {
|
|
32967
|
-
invocationBudgetMs: brightData ? BRIGHTDATA_INVOCATION_BUDGET_MS : PAA_INVOCATION_BUDGET_MS,
|
|
32968
|
-
workBudgetMs: brightData ? BRIGHTDATA_WORK_BUDGET_MS : 25e4,
|
|
32969
|
-
cleanupReserveMs: brightData ? 4e4 : 3e4
|
|
32970
|
-
}
|
|
32971
|
-
});
|
|
32972
|
-
if (!persisted) throw new Error("job_left_running_state_during_capture");
|
|
32973
|
-
};
|
|
32974
|
-
const query = options.query ?? job.query;
|
|
32975
|
-
const capture = useBrightData ? await (async () => {
|
|
32976
|
-
const navigating = {
|
|
32977
|
-
phase: "navigating",
|
|
32978
|
-
attempt: 1,
|
|
32979
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32980
|
-
records: [],
|
|
32981
|
-
clickedQuestions: [],
|
|
32982
|
-
finalUrl: null,
|
|
32983
|
-
captchaObserved: false,
|
|
32984
|
-
solverCleared: false,
|
|
32985
|
-
error: null
|
|
32986
|
-
};
|
|
32987
|
-
await persistCheckpoint(navigating, true);
|
|
32988
|
-
const result2 = await harvest({
|
|
32989
|
-
query,
|
|
32990
|
-
location: options.location ?? void 0,
|
|
32991
|
-
gl: options.gl ?? "us",
|
|
32992
|
-
hl: options.hl ?? "en",
|
|
32993
|
-
maxQuestions,
|
|
32994
|
-
serpOnly: options.serpOnly ?? false,
|
|
32995
|
-
maxAttempts: 1,
|
|
32996
|
-
headless: true,
|
|
32997
|
-
format: "json",
|
|
32998
|
-
outputDir: "/tmp/paa-output-inngest",
|
|
32999
|
-
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
33000
|
-
includeLocalPack: options.includeLocalPack ?? false,
|
|
33001
|
-
includeForums: options.includeForums ?? false,
|
|
33002
|
-
includeVideos: options.includeVideos ?? false,
|
|
33003
|
-
includeAiOverview: options.includeAiOverview ?? false,
|
|
33004
|
-
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
33005
|
-
softDeadlineMs: invocationStartedAt + BRIGHTDATA_WORK_BUDGET_MS
|
|
33006
|
-
});
|
|
33007
|
-
const records = durableRecordsFromResult(result2);
|
|
33008
|
-
const captured = {
|
|
33009
|
-
...navigating,
|
|
33010
|
-
phase: "captured",
|
|
33011
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33012
|
-
records,
|
|
33013
|
-
clickedQuestions: records.map((record) => record.question)
|
|
33014
|
-
};
|
|
33015
|
-
await persistCheckpoint(captured, true);
|
|
33016
|
-
return {
|
|
33017
|
-
result: {
|
|
33018
|
-
...result2,
|
|
33019
|
-
capture: {
|
|
33020
|
-
execution: "brightdata_browser_api_controller_fulfilled",
|
|
33021
|
-
provider: "bright_data_browser_api",
|
|
33022
|
-
sameQueryClickGraph: true
|
|
33023
|
-
}
|
|
33024
|
-
},
|
|
33025
|
-
rawDomGzip: null,
|
|
33026
|
-
rawDomSha256: null,
|
|
33027
|
-
attempt: 1,
|
|
33028
|
-
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
33029
|
-
checkpoint: captured
|
|
33030
|
-
};
|
|
33031
|
-
})() : await runDurablePaaCapture({
|
|
33032
|
-
apiKey,
|
|
33033
|
-
query,
|
|
33034
|
-
location: options.location ?? null,
|
|
33035
|
-
gl: options.gl ?? "us",
|
|
33036
|
-
hl: options.hl ?? "en",
|
|
33037
|
-
maxQuestions,
|
|
33038
|
-
...serpIdentity ? {
|
|
33039
|
-
proxyId: serpIdentity.kernel_proxy_id,
|
|
33040
|
-
profileName: serpIdentity.kernel_profile_name
|
|
33041
|
-
} : {},
|
|
33042
|
-
startedAtMs: invocationStartedAt,
|
|
33043
|
-
onCheckpoint: (checkpoint) => persistCheckpoint(checkpoint, false)
|
|
33044
|
-
});
|
|
33045
|
-
const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
|
|
33046
|
-
userId: job.user_id,
|
|
33047
|
-
jobId: job.id,
|
|
33048
|
-
attempt: capture.attempt,
|
|
33049
|
-
gzip: capture.rawDomGzip
|
|
33050
|
-
}) : { artifact: null, status: "unavailable" };
|
|
33051
|
-
const artifact = artifactResult.artifact;
|
|
33052
|
-
const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
|
|
33053
|
-
const captureResult = capture.result;
|
|
33054
|
-
const result = {
|
|
33055
|
-
...captureResult,
|
|
33056
|
-
diagnostics: {
|
|
33057
|
-
...captureResult.diagnostics,
|
|
33058
|
-
...artifactMissing ? {
|
|
33059
|
-
resultQuality: "partial",
|
|
33060
|
-
degradedResult: true,
|
|
33061
|
-
retryRecommended: false
|
|
33062
|
-
} : {}
|
|
33063
|
-
},
|
|
33064
|
-
capture: {
|
|
33065
|
-
...captureResult.capture,
|
|
33066
|
-
rawDomArtifact: artifact,
|
|
33067
|
-
rawDomArtifactStatus: artifactResult.status
|
|
33068
|
-
}
|
|
33069
|
-
};
|
|
33070
|
-
await completeJob(job.id, result);
|
|
33071
|
-
if (options.serpIdentity) await touchSerpIdentity(job.user_id, options.serpIdentity).catch(() => void 0);
|
|
33072
|
-
await finishHarvestAttempt({
|
|
33073
|
-
jobId: job.id,
|
|
33074
|
-
attemptNumber: 1,
|
|
33075
|
-
outcome: capture.partial ? "partial" : "paa_found",
|
|
33076
|
-
kernelSessionId: null,
|
|
33077
|
-
questionCount: questionCount(result),
|
|
33078
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
33079
|
-
error: capture.checkpoint.error,
|
|
33080
|
-
willRetry: false,
|
|
33081
|
-
kernelDeleteStarted: true,
|
|
33082
|
-
kernelDeleteSucceeded: true,
|
|
33083
|
-
kernelDeleteError: null,
|
|
33084
|
-
browserCloseSucceeded: true,
|
|
33085
|
-
browserCloseError: null,
|
|
33086
|
-
debug: {
|
|
33087
|
-
execution: useBrightData ? "brightdata_browser_api_controller_fulfilled" : "kernel_colocated_playwright",
|
|
33088
|
-
stealth: true,
|
|
33089
|
-
attemptUsed: capture.attempt,
|
|
33090
|
-
rawDomStored: Boolean(artifact)
|
|
33091
|
-
},
|
|
33092
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
33093
|
-
});
|
|
33094
|
-
await settlePaaHarvestJob(job.id);
|
|
33095
|
-
if (job.callback_url) {
|
|
33096
|
-
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
33097
|
-
}
|
|
33098
|
-
return {
|
|
33099
|
-
jobId: job.id,
|
|
33100
|
-
status: "done",
|
|
33101
|
-
totalQuestions: questionCount(result),
|
|
33102
|
-
partial: capture.partial,
|
|
33103
|
-
artifactStored: Boolean(artifact)
|
|
33104
|
-
};
|
|
33105
|
-
} catch (error) {
|
|
33106
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
33107
|
-
const problem = classifyHarvestProblem(error);
|
|
33108
|
-
const latestCheckpoint = checkpointState.value;
|
|
33109
|
-
await finishHarvestAttempt({
|
|
33110
|
-
jobId: job.id,
|
|
33111
|
-
attemptNumber: 1,
|
|
33112
|
-
outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
|
|
33113
|
-
kernelSessionId: null,
|
|
33114
|
-
questionCount: latestCheckpoint?.records.length ?? 0,
|
|
33115
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
33116
|
-
error: message.slice(0, 1e3),
|
|
33117
|
-
willRetry: false,
|
|
33118
|
-
kernelDeleteStarted: true,
|
|
33119
|
-
kernelDeleteSucceeded: null,
|
|
33120
|
-
kernelDeleteError: null,
|
|
33121
|
-
browserCloseSucceeded: null,
|
|
33122
|
-
browserCloseError: null,
|
|
33123
|
-
debug: latestCheckpoint,
|
|
33124
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
33125
|
-
});
|
|
33126
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
33127
|
-
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
33128
|
-
if (!settled) return;
|
|
33129
|
-
const settledJob = await getJob(job.id);
|
|
33130
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
33131
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
33132
|
-
}).catch((settlementError) => {
|
|
33133
|
-
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
33134
|
-
});
|
|
33135
|
-
if (job.callback_url) {
|
|
33136
|
-
await deliverWebhook(job.callback_url, {
|
|
33137
|
-
job_id: job.id,
|
|
33138
|
-
status: "failed",
|
|
33139
|
-
...harvestProblemResponse(problem),
|
|
33140
|
-
progress: latestCheckpoint
|
|
33141
|
-
});
|
|
33142
|
-
}
|
|
33143
|
-
throw error;
|
|
33144
|
-
}
|
|
33145
|
-
}
|
|
33146
|
-
);
|
|
33147
|
-
}
|
|
33148
|
-
});
|
|
33149
|
-
|
|
33150
31853
|
// src/api/local-sourcebook-repository.ts
|
|
33151
31854
|
function ensureLocalSourcebookSchema() {
|
|
33152
31855
|
const currentDb = getDb();
|
|
@@ -33259,7 +31962,7 @@ function mapSubmission(row) {
|
|
|
33259
31962
|
async function event(submissionId, eventType, actorKind, actorId, metadata = {}) {
|
|
33260
31963
|
await getDb().execute({
|
|
33261
31964
|
sql: `INSERT INTO local_sourcebook_events (id, submission_id, event_type, actor_kind, actor_id, metadata_json) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
33262
|
-
args: [(0,
|
|
31965
|
+
args: [(0, import_node_crypto15.randomUUID)(), submissionId, eventType, actorKind, actorId, JSON.stringify(metadata)]
|
|
33263
31966
|
});
|
|
33264
31967
|
}
|
|
33265
31968
|
async function createLocalSourcebookSubmission(input) {
|
|
@@ -33273,7 +31976,7 @@ async function createLocalSourcebookSubmission(input) {
|
|
|
33273
31976
|
return submission;
|
|
33274
31977
|
}
|
|
33275
31978
|
}
|
|
33276
|
-
const id = `lsb_${(0,
|
|
31979
|
+
const id = `lsb_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "")}`;
|
|
33277
31980
|
const coverage = {
|
|
33278
31981
|
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
33279
31982
|
crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
|
|
@@ -33414,11 +32117,11 @@ async function getPublicLocalSourcebook(category, state, slug4) {
|
|
|
33414
32117
|
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
|
|
33415
32118
|
return result.rows[0] ? parseJson3(result.rows[0].payload_json) : null;
|
|
33416
32119
|
}
|
|
33417
|
-
var
|
|
32120
|
+
var import_node_crypto15, LOCAL_SOURCEBOOK_CATEGORIES, schemaPromise4, schemaDb4;
|
|
33418
32121
|
var init_local_sourcebook_repository = __esm({
|
|
33419
32122
|
"src/api/local-sourcebook-repository.ts"() {
|
|
33420
32123
|
"use strict";
|
|
33421
|
-
|
|
32124
|
+
import_node_crypto15 = require("crypto");
|
|
33422
32125
|
init_db();
|
|
33423
32126
|
LOCAL_SOURCEBOOK_CATEGORIES = ["home", "professional", "restaurants", "financial", "realestate", "auto", "wellness"];
|
|
33424
32127
|
schemaPromise4 = null;
|
|
@@ -34570,7 +33273,7 @@ var init_local_sourcebook_schema = __esm({
|
|
|
34570
33273
|
|
|
34571
33274
|
// src/api/local-sourcebook-compiler.ts
|
|
34572
33275
|
function digest(value) {
|
|
34573
|
-
return (0,
|
|
33276
|
+
return (0, import_node_crypto16.createHash)("sha256").update(value).digest("hex").slice(0, 20);
|
|
34574
33277
|
}
|
|
34575
33278
|
function unique(values, limit = 100) {
|
|
34576
33279
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -35007,11 +33710,11 @@ function compileLocalSourcebookListing(input) {
|
|
|
35007
33710
|
}
|
|
35008
33711
|
};
|
|
35009
33712
|
}
|
|
35010
|
-
var
|
|
33713
|
+
var import_node_crypto16, CATEGORY_LABELS, STATE_NAMES, GENERIC_HEADINGS, SERVICE_PATH, PRODUCT_PATH, TEAM_PATH, WORK_PATH, CREDENTIAL_PATH, BLOG_PATH, LOCATION_PATH, REVIEW_THEME_RULES;
|
|
35011
33714
|
var init_local_sourcebook_compiler = __esm({
|
|
35012
33715
|
"src/api/local-sourcebook-compiler.ts"() {
|
|
35013
33716
|
"use strict";
|
|
35014
|
-
|
|
33717
|
+
import_node_crypto16 = require("crypto");
|
|
35015
33718
|
init_local_sourcebook_public_urls();
|
|
35016
33719
|
init_local_sourcebook_schema();
|
|
35017
33720
|
CATEGORY_LABELS = {
|
|
@@ -35186,7 +33889,7 @@ function createDefaultLocalSourcebookProvider() {
|
|
|
35186
33889
|
}
|
|
35187
33890
|
};
|
|
35188
33891
|
}
|
|
35189
|
-
function
|
|
33892
|
+
function errorMessage4(error) {
|
|
35190
33893
|
return (error instanceof Error ? error.message : String(error)).replace(/\s+/g, " ").trim().slice(0, 1e3);
|
|
35191
33894
|
}
|
|
35192
33895
|
async function runLocalSourcebookAcquisition(submission, provider = createDefaultLocalSourcebookProvider()) {
|
|
@@ -35219,7 +33922,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35219
33922
|
const successfulPages2 = site.pages.filter(isPageExtractionSuccessful).length;
|
|
35220
33923
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "done", query: submission.websiteUrl, resultCount: successfulPages2, result: { submissionId: submission.id, discovered: site.spider.totalFound, captured: successfulPages2, truncated: site.truncated } });
|
|
35221
33924
|
} catch (error) {
|
|
35222
|
-
siteError =
|
|
33925
|
+
siteError = errorMessage4(error);
|
|
35223
33926
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_website", status: "failed", query: submission.websiteUrl, error: siteError });
|
|
35224
33927
|
}
|
|
35225
33928
|
try {
|
|
@@ -35227,7 +33930,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35227
33930
|
validateLocalSourcebookMapsIdentity(submission, maps);
|
|
35228
33931
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "done", query: submission.businessName, location: submission.state.toUpperCase(), resultCount: maps.reviews.length, result: { submissionId: submission.id, placeUrl: maps.placeUrl, cid: maps.cidDecimal ?? maps.cid, reviewsStatus: maps.reviewsStatus, reviewsRetained: maps.reviews.length, services: maps.services.length, areasServed: maps.areasServed.length } });
|
|
35229
33932
|
} catch (error) {
|
|
35230
|
-
mapsError =
|
|
33933
|
+
mapsError = errorMessage4(error);
|
|
35231
33934
|
maps = null;
|
|
35232
33935
|
await logRequestEvent({ userId: submission.ownerUserId, source: "local_sourcebook_maps", status: "failed", query: submission.businessName, location: submission.state.toUpperCase(), error: mapsError });
|
|
35233
33936
|
}
|
|
@@ -35265,7 +33968,7 @@ async function runLocalSourcebookAcquisition(submission, provider = createDefaul
|
|
|
35265
33968
|
kind,
|
|
35266
33969
|
userId: submission.ownerUserId,
|
|
35267
33970
|
debitKey: debitKey2,
|
|
35268
|
-
reason: `Local Sourcebook acquisition failed: ${
|
|
33971
|
+
reason: `Local Sourcebook acquisition failed: ${errorMessage4(error)}`
|
|
35269
33972
|
}).catch(() => null);
|
|
35270
33973
|
}
|
|
35271
33974
|
throw error;
|
|
@@ -35278,7 +33981,7 @@ async function processLocalSourcebookSubmission(submissionId) {
|
|
|
35278
33981
|
const completed = await runLocalSourcebookAcquisition(submission);
|
|
35279
33982
|
return { claimed: true, submissionId: completed.id, status: completed.status };
|
|
35280
33983
|
} catch (error) {
|
|
35281
|
-
const message =
|
|
33984
|
+
const message = errorMessage4(error);
|
|
35282
33985
|
await failLocalSourcebookAcquisition(submission.id, message);
|
|
35283
33986
|
return { claimed: true, submissionId: submission.id, status: "failed" };
|
|
35284
33987
|
}
|
|
@@ -36117,15 +34820,15 @@ function normalizeIdempotencyKey(value) {
|
|
|
36117
34820
|
return key;
|
|
36118
34821
|
}
|
|
36119
34822
|
function tokenFor(id, idempotencyKey4) {
|
|
36120
|
-
return (0,
|
|
34823
|
+
return (0, import_node_crypto17.createHmac)("sha256", billingSecret()).update(id).update(":").update(idempotencyKey4).digest("base64url");
|
|
36121
34824
|
}
|
|
36122
34825
|
function tokenHash(token6) {
|
|
36123
|
-
return (0,
|
|
34826
|
+
return (0, import_node_crypto17.createHash)("sha256").update(token6).digest("hex");
|
|
36124
34827
|
}
|
|
36125
34828
|
function verifyToken(row, token6) {
|
|
36126
34829
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
36127
34830
|
const actual = Buffer.from(tokenHash(token6), "hex");
|
|
36128
|
-
if (expected.length !== actual.length || !(0,
|
|
34831
|
+
if (expected.length !== actual.length || !(0, import_node_crypto17.timingSafeEqual)(expected, actual)) {
|
|
36129
34832
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
36130
34833
|
}
|
|
36131
34834
|
}
|
|
@@ -36240,7 +34943,7 @@ async function authorizeScheduledRun(args) {
|
|
|
36240
34943
|
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
36241
34944
|
);
|
|
36242
34945
|
}
|
|
36243
|
-
const id = (0,
|
|
34946
|
+
const id = (0, import_node_crypto17.randomUUID)();
|
|
36244
34947
|
const token6 = tokenFor(id, idempotencyKey4);
|
|
36245
34948
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
36246
34949
|
const inserted = await getDb().execute({
|
|
@@ -36308,7 +35011,7 @@ async function startScheduledRun(args) {
|
|
|
36308
35011
|
if (!authorization || !["starting", "started"].includes(authorization.status)) {
|
|
36309
35012
|
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
|
|
36310
35013
|
}
|
|
36311
|
-
const eventId = existing?.id ?? (0,
|
|
35014
|
+
const eventId = existing?.id ?? (0, import_node_crypto17.randomUUID)();
|
|
36312
35015
|
if (!existing) {
|
|
36313
35016
|
await getDb().execute({
|
|
36314
35017
|
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
|
|
@@ -36447,7 +35150,7 @@ async function settleScheduledRun(args) {
|
|
|
36447
35150
|
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
36448
35151
|
...pendingReason ? { reason: pendingReason } : {}
|
|
36449
35152
|
});
|
|
36450
|
-
const eventId = existing?.id ?? (0,
|
|
35153
|
+
const eventId = existing?.id ?? (0, import_node_crypto17.randomUUID)();
|
|
36451
35154
|
const status = pendingReason ? "cost_pending" : "settling";
|
|
36452
35155
|
const eventValues = [
|
|
36453
35156
|
modelMc,
|
|
@@ -36572,11 +35275,11 @@ async function voidScheduledRunAuthorization(args) {
|
|
|
36572
35275
|
}
|
|
36573
35276
|
return { ok: true, status: authorization.status };
|
|
36574
35277
|
}
|
|
36575
|
-
var
|
|
35278
|
+
var import_node_crypto17, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, AUTHORIZATION_TTL_MS, UnifiedBillingError;
|
|
36576
35279
|
var init_unified_billing = __esm({
|
|
36577
35280
|
"src/api/unified-billing.ts"() {
|
|
36578
35281
|
"use strict";
|
|
36579
|
-
|
|
35282
|
+
import_node_crypto17 = require("crypto");
|
|
36580
35283
|
init_db();
|
|
36581
35284
|
init_rates();
|
|
36582
35285
|
init_scheduling_access();
|
|
@@ -36608,14 +35311,14 @@ function getSessionSecret() {
|
|
|
36608
35311
|
function safeEqualHex(a, b) {
|
|
36609
35312
|
if (a.length !== b.length) return false;
|
|
36610
35313
|
try {
|
|
36611
|
-
return (0,
|
|
35314
|
+
return (0, import_node_crypto18.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
36612
35315
|
} catch {
|
|
36613
35316
|
return false;
|
|
36614
35317
|
}
|
|
36615
35318
|
}
|
|
36616
35319
|
function signSession(userId) {
|
|
36617
35320
|
const payload = String(userId);
|
|
36618
|
-
const sig = (0,
|
|
35321
|
+
const sig = (0, import_node_crypto18.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
36619
35322
|
return `${payload}.${sig}`;
|
|
36620
35323
|
}
|
|
36621
35324
|
function verifySession(token6) {
|
|
@@ -36623,16 +35326,16 @@ function verifySession(token6) {
|
|
|
36623
35326
|
if (dot === -1) return null;
|
|
36624
35327
|
const payload = token6.slice(0, dot);
|
|
36625
35328
|
const sig = token6.slice(dot + 1);
|
|
36626
|
-
const expected = (0,
|
|
35329
|
+
const expected = (0, import_node_crypto18.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
36627
35330
|
if (!safeEqualHex(sig, expected)) return null;
|
|
36628
35331
|
const id = parseInt(payload);
|
|
36629
35332
|
return isNaN(id) ? null : id;
|
|
36630
35333
|
}
|
|
36631
|
-
var
|
|
35334
|
+
var import_node_crypto18, isProduction, secret;
|
|
36632
35335
|
var init_session = __esm({
|
|
36633
35336
|
"src/api/session.ts"() {
|
|
36634
35337
|
"use strict";
|
|
36635
|
-
|
|
35338
|
+
import_node_crypto18 = require("crypto");
|
|
36636
35339
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
36637
35340
|
secret = () => getSessionSecret();
|
|
36638
35341
|
}
|
|
@@ -36650,11 +35353,11 @@ function isMemoryOperator(email) {
|
|
|
36650
35353
|
return ops.includes(email.trim().toLowerCase());
|
|
36651
35354
|
}
|
|
36652
35355
|
function encKey() {
|
|
36653
|
-
return (0,
|
|
35356
|
+
return (0, import_node_crypto19.scryptSync)(getSessionSecret(), "mcp-memory-key-v1", 32);
|
|
36654
35357
|
}
|
|
36655
35358
|
function encryptMemoryKey(secret2) {
|
|
36656
|
-
const iv = (0,
|
|
36657
|
-
const cipher = (0,
|
|
35359
|
+
const iv = (0, import_node_crypto19.randomBytes)(12);
|
|
35360
|
+
const cipher = (0, import_node_crypto19.createCipheriv)("aes-256-gcm", encKey(), iv);
|
|
36658
35361
|
const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
|
|
36659
35362
|
const tag = cipher.getAuthTag();
|
|
36660
35363
|
return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
|
|
@@ -36662,7 +35365,7 @@ function encryptMemoryKey(secret2) {
|
|
|
36662
35365
|
function decryptMemoryKey(stored) {
|
|
36663
35366
|
try {
|
|
36664
35367
|
const [ivB, tagB, dataB] = stored.split(":");
|
|
36665
|
-
const decipher = (0,
|
|
35368
|
+
const decipher = (0, import_node_crypto19.createDecipheriv)("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
|
|
36666
35369
|
decipher.setAuthTag(Buffer.from(tagB, "base64"));
|
|
36667
35370
|
return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
|
|
36668
35371
|
} catch {
|
|
@@ -36776,11 +35479,11 @@ async function syncScheduledActionCredentials(user) {
|
|
|
36776
35479
|
}, {});
|
|
36777
35480
|
return { ok: res.ok, error: res.error };
|
|
36778
35481
|
}
|
|
36779
|
-
var
|
|
35482
|
+
var import_node_crypto19, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
|
|
36780
35483
|
var init_memory = __esm({
|
|
36781
35484
|
"src/api/memory.ts"() {
|
|
36782
35485
|
"use strict";
|
|
36783
|
-
|
|
35486
|
+
import_node_crypto19 = require("crypto");
|
|
36784
35487
|
init_session();
|
|
36785
35488
|
init_db();
|
|
36786
35489
|
init_rates();
|
|
@@ -36830,7 +35533,7 @@ var init_connected_cost_telemetry = __esm({
|
|
|
36830
35533
|
|
|
36831
35534
|
// src/api/connected-usage-billing.ts
|
|
36832
35535
|
function hash(value) {
|
|
36833
|
-
return (0,
|
|
35536
|
+
return (0, import_node_crypto20.createHash)("sha256").update(value).digest("hex");
|
|
36834
35537
|
}
|
|
36835
35538
|
function eventKey(idempotencyKey4) {
|
|
36836
35539
|
return `connected-usage:${hash(idempotencyKey4)}`;
|
|
@@ -37102,7 +35805,7 @@ async function settleConnectedUsage(rawInput) {
|
|
|
37102
35805
|
sql: `INSERT OR IGNORE INTO billing_events
|
|
37103
35806
|
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
37104
35807
|
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
37105
|
-
args: [(0,
|
|
35808
|
+
args: [(0, import_node_crypto20.randomUUID)(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
|
|
37106
35809
|
});
|
|
37107
35810
|
let event2 = await readEvent(key);
|
|
37108
35811
|
if (!event2) throw new Error("connected usage receipt insert completed without a readable event");
|
|
@@ -37191,11 +35894,11 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
37191
35894
|
}
|
|
37192
35895
|
return history;
|
|
37193
35896
|
}
|
|
37194
|
-
var
|
|
35897
|
+
var import_node_crypto20, import_zod17, CONNECTED_USAGE_BILLING_CLASS, CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE, ConnectedUsageSafeMetadataSchema, ConnectedUsageSettlementInputSchema, ConnectedUsagePreflightInputSchema, ConnectedUsageBillingError;
|
|
37195
35898
|
var init_connected_usage_billing = __esm({
|
|
37196
35899
|
"src/api/connected-usage-billing.ts"() {
|
|
37197
35900
|
"use strict";
|
|
37198
|
-
|
|
35901
|
+
import_node_crypto20 = require("crypto");
|
|
37199
35902
|
import_zod17 = require("zod");
|
|
37200
35903
|
init_connected_cost_telemetry();
|
|
37201
35904
|
init_db();
|
|
@@ -38894,6 +37597,171 @@ var init_server_schemas = __esm({
|
|
|
38894
37597
|
}
|
|
38895
37598
|
});
|
|
38896
37599
|
|
|
37600
|
+
// src/api/harvest-problems.ts
|
|
37601
|
+
function errorMessage5(err) {
|
|
37602
|
+
return err instanceof Error ? err.message : String(err);
|
|
37603
|
+
}
|
|
37604
|
+
function looksLikeTimeout(err, message) {
|
|
37605
|
+
if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
|
|
37606
|
+
return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
|
|
37607
|
+
}
|
|
37608
|
+
function looksLikeCaptcha(message) {
|
|
37609
|
+
return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
|
|
37610
|
+
}
|
|
37611
|
+
function looksLikeProxyTunnelFailure3(message) {
|
|
37612
|
+
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
37613
|
+
}
|
|
37614
|
+
function looksLikeProxyUnavailable3(message) {
|
|
37615
|
+
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
37616
|
+
}
|
|
37617
|
+
function looksLikeInternalServiceUnavailable(message) {
|
|
37618
|
+
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
37619
|
+
}
|
|
37620
|
+
function looksLikeBrowserSessionInterrupted(message) {
|
|
37621
|
+
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
37622
|
+
}
|
|
37623
|
+
function classifyHarvestProblem(err) {
|
|
37624
|
+
const message = errorMessage5(err);
|
|
37625
|
+
if (err instanceof RequestAbortedError) {
|
|
37626
|
+
return {
|
|
37627
|
+
error_code: "request_aborted",
|
|
37628
|
+
error_type: "request_aborted",
|
|
37629
|
+
message: publicErrorMessage("request_aborted"),
|
|
37630
|
+
retryable: true,
|
|
37631
|
+
httpStatus: 408,
|
|
37632
|
+
terminalStatus: "cancelled"
|
|
37633
|
+
};
|
|
37634
|
+
}
|
|
37635
|
+
if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
|
|
37636
|
+
return {
|
|
37637
|
+
error_code: "captcha_exhausted",
|
|
37638
|
+
error_type: "captcha",
|
|
37639
|
+
message: publicErrorMessage("captcha_exhausted"),
|
|
37640
|
+
retryable: true,
|
|
37641
|
+
httpStatus: 503,
|
|
37642
|
+
terminalStatus: "failed"
|
|
37643
|
+
};
|
|
37644
|
+
}
|
|
37645
|
+
if (isVendorUnavailableError(err, message)) {
|
|
37646
|
+
return {
|
|
37647
|
+
error_code: "vendor_unavailable",
|
|
37648
|
+
error_type: "service_unavailable",
|
|
37649
|
+
message: publicErrorMessage("vendor_unavailable"),
|
|
37650
|
+
retryable: true,
|
|
37651
|
+
httpStatus: 503,
|
|
37652
|
+
terminalStatus: "failed"
|
|
37653
|
+
};
|
|
37654
|
+
}
|
|
37655
|
+
if (err instanceof LocationMismatchError) {
|
|
37656
|
+
return {
|
|
37657
|
+
error_code: "location_mismatch",
|
|
37658
|
+
error_type: "location_mismatch",
|
|
37659
|
+
message: publicErrorMessage("location_mismatch"),
|
|
37660
|
+
retryable: true,
|
|
37661
|
+
httpStatus: 503,
|
|
37662
|
+
terminalStatus: "failed"
|
|
37663
|
+
};
|
|
37664
|
+
}
|
|
37665
|
+
if (looksLikeProxyTunnelFailure3(message)) {
|
|
37666
|
+
return {
|
|
37667
|
+
error_code: "proxy_tunnel_failed",
|
|
37668
|
+
error_type: "connection",
|
|
37669
|
+
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
37670
|
+
retryable: true,
|
|
37671
|
+
httpStatus: 503,
|
|
37672
|
+
terminalStatus: "failed"
|
|
37673
|
+
};
|
|
37674
|
+
}
|
|
37675
|
+
if (looksLikeProxyUnavailable3(message)) {
|
|
37676
|
+
return {
|
|
37677
|
+
error_code: "proxy_unavailable",
|
|
37678
|
+
error_type: "connection",
|
|
37679
|
+
message: publicErrorMessage("proxy_unavailable"),
|
|
37680
|
+
retryable: true,
|
|
37681
|
+
httpStatus: 503,
|
|
37682
|
+
terminalStatus: "failed"
|
|
37683
|
+
};
|
|
37684
|
+
}
|
|
37685
|
+
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
37686
|
+
return {
|
|
37687
|
+
error_code: "browser_session_interrupted",
|
|
37688
|
+
error_type: "extraction",
|
|
37689
|
+
message: publicErrorMessage("browser_session_interrupted"),
|
|
37690
|
+
retryable: true,
|
|
37691
|
+
httpStatus: 503,
|
|
37692
|
+
terminalStatus: "failed"
|
|
37693
|
+
};
|
|
37694
|
+
}
|
|
37695
|
+
if (looksLikeTimeout(err, message)) {
|
|
37696
|
+
return {
|
|
37697
|
+
error_code: "harvest_timeout",
|
|
37698
|
+
error_type: "timeout",
|
|
37699
|
+
message: publicErrorMessage("harvest_timeout"),
|
|
37700
|
+
retryable: true,
|
|
37701
|
+
httpStatus: 504,
|
|
37702
|
+
terminalStatus: "failed"
|
|
37703
|
+
};
|
|
37704
|
+
}
|
|
37705
|
+
if (looksLikeInternalServiceUnavailable(message)) {
|
|
37706
|
+
return {
|
|
37707
|
+
error_code: "service_unavailable",
|
|
37708
|
+
error_type: "service_unavailable",
|
|
37709
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
37710
|
+
retryable: true,
|
|
37711
|
+
httpStatus: 503,
|
|
37712
|
+
terminalStatus: "failed"
|
|
37713
|
+
};
|
|
37714
|
+
}
|
|
37715
|
+
return {
|
|
37716
|
+
error_code: "extraction_failed",
|
|
37717
|
+
error_type: "extraction",
|
|
37718
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
37719
|
+
retryable: false,
|
|
37720
|
+
httpStatus: 500,
|
|
37721
|
+
terminalStatus: "failed"
|
|
37722
|
+
};
|
|
37723
|
+
}
|
|
37724
|
+
function serializeHarvestProblem(problem) {
|
|
37725
|
+
return JSON.stringify({
|
|
37726
|
+
error_code: problem.error_code,
|
|
37727
|
+
error_type: problem.error_type,
|
|
37728
|
+
message: problem.message,
|
|
37729
|
+
retryable: problem.retryable
|
|
37730
|
+
});
|
|
37731
|
+
}
|
|
37732
|
+
function harvestProblemResponse(problem, context = {}) {
|
|
37733
|
+
const envelope = buildPublicErrorEnvelope({
|
|
37734
|
+
errorCode: problem.error_code,
|
|
37735
|
+
errorType: problem.error_type,
|
|
37736
|
+
retryable: problem.retryable,
|
|
37737
|
+
retryAfterSeconds: context.retryAfterSeconds,
|
|
37738
|
+
chargeStatus: context.chargeStatus,
|
|
37739
|
+
details: context.details
|
|
37740
|
+
});
|
|
37741
|
+
return {
|
|
37742
|
+
error: envelope.message,
|
|
37743
|
+
...envelope
|
|
37744
|
+
};
|
|
37745
|
+
}
|
|
37746
|
+
function harvestProblemEnvelope(problem, context = {}) {
|
|
37747
|
+
const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context);
|
|
37748
|
+
return envelope;
|
|
37749
|
+
}
|
|
37750
|
+
function publicErrorBoundary(err, context = {}) {
|
|
37751
|
+
const problem = classifyHarvestProblem(err);
|
|
37752
|
+
return {
|
|
37753
|
+
body: harvestProblemResponse(problem, context),
|
|
37754
|
+
status: problem.httpStatus
|
|
37755
|
+
};
|
|
37756
|
+
}
|
|
37757
|
+
var init_harvest_problems = __esm({
|
|
37758
|
+
"src/api/harvest-problems.ts"() {
|
|
37759
|
+
"use strict";
|
|
37760
|
+
init_errors();
|
|
37761
|
+
init_vendor_errors();
|
|
37762
|
+
}
|
|
37763
|
+
});
|
|
37764
|
+
|
|
38897
37765
|
// src/api/youtube-routes.ts
|
|
38898
37766
|
function buildTranscriptMarkdown(result) {
|
|
38899
37767
|
const lines = [];
|
|
@@ -40177,7 +39045,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
40177
39045
|
}
|
|
40178
39046
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
40179
39047
|
}
|
|
40180
|
-
var import_hono7, import_zod24,
|
|
39048
|
+
var import_hono7, import_zod24, import_client7, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
40181
39049
|
var init_facebook_ad_routes = __esm({
|
|
40182
39050
|
"src/api/facebook-ad-routes.ts"() {
|
|
40183
39051
|
"use strict";
|
|
@@ -40192,7 +39060,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40192
39060
|
init_FacebookOrganicVideoExtractor();
|
|
40193
39061
|
init_kernel_proxy_resolver();
|
|
40194
39062
|
init_schemas2();
|
|
40195
|
-
|
|
39063
|
+
import_client7 = require("@fal-ai/client");
|
|
40196
39064
|
init_api_auth();
|
|
40197
39065
|
init_url_utils();
|
|
40198
39066
|
init_concurrency_gates();
|
|
@@ -40340,7 +39208,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40340
39208
|
metadata: { videoUrl }
|
|
40341
39209
|
});
|
|
40342
39210
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40343
|
-
|
|
39211
|
+
import_client7.fal.config({ credentials: process.env.FAL_KEY });
|
|
40344
39212
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40345
39213
|
let debited = false;
|
|
40346
39214
|
try {
|
|
@@ -40389,7 +39257,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
40389
39257
|
metadata: { url: sourceUrl.href }
|
|
40390
39258
|
});
|
|
40391
39259
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40392
|
-
|
|
39260
|
+
import_client7.fal.config({ credentials: process.env.FAL_KEY });
|
|
40393
39261
|
const driver = new BrowserDriver();
|
|
40394
39262
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40395
39263
|
let debited = false;
|
|
@@ -40845,7 +39713,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
40845
39713
|
}
|
|
40846
39714
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
40847
39715
|
}
|
|
40848
|
-
var import_hono8, import_zod25,
|
|
39716
|
+
var import_hono8, import_zod25, import_client8, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
|
|
40849
39717
|
var init_google_ads_routes = __esm({
|
|
40850
39718
|
"src/api/google-ads-routes.ts"() {
|
|
40851
39719
|
"use strict";
|
|
@@ -40858,7 +39726,7 @@ var init_google_ads_routes = __esm({
|
|
|
40858
39726
|
init_GoogleAdsExtractor();
|
|
40859
39727
|
init_kernel_proxy_resolver();
|
|
40860
39728
|
init_schemas2();
|
|
40861
|
-
|
|
39729
|
+
import_client8 = require("@fal-ai/client");
|
|
40862
39730
|
init_api_auth();
|
|
40863
39731
|
init_url_utils();
|
|
40864
39732
|
init_concurrency_gates();
|
|
@@ -40983,7 +39851,7 @@ var init_google_ads_routes = __esm({
|
|
|
40983
39851
|
const user = c.get("user");
|
|
40984
39852
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
40985
39853
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
40986
|
-
|
|
39854
|
+
import_client8.fal.config({ credentials: process.env.FAL_KEY });
|
|
40987
39855
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
40988
39856
|
let debited = false;
|
|
40989
39857
|
try {
|
|
@@ -41164,7 +40032,7 @@ async function applyAdjustment(input) {
|
|
|
41164
40032
|
const credits = validateCredits(input.credits, input.confirmLarge === true);
|
|
41165
40033
|
const reason = validateReason(input.reason);
|
|
41166
40034
|
const actor = validateActor(input.actor);
|
|
41167
|
-
const reference = input.reference?.trim() || `adj_${(0,
|
|
40035
|
+
const reference = input.reference?.trim() || `adj_${(0, import_node_crypto21.randomUUID)()}`;
|
|
41168
40036
|
if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
|
|
41169
40037
|
const db = getDb();
|
|
41170
40038
|
const existing = await db.execute({
|
|
@@ -41309,11 +40177,11 @@ async function lookupAccount(target, limit = 20) {
|
|
|
41309
40177
|
}))
|
|
41310
40178
|
};
|
|
41311
40179
|
}
|
|
41312
|
-
var
|
|
40180
|
+
var import_node_crypto21, MIN_ADJUSTMENT_CREDITS, LARGE_ADJUSTMENT_CREDITS, MAX_ADJUSTMENT_CREDITS, MIN_REASON_LENGTH, MAX_REASON_LENGTH, MAX_ACTOR_LENGTH, AdminCreditError;
|
|
41313
40181
|
var init_admin_credits = __esm({
|
|
41314
40182
|
"src/api/admin-credits.ts"() {
|
|
41315
40183
|
"use strict";
|
|
41316
|
-
|
|
40184
|
+
import_node_crypto21 = require("crypto");
|
|
41317
40185
|
init_db();
|
|
41318
40186
|
init_rates();
|
|
41319
40187
|
MIN_ADJUSTMENT_CREDITS = 1;
|
|
@@ -43278,7 +42146,7 @@ async function packageMapsMedia(args) {
|
|
|
43278
42146
|
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
43279
42147
|
files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
|
|
43280
42148
|
const archive = await zipBuffer(files);
|
|
43281
|
-
const id = (0,
|
|
42149
|
+
const id = (0, import_node_crypto22.randomBytes)(6).toString("hex");
|
|
43282
42150
|
const pointer = await createPrivateArtifact({
|
|
43283
42151
|
policy: policy3(),
|
|
43284
42152
|
ownerId: args.ownerId,
|
|
@@ -43291,13 +42159,13 @@ async function packageMapsMedia(args) {
|
|
|
43291
42159
|
const localPath = token() ? null : (0, import_node_path13.join)(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path13.join)((0, import_node_os9.homedir)(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
43292
42160
|
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
43293
42161
|
}
|
|
43294
|
-
var import_node_os9, import_node_path13,
|
|
42162
|
+
var import_node_os9, import_node_path13, import_node_crypto22, import_yazl2, import_p_limit5, MAPS_MEDIA_ARTIFACT_PREFIX, MAPS_MEDIA_ARTIFACT_TTL_MS, MAPS_MEDIA_DOWNLOAD_TTL_MS, MAX_IMAGE_BYTES2, MAX_ARCHIVE_IMAGE_BYTES, MAX_INLINE_IMAGE_BYTES, MAX_INLINE_TOTAL_BYTES, DOWNLOAD_CONCURRENCY, MAX_REDIRECTS;
|
|
43295
42163
|
var init_maps_media_artifacts = __esm({
|
|
43296
42164
|
"src/api/maps-media-artifacts.ts"() {
|
|
43297
42165
|
"use strict";
|
|
43298
42166
|
import_node_os9 = require("os");
|
|
43299
42167
|
import_node_path13 = require("path");
|
|
43300
|
-
|
|
42168
|
+
import_node_crypto22 = require("crypto");
|
|
43301
42169
|
import_yazl2 = require("yazl");
|
|
43302
42170
|
import_p_limit5 = __toESM(require("p-limit"), 1);
|
|
43303
42171
|
init_private_artifacts();
|
|
@@ -43921,7 +42789,7 @@ function retryDelaySeconds(attempts) {
|
|
|
43921
42789
|
}
|
|
43922
42790
|
async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
43923
42791
|
const rows = await claimDirectoryWorkflowOutbox({
|
|
43924
|
-
workerId: `directory-dispatch-${process.pid}-${(0,
|
|
42792
|
+
workerId: `directory-dispatch-${process.pid}-${(0, import_node_crypto23.randomUUID)().slice(0, 8)}`,
|
|
43925
42793
|
limit
|
|
43926
42794
|
});
|
|
43927
42795
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
@@ -43943,11 +42811,11 @@ async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
|
43943
42811
|
}
|
|
43944
42812
|
return result;
|
|
43945
42813
|
}
|
|
43946
|
-
var
|
|
42814
|
+
var import_node_crypto23;
|
|
43947
42815
|
var init_directory_workflow_dispatch = __esm({
|
|
43948
42816
|
"src/api/directory-workflow-dispatch.ts"() {
|
|
43949
42817
|
"use strict";
|
|
43950
|
-
|
|
42818
|
+
import_node_crypto23 = require("crypto");
|
|
43951
42819
|
init_client();
|
|
43952
42820
|
init_directory_workflow_repository();
|
|
43953
42821
|
}
|
|
@@ -43959,7 +42827,7 @@ function safeOptions(options) {
|
|
|
43959
42827
|
return safe2;
|
|
43960
42828
|
}
|
|
43961
42829
|
function requestFingerprint(options) {
|
|
43962
|
-
return (0,
|
|
42830
|
+
return (0, import_node_crypto24.createHash)("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
|
|
43963
42831
|
}
|
|
43964
42832
|
function idempotencyKey(raw) {
|
|
43965
42833
|
if (raw !== void 0) {
|
|
@@ -43968,14 +42836,14 @@ function idempotencyKey(raw) {
|
|
|
43968
42836
|
if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
|
|
43969
42837
|
return { ok: true, key: trimmed };
|
|
43970
42838
|
}
|
|
43971
|
-
return { ok: true, key: `directory-${(0,
|
|
42839
|
+
return { ok: true, key: `directory-${(0, import_node_crypto24.randomUUID)()}` };
|
|
43972
42840
|
}
|
|
43973
42841
|
function debitKeyFor(userId, responseKey) {
|
|
43974
|
-
const digest2 = (0,
|
|
42842
|
+
const digest2 = (0, import_node_crypto24.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
43975
42843
|
return `directory-workflow:${userId}:${digest2}`;
|
|
43976
42844
|
}
|
|
43977
42845
|
function jobId() {
|
|
43978
|
-
return `dir_${(0,
|
|
42846
|
+
return `dir_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "")}`;
|
|
43979
42847
|
}
|
|
43980
42848
|
function publicStatus(job, result) {
|
|
43981
42849
|
if (job.status === "failed") return "failed";
|
|
@@ -44084,7 +42952,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
44084
42952
|
const csv = renderDirectoryWorkflowCsv(result);
|
|
44085
42953
|
const artifact = await createDirectoryCsvArtifact({
|
|
44086
42954
|
ownerId: String(user.id),
|
|
44087
|
-
jobId: (0,
|
|
42955
|
+
jobId: (0, import_node_crypto24.createHash)("sha256").update(debitKey2).digest("hex").slice(0, 32),
|
|
44088
42956
|
createdAt: result.extractedAt,
|
|
44089
42957
|
filename: `${options.state}-${options.query}-directory.csv`,
|
|
44090
42958
|
csv,
|
|
@@ -44140,11 +43008,11 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
44140
43008
|
await releaseConcurrencyGate(gate.lockId);
|
|
44141
43009
|
}
|
|
44142
43010
|
}
|
|
44143
|
-
var
|
|
43011
|
+
var import_node_crypto24, import_hono16, directoryApp;
|
|
44144
43012
|
var init_directory_routes = __esm({
|
|
44145
43013
|
"src/api/directory-routes.ts"() {
|
|
44146
43014
|
"use strict";
|
|
44147
|
-
|
|
43015
|
+
import_node_crypto24 = require("crypto");
|
|
44148
43016
|
import_hono16 = require("hono");
|
|
44149
43017
|
init_api_auth();
|
|
44150
43018
|
init_db();
|
|
@@ -44362,10 +43230,10 @@ function bounded(value, field, min, max) {
|
|
|
44362
43230
|
return normalized;
|
|
44363
43231
|
}
|
|
44364
43232
|
function newLeadListUploadId() {
|
|
44365
|
-
return `upl_${(0,
|
|
43233
|
+
return `upl_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
|
|
44366
43234
|
}
|
|
44367
43235
|
function newImportedLeadListId() {
|
|
44368
|
-
return `lst_${(0,
|
|
43236
|
+
return `lst_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
|
|
44369
43237
|
}
|
|
44370
43238
|
async function ensureLeadListImportRepositorySchema() {
|
|
44371
43239
|
const db = getDb();
|
|
@@ -44565,11 +43433,11 @@ async function deleteExpiredLeadListInputRecords(now = /* @__PURE__ */ new Date(
|
|
|
44565
43433
|
], "write");
|
|
44566
43434
|
return { uploads: Number(uploads.rowsAffected ?? 0), lists: Number(lists.rowsAffected ?? 0) };
|
|
44567
43435
|
}
|
|
44568
|
-
var
|
|
43436
|
+
var import_node_crypto25, schemaDb5, schemaPromise5;
|
|
44569
43437
|
var init_lead_list_import_repository = __esm({
|
|
44570
43438
|
"src/api/lead-list-import-repository.ts"() {
|
|
44571
43439
|
"use strict";
|
|
44572
|
-
|
|
43440
|
+
import_node_crypto25 = require("crypto");
|
|
44573
43441
|
init_db();
|
|
44574
43442
|
schemaDb5 = null;
|
|
44575
43443
|
schemaPromise5 = null;
|
|
@@ -44611,7 +43479,7 @@ function safeFilenameHint(value) {
|
|
|
44611
43479
|
return cleaned || null;
|
|
44612
43480
|
}
|
|
44613
43481
|
function requestFingerprint2(filenameHint) {
|
|
44614
|
-
return (0,
|
|
43482
|
+
return (0, import_node_crypto26.createHash)("sha256").update(JSON.stringify({ filenameHint })).digest("hex");
|
|
44615
43483
|
}
|
|
44616
43484
|
function publicBaseUrl() {
|
|
44617
43485
|
const configured = process.env.MCP_SCRAPER_PUBLIC_BASE_URL?.trim() || process.env.PUBLIC_BASE_URL?.trim() || (process.env.VERCEL_PROJECT_PRODUCTION_URL ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}` : "") || "https://mcpscraper.dev";
|
|
@@ -44767,7 +43635,7 @@ async function inspectLeadListUpload(uploadId, ownerId2) {
|
|
|
44767
43635
|
maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
|
|
44768
43636
|
});
|
|
44769
43637
|
if (!buffer) return null;
|
|
44770
|
-
const sha2565 = (0,
|
|
43638
|
+
const sha2565 = (0, import_node_crypto26.createHash)("sha256").update(buffer).digest("hex");
|
|
44771
43639
|
const completed = await completeLeadListUpload({
|
|
44772
43640
|
id: record.id,
|
|
44773
43641
|
ownerId: record.ownerId,
|
|
@@ -44891,11 +43759,11 @@ async function cleanupExpiredLeadListInputs(args = {}) {
|
|
|
44891
43759
|
const records = await deleteExpiredLeadListInputRecords(now);
|
|
44892
43760
|
return { deletedBlobs, deletedUploadRecords: records.uploads, deletedListRecords: records.lists };
|
|
44893
43761
|
}
|
|
44894
|
-
var
|
|
43762
|
+
var import_node_crypto26, import_promises11, import_node_os10, import_node_path14, LEAD_LIST_UPLOAD_PREFIX, LEAD_LIST_NORMALIZED_PREFIX, LEAD_LIST_UPLOAD_MAX_BYTES, LEAD_LIST_UPLOAD_URL_TTL_MS, LEAD_LIST_UPLOAD_SOURCE_TTL_MS, LEAD_LIST_NORMALIZED_TTL_MS, LEAD_LIST_DOWNLOAD_TTL_MS, LEAD_LIST_UPLOAD_MIME_TYPES;
|
|
44895
43763
|
var init_lead_list_input_artifacts = __esm({
|
|
44896
43764
|
"src/api/lead-list-input-artifacts.ts"() {
|
|
44897
43765
|
"use strict";
|
|
44898
|
-
|
|
43766
|
+
import_node_crypto26 = require("crypto");
|
|
44899
43767
|
import_promises11 = require("fs/promises");
|
|
44900
43768
|
import_node_os10 = require("os");
|
|
44901
43769
|
import_node_path14 = require("path");
|
|
@@ -45278,7 +44146,7 @@ function canonicalize(value) {
|
|
|
45278
44146
|
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalize(object[key])}`).join(",")}}`;
|
|
45279
44147
|
}
|
|
45280
44148
|
function fingerprint(value) {
|
|
45281
|
-
return (0,
|
|
44149
|
+
return (0, import_node_crypto27.createHash)("sha256").update(canonicalize(value)).digest("hex");
|
|
45282
44150
|
}
|
|
45283
44151
|
async function safeJson(c) {
|
|
45284
44152
|
try {
|
|
@@ -45355,11 +44223,11 @@ function textDelimiter(raw, text2, observedMime) {
|
|
|
45355
44223
|
if (observedMime === "text/tab-separated-values") return " ";
|
|
45356
44224
|
return detectLeadListTextDelimiter(text2);
|
|
45357
44225
|
}
|
|
45358
|
-
var
|
|
44226
|
+
var import_node_crypto27, import_hono17, leadListInputApp;
|
|
45359
44227
|
var init_lead_list_input_routes = __esm({
|
|
45360
44228
|
"src/api/lead-list-input-routes.ts"() {
|
|
45361
44229
|
"use strict";
|
|
45362
|
-
|
|
44230
|
+
import_node_crypto27 = require("crypto");
|
|
45363
44231
|
import_hono17 = require("hono");
|
|
45364
44232
|
init_api_auth();
|
|
45365
44233
|
init_lead_list_input_artifacts();
|
|
@@ -45551,7 +44419,7 @@ function retryDelaySeconds2(attempts) {
|
|
|
45551
44419
|
return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
45552
44420
|
}
|
|
45553
44421
|
async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
45554
|
-
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${(0,
|
|
44422
|
+
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${(0, import_node_crypto28.randomUUID)().slice(0, 8)}`, limit });
|
|
45555
44423
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
45556
44424
|
for (const row of rows) {
|
|
45557
44425
|
if (!row.claimToken) continue;
|
|
@@ -45570,11 +44438,11 @@ async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
|
45570
44438
|
}
|
|
45571
44439
|
return result;
|
|
45572
44440
|
}
|
|
45573
|
-
var
|
|
44441
|
+
var import_node_crypto28;
|
|
45574
44442
|
var init_lead_list_enrichment_dispatch = __esm({
|
|
45575
44443
|
"src/api/lead-list-enrichment-dispatch.ts"() {
|
|
45576
44444
|
"use strict";
|
|
45577
|
-
|
|
44445
|
+
import_node_crypto28 = require("crypto");
|
|
45578
44446
|
init_client();
|
|
45579
44447
|
init_lead_list_enrichment_repository();
|
|
45580
44448
|
}
|
|
@@ -45590,15 +44458,15 @@ function idempotencyKey2(raw) {
|
|
|
45590
44458
|
return { ok: true, value };
|
|
45591
44459
|
}
|
|
45592
44460
|
function newJobId() {
|
|
45593
|
-
return `lle_${(0,
|
|
44461
|
+
return `lle_${(0, import_node_crypto29.randomUUID)().replace(/-/g, "")}`;
|
|
45594
44462
|
}
|
|
45595
44463
|
function debitKeyFor2(userId, key) {
|
|
45596
|
-
const digest2 = (0,
|
|
44464
|
+
const digest2 = (0, import_node_crypto29.createHash)("sha256").update(String(userId)).update("\0").update(key).digest("hex");
|
|
45597
44465
|
return `lead-list-enrichment:${userId}:${digest2}`;
|
|
45598
44466
|
}
|
|
45599
44467
|
function inlineSourceDigest(headers, rows) {
|
|
45600
44468
|
const values = rows.map((row) => headers.map((header) => row[header] ?? null));
|
|
45601
|
-
return (0,
|
|
44469
|
+
return (0, import_node_crypto29.createHash)("sha256").update(stableCanonicalJson({ headers, values })).digest("hex");
|
|
45602
44470
|
}
|
|
45603
44471
|
function issueMessage(error) {
|
|
45604
44472
|
const issue = error.issues[0];
|
|
@@ -45796,11 +44664,11 @@ async function resolveInput(ownerId2, parsed) {
|
|
|
45796
44664
|
sourceDigest: sourceDigest ?? inlineSourceDigest(normalized.headers, rows)
|
|
45797
44665
|
};
|
|
45798
44666
|
}
|
|
45799
|
-
var
|
|
44667
|
+
var import_node_crypto29, import_hono18, import_zod31, SourceSchema, StartSchema, TERMINAL, leadListEnrichmentApp;
|
|
45800
44668
|
var init_lead_list_enrichment_routes = __esm({
|
|
45801
44669
|
"src/api/lead-list-enrichment-routes.ts"() {
|
|
45802
44670
|
"use strict";
|
|
45803
|
-
|
|
44671
|
+
import_node_crypto29 = require("crypto");
|
|
45804
44672
|
import_hono18 = require("hono");
|
|
45805
44673
|
import_zod31 = require("zod");
|
|
45806
44674
|
init_api_auth();
|
|
@@ -46755,50 +45623,324 @@ var init_location_data_routes = __esm({
|
|
|
46755
45623
|
}
|
|
46756
45624
|
});
|
|
46757
45625
|
|
|
46758
|
-
// src/api/
|
|
46759
|
-
|
|
46760
|
-
|
|
46761
|
-
|
|
46762
|
-
|
|
46763
|
-
|
|
46764
|
-
|
|
45626
|
+
// src/api/harvest-attempt-events.ts
|
|
45627
|
+
function createHarvestAttemptRecorder(jobId2, userId) {
|
|
45628
|
+
return async (event2) => {
|
|
45629
|
+
if (event2.type === "started") {
|
|
45630
|
+
await startHarvestAttempt({
|
|
45631
|
+
jobId: jobId2,
|
|
45632
|
+
userId,
|
|
45633
|
+
attemptNumber: event2.attemptNumber,
|
|
45634
|
+
maxAttempts: event2.maxAttempts,
|
|
45635
|
+
query: event2.query,
|
|
45636
|
+
location: event2.location,
|
|
45637
|
+
maxQuestions: event2.maxQuestions,
|
|
45638
|
+
startedAt: event2.startedAt
|
|
45639
|
+
});
|
|
45640
|
+
return;
|
|
45641
|
+
}
|
|
45642
|
+
await finishHarvestAttempt({
|
|
45643
|
+
jobId: jobId2,
|
|
45644
|
+
attemptNumber: event2.attemptNumber,
|
|
45645
|
+
outcome: event2.outcome,
|
|
45646
|
+
kernelSessionId: event2.kernelSessionId,
|
|
45647
|
+
questionCount: event2.questionCount,
|
|
45648
|
+
durationMs: event2.durationMs,
|
|
45649
|
+
error: event2.error,
|
|
45650
|
+
willRetry: event2.willRetry,
|
|
45651
|
+
kernelDeleteStarted: event2.cleanup.kernelDeleteStarted,
|
|
45652
|
+
kernelDeleteSucceeded: event2.cleanup.kernelDeleteSucceeded,
|
|
45653
|
+
kernelDeleteError: event2.cleanup.kernelDeleteError,
|
|
45654
|
+
browserCloseSucceeded: event2.cleanup.browserCloseSucceeded,
|
|
45655
|
+
browserCloseError: event2.cleanup.browserCloseError,
|
|
45656
|
+
debug: event2.debug,
|
|
45657
|
+
completedAt: event2.completedAt
|
|
45658
|
+
});
|
|
45659
|
+
};
|
|
46765
45660
|
}
|
|
46766
|
-
|
|
46767
|
-
|
|
46768
|
-
|
|
46769
|
-
|
|
46770
|
-
|
|
45661
|
+
var init_harvest_attempt_events = __esm({
|
|
45662
|
+
"src/api/harvest-attempt-events.ts"() {
|
|
45663
|
+
"use strict";
|
|
45664
|
+
init_db();
|
|
45665
|
+
}
|
|
45666
|
+
});
|
|
45667
|
+
|
|
45668
|
+
// src/api/paa-harvest-settlement.ts
|
|
45669
|
+
function capturedQuestionCount(result) {
|
|
45670
|
+
if (!result || typeof result !== "object") return 0;
|
|
45671
|
+
const value = result;
|
|
45672
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
45673
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
45674
|
+
}
|
|
45675
|
+
function finalCost(result, heldMc) {
|
|
45676
|
+
const questions = capturedQuestionCount(result);
|
|
45677
|
+
if (questions <= 0) return 0;
|
|
45678
|
+
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
45679
|
+
}
|
|
45680
|
+
async function settlePaaHarvestJob(jobOrId) {
|
|
45681
|
+
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
45682
|
+
if (!job) return false;
|
|
45683
|
+
const options = job.options;
|
|
45684
|
+
const debitKey2 = options.billingDebitKey;
|
|
45685
|
+
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
45686
|
+
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
45687
|
+
const settlement = await settleDebitMcIdempotent(
|
|
45688
|
+
job.user_id,
|
|
45689
|
+
debitKey2,
|
|
45690
|
+
finalCost(job.result, heldMc),
|
|
45691
|
+
LedgerOperation.PAA_REFUND,
|
|
45692
|
+
"durable PAA harvest settlement",
|
|
45693
|
+
"paa_harvest"
|
|
45694
|
+
);
|
|
45695
|
+
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
45696
|
+
return true;
|
|
45697
|
+
}
|
|
45698
|
+
var init_paa_harvest_settlement = __esm({
|
|
45699
|
+
"src/api/paa-harvest-settlement.ts"() {
|
|
45700
|
+
"use strict";
|
|
45701
|
+
init_db();
|
|
45702
|
+
init_rates();
|
|
45703
|
+
}
|
|
45704
|
+
});
|
|
45705
|
+
|
|
45706
|
+
// src/api/webhook.ts
|
|
45707
|
+
async function deliverWebhook(url, payload, retries = 3) {
|
|
45708
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
46771
45709
|
try {
|
|
46772
|
-
await
|
|
46773
|
-
|
|
46774
|
-
|
|
46775
|
-
|
|
45710
|
+
const res = await fetch(url, {
|
|
45711
|
+
method: "POST",
|
|
45712
|
+
headers: { "content-type": "application/json" },
|
|
45713
|
+
body: JSON.stringify(payload),
|
|
45714
|
+
signal: AbortSignal.timeout(1e4)
|
|
45715
|
+
});
|
|
45716
|
+
if (res.ok) return;
|
|
45717
|
+
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
45718
|
+
} catch (err) {
|
|
45719
|
+
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
46776
45720
|
}
|
|
45721
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
46777
45722
|
}
|
|
46778
|
-
|
|
45723
|
+
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
46779
45724
|
}
|
|
46780
|
-
var
|
|
46781
|
-
"src/api/
|
|
45725
|
+
var init_webhook = __esm({
|
|
45726
|
+
"src/api/webhook.ts"() {
|
|
46782
45727
|
"use strict";
|
|
46783
|
-
|
|
45728
|
+
}
|
|
45729
|
+
});
|
|
45730
|
+
|
|
45731
|
+
// src/api/paa-harvest-runner.ts
|
|
45732
|
+
function durableRecordsFromResult(result) {
|
|
45733
|
+
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
45734
|
+
return flat.flatMap((value) => {
|
|
45735
|
+
if (!value || typeof value !== "object") return [];
|
|
45736
|
+
const row = value;
|
|
45737
|
+
const question = typeof row.question === "string" ? row.question : "";
|
|
45738
|
+
if (!question) return [];
|
|
45739
|
+
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
45740
|
+
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
45741
|
+
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
45742
|
+
return [{
|
|
45743
|
+
question,
|
|
45744
|
+
answer: typeof row.answer === "string" ? row.answer : "",
|
|
45745
|
+
sourceTitle,
|
|
45746
|
+
sourceSite,
|
|
45747
|
+
sourceCite,
|
|
45748
|
+
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
45749
|
+
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
45750
|
+
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
45751
|
+
}];
|
|
45752
|
+
});
|
|
45753
|
+
}
|
|
45754
|
+
function questionCount(result) {
|
|
45755
|
+
if (!result || typeof result !== "object") return 0;
|
|
45756
|
+
const value = result;
|
|
45757
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
45758
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
45759
|
+
}
|
|
45760
|
+
async function runPaaHarvestJob(jobId2) {
|
|
45761
|
+
const invocationStartedAt = Date.now();
|
|
45762
|
+
const job = await claimDurablePaaJob(jobId2);
|
|
45763
|
+
if (!job) return { jobId: jobId2, status: "missing_or_not_paa" };
|
|
45764
|
+
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
45765
|
+
await settlePaaHarvestJob(job.id);
|
|
45766
|
+
return { jobId: job.id, status: job.status };
|
|
45767
|
+
}
|
|
45768
|
+
const options = job.options;
|
|
45769
|
+
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
45770
|
+
const timeoutBudget = harvestTimeoutBudget(maxQuestions);
|
|
45771
|
+
const workBudgetMs = Math.max(1, timeoutBudget.serverMs - CLEANUP_RESERVE_MS);
|
|
45772
|
+
const checkpointState = { value: null };
|
|
45773
|
+
try {
|
|
45774
|
+
if (!brightDataSerpEnabled()) throw new Error("SERP browser service is not configured");
|
|
45775
|
+
const navigating = {
|
|
45776
|
+
phase: "navigating",
|
|
45777
|
+
attempt: 1,
|
|
45778
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
45779
|
+
records: [],
|
|
45780
|
+
clickedQuestions: [],
|
|
45781
|
+
finalUrl: null,
|
|
45782
|
+
captchaObserved: false,
|
|
45783
|
+
solverCleared: false,
|
|
45784
|
+
error: null
|
|
45785
|
+
};
|
|
45786
|
+
checkpointState.value = navigating;
|
|
45787
|
+
if (!await checkpointJob(job.id, {
|
|
45788
|
+
progress: navigating,
|
|
45789
|
+
durable: {
|
|
45790
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
45791
|
+
workBudgetMs,
|
|
45792
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
45793
|
+
}
|
|
45794
|
+
})) {
|
|
45795
|
+
throw new Error("job_left_running_state_during_capture");
|
|
45796
|
+
}
|
|
45797
|
+
const persistAttempt = createHarvestAttemptRecorder(job.id, job.user_id);
|
|
45798
|
+
let lastAttempt = 1;
|
|
45799
|
+
const onAttemptEvent = async (event2) => {
|
|
45800
|
+
lastAttempt = event2.attemptNumber;
|
|
45801
|
+
await persistAttempt(event2);
|
|
45802
|
+
};
|
|
45803
|
+
const harvestCtx = currentCostContext();
|
|
45804
|
+
const harvested = await runWithCostContext(
|
|
45805
|
+
{ ...harvestCtx, op: "paa", userId: job.user_id },
|
|
45806
|
+
() => harvest({
|
|
45807
|
+
query: options.query ?? job.query,
|
|
45808
|
+
location: options.location ?? void 0,
|
|
45809
|
+
gl: options.gl ?? "us",
|
|
45810
|
+
hl: options.hl ?? "en",
|
|
45811
|
+
maxQuestions,
|
|
45812
|
+
maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
|
|
45813
|
+
proxyMode: "none",
|
|
45814
|
+
forceManagedSerp: true,
|
|
45815
|
+
headless: true,
|
|
45816
|
+
format: "json",
|
|
45817
|
+
outputDir: "/tmp/paa-output-direct",
|
|
45818
|
+
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
45819
|
+
includeLocalPack: options.includeLocalPack ?? false,
|
|
45820
|
+
includeForums: options.includeForums ?? false,
|
|
45821
|
+
includeVideos: options.includeVideos ?? false,
|
|
45822
|
+
includeAiOverview: options.includeAiOverview ?? false,
|
|
45823
|
+
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
45824
|
+
softDeadlineMs: invocationStartedAt + workBudgetMs,
|
|
45825
|
+
onAttemptEvent
|
|
45826
|
+
})
|
|
45827
|
+
);
|
|
45828
|
+
const harvestResult = harvested;
|
|
45829
|
+
const records = durableRecordsFromResult(harvestResult);
|
|
45830
|
+
const captured = {
|
|
45831
|
+
...navigating,
|
|
45832
|
+
phase: "captured",
|
|
45833
|
+
attempt: lastAttempt,
|
|
45834
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
45835
|
+
records,
|
|
45836
|
+
clickedQuestions: records.map((record) => record.question)
|
|
45837
|
+
};
|
|
45838
|
+
checkpointState.value = captured;
|
|
45839
|
+
if (!await checkpointJob(job.id, {
|
|
45840
|
+
progress: captured,
|
|
45841
|
+
durable: {
|
|
45842
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
45843
|
+
workBudgetMs,
|
|
45844
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
45845
|
+
}
|
|
45846
|
+
})) {
|
|
45847
|
+
throw new Error("job_left_running_state_during_capture");
|
|
45848
|
+
}
|
|
45849
|
+
const result = {
|
|
45850
|
+
...harvestResult,
|
|
45851
|
+
capture: {
|
|
45852
|
+
...harvestResult.capture,
|
|
45853
|
+
execution: "brightdata_browser_api_controller_fulfilled",
|
|
45854
|
+
provider: "bright_data_browser_api",
|
|
45855
|
+
sameQueryClickGraph: true,
|
|
45856
|
+
rawDomArtifact: null,
|
|
45857
|
+
rawDomArtifactStatus: "unavailable"
|
|
45858
|
+
}
|
|
45859
|
+
};
|
|
45860
|
+
await completeJob(job.id, result);
|
|
45861
|
+
await settlePaaHarvestJob(job.id);
|
|
45862
|
+
if (job.callback_url) {
|
|
45863
|
+
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
45864
|
+
}
|
|
45865
|
+
return {
|
|
45866
|
+
jobId: job.id,
|
|
45867
|
+
status: "done",
|
|
45868
|
+
totalQuestions: questionCount(result),
|
|
45869
|
+
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
45870
|
+
artifactStored: false
|
|
45871
|
+
};
|
|
45872
|
+
} catch (error) {
|
|
45873
|
+
const problem = classifyHarvestProblem(error);
|
|
45874
|
+
const latestCheckpoint = checkpointState.value;
|
|
45875
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
45876
|
+
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
45877
|
+
if (!settled) return;
|
|
45878
|
+
const settledJob = await getJob(job.id);
|
|
45879
|
+
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
45880
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
45881
|
+
}).catch((settlementError) => {
|
|
45882
|
+
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
45883
|
+
});
|
|
45884
|
+
if (job.callback_url) {
|
|
45885
|
+
await deliverWebhook(job.callback_url, {
|
|
45886
|
+
job_id: job.id,
|
|
45887
|
+
status: "failed",
|
|
45888
|
+
...harvestProblemResponse(problem),
|
|
45889
|
+
progress: latestCheckpoint
|
|
45890
|
+
});
|
|
45891
|
+
}
|
|
45892
|
+
throw error;
|
|
45893
|
+
}
|
|
45894
|
+
}
|
|
45895
|
+
var CLEANUP_RESERVE_MS, DIRECT_PAA_MAX_ATTEMPTS;
|
|
45896
|
+
var init_paa_harvest_runner = __esm({
|
|
45897
|
+
"src/api/paa-harvest-runner.ts"() {
|
|
45898
|
+
"use strict";
|
|
45899
|
+
init_BrightDataSerpDriver();
|
|
45900
|
+
init_harvest();
|
|
45901
|
+
init_harvest_timeout();
|
|
45902
|
+
init_cost_context();
|
|
46784
45903
|
init_db();
|
|
45904
|
+
init_harvest_attempt_events();
|
|
45905
|
+
init_harvest_problems();
|
|
45906
|
+
init_paa_harvest_settlement();
|
|
45907
|
+
init_webhook();
|
|
45908
|
+
CLEANUP_RESERVE_MS = 4e4;
|
|
45909
|
+
DIRECT_PAA_MAX_ATTEMPTS = 3;
|
|
45910
|
+
}
|
|
45911
|
+
});
|
|
45912
|
+
|
|
45913
|
+
// src/api/paa-harvest-direct.ts
|
|
45914
|
+
function scheduleDirectPaaHarvest(jobId2) {
|
|
45915
|
+
const work = runPaaHarvestJob(jobId2).catch((error) => {
|
|
45916
|
+
console.error("[paa-harvest/direct] run failed:", error instanceof Error ? error.message : String(error));
|
|
45917
|
+
});
|
|
45918
|
+
(0, import_functions.waitUntil)(work);
|
|
45919
|
+
}
|
|
45920
|
+
var import_functions;
|
|
45921
|
+
var init_paa_harvest_direct = __esm({
|
|
45922
|
+
"src/api/paa-harvest-direct.ts"() {
|
|
45923
|
+
"use strict";
|
|
45924
|
+
import_functions = require("@vercel/functions");
|
|
45925
|
+
init_paa_harvest_runner();
|
|
46785
45926
|
}
|
|
46786
45927
|
});
|
|
46787
45928
|
|
|
46788
45929
|
// src/api/paa-harvest-reconciliation.ts
|
|
46789
45930
|
async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
46790
|
-
const stale = await
|
|
45931
|
+
const stale = await listStaleRunningDurablePaaJobs(limit);
|
|
46791
45932
|
let staleFailed = 0;
|
|
46792
45933
|
for (const job of stale) {
|
|
46793
45934
|
try {
|
|
46794
|
-
const privateError = "
|
|
45935
|
+
const privateError = "direct PAA invocation exceeded its platform safety window; preserved progress remains in result";
|
|
46795
45936
|
const problem = classifyHarvestProblem(new Error("harvest timeout"));
|
|
46796
|
-
await
|
|
46797
|
-
|
|
45937
|
+
if (await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
|
|
45938
|
+
staleFailed += 1;
|
|
45939
|
+
}
|
|
46798
45940
|
} catch {
|
|
46799
45941
|
}
|
|
46800
45942
|
}
|
|
46801
|
-
const jobs = await
|
|
45943
|
+
const jobs = await listRecentTerminalDurablePaaJobs(limit);
|
|
46802
45944
|
let settled = 0;
|
|
46803
45945
|
let failed = 0;
|
|
46804
45946
|
for (const job of jobs) {
|
|
@@ -48993,7 +48135,7 @@ async function readManifestFromSummary(summary) {
|
|
|
48993
48135
|
function webhookSignature(body, timestamp2) {
|
|
48994
48136
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
48995
48137
|
if (!secret2) return null;
|
|
48996
|
-
return (0,
|
|
48138
|
+
return (0, import_node_crypto30.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
48997
48139
|
}
|
|
48998
48140
|
async function deliverWorkflowWebhook(input) {
|
|
48999
48141
|
if (!input.webhookUrl) return;
|
|
@@ -49200,11 +48342,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
49200
48342
|
}
|
|
49201
48343
|
return { dispatched: results.length, results };
|
|
49202
48344
|
}
|
|
49203
|
-
var
|
|
48345
|
+
var import_node_crypto30, import_promises14, import_hono21, import_zod41, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
49204
48346
|
var init_workflow_routes = __esm({
|
|
49205
48347
|
"src/api/workflow-routes.ts"() {
|
|
49206
48348
|
"use strict";
|
|
49207
|
-
|
|
48349
|
+
import_node_crypto30 = require("crypto");
|
|
49208
48350
|
import_promises14 = require("fs/promises");
|
|
49209
48351
|
import_hono21 = require("hono");
|
|
49210
48352
|
import_zod41 = require("zod");
|
|
@@ -49485,7 +48627,7 @@ var init_workflow_routes = __esm({
|
|
|
49485
48627
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
49486
48628
|
function sha2562(value) {
|
|
49487
48629
|
if (!value) return null;
|
|
49488
|
-
return (0,
|
|
48630
|
+
return (0, import_node_crypto31.createHash)("sha256").update(value).digest("hex");
|
|
49489
48631
|
}
|
|
49490
48632
|
function countWords(markdown) {
|
|
49491
48633
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -49785,11 +48927,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
49785
48927
|
}
|
|
49786
48928
|
};
|
|
49787
48929
|
}
|
|
49788
|
-
var
|
|
48930
|
+
var import_node_crypto31, import_p_limit6, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
49789
48931
|
var init_page_snapshot_extractor = __esm({
|
|
49790
48932
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
49791
48933
|
"use strict";
|
|
49792
|
-
|
|
48934
|
+
import_node_crypto31 = require("crypto");
|
|
49793
48935
|
import_p_limit6 = __toESM(require("p-limit"), 1);
|
|
49794
48936
|
init_kpo_extractor();
|
|
49795
48937
|
init_url_utils();
|
|
@@ -50286,21 +49428,21 @@ async function logRequestEventBestEffort(input) {
|
|
|
50286
49428
|
}
|
|
50287
49429
|
}
|
|
50288
49430
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
50289
|
-
const responseKey = suppliedKey?.trim() || (0,
|
|
50290
|
-
const requestFingerprint3 = (0,
|
|
50291
|
-
const keyDigest = (0,
|
|
49431
|
+
const responseKey = suppliedKey?.trim() || (0, import_node_crypto32.randomUUID)();
|
|
49432
|
+
const requestFingerprint3 = (0, import_node_crypto32.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
|
|
49433
|
+
const keyDigest = (0, import_node_crypto32.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
50292
49434
|
return {
|
|
50293
49435
|
responseKey,
|
|
50294
49436
|
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
50295
49437
|
debitDescription: `${body.query} [request:${requestFingerprint3}]`
|
|
50296
49438
|
};
|
|
50297
49439
|
}
|
|
50298
|
-
var import_hono22,
|
|
49440
|
+
var import_hono22, import_node_crypto32, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
|
|
50299
49441
|
var init_serp_intelligence_routes = __esm({
|
|
50300
49442
|
"src/api/serp-intelligence-routes.ts"() {
|
|
50301
49443
|
"use strict";
|
|
50302
49444
|
import_hono22 = require("hono");
|
|
50303
|
-
|
|
49445
|
+
import_node_crypto32 = require("crypto");
|
|
50304
49446
|
init_browser_service_env();
|
|
50305
49447
|
init_page_snapshot_extractor();
|
|
50306
49448
|
init_serp_capture_service();
|
|
@@ -50537,7 +49679,7 @@ var PACKAGE_VERSION;
|
|
|
50537
49679
|
var init_version = __esm({
|
|
50538
49680
|
"src/version.ts"() {
|
|
50539
49681
|
"use strict";
|
|
50540
|
-
PACKAGE_VERSION = "0.66.
|
|
49682
|
+
PACKAGE_VERSION = "0.66.3";
|
|
50541
49683
|
}
|
|
50542
49684
|
});
|
|
50543
49685
|
|
|
@@ -50968,63 +50110,29 @@ var init_server_instructions = __esm({
|
|
|
50968
50110
|
|
|
50969
50111
|
// src/mcp/output-schema-registry.ts
|
|
50970
50112
|
function resolveOutputSchemaMode(env = process.env) {
|
|
50971
|
-
const
|
|
50972
|
-
if (
|
|
50973
|
-
|
|
50113
|
+
const requested = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
|
|
50114
|
+
if (requested) {
|
|
50115
|
+
if (requested === "contract" || requested === "all") return "contract";
|
|
50116
|
+
if (requested === "runtime" || requested === "none" || requested === "essential") return "runtime";
|
|
50117
|
+
throw new Error(`Unsupported MCP_SCRAPER_OUTPUT_SCHEMA_MODE: ${requested}`);
|
|
50118
|
+
}
|
|
50119
|
+
const legacyAdvertisement = env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS?.trim().toLowerCase();
|
|
50120
|
+
if (legacyAdvertisement === "true") return "contract";
|
|
50121
|
+
if (legacyAdvertisement === void 0 || legacyAdvertisement === "" || legacyAdvertisement === "false") {
|
|
50122
|
+
return "runtime";
|
|
50123
|
+
}
|
|
50124
|
+
throw new Error(`Unsupported MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS value: ${legacyAdvertisement}`);
|
|
50974
50125
|
}
|
|
50975
50126
|
function recordOutputSchema(name, schema) {
|
|
50976
50127
|
OUTPUT_SCHEMAS[name] = schema;
|
|
50977
|
-
return
|
|
50128
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
50978
50129
|
}
|
|
50979
|
-
var
|
|
50130
|
+
var OUTPUT_SCHEMA_MODE, ADVERTISE_OUTPUT_SCHEMAS, OUTPUT_SCHEMAS;
|
|
50980
50131
|
var init_output_schema_registry = __esm({
|
|
50981
50132
|
"src/mcp/output-schema-registry.ts"() {
|
|
50982
50133
|
"use strict";
|
|
50983
|
-
ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
|
|
50984
|
-
"extract_url",
|
|
50985
|
-
"extract_site",
|
|
50986
|
-
"analyze_site_similarity",
|
|
50987
|
-
"audit_site",
|
|
50988
|
-
"check_site_export",
|
|
50989
|
-
"site_export_read",
|
|
50990
|
-
"site_export_image",
|
|
50991
|
-
"archive_read",
|
|
50992
|
-
"report_artifact_read",
|
|
50993
|
-
"directory_workflow",
|
|
50994
|
-
"directory_workflow_status",
|
|
50995
|
-
"lead_list_upload_start",
|
|
50996
|
-
"lead_list_import",
|
|
50997
|
-
"lead_list_enrich",
|
|
50998
|
-
"lead_list_enrich_status",
|
|
50999
|
-
"video_frame_analysis",
|
|
51000
|
-
"video_frame_analysis_status",
|
|
51001
|
-
"workflow_run",
|
|
51002
|
-
"workflow_step",
|
|
51003
|
-
"workflow_status",
|
|
51004
|
-
"workflow_artifact_read",
|
|
51005
|
-
"export_connected_service_data",
|
|
51006
|
-
"export_search_console_table_data",
|
|
51007
|
-
"renew_connected_data_download",
|
|
51008
|
-
"browser_open",
|
|
51009
|
-
"browser_screenshot",
|
|
51010
|
-
"browser_replay_start",
|
|
51011
|
-
"browser_replay_stop",
|
|
51012
|
-
"browser_replay_download",
|
|
51013
|
-
"create_scheduled_run_view_link",
|
|
51014
|
-
"get_scheduled_run",
|
|
51015
|
-
"library-ingest",
|
|
51016
|
-
"image_project_create",
|
|
51017
|
-
"image_project_list",
|
|
51018
|
-
"image_folder_create",
|
|
51019
|
-
"image_folder_list",
|
|
51020
|
-
"image_asset_save",
|
|
51021
|
-
"image_asset_get",
|
|
51022
|
-
"image_asset_list",
|
|
51023
|
-
"image_asset_search",
|
|
51024
|
-
"image_asset_move",
|
|
51025
|
-
"image_asset_delete"
|
|
51026
|
-
]);
|
|
51027
50134
|
OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
|
|
50135
|
+
ADVERTISE_OUTPUT_SCHEMAS = OUTPUT_SCHEMA_MODE === "contract";
|
|
51028
50136
|
OUTPUT_SCHEMAS = {};
|
|
51029
50137
|
}
|
|
51030
50138
|
});
|
|
@@ -52381,7 +51489,7 @@ var init_contracts = __esm({
|
|
|
52381
51489
|
});
|
|
52382
51490
|
|
|
52383
51491
|
// src/mcp/mcp-tool-schemas.ts
|
|
52384
|
-
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
51492
|
+
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, HarvestPaaHostedInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
52385
51493
|
var init_mcp_tool_schemas = __esm({
|
|
52386
51494
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
52387
51495
|
"use strict";
|
|
@@ -52429,7 +51537,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52429
51537
|
gl: import_zod45.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
52430
51538
|
hl: import_zod45.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
52431
51539
|
device: import_zod45.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
|
|
52432
|
-
idempotencyKey: import_zod45.z.string().trim().min(8).max(200).optional().describe("
|
|
51540
|
+
idempotencyKey: import_zod45.z.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
|
|
52433
51541
|
serpIdentity: import_zod45.z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
|
|
52434
51542
|
includeAllSerpFeatures: import_zod45.z.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
|
|
52435
51543
|
includeLocalPack: import_zod45.z.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
|
|
@@ -52439,6 +51547,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52439
51547
|
includeWhatPeopleSaying: import_zod45.z.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
|
|
52440
51548
|
debug: import_zod45.z.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
|
|
52441
51549
|
};
|
|
51550
|
+
HarvestPaaHostedInputSchema = Object.fromEntries(
|
|
51551
|
+
Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
|
|
51552
|
+
);
|
|
52442
51553
|
ExtractUrlBaseInputSchema = {
|
|
52443
51554
|
url: import_zod45.z.string().url().describe("Public http/https URL to extract."),
|
|
52444
51555
|
screenshot: import_zod45.z.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
|
|
@@ -55691,11 +54802,11 @@ function requireTasksCapability(capabilityValue) {
|
|
|
55691
54802
|
);
|
|
55692
54803
|
}
|
|
55693
54804
|
function taskKey(secret2) {
|
|
55694
|
-
return (0,
|
|
54805
|
+
return (0, import_node_crypto33.createHash)("sha256").update("mcp-scraper-task-handle\0", "utf8").update(secret2, "utf8").digest();
|
|
55695
54806
|
}
|
|
55696
54807
|
function encodeTaskHandle(payload, secret2) {
|
|
55697
|
-
const nonce = (0,
|
|
55698
|
-
const cipher = (0,
|
|
54808
|
+
const nonce = (0, import_node_crypto33.randomBytes)(12);
|
|
54809
|
+
const cipher = (0, import_node_crypto33.createCipheriv)("aes-256-gcm", taskKey(secret2), nonce);
|
|
55699
54810
|
const ciphertext = Buffer.concat([
|
|
55700
54811
|
cipher.update(JSON.stringify(payload), "utf8"),
|
|
55701
54812
|
cipher.final()
|
|
@@ -55712,7 +54823,7 @@ function decodeTaskHandle(taskId, secret2, ownerId2) {
|
|
|
55712
54823
|
const nonce = bytes.subarray(0, 12);
|
|
55713
54824
|
const tag = bytes.subarray(bytes.length - 16);
|
|
55714
54825
|
const ciphertext = bytes.subarray(12, bytes.length - 16);
|
|
55715
|
-
const decipher = (0,
|
|
54826
|
+
const decipher = (0, import_node_crypto33.createDecipheriv)("aes-256-gcm", taskKey(secret2), nonce);
|
|
55716
54827
|
decipher.setAuthTag(tag);
|
|
55717
54828
|
const parsed = JSON.parse(Buffer.concat([
|
|
55718
54829
|
decipher.update(ciphertext),
|
|
@@ -55770,6 +54881,73 @@ function taskSeed(kind, result) {
|
|
|
55770
54881
|
if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
|
|
55771
54882
|
return { kind, operationId, statusMessage: "Video analysis is running." };
|
|
55772
54883
|
}
|
|
54884
|
+
function createTaskResult(kind, result, options) {
|
|
54885
|
+
const seed = taskSeed(kind, result);
|
|
54886
|
+
if (!seed) return result;
|
|
54887
|
+
const createdAtMs2 = Date.now();
|
|
54888
|
+
const payload = {
|
|
54889
|
+
v: 1,
|
|
54890
|
+
ownerId: options.ownerId,
|
|
54891
|
+
kind: seed.kind,
|
|
54892
|
+
operationId: seed.operationId,
|
|
54893
|
+
createdAtMs: createdAtMs2,
|
|
54894
|
+
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
54895
|
+
};
|
|
54896
|
+
return {
|
|
54897
|
+
resultType: "task",
|
|
54898
|
+
taskId: encodeTaskHandle(payload, options.secret),
|
|
54899
|
+
status: "working",
|
|
54900
|
+
statusMessage: seed.statusMessage,
|
|
54901
|
+
createdAt: new Date(createdAtMs2).toISOString(),
|
|
54902
|
+
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
54903
|
+
ttlMs: TASK_TTL_MS,
|
|
54904
|
+
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
54905
|
+
// The registered SDK path validates structuredContent against the tool's
|
|
54906
|
+
// terminal output schema before it can serialize a Task envelope. Retain
|
|
54907
|
+
// the bounded start receipt for that path; the hosted HTTP Task seam runs
|
|
54908
|
+
// before terminal-output serialization.
|
|
54909
|
+
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
54910
|
+
};
|
|
54911
|
+
}
|
|
54912
|
+
async function startTaskTool(executor, toolName, rawArguments) {
|
|
54913
|
+
const args = objectValue(rawArguments) ?? {};
|
|
54914
|
+
if (toolName === "harvest_paa") {
|
|
54915
|
+
const parsed = import_zod46.z.object(HarvestPaaHostedInputSchema).strict().safeParse(args);
|
|
54916
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid harvest_paa arguments");
|
|
54917
|
+
return { kind: "paa_harvest", result: await executor.startHarvestPaa({ ...parsed.data, debug: false }) };
|
|
54918
|
+
}
|
|
54919
|
+
if (toolName === "extract_site") {
|
|
54920
|
+
const parsed = import_zod46.z.object(ExtractSiteInputSchema).safeParse(args);
|
|
54921
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid extract_site arguments");
|
|
54922
|
+
return { kind: "site_export", result: await executor.extractSite(parsed.data) };
|
|
54923
|
+
}
|
|
54924
|
+
if (toolName === "analyze_site_similarity") {
|
|
54925
|
+
const parsed = import_zod46.z.object(AnalyzeSiteSimilarityInputSchema).safeParse(args);
|
|
54926
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid analyze_site_similarity arguments");
|
|
54927
|
+
return { kind: "site_export", result: await executor.analyzeSiteSimilarity(parsed.data) };
|
|
54928
|
+
}
|
|
54929
|
+
if (toolName === "audit_site") {
|
|
54930
|
+
const parsed = import_zod46.z.object(AuditSiteInputSchema).safeParse(args);
|
|
54931
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid audit_site arguments");
|
|
54932
|
+
return { kind: "site_export", result: await executor.auditSite(parsed.data) };
|
|
54933
|
+
}
|
|
54934
|
+
if (toolName === "video_frame_analysis") {
|
|
54935
|
+
const parsed = import_zod46.z.object(VideoFrameAnalysisInputSchema).safeParse(args);
|
|
54936
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid video_frame_analysis arguments");
|
|
54937
|
+
return { kind: "video_frame_analysis", result: await executor.videoFrameAnalysis(parsed.data) };
|
|
54938
|
+
}
|
|
54939
|
+
if (toolName === "directory_workflow") {
|
|
54940
|
+
const parsed = import_zod46.z.object(DirectoryWorkflowInputSchema).safeParse(args);
|
|
54941
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid directory_workflow arguments");
|
|
54942
|
+
return { kind: "directory_workflow", result: await executor.directoryWorkflow(parsed.data) };
|
|
54943
|
+
}
|
|
54944
|
+
if (toolName === "lead_list_enrich") {
|
|
54945
|
+
const parsed = import_zod46.z.object(LeadListEnrichInputSchema).safeParse(args);
|
|
54946
|
+
if (!parsed.success) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid lead_list_enrich arguments");
|
|
54947
|
+
return { kind: "lead_list_enrichment", result: await executor.leadListEnrich(parsed.data) };
|
|
54948
|
+
}
|
|
54949
|
+
throw new import_server.ProtocolError(TASK_INVALID_PARAMS, `Unsupported Task-start tool: ${toolName}`);
|
|
54950
|
+
}
|
|
55773
54951
|
function taskBase(payload, taskId) {
|
|
55774
54952
|
return {
|
|
55775
54953
|
taskId,
|
|
@@ -55888,6 +55066,13 @@ function isWorking(kind, status) {
|
|
|
55888
55066
|
return status === "queued" || status === "running" || status === "processing" || status === "pending";
|
|
55889
55067
|
}
|
|
55890
55068
|
function registerMcpTasksExtension(server, executor, options) {
|
|
55069
|
+
const enabled = options.enableTaskStarts !== false;
|
|
55070
|
+
if (!enabled) {
|
|
55071
|
+
return {
|
|
55072
|
+
supportsTasks: () => false,
|
|
55073
|
+
taskify: (_kind, result) => result
|
|
55074
|
+
};
|
|
55075
|
+
}
|
|
55891
55076
|
server.server.registerCapabilities({
|
|
55892
55077
|
extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
|
|
55893
55078
|
});
|
|
@@ -55918,32 +55103,7 @@ function registerMcpTasksExtension(server, executor, options) {
|
|
|
55918
55103
|
},
|
|
55919
55104
|
taskify(kind, result) {
|
|
55920
55105
|
if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
|
|
55921
|
-
|
|
55922
|
-
if (!seed) return result;
|
|
55923
|
-
const createdAtMs2 = Date.now();
|
|
55924
|
-
const payload = {
|
|
55925
|
-
v: 1,
|
|
55926
|
-
ownerId: options.ownerId,
|
|
55927
|
-
kind: seed.kind,
|
|
55928
|
-
operationId: seed.operationId,
|
|
55929
|
-
createdAtMs: createdAtMs2,
|
|
55930
|
-
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
55931
|
-
};
|
|
55932
|
-
const taskId = encodeTaskHandle(payload, options.secret);
|
|
55933
|
-
return {
|
|
55934
|
-
resultType: "task",
|
|
55935
|
-
taskId,
|
|
55936
|
-
status: "working",
|
|
55937
|
-
statusMessage: seed.statusMessage,
|
|
55938
|
-
createdAt: new Date(createdAtMs2).toISOString(),
|
|
55939
|
-
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
55940
|
-
ttlMs: TASK_TTL_MS,
|
|
55941
|
-
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
55942
|
-
// The SDK validates advertised outputSchema before serializing the
|
|
55943
|
-
// Task envelope. Preserve the already-bounded start receipt so typed
|
|
55944
|
-
// tools remain task-capable when essential output schemas are on.
|
|
55945
|
-
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
55946
|
-
};
|
|
55106
|
+
return createTaskResult(kind, result, options);
|
|
55947
55107
|
}
|
|
55948
55108
|
};
|
|
55949
55109
|
}
|
|
@@ -55956,10 +55116,12 @@ function taskHttpError(id, error) {
|
|
|
55956
55116
|
}
|
|
55957
55117
|
async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
55958
55118
|
const routedMethod = request.headers.get("mcp-method");
|
|
55959
|
-
|
|
55119
|
+
const routedToolName = request.headers.get("mcp-name");
|
|
55120
|
+
const taskStartCall = routedMethod === "tools/call" && TASK_START_TOOL_NAMES.has(routedToolName ?? "");
|
|
55121
|
+
if (!taskStartCall && !["tasks/get", "tasks/update", "tasks/cancel"].includes(routedMethod ?? "")) return void 0;
|
|
55960
55122
|
let payload;
|
|
55961
55123
|
try {
|
|
55962
|
-
payload = await request.json();
|
|
55124
|
+
payload = await request.clone().json();
|
|
55963
55125
|
} catch {
|
|
55964
55126
|
return taskHttpError(null, new import_server.ProtocolError(-32700, "Parse error"));
|
|
55965
55127
|
}
|
|
@@ -55972,6 +55134,28 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
|
55972
55134
|
throw new import_server.ProtocolError(-32020, "Mcp-Method header does not match the JSON-RPC method");
|
|
55973
55135
|
}
|
|
55974
55136
|
const params = objectValue(payload.params);
|
|
55137
|
+
if (taskStartCall) {
|
|
55138
|
+
if (params?.name !== routedToolName || typeof routedToolName !== "string") {
|
|
55139
|
+
throw new import_server.ProtocolError(-32020, "Mcp-Name header must match params.name");
|
|
55140
|
+
}
|
|
55141
|
+
const meta2 = objectValue(params?._meta);
|
|
55142
|
+
if (!hasTasksCapability(meta2?.["io.modelcontextprotocol/clientCapabilities"])) return void 0;
|
|
55143
|
+
if (routedToolName === "harvest_paa" && objectValue(params?.arguments)?.serpIdentity !== void 0) {
|
|
55144
|
+
return void 0;
|
|
55145
|
+
}
|
|
55146
|
+
const started = await startTaskTool(executor, routedToolName, params?.arguments);
|
|
55147
|
+
const task = createTaskResult(started.kind, started.result, options);
|
|
55148
|
+
const taskObject = task;
|
|
55149
|
+
return new Response(JSON.stringify({
|
|
55150
|
+
jsonrpc: "2.0",
|
|
55151
|
+
id,
|
|
55152
|
+
result: {
|
|
55153
|
+
...taskObject,
|
|
55154
|
+
resultType: taskObject.resultType === "task" ? "task" : "complete",
|
|
55155
|
+
_meta: { "io.modelcontextprotocol/serverInfo": options.serverInfo }
|
|
55156
|
+
}
|
|
55157
|
+
}), { status: 200, headers: { "content-type": "application/json" } });
|
|
55158
|
+
}
|
|
55975
55159
|
const taskId = params?.taskId;
|
|
55976
55160
|
if (typeof taskId !== "string" || !taskId) throw new import_server.ProtocolError(TASK_INVALID_PARAMS, "Invalid taskId");
|
|
55977
55161
|
if (request.headers.get("mcp-name") !== taskId) {
|
|
@@ -56003,19 +55187,29 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
|
56003
55187
|
return taskHttpError(id, error instanceof import_server.ProtocolError ? error : new import_server.ProtocolError(-32603, error instanceof Error ? error.message : "Internal error"));
|
|
56004
55188
|
}
|
|
56005
55189
|
}
|
|
56006
|
-
var import_server,
|
|
55190
|
+
var import_server, import_node_crypto33, import_zod46, MCP_TASKS_EXTENSION_ID, TASK_HANDLE_VERSION, TASK_TTL_MS, TASK_POLL_INTERVAL_MS, TASK_INVALID_PARAMS, TASK_MISSING_CAPABILITY, TASK_START_TOOL_NAMES;
|
|
56007
55191
|
var init_mcp_tasks_extension = __esm({
|
|
56008
55192
|
"src/mcp/mcp-tasks-extension.ts"() {
|
|
56009
55193
|
"use strict";
|
|
56010
55194
|
import_server = require("@modelcontextprotocol/server");
|
|
56011
|
-
|
|
55195
|
+
import_node_crypto33 = require("crypto");
|
|
56012
55196
|
import_zod46 = require("zod");
|
|
55197
|
+
init_mcp_tool_schemas();
|
|
56013
55198
|
MCP_TASKS_EXTENSION_ID = "io.modelcontextprotocol/tasks";
|
|
56014
55199
|
TASK_HANDLE_VERSION = "mt1";
|
|
56015
55200
|
TASK_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
56016
55201
|
TASK_POLL_INTERVAL_MS = 5e3;
|
|
56017
55202
|
TASK_INVALID_PARAMS = -32602;
|
|
56018
55203
|
TASK_MISSING_CAPABILITY = -32003;
|
|
55204
|
+
TASK_START_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
55205
|
+
"harvest_paa",
|
|
55206
|
+
"extract_site",
|
|
55207
|
+
"analyze_site_similarity",
|
|
55208
|
+
"audit_site",
|
|
55209
|
+
"video_frame_analysis",
|
|
55210
|
+
"directory_workflow",
|
|
55211
|
+
"lead_list_enrich"
|
|
55212
|
+
]);
|
|
56019
55213
|
}
|
|
56020
55214
|
});
|
|
56021
55215
|
|
|
@@ -56306,7 +55500,7 @@ var init_analytics_mcp_tools = __esm({
|
|
|
56306
55500
|
|
|
56307
55501
|
// src/mcp/paa-mcp-server.ts
|
|
56308
55502
|
function hashOwnerId(callerKey) {
|
|
56309
|
-
return (0,
|
|
55503
|
+
return (0, import_node_crypto34.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
56310
55504
|
}
|
|
56311
55505
|
function liveWebToolAnnotations(title) {
|
|
56312
55506
|
return {
|
|
@@ -56446,19 +55640,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
56446
55640
|
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
56447
55641
|
const ownerId2 = options.ownerId ?? "local";
|
|
56448
55642
|
const ctx = { hosted: !savesReports, ownerId: ownerId2 };
|
|
55643
|
+
const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
|
|
56449
55644
|
registerAnalyticsMcpTools(server, executor);
|
|
56450
55645
|
const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
|
|
56451
55646
|
deploymentProfile: options.deploymentProfile ?? "production",
|
|
56452
|
-
transportProfile
|
|
55647
|
+
transportProfile,
|
|
56453
55648
|
baseUrl: options.baseUrl
|
|
56454
55649
|
});
|
|
56455
55650
|
const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
|
|
56456
55651
|
const tasks = registerMcpTasksExtension(server, executor, {
|
|
56457
55652
|
ownerId: ownerId2,
|
|
56458
|
-
secret: options.taskHandleSecret ?? `local:${ownerId2}
|
|
55653
|
+
secret: options.taskHandleSecret ?? `local:${ownerId2}`,
|
|
55654
|
+
enableTaskStarts: transportProfile === "hosted_http"
|
|
56459
55655
|
});
|
|
56460
55656
|
const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
|
|
56461
|
-
const harvestPaaInputSchema =
|
|
55657
|
+
const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
|
|
56462
55658
|
const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
|
|
56463
55659
|
const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
|
|
56464
55660
|
const directoryWorkflowInputSchema = schemaWithoutFields(
|
|
@@ -56473,10 +55669,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
56473
55669
|
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
56474
55670
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
56475
55671
|
}, async (input, requestContext) => {
|
|
56476
|
-
|
|
56477
|
-
|
|
55672
|
+
const executionInput = { ...input, debug: input.debug === true };
|
|
55673
|
+
if (tasks.supportsTasks() && !executionInput.serpIdentity) {
|
|
55674
|
+
return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
|
|
56478
55675
|
}
|
|
56479
|
-
return formatHarvestPaa(await executor.harvestPaa(
|
|
55676
|
+
return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
|
|
56480
55677
|
});
|
|
56481
55678
|
server.registerTool("search_serp", {
|
|
56482
55679
|
title: "Google SERP Lookup",
|
|
@@ -57291,7 +56488,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
57291
56488
|
annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
57292
56489
|
}, async (input) => executor.setScheduledActionConnections(input));
|
|
57293
56490
|
}
|
|
57294
|
-
var import_server2, import_zod48, import_node_fs11, import_node_path17,
|
|
56491
|
+
var import_server2, import_zod48, import_node_fs11, import_node_path17, import_node_crypto34, ACTION_CONFIRMATION_SCHEMA;
|
|
57295
56492
|
var init_paa_mcp_server = __esm({
|
|
57296
56493
|
"src/mcp/paa-mcp-server.ts"() {
|
|
57297
56494
|
"use strict";
|
|
@@ -57299,7 +56496,7 @@ var init_paa_mcp_server = __esm({
|
|
|
57299
56496
|
import_zod48 = require("zod");
|
|
57300
56497
|
import_node_fs11 = require("fs");
|
|
57301
56498
|
import_node_path17 = require("path");
|
|
57302
|
-
|
|
56499
|
+
import_node_crypto34 = require("crypto");
|
|
57303
56500
|
init_version();
|
|
57304
56501
|
init_rates();
|
|
57305
56502
|
init_mcp_response_formatter();
|
|
@@ -57420,11 +56617,11 @@ function analyticsReportPath(input, report) {
|
|
|
57420
56617
|
const suffix2 = query.size ? `?${query.toString()}` : "";
|
|
57421
56618
|
return `/analytics/sites/${encodeURIComponent(input.siteId)}/${report}${suffix2}`;
|
|
57422
56619
|
}
|
|
57423
|
-
var
|
|
56620
|
+
var import_node_crypto35, HttpMcpToolExecutor;
|
|
57424
56621
|
var init_http_mcp_tool_executor = __esm({
|
|
57425
56622
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
57426
56623
|
"use strict";
|
|
57427
|
-
|
|
56624
|
+
import_node_crypto35 = require("crypto");
|
|
57428
56625
|
init_harvest_timeout();
|
|
57429
56626
|
init_browser_service_env();
|
|
57430
56627
|
init_errors();
|
|
@@ -57493,13 +56690,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57493
56690
|
});
|
|
57494
56691
|
}
|
|
57495
56692
|
async callDirectoryWorkflowStart(body, explicitIdempotencyKey) {
|
|
57496
|
-
const idempotencyKey4 = `mcp-directory-${(0,
|
|
56693
|
+
const idempotencyKey4 = `mcp-directory-${(0, import_node_crypto35.createHash)("sha256").update(explicitIdempotencyKey).digest("hex")}`;
|
|
57497
56694
|
return this.call("/directory/run", body, this.timeoutMs, "POST", {
|
|
57498
56695
|
"Idempotency-Key": idempotencyKey4
|
|
57499
56696
|
});
|
|
57500
56697
|
}
|
|
57501
56698
|
async callSiteExtractStart(toolName, body, explicitIdempotencyKey) {
|
|
57502
|
-
const idempotencyKey4 = `mcp-site-${(0,
|
|
56699
|
+
const idempotencyKey4 = `mcp-site-${(0, import_node_crypto35.createHash)("sha256").update(toolName).update("\0").update(explicitIdempotencyKey).digest("hex")}`;
|
|
57503
56700
|
return this.call("/extract-site", body, this.timeoutMs, "POST", {
|
|
57504
56701
|
"Idempotency-Key": idempotencyKey4
|
|
57505
56702
|
});
|
|
@@ -57561,12 +56758,14 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57561
56758
|
}
|
|
57562
56759
|
harvestPaa(input) {
|
|
57563
56760
|
const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
|
|
57564
|
-
const
|
|
57565
|
-
|
|
56761
|
+
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
56762
|
+
const headers = idempotencyKey4 ? { "Idempotency-Key": idempotencyKey4 } : {};
|
|
56763
|
+
return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
|
|
57566
56764
|
}
|
|
57567
56765
|
startHarvestPaa(input) {
|
|
57568
|
-
const
|
|
57569
|
-
|
|
56766
|
+
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
56767
|
+
const headers = idempotencyKey4 ? { "Idempotency-Key": idempotencyKey4 } : {};
|
|
56768
|
+
return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
|
|
57570
56769
|
}
|
|
57571
56770
|
harvestPaaStatus(input) {
|
|
57572
56771
|
return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);
|
|
@@ -57941,7 +57140,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57941
57140
|
report: input.report,
|
|
57942
57141
|
format: input.format
|
|
57943
57142
|
}, this.timeoutMs, "POST", {
|
|
57944
|
-
"Idempotency-Key": `analytics-export-${(0,
|
|
57143
|
+
"Idempotency-Key": `analytics-export-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57945
57144
|
});
|
|
57946
57145
|
}
|
|
57947
57146
|
commonsSearchEntities(input) {
|
|
@@ -57980,7 +57179,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57980
57179
|
commonsSubmitEntity(input) {
|
|
57981
57180
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
57982
57181
|
return this.call("/commons/entities/propose", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
57983
|
-
"Idempotency-Key": `commons-${(0,
|
|
57182
|
+
"Idempotency-Key": `commons-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57984
57183
|
});
|
|
57985
57184
|
}
|
|
57986
57185
|
commonsGetEntityLedger(input) {
|
|
@@ -57995,7 +57194,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
57995
57194
|
commonsUpdateEditorialArticle(input) {
|
|
57996
57195
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
57997
57196
|
return this.call("/commons/publications/articles", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
57998
|
-
"Idempotency-Key": `commons-article-${(0,
|
|
57197
|
+
"Idempotency-Key": `commons-article-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
57999
57198
|
});
|
|
58000
57199
|
}
|
|
58001
57200
|
commonsSaveFilter(input) {
|
|
@@ -58016,13 +57215,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
58016
57215
|
commonsClaimPublication(input) {
|
|
58017
57216
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
58018
57217
|
return this.call("/commons/publications/claim", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
58019
|
-
"Idempotency-Key": `commons-publication-claim-${(0,
|
|
57218
|
+
"Idempotency-Key": `commons-publication-claim-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
58020
57219
|
});
|
|
58021
57220
|
}
|
|
58022
57221
|
commonsPublishEditorial(input) {
|
|
58023
57222
|
const { idempotencyKey: idempotencyKey4, ...body } = input;
|
|
58024
57223
|
return this.call("/commons/publications/publish", { ...body, idempotencyKey: idempotencyKey4 }, this.timeoutMs, "POST", {
|
|
58025
|
-
"Idempotency-Key": `commons-publication-publish-${(0,
|
|
57224
|
+
"Idempotency-Key": `commons-publication-publish-${(0, import_node_crypto35.createHash)("sha256").update(idempotencyKey4).digest("hex")}`
|
|
58026
57225
|
});
|
|
58027
57226
|
}
|
|
58028
57227
|
commonsGetPublication(input) {
|
|
@@ -58030,13 +57229,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
58030
57229
|
return this.getJson(input.subdomain ? `/commons/publications/${encodeURIComponent(input.subdomain)}?${query}` : `/commons/publications/me?${query}`);
|
|
58031
57230
|
}
|
|
58032
57231
|
async captureSerpSnapshot(input) {
|
|
58033
|
-
const fingerprint2 = (0,
|
|
57232
|
+
const fingerprint2 = (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
58034
57233
|
const now = Date.now();
|
|
58035
57234
|
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
58036
57235
|
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
58037
57236
|
}
|
|
58038
57237
|
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint2);
|
|
58039
|
-
const idempotencyKey4 = pending && pending.expiresAt > now ? pending.key : (0,
|
|
57238
|
+
const idempotencyKey4 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto35.randomUUID)();
|
|
58040
57239
|
this.pendingSerpCaptureBillingKeys.set(fingerprint2, {
|
|
58041
57240
|
key: idempotencyKey4,
|
|
58042
57241
|
expiresAt: now + 15 * 6e4
|
|
@@ -64046,7 +63245,7 @@ async function createImageSourceArtifact(args) {
|
|
|
64046
63245
|
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
64047
63246
|
throw new Error("image_source_size_invalid");
|
|
64048
63247
|
}
|
|
64049
|
-
const id = (0,
|
|
63248
|
+
const id = (0, import_node_crypto36.randomUUID)().replaceAll("-", "");
|
|
64050
63249
|
return createPrivateArtifact({
|
|
64051
63250
|
policy: policy4(),
|
|
64052
63251
|
ownerId: args.ownerId,
|
|
@@ -64065,11 +63264,11 @@ async function readOwnedImageSourceArtifact(args) {
|
|
|
64065
63264
|
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
64066
63265
|
});
|
|
64067
63266
|
}
|
|
64068
|
-
var
|
|
63267
|
+
var import_node_crypto36, IMAGE_SOURCE_ARTIFACT_PREFIX, IMAGE_SOURCE_ARTIFACT_TTL_MS, IMAGE_SOURCE_DOWNLOAD_TTL_MS, IMAGE_SOURCE_MAX_BYTES, ALLOWED_IMAGE_TYPES;
|
|
64069
63268
|
var init_image_source_artifacts = __esm({
|
|
64070
63269
|
"src/api/image-source-artifacts.ts"() {
|
|
64071
63270
|
"use strict";
|
|
64072
|
-
|
|
63271
|
+
import_node_crypto36 = require("crypto");
|
|
64073
63272
|
init_private_artifacts();
|
|
64074
63273
|
IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
64075
63274
|
IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
@@ -65683,7 +64882,7 @@ async function runBrowserAgentMigration() {
|
|
|
65683
64882
|
}
|
|
65684
64883
|
async function createExtensionRow(input) {
|
|
65685
64884
|
const db = getDb();
|
|
65686
|
-
const id = `bext_${(0,
|
|
64885
|
+
const id = `bext_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65687
64886
|
await db.execute({
|
|
65688
64887
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
65689
64888
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -65718,7 +64917,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
65718
64917
|
}
|
|
65719
64918
|
async function createAuthConnectionRow(input) {
|
|
65720
64919
|
const db = getDb();
|
|
65721
|
-
const connectionId = `authc_${(0,
|
|
64920
|
+
const connectionId = `authc_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65722
64921
|
await db.execute({
|
|
65723
64922
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
65724
64923
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -65803,7 +65002,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
65803
65002
|
}
|
|
65804
65003
|
async function createSessionRow(input) {
|
|
65805
65004
|
const db = getDb();
|
|
65806
|
-
const id = `bas_${(0,
|
|
65005
|
+
const id = `bas_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
65807
65006
|
await db.execute({
|
|
65808
65007
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
65809
65008
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -65880,7 +65079,7 @@ async function recordAction(input) {
|
|
|
65880
65079
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
65881
65080
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
65882
65081
|
args: [
|
|
65883
|
-
`baa_${(0,
|
|
65082
|
+
`baa_${(0, import_node_crypto37.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
65884
65083
|
input.sessionId,
|
|
65885
65084
|
input.type,
|
|
65886
65085
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -65920,11 +65119,11 @@ async function listReplayRows(sessionId) {
|
|
|
65920
65119
|
});
|
|
65921
65120
|
return res.rows;
|
|
65922
65121
|
}
|
|
65923
|
-
var
|
|
65122
|
+
var import_node_crypto37, _ready2, _migrationPromise2, ORPHANED_SESSION_STATUS;
|
|
65924
65123
|
var init_browser_agent_db = __esm({
|
|
65925
65124
|
"src/api/browser-agent-db.ts"() {
|
|
65926
65125
|
"use strict";
|
|
65927
|
-
|
|
65126
|
+
import_node_crypto37 = require("crypto");
|
|
65928
65127
|
init_db();
|
|
65929
65128
|
_ready2 = false;
|
|
65930
65129
|
_migrationPromise2 = null;
|
|
@@ -66693,7 +65892,7 @@ async function currentOrRuntimeContext(browser) {
|
|
|
66693
65892
|
function client() {
|
|
66694
65893
|
const apiKey = browserServiceApiKey();
|
|
66695
65894
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
66696
|
-
return new
|
|
65895
|
+
return new import_sdk9.default({ apiKey });
|
|
66697
65896
|
}
|
|
66698
65897
|
function isProfileConflict(err) {
|
|
66699
65898
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -67051,11 +66250,11 @@ async function replayList(runtimeSessionId) {
|
|
|
67051
66250
|
finishedAt: r.finished_at ?? null
|
|
67052
66251
|
}));
|
|
67053
66252
|
}
|
|
67054
|
-
var
|
|
66253
|
+
var import_sdk9, import_playwright6, DEFAULT_TIMEOUT_SECONDS;
|
|
67055
66254
|
var init_browser_agent_service = __esm({
|
|
67056
66255
|
"src/services/browser-agent/browser-agent-service.ts"() {
|
|
67057
66256
|
"use strict";
|
|
67058
|
-
|
|
66257
|
+
import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
|
|
67059
66258
|
import_playwright6 = require("playwright");
|
|
67060
66259
|
init_browser_service_env();
|
|
67061
66260
|
init_run_capture();
|
|
@@ -67064,15 +66263,71 @@ var init_browser_agent_service = __esm({
|
|
|
67064
66263
|
}
|
|
67065
66264
|
});
|
|
67066
66265
|
|
|
66266
|
+
// src/api/serp-identity-db.ts
|
|
66267
|
+
async function createSerpIdentityRow(input) {
|
|
66268
|
+
const id = `serpi_${(0, import_node_crypto38.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
66269
|
+
await getDb().execute({
|
|
66270
|
+
sql: `INSERT INTO serp_identities
|
|
66271
|
+
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
66272
|
+
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
66273
|
+
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
66274
|
+
});
|
|
66275
|
+
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
66276
|
+
if (!row) throw new Error("SERP identity insert failed");
|
|
66277
|
+
return row;
|
|
66278
|
+
}
|
|
66279
|
+
async function getSerpIdentityRow(userId, name) {
|
|
66280
|
+
const result = await getDb().execute({
|
|
66281
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
66282
|
+
args: [userId, name]
|
|
66283
|
+
});
|
|
66284
|
+
return result.rows[0] ?? null;
|
|
66285
|
+
}
|
|
66286
|
+
async function listSerpIdentityRows(userId) {
|
|
66287
|
+
const result = await getDb().execute({
|
|
66288
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
66289
|
+
args: [userId]
|
|
66290
|
+
});
|
|
66291
|
+
return result.rows;
|
|
66292
|
+
}
|
|
66293
|
+
async function touchSerpIdentity(userId, name) {
|
|
66294
|
+
await getDb().execute({
|
|
66295
|
+
sql: `UPDATE serp_identities
|
|
66296
|
+
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
66297
|
+
WHERE user_id = ? AND name = ?`,
|
|
66298
|
+
args: [userId, name]
|
|
66299
|
+
});
|
|
66300
|
+
}
|
|
66301
|
+
async function setSerpIdentityStatus(userId, name, status) {
|
|
66302
|
+
await getDb().execute({
|
|
66303
|
+
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
66304
|
+
args: [status, userId, name]
|
|
66305
|
+
});
|
|
66306
|
+
}
|
|
66307
|
+
async function deleteSerpIdentityRow(userId, name) {
|
|
66308
|
+
await getDb().execute({
|
|
66309
|
+
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
66310
|
+
args: [userId, name]
|
|
66311
|
+
});
|
|
66312
|
+
}
|
|
66313
|
+
var import_node_crypto38;
|
|
66314
|
+
var init_serp_identity_db = __esm({
|
|
66315
|
+
"src/api/serp-identity-db.ts"() {
|
|
66316
|
+
"use strict";
|
|
66317
|
+
import_node_crypto38 = require("crypto");
|
|
66318
|
+
init_db();
|
|
66319
|
+
}
|
|
66320
|
+
});
|
|
66321
|
+
|
|
67067
66322
|
// src/api/serp-identity-service.ts
|
|
67068
66323
|
function kernelClient() {
|
|
67069
66324
|
const apiKey = browserServiceApiKey();
|
|
67070
66325
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
67071
|
-
return new
|
|
66326
|
+
return new import_sdk10.default({ apiKey });
|
|
67072
66327
|
}
|
|
67073
66328
|
function backendName(userId, name, resource) {
|
|
67074
|
-
const digest2 = (0,
|
|
67075
|
-
const nonce = (0,
|
|
66329
|
+
const digest2 = (0, import_node_crypto39.createHash)("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
|
|
66330
|
+
const nonce = (0, import_node_crypto39.randomUUID)().replace(/-/g, "").slice(0, 8);
|
|
67076
66331
|
return `mcp-serp-${resource}-${digest2}-${nonce}`;
|
|
67077
66332
|
}
|
|
67078
66333
|
function isNotFound2(error) {
|
|
@@ -67150,12 +66405,12 @@ async function deleteSerpIdentity(userId, name) {
|
|
|
67150
66405
|
throw error;
|
|
67151
66406
|
}
|
|
67152
66407
|
}
|
|
67153
|
-
var
|
|
66408
|
+
var import_node_crypto39, import_sdk10, MAX_SERP_IDENTITIES_PER_USER;
|
|
67154
66409
|
var init_serp_identity_service = __esm({
|
|
67155
66410
|
"src/api/serp-identity-service.ts"() {
|
|
67156
66411
|
"use strict";
|
|
67157
|
-
|
|
67158
|
-
|
|
66412
|
+
import_node_crypto39 = require("crypto");
|
|
66413
|
+
import_sdk10 = __toESM(require("@onkernel/sdk"), 1);
|
|
67159
66414
|
init_browser_service_env();
|
|
67160
66415
|
init_serp_identity_db();
|
|
67161
66416
|
MAX_SERP_IDENTITIES_PER_USER = 3;
|
|
@@ -67865,7 +67120,7 @@ function buildBrowserAgentRoutes() {
|
|
|
67865
67120
|
}
|
|
67866
67121
|
const existing = await getExtensionRow(user.id, name);
|
|
67867
67122
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
67868
|
-
const backendName2 = `u${user.id}_${(0,
|
|
67123
|
+
const backendName2 = `u${user.id}_${(0, import_node_crypto40.randomUUID)().replace(/-/g, "")}`;
|
|
67869
67124
|
try {
|
|
67870
67125
|
const imported = await importExtensionFromStore(storeUrl, backendName2);
|
|
67871
67126
|
const row = await createExtensionRow({
|
|
@@ -67924,11 +67179,11 @@ function buildBrowserAgentRoutes() {
|
|
|
67924
67179
|
});
|
|
67925
67180
|
return app2;
|
|
67926
67181
|
}
|
|
67927
|
-
var
|
|
67182
|
+
var import_node_crypto40, import_hono24, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE, SERP_IDENTITY_NAME_RE;
|
|
67928
67183
|
var init_browser_agent_routes = __esm({
|
|
67929
67184
|
"src/api/browser-agent-routes.ts"() {
|
|
67930
67185
|
"use strict";
|
|
67931
|
-
|
|
67186
|
+
import_node_crypto40 = require("crypto");
|
|
67932
67187
|
import_hono24 = require("hono");
|
|
67933
67188
|
init_api_auth();
|
|
67934
67189
|
init_errors();
|
|
@@ -68486,7 +67741,7 @@ async function getKeys() {
|
|
|
68486
67741
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
68487
67742
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
68488
67743
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
68489
|
-
const kid = (0,
|
|
67744
|
+
const kid = (0, import_node_crypto41.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
68490
67745
|
publicJwk.kid = kid;
|
|
68491
67746
|
publicJwk.alg = "RS256";
|
|
68492
67747
|
publicJwk.use = "sig";
|
|
@@ -68665,23 +67920,23 @@ async function validateAuthRequest(p) {
|
|
|
68665
67920
|
}
|
|
68666
67921
|
function pkceMatches(verifier, challenge) {
|
|
68667
67922
|
if (!verifier) return false;
|
|
68668
|
-
const computed = (0,
|
|
67923
|
+
const computed = (0, import_node_crypto41.createHash)("sha256").update(verifier).digest("base64url");
|
|
68669
67924
|
return computed === challenge;
|
|
68670
67925
|
}
|
|
68671
67926
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
68672
67927
|
const { privateKey, kid } = await getKeys();
|
|
68673
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
67928
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto41.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
68674
67929
|
}
|
|
68675
67930
|
function tokenErrorResponse(c, error, description, status) {
|
|
68676
67931
|
return c.json({ error, error_description: description }, status);
|
|
68677
67932
|
}
|
|
68678
|
-
var import_hono26, import_cookie,
|
|
67933
|
+
var import_hono26, import_cookie, import_node_crypto41, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, OAUTH_DATABASE_TIMEOUT_MS, OAuthDatabaseUnavailableError, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
68679
67934
|
var init_oauth_routes = __esm({
|
|
68680
67935
|
"src/api/oauth-routes.ts"() {
|
|
68681
67936
|
"use strict";
|
|
68682
67937
|
import_hono26 = require("hono");
|
|
68683
67938
|
import_cookie = require("hono/cookie");
|
|
68684
|
-
|
|
67939
|
+
import_node_crypto41 = require("crypto");
|
|
68685
67940
|
import_jose2 = require("jose");
|
|
68686
67941
|
init_session();
|
|
68687
67942
|
init_db();
|
|
@@ -68788,7 +68043,7 @@ var init_oauth_routes = __esm({
|
|
|
68788
68043
|
}
|
|
68789
68044
|
}
|
|
68790
68045
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
68791
|
-
const clientId = `client_${(0,
|
|
68046
|
+
const clientId = `client_${(0, import_node_crypto41.randomBytes)(16).toString("hex")}`;
|
|
68792
68047
|
await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
|
|
68793
68048
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
68794
68049
|
return c.json({
|
|
@@ -68841,7 +68096,7 @@ var init_oauth_routes = __esm({
|
|
|
68841
68096
|
if (action3 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
68842
68097
|
if (action3 !== "approve") return c.text("unsupported action", 400);
|
|
68843
68098
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
68844
|
-
const code = `code_${(0,
|
|
68099
|
+
const code = `code_${(0, import_node_crypto41.randomBytes)(32).toString("base64url")}`;
|
|
68845
68100
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
68846
68101
|
await withOAuthDatabaseDeadline("put-authorization-code", putCode({
|
|
68847
68102
|
code,
|
|
@@ -68880,7 +68135,7 @@ var init_oauth_routes = __esm({
|
|
|
68880
68135
|
const plan = user ? resolvePlan(user) : "free";
|
|
68881
68136
|
const audience = record.resource ?? RESOURCE();
|
|
68882
68137
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
68883
|
-
const refreshToken = `rt_${(0,
|
|
68138
|
+
const refreshToken = `rt_${(0, import_node_crypto41.randomBytes)(40).toString("base64url")}`;
|
|
68884
68139
|
await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
|
|
68885
68140
|
refresh_token: refreshToken,
|
|
68886
68141
|
client_id: clientId,
|
|
@@ -68910,7 +68165,7 @@ var init_oauth_routes = __esm({
|
|
|
68910
68165
|
const plan = user ? resolvePlan(user) : "free";
|
|
68911
68166
|
const audience = record.resource ?? RESOURCE();
|
|
68912
68167
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
68913
|
-
const nextRefresh = `rt_${(0,
|
|
68168
|
+
const nextRefresh = `rt_${(0, import_node_crypto41.randomBytes)(40).toString("base64url")}`;
|
|
68914
68169
|
await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
|
|
68915
68170
|
refresh_token: nextRefresh,
|
|
68916
68171
|
client_id: record.client_id,
|
|
@@ -70330,7 +69585,7 @@ function quoteUntrusted(value) {
|
|
|
70330
69585
|
return clean3.split("\n").map((line) => `> ${line}`).join("\n");
|
|
70331
69586
|
}
|
|
70332
69587
|
function shortHash(value, length = 24) {
|
|
70333
|
-
return (0,
|
|
69588
|
+
return (0, import_node_crypto42.createHash)("sha256").update(value).digest("hex").slice(0, length);
|
|
70334
69589
|
}
|
|
70335
69590
|
function safePathPart(value) {
|
|
70336
69591
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
|
|
@@ -70818,11 +70073,11 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
70818
70073
|
direction: options.direction
|
|
70819
70074
|
};
|
|
70820
70075
|
}
|
|
70821
|
-
var
|
|
70076
|
+
var import_node_crypto42, SUPPORT_TAGS, ISSUE_TAGS;
|
|
70822
70077
|
var init_resend_support_thread = __esm({
|
|
70823
70078
|
"src/api/resend-support-thread.ts"() {
|
|
70824
70079
|
"use strict";
|
|
70825
|
-
|
|
70080
|
+
import_node_crypto42 = require("crypto");
|
|
70826
70081
|
init_memory();
|
|
70827
70082
|
SUPPORT_TAGS = [
|
|
70828
70083
|
{
|
|
@@ -71407,7 +70662,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
71407
70662
|
) < ?
|
|
71408
70663
|
`,
|
|
71409
70664
|
args: [
|
|
71410
|
-
(0,
|
|
70665
|
+
(0, import_node_crypto43.randomUUID)(),
|
|
71411
70666
|
input.eventSlug,
|
|
71412
70667
|
input.email,
|
|
71413
70668
|
input.firstName,
|
|
@@ -71424,7 +70679,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
71424
70679
|
id, event_slug, email, first_name, last_name, status
|
|
71425
70680
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
71426
70681
|
`,
|
|
71427
|
-
args: [(0,
|
|
70682
|
+
args: [(0, import_node_crypto43.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
71428
70683
|
});
|
|
71429
70684
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
71430
70685
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -71439,7 +70694,7 @@ async function registerWorkshopInterest(input) {
|
|
|
71439
70694
|
id, event_slug, email, first_name, last_name, status
|
|
71440
70695
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
71441
70696
|
`,
|
|
71442
|
-
args: [(0,
|
|
70697
|
+
args: [(0, import_node_crypto43.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
71443
70698
|
});
|
|
71444
70699
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
71445
70700
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -71469,11 +70724,11 @@ async function updateWorkshopRegistration(id, patch) {
|
|
|
71469
70724
|
args: [...entries.map(([, value]) => value), id]
|
|
71470
70725
|
});
|
|
71471
70726
|
}
|
|
71472
|
-
var
|
|
70727
|
+
var import_node_crypto43;
|
|
71473
70728
|
var init_workshop_registration_repository = __esm({
|
|
71474
70729
|
"src/api/workshop-registration-repository.ts"() {
|
|
71475
70730
|
"use strict";
|
|
71476
|
-
|
|
70731
|
+
import_node_crypto43 = require("crypto");
|
|
71477
70732
|
init_db();
|
|
71478
70733
|
}
|
|
71479
70734
|
});
|
|
@@ -72046,7 +71301,7 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
|
|
|
72046
71301
|
articleCount: articles.length,
|
|
72047
71302
|
wordCount: totalWordCount,
|
|
72048
71303
|
bytes,
|
|
72049
|
-
sha256: (0,
|
|
71304
|
+
sha256: (0, import_node_crypto44.createHash)("sha256").update(html).digest("hex"),
|
|
72050
71305
|
warnings
|
|
72051
71306
|
};
|
|
72052
71307
|
}
|
|
@@ -72129,14 +71384,14 @@ ${provenance}`;
|
|
|
72129
71384
|
...rendered,
|
|
72130
71385
|
html,
|
|
72131
71386
|
bytes: Buffer.byteLength(html),
|
|
72132
|
-
sha256: (0,
|
|
71387
|
+
sha256: (0, import_node_crypto44.createHash)("sha256").update(html).digest("hex")
|
|
72133
71388
|
};
|
|
72134
71389
|
}
|
|
72135
|
-
var
|
|
71390
|
+
var import_node_crypto44, import_node_fs14, import_node_path21, import_marked, runtimeEntryDir, assetCache;
|
|
72136
71391
|
var init_render = __esm({
|
|
72137
71392
|
"src/editorial-reading-room/render.ts"() {
|
|
72138
71393
|
"use strict";
|
|
72139
|
-
|
|
71394
|
+
import_node_crypto44 = require("crypto");
|
|
72140
71395
|
import_node_fs14 = require("fs");
|
|
72141
71396
|
import_node_path21 = require("path");
|
|
72142
71397
|
import_marked = require("marked");
|
|
@@ -72400,7 +71655,7 @@ async function hostCommonsImage(input) {
|
|
|
72400
71655
|
415
|
|
72401
71656
|
);
|
|
72402
71657
|
}
|
|
72403
|
-
const digest2 = (0,
|
|
71658
|
+
const digest2 = (0, import_node_crypto45.createHash)("sha256").update(bytes).digest("hex");
|
|
72404
71659
|
const existing = await getDb().execute({
|
|
72405
71660
|
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
72406
71661
|
args: [digest2]
|
|
@@ -72488,11 +71743,11 @@ async function hostEntityImages(input) {
|
|
|
72488
71743
|
}
|
|
72489
71744
|
return rewrite;
|
|
72490
71745
|
}
|
|
72491
|
-
var
|
|
71746
|
+
var import_node_crypto45, import_promises16, import_node_net2, COMMONS_IMAGE_MAX_BYTES, COMMONS_IMAGE_MAX_REDIRECTS, COMMONS_IMAGE_FETCH_TIMEOUT_MS, CommonsImageError, imageSchemaReady, hostOverride;
|
|
72492
71747
|
var init_commons_image_store = __esm({
|
|
72493
71748
|
"src/api/commons-image-store.ts"() {
|
|
72494
71749
|
"use strict";
|
|
72495
|
-
|
|
71750
|
+
import_node_crypto45 = require("crypto");
|
|
72496
71751
|
import_promises16 = require("dns/promises");
|
|
72497
71752
|
import_node_net2 = require("net");
|
|
72498
71753
|
init_blob_store();
|
|
@@ -72617,7 +71872,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
72617
71872
|
context[claim.predicate] = targets;
|
|
72618
71873
|
}
|
|
72619
71874
|
const document2 = { linkset: [context] };
|
|
72620
|
-
const etag = `"${(0,
|
|
71875
|
+
const etag = `"${(0, import_node_crypto46.createHash)("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
72621
71876
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
72622
71877
|
}
|
|
72623
71878
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -72685,11 +71940,11 @@ function normalizeHreflang(value) {
|
|
|
72685
71940
|
const languages = [...new Set(value.map((item) => optionalText(item, 80)).filter((item) => Boolean(item)))];
|
|
72686
71941
|
return languages.length ? languages : void 0;
|
|
72687
71942
|
}
|
|
72688
|
-
var
|
|
71943
|
+
var import_node_crypto46, COMMONS_LINKSET_MEDIA_TYPE, COMMONS_RELATIONSHIP_PROFILE, REGISTERED_RELATIONS;
|
|
72689
71944
|
var init_commons_linksets = __esm({
|
|
72690
71945
|
"src/api/commons-linksets.ts"() {
|
|
72691
71946
|
"use strict";
|
|
72692
|
-
|
|
71947
|
+
import_node_crypto46 = require("crypto");
|
|
72693
71948
|
COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
72694
71949
|
COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
72695
71950
|
REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -73329,7 +72584,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
73329
72584
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
73330
72585
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
73331
72586
|
const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
|
|
73332
|
-
const proposalId = `commons-proposal-${(0,
|
|
72587
|
+
const proposalId = `commons-proposal-${(0, import_node_crypto47.randomUUID)()}`;
|
|
73333
72588
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
73334
72589
|
const entityId = existing?.id ?? normalized.entityId ?? allocateEntityId();
|
|
73335
72590
|
const proposalStatus = shouldApply ? "accepted" : "pending_review";
|
|
@@ -73384,7 +72639,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
73384
72639
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
73385
72640
|
`,
|
|
73386
72641
|
args: [
|
|
73387
|
-
`commons-ledger-${(0,
|
|
72642
|
+
`commons-ledger-${(0, import_node_crypto47.randomUUID)()}`,
|
|
73388
72643
|
nextEntity.id,
|
|
73389
72644
|
proposalId,
|
|
73390
72645
|
user.id,
|
|
@@ -73490,7 +72745,7 @@ async function saveCommonsFilter(input, user) {
|
|
|
73490
72745
|
sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
|
|
73491
72746
|
args: [user.id, name]
|
|
73492
72747
|
});
|
|
73493
|
-
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0,
|
|
72748
|
+
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0, import_node_crypto47.randomUUID)()}`;
|
|
73494
72749
|
await getDb().execute({
|
|
73495
72750
|
sql: `
|
|
73496
72751
|
INSERT INTO commons_saved_filters (id, user_id, name, description, filter_json, created_at, updated_at)
|
|
@@ -74268,7 +73523,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
74268
73523
|
return documents;
|
|
74269
73524
|
}
|
|
74270
73525
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
74271
|
-
return `commons-index-${(0,
|
|
73526
|
+
return `commons-index-${(0, import_node_crypto47.createHash)("sha256").update(`${entityId}
|
|
74272
73527
|
${documentType}
|
|
74273
73528
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
74274
73529
|
}
|
|
@@ -74834,11 +74089,11 @@ function jsonLikeValue(value) {
|
|
|
74834
74089
|
function escapeLike(value) {
|
|
74835
74090
|
return value.replace(/[%_]/g, "");
|
|
74836
74091
|
}
|
|
74837
|
-
var
|
|
74092
|
+
var import_node_crypto47, COMMONS_SCHEMA_VERSION, DEFAULT_COMMONS_BASE_URL, DEFAULT_ENTITY_TYPE, COMMONS_ENTITY_PROFILES, schemaReady, COMMONS_SCHEMA_OBJECTS, SEARCH_CANDIDATE_LIMIT, CommonsRepositoryError;
|
|
74838
74093
|
var init_commons_repository = __esm({
|
|
74839
74094
|
"src/api/commons-repository.ts"() {
|
|
74840
74095
|
"use strict";
|
|
74841
|
-
|
|
74096
|
+
import_node_crypto47 = require("crypto");
|
|
74842
74097
|
init_db();
|
|
74843
74098
|
init_commons_image_store();
|
|
74844
74099
|
init_commons_embeddings();
|
|
@@ -75217,7 +74472,7 @@ async function claimCommonsPublication(input, user) {
|
|
|
75217
74472
|
}
|
|
75218
74473
|
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
75219
74474
|
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
75220
|
-
const id = `tcpub_${(0,
|
|
74475
|
+
const id = `tcpub_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75221
74476
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75222
74477
|
try {
|
|
75223
74478
|
await getDb().execute({
|
|
@@ -75269,7 +74524,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
75269
74524
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
75270
74525
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
75271
74526
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
75272
|
-
const editionId = `tced_${(0,
|
|
74527
|
+
const editionId = `tced_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75273
74528
|
const revision = (latest?.revision ?? 0) + 1;
|
|
75274
74529
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75275
74530
|
await getDb().batch([
|
|
@@ -75293,7 +74548,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
75293
74548
|
JSON.stringify(editionInput.articles),
|
|
75294
74549
|
html,
|
|
75295
74550
|
rendered.filename,
|
|
75296
|
-
(0,
|
|
74551
|
+
(0, import_node_crypto48.createHash)("sha256").update(html).digest("hex"),
|
|
75297
74552
|
rendered.articleCount,
|
|
75298
74553
|
rendered.wordCount,
|
|
75299
74554
|
Buffer.byteLength(html),
|
|
@@ -75351,7 +74606,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
75351
74606
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
75352
74607
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
75353
74608
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
75354
|
-
const editionId = `tced_${(0,
|
|
74609
|
+
const editionId = `tced_${(0, import_node_crypto48.randomUUID)()}`;
|
|
75355
74610
|
const revision = latest.revision + 1;
|
|
75356
74611
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
75357
74612
|
await getDb().batch([
|
|
@@ -75375,7 +74630,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
75375
74630
|
JSON.stringify(nextArticles),
|
|
75376
74631
|
html,
|
|
75377
74632
|
rendered.filename,
|
|
75378
|
-
(0,
|
|
74633
|
+
(0, import_node_crypto48.createHash)("sha256").update(html).digest("hex"),
|
|
75379
74634
|
rendered.articleCount,
|
|
75380
74635
|
rendered.wordCount,
|
|
75381
74636
|
Buffer.byteLength(html),
|
|
@@ -75580,11 +74835,11 @@ function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
|
75580
74835
|
"</head>"
|
|
75581
74836
|
].join("\n"));
|
|
75582
74837
|
}
|
|
75583
|
-
var
|
|
74838
|
+
var import_node_crypto48, PUBLICATION_ROOT_DOMAIN, RESERVED_SUBDOMAINS, CommonsPublicationError;
|
|
75584
74839
|
var init_commons_publication_repository = __esm({
|
|
75585
74840
|
"src/api/commons-publication-repository.ts"() {
|
|
75586
74841
|
"use strict";
|
|
75587
|
-
|
|
74842
|
+
import_node_crypto48 = require("crypto");
|
|
75588
74843
|
init_db();
|
|
75589
74844
|
init_commons_repository();
|
|
75590
74845
|
init_render();
|
|
@@ -76817,7 +76072,7 @@ async function migrateAnalytics() {
|
|
|
76817
76072
|
}
|
|
76818
76073
|
function normalizeSlug2(value) {
|
|
76819
76074
|
const slug4 = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
76820
|
-
return slug4 || `site-${(0,
|
|
76075
|
+
return slug4 || `site-${(0, import_node_crypto49.randomBytes)(4).toString("hex")}`;
|
|
76821
76076
|
}
|
|
76822
76077
|
function normalizeObservedHostname(origin) {
|
|
76823
76078
|
if (!origin || origin === "null") return null;
|
|
@@ -76830,7 +76085,7 @@ function normalizeObservedHostname(origin) {
|
|
|
76830
76085
|
}
|
|
76831
76086
|
}
|
|
76832
76087
|
function publicPixelId() {
|
|
76833
|
-
return `px_${(0,
|
|
76088
|
+
return `px_${(0, import_node_crypto49.randomBytes)(18).toString("base64url")}`;
|
|
76834
76089
|
}
|
|
76835
76090
|
function mapRows(rows) {
|
|
76836
76091
|
return rows;
|
|
@@ -76862,7 +76117,7 @@ async function requireEditor(client2, siteId, userId) {
|
|
|
76862
76117
|
async function createAnalyticsSite(input) {
|
|
76863
76118
|
const db = getAnalyticsPool();
|
|
76864
76119
|
const client2 = await db.connect();
|
|
76865
|
-
const id = (0,
|
|
76120
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
76866
76121
|
const baseSlug = normalizeSlug2(input.slug || input.name);
|
|
76867
76122
|
try {
|
|
76868
76123
|
await client2.query("BEGIN");
|
|
@@ -76878,7 +76133,7 @@ async function createAnalyticsSite(input) {
|
|
|
76878
76133
|
} catch (error) {
|
|
76879
76134
|
const code = error.code;
|
|
76880
76135
|
if (code !== "23505" || attempt === 2) throw error;
|
|
76881
|
-
slug4 = `${baseSlug}-${(0,
|
|
76136
|
+
slug4 = `${baseSlug}-${(0, import_node_crypto49.randomBytes)(2).toString("hex")}`;
|
|
76882
76137
|
}
|
|
76883
76138
|
}
|
|
76884
76139
|
await client2.query(
|
|
@@ -77093,7 +76348,7 @@ async function createAnalyticsPixel(input) {
|
|
|
77093
76348
|
VALUES ($1, $2, $3, $4, $5)
|
|
77094
76349
|
RETURNING id, site_id, public_id, name, environment, status, created_at::text, NULL::text AS last_event_at`,
|
|
77095
76350
|
[
|
|
77096
|
-
(0,
|
|
76351
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77097
76352
|
input.siteId,
|
|
77098
76353
|
publicPixelId(),
|
|
77099
76354
|
input.name.trim(),
|
|
@@ -77170,7 +76425,7 @@ async function setAnalyticsPixelDomainState(input) {
|
|
|
77170
76425
|
SELECT $1, p.id, $4, $5 FROM analytics_pixels p WHERE p.id = $2 AND p.site_id = $3
|
|
77171
76426
|
ON CONFLICT(pixel_id, hostname) DO UPDATE SET state = EXCLUDED.state, updated_at = now()
|
|
77172
76427
|
RETURNING id`,
|
|
77173
|
-
[(0,
|
|
76428
|
+
[(0, import_node_crypto49.randomUUID)(), input.pixelId, input.siteId, hostname, input.state]
|
|
77174
76429
|
);
|
|
77175
76430
|
if (!result.rowCount)
|
|
77176
76431
|
throw new AnalyticsRepositoryError(
|
|
@@ -77220,7 +76475,7 @@ function sanitizeAnalyticsProperties(value) {
|
|
|
77220
76475
|
}
|
|
77221
76476
|
async function ingestAnalyticsEvents(input) {
|
|
77222
76477
|
const db = getAnalyticsPool();
|
|
77223
|
-
const requestId = `air_${(0,
|
|
76478
|
+
const requestId = `air_${(0, import_node_crypto49.randomUUID)()}`;
|
|
77224
76479
|
const hostname = normalizeObservedHostname(input.origin);
|
|
77225
76480
|
const pixelResult = await db.query(
|
|
77226
76481
|
`SELECT p.id, p.site_id, p.status,
|
|
@@ -77241,7 +76496,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77241
76496
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77242
76497
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77243
76498
|
[
|
|
77244
|
-
(0,
|
|
76499
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77245
76500
|
requestId,
|
|
77246
76501
|
pixel?.site_id ?? null,
|
|
77247
76502
|
pixel?.id ?? null,
|
|
@@ -77266,7 +76521,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77266
76521
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77267
76522
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77268
76523
|
[
|
|
77269
|
-
(0,
|
|
76524
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77270
76525
|
requestId,
|
|
77271
76526
|
pixel.site_id,
|
|
77272
76527
|
pixel.id,
|
|
@@ -77291,7 +76546,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77291
76546
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, rejected_count, reason_codes)
|
|
77292
76547
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`,
|
|
77293
76548
|
[
|
|
77294
|
-
(0,
|
|
76549
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77295
76550
|
requestId,
|
|
77296
76551
|
pixel.site_id,
|
|
77297
76552
|
pixel.id,
|
|
@@ -77315,7 +76570,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77315
76570
|
ON CONFLICT(pixel_id, hostname) DO UPDATE
|
|
77316
76571
|
SET last_seen_at = now(), updated_at = now()
|
|
77317
76572
|
RETURNING state`,
|
|
77318
|
-
[(0,
|
|
76573
|
+
[(0, import_node_crypto49.randomUUID)(), pixel.id, hostname]
|
|
77319
76574
|
);
|
|
77320
76575
|
if (domainResult.rows[0]?.state !== "approved") {
|
|
77321
76576
|
await db.query(
|
|
@@ -77329,7 +76584,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77329
76584
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
77330
76585
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
77331
76586
|
[
|
|
77332
|
-
(0,
|
|
76587
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77333
76588
|
requestId,
|
|
77334
76589
|
pixel.site_id,
|
|
77335
76590
|
pixel.id,
|
|
@@ -77379,7 +76634,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77379
76634
|
$23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
|
|
77380
76635
|
) ON CONFLICT(site_id, event_id) DO NOTHING`,
|
|
77381
76636
|
[
|
|
77382
|
-
(0,
|
|
76637
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77383
76638
|
event2.eventId,
|
|
77384
76639
|
pixel.site_id,
|
|
77385
76640
|
pixel.id,
|
|
@@ -77437,7 +76692,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
77437
76692
|
id, request_id, site_id, pixel_id, hostname, accepted_count, rejected_count, reason_codes
|
|
77438
76693
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
|
77439
76694
|
[
|
|
77440
|
-
(0,
|
|
76695
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
77441
76696
|
requestId,
|
|
77442
76697
|
pixel.site_id,
|
|
77443
76698
|
pixel.id,
|
|
@@ -77469,7 +76724,7 @@ function normalizeGeographyCode(value, max) {
|
|
|
77469
76724
|
return normalized && /^[A-Z0-9-]+$/.test(normalized) ? normalized.slice(0, max) : null;
|
|
77470
76725
|
}
|
|
77471
76726
|
function pageFingerprint(scope, siteId, filters) {
|
|
77472
|
-
return (0,
|
|
76727
|
+
return (0, import_node_crypto49.createHash)("sha256").update(JSON.stringify({ scope, siteId, filters })).digest("base64url").slice(0, 18);
|
|
77473
76728
|
}
|
|
77474
76729
|
function decodePageOffset(cursor, fingerprint2) {
|
|
77475
76730
|
if (!cursor) return 0;
|
|
@@ -77519,7 +76774,7 @@ async function createAnalyticsConversion(input) {
|
|
|
77519
76774
|
404
|
|
77520
76775
|
);
|
|
77521
76776
|
}
|
|
77522
|
-
const id = (0,
|
|
76777
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
77523
76778
|
const resolvedPerson = input.sessionId ? await db.query(
|
|
77524
76779
|
`SELECT e.person_id FROM analytics_identity_nodes n JOIN analytics_identity_edges e ON e.identity_node_id = n.id
|
|
77525
76780
|
WHERE n.site_id = $1 AND n.kind = 'session_id' AND n.value_hmac = $2 ORDER BY e.confidence DESC LIMIT 1`,
|
|
@@ -78268,7 +77523,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
78268
77523
|
404
|
|
78269
77524
|
);
|
|
78270
77525
|
}
|
|
78271
|
-
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0,
|
|
77526
|
+
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0, import_node_crypto49.randomBytes)(5).toString("base64url").toLowerCase()).slice(0, 48);
|
|
78272
77527
|
try {
|
|
78273
77528
|
const result = await db.query(
|
|
78274
77529
|
`INSERT INTO analytics_campaign_links(
|
|
@@ -78277,7 +77532,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
78277
77532
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
|
|
78278
77533
|
RETURNING *, 0::int AS click_count`,
|
|
78279
77534
|
[
|
|
78280
|
-
(0,
|
|
77535
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78281
77536
|
input.siteId,
|
|
78282
77537
|
input.pixelId ?? null,
|
|
78283
77538
|
input.name.trim(),
|
|
@@ -78348,7 +77603,7 @@ async function resolveAnalyticsCampaignLink(shortCode, referrer) {
|
|
|
78348
77603
|
if (!row) return null;
|
|
78349
77604
|
await db.query(
|
|
78350
77605
|
`INSERT INTO analytics_campaign_clicks(id, link_id, referrer) VALUES ($1, $2, $3)`,
|
|
78351
|
-
[(0,
|
|
77606
|
+
[(0, import_node_crypto49.randomUUID)(), row.id, normalizeAnalyticsUrl(referrer || void 0)]
|
|
78352
77607
|
);
|
|
78353
77608
|
return buildTaggedCampaignUrl(row);
|
|
78354
77609
|
}
|
|
@@ -78366,14 +77621,14 @@ async function createAnalyticsForm(input) {
|
|
|
78366
77621
|
404
|
|
78367
77622
|
);
|
|
78368
77623
|
const baseSlug = normalizeSlug2(input.name);
|
|
78369
|
-
const slug4 = `${baseSlug}-${(0,
|
|
78370
|
-
const publicId = `form_${(0,
|
|
77624
|
+
const slug4 = `${baseSlug}-${(0, import_node_crypto49.randomBytes)(3).toString("hex")}`;
|
|
77625
|
+
const publicId = `form_${(0, import_node_crypto49.randomBytes)(18).toString("base64url")}`;
|
|
78371
77626
|
const result = await db.query(
|
|
78372
77627
|
`INSERT INTO analytics_forms(id, public_id, site_id, pixel_id, name, slug, fields, brand, submit_label, success_message, redirect_url, consent_text, status, created_by_user_id)
|
|
78373
77628
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10,$11,$12,$13,$14)
|
|
78374
77629
|
RETURNING *, 0::int AS submission_count`,
|
|
78375
77630
|
[
|
|
78376
|
-
(0,
|
|
77631
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78377
77632
|
publicId,
|
|
78378
77633
|
input.siteId,
|
|
78379
77634
|
input.pixelId,
|
|
@@ -78418,7 +77673,7 @@ async function getPublicAnalyticsForm(publicId) {
|
|
|
78418
77673
|
return result.rows[0] ?? null;
|
|
78419
77674
|
}
|
|
78420
77675
|
async function recordAnalyticsFormSubmission(input) {
|
|
78421
|
-
const id = (0,
|
|
77676
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78422
77677
|
await getAnalyticsPool().query(
|
|
78423
77678
|
`INSERT INTO analytics_form_submissions(id, form_id, site_id, pixel_id, visitor_id, session_id, crm_person_ref, source, medium, campaign, crm_delivery_status, click_ids)
|
|
78424
77679
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::jsonb)`,
|
|
@@ -78449,7 +77704,7 @@ function sanitizeClickIds(value) {
|
|
|
78449
77704
|
return output;
|
|
78450
77705
|
}
|
|
78451
77706
|
function identityHmac(kind, value) {
|
|
78452
|
-
return (0,
|
|
77707
|
+
return (0, import_node_crypto49.createHmac)("sha256", getSessionSecret()).update(`${kind}:${value.trim().toLowerCase()}`).digest("hex");
|
|
78453
77708
|
}
|
|
78454
77709
|
async function linkAnalyticsFormIdentity(input) {
|
|
78455
77710
|
const client2 = await getAnalyticsPool().connect();
|
|
@@ -78458,7 +77713,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78458
77713
|
const person = await client2.query(
|
|
78459
77714
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
78460
77715
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
78461
|
-
[(0,
|
|
77716
|
+
[(0, import_node_crypto49.randomUUID)(), input.siteId, input.crmPersonRef]
|
|
78462
77717
|
);
|
|
78463
77718
|
const personId = person.rows[0].id;
|
|
78464
77719
|
const signals = [];
|
|
@@ -78505,7 +77760,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78505
77760
|
`INSERT INTO analytics_identity_nodes(id, site_id, kind, value_hmac) VALUES ($1,$2,$3,$4)
|
|
78506
77761
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
78507
77762
|
[
|
|
78508
|
-
(0,
|
|
77763
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78509
77764
|
input.siteId,
|
|
78510
77765
|
signal.kind,
|
|
78511
77766
|
identityHmac(signal.kind, signal.value)
|
|
@@ -78516,7 +77771,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
78516
77771
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
78517
77772
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence)`,
|
|
78518
77773
|
[
|
|
78519
|
-
(0,
|
|
77774
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78520
77775
|
input.siteId,
|
|
78521
77776
|
personId,
|
|
78522
77777
|
node.rows[0].id,
|
|
@@ -78614,7 +77869,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
78614
77869
|
const db = getAnalyticsPool();
|
|
78615
77870
|
await requireEditor(db, input.siteId, input.userId);
|
|
78616
77871
|
const client2 = await db.connect();
|
|
78617
|
-
const id = (0,
|
|
77872
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78618
77873
|
try {
|
|
78619
77874
|
await client2.query("BEGIN");
|
|
78620
77875
|
await client2.query(
|
|
@@ -78634,7 +77889,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
78634
77889
|
await client2.query(
|
|
78635
77890
|
`INSERT INTO analytics_crm_import_rows(id, import_id, crm_person_ref, payload_ciphertext)
|
|
78636
77891
|
VALUES ($1,$2,$3,$4) ON CONFLICT(import_id, crm_person_ref) DO NOTHING`,
|
|
78637
|
-
[(0,
|
|
77892
|
+
[(0, import_node_crypto49.randomUUID)(), id, row.crmPersonRef, row.payloadCiphertext]
|
|
78638
77893
|
);
|
|
78639
77894
|
}
|
|
78640
77895
|
await client2.query("COMMIT");
|
|
@@ -78673,7 +77928,7 @@ async function createAnalyticsActivationDestination(input) {
|
|
|
78673
77928
|
`INSERT INTO analytics_activation_destinations(id, site_id, platform, name, connection_ref, external_dataset_id, event_mapping, created_by_user_id)
|
|
78674
77929
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8) RETURNING *`,
|
|
78675
77930
|
[
|
|
78676
|
-
(0,
|
|
77931
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78677
77932
|
input.siteId,
|
|
78678
77933
|
input.platform,
|
|
78679
77934
|
input.name.trim(),
|
|
@@ -78725,7 +77980,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
78725
77980
|
`INSERT INTO analytics_activation_jobs(id, destination_id, conversion_id, person_id, payload_ciphertext)
|
|
78726
77981
|
VALUES ($1,$2,$3,$4,$5) ON CONFLICT(destination_id, conversion_id) DO NOTHING`,
|
|
78727
77982
|
[
|
|
78728
|
-
(0,
|
|
77983
|
+
(0, import_node_crypto49.randomUUID)(),
|
|
78729
77984
|
destination.id,
|
|
78730
77985
|
input.conversionId,
|
|
78731
77986
|
input.personId ?? null,
|
|
@@ -78737,7 +77992,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
78737
77992
|
return queued;
|
|
78738
77993
|
}
|
|
78739
77994
|
async function queueAnalyticsFormDelivery(input) {
|
|
78740
|
-
const id = (0,
|
|
77995
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
78741
77996
|
await getAnalyticsPool().query(
|
|
78742
77997
|
`INSERT INTO analytics_form_delivery_jobs(id, submission_id, owner_user_id, payload_ciphertext, last_error_code)
|
|
78743
77998
|
VALUES ($1,$2,$3,$4,$5)`,
|
|
@@ -78864,7 +78119,7 @@ async function analyticsHealth(siteId, userId) {
|
|
|
78864
78119
|
}
|
|
78865
78120
|
async function refreshAnalyticsDailyRollups(input) {
|
|
78866
78121
|
const db = getAnalyticsPool();
|
|
78867
|
-
const runId = (0,
|
|
78122
|
+
const runId = (0, import_node_crypto49.randomUUID)();
|
|
78868
78123
|
await db.query(
|
|
78869
78124
|
`INSERT INTO analytics_rollup_runs(id, window_start, window_end, status) VALUES ($1, $2, $3, 'running')`,
|
|
78870
78125
|
[runId, input.start, input.end]
|
|
@@ -79061,7 +78316,7 @@ async function createAnalyticsExport(input) {
|
|
|
79061
78316
|
`Generated ${(/* @__PURE__ */ new Date()).toISOString()} from the governed ${input.report} report contract.`
|
|
79062
78317
|
].join("\n");
|
|
79063
78318
|
}
|
|
79064
|
-
const id = (0,
|
|
78319
|
+
const id = (0, import_node_crypto49.randomUUID)();
|
|
79065
78320
|
const inserted = await getAnalyticsPool().query(
|
|
79066
78321
|
`INSERT INTO analytics_exports(id, site_id, requested_by_user_id, idempotency_key, report, format, filters, content)
|
|
79067
78322
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)
|
|
@@ -79094,11 +78349,11 @@ async function createAnalyticsExport(input) {
|
|
|
79094
78349
|
}
|
|
79095
78350
|
return describeArtifact(replay.rows[0]);
|
|
79096
78351
|
}
|
|
79097
|
-
var
|
|
78352
|
+
var import_node_crypto49, import_pg, AnalyticsRepositoryError, pool2, MAX_ENGAGED_MS, ENGAGED_SESSION_MS, blockedPropertyName, inferredFamilySql, ANALYTICS_CONTENT_SORTS, clickIdKeys;
|
|
79098
78353
|
var init_analytics_repository = __esm({
|
|
79099
78354
|
"src/api/analytics-repository.ts"() {
|
|
79100
78355
|
"use strict";
|
|
79101
|
-
|
|
78356
|
+
import_node_crypto49 = require("crypto");
|
|
79102
78357
|
import_pg = require("pg");
|
|
79103
78358
|
init_session();
|
|
79104
78359
|
init_analytics_attribution();
|
|
@@ -79337,8 +78592,8 @@ function dashboardCallbackUrl() {
|
|
|
79337
78592
|
}
|
|
79338
78593
|
async function createThorbitConnectUrl(userId) {
|
|
79339
78594
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
79340
|
-
const state = (0,
|
|
79341
|
-
const stateHash = (0,
|
|
78595
|
+
const state = (0, import_node_crypto50.randomBytes)(32).toString("base64url");
|
|
78596
|
+
const stateHash = (0, import_node_crypto50.createHash)("sha256").update(state).digest("hex");
|
|
79342
78597
|
await getAnalyticsPool().query(
|
|
79343
78598
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
79344
78599
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -79353,9 +78608,9 @@ async function createThorbitConnectUrl(userId) {
|
|
|
79353
78608
|
function verifyThorbitAssertion(token6) {
|
|
79354
78609
|
const [encoded, signature] = token6.split(".");
|
|
79355
78610
|
if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
|
|
79356
|
-
const expected = (0,
|
|
78611
|
+
const expected = (0, import_node_crypto50.createHmac)("sha256", bridgeSecret()).update(encoded).digest();
|
|
79357
78612
|
const actual = Buffer.from(signature, "base64url");
|
|
79358
|
-
if (actual.length !== expected.length || !(0,
|
|
78613
|
+
if (actual.length !== expected.length || !(0, import_node_crypto50.timingSafeEqual)(actual, expected)) {
|
|
79359
78614
|
throw new Error("Invalid Thorbit assertion signature");
|
|
79360
78615
|
}
|
|
79361
78616
|
const parsed = AssertionSchema.safeParse(
|
|
@@ -79367,7 +78622,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
79367
78622
|
return parsed.data.entitlement;
|
|
79368
78623
|
}
|
|
79369
78624
|
async function consumeThorbitConnectCallback(input) {
|
|
79370
|
-
const stateHash = (0,
|
|
78625
|
+
const stateHash = (0, import_node_crypto50.createHash)("sha256").update(input.state).digest("hex");
|
|
79371
78626
|
const client2 = await getAnalyticsPool().connect();
|
|
79372
78627
|
try {
|
|
79373
78628
|
await client2.query("BEGIN");
|
|
@@ -79407,11 +78662,11 @@ async function disconnectAnalyticsEntitlement(userId) {
|
|
|
79407
78662
|
[userId]
|
|
79408
78663
|
);
|
|
79409
78664
|
}
|
|
79410
|
-
var
|
|
78665
|
+
var import_node_crypto50, import_zod55, THORBIT_PRODUCT_URL, REFRESH_INTERVAL_MS, ELIGIBLE_GRACE_MS, TRIAL_LENGTH_MS, ThorbitEntitlementSchema, AssertionSchema;
|
|
79411
78666
|
var init_analytics_entitlement = __esm({
|
|
79412
78667
|
"src/api/analytics-entitlement.ts"() {
|
|
79413
78668
|
"use strict";
|
|
79414
|
-
|
|
78669
|
+
import_node_crypto50 = require("crypto");
|
|
79415
78670
|
import_zod55 = require("zod");
|
|
79416
78671
|
init_analytics_repository();
|
|
79417
78672
|
THORBIT_PRODUCT_URL = "https://thorbit.ai";
|
|
@@ -79551,14 +78806,14 @@ function renderPublicForm(form, placementUrl) {
|
|
|
79551
78806
|
const brand = form.brand;
|
|
79552
78807
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><style>:root{font-family:ui-sans-serif,system-ui;color:${escapeHtml8(brand.textColor)};background:${escapeHtml8(brand.backgroundColor)}}*{box-sizing:border-box}body{margin:0;padding:18px}form{display:grid;gap:14px}label{display:grid;gap:6px;font-size:13px;font-weight:650}.check{grid-template-columns:auto 1fr;align-items:center}.check span{grid-column:2}.check input{grid-column:1;grid-row:1}input,textarea,select{width:100%;min-height:44px;padding:10px 12px;border:1px solid #ccd3df;border-radius:${brand.radius}px;background:white;color:inherit;font:inherit}textarea{min-height:110px;resize:vertical}button{min-height:46px;border:0;border-radius:${brand.radius}px;color:white;background:${escapeHtml8(brand.primaryColor)};font:inherit;font-weight:750;cursor:pointer}.consent{margin:0;color:#667085;font-size:11px;line-height:1.45}.notice{display:none;padding:12px;border-radius:${brand.radius}px;background:#edf8f2;color:#17613c}.hp{position:absolute!important;left:-9999px!important}</style></head><body><form id="mcp-form">${fields}<label class="hp">Website<input name="website" autocomplete="off" tabindex="-1"></label>${form.consent_text ? `<p class="consent">${escapeHtml8(form.consent_text)}</p>` : ""}<button>${escapeHtml8(form.submit_label)}</button><div class="notice" role="status"></div></form><script>(()=>{const form=document.querySelector('#mcp-form'),notice=form.querySelector('.notice'),q=new URLSearchParams(location.search);form.addEventListener('submit',async event=>{event.preventDefault();const button=form.querySelector('button');button.disabled=true;const raw=Object.fromEntries(new FormData(form).entries()),website=String(raw.website||''),clickIds={};delete raw.website;for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}try{const response=await fetch('/analytics/forms/${escapeHtml8(form.public_id)}/submissions',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({data:raw,website,placementUrl:${JSON.stringify(placementUrl)},visitorId:q.get('visitor')||undefined,sessionId:q.get('session')||undefined,source:q.get('source')||undefined,medium:q.get('medium')||undefined,campaign:q.get('campaign')||undefined,clickIds})});if(!response.ok)throw new Error('submit');notice.textContent=${JSON.stringify(form.success_message)};notice.style.display='block';form.reset();${form.redirect_url ? `setTimeout(()=>{top.location.href=${JSON.stringify(form.redirect_url)}},700);` : ""}}catch{notice.textContent='Your response could not be submitted. Please try again.';notice.style.display='block'}finally{button.disabled=false}})})()</script></body></html>`;
|
|
79553
78808
|
}
|
|
79554
|
-
var import_hono34, import_factory6, import_zod56,
|
|
78809
|
+
var import_hono34, import_factory6, import_zod56, import_node_crypto51, import_papaparse6, analyticsApp, auth3, entitlementGuard, ThorbitApiKeySchema, ThorbitCallbackSchema, SiteInputSchema, PixelInputSchema, PixelUpdateSchema, EventSchema, IngestionSchema, ConversionSchema, ExportSchema2, CampaignLinkSchema, FormFieldSchema, FormInputSchema, FormSubmissionSchema, CrmImportSchema, ActivationDestinationSchema, BusinessModelSchema, AdSpendSchema;
|
|
79555
78810
|
var init_analytics_routes = __esm({
|
|
79556
78811
|
"src/api/analytics-routes.ts"() {
|
|
79557
78812
|
"use strict";
|
|
79558
78813
|
import_hono34 = require("hono");
|
|
79559
78814
|
import_factory6 = require("hono/factory");
|
|
79560
78815
|
import_zod56 = require("zod");
|
|
79561
|
-
|
|
78816
|
+
import_node_crypto51 = require("crypto");
|
|
79562
78817
|
import_papaparse6 = __toESM(require("papaparse"), 1);
|
|
79563
78818
|
init_api_auth();
|
|
79564
78819
|
init_db();
|
|
@@ -79889,7 +79144,7 @@ var init_analytics_routes = __esm({
|
|
|
79889
79144
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
79890
79145
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
79891
79146
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
79892
|
-
const suffix2 = (0,
|
|
79147
|
+
const suffix2 = (0, import_node_crypto51.createHash)("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
79893
79148
|
const path6 = `Leads/person-${suffix2}`;
|
|
79894
79149
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
79895
79150
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path6 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -79988,9 +79243,9 @@ Submitted the published form.
|
|
|
79988
79243
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
79989
79244
|
});
|
|
79990
79245
|
const match = {
|
|
79991
|
-
...email ? { emailSha256: (0,
|
|
79246
|
+
...email ? { emailSha256: (0, import_node_crypto51.createHash)("sha256").update(email).digest("hex") } : {},
|
|
79992
79247
|
...phone ? {
|
|
79993
|
-
phoneSha256: (0,
|
|
79248
|
+
phoneSha256: (0, import_node_crypto51.createHash)("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
79994
79249
|
} : {}
|
|
79995
79250
|
};
|
|
79996
79251
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -80418,7 +79673,7 @@ Submitted the published form.
|
|
|
80418
79673
|
rejectedCount += 1;
|
|
80419
79674
|
continue;
|
|
80420
79675
|
}
|
|
80421
|
-
const digest2 = (0,
|
|
79676
|
+
const digest2 = (0, import_node_crypto51.createHash)("sha256").update(
|
|
80422
79677
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
80423
79678
|
).digest("hex").slice(0, 16);
|
|
80424
79679
|
const path6 = `Leads/person-${digest2}`;
|
|
@@ -80876,13 +80131,13 @@ var init_analytics_delivery = __esm({
|
|
|
80876
80131
|
|
|
80877
80132
|
// src/api/scheduled-artifact-owner.ts
|
|
80878
80133
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
80879
|
-
return (0,
|
|
80134
|
+
return (0, import_node_crypto52.createHash)("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
80880
80135
|
}
|
|
80881
|
-
var
|
|
80136
|
+
var import_node_crypto52;
|
|
80882
80137
|
var init_scheduled_artifact_owner = __esm({
|
|
80883
80138
|
"src/api/scheduled-artifact-owner.ts"() {
|
|
80884
80139
|
"use strict";
|
|
80885
|
-
|
|
80140
|
+
import_node_crypto52 = require("crypto");
|
|
80886
80141
|
}
|
|
80887
80142
|
});
|
|
80888
80143
|
|
|
@@ -80939,7 +80194,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
80939
80194
|
schemaReady2 = true;
|
|
80940
80195
|
}
|
|
80941
80196
|
function tokenHash2(token6) {
|
|
80942
|
-
return (0,
|
|
80197
|
+
return (0, import_node_crypto53.createHash)("sha256").update(token6).digest("hex");
|
|
80943
80198
|
}
|
|
80944
80199
|
function mapRow(row) {
|
|
80945
80200
|
return {
|
|
@@ -80959,9 +80214,9 @@ function mapRow(row) {
|
|
|
80959
80214
|
async function createScheduledRunViewLink(input) {
|
|
80960
80215
|
await ensureScheduledRunViewLinksSchema();
|
|
80961
80216
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
80962
|
-
const token6 = (0,
|
|
80217
|
+
const token6 = (0, import_node_crypto53.randomBytes)(32).toString("base64url");
|
|
80963
80218
|
const record = {
|
|
80964
|
-
shareId: (0,
|
|
80219
|
+
shareId: (0, import_node_crypto53.randomUUID)(),
|
|
80965
80220
|
ownerId: input.ownerId,
|
|
80966
80221
|
runId: input.runId,
|
|
80967
80222
|
artifactId: input.artifactId,
|
|
@@ -81024,11 +80279,11 @@ async function revokeScheduledRunViewLink(ownerId2, runId, shareId, now = /* @__
|
|
|
81024
80279
|
});
|
|
81025
80280
|
return result.rowsAffected > 0;
|
|
81026
80281
|
}
|
|
81027
|
-
var
|
|
80282
|
+
var import_node_crypto53, schemaReady2;
|
|
81028
80283
|
var init_scheduled_run_view_links = __esm({
|
|
81029
80284
|
"src/api/scheduled-run-view-links.ts"() {
|
|
81030
80285
|
"use strict";
|
|
81031
|
-
|
|
80286
|
+
import_node_crypto53 = require("crypto");
|
|
81032
80287
|
init_db();
|
|
81033
80288
|
schemaReady2 = false;
|
|
81034
80289
|
}
|
|
@@ -81094,15 +80349,15 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
81094
80349
|
html,
|
|
81095
80350
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
81096
80351
|
bytes,
|
|
81097
|
-
sha256: (0,
|
|
80352
|
+
sha256: (0, import_node_crypto54.createHash)("sha256").update(html).digest("hex"),
|
|
81098
80353
|
generatedAt: generatedAtIso
|
|
81099
80354
|
};
|
|
81100
80355
|
}
|
|
81101
|
-
var
|
|
80356
|
+
var import_node_crypto54;
|
|
81102
80357
|
var init_render2 = __esm({
|
|
81103
80358
|
"src/personal-authority/render.ts"() {
|
|
81104
80359
|
"use strict";
|
|
81105
|
-
|
|
80360
|
+
import_node_crypto54 = require("crypto");
|
|
81106
80361
|
}
|
|
81107
80362
|
});
|
|
81108
80363
|
|
|
@@ -81205,15 +80460,15 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
81205
80460
|
html,
|
|
81206
80461
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
81207
80462
|
bytes,
|
|
81208
|
-
sha256: (0,
|
|
80463
|
+
sha256: (0, import_node_crypto55.createHash)("sha256").update(html).digest("hex"),
|
|
81209
80464
|
generatedAt: generatedAtIso
|
|
81210
80465
|
};
|
|
81211
80466
|
}
|
|
81212
|
-
var
|
|
80467
|
+
var import_node_crypto55, FONT_STACKS;
|
|
81213
80468
|
var init_render_v2 = __esm({
|
|
81214
80469
|
"src/personal-authority/render-v2.ts"() {
|
|
81215
80470
|
"use strict";
|
|
81216
|
-
|
|
80471
|
+
import_node_crypto55 = require("crypto");
|
|
81217
80472
|
init_contracts();
|
|
81218
80473
|
FONT_STACKS = Object.freeze({
|
|
81219
80474
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
@@ -81329,15 +80584,15 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
81329
80584
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
81330
80585
|
</style></head><body data-theme="${escapeHtml11(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
81331
80586
|
const bytes = Buffer.byteLength(html);
|
|
81332
|
-
const sha2565 = (0,
|
|
80587
|
+
const sha2565 = (0, import_node_crypto56.createHash)("sha256").update(html).digest("hex");
|
|
81333
80588
|
const filename2 = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
81334
80589
|
return { html, filename: filename2, bytes, sha256: sha2565, generatedAt: generatedAtIso };
|
|
81335
80590
|
}
|
|
81336
|
-
var
|
|
80591
|
+
var import_node_crypto56;
|
|
81337
80592
|
var init_render3 = __esm({
|
|
81338
80593
|
"src/newsroom-publisher/render.ts"() {
|
|
81339
80594
|
"use strict";
|
|
81340
|
-
|
|
80595
|
+
import_node_crypto56 = require("crypto");
|
|
81341
80596
|
}
|
|
81342
80597
|
});
|
|
81343
80598
|
|
|
@@ -81429,7 +80684,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
81429
80684
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
81430
80685
|
</style></head><body data-theme="${escapeHtml12(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml12(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml12(input.eyebrow)}</span><h1>${escapeHtml12(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml12(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml12(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml12(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml12(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
81431
80686
|
const bytes = Buffer.byteLength(html);
|
|
81432
|
-
const sha2565 = (0,
|
|
80687
|
+
const sha2565 = (0, import_node_crypto57.createHash)("sha256").update(html).digest("hex");
|
|
81433
80688
|
return {
|
|
81434
80689
|
html,
|
|
81435
80690
|
filename: "blog-article.html",
|
|
@@ -81439,11 +80694,11 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
81439
80694
|
generatedAt: generatedAtIso
|
|
81440
80695
|
};
|
|
81441
80696
|
}
|
|
81442
|
-
var
|
|
80697
|
+
var import_node_crypto57;
|
|
81443
80698
|
var init_render4 = __esm({
|
|
81444
80699
|
"src/blog-article/render.ts"() {
|
|
81445
80700
|
"use strict";
|
|
81446
|
-
|
|
80701
|
+
import_node_crypto57 = require("crypto");
|
|
81447
80702
|
}
|
|
81448
80703
|
});
|
|
81449
80704
|
|
|
@@ -81774,7 +81029,7 @@ function policy6() {
|
|
|
81774
81029
|
};
|
|
81775
81030
|
}
|
|
81776
81031
|
function runStorageSegment(runId) {
|
|
81777
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0,
|
|
81032
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0, import_node_crypto58.createHash)("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
81778
81033
|
}
|
|
81779
81034
|
async function createScheduledRunArtifact(args) {
|
|
81780
81035
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -81815,12 +81070,12 @@ async function readScheduledRunArtifact(args) {
|
|
|
81815
81070
|
if (!window2 || window2.nextOffset !== null || window2.totalBytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) return null;
|
|
81816
81071
|
return window2.text;
|
|
81817
81072
|
}
|
|
81818
|
-
var
|
|
81073
|
+
var import_node_crypto58, SCHEDULED_RUN_ARTIFACT_PREFIX, SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS, SCHEDULED_RUN_ARTIFACT_MAX_BYTES;
|
|
81819
81074
|
var init_scheduled_run_artifact_store = __esm({
|
|
81820
81075
|
"src/scheduled-artifacts/scheduled-run-artifact-store.ts"() {
|
|
81821
81076
|
"use strict";
|
|
81822
81077
|
init_private_artifacts();
|
|
81823
|
-
|
|
81078
|
+
import_node_crypto58 = require("crypto");
|
|
81824
81079
|
SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
81825
81080
|
SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
81826
81081
|
SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -82038,7 +81293,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
82038
81293
|
updated_at = excluded.updated_at
|
|
82039
81294
|
`,
|
|
82040
81295
|
args: [
|
|
82041
|
-
(0,
|
|
81296
|
+
(0, import_node_crypto59.randomUUID)(),
|
|
82042
81297
|
userId,
|
|
82043
81298
|
connection.providerConfigKey,
|
|
82044
81299
|
connection.provider,
|
|
@@ -82109,7 +81364,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
82109
81364
|
});
|
|
82110
81365
|
await getDb().execute({
|
|
82111
81366
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
82112
|
-
args: [(0,
|
|
81367
|
+
args: [(0, import_node_crypto59.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
82113
81368
|
});
|
|
82114
81369
|
}
|
|
82115
81370
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -82149,7 +81404,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
82149
81404
|
if (!connection) throw new Error("service_connection_not_found");
|
|
82150
81405
|
const inserted = await getDb().execute({
|
|
82151
81406
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
82152
|
-
args: [(0,
|
|
81407
|
+
args: [(0, import_node_crypto59.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
82153
81408
|
});
|
|
82154
81409
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
82155
81410
|
const existing = await getDb().execute({
|
|
@@ -82174,11 +81429,11 @@ async function claimServiceConnectionAction(args) {
|
|
|
82174
81429
|
...result !== void 0 ? { result } : {}
|
|
82175
81430
|
};
|
|
82176
81431
|
}
|
|
82177
|
-
var
|
|
81432
|
+
var import_node_crypto59, schemaReady3, schemaDb6;
|
|
82178
81433
|
var init_service_connections = __esm({
|
|
82179
81434
|
"src/api/service-connections.ts"() {
|
|
82180
81435
|
"use strict";
|
|
82181
|
-
|
|
81436
|
+
import_node_crypto59 = require("crypto");
|
|
82182
81437
|
init_db();
|
|
82183
81438
|
schemaReady3 = null;
|
|
82184
81439
|
schemaDb6 = null;
|
|
@@ -82192,8 +81447,8 @@ function signingSecret() {
|
|
|
82192
81447
|
return secret2;
|
|
82193
81448
|
}
|
|
82194
81449
|
function schedulerIntegrationSignature(args) {
|
|
82195
|
-
const bodyHash = (0,
|
|
82196
|
-
return (0,
|
|
81450
|
+
const bodyHash = (0, import_node_crypto60.createHash)("sha256").update(args.body).digest("hex");
|
|
81451
|
+
return (0, import_node_crypto60.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
82197
81452
|
${args.path}
|
|
82198
81453
|
${args.timestamp}
|
|
82199
81454
|
${args.nonce}
|
|
@@ -82240,17 +81495,17 @@ async function verifySchedulerIntegrationRequest(request, rawBody) {
|
|
|
82240
81495
|
});
|
|
82241
81496
|
const suppliedBytes = Buffer.from(signature, "hex");
|
|
82242
81497
|
const expectedBytes = Buffer.from(expected, "hex");
|
|
82243
|
-
if (suppliedBytes.length !== expectedBytes.length || !(0,
|
|
81498
|
+
if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto60.timingSafeEqual)(suppliedBytes, expectedBytes)) {
|
|
82244
81499
|
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
82245
81500
|
}
|
|
82246
81501
|
await claimNonce(nonce, timestampMs);
|
|
82247
81502
|
return { requestId };
|
|
82248
81503
|
}
|
|
82249
|
-
var
|
|
81504
|
+
var import_node_crypto60, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
|
|
82250
81505
|
var init_scheduler_integration_auth = __esm({
|
|
82251
81506
|
"src/api/scheduler-integration-auth.ts"() {
|
|
82252
81507
|
"use strict";
|
|
82253
|
-
|
|
81508
|
+
import_node_crypto60 = require("crypto");
|
|
82254
81509
|
init_db();
|
|
82255
81510
|
init_service_connections();
|
|
82256
81511
|
MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
@@ -82558,7 +81813,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
82558
81813
|
finalUrl: checked.parsed.href,
|
|
82559
81814
|
width: dimensions?.width ?? null,
|
|
82560
81815
|
height: dimensions?.height ?? null,
|
|
82561
|
-
sha256: (0,
|
|
81816
|
+
sha256: (0, import_node_crypto61.createHash)("sha256").update(bytes).digest("hex")
|
|
82562
81817
|
};
|
|
82563
81818
|
}
|
|
82564
81819
|
throw new Error("media_redirect_rejected");
|
|
@@ -82672,7 +81927,7 @@ async function packagePageMedia(args) {
|
|
|
82672
81927
|
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
82673
81928
|
files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
|
|
82674
81929
|
const archive = await zipBuffer2(files);
|
|
82675
|
-
const id = (0,
|
|
81930
|
+
const id = (0, import_node_crypto61.randomBytes)(6).toString("hex");
|
|
82676
81931
|
const pointer = await createPrivateArtifact({
|
|
82677
81932
|
policy: policy7(),
|
|
82678
81933
|
ownerId: args.ownerId,
|
|
@@ -82685,11 +81940,11 @@ async function packagePageMedia(args) {
|
|
|
82685
81940
|
const localPath = token5() ? null : (0, import_node_path22.join)(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path22.join)((0, import_node_os15.homedir)(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
82686
81941
|
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
82687
81942
|
}
|
|
82688
|
-
var
|
|
81943
|
+
var import_node_crypto61, import_node_os15, import_node_path22, import_p_limit7, import_yazl3, PAGE_MEDIA_ARTIFACT_PREFIX, PAGE_MEDIA_ARTIFACT_TTL_MS, PAGE_MEDIA_DOWNLOAD_TTL_MS, MAX_FILE_BYTES, MAX_ARCHIVE_MEDIA_BYTES, MAX_INLINE_IMAGE_BYTES2, MAX_INLINE_TOTAL_BYTES2, DOWNLOAD_CONCURRENCY2, MAX_REDIRECTS3;
|
|
82689
81944
|
var init_page_media_artifacts = __esm({
|
|
82690
81945
|
"src/api/page-media-artifacts.ts"() {
|
|
82691
81946
|
"use strict";
|
|
82692
|
-
|
|
81947
|
+
import_node_crypto61 = require("crypto");
|
|
82693
81948
|
import_node_os15 = require("os");
|
|
82694
81949
|
import_node_path22 = require("path");
|
|
82695
81950
|
import_p_limit7 = __toESM(require("p-limit"), 1);
|
|
@@ -82798,7 +82053,7 @@ var init_site_extract_reconciliation = __esm({
|
|
|
82798
82053
|
|
|
82799
82054
|
// src/api/page-diff.ts
|
|
82800
82055
|
function sha256Hex(value) {
|
|
82801
|
-
return (0,
|
|
82056
|
+
return (0, import_node_crypto62.createHash)("sha256").update(value).digest("hex");
|
|
82802
82057
|
}
|
|
82803
82058
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
82804
82059
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -82855,11 +82110,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
82855
82110
|
totalChangedLineCount
|
|
82856
82111
|
};
|
|
82857
82112
|
}
|
|
82858
|
-
var
|
|
82113
|
+
var import_node_crypto62, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
82859
82114
|
var init_page_diff = __esm({
|
|
82860
82115
|
"src/api/page-diff.ts"() {
|
|
82861
82116
|
"use strict";
|
|
82862
|
-
|
|
82117
|
+
import_node_crypto62 = require("crypto");
|
|
82863
82118
|
import_diff = require("diff");
|
|
82864
82119
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
82865
82120
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -82933,7 +82188,7 @@ var init_scrape_vault_sink = __esm({
|
|
|
82933
82188
|
|
|
82934
82189
|
// src/api/scrape-image-sink.ts
|
|
82935
82190
|
function idempotencyKey3(userId, vault, input) {
|
|
82936
|
-
return `scrape-image-${(0,
|
|
82191
|
+
return `scrape-image-${(0, import_node_crypto63.createHash)("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
82937
82192
|
}
|
|
82938
82193
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
82939
82194
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -82975,11 +82230,11 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
|
82975
82230
|
assets
|
|
82976
82231
|
};
|
|
82977
82232
|
}
|
|
82978
|
-
var
|
|
82233
|
+
var import_node_crypto63, MAX_IMAGES_PER_SCRAPE;
|
|
82979
82234
|
var init_scrape_image_sink = __esm({
|
|
82980
82235
|
"src/api/scrape-image-sink.ts"() {
|
|
82981
82236
|
"use strict";
|
|
82982
|
-
|
|
82237
|
+
import_node_crypto63 = require("crypto");
|
|
82983
82238
|
init_memory();
|
|
82984
82239
|
MAX_IMAGES_PER_SCRAPE = 25;
|
|
82985
82240
|
}
|
|
@@ -83432,7 +82687,7 @@ function canonicalJson(value) {
|
|
|
83432
82687
|
return JSON.stringify(value);
|
|
83433
82688
|
}
|
|
83434
82689
|
function sha2563(value) {
|
|
83435
|
-
return (0,
|
|
82690
|
+
return (0, import_node_crypto64.createHash)("sha256").update(value).digest("hex");
|
|
83436
82691
|
}
|
|
83437
82692
|
function sensitiveKey(key) {
|
|
83438
82693
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -83647,11 +82902,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
83647
82902
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
83648
82903
|
};
|
|
83649
82904
|
}
|
|
83650
|
-
var
|
|
82905
|
+
var import_node_crypto64, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
83651
82906
|
var init_connection_memory_import = __esm({
|
|
83652
82907
|
"src/api/connection-memory-import.ts"() {
|
|
83653
82908
|
"use strict";
|
|
83654
|
-
|
|
82909
|
+
import_node_crypto64 = require("crypto");
|
|
83655
82910
|
init_slugify();
|
|
83656
82911
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
83657
82912
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -83735,7 +82990,7 @@ var init_scrape_blob_cleanup = __esm({
|
|
|
83735
82990
|
|
|
83736
82991
|
// src/api/site-export-reader.ts
|
|
83737
82992
|
function sha2564(value) {
|
|
83738
|
-
return (0,
|
|
82993
|
+
return (0, import_node_crypto65.createHash)("sha256").update(value).digest("hex");
|
|
83739
82994
|
}
|
|
83740
82995
|
function publicPageRecord(page) {
|
|
83741
82996
|
const { bodyMarkdown: _body, contentRef: _contentRef, discoveryLinks: _discovery, ...metadata } = page;
|
|
@@ -83843,11 +83098,11 @@ async function readOwnedSiteExportImage(input) {
|
|
|
83843
83098
|
if (!bytes || artifact.sha256 && sha2564(bytes) !== artifact.sha256) return null;
|
|
83844
83099
|
return { bytes, artifact };
|
|
83845
83100
|
}
|
|
83846
|
-
var
|
|
83101
|
+
var import_node_crypto65, SiteExportFormatUnavailableError;
|
|
83847
83102
|
var init_site_export_reader = __esm({
|
|
83848
83103
|
"src/api/site-export-reader.ts"() {
|
|
83849
83104
|
"use strict";
|
|
83850
|
-
|
|
83105
|
+
import_node_crypto65 = require("crypto");
|
|
83851
83106
|
init_site_extract_repository();
|
|
83852
83107
|
init_site_extract_content_store();
|
|
83853
83108
|
init_site_extract_artifacts();
|
|
@@ -83998,7 +83253,7 @@ function finitePositive(value, fallback) {
|
|
|
83998
83253
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
83999
83254
|
}
|
|
84000
83255
|
async function collectConnectedDataExport(args) {
|
|
84001
|
-
const exportId = (0,
|
|
83256
|
+
const exportId = (0, import_node_crypto66.randomUUID)();
|
|
84002
83257
|
const now = args.now ?? Date.now;
|
|
84003
83258
|
const startedAt = now();
|
|
84004
83259
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -84112,11 +83367,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
84112
83367
|
untrustedContent: true
|
|
84113
83368
|
};
|
|
84114
83369
|
}
|
|
84115
|
-
var
|
|
83370
|
+
var import_node_crypto66, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
84116
83371
|
var init_connected_data_export = __esm({
|
|
84117
83372
|
"src/api/connected-data-export.ts"() {
|
|
84118
83373
|
"use strict";
|
|
84119
|
-
|
|
83374
|
+
import_node_crypto66 = require("crypto");
|
|
84120
83375
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
84121
83376
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
84122
83377
|
);
|
|
@@ -84230,7 +83485,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
84230
83485
|
offset += rows.length;
|
|
84231
83486
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
84232
83487
|
}
|
|
84233
|
-
const exportId = (0,
|
|
83488
|
+
const exportId = (0, import_node_crypto67.randomUUID)();
|
|
84234
83489
|
const artifact = await args.writeArtifact({
|
|
84235
83490
|
ownerId: args.ownerId,
|
|
84236
83491
|
exportId,
|
|
@@ -84252,11 +83507,11 @@ async function exportSearchConsoleTableData(args) {
|
|
|
84252
83507
|
warnings
|
|
84253
83508
|
};
|
|
84254
83509
|
}
|
|
84255
|
-
var
|
|
83510
|
+
var import_node_crypto67, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
|
|
84256
83511
|
var init_search_console_table_export = __esm({
|
|
84257
83512
|
"src/api/search-console-table-export.ts"() {
|
|
84258
83513
|
"use strict";
|
|
84259
|
-
|
|
83514
|
+
import_node_crypto67 = require("crypto");
|
|
84260
83515
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
84261
83516
|
SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
84262
83517
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -84329,48 +83584,6 @@ var init_credit_operations = __esm({
|
|
|
84329
83584
|
}
|
|
84330
83585
|
});
|
|
84331
83586
|
|
|
84332
|
-
// src/api/harvest-attempt-events.ts
|
|
84333
|
-
function createHarvestAttemptRecorder(jobId2, userId) {
|
|
84334
|
-
return async (event2) => {
|
|
84335
|
-
if (event2.type === "started") {
|
|
84336
|
-
await startHarvestAttempt({
|
|
84337
|
-
jobId: jobId2,
|
|
84338
|
-
userId,
|
|
84339
|
-
attemptNumber: event2.attemptNumber,
|
|
84340
|
-
maxAttempts: event2.maxAttempts,
|
|
84341
|
-
query: event2.query,
|
|
84342
|
-
location: event2.location,
|
|
84343
|
-
maxQuestions: event2.maxQuestions,
|
|
84344
|
-
startedAt: event2.startedAt
|
|
84345
|
-
});
|
|
84346
|
-
return;
|
|
84347
|
-
}
|
|
84348
|
-
await finishHarvestAttempt({
|
|
84349
|
-
jobId: jobId2,
|
|
84350
|
-
attemptNumber: event2.attemptNumber,
|
|
84351
|
-
outcome: event2.outcome,
|
|
84352
|
-
kernelSessionId: event2.kernelSessionId,
|
|
84353
|
-
questionCount: event2.questionCount,
|
|
84354
|
-
durationMs: event2.durationMs,
|
|
84355
|
-
error: event2.error,
|
|
84356
|
-
willRetry: event2.willRetry,
|
|
84357
|
-
kernelDeleteStarted: event2.cleanup.kernelDeleteStarted,
|
|
84358
|
-
kernelDeleteSucceeded: event2.cleanup.kernelDeleteSucceeded,
|
|
84359
|
-
kernelDeleteError: event2.cleanup.kernelDeleteError,
|
|
84360
|
-
browserCloseSucceeded: event2.cleanup.browserCloseSucceeded,
|
|
84361
|
-
browserCloseError: event2.cleanup.browserCloseError,
|
|
84362
|
-
debug: event2.debug,
|
|
84363
|
-
completedAt: event2.completedAt
|
|
84364
|
-
});
|
|
84365
|
-
};
|
|
84366
|
-
}
|
|
84367
|
-
var init_harvest_attempt_events = __esm({
|
|
84368
|
-
"src/api/harvest-attempt-events.ts"() {
|
|
84369
|
-
"use strict";
|
|
84370
|
-
init_db();
|
|
84371
|
-
}
|
|
84372
|
-
});
|
|
84373
|
-
|
|
84374
83587
|
// src/api/memory-universe.ts
|
|
84375
83588
|
function stripExt(value) {
|
|
84376
83589
|
return value.replace(/\.(md|markdown|txt)$/i, "");
|
|
@@ -85368,7 +84581,7 @@ async function requireOwnedActiveConnection(identity, connectionId, allowNeedsRe
|
|
|
85368
84581
|
return connection;
|
|
85369
84582
|
}
|
|
85370
84583
|
async function withNangoClient(connection, run) {
|
|
85371
|
-
const transport = new
|
|
84584
|
+
const transport = new import_client13.StreamableHTTPClientTransport(nangoMcpUrl(), {
|
|
85372
84585
|
requestInit: {
|
|
85373
84586
|
headers: {
|
|
85374
84587
|
Authorization: `Bearer ${nangoSecret()}`,
|
|
@@ -85377,7 +84590,7 @@ async function withNangoClient(connection, run) {
|
|
|
85377
84590
|
}
|
|
85378
84591
|
}
|
|
85379
84592
|
});
|
|
85380
|
-
const client2 = new
|
|
84593
|
+
const client2 = new import_client13.Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
|
|
85381
84594
|
try {
|
|
85382
84595
|
await client2.connect(transport);
|
|
85383
84596
|
const value = await run(client2);
|
|
@@ -85435,7 +84648,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
85435
84648
|
});
|
|
85436
84649
|
const readTools = [...policies.values()].filter((policy8) => policy8.classification === "read").map((policy8) => policy8.name);
|
|
85437
84650
|
const actionTools = [...policies.values()].filter((policy8) => policy8.classification === "action").map((policy8) => policy8.name);
|
|
85438
|
-
const revision = (0,
|
|
84651
|
+
const revision = (0, import_node_crypto68.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
85439
84652
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
85440
84653
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
85441
84654
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -85462,8 +84675,8 @@ async function callNangoToolDirect(args) {
|
|
|
85462
84675
|
identity: args.identity,
|
|
85463
84676
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
85464
84677
|
});
|
|
85465
|
-
const requestId = args.requestId?.trim() || (0,
|
|
85466
|
-
const idempotencyKey4 = `main-nango:${(0,
|
|
84678
|
+
const requestId = args.requestId?.trim() || (0, import_node_crypto68.randomUUID)();
|
|
84679
|
+
const idempotencyKey4 = `main-nango:${(0, import_node_crypto68.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
85467
84680
|
const startedAt = /* @__PURE__ */ new Date();
|
|
85468
84681
|
const started = performance.now();
|
|
85469
84682
|
let result;
|
|
@@ -85488,7 +84701,7 @@ async function callNangoToolDirect(args) {
|
|
|
85488
84701
|
toolName: args.tool,
|
|
85489
84702
|
operationKind: args.operationKind ?? args.classification,
|
|
85490
84703
|
outcome: providerError ? "error" : "partial",
|
|
85491
|
-
requestId: requestId.length <= 200 ? requestId : (0,
|
|
84704
|
+
requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto68.createHash)("sha256").update(requestId).digest("hex"),
|
|
85492
84705
|
startedAt: startedAt.toISOString(),
|
|
85493
84706
|
completedAt: completedAt.toISOString()
|
|
85494
84707
|
}
|
|
@@ -85530,16 +84743,16 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
85530
84743
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
85531
84744
|
protocolVersion: null,
|
|
85532
84745
|
schemaSource: "live_tools_list",
|
|
85533
|
-
schemaHash: (0,
|
|
84746
|
+
schemaHash: (0, import_node_crypto68.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
85534
84747
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
85535
84748
|
};
|
|
85536
84749
|
}
|
|
85537
|
-
var
|
|
84750
|
+
var import_node_crypto68, import_client13, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
|
|
85538
84751
|
var init_main_nango_transport = __esm({
|
|
85539
84752
|
"src/api/main-nango-transport.ts"() {
|
|
85540
84753
|
"use strict";
|
|
85541
|
-
|
|
85542
|
-
|
|
84754
|
+
import_node_crypto68 = require("crypto");
|
|
84755
|
+
import_client13 = require("@modelcontextprotocol/client");
|
|
85543
84756
|
init_service_connections();
|
|
85544
84757
|
init_connected_usage_billing();
|
|
85545
84758
|
init_rates();
|
|
@@ -86162,8 +85375,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
86162
85375
|
return data.connection.actionsEnabled === true;
|
|
86163
85376
|
}
|
|
86164
85377
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey4) {
|
|
86165
|
-
const requestId = `main-connected-action:${(0,
|
|
86166
|
-
const requestDigest = (0,
|
|
85378
|
+
const requestId = `main-connected-action:${(0, import_node_crypto69.createHash)("sha256").update(identity).update("\0").update(idempotencyKey4?.trim() || (0, import_node_crypto69.randomUUID)()).digest("hex")}`;
|
|
85379
|
+
const requestDigest = (0, import_node_crypto69.createHash)("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
86167
85380
|
if (mainOwnsIntegrations()) {
|
|
86168
85381
|
const selectedTool = tool?.trim();
|
|
86169
85382
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -86314,7 +85527,7 @@ function canonicalJson2(value) {
|
|
|
86314
85527
|
return JSON.stringify(value);
|
|
86315
85528
|
}
|
|
86316
85529
|
function projectedToolSchemaHash(tool) {
|
|
86317
|
-
return (0,
|
|
85530
|
+
return (0, import_node_crypto69.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
86318
85531
|
}
|
|
86319
85532
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
86320
85533
|
if (mainOwnsIntegrations()) {
|
|
@@ -86591,11 +85804,11 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
86591
85804
|
untrustedContent: true
|
|
86592
85805
|
};
|
|
86593
85806
|
}
|
|
86594
|
-
var
|
|
85807
|
+
var import_node_crypto69, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
|
|
86595
85808
|
var init_nango_control = __esm({
|
|
86596
85809
|
"src/api/nango-control.ts"() {
|
|
86597
85810
|
"use strict";
|
|
86598
|
-
|
|
85811
|
+
import_node_crypto69 = require("crypto");
|
|
86599
85812
|
init_connected_data_export();
|
|
86600
85813
|
init_slack_connected_data_export();
|
|
86601
85814
|
init_main_nango_transport();
|
|
@@ -86951,7 +86164,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
86951
86164
|
return isRecord5(data) ? data.result ?? data : data;
|
|
86952
86165
|
}
|
|
86953
86166
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey4) {
|
|
86954
|
-
const requestId = `main-resend-action:${(0,
|
|
86167
|
+
const requestId = `main-resend-action:${(0, import_node_crypto70.createHash)("sha256").update(identity).update("\0").update(idempotencyKey4.trim()).digest("hex")}`;
|
|
86955
86168
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
86956
86169
|
method: "POST",
|
|
86957
86170
|
headers: { "x-request-id": requestId },
|
|
@@ -87016,12 +86229,12 @@ async function callResendExportPage(identity, input) {
|
|
|
87016
86229
|
untrustedContent: true
|
|
87017
86230
|
};
|
|
87018
86231
|
}
|
|
87019
|
-
var
|
|
86232
|
+
var import_node_crypto70, DEFAULT_CONNECTION_CONTROL_URL, RESEND_PROVIDER_CONFIG_KEY, RESEND_LOGO_URL, RESEND_DOCS_URL, RESEND_ADMIN_BLOCKED_TOOLS, RESEND_CONNECTION_SYNC_REQUIRED_TOOLS, ResendControlError;
|
|
87020
86233
|
var init_resend_control = __esm({
|
|
87021
86234
|
"src/api/resend-control.ts"() {
|
|
87022
86235
|
"use strict";
|
|
87023
86236
|
init_connected_data_export();
|
|
87024
|
-
|
|
86237
|
+
import_node_crypto70 = require("crypto");
|
|
87025
86238
|
DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
87026
86239
|
RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
87027
86240
|
RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -87628,7 +86841,7 @@ function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
|
|
|
87628
86841
|
);
|
|
87629
86842
|
});
|
|
87630
86843
|
}
|
|
87631
|
-
var import_resend3,
|
|
86844
|
+
var import_resend3, import_node_crypto71, import_hono38, import_hono39, import_factory8, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth4, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, deploymentProfile, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS, CRON_TICK_BUDGET_MS, CRON_TICK_DRAIN_BUDGET_MS;
|
|
87632
86845
|
var init_server = __esm({
|
|
87633
86846
|
"src/api/server.ts"() {
|
|
87634
86847
|
"use strict";
|
|
@@ -87641,7 +86854,7 @@ var init_server = __esm({
|
|
|
87641
86854
|
init_og();
|
|
87642
86855
|
import_resend3 = require("resend");
|
|
87643
86856
|
init_url_utils();
|
|
87644
|
-
|
|
86857
|
+
import_node_crypto71 = require("crypto");
|
|
87645
86858
|
init_kpo_extractor();
|
|
87646
86859
|
init_screenshot();
|
|
87647
86860
|
init_media_extractor();
|
|
@@ -87654,7 +86867,6 @@ var init_server = __esm({
|
|
|
87654
86867
|
init_site_extract();
|
|
87655
86868
|
init_directory_workflow2();
|
|
87656
86869
|
init_lead_list_enrichment2();
|
|
87657
|
-
init_paa_harvest();
|
|
87658
86870
|
init_local_sourcebook();
|
|
87659
86871
|
init_site_extract_repository();
|
|
87660
86872
|
init_site_extract_start();
|
|
@@ -87683,7 +86895,7 @@ var init_server = __esm({
|
|
|
87683
86895
|
init_local_sourcebook_routes();
|
|
87684
86896
|
init_location_data_routes();
|
|
87685
86897
|
init_directory_workflow_dispatch();
|
|
87686
|
-
|
|
86898
|
+
init_paa_harvest_direct();
|
|
87687
86899
|
init_paa_harvest_reconciliation();
|
|
87688
86900
|
init_unified_billing();
|
|
87689
86901
|
init_retention_sweeps();
|
|
@@ -89220,7 +88432,7 @@ var init_server = __esm({
|
|
|
89220
88432
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
89221
88433
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
89222
88434
|
} else {
|
|
89223
|
-
jobId2 = derivedJobId ?? (0,
|
|
88435
|
+
jobId2 = derivedJobId ?? (0, import_node_crypto71.randomUUID)();
|
|
89224
88436
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
89225
88437
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
89226
88438
|
const hold = await debitMcIdempotent(
|
|
@@ -89234,7 +88446,7 @@ var init_server = __esm({
|
|
|
89234
88446
|
try {
|
|
89235
88447
|
await createJobWithId(jobId2, user.id, options.query, {
|
|
89236
88448
|
...options,
|
|
89237
|
-
executionOwner: "
|
|
88449
|
+
executionOwner: "direct",
|
|
89238
88450
|
billingHoldMc: harvestCost,
|
|
89239
88451
|
billingDebitKey,
|
|
89240
88452
|
...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
|
|
@@ -89254,11 +88466,9 @@ var init_server = __esm({
|
|
|
89254
88466
|
).catch(() => void 0);
|
|
89255
88467
|
throw error;
|
|
89256
88468
|
}
|
|
89257
|
-
|
|
89258
|
-
console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
|
|
89259
|
-
});
|
|
88469
|
+
scheduleDirectPaaHarvest(jobId2);
|
|
89260
88470
|
}
|
|
89261
|
-
if (process.env.CRON_SECRET) {
|
|
88471
|
+
if (options.serpOnly && process.env.CRON_SECRET) {
|
|
89262
88472
|
const url = new URL(c.req.url);
|
|
89263
88473
|
void fetch(`${url.origin}/cron/tick`, {
|
|
89264
88474
|
headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
|
|
@@ -90726,7 +89936,7 @@ var init_server = __esm({
|
|
|
90726
89936
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
90727
89937
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
90728
89938
|
const unfinished = [];
|
|
90729
|
-
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation,
|
|
89939
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestReconciliation, commonsEmbeddingDrain, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
90730
89940
|
settleWithinTickBudget("drainQueue", unfinished, drainQueue2(budget), []),
|
|
90731
89941
|
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
90732
89942
|
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
@@ -90745,7 +89955,6 @@ var init_server = __esm({
|
|
|
90745
89955
|
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
90746
89956
|
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
90747
89957
|
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
90748
|
-
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
90749
89958
|
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
90750
89959
|
settleWithinTickBudget("commonsEmbeddingDrain", unfinished, embedQueuedCommonsDocuments(50), { claimed: 0, embedded: 0, failed: 0, remaining: 0 }),
|
|
90751
89960
|
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
@@ -90780,7 +89989,6 @@ var init_server = __esm({
|
|
|
90780
89989
|
siteExtractArtifactCleanup,
|
|
90781
89990
|
siteExtractRedispatch,
|
|
90782
89991
|
siteExtractReconciliation,
|
|
90783
|
-
paaHarvestRedispatch,
|
|
90784
89992
|
paaHarvestReconciliation,
|
|
90785
89993
|
commonsEmbeddingDrain,
|
|
90786
89994
|
localSourcebookRedispatch,
|
|
@@ -90847,7 +90055,7 @@ var init_server = __esm({
|
|
|
90847
90055
|
});
|
|
90848
90056
|
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono39.serve)({
|
|
90849
90057
|
client: inngest,
|
|
90850
|
-
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn,
|
|
90058
|
+
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn, localSourcebookFn]
|
|
90851
90059
|
}));
|
|
90852
90060
|
app.route("/admin/credits", adminCreditsApp);
|
|
90853
90061
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|