mcp-scraper 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
- package/dist/bin/api-server.cjs +1214 -328
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +75 -8
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
- package/dist/chunk-2XTYLJQK.js.map +1 -0
- package/dist/{chunk-PTABZABJ.js → chunk-44LNKZTE.js} +76 -8
- package/dist/chunk-44LNKZTE.js.map +1 -0
- package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
- package/dist/chunk-5BEPOFKG.js.map +1 -0
- package/dist/chunk-C7NN57WH.js +7 -0
- package/dist/chunk-C7NN57WH.js.map +1 -0
- package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
- package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
- package/dist/{chunk-QAH7WMYC.js → chunk-QH63IZTX.js} +3 -3
- package/dist/chunk-QH63IZTX.js.map +1 -0
- package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
- package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
- package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
- package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
- package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
- package/dist/{server-EVTCL3KB.js → server-3ND2EXYO.js} +882 -242
- package/dist/server-3ND2EXYO.js.map +1 -0
- package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
- package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
- package/package.json +1 -1
- package/dist/chunk-IAXQIHNS.js +0 -7
- package/dist/chunk-IAXQIHNS.js.map +0 -1
- package/dist/chunk-MFNGUM4L.js.map +0 -1
- package/dist/chunk-PTABZABJ.js.map +0 -1
- package/dist/chunk-QAH7WMYC.js.map +0 -1
- package/dist/chunk-YW2LXLDU.js.map +0 -1
- package/dist/server-EVTCL3KB.js.map +0 -1
- /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
- /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
- /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
- /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
- /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
- /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
- /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
- /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
- /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
- /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
- /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -41,11 +41,12 @@ function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
|
|
|
41
41
|
const clientMs = Math.min(serverMs + CLIENT_OVER_SERVER_MARGIN_MS, VERCEL_FUNCTION_MAX_MS - 5e3);
|
|
42
42
|
return { serverMs, clientMs };
|
|
43
43
|
}
|
|
44
|
-
var VERCEL_FUNCTION_MAX_MS, CLIENT_OVER_SERVER_MARGIN_MS;
|
|
44
|
+
var VERCEL_FUNCTION_MAX_MS, DEFAULT_TOOL_CLIENT_TIMEOUT_MS, CLIENT_OVER_SERVER_MARGIN_MS;
|
|
45
45
|
var init_harvest_timeout = __esm({
|
|
46
46
|
"src/harvest-timeout.ts"() {
|
|
47
47
|
"use strict";
|
|
48
48
|
VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
49
|
+
DEFAULT_TOOL_CLIENT_TIMEOUT_MS = 3e5;
|
|
49
50
|
CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
|
|
50
51
|
}
|
|
51
52
|
});
|
|
@@ -4067,6 +4068,7 @@ async function migrate(options = {}) {
|
|
|
4067
4068
|
`);
|
|
4068
4069
|
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
4069
4070
|
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
4071
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_active_user ON concurrency_locks(user_id, expires_at) WHERE status = 'active'`);
|
|
4070
4072
|
await db.execute(`
|
|
4071
4073
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
4072
4074
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -4253,6 +4255,16 @@ async function migrate(options = {}) {
|
|
|
4253
4255
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
4254
4256
|
)
|
|
4255
4257
|
`);
|
|
4258
|
+
await db.execute(`
|
|
4259
|
+
CREATE TABLE IF NOT EXISTS connected_account_charges (
|
|
4260
|
+
user_id INTEGER NOT NULL,
|
|
4261
|
+
connection_id TEXT NOT NULL,
|
|
4262
|
+
period_start TEXT NOT NULL,
|
|
4263
|
+
amount_mc INTEGER NOT NULL,
|
|
4264
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
4265
|
+
PRIMARY KEY (user_id, connection_id, period_start)
|
|
4266
|
+
)
|
|
4267
|
+
`);
|
|
4256
4268
|
await db.execute(`
|
|
4257
4269
|
CREATE TABLE IF NOT EXISTS billing_credits (
|
|
4258
4270
|
idempotency_key TEXT PRIMARY KEY,
|
|
@@ -4441,6 +4453,7 @@ async function migrate(options = {}) {
|
|
|
4441
4453
|
)
|
|
4442
4454
|
`);
|
|
4443
4455
|
await db.execute(`CREATE INDEX IF NOT EXISTS request_events_user_created_at ON request_events(user_id, created_at DESC)`);
|
|
4456
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS request_events_created_at ON request_events(created_at)`);
|
|
4444
4457
|
await db.execute(`
|
|
4445
4458
|
CREATE TABLE IF NOT EXISTS workflow_schedules (
|
|
4446
4459
|
id TEXT PRIMARY KEY,
|
|
@@ -6746,6 +6759,7 @@ var init_db = __esm({
|
|
|
6746
6759
|
"billing_debits_user_status",
|
|
6747
6760
|
"billing_events_authorization",
|
|
6748
6761
|
"billing_events_user_created_at",
|
|
6762
|
+
"concurrency_locks_active_user",
|
|
6749
6763
|
"concurrency_locks_operation",
|
|
6750
6764
|
"concurrency_locks_user_status",
|
|
6751
6765
|
"credit_lots_user_active",
|
|
@@ -6765,6 +6779,7 @@ var init_db = __esm({
|
|
|
6765
6779
|
"oauth_tokens_identity",
|
|
6766
6780
|
"page_snapshots_user_checked_at",
|
|
6767
6781
|
"prt_user_id",
|
|
6782
|
+
"request_events_created_at",
|
|
6768
6783
|
"request_events_user_created_at",
|
|
6769
6784
|
"scheduled_run_view_links_expiry",
|
|
6770
6785
|
"scheduled_run_view_links_owner_run",
|
|
@@ -23288,40 +23303,15 @@ function safeMetadata(metadata) {
|
|
|
23288
23303
|
return null;
|
|
23289
23304
|
}
|
|
23290
23305
|
}
|
|
23291
|
-
async function ensureConcurrencyGateSchema() {
|
|
23292
|
-
if (!schemaReady) {
|
|
23293
|
-
schemaReady = (async () => {
|
|
23294
|
-
const db = getDb();
|
|
23295
|
-
await db.execute(`
|
|
23296
|
-
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
23297
|
-
id TEXT PRIMARY KEY,
|
|
23298
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
23299
|
-
operation TEXT NOT NULL,
|
|
23300
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
23301
|
-
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
23302
|
-
expires_at TEXT NOT NULL,
|
|
23303
|
-
released_at TEXT,
|
|
23304
|
-
metadata TEXT
|
|
23305
|
-
)
|
|
23306
|
-
`);
|
|
23307
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
23308
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
23309
|
-
})();
|
|
23310
|
-
}
|
|
23311
|
-
return schemaReady;
|
|
23312
|
-
}
|
|
23313
23306
|
async function expireConcurrencyLocksForUser(userId) {
|
|
23314
|
-
await ensureConcurrencyGateSchema();
|
|
23315
23307
|
await getDb().execute({
|
|
23316
|
-
sql: `
|
|
23317
|
-
SET status = 'expired', released_at = datetime('now')
|
|
23308
|
+
sql: `DELETE FROM concurrency_locks
|
|
23318
23309
|
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
23319
23310
|
args: [userId]
|
|
23320
23311
|
});
|
|
23321
23312
|
}
|
|
23322
|
-
async function countActiveUsageForUser(userId) {
|
|
23323
|
-
await
|
|
23324
|
-
await expireConcurrencyLocksForUser(userId);
|
|
23313
|
+
async function countActiveUsageForUser(userId, alreadyExpired = false) {
|
|
23314
|
+
if (!alreadyExpired) await expireConcurrencyLocksForUser(userId);
|
|
23325
23315
|
const res = await getDb().execute({
|
|
23326
23316
|
sql: `SELECT
|
|
23327
23317
|
(
|
|
@@ -23341,7 +23331,6 @@ async function countActiveUsageForUser(userId) {
|
|
|
23341
23331
|
}
|
|
23342
23332
|
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
23343
23333
|
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
23344
|
-
await ensureConcurrencyGateSchema();
|
|
23345
23334
|
await expireConcurrencyLocksForUser(userId);
|
|
23346
23335
|
const res = await getDb().execute({
|
|
23347
23336
|
sql: `UPDATE concurrency_locks
|
|
@@ -23355,13 +23344,12 @@ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
|
23355
23344
|
return Number(res.rowsAffected ?? 0) > 0;
|
|
23356
23345
|
}
|
|
23357
23346
|
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
23358
|
-
await ensureConcurrencyGateSchema();
|
|
23359
23347
|
const limit = concurrencyLimitForUser(user);
|
|
23360
23348
|
if (isConcurrencyLimitBypassed(user)) {
|
|
23361
23349
|
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
23362
23350
|
}
|
|
23363
23351
|
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
23364
|
-
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
23352
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id, true), limit, operation, reused: true };
|
|
23365
23353
|
}
|
|
23366
23354
|
await expireConcurrencyLocksForUser(user.id);
|
|
23367
23355
|
const lockId = `cl_${(0, import_node_crypto10.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
|
|
@@ -23392,7 +23380,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
23392
23380
|
limit
|
|
23393
23381
|
]
|
|
23394
23382
|
});
|
|
23395
|
-
const active = await countActiveUsageForUser(user.id);
|
|
23383
|
+
const active = await countActiveUsageForUser(user.id, true);
|
|
23396
23384
|
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
23397
23385
|
return { ok: true, lockId, active, limit, operation };
|
|
23398
23386
|
}
|
|
@@ -23400,17 +23388,13 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
23400
23388
|
}
|
|
23401
23389
|
async function releaseConcurrencyGate(lockId) {
|
|
23402
23390
|
if (!lockId) return;
|
|
23403
|
-
await ensureConcurrencyGateSchema();
|
|
23404
23391
|
await getDb().execute({
|
|
23405
|
-
sql: `
|
|
23406
|
-
SET status = 'released', released_at = datetime('now')
|
|
23407
|
-
WHERE id = ? AND status = 'active'`,
|
|
23392
|
+
sql: `DELETE FROM concurrency_locks WHERE id = ?`,
|
|
23408
23393
|
args: [lockId]
|
|
23409
23394
|
});
|
|
23410
23395
|
}
|
|
23411
23396
|
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
23412
23397
|
if (!lockId) return;
|
|
23413
|
-
await ensureConcurrencyGateSchema();
|
|
23414
23398
|
await getDb().execute({
|
|
23415
23399
|
sql: `UPDATE concurrency_locks
|
|
23416
23400
|
SET expires_at = datetime('now', ?)
|
|
@@ -23418,7 +23402,7 @@ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECON
|
|
|
23418
23402
|
args: [lockTtlModifier(ttlSeconds), lockId]
|
|
23419
23403
|
});
|
|
23420
23404
|
}
|
|
23421
|
-
var import_node_crypto10, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS
|
|
23405
|
+
var import_node_crypto10, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS;
|
|
23422
23406
|
var init_concurrency_gates = __esm({
|
|
23423
23407
|
"src/api/concurrency-gates.ts"() {
|
|
23424
23408
|
"use strict";
|
|
@@ -23428,7 +23412,6 @@ var init_concurrency_gates = __esm({
|
|
|
23428
23412
|
DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
23429
23413
|
DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
23430
23414
|
MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
23431
|
-
schemaReady = null;
|
|
23432
23415
|
}
|
|
23433
23416
|
});
|
|
23434
23417
|
|
|
@@ -27530,7 +27513,7 @@ async function recoverSettlement(row, user, authorization, duplicate) {
|
|
|
27530
27513
|
}
|
|
27531
27514
|
}
|
|
27532
27515
|
await getDb().execute({
|
|
27533
|
-
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status
|
|
27516
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('settling', 'cost_pending')",
|
|
27534
27517
|
args: [row.id]
|
|
27535
27518
|
});
|
|
27536
27519
|
await getDb().execute({
|
|
@@ -27575,11 +27558,13 @@ async function settleScheduledRun(args) {
|
|
|
27575
27558
|
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
27576
27559
|
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
27577
27560
|
const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
27578
|
-
const
|
|
27561
|
+
const modelCostUnreported = args.modelCostStatus === "missing";
|
|
27562
|
+
const pendingReason = modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
|
|
27579
27563
|
const metadata = safeMetadata2({
|
|
27580
27564
|
...args.metadata,
|
|
27581
27565
|
billingPhase: "settlement",
|
|
27582
27566
|
chargeMode: "model_delta",
|
|
27567
|
+
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
27583
27568
|
...pendingReason ? { reason: pendingReason } : {}
|
|
27584
27569
|
});
|
|
27585
27570
|
const eventId = existing?.id ?? (0, import_node_crypto14.randomUUID)();
|
|
@@ -27623,6 +27608,72 @@ async function settleScheduledRun(args) {
|
|
|
27623
27608
|
}
|
|
27624
27609
|
return recoverSettlement(existing, user, authorization, duplicateClaim);
|
|
27625
27610
|
}
|
|
27611
|
+
async function reconcileScheduledRunSettlements(limit = 25, minimumAgeHours = 24) {
|
|
27612
|
+
const stale = await getDb().execute({
|
|
27613
|
+
sql: `SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata
|
|
27614
|
+
FROM billing_events
|
|
27615
|
+
WHERE billing_class = ? AND status = 'cost_pending'
|
|
27616
|
+
AND created_at < datetime('now', ?)
|
|
27617
|
+
ORDER BY created_at
|
|
27618
|
+
LIMIT ?`,
|
|
27619
|
+
args: [SCHEDULED_RUN_BILLING_CLASS, `-${Math.max(1, Math.trunc(minimumAgeHours))} hours`, Math.max(1, Math.trunc(limit))]
|
|
27620
|
+
});
|
|
27621
|
+
const result = { checked: stale.rows.length, settled: 0, chargedMc: 0, failed: 0 };
|
|
27622
|
+
for (const raw of stale.rows) {
|
|
27623
|
+
const row = raw;
|
|
27624
|
+
try {
|
|
27625
|
+
const authorization = await authorizationById(row.authorization_id);
|
|
27626
|
+
if (!authorization) {
|
|
27627
|
+
result.failed += 1;
|
|
27628
|
+
continue;
|
|
27629
|
+
}
|
|
27630
|
+
const cap = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
27631
|
+
const chargeMc = Math.min(Math.max(0, Math.round(Number(row.amount_mc))), cap);
|
|
27632
|
+
if (chargeMc > 0) {
|
|
27633
|
+
const debit = await debitMcIdempotent(
|
|
27634
|
+
Number(authorization.user_id),
|
|
27635
|
+
chargeMc,
|
|
27636
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
27637
|
+
"scheduled-run:" + authorization.id + ":model:reconciled",
|
|
27638
|
+
authorization.id + ":model"
|
|
27639
|
+
);
|
|
27640
|
+
if (!debit.ok) {
|
|
27641
|
+
result.failed += 1;
|
|
27642
|
+
continue;
|
|
27643
|
+
}
|
|
27644
|
+
}
|
|
27645
|
+
await getDb().execute({
|
|
27646
|
+
sql: `UPDATE billing_events
|
|
27647
|
+
SET status = 'settled', amount_mc = ?, settled_at = datetime('now'), metadata = ?
|
|
27648
|
+
WHERE id = ? AND status = 'cost_pending'`,
|
|
27649
|
+
args: [
|
|
27650
|
+
chargeMc,
|
|
27651
|
+
safeMetadata2({ ...metadataReason(row) ? { reason: metadataReason(row) } : {}, autoReconciled: true, cappedAtAuthorizedMaximum: chargeMc < Number(row.amount_mc) }),
|
|
27652
|
+
row.id
|
|
27653
|
+
]
|
|
27654
|
+
});
|
|
27655
|
+
await getDb().execute({
|
|
27656
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending')",
|
|
27657
|
+
args: [authorization.id]
|
|
27658
|
+
});
|
|
27659
|
+
result.settled += 1;
|
|
27660
|
+
result.chargedMc += chargeMc;
|
|
27661
|
+
console.error(JSON.stringify({
|
|
27662
|
+
event: "scheduled_run_settlement_reconciled",
|
|
27663
|
+
authorization_id: authorization.id,
|
|
27664
|
+
charged_mc: chargeMc,
|
|
27665
|
+
reason: metadataReason(row) ?? "unreported_cost"
|
|
27666
|
+
}));
|
|
27667
|
+
} catch (err) {
|
|
27668
|
+
result.failed += 1;
|
|
27669
|
+
console.error(JSON.stringify({
|
|
27670
|
+
event: "scheduled_run_reconciliation_failed",
|
|
27671
|
+
message: err instanceof Error ? err.message : String(err)
|
|
27672
|
+
}));
|
|
27673
|
+
}
|
|
27674
|
+
}
|
|
27675
|
+
return result;
|
|
27676
|
+
}
|
|
27626
27677
|
async function voidScheduledRunAuthorization(args) {
|
|
27627
27678
|
const user = await resolveSchedulingUser(args.identity);
|
|
27628
27679
|
const authorization = await authorizationById(args.authorizationId);
|
|
@@ -28609,28 +28660,6 @@ var init_inbox_routes = __esm({
|
|
|
28609
28660
|
});
|
|
28610
28661
|
|
|
28611
28662
|
// src/api/connected-account-billing.ts
|
|
28612
|
-
async function ensureConnectedAccountBillingSchema() {
|
|
28613
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
28614
|
-
user_id INTEGER PRIMARY KEY,
|
|
28615
|
-
stripe_subscription_id TEXT,
|
|
28616
|
-
stripe_subscription_item_id TEXT,
|
|
28617
|
-
price_id TEXT,
|
|
28618
|
-
quantity INTEGER NOT NULL DEFAULT 0,
|
|
28619
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
28620
|
-
last_error_code TEXT,
|
|
28621
|
-
checked_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
28622
|
-
synced_at TEXT,
|
|
28623
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
28624
|
-
)`);
|
|
28625
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_charges (
|
|
28626
|
-
user_id INTEGER NOT NULL,
|
|
28627
|
-
connection_id TEXT NOT NULL,
|
|
28628
|
-
period_start TEXT NOT NULL,
|
|
28629
|
-
amount_mc INTEGER NOT NULL,
|
|
28630
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
28631
|
-
PRIMARY KEY (user_id, connection_id, period_start)
|
|
28632
|
-
)`);
|
|
28633
|
-
}
|
|
28634
28663
|
function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
|
|
28635
28664
|
return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
28636
28665
|
}
|
|
@@ -28638,7 +28667,6 @@ function isBillableConnection(connection) {
|
|
|
28638
28667
|
return BILLABLE_LIFECYCLE.has(String(connection.lifecycleStatus ?? ""));
|
|
28639
28668
|
}
|
|
28640
28669
|
async function chargedConnectionIds(userId, periodStart) {
|
|
28641
|
-
await ensureConnectedAccountBillingSchema();
|
|
28642
28670
|
const result = await getDb().execute({
|
|
28643
28671
|
sql: "SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",
|
|
28644
28672
|
args: [Number(userId), periodStart]
|
|
@@ -28646,7 +28674,6 @@ async function chargedConnectionIds(userId, periodStart) {
|
|
|
28646
28674
|
return new Set(result.rows.map((row) => String(row.connection_id)));
|
|
28647
28675
|
}
|
|
28648
28676
|
async function getConnectedAccountBillingState2(userId) {
|
|
28649
|
-
await ensureConnectedAccountBillingSchema();
|
|
28650
28677
|
const result = await getDb().execute({
|
|
28651
28678
|
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
28652
28679
|
args: [Number(userId)]
|
|
@@ -28654,7 +28681,6 @@ async function getConnectedAccountBillingState2(userId) {
|
|
|
28654
28681
|
return result.rows[0] ? result.rows[0] : null;
|
|
28655
28682
|
}
|
|
28656
28683
|
async function setConnectedAccountBillingState2(input) {
|
|
28657
|
-
await ensureConnectedAccountBillingSchema();
|
|
28658
28684
|
await getDb().execute({
|
|
28659
28685
|
sql: `INSERT INTO connected_account_billing
|
|
28660
28686
|
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, checked_at, synced_at, updated_at)
|
|
@@ -37670,6 +37696,89 @@ var init_paa_harvest_reconciliation = __esm({
|
|
|
37670
37696
|
}
|
|
37671
37697
|
});
|
|
37672
37698
|
|
|
37699
|
+
// src/api/retention-sweeps.ts
|
|
37700
|
+
function cutoffFor(rule) {
|
|
37701
|
+
const cutoff = new Date(Date.now() - rule.retentionDays * 24 * 60 * 60 * 1e3);
|
|
37702
|
+
if (rule.timestampFormat === "iso8601") return cutoff.toISOString();
|
|
37703
|
+
return cutoff.toISOString().replace("T", " ").slice(0, 19);
|
|
37704
|
+
}
|
|
37705
|
+
async function tableExists(table) {
|
|
37706
|
+
const res = await getDb().execute({
|
|
37707
|
+
sql: "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
|
|
37708
|
+
args: [table]
|
|
37709
|
+
});
|
|
37710
|
+
return res.rows.length > 0;
|
|
37711
|
+
}
|
|
37712
|
+
async function sweepRule(rule, maxDeletes) {
|
|
37713
|
+
if (!await tableExists(rule.table)) return 0;
|
|
37714
|
+
const filter = rule.where ? ` AND ${rule.where}` : "";
|
|
37715
|
+
const cutoff = cutoffFor(rule);
|
|
37716
|
+
let deleted = 0;
|
|
37717
|
+
while (deleted < maxDeletes) {
|
|
37718
|
+
const res = await getDb().execute({
|
|
37719
|
+
sql: `DELETE FROM ${rule.table} WHERE rowid IN (
|
|
37720
|
+
SELECT rowid FROM ${rule.table}
|
|
37721
|
+
WHERE ${rule.timestampColumn} < ?${filter}
|
|
37722
|
+
LIMIT ?)`,
|
|
37723
|
+
args: [cutoff, Math.min(DEFAULT_BATCH, maxDeletes - deleted)]
|
|
37724
|
+
});
|
|
37725
|
+
const n = Number(res.rowsAffected ?? 0);
|
|
37726
|
+
deleted += n;
|
|
37727
|
+
if (n === 0) break;
|
|
37728
|
+
}
|
|
37729
|
+
return deleted;
|
|
37730
|
+
}
|
|
37731
|
+
async function sweepRetention(options = {}) {
|
|
37732
|
+
const maxDeletes = options.maxDeletesPerRule ?? DEFAULT_MAX_DELETES_PER_RULE;
|
|
37733
|
+
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_SWEEP_DEADLINE_MS);
|
|
37734
|
+
const perTable = {};
|
|
37735
|
+
let deleted = 0;
|
|
37736
|
+
let truncated = false;
|
|
37737
|
+
for (const rule of RETENTION_RULES) {
|
|
37738
|
+
const key = rule.where ? `${rule.table}:${rule.where}` : rule.table;
|
|
37739
|
+
if (Date.now() >= deadline) {
|
|
37740
|
+
truncated = true;
|
|
37741
|
+
break;
|
|
37742
|
+
}
|
|
37743
|
+
try {
|
|
37744
|
+
const n = await sweepRule(rule, maxDeletes);
|
|
37745
|
+
perTable[key] = n;
|
|
37746
|
+
deleted += n;
|
|
37747
|
+
if (n >= maxDeletes) truncated = true;
|
|
37748
|
+
} catch (err) {
|
|
37749
|
+
perTable[key] = -1;
|
|
37750
|
+
console.error(JSON.stringify({
|
|
37751
|
+
event: "retention_sweep_failed",
|
|
37752
|
+
table: rule.table,
|
|
37753
|
+
message: err instanceof Error ? err.message : String(err)
|
|
37754
|
+
}));
|
|
37755
|
+
}
|
|
37756
|
+
}
|
|
37757
|
+
return { deleted, perTable, truncated };
|
|
37758
|
+
}
|
|
37759
|
+
var REPORTING_WINDOW_DAYS, TELEMETRY_RETENTION_DAYS, FAILURE_RETENTION_DAYS, RETENTION_RULES, DEFAULT_BATCH, DEFAULT_MAX_DELETES_PER_RULE, DEFAULT_SWEEP_DEADLINE_MS;
|
|
37760
|
+
var init_retention_sweeps = __esm({
|
|
37761
|
+
"src/api/retention-sweeps.ts"() {
|
|
37762
|
+
"use strict";
|
|
37763
|
+
init_db();
|
|
37764
|
+
REPORTING_WINDOW_DAYS = 30;
|
|
37765
|
+
TELEMETRY_RETENTION_DAYS = 45;
|
|
37766
|
+
FAILURE_RETENTION_DAYS = 90;
|
|
37767
|
+
RETENTION_RULES = [
|
|
37768
|
+
{ table: "request_events", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
37769
|
+
{ table: "kernel_session_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
37770
|
+
{ table: "vendor_usage_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
37771
|
+
{ table: "harvest_attempts", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, timestampFormat: "iso8601" },
|
|
37772
|
+
{ table: "browser_agent_actions", timestampColumn: "created_at", retentionDays: REPORTING_WINDOW_DAYS },
|
|
37773
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, where: "status = 'done'" },
|
|
37774
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: FAILURE_RETENTION_DAYS, where: "status = 'failed'" }
|
|
37775
|
+
];
|
|
37776
|
+
DEFAULT_BATCH = 500;
|
|
37777
|
+
DEFAULT_MAX_DELETES_PER_RULE = 1e3;
|
|
37778
|
+
DEFAULT_SWEEP_DEADLINE_MS = 15e3;
|
|
37779
|
+
}
|
|
37780
|
+
});
|
|
37781
|
+
|
|
37673
37782
|
// src/api/directory-workflow-reconciliation.ts
|
|
37674
37783
|
async function reconcileDirectoryWorkflowSettlements(limit = 25) {
|
|
37675
37784
|
const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
|
|
@@ -41371,7 +41480,7 @@ var PACKAGE_VERSION;
|
|
|
41371
41480
|
var init_version = __esm({
|
|
41372
41481
|
"src/version.ts"() {
|
|
41373
41482
|
"use strict";
|
|
41374
|
-
PACKAGE_VERSION = "0.
|
|
41483
|
+
PACKAGE_VERSION = "0.50.0";
|
|
41375
41484
|
}
|
|
41376
41485
|
});
|
|
41377
41486
|
|
|
@@ -41429,6 +41538,8 @@ seam is noted so you can chain them.
|
|
|
41429
41538
|
JSON-LD, media, citations, source metadata, related entities, trails, and saved account filters.
|
|
41430
41539
|
- Read one published entity -> **commons_get_entity**. It returns the Wikidata-style backend fields,
|
|
41431
41540
|
JSON-LD, source provenance, media, ledger, and the Wikipedia-style page projection.
|
|
41541
|
+
- Read its machine-verifiable governed relationships -> **commons_get_entity_linkset**. It returns the
|
|
41542
|
+
RFC 9264 linkset projection; do not infer claim status from loose See Also labels.
|
|
41432
41543
|
- List unresolved See Also concepts -> **commons_list_needs_links**. Use it to find concepts that appear
|
|
41433
41544
|
in public See Also sections but do not yet have a Commons entity ID or /wiki/ slug.
|
|
41434
41545
|
- Plan a new or edited entity -> **commons_prepare_entity**. This is the Commons equivalent of
|
|
@@ -42565,7 +42676,7 @@ var init_contracts = __esm({
|
|
|
42565
42676
|
});
|
|
42566
42677
|
|
|
42567
42678
|
// src/mcp/mcp-tool-schemas.ts
|
|
42568
|
-
var import_zod42, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
42679
|
+
var import_zod42, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
42569
42680
|
var init_mcp_tool_schemas = __esm({
|
|
42570
42681
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
42571
42682
|
"use strict";
|
|
@@ -42870,6 +42981,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
42870
42981
|
idOrSlug: import_zod42.z.string().trim().min(1).max(180).describe("Transparent Public Wiki entity id such as TPW-Q... or a public /wiki/ slug."),
|
|
42871
42982
|
includeWikiPage: import_zod42.z.boolean().default(true).describe("Include the Wikipedia-style page projection used by transparent-commons.cc/wiki/.")
|
|
42872
42983
|
};
|
|
42984
|
+
CommonsGetEntityLinksetInputSchema = {
|
|
42985
|
+
idOrSlug: import_zod42.z.string().trim().min(1).max(180).describe("Transparent Commons TPW-Q entity id or /wiki/ slug whose governed RFC 9264 linkset should be read.")
|
|
42986
|
+
};
|
|
42873
42987
|
CommonsFeaturedImageInputSchema = import_zod42.z.object({
|
|
42874
42988
|
url: import_zod42.z.string().url().describe("Required public image URL for a publishable entity. Use extract_url includeFeaturedImage or preserved media when available."),
|
|
42875
42989
|
alt: import_zod42.z.string().trim().max(500).optional(),
|
|
@@ -42922,6 +43036,27 @@ var init_mcp_tool_schemas = __esm({
|
|
|
42922
43036
|
imageUrl: import_zod42.z.string().trim().max(2e3).optional().describe("Optional preview image for hover cards when available."),
|
|
42923
43037
|
needsLink: import_zod42.z.boolean().default(false).describe("Set true when the concept belongs in See Also but no existing Commons entity/page exists yet. The backend exposes these records through commons_list_needs_links.")
|
|
42924
43038
|
}).strict();
|
|
43039
|
+
CommonsClaimInputSchema = import_zod42.z.object({
|
|
43040
|
+
predicate: import_zod42.z.string().trim().min(1).max(500).describe("Registered link relation such as related, describedby, license, or type; alternatively an absolute HTTPS relation URI from a published vocabulary."),
|
|
43041
|
+
predicateLabel: import_zod42.z.string().trim().max(160).optional().describe("Human-readable label for display and review. It does not replace the machine predicate."),
|
|
43042
|
+
objectEntityId: import_zod42.z.string().trim().regex(/^TPW-Q[A-Za-z0-9._-]+$/).max(80).optional().describe("Existing Transparent Commons entity target. Use exactly one of objectEntityId or objectUrl."),
|
|
43043
|
+
objectUrl: import_zod42.z.string().url().refine((value) => value.startsWith("https://"), "objectUrl must use HTTPS.").optional().describe("External or third-party target. Use exactly one of objectUrl or objectEntityId."),
|
|
43044
|
+
title: import_zod42.z.string().trim().max(500).optional(),
|
|
43045
|
+
mediaType: import_zod42.z.string().trim().max(160).optional().describe("Optional RFC link target media type."),
|
|
43046
|
+
hreflang: import_zod42.z.array(import_zod42.z.string().trim().min(1).max(80)).max(20).optional().describe("Optional repeatable RFC link target language hints."),
|
|
43047
|
+
qualifiers: import_zod42.z.record(import_zod42.z.string().max(120), import_zod42.z.union([import_zod42.z.string(), import_zod42.z.number(), import_zod42.z.boolean(), import_zod42.z.null()])).optional(),
|
|
43048
|
+
references: import_zod42.z.array(CommonsCitationInputSchema).max(50).optional().describe("Evidence supporting the relationship assertion."),
|
|
43049
|
+
rank: import_zod42.z.enum(["preferred", "normal", "deprecated"]).default("normal"),
|
|
43050
|
+
confidence: import_zod42.z.number().min(0).max(1).default(1),
|
|
43051
|
+
confidenceReason: import_zod42.z.string().trim().max(1e3).optional(),
|
|
43052
|
+
disputeState: import_zod42.z.enum(["undisputed", "attributed", "disputed", "contested", "needs_review", "deprecated"]).default("undisputed"),
|
|
43053
|
+
validFrom: import_zod42.z.string().trim().max(80).optional(),
|
|
43054
|
+
validTo: import_zod42.z.string().trim().max(80).optional(),
|
|
43055
|
+
asOf: import_zod42.z.string().trim().max(80).optional(),
|
|
43056
|
+
sourceFamily: import_zod42.z.enum(["first_party", "partner", "independent", "community", "unknown"]).default("unknown")
|
|
43057
|
+
}).strict().refine((claim) => Boolean(claim.objectEntityId) !== Boolean(claim.objectUrl), {
|
|
43058
|
+
message: "Provide exactly one of objectEntityId or objectUrl."
|
|
43059
|
+
});
|
|
42925
43060
|
CommonsPrepareEntityInputSchema = {
|
|
42926
43061
|
title: import_zod42.z.string().trim().min(1).max(180).describe("Candidate public page title. Used to derive slug, choose a profile, and search duplicate entities."),
|
|
42927
43062
|
description: import_zod42.z.string().trim().max(600).optional().describe("Optional candidate summary. Include when known so the prepare pass can route and validate the article shape."),
|
|
@@ -42961,6 +43096,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
42961
43096
|
url: import_zod42.z.string().url().optional(),
|
|
42962
43097
|
description: import_zod42.z.string().trim().max(300).optional()
|
|
42963
43098
|
}).strict()).max(200).optional().describe("Related concepts, including trails. Existing entities should use entityId."),
|
|
43099
|
+
claims: import_zod42.z.array(CommonsClaimInputSchema).max(300).optional().describe("Governed relationship assertions. Supplying claims replaces the accepted claim set for this entity; omitting claims preserves it. Only approved, active, non-disputed claims project into the RFC 9264 linkset."),
|
|
42964
43100
|
bodyMarkdown: import_zod42.z.string().max(5e5).optional().describe("Structured encyclopedia body in neutral Markdown, not a raw scrape dump or blog essay. Use H2 for entity-profile sections and H3/H4/H5 for subtopics; every heading appears in the public page menu. Call commons_prepare_entity first for the live entity profile and commons_validate_entity before submit. Profile sections are adaptive: include History, Pricing, Reception, Timeline, Classification, or similar sections only when source evidence supports them; omit unsupported sections instead of adding empty/filler headings. Apply NPOV, verifiability, and no-original-research rules; self-published sources can support only uncontroversial source-owned facts."),
|
|
42965
43101
|
contentSections: import_zod42.z.array(import_zod42.z.object({
|
|
42966
43102
|
id: import_zod42.z.string().trim().max(180).optional(),
|
|
@@ -46230,6 +46366,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
46230
46366
|
openWorldHint: false
|
|
46231
46367
|
}
|
|
46232
46368
|
}, async (input) => executor.commonsGetEntity(input));
|
|
46369
|
+
server.registerTool("commons_get_entity_linkset", {
|
|
46370
|
+
title: "Transparent Commons RFC 9264 Linkset",
|
|
46371
|
+
description: "Read the governed RFC 9264 application/linkset+json projection for one published Transparent Commons entity. The anchor is the canonical /wiki/ page; only approved, active, non-disputed claims are included, while third-party targets remain explicit HTTPS links.",
|
|
46372
|
+
inputSchema: CommonsGetEntityLinksetInputSchema,
|
|
46373
|
+
outputSchema: recordOutputSchema("commons_get_entity_linkset", CommonsGenericOutputSchema),
|
|
46374
|
+
annotations: {
|
|
46375
|
+
title: "Transparent Commons RFC 9264 Linkset",
|
|
46376
|
+
readOnlyHint: true,
|
|
46377
|
+
destructiveHint: false,
|
|
46378
|
+
idempotentHint: true,
|
|
46379
|
+
openWorldHint: false
|
|
46380
|
+
}
|
|
46381
|
+
}, async (input) => executor.commonsGetEntityLinkset(input));
|
|
46233
46382
|
server.registerTool("commons_list_needs_links", {
|
|
46234
46383
|
title: "Transparent Commons Needs-Link Backlog",
|
|
46235
46384
|
description: "List unresolved See Also concepts from published Transparent Commons pages. Use this after search/lookup when an agent needs to grow the graph: each result names a concept that appears in See Also but does not yet have a resolved Commons entityId or /wiki/ slug. This is platform Commons graph data, not personal Memory storage.",
|
|
@@ -46722,6 +46871,29 @@ var init_paa_mcp_server = __esm({
|
|
|
46722
46871
|
});
|
|
46723
46872
|
|
|
46724
46873
|
// src/mcp/http-mcp-tool-executor.ts
|
|
46874
|
+
function unclassifiedTransportFailure(path6, err) {
|
|
46875
|
+
const name = err instanceof Error ? err.name : typeof err;
|
|
46876
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
46877
|
+
console.error(JSON.stringify({
|
|
46878
|
+
event: "mcp_executor_transport_failed",
|
|
46879
|
+
path: path6,
|
|
46880
|
+
error_name: name,
|
|
46881
|
+
error_message: sanitizeVendorName(detail).slice(0, 400)
|
|
46882
|
+
}));
|
|
46883
|
+
return {
|
|
46884
|
+
content: [{
|
|
46885
|
+
type: "text",
|
|
46886
|
+
text: JSON.stringify({
|
|
46887
|
+
error: "service_unavailable",
|
|
46888
|
+
error_type: name,
|
|
46889
|
+
retryable: true,
|
|
46890
|
+
path: path6,
|
|
46891
|
+
message: publicErrorMessage("service_unavailable")
|
|
46892
|
+
})
|
|
46893
|
+
}],
|
|
46894
|
+
isError: true
|
|
46895
|
+
};
|
|
46896
|
+
}
|
|
46725
46897
|
function youtubeVideoIdFromUrl(url) {
|
|
46726
46898
|
if (!url) return null;
|
|
46727
46899
|
try {
|
|
@@ -46831,7 +47003,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
46831
47003
|
const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
|
|
46832
47004
|
const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
|
|
46833
47005
|
this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
|
|
46834
|
-
this.timeoutMs = this.httpTimeoutOverrideMs ??
|
|
47006
|
+
this.timeoutMs = this.httpTimeoutOverrideMs ?? DEFAULT_TOOL_CLIENT_TIMEOUT_MS;
|
|
46835
47007
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
46836
47008
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
46837
47009
|
}
|
|
@@ -46870,7 +47042,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
46870
47042
|
isError: true
|
|
46871
47043
|
};
|
|
46872
47044
|
}
|
|
46873
|
-
return
|
|
47045
|
+
return unclassifiedTransportFailure(path6, err);
|
|
46874
47046
|
}
|
|
46875
47047
|
}
|
|
46876
47048
|
async callConnectedMutation(path6, body, idempotencyKey3, timeoutMs = this.timeoutMs) {
|
|
@@ -46904,8 +47076,8 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
46904
47076
|
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path6, res, data)) }], isError: true };
|
|
46905
47077
|
}
|
|
46906
47078
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
46907
|
-
} catch {
|
|
46908
|
-
return
|
|
47079
|
+
} catch (err) {
|
|
47080
|
+
return unclassifiedTransportFailure(path6, err);
|
|
46909
47081
|
}
|
|
46910
47082
|
}
|
|
46911
47083
|
async getTextArtifact(path6, maxBytes, timeoutMs = this.timeoutMs) {
|
|
@@ -46942,7 +47114,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
46942
47114
|
}]
|
|
46943
47115
|
};
|
|
46944
47116
|
} catch (err) {
|
|
46945
|
-
return
|
|
47117
|
+
return unclassifiedTransportFailure(path6, err);
|
|
46946
47118
|
}
|
|
46947
47119
|
}
|
|
46948
47120
|
harvestPaa(input) {
|
|
@@ -47299,6 +47471,9 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
47299
47471
|
return result;
|
|
47300
47472
|
}
|
|
47301
47473
|
}
|
|
47474
|
+
commonsGetEntityLinkset(input) {
|
|
47475
|
+
return this.getJson(`/commons/entities/${encodeURIComponent(input.idOrSlug)}/linkset?envelope=true`);
|
|
47476
|
+
}
|
|
47302
47477
|
commonsPrepareEntity(input) {
|
|
47303
47478
|
return this.call("/commons/entities/prepare", input);
|
|
47304
47479
|
}
|
|
@@ -53262,13 +53437,21 @@ async function setBilledMc(id, billedMc) {
|
|
|
53262
53437
|
args: [Math.round(billedMc), id]
|
|
53263
53438
|
});
|
|
53264
53439
|
}
|
|
53265
|
-
async function markSessionClosed(id) {
|
|
53440
|
+
async function markSessionClosed(id, runtimeClosed = true) {
|
|
53266
53441
|
const db = getDb();
|
|
53267
53442
|
await db.execute({
|
|
53268
|
-
sql: `UPDATE browser_agent_sessions SET status =
|
|
53269
|
-
args: [id]
|
|
53443
|
+
sql: `UPDATE browser_agent_sessions SET status = ?, closed_at = datetime('now') WHERE id = ?`,
|
|
53444
|
+
args: [runtimeClosed ? "closed" : ORPHANED_SESSION_STATUS, id]
|
|
53270
53445
|
});
|
|
53271
53446
|
}
|
|
53447
|
+
async function listOrphanedSessionRows(limit = 50) {
|
|
53448
|
+
const db = getDb();
|
|
53449
|
+
const res = await db.execute({
|
|
53450
|
+
sql: `SELECT * FROM browser_agent_sessions WHERE status = ? ORDER BY closed_at LIMIT ?`,
|
|
53451
|
+
args: [ORPHANED_SESSION_STATUS, Math.max(1, Math.min(200, Math.round(limit)))]
|
|
53452
|
+
});
|
|
53453
|
+
return res.rows;
|
|
53454
|
+
}
|
|
53272
53455
|
async function recordAction(input) {
|
|
53273
53456
|
const db = getDb();
|
|
53274
53457
|
await db.execute({
|
|
@@ -53315,7 +53498,7 @@ async function listReplayRows(sessionId) {
|
|
|
53315
53498
|
});
|
|
53316
53499
|
return res.rows;
|
|
53317
53500
|
}
|
|
53318
|
-
var import_node_crypto28, _ready2, _migrationPromise2;
|
|
53501
|
+
var import_node_crypto28, _ready2, _migrationPromise2, ORPHANED_SESSION_STATUS;
|
|
53319
53502
|
var init_browser_agent_db = __esm({
|
|
53320
53503
|
"src/api/browser-agent-db.ts"() {
|
|
53321
53504
|
"use strict";
|
|
@@ -53323,6 +53506,7 @@ var init_browser_agent_db = __esm({
|
|
|
53323
53506
|
init_db();
|
|
53324
53507
|
_ready2 = false;
|
|
53325
53508
|
_migrationPromise2 = null;
|
|
53509
|
+
ORPHANED_SESSION_STATUS = "runtime_close_pending";
|
|
53326
53510
|
}
|
|
53327
53511
|
});
|
|
53328
53512
|
|
|
@@ -54479,6 +54663,21 @@ async function charge(sessionId, userId, _startedAtMs) {
|
|
|
54479
54663
|
if (res.ok) await setBilledMc(sessionId, owed);
|
|
54480
54664
|
}
|
|
54481
54665
|
}
|
|
54666
|
+
async function closeRuntimeSession(runtimeSessionId, sessionId) {
|
|
54667
|
+
try {
|
|
54668
|
+
await closeSession(runtimeSessionId);
|
|
54669
|
+
return true;
|
|
54670
|
+
} catch (err) {
|
|
54671
|
+
console.error(JSON.stringify({
|
|
54672
|
+
event: "browser_runtime_close_failed",
|
|
54673
|
+
session_id: sessionId,
|
|
54674
|
+
runtime_session_id: runtimeSessionId,
|
|
54675
|
+
error_name: err instanceof Error ? err.name : typeof err,
|
|
54676
|
+
message: sanitizeVendorName(err instanceof Error ? err.message : String(err)).slice(0, 300)
|
|
54677
|
+
}));
|
|
54678
|
+
return false;
|
|
54679
|
+
}
|
|
54680
|
+
}
|
|
54482
54681
|
async function closeAndSettleSession(row) {
|
|
54483
54682
|
if (row.user_id != null) {
|
|
54484
54683
|
try {
|
|
@@ -54486,15 +54685,24 @@ async function closeAndSettleSession(row) {
|
|
|
54486
54685
|
} catch {
|
|
54487
54686
|
}
|
|
54488
54687
|
}
|
|
54489
|
-
|
|
54490
|
-
|
|
54491
|
-
} catch {
|
|
54492
|
-
}
|
|
54493
|
-
await markSessionClosed(row.id).catch(() => {
|
|
54688
|
+
const runtimeClosed = await closeRuntimeSession(row.runtime_session_id, row.id);
|
|
54689
|
+
await markSessionClosed(row.id, runtimeClosed).catch(() => {
|
|
54494
54690
|
});
|
|
54495
54691
|
await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
|
|
54496
54692
|
});
|
|
54497
54693
|
}
|
|
54694
|
+
async function retryOrphanedBrowserSessions(limit = 50) {
|
|
54695
|
+
const rows = await listOrphanedSessionRows(limit);
|
|
54696
|
+
let recovered = 0;
|
|
54697
|
+
for (const row of rows) {
|
|
54698
|
+
if (await closeRuntimeSession(row.runtime_session_id, row.id)) {
|
|
54699
|
+
await markSessionClosed(row.id, true).catch(() => {
|
|
54700
|
+
});
|
|
54701
|
+
recovered++;
|
|
54702
|
+
}
|
|
54703
|
+
}
|
|
54704
|
+
return { recovered, stillOpen: rows.length - recovered };
|
|
54705
|
+
}
|
|
54498
54706
|
async function completeAuthConnection(connectionId) {
|
|
54499
54707
|
const connection = await getAuthConnectionRow(connectionId);
|
|
54500
54708
|
if (!connection) return { ok: false, error: "not found" };
|
|
@@ -54519,7 +54727,7 @@ async function reapIdleBrowserSessions(idleSeconds = 120) {
|
|
|
54519
54727
|
function publicSession(row) {
|
|
54520
54728
|
return {
|
|
54521
54729
|
session_id: row.id,
|
|
54522
|
-
status: row.status,
|
|
54730
|
+
status: row.status === ORPHANED_SESSION_STATUS ? "closed" : row.status,
|
|
54523
54731
|
label: row.label,
|
|
54524
54732
|
watch_url: `/console/${row.id}`,
|
|
54525
54733
|
created_at: row.created_at,
|
|
@@ -59404,9 +59612,228 @@ var init_editorial_reading_room_routes = __esm({
|
|
|
59404
59612
|
}
|
|
59405
59613
|
});
|
|
59406
59614
|
|
|
59615
|
+
// src/api/schema-presence.ts
|
|
59616
|
+
async function loadSchemaObjects() {
|
|
59617
|
+
const res = await getDb().execute(
|
|
59618
|
+
"SELECT name FROM sqlite_master WHERE type IN ('table', 'index')"
|
|
59619
|
+
);
|
|
59620
|
+
return new Set(res.rows.map((row) => String(row.name)));
|
|
59621
|
+
}
|
|
59622
|
+
function invalidateSchemaPresence() {
|
|
59623
|
+
cache = null;
|
|
59624
|
+
}
|
|
59625
|
+
async function schemaObjectsPresent(names) {
|
|
59626
|
+
if (names.length === 0) return true;
|
|
59627
|
+
try {
|
|
59628
|
+
if (!cache) cache = loadSchemaObjects();
|
|
59629
|
+
const present = await cache;
|
|
59630
|
+
if (names.every((name) => present.has(name))) return true;
|
|
59631
|
+
invalidateSchemaPresence();
|
|
59632
|
+
return false;
|
|
59633
|
+
} catch {
|
|
59634
|
+
invalidateSchemaPresence();
|
|
59635
|
+
return false;
|
|
59636
|
+
}
|
|
59637
|
+
}
|
|
59638
|
+
var cache;
|
|
59639
|
+
var init_schema_presence = __esm({
|
|
59640
|
+
"src/api/schema-presence.ts"() {
|
|
59641
|
+
"use strict";
|
|
59642
|
+
init_db();
|
|
59643
|
+
cache = null;
|
|
59644
|
+
}
|
|
59645
|
+
});
|
|
59646
|
+
|
|
59647
|
+
// src/api/commons-linksets.ts
|
|
59648
|
+
function normalizeCommonsClaims(input) {
|
|
59649
|
+
if (input === void 0) return void 0;
|
|
59650
|
+
if (!Array.isArray(input)) throw new Error("claims must be an array.");
|
|
59651
|
+
if (input.length > 300) throw new Error("claims must contain no more than 300 assertions.");
|
|
59652
|
+
const seen = /* @__PURE__ */ new Set();
|
|
59653
|
+
return input.map((claim, index) => {
|
|
59654
|
+
const predicate = String(claim.predicate ?? "").trim();
|
|
59655
|
+
if (!isSupportedPredicate(predicate)) {
|
|
59656
|
+
throw new Error(`claims[${index}].predicate must be a supported registered relation or an absolute HTTPS relation URI.`);
|
|
59657
|
+
}
|
|
59658
|
+
const objectEntityId = optionalText(claim.objectEntityId, 80);
|
|
59659
|
+
const objectUrl = optionalHttpsUrl(claim.objectUrl, `claims[${index}].objectUrl`);
|
|
59660
|
+
if ((objectEntityId ? 1 : 0) + (objectUrl ? 1 : 0) !== 1) {
|
|
59661
|
+
throw new Error(`claims[${index}] must provide exactly one of objectEntityId or objectUrl.`);
|
|
59662
|
+
}
|
|
59663
|
+
if (objectEntityId && !/^TPW-Q[A-Za-z0-9._-]+$/.test(objectEntityId)) {
|
|
59664
|
+
throw new Error(`claims[${index}].objectEntityId must be a Transparent Commons TPW-Q identifier.`);
|
|
59665
|
+
}
|
|
59666
|
+
const confidence = claim.confidence ?? 1;
|
|
59667
|
+
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
|
|
59668
|
+
throw new Error(`claims[${index}].confidence must be between 0 and 1.`);
|
|
59669
|
+
}
|
|
59670
|
+
const key = `${predicate}\0${objectEntityId ?? objectUrl}`;
|
|
59671
|
+
if (seen.has(key)) throw new Error(`claims[${index}] duplicates another predicate and object pair.`);
|
|
59672
|
+
seen.add(key);
|
|
59673
|
+
return {
|
|
59674
|
+
predicate,
|
|
59675
|
+
predicateLabel: optionalText(claim.predicateLabel, 160),
|
|
59676
|
+
objectEntityId,
|
|
59677
|
+
objectUrl,
|
|
59678
|
+
title: optionalText(claim.title, 500),
|
|
59679
|
+
mediaType: optionalText(claim.mediaType, 160),
|
|
59680
|
+
hreflang: normalizeHreflang(claim.hreflang),
|
|
59681
|
+
qualifiers: normalizeScalarRecord(claim.qualifiers),
|
|
59682
|
+
references: normalizeReferences(claim.references),
|
|
59683
|
+
rank: claim.rank ?? "normal",
|
|
59684
|
+
confidence,
|
|
59685
|
+
confidenceReason: optionalText(claim.confidenceReason, 1e3),
|
|
59686
|
+
disputeState: claim.disputeState ?? "undisputed",
|
|
59687
|
+
validFrom: optionalDate(claim.validFrom, `claims[${index}].validFrom`),
|
|
59688
|
+
validTo: optionalDate(claim.validTo, `claims[${index}].validTo`),
|
|
59689
|
+
asOf: optionalDate(claim.asOf, `claims[${index}].asOf`),
|
|
59690
|
+
sourceFamily: claim.sourceFamily ?? "unknown"
|
|
59691
|
+
};
|
|
59692
|
+
});
|
|
59693
|
+
}
|
|
59694
|
+
function claimIsPublic(claim, now = /* @__PURE__ */ new Date()) {
|
|
59695
|
+
if (claim.reviewState !== "approved" || claim.rank === "deprecated") return false;
|
|
59696
|
+
if (!["undisputed", "attributed"].includes(claim.disputeState ?? "undisputed")) return false;
|
|
59697
|
+
const timestamp2 = now.getTime();
|
|
59698
|
+
if (claim.validFrom && Date.parse(claim.validFrom) > timestamp2) return false;
|
|
59699
|
+
if (claim.validTo && Date.parse(claim.validTo) < timestamp2) return false;
|
|
59700
|
+
return true;
|
|
59701
|
+
}
|
|
59702
|
+
function buildCommonsLinkset(entity, claims) {
|
|
59703
|
+
const publicUrl = `${commonsPublicWikiBaseUrl()}/wiki/${encodeURIComponent(entity.slug)}`;
|
|
59704
|
+
const linksetUrl = commonsLinksetUrl(entity.id);
|
|
59705
|
+
const context = { anchor: publicUrl };
|
|
59706
|
+
const publicClaims = claims.filter((claim) => claimIsPublic(claim)).sort((a, b) => a.predicate.localeCompare(b.predicate) || a.href.localeCompare(b.href) || a.id.localeCompare(b.id));
|
|
59707
|
+
for (const claim of publicClaims) {
|
|
59708
|
+
const targets = context[claim.predicate] ?? [];
|
|
59709
|
+
const target = { href: claim.href };
|
|
59710
|
+
if (claim.mediaType) target.type = claim.mediaType;
|
|
59711
|
+
if (claim.hreflang) target.hreflang = claim.hreflang;
|
|
59712
|
+
if (claim.title) target.title = claim.title;
|
|
59713
|
+
targets.push(target);
|
|
59714
|
+
context[claim.predicate] = targets;
|
|
59715
|
+
}
|
|
59716
|
+
const document2 = { linkset: [context] };
|
|
59717
|
+
const etag = `"${(0, import_node_crypto34.createHash)("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
59718
|
+
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
59719
|
+
}
|
|
59720
|
+
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
59721
|
+
const url = commonsLinksetUrl(idOrSlug);
|
|
59722
|
+
return `<${url}>; rel="linkset"; type="${COMMONS_LINKSET_MEDIA_TYPE}"; profile="${COMMONS_RELATIONSHIP_PROFILE}"`;
|
|
59723
|
+
}
|
|
59724
|
+
function commonsLinksetUrl(idOrSlug) {
|
|
59725
|
+
const base = (process.env.APP_URL?.trim() || "https://mcpscraper.dev").replace(/\/+$/, "");
|
|
59726
|
+
return `${base}/commons/entities/${encodeURIComponent(idOrSlug)}/linkset`;
|
|
59727
|
+
}
|
|
59728
|
+
function commonsClaimIdentityKey(claim) {
|
|
59729
|
+
return `${claim.predicate}\0${claim.objectEntityId ?? claim.objectUrl ?? ""}`;
|
|
59730
|
+
}
|
|
59731
|
+
function commonsPublicWikiBaseUrl() {
|
|
59732
|
+
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || "https://transparent-commons.cc").replace(/\/+$/, "");
|
|
59733
|
+
}
|
|
59734
|
+
function isSupportedPredicate(value) {
|
|
59735
|
+
if (REGISTERED_RELATIONS.has(value)) return true;
|
|
59736
|
+
try {
|
|
59737
|
+
const url = new URL(value);
|
|
59738
|
+
return url.protocol === "https:" && Boolean(url.hostname);
|
|
59739
|
+
} catch {
|
|
59740
|
+
return false;
|
|
59741
|
+
}
|
|
59742
|
+
}
|
|
59743
|
+
function optionalText(value, max) {
|
|
59744
|
+
const text2 = String(value ?? "").trim();
|
|
59745
|
+
return text2 ? text2.slice(0, max) : void 0;
|
|
59746
|
+
}
|
|
59747
|
+
function optionalHttpsUrl(value, field) {
|
|
59748
|
+
const text2 = optionalText(value, 2e3);
|
|
59749
|
+
if (!text2) return void 0;
|
|
59750
|
+
try {
|
|
59751
|
+
const url = new URL(text2);
|
|
59752
|
+
if (url.protocol !== "https:") throw new Error();
|
|
59753
|
+
url.hash = "";
|
|
59754
|
+
return url.href;
|
|
59755
|
+
} catch {
|
|
59756
|
+
throw new Error(`${field} must be an absolute HTTPS URL.`);
|
|
59757
|
+
}
|
|
59758
|
+
}
|
|
59759
|
+
function optionalDate(value, field) {
|
|
59760
|
+
const text2 = optionalText(value, 80);
|
|
59761
|
+
if (!text2) return void 0;
|
|
59762
|
+
const timestamp2 = Date.parse(text2);
|
|
59763
|
+
if (!Number.isFinite(timestamp2)) throw new Error(`${field} must be an ISO-compatible date.`);
|
|
59764
|
+
return new Date(timestamp2).toISOString();
|
|
59765
|
+
}
|
|
59766
|
+
function normalizeScalarRecord(value) {
|
|
59767
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
59768
|
+
return Object.fromEntries(Object.entries(value).slice(0, 50).map(([key, item]) => [key.slice(0, 120), item]));
|
|
59769
|
+
}
|
|
59770
|
+
function normalizeReferences(value) {
|
|
59771
|
+
if (!Array.isArray(value)) return void 0;
|
|
59772
|
+
return value.slice(0, 50).map((reference) => ({
|
|
59773
|
+
title: String(reference.title ?? "").trim().slice(0, 240),
|
|
59774
|
+
url: optionalHttpsUrl(reference.url, "claim reference URL"),
|
|
59775
|
+
source: optionalText(reference.source, 240),
|
|
59776
|
+
note: optionalText(reference.note, 1e3),
|
|
59777
|
+
accessedAt: optionalDate(reference.accessedAt, "claim reference accessedAt")
|
|
59778
|
+
})).filter((reference) => reference.title);
|
|
59779
|
+
}
|
|
59780
|
+
function normalizeHreflang(value) {
|
|
59781
|
+
if (!Array.isArray(value)) return void 0;
|
|
59782
|
+
const languages = [...new Set(value.map((item) => optionalText(item, 80)).filter((item) => Boolean(item)))];
|
|
59783
|
+
return languages.length ? languages : void 0;
|
|
59784
|
+
}
|
|
59785
|
+
var import_node_crypto34, COMMONS_LINKSET_MEDIA_TYPE, COMMONS_RELATIONSHIP_PROFILE, REGISTERED_RELATIONS;
|
|
59786
|
+
var init_commons_linksets = __esm({
|
|
59787
|
+
"src/api/commons-linksets.ts"() {
|
|
59788
|
+
"use strict";
|
|
59789
|
+
import_node_crypto34 = require("crypto");
|
|
59790
|
+
COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
59791
|
+
COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
59792
|
+
REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
59793
|
+
"about",
|
|
59794
|
+
"alternate",
|
|
59795
|
+
"author",
|
|
59796
|
+
"canonical",
|
|
59797
|
+
"cite-as",
|
|
59798
|
+
"collection",
|
|
59799
|
+
"contents",
|
|
59800
|
+
"copyright",
|
|
59801
|
+
"describedby",
|
|
59802
|
+
"describes",
|
|
59803
|
+
"first",
|
|
59804
|
+
"glossary",
|
|
59805
|
+
"help",
|
|
59806
|
+
"icon",
|
|
59807
|
+
"index",
|
|
59808
|
+
"item",
|
|
59809
|
+
"last",
|
|
59810
|
+
"latest-version",
|
|
59811
|
+
"license",
|
|
59812
|
+
"memento",
|
|
59813
|
+
"next",
|
|
59814
|
+
"original",
|
|
59815
|
+
"predecessor-version",
|
|
59816
|
+
"prev",
|
|
59817
|
+
"related",
|
|
59818
|
+
"replies",
|
|
59819
|
+
"service-desc",
|
|
59820
|
+
"service-doc",
|
|
59821
|
+
"successor-version",
|
|
59822
|
+
"terms-of-service",
|
|
59823
|
+
"type",
|
|
59824
|
+
"via",
|
|
59825
|
+
"version-history"
|
|
59826
|
+
]);
|
|
59827
|
+
}
|
|
59828
|
+
});
|
|
59829
|
+
|
|
59407
59830
|
// src/api/commons-repository.ts
|
|
59408
59831
|
async function ensureCommonsSchema() {
|
|
59409
|
-
if (
|
|
59832
|
+
if (schemaReady) return;
|
|
59833
|
+
if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
|
|
59834
|
+
schemaReady = true;
|
|
59835
|
+
return;
|
|
59836
|
+
}
|
|
59410
59837
|
const db = getDb();
|
|
59411
59838
|
await db.batch([
|
|
59412
59839
|
{
|
|
@@ -59453,6 +59880,42 @@ async function ensureCommonsSchema() {
|
|
|
59453
59880
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_status_updated ON commons_entities(status, updated_at DESC)`, args: [] },
|
|
59454
59881
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_type_updated ON commons_entities(entity_type, updated_at DESC)`, args: [] },
|
|
59455
59882
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_source_domain ON commons_entities(source_json)`, args: [] },
|
|
59883
|
+
{
|
|
59884
|
+
sql: `
|
|
59885
|
+
CREATE TABLE IF NOT EXISTS commons_claims (
|
|
59886
|
+
id TEXT PRIMARY KEY,
|
|
59887
|
+
subject_entity_id TEXT NOT NULL REFERENCES commons_entities(id),
|
|
59888
|
+
predicate TEXT NOT NULL,
|
|
59889
|
+
predicate_label TEXT,
|
|
59890
|
+
object_entity_id TEXT,
|
|
59891
|
+
object_url TEXT,
|
|
59892
|
+
title TEXT,
|
|
59893
|
+
media_type TEXT,
|
|
59894
|
+
hreflang TEXT,
|
|
59895
|
+
qualifiers_json TEXT NOT NULL DEFAULT '{}',
|
|
59896
|
+
references_json TEXT NOT NULL DEFAULT '[]',
|
|
59897
|
+
rank TEXT NOT NULL DEFAULT 'normal',
|
|
59898
|
+
confidence REAL NOT NULL DEFAULT 1,
|
|
59899
|
+
confidence_reason TEXT,
|
|
59900
|
+
dispute_state TEXT NOT NULL DEFAULT 'undisputed',
|
|
59901
|
+
valid_from TEXT,
|
|
59902
|
+
valid_to TEXT,
|
|
59903
|
+
as_of TEXT,
|
|
59904
|
+
source_family TEXT NOT NULL DEFAULT 'unknown',
|
|
59905
|
+
review_state TEXT NOT NULL DEFAULT 'approved',
|
|
59906
|
+
proposal_id TEXT NOT NULL REFERENCES commons_proposals(id),
|
|
59907
|
+
created_by_user_id INTEGER,
|
|
59908
|
+
updated_by_user_id INTEGER,
|
|
59909
|
+
created_at TEXT NOT NULL,
|
|
59910
|
+
updated_at TEXT NOT NULL,
|
|
59911
|
+
CHECK ((object_entity_id IS NOT NULL) != (object_url IS NOT NULL)),
|
|
59912
|
+
UNIQUE(subject_entity_id, predicate, object_entity_id, object_url)
|
|
59913
|
+
)
|
|
59914
|
+
`,
|
|
59915
|
+
args: []
|
|
59916
|
+
},
|
|
59917
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_claims_subject_predicate ON commons_claims(subject_entity_id, predicate)`, args: [] },
|
|
59918
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_claims_object_entity ON commons_claims(object_entity_id)`, args: [] },
|
|
59456
59919
|
{
|
|
59457
59920
|
sql: `
|
|
59458
59921
|
CREATE TABLE IF NOT EXISTS commons_proposals (
|
|
@@ -59591,7 +60054,7 @@ async function ensureCommonsSchema() {
|
|
|
59591
60054
|
args: [COMMONS_SCHEMA_VERSION]
|
|
59592
60055
|
}
|
|
59593
60056
|
], "write");
|
|
59594
|
-
|
|
60057
|
+
schemaReady = true;
|
|
59595
60058
|
}
|
|
59596
60059
|
async function searchCommonsEntities(filters = {}, userId) {
|
|
59597
60060
|
await ensureCommonsSchema();
|
|
@@ -59613,8 +60076,15 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
59613
60076
|
args.push(jsonLikeValue(keyword));
|
|
59614
60077
|
}
|
|
59615
60078
|
if (mergedFilters.relatedEntityId) {
|
|
59616
|
-
clauses.push(
|
|
59617
|
-
|
|
60079
|
+
clauses.push(`(
|
|
60080
|
+
related_entities_json LIKE ?
|
|
60081
|
+
OR EXISTS (
|
|
60082
|
+
SELECT 1 FROM commons_claims
|
|
60083
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
60084
|
+
AND commons_claims.object_entity_id = ?
|
|
60085
|
+
)
|
|
60086
|
+
)`);
|
|
60087
|
+
args.push(jsonLikeValue(mergedFilters.relatedEntityId), mergedFilters.relatedEntityId);
|
|
59618
60088
|
}
|
|
59619
60089
|
if (mergedFilters.trailId) {
|
|
59620
60090
|
clauses.push("related_entities_json LIKE ?");
|
|
@@ -59636,8 +60106,16 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
59636
60106
|
}
|
|
59637
60107
|
const terms = tokenize(mergedFilters.query);
|
|
59638
60108
|
for (const term of terms) {
|
|
59639
|
-
clauses.push(
|
|
59640
|
-
|
|
60109
|
+
clauses.push(`(
|
|
60110
|
+
search_text LIKE ?
|
|
60111
|
+
OR EXISTS (
|
|
60112
|
+
SELECT 1 FROM commons_claims
|
|
60113
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
60114
|
+
AND (commons_claims.predicate LIKE ? OR commons_claims.predicate_label LIKE ? OR commons_claims.title LIKE ? OR commons_claims.object_url LIKE ?)
|
|
60115
|
+
)
|
|
60116
|
+
)`);
|
|
60117
|
+
const pattern = `%${escapeLike(term)}%`;
|
|
60118
|
+
args.push(pattern, pattern, pattern, pattern, pattern);
|
|
59641
60119
|
}
|
|
59642
60120
|
args.push(limit, offset);
|
|
59643
60121
|
const result = await getDb().execute({
|
|
@@ -59659,9 +60137,9 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
59659
60137
|
}
|
|
59660
60138
|
async function prepareCommonsEntity(input, user) {
|
|
59661
60139
|
await ensureCommonsSchema();
|
|
59662
|
-
const title =
|
|
60140
|
+
const title = optionalText2(input.title, 180) ?? "";
|
|
59663
60141
|
const slug2 = normalizeSlug(input.slug || title);
|
|
59664
|
-
const schemaOrgType =
|
|
60142
|
+
const schemaOrgType = optionalText2(input.schemaOrgType, 80);
|
|
59665
60143
|
const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
|
|
59666
60144
|
const entityType = contract.entityType;
|
|
59667
60145
|
const bodyMarkdown = safeCandidateBody(input.bodyMarkdown, input.contentSections);
|
|
@@ -59719,8 +60197,8 @@ async function validateCommonsEntity(input, user) {
|
|
|
59719
60197
|
await ensureCommonsSchema();
|
|
59720
60198
|
const errors = [];
|
|
59721
60199
|
const warnings = [];
|
|
59722
|
-
const title =
|
|
59723
|
-
const description =
|
|
60200
|
+
const title = optionalText2(input.title, 180);
|
|
60201
|
+
const description = optionalText2(input.description, 600);
|
|
59724
60202
|
const slug2 = normalizeSlug(input.slug || title || "");
|
|
59725
60203
|
const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
|
|
59726
60204
|
const source = normalizeSource(input.source);
|
|
@@ -59735,6 +60213,11 @@ async function validateCommonsEntity(input, user) {
|
|
|
59735
60213
|
maxCandidates: 8
|
|
59736
60214
|
});
|
|
59737
60215
|
const existing = slug2 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug2, input.entityId) : null;
|
|
60216
|
+
try {
|
|
60217
|
+
normalizeCommonsClaims(input.claims);
|
|
60218
|
+
} catch (error) {
|
|
60219
|
+
errors.push(error instanceof Error ? error.message : "Claims are invalid.");
|
|
60220
|
+
}
|
|
59738
60221
|
if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
|
|
59739
60222
|
if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
|
|
59740
60223
|
if (!slug2) errors.push("A Commons entity needs a URL slug.");
|
|
@@ -59824,9 +60307,10 @@ async function submitCommonsEntity(input, user) {
|
|
|
59824
60307
|
};
|
|
59825
60308
|
}
|
|
59826
60309
|
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
60310
|
+
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
59827
60311
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
59828
60312
|
const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
|
|
59829
|
-
const proposalId = `commons-proposal-${(0,
|
|
60313
|
+
const proposalId = `commons-proposal-${(0, import_node_crypto35.randomUUID)()}`;
|
|
59830
60314
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
59831
60315
|
const entityId = existing?.id ?? normalized.entityId ?? allocateEntityId();
|
|
59832
60316
|
const proposalStatus = shouldApply ? "accepted" : "pending_review";
|
|
@@ -59868,6 +60352,10 @@ async function submitCommonsEntity(input, user) {
|
|
|
59868
60352
|
];
|
|
59869
60353
|
if (nextEntity) {
|
|
59870
60354
|
batch.push(entityUpsertStatement(nextEntity, existing));
|
|
60355
|
+
if (normalized.claims !== void 0) {
|
|
60356
|
+
batch.push({ sql: "DELETE FROM commons_claims WHERE subject_entity_id = ?", args: [nextEntity.id] });
|
|
60357
|
+
batch.push(...claimInsertStatements(normalized.claims, existingClaims, nextEntity.id, proposalId, Number(user.id), now));
|
|
60358
|
+
}
|
|
59871
60359
|
batch.push(...commonsIndexStatements(nextEntity, now));
|
|
59872
60360
|
batch.push({
|
|
59873
60361
|
sql: `
|
|
@@ -59877,17 +60365,26 @@ async function submitCommonsEntity(input, user) {
|
|
|
59877
60365
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
59878
60366
|
`,
|
|
59879
60367
|
args: [
|
|
59880
|
-
`commons-ledger-${(0,
|
|
60368
|
+
`commons-ledger-${(0, import_node_crypto35.randomUUID)()}`,
|
|
59881
60369
|
nextEntity.id,
|
|
59882
60370
|
proposalId,
|
|
59883
60371
|
user.id,
|
|
59884
60372
|
actorLabel(input, user),
|
|
59885
60373
|
normalized.actorChannel,
|
|
59886
60374
|
existing ? "entity_updated" : "entity_created",
|
|
59887
|
-
JSON.stringify(
|
|
60375
|
+
JSON.stringify([
|
|
60376
|
+
...changedFieldPaths(existing, nextEntity),
|
|
60377
|
+
...normalized.claims !== void 0 ? ["claims"] : []
|
|
60378
|
+
]),
|
|
59888
60379
|
normalized.changeSummary || (existing ? "Updated Commons entity." : "Created Commons entity."),
|
|
59889
|
-
existing ? JSON.stringify(
|
|
59890
|
-
|
|
60380
|
+
existing ? JSON.stringify({
|
|
60381
|
+
...entityLedgerSnapshot(existing),
|
|
60382
|
+
...normalized.claims !== void 0 ? { claims: existingClaims } : {}
|
|
60383
|
+
}) : null,
|
|
60384
|
+
JSON.stringify({
|
|
60385
|
+
...entityLedgerSnapshot(nextEntity),
|
|
60386
|
+
...normalized.claims !== void 0 ? { claims: normalized.claims } : {}
|
|
60387
|
+
}),
|
|
59891
60388
|
now
|
|
59892
60389
|
]
|
|
59893
60390
|
});
|
|
@@ -59938,17 +60435,41 @@ async function getCommonsEntityLedger(idOrSlug) {
|
|
|
59938
60435
|
ledger: result.rows.map((row) => rowToLedgerEntry(row))
|
|
59939
60436
|
};
|
|
59940
60437
|
}
|
|
60438
|
+
async function listCommonsClaims(idOrSlug) {
|
|
60439
|
+
await ensureCommonsSchema();
|
|
60440
|
+
const entity = await getCommonsEntity(idOrSlug);
|
|
60441
|
+
if (!entity) return null;
|
|
60442
|
+
return { entity, claims: await getCommonsClaimsByEntityId(entity.id) };
|
|
60443
|
+
}
|
|
60444
|
+
async function getCommonsEntityLinkset(idOrSlug) {
|
|
60445
|
+
const result = await listCommonsClaims(idOrSlug);
|
|
60446
|
+
if (!result) return null;
|
|
60447
|
+
const targetIds = [...new Set(result.claims.map((claim) => claim.objectEntityId).filter((id) => Boolean(id)))];
|
|
60448
|
+
const targetSlugs = /* @__PURE__ */ new Map();
|
|
60449
|
+
if (targetIds.length) {
|
|
60450
|
+
const targets = await getDb().execute({
|
|
60451
|
+
sql: `SELECT id, slug FROM commons_entities WHERE status = 'published' AND id IN (${targetIds.map(() => "?").join(", ")})`,
|
|
60452
|
+
args: targetIds
|
|
60453
|
+
});
|
|
60454
|
+
for (const row of targets.rows) targetSlugs.set(String(row.id), String(row.slug));
|
|
60455
|
+
}
|
|
60456
|
+
const resolved = result.claims.map((claim) => {
|
|
60457
|
+
const href = claim.objectUrl ?? (claim.objectEntityId && targetSlugs.has(claim.objectEntityId) ? publicUrlForSlug(targetSlugs.get(claim.objectEntityId)) : null);
|
|
60458
|
+
return href ? { ...claim, href } : null;
|
|
60459
|
+
}).filter((claim) => claim !== null);
|
|
60460
|
+
return { entity: result.entity, claims: result.claims, ...buildCommonsLinkset(result.entity, resolved) };
|
|
60461
|
+
}
|
|
59941
60462
|
async function saveCommonsFilter(input, user) {
|
|
59942
60463
|
await ensureCommonsSchema();
|
|
59943
60464
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
59944
60465
|
const name = requiredText(input.name, "name", 2, 80);
|
|
59945
|
-
const description =
|
|
60466
|
+
const description = optionalText2(input.description, 500);
|
|
59946
60467
|
const filter = sanitizeSearchFilters(input.filter);
|
|
59947
60468
|
const existing = await getDb().execute({
|
|
59948
60469
|
sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
|
|
59949
60470
|
args: [user.id, name]
|
|
59950
60471
|
});
|
|
59951
|
-
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0,
|
|
60472
|
+
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${(0, import_node_crypto35.randomUUID)()}`;
|
|
59952
60473
|
await getDb().execute({
|
|
59953
60474
|
sql: `
|
|
59954
60475
|
INSERT INTO commons_saved_filters (id, user_id, name, description, filter_json, created_at, updated_at)
|
|
@@ -60075,9 +60596,9 @@ function commonsDatabaseReport() {
|
|
|
60075
60596
|
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review.",
|
|
60076
60597
|
billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
|
|
60077
60598
|
},
|
|
60078
|
-
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
60599
|
+
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "relationship claims", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
60079
60600
|
indexing: commonsIndexingPolicy(),
|
|
60080
|
-
tables: ["commons_entities", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
|
|
60601
|
+
tables: ["commons_entities", "commons_claims", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
|
|
60081
60602
|
};
|
|
60082
60603
|
}
|
|
60083
60604
|
function entityToWikiPage(entity) {
|
|
@@ -60097,6 +60618,13 @@ function entityToWikiPage(entity) {
|
|
|
60097
60618
|
media: entity.media,
|
|
60098
60619
|
sections: entity.articleSections,
|
|
60099
60620
|
metadata: {
|
|
60621
|
+
linksetUrl: commonsLinksetUrl(entity.id),
|
|
60622
|
+
linksetDiscovery: {
|
|
60623
|
+
rel: "linkset",
|
|
60624
|
+
type: COMMONS_LINKSET_MEDIA_TYPE,
|
|
60625
|
+
profile: COMMONS_RELATIONSHIP_PROFILE,
|
|
60626
|
+
href: commonsLinksetUrl(entity.id)
|
|
60627
|
+
},
|
|
60100
60628
|
entityType: friendlyEntityType(entity.entityType),
|
|
60101
60629
|
disambiguationName: entity.disambiguationName,
|
|
60102
60630
|
keywords: entity.keywords,
|
|
@@ -60253,7 +60781,7 @@ async function commonsTagVocabulary() {
|
|
|
60253
60781
|
...parseJson3(record.tags_json, []),
|
|
60254
60782
|
...parseJson3(record.keywords_json, [])
|
|
60255
60783
|
]) {
|
|
60256
|
-
const cleaned =
|
|
60784
|
+
const cleaned = optionalText2(tag, 120);
|
|
60257
60785
|
if (!cleaned) continue;
|
|
60258
60786
|
counts.set(cleaned, (counts.get(cleaned) ?? 0) + 1);
|
|
60259
60787
|
}
|
|
@@ -60263,7 +60791,7 @@ async function commonsTagVocabulary() {
|
|
|
60263
60791
|
function resolveCommonsTags(candidates, vocabulary) {
|
|
60264
60792
|
const vocabularyByKey = new Map(vocabulary.map((entry) => [entry.tag.toLowerCase(), entry]));
|
|
60265
60793
|
return candidates.map((candidate) => {
|
|
60266
|
-
const cleaned =
|
|
60794
|
+
const cleaned = optionalText2(candidate, 120) ?? "";
|
|
60267
60795
|
const key = cleaned.toLowerCase();
|
|
60268
60796
|
const exact = vocabularyByKey.get(key);
|
|
60269
60797
|
if (!cleaned || cleaned.length < 2) {
|
|
@@ -60360,12 +60888,13 @@ function normalizeSubmitInput(input) {
|
|
|
60360
60888
|
slug: slug2,
|
|
60361
60889
|
entityId: normalizeEntityId(input.entityId),
|
|
60362
60890
|
entityType: normalizeEntityType(input.entityType),
|
|
60363
|
-
disambiguationName:
|
|
60891
|
+
disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
|
|
60364
60892
|
featuredImage: normalizeFeaturedImage(input.featuredImage),
|
|
60365
60893
|
source: normalizeSource(input.source),
|
|
60366
60894
|
tags: uniqueStrings(input.tags ?? []).slice(0, 24),
|
|
60367
60895
|
keywords: uniqueStrings(input.keywords ?? []).slice(0, 50),
|
|
60368
60896
|
relatedEntities: normalizeRelatedEntities(input.relatedEntities),
|
|
60897
|
+
claims: normalizeClaimsForRepository(input.claims),
|
|
60369
60898
|
bodyMarkdown: normalizeBody(input.bodyMarkdown, input.contentSections),
|
|
60370
60899
|
contentSections: normalizeContentSections(input.contentSections),
|
|
60371
60900
|
articleSections: normalizeArticleSections(input.articleSections),
|
|
@@ -60374,11 +60903,18 @@ function normalizeSubmitInput(input) {
|
|
|
60374
60903
|
seo: normalizeSeo(input.seo),
|
|
60375
60904
|
actorChannel: input.actorChannel ?? "mcp",
|
|
60376
60905
|
actorLabel: input.actorLabel ?? "",
|
|
60377
|
-
changeSummary:
|
|
60906
|
+
changeSummary: optionalText2(input.changeSummary, 500) ?? "",
|
|
60378
60907
|
reviewPolicy: input.reviewPolicy ?? "auto_publish_if_safe",
|
|
60379
60908
|
baseRevision: input.baseRevision
|
|
60380
60909
|
};
|
|
60381
60910
|
}
|
|
60911
|
+
function normalizeClaimsForRepository(input) {
|
|
60912
|
+
try {
|
|
60913
|
+
return normalizeCommonsClaims(input);
|
|
60914
|
+
} catch (error) {
|
|
60915
|
+
throw new CommonsRepositoryError("invalid_claims", error instanceof Error ? error.message : "Claims are invalid.");
|
|
60916
|
+
}
|
|
60917
|
+
}
|
|
60382
60918
|
function evaluatePublishSafety(input, existing) {
|
|
60383
60919
|
if (!input.featuredImage?.url) return { safe: false, reason: "A featured image is required before a Commons entity can publish." };
|
|
60384
60920
|
if (!input.bodyMarkdown.trim() && !input.source.originalUrl && !input.source.resolvedUrl) {
|
|
@@ -60524,6 +61060,84 @@ function entityStorageValues(entity) {
|
|
|
60524
61060
|
entity.revision
|
|
60525
61061
|
];
|
|
60526
61062
|
}
|
|
61063
|
+
function claimInsertStatements(claims, existingClaims, subjectEntityId, proposalId, userId, now) {
|
|
61064
|
+
const existingByIdentity = new Map(existingClaims.map((claim) => [commonsClaimIdentityKey(claim), claim]));
|
|
61065
|
+
return claims.map((claim) => {
|
|
61066
|
+
const existing = existingByIdentity.get(commonsClaimIdentityKey(claim));
|
|
61067
|
+
return {
|
|
61068
|
+
sql: `
|
|
61069
|
+
INSERT INTO commons_claims (
|
|
61070
|
+
id, subject_entity_id, predicate, predicate_label, object_entity_id, object_url,
|
|
61071
|
+
title, media_type, hreflang, qualifiers_json, references_json, rank, confidence,
|
|
61072
|
+
confidence_reason, dispute_state, valid_from, valid_to, as_of, source_family,
|
|
61073
|
+
review_state, proposal_id, created_by_user_id, updated_by_user_id, created_at, updated_at
|
|
61074
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'approved', ?, ?, ?, ?, ?)
|
|
61075
|
+
`,
|
|
61076
|
+
args: [
|
|
61077
|
+
existing?.id ?? allocateClaimId(),
|
|
61078
|
+
subjectEntityId,
|
|
61079
|
+
claim.predicate,
|
|
61080
|
+
claim.predicateLabel ?? null,
|
|
61081
|
+
claim.objectEntityId ?? null,
|
|
61082
|
+
claim.objectUrl ?? null,
|
|
61083
|
+
claim.title ?? null,
|
|
61084
|
+
claim.mediaType ?? null,
|
|
61085
|
+
claim.hreflang ? JSON.stringify(claim.hreflang) : null,
|
|
61086
|
+
JSON.stringify(claim.qualifiers ?? {}),
|
|
61087
|
+
JSON.stringify(claim.references ?? []),
|
|
61088
|
+
claim.rank ?? "normal",
|
|
61089
|
+
claim.confidence ?? 1,
|
|
61090
|
+
claim.confidenceReason ?? null,
|
|
61091
|
+
claim.disputeState ?? "undisputed",
|
|
61092
|
+
claim.validFrom ?? null,
|
|
61093
|
+
claim.validTo ?? null,
|
|
61094
|
+
claim.asOf ?? null,
|
|
61095
|
+
claim.sourceFamily ?? "unknown",
|
|
61096
|
+
proposalId,
|
|
61097
|
+
existing?.createdByUserId ?? userId,
|
|
61098
|
+
userId,
|
|
61099
|
+
existing?.createdAt ?? now,
|
|
61100
|
+
now
|
|
61101
|
+
]
|
|
61102
|
+
};
|
|
61103
|
+
});
|
|
61104
|
+
}
|
|
61105
|
+
async function getCommonsClaimsByEntityId(entityId) {
|
|
61106
|
+
const result = await getDb().execute({
|
|
61107
|
+
sql: "SELECT * FROM commons_claims WHERE subject_entity_id = ? ORDER BY predicate ASC, object_url ASC, object_entity_id ASC, id ASC",
|
|
61108
|
+
args: [entityId]
|
|
61109
|
+
});
|
|
61110
|
+
return result.rows.map((row) => rowToClaim(row));
|
|
61111
|
+
}
|
|
61112
|
+
function rowToClaim(row) {
|
|
61113
|
+
return {
|
|
61114
|
+
id: String(row.id),
|
|
61115
|
+
subjectEntityId: String(row.subject_entity_id),
|
|
61116
|
+
predicate: String(row.predicate),
|
|
61117
|
+
predicateLabel: row.predicate_label == null ? void 0 : String(row.predicate_label),
|
|
61118
|
+
objectEntityId: row.object_entity_id == null ? void 0 : String(row.object_entity_id),
|
|
61119
|
+
objectUrl: row.object_url == null ? void 0 : String(row.object_url),
|
|
61120
|
+
title: row.title == null ? void 0 : String(row.title),
|
|
61121
|
+
mediaType: row.media_type == null ? void 0 : String(row.media_type),
|
|
61122
|
+
hreflang: row.hreflang == null ? void 0 : parseJson3(row.hreflang, []),
|
|
61123
|
+
qualifiers: parseJson3(row.qualifiers_json, {}),
|
|
61124
|
+
references: parseJson3(row.references_json, []),
|
|
61125
|
+
rank: String(row.rank),
|
|
61126
|
+
confidence: Number(row.confidence),
|
|
61127
|
+
confidenceReason: row.confidence_reason == null ? void 0 : String(row.confidence_reason),
|
|
61128
|
+
disputeState: String(row.dispute_state),
|
|
61129
|
+
validFrom: row.valid_from == null ? void 0 : String(row.valid_from),
|
|
61130
|
+
validTo: row.valid_to == null ? void 0 : String(row.valid_to),
|
|
61131
|
+
asOf: row.as_of == null ? void 0 : String(row.as_of),
|
|
61132
|
+
sourceFamily: String(row.source_family),
|
|
61133
|
+
reviewState: String(row.review_state),
|
|
61134
|
+
proposalId: String(row.proposal_id),
|
|
61135
|
+
createdByUserId: row.created_by_user_id == null ? null : Number(row.created_by_user_id),
|
|
61136
|
+
updatedByUserId: row.updated_by_user_id == null ? null : Number(row.updated_by_user_id),
|
|
61137
|
+
createdAt: String(row.created_at),
|
|
61138
|
+
updatedAt: String(row.updated_at)
|
|
61139
|
+
};
|
|
61140
|
+
}
|
|
60527
61141
|
function commonsIndexStatements(entity, now) {
|
|
60528
61142
|
return commonsIndexDocuments(entity, now).map((document2) => ({
|
|
60529
61143
|
sql: `
|
|
@@ -60628,7 +61242,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
60628
61242
|
return documents;
|
|
60629
61243
|
}
|
|
60630
61244
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
60631
|
-
return `commons-index-${(0,
|
|
61245
|
+
return `commons-index-${(0, import_node_crypto35.createHash)("sha256").update(`${entityId}
|
|
60632
61246
|
${documentType}
|
|
60633
61247
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
60634
61248
|
}
|
|
@@ -60803,14 +61417,14 @@ function normalizeArticleSections(sections) {
|
|
|
60803
61417
|
return {
|
|
60804
61418
|
relatedLinks: normalizeRelatedLinks(sections?.relatedLinks),
|
|
60805
61419
|
notes: Array.isArray(sections?.notes) ? sections.notes.map((note) => ({
|
|
60806
|
-
marker:
|
|
61420
|
+
marker: optionalText2(note.marker, 20) ?? void 0,
|
|
60807
61421
|
body: requiredText(note.body, "notes.body", 1, 2e3)
|
|
60808
61422
|
})).slice(0, 100) : [],
|
|
60809
61423
|
citations: normalizeCitations(sections?.citations),
|
|
60810
61424
|
externalLinks: Array.isArray(sections?.externalLinks) ? sections.externalLinks.map((link2) => ({
|
|
60811
61425
|
title: requiredText(link2.title, "externalLinks.title", 1, 180),
|
|
60812
61426
|
url: requiredText(link2.url, "externalLinks.url", 1, 2e3),
|
|
60813
|
-
summary:
|
|
61427
|
+
summary: optionalText2(link2.summary, 300) ?? void 0
|
|
60814
61428
|
})).slice(0, 100) : [],
|
|
60815
61429
|
categories: uniqueStrings(sections?.categories ?? []).slice(0, 50)
|
|
60816
61430
|
};
|
|
@@ -60819,7 +61433,7 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
60819
61433
|
if (!Array.isArray(relatedLinks)) return [];
|
|
60820
61434
|
return relatedLinks.map((link2) => {
|
|
60821
61435
|
const title = requiredText(link2.title, "relatedLinks.title", 1, 180);
|
|
60822
|
-
const url =
|
|
61436
|
+
const url = optionalText2(link2.url, 2e3);
|
|
60823
61437
|
const slug2 = link2.slug ? normalizeSlug(link2.slug) : void 0;
|
|
60824
61438
|
const entityId = normalizeEntityId(link2.entityId);
|
|
60825
61439
|
const needsLink = link2.needsLink === true || !url && !slug2 && !entityId;
|
|
@@ -60828,9 +61442,9 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
60828
61442
|
url,
|
|
60829
61443
|
slug: slug2,
|
|
60830
61444
|
entityId,
|
|
60831
|
-
relationship:
|
|
60832
|
-
summary:
|
|
60833
|
-
imageUrl:
|
|
61445
|
+
relationship: optionalText2(link2.relationship, 120) ?? void 0,
|
|
61446
|
+
summary: optionalText2(link2.summary, 300) ?? void 0,
|
|
61447
|
+
imageUrl: optionalText2(link2.imageUrl, 2e3) ?? void 0,
|
|
60834
61448
|
needsLink
|
|
60835
61449
|
};
|
|
60836
61450
|
}).slice(0, 100);
|
|
@@ -60839,20 +61453,20 @@ function normalizeCitations(citations) {
|
|
|
60839
61453
|
if (!Array.isArray(citations)) return [];
|
|
60840
61454
|
return citations.map((citation) => ({
|
|
60841
61455
|
title: requiredText(citation.title, "citation.title", 1, 240),
|
|
60842
|
-
url:
|
|
60843
|
-
source:
|
|
60844
|
-
note:
|
|
60845
|
-
accessedAt:
|
|
61456
|
+
url: optionalText2(citation.url, 2e3) ?? void 0,
|
|
61457
|
+
source: optionalText2(citation.source, 240) ?? void 0,
|
|
61458
|
+
note: optionalText2(citation.note, 1e3) ?? void 0,
|
|
61459
|
+
accessedAt: optionalText2(citation.accessedAt, 80) ?? void 0
|
|
60846
61460
|
})).slice(0, 300);
|
|
60847
61461
|
}
|
|
60848
61462
|
function normalizeFeaturedImage(image) {
|
|
60849
61463
|
if (!image?.url) return void 0;
|
|
60850
61464
|
return {
|
|
60851
61465
|
url: requiredText(image.url, "featuredImage.url", 1, 2e3),
|
|
60852
|
-
alt:
|
|
60853
|
-
caption:
|
|
60854
|
-
sourceUrl:
|
|
60855
|
-
license:
|
|
61466
|
+
alt: optionalText2(image.alt, 500) ?? void 0,
|
|
61467
|
+
caption: optionalText2(image.caption, 1e3) ?? void 0,
|
|
61468
|
+
sourceUrl: optionalText2(image.sourceUrl, 2e3) ?? void 0,
|
|
61469
|
+
license: optionalText2(image.license, 240) ?? void 0,
|
|
60856
61470
|
width: positiveInt(image.width),
|
|
60857
61471
|
height: positiveInt(image.height)
|
|
60858
61472
|
};
|
|
@@ -60861,11 +61475,11 @@ function normalizeMedia(media, featuredImage) {
|
|
|
60861
61475
|
const normalized = Array.isArray(media) ? media.map((item) => ({
|
|
60862
61476
|
type: item.type,
|
|
60863
61477
|
url: requiredText(item.url, "media.url", 1, 2e3),
|
|
60864
|
-
alt:
|
|
60865
|
-
caption:
|
|
60866
|
-
posterUrl:
|
|
60867
|
-
sourceUrl:
|
|
60868
|
-
license:
|
|
61478
|
+
alt: optionalText2(item.alt, 500) ?? void 0,
|
|
61479
|
+
caption: optionalText2(item.caption, 1e3) ?? void 0,
|
|
61480
|
+
posterUrl: optionalText2(item.posterUrl, 2e3) ?? void 0,
|
|
61481
|
+
sourceUrl: optionalText2(item.sourceUrl, 2e3) ?? void 0,
|
|
61482
|
+
license: optionalText2(item.license, 240) ?? void 0,
|
|
60869
61483
|
width: positiveInt(item.width),
|
|
60870
61484
|
height: positiveInt(item.height),
|
|
60871
61485
|
durationSeconds: positiveInt(item.durationSeconds)
|
|
@@ -60895,28 +61509,28 @@ function normalizeMedia(media, featuredImage) {
|
|
|
60895
61509
|
}
|
|
60896
61510
|
function normalizeSource(source) {
|
|
60897
61511
|
return {
|
|
60898
|
-
originalUrl:
|
|
60899
|
-
resolvedUrl:
|
|
60900
|
-
sourceCanonicalUrl:
|
|
60901
|
-
relCanonicalHref:
|
|
60902
|
-
sourceByline:
|
|
60903
|
-
publisher:
|
|
61512
|
+
originalUrl: optionalText2(source?.originalUrl, 2e3) ?? void 0,
|
|
61513
|
+
resolvedUrl: optionalText2(source?.resolvedUrl, 2e3) ?? void 0,
|
|
61514
|
+
sourceCanonicalUrl: optionalText2(source?.sourceCanonicalUrl, 2e3) ?? void 0,
|
|
61515
|
+
relCanonicalHref: optionalText2(source?.relCanonicalHref, 2e3) ?? void 0,
|
|
61516
|
+
sourceByline: optionalText2(source?.sourceByline, 240) ?? void 0,
|
|
61517
|
+
publisher: optionalText2(source?.publisher, 240) ?? void 0,
|
|
60904
61518
|
authors: uniqueStrings(source?.authors ?? []).slice(0, 20),
|
|
60905
|
-
publishedAt:
|
|
60906
|
-
capturedAt:
|
|
60907
|
-
license:
|
|
60908
|
-
rightsSummary:
|
|
60909
|
-
contentHash:
|
|
61519
|
+
publishedAt: optionalText2(source?.publishedAt, 80) ?? void 0,
|
|
61520
|
+
capturedAt: optionalText2(source?.capturedAt, 80) ?? void 0,
|
|
61521
|
+
license: optionalText2(source?.license, 240) ?? void 0,
|
|
61522
|
+
rightsSummary: optionalText2(source?.rightsSummary, 1e3) ?? void 0,
|
|
61523
|
+
contentHash: optionalText2(source?.contentHash, 200) ?? void 0
|
|
60910
61524
|
};
|
|
60911
61525
|
}
|
|
60912
61526
|
function normalizeSeo(seo) {
|
|
60913
61527
|
return {
|
|
60914
|
-
canonicalUrl:
|
|
60915
|
-
relCanonical:
|
|
60916
|
-
metaTitle:
|
|
60917
|
-
metaDescription:
|
|
60918
|
-
ogImage:
|
|
60919
|
-
schemaOrgType:
|
|
61528
|
+
canonicalUrl: optionalText2(seo?.canonicalUrl, 2e3) ?? void 0,
|
|
61529
|
+
relCanonical: optionalText2(seo?.relCanonical, 2e3) ?? void 0,
|
|
61530
|
+
metaTitle: optionalText2(seo?.metaTitle, 240) ?? void 0,
|
|
61531
|
+
metaDescription: optionalText2(seo?.metaDescription, 500) ?? void 0,
|
|
61532
|
+
ogImage: optionalText2(seo?.ogImage, 2e3) ?? void 0,
|
|
61533
|
+
schemaOrgType: optionalText2(seo?.schemaOrgType, 80) ?? void 0,
|
|
60920
61534
|
noIndex: seo?.noIndex === true
|
|
60921
61535
|
};
|
|
60922
61536
|
}
|
|
@@ -60925,37 +61539,37 @@ function normalizeRelatedEntities(related) {
|
|
|
60925
61539
|
return related.map((item) => ({
|
|
60926
61540
|
entityId: normalizeEntityId(item.entityId),
|
|
60927
61541
|
title: requiredText(item.title, "relatedEntities.title", 1, 180),
|
|
60928
|
-
relationship:
|
|
61542
|
+
relationship: optionalText2(item.relationship, 120) ?? void 0,
|
|
60929
61543
|
slug: item.slug ? normalizeSlug(item.slug) : void 0,
|
|
60930
|
-
url:
|
|
60931
|
-
description:
|
|
61544
|
+
url: optionalText2(item.url, 2e3) ?? void 0,
|
|
61545
|
+
description: optionalText2(item.description, 300) ?? void 0
|
|
60932
61546
|
})).slice(0, 200);
|
|
60933
61547
|
}
|
|
60934
61548
|
function sanitizeSearchFilters(filters) {
|
|
60935
61549
|
return {
|
|
60936
|
-
query:
|
|
61550
|
+
query: optionalText2(filters.query, 300) ?? void 0,
|
|
60937
61551
|
entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
|
|
60938
|
-
tag:
|
|
61552
|
+
tag: optionalText2(filters.tag, 80) ?? void 0,
|
|
60939
61553
|
tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
|
|
60940
|
-
keyword:
|
|
61554
|
+
keyword: optionalText2(filters.keyword, 120) ?? void 0,
|
|
60941
61555
|
keywords: uniqueStrings(filters.keywords ?? []).slice(0, 20),
|
|
60942
61556
|
relatedEntityId: normalizeEntityId(filters.relatedEntityId),
|
|
60943
61557
|
trailId: normalizeEntityId(filters.trailId),
|
|
60944
|
-
sourceDomain:
|
|
61558
|
+
sourceDomain: optionalText2(filters.sourceDomain, 240) ?? void 0,
|
|
60945
61559
|
hasMedia: filters.hasMedia === true ? true : void 0,
|
|
60946
61560
|
hasVideo: filters.hasVideo === true ? true : void 0,
|
|
60947
|
-
publishedAfter:
|
|
60948
|
-
updatedAfter:
|
|
60949
|
-
filterId:
|
|
61561
|
+
publishedAfter: optionalText2(filters.publishedAfter, 80) ?? void 0,
|
|
61562
|
+
updatedAfter: optionalText2(filters.updatedAfter, 80) ?? void 0,
|
|
61563
|
+
filterId: optionalText2(filters.filterId, 200) ?? void 0,
|
|
60950
61564
|
limit: clampNumber(filters.limit, 1, 100, 20),
|
|
60951
61565
|
offset: clampNumber(filters.offset, 0, 1e4, 0)
|
|
60952
61566
|
};
|
|
60953
61567
|
}
|
|
60954
61568
|
function sanitizeNeedsLinkFilters(filters) {
|
|
60955
61569
|
return {
|
|
60956
|
-
query:
|
|
61570
|
+
query: optionalText2(filters.query, 300) ?? void 0,
|
|
60957
61571
|
entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
|
|
60958
|
-
tag:
|
|
61572
|
+
tag: optionalText2(filters.tag, 80) ?? void 0,
|
|
60959
61573
|
tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
|
|
60960
61574
|
sourceEntityId: normalizeEntityId(filters.sourceEntityId),
|
|
60961
61575
|
sourceSlug: filters.sourceSlug ? normalizeSlug(filters.sourceSlug) : void 0,
|
|
@@ -60976,7 +61590,7 @@ function commonsIndexingPolicy() {
|
|
|
60976
61590
|
storageScope: "platform_commons_graph",
|
|
60977
61591
|
countsAgainstUserStorage: false,
|
|
60978
61592
|
embedsPrivateUserVaults: false,
|
|
60979
|
-
corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "See Also concepts marked needsLink"],
|
|
61593
|
+
corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "governed relationship claims", "See Also concepts marked needsLink"],
|
|
60980
61594
|
excluded: ["comments", "comment threads", "reader alerts", "private reading-room state"],
|
|
60981
61595
|
retrievalPattern: {
|
|
60982
61596
|
lexicalFirst: ["title", "tags", "keywords", "source metadata", "article body", "See Also concepts"],
|
|
@@ -61052,7 +61666,7 @@ function searchableText(entity) {
|
|
|
61052
61666
|
].filter(Boolean).join("\n").toLowerCase();
|
|
61053
61667
|
}
|
|
61054
61668
|
function actorLabel(input, user) {
|
|
61055
|
-
return
|
|
61669
|
+
return optionalText2(input.actorLabel, 120) || optionalText2(user.name, 120) || `mcp-user-${user.id}`;
|
|
61056
61670
|
}
|
|
61057
61671
|
function publicBaseUrl() {
|
|
61058
61672
|
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
|
|
@@ -61063,13 +61677,16 @@ function publicUrlForSlug(slug2) {
|
|
|
61063
61677
|
function allocateEntityId() {
|
|
61064
61678
|
return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
|
|
61065
61679
|
}
|
|
61680
|
+
function allocateClaimId() {
|
|
61681
|
+
return `TPW-S${Date.now()}${Math.floor(Math.random() * 1e6).toString().padStart(6, "0")}`;
|
|
61682
|
+
}
|
|
61066
61683
|
function normalizeEntityId(value) {
|
|
61067
|
-
const text2 =
|
|
61684
|
+
const text2 = optionalText2(value, 80);
|
|
61068
61685
|
if (!text2) return void 0;
|
|
61069
61686
|
return text2.replace(/\s+/g, "").toUpperCase();
|
|
61070
61687
|
}
|
|
61071
61688
|
function normalizeEntityType(value) {
|
|
61072
|
-
const raw =
|
|
61689
|
+
const raw = optionalText2(value, 120) || DEFAULT_ENTITY_TYPE;
|
|
61073
61690
|
const squashed = raw.replace(/[\s_-]+/g, "").toLowerCase();
|
|
61074
61691
|
if (squashed === "publicarticle" || squashed === "article" || squashed === "item") return DEFAULT_ENTITY_TYPE;
|
|
61075
61692
|
if (squashed === "trail" || squashed === "readingtrail") return "Trail";
|
|
@@ -61097,7 +61714,7 @@ function requiredText(value, field, min, max) {
|
|
|
61097
61714
|
if (text2.length < min) throw new CommonsRepositoryError(`${field.replaceAll(".", "_")}_required`, `${field} must be at least ${min} characters.`);
|
|
61098
61715
|
return text2.slice(0, max);
|
|
61099
61716
|
}
|
|
61100
|
-
function
|
|
61717
|
+
function optionalText2(value, max) {
|
|
61101
61718
|
const text2 = String(value ?? "").trim();
|
|
61102
61719
|
return text2 ? text2.slice(0, max) : void 0;
|
|
61103
61720
|
}
|
|
@@ -61117,7 +61734,7 @@ function uniqueStrings(values) {
|
|
|
61117
61734
|
const seen = /* @__PURE__ */ new Set();
|
|
61118
61735
|
const out = [];
|
|
61119
61736
|
for (const value of values) {
|
|
61120
|
-
const text2 =
|
|
61737
|
+
const text2 = optionalText2(value, 120);
|
|
61121
61738
|
if (!text2) continue;
|
|
61122
61739
|
const key = text2.toLowerCase();
|
|
61123
61740
|
if (seen.has(key)) continue;
|
|
@@ -61152,14 +61769,16 @@ function jsonLikeValue(value) {
|
|
|
61152
61769
|
function escapeLike(value) {
|
|
61153
61770
|
return value.replace(/[%_]/g, "");
|
|
61154
61771
|
}
|
|
61155
|
-
var
|
|
61772
|
+
var import_node_crypto35, COMMONS_SCHEMA_VERSION, DEFAULT_COMMONS_BASE_URL, DEFAULT_ENTITY_TYPE, COMMONS_ENTITY_PROFILES, schemaReady, COMMONS_SCHEMA_OBJECTS, CommonsRepositoryError;
|
|
61156
61773
|
var init_commons_repository = __esm({
|
|
61157
61774
|
"src/api/commons-repository.ts"() {
|
|
61158
61775
|
"use strict";
|
|
61159
|
-
|
|
61776
|
+
import_node_crypto35 = require("crypto");
|
|
61160
61777
|
init_db();
|
|
61778
|
+
init_schema_presence();
|
|
61161
61779
|
init_rates();
|
|
61162
|
-
|
|
61780
|
+
init_commons_linksets();
|
|
61781
|
+
COMMONS_SCHEMA_VERSION = "2026-08-05.1";
|
|
61163
61782
|
DEFAULT_COMMONS_BASE_URL = "https://transparent-commons.cc";
|
|
61164
61783
|
DEFAULT_ENTITY_TYPE = "PublicArticle";
|
|
61165
61784
|
COMMONS_ENTITY_PROFILES = {
|
|
@@ -61307,7 +61926,32 @@ var init_commons_repository = __esm({
|
|
|
61307
61926
|
]
|
|
61308
61927
|
}
|
|
61309
61928
|
};
|
|
61310
|
-
|
|
61929
|
+
schemaReady = false;
|
|
61930
|
+
COMMONS_SCHEMA_OBJECTS = [
|
|
61931
|
+
"schema_migrations",
|
|
61932
|
+
"commons_entities",
|
|
61933
|
+
"commons_claims",
|
|
61934
|
+
"commons_proposals",
|
|
61935
|
+
"commons_ledger",
|
|
61936
|
+
"commons_saved_filters",
|
|
61937
|
+
"commons_index_documents",
|
|
61938
|
+
"commons_publications",
|
|
61939
|
+
"commons_publication_editions",
|
|
61940
|
+
"commons_entities_status_updated",
|
|
61941
|
+
"commons_entities_type_updated",
|
|
61942
|
+
"commons_entities_source_domain",
|
|
61943
|
+
"commons_claims_subject_predicate",
|
|
61944
|
+
"commons_claims_object_entity",
|
|
61945
|
+
"commons_proposals_user_created",
|
|
61946
|
+
"commons_proposals_entity_created",
|
|
61947
|
+
"commons_ledger_entity_created",
|
|
61948
|
+
"commons_ledger_user_created",
|
|
61949
|
+
"commons_saved_filters_user_updated",
|
|
61950
|
+
"commons_index_documents_status_updated",
|
|
61951
|
+
"commons_index_documents_entity_type",
|
|
61952
|
+
"commons_publications_updated",
|
|
61953
|
+
"commons_publication_editions_publication_published"
|
|
61954
|
+
];
|
|
61311
61955
|
CommonsRepositoryError = class extends Error {
|
|
61312
61956
|
constructor(code, message, httpStatus = 400) {
|
|
61313
61957
|
super(message);
|
|
@@ -61410,7 +62054,7 @@ async function claimCommonsPublication(input, user) {
|
|
|
61410
62054
|
}
|
|
61411
62055
|
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
61412
62056
|
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
61413
|
-
const id = `tcpub_${(0,
|
|
62057
|
+
const id = `tcpub_${(0, import_node_crypto36.randomUUID)()}`;
|
|
61414
62058
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
61415
62059
|
try {
|
|
61416
62060
|
await getDb().execute({
|
|
@@ -61462,7 +62106,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
61462
62106
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
61463
62107
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
61464
62108
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
61465
|
-
const editionId = `tced_${(0,
|
|
62109
|
+
const editionId = `tced_${(0, import_node_crypto36.randomUUID)()}`;
|
|
61466
62110
|
const revision = (latest?.revision ?? 0) + 1;
|
|
61467
62111
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
61468
62112
|
await getDb().batch([
|
|
@@ -61486,7 +62130,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
61486
62130
|
JSON.stringify(editionInput.articles),
|
|
61487
62131
|
html,
|
|
61488
62132
|
rendered.filename,
|
|
61489
|
-
(0,
|
|
62133
|
+
(0, import_node_crypto36.createHash)("sha256").update(html).digest("hex"),
|
|
61490
62134
|
rendered.articleCount,
|
|
61491
62135
|
rendered.wordCount,
|
|
61492
62136
|
Buffer.byteLength(html),
|
|
@@ -61672,11 +62316,11 @@ function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
|
61672
62316
|
"</head>"
|
|
61673
62317
|
].join("\n"));
|
|
61674
62318
|
}
|
|
61675
|
-
var
|
|
62319
|
+
var import_node_crypto36, PUBLICATION_ROOT_DOMAIN, RESERVED_SUBDOMAINS, CommonsPublicationError;
|
|
61676
62320
|
var init_commons_publication_repository = __esm({
|
|
61677
62321
|
"src/api/commons-publication-repository.ts"() {
|
|
61678
62322
|
"use strict";
|
|
61679
|
-
|
|
62323
|
+
import_node_crypto36 = require("crypto");
|
|
61680
62324
|
init_db();
|
|
61681
62325
|
init_commons_repository();
|
|
61682
62326
|
init_render();
|
|
@@ -61785,6 +62429,7 @@ var init_commons_routes = __esm({
|
|
|
61785
62429
|
import_zod51 = require("zod");
|
|
61786
62430
|
init_api_auth();
|
|
61787
62431
|
init_commons_repository();
|
|
62432
|
+
init_commons_linksets();
|
|
61788
62433
|
init_listing_write_billing();
|
|
61789
62434
|
init_commons_publication_repository();
|
|
61790
62435
|
init_mcp_tool_schemas();
|
|
@@ -61879,6 +62524,7 @@ var init_commons_routes = __esm({
|
|
|
61879
62524
|
url: import_zod51.z.string().url().optional(),
|
|
61880
62525
|
description: import_zod51.z.string().max(300).optional()
|
|
61881
62526
|
}).strict()).max(200).optional(),
|
|
62527
|
+
claims: import_zod51.z.array(CommonsClaimInputSchema).max(300).optional(),
|
|
61882
62528
|
bodyMarkdown: import_zod51.z.string().max(5e5).optional(),
|
|
61883
62529
|
contentSections: import_zod51.z.array(import_zod51.z.object({
|
|
61884
62530
|
id: import_zod51.z.string().max(180).optional(),
|
|
@@ -61956,6 +62602,17 @@ var init_commons_routes = __esm({
|
|
|
61956
62602
|
await ensureCommonsSchema();
|
|
61957
62603
|
return c.json({ ok: true, data: commonsDatabaseReport() });
|
|
61958
62604
|
});
|
|
62605
|
+
commonsApp.get("/profiles/relationships/v1", (c) => c.json({
|
|
62606
|
+
id: COMMONS_RELATIONSHIP_PROFILE,
|
|
62607
|
+
title: "Transparent Commons governed relationship linksets",
|
|
62608
|
+
mediaType: COMMONS_LINKSET_MEDIA_TYPE,
|
|
62609
|
+
claimAuthority: "MCP Scraper authenticated proposal and contribution ledger",
|
|
62610
|
+
anchorPolicy: "Every link context is an absolute canonical Transparent Commons wiki URL.",
|
|
62611
|
+
predicatePolicy: "Predicates are curated registered link relations or absolute HTTPS extension relation URIs.",
|
|
62612
|
+
publicationPolicy: "Only approved, active, non-deprecated claims whose dispute state is undisputed or attributed are projected.",
|
|
62613
|
+
trustPolicy: "A third-party target is an asserted relationship, not an endorsement or proof that the target agrees.",
|
|
62614
|
+
source: "https://www.rfc-editor.org/rfc/rfc9264.html"
|
|
62615
|
+
}));
|
|
61959
62616
|
commonsApp.post("/publications/prepare", auth2, async (c) => {
|
|
61960
62617
|
const parsed = PreparePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
61961
62618
|
if (!parsed.success) return validationError(c, parsed.error);
|
|
@@ -62086,11 +62743,32 @@ var init_commons_routes = __esm({
|
|
|
62086
62743
|
commonsApp.get("/entities/:idOrSlug", async (c) => {
|
|
62087
62744
|
const entity = await getCommonsEntity(c.req.param("idOrSlug"));
|
|
62088
62745
|
if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
62746
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
62089
62747
|
return c.json({ ok: true, data: entity });
|
|
62090
62748
|
});
|
|
62749
|
+
commonsApp.get("/entities/:idOrSlug/claims", async (c) => {
|
|
62750
|
+
const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
|
|
62751
|
+
if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
62752
|
+
c.header("Link", commonsLinksetDiscoveryHeader(result.entity.id));
|
|
62753
|
+
return c.json({ ok: true, data: { entity: result.entity, claims: result.claims, linksetUrl: result.linksetUrl } });
|
|
62754
|
+
});
|
|
62755
|
+
commonsApp.get("/entities/:idOrSlug/linkset", async (c) => {
|
|
62756
|
+
const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
|
|
62757
|
+
if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
62758
|
+
c.header("Content-Type", `${COMMONS_LINKSET_MEDIA_TYPE}; profile="${COMMONS_RELATIONSHIP_PROFILE}"`);
|
|
62759
|
+
c.header("ETag", result.etag);
|
|
62760
|
+
c.header("Link", `<${COMMONS_RELATIONSHIP_PROFILE}>; rel="profile"`);
|
|
62761
|
+
c.header("Cache-Control", "public, max-age=60, s-maxage=300");
|
|
62762
|
+
if (c.req.header("If-None-Match") === result.etag) return c.body(null, 304);
|
|
62763
|
+
if (c.req.query("envelope") === "true") {
|
|
62764
|
+
return c.json({ ok: true, data: { ...result, document: result.document } });
|
|
62765
|
+
}
|
|
62766
|
+
return c.body(JSON.stringify(result.document));
|
|
62767
|
+
});
|
|
62091
62768
|
commonsApp.get("/entities/:idOrSlug/wiki-page", async (c) => {
|
|
62092
62769
|
const entity = await getCommonsEntity(c.req.param("idOrSlug"));
|
|
62093
62770
|
if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
62771
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
62094
62772
|
return c.json({ ok: true, data: entityToWikiPage(entity) });
|
|
62095
62773
|
});
|
|
62096
62774
|
commonsApp.get("/entities/:idOrSlug/json-ld", async (c) => {
|
|
@@ -62106,6 +62784,7 @@ var init_commons_routes = __esm({
|
|
|
62106
62784
|
commonsApp.get("/wiki/:slug", async (c) => {
|
|
62107
62785
|
const entity = await getCommonsEntity(c.req.param("slug"));
|
|
62108
62786
|
if (!entity) return c.json({ ok: false, error: "page_not_found", message: "No published Commons wiki page matched that slug." }, 404);
|
|
62787
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
62109
62788
|
return c.json({
|
|
62110
62789
|
ok: true,
|
|
62111
62790
|
data: {
|
|
@@ -62229,7 +62908,10 @@ var init_analytics_attribution = __esm({
|
|
|
62229
62908
|
// src/api/analytics-repository.ts
|
|
62230
62909
|
var analytics_repository_exports = {};
|
|
62231
62910
|
__export(analytics_repository_exports, {
|
|
62911
|
+
ANALYTICS_CONTENT_SORTS: () => ANALYTICS_CONTENT_SORTS,
|
|
62232
62912
|
AnalyticsRepositoryError: () => AnalyticsRepositoryError,
|
|
62913
|
+
ENGAGED_SESSION_MS: () => ENGAGED_SESSION_MS,
|
|
62914
|
+
MAX_ENGAGED_MS: () => MAX_ENGAGED_MS,
|
|
62233
62915
|
analyticsAcquisition: () => analyticsAcquisition,
|
|
62234
62916
|
analyticsBusinessMetrics: () => analyticsBusinessMetrics,
|
|
62235
62917
|
analyticsChannelBreakdown: () => analyticsChannelBreakdown,
|
|
@@ -62273,6 +62955,7 @@ __export(analytics_repository_exports, {
|
|
|
62273
62955
|
migrateAnalytics: () => migrateAnalytics,
|
|
62274
62956
|
normalizeAnalyticsPath: () => normalizeAnalyticsPath,
|
|
62275
62957
|
normalizeAnalyticsUrl: () => normalizeAnalyticsUrl,
|
|
62958
|
+
normalizeContentOptions: () => normalizeContentOptions,
|
|
62276
62959
|
normalizeObservedHostname: () => normalizeObservedHostname,
|
|
62277
62960
|
queueAnalyticsActivation: () => queueAnalyticsActivation,
|
|
62278
62961
|
queueAnalyticsFormDelivery: () => queueAnalyticsFormDelivery,
|
|
@@ -62623,6 +63306,12 @@ async function migrateAnalytics() {
|
|
|
62623
63306
|
await client2.query(
|
|
62624
63307
|
`ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS utm_content text`
|
|
62625
63308
|
);
|
|
63309
|
+
await client2.query(
|
|
63310
|
+
`ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS engaged_ms integer`
|
|
63311
|
+
);
|
|
63312
|
+
await client2.query(
|
|
63313
|
+
`ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS scroll_depth smallint`
|
|
63314
|
+
);
|
|
62626
63315
|
await client2.query(
|
|
62627
63316
|
`ALTER TABLE analytics_conversions ADD COLUMN IF NOT EXISTS person_id uuid REFERENCES analytics_people(id) ON DELETE SET NULL`
|
|
62628
63317
|
);
|
|
@@ -62786,7 +63475,7 @@ async function migrateAnalytics() {
|
|
|
62786
63475
|
}
|
|
62787
63476
|
function normalizeSlug2(value) {
|
|
62788
63477
|
const slug2 = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
62789
|
-
return slug2 || `site-${(0,
|
|
63478
|
+
return slug2 || `site-${(0, import_node_crypto37.randomBytes)(4).toString("hex")}`;
|
|
62790
63479
|
}
|
|
62791
63480
|
function normalizeObservedHostname(origin) {
|
|
62792
63481
|
if (!origin || origin === "null") return null;
|
|
@@ -62799,7 +63488,7 @@ function normalizeObservedHostname(origin) {
|
|
|
62799
63488
|
}
|
|
62800
63489
|
}
|
|
62801
63490
|
function publicPixelId() {
|
|
62802
|
-
return `px_${(0,
|
|
63491
|
+
return `px_${(0, import_node_crypto37.randomBytes)(18).toString("base64url")}`;
|
|
62803
63492
|
}
|
|
62804
63493
|
function mapRows(rows) {
|
|
62805
63494
|
return rows;
|
|
@@ -62831,7 +63520,7 @@ async function requireEditor(client2, siteId, userId) {
|
|
|
62831
63520
|
async function createAnalyticsSite(input) {
|
|
62832
63521
|
const db = getAnalyticsPool();
|
|
62833
63522
|
const client2 = await db.connect();
|
|
62834
|
-
const id = (0,
|
|
63523
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
62835
63524
|
const baseSlug = normalizeSlug2(input.slug || input.name);
|
|
62836
63525
|
try {
|
|
62837
63526
|
await client2.query("BEGIN");
|
|
@@ -62847,7 +63536,7 @@ async function createAnalyticsSite(input) {
|
|
|
62847
63536
|
} catch (error) {
|
|
62848
63537
|
const code = error.code;
|
|
62849
63538
|
if (code !== "23505" || attempt === 2) throw error;
|
|
62850
|
-
slug2 = `${baseSlug}-${(0,
|
|
63539
|
+
slug2 = `${baseSlug}-${(0, import_node_crypto37.randomBytes)(2).toString("hex")}`;
|
|
62851
63540
|
}
|
|
62852
63541
|
}
|
|
62853
63542
|
await client2.query(
|
|
@@ -63062,7 +63751,7 @@ async function createAnalyticsPixel(input) {
|
|
|
63062
63751
|
VALUES ($1, $2, $3, $4, $5)
|
|
63063
63752
|
RETURNING id, site_id, public_id, name, environment, status, created_at::text, NULL::text AS last_event_at`,
|
|
63064
63753
|
[
|
|
63065
|
-
(0,
|
|
63754
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63066
63755
|
input.siteId,
|
|
63067
63756
|
publicPixelId(),
|
|
63068
63757
|
input.name.trim(),
|
|
@@ -63139,7 +63828,7 @@ async function setAnalyticsPixelDomainState(input) {
|
|
|
63139
63828
|
SELECT $1, p.id, $4, $5 FROM analytics_pixels p WHERE p.id = $2 AND p.site_id = $3
|
|
63140
63829
|
ON CONFLICT(pixel_id, hostname) DO UPDATE SET state = EXCLUDED.state, updated_at = now()
|
|
63141
63830
|
RETURNING id`,
|
|
63142
|
-
[(0,
|
|
63831
|
+
[(0, import_node_crypto37.randomUUID)(), input.pixelId, input.siteId, hostname, input.state]
|
|
63143
63832
|
);
|
|
63144
63833
|
if (!result.rowCount)
|
|
63145
63834
|
throw new AnalyticsRepositoryError(
|
|
@@ -63148,6 +63837,12 @@ async function setAnalyticsPixelDomainState(input) {
|
|
|
63148
63837
|
404
|
|
63149
63838
|
);
|
|
63150
63839
|
}
|
|
63840
|
+
function boundedEngagedMs(value) {
|
|
63841
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.min(Math.trunc(value), MAX_ENGAGED_MS) : null;
|
|
63842
|
+
}
|
|
63843
|
+
function boundedScrollDepth(value) {
|
|
63844
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.min(Math.trunc(value), 100) : null;
|
|
63845
|
+
}
|
|
63151
63846
|
function normalizeAnalyticsPath(value) {
|
|
63152
63847
|
if (!value) return "/";
|
|
63153
63848
|
try {
|
|
@@ -63183,7 +63878,7 @@ function sanitizeAnalyticsProperties(value) {
|
|
|
63183
63878
|
}
|
|
63184
63879
|
async function ingestAnalyticsEvents(input) {
|
|
63185
63880
|
const db = getAnalyticsPool();
|
|
63186
|
-
const requestId = `air_${(0,
|
|
63881
|
+
const requestId = `air_${(0, import_node_crypto37.randomUUID)()}`;
|
|
63187
63882
|
const hostname = normalizeObservedHostname(input.origin);
|
|
63188
63883
|
const pixelResult = await db.query(
|
|
63189
63884
|
`SELECT p.id, p.site_id, p.status,
|
|
@@ -63204,7 +63899,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63204
63899
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
63205
63900
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
63206
63901
|
[
|
|
63207
|
-
(0,
|
|
63902
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63208
63903
|
requestId,
|
|
63209
63904
|
pixel?.site_id ?? null,
|
|
63210
63905
|
pixel?.id ?? null,
|
|
@@ -63229,7 +63924,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63229
63924
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
63230
63925
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
63231
63926
|
[
|
|
63232
|
-
(0,
|
|
63927
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63233
63928
|
requestId,
|
|
63234
63929
|
pixel.site_id,
|
|
63235
63930
|
pixel.id,
|
|
@@ -63254,7 +63949,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63254
63949
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, rejected_count, reason_codes)
|
|
63255
63950
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`,
|
|
63256
63951
|
[
|
|
63257
|
-
(0,
|
|
63952
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63258
63953
|
requestId,
|
|
63259
63954
|
pixel.site_id,
|
|
63260
63955
|
pixel.id,
|
|
@@ -63278,7 +63973,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63278
63973
|
ON CONFLICT(pixel_id, hostname) DO UPDATE
|
|
63279
63974
|
SET last_seen_at = now(), updated_at = now()
|
|
63280
63975
|
RETURNING state`,
|
|
63281
|
-
[(0,
|
|
63976
|
+
[(0, import_node_crypto37.randomUUID)(), pixel.id, hostname]
|
|
63282
63977
|
);
|
|
63283
63978
|
if (domainResult.rows[0]?.state !== "approved") {
|
|
63284
63979
|
await db.query(
|
|
@@ -63292,7 +63987,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63292
63987
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
63293
63988
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
63294
63989
|
[
|
|
63295
|
-
(0,
|
|
63990
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63296
63991
|
requestId,
|
|
63297
63992
|
pixel.site_id,
|
|
63298
63993
|
pixel.id,
|
|
@@ -63335,13 +64030,14 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63335
64030
|
id, event_id, site_id, pixel_id, hostname, visitor_id, session_id, event_name,
|
|
63336
64031
|
occurred_at, path, canonical_url, title, referrer, source, medium, campaign,
|
|
63337
64032
|
device_class, properties, country_code, region_code, click_ids, person_id,
|
|
63338
|
-
channel_family, platform, campaign_id, ad_set_id, ad_id, creative_id, placement, utm_term, utm_content
|
|
64033
|
+
channel_family, platform, campaign_id, ad_set_id, ad_id, creative_id, placement, utm_term, utm_content,
|
|
64034
|
+
engaged_ms, scroll_depth
|
|
63339
64035
|
) VALUES (
|
|
63340
64036
|
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18::jsonb, $19, $20, $21::jsonb, $22,
|
|
63341
|
-
$23, $24, $25, $26, $27, $28, $29, $30, $31
|
|
64037
|
+
$23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
|
|
63342
64038
|
) ON CONFLICT(site_id, event_id) DO NOTHING`,
|
|
63343
64039
|
[
|
|
63344
|
-
(0,
|
|
64040
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63345
64041
|
event2.eventId,
|
|
63346
64042
|
pixel.site_id,
|
|
63347
64043
|
pixel.id,
|
|
@@ -63371,7 +64067,9 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63371
64067
|
attribution.creativeId,
|
|
63372
64068
|
attribution.placement,
|
|
63373
64069
|
attribution.term,
|
|
63374
|
-
attribution.content
|
|
64070
|
+
attribution.content,
|
|
64071
|
+
boundedEngagedMs(event2.engagedMs),
|
|
64072
|
+
boundedScrollDepth(event2.scrollDepth)
|
|
63375
64073
|
]
|
|
63376
64074
|
);
|
|
63377
64075
|
if (inserted.rowCount) accepted += 1;
|
|
@@ -63397,7 +64095,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
63397
64095
|
id, request_id, site_id, pixel_id, hostname, accepted_count, rejected_count, reason_codes
|
|
63398
64096
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
|
63399
64097
|
[
|
|
63400
|
-
(0,
|
|
64098
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
63401
64099
|
requestId,
|
|
63402
64100
|
pixel.site_id,
|
|
63403
64101
|
pixel.id,
|
|
@@ -63429,7 +64127,7 @@ function normalizeGeographyCode(value, max) {
|
|
|
63429
64127
|
return normalized && /^[A-Z0-9-]+$/.test(normalized) ? normalized.slice(0, max) : null;
|
|
63430
64128
|
}
|
|
63431
64129
|
function pageFingerprint(scope, siteId, filters) {
|
|
63432
|
-
return (0,
|
|
64130
|
+
return (0, import_node_crypto37.createHash)("sha256").update(JSON.stringify({ scope, siteId, filters })).digest("base64url").slice(0, 18);
|
|
63433
64131
|
}
|
|
63434
64132
|
function decodePageOffset(cursor, fingerprint) {
|
|
63435
64133
|
if (!cursor) return 0;
|
|
@@ -63479,7 +64177,7 @@ async function createAnalyticsConversion(input) {
|
|
|
63479
64177
|
404
|
|
63480
64178
|
);
|
|
63481
64179
|
}
|
|
63482
|
-
const id = (0,
|
|
64180
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
63483
64181
|
const resolvedPerson = input.sessionId ? await db.query(
|
|
63484
64182
|
`SELECT e.person_id FROM analytics_identity_nodes n JOIN analytics_identity_edges e ON e.identity_node_id = n.id
|
|
63485
64183
|
WHERE n.site_id = $1 AND n.kind = 'session_id' AND n.value_hmac = $2 ORDER BY e.confidence DESC LIMIT 1`,
|
|
@@ -63882,10 +64580,21 @@ async function analyticsChannelBreakdown(siteId, userId, filters) {
|
|
|
63882
64580
|
dataFreshThrough: (/* @__PURE__ */ new Date()).toISOString()
|
|
63883
64581
|
};
|
|
63884
64582
|
}
|
|
63885
|
-
|
|
64583
|
+
function normalizeContentOptions(options) {
|
|
64584
|
+
const sort = ANALYTICS_CONTENT_SORTS.includes(options?.sort) ? options.sort : "pageviews";
|
|
64585
|
+
return {
|
|
64586
|
+
sort,
|
|
64587
|
+
direction: options?.direction === "asc" ? "asc" : "desc",
|
|
64588
|
+
search: options?.search?.trim().slice(0, 300) || null
|
|
64589
|
+
};
|
|
64590
|
+
}
|
|
64591
|
+
async function analyticsContent(siteId, userId, filters, page, options) {
|
|
63886
64592
|
const db = getAnalyticsPool();
|
|
63887
64593
|
await accessFor(db, siteId, userId);
|
|
63888
64594
|
const query = eventFilterSql(siteId, filters);
|
|
64595
|
+
const table = normalizeContentOptions(options);
|
|
64596
|
+
const searchParam = `$${query.values.length + 1}`;
|
|
64597
|
+
const order = table.sort === "path" ? `metrics.path ${table.direction === "asc" ? "ASC" : "DESC"}` : `${table.sort} ${table.direction === "asc" ? "ASC" : "DESC"} NULLS LAST, metrics.path ASC`;
|
|
63889
64598
|
const result = await db.query(
|
|
63890
64599
|
`WITH filtered AS (
|
|
63891
64600
|
SELECT e.* FROM analytics_events e WHERE ${query.where}
|
|
@@ -63893,7 +64602,7 @@ async function analyticsContent(siteId, userId, filters, page) {
|
|
|
63893
64602
|
SELECT path, count(DISTINCT visitor_id)::int AS visitors,
|
|
63894
64603
|
count(DISTINCT session_id)::int AS sessions,
|
|
63895
64604
|
count(*) FILTER (WHERE event_name = 'page_view')::int AS pageviews,
|
|
63896
|
-
count(*) FILTER (WHERE event_name
|
|
64605
|
+
count(*) FILTER (WHERE event_name NOT IN ('page_view', 'page_engagement'))::int AS engagement_events,
|
|
63897
64606
|
count(*)::int AS events
|
|
63898
64607
|
FROM filtered GROUP BY path
|
|
63899
64608
|
), session_bounds AS (
|
|
@@ -63901,35 +64610,98 @@ async function analyticsContent(siteId, userId, filters, page) {
|
|
|
63901
64610
|
(array_agg(path ORDER BY occurred_at ASC, id ASC))[1] AS entrance_path,
|
|
63902
64611
|
(array_agg(path ORDER BY occurred_at DESC, id DESC))[1] AS exit_path
|
|
63903
64612
|
FROM filtered WHERE session_id IS NOT NULL GROUP BY session_id
|
|
64613
|
+
), session_pageviews AS (
|
|
64614
|
+
SELECT session_id, count(*) FILTER (WHERE event_name = 'page_view')::int AS pageviews
|
|
64615
|
+
FROM filtered WHERE session_id IS NOT NULL GROUP BY session_id
|
|
64616
|
+
), page_dwell AS (
|
|
64617
|
+
SELECT session_id, path,
|
|
64618
|
+
sum(engaged_ms)::bigint AS engaged_ms,
|
|
64619
|
+
max(scroll_depth)::int AS scroll_depth
|
|
64620
|
+
FROM filtered
|
|
64621
|
+
WHERE event_name = 'page_engagement' AND session_id IS NOT NULL
|
|
64622
|
+
GROUP BY session_id, path
|
|
64623
|
+
), dwell_metrics AS (
|
|
64624
|
+
SELECT path,
|
|
64625
|
+
avg(engaged_ms) FILTER (WHERE engaged_ms IS NOT NULL) AS avg_engaged_ms,
|
|
64626
|
+
avg(scroll_depth) FILTER (WHERE scroll_depth IS NOT NULL) AS avg_scroll,
|
|
64627
|
+
count(*) FILTER (WHERE engaged_ms IS NOT NULL)::int AS measured_pageviews
|
|
64628
|
+
FROM page_dwell GROUP BY path
|
|
63904
64629
|
), entrances AS (
|
|
63905
64630
|
SELECT entrance_path AS path, count(*)::int AS entrances FROM session_bounds GROUP BY entrance_path
|
|
63906
64631
|
), exits AS (
|
|
63907
64632
|
SELECT exit_path AS path, count(*)::int AS exits FROM session_bounds GROUP BY exit_path
|
|
64633
|
+
), bounces AS (
|
|
64634
|
+
SELECT b.entrance_path AS path, count(*)::int AS bounces
|
|
64635
|
+
FROM session_bounds b JOIN session_pageviews p USING(session_id)
|
|
64636
|
+
WHERE p.pageviews <= 1
|
|
64637
|
+
GROUP BY b.entrance_path
|
|
64638
|
+
), converted_sessions AS (
|
|
64639
|
+
SELECT DISTINCT c.session_id
|
|
64640
|
+
FROM analytics_conversions c
|
|
64641
|
+
WHERE c.site_id = $1 AND c.occurred_at >= $2::timestamptz AND c.occurred_at < $3::timestamptz
|
|
64642
|
+
AND c.session_id IS NOT NULL
|
|
64643
|
+
), path_sessions AS (
|
|
64644
|
+
SELECT DISTINCT path, session_id FROM filtered WHERE session_id IS NOT NULL
|
|
64645
|
+
), engaged AS (
|
|
64646
|
+
SELECT ps.path,
|
|
64647
|
+
count(*) FILTER (
|
|
64648
|
+
WHERE sp.pageviews >= 2
|
|
64649
|
+
OR COALESCE(pd.engaged_ms, 0) >= ${ENGAGED_SESSION_MS}
|
|
64650
|
+
OR cs.session_id IS NOT NULL
|
|
64651
|
+
)::int AS engaged_sessions
|
|
64652
|
+
FROM path_sessions ps
|
|
64653
|
+
JOIN session_pageviews sp USING(session_id)
|
|
64654
|
+
LEFT JOIN page_dwell pd ON pd.session_id = ps.session_id AND pd.path = ps.path
|
|
64655
|
+
LEFT JOIN converted_sessions cs ON cs.session_id = ps.session_id
|
|
64656
|
+
GROUP BY ps.path
|
|
63908
64657
|
), assisted AS (
|
|
63909
64658
|
SELECT f.path, count(DISTINCT c.id)::int AS conversions
|
|
63910
64659
|
FROM analytics_conversions c JOIN filtered f ON f.session_id = c.session_id
|
|
63911
64660
|
WHERE c.site_id = $1 AND c.occurred_at >= $2::timestamptz AND c.occurred_at < $3::timestamptz
|
|
63912
64661
|
GROUP BY f.path
|
|
63913
64662
|
)
|
|
63914
|
-
SELECT metrics.*,
|
|
64663
|
+
SELECT metrics.*,
|
|
64664
|
+
COALESCE(entrances.entrances, 0)::int AS entrances,
|
|
63915
64665
|
COALESCE(exits.exits, 0)::int AS exits,
|
|
63916
|
-
COALESCE(assisted.conversions, 0)::int AS conversions
|
|
64666
|
+
COALESCE(assisted.conversions, 0)::int AS conversions,
|
|
64667
|
+
COALESCE(dwell.measured_pageviews, 0)::int AS measured_pageviews,
|
|
64668
|
+
CASE WHEN dwell.avg_engaged_ms IS NULL THEN NULL
|
|
64669
|
+
ELSE round((dwell.avg_engaged_ms / 1000.0)::numeric, 1) END AS avg_seconds_on_page,
|
|
64670
|
+
CASE WHEN dwell.avg_scroll IS NULL THEN NULL
|
|
64671
|
+
ELSE round(dwell.avg_scroll::numeric, 1) END AS avg_scroll_depth,
|
|
64672
|
+
CASE WHEN COALESCE(entrances.entrances, 0) = 0 THEN NULL
|
|
64673
|
+
ELSE round(COALESCE(bounces.bounces, 0)::numeric * 100 / entrances.entrances, 1) END AS bounce_rate,
|
|
64674
|
+
CASE WHEN metrics.pageviews = 0 THEN NULL
|
|
64675
|
+
ELSE round(COALESCE(exits.exits, 0)::numeric * 100 / metrics.pageviews, 1) END AS exit_rate,
|
|
64676
|
+
CASE WHEN metrics.sessions = 0 THEN NULL
|
|
64677
|
+
ELSE round(COALESCE(engaged.engaged_sessions, 0)::numeric * 100 / metrics.sessions, 1) END AS engagement_rate,
|
|
64678
|
+
CASE WHEN metrics.sessions = 0 THEN NULL
|
|
64679
|
+
ELSE round(metrics.pageviews::numeric / metrics.sessions, 2) END AS pageviews_per_session
|
|
63917
64680
|
FROM page_metrics metrics
|
|
63918
64681
|
LEFT JOIN entrances USING(path)
|
|
63919
64682
|
LEFT JOIN exits USING(path)
|
|
63920
64683
|
LEFT JOIN assisted USING(path)
|
|
63921
|
-
|
|
64684
|
+
LEFT JOIN bounces USING(path)
|
|
64685
|
+
LEFT JOIN engaged USING(path)
|
|
64686
|
+
LEFT JOIN dwell_metrics dwell USING(path)
|
|
64687
|
+
WHERE ${searchParam}::text IS NULL OR metrics.path ILIKE '%' || ${searchParam} || '%'
|
|
64688
|
+
ORDER BY ${order}
|
|
63922
64689
|
LIMIT 1000`,
|
|
63923
|
-
query.values
|
|
64690
|
+
[...query.values, table.search]
|
|
63924
64691
|
);
|
|
63925
64692
|
const paged = paginateRows(
|
|
63926
64693
|
result.rows,
|
|
63927
64694
|
page,
|
|
63928
|
-
pageFingerprint(
|
|
64695
|
+
pageFingerprint(
|
|
64696
|
+
`content:${table.sort}:${table.direction}:${table.search ?? ""}`,
|
|
64697
|
+
siteId,
|
|
64698
|
+
filters
|
|
64699
|
+
)
|
|
63929
64700
|
);
|
|
63930
64701
|
return {
|
|
63931
64702
|
content: paged.items,
|
|
63932
64703
|
pageInfo: paged.pageInfo,
|
|
64704
|
+
table,
|
|
63933
64705
|
dataFreshThrough: (/* @__PURE__ */ new Date()).toISOString()
|
|
63934
64706
|
};
|
|
63935
64707
|
}
|
|
@@ -64154,7 +64926,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
64154
64926
|
404
|
|
64155
64927
|
);
|
|
64156
64928
|
}
|
|
64157
|
-
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0,
|
|
64929
|
+
const shortCode = (input.shortCode?.trim().toLowerCase().replace(/[^a-z0-9_-]/g, "-") || (0, import_node_crypto37.randomBytes)(5).toString("base64url").toLowerCase()).slice(0, 48);
|
|
64158
64930
|
try {
|
|
64159
64931
|
const result = await db.query(
|
|
64160
64932
|
`INSERT INTO analytics_campaign_links(
|
|
@@ -64163,7 +64935,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
64163
64935
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
|
|
64164
64936
|
RETURNING *, 0::int AS click_count`,
|
|
64165
64937
|
[
|
|
64166
|
-
(0,
|
|
64938
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64167
64939
|
input.siteId,
|
|
64168
64940
|
input.pixelId ?? null,
|
|
64169
64941
|
input.name.trim(),
|
|
@@ -64234,7 +65006,7 @@ async function resolveAnalyticsCampaignLink(shortCode, referrer) {
|
|
|
64234
65006
|
if (!row) return null;
|
|
64235
65007
|
await db.query(
|
|
64236
65008
|
`INSERT INTO analytics_campaign_clicks(id, link_id, referrer) VALUES ($1, $2, $3)`,
|
|
64237
|
-
[(0,
|
|
65009
|
+
[(0, import_node_crypto37.randomUUID)(), row.id, normalizeAnalyticsUrl(referrer || void 0)]
|
|
64238
65010
|
);
|
|
64239
65011
|
return buildTaggedCampaignUrl(row);
|
|
64240
65012
|
}
|
|
@@ -64252,14 +65024,14 @@ async function createAnalyticsForm(input) {
|
|
|
64252
65024
|
404
|
|
64253
65025
|
);
|
|
64254
65026
|
const baseSlug = normalizeSlug2(input.name);
|
|
64255
|
-
const slug2 = `${baseSlug}-${(0,
|
|
64256
|
-
const publicId = `form_${(0,
|
|
65027
|
+
const slug2 = `${baseSlug}-${(0, import_node_crypto37.randomBytes)(3).toString("hex")}`;
|
|
65028
|
+
const publicId = `form_${(0, import_node_crypto37.randomBytes)(18).toString("base64url")}`;
|
|
64257
65029
|
const result = await db.query(
|
|
64258
65030
|
`INSERT INTO analytics_forms(id, public_id, site_id, pixel_id, name, slug, fields, brand, submit_label, success_message, redirect_url, consent_text, status, created_by_user_id)
|
|
64259
65031
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10,$11,$12,$13,$14)
|
|
64260
65032
|
RETURNING *, 0::int AS submission_count`,
|
|
64261
65033
|
[
|
|
64262
|
-
(0,
|
|
65034
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64263
65035
|
publicId,
|
|
64264
65036
|
input.siteId,
|
|
64265
65037
|
input.pixelId,
|
|
@@ -64304,7 +65076,7 @@ async function getPublicAnalyticsForm(publicId) {
|
|
|
64304
65076
|
return result.rows[0] ?? null;
|
|
64305
65077
|
}
|
|
64306
65078
|
async function recordAnalyticsFormSubmission(input) {
|
|
64307
|
-
const id = (0,
|
|
65079
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
64308
65080
|
await getAnalyticsPool().query(
|
|
64309
65081
|
`INSERT INTO analytics_form_submissions(id, form_id, site_id, pixel_id, visitor_id, session_id, crm_person_ref, source, medium, campaign, crm_delivery_status, click_ids)
|
|
64310
65082
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::jsonb)`,
|
|
@@ -64335,7 +65107,7 @@ function sanitizeClickIds(value) {
|
|
|
64335
65107
|
return output;
|
|
64336
65108
|
}
|
|
64337
65109
|
function identityHmac(kind, value) {
|
|
64338
|
-
return (0,
|
|
65110
|
+
return (0, import_node_crypto37.createHmac)("sha256", getSessionSecret()).update(`${kind}:${value.trim().toLowerCase()}`).digest("hex");
|
|
64339
65111
|
}
|
|
64340
65112
|
async function linkAnalyticsFormIdentity(input) {
|
|
64341
65113
|
const client2 = await getAnalyticsPool().connect();
|
|
@@ -64344,7 +65116,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
64344
65116
|
const person = await client2.query(
|
|
64345
65117
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
64346
65118
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
64347
|
-
[(0,
|
|
65119
|
+
[(0, import_node_crypto37.randomUUID)(), input.siteId, input.crmPersonRef]
|
|
64348
65120
|
);
|
|
64349
65121
|
const personId = person.rows[0].id;
|
|
64350
65122
|
const signals = [];
|
|
@@ -64391,7 +65163,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
64391
65163
|
`INSERT INTO analytics_identity_nodes(id, site_id, kind, value_hmac) VALUES ($1,$2,$3,$4)
|
|
64392
65164
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
64393
65165
|
[
|
|
64394
|
-
(0,
|
|
65166
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64395
65167
|
input.siteId,
|
|
64396
65168
|
signal.kind,
|
|
64397
65169
|
identityHmac(signal.kind, signal.value)
|
|
@@ -64402,7 +65174,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
64402
65174
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
64403
65175
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence)`,
|
|
64404
65176
|
[
|
|
64405
|
-
(0,
|
|
65177
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64406
65178
|
input.siteId,
|
|
64407
65179
|
personId,
|
|
64408
65180
|
node.rows[0].id,
|
|
@@ -64500,7 +65272,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
64500
65272
|
const db = getAnalyticsPool();
|
|
64501
65273
|
await requireEditor(db, input.siteId, input.userId);
|
|
64502
65274
|
const client2 = await db.connect();
|
|
64503
|
-
const id = (0,
|
|
65275
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
64504
65276
|
try {
|
|
64505
65277
|
await client2.query("BEGIN");
|
|
64506
65278
|
await client2.query(
|
|
@@ -64520,7 +65292,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
64520
65292
|
await client2.query(
|
|
64521
65293
|
`INSERT INTO analytics_crm_import_rows(id, import_id, crm_person_ref, payload_ciphertext)
|
|
64522
65294
|
VALUES ($1,$2,$3,$4) ON CONFLICT(import_id, crm_person_ref) DO NOTHING`,
|
|
64523
|
-
[(0,
|
|
65295
|
+
[(0, import_node_crypto37.randomUUID)(), id, row.crmPersonRef, row.payloadCiphertext]
|
|
64524
65296
|
);
|
|
64525
65297
|
}
|
|
64526
65298
|
await client2.query("COMMIT");
|
|
@@ -64559,7 +65331,7 @@ async function createAnalyticsActivationDestination(input) {
|
|
|
64559
65331
|
`INSERT INTO analytics_activation_destinations(id, site_id, platform, name, connection_ref, external_dataset_id, event_mapping, created_by_user_id)
|
|
64560
65332
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8) RETURNING *`,
|
|
64561
65333
|
[
|
|
64562
|
-
(0,
|
|
65334
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64563
65335
|
input.siteId,
|
|
64564
65336
|
input.platform,
|
|
64565
65337
|
input.name.trim(),
|
|
@@ -64611,7 +65383,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
64611
65383
|
`INSERT INTO analytics_activation_jobs(id, destination_id, conversion_id, person_id, payload_ciphertext)
|
|
64612
65384
|
VALUES ($1,$2,$3,$4,$5) ON CONFLICT(destination_id, conversion_id) DO NOTHING`,
|
|
64613
65385
|
[
|
|
64614
|
-
(0,
|
|
65386
|
+
(0, import_node_crypto37.randomUUID)(),
|
|
64615
65387
|
destination.id,
|
|
64616
65388
|
input.conversionId,
|
|
64617
65389
|
input.personId ?? null,
|
|
@@ -64623,7 +65395,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
64623
65395
|
return queued;
|
|
64624
65396
|
}
|
|
64625
65397
|
async function queueAnalyticsFormDelivery(input) {
|
|
64626
|
-
const id = (0,
|
|
65398
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
64627
65399
|
await getAnalyticsPool().query(
|
|
64628
65400
|
`INSERT INTO analytics_form_delivery_jobs(id, submission_id, owner_user_id, payload_ciphertext, last_error_code)
|
|
64629
65401
|
VALUES ($1,$2,$3,$4,$5)`,
|
|
@@ -64750,7 +65522,7 @@ async function analyticsHealth(siteId, userId) {
|
|
|
64750
65522
|
}
|
|
64751
65523
|
async function refreshAnalyticsDailyRollups(input) {
|
|
64752
65524
|
const db = getAnalyticsPool();
|
|
64753
|
-
const runId = (0,
|
|
65525
|
+
const runId = (0, import_node_crypto37.randomUUID)();
|
|
64754
65526
|
await db.query(
|
|
64755
65527
|
`INSERT INTO analytics_rollup_runs(id, window_start, window_end, status) VALUES ($1, $2, $3, 'running')`,
|
|
64756
65528
|
[runId, input.start, input.end]
|
|
@@ -64759,7 +65531,7 @@ async function refreshAnalyticsDailyRollups(input) {
|
|
|
64759
65531
|
try {
|
|
64760
65532
|
await client2.query("BEGIN");
|
|
64761
65533
|
await client2.query(
|
|
64762
|
-
`DELETE FROM analytics_rollups_daily WHERE bucket >= $1::date AND bucket
|
|
65534
|
+
`DELETE FROM analytics_rollups_daily WHERE bucket >= $1::date AND bucket <= $2::date`,
|
|
64763
65535
|
[input.start, input.end]
|
|
64764
65536
|
);
|
|
64765
65537
|
const inserted = await client2.query(
|
|
@@ -64776,7 +65548,14 @@ async function refreshAnalyticsDailyRollups(input) {
|
|
|
64776
65548
|
now()
|
|
64777
65549
|
FROM analytics_events
|
|
64778
65550
|
WHERE occurred_at >= $1::timestamptz AND occurred_at < $2::timestamptz
|
|
64779
|
-
GROUP BY 1, 2, 3, 4, 5, 6, 7
|
|
65551
|
+
GROUP BY 1, 2, 3, 4, 5, 6, 7
|
|
65552
|
+
ON CONFLICT (site_id, pixel_id, hostname, bucket, source, medium, campaign)
|
|
65553
|
+
DO UPDATE SET
|
|
65554
|
+
pageviews = EXCLUDED.pageviews,
|
|
65555
|
+
events = EXCLUDED.events,
|
|
65556
|
+
visitors = EXCLUDED.visitors,
|
|
65557
|
+
sessions = EXCLUDED.sessions,
|
|
65558
|
+
refreshed_at = EXCLUDED.refreshed_at`,
|
|
64780
65559
|
[input.start, input.end]
|
|
64781
65560
|
);
|
|
64782
65561
|
await client2.query("COMMIT");
|
|
@@ -64788,9 +65567,17 @@ async function refreshAnalyticsDailyRollups(input) {
|
|
|
64788
65567
|
return { runId, rows };
|
|
64789
65568
|
} catch (error) {
|
|
64790
65569
|
await client2.query("ROLLBACK");
|
|
65570
|
+
const pgCode = error?.code;
|
|
65571
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
65572
|
+
console.error(JSON.stringify({
|
|
65573
|
+
event: "analytics_rollup_failed",
|
|
65574
|
+
runId,
|
|
65575
|
+
pgCode: pgCode ?? null,
|
|
65576
|
+
message: detail.slice(0, 400)
|
|
65577
|
+
}));
|
|
64791
65578
|
await db.query(
|
|
64792
|
-
`UPDATE analytics_rollup_runs SET status = 'failed', completed_at = now(), error_code =
|
|
64793
|
-
[runId]
|
|
65579
|
+
`UPDATE analytics_rollup_runs SET status = 'failed', completed_at = now(), error_code = $2 WHERE id = $1`,
|
|
65580
|
+
[runId, (pgCode ? `pg_${pgCode}` : "rollup_failed").slice(0, 60)]
|
|
64794
65581
|
);
|
|
64795
65582
|
throw error;
|
|
64796
65583
|
} finally {
|
|
@@ -64843,6 +65630,24 @@ function rowsToCsv2(rows) {
|
|
|
64843
65630
|
)
|
|
64844
65631
|
].join("\n");
|
|
64845
65632
|
}
|
|
65633
|
+
function markdownHeading(column) {
|
|
65634
|
+
return column.split("_").map((word, index) => index ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
65635
|
+
}
|
|
65636
|
+
function markdownCell(value) {
|
|
65637
|
+
if (value === null || value === void 0) return "\u2014";
|
|
65638
|
+
return String(value).replaceAll("|", "\\|").replaceAll("\n", " ");
|
|
65639
|
+
}
|
|
65640
|
+
function rowsToMarkdown(rows) {
|
|
65641
|
+
const columns = [...new Set(rows.flatMap((row) => Object.keys(row)))];
|
|
65642
|
+
if (!columns.length) return ["No rows matched this window and filter set."];
|
|
65643
|
+
return [
|
|
65644
|
+
`| ${columns.map(markdownHeading).join(" | ")} |`,
|
|
65645
|
+
`| ${columns.map(() => "---").join(" | ")} |`,
|
|
65646
|
+
...rows.map(
|
|
65647
|
+
(row) => `| ${columns.map((column) => markdownCell(row[column])).join(" | ")} |`
|
|
65648
|
+
)
|
|
65649
|
+
];
|
|
65650
|
+
}
|
|
64846
65651
|
async function createAnalyticsExport(input) {
|
|
64847
65652
|
const site = (await listAnalyticsSites(input.userId)).find(
|
|
64848
65653
|
(candidate) => candidate.id === input.siteId
|
|
@@ -64910,10 +65715,11 @@ async function createAnalyticsExport(input) {
|
|
|
64910
65715
|
`- Verified conversions: ${overview.conversions}`,
|
|
64911
65716
|
`- Verified revenue: ${(overview.revenueMinor / 100).toFixed(2)}`,
|
|
64912
65717
|
"",
|
|
65718
|
+
...input.report === "overview" ? [] : [`## ${markdownHeading(input.report)} rows`, "", ...rowsToMarkdown(rows), ""],
|
|
64913
65719
|
`Generated ${(/* @__PURE__ */ new Date()).toISOString()} from the governed ${input.report} report contract.`
|
|
64914
65720
|
].join("\n");
|
|
64915
65721
|
}
|
|
64916
|
-
const id = (0,
|
|
65722
|
+
const id = (0, import_node_crypto37.randomUUID)();
|
|
64917
65723
|
const inserted = await getAnalyticsPool().query(
|
|
64918
65724
|
`INSERT INTO analytics_exports(id, site_id, requested_by_user_id, idempotency_key, report, format, filters, content)
|
|
64919
65725
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)
|
|
@@ -64946,11 +65752,11 @@ async function createAnalyticsExport(input) {
|
|
|
64946
65752
|
}
|
|
64947
65753
|
return describeArtifact(replay.rows[0]);
|
|
64948
65754
|
}
|
|
64949
|
-
var
|
|
65755
|
+
var import_node_crypto37, import_pg, AnalyticsRepositoryError, pool2, MAX_ENGAGED_MS, ENGAGED_SESSION_MS, blockedPropertyName, inferredFamilySql, ANALYTICS_CONTENT_SORTS, clickIdKeys;
|
|
64950
65756
|
var init_analytics_repository = __esm({
|
|
64951
65757
|
"src/api/analytics-repository.ts"() {
|
|
64952
65758
|
"use strict";
|
|
64953
|
-
|
|
65759
|
+
import_node_crypto37 = require("crypto");
|
|
64954
65760
|
import_pg = require("pg");
|
|
64955
65761
|
init_session();
|
|
64956
65762
|
init_analytics_attribution();
|
|
@@ -64965,6 +65771,8 @@ var init_analytics_repository = __esm({
|
|
|
64965
65771
|
status;
|
|
64966
65772
|
};
|
|
64967
65773
|
pool2 = null;
|
|
65774
|
+
MAX_ENGAGED_MS = 30 * 60 * 1e3;
|
|
65775
|
+
ENGAGED_SESSION_MS = 1e4;
|
|
64968
65776
|
blockedPropertyName = /(email|phone|password|token|secret|authorization|cookie|name|address|form|body)/i;
|
|
64969
65777
|
inferredFamilySql = `COALESCE(e.channel_family, CASE
|
|
64970
65778
|
WHEN lower(COALESCE(e.source,'') || ' ' || COALESCE(e.referrer,'')) ~ '(perplexity|chatgpt|openai|claude|anthropic|grok|xai|ai[_ -]?overview|google[_ -]?ai|gemini)' THEN 'llm'
|
|
@@ -64973,6 +65781,22 @@ var init_analytics_repository = __esm({
|
|
|
64973
65781
|
WHEN COALESCE(e.source,'') = '' THEN 'direct'
|
|
64974
65782
|
WHEN lower(COALESCE(e.medium,'')) ~ 'email|newsletter' THEN 'email'
|
|
64975
65783
|
ELSE 'other' END)`;
|
|
65784
|
+
ANALYTICS_CONTENT_SORTS = [
|
|
65785
|
+
"path",
|
|
65786
|
+
"visitors",
|
|
65787
|
+
"sessions",
|
|
65788
|
+
"pageviews",
|
|
65789
|
+
"entrances",
|
|
65790
|
+
"exits",
|
|
65791
|
+
"engagement_events",
|
|
65792
|
+
"conversions",
|
|
65793
|
+
"avg_seconds_on_page",
|
|
65794
|
+
"avg_scroll_depth",
|
|
65795
|
+
"bounce_rate",
|
|
65796
|
+
"exit_rate",
|
|
65797
|
+
"engagement_rate",
|
|
65798
|
+
"pageviews_per_session"
|
|
65799
|
+
];
|
|
64976
65800
|
clickIdKeys = /* @__PURE__ */ new Set([
|
|
64977
65801
|
"fbclid",
|
|
64978
65802
|
"gclid",
|
|
@@ -65171,8 +65995,8 @@ function dashboardCallbackUrl() {
|
|
|
65171
65995
|
}
|
|
65172
65996
|
async function createThorbitConnectUrl(userId) {
|
|
65173
65997
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
65174
|
-
const state = (0,
|
|
65175
|
-
const stateHash = (0,
|
|
65998
|
+
const state = (0, import_node_crypto38.randomBytes)(32).toString("base64url");
|
|
65999
|
+
const stateHash = (0, import_node_crypto38.createHash)("sha256").update(state).digest("hex");
|
|
65176
66000
|
await getAnalyticsPool().query(
|
|
65177
66001
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
65178
66002
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -65187,9 +66011,9 @@ async function createThorbitConnectUrl(userId) {
|
|
|
65187
66011
|
function verifyThorbitAssertion(token4) {
|
|
65188
66012
|
const [encoded, signature] = token4.split(".");
|
|
65189
66013
|
if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
|
|
65190
|
-
const expected = (0,
|
|
66014
|
+
const expected = (0, import_node_crypto38.createHmac)("sha256", bridgeSecret()).update(encoded).digest();
|
|
65191
66015
|
const actual = Buffer.from(signature, "base64url");
|
|
65192
|
-
if (actual.length !== expected.length || !(0,
|
|
66016
|
+
if (actual.length !== expected.length || !(0, import_node_crypto38.timingSafeEqual)(actual, expected)) {
|
|
65193
66017
|
throw new Error("Invalid Thorbit assertion signature");
|
|
65194
66018
|
}
|
|
65195
66019
|
const parsed = AssertionSchema.safeParse(
|
|
@@ -65201,7 +66025,7 @@ function verifyThorbitAssertion(token4) {
|
|
|
65201
66025
|
return parsed.data.entitlement;
|
|
65202
66026
|
}
|
|
65203
66027
|
async function consumeThorbitConnectCallback(input) {
|
|
65204
|
-
const stateHash = (0,
|
|
66028
|
+
const stateHash = (0, import_node_crypto38.createHash)("sha256").update(input.state).digest("hex");
|
|
65205
66029
|
const client2 = await getAnalyticsPool().connect();
|
|
65206
66030
|
try {
|
|
65207
66031
|
await client2.query("BEGIN");
|
|
@@ -65241,11 +66065,11 @@ async function disconnectAnalyticsEntitlement(userId) {
|
|
|
65241
66065
|
[userId]
|
|
65242
66066
|
);
|
|
65243
66067
|
}
|
|
65244
|
-
var
|
|
66068
|
+
var import_node_crypto38, import_zod52, THORBIT_PRODUCT_URL, REFRESH_INTERVAL_MS, ELIGIBLE_GRACE_MS, TRIAL_LENGTH_MS, ThorbitEntitlementSchema, AssertionSchema;
|
|
65245
66069
|
var init_analytics_entitlement = __esm({
|
|
65246
66070
|
"src/api/analytics-entitlement.ts"() {
|
|
65247
66071
|
"use strict";
|
|
65248
|
-
|
|
66072
|
+
import_node_crypto38 = require("crypto");
|
|
65249
66073
|
import_zod52 = require("zod");
|
|
65250
66074
|
init_analytics_repository();
|
|
65251
66075
|
THORBIT_PRODUCT_URL = "https://thorbit.ai";
|
|
@@ -65292,7 +66116,10 @@ function dashboardOriginAllowed(origin) {
|
|
|
65292
66116
|
return (process.env.ALLOWED_ORIGINS ?? process.env.APP_ORIGIN ?? "").split(",").map((value) => value.trim()).includes(origin);
|
|
65293
66117
|
}
|
|
65294
66118
|
function analyticsPixelSdk() {
|
|
65295
|
-
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties}}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){current=location.href;api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
66119
|
+
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={},over={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties},...over}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};let visibleSince=document.visibilityState==='visible'?Date.now():0,dwellMs=0,maxScroll=0,dwellPath=location.pathname;const depth=()=>{const d=document.documentElement,h=d.scrollHeight||1;return Math.max(0,Math.min(100,Math.round(((d.scrollTop||document.body.scrollTop||0)+innerHeight)/h*100)))};const accrue=()=>{if(visibleSince){dwellMs+=Date.now()-visibleSince;visibleSince=0}};const resume=()=>{if(!visibleSince)visibleSince=Date.now()};const flush=()=>{accrue();const ms=Math.min(dwellMs,18e5),sd=Math.max(maxScroll,depth());dwellMs=0;maxScroll=0;if(ms<250)return;send('page_engagement',{},{path:dwellPath,canonicalUrl:location.origin+dwellPath,engagedMs:ms,scrollDepth:sd})};addEventListener('scroll',()=>{const d=depth();if(d>maxScroll)maxScroll=d},{passive:true});document.addEventListener('visibilitychange',()=>{if(document.visibilityState==='hidden')flush();else resume()});addEventListener('pagehide',flush);const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();resume();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,engagedMs:dwellMs+(visibleSince?Date.now()-visibleSince:0),scrollDepth:Math.max(maxScroll,depth()),...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){flush();current=location.href;dwellPath=location.pathname;resume();api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
66120
|
+
}
|
|
66121
|
+
function pagePathFilter(value) {
|
|
66122
|
+
return value && value.startsWith("/") ? value : void 0;
|
|
65296
66123
|
}
|
|
65297
66124
|
function reportFilters(url) {
|
|
65298
66125
|
const end = url.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -65314,7 +66141,7 @@ function reportFilters(url) {
|
|
|
65314
66141
|
channelFamily: url.searchParams.get("channelFamily") || void 0,
|
|
65315
66142
|
platform: url.searchParams.get("platform") || void 0,
|
|
65316
66143
|
referrer: url.searchParams.get("referrer") || void 0,
|
|
65317
|
-
path: url.searchParams.get("path")
|
|
66144
|
+
path: pagePathFilter(url.searchParams.get("path")),
|
|
65318
66145
|
deviceClass: url.searchParams.get("deviceClass") || void 0,
|
|
65319
66146
|
countryCode: url.searchParams.get("countryCode") || void 0,
|
|
65320
66147
|
regionCode: url.searchParams.get("regionCode") || void 0,
|
|
@@ -65382,14 +66209,14 @@ function renderPublicForm(form, placementUrl) {
|
|
|
65382
66209
|
const brand = form.brand;
|
|
65383
66210
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><style>:root{font-family:ui-sans-serif,system-ui;color:${escapeHtml7(brand.textColor)};background:${escapeHtml7(brand.backgroundColor)}}*{box-sizing:border-box}body{margin:0;padding:18px}form{display:grid;gap:14px}label{display:grid;gap:6px;font-size:13px;font-weight:650}.check{grid-template-columns:auto 1fr;align-items:center}.check span{grid-column:2}.check input{grid-column:1;grid-row:1}input,textarea,select{width:100%;min-height:44px;padding:10px 12px;border:1px solid #ccd3df;border-radius:${brand.radius}px;background:white;color:inherit;font:inherit}textarea{min-height:110px;resize:vertical}button{min-height:46px;border:0;border-radius:${brand.radius}px;color:white;background:${escapeHtml7(brand.primaryColor)};font:inherit;font-weight:750;cursor:pointer}.consent{margin:0;color:#667085;font-size:11px;line-height:1.45}.notice{display:none;padding:12px;border-radius:${brand.radius}px;background:#edf8f2;color:#17613c}.hp{position:absolute!important;left:-9999px!important}</style></head><body><form id="mcp-form">${fields}<label class="hp">Website<input name="website" autocomplete="off" tabindex="-1"></label>${form.consent_text ? `<p class="consent">${escapeHtml7(form.consent_text)}</p>` : ""}<button>${escapeHtml7(form.submit_label)}</button><div class="notice" role="status"></div></form><script>(()=>{const form=document.querySelector('#mcp-form'),notice=form.querySelector('.notice'),q=new URLSearchParams(location.search);form.addEventListener('submit',async event=>{event.preventDefault();const button=form.querySelector('button');button.disabled=true;const raw=Object.fromEntries(new FormData(form).entries()),website=String(raw.website||''),clickIds={};delete raw.website;for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}try{const response=await fetch('/analytics/forms/${escapeHtml7(form.public_id)}/submissions',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({data:raw,website,placementUrl:${JSON.stringify(placementUrl)},visitorId:q.get('visitor')||undefined,sessionId:q.get('session')||undefined,source:q.get('source')||undefined,medium:q.get('medium')||undefined,campaign:q.get('campaign')||undefined,clickIds})});if(!response.ok)throw new Error('submit');notice.textContent=${JSON.stringify(form.success_message)};notice.style.display='block';form.reset();${form.redirect_url ? `setTimeout(()=>{top.location.href=${JSON.stringify(form.redirect_url)}},700);` : ""}}catch{notice.textContent='Your response could not be submitted. Please try again.';notice.style.display='block'}finally{button.disabled=false}})})()</script></body></html>`;
|
|
65384
66211
|
}
|
|
65385
|
-
var import_hono32, import_factory6, import_zod53,
|
|
66212
|
+
var import_hono32, import_factory6, import_zod53, import_node_crypto39, import_papaparse4, analyticsApp, auth3, entitlementGuard, ThorbitApiKeySchema, ThorbitCallbackSchema, SiteInputSchema, PixelInputSchema, PixelUpdateSchema, EventSchema, IngestionSchema, ConversionSchema, ExportSchema2, CampaignLinkSchema, FormFieldSchema, FormInputSchema, FormSubmissionSchema, CrmImportSchema, ActivationDestinationSchema, BusinessModelSchema, AdSpendSchema;
|
|
65386
66213
|
var init_analytics_routes = __esm({
|
|
65387
66214
|
"src/api/analytics-routes.ts"() {
|
|
65388
66215
|
"use strict";
|
|
65389
66216
|
import_hono32 = require("hono");
|
|
65390
66217
|
import_factory6 = require("hono/factory");
|
|
65391
66218
|
import_zod53 = require("zod");
|
|
65392
|
-
|
|
66219
|
+
import_node_crypto39 = require("crypto");
|
|
65393
66220
|
import_papaparse4 = __toESM(require("papaparse"), 1);
|
|
65394
66221
|
init_api_auth();
|
|
65395
66222
|
init_db();
|
|
@@ -65450,7 +66277,9 @@ var init_analytics_routes = __esm({
|
|
|
65450
66277
|
campaign: import_zod53.z.string().trim().max(240).optional(),
|
|
65451
66278
|
clickIds: import_zod53.z.record(import_zod53.z.string(), import_zod53.z.string().trim().max(500)).optional(),
|
|
65452
66279
|
deviceClass: import_zod53.z.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
|
|
65453
|
-
properties: import_zod53.z.record(import_zod53.z.string(), import_zod53.z.unknown()).optional()
|
|
66280
|
+
properties: import_zod53.z.record(import_zod53.z.string(), import_zod53.z.unknown()).optional(),
|
|
66281
|
+
engagedMs: import_zod53.z.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
|
|
66282
|
+
scrollDepth: import_zod53.z.number().int().min(0).max(100).optional()
|
|
65454
66283
|
}).strict();
|
|
65455
66284
|
IngestionSchema = import_zod53.z.object({
|
|
65456
66285
|
pixelId: import_zod53.z.string().trim().min(8).max(160),
|
|
@@ -65718,7 +66547,7 @@ var init_analytics_routes = __esm({
|
|
|
65718
66547
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
65719
66548
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
65720
66549
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
65721
|
-
const suffix2 = (0,
|
|
66550
|
+
const suffix2 = (0, import_node_crypto39.createHash)("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
65722
66551
|
const path6 = `Leads/person-${suffix2}`;
|
|
65723
66552
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
65724
66553
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path6 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -65817,9 +66646,9 @@ Submitted the published form.
|
|
|
65817
66646
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
65818
66647
|
});
|
|
65819
66648
|
const match = {
|
|
65820
|
-
...email ? { emailSha256: (0,
|
|
66649
|
+
...email ? { emailSha256: (0, import_node_crypto39.createHash)("sha256").update(email).digest("hex") } : {},
|
|
65821
66650
|
...phone ? {
|
|
65822
|
-
phoneSha256: (0,
|
|
66651
|
+
phoneSha256: (0, import_node_crypto39.createHash)("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
65823
66652
|
} : {}
|
|
65824
66653
|
};
|
|
65825
66654
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -66247,7 +67076,7 @@ Submitted the published form.
|
|
|
66247
67076
|
rejectedCount += 1;
|
|
66248
67077
|
continue;
|
|
66249
67078
|
}
|
|
66250
|
-
const digest2 = (0,
|
|
67079
|
+
const digest2 = (0, import_node_crypto39.createHash)("sha256").update(
|
|
66251
67080
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
66252
67081
|
).digest("hex").slice(0, 16);
|
|
66253
67082
|
const path6 = `Leads/person-${digest2}`;
|
|
@@ -66450,14 +67279,13 @@ Imported through the Site Analytics CRM workflow.
|
|
|
66450
67279
|
["/sites/:siteId/overview", analyticsOverview],
|
|
66451
67280
|
["/sites/:siteId/timeseries", analyticsTimeseries],
|
|
66452
67281
|
["/sites/:siteId/acquisition", analyticsAcquisition],
|
|
66453
|
-
["/sites/:siteId/content", analyticsContent],
|
|
66454
67282
|
["/sites/:siteId/events", analyticsEventCounts]
|
|
66455
67283
|
]) {
|
|
66456
67284
|
analyticsApp.get(path6, auth3, async (c) => {
|
|
66457
67285
|
const url = new URL(c.req.url);
|
|
66458
67286
|
const filters = reportFilters(url);
|
|
66459
67287
|
validateReportWindow(filters);
|
|
66460
|
-
const paginated = path6.endsWith("/acquisition") || path6.endsWith("/
|
|
67288
|
+
const paginated = path6.endsWith("/acquisition") || path6.endsWith("/events");
|
|
66461
67289
|
const data = await load(
|
|
66462
67290
|
c.req.param("siteId"),
|
|
66463
67291
|
Number(c.get("user").id),
|
|
@@ -66467,6 +67295,24 @@ Imported through the Site Analytics CRM workflow.
|
|
|
66467
67295
|
return c.json({ ok: true, data, filters });
|
|
66468
67296
|
});
|
|
66469
67297
|
}
|
|
67298
|
+
analyticsApp.get("/sites/:siteId/content", auth3, async (c) => {
|
|
67299
|
+
const url = new URL(c.req.url);
|
|
67300
|
+
const filters = reportFilters(url);
|
|
67301
|
+
validateReportWindow(filters);
|
|
67302
|
+
const requestedSort = url.searchParams.get("sort");
|
|
67303
|
+
const data = await analyticsContent(
|
|
67304
|
+
c.req.param("siteId"),
|
|
67305
|
+
Number(c.get("user").id),
|
|
67306
|
+
filters,
|
|
67307
|
+
reportPage(url),
|
|
67308
|
+
{
|
|
67309
|
+
sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
|
|
67310
|
+
direction: url.searchParams.get("direction") === "asc" ? "asc" : "desc",
|
|
67311
|
+
search: url.searchParams.get("search") || void 0
|
|
67312
|
+
}
|
|
67313
|
+
);
|
|
67314
|
+
return c.json({ ok: true, data, filters });
|
|
67315
|
+
});
|
|
66470
67316
|
analyticsApp.get("/sites/:siteId/dashboard", auth3, async (c) => {
|
|
66471
67317
|
const url = new URL(c.req.url);
|
|
66472
67318
|
const filters = reportFilters(url);
|
|
@@ -66688,19 +67534,19 @@ var init_analytics_delivery = __esm({
|
|
|
66688
67534
|
|
|
66689
67535
|
// src/api/scheduled-artifact-owner.ts
|
|
66690
67536
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
66691
|
-
return (0,
|
|
67537
|
+
return (0, import_node_crypto40.createHash)("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
66692
67538
|
}
|
|
66693
|
-
var
|
|
67539
|
+
var import_node_crypto40;
|
|
66694
67540
|
var init_scheduled_artifact_owner = __esm({
|
|
66695
67541
|
"src/api/scheduled-artifact-owner.ts"() {
|
|
66696
67542
|
"use strict";
|
|
66697
|
-
|
|
67543
|
+
import_node_crypto40 = require("crypto");
|
|
66698
67544
|
}
|
|
66699
67545
|
});
|
|
66700
67546
|
|
|
66701
67547
|
// src/api/scheduled-run-view-links.ts
|
|
66702
67548
|
async function ensureScheduledRunViewLinksSchema() {
|
|
66703
|
-
if (
|
|
67549
|
+
if (schemaReady2) return;
|
|
66704
67550
|
const db = getDb();
|
|
66705
67551
|
await db.execute(`
|
|
66706
67552
|
CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
|
|
@@ -66748,10 +67594,10 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
66748
67594
|
CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
|
|
66749
67595
|
ON scheduled_run_view_links(expires_at, revoked_at)
|
|
66750
67596
|
`);
|
|
66751
|
-
|
|
67597
|
+
schemaReady2 = true;
|
|
66752
67598
|
}
|
|
66753
67599
|
function tokenHash2(token4) {
|
|
66754
|
-
return (0,
|
|
67600
|
+
return (0, import_node_crypto41.createHash)("sha256").update(token4).digest("hex");
|
|
66755
67601
|
}
|
|
66756
67602
|
function mapRow(row) {
|
|
66757
67603
|
return {
|
|
@@ -66771,9 +67617,9 @@ function mapRow(row) {
|
|
|
66771
67617
|
async function createScheduledRunViewLink(input) {
|
|
66772
67618
|
await ensureScheduledRunViewLinksSchema();
|
|
66773
67619
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
66774
|
-
const token4 = (0,
|
|
67620
|
+
const token4 = (0, import_node_crypto41.randomBytes)(32).toString("base64url");
|
|
66775
67621
|
const record = {
|
|
66776
|
-
shareId: (0,
|
|
67622
|
+
shareId: (0, import_node_crypto41.randomUUID)(),
|
|
66777
67623
|
ownerId: input.ownerId,
|
|
66778
67624
|
runId: input.runId,
|
|
66779
67625
|
artifactId: input.artifactId,
|
|
@@ -66836,13 +67682,13 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
|
|
|
66836
67682
|
});
|
|
66837
67683
|
return result.rowsAffected > 0;
|
|
66838
67684
|
}
|
|
66839
|
-
var
|
|
67685
|
+
var import_node_crypto41, schemaReady2;
|
|
66840
67686
|
var init_scheduled_run_view_links = __esm({
|
|
66841
67687
|
"src/api/scheduled-run-view-links.ts"() {
|
|
66842
67688
|
"use strict";
|
|
66843
|
-
|
|
67689
|
+
import_node_crypto41 = require("crypto");
|
|
66844
67690
|
init_db();
|
|
66845
|
-
|
|
67691
|
+
schemaReady2 = false;
|
|
66846
67692
|
}
|
|
66847
67693
|
});
|
|
66848
67694
|
|
|
@@ -66906,15 +67752,15 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
66906
67752
|
html,
|
|
66907
67753
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
66908
67754
|
bytes,
|
|
66909
|
-
sha256: (0,
|
|
67755
|
+
sha256: (0, import_node_crypto42.createHash)("sha256").update(html).digest("hex"),
|
|
66910
67756
|
generatedAt: generatedAtIso
|
|
66911
67757
|
};
|
|
66912
67758
|
}
|
|
66913
|
-
var
|
|
67759
|
+
var import_node_crypto42;
|
|
66914
67760
|
var init_render2 = __esm({
|
|
66915
67761
|
"src/personal-authority/render.ts"() {
|
|
66916
67762
|
"use strict";
|
|
66917
|
-
|
|
67763
|
+
import_node_crypto42 = require("crypto");
|
|
66918
67764
|
}
|
|
66919
67765
|
});
|
|
66920
67766
|
|
|
@@ -67024,15 +67870,15 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
67024
67870
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
67025
67871
|
</style></head><body data-theme="${escapeHtml9(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
67026
67872
|
const bytes = Buffer.byteLength(html);
|
|
67027
|
-
const sha2563 = (0,
|
|
67873
|
+
const sha2563 = (0, import_node_crypto43.createHash)("sha256").update(html).digest("hex");
|
|
67028
67874
|
const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
67029
67875
|
return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
|
|
67030
67876
|
}
|
|
67031
|
-
var
|
|
67877
|
+
var import_node_crypto43;
|
|
67032
67878
|
var init_render3 = __esm({
|
|
67033
67879
|
"src/newsroom-publisher/render.ts"() {
|
|
67034
67880
|
"use strict";
|
|
67035
|
-
|
|
67881
|
+
import_node_crypto43 = require("crypto");
|
|
67036
67882
|
}
|
|
67037
67883
|
});
|
|
67038
67884
|
|
|
@@ -67325,7 +68171,7 @@ function policy4() {
|
|
|
67325
68171
|
};
|
|
67326
68172
|
}
|
|
67327
68173
|
function runStorageSegment(runId) {
|
|
67328
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0,
|
|
68174
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${(0, import_node_crypto44.createHash)("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
67329
68175
|
}
|
|
67330
68176
|
async function createScheduledRunArtifact(args) {
|
|
67331
68177
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -67366,12 +68212,12 @@ async function readScheduledRunArtifact(args) {
|
|
|
67366
68212
|
if (!window2 || window2.nextOffset !== null || window2.totalBytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) return null;
|
|
67367
68213
|
return window2.text;
|
|
67368
68214
|
}
|
|
67369
|
-
var
|
|
68215
|
+
var import_node_crypto44, SCHEDULED_RUN_ARTIFACT_PREFIX, SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS, SCHEDULED_RUN_ARTIFACT_MAX_BYTES;
|
|
67370
68216
|
var init_scheduled_run_artifact_store = __esm({
|
|
67371
68217
|
"src/scheduled-artifacts/scheduled-run-artifact-store.ts"() {
|
|
67372
68218
|
"use strict";
|
|
67373
68219
|
init_private_artifacts();
|
|
67374
|
-
|
|
68220
|
+
import_node_crypto44 = require("crypto");
|
|
67375
68221
|
SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
67376
68222
|
SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
67377
68223
|
SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -67452,9 +68298,9 @@ var init_public_scheduled_result_routes = __esm({
|
|
|
67452
68298
|
// src/api/service-connections.ts
|
|
67453
68299
|
function ensureServiceConnectionsSchema() {
|
|
67454
68300
|
const currentDb = getDb();
|
|
67455
|
-
if (
|
|
68301
|
+
if (schemaReady3 && schemaDb4 === currentDb) return schemaReady3;
|
|
67456
68302
|
schemaDb4 = currentDb;
|
|
67457
|
-
|
|
68303
|
+
schemaReady3 = (async () => {
|
|
67458
68304
|
const db = currentDb;
|
|
67459
68305
|
await db.execute(`
|
|
67460
68306
|
CREATE TABLE IF NOT EXISTS service_connections (
|
|
@@ -67522,11 +68368,11 @@ function ensureServiceConnectionsSchema() {
|
|
|
67522
68368
|
} catch {
|
|
67523
68369
|
}
|
|
67524
68370
|
})().catch((error) => {
|
|
67525
|
-
|
|
68371
|
+
schemaReady3 = null;
|
|
67526
68372
|
schemaDb4 = null;
|
|
67527
68373
|
throw error;
|
|
67528
68374
|
});
|
|
67529
|
-
return
|
|
68375
|
+
return schemaReady3;
|
|
67530
68376
|
}
|
|
67531
68377
|
function parseTools(value) {
|
|
67532
68378
|
if (typeof value !== "string") return [];
|
|
@@ -67589,7 +68435,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
67589
68435
|
updated_at = excluded.updated_at
|
|
67590
68436
|
`,
|
|
67591
68437
|
args: [
|
|
67592
|
-
(0,
|
|
68438
|
+
(0, import_node_crypto45.randomUUID)(),
|
|
67593
68439
|
userId,
|
|
67594
68440
|
connection.providerConfigKey,
|
|
67595
68441
|
connection.provider,
|
|
@@ -67660,7 +68506,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
67660
68506
|
});
|
|
67661
68507
|
await getDb().execute({
|
|
67662
68508
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
67663
|
-
args: [(0,
|
|
68509
|
+
args: [(0, import_node_crypto45.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
67664
68510
|
});
|
|
67665
68511
|
}
|
|
67666
68512
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -67700,7 +68546,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
67700
68546
|
if (!connection) throw new Error("service_connection_not_found");
|
|
67701
68547
|
const inserted = await getDb().execute({
|
|
67702
68548
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
67703
|
-
args: [(0,
|
|
68549
|
+
args: [(0, import_node_crypto45.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
67704
68550
|
});
|
|
67705
68551
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
67706
68552
|
const existing = await getDb().execute({
|
|
@@ -67725,13 +68571,13 @@ async function claimServiceConnectionAction(args) {
|
|
|
67725
68571
|
...result !== void 0 ? { result } : {}
|
|
67726
68572
|
};
|
|
67727
68573
|
}
|
|
67728
|
-
var
|
|
68574
|
+
var import_node_crypto45, schemaReady3, schemaDb4;
|
|
67729
68575
|
var init_service_connections = __esm({
|
|
67730
68576
|
"src/api/service-connections.ts"() {
|
|
67731
68577
|
"use strict";
|
|
67732
|
-
|
|
68578
|
+
import_node_crypto45 = require("crypto");
|
|
67733
68579
|
init_db();
|
|
67734
|
-
|
|
68580
|
+
schemaReady3 = null;
|
|
67735
68581
|
schemaDb4 = null;
|
|
67736
68582
|
}
|
|
67737
68583
|
});
|
|
@@ -67743,8 +68589,8 @@ function signingSecret() {
|
|
|
67743
68589
|
return secret2;
|
|
67744
68590
|
}
|
|
67745
68591
|
function schedulerIntegrationSignature(args) {
|
|
67746
|
-
const bodyHash = (0,
|
|
67747
|
-
return (0,
|
|
68592
|
+
const bodyHash = (0, import_node_crypto46.createHash)("sha256").update(args.body).digest("hex");
|
|
68593
|
+
return (0, import_node_crypto46.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
67748
68594
|
${args.path}
|
|
67749
68595
|
${args.timestamp}
|
|
67750
68596
|
${args.nonce}
|
|
@@ -67791,17 +68637,17 @@ async function verifySchedulerIntegrationRequest(request, rawBody) {
|
|
|
67791
68637
|
});
|
|
67792
68638
|
const suppliedBytes = Buffer.from(signature, "hex");
|
|
67793
68639
|
const expectedBytes = Buffer.from(expected, "hex");
|
|
67794
|
-
if (suppliedBytes.length !== expectedBytes.length || !(0,
|
|
68640
|
+
if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto46.timingSafeEqual)(suppliedBytes, expectedBytes)) {
|
|
67795
68641
|
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
67796
68642
|
}
|
|
67797
68643
|
await claimNonce(nonce, timestampMs);
|
|
67798
68644
|
return { requestId };
|
|
67799
68645
|
}
|
|
67800
|
-
var
|
|
68646
|
+
var import_node_crypto46, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
|
|
67801
68647
|
var init_scheduler_integration_auth = __esm({
|
|
67802
68648
|
"src/api/scheduler-integration-auth.ts"() {
|
|
67803
68649
|
"use strict";
|
|
67804
|
-
|
|
68650
|
+
import_node_crypto46 = require("crypto");
|
|
67805
68651
|
init_db();
|
|
67806
68652
|
init_service_connections();
|
|
67807
68653
|
MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
@@ -68049,7 +68895,7 @@ var init_site_extract_reconciliation = __esm({
|
|
|
68049
68895
|
|
|
68050
68896
|
// src/api/page-diff.ts
|
|
68051
68897
|
function sha256Hex(value) {
|
|
68052
|
-
return (0,
|
|
68898
|
+
return (0, import_node_crypto47.createHash)("sha256").update(value).digest("hex");
|
|
68053
68899
|
}
|
|
68054
68900
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
68055
68901
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -68106,11 +68952,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
68106
68952
|
totalChangedLineCount
|
|
68107
68953
|
};
|
|
68108
68954
|
}
|
|
68109
|
-
var
|
|
68955
|
+
var import_node_crypto47, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
68110
68956
|
var init_page_diff = __esm({
|
|
68111
68957
|
"src/api/page-diff.ts"() {
|
|
68112
68958
|
"use strict";
|
|
68113
|
-
|
|
68959
|
+
import_node_crypto47 = require("crypto");
|
|
68114
68960
|
import_diff = require("diff");
|
|
68115
68961
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
68116
68962
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -68184,7 +69030,7 @@ var init_scrape_vault_sink = __esm({
|
|
|
68184
69030
|
|
|
68185
69031
|
// src/api/scrape-image-sink.ts
|
|
68186
69032
|
function idempotencyKey2(userId, vault, input) {
|
|
68187
|
-
return `scrape-image-${(0,
|
|
69033
|
+
return `scrape-image-${(0, import_node_crypto48.createHash)("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
68188
69034
|
}
|
|
68189
69035
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
68190
69036
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -68226,11 +69072,11 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
|
68226
69072
|
assets
|
|
68227
69073
|
};
|
|
68228
69074
|
}
|
|
68229
|
-
var
|
|
69075
|
+
var import_node_crypto48, MAX_IMAGES_PER_SCRAPE;
|
|
68230
69076
|
var init_scrape_image_sink = __esm({
|
|
68231
69077
|
"src/api/scrape-image-sink.ts"() {
|
|
68232
69078
|
"use strict";
|
|
68233
|
-
|
|
69079
|
+
import_node_crypto48 = require("crypto");
|
|
68234
69080
|
init_memory();
|
|
68235
69081
|
MAX_IMAGES_PER_SCRAPE = 25;
|
|
68236
69082
|
}
|
|
@@ -68683,7 +69529,7 @@ function canonicalJson(value) {
|
|
|
68683
69529
|
return JSON.stringify(value);
|
|
68684
69530
|
}
|
|
68685
69531
|
function sha2562(value) {
|
|
68686
|
-
return (0,
|
|
69532
|
+
return (0, import_node_crypto49.createHash)("sha256").update(value).digest("hex");
|
|
68687
69533
|
}
|
|
68688
69534
|
function sensitiveKey(key) {
|
|
68689
69535
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -68898,11 +69744,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
68898
69744
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
68899
69745
|
};
|
|
68900
69746
|
}
|
|
68901
|
-
var
|
|
69747
|
+
var import_node_crypto49, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
68902
69748
|
var init_connection_memory_import = __esm({
|
|
68903
69749
|
"src/api/connection-memory-import.ts"() {
|
|
68904
69750
|
"use strict";
|
|
68905
|
-
|
|
69751
|
+
import_node_crypto49 = require("crypto");
|
|
68906
69752
|
init_slugify();
|
|
68907
69753
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
68908
69754
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -69120,7 +69966,7 @@ function finitePositive(value, fallback) {
|
|
|
69120
69966
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
69121
69967
|
}
|
|
69122
69968
|
async function collectConnectedDataExport(args) {
|
|
69123
|
-
const exportId = (0,
|
|
69969
|
+
const exportId = (0, import_node_crypto50.randomUUID)();
|
|
69124
69970
|
const now = args.now ?? Date.now;
|
|
69125
69971
|
const startedAt = now();
|
|
69126
69972
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -69234,11 +70080,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
69234
70080
|
untrustedContent: true
|
|
69235
70081
|
};
|
|
69236
70082
|
}
|
|
69237
|
-
var
|
|
70083
|
+
var import_node_crypto50, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
69238
70084
|
var init_connected_data_export = __esm({
|
|
69239
70085
|
"src/api/connected-data-export.ts"() {
|
|
69240
70086
|
"use strict";
|
|
69241
|
-
|
|
70087
|
+
import_node_crypto50 = require("crypto");
|
|
69242
70088
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
69243
70089
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
69244
70090
|
);
|
|
@@ -69352,7 +70198,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
69352
70198
|
offset += rows.length;
|
|
69353
70199
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
69354
70200
|
}
|
|
69355
|
-
const exportId = (0,
|
|
70201
|
+
const exportId = (0, import_node_crypto51.randomUUID)();
|
|
69356
70202
|
const artifact = await args.writeArtifact({
|
|
69357
70203
|
ownerId: args.ownerId,
|
|
69358
70204
|
exportId,
|
|
@@ -69374,11 +70220,11 @@ async function exportSearchConsoleTableData(args) {
|
|
|
69374
70220
|
warnings
|
|
69375
70221
|
};
|
|
69376
70222
|
}
|
|
69377
|
-
var
|
|
70223
|
+
var import_node_crypto51, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
|
|
69378
70224
|
var init_search_console_table_export = __esm({
|
|
69379
70225
|
"src/api/search-console-table-export.ts"() {
|
|
69380
70226
|
"use strict";
|
|
69381
|
-
|
|
70227
|
+
import_node_crypto51 = require("crypto");
|
|
69382
70228
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
69383
70229
|
SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
69384
70230
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -70557,7 +71403,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
70557
71403
|
});
|
|
70558
71404
|
const readTools = [...policies.values()].filter((policy5) => policy5.classification === "read").map((policy5) => policy5.name);
|
|
70559
71405
|
const actionTools = [...policies.values()].filter((policy5) => policy5.classification === "action").map((policy5) => policy5.name);
|
|
70560
|
-
const revision = (0,
|
|
71406
|
+
const revision = (0, import_node_crypto52.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
70561
71407
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
70562
71408
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
70563
71409
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -70584,8 +71430,8 @@ async function callNangoToolDirect(args) {
|
|
|
70584
71430
|
identity: args.identity,
|
|
70585
71431
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
70586
71432
|
});
|
|
70587
|
-
const requestId = args.requestId?.trim() || (0,
|
|
70588
|
-
const idempotencyKey3 = `main-nango:${(0,
|
|
71433
|
+
const requestId = args.requestId?.trim() || (0, import_node_crypto52.randomUUID)();
|
|
71434
|
+
const idempotencyKey3 = `main-nango:${(0, import_node_crypto52.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
70589
71435
|
const startedAt = /* @__PURE__ */ new Date();
|
|
70590
71436
|
const started = performance.now();
|
|
70591
71437
|
let result;
|
|
@@ -70610,7 +71456,7 @@ async function callNangoToolDirect(args) {
|
|
|
70610
71456
|
toolName: args.tool,
|
|
70611
71457
|
operationKind: args.operationKind ?? args.classification,
|
|
70612
71458
|
outcome: providerError ? "error" : "partial",
|
|
70613
|
-
requestId: requestId.length <= 200 ? requestId : (0,
|
|
71459
|
+
requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto52.createHash)("sha256").update(requestId).digest("hex"),
|
|
70614
71460
|
startedAt: startedAt.toISOString(),
|
|
70615
71461
|
completedAt: completedAt.toISOString()
|
|
70616
71462
|
}
|
|
@@ -70652,15 +71498,15 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
70652
71498
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
70653
71499
|
protocolVersion: null,
|
|
70654
71500
|
schemaSource: "live_tools_list",
|
|
70655
|
-
schemaHash: (0,
|
|
71501
|
+
schemaHash: (0, import_node_crypto52.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
70656
71502
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
70657
71503
|
};
|
|
70658
71504
|
}
|
|
70659
|
-
var
|
|
71505
|
+
var import_node_crypto52, import_client13, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
|
|
70660
71506
|
var init_main_nango_transport = __esm({
|
|
70661
71507
|
"src/api/main-nango-transport.ts"() {
|
|
70662
71508
|
"use strict";
|
|
70663
|
-
|
|
71509
|
+
import_node_crypto52 = require("crypto");
|
|
70664
71510
|
import_client13 = require("@modelcontextprotocol/client");
|
|
70665
71511
|
init_service_connections();
|
|
70666
71512
|
init_connected_usage_billing();
|
|
@@ -71284,8 +72130,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
71284
72130
|
return data.connection.actionsEnabled === true;
|
|
71285
72131
|
}
|
|
71286
72132
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
71287
|
-
const requestId = `main-connected-action:${(0,
|
|
71288
|
-
const requestDigest = (0,
|
|
72133
|
+
const requestId = `main-connected-action:${(0, import_node_crypto53.createHash)("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || (0, import_node_crypto53.randomUUID)()).digest("hex")}`;
|
|
72134
|
+
const requestDigest = (0, import_node_crypto53.createHash)("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
71289
72135
|
if (mainOwnsIntegrations()) {
|
|
71290
72136
|
const selectedTool = tool?.trim();
|
|
71291
72137
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -71436,7 +72282,7 @@ function canonicalJson2(value) {
|
|
|
71436
72282
|
return JSON.stringify(value);
|
|
71437
72283
|
}
|
|
71438
72284
|
function projectedToolSchemaHash(tool) {
|
|
71439
|
-
return (0,
|
|
72285
|
+
return (0, import_node_crypto53.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
71440
72286
|
}
|
|
71441
72287
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
71442
72288
|
if (mainOwnsIntegrations()) {
|
|
@@ -71713,11 +72559,11 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
71713
72559
|
untrustedContent: true
|
|
71714
72560
|
};
|
|
71715
72561
|
}
|
|
71716
|
-
var
|
|
72562
|
+
var import_node_crypto53, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
|
|
71717
72563
|
var init_nango_control = __esm({
|
|
71718
72564
|
"src/api/nango-control.ts"() {
|
|
71719
72565
|
"use strict";
|
|
71720
|
-
|
|
72566
|
+
import_node_crypto53 = require("crypto");
|
|
71721
72567
|
init_connected_data_export();
|
|
71722
72568
|
init_slack_connected_data_export();
|
|
71723
72569
|
init_main_nango_transport();
|
|
@@ -72073,7 +72919,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
72073
72919
|
return isRecord5(data) ? data.result ?? data : data;
|
|
72074
72920
|
}
|
|
72075
72921
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
72076
|
-
const requestId = `main-resend-action:${(0,
|
|
72922
|
+
const requestId = `main-resend-action:${(0, import_node_crypto54.createHash)("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
72077
72923
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
72078
72924
|
method: "POST",
|
|
72079
72925
|
headers: { "x-request-id": requestId },
|
|
@@ -72138,12 +72984,12 @@ async function callResendExportPage(identity, input) {
|
|
|
72138
72984
|
untrustedContent: true
|
|
72139
72985
|
};
|
|
72140
72986
|
}
|
|
72141
|
-
var
|
|
72987
|
+
var import_node_crypto54, DEFAULT_CONNECTION_CONTROL_URL, RESEND_PROVIDER_CONFIG_KEY, RESEND_LOGO_URL, RESEND_DOCS_URL, RESEND_ADMIN_BLOCKED_TOOLS, RESEND_CONNECTION_SYNC_REQUIRED_TOOLS, ResendControlError;
|
|
72142
72988
|
var init_resend_control = __esm({
|
|
72143
72989
|
"src/api/resend-control.ts"() {
|
|
72144
72990
|
"use strict";
|
|
72145
72991
|
init_connected_data_export();
|
|
72146
|
-
|
|
72992
|
+
import_node_crypto54 = require("crypto");
|
|
72147
72993
|
DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
72148
72994
|
RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
72149
72995
|
RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -72661,7 +73507,31 @@ async function chargeTierChangeNow(stripeClient, subscriptionId, customerId) {
|
|
|
72661
73507
|
return { ok: false, amountDue: 0, error: err instanceof Error ? err.message : "Unable to charge the plan change immediately." };
|
|
72662
73508
|
}
|
|
72663
73509
|
}
|
|
72664
|
-
|
|
73510
|
+
function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
|
|
73511
|
+
return new Promise((resolve2) => {
|
|
73512
|
+
const timer = setTimeout(() => {
|
|
73513
|
+
unfinished.push(`${label}:deadline`);
|
|
73514
|
+
resolve2(onDeadlineOrError);
|
|
73515
|
+
}, CRON_TICK_BUDGET_MS);
|
|
73516
|
+
work.then(
|
|
73517
|
+
(value) => {
|
|
73518
|
+
clearTimeout(timer);
|
|
73519
|
+
resolve2(value);
|
|
73520
|
+
},
|
|
73521
|
+
(err) => {
|
|
73522
|
+
clearTimeout(timer);
|
|
73523
|
+
unfinished.push(`${label}:error`);
|
|
73524
|
+
console.error(JSON.stringify({
|
|
73525
|
+
event: "cron_tick_job_failed",
|
|
73526
|
+
job: label,
|
|
73527
|
+
message: err instanceof Error ? err.message : String(err)
|
|
73528
|
+
}));
|
|
73529
|
+
resolve2(onDeadlineOrError);
|
|
73530
|
+
}
|
|
73531
|
+
);
|
|
73532
|
+
});
|
|
73533
|
+
}
|
|
73534
|
+
var import_resend3, import_node_crypto55, import_hono36, import_hono37, import_factory8, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth4, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, deploymentProfile, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS, CRON_TICK_BUDGET_MS, CRON_TICK_DRAIN_BUDGET_MS;
|
|
72665
73535
|
var init_server = __esm({
|
|
72666
73536
|
"src/api/server.ts"() {
|
|
72667
73537
|
"use strict";
|
|
@@ -72674,7 +73544,7 @@ var init_server = __esm({
|
|
|
72674
73544
|
init_og();
|
|
72675
73545
|
import_resend3 = require("resend");
|
|
72676
73546
|
init_url_utils();
|
|
72677
|
-
|
|
73547
|
+
import_node_crypto55 = require("crypto");
|
|
72678
73548
|
init_kpo_extractor();
|
|
72679
73549
|
init_screenshot();
|
|
72680
73550
|
init_media_extractor();
|
|
@@ -72715,6 +73585,8 @@ var init_server = __esm({
|
|
|
72715
73585
|
init_directory_workflow_dispatch();
|
|
72716
73586
|
init_paa_harvest_dispatch();
|
|
72717
73587
|
init_paa_harvest_reconciliation();
|
|
73588
|
+
init_unified_billing();
|
|
73589
|
+
init_retention_sweeps();
|
|
72718
73590
|
init_directory_workflow_reconciliation();
|
|
72719
73591
|
init_directory_artifacts();
|
|
72720
73592
|
init_workflow_routes();
|
|
@@ -74209,7 +75081,7 @@ var init_server = __esm({
|
|
|
74209
75081
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
74210
75082
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
74211
75083
|
} else {
|
|
74212
|
-
jobId2 = (0,
|
|
75084
|
+
jobId2 = (0, import_node_crypto55.randomUUID)();
|
|
74213
75085
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
74214
75086
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
74215
75087
|
const hold = await debitMcIdempotent(
|
|
@@ -75406,35 +76278,49 @@ var init_server = __esm({
|
|
|
75406
76278
|
ledger
|
|
75407
76279
|
});
|
|
75408
76280
|
});
|
|
76281
|
+
CRON_TICK_BUDGET_MS = 5e4;
|
|
76282
|
+
CRON_TICK_DRAIN_BUDGET_MS = 4e4;
|
|
75409
76283
|
app.get("/cron/tick", async (c) => {
|
|
75410
76284
|
const secret2 = c.req.header("authorization");
|
|
75411
76285
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
75412
76286
|
return c.json({ error: "Unauthorized" }, 401);
|
|
75413
76287
|
}
|
|
76288
|
+
const startedAt = Date.now();
|
|
75414
76289
|
const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
|
|
75415
|
-
const budget = { maxJobs: 10, deadlineMs:
|
|
75416
|
-
const
|
|
75417
|
-
const
|
|
75418
|
-
|
|
75419
|
-
|
|
75420
|
-
|
|
75421
|
-
|
|
75422
|
-
|
|
75423
|
-
|
|
75424
|
-
|
|
75425
|
-
|
|
75426
|
-
|
|
75427
|
-
|
|
75428
|
-
|
|
75429
|
-
|
|
75430
|
-
|
|
75431
|
-
|
|
75432
|
-
|
|
75433
|
-
|
|
75434
|
-
|
|
75435
|
-
|
|
76290
|
+
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
76291
|
+
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
76292
|
+
const unfinished = [];
|
|
76293
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
76294
|
+
settleWithinTickBudget("drainQueue", unfinished, drainQueue2(budget), []),
|
|
76295
|
+
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
76296
|
+
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
76297
|
+
settleWithinTickBudget("retryOrphanedBrowserSessions", unfinished, retryOrphanedBrowserSessions(50), { recovered: 0, stillOpen: 0 }),
|
|
76298
|
+
settleWithinTickBudget("expireLots", unfinished, expireOldLots(), { expired_lots: 0, expired_mc: 0 }),
|
|
76299
|
+
settleWithinTickBudget("scrapeBlobCleanup", unfinished, cleanupExpiredScrapeBlobs(), { deleted: 0, store: "none" }),
|
|
76300
|
+
settleWithinTickBudget("connectedDataCleanup", unfinished, cleanupExpiredConnectedDataArtifacts(), { deleted: 0, store: "none" }),
|
|
76301
|
+
settleWithinTickBudget("connectedBillingSweep", unfinished, reconcileConnectedAccountBillingSweep(), { checked: 0, synced: 0, failed: 0 }),
|
|
76302
|
+
settleWithinTickBudget("directoryDispatch", unfinished, dispatchPendingDirectoryWorkflows(5), { claimed: 0, dispatched: 0, failed: 0 }),
|
|
76303
|
+
settleWithinTickBudget("directoryReconcile", unfinished, reconcileDirectoryWorkflowSettlements(5), { checked: 0, settled: 0, failed: 0 }),
|
|
76304
|
+
settleWithinTickBudget("directoryArtifactCleanup", unfinished, cleanupExpiredDirectoryArtifacts(), { deleted: 0, store: "none" }),
|
|
76305
|
+
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
76306
|
+
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
76307
|
+
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
76308
|
+
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
76309
|
+
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
76310
|
+
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
76311
|
+
settleWithinTickBudget("analyticsRollup", unfinished, refreshAnalyticsDailyRollupsIfDue(), { status: "failed", rows: 0 }),
|
|
76312
|
+
settleWithinTickBudget("analyticsForms", unfinished, deliverPendingAnalyticsForms(10), { claimed: 0, delivered: 0, deferred: 0 }),
|
|
76313
|
+
settleWithinTickBudget("workflowDispatch", unfinished, dispatchDueWorkflowSchedules(origin), { dispatched: 0, status: "skipped_deadline" }),
|
|
76314
|
+
settleWithinTickBudget("scheduledRunReconcile", unfinished, reconcileScheduledRunSettlements(25), { checked: 0, settled: 0, chargedMc: 0, failed: 0 }),
|
|
76315
|
+
settleWithinTickBudget("retentionSweep", unfinished, sweepRetention(), { deleted: 0, perTable: {}, truncated: false })
|
|
75436
76316
|
]);
|
|
75437
76317
|
return c.json({
|
|
76318
|
+
durationMs: Date.now() - startedAt,
|
|
76319
|
+
tickBudgetMs: CRON_TICK_BUDGET_MS,
|
|
76320
|
+
unfinished,
|
|
76321
|
+
complete: unfinished.length === 0,
|
|
76322
|
+
scheduledRunReconciliation,
|
|
76323
|
+
retentionSweep,
|
|
75438
76324
|
drained: results.length,
|
|
75439
76325
|
results,
|
|
75440
76326
|
sweepResult,
|