mcp-scraper 0.48.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +2 -2
  2. package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
  3. package/dist/bin/api-server.cjs +1214 -328
  4. package/dist/bin/api-server.cjs.map +1 -1
  5. package/dist/bin/api-server.js +4 -4
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  7. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  8. package/dist/bin/mcp-scraper-cli.js +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +3 -3
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +75 -8
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +5 -5
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
  18. package/dist/chunk-2XTYLJQK.js.map +1 -0
  19. package/dist/{chunk-PTABZABJ.js → chunk-44LNKZTE.js} +76 -8
  20. package/dist/chunk-44LNKZTE.js.map +1 -0
  21. package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
  22. package/dist/chunk-5BEPOFKG.js.map +1 -0
  23. package/dist/chunk-C7NN57WH.js +7 -0
  24. package/dist/chunk-C7NN57WH.js.map +1 -0
  25. package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
  26. package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
  27. package/dist/{chunk-QAH7WMYC.js → chunk-QH63IZTX.js} +3 -3
  28. package/dist/chunk-QH63IZTX.js.map +1 -0
  29. package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
  30. package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
  31. package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
  32. package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
  33. package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
  34. package/dist/index.cjs.map +1 -1
  35. package/dist/index.js +3 -3
  36. package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
  37. package/dist/{server-EVTCL3KB.js → server-3ND2EXYO.js} +882 -242
  38. package/dist/server-3ND2EXYO.js.map +1 -0
  39. package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
  40. package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
  41. package/package.json +1 -1
  42. package/dist/chunk-IAXQIHNS.js +0 -7
  43. package/dist/chunk-IAXQIHNS.js.map +0 -1
  44. package/dist/chunk-MFNGUM4L.js.map +0 -1
  45. package/dist/chunk-PTABZABJ.js.map +0 -1
  46. package/dist/chunk-QAH7WMYC.js.map +0 -1
  47. package/dist/chunk-YW2LXLDU.js.map +0 -1
  48. package/dist/server-EVTCL3KB.js.map +0 -1
  49. /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
  50. /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
  51. /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
  52. /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
  53. /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
  54. /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
  55. /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
  56. /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
  57. /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
  58. /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
  59. /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
@@ -10,9 +10,11 @@ import {
10
10
  importHostedCensusPlacesCsv,
11
11
  importHostedZipGroupsCsv,
12
12
  listActiveHostedLocationDatasets
13
- } from "./chunk-R66PJOZW.js";
13
+ } from "./chunk-NNZXZTJ2.js";
14
14
  import {
15
+ ANALYTICS_CONTENT_SORTS,
15
16
  AnalyticsRepositoryError,
17
+ MAX_ENGAGED_MS,
16
18
  analyticsAcquisition,
17
19
  analyticsBusinessMetrics,
18
20
  analyticsChannelBreakdown,
@@ -64,14 +66,14 @@ import {
64
66
  updateAnalyticsPixel,
65
67
  upsertAnalyticsAdSpend,
66
68
  verifySession
67
- } from "./chunk-YW2LXLDU.js";
69
+ } from "./chunk-5BEPOFKG.js";
68
70
  import {
69
71
  classifyHarvestProblem,
70
72
  createHarvestAttemptRecorder,
71
73
  deliverWebhook,
72
74
  harvestProblemResponse,
73
75
  serializeHarvestProblem
74
- } from "./chunk-ZEMHL36J.js";
76
+ } from "./chunk-YMSZO62N.js";
75
77
  import {
76
78
  AIModeSelectors,
77
79
  AIOverviewSelectors,
@@ -98,10 +100,11 @@ import {
98
100
  resolveKernelProxyId,
99
101
  serpArrivalUrl,
100
102
  withRuntimeUserAgent
101
- } from "./chunk-MFE6RMIA.js";
103
+ } from "./chunk-GOZIG6HD.js";
102
104
  import {
103
105
  ApiErrorResponseSchema,
104
106
  ArtifactTemplatePresetKeySchema,
107
+ CommonsClaimInputSchema,
105
108
  CommonsClaimPublicationInputSchema,
106
109
  CommonsPreparePublicationInputSchema,
107
110
  CommonsPublishEditorialInputSchema,
@@ -155,7 +158,7 @@ import {
155
158
  resolveDeploymentProfile,
156
159
  resolveLocalSourcebookSchemaType,
157
160
  transcribeMediaUrl
158
- } from "./chunk-PTABZABJ.js";
161
+ } from "./chunk-44LNKZTE.js";
159
162
  import {
160
163
  auditImageUrls,
161
164
  auditImages,
@@ -183,7 +186,7 @@ import {
183
186
  recordKernelSession,
184
187
  recordVendorUsage,
185
188
  runWithCostContext
186
- } from "./chunk-PODASLGT.js";
189
+ } from "./chunk-QXAY44SA.js";
187
190
  import {
188
191
  listWorkflowDefinitions,
189
192
  runWorkflow,
@@ -211,7 +214,7 @@ import {
211
214
  } from "./chunk-3ZUBQQPQ.js";
212
215
  import {
213
216
  PACKAGE_VERSION
214
- } from "./chunk-IAXQIHNS.js";
217
+ } from "./chunk-C7NN57WH.js";
215
218
  import {
216
219
  abandonExtractSettlement,
217
220
  countSuccessfulPages,
@@ -234,7 +237,7 @@ import {
234
237
  setExtractJobTotal,
235
238
  settleExtractJob,
236
239
  terminalExtractJobStatus
237
- } from "./chunk-6DPIE262.js";
240
+ } from "./chunk-QHVFKFME.js";
238
241
  import {
239
242
  BROWSER_OPEN_MIN_BALANCE_MC,
240
243
  CONCURRENCY_PRICE_ID,
@@ -391,7 +394,7 @@ import {
391
394
  startHarvestAttempt,
392
395
  upsertPageSnapshot,
393
396
  verifyPassword
394
- } from "./chunk-MFNGUM4L.js";
397
+ } from "./chunk-2XTYLJQK.js";
395
398
 
396
399
  // src/blog/registry.ts
397
400
  var posts = [
@@ -8737,7 +8740,7 @@ var siteExtractFn = inngest.createFunction(
8737
8740
  });
8738
8741
  }) : null;
8739
8742
  const finalized = await step.run("finalize", async () => {
8740
- const { assembleExtractArtifacts } = await import("./extract-bundle-2HONVC5T.js");
8743
+ const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
8741
8744
  const current = await getExtractJob(jobId2);
8742
8745
  if (!current) throw new Error("extract job disappeared before finalization");
8743
8746
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -9679,7 +9682,7 @@ async function resolveUsZipsPath(requestedPath) {
9679
9682
  async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
9680
9683
  const mode = resolvedLocationDataMode({ zipSource: sourceMode });
9681
9684
  if (mode === "hosted") {
9682
- const { getHostedZipGroups } = await import("./location-data-repository-YWFJGNIP.js");
9685
+ const { getHostedZipGroups } = await import("./location-data-repository-UUSHH5M2.js");
9683
9686
  const hosted = await getHostedZipGroups(stateAbbr);
9684
9687
  if (!hosted.dataset) {
9685
9688
  throw new DirectoryLocationError(
@@ -9742,7 +9745,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
9742
9745
  const sourceUrl = censusStateUrl(state.fips);
9743
9746
  const warnings = [];
9744
9747
  if (resolvedLocationDataMode(config) === "hosted") {
9745
- const { queryHostedLocationMarkets } = await import("./location-data-repository-YWFJGNIP.js");
9748
+ const { queryHostedLocationMarkets } = await import("./location-data-repository-UUSHH5M2.js");
9746
9749
  const hosted = await queryHostedLocationMarkets({
9747
9750
  state: state.abbr,
9748
9751
  minPopulation: options.minPopulation,
@@ -10731,7 +10734,6 @@ import { randomUUID as randomUUID2 } from "crypto";
10731
10734
  var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
10732
10735
  var DEFAULT_RETRY_AFTER_SECONDS = 30;
10733
10736
  var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
10734
- var schemaReady = null;
10735
10737
  function concurrencyLimitForUser(user) {
10736
10738
  const tier = Math.max(0, Number(user.subscription_concurrency ?? 0));
10737
10739
  return Math.max(1, tier) + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
@@ -10769,40 +10771,15 @@ function safeMetadata(metadata) {
10769
10771
  return null;
10770
10772
  }
10771
10773
  }
10772
- async function ensureConcurrencyGateSchema() {
10773
- if (!schemaReady) {
10774
- schemaReady = (async () => {
10775
- const db = getDb();
10776
- await db.execute(`
10777
- CREATE TABLE IF NOT EXISTS concurrency_locks (
10778
- id TEXT PRIMARY KEY,
10779
- user_id INTEGER NOT NULL REFERENCES users(id),
10780
- operation TEXT NOT NULL,
10781
- status TEXT NOT NULL DEFAULT 'active',
10782
- acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
10783
- expires_at TEXT NOT NULL,
10784
- released_at TEXT,
10785
- metadata TEXT
10786
- )
10787
- `);
10788
- await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
10789
- await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
10790
- })();
10791
- }
10792
- return schemaReady;
10793
- }
10794
10774
  async function expireConcurrencyLocksForUser(userId) {
10795
- await ensureConcurrencyGateSchema();
10796
10775
  await getDb().execute({
10797
- sql: `UPDATE concurrency_locks
10798
- SET status = 'expired', released_at = datetime('now')
10776
+ sql: `DELETE FROM concurrency_locks
10799
10777
  WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
10800
10778
  args: [userId]
10801
10779
  });
10802
10780
  }
10803
- async function countActiveUsageForUser(userId) {
10804
- await ensureConcurrencyGateSchema();
10805
- await expireConcurrencyLocksForUser(userId);
10781
+ async function countActiveUsageForUser(userId, alreadyExpired = false) {
10782
+ if (!alreadyExpired) await expireConcurrencyLocksForUser(userId);
10806
10783
  const res = await getDb().execute({
10807
10784
  sql: `SELECT
10808
10785
  (
@@ -10822,7 +10799,6 @@ async function countActiveUsageForUser(userId) {
10822
10799
  }
10823
10800
  async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
10824
10801
  if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
10825
- await ensureConcurrencyGateSchema();
10826
10802
  await expireConcurrencyLocksForUser(userId);
10827
10803
  const res = await getDb().execute({
10828
10804
  sql: `UPDATE concurrency_locks
@@ -10836,13 +10812,12 @@ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
10836
10812
  return Number(res.rowsAffected ?? 0) > 0;
10837
10813
  }
10838
10814
  async function acquireConcurrencyGate(user, operation, options = {}) {
10839
- await ensureConcurrencyGateSchema();
10840
10815
  const limit = concurrencyLimitForUser(user);
10841
10816
  if (isConcurrencyLimitBypassed(user)) {
10842
10817
  return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
10843
10818
  }
10844
10819
  if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
10845
- return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
10820
+ return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id, true), limit, operation, reused: true };
10846
10821
  }
10847
10822
  await expireConcurrencyLocksForUser(user.id);
10848
10823
  const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
@@ -10873,7 +10848,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
10873
10848
  limit
10874
10849
  ]
10875
10850
  });
10876
- const active = await countActiveUsageForUser(user.id);
10851
+ const active = await countActiveUsageForUser(user.id, true);
10877
10852
  if (Number(res.rowsAffected ?? 0) > 0) {
10878
10853
  return { ok: true, lockId, active, limit, operation };
10879
10854
  }
@@ -10881,17 +10856,13 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
10881
10856
  }
10882
10857
  async function releaseConcurrencyGate(lockId) {
10883
10858
  if (!lockId) return;
10884
- await ensureConcurrencyGateSchema();
10885
10859
  await getDb().execute({
10886
- sql: `UPDATE concurrency_locks
10887
- SET status = 'released', released_at = datetime('now')
10888
- WHERE id = ? AND status = 'active'`,
10860
+ sql: `DELETE FROM concurrency_locks WHERE id = ?`,
10889
10861
  args: [lockId]
10890
10862
  });
10891
10863
  }
10892
10864
  async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
10893
10865
  if (!lockId) return;
10894
- await ensureConcurrencyGateSchema();
10895
10866
  await getDb().execute({
10896
10867
  sql: `UPDATE concurrency_locks
10897
10868
  SET expires_at = datetime('now', ?)
@@ -14708,7 +14679,7 @@ async function recoverSettlement(row, user, authorization, duplicate) {
14708
14679
  }
14709
14680
  }
14710
14681
  await getDb().execute({
14711
- sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status = 'settling'",
14682
+ sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('settling', 'cost_pending')",
14712
14683
  args: [row.id]
14713
14684
  });
14714
14685
  await getDb().execute({
@@ -14753,11 +14724,13 @@ async function settleScheduledRun(args) {
14753
14724
  const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
14754
14725
  const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
14755
14726
  const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
14756
- const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
14727
+ const modelCostUnreported = args.modelCostStatus === "missing";
14728
+ const pendingReason = modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
14757
14729
  const metadata = safeMetadata2({
14758
14730
  ...args.metadata,
14759
14731
  billingPhase: "settlement",
14760
14732
  chargeMode: "model_delta",
14733
+ ...modelCostUnreported ? { modelCostUnreported: true } : {},
14761
14734
  ...pendingReason ? { reason: pendingReason } : {}
14762
14735
  });
14763
14736
  const eventId = existing?.id ?? randomUUID4();
@@ -14801,6 +14774,72 @@ async function settleScheduledRun(args) {
14801
14774
  }
14802
14775
  return recoverSettlement(existing, user, authorization, duplicateClaim);
14803
14776
  }
14777
+ async function reconcileScheduledRunSettlements(limit = 25, minimumAgeHours = 24) {
14778
+ const stale = await getDb().execute({
14779
+ sql: `SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata
14780
+ FROM billing_events
14781
+ WHERE billing_class = ? AND status = 'cost_pending'
14782
+ AND created_at < datetime('now', ?)
14783
+ ORDER BY created_at
14784
+ LIMIT ?`,
14785
+ args: [SCHEDULED_RUN_BILLING_CLASS, `-${Math.max(1, Math.trunc(minimumAgeHours))} hours`, Math.max(1, Math.trunc(limit))]
14786
+ });
14787
+ const result = { checked: stale.rows.length, settled: 0, chargedMc: 0, failed: 0 };
14788
+ for (const raw of stale.rows) {
14789
+ const row = raw;
14790
+ try {
14791
+ const authorization = await authorizationById(row.authorization_id);
14792
+ if (!authorization) {
14793
+ result.failed += 1;
14794
+ continue;
14795
+ }
14796
+ const cap = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
14797
+ const chargeMc = Math.min(Math.max(0, Math.round(Number(row.amount_mc))), cap);
14798
+ if (chargeMc > 0) {
14799
+ const debit = await debitMcIdempotent(
14800
+ Number(authorization.user_id),
14801
+ chargeMc,
14802
+ LedgerOperation.SCHEDULED_RUN,
14803
+ "scheduled-run:" + authorization.id + ":model:reconciled",
14804
+ authorization.id + ":model"
14805
+ );
14806
+ if (!debit.ok) {
14807
+ result.failed += 1;
14808
+ continue;
14809
+ }
14810
+ }
14811
+ await getDb().execute({
14812
+ sql: `UPDATE billing_events
14813
+ SET status = 'settled', amount_mc = ?, settled_at = datetime('now'), metadata = ?
14814
+ WHERE id = ? AND status = 'cost_pending'`,
14815
+ args: [
14816
+ chargeMc,
14817
+ safeMetadata2({ ...metadataReason(row) ? { reason: metadataReason(row) } : {}, autoReconciled: true, cappedAtAuthorizedMaximum: chargeMc < Number(row.amount_mc) }),
14818
+ row.id
14819
+ ]
14820
+ });
14821
+ await getDb().execute({
14822
+ sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending')",
14823
+ args: [authorization.id]
14824
+ });
14825
+ result.settled += 1;
14826
+ result.chargedMc += chargeMc;
14827
+ console.error(JSON.stringify({
14828
+ event: "scheduled_run_settlement_reconciled",
14829
+ authorization_id: authorization.id,
14830
+ charged_mc: chargeMc,
14831
+ reason: metadataReason(row) ?? "unreported_cost"
14832
+ }));
14833
+ } catch (err) {
14834
+ result.failed += 1;
14835
+ console.error(JSON.stringify({
14836
+ event: "scheduled_run_reconciliation_failed",
14837
+ message: err instanceof Error ? err.message : String(err)
14838
+ }));
14839
+ }
14840
+ }
14841
+ return result;
14842
+ }
14804
14843
  async function voidScheduledRunAuthorization(args) {
14805
14844
  const user = await resolveSchedulingUser(args.identity);
14806
14845
  const authorization = await authorizationById(args.authorizationId);
@@ -15672,28 +15711,6 @@ internalInboxApp.post("/broadcast", async (c) => {
15672
15711
 
15673
15712
  // src/api/connected-account-billing.ts
15674
15713
  var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
15675
- async function ensureConnectedAccountBillingSchema() {
15676
- await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
15677
- user_id INTEGER PRIMARY KEY,
15678
- stripe_subscription_id TEXT,
15679
- stripe_subscription_item_id TEXT,
15680
- price_id TEXT,
15681
- quantity INTEGER NOT NULL DEFAULT 0,
15682
- status TEXT NOT NULL DEFAULT 'pending',
15683
- last_error_code TEXT,
15684
- checked_at TEXT NOT NULL DEFAULT (datetime('now')),
15685
- synced_at TEXT,
15686
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
15687
- )`);
15688
- await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_charges (
15689
- user_id INTEGER NOT NULL,
15690
- connection_id TEXT NOT NULL,
15691
- period_start TEXT NOT NULL,
15692
- amount_mc INTEGER NOT NULL,
15693
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
15694
- PRIMARY KEY (user_id, connection_id, period_start)
15695
- )`);
15696
- }
15697
15714
  function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
15698
15715
  return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
15699
15716
  }
@@ -15702,7 +15719,6 @@ function isBillableConnection(connection) {
15702
15719
  return BILLABLE_LIFECYCLE.has(String(connection.lifecycleStatus ?? ""));
15703
15720
  }
15704
15721
  async function chargedConnectionIds(userId, periodStart) {
15705
- await ensureConnectedAccountBillingSchema();
15706
15722
  const result = await getDb().execute({
15707
15723
  sql: "SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",
15708
15724
  args: [Number(userId), periodStart]
@@ -15710,7 +15726,6 @@ async function chargedConnectionIds(userId, periodStart) {
15710
15726
  return new Set(result.rows.map((row) => String(row.connection_id)));
15711
15727
  }
15712
15728
  async function getConnectedAccountBillingState(userId) {
15713
- await ensureConnectedAccountBillingSchema();
15714
15729
  const result = await getDb().execute({
15715
15730
  sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
15716
15731
  args: [Number(userId)]
@@ -15718,7 +15733,6 @@ async function getConnectedAccountBillingState(userId) {
15718
15733
  return result.rows[0] ? result.rows[0] : null;
15719
15734
  }
15720
15735
  async function setConnectedAccountBillingState(input) {
15721
- await ensureConnectedAccountBillingSchema();
15722
15736
  await getDb().execute({
15723
15737
  sql: `INSERT INTO connected_account_billing
15724
15738
  (user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, checked_at, synced_at, updated_at)
@@ -22184,6 +22198,82 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
22184
22198
  return { checked: jobs.length, staleFailed, settled, failed };
22185
22199
  }
22186
22200
 
22201
+ // src/api/retention-sweeps.ts
22202
+ var REPORTING_WINDOW_DAYS = 30;
22203
+ var TELEMETRY_RETENTION_DAYS = 45;
22204
+ var FAILURE_RETENTION_DAYS = 90;
22205
+ var RETENTION_RULES = [
22206
+ { table: "request_events", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22207
+ { table: "kernel_session_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22208
+ { table: "vendor_usage_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22209
+ { table: "harvest_attempts", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, timestampFormat: "iso8601" },
22210
+ { table: "browser_agent_actions", timestampColumn: "created_at", retentionDays: REPORTING_WINDOW_DAYS },
22211
+ { table: "jobs", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, where: "status = 'done'" },
22212
+ { table: "jobs", timestampColumn: "created_at", retentionDays: FAILURE_RETENTION_DAYS, where: "status = 'failed'" }
22213
+ ];
22214
+ function cutoffFor(rule) {
22215
+ const cutoff = new Date(Date.now() - rule.retentionDays * 24 * 60 * 60 * 1e3);
22216
+ if (rule.timestampFormat === "iso8601") return cutoff.toISOString();
22217
+ return cutoff.toISOString().replace("T", " ").slice(0, 19);
22218
+ }
22219
+ var DEFAULT_BATCH = 500;
22220
+ var DEFAULT_MAX_DELETES_PER_RULE = 1e3;
22221
+ var DEFAULT_SWEEP_DEADLINE_MS = 15e3;
22222
+ async function tableExists(table) {
22223
+ const res = await getDb().execute({
22224
+ sql: "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
22225
+ args: [table]
22226
+ });
22227
+ return res.rows.length > 0;
22228
+ }
22229
+ async function sweepRule(rule, maxDeletes) {
22230
+ if (!await tableExists(rule.table)) return 0;
22231
+ const filter = rule.where ? ` AND ${rule.where}` : "";
22232
+ const cutoff = cutoffFor(rule);
22233
+ let deleted = 0;
22234
+ while (deleted < maxDeletes) {
22235
+ const res = await getDb().execute({
22236
+ sql: `DELETE FROM ${rule.table} WHERE rowid IN (
22237
+ SELECT rowid FROM ${rule.table}
22238
+ WHERE ${rule.timestampColumn} < ?${filter}
22239
+ LIMIT ?)`,
22240
+ args: [cutoff, Math.min(DEFAULT_BATCH, maxDeletes - deleted)]
22241
+ });
22242
+ const n = Number(res.rowsAffected ?? 0);
22243
+ deleted += n;
22244
+ if (n === 0) break;
22245
+ }
22246
+ return deleted;
22247
+ }
22248
+ async function sweepRetention(options = {}) {
22249
+ const maxDeletes = options.maxDeletesPerRule ?? DEFAULT_MAX_DELETES_PER_RULE;
22250
+ const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_SWEEP_DEADLINE_MS);
22251
+ const perTable = {};
22252
+ let deleted = 0;
22253
+ let truncated = false;
22254
+ for (const rule of RETENTION_RULES) {
22255
+ const key = rule.where ? `${rule.table}:${rule.where}` : rule.table;
22256
+ if (Date.now() >= deadline) {
22257
+ truncated = true;
22258
+ break;
22259
+ }
22260
+ try {
22261
+ const n = await sweepRule(rule, maxDeletes);
22262
+ perTable[key] = n;
22263
+ deleted += n;
22264
+ if (n >= maxDeletes) truncated = true;
22265
+ } catch (err) {
22266
+ perTable[key] = -1;
22267
+ console.error(JSON.stringify({
22268
+ event: "retention_sweep_failed",
22269
+ table: rule.table,
22270
+ message: err instanceof Error ? err.message : String(err)
22271
+ }));
22272
+ }
22273
+ }
22274
+ return { deleted, perTable, truncated };
22275
+ }
22276
+
22187
22277
  // src/api/directory-workflow-reconciliation.ts
22188
22278
  async function reconcileDirectoryWorkflowSettlements(limit = 25) {
22189
22279
  const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
@@ -24893,6 +24983,7 @@ async function listSessionRows(userId, includeClosed = false) {
24893
24983
  });
24894
24984
  return res.rows;
24895
24985
  }
24986
+ var ORPHANED_SESSION_STATUS = "runtime_close_pending";
24896
24987
  async function listIdleOpenSessionRows(idleSeconds) {
24897
24988
  const db = getDb();
24898
24989
  const res = await db.execute({
@@ -24918,13 +25009,21 @@ async function setBilledMc(id, billedMc) {
24918
25009
  args: [Math.round(billedMc), id]
24919
25010
  });
24920
25011
  }
24921
- async function markSessionClosed(id) {
25012
+ async function markSessionClosed(id, runtimeClosed = true) {
24922
25013
  const db = getDb();
24923
25014
  await db.execute({
24924
- sql: `UPDATE browser_agent_sessions SET status = 'closed', closed_at = datetime('now') WHERE id = ?`,
24925
- args: [id]
25015
+ sql: `UPDATE browser_agent_sessions SET status = ?, closed_at = datetime('now') WHERE id = ?`,
25016
+ args: [runtimeClosed ? "closed" : ORPHANED_SESSION_STATUS, id]
24926
25017
  });
24927
25018
  }
25019
+ async function listOrphanedSessionRows(limit = 50) {
25020
+ const db = getDb();
25021
+ const res = await db.execute({
25022
+ sql: `SELECT * FROM browser_agent_sessions WHERE status = ? ORDER BY closed_at LIMIT ?`,
25023
+ args: [ORPHANED_SESSION_STATUS, Math.max(1, Math.min(200, Math.round(limit)))]
25024
+ });
25025
+ return res.rows;
25026
+ }
24928
25027
  async function recordAction(input) {
24929
25028
  const db = getDb();
24930
25029
  await db.execute({
@@ -26064,6 +26163,21 @@ async function charge(sessionId, userId, _startedAtMs) {
26064
26163
  if (res.ok) await setBilledMc(sessionId, owed);
26065
26164
  }
26066
26165
  }
26166
+ async function closeRuntimeSession(runtimeSessionId, sessionId) {
26167
+ try {
26168
+ await closeSession(runtimeSessionId);
26169
+ return true;
26170
+ } catch (err) {
26171
+ console.error(JSON.stringify({
26172
+ event: "browser_runtime_close_failed",
26173
+ session_id: sessionId,
26174
+ runtime_session_id: runtimeSessionId,
26175
+ error_name: err instanceof Error ? err.name : typeof err,
26176
+ message: sanitizeVendorName(err instanceof Error ? err.message : String(err)).slice(0, 300)
26177
+ }));
26178
+ return false;
26179
+ }
26180
+ }
26067
26181
  async function closeAndSettleSession(row) {
26068
26182
  if (row.user_id != null) {
26069
26183
  try {
@@ -26071,15 +26185,24 @@ async function closeAndSettleSession(row) {
26071
26185
  } catch {
26072
26186
  }
26073
26187
  }
26074
- try {
26075
- await closeSession(row.runtime_session_id);
26076
- } catch {
26077
- }
26078
- await markSessionClosed(row.id).catch(() => {
26188
+ const runtimeClosed = await closeRuntimeSession(row.runtime_session_id, row.id);
26189
+ await markSessionClosed(row.id, runtimeClosed).catch(() => {
26079
26190
  });
26080
26191
  await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
26081
26192
  });
26082
26193
  }
26194
+ async function retryOrphanedBrowserSessions(limit = 50) {
26195
+ const rows = await listOrphanedSessionRows(limit);
26196
+ let recovered = 0;
26197
+ for (const row of rows) {
26198
+ if (await closeRuntimeSession(row.runtime_session_id, row.id)) {
26199
+ await markSessionClosed(row.id, true).catch(() => {
26200
+ });
26201
+ recovered++;
26202
+ }
26203
+ }
26204
+ return { recovered, stillOpen: rows.length - recovered };
26205
+ }
26083
26206
  async function completeAuthConnection(connectionId) {
26084
26207
  const connection = await getAuthConnectionRow(connectionId);
26085
26208
  if (!connection) return { ok: false, error: "not found" };
@@ -26104,7 +26227,7 @@ async function reapIdleBrowserSessions(idleSeconds = 120) {
26104
26227
  function publicSession(row) {
26105
26228
  return {
26106
26229
  session_id: row.id,
26107
- status: row.status,
26230
+ status: row.status === ORPHANED_SESSION_STATUS ? "closed" : row.status,
26108
26231
  label: row.label,
26109
26232
  watch_url: `/console/${row.id}`,
26110
26233
  created_at: row.created_at,
@@ -30873,8 +30996,212 @@ import { Hono as Hono31 } from "hono";
30873
30996
  import { z as z32 } from "zod";
30874
30997
 
30875
30998
  // src/api/commons-repository.ts
30876
- import { createHash as createHash12, randomUUID as randomUUID15 } from "crypto";
30877
- var COMMONS_SCHEMA_VERSION = "2026-08-04.3";
30999
+ import { createHash as createHash13, randomUUID as randomUUID15 } from "crypto";
31000
+
31001
+ // src/api/schema-presence.ts
31002
+ var cache = null;
31003
+ async function loadSchemaObjects() {
31004
+ const res = await getDb().execute(
31005
+ "SELECT name FROM sqlite_master WHERE type IN ('table', 'index')"
31006
+ );
31007
+ return new Set(res.rows.map((row) => String(row.name)));
31008
+ }
31009
+ function invalidateSchemaPresence() {
31010
+ cache = null;
31011
+ }
31012
+ async function schemaObjectsPresent(names) {
31013
+ if (names.length === 0) return true;
31014
+ try {
31015
+ if (!cache) cache = loadSchemaObjects();
31016
+ const present = await cache;
31017
+ if (names.every((name) => present.has(name))) return true;
31018
+ invalidateSchemaPresence();
31019
+ return false;
31020
+ } catch {
31021
+ invalidateSchemaPresence();
31022
+ return false;
31023
+ }
31024
+ }
31025
+
31026
+ // src/api/commons-linksets.ts
31027
+ import { createHash as createHash12 } from "crypto";
31028
+ var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
31029
+ var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
31030
+ var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
31031
+ "about",
31032
+ "alternate",
31033
+ "author",
31034
+ "canonical",
31035
+ "cite-as",
31036
+ "collection",
31037
+ "contents",
31038
+ "copyright",
31039
+ "describedby",
31040
+ "describes",
31041
+ "first",
31042
+ "glossary",
31043
+ "help",
31044
+ "icon",
31045
+ "index",
31046
+ "item",
31047
+ "last",
31048
+ "latest-version",
31049
+ "license",
31050
+ "memento",
31051
+ "next",
31052
+ "original",
31053
+ "predecessor-version",
31054
+ "prev",
31055
+ "related",
31056
+ "replies",
31057
+ "service-desc",
31058
+ "service-doc",
31059
+ "successor-version",
31060
+ "terms-of-service",
31061
+ "type",
31062
+ "via",
31063
+ "version-history"
31064
+ ]);
31065
+ function normalizeCommonsClaims(input) {
31066
+ if (input === void 0) return void 0;
31067
+ if (!Array.isArray(input)) throw new Error("claims must be an array.");
31068
+ if (input.length > 300) throw new Error("claims must contain no more than 300 assertions.");
31069
+ const seen = /* @__PURE__ */ new Set();
31070
+ return input.map((claim, index) => {
31071
+ const predicate = String(claim.predicate ?? "").trim();
31072
+ if (!isSupportedPredicate(predicate)) {
31073
+ throw new Error(`claims[${index}].predicate must be a supported registered relation or an absolute HTTPS relation URI.`);
31074
+ }
31075
+ const objectEntityId = optionalText(claim.objectEntityId, 80);
31076
+ const objectUrl = optionalHttpsUrl(claim.objectUrl, `claims[${index}].objectUrl`);
31077
+ if ((objectEntityId ? 1 : 0) + (objectUrl ? 1 : 0) !== 1) {
31078
+ throw new Error(`claims[${index}] must provide exactly one of objectEntityId or objectUrl.`);
31079
+ }
31080
+ if (objectEntityId && !/^TPW-Q[A-Za-z0-9._-]+$/.test(objectEntityId)) {
31081
+ throw new Error(`claims[${index}].objectEntityId must be a Transparent Commons TPW-Q identifier.`);
31082
+ }
31083
+ const confidence = claim.confidence ?? 1;
31084
+ if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
31085
+ throw new Error(`claims[${index}].confidence must be between 0 and 1.`);
31086
+ }
31087
+ const key = `${predicate}\0${objectEntityId ?? objectUrl}`;
31088
+ if (seen.has(key)) throw new Error(`claims[${index}] duplicates another predicate and object pair.`);
31089
+ seen.add(key);
31090
+ return {
31091
+ predicate,
31092
+ predicateLabel: optionalText(claim.predicateLabel, 160),
31093
+ objectEntityId,
31094
+ objectUrl,
31095
+ title: optionalText(claim.title, 500),
31096
+ mediaType: optionalText(claim.mediaType, 160),
31097
+ hreflang: normalizeHreflang(claim.hreflang),
31098
+ qualifiers: normalizeScalarRecord(claim.qualifiers),
31099
+ references: normalizeReferences(claim.references),
31100
+ rank: claim.rank ?? "normal",
31101
+ confidence,
31102
+ confidenceReason: optionalText(claim.confidenceReason, 1e3),
31103
+ disputeState: claim.disputeState ?? "undisputed",
31104
+ validFrom: optionalDate(claim.validFrom, `claims[${index}].validFrom`),
31105
+ validTo: optionalDate(claim.validTo, `claims[${index}].validTo`),
31106
+ asOf: optionalDate(claim.asOf, `claims[${index}].asOf`),
31107
+ sourceFamily: claim.sourceFamily ?? "unknown"
31108
+ };
31109
+ });
31110
+ }
31111
+ function claimIsPublic(claim, now = /* @__PURE__ */ new Date()) {
31112
+ if (claim.reviewState !== "approved" || claim.rank === "deprecated") return false;
31113
+ if (!["undisputed", "attributed"].includes(claim.disputeState ?? "undisputed")) return false;
31114
+ const timestamp = now.getTime();
31115
+ if (claim.validFrom && Date.parse(claim.validFrom) > timestamp) return false;
31116
+ if (claim.validTo && Date.parse(claim.validTo) < timestamp) return false;
31117
+ return true;
31118
+ }
31119
+ function buildCommonsLinkset(entity, claims) {
31120
+ const publicUrl = `${commonsPublicWikiBaseUrl()}/wiki/${encodeURIComponent(entity.slug)}`;
31121
+ const linksetUrl = commonsLinksetUrl(entity.id);
31122
+ const context = { anchor: publicUrl };
31123
+ const publicClaims = claims.filter((claim) => claimIsPublic(claim)).sort((a, b) => a.predicate.localeCompare(b.predicate) || a.href.localeCompare(b.href) || a.id.localeCompare(b.id));
31124
+ for (const claim of publicClaims) {
31125
+ const targets = context[claim.predicate] ?? [];
31126
+ const target = { href: claim.href };
31127
+ if (claim.mediaType) target.type = claim.mediaType;
31128
+ if (claim.hreflang) target.hreflang = claim.hreflang;
31129
+ if (claim.title) target.title = claim.title;
31130
+ targets.push(target);
31131
+ context[claim.predicate] = targets;
31132
+ }
31133
+ const document2 = { linkset: [context] };
31134
+ const etag = `"${createHash12("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
31135
+ return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
31136
+ }
31137
+ function commonsLinksetDiscoveryHeader(idOrSlug) {
31138
+ const url = commonsLinksetUrl(idOrSlug);
31139
+ return `<${url}>; rel="linkset"; type="${COMMONS_LINKSET_MEDIA_TYPE}"; profile="${COMMONS_RELATIONSHIP_PROFILE}"`;
31140
+ }
31141
+ function commonsLinksetUrl(idOrSlug) {
31142
+ const base = (process.env.APP_URL?.trim() || "https://mcpscraper.dev").replace(/\/+$/, "");
31143
+ return `${base}/commons/entities/${encodeURIComponent(idOrSlug)}/linkset`;
31144
+ }
31145
+ function commonsClaimIdentityKey(claim) {
31146
+ return `${claim.predicate}\0${claim.objectEntityId ?? claim.objectUrl ?? ""}`;
31147
+ }
31148
+ function commonsPublicWikiBaseUrl() {
31149
+ return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || "https://transparent-commons.cc").replace(/\/+$/, "");
31150
+ }
31151
+ function isSupportedPredicate(value) {
31152
+ if (REGISTERED_RELATIONS.has(value)) return true;
31153
+ try {
31154
+ const url = new URL(value);
31155
+ return url.protocol === "https:" && Boolean(url.hostname);
31156
+ } catch {
31157
+ return false;
31158
+ }
31159
+ }
31160
+ function optionalText(value, max) {
31161
+ const text2 = String(value ?? "").trim();
31162
+ return text2 ? text2.slice(0, max) : void 0;
31163
+ }
31164
+ function optionalHttpsUrl(value, field) {
31165
+ const text2 = optionalText(value, 2e3);
31166
+ if (!text2) return void 0;
31167
+ try {
31168
+ const url = new URL(text2);
31169
+ if (url.protocol !== "https:") throw new Error();
31170
+ url.hash = "";
31171
+ return url.href;
31172
+ } catch {
31173
+ throw new Error(`${field} must be an absolute HTTPS URL.`);
31174
+ }
31175
+ }
31176
+ function optionalDate(value, field) {
31177
+ const text2 = optionalText(value, 80);
31178
+ if (!text2) return void 0;
31179
+ const timestamp = Date.parse(text2);
31180
+ if (!Number.isFinite(timestamp)) throw new Error(`${field} must be an ISO-compatible date.`);
31181
+ return new Date(timestamp).toISOString();
31182
+ }
31183
+ function normalizeScalarRecord(value) {
31184
+ if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
31185
+ return Object.fromEntries(Object.entries(value).slice(0, 50).map(([key, item]) => [key.slice(0, 120), item]));
31186
+ }
31187
+ function normalizeReferences(value) {
31188
+ if (!Array.isArray(value)) return void 0;
31189
+ return value.slice(0, 50).map((reference) => ({
31190
+ title: String(reference.title ?? "").trim().slice(0, 240),
31191
+ url: optionalHttpsUrl(reference.url, "claim reference URL"),
31192
+ source: optionalText(reference.source, 240),
31193
+ note: optionalText(reference.note, 1e3),
31194
+ accessedAt: optionalDate(reference.accessedAt, "claim reference accessedAt")
31195
+ })).filter((reference) => reference.title);
31196
+ }
31197
+ function normalizeHreflang(value) {
31198
+ if (!Array.isArray(value)) return void 0;
31199
+ const languages = [...new Set(value.map((item) => optionalText(item, 80)).filter((item) => Boolean(item)))];
31200
+ return languages.length ? languages : void 0;
31201
+ }
31202
+
31203
+ // src/api/commons-repository.ts
31204
+ var COMMONS_SCHEMA_VERSION = "2026-08-05.1";
30878
31205
  var DEFAULT_COMMONS_BASE_URL = "https://transparent-commons.cc";
30879
31206
  var DEFAULT_ENTITY_TYPE = "PublicArticle";
30880
31207
  var COMMONS_ENTITY_PROFILES = {
@@ -31022,9 +31349,38 @@ var COMMONS_ENTITY_PROFILES = {
31022
31349
  ]
31023
31350
  }
31024
31351
  };
31025
- var schemaReady2 = false;
31352
+ var schemaReady = false;
31353
+ var COMMONS_SCHEMA_OBJECTS = [
31354
+ "schema_migrations",
31355
+ "commons_entities",
31356
+ "commons_claims",
31357
+ "commons_proposals",
31358
+ "commons_ledger",
31359
+ "commons_saved_filters",
31360
+ "commons_index_documents",
31361
+ "commons_publications",
31362
+ "commons_publication_editions",
31363
+ "commons_entities_status_updated",
31364
+ "commons_entities_type_updated",
31365
+ "commons_entities_source_domain",
31366
+ "commons_claims_subject_predicate",
31367
+ "commons_claims_object_entity",
31368
+ "commons_proposals_user_created",
31369
+ "commons_proposals_entity_created",
31370
+ "commons_ledger_entity_created",
31371
+ "commons_ledger_user_created",
31372
+ "commons_saved_filters_user_updated",
31373
+ "commons_index_documents_status_updated",
31374
+ "commons_index_documents_entity_type",
31375
+ "commons_publications_updated",
31376
+ "commons_publication_editions_publication_published"
31377
+ ];
31026
31378
  async function ensureCommonsSchema() {
31027
- if (schemaReady2) return;
31379
+ if (schemaReady) return;
31380
+ if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
31381
+ schemaReady = true;
31382
+ return;
31383
+ }
31028
31384
  const db = getDb();
31029
31385
  await db.batch([
31030
31386
  {
@@ -31071,6 +31427,42 @@ async function ensureCommonsSchema() {
31071
31427
  { sql: `CREATE INDEX IF NOT EXISTS commons_entities_status_updated ON commons_entities(status, updated_at DESC)`, args: [] },
31072
31428
  { sql: `CREATE INDEX IF NOT EXISTS commons_entities_type_updated ON commons_entities(entity_type, updated_at DESC)`, args: [] },
31073
31429
  { sql: `CREATE INDEX IF NOT EXISTS commons_entities_source_domain ON commons_entities(source_json)`, args: [] },
31430
+ {
31431
+ sql: `
31432
+ CREATE TABLE IF NOT EXISTS commons_claims (
31433
+ id TEXT PRIMARY KEY,
31434
+ subject_entity_id TEXT NOT NULL REFERENCES commons_entities(id),
31435
+ predicate TEXT NOT NULL,
31436
+ predicate_label TEXT,
31437
+ object_entity_id TEXT,
31438
+ object_url TEXT,
31439
+ title TEXT,
31440
+ media_type TEXT,
31441
+ hreflang TEXT,
31442
+ qualifiers_json TEXT NOT NULL DEFAULT '{}',
31443
+ references_json TEXT NOT NULL DEFAULT '[]',
31444
+ rank TEXT NOT NULL DEFAULT 'normal',
31445
+ confidence REAL NOT NULL DEFAULT 1,
31446
+ confidence_reason TEXT,
31447
+ dispute_state TEXT NOT NULL DEFAULT 'undisputed',
31448
+ valid_from TEXT,
31449
+ valid_to TEXT,
31450
+ as_of TEXT,
31451
+ source_family TEXT NOT NULL DEFAULT 'unknown',
31452
+ review_state TEXT NOT NULL DEFAULT 'approved',
31453
+ proposal_id TEXT NOT NULL REFERENCES commons_proposals(id),
31454
+ created_by_user_id INTEGER,
31455
+ updated_by_user_id INTEGER,
31456
+ created_at TEXT NOT NULL,
31457
+ updated_at TEXT NOT NULL,
31458
+ CHECK ((object_entity_id IS NOT NULL) != (object_url IS NOT NULL)),
31459
+ UNIQUE(subject_entity_id, predicate, object_entity_id, object_url)
31460
+ )
31461
+ `,
31462
+ args: []
31463
+ },
31464
+ { sql: `CREATE INDEX IF NOT EXISTS commons_claims_subject_predicate ON commons_claims(subject_entity_id, predicate)`, args: [] },
31465
+ { sql: `CREATE INDEX IF NOT EXISTS commons_claims_object_entity ON commons_claims(object_entity_id)`, args: [] },
31074
31466
  {
31075
31467
  sql: `
31076
31468
  CREATE TABLE IF NOT EXISTS commons_proposals (
@@ -31209,7 +31601,7 @@ async function ensureCommonsSchema() {
31209
31601
  args: [COMMONS_SCHEMA_VERSION]
31210
31602
  }
31211
31603
  ], "write");
31212
- schemaReady2 = true;
31604
+ schemaReady = true;
31213
31605
  }
31214
31606
  async function searchCommonsEntities(filters = {}, userId) {
31215
31607
  await ensureCommonsSchema();
@@ -31231,8 +31623,15 @@ async function searchCommonsEntities(filters = {}, userId) {
31231
31623
  args.push(jsonLikeValue(keyword));
31232
31624
  }
31233
31625
  if (mergedFilters.relatedEntityId) {
31234
- clauses.push("related_entities_json LIKE ?");
31235
- args.push(jsonLikeValue(mergedFilters.relatedEntityId));
31626
+ clauses.push(`(
31627
+ related_entities_json LIKE ?
31628
+ OR EXISTS (
31629
+ SELECT 1 FROM commons_claims
31630
+ WHERE commons_claims.subject_entity_id = commons_entities.id
31631
+ AND commons_claims.object_entity_id = ?
31632
+ )
31633
+ )`);
31634
+ args.push(jsonLikeValue(mergedFilters.relatedEntityId), mergedFilters.relatedEntityId);
31236
31635
  }
31237
31636
  if (mergedFilters.trailId) {
31238
31637
  clauses.push("related_entities_json LIKE ?");
@@ -31254,8 +31653,16 @@ async function searchCommonsEntities(filters = {}, userId) {
31254
31653
  }
31255
31654
  const terms = tokenize(mergedFilters.query);
31256
31655
  for (const term of terms) {
31257
- clauses.push("search_text LIKE ?");
31258
- args.push(`%${escapeLike(term)}%`);
31656
+ clauses.push(`(
31657
+ search_text LIKE ?
31658
+ OR EXISTS (
31659
+ SELECT 1 FROM commons_claims
31660
+ WHERE commons_claims.subject_entity_id = commons_entities.id
31661
+ AND (commons_claims.predicate LIKE ? OR commons_claims.predicate_label LIKE ? OR commons_claims.title LIKE ? OR commons_claims.object_url LIKE ?)
31662
+ )
31663
+ )`);
31664
+ const pattern = `%${escapeLike(term)}%`;
31665
+ args.push(pattern, pattern, pattern, pattern, pattern);
31259
31666
  }
31260
31667
  args.push(limit, offset);
31261
31668
  const result = await getDb().execute({
@@ -31277,9 +31684,9 @@ async function searchCommonsEntities(filters = {}, userId) {
31277
31684
  }
31278
31685
  async function prepareCommonsEntity(input, user) {
31279
31686
  await ensureCommonsSchema();
31280
- const title = optionalText(input.title, 180) ?? "";
31687
+ const title = optionalText2(input.title, 180) ?? "";
31281
31688
  const slug2 = normalizeSlug(input.slug || title);
31282
- const schemaOrgType = optionalText(input.schemaOrgType, 80);
31689
+ const schemaOrgType = optionalText2(input.schemaOrgType, 80);
31283
31690
  const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
31284
31691
  const entityType = contract.entityType;
31285
31692
  const bodyMarkdown = safeCandidateBody(input.bodyMarkdown, input.contentSections);
@@ -31337,8 +31744,8 @@ async function validateCommonsEntity(input, user) {
31337
31744
  await ensureCommonsSchema();
31338
31745
  const errors = [];
31339
31746
  const warnings = [];
31340
- const title = optionalText(input.title, 180);
31341
- const description = optionalText(input.description, 600);
31747
+ const title = optionalText2(input.title, 180);
31748
+ const description = optionalText2(input.description, 600);
31342
31749
  const slug2 = normalizeSlug(input.slug || title || "");
31343
31750
  const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
31344
31751
  const source = normalizeSource(input.source);
@@ -31353,6 +31760,11 @@ async function validateCommonsEntity(input, user) {
31353
31760
  maxCandidates: 8
31354
31761
  });
31355
31762
  const existing = slug2 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug2, input.entityId) : null;
31763
+ try {
31764
+ normalizeCommonsClaims(input.claims);
31765
+ } catch (error) {
31766
+ errors.push(error instanceof Error ? error.message : "Claims are invalid.");
31767
+ }
31356
31768
  if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
31357
31769
  if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
31358
31770
  if (!slug2) errors.push("A Commons entity needs a URL slug.");
@@ -31442,6 +31854,7 @@ async function submitCommonsEntity(input, user) {
31442
31854
  };
31443
31855
  }
31444
31856
  const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
31857
+ const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
31445
31858
  const safety = evaluatePublishSafety(normalized, existing);
31446
31859
  const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
31447
31860
  const proposalId = `commons-proposal-${randomUUID15()}`;
@@ -31486,6 +31899,10 @@ async function submitCommonsEntity(input, user) {
31486
31899
  ];
31487
31900
  if (nextEntity) {
31488
31901
  batch.push(entityUpsertStatement(nextEntity, existing));
31902
+ if (normalized.claims !== void 0) {
31903
+ batch.push({ sql: "DELETE FROM commons_claims WHERE subject_entity_id = ?", args: [nextEntity.id] });
31904
+ batch.push(...claimInsertStatements(normalized.claims, existingClaims, nextEntity.id, proposalId, Number(user.id), now));
31905
+ }
31489
31906
  batch.push(...commonsIndexStatements(nextEntity, now));
31490
31907
  batch.push({
31491
31908
  sql: `
@@ -31502,10 +31919,19 @@ async function submitCommonsEntity(input, user) {
31502
31919
  actorLabel(input, user),
31503
31920
  normalized.actorChannel,
31504
31921
  existing ? "entity_updated" : "entity_created",
31505
- JSON.stringify(changedFieldPaths(existing, nextEntity)),
31922
+ JSON.stringify([
31923
+ ...changedFieldPaths(existing, nextEntity),
31924
+ ...normalized.claims !== void 0 ? ["claims"] : []
31925
+ ]),
31506
31926
  normalized.changeSummary || (existing ? "Updated Commons entity." : "Created Commons entity."),
31507
- existing ? JSON.stringify(entityLedgerSnapshot(existing)) : null,
31508
- JSON.stringify(entityLedgerSnapshot(nextEntity)),
31927
+ existing ? JSON.stringify({
31928
+ ...entityLedgerSnapshot(existing),
31929
+ ...normalized.claims !== void 0 ? { claims: existingClaims } : {}
31930
+ }) : null,
31931
+ JSON.stringify({
31932
+ ...entityLedgerSnapshot(nextEntity),
31933
+ ...normalized.claims !== void 0 ? { claims: normalized.claims } : {}
31934
+ }),
31509
31935
  now
31510
31936
  ]
31511
31937
  });
@@ -31556,11 +31982,35 @@ async function getCommonsEntityLedger(idOrSlug) {
31556
31982
  ledger: result.rows.map((row) => rowToLedgerEntry(row))
31557
31983
  };
31558
31984
  }
31985
+ async function listCommonsClaims(idOrSlug) {
31986
+ await ensureCommonsSchema();
31987
+ const entity = await getCommonsEntity(idOrSlug);
31988
+ if (!entity) return null;
31989
+ return { entity, claims: await getCommonsClaimsByEntityId(entity.id) };
31990
+ }
31991
+ async function getCommonsEntityLinkset(idOrSlug) {
31992
+ const result = await listCommonsClaims(idOrSlug);
31993
+ if (!result) return null;
31994
+ const targetIds = [...new Set(result.claims.map((claim) => claim.objectEntityId).filter((id) => Boolean(id)))];
31995
+ const targetSlugs = /* @__PURE__ */ new Map();
31996
+ if (targetIds.length) {
31997
+ const targets = await getDb().execute({
31998
+ sql: `SELECT id, slug FROM commons_entities WHERE status = 'published' AND id IN (${targetIds.map(() => "?").join(", ")})`,
31999
+ args: targetIds
32000
+ });
32001
+ for (const row of targets.rows) targetSlugs.set(String(row.id), String(row.slug));
32002
+ }
32003
+ const resolved = result.claims.map((claim) => {
32004
+ const href = claim.objectUrl ?? (claim.objectEntityId && targetSlugs.has(claim.objectEntityId) ? publicUrlForSlug(targetSlugs.get(claim.objectEntityId)) : null);
32005
+ return href ? { ...claim, href } : null;
32006
+ }).filter((claim) => claim !== null);
32007
+ return { entity: result.entity, claims: result.claims, ...buildCommonsLinkset(result.entity, resolved) };
32008
+ }
31559
32009
  async function saveCommonsFilter(input, user) {
31560
32010
  await ensureCommonsSchema();
31561
32011
  const now = (/* @__PURE__ */ new Date()).toISOString();
31562
32012
  const name = requiredText(input.name, "name", 2, 80);
31563
- const description = optionalText(input.description, 500);
32013
+ const description = optionalText2(input.description, 500);
31564
32014
  const filter = sanitizeSearchFilters(input.filter);
31565
32015
  const existing = await getDb().execute({
31566
32016
  sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
@@ -31693,9 +32143,9 @@ function commonsDatabaseReport() {
31693
32143
  conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review.",
31694
32144
  billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
31695
32145
  },
31696
- searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
32146
+ searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "relationship claims", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
31697
32147
  indexing: commonsIndexingPolicy(),
31698
- tables: ["commons_entities", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
32148
+ tables: ["commons_entities", "commons_claims", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
31699
32149
  };
31700
32150
  }
31701
32151
  function entityToWikiPage(entity) {
@@ -31715,6 +32165,13 @@ function entityToWikiPage(entity) {
31715
32165
  media: entity.media,
31716
32166
  sections: entity.articleSections,
31717
32167
  metadata: {
32168
+ linksetUrl: commonsLinksetUrl(entity.id),
32169
+ linksetDiscovery: {
32170
+ rel: "linkset",
32171
+ type: COMMONS_LINKSET_MEDIA_TYPE,
32172
+ profile: COMMONS_RELATIONSHIP_PROFILE,
32173
+ href: commonsLinksetUrl(entity.id)
32174
+ },
31718
32175
  entityType: friendlyEntityType(entity.entityType),
31719
32176
  disambiguationName: entity.disambiguationName,
31720
32177
  keywords: entity.keywords,
@@ -31871,7 +32328,7 @@ async function commonsTagVocabulary() {
31871
32328
  ...parseJson3(record.tags_json, []),
31872
32329
  ...parseJson3(record.keywords_json, [])
31873
32330
  ]) {
31874
- const cleaned = optionalText(tag, 120);
32331
+ const cleaned = optionalText2(tag, 120);
31875
32332
  if (!cleaned) continue;
31876
32333
  counts.set(cleaned, (counts.get(cleaned) ?? 0) + 1);
31877
32334
  }
@@ -31881,7 +32338,7 @@ async function commonsTagVocabulary() {
31881
32338
  function resolveCommonsTags(candidates, vocabulary) {
31882
32339
  const vocabularyByKey = new Map(vocabulary.map((entry) => [entry.tag.toLowerCase(), entry]));
31883
32340
  return candidates.map((candidate) => {
31884
- const cleaned = optionalText(candidate, 120) ?? "";
32341
+ const cleaned = optionalText2(candidate, 120) ?? "";
31885
32342
  const key = cleaned.toLowerCase();
31886
32343
  const exact = vocabularyByKey.get(key);
31887
32344
  if (!cleaned || cleaned.length < 2) {
@@ -31978,12 +32435,13 @@ function normalizeSubmitInput(input) {
31978
32435
  slug: slug2,
31979
32436
  entityId: normalizeEntityId(input.entityId),
31980
32437
  entityType: normalizeEntityType(input.entityType),
31981
- disambiguationName: optionalText(input.disambiguationName, 240) ?? void 0,
32438
+ disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
31982
32439
  featuredImage: normalizeFeaturedImage(input.featuredImage),
31983
32440
  source: normalizeSource(input.source),
31984
32441
  tags: uniqueStrings(input.tags ?? []).slice(0, 24),
31985
32442
  keywords: uniqueStrings(input.keywords ?? []).slice(0, 50),
31986
32443
  relatedEntities: normalizeRelatedEntities(input.relatedEntities),
32444
+ claims: normalizeClaimsForRepository(input.claims),
31987
32445
  bodyMarkdown: normalizeBody(input.bodyMarkdown, input.contentSections),
31988
32446
  contentSections: normalizeContentSections(input.contentSections),
31989
32447
  articleSections: normalizeArticleSections(input.articleSections),
@@ -31992,11 +32450,18 @@ function normalizeSubmitInput(input) {
31992
32450
  seo: normalizeSeo(input.seo),
31993
32451
  actorChannel: input.actorChannel ?? "mcp",
31994
32452
  actorLabel: input.actorLabel ?? "",
31995
- changeSummary: optionalText(input.changeSummary, 500) ?? "",
32453
+ changeSummary: optionalText2(input.changeSummary, 500) ?? "",
31996
32454
  reviewPolicy: input.reviewPolicy ?? "auto_publish_if_safe",
31997
32455
  baseRevision: input.baseRevision
31998
32456
  };
31999
32457
  }
32458
+ function normalizeClaimsForRepository(input) {
32459
+ try {
32460
+ return normalizeCommonsClaims(input);
32461
+ } catch (error) {
32462
+ throw new CommonsRepositoryError("invalid_claims", error instanceof Error ? error.message : "Claims are invalid.");
32463
+ }
32464
+ }
32000
32465
  function evaluatePublishSafety(input, existing) {
32001
32466
  if (!input.featuredImage?.url) return { safe: false, reason: "A featured image is required before a Commons entity can publish." };
32002
32467
  if (!input.bodyMarkdown.trim() && !input.source.originalUrl && !input.source.resolvedUrl) {
@@ -32142,6 +32607,84 @@ function entityStorageValues(entity) {
32142
32607
  entity.revision
32143
32608
  ];
32144
32609
  }
32610
+ function claimInsertStatements(claims, existingClaims, subjectEntityId, proposalId, userId, now) {
32611
+ const existingByIdentity = new Map(existingClaims.map((claim) => [commonsClaimIdentityKey(claim), claim]));
32612
+ return claims.map((claim) => {
32613
+ const existing = existingByIdentity.get(commonsClaimIdentityKey(claim));
32614
+ return {
32615
+ sql: `
32616
+ INSERT INTO commons_claims (
32617
+ id, subject_entity_id, predicate, predicate_label, object_entity_id, object_url,
32618
+ title, media_type, hreflang, qualifiers_json, references_json, rank, confidence,
32619
+ confidence_reason, dispute_state, valid_from, valid_to, as_of, source_family,
32620
+ review_state, proposal_id, created_by_user_id, updated_by_user_id, created_at, updated_at
32621
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'approved', ?, ?, ?, ?, ?)
32622
+ `,
32623
+ args: [
32624
+ existing?.id ?? allocateClaimId(),
32625
+ subjectEntityId,
32626
+ claim.predicate,
32627
+ claim.predicateLabel ?? null,
32628
+ claim.objectEntityId ?? null,
32629
+ claim.objectUrl ?? null,
32630
+ claim.title ?? null,
32631
+ claim.mediaType ?? null,
32632
+ claim.hreflang ? JSON.stringify(claim.hreflang) : null,
32633
+ JSON.stringify(claim.qualifiers ?? {}),
32634
+ JSON.stringify(claim.references ?? []),
32635
+ claim.rank ?? "normal",
32636
+ claim.confidence ?? 1,
32637
+ claim.confidenceReason ?? null,
32638
+ claim.disputeState ?? "undisputed",
32639
+ claim.validFrom ?? null,
32640
+ claim.validTo ?? null,
32641
+ claim.asOf ?? null,
32642
+ claim.sourceFamily ?? "unknown",
32643
+ proposalId,
32644
+ existing?.createdByUserId ?? userId,
32645
+ userId,
32646
+ existing?.createdAt ?? now,
32647
+ now
32648
+ ]
32649
+ };
32650
+ });
32651
+ }
32652
+ async function getCommonsClaimsByEntityId(entityId) {
32653
+ const result = await getDb().execute({
32654
+ sql: "SELECT * FROM commons_claims WHERE subject_entity_id = ? ORDER BY predicate ASC, object_url ASC, object_entity_id ASC, id ASC",
32655
+ args: [entityId]
32656
+ });
32657
+ return result.rows.map((row) => rowToClaim(row));
32658
+ }
32659
+ function rowToClaim(row) {
32660
+ return {
32661
+ id: String(row.id),
32662
+ subjectEntityId: String(row.subject_entity_id),
32663
+ predicate: String(row.predicate),
32664
+ predicateLabel: row.predicate_label == null ? void 0 : String(row.predicate_label),
32665
+ objectEntityId: row.object_entity_id == null ? void 0 : String(row.object_entity_id),
32666
+ objectUrl: row.object_url == null ? void 0 : String(row.object_url),
32667
+ title: row.title == null ? void 0 : String(row.title),
32668
+ mediaType: row.media_type == null ? void 0 : String(row.media_type),
32669
+ hreflang: row.hreflang == null ? void 0 : parseJson3(row.hreflang, []),
32670
+ qualifiers: parseJson3(row.qualifiers_json, {}),
32671
+ references: parseJson3(row.references_json, []),
32672
+ rank: String(row.rank),
32673
+ confidence: Number(row.confidence),
32674
+ confidenceReason: row.confidence_reason == null ? void 0 : String(row.confidence_reason),
32675
+ disputeState: String(row.dispute_state),
32676
+ validFrom: row.valid_from == null ? void 0 : String(row.valid_from),
32677
+ validTo: row.valid_to == null ? void 0 : String(row.valid_to),
32678
+ asOf: row.as_of == null ? void 0 : String(row.as_of),
32679
+ sourceFamily: String(row.source_family),
32680
+ reviewState: String(row.review_state),
32681
+ proposalId: String(row.proposal_id),
32682
+ createdByUserId: row.created_by_user_id == null ? null : Number(row.created_by_user_id),
32683
+ updatedByUserId: row.updated_by_user_id == null ? null : Number(row.updated_by_user_id),
32684
+ createdAt: String(row.created_at),
32685
+ updatedAt: String(row.updated_at)
32686
+ };
32687
+ }
32145
32688
  function commonsIndexStatements(entity, now) {
32146
32689
  return commonsIndexDocuments(entity, now).map((document2) => ({
32147
32690
  sql: `
@@ -32246,7 +32789,7 @@ function commonsIndexDocuments(entity, now) {
32246
32789
  return documents;
32247
32790
  }
32248
32791
  function commonsIndexDocumentId(entityId, documentType, documentKey) {
32249
- return `commons-index-${createHash12("sha256").update(`${entityId}
32792
+ return `commons-index-${createHash13("sha256").update(`${entityId}
32250
32793
  ${documentType}
32251
32794
  ${documentKey}`).digest("hex").slice(0, 32)}`;
32252
32795
  }
@@ -32421,14 +32964,14 @@ function normalizeArticleSections(sections) {
32421
32964
  return {
32422
32965
  relatedLinks: normalizeRelatedLinks(sections?.relatedLinks),
32423
32966
  notes: Array.isArray(sections?.notes) ? sections.notes.map((note) => ({
32424
- marker: optionalText(note.marker, 20) ?? void 0,
32967
+ marker: optionalText2(note.marker, 20) ?? void 0,
32425
32968
  body: requiredText(note.body, "notes.body", 1, 2e3)
32426
32969
  })).slice(0, 100) : [],
32427
32970
  citations: normalizeCitations(sections?.citations),
32428
32971
  externalLinks: Array.isArray(sections?.externalLinks) ? sections.externalLinks.map((link) => ({
32429
32972
  title: requiredText(link.title, "externalLinks.title", 1, 180),
32430
32973
  url: requiredText(link.url, "externalLinks.url", 1, 2e3),
32431
- summary: optionalText(link.summary, 300) ?? void 0
32974
+ summary: optionalText2(link.summary, 300) ?? void 0
32432
32975
  })).slice(0, 100) : [],
32433
32976
  categories: uniqueStrings(sections?.categories ?? []).slice(0, 50)
32434
32977
  };
@@ -32437,7 +32980,7 @@ function normalizeRelatedLinks(relatedLinks) {
32437
32980
  if (!Array.isArray(relatedLinks)) return [];
32438
32981
  return relatedLinks.map((link) => {
32439
32982
  const title = requiredText(link.title, "relatedLinks.title", 1, 180);
32440
- const url = optionalText(link.url, 2e3);
32983
+ const url = optionalText2(link.url, 2e3);
32441
32984
  const slug2 = link.slug ? normalizeSlug(link.slug) : void 0;
32442
32985
  const entityId = normalizeEntityId(link.entityId);
32443
32986
  const needsLink = link.needsLink === true || !url && !slug2 && !entityId;
@@ -32446,9 +32989,9 @@ function normalizeRelatedLinks(relatedLinks) {
32446
32989
  url,
32447
32990
  slug: slug2,
32448
32991
  entityId,
32449
- relationship: optionalText(link.relationship, 120) ?? void 0,
32450
- summary: optionalText(link.summary, 300) ?? void 0,
32451
- imageUrl: optionalText(link.imageUrl, 2e3) ?? void 0,
32992
+ relationship: optionalText2(link.relationship, 120) ?? void 0,
32993
+ summary: optionalText2(link.summary, 300) ?? void 0,
32994
+ imageUrl: optionalText2(link.imageUrl, 2e3) ?? void 0,
32452
32995
  needsLink
32453
32996
  };
32454
32997
  }).slice(0, 100);
@@ -32457,20 +33000,20 @@ function normalizeCitations(citations) {
32457
33000
  if (!Array.isArray(citations)) return [];
32458
33001
  return citations.map((citation) => ({
32459
33002
  title: requiredText(citation.title, "citation.title", 1, 240),
32460
- url: optionalText(citation.url, 2e3) ?? void 0,
32461
- source: optionalText(citation.source, 240) ?? void 0,
32462
- note: optionalText(citation.note, 1e3) ?? void 0,
32463
- accessedAt: optionalText(citation.accessedAt, 80) ?? void 0
33003
+ url: optionalText2(citation.url, 2e3) ?? void 0,
33004
+ source: optionalText2(citation.source, 240) ?? void 0,
33005
+ note: optionalText2(citation.note, 1e3) ?? void 0,
33006
+ accessedAt: optionalText2(citation.accessedAt, 80) ?? void 0
32464
33007
  })).slice(0, 300);
32465
33008
  }
32466
33009
  function normalizeFeaturedImage(image) {
32467
33010
  if (!image?.url) return void 0;
32468
33011
  return {
32469
33012
  url: requiredText(image.url, "featuredImage.url", 1, 2e3),
32470
- alt: optionalText(image.alt, 500) ?? void 0,
32471
- caption: optionalText(image.caption, 1e3) ?? void 0,
32472
- sourceUrl: optionalText(image.sourceUrl, 2e3) ?? void 0,
32473
- license: optionalText(image.license, 240) ?? void 0,
33013
+ alt: optionalText2(image.alt, 500) ?? void 0,
33014
+ caption: optionalText2(image.caption, 1e3) ?? void 0,
33015
+ sourceUrl: optionalText2(image.sourceUrl, 2e3) ?? void 0,
33016
+ license: optionalText2(image.license, 240) ?? void 0,
32474
33017
  width: positiveInt(image.width),
32475
33018
  height: positiveInt(image.height)
32476
33019
  };
@@ -32479,11 +33022,11 @@ function normalizeMedia(media, featuredImage) {
32479
33022
  const normalized = Array.isArray(media) ? media.map((item) => ({
32480
33023
  type: item.type,
32481
33024
  url: requiredText(item.url, "media.url", 1, 2e3),
32482
- alt: optionalText(item.alt, 500) ?? void 0,
32483
- caption: optionalText(item.caption, 1e3) ?? void 0,
32484
- posterUrl: optionalText(item.posterUrl, 2e3) ?? void 0,
32485
- sourceUrl: optionalText(item.sourceUrl, 2e3) ?? void 0,
32486
- license: optionalText(item.license, 240) ?? void 0,
33025
+ alt: optionalText2(item.alt, 500) ?? void 0,
33026
+ caption: optionalText2(item.caption, 1e3) ?? void 0,
33027
+ posterUrl: optionalText2(item.posterUrl, 2e3) ?? void 0,
33028
+ sourceUrl: optionalText2(item.sourceUrl, 2e3) ?? void 0,
33029
+ license: optionalText2(item.license, 240) ?? void 0,
32487
33030
  width: positiveInt(item.width),
32488
33031
  height: positiveInt(item.height),
32489
33032
  durationSeconds: positiveInt(item.durationSeconds)
@@ -32513,28 +33056,28 @@ function normalizeMedia(media, featuredImage) {
32513
33056
  }
32514
33057
  function normalizeSource(source) {
32515
33058
  return {
32516
- originalUrl: optionalText(source?.originalUrl, 2e3) ?? void 0,
32517
- resolvedUrl: optionalText(source?.resolvedUrl, 2e3) ?? void 0,
32518
- sourceCanonicalUrl: optionalText(source?.sourceCanonicalUrl, 2e3) ?? void 0,
32519
- relCanonicalHref: optionalText(source?.relCanonicalHref, 2e3) ?? void 0,
32520
- sourceByline: optionalText(source?.sourceByline, 240) ?? void 0,
32521
- publisher: optionalText(source?.publisher, 240) ?? void 0,
33059
+ originalUrl: optionalText2(source?.originalUrl, 2e3) ?? void 0,
33060
+ resolvedUrl: optionalText2(source?.resolvedUrl, 2e3) ?? void 0,
33061
+ sourceCanonicalUrl: optionalText2(source?.sourceCanonicalUrl, 2e3) ?? void 0,
33062
+ relCanonicalHref: optionalText2(source?.relCanonicalHref, 2e3) ?? void 0,
33063
+ sourceByline: optionalText2(source?.sourceByline, 240) ?? void 0,
33064
+ publisher: optionalText2(source?.publisher, 240) ?? void 0,
32522
33065
  authors: uniqueStrings(source?.authors ?? []).slice(0, 20),
32523
- publishedAt: optionalText(source?.publishedAt, 80) ?? void 0,
32524
- capturedAt: optionalText(source?.capturedAt, 80) ?? void 0,
32525
- license: optionalText(source?.license, 240) ?? void 0,
32526
- rightsSummary: optionalText(source?.rightsSummary, 1e3) ?? void 0,
32527
- contentHash: optionalText(source?.contentHash, 200) ?? void 0
33066
+ publishedAt: optionalText2(source?.publishedAt, 80) ?? void 0,
33067
+ capturedAt: optionalText2(source?.capturedAt, 80) ?? void 0,
33068
+ license: optionalText2(source?.license, 240) ?? void 0,
33069
+ rightsSummary: optionalText2(source?.rightsSummary, 1e3) ?? void 0,
33070
+ contentHash: optionalText2(source?.contentHash, 200) ?? void 0
32528
33071
  };
32529
33072
  }
32530
33073
  function normalizeSeo(seo) {
32531
33074
  return {
32532
- canonicalUrl: optionalText(seo?.canonicalUrl, 2e3) ?? void 0,
32533
- relCanonical: optionalText(seo?.relCanonical, 2e3) ?? void 0,
32534
- metaTitle: optionalText(seo?.metaTitle, 240) ?? void 0,
32535
- metaDescription: optionalText(seo?.metaDescription, 500) ?? void 0,
32536
- ogImage: optionalText(seo?.ogImage, 2e3) ?? void 0,
32537
- schemaOrgType: optionalText(seo?.schemaOrgType, 80) ?? void 0,
33075
+ canonicalUrl: optionalText2(seo?.canonicalUrl, 2e3) ?? void 0,
33076
+ relCanonical: optionalText2(seo?.relCanonical, 2e3) ?? void 0,
33077
+ metaTitle: optionalText2(seo?.metaTitle, 240) ?? void 0,
33078
+ metaDescription: optionalText2(seo?.metaDescription, 500) ?? void 0,
33079
+ ogImage: optionalText2(seo?.ogImage, 2e3) ?? void 0,
33080
+ schemaOrgType: optionalText2(seo?.schemaOrgType, 80) ?? void 0,
32538
33081
  noIndex: seo?.noIndex === true
32539
33082
  };
32540
33083
  }
@@ -32543,37 +33086,37 @@ function normalizeRelatedEntities(related) {
32543
33086
  return related.map((item) => ({
32544
33087
  entityId: normalizeEntityId(item.entityId),
32545
33088
  title: requiredText(item.title, "relatedEntities.title", 1, 180),
32546
- relationship: optionalText(item.relationship, 120) ?? void 0,
33089
+ relationship: optionalText2(item.relationship, 120) ?? void 0,
32547
33090
  slug: item.slug ? normalizeSlug(item.slug) : void 0,
32548
- url: optionalText(item.url, 2e3) ?? void 0,
32549
- description: optionalText(item.description, 300) ?? void 0
33091
+ url: optionalText2(item.url, 2e3) ?? void 0,
33092
+ description: optionalText2(item.description, 300) ?? void 0
32550
33093
  })).slice(0, 200);
32551
33094
  }
32552
33095
  function sanitizeSearchFilters(filters) {
32553
33096
  return {
32554
- query: optionalText(filters.query, 300) ?? void 0,
33097
+ query: optionalText2(filters.query, 300) ?? void 0,
32555
33098
  entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
32556
- tag: optionalText(filters.tag, 80) ?? void 0,
33099
+ tag: optionalText2(filters.tag, 80) ?? void 0,
32557
33100
  tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
32558
- keyword: optionalText(filters.keyword, 120) ?? void 0,
33101
+ keyword: optionalText2(filters.keyword, 120) ?? void 0,
32559
33102
  keywords: uniqueStrings(filters.keywords ?? []).slice(0, 20),
32560
33103
  relatedEntityId: normalizeEntityId(filters.relatedEntityId),
32561
33104
  trailId: normalizeEntityId(filters.trailId),
32562
- sourceDomain: optionalText(filters.sourceDomain, 240) ?? void 0,
33105
+ sourceDomain: optionalText2(filters.sourceDomain, 240) ?? void 0,
32563
33106
  hasMedia: filters.hasMedia === true ? true : void 0,
32564
33107
  hasVideo: filters.hasVideo === true ? true : void 0,
32565
- publishedAfter: optionalText(filters.publishedAfter, 80) ?? void 0,
32566
- updatedAfter: optionalText(filters.updatedAfter, 80) ?? void 0,
32567
- filterId: optionalText(filters.filterId, 200) ?? void 0,
33108
+ publishedAfter: optionalText2(filters.publishedAfter, 80) ?? void 0,
33109
+ updatedAfter: optionalText2(filters.updatedAfter, 80) ?? void 0,
33110
+ filterId: optionalText2(filters.filterId, 200) ?? void 0,
32568
33111
  limit: clampNumber(filters.limit, 1, 100, 20),
32569
33112
  offset: clampNumber(filters.offset, 0, 1e4, 0)
32570
33113
  };
32571
33114
  }
32572
33115
  function sanitizeNeedsLinkFilters(filters) {
32573
33116
  return {
32574
- query: optionalText(filters.query, 300) ?? void 0,
33117
+ query: optionalText2(filters.query, 300) ?? void 0,
32575
33118
  entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
32576
- tag: optionalText(filters.tag, 80) ?? void 0,
33119
+ tag: optionalText2(filters.tag, 80) ?? void 0,
32577
33120
  tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
32578
33121
  sourceEntityId: normalizeEntityId(filters.sourceEntityId),
32579
33122
  sourceSlug: filters.sourceSlug ? normalizeSlug(filters.sourceSlug) : void 0,
@@ -32594,7 +33137,7 @@ function commonsIndexingPolicy() {
32594
33137
  storageScope: "platform_commons_graph",
32595
33138
  countsAgainstUserStorage: false,
32596
33139
  embedsPrivateUserVaults: false,
32597
- corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "See Also concepts marked needsLink"],
33140
+ corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "governed relationship claims", "See Also concepts marked needsLink"],
32598
33141
  excluded: ["comments", "comment threads", "reader alerts", "private reading-room state"],
32599
33142
  retrievalPattern: {
32600
33143
  lexicalFirst: ["title", "tags", "keywords", "source metadata", "article body", "See Also concepts"],
@@ -32670,7 +33213,7 @@ function searchableText(entity) {
32670
33213
  ].filter(Boolean).join("\n").toLowerCase();
32671
33214
  }
32672
33215
  function actorLabel(input, user) {
32673
- return optionalText(input.actorLabel, 120) || optionalText(user.name, 120) || `mcp-user-${user.id}`;
33216
+ return optionalText2(input.actorLabel, 120) || optionalText2(user.name, 120) || `mcp-user-${user.id}`;
32674
33217
  }
32675
33218
  function publicBaseUrl() {
32676
33219
  return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
@@ -32681,13 +33224,16 @@ function publicUrlForSlug(slug2) {
32681
33224
  function allocateEntityId() {
32682
33225
  return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
32683
33226
  }
33227
+ function allocateClaimId() {
33228
+ return `TPW-S${Date.now()}${Math.floor(Math.random() * 1e6).toString().padStart(6, "0")}`;
33229
+ }
32684
33230
  function normalizeEntityId(value) {
32685
- const text2 = optionalText(value, 80);
33231
+ const text2 = optionalText2(value, 80);
32686
33232
  if (!text2) return void 0;
32687
33233
  return text2.replace(/\s+/g, "").toUpperCase();
32688
33234
  }
32689
33235
  function normalizeEntityType(value) {
32690
- const raw = optionalText(value, 120) || DEFAULT_ENTITY_TYPE;
33236
+ const raw = optionalText2(value, 120) || DEFAULT_ENTITY_TYPE;
32691
33237
  const squashed = raw.replace(/[\s_-]+/g, "").toLowerCase();
32692
33238
  if (squashed === "publicarticle" || squashed === "article" || squashed === "item") return DEFAULT_ENTITY_TYPE;
32693
33239
  if (squashed === "trail" || squashed === "readingtrail") return "Trail";
@@ -32715,7 +33261,7 @@ function requiredText(value, field, min, max) {
32715
33261
  if (text2.length < min) throw new CommonsRepositoryError(`${field.replaceAll(".", "_")}_required`, `${field} must be at least ${min} characters.`);
32716
33262
  return text2.slice(0, max);
32717
33263
  }
32718
- function optionalText(value, max) {
33264
+ function optionalText2(value, max) {
32719
33265
  const text2 = String(value ?? "").trim();
32720
33266
  return text2 ? text2.slice(0, max) : void 0;
32721
33267
  }
@@ -32735,7 +33281,7 @@ function uniqueStrings(values) {
32735
33281
  const seen = /* @__PURE__ */ new Set();
32736
33282
  const out = [];
32737
33283
  for (const value of values) {
32738
- const text2 = optionalText(value, 120);
33284
+ const text2 = optionalText2(value, 120);
32739
33285
  if (!text2) continue;
32740
33286
  const key = text2.toLowerCase();
32741
33287
  if (seen.has(key)) continue;
@@ -32782,7 +33328,7 @@ var CommonsRepositoryError = class extends Error {
32782
33328
  };
32783
33329
 
32784
33330
  // src/api/commons-publication-repository.ts
32785
- import { createHash as createHash13, randomUUID as randomUUID16 } from "crypto";
33331
+ import { createHash as createHash14, randomUUID as randomUUID16 } from "crypto";
32786
33332
  var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
32787
33333
  var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
32788
33334
  "admin",
@@ -32980,7 +33526,7 @@ async function publishCommonsEditorial(input, user) {
32980
33526
  JSON.stringify(editionInput.articles),
32981
33527
  html,
32982
33528
  rendered.filename,
32983
- createHash13("sha256").update(html).digest("hex"),
33529
+ createHash14("sha256").update(html).digest("hex"),
32984
33530
  rendered.articleCount,
32985
33531
  rendered.wordCount,
32986
33532
  Buffer.byteLength(html),
@@ -33259,6 +33805,7 @@ var SubmitEntitySchema = z32.object({
33259
33805
  url: z32.string().url().optional(),
33260
33806
  description: z32.string().max(300).optional()
33261
33807
  }).strict()).max(200).optional(),
33808
+ claims: z32.array(CommonsClaimInputSchema).max(300).optional(),
33262
33809
  bodyMarkdown: z32.string().max(5e5).optional(),
33263
33810
  contentSections: z32.array(z32.object({
33264
33811
  id: z32.string().max(180).optional(),
@@ -33336,6 +33883,17 @@ commonsApp.get("/health", async (c) => {
33336
33883
  await ensureCommonsSchema();
33337
33884
  return c.json({ ok: true, data: commonsDatabaseReport() });
33338
33885
  });
33886
+ commonsApp.get("/profiles/relationships/v1", (c) => c.json({
33887
+ id: COMMONS_RELATIONSHIP_PROFILE,
33888
+ title: "Transparent Commons governed relationship linksets",
33889
+ mediaType: COMMONS_LINKSET_MEDIA_TYPE,
33890
+ claimAuthority: "MCP Scraper authenticated proposal and contribution ledger",
33891
+ anchorPolicy: "Every link context is an absolute canonical Transparent Commons wiki URL.",
33892
+ predicatePolicy: "Predicates are curated registered link relations or absolute HTTPS extension relation URIs.",
33893
+ publicationPolicy: "Only approved, active, non-deprecated claims whose dispute state is undisputed or attributed are projected.",
33894
+ trustPolicy: "A third-party target is an asserted relationship, not an endorsement or proof that the target agrees.",
33895
+ source: "https://www.rfc-editor.org/rfc/rfc9264.html"
33896
+ }));
33339
33897
  commonsApp.post("/publications/prepare", auth2, async (c) => {
33340
33898
  const parsed = PreparePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
33341
33899
  if (!parsed.success) return validationError(c, parsed.error);
@@ -33466,11 +34024,32 @@ commonsApp.post("/entities/propose", auth2, async (c) => {
33466
34024
  commonsApp.get("/entities/:idOrSlug", async (c) => {
33467
34025
  const entity = await getCommonsEntity(c.req.param("idOrSlug"));
33468
34026
  if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
34027
+ c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
33469
34028
  return c.json({ ok: true, data: entity });
33470
34029
  });
34030
+ commonsApp.get("/entities/:idOrSlug/claims", async (c) => {
34031
+ const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
34032
+ if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
34033
+ c.header("Link", commonsLinksetDiscoveryHeader(result.entity.id));
34034
+ return c.json({ ok: true, data: { entity: result.entity, claims: result.claims, linksetUrl: result.linksetUrl } });
34035
+ });
34036
+ commonsApp.get("/entities/:idOrSlug/linkset", async (c) => {
34037
+ const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
34038
+ if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
34039
+ c.header("Content-Type", `${COMMONS_LINKSET_MEDIA_TYPE}; profile="${COMMONS_RELATIONSHIP_PROFILE}"`);
34040
+ c.header("ETag", result.etag);
34041
+ c.header("Link", `<${COMMONS_RELATIONSHIP_PROFILE}>; rel="profile"`);
34042
+ c.header("Cache-Control", "public, max-age=60, s-maxage=300");
34043
+ if (c.req.header("If-None-Match") === result.etag) return c.body(null, 304);
34044
+ if (c.req.query("envelope") === "true") {
34045
+ return c.json({ ok: true, data: { ...result, document: result.document } });
34046
+ }
34047
+ return c.body(JSON.stringify(result.document));
34048
+ });
33471
34049
  commonsApp.get("/entities/:idOrSlug/wiki-page", async (c) => {
33472
34050
  const entity = await getCommonsEntity(c.req.param("idOrSlug"));
33473
34051
  if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
34052
+ c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
33474
34053
  return c.json({ ok: true, data: entityToWikiPage(entity) });
33475
34054
  });
33476
34055
  commonsApp.get("/entities/:idOrSlug/json-ld", async (c) => {
@@ -33486,6 +34065,7 @@ commonsApp.get("/entities/:idOrSlug/ledger", async (c) => {
33486
34065
  commonsApp.get("/wiki/:slug", async (c) => {
33487
34066
  const entity = await getCommonsEntity(c.req.param("slug"));
33488
34067
  if (!entity) return c.json({ ok: false, error: "page_not_found", message: "No published Commons wiki page matched that slug." }, 404);
34068
+ c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
33489
34069
  return c.json({
33490
34070
  ok: true,
33491
34071
  data: {
@@ -33587,12 +34167,12 @@ function xmlEscape(value) {
33587
34167
  import { Hono as Hono32 } from "hono";
33588
34168
  import { createMiddleware as createMiddleware4 } from "hono/factory";
33589
34169
  import { z as z34 } from "zod";
33590
- import { createHash as createHash15 } from "crypto";
34170
+ import { createHash as createHash16 } from "crypto";
33591
34171
  import Papa2 from "papaparse";
33592
34172
 
33593
34173
  // src/api/analytics-entitlement.ts
33594
34174
  import {
33595
- createHash as createHash14,
34175
+ createHash as createHash15,
33596
34176
  createHmac as createHmac3,
33597
34177
  randomBytes as randomBytes3,
33598
34178
  timingSafeEqual as timingSafeEqual2
@@ -33807,7 +34387,7 @@ function dashboardCallbackUrl() {
33807
34387
  async function createThorbitConnectUrl(userId) {
33808
34388
  if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
33809
34389
  const state = randomBytes3(32).toString("base64url");
33810
- const stateHash = createHash14("sha256").update(state).digest("hex");
34390
+ const stateHash = createHash15("sha256").update(state).digest("hex");
33811
34391
  await getAnalyticsPool().query(
33812
34392
  `INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
33813
34393
  VALUES ($1,$2,now()+interval '10 minutes')
@@ -33836,7 +34416,7 @@ function verifyThorbitAssertion(token4) {
33836
34416
  return parsed.data.entitlement;
33837
34417
  }
33838
34418
  async function consumeThorbitConnectCallback(input) {
33839
- const stateHash = createHash14("sha256").update(input.state).digest("hex");
34419
+ const stateHash = createHash15("sha256").update(input.state).digest("hex");
33840
34420
  const client2 = await getAnalyticsPool().connect();
33841
34421
  try {
33842
34422
  await client2.query("BEGIN");
@@ -33932,7 +34512,9 @@ var EventSchema = z34.object({
33932
34512
  campaign: z34.string().trim().max(240).optional(),
33933
34513
  clickIds: z34.record(z34.string(), z34.string().trim().max(500)).optional(),
33934
34514
  deviceClass: z34.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
33935
- properties: z34.record(z34.string(), z34.unknown()).optional()
34515
+ properties: z34.record(z34.string(), z34.unknown()).optional(),
34516
+ engagedMs: z34.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
34517
+ scrollDepth: z34.number().int().min(0).max(100).optional()
33936
34518
  }).strict();
33937
34519
  var IngestionSchema = z34.object({
33938
34520
  pixelId: z34.string().trim().min(8).max(160),
@@ -34091,7 +34673,10 @@ analyticsApp.use("*", async (c, next) => {
34091
34673
  await next();
34092
34674
  });
34093
34675
  function analyticsPixelSdk() {
34094
- return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties}}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){current=location.href;api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
34676
+ return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={},over={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties},...over}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};let visibleSince=document.visibilityState==='visible'?Date.now():0,dwellMs=0,maxScroll=0,dwellPath=location.pathname;const depth=()=>{const d=document.documentElement,h=d.scrollHeight||1;return Math.max(0,Math.min(100,Math.round(((d.scrollTop||document.body.scrollTop||0)+innerHeight)/h*100)))};const accrue=()=>{if(visibleSince){dwellMs+=Date.now()-visibleSince;visibleSince=0}};const resume=()=>{if(!visibleSince)visibleSince=Date.now()};const flush=()=>{accrue();const ms=Math.min(dwellMs,18e5),sd=Math.max(maxScroll,depth());dwellMs=0;maxScroll=0;if(ms<250)return;send('page_engagement',{},{path:dwellPath,canonicalUrl:location.origin+dwellPath,engagedMs:ms,scrollDepth:sd})};addEventListener('scroll',()=>{const d=depth();if(d>maxScroll)maxScroll=d},{passive:true});document.addEventListener('visibilitychange',()=>{if(document.visibilityState==='hidden')flush();else resume()});addEventListener('pagehide',flush);const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();resume();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,engagedMs:dwellMs+(visibleSince?Date.now()-visibleSince:0),scrollDepth:Math.max(maxScroll,depth()),...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){flush();current=location.href;dwellPath=location.pathname;resume();api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
34677
+ }
34678
+ function pagePathFilter(value) {
34679
+ return value && value.startsWith("/") ? value : void 0;
34095
34680
  }
34096
34681
  function reportFilters(url) {
34097
34682
  const end = url.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
@@ -34113,7 +34698,7 @@ function reportFilters(url) {
34113
34698
  channelFamily: url.searchParams.get("channelFamily") || void 0,
34114
34699
  platform: url.searchParams.get("platform") || void 0,
34115
34700
  referrer: url.searchParams.get("referrer") || void 0,
34116
- path: url.searchParams.get("path") || void 0,
34701
+ path: pagePathFilter(url.searchParams.get("path")),
34117
34702
  deviceClass: url.searchParams.get("deviceClass") || void 0,
34118
34703
  countryCode: url.searchParams.get("countryCode") || void 0,
34119
34704
  regionCode: url.searchParams.get("regionCode") || void 0,
@@ -34307,7 +34892,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
34307
34892
  const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
34308
34893
  const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
34309
34894
  const identitySeed = email || `${fullName}:${Date.now()}`;
34310
- const suffix2 = createHash15("sha256").update(identitySeed).digest("hex").slice(0, 12);
34895
+ const suffix2 = createHash16("sha256").update(identitySeed).digest("hex").slice(0, 12);
34311
34896
  const path5 = `Leads/person-${suffix2}`;
34312
34897
  const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
34313
34898
  const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
@@ -34406,9 +34991,9 @@ Submitted the published form.
34406
34991
  occurredAt: (/* @__PURE__ */ new Date()).toISOString()
34407
34992
  });
34408
34993
  const match = {
34409
- ...email ? { emailSha256: createHash15("sha256").update(email).digest("hex") } : {},
34994
+ ...email ? { emailSha256: createHash16("sha256").update(email).digest("hex") } : {},
34410
34995
  ...phone ? {
34411
- phoneSha256: createHash15("sha256").update(phone.replace(/\D/g, "")).digest("hex")
34996
+ phoneSha256: createHash16("sha256").update(phone.replace(/\D/g, "")).digest("hex")
34412
34997
  } : {}
34413
34998
  };
34414
34999
  const activationQueued = await queueAnalyticsActivation({
@@ -34836,7 +35421,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
34836
35421
  rejectedCount += 1;
34837
35422
  continue;
34838
35423
  }
34839
- const digest2 = createHash15("sha256").update(
35424
+ const digest2 = createHash16("sha256").update(
34840
35425
  `${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
34841
35426
  ).digest("hex").slice(0, 16);
34842
35427
  const path5 = `Leads/person-${digest2}`;
@@ -35039,14 +35624,13 @@ for (const [path5, load] of [
35039
35624
  ["/sites/:siteId/overview", analyticsOverview],
35040
35625
  ["/sites/:siteId/timeseries", analyticsTimeseries],
35041
35626
  ["/sites/:siteId/acquisition", analyticsAcquisition],
35042
- ["/sites/:siteId/content", analyticsContent],
35043
35627
  ["/sites/:siteId/events", analyticsEventCounts]
35044
35628
  ]) {
35045
35629
  analyticsApp.get(path5, auth3, async (c) => {
35046
35630
  const url = new URL(c.req.url);
35047
35631
  const filters = reportFilters(url);
35048
35632
  validateReportWindow(filters);
35049
- const paginated = path5.endsWith("/acquisition") || path5.endsWith("/content") || path5.endsWith("/events");
35633
+ const paginated = path5.endsWith("/acquisition") || path5.endsWith("/events");
35050
35634
  const data = await load(
35051
35635
  c.req.param("siteId"),
35052
35636
  Number(c.get("user").id),
@@ -35056,6 +35640,24 @@ for (const [path5, load] of [
35056
35640
  return c.json({ ok: true, data, filters });
35057
35641
  });
35058
35642
  }
35643
+ analyticsApp.get("/sites/:siteId/content", auth3, async (c) => {
35644
+ const url = new URL(c.req.url);
35645
+ const filters = reportFilters(url);
35646
+ validateReportWindow(filters);
35647
+ const requestedSort = url.searchParams.get("sort");
35648
+ const data = await analyticsContent(
35649
+ c.req.param("siteId"),
35650
+ Number(c.get("user").id),
35651
+ filters,
35652
+ reportPage(url),
35653
+ {
35654
+ sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
35655
+ direction: url.searchParams.get("direction") === "asc" ? "asc" : "desc",
35656
+ search: url.searchParams.get("search") || void 0
35657
+ }
35658
+ );
35659
+ return c.json({ ok: true, data, filters });
35660
+ });
35059
35661
  analyticsApp.get("/sites/:siteId/dashboard", auth3, async (c) => {
35060
35662
  const url = new URL(c.req.url);
35061
35663
  const filters = reportFilters(url);
@@ -35270,16 +35872,16 @@ import { Hono as Hono33 } from "hono";
35270
35872
  import { z as z35 } from "zod";
35271
35873
 
35272
35874
  // src/api/scheduled-artifact-owner.ts
35273
- import { createHash as createHash16 } from "crypto";
35875
+ import { createHash as createHash17 } from "crypto";
35274
35876
  function scheduledArtifactOwnerIdForApiKey(apiKey) {
35275
- return createHash16("sha256").update(apiKey).digest("hex").slice(0, 24);
35877
+ return createHash17("sha256").update(apiKey).digest("hex").slice(0, 24);
35276
35878
  }
35277
35879
 
35278
35880
  // src/api/scheduled-run-view-links.ts
35279
- import { createHash as createHash17, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
35280
- var schemaReady3 = false;
35881
+ import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
35882
+ var schemaReady2 = false;
35281
35883
  async function ensureScheduledRunViewLinksSchema() {
35282
- if (schemaReady3) return;
35884
+ if (schemaReady2) return;
35283
35885
  const db = getDb();
35284
35886
  await db.execute(`
35285
35887
  CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
@@ -35327,10 +35929,10 @@ async function ensureScheduledRunViewLinksSchema() {
35327
35929
  CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
35328
35930
  ON scheduled_run_view_links(expires_at, revoked_at)
35329
35931
  `);
35330
- schemaReady3 = true;
35932
+ schemaReady2 = true;
35331
35933
  }
35332
35934
  function tokenHash2(token4) {
35333
- return createHash17("sha256").update(token4).digest("hex");
35935
+ return createHash18("sha256").update(token4).digest("hex");
35334
35936
  }
35335
35937
  function mapRow(row) {
35336
35938
  return {
@@ -35417,7 +36019,7 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
35417
36019
  }
35418
36020
 
35419
36021
  // src/personal-authority/render.ts
35420
- import { createHash as createHash18 } from "crypto";
36022
+ import { createHash as createHash19 } from "crypto";
35421
36023
  function escapeHtml7(value) {
35422
36024
  return value.replace(/[&<>"']/g, (character) => ({
35423
36025
  "&": "&amp;",
@@ -35477,13 +36079,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
35477
36079
  html,
35478
36080
  filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
35479
36081
  bytes,
35480
- sha256: createHash18("sha256").update(html).digest("hex"),
36082
+ sha256: createHash19("sha256").update(html).digest("hex"),
35481
36083
  generatedAt: generatedAtIso
35482
36084
  };
35483
36085
  }
35484
36086
 
35485
36087
  // src/newsroom-publisher/render.ts
35486
- import { createHash as createHash19 } from "crypto";
36088
+ import { createHash as createHash20 } from "crypto";
35487
36089
  function escapeHtml8(value) {
35488
36090
  return value.replace(/[&<>"']/g, (character) => ({
35489
36091
  "&": "&amp;",
@@ -35589,7 +36191,7 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
35589
36191
  @media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
35590
36192
  </style></head><body data-theme="${escapeHtml8(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
35591
36193
  const bytes = Buffer.byteLength(html);
35592
- const sha2563 = createHash19("sha256").update(html).digest("hex");
36194
+ const sha2563 = createHash20("sha256").update(html).digest("hex");
35593
36195
  const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
35594
36196
  return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
35595
36197
  }
@@ -35850,7 +36452,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
35850
36452
  import { Hono as Hono34 } from "hono";
35851
36453
 
35852
36454
  // src/scheduled-artifacts/scheduled-run-artifact-store.ts
35853
- import { createHash as createHash20 } from "crypto";
36455
+ import { createHash as createHash21 } from "crypto";
35854
36456
  var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
35855
36457
  var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
35856
36458
  var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
@@ -35866,7 +36468,7 @@ function policy3() {
35866
36468
  };
35867
36469
  }
35868
36470
  function runStorageSegment(runId) {
35869
- return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash20("sha256").update(runId).digest("hex").slice(0, 32)}`;
36471
+ return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash21("sha256").update(runId).digest("hex").slice(0, 32)}`;
35870
36472
  }
35871
36473
  async function createScheduledRunArtifact(args) {
35872
36474
  if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
@@ -35973,17 +36575,17 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
35973
36575
  import { Hono as Hono35 } from "hono";
35974
36576
 
35975
36577
  // src/api/scheduler-integration-auth.ts
35976
- import { createHash as createHash21, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
36578
+ import { createHash as createHash22, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
35977
36579
 
35978
36580
  // src/api/service-connections.ts
35979
36581
  import { randomUUID as randomUUID18 } from "crypto";
35980
- var schemaReady4 = null;
36582
+ var schemaReady3 = null;
35981
36583
  var schemaDb3 = null;
35982
36584
  function ensureServiceConnectionsSchema() {
35983
36585
  const currentDb = getDb();
35984
- if (schemaReady4 && schemaDb3 === currentDb) return schemaReady4;
36586
+ if (schemaReady3 && schemaDb3 === currentDb) return schemaReady3;
35985
36587
  schemaDb3 = currentDb;
35986
- schemaReady4 = (async () => {
36588
+ schemaReady3 = (async () => {
35987
36589
  const db = currentDb;
35988
36590
  await db.execute(`
35989
36591
  CREATE TABLE IF NOT EXISTS service_connections (
@@ -36051,11 +36653,11 @@ function ensureServiceConnectionsSchema() {
36051
36653
  } catch {
36052
36654
  }
36053
36655
  })().catch((error) => {
36054
- schemaReady4 = null;
36656
+ schemaReady3 = null;
36055
36657
  schemaDb3 = null;
36056
36658
  throw error;
36057
36659
  });
36058
- return schemaReady4;
36660
+ return schemaReady3;
36059
36661
  }
36060
36662
  function parseTools(value) {
36061
36663
  if (typeof value !== "string") return [];
@@ -36271,7 +36873,7 @@ function signingSecret() {
36271
36873
  return secret;
36272
36874
  }
36273
36875
  function schedulerIntegrationSignature(args) {
36274
- const bodyHash = createHash21("sha256").update(args.body).digest("hex");
36876
+ const bodyHash = createHash22("sha256").update(args.body).digest("hex");
36275
36877
  return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
36276
36878
  ${args.path}
36277
36879
  ${args.timestamp}
@@ -36527,13 +37129,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
36527
37129
  }
36528
37130
 
36529
37131
  // src/api/page-diff.ts
36530
- import { createHash as createHash22 } from "crypto";
37132
+ import { createHash as createHash23 } from "crypto";
36531
37133
  import { diffLines } from "diff";
36532
37134
  var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
36533
37135
  var MAX_DIFF_HUNKS = 200;
36534
37136
  var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
36535
37137
  function sha256Hex(value) {
36536
- return createHash22("sha256").update(value).digest("hex");
37138
+ return createHash23("sha256").update(value).digest("hex");
36537
37139
  }
36538
37140
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
36539
37141
  if (value.length <= maxChars) return { value, truncated: false };
@@ -36647,10 +37249,10 @@ async function persistScrapeBody(user, opts) {
36647
37249
  }
36648
37250
 
36649
37251
  // src/api/scrape-image-sink.ts
36650
- import { createHash as createHash23 } from "crypto";
37252
+ import { createHash as createHash24 } from "crypto";
36651
37253
  var MAX_IMAGES_PER_SCRAPE = 25;
36652
37254
  function idempotencyKey2(userId, vault, input) {
36653
- return `scrape-image-${createHash23("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
37255
+ return `scrape-image-${createHash24("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
36654
37256
  }
36655
37257
  async function persistScrapeImagesToMemory(user, inputs, vault) {
36656
37258
  const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
@@ -37122,7 +37724,7 @@ function inferredWaybackCapturedAt(content) {
37122
37724
  }
37123
37725
 
37124
37726
  // src/api/connection-memory-import.ts
37125
- import { createHash as createHash24 } from "crypto";
37727
+ import { createHash as createHash25 } from "crypto";
37126
37728
  var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
37127
37729
  var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
37128
37730
  var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
@@ -37152,7 +37754,7 @@ function canonicalJson(value) {
37152
37754
  return JSON.stringify(value);
37153
37755
  }
37154
37756
  function sha2562(value) {
37155
- return createHash24("sha256").update(value).digest("hex");
37757
+ return createHash25("sha256").update(value).digest("hex");
37156
37758
  }
37157
37759
  function sensitiveKey(key) {
37158
37760
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -38163,7 +38765,7 @@ async function dbUsage(identity, plan) {
38163
38765
  }
38164
38766
 
38165
38767
  // src/api/nango-control.ts
38166
- import { createHash as createHash26, randomUUID as randomUUID22 } from "crypto";
38768
+ import { createHash as createHash27, randomUUID as randomUUID22 } from "crypto";
38167
38769
 
38168
38770
  // src/api/slack-archive-analysis.ts
38169
38771
  function isRecord2(value) {
@@ -38537,7 +39139,7 @@ async function exportSlackChannelPage(input, dependencies) {
38537
39139
  }
38538
39140
 
38539
39141
  // src/api/main-nango-transport.ts
38540
- import { createHash as createHash25, randomUUID as randomUUID21 } from "crypto";
39142
+ import { createHash as createHash26, randomUUID as randomUUID21 } from "crypto";
38541
39143
  import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
38542
39144
 
38543
39145
  // src/api/connected-tool-policy.ts
@@ -38930,7 +39532,7 @@ async function listNangoToolsDirect(identity, connectionId) {
38930
39532
  });
38931
39533
  const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
38932
39534
  const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
38933
- const revision = createHash25("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
39535
+ const revision = createHash26("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
38934
39536
  await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
38935
39537
  const refreshed = await getOwnedServiceConnection(identity, connection.id);
38936
39538
  return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
@@ -38958,7 +39560,7 @@ async function callNangoToolDirect(args) {
38958
39560
  ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
38959
39561
  });
38960
39562
  const requestId = args.requestId?.trim() || randomUUID21();
38961
- const idempotencyKey3 = `main-nango:${createHash25("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
39563
+ const idempotencyKey3 = `main-nango:${createHash26("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
38962
39564
  const startedAt = /* @__PURE__ */ new Date();
38963
39565
  const started = performance.now();
38964
39566
  let result;
@@ -38983,7 +39585,7 @@ async function callNangoToolDirect(args) {
38983
39585
  toolName: args.tool,
38984
39586
  operationKind: args.operationKind ?? args.classification,
38985
39587
  outcome: providerError ? "error" : "partial",
38986
- requestId: requestId.length <= 200 ? requestId : createHash25("sha256").update(requestId).digest("hex"),
39588
+ requestId: requestId.length <= 200 ? requestId : createHash26("sha256").update(requestId).digest("hex"),
38987
39589
  startedAt: startedAt.toISOString(),
38988
39590
  completedAt: completedAt.toISOString()
38989
39591
  }
@@ -39025,7 +39627,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
39025
39627
  providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
39026
39628
  protocolVersion: null,
39027
39629
  schemaSource: "live_tools_list",
39028
- schemaHash: createHash25("sha256").update(JSON.stringify(projected)).digest("hex"),
39630
+ schemaHash: createHash26("sha256").update(JSON.stringify(projected)).digest("hex"),
39029
39631
  fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
39030
39632
  };
39031
39633
  }
@@ -39749,8 +40351,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
39749
40351
  return data.connection.actionsEnabled === true;
39750
40352
  }
39751
40353
  async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
39752
- const requestId = `main-connected-action:${createHash26("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
39753
- const requestDigest = createHash26("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
40354
+ const requestId = `main-connected-action:${createHash27("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
40355
+ const requestDigest = createHash27("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
39754
40356
  if (mainOwnsIntegrations()) {
39755
40357
  const selectedTool = tool?.trim();
39756
40358
  if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
@@ -39901,7 +40503,7 @@ function canonicalJson2(value) {
39901
40503
  return JSON.stringify(value);
39902
40504
  }
39903
40505
  function projectedToolSchemaHash(tool) {
39904
- return createHash26("sha256").update(canonicalJson2(tool)).digest("hex");
40506
+ return createHash27("sha256").update(canonicalJson2(tool)).digest("hex");
39905
40507
  }
39906
40508
  async function describeNangoTool(identity, connectionId, tool, fresh) {
39907
40509
  if (mainOwnsIntegrations()) {
@@ -40180,7 +40782,7 @@ async function callMainOwnedExportPage(identity, input) {
40180
40782
  }
40181
40783
 
40182
40784
  // src/api/resend-control.ts
40183
- import { createHash as createHash27 } from "crypto";
40785
+ import { createHash as createHash28 } from "crypto";
40184
40786
  var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
40185
40787
  var RESEND_PROVIDER_CONFIG_KEY = "resend";
40186
40788
  var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
@@ -40439,7 +41041,7 @@ async function callResendRead(identity, connectionId, tool, args) {
40439
41041
  return isRecord5(data) ? data.result ?? data : data;
40440
41042
  }
40441
41043
  async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
40442
- const requestId = `main-resend-action:${createHash27("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
41044
+ const requestId = `main-resend-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
40443
41045
  const body = await controlRequest2("/api/internal/resend/actions/call", {
40444
41046
  method: "POST",
40445
41047
  headers: { "x-request-id": requestId },
@@ -43486,35 +44088,73 @@ app.post("/billing/credits", auth4, async (c) => {
43486
44088
  ledger
43487
44089
  });
43488
44090
  });
44091
+ var CRON_TICK_BUDGET_MS = 5e4;
44092
+ var CRON_TICK_DRAIN_BUDGET_MS = 4e4;
44093
+ function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
44094
+ return new Promise((resolve2) => {
44095
+ const timer = setTimeout(() => {
44096
+ unfinished.push(`${label}:deadline`);
44097
+ resolve2(onDeadlineOrError);
44098
+ }, CRON_TICK_BUDGET_MS);
44099
+ work.then(
44100
+ (value) => {
44101
+ clearTimeout(timer);
44102
+ resolve2(value);
44103
+ },
44104
+ (err) => {
44105
+ clearTimeout(timer);
44106
+ unfinished.push(`${label}:error`);
44107
+ console.error(JSON.stringify({
44108
+ event: "cron_tick_job_failed",
44109
+ job: label,
44110
+ message: err instanceof Error ? err.message : String(err)
44111
+ }));
44112
+ resolve2(onDeadlineOrError);
44113
+ }
44114
+ );
44115
+ });
44116
+ }
43489
44117
  app.get("/cron/tick", async (c) => {
43490
44118
  const secret = c.req.header("authorization");
43491
44119
  if (!process.env.CRON_SECRET || secret !== `Bearer ${process.env.CRON_SECRET}`) {
43492
44120
  return c.json({ error: "Unauthorized" }, 401);
43493
44121
  }
43494
- const { drainQueue } = await import("./worker-4MHZS3OB.js");
43495
- const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
43496
- const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
43497
- const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery] = await Promise.all([
43498
- drainQueue(budget),
43499
- runMonthlyRefreshSweep(),
43500
- reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
43501
- expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
43502
- cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" })),
43503
- cleanupExpiredConnectedDataArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43504
- reconcileConnectedAccountBillingSweep().catch(() => ({ checked: 0, synced: 0, failed: 0 })),
43505
- dispatchPendingDirectoryWorkflows(25).catch(() => ({ claimed: 0, dispatched: 0, failed: 0 })),
43506
- reconcileDirectoryWorkflowSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
43507
- cleanupExpiredDirectoryArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43508
- cleanupExpiredSiteExtractArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43509
- redispatchFundedSiteExtractJobs(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43510
- reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
43511
- redispatchPendingPaaHarvests(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43512
- reconcilePaaHarvestSettlements(25).catch(() => ({ checked: 0, staleFailed: 0, settled: 0, failed: 0 })),
43513
- redispatchQueuedLocalSourcebookAcquisitions(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43514
- refreshAnalyticsDailyRollupsIfDue().catch(() => ({ status: "failed", rows: 0 })),
43515
- deliverPendingAnalyticsForms(10).catch(() => ({ claimed: 0, delivered: 0, deferred: 0 }))
44122
+ const startedAt = Date.now();
44123
+ const { drainQueue } = await import("./worker-KQN673JF.js");
44124
+ const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
44125
+ const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
44126
+ const unfinished = [];
44127
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
44128
+ settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
44129
+ settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
44130
+ settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
44131
+ settleWithinTickBudget("retryOrphanedBrowserSessions", unfinished, retryOrphanedBrowserSessions(50), { recovered: 0, stillOpen: 0 }),
44132
+ settleWithinTickBudget("expireLots", unfinished, expireOldLots(), { expired_lots: 0, expired_mc: 0 }),
44133
+ settleWithinTickBudget("scrapeBlobCleanup", unfinished, cleanupExpiredScrapeBlobs(), { deleted: 0, store: "none" }),
44134
+ settleWithinTickBudget("connectedDataCleanup", unfinished, cleanupExpiredConnectedDataArtifacts(), { deleted: 0, store: "none" }),
44135
+ settleWithinTickBudget("connectedBillingSweep", unfinished, reconcileConnectedAccountBillingSweep(), { checked: 0, synced: 0, failed: 0 }),
44136
+ settleWithinTickBudget("directoryDispatch", unfinished, dispatchPendingDirectoryWorkflows(5), { claimed: 0, dispatched: 0, failed: 0 }),
44137
+ settleWithinTickBudget("directoryReconcile", unfinished, reconcileDirectoryWorkflowSettlements(5), { checked: 0, settled: 0, failed: 0 }),
44138
+ settleWithinTickBudget("directoryArtifactCleanup", unfinished, cleanupExpiredDirectoryArtifacts(), { deleted: 0, store: "none" }),
44139
+ settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
44140
+ settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
44141
+ settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
44142
+ settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
44143
+ settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
44144
+ settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
44145
+ settleWithinTickBudget("analyticsRollup", unfinished, refreshAnalyticsDailyRollupsIfDue(), { status: "failed", rows: 0 }),
44146
+ settleWithinTickBudget("analyticsForms", unfinished, deliverPendingAnalyticsForms(10), { claimed: 0, delivered: 0, deferred: 0 }),
44147
+ settleWithinTickBudget("workflowDispatch", unfinished, dispatchDueWorkflowSchedules(origin), { dispatched: 0, status: "skipped_deadline" }),
44148
+ settleWithinTickBudget("scheduledRunReconcile", unfinished, reconcileScheduledRunSettlements(25), { checked: 0, settled: 0, chargedMc: 0, failed: 0 }),
44149
+ settleWithinTickBudget("retentionSweep", unfinished, sweepRetention(), { deleted: 0, perTable: {}, truncated: false })
43516
44150
  ]);
43517
44151
  return c.json({
44152
+ durationMs: Date.now() - startedAt,
44153
+ tickBudgetMs: CRON_TICK_BUDGET_MS,
44154
+ unfinished,
44155
+ complete: unfinished.length === 0,
44156
+ scheduledRunReconciliation,
44157
+ retentionSweep,
43518
44158
  drained: results.length,
43519
44159
  results,
43520
44160
  sweepResult,
@@ -43550,8 +44190,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
43550
44190
  finishExtractJob: finishExtractJob2,
43551
44191
  terminalExtractJobStatus: terminalExtractJobStatus2,
43552
44192
  extractJobLimitInfo: extractJobLimitInfo2
43553
- } = await import("./site-extract-repository-STNOI5OH.js");
43554
- const { assembleExtractArtifacts } = await import("./extract-bundle-2HONVC5T.js");
44193
+ } = await import("./site-extract-repository-3VQAALNR.js");
44194
+ const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
43555
44195
  const existing = await getExtractJob2(jobId2);
43556
44196
  if (!existing) return c.json({ error: "job not found" }, 404);
43557
44197
  if (existing.status !== "failed") {
@@ -43578,7 +44218,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
43578
44218
  if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
43579
44219
  let settlement = "already_settled_or_refunded";
43580
44220
  if (job.billedMc == null && job.userId != null) {
43581
- const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-STNOI5OH.js");
44221
+ const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-3VQAALNR.js");
43582
44222
  const heldMc = Number(job.options.heldMc ?? 0);
43583
44223
  const successful = await countSuccessfulPages2(jobId2);
43584
44224
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -43763,4 +44403,4 @@ app.get("/blog/:slug/", (c) => {
43763
44403
  export {
43764
44404
  app
43765
44405
  };
43766
- //# sourceMappingURL=server-EVTCL3KB.js.map
44406
+ //# sourceMappingURL=server-3ND2EXYO.js.map