mcp-scraper 0.49.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +1 -1
  2. package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
  3. package/dist/bin/api-server.cjs +573 -139
  4. package/dist/bin/api-server.cjs.map +1 -1
  5. package/dist/bin/api-server.js +4 -4
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  7. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  8. package/dist/bin/mcp-scraper-cli.js +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +1 -1
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +1 -1
  12. package/dist/bin/mcp-stdio-server.cjs +30 -6
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +4 -4
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
  18. package/dist/chunk-2XTYLJQK.js.map +1 -0
  19. package/dist/{chunk-64WBUDPC.js → chunk-44LNKZTE.js} +32 -8
  20. package/dist/chunk-44LNKZTE.js.map +1 -0
  21. package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
  22. package/dist/chunk-5BEPOFKG.js.map +1 -0
  23. package/dist/chunk-C7NN57WH.js +7 -0
  24. package/dist/chunk-C7NN57WH.js.map +1 -0
  25. package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
  26. package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
  27. package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
  28. package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
  29. package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
  30. package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
  31. package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
  32. package/dist/index.cjs.map +1 -1
  33. package/dist/index.js +3 -3
  34. package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
  35. package/dist/{server-JE7SLZLJ.js → server-3ND2EXYO.js} +364 -128
  36. package/dist/server-3ND2EXYO.js.map +1 -0
  37. package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
  38. package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
  39. package/package.json +1 -1
  40. package/dist/chunk-64WBUDPC.js.map +0 -1
  41. package/dist/chunk-MFNGUM4L.js.map +0 -1
  42. package/dist/chunk-XLEDHV3J.js +0 -7
  43. package/dist/chunk-XLEDHV3J.js.map +0 -1
  44. package/dist/chunk-YW2LXLDU.js.map +0 -1
  45. package/dist/server-JE7SLZLJ.js.map +0 -1
  46. /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
  47. /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
  48. /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
  49. /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
  50. /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
  51. /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
  52. /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
  53. /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
  54. /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
  55. /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
  56. /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
@@ -10,9 +10,11 @@ import {
10
10
  importHostedCensusPlacesCsv,
11
11
  importHostedZipGroupsCsv,
12
12
  listActiveHostedLocationDatasets
13
- } from "./chunk-R66PJOZW.js";
13
+ } from "./chunk-NNZXZTJ2.js";
14
14
  import {
15
+ ANALYTICS_CONTENT_SORTS,
15
16
  AnalyticsRepositoryError,
17
+ MAX_ENGAGED_MS,
16
18
  analyticsAcquisition,
17
19
  analyticsBusinessMetrics,
18
20
  analyticsChannelBreakdown,
@@ -64,14 +66,14 @@ import {
64
66
  updateAnalyticsPixel,
65
67
  upsertAnalyticsAdSpend,
66
68
  verifySession
67
- } from "./chunk-YW2LXLDU.js";
69
+ } from "./chunk-5BEPOFKG.js";
68
70
  import {
69
71
  classifyHarvestProblem,
70
72
  createHarvestAttemptRecorder,
71
73
  deliverWebhook,
72
74
  harvestProblemResponse,
73
75
  serializeHarvestProblem
74
- } from "./chunk-ZEMHL36J.js";
76
+ } from "./chunk-YMSZO62N.js";
75
77
  import {
76
78
  AIModeSelectors,
77
79
  AIOverviewSelectors,
@@ -98,7 +100,7 @@ import {
98
100
  resolveKernelProxyId,
99
101
  serpArrivalUrl,
100
102
  withRuntimeUserAgent
101
- } from "./chunk-MFE6RMIA.js";
103
+ } from "./chunk-GOZIG6HD.js";
102
104
  import {
103
105
  ApiErrorResponseSchema,
104
106
  ArtifactTemplatePresetKeySchema,
@@ -156,7 +158,7 @@ import {
156
158
  resolveDeploymentProfile,
157
159
  resolveLocalSourcebookSchemaType,
158
160
  transcribeMediaUrl
159
- } from "./chunk-64WBUDPC.js";
161
+ } from "./chunk-44LNKZTE.js";
160
162
  import {
161
163
  auditImageUrls,
162
164
  auditImages,
@@ -184,7 +186,7 @@ import {
184
186
  recordKernelSession,
185
187
  recordVendorUsage,
186
188
  runWithCostContext
187
- } from "./chunk-PODASLGT.js";
189
+ } from "./chunk-QXAY44SA.js";
188
190
  import {
189
191
  listWorkflowDefinitions,
190
192
  runWorkflow,
@@ -212,7 +214,7 @@ import {
212
214
  } from "./chunk-3ZUBQQPQ.js";
213
215
  import {
214
216
  PACKAGE_VERSION
215
- } from "./chunk-XLEDHV3J.js";
217
+ } from "./chunk-C7NN57WH.js";
216
218
  import {
217
219
  abandonExtractSettlement,
218
220
  countSuccessfulPages,
@@ -235,7 +237,7 @@ import {
235
237
  setExtractJobTotal,
236
238
  settleExtractJob,
237
239
  terminalExtractJobStatus
238
- } from "./chunk-6DPIE262.js";
240
+ } from "./chunk-QHVFKFME.js";
239
241
  import {
240
242
  BROWSER_OPEN_MIN_BALANCE_MC,
241
243
  CONCURRENCY_PRICE_ID,
@@ -392,7 +394,7 @@ import {
392
394
  startHarvestAttempt,
393
395
  upsertPageSnapshot,
394
396
  verifyPassword
395
- } from "./chunk-MFNGUM4L.js";
397
+ } from "./chunk-2XTYLJQK.js";
396
398
 
397
399
  // src/blog/registry.ts
398
400
  var posts = [
@@ -8738,7 +8740,7 @@ var siteExtractFn = inngest.createFunction(
8738
8740
  });
8739
8741
  }) : null;
8740
8742
  const finalized = await step.run("finalize", async () => {
8741
- const { assembleExtractArtifacts } = await import("./extract-bundle-2HONVC5T.js");
8743
+ const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
8742
8744
  const current = await getExtractJob(jobId2);
8743
8745
  if (!current) throw new Error("extract job disappeared before finalization");
8744
8746
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -9680,7 +9682,7 @@ async function resolveUsZipsPath(requestedPath) {
9680
9682
  async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
9681
9683
  const mode = resolvedLocationDataMode({ zipSource: sourceMode });
9682
9684
  if (mode === "hosted") {
9683
- const { getHostedZipGroups } = await import("./location-data-repository-YWFJGNIP.js");
9685
+ const { getHostedZipGroups } = await import("./location-data-repository-UUSHH5M2.js");
9684
9686
  const hosted = await getHostedZipGroups(stateAbbr);
9685
9687
  if (!hosted.dataset) {
9686
9688
  throw new DirectoryLocationError(
@@ -9743,7 +9745,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
9743
9745
  const sourceUrl = censusStateUrl(state.fips);
9744
9746
  const warnings = [];
9745
9747
  if (resolvedLocationDataMode(config) === "hosted") {
9746
- const { queryHostedLocationMarkets } = await import("./location-data-repository-YWFJGNIP.js");
9748
+ const { queryHostedLocationMarkets } = await import("./location-data-repository-UUSHH5M2.js");
9747
9749
  const hosted = await queryHostedLocationMarkets({
9748
9750
  state: state.abbr,
9749
9751
  minPopulation: options.minPopulation,
@@ -10732,7 +10734,6 @@ import { randomUUID as randomUUID2 } from "crypto";
10732
10734
  var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
10733
10735
  var DEFAULT_RETRY_AFTER_SECONDS = 30;
10734
10736
  var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
10735
- var schemaReady = null;
10736
10737
  function concurrencyLimitForUser(user) {
10737
10738
  const tier = Math.max(0, Number(user.subscription_concurrency ?? 0));
10738
10739
  return Math.max(1, tier) + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
@@ -10770,40 +10771,15 @@ function safeMetadata(metadata) {
10770
10771
  return null;
10771
10772
  }
10772
10773
  }
10773
- async function ensureConcurrencyGateSchema() {
10774
- if (!schemaReady) {
10775
- schemaReady = (async () => {
10776
- const db = getDb();
10777
- await db.execute(`
10778
- CREATE TABLE IF NOT EXISTS concurrency_locks (
10779
- id TEXT PRIMARY KEY,
10780
- user_id INTEGER NOT NULL REFERENCES users(id),
10781
- operation TEXT NOT NULL,
10782
- status TEXT NOT NULL DEFAULT 'active',
10783
- acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
10784
- expires_at TEXT NOT NULL,
10785
- released_at TEXT,
10786
- metadata TEXT
10787
- )
10788
- `);
10789
- await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
10790
- await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
10791
- })();
10792
- }
10793
- return schemaReady;
10794
- }
10795
10774
  async function expireConcurrencyLocksForUser(userId) {
10796
- await ensureConcurrencyGateSchema();
10797
10775
  await getDb().execute({
10798
- sql: `UPDATE concurrency_locks
10799
- SET status = 'expired', released_at = datetime('now')
10776
+ sql: `DELETE FROM concurrency_locks
10800
10777
  WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
10801
10778
  args: [userId]
10802
10779
  });
10803
10780
  }
10804
- async function countActiveUsageForUser(userId) {
10805
- await ensureConcurrencyGateSchema();
10806
- await expireConcurrencyLocksForUser(userId);
10781
+ async function countActiveUsageForUser(userId, alreadyExpired = false) {
10782
+ if (!alreadyExpired) await expireConcurrencyLocksForUser(userId);
10807
10783
  const res = await getDb().execute({
10808
10784
  sql: `SELECT
10809
10785
  (
@@ -10823,7 +10799,6 @@ async function countActiveUsageForUser(userId) {
10823
10799
  }
10824
10800
  async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
10825
10801
  if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
10826
- await ensureConcurrencyGateSchema();
10827
10802
  await expireConcurrencyLocksForUser(userId);
10828
10803
  const res = await getDb().execute({
10829
10804
  sql: `UPDATE concurrency_locks
@@ -10837,13 +10812,12 @@ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
10837
10812
  return Number(res.rowsAffected ?? 0) > 0;
10838
10813
  }
10839
10814
  async function acquireConcurrencyGate(user, operation, options = {}) {
10840
- await ensureConcurrencyGateSchema();
10841
10815
  const limit = concurrencyLimitForUser(user);
10842
10816
  if (isConcurrencyLimitBypassed(user)) {
10843
10817
  return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
10844
10818
  }
10845
10819
  if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
10846
- return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
10820
+ return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id, true), limit, operation, reused: true };
10847
10821
  }
10848
10822
  await expireConcurrencyLocksForUser(user.id);
10849
10823
  const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
@@ -10874,7 +10848,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
10874
10848
  limit
10875
10849
  ]
10876
10850
  });
10877
- const active = await countActiveUsageForUser(user.id);
10851
+ const active = await countActiveUsageForUser(user.id, true);
10878
10852
  if (Number(res.rowsAffected ?? 0) > 0) {
10879
10853
  return { ok: true, lockId, active, limit, operation };
10880
10854
  }
@@ -10882,17 +10856,13 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
10882
10856
  }
10883
10857
  async function releaseConcurrencyGate(lockId) {
10884
10858
  if (!lockId) return;
10885
- await ensureConcurrencyGateSchema();
10886
10859
  await getDb().execute({
10887
- sql: `UPDATE concurrency_locks
10888
- SET status = 'released', released_at = datetime('now')
10889
- WHERE id = ? AND status = 'active'`,
10860
+ sql: `DELETE FROM concurrency_locks WHERE id = ?`,
10890
10861
  args: [lockId]
10891
10862
  });
10892
10863
  }
10893
10864
  async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
10894
10865
  if (!lockId) return;
10895
- await ensureConcurrencyGateSchema();
10896
10866
  await getDb().execute({
10897
10867
  sql: `UPDATE concurrency_locks
10898
10868
  SET expires_at = datetime('now', ?)
@@ -14709,7 +14679,7 @@ async function recoverSettlement(row, user, authorization, duplicate) {
14709
14679
  }
14710
14680
  }
14711
14681
  await getDb().execute({
14712
- sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status = 'settling'",
14682
+ sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('settling', 'cost_pending')",
14713
14683
  args: [row.id]
14714
14684
  });
14715
14685
  await getDb().execute({
@@ -14754,11 +14724,13 @@ async function settleScheduledRun(args) {
14754
14724
  const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
14755
14725
  const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
14756
14726
  const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
14757
- const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
14727
+ const modelCostUnreported = args.modelCostStatus === "missing";
14728
+ const pendingReason = modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
14758
14729
  const metadata = safeMetadata2({
14759
14730
  ...args.metadata,
14760
14731
  billingPhase: "settlement",
14761
14732
  chargeMode: "model_delta",
14733
+ ...modelCostUnreported ? { modelCostUnreported: true } : {},
14762
14734
  ...pendingReason ? { reason: pendingReason } : {}
14763
14735
  });
14764
14736
  const eventId = existing?.id ?? randomUUID4();
@@ -14802,6 +14774,72 @@ async function settleScheduledRun(args) {
14802
14774
  }
14803
14775
  return recoverSettlement(existing, user, authorization, duplicateClaim);
14804
14776
  }
14777
+ async function reconcileScheduledRunSettlements(limit = 25, minimumAgeHours = 24) {
14778
+ const stale = await getDb().execute({
14779
+ sql: `SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata
14780
+ FROM billing_events
14781
+ WHERE billing_class = ? AND status = 'cost_pending'
14782
+ AND created_at < datetime('now', ?)
14783
+ ORDER BY created_at
14784
+ LIMIT ?`,
14785
+ args: [SCHEDULED_RUN_BILLING_CLASS, `-${Math.max(1, Math.trunc(minimumAgeHours))} hours`, Math.max(1, Math.trunc(limit))]
14786
+ });
14787
+ const result = { checked: stale.rows.length, settled: 0, chargedMc: 0, failed: 0 };
14788
+ for (const raw of stale.rows) {
14789
+ const row = raw;
14790
+ try {
14791
+ const authorization = await authorizationById(row.authorization_id);
14792
+ if (!authorization) {
14793
+ result.failed += 1;
14794
+ continue;
14795
+ }
14796
+ const cap = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
14797
+ const chargeMc = Math.min(Math.max(0, Math.round(Number(row.amount_mc))), cap);
14798
+ if (chargeMc > 0) {
14799
+ const debit = await debitMcIdempotent(
14800
+ Number(authorization.user_id),
14801
+ chargeMc,
14802
+ LedgerOperation.SCHEDULED_RUN,
14803
+ "scheduled-run:" + authorization.id + ":model:reconciled",
14804
+ authorization.id + ":model"
14805
+ );
14806
+ if (!debit.ok) {
14807
+ result.failed += 1;
14808
+ continue;
14809
+ }
14810
+ }
14811
+ await getDb().execute({
14812
+ sql: `UPDATE billing_events
14813
+ SET status = 'settled', amount_mc = ?, settled_at = datetime('now'), metadata = ?
14814
+ WHERE id = ? AND status = 'cost_pending'`,
14815
+ args: [
14816
+ chargeMc,
14817
+ safeMetadata2({ ...metadataReason(row) ? { reason: metadataReason(row) } : {}, autoReconciled: true, cappedAtAuthorizedMaximum: chargeMc < Number(row.amount_mc) }),
14818
+ row.id
14819
+ ]
14820
+ });
14821
+ await getDb().execute({
14822
+ sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending')",
14823
+ args: [authorization.id]
14824
+ });
14825
+ result.settled += 1;
14826
+ result.chargedMc += chargeMc;
14827
+ console.error(JSON.stringify({
14828
+ event: "scheduled_run_settlement_reconciled",
14829
+ authorization_id: authorization.id,
14830
+ charged_mc: chargeMc,
14831
+ reason: metadataReason(row) ?? "unreported_cost"
14832
+ }));
14833
+ } catch (err) {
14834
+ result.failed += 1;
14835
+ console.error(JSON.stringify({
14836
+ event: "scheduled_run_reconciliation_failed",
14837
+ message: err instanceof Error ? err.message : String(err)
14838
+ }));
14839
+ }
14840
+ }
14841
+ return result;
14842
+ }
14805
14843
  async function voidScheduledRunAuthorization(args) {
14806
14844
  const user = await resolveSchedulingUser(args.identity);
14807
14845
  const authorization = await authorizationById(args.authorizationId);
@@ -15673,28 +15711,6 @@ internalInboxApp.post("/broadcast", async (c) => {
15673
15711
 
15674
15712
  // src/api/connected-account-billing.ts
15675
15713
  var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
15676
- async function ensureConnectedAccountBillingSchema() {
15677
- await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
15678
- user_id INTEGER PRIMARY KEY,
15679
- stripe_subscription_id TEXT,
15680
- stripe_subscription_item_id TEXT,
15681
- price_id TEXT,
15682
- quantity INTEGER NOT NULL DEFAULT 0,
15683
- status TEXT NOT NULL DEFAULT 'pending',
15684
- last_error_code TEXT,
15685
- checked_at TEXT NOT NULL DEFAULT (datetime('now')),
15686
- synced_at TEXT,
15687
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
15688
- )`);
15689
- await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_charges (
15690
- user_id INTEGER NOT NULL,
15691
- connection_id TEXT NOT NULL,
15692
- period_start TEXT NOT NULL,
15693
- amount_mc INTEGER NOT NULL,
15694
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
15695
- PRIMARY KEY (user_id, connection_id, period_start)
15696
- )`);
15697
- }
15698
15714
  function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
15699
15715
  return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
15700
15716
  }
@@ -15703,7 +15719,6 @@ function isBillableConnection(connection) {
15703
15719
  return BILLABLE_LIFECYCLE.has(String(connection.lifecycleStatus ?? ""));
15704
15720
  }
15705
15721
  async function chargedConnectionIds(userId, periodStart) {
15706
- await ensureConnectedAccountBillingSchema();
15707
15722
  const result = await getDb().execute({
15708
15723
  sql: "SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",
15709
15724
  args: [Number(userId), periodStart]
@@ -15711,7 +15726,6 @@ async function chargedConnectionIds(userId, periodStart) {
15711
15726
  return new Set(result.rows.map((row) => String(row.connection_id)));
15712
15727
  }
15713
15728
  async function getConnectedAccountBillingState(userId) {
15714
- await ensureConnectedAccountBillingSchema();
15715
15729
  const result = await getDb().execute({
15716
15730
  sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
15717
15731
  args: [Number(userId)]
@@ -15719,7 +15733,6 @@ async function getConnectedAccountBillingState(userId) {
15719
15733
  return result.rows[0] ? result.rows[0] : null;
15720
15734
  }
15721
15735
  async function setConnectedAccountBillingState(input) {
15722
- await ensureConnectedAccountBillingSchema();
15723
15736
  await getDb().execute({
15724
15737
  sql: `INSERT INTO connected_account_billing
15725
15738
  (user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, checked_at, synced_at, updated_at)
@@ -22185,6 +22198,82 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
22185
22198
  return { checked: jobs.length, staleFailed, settled, failed };
22186
22199
  }
22187
22200
 
22201
+ // src/api/retention-sweeps.ts
22202
+ var REPORTING_WINDOW_DAYS = 30;
22203
+ var TELEMETRY_RETENTION_DAYS = 45;
22204
+ var FAILURE_RETENTION_DAYS = 90;
22205
+ var RETENTION_RULES = [
22206
+ { table: "request_events", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22207
+ { table: "kernel_session_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22208
+ { table: "vendor_usage_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
22209
+ { table: "harvest_attempts", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, timestampFormat: "iso8601" },
22210
+ { table: "browser_agent_actions", timestampColumn: "created_at", retentionDays: REPORTING_WINDOW_DAYS },
22211
+ { table: "jobs", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, where: "status = 'done'" },
22212
+ { table: "jobs", timestampColumn: "created_at", retentionDays: FAILURE_RETENTION_DAYS, where: "status = 'failed'" }
22213
+ ];
22214
+ function cutoffFor(rule) {
22215
+ const cutoff = new Date(Date.now() - rule.retentionDays * 24 * 60 * 60 * 1e3);
22216
+ if (rule.timestampFormat === "iso8601") return cutoff.toISOString();
22217
+ return cutoff.toISOString().replace("T", " ").slice(0, 19);
22218
+ }
22219
+ var DEFAULT_BATCH = 500;
22220
+ var DEFAULT_MAX_DELETES_PER_RULE = 1e3;
22221
+ var DEFAULT_SWEEP_DEADLINE_MS = 15e3;
22222
+ async function tableExists(table) {
22223
+ const res = await getDb().execute({
22224
+ sql: "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
22225
+ args: [table]
22226
+ });
22227
+ return res.rows.length > 0;
22228
+ }
22229
+ async function sweepRule(rule, maxDeletes) {
22230
+ if (!await tableExists(rule.table)) return 0;
22231
+ const filter = rule.where ? ` AND ${rule.where}` : "";
22232
+ const cutoff = cutoffFor(rule);
22233
+ let deleted = 0;
22234
+ while (deleted < maxDeletes) {
22235
+ const res = await getDb().execute({
22236
+ sql: `DELETE FROM ${rule.table} WHERE rowid IN (
22237
+ SELECT rowid FROM ${rule.table}
22238
+ WHERE ${rule.timestampColumn} < ?${filter}
22239
+ LIMIT ?)`,
22240
+ args: [cutoff, Math.min(DEFAULT_BATCH, maxDeletes - deleted)]
22241
+ });
22242
+ const n = Number(res.rowsAffected ?? 0);
22243
+ deleted += n;
22244
+ if (n === 0) break;
22245
+ }
22246
+ return deleted;
22247
+ }
22248
+ async function sweepRetention(options = {}) {
22249
+ const maxDeletes = options.maxDeletesPerRule ?? DEFAULT_MAX_DELETES_PER_RULE;
22250
+ const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_SWEEP_DEADLINE_MS);
22251
+ const perTable = {};
22252
+ let deleted = 0;
22253
+ let truncated = false;
22254
+ for (const rule of RETENTION_RULES) {
22255
+ const key = rule.where ? `${rule.table}:${rule.where}` : rule.table;
22256
+ if (Date.now() >= deadline) {
22257
+ truncated = true;
22258
+ break;
22259
+ }
22260
+ try {
22261
+ const n = await sweepRule(rule, maxDeletes);
22262
+ perTable[key] = n;
22263
+ deleted += n;
22264
+ if (n >= maxDeletes) truncated = true;
22265
+ } catch (err) {
22266
+ perTable[key] = -1;
22267
+ console.error(JSON.stringify({
22268
+ event: "retention_sweep_failed",
22269
+ table: rule.table,
22270
+ message: err instanceof Error ? err.message : String(err)
22271
+ }));
22272
+ }
22273
+ }
22274
+ return { deleted, perTable, truncated };
22275
+ }
22276
+
22188
22277
  // src/api/directory-workflow-reconciliation.ts
22189
22278
  async function reconcileDirectoryWorkflowSettlements(limit = 25) {
22190
22279
  const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
@@ -24894,6 +24983,7 @@ async function listSessionRows(userId, includeClosed = false) {
24894
24983
  });
24895
24984
  return res.rows;
24896
24985
  }
24986
+ var ORPHANED_SESSION_STATUS = "runtime_close_pending";
24897
24987
  async function listIdleOpenSessionRows(idleSeconds) {
24898
24988
  const db = getDb();
24899
24989
  const res = await db.execute({
@@ -24919,12 +25009,20 @@ async function setBilledMc(id, billedMc) {
24919
25009
  args: [Math.round(billedMc), id]
24920
25010
  });
24921
25011
  }
24922
- async function markSessionClosed(id) {
25012
+ async function markSessionClosed(id, runtimeClosed = true) {
24923
25013
  const db = getDb();
24924
25014
  await db.execute({
24925
- sql: `UPDATE browser_agent_sessions SET status = 'closed', closed_at = datetime('now') WHERE id = ?`,
24926
- args: [id]
25015
+ sql: `UPDATE browser_agent_sessions SET status = ?, closed_at = datetime('now') WHERE id = ?`,
25016
+ args: [runtimeClosed ? "closed" : ORPHANED_SESSION_STATUS, id]
25017
+ });
25018
+ }
25019
+ async function listOrphanedSessionRows(limit = 50) {
25020
+ const db = getDb();
25021
+ const res = await db.execute({
25022
+ sql: `SELECT * FROM browser_agent_sessions WHERE status = ? ORDER BY closed_at LIMIT ?`,
25023
+ args: [ORPHANED_SESSION_STATUS, Math.max(1, Math.min(200, Math.round(limit)))]
24927
25024
  });
25025
+ return res.rows;
24928
25026
  }
24929
25027
  async function recordAction(input) {
24930
25028
  const db = getDb();
@@ -26065,6 +26163,21 @@ async function charge(sessionId, userId, _startedAtMs) {
26065
26163
  if (res.ok) await setBilledMc(sessionId, owed);
26066
26164
  }
26067
26165
  }
26166
+ async function closeRuntimeSession(runtimeSessionId, sessionId) {
26167
+ try {
26168
+ await closeSession(runtimeSessionId);
26169
+ return true;
26170
+ } catch (err) {
26171
+ console.error(JSON.stringify({
26172
+ event: "browser_runtime_close_failed",
26173
+ session_id: sessionId,
26174
+ runtime_session_id: runtimeSessionId,
26175
+ error_name: err instanceof Error ? err.name : typeof err,
26176
+ message: sanitizeVendorName(err instanceof Error ? err.message : String(err)).slice(0, 300)
26177
+ }));
26178
+ return false;
26179
+ }
26180
+ }
26068
26181
  async function closeAndSettleSession(row) {
26069
26182
  if (row.user_id != null) {
26070
26183
  try {
@@ -26072,15 +26185,24 @@ async function closeAndSettleSession(row) {
26072
26185
  } catch {
26073
26186
  }
26074
26187
  }
26075
- try {
26076
- await closeSession(row.runtime_session_id);
26077
- } catch {
26078
- }
26079
- await markSessionClosed(row.id).catch(() => {
26188
+ const runtimeClosed = await closeRuntimeSession(row.runtime_session_id, row.id);
26189
+ await markSessionClosed(row.id, runtimeClosed).catch(() => {
26080
26190
  });
26081
26191
  await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
26082
26192
  });
26083
26193
  }
26194
+ async function retryOrphanedBrowserSessions(limit = 50) {
26195
+ const rows = await listOrphanedSessionRows(limit);
26196
+ let recovered = 0;
26197
+ for (const row of rows) {
26198
+ if (await closeRuntimeSession(row.runtime_session_id, row.id)) {
26199
+ await markSessionClosed(row.id, true).catch(() => {
26200
+ });
26201
+ recovered++;
26202
+ }
26203
+ }
26204
+ return { recovered, stillOpen: rows.length - recovered };
26205
+ }
26084
26206
  async function completeAuthConnection(connectionId) {
26085
26207
  const connection = await getAuthConnectionRow(connectionId);
26086
26208
  if (!connection) return { ok: false, error: "not found" };
@@ -26105,7 +26227,7 @@ async function reapIdleBrowserSessions(idleSeconds = 120) {
26105
26227
  function publicSession(row) {
26106
26228
  return {
26107
26229
  session_id: row.id,
26108
- status: row.status,
26230
+ status: row.status === ORPHANED_SESSION_STATUS ? "closed" : row.status,
26109
26231
  label: row.label,
26110
26232
  watch_url: `/console/${row.id}`,
26111
26233
  created_at: row.created_at,
@@ -30876,6 +30998,31 @@ import { z as z32 } from "zod";
30876
30998
  // src/api/commons-repository.ts
30877
30999
  import { createHash as createHash13, randomUUID as randomUUID15 } from "crypto";
30878
31000
 
31001
+ // src/api/schema-presence.ts
31002
+ var cache = null;
31003
+ async function loadSchemaObjects() {
31004
+ const res = await getDb().execute(
31005
+ "SELECT name FROM sqlite_master WHERE type IN ('table', 'index')"
31006
+ );
31007
+ return new Set(res.rows.map((row) => String(row.name)));
31008
+ }
31009
+ function invalidateSchemaPresence() {
31010
+ cache = null;
31011
+ }
31012
+ async function schemaObjectsPresent(names) {
31013
+ if (names.length === 0) return true;
31014
+ try {
31015
+ if (!cache) cache = loadSchemaObjects();
31016
+ const present = await cache;
31017
+ if (names.every((name) => present.has(name))) return true;
31018
+ invalidateSchemaPresence();
31019
+ return false;
31020
+ } catch {
31021
+ invalidateSchemaPresence();
31022
+ return false;
31023
+ }
31024
+ }
31025
+
30879
31026
  // src/api/commons-linksets.ts
30880
31027
  import { createHash as createHash12 } from "crypto";
30881
31028
  var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
@@ -31202,9 +31349,38 @@ var COMMONS_ENTITY_PROFILES = {
31202
31349
  ]
31203
31350
  }
31204
31351
  };
31205
- var schemaReady2 = false;
31352
+ var schemaReady = false;
31353
+ var COMMONS_SCHEMA_OBJECTS = [
31354
+ "schema_migrations",
31355
+ "commons_entities",
31356
+ "commons_claims",
31357
+ "commons_proposals",
31358
+ "commons_ledger",
31359
+ "commons_saved_filters",
31360
+ "commons_index_documents",
31361
+ "commons_publications",
31362
+ "commons_publication_editions",
31363
+ "commons_entities_status_updated",
31364
+ "commons_entities_type_updated",
31365
+ "commons_entities_source_domain",
31366
+ "commons_claims_subject_predicate",
31367
+ "commons_claims_object_entity",
31368
+ "commons_proposals_user_created",
31369
+ "commons_proposals_entity_created",
31370
+ "commons_ledger_entity_created",
31371
+ "commons_ledger_user_created",
31372
+ "commons_saved_filters_user_updated",
31373
+ "commons_index_documents_status_updated",
31374
+ "commons_index_documents_entity_type",
31375
+ "commons_publications_updated",
31376
+ "commons_publication_editions_publication_published"
31377
+ ];
31206
31378
  async function ensureCommonsSchema() {
31207
- if (schemaReady2) return;
31379
+ if (schemaReady) return;
31380
+ if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
31381
+ schemaReady = true;
31382
+ return;
31383
+ }
31208
31384
  const db = getDb();
31209
31385
  await db.batch([
31210
31386
  {
@@ -31425,7 +31601,7 @@ async function ensureCommonsSchema() {
31425
31601
  args: [COMMONS_SCHEMA_VERSION]
31426
31602
  }
31427
31603
  ], "write");
31428
- schemaReady2 = true;
31604
+ schemaReady = true;
31429
31605
  }
31430
31606
  async function searchCommonsEntities(filters = {}, userId) {
31431
31607
  await ensureCommonsSchema();
@@ -34336,7 +34512,9 @@ var EventSchema = z34.object({
34336
34512
  campaign: z34.string().trim().max(240).optional(),
34337
34513
  clickIds: z34.record(z34.string(), z34.string().trim().max(500)).optional(),
34338
34514
  deviceClass: z34.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
34339
- properties: z34.record(z34.string(), z34.unknown()).optional()
34515
+ properties: z34.record(z34.string(), z34.unknown()).optional(),
34516
+ engagedMs: z34.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
34517
+ scrollDepth: z34.number().int().min(0).max(100).optional()
34340
34518
  }).strict();
34341
34519
  var IngestionSchema = z34.object({
34342
34520
  pixelId: z34.string().trim().min(8).max(160),
@@ -34495,7 +34673,10 @@ analyticsApp.use("*", async (c, next) => {
34495
34673
  await next();
34496
34674
  });
34497
34675
  function analyticsPixelSdk() {
34498
- return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties}}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){current=location.href;api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
34676
+ return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={},over={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties},...over}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};let visibleSince=document.visibilityState==='visible'?Date.now():0,dwellMs=0,maxScroll=0,dwellPath=location.pathname;const depth=()=>{const d=document.documentElement,h=d.scrollHeight||1;return Math.max(0,Math.min(100,Math.round(((d.scrollTop||document.body.scrollTop||0)+innerHeight)/h*100)))};const accrue=()=>{if(visibleSince){dwellMs+=Date.now()-visibleSince;visibleSince=0}};const resume=()=>{if(!visibleSince)visibleSince=Date.now()};const flush=()=>{accrue();const ms=Math.min(dwellMs,18e5),sd=Math.max(maxScroll,depth());dwellMs=0;maxScroll=0;if(ms<250)return;send('page_engagement',{},{path:dwellPath,canonicalUrl:location.origin+dwellPath,engagedMs:ms,scrollDepth:sd})};addEventListener('scroll',()=>{const d=depth();if(d>maxScroll)maxScroll=d},{passive:true});document.addEventListener('visibilitychange',()=>{if(document.visibilityState==='hidden')flush();else resume()});addEventListener('pagehide',flush);const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();resume();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,engagedMs:dwellMs+(visibleSince?Date.now()-visibleSince:0),scrollDepth:Math.max(maxScroll,depth()),...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){flush();current=location.href;dwellPath=location.pathname;resume();api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
34677
+ }
34678
+ function pagePathFilter(value) {
34679
+ return value && value.startsWith("/") ? value : void 0;
34499
34680
  }
34500
34681
  function reportFilters(url) {
34501
34682
  const end = url.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
@@ -34517,7 +34698,7 @@ function reportFilters(url) {
34517
34698
  channelFamily: url.searchParams.get("channelFamily") || void 0,
34518
34699
  platform: url.searchParams.get("platform") || void 0,
34519
34700
  referrer: url.searchParams.get("referrer") || void 0,
34520
- path: url.searchParams.get("path") || void 0,
34701
+ path: pagePathFilter(url.searchParams.get("path")),
34521
34702
  deviceClass: url.searchParams.get("deviceClass") || void 0,
34522
34703
  countryCode: url.searchParams.get("countryCode") || void 0,
34523
34704
  regionCode: url.searchParams.get("regionCode") || void 0,
@@ -35443,14 +35624,13 @@ for (const [path5, load] of [
35443
35624
  ["/sites/:siteId/overview", analyticsOverview],
35444
35625
  ["/sites/:siteId/timeseries", analyticsTimeseries],
35445
35626
  ["/sites/:siteId/acquisition", analyticsAcquisition],
35446
- ["/sites/:siteId/content", analyticsContent],
35447
35627
  ["/sites/:siteId/events", analyticsEventCounts]
35448
35628
  ]) {
35449
35629
  analyticsApp.get(path5, auth3, async (c) => {
35450
35630
  const url = new URL(c.req.url);
35451
35631
  const filters = reportFilters(url);
35452
35632
  validateReportWindow(filters);
35453
- const paginated = path5.endsWith("/acquisition") || path5.endsWith("/content") || path5.endsWith("/events");
35633
+ const paginated = path5.endsWith("/acquisition") || path5.endsWith("/events");
35454
35634
  const data = await load(
35455
35635
  c.req.param("siteId"),
35456
35636
  Number(c.get("user").id),
@@ -35460,6 +35640,24 @@ for (const [path5, load] of [
35460
35640
  return c.json({ ok: true, data, filters });
35461
35641
  });
35462
35642
  }
35643
+ analyticsApp.get("/sites/:siteId/content", auth3, async (c) => {
35644
+ const url = new URL(c.req.url);
35645
+ const filters = reportFilters(url);
35646
+ validateReportWindow(filters);
35647
+ const requestedSort = url.searchParams.get("sort");
35648
+ const data = await analyticsContent(
35649
+ c.req.param("siteId"),
35650
+ Number(c.get("user").id),
35651
+ filters,
35652
+ reportPage(url),
35653
+ {
35654
+ sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
35655
+ direction: url.searchParams.get("direction") === "asc" ? "asc" : "desc",
35656
+ search: url.searchParams.get("search") || void 0
35657
+ }
35658
+ );
35659
+ return c.json({ ok: true, data, filters });
35660
+ });
35463
35661
  analyticsApp.get("/sites/:siteId/dashboard", auth3, async (c) => {
35464
35662
  const url = new URL(c.req.url);
35465
35663
  const filters = reportFilters(url);
@@ -35681,9 +35879,9 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
35681
35879
 
35682
35880
  // src/api/scheduled-run-view-links.ts
35683
35881
  import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
35684
- var schemaReady3 = false;
35882
+ var schemaReady2 = false;
35685
35883
  async function ensureScheduledRunViewLinksSchema() {
35686
- if (schemaReady3) return;
35884
+ if (schemaReady2) return;
35687
35885
  const db = getDb();
35688
35886
  await db.execute(`
35689
35887
  CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
@@ -35731,7 +35929,7 @@ async function ensureScheduledRunViewLinksSchema() {
35731
35929
  CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
35732
35930
  ON scheduled_run_view_links(expires_at, revoked_at)
35733
35931
  `);
35734
- schemaReady3 = true;
35932
+ schemaReady2 = true;
35735
35933
  }
35736
35934
  function tokenHash2(token4) {
35737
35935
  return createHash18("sha256").update(token4).digest("hex");
@@ -36381,13 +36579,13 @@ import { createHash as createHash22, createHmac as createHmac4, timingSafeEqual
36381
36579
 
36382
36580
  // src/api/service-connections.ts
36383
36581
  import { randomUUID as randomUUID18 } from "crypto";
36384
- var schemaReady4 = null;
36582
+ var schemaReady3 = null;
36385
36583
  var schemaDb3 = null;
36386
36584
  function ensureServiceConnectionsSchema() {
36387
36585
  const currentDb = getDb();
36388
- if (schemaReady4 && schemaDb3 === currentDb) return schemaReady4;
36586
+ if (schemaReady3 && schemaDb3 === currentDb) return schemaReady3;
36389
36587
  schemaDb3 = currentDb;
36390
- schemaReady4 = (async () => {
36588
+ schemaReady3 = (async () => {
36391
36589
  const db = currentDb;
36392
36590
  await db.execute(`
36393
36591
  CREATE TABLE IF NOT EXISTS service_connections (
@@ -36455,11 +36653,11 @@ function ensureServiceConnectionsSchema() {
36455
36653
  } catch {
36456
36654
  }
36457
36655
  })().catch((error) => {
36458
- schemaReady4 = null;
36656
+ schemaReady3 = null;
36459
36657
  schemaDb3 = null;
36460
36658
  throw error;
36461
36659
  });
36462
- return schemaReady4;
36660
+ return schemaReady3;
36463
36661
  }
36464
36662
  function parseTools(value) {
36465
36663
  if (typeof value !== "string") return [];
@@ -43890,35 +44088,73 @@ app.post("/billing/credits", auth4, async (c) => {
43890
44088
  ledger
43891
44089
  });
43892
44090
  });
44091
+ var CRON_TICK_BUDGET_MS = 5e4;
44092
+ var CRON_TICK_DRAIN_BUDGET_MS = 4e4;
44093
+ function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
44094
+ return new Promise((resolve2) => {
44095
+ const timer = setTimeout(() => {
44096
+ unfinished.push(`${label}:deadline`);
44097
+ resolve2(onDeadlineOrError);
44098
+ }, CRON_TICK_BUDGET_MS);
44099
+ work.then(
44100
+ (value) => {
44101
+ clearTimeout(timer);
44102
+ resolve2(value);
44103
+ },
44104
+ (err) => {
44105
+ clearTimeout(timer);
44106
+ unfinished.push(`${label}:error`);
44107
+ console.error(JSON.stringify({
44108
+ event: "cron_tick_job_failed",
44109
+ job: label,
44110
+ message: err instanceof Error ? err.message : String(err)
44111
+ }));
44112
+ resolve2(onDeadlineOrError);
44113
+ }
44114
+ );
44115
+ });
44116
+ }
43893
44117
  app.get("/cron/tick", async (c) => {
43894
44118
  const secret = c.req.header("authorization");
43895
44119
  if (!process.env.CRON_SECRET || secret !== `Bearer ${process.env.CRON_SECRET}`) {
43896
44120
  return c.json({ error: "Unauthorized" }, 401);
43897
44121
  }
43898
- const { drainQueue } = await import("./worker-4MHZS3OB.js");
43899
- const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
43900
- const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
43901
- const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery] = await Promise.all([
43902
- drainQueue(budget),
43903
- runMonthlyRefreshSweep(),
43904
- reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
43905
- expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
43906
- cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" })),
43907
- cleanupExpiredConnectedDataArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43908
- reconcileConnectedAccountBillingSweep().catch(() => ({ checked: 0, synced: 0, failed: 0 })),
43909
- dispatchPendingDirectoryWorkflows(25).catch(() => ({ claimed: 0, dispatched: 0, failed: 0 })),
43910
- reconcileDirectoryWorkflowSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
43911
- cleanupExpiredDirectoryArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43912
- cleanupExpiredSiteExtractArtifacts().catch(() => ({ deleted: 0, store: "none" })),
43913
- redispatchFundedSiteExtractJobs(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43914
- reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
43915
- redispatchPendingPaaHarvests(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43916
- reconcilePaaHarvestSettlements(25).catch(() => ({ checked: 0, staleFailed: 0, settled: 0, failed: 0 })),
43917
- redispatchQueuedLocalSourcebookAcquisitions(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
43918
- refreshAnalyticsDailyRollupsIfDue().catch(() => ({ status: "failed", rows: 0 })),
43919
- deliverPendingAnalyticsForms(10).catch(() => ({ claimed: 0, delivered: 0, deferred: 0 }))
44122
+ const startedAt = Date.now();
44123
+ const { drainQueue } = await import("./worker-KQN673JF.js");
44124
+ const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
44125
+ const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
44126
+ const unfinished = [];
44127
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
44128
+ settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
44129
+ settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
44130
+ settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
44131
+ settleWithinTickBudget("retryOrphanedBrowserSessions", unfinished, retryOrphanedBrowserSessions(50), { recovered: 0, stillOpen: 0 }),
44132
+ settleWithinTickBudget("expireLots", unfinished, expireOldLots(), { expired_lots: 0, expired_mc: 0 }),
44133
+ settleWithinTickBudget("scrapeBlobCleanup", unfinished, cleanupExpiredScrapeBlobs(), { deleted: 0, store: "none" }),
44134
+ settleWithinTickBudget("connectedDataCleanup", unfinished, cleanupExpiredConnectedDataArtifacts(), { deleted: 0, store: "none" }),
44135
+ settleWithinTickBudget("connectedBillingSweep", unfinished, reconcileConnectedAccountBillingSweep(), { checked: 0, synced: 0, failed: 0 }),
44136
+ settleWithinTickBudget("directoryDispatch", unfinished, dispatchPendingDirectoryWorkflows(5), { claimed: 0, dispatched: 0, failed: 0 }),
44137
+ settleWithinTickBudget("directoryReconcile", unfinished, reconcileDirectoryWorkflowSettlements(5), { checked: 0, settled: 0, failed: 0 }),
44138
+ settleWithinTickBudget("directoryArtifactCleanup", unfinished, cleanupExpiredDirectoryArtifacts(), { deleted: 0, store: "none" }),
44139
+ settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
44140
+ settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
44141
+ settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
44142
+ settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
44143
+ settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
44144
+ settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
44145
+ settleWithinTickBudget("analyticsRollup", unfinished, refreshAnalyticsDailyRollupsIfDue(), { status: "failed", rows: 0 }),
44146
+ settleWithinTickBudget("analyticsForms", unfinished, deliverPendingAnalyticsForms(10), { claimed: 0, delivered: 0, deferred: 0 }),
44147
+ settleWithinTickBudget("workflowDispatch", unfinished, dispatchDueWorkflowSchedules(origin), { dispatched: 0, status: "skipped_deadline" }),
44148
+ settleWithinTickBudget("scheduledRunReconcile", unfinished, reconcileScheduledRunSettlements(25), { checked: 0, settled: 0, chargedMc: 0, failed: 0 }),
44149
+ settleWithinTickBudget("retentionSweep", unfinished, sweepRetention(), { deleted: 0, perTable: {}, truncated: false })
43920
44150
  ]);
43921
44151
  return c.json({
44152
+ durationMs: Date.now() - startedAt,
44153
+ tickBudgetMs: CRON_TICK_BUDGET_MS,
44154
+ unfinished,
44155
+ complete: unfinished.length === 0,
44156
+ scheduledRunReconciliation,
44157
+ retentionSweep,
43922
44158
  drained: results.length,
43923
44159
  results,
43924
44160
  sweepResult,
@@ -43954,8 +44190,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
43954
44190
  finishExtractJob: finishExtractJob2,
43955
44191
  terminalExtractJobStatus: terminalExtractJobStatus2,
43956
44192
  extractJobLimitInfo: extractJobLimitInfo2
43957
- } = await import("./site-extract-repository-STNOI5OH.js");
43958
- const { assembleExtractArtifacts } = await import("./extract-bundle-2HONVC5T.js");
44193
+ } = await import("./site-extract-repository-3VQAALNR.js");
44194
+ const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
43959
44195
  const existing = await getExtractJob2(jobId2);
43960
44196
  if (!existing) return c.json({ error: "job not found" }, 404);
43961
44197
  if (existing.status !== "failed") {
@@ -43982,7 +44218,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
43982
44218
  if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
43983
44219
  let settlement = "already_settled_or_refunded";
43984
44220
  if (job.billedMc == null && job.userId != null) {
43985
- const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-STNOI5OH.js");
44221
+ const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-3VQAALNR.js");
43986
44222
  const heldMc = Number(job.options.heldMc ?? 0);
43987
44223
  const successful = await countSuccessfulPages2(jobId2);
43988
44224
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -44167,4 +44403,4 @@ app.get("/blog/:slug/", (c) => {
44167
44403
  export {
44168
44404
  app
44169
44405
  };
44170
- //# sourceMappingURL=server-JE7SLZLJ.js.map
44406
+ //# sourceMappingURL=server-3ND2EXYO.js.map