mcp-scraper 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
- package/dist/bin/api-server.cjs +573 -139
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +30 -6
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
- package/dist/chunk-2XTYLJQK.js.map +1 -0
- package/dist/{chunk-64WBUDPC.js → chunk-44LNKZTE.js} +32 -8
- package/dist/chunk-44LNKZTE.js.map +1 -0
- package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
- package/dist/chunk-5BEPOFKG.js.map +1 -0
- package/dist/chunk-C7NN57WH.js +7 -0
- package/dist/chunk-C7NN57WH.js.map +1 -0
- package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
- package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
- package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
- package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
- package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
- package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
- package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
- package/dist/{server-JE7SLZLJ.js → server-3ND2EXYO.js} +364 -128
- package/dist/server-3ND2EXYO.js.map +1 -0
- package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
- package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
- package/package.json +1 -1
- package/dist/chunk-64WBUDPC.js.map +0 -1
- package/dist/chunk-MFNGUM4L.js.map +0 -1
- package/dist/chunk-XLEDHV3J.js +0 -7
- package/dist/chunk-XLEDHV3J.js.map +0 -1
- package/dist/chunk-YW2LXLDU.js.map +0 -1
- package/dist/server-JE7SLZLJ.js.map +0 -1
- /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
- /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
- /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
- /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
- /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
- /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
- /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
- /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
- /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
- /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
- /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
|
@@ -10,9 +10,11 @@ import {
|
|
|
10
10
|
importHostedCensusPlacesCsv,
|
|
11
11
|
importHostedZipGroupsCsv,
|
|
12
12
|
listActiveHostedLocationDatasets
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-NNZXZTJ2.js";
|
|
14
14
|
import {
|
|
15
|
+
ANALYTICS_CONTENT_SORTS,
|
|
15
16
|
AnalyticsRepositoryError,
|
|
17
|
+
MAX_ENGAGED_MS,
|
|
16
18
|
analyticsAcquisition,
|
|
17
19
|
analyticsBusinessMetrics,
|
|
18
20
|
analyticsChannelBreakdown,
|
|
@@ -64,14 +66,14 @@ import {
|
|
|
64
66
|
updateAnalyticsPixel,
|
|
65
67
|
upsertAnalyticsAdSpend,
|
|
66
68
|
verifySession
|
|
67
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-5BEPOFKG.js";
|
|
68
70
|
import {
|
|
69
71
|
classifyHarvestProblem,
|
|
70
72
|
createHarvestAttemptRecorder,
|
|
71
73
|
deliverWebhook,
|
|
72
74
|
harvestProblemResponse,
|
|
73
75
|
serializeHarvestProblem
|
|
74
|
-
} from "./chunk-
|
|
76
|
+
} from "./chunk-YMSZO62N.js";
|
|
75
77
|
import {
|
|
76
78
|
AIModeSelectors,
|
|
77
79
|
AIOverviewSelectors,
|
|
@@ -98,7 +100,7 @@ import {
|
|
|
98
100
|
resolveKernelProxyId,
|
|
99
101
|
serpArrivalUrl,
|
|
100
102
|
withRuntimeUserAgent
|
|
101
|
-
} from "./chunk-
|
|
103
|
+
} from "./chunk-GOZIG6HD.js";
|
|
102
104
|
import {
|
|
103
105
|
ApiErrorResponseSchema,
|
|
104
106
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -156,7 +158,7 @@ import {
|
|
|
156
158
|
resolveDeploymentProfile,
|
|
157
159
|
resolveLocalSourcebookSchemaType,
|
|
158
160
|
transcribeMediaUrl
|
|
159
|
-
} from "./chunk-
|
|
161
|
+
} from "./chunk-44LNKZTE.js";
|
|
160
162
|
import {
|
|
161
163
|
auditImageUrls,
|
|
162
164
|
auditImages,
|
|
@@ -184,7 +186,7 @@ import {
|
|
|
184
186
|
recordKernelSession,
|
|
185
187
|
recordVendorUsage,
|
|
186
188
|
runWithCostContext
|
|
187
|
-
} from "./chunk-
|
|
189
|
+
} from "./chunk-QXAY44SA.js";
|
|
188
190
|
import {
|
|
189
191
|
listWorkflowDefinitions,
|
|
190
192
|
runWorkflow,
|
|
@@ -212,7 +214,7 @@ import {
|
|
|
212
214
|
} from "./chunk-3ZUBQQPQ.js";
|
|
213
215
|
import {
|
|
214
216
|
PACKAGE_VERSION
|
|
215
|
-
} from "./chunk-
|
|
217
|
+
} from "./chunk-C7NN57WH.js";
|
|
216
218
|
import {
|
|
217
219
|
abandonExtractSettlement,
|
|
218
220
|
countSuccessfulPages,
|
|
@@ -235,7 +237,7 @@ import {
|
|
|
235
237
|
setExtractJobTotal,
|
|
236
238
|
settleExtractJob,
|
|
237
239
|
terminalExtractJobStatus
|
|
238
|
-
} from "./chunk-
|
|
240
|
+
} from "./chunk-QHVFKFME.js";
|
|
239
241
|
import {
|
|
240
242
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
241
243
|
CONCURRENCY_PRICE_ID,
|
|
@@ -392,7 +394,7 @@ import {
|
|
|
392
394
|
startHarvestAttempt,
|
|
393
395
|
upsertPageSnapshot,
|
|
394
396
|
verifyPassword
|
|
395
|
-
} from "./chunk-
|
|
397
|
+
} from "./chunk-2XTYLJQK.js";
|
|
396
398
|
|
|
397
399
|
// src/blog/registry.ts
|
|
398
400
|
var posts = [
|
|
@@ -8738,7 +8740,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8738
8740
|
});
|
|
8739
8741
|
}) : null;
|
|
8740
8742
|
const finalized = await step.run("finalize", async () => {
|
|
8741
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8743
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
|
|
8742
8744
|
const current = await getExtractJob(jobId2);
|
|
8743
8745
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8744
8746
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -9680,7 +9682,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
9680
9682
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
9681
9683
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
9682
9684
|
if (mode === "hosted") {
|
|
9683
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
9685
|
+
const { getHostedZipGroups } = await import("./location-data-repository-UUSHH5M2.js");
|
|
9684
9686
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
9685
9687
|
if (!hosted.dataset) {
|
|
9686
9688
|
throw new DirectoryLocationError(
|
|
@@ -9743,7 +9745,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
9743
9745
|
const sourceUrl = censusStateUrl(state.fips);
|
|
9744
9746
|
const warnings = [];
|
|
9745
9747
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
9746
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
9748
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-UUSHH5M2.js");
|
|
9747
9749
|
const hosted = await queryHostedLocationMarkets({
|
|
9748
9750
|
state: state.abbr,
|
|
9749
9751
|
minPopulation: options.minPopulation,
|
|
@@ -10732,7 +10734,6 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
10732
10734
|
var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
10733
10735
|
var DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
10734
10736
|
var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
10735
|
-
var schemaReady = null;
|
|
10736
10737
|
function concurrencyLimitForUser(user) {
|
|
10737
10738
|
const tier = Math.max(0, Number(user.subscription_concurrency ?? 0));
|
|
10738
10739
|
return Math.max(1, tier) + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
|
|
@@ -10770,40 +10771,15 @@ function safeMetadata(metadata) {
|
|
|
10770
10771
|
return null;
|
|
10771
10772
|
}
|
|
10772
10773
|
}
|
|
10773
|
-
async function ensureConcurrencyGateSchema() {
|
|
10774
|
-
if (!schemaReady) {
|
|
10775
|
-
schemaReady = (async () => {
|
|
10776
|
-
const db = getDb();
|
|
10777
|
-
await db.execute(`
|
|
10778
|
-
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
10779
|
-
id TEXT PRIMARY KEY,
|
|
10780
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
10781
|
-
operation TEXT NOT NULL,
|
|
10782
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
10783
|
-
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
10784
|
-
expires_at TEXT NOT NULL,
|
|
10785
|
-
released_at TEXT,
|
|
10786
|
-
metadata TEXT
|
|
10787
|
-
)
|
|
10788
|
-
`);
|
|
10789
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
10790
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
10791
|
-
})();
|
|
10792
|
-
}
|
|
10793
|
-
return schemaReady;
|
|
10794
|
-
}
|
|
10795
10774
|
async function expireConcurrencyLocksForUser(userId) {
|
|
10796
|
-
await ensureConcurrencyGateSchema();
|
|
10797
10775
|
await getDb().execute({
|
|
10798
|
-
sql: `
|
|
10799
|
-
SET status = 'expired', released_at = datetime('now')
|
|
10776
|
+
sql: `DELETE FROM concurrency_locks
|
|
10800
10777
|
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
10801
10778
|
args: [userId]
|
|
10802
10779
|
});
|
|
10803
10780
|
}
|
|
10804
|
-
async function countActiveUsageForUser(userId) {
|
|
10805
|
-
await
|
|
10806
|
-
await expireConcurrencyLocksForUser(userId);
|
|
10781
|
+
async function countActiveUsageForUser(userId, alreadyExpired = false) {
|
|
10782
|
+
if (!alreadyExpired) await expireConcurrencyLocksForUser(userId);
|
|
10807
10783
|
const res = await getDb().execute({
|
|
10808
10784
|
sql: `SELECT
|
|
10809
10785
|
(
|
|
@@ -10823,7 +10799,6 @@ async function countActiveUsageForUser(userId) {
|
|
|
10823
10799
|
}
|
|
10824
10800
|
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
10825
10801
|
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
10826
|
-
await ensureConcurrencyGateSchema();
|
|
10827
10802
|
await expireConcurrencyLocksForUser(userId);
|
|
10828
10803
|
const res = await getDb().execute({
|
|
10829
10804
|
sql: `UPDATE concurrency_locks
|
|
@@ -10837,13 +10812,12 @@ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
|
10837
10812
|
return Number(res.rowsAffected ?? 0) > 0;
|
|
10838
10813
|
}
|
|
10839
10814
|
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
10840
|
-
await ensureConcurrencyGateSchema();
|
|
10841
10815
|
const limit = concurrencyLimitForUser(user);
|
|
10842
10816
|
if (isConcurrencyLimitBypassed(user)) {
|
|
10843
10817
|
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
10844
10818
|
}
|
|
10845
10819
|
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
10846
|
-
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
10820
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id, true), limit, operation, reused: true };
|
|
10847
10821
|
}
|
|
10848
10822
|
await expireConcurrencyLocksForUser(user.id);
|
|
10849
10823
|
const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
|
|
@@ -10874,7 +10848,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
10874
10848
|
limit
|
|
10875
10849
|
]
|
|
10876
10850
|
});
|
|
10877
|
-
const active = await countActiveUsageForUser(user.id);
|
|
10851
|
+
const active = await countActiveUsageForUser(user.id, true);
|
|
10878
10852
|
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
10879
10853
|
return { ok: true, lockId, active, limit, operation };
|
|
10880
10854
|
}
|
|
@@ -10882,17 +10856,13 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
10882
10856
|
}
|
|
10883
10857
|
async function releaseConcurrencyGate(lockId) {
|
|
10884
10858
|
if (!lockId) return;
|
|
10885
|
-
await ensureConcurrencyGateSchema();
|
|
10886
10859
|
await getDb().execute({
|
|
10887
|
-
sql: `
|
|
10888
|
-
SET status = 'released', released_at = datetime('now')
|
|
10889
|
-
WHERE id = ? AND status = 'active'`,
|
|
10860
|
+
sql: `DELETE FROM concurrency_locks WHERE id = ?`,
|
|
10890
10861
|
args: [lockId]
|
|
10891
10862
|
});
|
|
10892
10863
|
}
|
|
10893
10864
|
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
10894
10865
|
if (!lockId) return;
|
|
10895
|
-
await ensureConcurrencyGateSchema();
|
|
10896
10866
|
await getDb().execute({
|
|
10897
10867
|
sql: `UPDATE concurrency_locks
|
|
10898
10868
|
SET expires_at = datetime('now', ?)
|
|
@@ -14709,7 +14679,7 @@ async function recoverSettlement(row, user, authorization, duplicate) {
|
|
|
14709
14679
|
}
|
|
14710
14680
|
}
|
|
14711
14681
|
await getDb().execute({
|
|
14712
|
-
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status
|
|
14682
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('settling', 'cost_pending')",
|
|
14713
14683
|
args: [row.id]
|
|
14714
14684
|
});
|
|
14715
14685
|
await getDb().execute({
|
|
@@ -14754,11 +14724,13 @@ async function settleScheduledRun(args) {
|
|
|
14754
14724
|
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
14755
14725
|
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
14756
14726
|
const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
14757
|
-
const
|
|
14727
|
+
const modelCostUnreported = args.modelCostStatus === "missing";
|
|
14728
|
+
const pendingReason = modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
|
|
14758
14729
|
const metadata = safeMetadata2({
|
|
14759
14730
|
...args.metadata,
|
|
14760
14731
|
billingPhase: "settlement",
|
|
14761
14732
|
chargeMode: "model_delta",
|
|
14733
|
+
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
14762
14734
|
...pendingReason ? { reason: pendingReason } : {}
|
|
14763
14735
|
});
|
|
14764
14736
|
const eventId = existing?.id ?? randomUUID4();
|
|
@@ -14802,6 +14774,72 @@ async function settleScheduledRun(args) {
|
|
|
14802
14774
|
}
|
|
14803
14775
|
return recoverSettlement(existing, user, authorization, duplicateClaim);
|
|
14804
14776
|
}
|
|
14777
|
+
async function reconcileScheduledRunSettlements(limit = 25, minimumAgeHours = 24) {
|
|
14778
|
+
const stale = await getDb().execute({
|
|
14779
|
+
sql: `SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata
|
|
14780
|
+
FROM billing_events
|
|
14781
|
+
WHERE billing_class = ? AND status = 'cost_pending'
|
|
14782
|
+
AND created_at < datetime('now', ?)
|
|
14783
|
+
ORDER BY created_at
|
|
14784
|
+
LIMIT ?`,
|
|
14785
|
+
args: [SCHEDULED_RUN_BILLING_CLASS, `-${Math.max(1, Math.trunc(minimumAgeHours))} hours`, Math.max(1, Math.trunc(limit))]
|
|
14786
|
+
});
|
|
14787
|
+
const result = { checked: stale.rows.length, settled: 0, chargedMc: 0, failed: 0 };
|
|
14788
|
+
for (const raw of stale.rows) {
|
|
14789
|
+
const row = raw;
|
|
14790
|
+
try {
|
|
14791
|
+
const authorization = await authorizationById(row.authorization_id);
|
|
14792
|
+
if (!authorization) {
|
|
14793
|
+
result.failed += 1;
|
|
14794
|
+
continue;
|
|
14795
|
+
}
|
|
14796
|
+
const cap = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
14797
|
+
const chargeMc = Math.min(Math.max(0, Math.round(Number(row.amount_mc))), cap);
|
|
14798
|
+
if (chargeMc > 0) {
|
|
14799
|
+
const debit = await debitMcIdempotent(
|
|
14800
|
+
Number(authorization.user_id),
|
|
14801
|
+
chargeMc,
|
|
14802
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
14803
|
+
"scheduled-run:" + authorization.id + ":model:reconciled",
|
|
14804
|
+
authorization.id + ":model"
|
|
14805
|
+
);
|
|
14806
|
+
if (!debit.ok) {
|
|
14807
|
+
result.failed += 1;
|
|
14808
|
+
continue;
|
|
14809
|
+
}
|
|
14810
|
+
}
|
|
14811
|
+
await getDb().execute({
|
|
14812
|
+
sql: `UPDATE billing_events
|
|
14813
|
+
SET status = 'settled', amount_mc = ?, settled_at = datetime('now'), metadata = ?
|
|
14814
|
+
WHERE id = ? AND status = 'cost_pending'`,
|
|
14815
|
+
args: [
|
|
14816
|
+
chargeMc,
|
|
14817
|
+
safeMetadata2({ ...metadataReason(row) ? { reason: metadataReason(row) } : {}, autoReconciled: true, cappedAtAuthorizedMaximum: chargeMc < Number(row.amount_mc) }),
|
|
14818
|
+
row.id
|
|
14819
|
+
]
|
|
14820
|
+
});
|
|
14821
|
+
await getDb().execute({
|
|
14822
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending')",
|
|
14823
|
+
args: [authorization.id]
|
|
14824
|
+
});
|
|
14825
|
+
result.settled += 1;
|
|
14826
|
+
result.chargedMc += chargeMc;
|
|
14827
|
+
console.error(JSON.stringify({
|
|
14828
|
+
event: "scheduled_run_settlement_reconciled",
|
|
14829
|
+
authorization_id: authorization.id,
|
|
14830
|
+
charged_mc: chargeMc,
|
|
14831
|
+
reason: metadataReason(row) ?? "unreported_cost"
|
|
14832
|
+
}));
|
|
14833
|
+
} catch (err) {
|
|
14834
|
+
result.failed += 1;
|
|
14835
|
+
console.error(JSON.stringify({
|
|
14836
|
+
event: "scheduled_run_reconciliation_failed",
|
|
14837
|
+
message: err instanceof Error ? err.message : String(err)
|
|
14838
|
+
}));
|
|
14839
|
+
}
|
|
14840
|
+
}
|
|
14841
|
+
return result;
|
|
14842
|
+
}
|
|
14805
14843
|
async function voidScheduledRunAuthorization(args) {
|
|
14806
14844
|
const user = await resolveSchedulingUser(args.identity);
|
|
14807
14845
|
const authorization = await authorizationById(args.authorizationId);
|
|
@@ -15673,28 +15711,6 @@ internalInboxApp.post("/broadcast", async (c) => {
|
|
|
15673
15711
|
|
|
15674
15712
|
// src/api/connected-account-billing.ts
|
|
15675
15713
|
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
15676
|
-
async function ensureConnectedAccountBillingSchema() {
|
|
15677
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
15678
|
-
user_id INTEGER PRIMARY KEY,
|
|
15679
|
-
stripe_subscription_id TEXT,
|
|
15680
|
-
stripe_subscription_item_id TEXT,
|
|
15681
|
-
price_id TEXT,
|
|
15682
|
-
quantity INTEGER NOT NULL DEFAULT 0,
|
|
15683
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
15684
|
-
last_error_code TEXT,
|
|
15685
|
-
checked_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
15686
|
-
synced_at TEXT,
|
|
15687
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
15688
|
-
)`);
|
|
15689
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_charges (
|
|
15690
|
-
user_id INTEGER NOT NULL,
|
|
15691
|
-
connection_id TEXT NOT NULL,
|
|
15692
|
-
period_start TEXT NOT NULL,
|
|
15693
|
-
amount_mc INTEGER NOT NULL,
|
|
15694
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
15695
|
-
PRIMARY KEY (user_id, connection_id, period_start)
|
|
15696
|
-
)`);
|
|
15697
|
-
}
|
|
15698
15714
|
function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
|
|
15699
15715
|
return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
15700
15716
|
}
|
|
@@ -15703,7 +15719,6 @@ function isBillableConnection(connection) {
|
|
|
15703
15719
|
return BILLABLE_LIFECYCLE.has(String(connection.lifecycleStatus ?? ""));
|
|
15704
15720
|
}
|
|
15705
15721
|
async function chargedConnectionIds(userId, periodStart) {
|
|
15706
|
-
await ensureConnectedAccountBillingSchema();
|
|
15707
15722
|
const result = await getDb().execute({
|
|
15708
15723
|
sql: "SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",
|
|
15709
15724
|
args: [Number(userId), periodStart]
|
|
@@ -15711,7 +15726,6 @@ async function chargedConnectionIds(userId, periodStart) {
|
|
|
15711
15726
|
return new Set(result.rows.map((row) => String(row.connection_id)));
|
|
15712
15727
|
}
|
|
15713
15728
|
async function getConnectedAccountBillingState(userId) {
|
|
15714
|
-
await ensureConnectedAccountBillingSchema();
|
|
15715
15729
|
const result = await getDb().execute({
|
|
15716
15730
|
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
15717
15731
|
args: [Number(userId)]
|
|
@@ -15719,7 +15733,6 @@ async function getConnectedAccountBillingState(userId) {
|
|
|
15719
15733
|
return result.rows[0] ? result.rows[0] : null;
|
|
15720
15734
|
}
|
|
15721
15735
|
async function setConnectedAccountBillingState(input) {
|
|
15722
|
-
await ensureConnectedAccountBillingSchema();
|
|
15723
15736
|
await getDb().execute({
|
|
15724
15737
|
sql: `INSERT INTO connected_account_billing
|
|
15725
15738
|
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, checked_at, synced_at, updated_at)
|
|
@@ -22185,6 +22198,82 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
|
22185
22198
|
return { checked: jobs.length, staleFailed, settled, failed };
|
|
22186
22199
|
}
|
|
22187
22200
|
|
|
22201
|
+
// src/api/retention-sweeps.ts
|
|
22202
|
+
var REPORTING_WINDOW_DAYS = 30;
|
|
22203
|
+
var TELEMETRY_RETENTION_DAYS = 45;
|
|
22204
|
+
var FAILURE_RETENTION_DAYS = 90;
|
|
22205
|
+
var RETENTION_RULES = [
|
|
22206
|
+
{ table: "request_events", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22207
|
+
{ table: "kernel_session_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22208
|
+
{ table: "vendor_usage_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22209
|
+
{ table: "harvest_attempts", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, timestampFormat: "iso8601" },
|
|
22210
|
+
{ table: "browser_agent_actions", timestampColumn: "created_at", retentionDays: REPORTING_WINDOW_DAYS },
|
|
22211
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, where: "status = 'done'" },
|
|
22212
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: FAILURE_RETENTION_DAYS, where: "status = 'failed'" }
|
|
22213
|
+
];
|
|
22214
|
+
function cutoffFor(rule) {
|
|
22215
|
+
const cutoff = new Date(Date.now() - rule.retentionDays * 24 * 60 * 60 * 1e3);
|
|
22216
|
+
if (rule.timestampFormat === "iso8601") return cutoff.toISOString();
|
|
22217
|
+
return cutoff.toISOString().replace("T", " ").slice(0, 19);
|
|
22218
|
+
}
|
|
22219
|
+
var DEFAULT_BATCH = 500;
|
|
22220
|
+
var DEFAULT_MAX_DELETES_PER_RULE = 1e3;
|
|
22221
|
+
var DEFAULT_SWEEP_DEADLINE_MS = 15e3;
|
|
22222
|
+
async function tableExists(table) {
|
|
22223
|
+
const res = await getDb().execute({
|
|
22224
|
+
sql: "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
|
|
22225
|
+
args: [table]
|
|
22226
|
+
});
|
|
22227
|
+
return res.rows.length > 0;
|
|
22228
|
+
}
|
|
22229
|
+
async function sweepRule(rule, maxDeletes) {
|
|
22230
|
+
if (!await tableExists(rule.table)) return 0;
|
|
22231
|
+
const filter = rule.where ? ` AND ${rule.where}` : "";
|
|
22232
|
+
const cutoff = cutoffFor(rule);
|
|
22233
|
+
let deleted = 0;
|
|
22234
|
+
while (deleted < maxDeletes) {
|
|
22235
|
+
const res = await getDb().execute({
|
|
22236
|
+
sql: `DELETE FROM ${rule.table} WHERE rowid IN (
|
|
22237
|
+
SELECT rowid FROM ${rule.table}
|
|
22238
|
+
WHERE ${rule.timestampColumn} < ?${filter}
|
|
22239
|
+
LIMIT ?)`,
|
|
22240
|
+
args: [cutoff, Math.min(DEFAULT_BATCH, maxDeletes - deleted)]
|
|
22241
|
+
});
|
|
22242
|
+
const n = Number(res.rowsAffected ?? 0);
|
|
22243
|
+
deleted += n;
|
|
22244
|
+
if (n === 0) break;
|
|
22245
|
+
}
|
|
22246
|
+
return deleted;
|
|
22247
|
+
}
|
|
22248
|
+
async function sweepRetention(options = {}) {
|
|
22249
|
+
const maxDeletes = options.maxDeletesPerRule ?? DEFAULT_MAX_DELETES_PER_RULE;
|
|
22250
|
+
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_SWEEP_DEADLINE_MS);
|
|
22251
|
+
const perTable = {};
|
|
22252
|
+
let deleted = 0;
|
|
22253
|
+
let truncated = false;
|
|
22254
|
+
for (const rule of RETENTION_RULES) {
|
|
22255
|
+
const key = rule.where ? `${rule.table}:${rule.where}` : rule.table;
|
|
22256
|
+
if (Date.now() >= deadline) {
|
|
22257
|
+
truncated = true;
|
|
22258
|
+
break;
|
|
22259
|
+
}
|
|
22260
|
+
try {
|
|
22261
|
+
const n = await sweepRule(rule, maxDeletes);
|
|
22262
|
+
perTable[key] = n;
|
|
22263
|
+
deleted += n;
|
|
22264
|
+
if (n >= maxDeletes) truncated = true;
|
|
22265
|
+
} catch (err) {
|
|
22266
|
+
perTable[key] = -1;
|
|
22267
|
+
console.error(JSON.stringify({
|
|
22268
|
+
event: "retention_sweep_failed",
|
|
22269
|
+
table: rule.table,
|
|
22270
|
+
message: err instanceof Error ? err.message : String(err)
|
|
22271
|
+
}));
|
|
22272
|
+
}
|
|
22273
|
+
}
|
|
22274
|
+
return { deleted, perTable, truncated };
|
|
22275
|
+
}
|
|
22276
|
+
|
|
22188
22277
|
// src/api/directory-workflow-reconciliation.ts
|
|
22189
22278
|
async function reconcileDirectoryWorkflowSettlements(limit = 25) {
|
|
22190
22279
|
const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
|
|
@@ -24894,6 +24983,7 @@ async function listSessionRows(userId, includeClosed = false) {
|
|
|
24894
24983
|
});
|
|
24895
24984
|
return res.rows;
|
|
24896
24985
|
}
|
|
24986
|
+
var ORPHANED_SESSION_STATUS = "runtime_close_pending";
|
|
24897
24987
|
async function listIdleOpenSessionRows(idleSeconds) {
|
|
24898
24988
|
const db = getDb();
|
|
24899
24989
|
const res = await db.execute({
|
|
@@ -24919,12 +25009,20 @@ async function setBilledMc(id, billedMc) {
|
|
|
24919
25009
|
args: [Math.round(billedMc), id]
|
|
24920
25010
|
});
|
|
24921
25011
|
}
|
|
24922
|
-
async function markSessionClosed(id) {
|
|
25012
|
+
async function markSessionClosed(id, runtimeClosed = true) {
|
|
24923
25013
|
const db = getDb();
|
|
24924
25014
|
await db.execute({
|
|
24925
|
-
sql: `UPDATE browser_agent_sessions SET status =
|
|
24926
|
-
args: [id]
|
|
25015
|
+
sql: `UPDATE browser_agent_sessions SET status = ?, closed_at = datetime('now') WHERE id = ?`,
|
|
25016
|
+
args: [runtimeClosed ? "closed" : ORPHANED_SESSION_STATUS, id]
|
|
25017
|
+
});
|
|
25018
|
+
}
|
|
25019
|
+
async function listOrphanedSessionRows(limit = 50) {
|
|
25020
|
+
const db = getDb();
|
|
25021
|
+
const res = await db.execute({
|
|
25022
|
+
sql: `SELECT * FROM browser_agent_sessions WHERE status = ? ORDER BY closed_at LIMIT ?`,
|
|
25023
|
+
args: [ORPHANED_SESSION_STATUS, Math.max(1, Math.min(200, Math.round(limit)))]
|
|
24927
25024
|
});
|
|
25025
|
+
return res.rows;
|
|
24928
25026
|
}
|
|
24929
25027
|
async function recordAction(input) {
|
|
24930
25028
|
const db = getDb();
|
|
@@ -26065,6 +26163,21 @@ async function charge(sessionId, userId, _startedAtMs) {
|
|
|
26065
26163
|
if (res.ok) await setBilledMc(sessionId, owed);
|
|
26066
26164
|
}
|
|
26067
26165
|
}
|
|
26166
|
+
async function closeRuntimeSession(runtimeSessionId, sessionId) {
|
|
26167
|
+
try {
|
|
26168
|
+
await closeSession(runtimeSessionId);
|
|
26169
|
+
return true;
|
|
26170
|
+
} catch (err) {
|
|
26171
|
+
console.error(JSON.stringify({
|
|
26172
|
+
event: "browser_runtime_close_failed",
|
|
26173
|
+
session_id: sessionId,
|
|
26174
|
+
runtime_session_id: runtimeSessionId,
|
|
26175
|
+
error_name: err instanceof Error ? err.name : typeof err,
|
|
26176
|
+
message: sanitizeVendorName(err instanceof Error ? err.message : String(err)).slice(0, 300)
|
|
26177
|
+
}));
|
|
26178
|
+
return false;
|
|
26179
|
+
}
|
|
26180
|
+
}
|
|
26068
26181
|
async function closeAndSettleSession(row) {
|
|
26069
26182
|
if (row.user_id != null) {
|
|
26070
26183
|
try {
|
|
@@ -26072,15 +26185,24 @@ async function closeAndSettleSession(row) {
|
|
|
26072
26185
|
} catch {
|
|
26073
26186
|
}
|
|
26074
26187
|
}
|
|
26075
|
-
|
|
26076
|
-
|
|
26077
|
-
} catch {
|
|
26078
|
-
}
|
|
26079
|
-
await markSessionClosed(row.id).catch(() => {
|
|
26188
|
+
const runtimeClosed = await closeRuntimeSession(row.runtime_session_id, row.id);
|
|
26189
|
+
await markSessionClosed(row.id, runtimeClosed).catch(() => {
|
|
26080
26190
|
});
|
|
26081
26191
|
await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
|
|
26082
26192
|
});
|
|
26083
26193
|
}
|
|
26194
|
+
async function retryOrphanedBrowserSessions(limit = 50) {
|
|
26195
|
+
const rows = await listOrphanedSessionRows(limit);
|
|
26196
|
+
let recovered = 0;
|
|
26197
|
+
for (const row of rows) {
|
|
26198
|
+
if (await closeRuntimeSession(row.runtime_session_id, row.id)) {
|
|
26199
|
+
await markSessionClosed(row.id, true).catch(() => {
|
|
26200
|
+
});
|
|
26201
|
+
recovered++;
|
|
26202
|
+
}
|
|
26203
|
+
}
|
|
26204
|
+
return { recovered, stillOpen: rows.length - recovered };
|
|
26205
|
+
}
|
|
26084
26206
|
async function completeAuthConnection(connectionId) {
|
|
26085
26207
|
const connection = await getAuthConnectionRow(connectionId);
|
|
26086
26208
|
if (!connection) return { ok: false, error: "not found" };
|
|
@@ -26105,7 +26227,7 @@ async function reapIdleBrowserSessions(idleSeconds = 120) {
|
|
|
26105
26227
|
function publicSession(row) {
|
|
26106
26228
|
return {
|
|
26107
26229
|
session_id: row.id,
|
|
26108
|
-
status: row.status,
|
|
26230
|
+
status: row.status === ORPHANED_SESSION_STATUS ? "closed" : row.status,
|
|
26109
26231
|
label: row.label,
|
|
26110
26232
|
watch_url: `/console/${row.id}`,
|
|
26111
26233
|
created_at: row.created_at,
|
|
@@ -30876,6 +30998,31 @@ import { z as z32 } from "zod";
|
|
|
30876
30998
|
// src/api/commons-repository.ts
|
|
30877
30999
|
import { createHash as createHash13, randomUUID as randomUUID15 } from "crypto";
|
|
30878
31000
|
|
|
31001
|
+
// src/api/schema-presence.ts
|
|
31002
|
+
var cache = null;
|
|
31003
|
+
async function loadSchemaObjects() {
|
|
31004
|
+
const res = await getDb().execute(
|
|
31005
|
+
"SELECT name FROM sqlite_master WHERE type IN ('table', 'index')"
|
|
31006
|
+
);
|
|
31007
|
+
return new Set(res.rows.map((row) => String(row.name)));
|
|
31008
|
+
}
|
|
31009
|
+
function invalidateSchemaPresence() {
|
|
31010
|
+
cache = null;
|
|
31011
|
+
}
|
|
31012
|
+
async function schemaObjectsPresent(names) {
|
|
31013
|
+
if (names.length === 0) return true;
|
|
31014
|
+
try {
|
|
31015
|
+
if (!cache) cache = loadSchemaObjects();
|
|
31016
|
+
const present = await cache;
|
|
31017
|
+
if (names.every((name) => present.has(name))) return true;
|
|
31018
|
+
invalidateSchemaPresence();
|
|
31019
|
+
return false;
|
|
31020
|
+
} catch {
|
|
31021
|
+
invalidateSchemaPresence();
|
|
31022
|
+
return false;
|
|
31023
|
+
}
|
|
31024
|
+
}
|
|
31025
|
+
|
|
30879
31026
|
// src/api/commons-linksets.ts
|
|
30880
31027
|
import { createHash as createHash12 } from "crypto";
|
|
30881
31028
|
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
@@ -31202,9 +31349,38 @@ var COMMONS_ENTITY_PROFILES = {
|
|
|
31202
31349
|
]
|
|
31203
31350
|
}
|
|
31204
31351
|
};
|
|
31205
|
-
var
|
|
31352
|
+
var schemaReady = false;
|
|
31353
|
+
var COMMONS_SCHEMA_OBJECTS = [
|
|
31354
|
+
"schema_migrations",
|
|
31355
|
+
"commons_entities",
|
|
31356
|
+
"commons_claims",
|
|
31357
|
+
"commons_proposals",
|
|
31358
|
+
"commons_ledger",
|
|
31359
|
+
"commons_saved_filters",
|
|
31360
|
+
"commons_index_documents",
|
|
31361
|
+
"commons_publications",
|
|
31362
|
+
"commons_publication_editions",
|
|
31363
|
+
"commons_entities_status_updated",
|
|
31364
|
+
"commons_entities_type_updated",
|
|
31365
|
+
"commons_entities_source_domain",
|
|
31366
|
+
"commons_claims_subject_predicate",
|
|
31367
|
+
"commons_claims_object_entity",
|
|
31368
|
+
"commons_proposals_user_created",
|
|
31369
|
+
"commons_proposals_entity_created",
|
|
31370
|
+
"commons_ledger_entity_created",
|
|
31371
|
+
"commons_ledger_user_created",
|
|
31372
|
+
"commons_saved_filters_user_updated",
|
|
31373
|
+
"commons_index_documents_status_updated",
|
|
31374
|
+
"commons_index_documents_entity_type",
|
|
31375
|
+
"commons_publications_updated",
|
|
31376
|
+
"commons_publication_editions_publication_published"
|
|
31377
|
+
];
|
|
31206
31378
|
async function ensureCommonsSchema() {
|
|
31207
|
-
if (
|
|
31379
|
+
if (schemaReady) return;
|
|
31380
|
+
if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
|
|
31381
|
+
schemaReady = true;
|
|
31382
|
+
return;
|
|
31383
|
+
}
|
|
31208
31384
|
const db = getDb();
|
|
31209
31385
|
await db.batch([
|
|
31210
31386
|
{
|
|
@@ -31425,7 +31601,7 @@ async function ensureCommonsSchema() {
|
|
|
31425
31601
|
args: [COMMONS_SCHEMA_VERSION]
|
|
31426
31602
|
}
|
|
31427
31603
|
], "write");
|
|
31428
|
-
|
|
31604
|
+
schemaReady = true;
|
|
31429
31605
|
}
|
|
31430
31606
|
async function searchCommonsEntities(filters = {}, userId) {
|
|
31431
31607
|
await ensureCommonsSchema();
|
|
@@ -34336,7 +34512,9 @@ var EventSchema = z34.object({
|
|
|
34336
34512
|
campaign: z34.string().trim().max(240).optional(),
|
|
34337
34513
|
clickIds: z34.record(z34.string(), z34.string().trim().max(500)).optional(),
|
|
34338
34514
|
deviceClass: z34.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
|
|
34339
|
-
properties: z34.record(z34.string(), z34.unknown()).optional()
|
|
34515
|
+
properties: z34.record(z34.string(), z34.unknown()).optional(),
|
|
34516
|
+
engagedMs: z34.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
|
|
34517
|
+
scrollDepth: z34.number().int().min(0).max(100).optional()
|
|
34340
34518
|
}).strict();
|
|
34341
34519
|
var IngestionSchema = z34.object({
|
|
34342
34520
|
pixelId: z34.string().trim().min(8).max(160),
|
|
@@ -34495,7 +34673,10 @@ analyticsApp.use("*", async (c, next) => {
|
|
|
34495
34673
|
await next();
|
|
34496
34674
|
});
|
|
34497
34675
|
function analyticsPixelSdk() {
|
|
34498
|
-
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties}}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){current=location.href;api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
34676
|
+
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={},over={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties},...over}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};let visibleSince=document.visibilityState==='visible'?Date.now():0,dwellMs=0,maxScroll=0,dwellPath=location.pathname;const depth=()=>{const d=document.documentElement,h=d.scrollHeight||1;return Math.max(0,Math.min(100,Math.round(((d.scrollTop||document.body.scrollTop||0)+innerHeight)/h*100)))};const accrue=()=>{if(visibleSince){dwellMs+=Date.now()-visibleSince;visibleSince=0}};const resume=()=>{if(!visibleSince)visibleSince=Date.now()};const flush=()=>{accrue();const ms=Math.min(dwellMs,18e5),sd=Math.max(maxScroll,depth());dwellMs=0;maxScroll=0;if(ms<250)return;send('page_engagement',{},{path:dwellPath,canonicalUrl:location.origin+dwellPath,engagedMs:ms,scrollDepth:sd})};addEventListener('scroll',()=>{const d=depth();if(d>maxScroll)maxScroll=d},{passive:true});document.addEventListener('visibilitychange',()=>{if(document.visibilityState==='hidden')flush();else resume()});addEventListener('pagehide',flush);const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();resume();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,engagedMs:dwellMs+(visibleSince?Date.now()-visibleSince:0),scrollDepth:Math.max(maxScroll,depth()),...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){flush();current=location.href;dwellPath=location.pathname;resume();api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
34677
|
+
}
|
|
34678
|
+
function pagePathFilter(value) {
|
|
34679
|
+
return value && value.startsWith("/") ? value : void 0;
|
|
34499
34680
|
}
|
|
34500
34681
|
function reportFilters(url) {
|
|
34501
34682
|
const end = url.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -34517,7 +34698,7 @@ function reportFilters(url) {
|
|
|
34517
34698
|
channelFamily: url.searchParams.get("channelFamily") || void 0,
|
|
34518
34699
|
platform: url.searchParams.get("platform") || void 0,
|
|
34519
34700
|
referrer: url.searchParams.get("referrer") || void 0,
|
|
34520
|
-
path: url.searchParams.get("path")
|
|
34701
|
+
path: pagePathFilter(url.searchParams.get("path")),
|
|
34521
34702
|
deviceClass: url.searchParams.get("deviceClass") || void 0,
|
|
34522
34703
|
countryCode: url.searchParams.get("countryCode") || void 0,
|
|
34523
34704
|
regionCode: url.searchParams.get("regionCode") || void 0,
|
|
@@ -35443,14 +35624,13 @@ for (const [path5, load] of [
|
|
|
35443
35624
|
["/sites/:siteId/overview", analyticsOverview],
|
|
35444
35625
|
["/sites/:siteId/timeseries", analyticsTimeseries],
|
|
35445
35626
|
["/sites/:siteId/acquisition", analyticsAcquisition],
|
|
35446
|
-
["/sites/:siteId/content", analyticsContent],
|
|
35447
35627
|
["/sites/:siteId/events", analyticsEventCounts]
|
|
35448
35628
|
]) {
|
|
35449
35629
|
analyticsApp.get(path5, auth3, async (c) => {
|
|
35450
35630
|
const url = new URL(c.req.url);
|
|
35451
35631
|
const filters = reportFilters(url);
|
|
35452
35632
|
validateReportWindow(filters);
|
|
35453
|
-
const paginated = path5.endsWith("/acquisition") || path5.endsWith("/
|
|
35633
|
+
const paginated = path5.endsWith("/acquisition") || path5.endsWith("/events");
|
|
35454
35634
|
const data = await load(
|
|
35455
35635
|
c.req.param("siteId"),
|
|
35456
35636
|
Number(c.get("user").id),
|
|
@@ -35460,6 +35640,24 @@ for (const [path5, load] of [
|
|
|
35460
35640
|
return c.json({ ok: true, data, filters });
|
|
35461
35641
|
});
|
|
35462
35642
|
}
|
|
35643
|
+
analyticsApp.get("/sites/:siteId/content", auth3, async (c) => {
|
|
35644
|
+
const url = new URL(c.req.url);
|
|
35645
|
+
const filters = reportFilters(url);
|
|
35646
|
+
validateReportWindow(filters);
|
|
35647
|
+
const requestedSort = url.searchParams.get("sort");
|
|
35648
|
+
const data = await analyticsContent(
|
|
35649
|
+
c.req.param("siteId"),
|
|
35650
|
+
Number(c.get("user").id),
|
|
35651
|
+
filters,
|
|
35652
|
+
reportPage(url),
|
|
35653
|
+
{
|
|
35654
|
+
sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
|
|
35655
|
+
direction: url.searchParams.get("direction") === "asc" ? "asc" : "desc",
|
|
35656
|
+
search: url.searchParams.get("search") || void 0
|
|
35657
|
+
}
|
|
35658
|
+
);
|
|
35659
|
+
return c.json({ ok: true, data, filters });
|
|
35660
|
+
});
|
|
35463
35661
|
analyticsApp.get("/sites/:siteId/dashboard", auth3, async (c) => {
|
|
35464
35662
|
const url = new URL(c.req.url);
|
|
35465
35663
|
const filters = reportFilters(url);
|
|
@@ -35681,9 +35879,9 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
35681
35879
|
|
|
35682
35880
|
// src/api/scheduled-run-view-links.ts
|
|
35683
35881
|
import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
|
|
35684
|
-
var
|
|
35882
|
+
var schemaReady2 = false;
|
|
35685
35883
|
async function ensureScheduledRunViewLinksSchema() {
|
|
35686
|
-
if (
|
|
35884
|
+
if (schemaReady2) return;
|
|
35687
35885
|
const db = getDb();
|
|
35688
35886
|
await db.execute(`
|
|
35689
35887
|
CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
|
|
@@ -35731,7 +35929,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
35731
35929
|
CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
|
|
35732
35930
|
ON scheduled_run_view_links(expires_at, revoked_at)
|
|
35733
35931
|
`);
|
|
35734
|
-
|
|
35932
|
+
schemaReady2 = true;
|
|
35735
35933
|
}
|
|
35736
35934
|
function tokenHash2(token4) {
|
|
35737
35935
|
return createHash18("sha256").update(token4).digest("hex");
|
|
@@ -36381,13 +36579,13 @@ import { createHash as createHash22, createHmac as createHmac4, timingSafeEqual
|
|
|
36381
36579
|
|
|
36382
36580
|
// src/api/service-connections.ts
|
|
36383
36581
|
import { randomUUID as randomUUID18 } from "crypto";
|
|
36384
|
-
var
|
|
36582
|
+
var schemaReady3 = null;
|
|
36385
36583
|
var schemaDb3 = null;
|
|
36386
36584
|
function ensureServiceConnectionsSchema() {
|
|
36387
36585
|
const currentDb = getDb();
|
|
36388
|
-
if (
|
|
36586
|
+
if (schemaReady3 && schemaDb3 === currentDb) return schemaReady3;
|
|
36389
36587
|
schemaDb3 = currentDb;
|
|
36390
|
-
|
|
36588
|
+
schemaReady3 = (async () => {
|
|
36391
36589
|
const db = currentDb;
|
|
36392
36590
|
await db.execute(`
|
|
36393
36591
|
CREATE TABLE IF NOT EXISTS service_connections (
|
|
@@ -36455,11 +36653,11 @@ function ensureServiceConnectionsSchema() {
|
|
|
36455
36653
|
} catch {
|
|
36456
36654
|
}
|
|
36457
36655
|
})().catch((error) => {
|
|
36458
|
-
|
|
36656
|
+
schemaReady3 = null;
|
|
36459
36657
|
schemaDb3 = null;
|
|
36460
36658
|
throw error;
|
|
36461
36659
|
});
|
|
36462
|
-
return
|
|
36660
|
+
return schemaReady3;
|
|
36463
36661
|
}
|
|
36464
36662
|
function parseTools(value) {
|
|
36465
36663
|
if (typeof value !== "string") return [];
|
|
@@ -43890,35 +44088,73 @@ app.post("/billing/credits", auth4, async (c) => {
|
|
|
43890
44088
|
ledger
|
|
43891
44089
|
});
|
|
43892
44090
|
});
|
|
44091
|
+
var CRON_TICK_BUDGET_MS = 5e4;
|
|
44092
|
+
var CRON_TICK_DRAIN_BUDGET_MS = 4e4;
|
|
44093
|
+
function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
|
|
44094
|
+
return new Promise((resolve2) => {
|
|
44095
|
+
const timer = setTimeout(() => {
|
|
44096
|
+
unfinished.push(`${label}:deadline`);
|
|
44097
|
+
resolve2(onDeadlineOrError);
|
|
44098
|
+
}, CRON_TICK_BUDGET_MS);
|
|
44099
|
+
work.then(
|
|
44100
|
+
(value) => {
|
|
44101
|
+
clearTimeout(timer);
|
|
44102
|
+
resolve2(value);
|
|
44103
|
+
},
|
|
44104
|
+
(err) => {
|
|
44105
|
+
clearTimeout(timer);
|
|
44106
|
+
unfinished.push(`${label}:error`);
|
|
44107
|
+
console.error(JSON.stringify({
|
|
44108
|
+
event: "cron_tick_job_failed",
|
|
44109
|
+
job: label,
|
|
44110
|
+
message: err instanceof Error ? err.message : String(err)
|
|
44111
|
+
}));
|
|
44112
|
+
resolve2(onDeadlineOrError);
|
|
44113
|
+
}
|
|
44114
|
+
);
|
|
44115
|
+
});
|
|
44116
|
+
}
|
|
43893
44117
|
app.get("/cron/tick", async (c) => {
|
|
43894
44118
|
const secret = c.req.header("authorization");
|
|
43895
44119
|
if (!process.env.CRON_SECRET || secret !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
43896
44120
|
return c.json({ error: "Unauthorized" }, 401);
|
|
43897
44121
|
}
|
|
43898
|
-
const
|
|
43899
|
-
const
|
|
43900
|
-
const
|
|
43901
|
-
const
|
|
43902
|
-
|
|
43903
|
-
|
|
43904
|
-
|
|
43905
|
-
|
|
43906
|
-
|
|
43907
|
-
|
|
43908
|
-
|
|
43909
|
-
|
|
43910
|
-
|
|
43911
|
-
|
|
43912
|
-
|
|
43913
|
-
|
|
43914
|
-
|
|
43915
|
-
|
|
43916
|
-
|
|
43917
|
-
|
|
43918
|
-
|
|
43919
|
-
|
|
44122
|
+
const startedAt = Date.now();
|
|
44123
|
+
const { drainQueue } = await import("./worker-KQN673JF.js");
|
|
44124
|
+
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
44125
|
+
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
44126
|
+
const unfinished = [];
|
|
44127
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
44128
|
+
settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
|
|
44129
|
+
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
44130
|
+
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
44131
|
+
settleWithinTickBudget("retryOrphanedBrowserSessions", unfinished, retryOrphanedBrowserSessions(50), { recovered: 0, stillOpen: 0 }),
|
|
44132
|
+
settleWithinTickBudget("expireLots", unfinished, expireOldLots(), { expired_lots: 0, expired_mc: 0 }),
|
|
44133
|
+
settleWithinTickBudget("scrapeBlobCleanup", unfinished, cleanupExpiredScrapeBlobs(), { deleted: 0, store: "none" }),
|
|
44134
|
+
settleWithinTickBudget("connectedDataCleanup", unfinished, cleanupExpiredConnectedDataArtifacts(), { deleted: 0, store: "none" }),
|
|
44135
|
+
settleWithinTickBudget("connectedBillingSweep", unfinished, reconcileConnectedAccountBillingSweep(), { checked: 0, synced: 0, failed: 0 }),
|
|
44136
|
+
settleWithinTickBudget("directoryDispatch", unfinished, dispatchPendingDirectoryWorkflows(5), { claimed: 0, dispatched: 0, failed: 0 }),
|
|
44137
|
+
settleWithinTickBudget("directoryReconcile", unfinished, reconcileDirectoryWorkflowSettlements(5), { checked: 0, settled: 0, failed: 0 }),
|
|
44138
|
+
settleWithinTickBudget("directoryArtifactCleanup", unfinished, cleanupExpiredDirectoryArtifacts(), { deleted: 0, store: "none" }),
|
|
44139
|
+
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
44140
|
+
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44141
|
+
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
44142
|
+
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44143
|
+
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
44144
|
+
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44145
|
+
settleWithinTickBudget("analyticsRollup", unfinished, refreshAnalyticsDailyRollupsIfDue(), { status: "failed", rows: 0 }),
|
|
44146
|
+
settleWithinTickBudget("analyticsForms", unfinished, deliverPendingAnalyticsForms(10), { claimed: 0, delivered: 0, deferred: 0 }),
|
|
44147
|
+
settleWithinTickBudget("workflowDispatch", unfinished, dispatchDueWorkflowSchedules(origin), { dispatched: 0, status: "skipped_deadline" }),
|
|
44148
|
+
settleWithinTickBudget("scheduledRunReconcile", unfinished, reconcileScheduledRunSettlements(25), { checked: 0, settled: 0, chargedMc: 0, failed: 0 }),
|
|
44149
|
+
settleWithinTickBudget("retentionSweep", unfinished, sweepRetention(), { deleted: 0, perTable: {}, truncated: false })
|
|
43920
44150
|
]);
|
|
43921
44151
|
return c.json({
|
|
44152
|
+
durationMs: Date.now() - startedAt,
|
|
44153
|
+
tickBudgetMs: CRON_TICK_BUDGET_MS,
|
|
44154
|
+
unfinished,
|
|
44155
|
+
complete: unfinished.length === 0,
|
|
44156
|
+
scheduledRunReconciliation,
|
|
44157
|
+
retentionSweep,
|
|
43922
44158
|
drained: results.length,
|
|
43923
44159
|
results,
|
|
43924
44160
|
sweepResult,
|
|
@@ -43954,8 +44190,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
43954
44190
|
finishExtractJob: finishExtractJob2,
|
|
43955
44191
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
43956
44192
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
43957
|
-
} = await import("./site-extract-repository-
|
|
43958
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
44193
|
+
} = await import("./site-extract-repository-3VQAALNR.js");
|
|
44194
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
|
|
43959
44195
|
const existing = await getExtractJob2(jobId2);
|
|
43960
44196
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
43961
44197
|
if (existing.status !== "failed") {
|
|
@@ -43982,7 +44218,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
43982
44218
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
43983
44219
|
let settlement = "already_settled_or_refunded";
|
|
43984
44220
|
if (job.billedMc == null && job.userId != null) {
|
|
43985
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
44221
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-3VQAALNR.js");
|
|
43986
44222
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
43987
44223
|
const successful = await countSuccessfulPages2(jobId2);
|
|
43988
44224
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -44167,4 +44403,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
44167
44403
|
export {
|
|
44168
44404
|
app
|
|
44169
44405
|
};
|
|
44170
|
-
//# sourceMappingURL=server-
|
|
44406
|
+
//# sourceMappingURL=server-3ND2EXYO.js.map
|