mcp-scraper 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
- package/dist/bin/api-server.cjs +1214 -328
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +75 -8
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
- package/dist/chunk-2XTYLJQK.js.map +1 -0
- package/dist/{chunk-PTABZABJ.js → chunk-44LNKZTE.js} +76 -8
- package/dist/chunk-44LNKZTE.js.map +1 -0
- package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
- package/dist/chunk-5BEPOFKG.js.map +1 -0
- package/dist/chunk-C7NN57WH.js +7 -0
- package/dist/chunk-C7NN57WH.js.map +1 -0
- package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
- package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
- package/dist/{chunk-QAH7WMYC.js → chunk-QH63IZTX.js} +3 -3
- package/dist/chunk-QH63IZTX.js.map +1 -0
- package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
- package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
- package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
- package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
- package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
- package/dist/{server-EVTCL3KB.js → server-3ND2EXYO.js} +882 -242
- package/dist/server-3ND2EXYO.js.map +1 -0
- package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
- package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
- package/package.json +1 -1
- package/dist/chunk-IAXQIHNS.js +0 -7
- package/dist/chunk-IAXQIHNS.js.map +0 -1
- package/dist/chunk-MFNGUM4L.js.map +0 -1
- package/dist/chunk-PTABZABJ.js.map +0 -1
- package/dist/chunk-QAH7WMYC.js.map +0 -1
- package/dist/chunk-YW2LXLDU.js.map +0 -1
- package/dist/server-EVTCL3KB.js.map +0 -1
- /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
- /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
- /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
- /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
- /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
- /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
- /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
- /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
- /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
- /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
- /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
|
@@ -10,9 +10,11 @@ import {
|
|
|
10
10
|
importHostedCensusPlacesCsv,
|
|
11
11
|
importHostedZipGroupsCsv,
|
|
12
12
|
listActiveHostedLocationDatasets
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-NNZXZTJ2.js";
|
|
14
14
|
import {
|
|
15
|
+
ANALYTICS_CONTENT_SORTS,
|
|
15
16
|
AnalyticsRepositoryError,
|
|
17
|
+
MAX_ENGAGED_MS,
|
|
16
18
|
analyticsAcquisition,
|
|
17
19
|
analyticsBusinessMetrics,
|
|
18
20
|
analyticsChannelBreakdown,
|
|
@@ -64,14 +66,14 @@ import {
|
|
|
64
66
|
updateAnalyticsPixel,
|
|
65
67
|
upsertAnalyticsAdSpend,
|
|
66
68
|
verifySession
|
|
67
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-5BEPOFKG.js";
|
|
68
70
|
import {
|
|
69
71
|
classifyHarvestProblem,
|
|
70
72
|
createHarvestAttemptRecorder,
|
|
71
73
|
deliverWebhook,
|
|
72
74
|
harvestProblemResponse,
|
|
73
75
|
serializeHarvestProblem
|
|
74
|
-
} from "./chunk-
|
|
76
|
+
} from "./chunk-YMSZO62N.js";
|
|
75
77
|
import {
|
|
76
78
|
AIModeSelectors,
|
|
77
79
|
AIOverviewSelectors,
|
|
@@ -98,10 +100,11 @@ import {
|
|
|
98
100
|
resolveKernelProxyId,
|
|
99
101
|
serpArrivalUrl,
|
|
100
102
|
withRuntimeUserAgent
|
|
101
|
-
} from "./chunk-
|
|
103
|
+
} from "./chunk-GOZIG6HD.js";
|
|
102
104
|
import {
|
|
103
105
|
ApiErrorResponseSchema,
|
|
104
106
|
ArtifactTemplatePresetKeySchema,
|
|
107
|
+
CommonsClaimInputSchema,
|
|
105
108
|
CommonsClaimPublicationInputSchema,
|
|
106
109
|
CommonsPreparePublicationInputSchema,
|
|
107
110
|
CommonsPublishEditorialInputSchema,
|
|
@@ -155,7 +158,7 @@ import {
|
|
|
155
158
|
resolveDeploymentProfile,
|
|
156
159
|
resolveLocalSourcebookSchemaType,
|
|
157
160
|
transcribeMediaUrl
|
|
158
|
-
} from "./chunk-
|
|
161
|
+
} from "./chunk-44LNKZTE.js";
|
|
159
162
|
import {
|
|
160
163
|
auditImageUrls,
|
|
161
164
|
auditImages,
|
|
@@ -183,7 +186,7 @@ import {
|
|
|
183
186
|
recordKernelSession,
|
|
184
187
|
recordVendorUsage,
|
|
185
188
|
runWithCostContext
|
|
186
|
-
} from "./chunk-
|
|
189
|
+
} from "./chunk-QXAY44SA.js";
|
|
187
190
|
import {
|
|
188
191
|
listWorkflowDefinitions,
|
|
189
192
|
runWorkflow,
|
|
@@ -211,7 +214,7 @@ import {
|
|
|
211
214
|
} from "./chunk-3ZUBQQPQ.js";
|
|
212
215
|
import {
|
|
213
216
|
PACKAGE_VERSION
|
|
214
|
-
} from "./chunk-
|
|
217
|
+
} from "./chunk-C7NN57WH.js";
|
|
215
218
|
import {
|
|
216
219
|
abandonExtractSettlement,
|
|
217
220
|
countSuccessfulPages,
|
|
@@ -234,7 +237,7 @@ import {
|
|
|
234
237
|
setExtractJobTotal,
|
|
235
238
|
settleExtractJob,
|
|
236
239
|
terminalExtractJobStatus
|
|
237
|
-
} from "./chunk-
|
|
240
|
+
} from "./chunk-QHVFKFME.js";
|
|
238
241
|
import {
|
|
239
242
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
240
243
|
CONCURRENCY_PRICE_ID,
|
|
@@ -391,7 +394,7 @@ import {
|
|
|
391
394
|
startHarvestAttempt,
|
|
392
395
|
upsertPageSnapshot,
|
|
393
396
|
verifyPassword
|
|
394
|
-
} from "./chunk-
|
|
397
|
+
} from "./chunk-2XTYLJQK.js";
|
|
395
398
|
|
|
396
399
|
// src/blog/registry.ts
|
|
397
400
|
var posts = [
|
|
@@ -8737,7 +8740,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8737
8740
|
});
|
|
8738
8741
|
}) : null;
|
|
8739
8742
|
const finalized = await step.run("finalize", async () => {
|
|
8740
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
8743
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
|
|
8741
8744
|
const current = await getExtractJob(jobId2);
|
|
8742
8745
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8743
8746
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -9679,7 +9682,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
9679
9682
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
9680
9683
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
9681
9684
|
if (mode === "hosted") {
|
|
9682
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
9685
|
+
const { getHostedZipGroups } = await import("./location-data-repository-UUSHH5M2.js");
|
|
9683
9686
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
9684
9687
|
if (!hosted.dataset) {
|
|
9685
9688
|
throw new DirectoryLocationError(
|
|
@@ -9742,7 +9745,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
9742
9745
|
const sourceUrl = censusStateUrl(state.fips);
|
|
9743
9746
|
const warnings = [];
|
|
9744
9747
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
9745
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
9748
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-UUSHH5M2.js");
|
|
9746
9749
|
const hosted = await queryHostedLocationMarkets({
|
|
9747
9750
|
state: state.abbr,
|
|
9748
9751
|
minPopulation: options.minPopulation,
|
|
@@ -10731,7 +10734,6 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
10731
10734
|
var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
10732
10735
|
var DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
10733
10736
|
var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
10734
|
-
var schemaReady = null;
|
|
10735
10737
|
function concurrencyLimitForUser(user) {
|
|
10736
10738
|
const tier = Math.max(0, Number(user.subscription_concurrency ?? 0));
|
|
10737
10739
|
return Math.max(1, tier) + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
|
|
@@ -10769,40 +10771,15 @@ function safeMetadata(metadata) {
|
|
|
10769
10771
|
return null;
|
|
10770
10772
|
}
|
|
10771
10773
|
}
|
|
10772
|
-
async function ensureConcurrencyGateSchema() {
|
|
10773
|
-
if (!schemaReady) {
|
|
10774
|
-
schemaReady = (async () => {
|
|
10775
|
-
const db = getDb();
|
|
10776
|
-
await db.execute(`
|
|
10777
|
-
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
10778
|
-
id TEXT PRIMARY KEY,
|
|
10779
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
10780
|
-
operation TEXT NOT NULL,
|
|
10781
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
10782
|
-
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
10783
|
-
expires_at TEXT NOT NULL,
|
|
10784
|
-
released_at TEXT,
|
|
10785
|
-
metadata TEXT
|
|
10786
|
-
)
|
|
10787
|
-
`);
|
|
10788
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
10789
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
10790
|
-
})();
|
|
10791
|
-
}
|
|
10792
|
-
return schemaReady;
|
|
10793
|
-
}
|
|
10794
10774
|
async function expireConcurrencyLocksForUser(userId) {
|
|
10795
|
-
await ensureConcurrencyGateSchema();
|
|
10796
10775
|
await getDb().execute({
|
|
10797
|
-
sql: `
|
|
10798
|
-
SET status = 'expired', released_at = datetime('now')
|
|
10776
|
+
sql: `DELETE FROM concurrency_locks
|
|
10799
10777
|
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
10800
10778
|
args: [userId]
|
|
10801
10779
|
});
|
|
10802
10780
|
}
|
|
10803
|
-
async function countActiveUsageForUser(userId) {
|
|
10804
|
-
await
|
|
10805
|
-
await expireConcurrencyLocksForUser(userId);
|
|
10781
|
+
async function countActiveUsageForUser(userId, alreadyExpired = false) {
|
|
10782
|
+
if (!alreadyExpired) await expireConcurrencyLocksForUser(userId);
|
|
10806
10783
|
const res = await getDb().execute({
|
|
10807
10784
|
sql: `SELECT
|
|
10808
10785
|
(
|
|
@@ -10822,7 +10799,6 @@ async function countActiveUsageForUser(userId) {
|
|
|
10822
10799
|
}
|
|
10823
10800
|
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
10824
10801
|
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
10825
|
-
await ensureConcurrencyGateSchema();
|
|
10826
10802
|
await expireConcurrencyLocksForUser(userId);
|
|
10827
10803
|
const res = await getDb().execute({
|
|
10828
10804
|
sql: `UPDATE concurrency_locks
|
|
@@ -10836,13 +10812,12 @@ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
|
10836
10812
|
return Number(res.rowsAffected ?? 0) > 0;
|
|
10837
10813
|
}
|
|
10838
10814
|
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
10839
|
-
await ensureConcurrencyGateSchema();
|
|
10840
10815
|
const limit = concurrencyLimitForUser(user);
|
|
10841
10816
|
if (isConcurrencyLimitBypassed(user)) {
|
|
10842
10817
|
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
10843
10818
|
}
|
|
10844
10819
|
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
10845
|
-
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
10820
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id, true), limit, operation, reused: true };
|
|
10846
10821
|
}
|
|
10847
10822
|
await expireConcurrencyLocksForUser(user.id);
|
|
10848
10823
|
const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
|
|
@@ -10873,7 +10848,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
10873
10848
|
limit
|
|
10874
10849
|
]
|
|
10875
10850
|
});
|
|
10876
|
-
const active = await countActiveUsageForUser(user.id);
|
|
10851
|
+
const active = await countActiveUsageForUser(user.id, true);
|
|
10877
10852
|
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
10878
10853
|
return { ok: true, lockId, active, limit, operation };
|
|
10879
10854
|
}
|
|
@@ -10881,17 +10856,13 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
10881
10856
|
}
|
|
10882
10857
|
async function releaseConcurrencyGate(lockId) {
|
|
10883
10858
|
if (!lockId) return;
|
|
10884
|
-
await ensureConcurrencyGateSchema();
|
|
10885
10859
|
await getDb().execute({
|
|
10886
|
-
sql: `
|
|
10887
|
-
SET status = 'released', released_at = datetime('now')
|
|
10888
|
-
WHERE id = ? AND status = 'active'`,
|
|
10860
|
+
sql: `DELETE FROM concurrency_locks WHERE id = ?`,
|
|
10889
10861
|
args: [lockId]
|
|
10890
10862
|
});
|
|
10891
10863
|
}
|
|
10892
10864
|
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
10893
10865
|
if (!lockId) return;
|
|
10894
|
-
await ensureConcurrencyGateSchema();
|
|
10895
10866
|
await getDb().execute({
|
|
10896
10867
|
sql: `UPDATE concurrency_locks
|
|
10897
10868
|
SET expires_at = datetime('now', ?)
|
|
@@ -14708,7 +14679,7 @@ async function recoverSettlement(row, user, authorization, duplicate) {
|
|
|
14708
14679
|
}
|
|
14709
14680
|
}
|
|
14710
14681
|
await getDb().execute({
|
|
14711
|
-
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status
|
|
14682
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('settling', 'cost_pending')",
|
|
14712
14683
|
args: [row.id]
|
|
14713
14684
|
});
|
|
14714
14685
|
await getDb().execute({
|
|
@@ -14753,11 +14724,13 @@ async function settleScheduledRun(args) {
|
|
|
14753
14724
|
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
14754
14725
|
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
14755
14726
|
const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
14756
|
-
const
|
|
14727
|
+
const modelCostUnreported = args.modelCostStatus === "missing";
|
|
14728
|
+
const pendingReason = modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
|
|
14757
14729
|
const metadata = safeMetadata2({
|
|
14758
14730
|
...args.metadata,
|
|
14759
14731
|
billingPhase: "settlement",
|
|
14760
14732
|
chargeMode: "model_delta",
|
|
14733
|
+
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
14761
14734
|
...pendingReason ? { reason: pendingReason } : {}
|
|
14762
14735
|
});
|
|
14763
14736
|
const eventId = existing?.id ?? randomUUID4();
|
|
@@ -14801,6 +14774,72 @@ async function settleScheduledRun(args) {
|
|
|
14801
14774
|
}
|
|
14802
14775
|
return recoverSettlement(existing, user, authorization, duplicateClaim);
|
|
14803
14776
|
}
|
|
14777
|
+
async function reconcileScheduledRunSettlements(limit = 25, minimumAgeHours = 24) {
|
|
14778
|
+
const stale = await getDb().execute({
|
|
14779
|
+
sql: `SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata
|
|
14780
|
+
FROM billing_events
|
|
14781
|
+
WHERE billing_class = ? AND status = 'cost_pending'
|
|
14782
|
+
AND created_at < datetime('now', ?)
|
|
14783
|
+
ORDER BY created_at
|
|
14784
|
+
LIMIT ?`,
|
|
14785
|
+
args: [SCHEDULED_RUN_BILLING_CLASS, `-${Math.max(1, Math.trunc(minimumAgeHours))} hours`, Math.max(1, Math.trunc(limit))]
|
|
14786
|
+
});
|
|
14787
|
+
const result = { checked: stale.rows.length, settled: 0, chargedMc: 0, failed: 0 };
|
|
14788
|
+
for (const raw of stale.rows) {
|
|
14789
|
+
const row = raw;
|
|
14790
|
+
try {
|
|
14791
|
+
const authorization = await authorizationById(row.authorization_id);
|
|
14792
|
+
if (!authorization) {
|
|
14793
|
+
result.failed += 1;
|
|
14794
|
+
continue;
|
|
14795
|
+
}
|
|
14796
|
+
const cap = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
14797
|
+
const chargeMc = Math.min(Math.max(0, Math.round(Number(row.amount_mc))), cap);
|
|
14798
|
+
if (chargeMc > 0) {
|
|
14799
|
+
const debit = await debitMcIdempotent(
|
|
14800
|
+
Number(authorization.user_id),
|
|
14801
|
+
chargeMc,
|
|
14802
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
14803
|
+
"scheduled-run:" + authorization.id + ":model:reconciled",
|
|
14804
|
+
authorization.id + ":model"
|
|
14805
|
+
);
|
|
14806
|
+
if (!debit.ok) {
|
|
14807
|
+
result.failed += 1;
|
|
14808
|
+
continue;
|
|
14809
|
+
}
|
|
14810
|
+
}
|
|
14811
|
+
await getDb().execute({
|
|
14812
|
+
sql: `UPDATE billing_events
|
|
14813
|
+
SET status = 'settled', amount_mc = ?, settled_at = datetime('now'), metadata = ?
|
|
14814
|
+
WHERE id = ? AND status = 'cost_pending'`,
|
|
14815
|
+
args: [
|
|
14816
|
+
chargeMc,
|
|
14817
|
+
safeMetadata2({ ...metadataReason(row) ? { reason: metadataReason(row) } : {}, autoReconciled: true, cappedAtAuthorizedMaximum: chargeMc < Number(row.amount_mc) }),
|
|
14818
|
+
row.id
|
|
14819
|
+
]
|
|
14820
|
+
});
|
|
14821
|
+
await getDb().execute({
|
|
14822
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending')",
|
|
14823
|
+
args: [authorization.id]
|
|
14824
|
+
});
|
|
14825
|
+
result.settled += 1;
|
|
14826
|
+
result.chargedMc += chargeMc;
|
|
14827
|
+
console.error(JSON.stringify({
|
|
14828
|
+
event: "scheduled_run_settlement_reconciled",
|
|
14829
|
+
authorization_id: authorization.id,
|
|
14830
|
+
charged_mc: chargeMc,
|
|
14831
|
+
reason: metadataReason(row) ?? "unreported_cost"
|
|
14832
|
+
}));
|
|
14833
|
+
} catch (err) {
|
|
14834
|
+
result.failed += 1;
|
|
14835
|
+
console.error(JSON.stringify({
|
|
14836
|
+
event: "scheduled_run_reconciliation_failed",
|
|
14837
|
+
message: err instanceof Error ? err.message : String(err)
|
|
14838
|
+
}));
|
|
14839
|
+
}
|
|
14840
|
+
}
|
|
14841
|
+
return result;
|
|
14842
|
+
}
|
|
14804
14843
|
async function voidScheduledRunAuthorization(args) {
|
|
14805
14844
|
const user = await resolveSchedulingUser(args.identity);
|
|
14806
14845
|
const authorization = await authorizationById(args.authorizationId);
|
|
@@ -15672,28 +15711,6 @@ internalInboxApp.post("/broadcast", async (c) => {
|
|
|
15672
15711
|
|
|
15673
15712
|
// src/api/connected-account-billing.ts
|
|
15674
15713
|
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
15675
|
-
async function ensureConnectedAccountBillingSchema() {
|
|
15676
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_billing (
|
|
15677
|
-
user_id INTEGER PRIMARY KEY,
|
|
15678
|
-
stripe_subscription_id TEXT,
|
|
15679
|
-
stripe_subscription_item_id TEXT,
|
|
15680
|
-
price_id TEXT,
|
|
15681
|
-
quantity INTEGER NOT NULL DEFAULT 0,
|
|
15682
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
15683
|
-
last_error_code TEXT,
|
|
15684
|
-
checked_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
15685
|
-
synced_at TEXT,
|
|
15686
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
15687
|
-
)`);
|
|
15688
|
-
await getDb().execute(`CREATE TABLE IF NOT EXISTS connected_account_charges (
|
|
15689
|
-
user_id INTEGER NOT NULL,
|
|
15690
|
-
connection_id TEXT NOT NULL,
|
|
15691
|
-
period_start TEXT NOT NULL,
|
|
15692
|
-
amount_mc INTEGER NOT NULL,
|
|
15693
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
15694
|
-
PRIMARY KEY (user_id, connection_id, period_start)
|
|
15695
|
-
)`);
|
|
15696
|
-
}
|
|
15697
15714
|
function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
|
|
15698
15715
|
return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
15699
15716
|
}
|
|
@@ -15702,7 +15719,6 @@ function isBillableConnection(connection) {
|
|
|
15702
15719
|
return BILLABLE_LIFECYCLE.has(String(connection.lifecycleStatus ?? ""));
|
|
15703
15720
|
}
|
|
15704
15721
|
async function chargedConnectionIds(userId, periodStart) {
|
|
15705
|
-
await ensureConnectedAccountBillingSchema();
|
|
15706
15722
|
const result = await getDb().execute({
|
|
15707
15723
|
sql: "SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",
|
|
15708
15724
|
args: [Number(userId), periodStart]
|
|
@@ -15710,7 +15726,6 @@ async function chargedConnectionIds(userId, periodStart) {
|
|
|
15710
15726
|
return new Set(result.rows.map((row) => String(row.connection_id)));
|
|
15711
15727
|
}
|
|
15712
15728
|
async function getConnectedAccountBillingState(userId) {
|
|
15713
|
-
await ensureConnectedAccountBillingSchema();
|
|
15714
15729
|
const result = await getDb().execute({
|
|
15715
15730
|
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
15716
15731
|
args: [Number(userId)]
|
|
@@ -15718,7 +15733,6 @@ async function getConnectedAccountBillingState(userId) {
|
|
|
15718
15733
|
return result.rows[0] ? result.rows[0] : null;
|
|
15719
15734
|
}
|
|
15720
15735
|
async function setConnectedAccountBillingState(input) {
|
|
15721
|
-
await ensureConnectedAccountBillingSchema();
|
|
15722
15736
|
await getDb().execute({
|
|
15723
15737
|
sql: `INSERT INTO connected_account_billing
|
|
15724
15738
|
(user_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity, status, last_error_code, checked_at, synced_at, updated_at)
|
|
@@ -22184,6 +22198,82 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
|
22184
22198
|
return { checked: jobs.length, staleFailed, settled, failed };
|
|
22185
22199
|
}
|
|
22186
22200
|
|
|
22201
|
+
// src/api/retention-sweeps.ts
|
|
22202
|
+
var REPORTING_WINDOW_DAYS = 30;
|
|
22203
|
+
var TELEMETRY_RETENTION_DAYS = 45;
|
|
22204
|
+
var FAILURE_RETENTION_DAYS = 90;
|
|
22205
|
+
var RETENTION_RULES = [
|
|
22206
|
+
{ table: "request_events", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22207
|
+
{ table: "kernel_session_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22208
|
+
{ table: "vendor_usage_log", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS },
|
|
22209
|
+
{ table: "harvest_attempts", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, timestampFormat: "iso8601" },
|
|
22210
|
+
{ table: "browser_agent_actions", timestampColumn: "created_at", retentionDays: REPORTING_WINDOW_DAYS },
|
|
22211
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: TELEMETRY_RETENTION_DAYS, where: "status = 'done'" },
|
|
22212
|
+
{ table: "jobs", timestampColumn: "created_at", retentionDays: FAILURE_RETENTION_DAYS, where: "status = 'failed'" }
|
|
22213
|
+
];
|
|
22214
|
+
function cutoffFor(rule) {
|
|
22215
|
+
const cutoff = new Date(Date.now() - rule.retentionDays * 24 * 60 * 60 * 1e3);
|
|
22216
|
+
if (rule.timestampFormat === "iso8601") return cutoff.toISOString();
|
|
22217
|
+
return cutoff.toISOString().replace("T", " ").slice(0, 19);
|
|
22218
|
+
}
|
|
22219
|
+
var DEFAULT_BATCH = 500;
|
|
22220
|
+
var DEFAULT_MAX_DELETES_PER_RULE = 1e3;
|
|
22221
|
+
var DEFAULT_SWEEP_DEADLINE_MS = 15e3;
|
|
22222
|
+
async function tableExists(table) {
|
|
22223
|
+
const res = await getDb().execute({
|
|
22224
|
+
sql: "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
|
|
22225
|
+
args: [table]
|
|
22226
|
+
});
|
|
22227
|
+
return res.rows.length > 0;
|
|
22228
|
+
}
|
|
22229
|
+
async function sweepRule(rule, maxDeletes) {
|
|
22230
|
+
if (!await tableExists(rule.table)) return 0;
|
|
22231
|
+
const filter = rule.where ? ` AND ${rule.where}` : "";
|
|
22232
|
+
const cutoff = cutoffFor(rule);
|
|
22233
|
+
let deleted = 0;
|
|
22234
|
+
while (deleted < maxDeletes) {
|
|
22235
|
+
const res = await getDb().execute({
|
|
22236
|
+
sql: `DELETE FROM ${rule.table} WHERE rowid IN (
|
|
22237
|
+
SELECT rowid FROM ${rule.table}
|
|
22238
|
+
WHERE ${rule.timestampColumn} < ?${filter}
|
|
22239
|
+
LIMIT ?)`,
|
|
22240
|
+
args: [cutoff, Math.min(DEFAULT_BATCH, maxDeletes - deleted)]
|
|
22241
|
+
});
|
|
22242
|
+
const n = Number(res.rowsAffected ?? 0);
|
|
22243
|
+
deleted += n;
|
|
22244
|
+
if (n === 0) break;
|
|
22245
|
+
}
|
|
22246
|
+
return deleted;
|
|
22247
|
+
}
|
|
22248
|
+
async function sweepRetention(options = {}) {
|
|
22249
|
+
const maxDeletes = options.maxDeletesPerRule ?? DEFAULT_MAX_DELETES_PER_RULE;
|
|
22250
|
+
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_SWEEP_DEADLINE_MS);
|
|
22251
|
+
const perTable = {};
|
|
22252
|
+
let deleted = 0;
|
|
22253
|
+
let truncated = false;
|
|
22254
|
+
for (const rule of RETENTION_RULES) {
|
|
22255
|
+
const key = rule.where ? `${rule.table}:${rule.where}` : rule.table;
|
|
22256
|
+
if (Date.now() >= deadline) {
|
|
22257
|
+
truncated = true;
|
|
22258
|
+
break;
|
|
22259
|
+
}
|
|
22260
|
+
try {
|
|
22261
|
+
const n = await sweepRule(rule, maxDeletes);
|
|
22262
|
+
perTable[key] = n;
|
|
22263
|
+
deleted += n;
|
|
22264
|
+
if (n >= maxDeletes) truncated = true;
|
|
22265
|
+
} catch (err) {
|
|
22266
|
+
perTable[key] = -1;
|
|
22267
|
+
console.error(JSON.stringify({
|
|
22268
|
+
event: "retention_sweep_failed",
|
|
22269
|
+
table: rule.table,
|
|
22270
|
+
message: err instanceof Error ? err.message : String(err)
|
|
22271
|
+
}));
|
|
22272
|
+
}
|
|
22273
|
+
}
|
|
22274
|
+
return { deleted, perTable, truncated };
|
|
22275
|
+
}
|
|
22276
|
+
|
|
22187
22277
|
// src/api/directory-workflow-reconciliation.ts
|
|
22188
22278
|
async function reconcileDirectoryWorkflowSettlements(limit = 25) {
|
|
22189
22279
|
const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
|
|
@@ -24893,6 +24983,7 @@ async function listSessionRows(userId, includeClosed = false) {
|
|
|
24893
24983
|
});
|
|
24894
24984
|
return res.rows;
|
|
24895
24985
|
}
|
|
24986
|
+
var ORPHANED_SESSION_STATUS = "runtime_close_pending";
|
|
24896
24987
|
async function listIdleOpenSessionRows(idleSeconds) {
|
|
24897
24988
|
const db = getDb();
|
|
24898
24989
|
const res = await db.execute({
|
|
@@ -24918,13 +25009,21 @@ async function setBilledMc(id, billedMc) {
|
|
|
24918
25009
|
args: [Math.round(billedMc), id]
|
|
24919
25010
|
});
|
|
24920
25011
|
}
|
|
24921
|
-
async function markSessionClosed(id) {
|
|
25012
|
+
async function markSessionClosed(id, runtimeClosed = true) {
|
|
24922
25013
|
const db = getDb();
|
|
24923
25014
|
await db.execute({
|
|
24924
|
-
sql: `UPDATE browser_agent_sessions SET status =
|
|
24925
|
-
args: [id]
|
|
25015
|
+
sql: `UPDATE browser_agent_sessions SET status = ?, closed_at = datetime('now') WHERE id = ?`,
|
|
25016
|
+
args: [runtimeClosed ? "closed" : ORPHANED_SESSION_STATUS, id]
|
|
24926
25017
|
});
|
|
24927
25018
|
}
|
|
25019
|
+
async function listOrphanedSessionRows(limit = 50) {
|
|
25020
|
+
const db = getDb();
|
|
25021
|
+
const res = await db.execute({
|
|
25022
|
+
sql: `SELECT * FROM browser_agent_sessions WHERE status = ? ORDER BY closed_at LIMIT ?`,
|
|
25023
|
+
args: [ORPHANED_SESSION_STATUS, Math.max(1, Math.min(200, Math.round(limit)))]
|
|
25024
|
+
});
|
|
25025
|
+
return res.rows;
|
|
25026
|
+
}
|
|
24928
25027
|
async function recordAction(input) {
|
|
24929
25028
|
const db = getDb();
|
|
24930
25029
|
await db.execute({
|
|
@@ -26064,6 +26163,21 @@ async function charge(sessionId, userId, _startedAtMs) {
|
|
|
26064
26163
|
if (res.ok) await setBilledMc(sessionId, owed);
|
|
26065
26164
|
}
|
|
26066
26165
|
}
|
|
26166
|
+
async function closeRuntimeSession(runtimeSessionId, sessionId) {
|
|
26167
|
+
try {
|
|
26168
|
+
await closeSession(runtimeSessionId);
|
|
26169
|
+
return true;
|
|
26170
|
+
} catch (err) {
|
|
26171
|
+
console.error(JSON.stringify({
|
|
26172
|
+
event: "browser_runtime_close_failed",
|
|
26173
|
+
session_id: sessionId,
|
|
26174
|
+
runtime_session_id: runtimeSessionId,
|
|
26175
|
+
error_name: err instanceof Error ? err.name : typeof err,
|
|
26176
|
+
message: sanitizeVendorName(err instanceof Error ? err.message : String(err)).slice(0, 300)
|
|
26177
|
+
}));
|
|
26178
|
+
return false;
|
|
26179
|
+
}
|
|
26180
|
+
}
|
|
26067
26181
|
async function closeAndSettleSession(row) {
|
|
26068
26182
|
if (row.user_id != null) {
|
|
26069
26183
|
try {
|
|
@@ -26071,15 +26185,24 @@ async function closeAndSettleSession(row) {
|
|
|
26071
26185
|
} catch {
|
|
26072
26186
|
}
|
|
26073
26187
|
}
|
|
26074
|
-
|
|
26075
|
-
|
|
26076
|
-
} catch {
|
|
26077
|
-
}
|
|
26078
|
-
await markSessionClosed(row.id).catch(() => {
|
|
26188
|
+
const runtimeClosed = await closeRuntimeSession(row.runtime_session_id, row.id);
|
|
26189
|
+
await markSessionClosed(row.id, runtimeClosed).catch(() => {
|
|
26079
26190
|
});
|
|
26080
26191
|
await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
|
|
26081
26192
|
});
|
|
26082
26193
|
}
|
|
26194
|
+
async function retryOrphanedBrowserSessions(limit = 50) {
|
|
26195
|
+
const rows = await listOrphanedSessionRows(limit);
|
|
26196
|
+
let recovered = 0;
|
|
26197
|
+
for (const row of rows) {
|
|
26198
|
+
if (await closeRuntimeSession(row.runtime_session_id, row.id)) {
|
|
26199
|
+
await markSessionClosed(row.id, true).catch(() => {
|
|
26200
|
+
});
|
|
26201
|
+
recovered++;
|
|
26202
|
+
}
|
|
26203
|
+
}
|
|
26204
|
+
return { recovered, stillOpen: rows.length - recovered };
|
|
26205
|
+
}
|
|
26083
26206
|
async function completeAuthConnection(connectionId) {
|
|
26084
26207
|
const connection = await getAuthConnectionRow(connectionId);
|
|
26085
26208
|
if (!connection) return { ok: false, error: "not found" };
|
|
@@ -26104,7 +26227,7 @@ async function reapIdleBrowserSessions(idleSeconds = 120) {
|
|
|
26104
26227
|
function publicSession(row) {
|
|
26105
26228
|
return {
|
|
26106
26229
|
session_id: row.id,
|
|
26107
|
-
status: row.status,
|
|
26230
|
+
status: row.status === ORPHANED_SESSION_STATUS ? "closed" : row.status,
|
|
26108
26231
|
label: row.label,
|
|
26109
26232
|
watch_url: `/console/${row.id}`,
|
|
26110
26233
|
created_at: row.created_at,
|
|
@@ -30873,8 +30996,212 @@ import { Hono as Hono31 } from "hono";
|
|
|
30873
30996
|
import { z as z32 } from "zod";
|
|
30874
30997
|
|
|
30875
30998
|
// src/api/commons-repository.ts
|
|
30876
|
-
import { createHash as
|
|
30877
|
-
|
|
30999
|
+
import { createHash as createHash13, randomUUID as randomUUID15 } from "crypto";
|
|
31000
|
+
|
|
31001
|
+
// src/api/schema-presence.ts
|
|
31002
|
+
var cache = null;
|
|
31003
|
+
async function loadSchemaObjects() {
|
|
31004
|
+
const res = await getDb().execute(
|
|
31005
|
+
"SELECT name FROM sqlite_master WHERE type IN ('table', 'index')"
|
|
31006
|
+
);
|
|
31007
|
+
return new Set(res.rows.map((row) => String(row.name)));
|
|
31008
|
+
}
|
|
31009
|
+
function invalidateSchemaPresence() {
|
|
31010
|
+
cache = null;
|
|
31011
|
+
}
|
|
31012
|
+
async function schemaObjectsPresent(names) {
|
|
31013
|
+
if (names.length === 0) return true;
|
|
31014
|
+
try {
|
|
31015
|
+
if (!cache) cache = loadSchemaObjects();
|
|
31016
|
+
const present = await cache;
|
|
31017
|
+
if (names.every((name) => present.has(name))) return true;
|
|
31018
|
+
invalidateSchemaPresence();
|
|
31019
|
+
return false;
|
|
31020
|
+
} catch {
|
|
31021
|
+
invalidateSchemaPresence();
|
|
31022
|
+
return false;
|
|
31023
|
+
}
|
|
31024
|
+
}
|
|
31025
|
+
|
|
31026
|
+
// src/api/commons-linksets.ts
|
|
31027
|
+
import { createHash as createHash12 } from "crypto";
|
|
31028
|
+
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
31029
|
+
var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
31030
|
+
var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
31031
|
+
"about",
|
|
31032
|
+
"alternate",
|
|
31033
|
+
"author",
|
|
31034
|
+
"canonical",
|
|
31035
|
+
"cite-as",
|
|
31036
|
+
"collection",
|
|
31037
|
+
"contents",
|
|
31038
|
+
"copyright",
|
|
31039
|
+
"describedby",
|
|
31040
|
+
"describes",
|
|
31041
|
+
"first",
|
|
31042
|
+
"glossary",
|
|
31043
|
+
"help",
|
|
31044
|
+
"icon",
|
|
31045
|
+
"index",
|
|
31046
|
+
"item",
|
|
31047
|
+
"last",
|
|
31048
|
+
"latest-version",
|
|
31049
|
+
"license",
|
|
31050
|
+
"memento",
|
|
31051
|
+
"next",
|
|
31052
|
+
"original",
|
|
31053
|
+
"predecessor-version",
|
|
31054
|
+
"prev",
|
|
31055
|
+
"related",
|
|
31056
|
+
"replies",
|
|
31057
|
+
"service-desc",
|
|
31058
|
+
"service-doc",
|
|
31059
|
+
"successor-version",
|
|
31060
|
+
"terms-of-service",
|
|
31061
|
+
"type",
|
|
31062
|
+
"via",
|
|
31063
|
+
"version-history"
|
|
31064
|
+
]);
|
|
31065
|
+
function normalizeCommonsClaims(input) {
|
|
31066
|
+
if (input === void 0) return void 0;
|
|
31067
|
+
if (!Array.isArray(input)) throw new Error("claims must be an array.");
|
|
31068
|
+
if (input.length > 300) throw new Error("claims must contain no more than 300 assertions.");
|
|
31069
|
+
const seen = /* @__PURE__ */ new Set();
|
|
31070
|
+
return input.map((claim, index) => {
|
|
31071
|
+
const predicate = String(claim.predicate ?? "").trim();
|
|
31072
|
+
if (!isSupportedPredicate(predicate)) {
|
|
31073
|
+
throw new Error(`claims[${index}].predicate must be a supported registered relation or an absolute HTTPS relation URI.`);
|
|
31074
|
+
}
|
|
31075
|
+
const objectEntityId = optionalText(claim.objectEntityId, 80);
|
|
31076
|
+
const objectUrl = optionalHttpsUrl(claim.objectUrl, `claims[${index}].objectUrl`);
|
|
31077
|
+
if ((objectEntityId ? 1 : 0) + (objectUrl ? 1 : 0) !== 1) {
|
|
31078
|
+
throw new Error(`claims[${index}] must provide exactly one of objectEntityId or objectUrl.`);
|
|
31079
|
+
}
|
|
31080
|
+
if (objectEntityId && !/^TPW-Q[A-Za-z0-9._-]+$/.test(objectEntityId)) {
|
|
31081
|
+
throw new Error(`claims[${index}].objectEntityId must be a Transparent Commons TPW-Q identifier.`);
|
|
31082
|
+
}
|
|
31083
|
+
const confidence = claim.confidence ?? 1;
|
|
31084
|
+
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
|
|
31085
|
+
throw new Error(`claims[${index}].confidence must be between 0 and 1.`);
|
|
31086
|
+
}
|
|
31087
|
+
const key = `${predicate}\0${objectEntityId ?? objectUrl}`;
|
|
31088
|
+
if (seen.has(key)) throw new Error(`claims[${index}] duplicates another predicate and object pair.`);
|
|
31089
|
+
seen.add(key);
|
|
31090
|
+
return {
|
|
31091
|
+
predicate,
|
|
31092
|
+
predicateLabel: optionalText(claim.predicateLabel, 160),
|
|
31093
|
+
objectEntityId,
|
|
31094
|
+
objectUrl,
|
|
31095
|
+
title: optionalText(claim.title, 500),
|
|
31096
|
+
mediaType: optionalText(claim.mediaType, 160),
|
|
31097
|
+
hreflang: normalizeHreflang(claim.hreflang),
|
|
31098
|
+
qualifiers: normalizeScalarRecord(claim.qualifiers),
|
|
31099
|
+
references: normalizeReferences(claim.references),
|
|
31100
|
+
rank: claim.rank ?? "normal",
|
|
31101
|
+
confidence,
|
|
31102
|
+
confidenceReason: optionalText(claim.confidenceReason, 1e3),
|
|
31103
|
+
disputeState: claim.disputeState ?? "undisputed",
|
|
31104
|
+
validFrom: optionalDate(claim.validFrom, `claims[${index}].validFrom`),
|
|
31105
|
+
validTo: optionalDate(claim.validTo, `claims[${index}].validTo`),
|
|
31106
|
+
asOf: optionalDate(claim.asOf, `claims[${index}].asOf`),
|
|
31107
|
+
sourceFamily: claim.sourceFamily ?? "unknown"
|
|
31108
|
+
};
|
|
31109
|
+
});
|
|
31110
|
+
}
|
|
31111
|
+
function claimIsPublic(claim, now = /* @__PURE__ */ new Date()) {
|
|
31112
|
+
if (claim.reviewState !== "approved" || claim.rank === "deprecated") return false;
|
|
31113
|
+
if (!["undisputed", "attributed"].includes(claim.disputeState ?? "undisputed")) return false;
|
|
31114
|
+
const timestamp = now.getTime();
|
|
31115
|
+
if (claim.validFrom && Date.parse(claim.validFrom) > timestamp) return false;
|
|
31116
|
+
if (claim.validTo && Date.parse(claim.validTo) < timestamp) return false;
|
|
31117
|
+
return true;
|
|
31118
|
+
}
|
|
31119
|
+
function buildCommonsLinkset(entity, claims) {
|
|
31120
|
+
const publicUrl = `${commonsPublicWikiBaseUrl()}/wiki/${encodeURIComponent(entity.slug)}`;
|
|
31121
|
+
const linksetUrl = commonsLinksetUrl(entity.id);
|
|
31122
|
+
const context = { anchor: publicUrl };
|
|
31123
|
+
const publicClaims = claims.filter((claim) => claimIsPublic(claim)).sort((a, b) => a.predicate.localeCompare(b.predicate) || a.href.localeCompare(b.href) || a.id.localeCompare(b.id));
|
|
31124
|
+
for (const claim of publicClaims) {
|
|
31125
|
+
const targets = context[claim.predicate] ?? [];
|
|
31126
|
+
const target = { href: claim.href };
|
|
31127
|
+
if (claim.mediaType) target.type = claim.mediaType;
|
|
31128
|
+
if (claim.hreflang) target.hreflang = claim.hreflang;
|
|
31129
|
+
if (claim.title) target.title = claim.title;
|
|
31130
|
+
targets.push(target);
|
|
31131
|
+
context[claim.predicate] = targets;
|
|
31132
|
+
}
|
|
31133
|
+
const document2 = { linkset: [context] };
|
|
31134
|
+
const etag = `"${createHash12("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
31135
|
+
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
31136
|
+
}
|
|
31137
|
+
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
31138
|
+
const url = commonsLinksetUrl(idOrSlug);
|
|
31139
|
+
return `<${url}>; rel="linkset"; type="${COMMONS_LINKSET_MEDIA_TYPE}"; profile="${COMMONS_RELATIONSHIP_PROFILE}"`;
|
|
31140
|
+
}
|
|
31141
|
+
function commonsLinksetUrl(idOrSlug) {
|
|
31142
|
+
const base = (process.env.APP_URL?.trim() || "https://mcpscraper.dev").replace(/\/+$/, "");
|
|
31143
|
+
return `${base}/commons/entities/${encodeURIComponent(idOrSlug)}/linkset`;
|
|
31144
|
+
}
|
|
31145
|
+
function commonsClaimIdentityKey(claim) {
|
|
31146
|
+
return `${claim.predicate}\0${claim.objectEntityId ?? claim.objectUrl ?? ""}`;
|
|
31147
|
+
}
|
|
31148
|
+
function commonsPublicWikiBaseUrl() {
|
|
31149
|
+
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || "https://transparent-commons.cc").replace(/\/+$/, "");
|
|
31150
|
+
}
|
|
31151
|
+
function isSupportedPredicate(value) {
|
|
31152
|
+
if (REGISTERED_RELATIONS.has(value)) return true;
|
|
31153
|
+
try {
|
|
31154
|
+
const url = new URL(value);
|
|
31155
|
+
return url.protocol === "https:" && Boolean(url.hostname);
|
|
31156
|
+
} catch {
|
|
31157
|
+
return false;
|
|
31158
|
+
}
|
|
31159
|
+
}
|
|
31160
|
+
function optionalText(value, max) {
|
|
31161
|
+
const text2 = String(value ?? "").trim();
|
|
31162
|
+
return text2 ? text2.slice(0, max) : void 0;
|
|
31163
|
+
}
|
|
31164
|
+
function optionalHttpsUrl(value, field) {
|
|
31165
|
+
const text2 = optionalText(value, 2e3);
|
|
31166
|
+
if (!text2) return void 0;
|
|
31167
|
+
try {
|
|
31168
|
+
const url = new URL(text2);
|
|
31169
|
+
if (url.protocol !== "https:") throw new Error();
|
|
31170
|
+
url.hash = "";
|
|
31171
|
+
return url.href;
|
|
31172
|
+
} catch {
|
|
31173
|
+
throw new Error(`${field} must be an absolute HTTPS URL.`);
|
|
31174
|
+
}
|
|
31175
|
+
}
|
|
31176
|
+
function optionalDate(value, field) {
|
|
31177
|
+
const text2 = optionalText(value, 80);
|
|
31178
|
+
if (!text2) return void 0;
|
|
31179
|
+
const timestamp = Date.parse(text2);
|
|
31180
|
+
if (!Number.isFinite(timestamp)) throw new Error(`${field} must be an ISO-compatible date.`);
|
|
31181
|
+
return new Date(timestamp).toISOString();
|
|
31182
|
+
}
|
|
31183
|
+
function normalizeScalarRecord(value) {
|
|
31184
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
31185
|
+
return Object.fromEntries(Object.entries(value).slice(0, 50).map(([key, item]) => [key.slice(0, 120), item]));
|
|
31186
|
+
}
|
|
31187
|
+
function normalizeReferences(value) {
|
|
31188
|
+
if (!Array.isArray(value)) return void 0;
|
|
31189
|
+
return value.slice(0, 50).map((reference) => ({
|
|
31190
|
+
title: String(reference.title ?? "").trim().slice(0, 240),
|
|
31191
|
+
url: optionalHttpsUrl(reference.url, "claim reference URL"),
|
|
31192
|
+
source: optionalText(reference.source, 240),
|
|
31193
|
+
note: optionalText(reference.note, 1e3),
|
|
31194
|
+
accessedAt: optionalDate(reference.accessedAt, "claim reference accessedAt")
|
|
31195
|
+
})).filter((reference) => reference.title);
|
|
31196
|
+
}
|
|
31197
|
+
function normalizeHreflang(value) {
|
|
31198
|
+
if (!Array.isArray(value)) return void 0;
|
|
31199
|
+
const languages = [...new Set(value.map((item) => optionalText(item, 80)).filter((item) => Boolean(item)))];
|
|
31200
|
+
return languages.length ? languages : void 0;
|
|
31201
|
+
}
|
|
31202
|
+
|
|
31203
|
+
// src/api/commons-repository.ts
|
|
31204
|
+
var COMMONS_SCHEMA_VERSION = "2026-08-05.1";
|
|
30878
31205
|
var DEFAULT_COMMONS_BASE_URL = "https://transparent-commons.cc";
|
|
30879
31206
|
var DEFAULT_ENTITY_TYPE = "PublicArticle";
|
|
30880
31207
|
var COMMONS_ENTITY_PROFILES = {
|
|
@@ -31022,9 +31349,38 @@ var COMMONS_ENTITY_PROFILES = {
|
|
|
31022
31349
|
]
|
|
31023
31350
|
}
|
|
31024
31351
|
};
|
|
31025
|
-
var
|
|
31352
|
+
var schemaReady = false;
|
|
31353
|
+
var COMMONS_SCHEMA_OBJECTS = [
|
|
31354
|
+
"schema_migrations",
|
|
31355
|
+
"commons_entities",
|
|
31356
|
+
"commons_claims",
|
|
31357
|
+
"commons_proposals",
|
|
31358
|
+
"commons_ledger",
|
|
31359
|
+
"commons_saved_filters",
|
|
31360
|
+
"commons_index_documents",
|
|
31361
|
+
"commons_publications",
|
|
31362
|
+
"commons_publication_editions",
|
|
31363
|
+
"commons_entities_status_updated",
|
|
31364
|
+
"commons_entities_type_updated",
|
|
31365
|
+
"commons_entities_source_domain",
|
|
31366
|
+
"commons_claims_subject_predicate",
|
|
31367
|
+
"commons_claims_object_entity",
|
|
31368
|
+
"commons_proposals_user_created",
|
|
31369
|
+
"commons_proposals_entity_created",
|
|
31370
|
+
"commons_ledger_entity_created",
|
|
31371
|
+
"commons_ledger_user_created",
|
|
31372
|
+
"commons_saved_filters_user_updated",
|
|
31373
|
+
"commons_index_documents_status_updated",
|
|
31374
|
+
"commons_index_documents_entity_type",
|
|
31375
|
+
"commons_publications_updated",
|
|
31376
|
+
"commons_publication_editions_publication_published"
|
|
31377
|
+
];
|
|
31026
31378
|
async function ensureCommonsSchema() {
|
|
31027
|
-
if (
|
|
31379
|
+
if (schemaReady) return;
|
|
31380
|
+
if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
|
|
31381
|
+
schemaReady = true;
|
|
31382
|
+
return;
|
|
31383
|
+
}
|
|
31028
31384
|
const db = getDb();
|
|
31029
31385
|
await db.batch([
|
|
31030
31386
|
{
|
|
@@ -31071,6 +31427,42 @@ async function ensureCommonsSchema() {
|
|
|
31071
31427
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_status_updated ON commons_entities(status, updated_at DESC)`, args: [] },
|
|
31072
31428
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_type_updated ON commons_entities(entity_type, updated_at DESC)`, args: [] },
|
|
31073
31429
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_entities_source_domain ON commons_entities(source_json)`, args: [] },
|
|
31430
|
+
{
|
|
31431
|
+
sql: `
|
|
31432
|
+
CREATE TABLE IF NOT EXISTS commons_claims (
|
|
31433
|
+
id TEXT PRIMARY KEY,
|
|
31434
|
+
subject_entity_id TEXT NOT NULL REFERENCES commons_entities(id),
|
|
31435
|
+
predicate TEXT NOT NULL,
|
|
31436
|
+
predicate_label TEXT,
|
|
31437
|
+
object_entity_id TEXT,
|
|
31438
|
+
object_url TEXT,
|
|
31439
|
+
title TEXT,
|
|
31440
|
+
media_type TEXT,
|
|
31441
|
+
hreflang TEXT,
|
|
31442
|
+
qualifiers_json TEXT NOT NULL DEFAULT '{}',
|
|
31443
|
+
references_json TEXT NOT NULL DEFAULT '[]',
|
|
31444
|
+
rank TEXT NOT NULL DEFAULT 'normal',
|
|
31445
|
+
confidence REAL NOT NULL DEFAULT 1,
|
|
31446
|
+
confidence_reason TEXT,
|
|
31447
|
+
dispute_state TEXT NOT NULL DEFAULT 'undisputed',
|
|
31448
|
+
valid_from TEXT,
|
|
31449
|
+
valid_to TEXT,
|
|
31450
|
+
as_of TEXT,
|
|
31451
|
+
source_family TEXT NOT NULL DEFAULT 'unknown',
|
|
31452
|
+
review_state TEXT NOT NULL DEFAULT 'approved',
|
|
31453
|
+
proposal_id TEXT NOT NULL REFERENCES commons_proposals(id),
|
|
31454
|
+
created_by_user_id INTEGER,
|
|
31455
|
+
updated_by_user_id INTEGER,
|
|
31456
|
+
created_at TEXT NOT NULL,
|
|
31457
|
+
updated_at TEXT NOT NULL,
|
|
31458
|
+
CHECK ((object_entity_id IS NOT NULL) != (object_url IS NOT NULL)),
|
|
31459
|
+
UNIQUE(subject_entity_id, predicate, object_entity_id, object_url)
|
|
31460
|
+
)
|
|
31461
|
+
`,
|
|
31462
|
+
args: []
|
|
31463
|
+
},
|
|
31464
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_claims_subject_predicate ON commons_claims(subject_entity_id, predicate)`, args: [] },
|
|
31465
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_claims_object_entity ON commons_claims(object_entity_id)`, args: [] },
|
|
31074
31466
|
{
|
|
31075
31467
|
sql: `
|
|
31076
31468
|
CREATE TABLE IF NOT EXISTS commons_proposals (
|
|
@@ -31209,7 +31601,7 @@ async function ensureCommonsSchema() {
|
|
|
31209
31601
|
args: [COMMONS_SCHEMA_VERSION]
|
|
31210
31602
|
}
|
|
31211
31603
|
], "write");
|
|
31212
|
-
|
|
31604
|
+
schemaReady = true;
|
|
31213
31605
|
}
|
|
31214
31606
|
async function searchCommonsEntities(filters = {}, userId) {
|
|
31215
31607
|
await ensureCommonsSchema();
|
|
@@ -31231,8 +31623,15 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
31231
31623
|
args.push(jsonLikeValue(keyword));
|
|
31232
31624
|
}
|
|
31233
31625
|
if (mergedFilters.relatedEntityId) {
|
|
31234
|
-
clauses.push(
|
|
31235
|
-
|
|
31626
|
+
clauses.push(`(
|
|
31627
|
+
related_entities_json LIKE ?
|
|
31628
|
+
OR EXISTS (
|
|
31629
|
+
SELECT 1 FROM commons_claims
|
|
31630
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
31631
|
+
AND commons_claims.object_entity_id = ?
|
|
31632
|
+
)
|
|
31633
|
+
)`);
|
|
31634
|
+
args.push(jsonLikeValue(mergedFilters.relatedEntityId), mergedFilters.relatedEntityId);
|
|
31236
31635
|
}
|
|
31237
31636
|
if (mergedFilters.trailId) {
|
|
31238
31637
|
clauses.push("related_entities_json LIKE ?");
|
|
@@ -31254,8 +31653,16 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
31254
31653
|
}
|
|
31255
31654
|
const terms = tokenize(mergedFilters.query);
|
|
31256
31655
|
for (const term of terms) {
|
|
31257
|
-
clauses.push(
|
|
31258
|
-
|
|
31656
|
+
clauses.push(`(
|
|
31657
|
+
search_text LIKE ?
|
|
31658
|
+
OR EXISTS (
|
|
31659
|
+
SELECT 1 FROM commons_claims
|
|
31660
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
31661
|
+
AND (commons_claims.predicate LIKE ? OR commons_claims.predicate_label LIKE ? OR commons_claims.title LIKE ? OR commons_claims.object_url LIKE ?)
|
|
31662
|
+
)
|
|
31663
|
+
)`);
|
|
31664
|
+
const pattern = `%${escapeLike(term)}%`;
|
|
31665
|
+
args.push(pattern, pattern, pattern, pattern, pattern);
|
|
31259
31666
|
}
|
|
31260
31667
|
args.push(limit, offset);
|
|
31261
31668
|
const result = await getDb().execute({
|
|
@@ -31277,9 +31684,9 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
31277
31684
|
}
|
|
31278
31685
|
async function prepareCommonsEntity(input, user) {
|
|
31279
31686
|
await ensureCommonsSchema();
|
|
31280
|
-
const title =
|
|
31687
|
+
const title = optionalText2(input.title, 180) ?? "";
|
|
31281
31688
|
const slug2 = normalizeSlug(input.slug || title);
|
|
31282
|
-
const schemaOrgType =
|
|
31689
|
+
const schemaOrgType = optionalText2(input.schemaOrgType, 80);
|
|
31283
31690
|
const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
|
|
31284
31691
|
const entityType = contract.entityType;
|
|
31285
31692
|
const bodyMarkdown = safeCandidateBody(input.bodyMarkdown, input.contentSections);
|
|
@@ -31337,8 +31744,8 @@ async function validateCommonsEntity(input, user) {
|
|
|
31337
31744
|
await ensureCommonsSchema();
|
|
31338
31745
|
const errors = [];
|
|
31339
31746
|
const warnings = [];
|
|
31340
|
-
const title =
|
|
31341
|
-
const description =
|
|
31747
|
+
const title = optionalText2(input.title, 180);
|
|
31748
|
+
const description = optionalText2(input.description, 600);
|
|
31342
31749
|
const slug2 = normalizeSlug(input.slug || title || "");
|
|
31343
31750
|
const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
|
|
31344
31751
|
const source = normalizeSource(input.source);
|
|
@@ -31353,6 +31760,11 @@ async function validateCommonsEntity(input, user) {
|
|
|
31353
31760
|
maxCandidates: 8
|
|
31354
31761
|
});
|
|
31355
31762
|
const existing = slug2 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug2, input.entityId) : null;
|
|
31763
|
+
try {
|
|
31764
|
+
normalizeCommonsClaims(input.claims);
|
|
31765
|
+
} catch (error) {
|
|
31766
|
+
errors.push(error instanceof Error ? error.message : "Claims are invalid.");
|
|
31767
|
+
}
|
|
31356
31768
|
if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
|
|
31357
31769
|
if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
|
|
31358
31770
|
if (!slug2) errors.push("A Commons entity needs a URL slug.");
|
|
@@ -31442,6 +31854,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
31442
31854
|
};
|
|
31443
31855
|
}
|
|
31444
31856
|
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
31857
|
+
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
31445
31858
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
31446
31859
|
const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
|
|
31447
31860
|
const proposalId = `commons-proposal-${randomUUID15()}`;
|
|
@@ -31486,6 +31899,10 @@ async function submitCommonsEntity(input, user) {
|
|
|
31486
31899
|
];
|
|
31487
31900
|
if (nextEntity) {
|
|
31488
31901
|
batch.push(entityUpsertStatement(nextEntity, existing));
|
|
31902
|
+
if (normalized.claims !== void 0) {
|
|
31903
|
+
batch.push({ sql: "DELETE FROM commons_claims WHERE subject_entity_id = ?", args: [nextEntity.id] });
|
|
31904
|
+
batch.push(...claimInsertStatements(normalized.claims, existingClaims, nextEntity.id, proposalId, Number(user.id), now));
|
|
31905
|
+
}
|
|
31489
31906
|
batch.push(...commonsIndexStatements(nextEntity, now));
|
|
31490
31907
|
batch.push({
|
|
31491
31908
|
sql: `
|
|
@@ -31502,10 +31919,19 @@ async function submitCommonsEntity(input, user) {
|
|
|
31502
31919
|
actorLabel(input, user),
|
|
31503
31920
|
normalized.actorChannel,
|
|
31504
31921
|
existing ? "entity_updated" : "entity_created",
|
|
31505
|
-
JSON.stringify(
|
|
31922
|
+
JSON.stringify([
|
|
31923
|
+
...changedFieldPaths(existing, nextEntity),
|
|
31924
|
+
...normalized.claims !== void 0 ? ["claims"] : []
|
|
31925
|
+
]),
|
|
31506
31926
|
normalized.changeSummary || (existing ? "Updated Commons entity." : "Created Commons entity."),
|
|
31507
|
-
existing ? JSON.stringify(
|
|
31508
|
-
|
|
31927
|
+
existing ? JSON.stringify({
|
|
31928
|
+
...entityLedgerSnapshot(existing),
|
|
31929
|
+
...normalized.claims !== void 0 ? { claims: existingClaims } : {}
|
|
31930
|
+
}) : null,
|
|
31931
|
+
JSON.stringify({
|
|
31932
|
+
...entityLedgerSnapshot(nextEntity),
|
|
31933
|
+
...normalized.claims !== void 0 ? { claims: normalized.claims } : {}
|
|
31934
|
+
}),
|
|
31509
31935
|
now
|
|
31510
31936
|
]
|
|
31511
31937
|
});
|
|
@@ -31556,11 +31982,35 @@ async function getCommonsEntityLedger(idOrSlug) {
|
|
|
31556
31982
|
ledger: result.rows.map((row) => rowToLedgerEntry(row))
|
|
31557
31983
|
};
|
|
31558
31984
|
}
|
|
31985
|
+
async function listCommonsClaims(idOrSlug) {
|
|
31986
|
+
await ensureCommonsSchema();
|
|
31987
|
+
const entity = await getCommonsEntity(idOrSlug);
|
|
31988
|
+
if (!entity) return null;
|
|
31989
|
+
return { entity, claims: await getCommonsClaimsByEntityId(entity.id) };
|
|
31990
|
+
}
|
|
31991
|
+
async function getCommonsEntityLinkset(idOrSlug) {
|
|
31992
|
+
const result = await listCommonsClaims(idOrSlug);
|
|
31993
|
+
if (!result) return null;
|
|
31994
|
+
const targetIds = [...new Set(result.claims.map((claim) => claim.objectEntityId).filter((id) => Boolean(id)))];
|
|
31995
|
+
const targetSlugs = /* @__PURE__ */ new Map();
|
|
31996
|
+
if (targetIds.length) {
|
|
31997
|
+
const targets = await getDb().execute({
|
|
31998
|
+
sql: `SELECT id, slug FROM commons_entities WHERE status = 'published' AND id IN (${targetIds.map(() => "?").join(", ")})`,
|
|
31999
|
+
args: targetIds
|
|
32000
|
+
});
|
|
32001
|
+
for (const row of targets.rows) targetSlugs.set(String(row.id), String(row.slug));
|
|
32002
|
+
}
|
|
32003
|
+
const resolved = result.claims.map((claim) => {
|
|
32004
|
+
const href = claim.objectUrl ?? (claim.objectEntityId && targetSlugs.has(claim.objectEntityId) ? publicUrlForSlug(targetSlugs.get(claim.objectEntityId)) : null);
|
|
32005
|
+
return href ? { ...claim, href } : null;
|
|
32006
|
+
}).filter((claim) => claim !== null);
|
|
32007
|
+
return { entity: result.entity, claims: result.claims, ...buildCommonsLinkset(result.entity, resolved) };
|
|
32008
|
+
}
|
|
31559
32009
|
async function saveCommonsFilter(input, user) {
|
|
31560
32010
|
await ensureCommonsSchema();
|
|
31561
32011
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
31562
32012
|
const name = requiredText(input.name, "name", 2, 80);
|
|
31563
|
-
const description =
|
|
32013
|
+
const description = optionalText2(input.description, 500);
|
|
31564
32014
|
const filter = sanitizeSearchFilters(input.filter);
|
|
31565
32015
|
const existing = await getDb().execute({
|
|
31566
32016
|
sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
|
|
@@ -31693,9 +32143,9 @@ function commonsDatabaseReport() {
|
|
|
31693
32143
|
conflictBehavior: "Existing entity edits require baseRevision. Otherwise the proposal is pending_review.",
|
|
31694
32144
|
billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
|
|
31695
32145
|
},
|
|
31696
|
-
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
32146
|
+
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "relationship claims", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
31697
32147
|
indexing: commonsIndexingPolicy(),
|
|
31698
|
-
tables: ["commons_entities", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
|
|
32148
|
+
tables: ["commons_entities", "commons_claims", "commons_proposals", "commons_ledger", "commons_saved_filters", "commons_index_documents"]
|
|
31699
32149
|
};
|
|
31700
32150
|
}
|
|
31701
32151
|
function entityToWikiPage(entity) {
|
|
@@ -31715,6 +32165,13 @@ function entityToWikiPage(entity) {
|
|
|
31715
32165
|
media: entity.media,
|
|
31716
32166
|
sections: entity.articleSections,
|
|
31717
32167
|
metadata: {
|
|
32168
|
+
linksetUrl: commonsLinksetUrl(entity.id),
|
|
32169
|
+
linksetDiscovery: {
|
|
32170
|
+
rel: "linkset",
|
|
32171
|
+
type: COMMONS_LINKSET_MEDIA_TYPE,
|
|
32172
|
+
profile: COMMONS_RELATIONSHIP_PROFILE,
|
|
32173
|
+
href: commonsLinksetUrl(entity.id)
|
|
32174
|
+
},
|
|
31718
32175
|
entityType: friendlyEntityType(entity.entityType),
|
|
31719
32176
|
disambiguationName: entity.disambiguationName,
|
|
31720
32177
|
keywords: entity.keywords,
|
|
@@ -31871,7 +32328,7 @@ async function commonsTagVocabulary() {
|
|
|
31871
32328
|
...parseJson3(record.tags_json, []),
|
|
31872
32329
|
...parseJson3(record.keywords_json, [])
|
|
31873
32330
|
]) {
|
|
31874
|
-
const cleaned =
|
|
32331
|
+
const cleaned = optionalText2(tag, 120);
|
|
31875
32332
|
if (!cleaned) continue;
|
|
31876
32333
|
counts.set(cleaned, (counts.get(cleaned) ?? 0) + 1);
|
|
31877
32334
|
}
|
|
@@ -31881,7 +32338,7 @@ async function commonsTagVocabulary() {
|
|
|
31881
32338
|
function resolveCommonsTags(candidates, vocabulary) {
|
|
31882
32339
|
const vocabularyByKey = new Map(vocabulary.map((entry) => [entry.tag.toLowerCase(), entry]));
|
|
31883
32340
|
return candidates.map((candidate) => {
|
|
31884
|
-
const cleaned =
|
|
32341
|
+
const cleaned = optionalText2(candidate, 120) ?? "";
|
|
31885
32342
|
const key = cleaned.toLowerCase();
|
|
31886
32343
|
const exact = vocabularyByKey.get(key);
|
|
31887
32344
|
if (!cleaned || cleaned.length < 2) {
|
|
@@ -31978,12 +32435,13 @@ function normalizeSubmitInput(input) {
|
|
|
31978
32435
|
slug: slug2,
|
|
31979
32436
|
entityId: normalizeEntityId(input.entityId),
|
|
31980
32437
|
entityType: normalizeEntityType(input.entityType),
|
|
31981
|
-
disambiguationName:
|
|
32438
|
+
disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
|
|
31982
32439
|
featuredImage: normalizeFeaturedImage(input.featuredImage),
|
|
31983
32440
|
source: normalizeSource(input.source),
|
|
31984
32441
|
tags: uniqueStrings(input.tags ?? []).slice(0, 24),
|
|
31985
32442
|
keywords: uniqueStrings(input.keywords ?? []).slice(0, 50),
|
|
31986
32443
|
relatedEntities: normalizeRelatedEntities(input.relatedEntities),
|
|
32444
|
+
claims: normalizeClaimsForRepository(input.claims),
|
|
31987
32445
|
bodyMarkdown: normalizeBody(input.bodyMarkdown, input.contentSections),
|
|
31988
32446
|
contentSections: normalizeContentSections(input.contentSections),
|
|
31989
32447
|
articleSections: normalizeArticleSections(input.articleSections),
|
|
@@ -31992,11 +32450,18 @@ function normalizeSubmitInput(input) {
|
|
|
31992
32450
|
seo: normalizeSeo(input.seo),
|
|
31993
32451
|
actorChannel: input.actorChannel ?? "mcp",
|
|
31994
32452
|
actorLabel: input.actorLabel ?? "",
|
|
31995
|
-
changeSummary:
|
|
32453
|
+
changeSummary: optionalText2(input.changeSummary, 500) ?? "",
|
|
31996
32454
|
reviewPolicy: input.reviewPolicy ?? "auto_publish_if_safe",
|
|
31997
32455
|
baseRevision: input.baseRevision
|
|
31998
32456
|
};
|
|
31999
32457
|
}
|
|
32458
|
+
function normalizeClaimsForRepository(input) {
|
|
32459
|
+
try {
|
|
32460
|
+
return normalizeCommonsClaims(input);
|
|
32461
|
+
} catch (error) {
|
|
32462
|
+
throw new CommonsRepositoryError("invalid_claims", error instanceof Error ? error.message : "Claims are invalid.");
|
|
32463
|
+
}
|
|
32464
|
+
}
|
|
32000
32465
|
function evaluatePublishSafety(input, existing) {
|
|
32001
32466
|
if (!input.featuredImage?.url) return { safe: false, reason: "A featured image is required before a Commons entity can publish." };
|
|
32002
32467
|
if (!input.bodyMarkdown.trim() && !input.source.originalUrl && !input.source.resolvedUrl) {
|
|
@@ -32142,6 +32607,84 @@ function entityStorageValues(entity) {
|
|
|
32142
32607
|
entity.revision
|
|
32143
32608
|
];
|
|
32144
32609
|
}
|
|
32610
|
+
function claimInsertStatements(claims, existingClaims, subjectEntityId, proposalId, userId, now) {
|
|
32611
|
+
const existingByIdentity = new Map(existingClaims.map((claim) => [commonsClaimIdentityKey(claim), claim]));
|
|
32612
|
+
return claims.map((claim) => {
|
|
32613
|
+
const existing = existingByIdentity.get(commonsClaimIdentityKey(claim));
|
|
32614
|
+
return {
|
|
32615
|
+
sql: `
|
|
32616
|
+
INSERT INTO commons_claims (
|
|
32617
|
+
id, subject_entity_id, predicate, predicate_label, object_entity_id, object_url,
|
|
32618
|
+
title, media_type, hreflang, qualifiers_json, references_json, rank, confidence,
|
|
32619
|
+
confidence_reason, dispute_state, valid_from, valid_to, as_of, source_family,
|
|
32620
|
+
review_state, proposal_id, created_by_user_id, updated_by_user_id, created_at, updated_at
|
|
32621
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'approved', ?, ?, ?, ?, ?)
|
|
32622
|
+
`,
|
|
32623
|
+
args: [
|
|
32624
|
+
existing?.id ?? allocateClaimId(),
|
|
32625
|
+
subjectEntityId,
|
|
32626
|
+
claim.predicate,
|
|
32627
|
+
claim.predicateLabel ?? null,
|
|
32628
|
+
claim.objectEntityId ?? null,
|
|
32629
|
+
claim.objectUrl ?? null,
|
|
32630
|
+
claim.title ?? null,
|
|
32631
|
+
claim.mediaType ?? null,
|
|
32632
|
+
claim.hreflang ? JSON.stringify(claim.hreflang) : null,
|
|
32633
|
+
JSON.stringify(claim.qualifiers ?? {}),
|
|
32634
|
+
JSON.stringify(claim.references ?? []),
|
|
32635
|
+
claim.rank ?? "normal",
|
|
32636
|
+
claim.confidence ?? 1,
|
|
32637
|
+
claim.confidenceReason ?? null,
|
|
32638
|
+
claim.disputeState ?? "undisputed",
|
|
32639
|
+
claim.validFrom ?? null,
|
|
32640
|
+
claim.validTo ?? null,
|
|
32641
|
+
claim.asOf ?? null,
|
|
32642
|
+
claim.sourceFamily ?? "unknown",
|
|
32643
|
+
proposalId,
|
|
32644
|
+
existing?.createdByUserId ?? userId,
|
|
32645
|
+
userId,
|
|
32646
|
+
existing?.createdAt ?? now,
|
|
32647
|
+
now
|
|
32648
|
+
]
|
|
32649
|
+
};
|
|
32650
|
+
});
|
|
32651
|
+
}
|
|
32652
|
+
async function getCommonsClaimsByEntityId(entityId) {
|
|
32653
|
+
const result = await getDb().execute({
|
|
32654
|
+
sql: "SELECT * FROM commons_claims WHERE subject_entity_id = ? ORDER BY predicate ASC, object_url ASC, object_entity_id ASC, id ASC",
|
|
32655
|
+
args: [entityId]
|
|
32656
|
+
});
|
|
32657
|
+
return result.rows.map((row) => rowToClaim(row));
|
|
32658
|
+
}
|
|
32659
|
+
function rowToClaim(row) {
|
|
32660
|
+
return {
|
|
32661
|
+
id: String(row.id),
|
|
32662
|
+
subjectEntityId: String(row.subject_entity_id),
|
|
32663
|
+
predicate: String(row.predicate),
|
|
32664
|
+
predicateLabel: row.predicate_label == null ? void 0 : String(row.predicate_label),
|
|
32665
|
+
objectEntityId: row.object_entity_id == null ? void 0 : String(row.object_entity_id),
|
|
32666
|
+
objectUrl: row.object_url == null ? void 0 : String(row.object_url),
|
|
32667
|
+
title: row.title == null ? void 0 : String(row.title),
|
|
32668
|
+
mediaType: row.media_type == null ? void 0 : String(row.media_type),
|
|
32669
|
+
hreflang: row.hreflang == null ? void 0 : parseJson3(row.hreflang, []),
|
|
32670
|
+
qualifiers: parseJson3(row.qualifiers_json, {}),
|
|
32671
|
+
references: parseJson3(row.references_json, []),
|
|
32672
|
+
rank: String(row.rank),
|
|
32673
|
+
confidence: Number(row.confidence),
|
|
32674
|
+
confidenceReason: row.confidence_reason == null ? void 0 : String(row.confidence_reason),
|
|
32675
|
+
disputeState: String(row.dispute_state),
|
|
32676
|
+
validFrom: row.valid_from == null ? void 0 : String(row.valid_from),
|
|
32677
|
+
validTo: row.valid_to == null ? void 0 : String(row.valid_to),
|
|
32678
|
+
asOf: row.as_of == null ? void 0 : String(row.as_of),
|
|
32679
|
+
sourceFamily: String(row.source_family),
|
|
32680
|
+
reviewState: String(row.review_state),
|
|
32681
|
+
proposalId: String(row.proposal_id),
|
|
32682
|
+
createdByUserId: row.created_by_user_id == null ? null : Number(row.created_by_user_id),
|
|
32683
|
+
updatedByUserId: row.updated_by_user_id == null ? null : Number(row.updated_by_user_id),
|
|
32684
|
+
createdAt: String(row.created_at),
|
|
32685
|
+
updatedAt: String(row.updated_at)
|
|
32686
|
+
};
|
|
32687
|
+
}
|
|
32145
32688
|
function commonsIndexStatements(entity, now) {
|
|
32146
32689
|
return commonsIndexDocuments(entity, now).map((document2) => ({
|
|
32147
32690
|
sql: `
|
|
@@ -32246,7 +32789,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
32246
32789
|
return documents;
|
|
32247
32790
|
}
|
|
32248
32791
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
32249
|
-
return `commons-index-${
|
|
32792
|
+
return `commons-index-${createHash13("sha256").update(`${entityId}
|
|
32250
32793
|
${documentType}
|
|
32251
32794
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
32252
32795
|
}
|
|
@@ -32421,14 +32964,14 @@ function normalizeArticleSections(sections) {
|
|
|
32421
32964
|
return {
|
|
32422
32965
|
relatedLinks: normalizeRelatedLinks(sections?.relatedLinks),
|
|
32423
32966
|
notes: Array.isArray(sections?.notes) ? sections.notes.map((note) => ({
|
|
32424
|
-
marker:
|
|
32967
|
+
marker: optionalText2(note.marker, 20) ?? void 0,
|
|
32425
32968
|
body: requiredText(note.body, "notes.body", 1, 2e3)
|
|
32426
32969
|
})).slice(0, 100) : [],
|
|
32427
32970
|
citations: normalizeCitations(sections?.citations),
|
|
32428
32971
|
externalLinks: Array.isArray(sections?.externalLinks) ? sections.externalLinks.map((link) => ({
|
|
32429
32972
|
title: requiredText(link.title, "externalLinks.title", 1, 180),
|
|
32430
32973
|
url: requiredText(link.url, "externalLinks.url", 1, 2e3),
|
|
32431
|
-
summary:
|
|
32974
|
+
summary: optionalText2(link.summary, 300) ?? void 0
|
|
32432
32975
|
})).slice(0, 100) : [],
|
|
32433
32976
|
categories: uniqueStrings(sections?.categories ?? []).slice(0, 50)
|
|
32434
32977
|
};
|
|
@@ -32437,7 +32980,7 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
32437
32980
|
if (!Array.isArray(relatedLinks)) return [];
|
|
32438
32981
|
return relatedLinks.map((link) => {
|
|
32439
32982
|
const title = requiredText(link.title, "relatedLinks.title", 1, 180);
|
|
32440
|
-
const url =
|
|
32983
|
+
const url = optionalText2(link.url, 2e3);
|
|
32441
32984
|
const slug2 = link.slug ? normalizeSlug(link.slug) : void 0;
|
|
32442
32985
|
const entityId = normalizeEntityId(link.entityId);
|
|
32443
32986
|
const needsLink = link.needsLink === true || !url && !slug2 && !entityId;
|
|
@@ -32446,9 +32989,9 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
32446
32989
|
url,
|
|
32447
32990
|
slug: slug2,
|
|
32448
32991
|
entityId,
|
|
32449
|
-
relationship:
|
|
32450
|
-
summary:
|
|
32451
|
-
imageUrl:
|
|
32992
|
+
relationship: optionalText2(link.relationship, 120) ?? void 0,
|
|
32993
|
+
summary: optionalText2(link.summary, 300) ?? void 0,
|
|
32994
|
+
imageUrl: optionalText2(link.imageUrl, 2e3) ?? void 0,
|
|
32452
32995
|
needsLink
|
|
32453
32996
|
};
|
|
32454
32997
|
}).slice(0, 100);
|
|
@@ -32457,20 +33000,20 @@ function normalizeCitations(citations) {
|
|
|
32457
33000
|
if (!Array.isArray(citations)) return [];
|
|
32458
33001
|
return citations.map((citation) => ({
|
|
32459
33002
|
title: requiredText(citation.title, "citation.title", 1, 240),
|
|
32460
|
-
url:
|
|
32461
|
-
source:
|
|
32462
|
-
note:
|
|
32463
|
-
accessedAt:
|
|
33003
|
+
url: optionalText2(citation.url, 2e3) ?? void 0,
|
|
33004
|
+
source: optionalText2(citation.source, 240) ?? void 0,
|
|
33005
|
+
note: optionalText2(citation.note, 1e3) ?? void 0,
|
|
33006
|
+
accessedAt: optionalText2(citation.accessedAt, 80) ?? void 0
|
|
32464
33007
|
})).slice(0, 300);
|
|
32465
33008
|
}
|
|
32466
33009
|
function normalizeFeaturedImage(image) {
|
|
32467
33010
|
if (!image?.url) return void 0;
|
|
32468
33011
|
return {
|
|
32469
33012
|
url: requiredText(image.url, "featuredImage.url", 1, 2e3),
|
|
32470
|
-
alt:
|
|
32471
|
-
caption:
|
|
32472
|
-
sourceUrl:
|
|
32473
|
-
license:
|
|
33013
|
+
alt: optionalText2(image.alt, 500) ?? void 0,
|
|
33014
|
+
caption: optionalText2(image.caption, 1e3) ?? void 0,
|
|
33015
|
+
sourceUrl: optionalText2(image.sourceUrl, 2e3) ?? void 0,
|
|
33016
|
+
license: optionalText2(image.license, 240) ?? void 0,
|
|
32474
33017
|
width: positiveInt(image.width),
|
|
32475
33018
|
height: positiveInt(image.height)
|
|
32476
33019
|
};
|
|
@@ -32479,11 +33022,11 @@ function normalizeMedia(media, featuredImage) {
|
|
|
32479
33022
|
const normalized = Array.isArray(media) ? media.map((item) => ({
|
|
32480
33023
|
type: item.type,
|
|
32481
33024
|
url: requiredText(item.url, "media.url", 1, 2e3),
|
|
32482
|
-
alt:
|
|
32483
|
-
caption:
|
|
32484
|
-
posterUrl:
|
|
32485
|
-
sourceUrl:
|
|
32486
|
-
license:
|
|
33025
|
+
alt: optionalText2(item.alt, 500) ?? void 0,
|
|
33026
|
+
caption: optionalText2(item.caption, 1e3) ?? void 0,
|
|
33027
|
+
posterUrl: optionalText2(item.posterUrl, 2e3) ?? void 0,
|
|
33028
|
+
sourceUrl: optionalText2(item.sourceUrl, 2e3) ?? void 0,
|
|
33029
|
+
license: optionalText2(item.license, 240) ?? void 0,
|
|
32487
33030
|
width: positiveInt(item.width),
|
|
32488
33031
|
height: positiveInt(item.height),
|
|
32489
33032
|
durationSeconds: positiveInt(item.durationSeconds)
|
|
@@ -32513,28 +33056,28 @@ function normalizeMedia(media, featuredImage) {
|
|
|
32513
33056
|
}
|
|
32514
33057
|
function normalizeSource(source) {
|
|
32515
33058
|
return {
|
|
32516
|
-
originalUrl:
|
|
32517
|
-
resolvedUrl:
|
|
32518
|
-
sourceCanonicalUrl:
|
|
32519
|
-
relCanonicalHref:
|
|
32520
|
-
sourceByline:
|
|
32521
|
-
publisher:
|
|
33059
|
+
originalUrl: optionalText2(source?.originalUrl, 2e3) ?? void 0,
|
|
33060
|
+
resolvedUrl: optionalText2(source?.resolvedUrl, 2e3) ?? void 0,
|
|
33061
|
+
sourceCanonicalUrl: optionalText2(source?.sourceCanonicalUrl, 2e3) ?? void 0,
|
|
33062
|
+
relCanonicalHref: optionalText2(source?.relCanonicalHref, 2e3) ?? void 0,
|
|
33063
|
+
sourceByline: optionalText2(source?.sourceByline, 240) ?? void 0,
|
|
33064
|
+
publisher: optionalText2(source?.publisher, 240) ?? void 0,
|
|
32522
33065
|
authors: uniqueStrings(source?.authors ?? []).slice(0, 20),
|
|
32523
|
-
publishedAt:
|
|
32524
|
-
capturedAt:
|
|
32525
|
-
license:
|
|
32526
|
-
rightsSummary:
|
|
32527
|
-
contentHash:
|
|
33066
|
+
publishedAt: optionalText2(source?.publishedAt, 80) ?? void 0,
|
|
33067
|
+
capturedAt: optionalText2(source?.capturedAt, 80) ?? void 0,
|
|
33068
|
+
license: optionalText2(source?.license, 240) ?? void 0,
|
|
33069
|
+
rightsSummary: optionalText2(source?.rightsSummary, 1e3) ?? void 0,
|
|
33070
|
+
contentHash: optionalText2(source?.contentHash, 200) ?? void 0
|
|
32528
33071
|
};
|
|
32529
33072
|
}
|
|
32530
33073
|
function normalizeSeo(seo) {
|
|
32531
33074
|
return {
|
|
32532
|
-
canonicalUrl:
|
|
32533
|
-
relCanonical:
|
|
32534
|
-
metaTitle:
|
|
32535
|
-
metaDescription:
|
|
32536
|
-
ogImage:
|
|
32537
|
-
schemaOrgType:
|
|
33075
|
+
canonicalUrl: optionalText2(seo?.canonicalUrl, 2e3) ?? void 0,
|
|
33076
|
+
relCanonical: optionalText2(seo?.relCanonical, 2e3) ?? void 0,
|
|
33077
|
+
metaTitle: optionalText2(seo?.metaTitle, 240) ?? void 0,
|
|
33078
|
+
metaDescription: optionalText2(seo?.metaDescription, 500) ?? void 0,
|
|
33079
|
+
ogImage: optionalText2(seo?.ogImage, 2e3) ?? void 0,
|
|
33080
|
+
schemaOrgType: optionalText2(seo?.schemaOrgType, 80) ?? void 0,
|
|
32538
33081
|
noIndex: seo?.noIndex === true
|
|
32539
33082
|
};
|
|
32540
33083
|
}
|
|
@@ -32543,37 +33086,37 @@ function normalizeRelatedEntities(related) {
|
|
|
32543
33086
|
return related.map((item) => ({
|
|
32544
33087
|
entityId: normalizeEntityId(item.entityId),
|
|
32545
33088
|
title: requiredText(item.title, "relatedEntities.title", 1, 180),
|
|
32546
|
-
relationship:
|
|
33089
|
+
relationship: optionalText2(item.relationship, 120) ?? void 0,
|
|
32547
33090
|
slug: item.slug ? normalizeSlug(item.slug) : void 0,
|
|
32548
|
-
url:
|
|
32549
|
-
description:
|
|
33091
|
+
url: optionalText2(item.url, 2e3) ?? void 0,
|
|
33092
|
+
description: optionalText2(item.description, 300) ?? void 0
|
|
32550
33093
|
})).slice(0, 200);
|
|
32551
33094
|
}
|
|
32552
33095
|
function sanitizeSearchFilters(filters) {
|
|
32553
33096
|
return {
|
|
32554
|
-
query:
|
|
33097
|
+
query: optionalText2(filters.query, 300) ?? void 0,
|
|
32555
33098
|
entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
|
|
32556
|
-
tag:
|
|
33099
|
+
tag: optionalText2(filters.tag, 80) ?? void 0,
|
|
32557
33100
|
tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
|
|
32558
|
-
keyword:
|
|
33101
|
+
keyword: optionalText2(filters.keyword, 120) ?? void 0,
|
|
32559
33102
|
keywords: uniqueStrings(filters.keywords ?? []).slice(0, 20),
|
|
32560
33103
|
relatedEntityId: normalizeEntityId(filters.relatedEntityId),
|
|
32561
33104
|
trailId: normalizeEntityId(filters.trailId),
|
|
32562
|
-
sourceDomain:
|
|
33105
|
+
sourceDomain: optionalText2(filters.sourceDomain, 240) ?? void 0,
|
|
32563
33106
|
hasMedia: filters.hasMedia === true ? true : void 0,
|
|
32564
33107
|
hasVideo: filters.hasVideo === true ? true : void 0,
|
|
32565
|
-
publishedAfter:
|
|
32566
|
-
updatedAfter:
|
|
32567
|
-
filterId:
|
|
33108
|
+
publishedAfter: optionalText2(filters.publishedAfter, 80) ?? void 0,
|
|
33109
|
+
updatedAfter: optionalText2(filters.updatedAfter, 80) ?? void 0,
|
|
33110
|
+
filterId: optionalText2(filters.filterId, 200) ?? void 0,
|
|
32568
33111
|
limit: clampNumber(filters.limit, 1, 100, 20),
|
|
32569
33112
|
offset: clampNumber(filters.offset, 0, 1e4, 0)
|
|
32570
33113
|
};
|
|
32571
33114
|
}
|
|
32572
33115
|
function sanitizeNeedsLinkFilters(filters) {
|
|
32573
33116
|
return {
|
|
32574
|
-
query:
|
|
33117
|
+
query: optionalText2(filters.query, 300) ?? void 0,
|
|
32575
33118
|
entityType: filters.entityType ? normalizeEntityType(filters.entityType) : void 0,
|
|
32576
|
-
tag:
|
|
33119
|
+
tag: optionalText2(filters.tag, 80) ?? void 0,
|
|
32577
33120
|
tags: uniqueStrings(filters.tags ?? []).slice(0, 20),
|
|
32578
33121
|
sourceEntityId: normalizeEntityId(filters.sourceEntityId),
|
|
32579
33122
|
sourceSlug: filters.sourceSlug ? normalizeSlug(filters.sourceSlug) : void 0,
|
|
@@ -32594,7 +33137,7 @@ function commonsIndexingPolicy() {
|
|
|
32594
33137
|
storageScope: "platform_commons_graph",
|
|
32595
33138
|
countsAgainstUserStorage: false,
|
|
32596
33139
|
embedsPrivateUserVaults: false,
|
|
32597
|
-
corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "See Also concepts marked needsLink"],
|
|
33140
|
+
corpus: ["published Commons article body", "tags", "keywords", "JSON-LD metadata", "related entities", "governed relationship claims", "See Also concepts marked needsLink"],
|
|
32598
33141
|
excluded: ["comments", "comment threads", "reader alerts", "private reading-room state"],
|
|
32599
33142
|
retrievalPattern: {
|
|
32600
33143
|
lexicalFirst: ["title", "tags", "keywords", "source metadata", "article body", "See Also concepts"],
|
|
@@ -32670,7 +33213,7 @@ function searchableText(entity) {
|
|
|
32670
33213
|
].filter(Boolean).join("\n").toLowerCase();
|
|
32671
33214
|
}
|
|
32672
33215
|
function actorLabel(input, user) {
|
|
32673
|
-
return
|
|
33216
|
+
return optionalText2(input.actorLabel, 120) || optionalText2(user.name, 120) || `mcp-user-${user.id}`;
|
|
32674
33217
|
}
|
|
32675
33218
|
function publicBaseUrl() {
|
|
32676
33219
|
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
|
|
@@ -32681,13 +33224,16 @@ function publicUrlForSlug(slug2) {
|
|
|
32681
33224
|
function allocateEntityId() {
|
|
32682
33225
|
return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
|
|
32683
33226
|
}
|
|
33227
|
+
function allocateClaimId() {
|
|
33228
|
+
return `TPW-S${Date.now()}${Math.floor(Math.random() * 1e6).toString().padStart(6, "0")}`;
|
|
33229
|
+
}
|
|
32684
33230
|
function normalizeEntityId(value) {
|
|
32685
|
-
const text2 =
|
|
33231
|
+
const text2 = optionalText2(value, 80);
|
|
32686
33232
|
if (!text2) return void 0;
|
|
32687
33233
|
return text2.replace(/\s+/g, "").toUpperCase();
|
|
32688
33234
|
}
|
|
32689
33235
|
function normalizeEntityType(value) {
|
|
32690
|
-
const raw =
|
|
33236
|
+
const raw = optionalText2(value, 120) || DEFAULT_ENTITY_TYPE;
|
|
32691
33237
|
const squashed = raw.replace(/[\s_-]+/g, "").toLowerCase();
|
|
32692
33238
|
if (squashed === "publicarticle" || squashed === "article" || squashed === "item") return DEFAULT_ENTITY_TYPE;
|
|
32693
33239
|
if (squashed === "trail" || squashed === "readingtrail") return "Trail";
|
|
@@ -32715,7 +33261,7 @@ function requiredText(value, field, min, max) {
|
|
|
32715
33261
|
if (text2.length < min) throw new CommonsRepositoryError(`${field.replaceAll(".", "_")}_required`, `${field} must be at least ${min} characters.`);
|
|
32716
33262
|
return text2.slice(0, max);
|
|
32717
33263
|
}
|
|
32718
|
-
function
|
|
33264
|
+
function optionalText2(value, max) {
|
|
32719
33265
|
const text2 = String(value ?? "").trim();
|
|
32720
33266
|
return text2 ? text2.slice(0, max) : void 0;
|
|
32721
33267
|
}
|
|
@@ -32735,7 +33281,7 @@ function uniqueStrings(values) {
|
|
|
32735
33281
|
const seen = /* @__PURE__ */ new Set();
|
|
32736
33282
|
const out = [];
|
|
32737
33283
|
for (const value of values) {
|
|
32738
|
-
const text2 =
|
|
33284
|
+
const text2 = optionalText2(value, 120);
|
|
32739
33285
|
if (!text2) continue;
|
|
32740
33286
|
const key = text2.toLowerCase();
|
|
32741
33287
|
if (seen.has(key)) continue;
|
|
@@ -32782,7 +33328,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
32782
33328
|
};
|
|
32783
33329
|
|
|
32784
33330
|
// src/api/commons-publication-repository.ts
|
|
32785
|
-
import { createHash as
|
|
33331
|
+
import { createHash as createHash14, randomUUID as randomUUID16 } from "crypto";
|
|
32786
33332
|
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
32787
33333
|
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
32788
33334
|
"admin",
|
|
@@ -32980,7 +33526,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
32980
33526
|
JSON.stringify(editionInput.articles),
|
|
32981
33527
|
html,
|
|
32982
33528
|
rendered.filename,
|
|
32983
|
-
|
|
33529
|
+
createHash14("sha256").update(html).digest("hex"),
|
|
32984
33530
|
rendered.articleCount,
|
|
32985
33531
|
rendered.wordCount,
|
|
32986
33532
|
Buffer.byteLength(html),
|
|
@@ -33259,6 +33805,7 @@ var SubmitEntitySchema = z32.object({
|
|
|
33259
33805
|
url: z32.string().url().optional(),
|
|
33260
33806
|
description: z32.string().max(300).optional()
|
|
33261
33807
|
}).strict()).max(200).optional(),
|
|
33808
|
+
claims: z32.array(CommonsClaimInputSchema).max(300).optional(),
|
|
33262
33809
|
bodyMarkdown: z32.string().max(5e5).optional(),
|
|
33263
33810
|
contentSections: z32.array(z32.object({
|
|
33264
33811
|
id: z32.string().max(180).optional(),
|
|
@@ -33336,6 +33883,17 @@ commonsApp.get("/health", async (c) => {
|
|
|
33336
33883
|
await ensureCommonsSchema();
|
|
33337
33884
|
return c.json({ ok: true, data: commonsDatabaseReport() });
|
|
33338
33885
|
});
|
|
33886
|
+
commonsApp.get("/profiles/relationships/v1", (c) => c.json({
|
|
33887
|
+
id: COMMONS_RELATIONSHIP_PROFILE,
|
|
33888
|
+
title: "Transparent Commons governed relationship linksets",
|
|
33889
|
+
mediaType: COMMONS_LINKSET_MEDIA_TYPE,
|
|
33890
|
+
claimAuthority: "MCP Scraper authenticated proposal and contribution ledger",
|
|
33891
|
+
anchorPolicy: "Every link context is an absolute canonical Transparent Commons wiki URL.",
|
|
33892
|
+
predicatePolicy: "Predicates are curated registered link relations or absolute HTTPS extension relation URIs.",
|
|
33893
|
+
publicationPolicy: "Only approved, active, non-deprecated claims whose dispute state is undisputed or attributed are projected.",
|
|
33894
|
+
trustPolicy: "A third-party target is an asserted relationship, not an endorsement or proof that the target agrees.",
|
|
33895
|
+
source: "https://www.rfc-editor.org/rfc/rfc9264.html"
|
|
33896
|
+
}));
|
|
33339
33897
|
commonsApp.post("/publications/prepare", auth2, async (c) => {
|
|
33340
33898
|
const parsed = PreparePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33341
33899
|
if (!parsed.success) return validationError(c, parsed.error);
|
|
@@ -33466,11 +34024,32 @@ commonsApp.post("/entities/propose", auth2, async (c) => {
|
|
|
33466
34024
|
commonsApp.get("/entities/:idOrSlug", async (c) => {
|
|
33467
34025
|
const entity = await getCommonsEntity(c.req.param("idOrSlug"));
|
|
33468
34026
|
if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
34027
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
33469
34028
|
return c.json({ ok: true, data: entity });
|
|
33470
34029
|
});
|
|
34030
|
+
commonsApp.get("/entities/:idOrSlug/claims", async (c) => {
|
|
34031
|
+
const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
|
|
34032
|
+
if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
34033
|
+
c.header("Link", commonsLinksetDiscoveryHeader(result.entity.id));
|
|
34034
|
+
return c.json({ ok: true, data: { entity: result.entity, claims: result.claims, linksetUrl: result.linksetUrl } });
|
|
34035
|
+
});
|
|
34036
|
+
commonsApp.get("/entities/:idOrSlug/linkset", async (c) => {
|
|
34037
|
+
const result = await getCommonsEntityLinkset(c.req.param("idOrSlug"));
|
|
34038
|
+
if (!result) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
34039
|
+
c.header("Content-Type", `${COMMONS_LINKSET_MEDIA_TYPE}; profile="${COMMONS_RELATIONSHIP_PROFILE}"`);
|
|
34040
|
+
c.header("ETag", result.etag);
|
|
34041
|
+
c.header("Link", `<${COMMONS_RELATIONSHIP_PROFILE}>; rel="profile"`);
|
|
34042
|
+
c.header("Cache-Control", "public, max-age=60, s-maxage=300");
|
|
34043
|
+
if (c.req.header("If-None-Match") === result.etag) return c.body(null, 304);
|
|
34044
|
+
if (c.req.query("envelope") === "true") {
|
|
34045
|
+
return c.json({ ok: true, data: { ...result, document: result.document } });
|
|
34046
|
+
}
|
|
34047
|
+
return c.body(JSON.stringify(result.document));
|
|
34048
|
+
});
|
|
33471
34049
|
commonsApp.get("/entities/:idOrSlug/wiki-page", async (c) => {
|
|
33472
34050
|
const entity = await getCommonsEntity(c.req.param("idOrSlug"));
|
|
33473
34051
|
if (!entity) return c.json({ ok: false, error: "entity_not_found", message: "No published Commons entity matched that id or slug." }, 404);
|
|
34052
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
33474
34053
|
return c.json({ ok: true, data: entityToWikiPage(entity) });
|
|
33475
34054
|
});
|
|
33476
34055
|
commonsApp.get("/entities/:idOrSlug/json-ld", async (c) => {
|
|
@@ -33486,6 +34065,7 @@ commonsApp.get("/entities/:idOrSlug/ledger", async (c) => {
|
|
|
33486
34065
|
commonsApp.get("/wiki/:slug", async (c) => {
|
|
33487
34066
|
const entity = await getCommonsEntity(c.req.param("slug"));
|
|
33488
34067
|
if (!entity) return c.json({ ok: false, error: "page_not_found", message: "No published Commons wiki page matched that slug." }, 404);
|
|
34068
|
+
c.header("Link", commonsLinksetDiscoveryHeader(entity.id));
|
|
33489
34069
|
return c.json({
|
|
33490
34070
|
ok: true,
|
|
33491
34071
|
data: {
|
|
@@ -33587,12 +34167,12 @@ function xmlEscape(value) {
|
|
|
33587
34167
|
import { Hono as Hono32 } from "hono";
|
|
33588
34168
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
33589
34169
|
import { z as z34 } from "zod";
|
|
33590
|
-
import { createHash as
|
|
34170
|
+
import { createHash as createHash16 } from "crypto";
|
|
33591
34171
|
import Papa2 from "papaparse";
|
|
33592
34172
|
|
|
33593
34173
|
// src/api/analytics-entitlement.ts
|
|
33594
34174
|
import {
|
|
33595
|
-
createHash as
|
|
34175
|
+
createHash as createHash15,
|
|
33596
34176
|
createHmac as createHmac3,
|
|
33597
34177
|
randomBytes as randomBytes3,
|
|
33598
34178
|
timingSafeEqual as timingSafeEqual2
|
|
@@ -33807,7 +34387,7 @@ function dashboardCallbackUrl() {
|
|
|
33807
34387
|
async function createThorbitConnectUrl(userId) {
|
|
33808
34388
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
33809
34389
|
const state = randomBytes3(32).toString("base64url");
|
|
33810
|
-
const stateHash =
|
|
34390
|
+
const stateHash = createHash15("sha256").update(state).digest("hex");
|
|
33811
34391
|
await getAnalyticsPool().query(
|
|
33812
34392
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
33813
34393
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -33836,7 +34416,7 @@ function verifyThorbitAssertion(token4) {
|
|
|
33836
34416
|
return parsed.data.entitlement;
|
|
33837
34417
|
}
|
|
33838
34418
|
async function consumeThorbitConnectCallback(input) {
|
|
33839
|
-
const stateHash =
|
|
34419
|
+
const stateHash = createHash15("sha256").update(input.state).digest("hex");
|
|
33840
34420
|
const client2 = await getAnalyticsPool().connect();
|
|
33841
34421
|
try {
|
|
33842
34422
|
await client2.query("BEGIN");
|
|
@@ -33932,7 +34512,9 @@ var EventSchema = z34.object({
|
|
|
33932
34512
|
campaign: z34.string().trim().max(240).optional(),
|
|
33933
34513
|
clickIds: z34.record(z34.string(), z34.string().trim().max(500)).optional(),
|
|
33934
34514
|
deviceClass: z34.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
|
|
33935
|
-
properties: z34.record(z34.string(), z34.unknown()).optional()
|
|
34515
|
+
properties: z34.record(z34.string(), z34.unknown()).optional(),
|
|
34516
|
+
engagedMs: z34.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
|
|
34517
|
+
scrollDepth: z34.number().int().min(0).max(100).optional()
|
|
33936
34518
|
}).strict();
|
|
33937
34519
|
var IngestionSchema = z34.object({
|
|
33938
34520
|
pixelId: z34.string().trim().min(8).max(160),
|
|
@@ -34091,7 +34673,10 @@ analyticsApp.use("*", async (c, next) => {
|
|
|
34091
34673
|
await next();
|
|
34092
34674
|
});
|
|
34093
34675
|
function analyticsPixelSdk() {
|
|
34094
|
-
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties}}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){current=location.href;api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
34676
|
+
return `(()=>{const s=document.currentScript;if(!s)return;const pixel=s.dataset.pixel;if(!pixel)return;const endpoint=new URL('/analytics/v1/events',s.src).toString(),ttl=18e5;const key='mcp_analytics_visitor',sessionKey='mcp_analytics_session',touchKey='mcp_analytics_touch';const id=()=>crypto.randomUUID?crypto.randomUUID():'e_'+Date.now().toString(36)+Math.random().toString(36).slice(2);let consent=s.dataset.consent==='granted'&&!navigator.globalPrivacyControl,visitor=null,session=null;const read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}};const ensure=()=>{if(!consent)return;visitor=read(key)||id();const now=Date.now();let saved={};try{saved=JSON.parse(read(sessionKey)||'{}')}catch{}session=saved.id&&now-Number(saved.seen||0)<ttl?saved.id:id();write(key,visitor);write(sessionKey,JSON.stringify({id:session,seen:now}))};const attribution=()=>{const q=new URLSearchParams(location.search),clickIds={},properties={},map={utm_term:'utm_term',utm_content:'utm_content',utm_id:'campaign_id',campaign_id:'campaign_id',adset_id:'ad_set_id',ad_set_id:'ad_set_id',adgroup_id:'ad_group_id',ad_group_id:'ad_group_id',ad_id:'ad_id',creative_id:'creative_id',placement:'placement'};for(const k of ['fbclid','gclid','gbraid','wbraid','ttclid','rdt_cid','msclkid']){const v=q.get(k);if(v)clickIds[k]=v}for(const [from,to] of Object.entries(map)){const v=q.get(from);if(v)properties[to]=v}const current={source:q.get('utm_source')||undefined,medium:q.get('utm_medium')||undefined,campaign:q.get('utm_campaign')||undefined,clickIds,properties};if(current.source||Object.keys(clickIds).length)write(touchKey,JSON.stringify(current));else{try{return JSON.parse(read(touchKey)||'null')||current}catch{}}return current};const send=(name,properties={},over={})=>{if(!consent||navigator.globalPrivacyControl)return;ensure();const a=attribution();const body=JSON.stringify({pixelId:pixel,events:[{eventId:id(),eventName:name,occurredAt:new Date().toISOString(),visitorId:visitor,sessionId:session,path:location.pathname,canonicalUrl:location.origin+location.pathname,title:document.title,referrer:document.referrer||undefined,...a,deviceClass:innerWidth<640?'mobile':innerWidth<1024?'tablet':'desktop',properties:{...a.properties,...properties},...over}]});if(navigator.sendBeacon&&navigator.sendBeacon(endpoint,new Blob([body],{type:'application/json'})))return;fetch(endpoint,{method:'POST',headers:{'content-type':'application/json'},body,keepalive:true,credentials:'omit'}).catch(()=>{})};let visibleSince=document.visibilityState==='visible'?Date.now():0,dwellMs=0,maxScroll=0,dwellPath=location.pathname;const depth=()=>{const d=document.documentElement,h=d.scrollHeight||1;return Math.max(0,Math.min(100,Math.round(((d.scrollTop||document.body.scrollTop||0)+innerHeight)/h*100)))};const accrue=()=>{if(visibleSince){dwellMs+=Date.now()-visibleSince;visibleSince=0}};const resume=()=>{if(!visibleSince)visibleSince=Date.now()};const flush=()=>{accrue();const ms=Math.min(dwellMs,18e5),sd=Math.max(maxScroll,depth());dwellMs=0;maxScroll=0;if(ms<250)return;send('page_engagement',{},{path:dwellPath,canonicalUrl:location.origin+dwellPath,engagedMs:ms,scrollDepth:sd})};addEventListener('scroll',()=>{const d=depth();if(d>maxScroll)maxScroll=d},{passive:true});document.addEventListener('visibilitychange',()=>{if(document.visibilityState==='hidden')flush();else resume()});addEventListener('pagehide',flush);const api={page:()=>send('page_view'),track:send,consent:state=>{consent=(state==='granted'||state===true)&&!navigator.globalPrivacyControl;if(consent){ensure();resume();api.page()}},debug:()=>({pixel,endpoint,consent,gpc:Boolean(navigator.globalPrivacyControl),visitorId:visitor,sessionId:session,hostname:location.hostname,path:location.pathname,engagedMs:dwellMs+(visibleSince?Date.now()-visibleSince:0),scrollDepth:Math.max(maxScroll,depth()),...attribution()})};window.mcpAnalytics=api;ensure();if(consent)api.page();let current=location.href;const changed=()=>{if(location.href!==current){flush();current=location.href;dwellPath=location.pathname;resume();api.page()}};for(const method of ['pushState','replaceState']){const original=history[method];history[method]=function(...args){const result=original.apply(this,args);queueMicrotask(changed);return result}}addEventListener('popstate',changed)})();`;
|
|
34677
|
+
}
|
|
34678
|
+
function pagePathFilter(value) {
|
|
34679
|
+
return value && value.startsWith("/") ? value : void 0;
|
|
34095
34680
|
}
|
|
34096
34681
|
function reportFilters(url) {
|
|
34097
34682
|
const end = url.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -34113,7 +34698,7 @@ function reportFilters(url) {
|
|
|
34113
34698
|
channelFamily: url.searchParams.get("channelFamily") || void 0,
|
|
34114
34699
|
platform: url.searchParams.get("platform") || void 0,
|
|
34115
34700
|
referrer: url.searchParams.get("referrer") || void 0,
|
|
34116
|
-
path: url.searchParams.get("path")
|
|
34701
|
+
path: pagePathFilter(url.searchParams.get("path")),
|
|
34117
34702
|
deviceClass: url.searchParams.get("deviceClass") || void 0,
|
|
34118
34703
|
countryCode: url.searchParams.get("countryCode") || void 0,
|
|
34119
34704
|
regionCode: url.searchParams.get("regionCode") || void 0,
|
|
@@ -34307,7 +34892,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
|
|
|
34307
34892
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
34308
34893
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
34309
34894
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
34310
|
-
const suffix2 =
|
|
34895
|
+
const suffix2 = createHash16("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
34311
34896
|
const path5 = `Leads/person-${suffix2}`;
|
|
34312
34897
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
34313
34898
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -34406,9 +34991,9 @@ Submitted the published form.
|
|
|
34406
34991
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
34407
34992
|
});
|
|
34408
34993
|
const match = {
|
|
34409
|
-
...email ? { emailSha256:
|
|
34994
|
+
...email ? { emailSha256: createHash16("sha256").update(email).digest("hex") } : {},
|
|
34410
34995
|
...phone ? {
|
|
34411
|
-
phoneSha256:
|
|
34996
|
+
phoneSha256: createHash16("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
34412
34997
|
} : {}
|
|
34413
34998
|
};
|
|
34414
34999
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -34836,7 +35421,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
|
|
|
34836
35421
|
rejectedCount += 1;
|
|
34837
35422
|
continue;
|
|
34838
35423
|
}
|
|
34839
|
-
const digest2 =
|
|
35424
|
+
const digest2 = createHash16("sha256").update(
|
|
34840
35425
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
34841
35426
|
).digest("hex").slice(0, 16);
|
|
34842
35427
|
const path5 = `Leads/person-${digest2}`;
|
|
@@ -35039,14 +35624,13 @@ for (const [path5, load] of [
|
|
|
35039
35624
|
["/sites/:siteId/overview", analyticsOverview],
|
|
35040
35625
|
["/sites/:siteId/timeseries", analyticsTimeseries],
|
|
35041
35626
|
["/sites/:siteId/acquisition", analyticsAcquisition],
|
|
35042
|
-
["/sites/:siteId/content", analyticsContent],
|
|
35043
35627
|
["/sites/:siteId/events", analyticsEventCounts]
|
|
35044
35628
|
]) {
|
|
35045
35629
|
analyticsApp.get(path5, auth3, async (c) => {
|
|
35046
35630
|
const url = new URL(c.req.url);
|
|
35047
35631
|
const filters = reportFilters(url);
|
|
35048
35632
|
validateReportWindow(filters);
|
|
35049
|
-
const paginated = path5.endsWith("/acquisition") || path5.endsWith("/
|
|
35633
|
+
const paginated = path5.endsWith("/acquisition") || path5.endsWith("/events");
|
|
35050
35634
|
const data = await load(
|
|
35051
35635
|
c.req.param("siteId"),
|
|
35052
35636
|
Number(c.get("user").id),
|
|
@@ -35056,6 +35640,24 @@ for (const [path5, load] of [
|
|
|
35056
35640
|
return c.json({ ok: true, data, filters });
|
|
35057
35641
|
});
|
|
35058
35642
|
}
|
|
35643
|
+
analyticsApp.get("/sites/:siteId/content", auth3, async (c) => {
|
|
35644
|
+
const url = new URL(c.req.url);
|
|
35645
|
+
const filters = reportFilters(url);
|
|
35646
|
+
validateReportWindow(filters);
|
|
35647
|
+
const requestedSort = url.searchParams.get("sort");
|
|
35648
|
+
const data = await analyticsContent(
|
|
35649
|
+
c.req.param("siteId"),
|
|
35650
|
+
Number(c.get("user").id),
|
|
35651
|
+
filters,
|
|
35652
|
+
reportPage(url),
|
|
35653
|
+
{
|
|
35654
|
+
sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
|
|
35655
|
+
direction: url.searchParams.get("direction") === "asc" ? "asc" : "desc",
|
|
35656
|
+
search: url.searchParams.get("search") || void 0
|
|
35657
|
+
}
|
|
35658
|
+
);
|
|
35659
|
+
return c.json({ ok: true, data, filters });
|
|
35660
|
+
});
|
|
35059
35661
|
analyticsApp.get("/sites/:siteId/dashboard", auth3, async (c) => {
|
|
35060
35662
|
const url = new URL(c.req.url);
|
|
35061
35663
|
const filters = reportFilters(url);
|
|
@@ -35270,16 +35872,16 @@ import { Hono as Hono33 } from "hono";
|
|
|
35270
35872
|
import { z as z35 } from "zod";
|
|
35271
35873
|
|
|
35272
35874
|
// src/api/scheduled-artifact-owner.ts
|
|
35273
|
-
import { createHash as
|
|
35875
|
+
import { createHash as createHash17 } from "crypto";
|
|
35274
35876
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
35275
|
-
return
|
|
35877
|
+
return createHash17("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
35276
35878
|
}
|
|
35277
35879
|
|
|
35278
35880
|
// src/api/scheduled-run-view-links.ts
|
|
35279
|
-
import { createHash as
|
|
35280
|
-
var
|
|
35881
|
+
import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
|
|
35882
|
+
var schemaReady2 = false;
|
|
35281
35883
|
async function ensureScheduledRunViewLinksSchema() {
|
|
35282
|
-
if (
|
|
35884
|
+
if (schemaReady2) return;
|
|
35283
35885
|
const db = getDb();
|
|
35284
35886
|
await db.execute(`
|
|
35285
35887
|
CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
|
|
@@ -35327,10 +35929,10 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
35327
35929
|
CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
|
|
35328
35930
|
ON scheduled_run_view_links(expires_at, revoked_at)
|
|
35329
35931
|
`);
|
|
35330
|
-
|
|
35932
|
+
schemaReady2 = true;
|
|
35331
35933
|
}
|
|
35332
35934
|
function tokenHash2(token4) {
|
|
35333
|
-
return
|
|
35935
|
+
return createHash18("sha256").update(token4).digest("hex");
|
|
35334
35936
|
}
|
|
35335
35937
|
function mapRow(row) {
|
|
35336
35938
|
return {
|
|
@@ -35417,7 +36019,7 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
|
|
|
35417
36019
|
}
|
|
35418
36020
|
|
|
35419
36021
|
// src/personal-authority/render.ts
|
|
35420
|
-
import { createHash as
|
|
36022
|
+
import { createHash as createHash19 } from "crypto";
|
|
35421
36023
|
function escapeHtml7(value) {
|
|
35422
36024
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
35423
36025
|
"&": "&",
|
|
@@ -35477,13 +36079,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
35477
36079
|
html,
|
|
35478
36080
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
35479
36081
|
bytes,
|
|
35480
|
-
sha256:
|
|
36082
|
+
sha256: createHash19("sha256").update(html).digest("hex"),
|
|
35481
36083
|
generatedAt: generatedAtIso
|
|
35482
36084
|
};
|
|
35483
36085
|
}
|
|
35484
36086
|
|
|
35485
36087
|
// src/newsroom-publisher/render.ts
|
|
35486
|
-
import { createHash as
|
|
36088
|
+
import { createHash as createHash20 } from "crypto";
|
|
35487
36089
|
function escapeHtml8(value) {
|
|
35488
36090
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
35489
36091
|
"&": "&",
|
|
@@ -35589,7 +36191,7 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
35589
36191
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
35590
36192
|
</style></head><body data-theme="${escapeHtml8(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
35591
36193
|
const bytes = Buffer.byteLength(html);
|
|
35592
|
-
const sha2563 =
|
|
36194
|
+
const sha2563 = createHash20("sha256").update(html).digest("hex");
|
|
35593
36195
|
const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
35594
36196
|
return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
|
|
35595
36197
|
}
|
|
@@ -35850,7 +36452,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
35850
36452
|
import { Hono as Hono34 } from "hono";
|
|
35851
36453
|
|
|
35852
36454
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
35853
|
-
import { createHash as
|
|
36455
|
+
import { createHash as createHash21 } from "crypto";
|
|
35854
36456
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
35855
36457
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
35856
36458
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -35866,7 +36468,7 @@ function policy3() {
|
|
|
35866
36468
|
};
|
|
35867
36469
|
}
|
|
35868
36470
|
function runStorageSegment(runId) {
|
|
35869
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
36471
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash21("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
35870
36472
|
}
|
|
35871
36473
|
async function createScheduledRunArtifact(args) {
|
|
35872
36474
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -35973,17 +36575,17 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
35973
36575
|
import { Hono as Hono35 } from "hono";
|
|
35974
36576
|
|
|
35975
36577
|
// src/api/scheduler-integration-auth.ts
|
|
35976
|
-
import { createHash as
|
|
36578
|
+
import { createHash as createHash22, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
35977
36579
|
|
|
35978
36580
|
// src/api/service-connections.ts
|
|
35979
36581
|
import { randomUUID as randomUUID18 } from "crypto";
|
|
35980
|
-
var
|
|
36582
|
+
var schemaReady3 = null;
|
|
35981
36583
|
var schemaDb3 = null;
|
|
35982
36584
|
function ensureServiceConnectionsSchema() {
|
|
35983
36585
|
const currentDb = getDb();
|
|
35984
|
-
if (
|
|
36586
|
+
if (schemaReady3 && schemaDb3 === currentDb) return schemaReady3;
|
|
35985
36587
|
schemaDb3 = currentDb;
|
|
35986
|
-
|
|
36588
|
+
schemaReady3 = (async () => {
|
|
35987
36589
|
const db = currentDb;
|
|
35988
36590
|
await db.execute(`
|
|
35989
36591
|
CREATE TABLE IF NOT EXISTS service_connections (
|
|
@@ -36051,11 +36653,11 @@ function ensureServiceConnectionsSchema() {
|
|
|
36051
36653
|
} catch {
|
|
36052
36654
|
}
|
|
36053
36655
|
})().catch((error) => {
|
|
36054
|
-
|
|
36656
|
+
schemaReady3 = null;
|
|
36055
36657
|
schemaDb3 = null;
|
|
36056
36658
|
throw error;
|
|
36057
36659
|
});
|
|
36058
|
-
return
|
|
36660
|
+
return schemaReady3;
|
|
36059
36661
|
}
|
|
36060
36662
|
function parseTools(value) {
|
|
36061
36663
|
if (typeof value !== "string") return [];
|
|
@@ -36271,7 +36873,7 @@ function signingSecret() {
|
|
|
36271
36873
|
return secret;
|
|
36272
36874
|
}
|
|
36273
36875
|
function schedulerIntegrationSignature(args) {
|
|
36274
|
-
const bodyHash =
|
|
36876
|
+
const bodyHash = createHash22("sha256").update(args.body).digest("hex");
|
|
36275
36877
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
36276
36878
|
${args.path}
|
|
36277
36879
|
${args.timestamp}
|
|
@@ -36527,13 +37129,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
36527
37129
|
}
|
|
36528
37130
|
|
|
36529
37131
|
// src/api/page-diff.ts
|
|
36530
|
-
import { createHash as
|
|
37132
|
+
import { createHash as createHash23 } from "crypto";
|
|
36531
37133
|
import { diffLines } from "diff";
|
|
36532
37134
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
36533
37135
|
var MAX_DIFF_HUNKS = 200;
|
|
36534
37136
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
36535
37137
|
function sha256Hex(value) {
|
|
36536
|
-
return
|
|
37138
|
+
return createHash23("sha256").update(value).digest("hex");
|
|
36537
37139
|
}
|
|
36538
37140
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
36539
37141
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -36647,10 +37249,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
36647
37249
|
}
|
|
36648
37250
|
|
|
36649
37251
|
// src/api/scrape-image-sink.ts
|
|
36650
|
-
import { createHash as
|
|
37252
|
+
import { createHash as createHash24 } from "crypto";
|
|
36651
37253
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
36652
37254
|
function idempotencyKey2(userId, vault, input) {
|
|
36653
|
-
return `scrape-image-${
|
|
37255
|
+
return `scrape-image-${createHash24("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
36654
37256
|
}
|
|
36655
37257
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
36656
37258
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -37122,7 +37724,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
37122
37724
|
}
|
|
37123
37725
|
|
|
37124
37726
|
// src/api/connection-memory-import.ts
|
|
37125
|
-
import { createHash as
|
|
37727
|
+
import { createHash as createHash25 } from "crypto";
|
|
37126
37728
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
37127
37729
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
37128
37730
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -37152,7 +37754,7 @@ function canonicalJson(value) {
|
|
|
37152
37754
|
return JSON.stringify(value);
|
|
37153
37755
|
}
|
|
37154
37756
|
function sha2562(value) {
|
|
37155
|
-
return
|
|
37757
|
+
return createHash25("sha256").update(value).digest("hex");
|
|
37156
37758
|
}
|
|
37157
37759
|
function sensitiveKey(key) {
|
|
37158
37760
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -38163,7 +38765,7 @@ async function dbUsage(identity, plan) {
|
|
|
38163
38765
|
}
|
|
38164
38766
|
|
|
38165
38767
|
// src/api/nango-control.ts
|
|
38166
|
-
import { createHash as
|
|
38768
|
+
import { createHash as createHash27, randomUUID as randomUUID22 } from "crypto";
|
|
38167
38769
|
|
|
38168
38770
|
// src/api/slack-archive-analysis.ts
|
|
38169
38771
|
function isRecord2(value) {
|
|
@@ -38537,7 +39139,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
38537
39139
|
}
|
|
38538
39140
|
|
|
38539
39141
|
// src/api/main-nango-transport.ts
|
|
38540
|
-
import { createHash as
|
|
39142
|
+
import { createHash as createHash26, randomUUID as randomUUID21 } from "crypto";
|
|
38541
39143
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
38542
39144
|
|
|
38543
39145
|
// src/api/connected-tool-policy.ts
|
|
@@ -38930,7 +39532,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
38930
39532
|
});
|
|
38931
39533
|
const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
|
|
38932
39534
|
const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
|
|
38933
|
-
const revision =
|
|
39535
|
+
const revision = createHash26("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
38934
39536
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
38935
39537
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
38936
39538
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -38958,7 +39560,7 @@ async function callNangoToolDirect(args) {
|
|
|
38958
39560
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
38959
39561
|
});
|
|
38960
39562
|
const requestId = args.requestId?.trim() || randomUUID21();
|
|
38961
|
-
const idempotencyKey3 = `main-nango:${
|
|
39563
|
+
const idempotencyKey3 = `main-nango:${createHash26("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
38962
39564
|
const startedAt = /* @__PURE__ */ new Date();
|
|
38963
39565
|
const started = performance.now();
|
|
38964
39566
|
let result;
|
|
@@ -38983,7 +39585,7 @@ async function callNangoToolDirect(args) {
|
|
|
38983
39585
|
toolName: args.tool,
|
|
38984
39586
|
operationKind: args.operationKind ?? args.classification,
|
|
38985
39587
|
outcome: providerError ? "error" : "partial",
|
|
38986
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
39588
|
+
requestId: requestId.length <= 200 ? requestId : createHash26("sha256").update(requestId).digest("hex"),
|
|
38987
39589
|
startedAt: startedAt.toISOString(),
|
|
38988
39590
|
completedAt: completedAt.toISOString()
|
|
38989
39591
|
}
|
|
@@ -39025,7 +39627,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
39025
39627
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
39026
39628
|
protocolVersion: null,
|
|
39027
39629
|
schemaSource: "live_tools_list",
|
|
39028
|
-
schemaHash:
|
|
39630
|
+
schemaHash: createHash26("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
39029
39631
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
39030
39632
|
};
|
|
39031
39633
|
}
|
|
@@ -39749,8 +40351,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
39749
40351
|
return data.connection.actionsEnabled === true;
|
|
39750
40352
|
}
|
|
39751
40353
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
39752
|
-
const requestId = `main-connected-action:${
|
|
39753
|
-
const requestDigest =
|
|
40354
|
+
const requestId = `main-connected-action:${createHash27("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
|
|
40355
|
+
const requestDigest = createHash27("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
39754
40356
|
if (mainOwnsIntegrations()) {
|
|
39755
40357
|
const selectedTool = tool?.trim();
|
|
39756
40358
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -39901,7 +40503,7 @@ function canonicalJson2(value) {
|
|
|
39901
40503
|
return JSON.stringify(value);
|
|
39902
40504
|
}
|
|
39903
40505
|
function projectedToolSchemaHash(tool) {
|
|
39904
|
-
return
|
|
40506
|
+
return createHash27("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
39905
40507
|
}
|
|
39906
40508
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
39907
40509
|
if (mainOwnsIntegrations()) {
|
|
@@ -40180,7 +40782,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
40180
40782
|
}
|
|
40181
40783
|
|
|
40182
40784
|
// src/api/resend-control.ts
|
|
40183
|
-
import { createHash as
|
|
40785
|
+
import { createHash as createHash28 } from "crypto";
|
|
40184
40786
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
40185
40787
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
40186
40788
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -40439,7 +41041,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
40439
41041
|
return isRecord5(data) ? data.result ?? data : data;
|
|
40440
41042
|
}
|
|
40441
41043
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
40442
|
-
const requestId = `main-resend-action:${
|
|
41044
|
+
const requestId = `main-resend-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
40443
41045
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
40444
41046
|
method: "POST",
|
|
40445
41047
|
headers: { "x-request-id": requestId },
|
|
@@ -43486,35 +44088,73 @@ app.post("/billing/credits", auth4, async (c) => {
|
|
|
43486
44088
|
ledger
|
|
43487
44089
|
});
|
|
43488
44090
|
});
|
|
44091
|
+
var CRON_TICK_BUDGET_MS = 5e4;
|
|
44092
|
+
var CRON_TICK_DRAIN_BUDGET_MS = 4e4;
|
|
44093
|
+
function settleWithinTickBudget(label, unfinished, work, onDeadlineOrError) {
|
|
44094
|
+
return new Promise((resolve2) => {
|
|
44095
|
+
const timer = setTimeout(() => {
|
|
44096
|
+
unfinished.push(`${label}:deadline`);
|
|
44097
|
+
resolve2(onDeadlineOrError);
|
|
44098
|
+
}, CRON_TICK_BUDGET_MS);
|
|
44099
|
+
work.then(
|
|
44100
|
+
(value) => {
|
|
44101
|
+
clearTimeout(timer);
|
|
44102
|
+
resolve2(value);
|
|
44103
|
+
},
|
|
44104
|
+
(err) => {
|
|
44105
|
+
clearTimeout(timer);
|
|
44106
|
+
unfinished.push(`${label}:error`);
|
|
44107
|
+
console.error(JSON.stringify({
|
|
44108
|
+
event: "cron_tick_job_failed",
|
|
44109
|
+
job: label,
|
|
44110
|
+
message: err instanceof Error ? err.message : String(err)
|
|
44111
|
+
}));
|
|
44112
|
+
resolve2(onDeadlineOrError);
|
|
44113
|
+
}
|
|
44114
|
+
);
|
|
44115
|
+
});
|
|
44116
|
+
}
|
|
43489
44117
|
app.get("/cron/tick", async (c) => {
|
|
43490
44118
|
const secret = c.req.header("authorization");
|
|
43491
44119
|
if (!process.env.CRON_SECRET || secret !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
43492
44120
|
return c.json({ error: "Unauthorized" }, 401);
|
|
43493
44121
|
}
|
|
43494
|
-
const
|
|
43495
|
-
const
|
|
43496
|
-
const
|
|
43497
|
-
const
|
|
43498
|
-
|
|
43499
|
-
|
|
43500
|
-
|
|
43501
|
-
|
|
43502
|
-
|
|
43503
|
-
|
|
43504
|
-
|
|
43505
|
-
|
|
43506
|
-
|
|
43507
|
-
|
|
43508
|
-
|
|
43509
|
-
|
|
43510
|
-
|
|
43511
|
-
|
|
43512
|
-
|
|
43513
|
-
|
|
43514
|
-
|
|
43515
|
-
|
|
44122
|
+
const startedAt = Date.now();
|
|
44123
|
+
const { drainQueue } = await import("./worker-KQN673JF.js");
|
|
44124
|
+
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
44125
|
+
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
44126
|
+
const unfinished = [];
|
|
44127
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
44128
|
+
settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
|
|
44129
|
+
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
44130
|
+
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
44131
|
+
settleWithinTickBudget("retryOrphanedBrowserSessions", unfinished, retryOrphanedBrowserSessions(50), { recovered: 0, stillOpen: 0 }),
|
|
44132
|
+
settleWithinTickBudget("expireLots", unfinished, expireOldLots(), { expired_lots: 0, expired_mc: 0 }),
|
|
44133
|
+
settleWithinTickBudget("scrapeBlobCleanup", unfinished, cleanupExpiredScrapeBlobs(), { deleted: 0, store: "none" }),
|
|
44134
|
+
settleWithinTickBudget("connectedDataCleanup", unfinished, cleanupExpiredConnectedDataArtifacts(), { deleted: 0, store: "none" }),
|
|
44135
|
+
settleWithinTickBudget("connectedBillingSweep", unfinished, reconcileConnectedAccountBillingSweep(), { checked: 0, synced: 0, failed: 0 }),
|
|
44136
|
+
settleWithinTickBudget("directoryDispatch", unfinished, dispatchPendingDirectoryWorkflows(5), { claimed: 0, dispatched: 0, failed: 0 }),
|
|
44137
|
+
settleWithinTickBudget("directoryReconcile", unfinished, reconcileDirectoryWorkflowSettlements(5), { checked: 0, settled: 0, failed: 0 }),
|
|
44138
|
+
settleWithinTickBudget("directoryArtifactCleanup", unfinished, cleanupExpiredDirectoryArtifacts(), { deleted: 0, store: "none" }),
|
|
44139
|
+
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
44140
|
+
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44141
|
+
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
44142
|
+
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44143
|
+
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
44144
|
+
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
44145
|
+
settleWithinTickBudget("analyticsRollup", unfinished, refreshAnalyticsDailyRollupsIfDue(), { status: "failed", rows: 0 }),
|
|
44146
|
+
settleWithinTickBudget("analyticsForms", unfinished, deliverPendingAnalyticsForms(10), { claimed: 0, delivered: 0, deferred: 0 }),
|
|
44147
|
+
settleWithinTickBudget("workflowDispatch", unfinished, dispatchDueWorkflowSchedules(origin), { dispatched: 0, status: "skipped_deadline" }),
|
|
44148
|
+
settleWithinTickBudget("scheduledRunReconcile", unfinished, reconcileScheduledRunSettlements(25), { checked: 0, settled: 0, chargedMc: 0, failed: 0 }),
|
|
44149
|
+
settleWithinTickBudget("retentionSweep", unfinished, sweepRetention(), { deleted: 0, perTable: {}, truncated: false })
|
|
43516
44150
|
]);
|
|
43517
44151
|
return c.json({
|
|
44152
|
+
durationMs: Date.now() - startedAt,
|
|
44153
|
+
tickBudgetMs: CRON_TICK_BUDGET_MS,
|
|
44154
|
+
unfinished,
|
|
44155
|
+
complete: unfinished.length === 0,
|
|
44156
|
+
scheduledRunReconciliation,
|
|
44157
|
+
retentionSweep,
|
|
43518
44158
|
drained: results.length,
|
|
43519
44159
|
results,
|
|
43520
44160
|
sweepResult,
|
|
@@ -43550,8 +44190,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
43550
44190
|
finishExtractJob: finishExtractJob2,
|
|
43551
44191
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
43552
44192
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
43553
|
-
} = await import("./site-extract-repository-
|
|
43554
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
44193
|
+
} = await import("./site-extract-repository-3VQAALNR.js");
|
|
44194
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
|
|
43555
44195
|
const existing = await getExtractJob2(jobId2);
|
|
43556
44196
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
43557
44197
|
if (existing.status !== "failed") {
|
|
@@ -43578,7 +44218,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
43578
44218
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
43579
44219
|
let settlement = "already_settled_or_refunded";
|
|
43580
44220
|
if (job.billedMc == null && job.userId != null) {
|
|
43581
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
44221
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-3VQAALNR.js");
|
|
43582
44222
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
43583
44223
|
const successful = await countSuccessfulPages2(jobId2);
|
|
43584
44224
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -43763,4 +44403,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
43763
44403
|
export {
|
|
43764
44404
|
app
|
|
43765
44405
|
};
|
|
43766
|
-
//# sourceMappingURL=server-
|
|
44406
|
+
//# sourceMappingURL=server-3ND2EXYO.js.map
|