mcp-scraper 0.66.6 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -1
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1122 -625
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +147 -17
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.cjs +20 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-3EYALCYK.js → chunk-2L4C4DAZ.js} +1 -1
- package/dist/{chunk-CETDPNMZ.js → chunk-2LXTOAKS.js} +1 -1
- package/dist/{chunk-K7Y27ND3.js → chunk-62LEXS5O.js} +2 -2
- package/dist/{chunk-55B3T5L2.js → chunk-65FTMB7G.js} +92 -1
- package/dist/{chunk-3TKKJQB4.js → chunk-ES25GP6C.js} +1 -1
- package/dist/{chunk-NLEAYKOC.js → chunk-GT63KEUB.js} +1 -1
- package/dist/{chunk-KPE52NPT.js → chunk-IHDTONZV.js} +1 -1
- package/dist/{chunk-KXRWY7DZ.js → chunk-LCGTNWOT.js} +1 -1
- package/dist/{chunk-DBWWIK7O.js → chunk-LN6N3YLK.js} +1 -1
- package/dist/{chunk-ABYBPODM.js → chunk-UL4ZKAWZ.js} +1 -1
- package/dist/{chunk-VMWVEKZT.js → chunk-XOOKZYZG.js} +71 -18
- package/dist/{chunk-PMCT2YQM.js → chunk-Y46YNQMM.js} +1 -1
- package/dist/{db-4RVZ3NYZ.js → db-ZGZAKYBW.js} +1 -1
- package/dist/{extract-bundle-EA5URLXN.js → extract-bundle-QQIYJSBW.js} +3 -3
- package/dist/index.cjs +20 -1
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-IBXNWM6H.js → lead-list-enrichment-repository-CXLVIV3R.js} +2 -2
- package/dist/{location-data-repository-WHHQUCUI.js → location-data-repository-G72M7DTN.js} +2 -2
- package/dist/{server-5WPS3LZD.js → server-MFVYRB6I.js} +818 -471
- package/dist/{site-extract-repository-NTM4MDQN.js → site-extract-repository-7EP6UX6I.js} +2 -2
- package/dist/{worker-UTBUWEXQ.js → worker-JGWV2LID.js} +5 -5
- package/package.json +1 -1
|
@@ -10,13 +10,13 @@ import {
|
|
|
10
10
|
queuedCommonsDocumentCount,
|
|
11
11
|
semanticCommonsEntityScores,
|
|
12
12
|
typeFromMime
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-LN6N3YLK.js";
|
|
14
14
|
import {
|
|
15
15
|
REQUIRED_LOCATION_STATE_CODES,
|
|
16
16
|
importHostedCensusPlacesCsv,
|
|
17
17
|
importHostedZipGroupsCsv,
|
|
18
18
|
listActiveHostedLocationDatasets
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-Y46YNQMM.js";
|
|
20
20
|
import {
|
|
21
21
|
cancelPendingLeadListEnrichmentRows,
|
|
22
22
|
claimLeadListEnrichmentOutbox,
|
|
@@ -43,7 +43,7 @@ import {
|
|
|
43
43
|
saveLeadListEnrichmentRowResult,
|
|
44
44
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
45
45
|
stableCanonicalJson
|
|
46
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-UL4ZKAWZ.js";
|
|
47
47
|
import {
|
|
48
48
|
ANALYTICS_CONTENT_SORTS,
|
|
49
49
|
AnalyticsRepositoryError,
|
|
@@ -108,7 +108,7 @@ import {
|
|
|
108
108
|
harvestProblemResponse,
|
|
109
109
|
publicErrorBoundary,
|
|
110
110
|
serializeHarvestProblem
|
|
111
|
-
} from "./chunk-
|
|
111
|
+
} from "./chunk-2LXTOAKS.js";
|
|
112
112
|
import "./chunk-OZJMVCDK.js";
|
|
113
113
|
import {
|
|
114
114
|
BACKUP_LAST_RESORT_ATTEMPTS,
|
|
@@ -127,7 +127,7 @@ import {
|
|
|
127
127
|
resolveKernelProxyId,
|
|
128
128
|
serpArrivalUrl,
|
|
129
129
|
withRuntimeUserAgent
|
|
130
|
-
} from "./chunk-
|
|
130
|
+
} from "./chunk-62LEXS5O.js";
|
|
131
131
|
import {
|
|
132
132
|
ApiErrorResponseSchema,
|
|
133
133
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -199,7 +199,7 @@ import {
|
|
|
199
199
|
resolveDeploymentProfile,
|
|
200
200
|
resolveLocalSourcebookSchemaType,
|
|
201
201
|
transcribeMediaUrl
|
|
202
|
-
} from "./chunk-
|
|
202
|
+
} from "./chunk-XOOKZYZG.js";
|
|
203
203
|
import {
|
|
204
204
|
auditImageUrls,
|
|
205
205
|
auditImages,
|
|
@@ -210,7 +210,7 @@ import {
|
|
|
210
210
|
sanitizeAttempts,
|
|
211
211
|
sanitizeHarvestResult,
|
|
212
212
|
sanitizeOutboundDiagnostics
|
|
213
|
-
} from "./chunk-
|
|
213
|
+
} from "./chunk-2L4C4DAZ.js";
|
|
214
214
|
import {
|
|
215
215
|
browserServiceApiKey,
|
|
216
216
|
browserServiceProfileName,
|
|
@@ -220,7 +220,7 @@ import {
|
|
|
220
220
|
recordKernelSession,
|
|
221
221
|
recordVendorUsage,
|
|
222
222
|
runWithCostContext
|
|
223
|
-
} from "./chunk-
|
|
223
|
+
} from "./chunk-ES25GP6C.js";
|
|
224
224
|
import {
|
|
225
225
|
listWorkflowDefinitions,
|
|
226
226
|
runWorkflow,
|
|
@@ -256,7 +256,7 @@ import {
|
|
|
256
256
|
} from "./chunk-GGZEC22A.js";
|
|
257
257
|
import {
|
|
258
258
|
PACKAGE_VERSION
|
|
259
|
-
} from "./chunk-
|
|
259
|
+
} from "./chunk-GT63KEUB.js";
|
|
260
260
|
import {
|
|
261
261
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
262
262
|
abandonExtractSettlement,
|
|
@@ -287,7 +287,7 @@ import {
|
|
|
287
287
|
setExtractJobTotal,
|
|
288
288
|
settleExtractJob,
|
|
289
289
|
terminalExtractJobStatus
|
|
290
|
-
} from "./chunk-
|
|
290
|
+
} from "./chunk-LCGTNWOT.js";
|
|
291
291
|
import {
|
|
292
292
|
createPrivateArtifact,
|
|
293
293
|
privateArtifactOwnerId,
|
|
@@ -451,7 +451,7 @@ import {
|
|
|
451
451
|
softDeleteInboxMessage,
|
|
452
452
|
upsertPageSnapshot,
|
|
453
453
|
verifyPassword
|
|
454
|
-
} from "./chunk-
|
|
454
|
+
} from "./chunk-65FTMB7G.js";
|
|
455
455
|
|
|
456
456
|
// src/blog/registry.ts
|
|
457
457
|
var posts = [
|
|
@@ -6312,7 +6312,8 @@ function firstJsonLdImage(value) {
|
|
|
6312
6312
|
}
|
|
6313
6313
|
return null;
|
|
6314
6314
|
}
|
|
6315
|
-
function
|
|
6315
|
+
function contentImages(html) {
|
|
6316
|
+
const images = [];
|
|
6316
6317
|
for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
|
|
6317
6318
|
const tag = match[0];
|
|
6318
6319
|
const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
|
|
@@ -6323,30 +6324,36 @@ function firstContentImage(html) {
|
|
|
6323
6324
|
const width = Number(attr("width") ?? 0);
|
|
6324
6325
|
const height = Number(attr("height") ?? 0);
|
|
6325
6326
|
if (width > 0 && width < 200 || height > 0 && height < 120) continue;
|
|
6326
|
-
|
|
6327
|
+
images.push(candidate);
|
|
6327
6328
|
}
|
|
6328
|
-
return
|
|
6329
|
+
return images;
|
|
6329
6330
|
}
|
|
6330
|
-
function
|
|
6331
|
+
function resolveFeaturedImageCandidates(input) {
|
|
6331
6332
|
const replay = parseWaybackReplayUrl(input.pageUrl);
|
|
6332
6333
|
const baseUrl = replay?.originalUrl ?? input.pageUrl;
|
|
6333
6334
|
const candidates = [
|
|
6334
6335
|
{ value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
|
|
6335
6336
|
{ value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
|
|
6336
6337
|
{ value: firstJsonLdImage(input.schema), source: "json-ld" },
|
|
6337
|
-
|
|
6338
|
+
...contentImages(input.html).map((value) => ({ value, source: "content-image" }))
|
|
6338
6339
|
];
|
|
6340
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6341
|
+
const resolved = [];
|
|
6339
6342
|
for (const candidate of candidates) {
|
|
6340
6343
|
if (!candidate.value) continue;
|
|
6341
6344
|
const url = safeHttpUrl(candidate.value, baseUrl);
|
|
6342
|
-
if (!url) continue;
|
|
6343
|
-
|
|
6345
|
+
if (!url || seen.has(url)) continue;
|
|
6346
|
+
seen.add(url);
|
|
6347
|
+
resolved.push({
|
|
6344
6348
|
url,
|
|
6345
6349
|
archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
|
|
6346
6350
|
source: candidate.source
|
|
6347
|
-
};
|
|
6351
|
+
});
|
|
6348
6352
|
}
|
|
6349
|
-
return
|
|
6353
|
+
return resolved;
|
|
6354
|
+
}
|
|
6355
|
+
function resolveFeaturedImage(input) {
|
|
6356
|
+
return resolveFeaturedImageCandidates(input)[0] ?? null;
|
|
6350
6357
|
}
|
|
6351
6358
|
|
|
6352
6359
|
// src/api/site-extractor.ts
|
|
@@ -9545,7 +9552,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9545
9552
|
});
|
|
9546
9553
|
}) : null;
|
|
9547
9554
|
const finalized = await step.run("finalize", async () => {
|
|
9548
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9555
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
|
|
9549
9556
|
const current = await getExtractJob(jobId2);
|
|
9550
9557
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9551
9558
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10505,7 +10512,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
10505
10512
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
10506
10513
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
10507
10514
|
if (mode === "hosted") {
|
|
10508
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
10515
|
+
const { getHostedZipGroups } = await import("./location-data-repository-G72M7DTN.js");
|
|
10509
10516
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
10510
10517
|
if (!hosted.dataset) {
|
|
10511
10518
|
throw new DirectoryLocationError(
|
|
@@ -10568,7 +10575,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10568
10575
|
const sourceUrl = censusStateUrl(state.fips);
|
|
10569
10576
|
const warnings = [];
|
|
10570
10577
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
10571
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
10578
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-G72M7DTN.js");
|
|
10572
10579
|
const hosted = await queryHostedLocationMarkets({
|
|
10573
10580
|
state: state.abbr,
|
|
10574
10581
|
minPopulation: options.minPopulation,
|
|
@@ -12055,7 +12062,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
|
|
|
12055
12062
|
async function settlePersistedLeadListEnrichment(jobId2) {
|
|
12056
12063
|
const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
|
|
12057
12064
|
const job = jobs.find((candidate) => candidate.id === jobId2);
|
|
12058
|
-
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12065
|
+
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
|
|
12059
12066
|
const current = job ?? await getLeadListEnrichmentJob2(jobId2);
|
|
12060
12067
|
if (!current) return null;
|
|
12061
12068
|
const rows = await listLeadListEnrichmentRows(jobId2);
|
|
@@ -12111,7 +12118,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
|
|
|
12111
12118
|
const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
|
|
12112
12119
|
const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
|
|
12113
12120
|
if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
|
|
12114
|
-
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12121
|
+
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
|
|
12115
12122
|
await failLeadListEnrichmentJob2({
|
|
12116
12123
|
jobId: job.id,
|
|
12117
12124
|
errorCode: "lead_list_all_rows_failed",
|
|
@@ -16505,6 +16512,350 @@ function findPreferredEntitledSubscription(subscriptions, predicate) {
|
|
|
16505
16512
|
return void 0;
|
|
16506
16513
|
}
|
|
16507
16514
|
|
|
16515
|
+
// src/api/service-connections.ts
|
|
16516
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
16517
|
+
var schemaReady = null;
|
|
16518
|
+
var schemaDb3 = null;
|
|
16519
|
+
function ensureServiceConnectionsSchema() {
|
|
16520
|
+
const currentDb = getDb();
|
|
16521
|
+
if (schemaReady && schemaDb3 === currentDb) return schemaReady;
|
|
16522
|
+
schemaDb3 = currentDb;
|
|
16523
|
+
schemaReady = (async () => {
|
|
16524
|
+
const db = currentDb;
|
|
16525
|
+
await db.execute(`
|
|
16526
|
+
CREATE TABLE IF NOT EXISTS service_connections (
|
|
16527
|
+
id TEXT PRIMARY KEY,
|
|
16528
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
16529
|
+
provider_config_key TEXT NOT NULL,
|
|
16530
|
+
provider TEXT,
|
|
16531
|
+
transport TEXT NOT NULL,
|
|
16532
|
+
upstream_connection_id TEXT,
|
|
16533
|
+
label TEXT,
|
|
16534
|
+
user_label TEXT,
|
|
16535
|
+
provider_account_id TEXT,
|
|
16536
|
+
provider_account_email TEXT,
|
|
16537
|
+
provider_account_name TEXT,
|
|
16538
|
+
provider_identity_checked_at TEXT,
|
|
16539
|
+
provider_identity_source_updated_at TEXT,
|
|
16540
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
16541
|
+
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
16542
|
+
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
16543
|
+
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
16544
|
+
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
16545
|
+
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
16546
|
+
tool_revision TEXT,
|
|
16547
|
+
last_checked_at TEXT,
|
|
16548
|
+
last_successful_call_at TEXT,
|
|
16549
|
+
last_failure_at TEXT,
|
|
16550
|
+
last_failure_code TEXT,
|
|
16551
|
+
last_failure_retryable INTEGER,
|
|
16552
|
+
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
16553
|
+
vault_name TEXT,
|
|
16554
|
+
table_name TEXT,
|
|
16555
|
+
legacy_owner_identity TEXT,
|
|
16556
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
16557
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
16558
|
+
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
16559
|
+
)
|
|
16560
|
+
`);
|
|
16561
|
+
try {
|
|
16562
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN user_label TEXT`);
|
|
16563
|
+
} catch {
|
|
16564
|
+
}
|
|
16565
|
+
try {
|
|
16566
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT`);
|
|
16567
|
+
} catch {
|
|
16568
|
+
}
|
|
16569
|
+
try {
|
|
16570
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT`);
|
|
16571
|
+
} catch {
|
|
16572
|
+
}
|
|
16573
|
+
try {
|
|
16574
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT`);
|
|
16575
|
+
} catch {
|
|
16576
|
+
}
|
|
16577
|
+
try {
|
|
16578
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT`);
|
|
16579
|
+
} catch {
|
|
16580
|
+
}
|
|
16581
|
+
try {
|
|
16582
|
+
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT`);
|
|
16583
|
+
} catch {
|
|
16584
|
+
}
|
|
16585
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
|
|
16586
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
|
|
16587
|
+
await db.execute(`
|
|
16588
|
+
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
16589
|
+
id TEXT PRIMARY KEY,
|
|
16590
|
+
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
16591
|
+
operational_status TEXT NOT NULL,
|
|
16592
|
+
failure_code TEXT,
|
|
16593
|
+
retryable INTEGER,
|
|
16594
|
+
evidence_source TEXT NOT NULL,
|
|
16595
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
16596
|
+
)
|
|
16597
|
+
`);
|
|
16598
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
|
|
16599
|
+
await db.execute(`
|
|
16600
|
+
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
16601
|
+
id TEXT PRIMARY KEY,
|
|
16602
|
+
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
16603
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
16604
|
+
tool TEXT NOT NULL,
|
|
16605
|
+
request_id TEXT NOT NULL,
|
|
16606
|
+
status TEXT NOT NULL,
|
|
16607
|
+
error_code TEXT,
|
|
16608
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
16609
|
+
UNIQUE(user_id, request_id)
|
|
16610
|
+
)
|
|
16611
|
+
`);
|
|
16612
|
+
try {
|
|
16613
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
|
|
16614
|
+
} catch {
|
|
16615
|
+
}
|
|
16616
|
+
try {
|
|
16617
|
+
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
|
|
16618
|
+
} catch {
|
|
16619
|
+
}
|
|
16620
|
+
})().catch((error) => {
|
|
16621
|
+
schemaReady = null;
|
|
16622
|
+
schemaDb3 = null;
|
|
16623
|
+
throw error;
|
|
16624
|
+
});
|
|
16625
|
+
return schemaReady;
|
|
16626
|
+
}
|
|
16627
|
+
function parseTools(value) {
|
|
16628
|
+
if (typeof value !== "string") return [];
|
|
16629
|
+
try {
|
|
16630
|
+
const parsed = JSON.parse(value);
|
|
16631
|
+
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
|
|
16632
|
+
} catch {
|
|
16633
|
+
return [];
|
|
16634
|
+
}
|
|
16635
|
+
}
|
|
16636
|
+
function rowToRecord(row) {
|
|
16637
|
+
return {
|
|
16638
|
+
id: String(row.id),
|
|
16639
|
+
userId: Number(row.user_id),
|
|
16640
|
+
providerConfigKey: String(row.provider_config_key),
|
|
16641
|
+
provider: typeof row.provider === "string" ? row.provider : null,
|
|
16642
|
+
transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
|
|
16643
|
+
upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
|
|
16644
|
+
label: typeof row.label === "string" ? row.label : null,
|
|
16645
|
+
userLabel: typeof row.user_label === "string" ? row.user_label : null,
|
|
16646
|
+
providerAccountId: typeof row.provider_account_id === "string" ? row.provider_account_id : null,
|
|
16647
|
+
providerAccountEmail: typeof row.provider_account_email === "string" ? row.provider_account_email : null,
|
|
16648
|
+
providerAccountName: typeof row.provider_account_name === "string" ? row.provider_account_name : null,
|
|
16649
|
+
providerIdentityCheckedAt: typeof row.provider_identity_checked_at === "string" ? row.provider_identity_checked_at : null,
|
|
16650
|
+
providerIdentitySourceUpdatedAt: typeof row.provider_identity_source_updated_at === "string" ? row.provider_identity_source_updated_at : null,
|
|
16651
|
+
lifecycleStatus: String(row.lifecycle_status),
|
|
16652
|
+
operationalStatus: String(row.operational_status),
|
|
16653
|
+
reconnectRequired: Number(row.reconnect_required) === 1,
|
|
16654
|
+
actionsEnabled: Number(row.actions_enabled) === 1,
|
|
16655
|
+
readTools: parseTools(row.read_tools_json),
|
|
16656
|
+
actionTools: parseTools(row.action_tools_json),
|
|
16657
|
+
toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
|
|
16658
|
+
lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
|
|
16659
|
+
lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
|
|
16660
|
+
lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
|
|
16661
|
+
lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
|
|
16662
|
+
lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
|
|
16663
|
+
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
|
16664
|
+
vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
|
|
16665
|
+
tableName: typeof row.table_name === "string" ? row.table_name : null,
|
|
16666
|
+
createdAt: String(row.created_at),
|
|
16667
|
+
updatedAt: String(row.updated_at)
|
|
16668
|
+
};
|
|
16669
|
+
}
|
|
16670
|
+
async function userIdForIdentity(identity) {
|
|
16671
|
+
const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
|
|
16672
|
+
if (!user) throw new Error("service_connection_user_not_found");
|
|
16673
|
+
return Number(user.id);
|
|
16674
|
+
}
|
|
16675
|
+
async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
16676
|
+
await ensureServiceConnectionsSchema();
|
|
16677
|
+
const db = getDb();
|
|
16678
|
+
const userId = await userIdForIdentity(identity);
|
|
16679
|
+
for (const connection of discovered) {
|
|
16680
|
+
await db.execute({
|
|
16681
|
+
sql: `
|
|
16682
|
+
INSERT INTO service_connections (
|
|
16683
|
+
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
16684
|
+
label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
|
|
16685
|
+
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
16686
|
+
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
16687
|
+
provider = excluded.provider,
|
|
16688
|
+
label = COALESCE(excluded.label, service_connections.label),
|
|
16689
|
+
lifecycle_status = excluded.lifecycle_status,
|
|
16690
|
+
reconnect_required = excluded.reconnect_required,
|
|
16691
|
+
updated_at = excluded.updated_at
|
|
16692
|
+
`,
|
|
16693
|
+
args: [
|
|
16694
|
+
randomUUID6(),
|
|
16695
|
+
userId,
|
|
16696
|
+
connection.providerConfigKey,
|
|
16697
|
+
connection.provider,
|
|
16698
|
+
connection.upstreamConnectionId,
|
|
16699
|
+
connection.label,
|
|
16700
|
+
connection.lifecycleStatus,
|
|
16701
|
+
connection.reconnectRequired ? 1 : 0,
|
|
16702
|
+
identity,
|
|
16703
|
+
connection.createdAt,
|
|
16704
|
+
connection.updatedAt
|
|
16705
|
+
]
|
|
16706
|
+
});
|
|
16707
|
+
}
|
|
16708
|
+
return listServiceConnections(identity, "nango");
|
|
16709
|
+
}
|
|
16710
|
+
async function listServiceConnections(identity, transport) {
|
|
16711
|
+
await ensureServiceConnectionsSchema();
|
|
16712
|
+
const userId = await userIdForIdentity(identity);
|
|
16713
|
+
const result = await getDb().execute({
|
|
16714
|
+
sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
|
|
16715
|
+
args: transport ? [userId, transport] : [userId]
|
|
16716
|
+
});
|
|
16717
|
+
return result.rows.map((row) => rowToRecord(row));
|
|
16718
|
+
}
|
|
16719
|
+
async function getOwnedServiceConnection(identity, connectionId) {
|
|
16720
|
+
await ensureServiceConnectionsSchema();
|
|
16721
|
+
const userId = await userIdForIdentity(identity);
|
|
16722
|
+
const result = await getDb().execute({
|
|
16723
|
+
sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
|
|
16724
|
+
args: [connectionId, userId]
|
|
16725
|
+
});
|
|
16726
|
+
const row = result.rows[0];
|
|
16727
|
+
return row ? rowToRecord(row) : null;
|
|
16728
|
+
}
|
|
16729
|
+
async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
|
|
16730
|
+
await ensureServiceConnectionsSchema();
|
|
16731
|
+
await getDb().execute({
|
|
16732
|
+
sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
16733
|
+
args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
|
|
16734
|
+
});
|
|
16735
|
+
}
|
|
16736
|
+
async function setServiceConnectionProviderIdentity(args) {
|
|
16737
|
+
await ensureServiceConnectionsSchema();
|
|
16738
|
+
await getDb().execute({
|
|
16739
|
+
sql: `
|
|
16740
|
+
UPDATE service_connections SET
|
|
16741
|
+
provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
|
|
16742
|
+
provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
|
|
16743
|
+
updated_at = datetime('now')
|
|
16744
|
+
WHERE id = ?
|
|
16745
|
+
`,
|
|
16746
|
+
args: [
|
|
16747
|
+
args.providerAccountId ?? null,
|
|
16748
|
+
args.providerAccountEmail?.toLowerCase() ?? null,
|
|
16749
|
+
args.providerAccountName ?? null,
|
|
16750
|
+
args.sourceUpdatedAt,
|
|
16751
|
+
args.connectionId
|
|
16752
|
+
]
|
|
16753
|
+
});
|
|
16754
|
+
}
|
|
16755
|
+
async function setServiceConnectionUserLabel(identity, connectionId, userLabel) {
|
|
16756
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
16757
|
+
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
16758
|
+
await getDb().execute({
|
|
16759
|
+
sql: `UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
16760
|
+
args: [userLabel, connectionId, connection.userId]
|
|
16761
|
+
});
|
|
16762
|
+
return userLabel;
|
|
16763
|
+
}
|
|
16764
|
+
async function recordServiceConnectionHealth(args) {
|
|
16765
|
+
await ensureServiceConnectionsSchema();
|
|
16766
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16767
|
+
await getDb().execute({
|
|
16768
|
+
sql: `
|
|
16769
|
+
UPDATE service_connections SET
|
|
16770
|
+
operational_status = ?, last_checked_at = ?,
|
|
16771
|
+
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
16772
|
+
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
16773
|
+
last_failure_code = ?, last_failure_retryable = ?,
|
|
16774
|
+
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
16775
|
+
updated_at = datetime('now')
|
|
16776
|
+
WHERE id = ?
|
|
16777
|
+
`,
|
|
16778
|
+
args: [
|
|
16779
|
+
args.operationalStatus,
|
|
16780
|
+
now,
|
|
16781
|
+
args.operationalStatus,
|
|
16782
|
+
now,
|
|
16783
|
+
args.operationalStatus,
|
|
16784
|
+
now,
|
|
16785
|
+
args.failureCode ?? null,
|
|
16786
|
+
args.retryable == null ? null : args.retryable ? 1 : 0,
|
|
16787
|
+
args.operationalStatus,
|
|
16788
|
+
args.connectionId
|
|
16789
|
+
]
|
|
16790
|
+
});
|
|
16791
|
+
await getDb().execute({
|
|
16792
|
+
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
16793
|
+
args: [randomUUID6(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
16794
|
+
});
|
|
16795
|
+
}
|
|
16796
|
+
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
16797
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
16798
|
+
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
16799
|
+
await getDb().execute({
|
|
16800
|
+
sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
16801
|
+
args: [enabled ? 1 : 0, connectionId, connection.userId]
|
|
16802
|
+
});
|
|
16803
|
+
return enabled;
|
|
16804
|
+
}
|
|
16805
|
+
async function markServiceConnectionDisconnected(identity, connectionId) {
|
|
16806
|
+
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
16807
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
16808
|
+
await getDb().execute({
|
|
16809
|
+
sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
16810
|
+
args: [connectionId, connection.userId]
|
|
16811
|
+
});
|
|
16812
|
+
const remaining = await getDb().execute({
|
|
16813
|
+
sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
|
|
16814
|
+
args: [connection.userId]
|
|
16815
|
+
});
|
|
16816
|
+
return Number(remaining.rows[0]?.count ?? 0);
|
|
16817
|
+
}
|
|
16818
|
+
async function recordServiceConnectionAction(args) {
|
|
16819
|
+
await ensureServiceConnectionsSchema();
|
|
16820
|
+
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
16821
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
16822
|
+
await getDb().execute({
|
|
16823
|
+
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
16824
|
+
args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
|
|
16825
|
+
});
|
|
16826
|
+
}
|
|
16827
|
+
async function claimServiceConnectionAction(args) {
|
|
16828
|
+
await ensureServiceConnectionsSchema();
|
|
16829
|
+
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
16830
|
+
if (!connection) throw new Error("service_connection_not_found");
|
|
16831
|
+
const inserted = await getDb().execute({
|
|
16832
|
+
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
16833
|
+
args: [randomUUID6(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
16834
|
+
});
|
|
16835
|
+
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
16836
|
+
const existing = await getDb().execute({
|
|
16837
|
+
sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
16838
|
+
args: [connection.userId, args.requestId]
|
|
16839
|
+
});
|
|
16840
|
+
const row = existing.rows[0];
|
|
16841
|
+
const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
|
|
16842
|
+
let result = void 0;
|
|
16843
|
+
if (!conflict && typeof row?.result_json === "string") {
|
|
16844
|
+
try {
|
|
16845
|
+
result = JSON.parse(row.result_json);
|
|
16846
|
+
} catch {
|
|
16847
|
+
result = void 0;
|
|
16848
|
+
}
|
|
16849
|
+
}
|
|
16850
|
+
return {
|
|
16851
|
+
claimed: false,
|
|
16852
|
+
status: String(row?.status ?? "unknown"),
|
|
16853
|
+
errorCode: typeof row?.error_code === "string" ? row.error_code : null,
|
|
16854
|
+
conflict,
|
|
16855
|
+
...result !== void 0 ? { result } : {}
|
|
16856
|
+
};
|
|
16857
|
+
}
|
|
16858
|
+
|
|
16508
16859
|
// src/api/connected-account-billing.ts
|
|
16509
16860
|
var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
|
|
16510
16861
|
function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
|
|
@@ -16521,6 +16872,87 @@ async function chargedConnectionIds(userId, periodStart) {
|
|
|
16521
16872
|
});
|
|
16522
16873
|
return new Set(result.rows.map((row) => String(row.connection_id)));
|
|
16523
16874
|
}
|
|
16875
|
+
async function inheritDisconnectedConnectionCharge(userId, connectionId, periodStart) {
|
|
16876
|
+
await ensureServiceConnectionsSchema();
|
|
16877
|
+
const normalizedUserId = Number(userId);
|
|
16878
|
+
const db = getDb();
|
|
16879
|
+
await db.batch([
|
|
16880
|
+
{
|
|
16881
|
+
sql: `
|
|
16882
|
+
INSERT OR IGNORE INTO connected_account_charge_transfers (
|
|
16883
|
+
user_id, period_start, from_connection_id, to_connection_id, provider_config_key
|
|
16884
|
+
)
|
|
16885
|
+
SELECT ?, ?, previous.id, replacement.id, replacement.provider_config_key
|
|
16886
|
+
FROM service_connections AS replacement
|
|
16887
|
+
JOIN service_connections AS previous
|
|
16888
|
+
ON previous.user_id = replacement.user_id
|
|
16889
|
+
AND previous.transport = 'nango'
|
|
16890
|
+
AND previous.provider_config_key = replacement.provider_config_key
|
|
16891
|
+
AND previous.lifecycle_status = 'disconnected'
|
|
16892
|
+
JOIN connected_account_charges AS charge
|
|
16893
|
+
ON charge.user_id = previous.user_id
|
|
16894
|
+
AND charge.connection_id = previous.id
|
|
16895
|
+
AND charge.period_start = ?
|
|
16896
|
+
WHERE replacement.id = ?
|
|
16897
|
+
AND replacement.user_id = ?
|
|
16898
|
+
AND replacement.transport = 'nango'
|
|
16899
|
+
AND replacement.lifecycle_status IN ('connected', 'needs_reauth')
|
|
16900
|
+
AND NOT EXISTS (
|
|
16901
|
+
SELECT 1 FROM connected_account_charges AS existing
|
|
16902
|
+
WHERE existing.user_id = replacement.user_id
|
|
16903
|
+
AND existing.connection_id = replacement.id
|
|
16904
|
+
AND existing.period_start = ?
|
|
16905
|
+
)
|
|
16906
|
+
ORDER BY previous.updated_at DESC, previous.id ASC
|
|
16907
|
+
LIMIT 1
|
|
16908
|
+
`,
|
|
16909
|
+
args: [normalizedUserId, periodStart, periodStart, connectionId, normalizedUserId, periodStart]
|
|
16910
|
+
},
|
|
16911
|
+
{
|
|
16912
|
+
sql: `
|
|
16913
|
+
UPDATE connected_account_charges
|
|
16914
|
+
SET connection_id = ?
|
|
16915
|
+
WHERE user_id = ?
|
|
16916
|
+
AND period_start = ?
|
|
16917
|
+
AND connection_id = (
|
|
16918
|
+
SELECT transfer.from_connection_id
|
|
16919
|
+
FROM connected_account_charge_transfers AS transfer
|
|
16920
|
+
JOIN connected_account_charges AS current_charge
|
|
16921
|
+
ON current_charge.user_id = transfer.user_id
|
|
16922
|
+
AND current_charge.period_start = transfer.period_start
|
|
16923
|
+
AND current_charge.connection_id = transfer.from_connection_id
|
|
16924
|
+
WHERE transfer.user_id = ?
|
|
16925
|
+
AND transfer.period_start = ?
|
|
16926
|
+
AND transfer.to_connection_id = ?
|
|
16927
|
+
ORDER BY transfer.transferred_at DESC, transfer.id DESC
|
|
16928
|
+
LIMIT 1
|
|
16929
|
+
)
|
|
16930
|
+
AND NOT EXISTS (
|
|
16931
|
+
SELECT 1 FROM connected_account_charges AS existing
|
|
16932
|
+
WHERE existing.user_id = ?
|
|
16933
|
+
AND existing.connection_id = ?
|
|
16934
|
+
AND existing.period_start = ?
|
|
16935
|
+
)
|
|
16936
|
+
`,
|
|
16937
|
+
args: [
|
|
16938
|
+
connectionId,
|
|
16939
|
+
normalizedUserId,
|
|
16940
|
+
periodStart,
|
|
16941
|
+
normalizedUserId,
|
|
16942
|
+
periodStart,
|
|
16943
|
+
connectionId,
|
|
16944
|
+
normalizedUserId,
|
|
16945
|
+
connectionId,
|
|
16946
|
+
periodStart
|
|
16947
|
+
]
|
|
16948
|
+
}
|
|
16949
|
+
], "write");
|
|
16950
|
+
const inherited = await db.execute({
|
|
16951
|
+
sql: "SELECT 1 FROM connected_account_charges WHERE user_id = ? AND connection_id = ? AND period_start = ? LIMIT 1",
|
|
16952
|
+
args: [normalizedUserId, connectionId, periodStart]
|
|
16953
|
+
});
|
|
16954
|
+
return inherited.rows.length > 0;
|
|
16955
|
+
}
|
|
16524
16956
|
async function getConnectedAccountBillingState(userId) {
|
|
16525
16957
|
const result = await getDb().execute({
|
|
16526
16958
|
sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
|
|
@@ -16661,6 +17093,7 @@ async function reconcileConnectedAccountBilling(user, connections) {
|
|
|
16661
17093
|
const alreadyCharged = await chargedConnectionIds(user.id, periodStart);
|
|
16662
17094
|
const unpaid = billableConnectionIds.filter((id) => !alreadyCharged.has(id));
|
|
16663
17095
|
for (const connectionId of unpaid) {
|
|
17096
|
+
if (await inheritDisconnectedConnectionCharge(user.id, connectionId, periodStart)) continue;
|
|
16664
17097
|
const feeMc = CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC;
|
|
16665
17098
|
const debit = await debitMcIdempotent(
|
|
16666
17099
|
user.id,
|
|
@@ -19752,7 +20185,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
|
|
|
19752
20185
|
import { Hono as Hono9 } from "hono";
|
|
19753
20186
|
|
|
19754
20187
|
// src/api/admin-credits.ts
|
|
19755
|
-
import { randomUUID as
|
|
20188
|
+
import { randomUUID as randomUUID7 } from "crypto";
|
|
19756
20189
|
var MIN_ADJUSTMENT_CREDITS = 1;
|
|
19757
20190
|
var LARGE_ADJUSTMENT_CREDITS = 1e5;
|
|
19758
20191
|
var MAX_ADJUSTMENT_CREDITS = 1e7;
|
|
@@ -19902,7 +20335,7 @@ async function applyAdjustment(input) {
|
|
|
19902
20335
|
const credits = validateCredits(input.credits, input.confirmLarge === true);
|
|
19903
20336
|
const reason = validateReason(input.reason);
|
|
19904
20337
|
const actor = validateActor(input.actor);
|
|
19905
|
-
const reference = input.reference?.trim() || `adj_${
|
|
20338
|
+
const reference = input.reference?.trim() || `adj_${randomUUID7()}`;
|
|
19906
20339
|
if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
|
|
19907
20340
|
const db = getDb();
|
|
19908
20341
|
const existing = await db.execute({
|
|
@@ -22509,17 +22942,17 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
22509
22942
|
});
|
|
22510
22943
|
|
|
22511
22944
|
// src/api/directory-routes.ts
|
|
22512
|
-
import { createHash as createHash5, randomUUID as
|
|
22945
|
+
import { createHash as createHash5, randomUUID as randomUUID9 } from "crypto";
|
|
22513
22946
|
import { Hono as Hono16 } from "hono";
|
|
22514
22947
|
|
|
22515
22948
|
// src/api/directory-workflow-dispatch.ts
|
|
22516
|
-
import { randomUUID as
|
|
22949
|
+
import { randomUUID as randomUUID8 } from "crypto";
|
|
22517
22950
|
function retryDelaySeconds(attempts) {
|
|
22518
22951
|
return Math.min(60 * 60, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
22519
22952
|
}
|
|
22520
22953
|
async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
22521
22954
|
const rows = await claimDirectoryWorkflowOutbox({
|
|
22522
|
-
workerId: `directory-dispatch-${process.pid}-${
|
|
22955
|
+
workerId: `directory-dispatch-${process.pid}-${randomUUID8().slice(0, 8)}`,
|
|
22523
22956
|
limit
|
|
22524
22957
|
});
|
|
22525
22958
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
@@ -22558,14 +22991,14 @@ function idempotencyKey(raw) {
|
|
|
22558
22991
|
if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
|
|
22559
22992
|
return { ok: true, key: trimmed };
|
|
22560
22993
|
}
|
|
22561
|
-
return { ok: true, key: `directory-${
|
|
22994
|
+
return { ok: true, key: `directory-${randomUUID9()}` };
|
|
22562
22995
|
}
|
|
22563
22996
|
function debitKeyFor(userId, responseKey) {
|
|
22564
22997
|
const digest2 = createHash5("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
22565
22998
|
return `directory-workflow:${userId}:${digest2}`;
|
|
22566
22999
|
}
|
|
22567
23000
|
function jobId() {
|
|
22568
|
-
return `dir_${
|
|
23001
|
+
return `dir_${randomUUID9().replace(/-/g, "")}`;
|
|
22569
23002
|
}
|
|
22570
23003
|
function publicStatus(job, result) {
|
|
22571
23004
|
if (job.status === "failed") return "failed";
|
|
@@ -22879,8 +23312,8 @@ import { homedir as homedir3 } from "os";
|
|
|
22879
23312
|
import { join as join7 } from "path";
|
|
22880
23313
|
|
|
22881
23314
|
// src/api/lead-list-import-repository.ts
|
|
22882
|
-
import { randomUUID as
|
|
22883
|
-
var
|
|
23315
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
23316
|
+
var schemaDb4 = null;
|
|
22884
23317
|
var schemaPromise3 = null;
|
|
22885
23318
|
function parseJson3(value, fallback) {
|
|
22886
23319
|
if (value == null) return fallback;
|
|
@@ -22945,15 +23378,15 @@ function bounded(value, field, min, max) {
|
|
|
22945
23378
|
return normalized;
|
|
22946
23379
|
}
|
|
22947
23380
|
function newLeadListUploadId() {
|
|
22948
|
-
return `upl_${
|
|
23381
|
+
return `upl_${randomUUID10().replace(/-/g, "")}`;
|
|
22949
23382
|
}
|
|
22950
23383
|
function newImportedLeadListId() {
|
|
22951
|
-
return `lst_${
|
|
23384
|
+
return `lst_${randomUUID10().replace(/-/g, "")}`;
|
|
22952
23385
|
}
|
|
22953
23386
|
async function ensureLeadListImportRepositorySchema() {
|
|
22954
23387
|
const db = getDb();
|
|
22955
|
-
if (
|
|
22956
|
-
|
|
23388
|
+
if (schemaDb4 !== db) {
|
|
23389
|
+
schemaDb4 = db;
|
|
22957
23390
|
schemaPromise3 = null;
|
|
22958
23391
|
}
|
|
22959
23392
|
if (!schemaPromise3) {
|
|
@@ -24090,17 +24523,17 @@ leadListInputApp.post("/imports", async (c) => {
|
|
|
24090
24523
|
});
|
|
24091
24524
|
|
|
24092
24525
|
// src/api/lead-list-enrichment-routes.ts
|
|
24093
|
-
import { createHash as createHash8, randomUUID as
|
|
24526
|
+
import { createHash as createHash8, randomUUID as randomUUID12 } from "crypto";
|
|
24094
24527
|
import { Hono as Hono18 } from "hono";
|
|
24095
24528
|
import { z as z25 } from "zod";
|
|
24096
24529
|
|
|
24097
24530
|
// src/api/lead-list-enrichment-dispatch.ts
|
|
24098
|
-
import { randomUUID as
|
|
24531
|
+
import { randomUUID as randomUUID11 } from "crypto";
|
|
24099
24532
|
function retryDelaySeconds2(attempts) {
|
|
24100
24533
|
return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
24101
24534
|
}
|
|
24102
24535
|
async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
24103
|
-
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${
|
|
24536
|
+
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID11().slice(0, 8)}`, limit });
|
|
24104
24537
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
24105
24538
|
for (const row of rows) {
|
|
24106
24539
|
if (!row.claimToken) continue;
|
|
@@ -24148,7 +24581,7 @@ function idempotencyKey2(raw) {
|
|
|
24148
24581
|
return { ok: true, value };
|
|
24149
24582
|
}
|
|
24150
24583
|
function newJobId() {
|
|
24151
|
-
return `lle_${
|
|
24584
|
+
return `lle_${randomUUID12().replace(/-/g, "")}`;
|
|
24152
24585
|
}
|
|
24153
24586
|
function debitKeyFor2(userId, key) {
|
|
24154
24587
|
const digest2 = createHash8("sha256").update(String(userId)).update("\0").update(key).digest("hex");
|
|
@@ -26158,7 +26591,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
26158
26591
|
|
|
26159
26592
|
// src/api/serp-intelligence-routes.ts
|
|
26160
26593
|
import { Hono as Hono22 } from "hono";
|
|
26161
|
-
import { createHash as createHash10, randomUUID as
|
|
26594
|
+
import { createHash as createHash10, randomUUID as randomUUID13 } from "crypto";
|
|
26162
26595
|
|
|
26163
26596
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
26164
26597
|
import { createHash as createHash9 } from "crypto";
|
|
@@ -26944,7 +27377,7 @@ async function logRequestEventBestEffort(input) {
|
|
|
26944
27377
|
}
|
|
26945
27378
|
}
|
|
26946
27379
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
26947
|
-
const responseKey = suppliedKey?.trim() ||
|
|
27380
|
+
const responseKey = suppliedKey?.trim() || randomUUID13();
|
|
26948
27381
|
const requestFingerprint3 = createHash10("sha256").update(JSON.stringify(body)).digest("hex");
|
|
26949
27382
|
const keyDigest = createHash10("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
26950
27383
|
return {
|
|
@@ -27274,7 +27707,7 @@ var MemoryMcpToolExecutor = class {
|
|
|
27274
27707
|
};
|
|
27275
27708
|
|
|
27276
27709
|
// src/api/image-source-artifacts.ts
|
|
27277
|
-
import { randomUUID as
|
|
27710
|
+
import { randomUUID as randomUUID14 } from "crypto";
|
|
27278
27711
|
var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
27279
27712
|
var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
27280
27713
|
var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
@@ -27305,7 +27738,7 @@ async function createImageSourceArtifact(args) {
|
|
|
27305
27738
|
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
27306
27739
|
throw new Error("image_source_size_invalid");
|
|
27307
27740
|
}
|
|
27308
|
-
const id =
|
|
27741
|
+
const id = randomUUID14().replaceAll("-", "");
|
|
27309
27742
|
return createPrivateArtifact({
|
|
27310
27743
|
policy: policy3(),
|
|
27311
27744
|
ownerId: args.ownerId,
|
|
@@ -28050,11 +28483,11 @@ mcpApp.all("/", async (c) => {
|
|
|
28050
28483
|
});
|
|
28051
28484
|
|
|
28052
28485
|
// src/api/browser-agent-routes.ts
|
|
28053
|
-
import { randomUUID as
|
|
28486
|
+
import { randomUUID as randomUUID18 } from "crypto";
|
|
28054
28487
|
import { Hono as Hono24 } from "hono";
|
|
28055
28488
|
|
|
28056
28489
|
// src/api/browser-agent-db.ts
|
|
28057
|
-
import { randomUUID as
|
|
28490
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
28058
28491
|
var _ready = false;
|
|
28059
28492
|
var _migrationPromise = null;
|
|
28060
28493
|
async function migrateBrowserAgent() {
|
|
@@ -28185,7 +28618,7 @@ async function runBrowserAgentMigration() {
|
|
|
28185
28618
|
}
|
|
28186
28619
|
async function createExtensionRow(input) {
|
|
28187
28620
|
const db = getDb();
|
|
28188
|
-
const id = `bext_${
|
|
28621
|
+
const id = `bext_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
|
|
28189
28622
|
await db.execute({
|
|
28190
28623
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
28191
28624
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -28220,7 +28653,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
28220
28653
|
}
|
|
28221
28654
|
async function createAuthConnectionRow(input) {
|
|
28222
28655
|
const db = getDb();
|
|
28223
|
-
const connectionId = `authc_${
|
|
28656
|
+
const connectionId = `authc_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
|
|
28224
28657
|
await db.execute({
|
|
28225
28658
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
28226
28659
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -28305,7 +28738,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
28305
28738
|
}
|
|
28306
28739
|
async function createSessionRow(input) {
|
|
28307
28740
|
const db = getDb();
|
|
28308
|
-
const id = `bas_${
|
|
28741
|
+
const id = `bas_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
|
|
28309
28742
|
await db.execute({
|
|
28310
28743
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
28311
28744
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -28383,7 +28816,7 @@ async function recordAction(input) {
|
|
|
28383
28816
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
28384
28817
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
28385
28818
|
args: [
|
|
28386
|
-
`baa_${
|
|
28819
|
+
`baa_${randomUUID15().replace(/-/g, "").slice(0, 20)}`,
|
|
28387
28820
|
input.sessionId,
|
|
28388
28821
|
input.type,
|
|
28389
28822
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -29494,9 +29927,9 @@ async function replayList(runtimeSessionId) {
|
|
|
29494
29927
|
}
|
|
29495
29928
|
|
|
29496
29929
|
// src/api/serp-identity-db.ts
|
|
29497
|
-
import { randomUUID as
|
|
29930
|
+
import { randomUUID as randomUUID16 } from "crypto";
|
|
29498
29931
|
async function createSerpIdentityRow(input) {
|
|
29499
|
-
const id = `serpi_${
|
|
29932
|
+
const id = `serpi_${randomUUID16().replace(/-/g, "").slice(0, 20)}`;
|
|
29500
29933
|
await getDb().execute({
|
|
29501
29934
|
sql: `INSERT INTO serp_identities
|
|
29502
29935
|
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
@@ -29543,7 +29976,7 @@ async function deleteSerpIdentityRow(userId, name) {
|
|
|
29543
29976
|
}
|
|
29544
29977
|
|
|
29545
29978
|
// src/api/serp-identity-service.ts
|
|
29546
|
-
import { createHash as createHash11, randomUUID as
|
|
29979
|
+
import { createHash as createHash11, randomUUID as randomUUID17 } from "crypto";
|
|
29547
29980
|
import Kernel5 from "@onkernel/sdk";
|
|
29548
29981
|
var MAX_SERP_IDENTITIES_PER_USER = 3;
|
|
29549
29982
|
function kernelClient() {
|
|
@@ -29553,7 +29986,7 @@ function kernelClient() {
|
|
|
29553
29986
|
}
|
|
29554
29987
|
function backendName(userId, name, resource) {
|
|
29555
29988
|
const digest2 = createHash11("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
|
|
29556
|
-
const nonce =
|
|
29989
|
+
const nonce = randomUUID17().replace(/-/g, "").slice(0, 8);
|
|
29557
29990
|
return `mcp-serp-${resource}-${digest2}-${nonce}`;
|
|
29558
29991
|
}
|
|
29559
29992
|
function isNotFound2(error) {
|
|
@@ -30339,7 +30772,7 @@ function buildBrowserAgentRoutes() {
|
|
|
30339
30772
|
}
|
|
30340
30773
|
const existing = await getExtensionRow(user.id, name);
|
|
30341
30774
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
30342
|
-
const backendName2 = `u${user.id}_${
|
|
30775
|
+
const backendName2 = `u${user.id}_${randomUUID18().replace(/-/g, "")}`;
|
|
30343
30776
|
try {
|
|
30344
30777
|
const imported = await importExtensionFromStore(storeUrl, backendName2);
|
|
30345
30778
|
const row = await createExtensionRow({
|
|
@@ -30892,7 +31325,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
30892
31325
|
// src/api/oauth-routes.ts
|
|
30893
31326
|
import { Hono as Hono26 } from "hono";
|
|
30894
31327
|
import { getCookie, setCookie } from "hono/cookie";
|
|
30895
|
-
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as
|
|
31328
|
+
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID19 } from "crypto";
|
|
30896
31329
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
30897
31330
|
var ISSUER = "https://mcpscraper.dev";
|
|
30898
31331
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -31277,7 +31710,7 @@ function pkceMatches(verifier, challenge) {
|
|
|
31277
31710
|
}
|
|
31278
31711
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
31279
31712
|
const { privateKey, kid } = await getKeys();
|
|
31280
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
31713
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID19()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
31281
31714
|
}
|
|
31282
31715
|
function tokenErrorResponse(c, error, description, status) {
|
|
31283
31716
|
return c.json({ error, error_description: description }, status);
|
|
@@ -33727,7 +34160,7 @@ function renderWorkshopFieldNotesBody() {
|
|
|
33727
34160
|
}
|
|
33728
34161
|
|
|
33729
34162
|
// src/api/workshop-registration-repository.ts
|
|
33730
|
-
import { randomUUID as
|
|
34163
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
33731
34164
|
function rowToRegistration(row) {
|
|
33732
34165
|
return {
|
|
33733
34166
|
id: String(row.id),
|
|
@@ -33796,7 +34229,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
33796
34229
|
) < ?
|
|
33797
34230
|
`,
|
|
33798
34231
|
args: [
|
|
33799
|
-
|
|
34232
|
+
randomUUID20(),
|
|
33800
34233
|
input.eventSlug,
|
|
33801
34234
|
input.email,
|
|
33802
34235
|
input.firstName,
|
|
@@ -33813,7 +34246,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
33813
34246
|
id, event_slug, email, first_name, last_name, status
|
|
33814
34247
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
33815
34248
|
`,
|
|
33816
|
-
args: [
|
|
34249
|
+
args: [randomUUID20(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
33817
34250
|
});
|
|
33818
34251
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
33819
34252
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -33828,7 +34261,7 @@ async function registerWorkshopInterest(input) {
|
|
|
33828
34261
|
id, event_slug, email, first_name, last_name, status
|
|
33829
34262
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
33830
34263
|
`,
|
|
33831
|
-
args: [
|
|
34264
|
+
args: [randomUUID20(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
33832
34265
|
});
|
|
33833
34266
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
33834
34267
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -34631,7 +35064,7 @@ import { Hono as Hono33 } from "hono";
|
|
|
34631
35064
|
import { z as z33 } from "zod";
|
|
34632
35065
|
|
|
34633
35066
|
// src/api/commons-repository.ts
|
|
34634
|
-
import { createHash as createHash17, randomUUID as
|
|
35067
|
+
import { createHash as createHash17, randomUUID as randomUUID21 } from "crypto";
|
|
34635
35068
|
|
|
34636
35069
|
// src/api/commons-image-store.ts
|
|
34637
35070
|
import { createHash as createHash15 } from "crypto";
|
|
@@ -34640,20 +35073,29 @@ import { isIP } from "net";
|
|
|
34640
35073
|
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
34641
35074
|
var COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
34642
35075
|
var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
35076
|
+
var COMMONS_IMAGE_PAGE_MAX_BYTES = 2 * 1024 * 1024;
|
|
35077
|
+
var COMMONS_IMAGE_MAX_PAGE_CANDIDATES = 12;
|
|
35078
|
+
var COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with imageBase64, an owner-authorized artifactId, a public HTTPS image URL, or a public HTTPS webpage containing a usable featured image. Replace featuredImage.url with the returned permanent URL, validate again, then retry commons_submit_entity with the same idempotency key.";
|
|
34643
35079
|
var CommonsImageError = class extends Error {
|
|
34644
|
-
constructor(code, message, httpStatus = 400) {
|
|
35080
|
+
constructor(code, message, httpStatus = 400, retryable = false, retryGuidance = COMMONS_IMAGE_RETRY_GUIDANCE) {
|
|
34645
35081
|
super(message);
|
|
34646
35082
|
this.code = code;
|
|
34647
35083
|
this.httpStatus = httpStatus;
|
|
35084
|
+
this.retryable = retryable;
|
|
35085
|
+
this.retryGuidance = retryGuidance;
|
|
34648
35086
|
this.name = "CommonsImageError";
|
|
34649
35087
|
}
|
|
34650
35088
|
code;
|
|
34651
35089
|
httpStatus;
|
|
35090
|
+
retryable;
|
|
35091
|
+
retryGuidance;
|
|
34652
35092
|
};
|
|
34653
35093
|
var imageSchemaReady = false;
|
|
35094
|
+
var imageSchemaDb = null;
|
|
34654
35095
|
async function ensureCommonsImageSchema() {
|
|
34655
|
-
|
|
34656
|
-
|
|
35096
|
+
const db = getDb();
|
|
35097
|
+
if (imageSchemaReady && imageSchemaDb === db) return;
|
|
35098
|
+
await db.execute(`
|
|
34657
35099
|
CREATE TABLE IF NOT EXISTS commons_images (
|
|
34658
35100
|
id TEXT PRIMARY KEY,
|
|
34659
35101
|
user_id INTEGER NOT NULL,
|
|
@@ -34669,8 +35111,9 @@ async function ensureCommonsImageSchema() {
|
|
|
34669
35111
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
34670
35112
|
)
|
|
34671
35113
|
`);
|
|
34672
|
-
await
|
|
35114
|
+
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
|
|
34673
35115
|
imageSchemaReady = true;
|
|
35116
|
+
imageSchemaDb = db;
|
|
34674
35117
|
}
|
|
34675
35118
|
function sniffImageFormat(bytes) {
|
|
34676
35119
|
if (bytes.length < 12) return null;
|
|
@@ -34700,44 +35143,85 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
34700
35143
|
try {
|
|
34701
35144
|
url = new URL(rawUrl);
|
|
34702
35145
|
} catch {
|
|
34703
|
-
throw new CommonsImageError("image_url_invalid",
|
|
35146
|
+
throw new CommonsImageError("image_url_invalid", "The image source is not a valid URL.");
|
|
34704
35147
|
}
|
|
34705
35148
|
if (url.protocol !== "https:") {
|
|
34706
35149
|
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
34707
35150
|
}
|
|
35151
|
+
if (url.username || url.password) {
|
|
35152
|
+
throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
|
|
35153
|
+
}
|
|
34708
35154
|
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
34709
35155
|
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
34710
35156
|
if (!addresses.length) {
|
|
34711
|
-
throw new CommonsImageError("image_host_unresolvable",
|
|
35157
|
+
throw new CommonsImageError("image_host_unresolvable", "The image source host could not be resolved.");
|
|
34712
35158
|
}
|
|
34713
35159
|
if (addresses.some((entry) => isBlockedAddress(entry.address))) {
|
|
34714
35160
|
throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
|
|
34715
35161
|
}
|
|
34716
35162
|
return url;
|
|
34717
35163
|
}
|
|
34718
|
-
async function
|
|
35164
|
+
async function readResponseBytes(response, maxBytes) {
|
|
35165
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
35166
|
+
if (declaredLength > maxBytes) {
|
|
35167
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
35168
|
+
}
|
|
35169
|
+
if (!response.body) return Buffer.alloc(0);
|
|
35170
|
+
const reader = response.body.getReader();
|
|
35171
|
+
const chunks = [];
|
|
35172
|
+
let total = 0;
|
|
35173
|
+
while (true) {
|
|
35174
|
+
const { done, value } = await reader.read();
|
|
35175
|
+
if (done) break;
|
|
35176
|
+
total += value.byteLength;
|
|
35177
|
+
if (total > maxBytes) {
|
|
35178
|
+
await reader.cancel().catch(() => void 0);
|
|
35179
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
35180
|
+
}
|
|
35181
|
+
chunks.push(Buffer.from(value));
|
|
35182
|
+
}
|
|
35183
|
+
return Buffer.concat(chunks, total);
|
|
35184
|
+
}
|
|
35185
|
+
async function downloadPublicSource(sourceUrl) {
|
|
34719
35186
|
let current = sourceUrl;
|
|
34720
35187
|
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
34721
35188
|
const url = await assertPublicHttpsUrl(current);
|
|
34722
|
-
|
|
35189
|
+
let response;
|
|
35190
|
+
try {
|
|
35191
|
+
response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
|
|
35192
|
+
} catch (error) {
|
|
35193
|
+
if (error instanceof CommonsImageError) throw error;
|
|
35194
|
+
throw new CommonsImageError(
|
|
35195
|
+
"image_source_unavailable",
|
|
35196
|
+
"The image source could not be reached by Commons.",
|
|
35197
|
+
422,
|
|
35198
|
+
true
|
|
35199
|
+
);
|
|
35200
|
+
}
|
|
34723
35201
|
if (response.status >= 300 && response.status < 400) {
|
|
34724
35202
|
const location2 = response.headers.get("location");
|
|
34725
35203
|
if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
|
|
34726
|
-
|
|
35204
|
+
try {
|
|
35205
|
+
current = new URL(location2, url).toString();
|
|
35206
|
+
} catch {
|
|
35207
|
+
throw new CommonsImageError("image_redirect_invalid", "Image source redirected to an invalid destination.");
|
|
35208
|
+
}
|
|
34727
35209
|
continue;
|
|
34728
35210
|
}
|
|
34729
35211
|
if (!response.ok) {
|
|
34730
|
-
|
|
34731
|
-
|
|
34732
|
-
|
|
34733
|
-
|
|
34734
|
-
|
|
34735
|
-
|
|
34736
|
-
|
|
34737
|
-
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
34738
|
-
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
35212
|
+
const retryable = response.status === 408 || response.status === 425 || response.status === 429 || response.status >= 500;
|
|
35213
|
+
throw new CommonsImageError(
|
|
35214
|
+
"image_source_unavailable",
|
|
35215
|
+
`The image source returned HTTP ${response.status} and could not be hosted.`,
|
|
35216
|
+
422,
|
|
35217
|
+
retryable
|
|
35218
|
+
);
|
|
34739
35219
|
}
|
|
34740
|
-
return
|
|
35220
|
+
return {
|
|
35221
|
+
bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
|
|
35222
|
+
finalUrl: url.toString(),
|
|
35223
|
+
contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
|
|
35224
|
+
};
|
|
34741
35225
|
}
|
|
34742
35226
|
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
34743
35227
|
}
|
|
@@ -34750,21 +35234,161 @@ function decodeBase64Image(value) {
|
|
|
34750
35234
|
}
|
|
34751
35235
|
return bytes;
|
|
34752
35236
|
}
|
|
35237
|
+
function safeProvenanceUrl(rawUrl) {
|
|
35238
|
+
try {
|
|
35239
|
+
const url = new URL(rawUrl);
|
|
35240
|
+
url.username = "";
|
|
35241
|
+
url.password = "";
|
|
35242
|
+
url.search = "";
|
|
35243
|
+
url.hash = "";
|
|
35244
|
+
return url.toString();
|
|
35245
|
+
} catch {
|
|
35246
|
+
return null;
|
|
35247
|
+
}
|
|
35248
|
+
}
|
|
35249
|
+
function looksLikeHtml(bytes, declaredType) {
|
|
35250
|
+
if (declaredType === "text/html" || declaredType === "application/xhtml+xml") return true;
|
|
35251
|
+
const prefix = bytes.subarray(0, 512).toString("utf8").trimStart().toLowerCase();
|
|
35252
|
+
return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
|
|
35253
|
+
}
|
|
35254
|
+
async function resolveCommonsImageSource(input) {
|
|
35255
|
+
const supplied = [input.sourceUrl, input.imageBase64, input.artifactId].filter((value) => Boolean(value)).length;
|
|
35256
|
+
if (supplied === 0) {
|
|
35257
|
+
throw new CommonsImageError("image_source_missing", "Provide exactly one of sourceUrl, imageBase64, or artifactId.");
|
|
35258
|
+
}
|
|
35259
|
+
if (supplied !== 1) {
|
|
35260
|
+
throw new CommonsImageError("image_source_ambiguous", "Provide only one image source: sourceUrl, imageBase64, or artifactId.");
|
|
35261
|
+
}
|
|
35262
|
+
if (input.sourceUrl && input.sourceType) {
|
|
35263
|
+
throw new CommonsImageError("image_source_type_invalid", "sourceType applies only to imageBase64 or artifactId, not sourceUrl.");
|
|
35264
|
+
}
|
|
35265
|
+
if (input.imageBase64) {
|
|
35266
|
+
return {
|
|
35267
|
+
bytes: decodeBase64Image(input.imageBase64),
|
|
35268
|
+
sourceUrl: null,
|
|
35269
|
+
sourcePageUrl: null,
|
|
35270
|
+
sourceKind: input.sourceType ?? "uploaded",
|
|
35271
|
+
selectionSource: "provided_bytes"
|
|
35272
|
+
};
|
|
35273
|
+
}
|
|
35274
|
+
if (input.artifactId) {
|
|
35275
|
+
const bytes = await readOwnedImageSourceArtifact({ artifactId: input.artifactId, ownerId: String(input.userId) });
|
|
35276
|
+
if (!bytes) {
|
|
35277
|
+
throw new CommonsImageError("image_artifact_not_found", "The image artifact was not found, expired, or is not owned by this account.", 422);
|
|
35278
|
+
}
|
|
35279
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
35280
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
35281
|
+
}
|
|
35282
|
+
return {
|
|
35283
|
+
bytes,
|
|
35284
|
+
sourceUrl: null,
|
|
35285
|
+
sourcePageUrl: null,
|
|
35286
|
+
sourceKind: input.sourceType ?? "artifact",
|
|
35287
|
+
selectionSource: "artifact"
|
|
35288
|
+
};
|
|
35289
|
+
}
|
|
35290
|
+
const requestedUrl = String(input.sourceUrl);
|
|
35291
|
+
const downloaded = await downloadPublicSource(requestedUrl);
|
|
35292
|
+
if (sniffImageFormat(downloaded.bytes)) {
|
|
35293
|
+
return {
|
|
35294
|
+
bytes: downloaded.bytes,
|
|
35295
|
+
sourceUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
35296
|
+
sourcePageUrl: null,
|
|
35297
|
+
sourceKind: "direct_url",
|
|
35298
|
+
selectionSource: "direct_url"
|
|
35299
|
+
};
|
|
35300
|
+
}
|
|
35301
|
+
if (!looksLikeHtml(downloaded.bytes, downloaded.contentType)) {
|
|
35302
|
+
throw new CommonsImageError(
|
|
35303
|
+
"image_format_unsupported",
|
|
35304
|
+
"The supplied URL returned neither a supported image nor an HTML webpage with a discoverable image.",
|
|
35305
|
+
415
|
|
35306
|
+
);
|
|
35307
|
+
}
|
|
35308
|
+
if (downloaded.bytes.length > COMMONS_IMAGE_PAGE_MAX_BYTES) {
|
|
35309
|
+
throw new CommonsImageError("image_page_too_large", `Webpage exceeds the ${COMMONS_IMAGE_PAGE_MAX_BYTES} byte discovery limit.`, 413);
|
|
35310
|
+
}
|
|
35311
|
+
const html = downloaded.bytes.toString("utf8");
|
|
35312
|
+
const document2 = loadHtmlDocument(html);
|
|
35313
|
+
const candidates = resolveFeaturedImageCandidates({
|
|
35314
|
+
html,
|
|
35315
|
+
meta: document2.meta,
|
|
35316
|
+
schema: document2.schema,
|
|
35317
|
+
pageUrl: downloaded.finalUrl
|
|
35318
|
+
}).slice(0, COMMONS_IMAGE_MAX_PAGE_CANDIDATES);
|
|
35319
|
+
if (!candidates.length) {
|
|
35320
|
+
throw new CommonsImageError("page_image_not_found", "The webpage did not expose a usable featured or content image.", 422);
|
|
35321
|
+
}
|
|
35322
|
+
let sawRetryableFailure = false;
|
|
35323
|
+
for (const candidate of candidates) {
|
|
35324
|
+
try {
|
|
35325
|
+
const image = await downloadPublicSource(candidate.url);
|
|
35326
|
+
if (!sniffImageFormat(image.bytes)) continue;
|
|
35327
|
+
return {
|
|
35328
|
+
bytes: image.bytes,
|
|
35329
|
+
sourceUrl: safeProvenanceUrl(image.finalUrl),
|
|
35330
|
+
sourcePageUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
35331
|
+
sourceKind: "webpage",
|
|
35332
|
+
selectionSource: candidate.source
|
|
35333
|
+
};
|
|
35334
|
+
} catch (error) {
|
|
35335
|
+
if (error instanceof CommonsImageError && error.retryable) sawRetryableFailure = true;
|
|
35336
|
+
}
|
|
35337
|
+
}
|
|
35338
|
+
throw new CommonsImageError(
|
|
35339
|
+
"page_image_candidates_unusable",
|
|
35340
|
+
"The webpage exposed image candidates, but none returned supported public image bytes.",
|
|
35341
|
+
422,
|
|
35342
|
+
sawRetryableFailure
|
|
35343
|
+
);
|
|
35344
|
+
}
|
|
34753
35345
|
function commonsImageBlobStore() {
|
|
34754
35346
|
const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
|
|
34755
35347
|
return token6 ? new VercelBlobStore(token6) : getBlobStore();
|
|
34756
35348
|
}
|
|
34757
|
-
function
|
|
34758
|
-
|
|
35349
|
+
function normalizeCommonsImageUrl(rawUrl) {
|
|
35350
|
+
try {
|
|
35351
|
+
const url = new URL(rawUrl.trim());
|
|
35352
|
+
if (url.protocol !== "https:") return null;
|
|
35353
|
+
url.hash = "";
|
|
35354
|
+
return url.toString();
|
|
35355
|
+
} catch {
|
|
35356
|
+
return null;
|
|
35357
|
+
}
|
|
35358
|
+
}
|
|
35359
|
+
async function findCommonsHostedImageByUrl(rawUrl) {
|
|
35360
|
+
const url = normalizeCommonsImageUrl(rawUrl);
|
|
35361
|
+
if (!url) return null;
|
|
35362
|
+
await ensureCommonsImageSchema();
|
|
35363
|
+
const result = await getDb().execute({
|
|
35364
|
+
sql: "SELECT id, url, content_type, bytes, digest, source_url FROM commons_images WHERE url = ? LIMIT 1",
|
|
35365
|
+
args: [url]
|
|
35366
|
+
});
|
|
35367
|
+
const row = result.rows[0];
|
|
35368
|
+
if (!row) return null;
|
|
35369
|
+
return {
|
|
35370
|
+
id: String(row.id),
|
|
35371
|
+
url: String(row.url),
|
|
35372
|
+
contentType: String(row.content_type),
|
|
35373
|
+
bytes: Number(row.bytes),
|
|
35374
|
+
digest: String(row.digest),
|
|
35375
|
+
sourceUrl: row.source_url ? String(row.source_url) : null,
|
|
35376
|
+
sourcePageUrl: null,
|
|
35377
|
+
sourceKind: "registered",
|
|
35378
|
+
selectionSource: "registered",
|
|
35379
|
+
reused: true
|
|
35380
|
+
};
|
|
34759
35381
|
}
|
|
34760
35382
|
var hostOverride = null;
|
|
34761
35383
|
async function hostCommonsImage(input) {
|
|
34762
35384
|
if (hostOverride) return hostOverride(input);
|
|
34763
35385
|
await ensureCommonsImageSchema();
|
|
34764
|
-
if (
|
|
34765
|
-
|
|
35386
|
+
if (input.sourceUrl && !input.imageBase64 && !input.artifactId && !input.sourceType) {
|
|
35387
|
+
const registered6 = await findCommonsHostedImageByUrl(input.sourceUrl);
|
|
35388
|
+
if (registered6) return registered6;
|
|
34766
35389
|
}
|
|
34767
|
-
const
|
|
35390
|
+
const resolved = await resolveCommonsImageSource(input);
|
|
35391
|
+
const bytes = resolved.bytes;
|
|
34768
35392
|
const contentType = sniffImageFormat(bytes);
|
|
34769
35393
|
if (!contentType) {
|
|
34770
35394
|
throw new CommonsImageError(
|
|
@@ -34786,7 +35410,10 @@ async function hostCommonsImage(input) {
|
|
|
34786
35410
|
contentType: String(reusable.content_type),
|
|
34787
35411
|
bytes: Number(reusable.bytes),
|
|
34788
35412
|
digest: digest2,
|
|
34789
|
-
sourceUrl:
|
|
35413
|
+
sourceUrl: resolved.sourceUrl,
|
|
35414
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
35415
|
+
sourceKind: resolved.sourceKind,
|
|
35416
|
+
selectionSource: resolved.selectionSource,
|
|
34790
35417
|
reused: true
|
|
34791
35418
|
};
|
|
34792
35419
|
}
|
|
@@ -34815,19 +35442,31 @@ async function hostCommonsImage(input) {
|
|
|
34815
35442
|
stored.url,
|
|
34816
35443
|
contentType,
|
|
34817
35444
|
bytes.length,
|
|
34818
|
-
|
|
35445
|
+
resolved.sourceUrl,
|
|
34819
35446
|
input.alt ?? null,
|
|
34820
35447
|
input.license ?? null,
|
|
34821
35448
|
input.attribution ?? null,
|
|
34822
35449
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
34823
35450
|
]
|
|
34824
35451
|
});
|
|
34825
|
-
return {
|
|
35452
|
+
return {
|
|
35453
|
+
id,
|
|
35454
|
+
url: stored.url,
|
|
35455
|
+
contentType,
|
|
35456
|
+
bytes: bytes.length,
|
|
35457
|
+
digest: digest2,
|
|
35458
|
+
sourceUrl: resolved.sourceUrl,
|
|
35459
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
35460
|
+
sourceKind: resolved.sourceKind,
|
|
35461
|
+
selectionSource: resolved.selectionSource,
|
|
35462
|
+
reused: false
|
|
35463
|
+
};
|
|
34826
35464
|
}
|
|
34827
35465
|
async function hostEntityImages(input) {
|
|
34828
35466
|
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
34829
35467
|
const featuredUrl = input.featuredImage?.url;
|
|
34830
|
-
|
|
35468
|
+
const registeredFeatured = featuredUrl ? await findCommonsHostedImageByUrl(featuredUrl) : null;
|
|
35469
|
+
if (featuredUrl && !registeredFeatured) {
|
|
34831
35470
|
const hosted = await hostCommonsImage({
|
|
34832
35471
|
userId: input.userId,
|
|
34833
35472
|
sourceUrl: featuredUrl,
|
|
@@ -34835,17 +35474,19 @@ async function hostEntityImages(input) {
|
|
|
34835
35474
|
license: input.featuredImage?.license,
|
|
34836
35475
|
attribution: input.featuredImage?.sourceUrl
|
|
34837
35476
|
}).catch((error) => {
|
|
34838
|
-
|
|
35477
|
+
if (error instanceof CommonsImageError) throw error;
|
|
34839
35478
|
throw new CommonsImageError(
|
|
34840
|
-
|
|
34841
|
-
|
|
35479
|
+
"featured_image_unhostable",
|
|
35480
|
+
"The featured image could not be hosted, so the entity was not persisted with a link that could break.",
|
|
34842
35481
|
422
|
|
34843
35482
|
);
|
|
34844
35483
|
});
|
|
34845
35484
|
rewrite.featuredImageUrl = hosted.url;
|
|
35485
|
+
rewrite.mediaUrls.set(featuredUrl, hosted.url);
|
|
34846
35486
|
}
|
|
34847
35487
|
for (const item of input.media ?? []) {
|
|
34848
|
-
if (
|
|
35488
|
+
if (item.url && rewrite.mediaUrls.has(item.url)) continue;
|
|
35489
|
+
if (!item.url || await findCommonsHostedImageByUrl(item.url)) continue;
|
|
34849
35490
|
try {
|
|
34850
35491
|
const hosted = await hostCommonsImage({
|
|
34851
35492
|
userId: input.userId,
|
|
@@ -35213,7 +35854,7 @@ var COMMONS_ENTITY_PROFILES = {
|
|
|
35213
35854
|
]
|
|
35214
35855
|
}
|
|
35215
35856
|
};
|
|
35216
|
-
var
|
|
35857
|
+
var schemaReady2 = false;
|
|
35217
35858
|
var COMMONS_SCHEMA_OBJECTS = [
|
|
35218
35859
|
"schema_migrations",
|
|
35219
35860
|
"commons_entities",
|
|
@@ -35240,9 +35881,9 @@ var COMMONS_SCHEMA_OBJECTS = [
|
|
|
35240
35881
|
"commons_publication_editions_publication_published"
|
|
35241
35882
|
];
|
|
35242
35883
|
async function ensureCommonsSchema() {
|
|
35243
|
-
if (
|
|
35884
|
+
if (schemaReady2) return;
|
|
35244
35885
|
if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
|
|
35245
|
-
|
|
35886
|
+
schemaReady2 = true;
|
|
35246
35887
|
return;
|
|
35247
35888
|
}
|
|
35248
35889
|
const db = getDb();
|
|
@@ -35465,7 +36106,7 @@ async function ensureCommonsSchema() {
|
|
|
35465
36106
|
args: [COMMONS_SCHEMA_VERSION]
|
|
35466
36107
|
}
|
|
35467
36108
|
], "write");
|
|
35468
|
-
|
|
36109
|
+
schemaReady2 = true;
|
|
35469
36110
|
}
|
|
35470
36111
|
async function searchCommonsEntities(filters = {}, userId) {
|
|
35471
36112
|
await ensureCommonsSchema();
|
|
@@ -35674,7 +36315,7 @@ async function prepareCommonsEntity(input, user) {
|
|
|
35674
36315
|
normalizedEntityType: entityType,
|
|
35675
36316
|
proposedSlug: slug4 || null,
|
|
35676
36317
|
publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
|
|
35677
|
-
writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
|
|
36318
|
+
writePath: "commons_prepare_entity -> compose the entity -> commons_host_image -> commons_validate_entity -> commons_submit_entity"
|
|
35678
36319
|
},
|
|
35679
36320
|
contract,
|
|
35680
36321
|
billing: {
|
|
@@ -35696,8 +36337,8 @@ async function prepareCommonsEntity(input, user) {
|
|
|
35696
36337
|
"Search likely duplicates before submitting a new entity. If one matches, edit it with entityId and baseRevision instead of creating another page.",
|
|
35697
36338
|
"Use the returned profile as a contract, not a rigid empty template: do not include History, Reception, Pricing, Timeline, or similar sections unless source evidence supports them.",
|
|
35698
36339
|
"Write neutral encyclopedia prose. Do not paste raw scrape Markdown, marketing layout fragments, or empty/filler headings.",
|
|
35699
|
-
"
|
|
35700
|
-
"Call commons_validate_entity with the composed payload before commons_submit_entity.",
|
|
36340
|
+
"For a featured image not already hosted by Commons, call commons_host_image and replace featuredImage.url with its returned permanent URL. Preserve original URL/source byline/relCanonical when republishing, and put media assets in the media array.",
|
|
36341
|
+
"Call commons_validate_entity with the composed, image-ready payload before commons_submit_entity.",
|
|
35701
36342
|
"Validation is free. Submitting a new entity costs 10 Credits; submitting an edit to an existing entity costs 2 Credits."
|
|
35702
36343
|
]
|
|
35703
36344
|
};
|
|
@@ -35712,6 +36353,8 @@ async function validateCommonsEntity(input, user) {
|
|
|
35712
36353
|
const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
|
|
35713
36354
|
const source = normalizeSource(input.source);
|
|
35714
36355
|
const featuredImage = normalizeFeaturedImage(input.featuredImage);
|
|
36356
|
+
const normalizedFeaturedImageUrl = featuredImage?.url ? normalizeCommonsImageUrl(featuredImage.url) : null;
|
|
36357
|
+
const registeredFeaturedImage = normalizedFeaturedImageUrl ? await findCommonsHostedImageByUrl(normalizedFeaturedImageUrl) : null;
|
|
35715
36358
|
const bodyMarkdown = safeCandidateBody(input.bodyMarkdown, input.contentSections);
|
|
35716
36359
|
const headings = extractMarkdownHeadings(bodyMarkdown);
|
|
35717
36360
|
const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
|
|
@@ -35749,15 +36392,44 @@ async function validateCommonsEntity(input, user) {
|
|
|
35749
36392
|
if (input.reviewPolicy === "always_review") {
|
|
35750
36393
|
warnings.push("reviewPolicy is always_review, so the proposal will not auto-publish even if the payload is otherwise valid.");
|
|
35751
36394
|
}
|
|
36395
|
+
if (featuredImage?.url && !registeredFeaturedImage) {
|
|
36396
|
+
warnings.push("The featured image is not registered in permanent Commons image storage. Call commons_host_image, replace featuredImage.url with its returned URL, and validate again before submitting.");
|
|
36397
|
+
}
|
|
35752
36398
|
if (user?.id == null) warnings.push("No authenticated user was supplied to validation; submit still requires API-key authenticated user context.");
|
|
35753
36399
|
const valid = errors.length === 0;
|
|
35754
|
-
const
|
|
36400
|
+
const imageReadyForPublish = Boolean(registeredFeaturedImage);
|
|
36401
|
+
const publishable = valid && imageReadyForPublish;
|
|
36402
|
+
const autoPublishEligible = publishable && input.reviewPolicy !== "always_review";
|
|
35755
36403
|
const tagVocabulary = await commonsTagVocabulary();
|
|
35756
36404
|
return {
|
|
35757
36405
|
ok: true,
|
|
35758
36406
|
valid,
|
|
35759
|
-
publishable
|
|
36407
|
+
publishable,
|
|
35760
36408
|
autoPublishEligible,
|
|
36409
|
+
imageDiagnostics: {
|
|
36410
|
+
featuredImage: registeredFeaturedImage ? {
|
|
36411
|
+
state: "commons_hosted",
|
|
36412
|
+
url: registeredFeaturedImage.url,
|
|
36413
|
+
commonsImageId: registeredFeaturedImage.id,
|
|
36414
|
+
readyForPublish: true,
|
|
36415
|
+
submitWillAttemptAutoHost: false,
|
|
36416
|
+
requiredAction: null
|
|
36417
|
+
} : featuredImage?.url ? {
|
|
36418
|
+
state: "external_unhosted",
|
|
36419
|
+
url: normalizedFeaturedImageUrl,
|
|
36420
|
+
commonsImageId: null,
|
|
36421
|
+
readyForPublish: false,
|
|
36422
|
+
submitWillAttemptAutoHost: true,
|
|
36423
|
+
requiredAction: "Call commons_host_image, replace featuredImage.url with its returned permanent URL, and validate again."
|
|
36424
|
+
} : {
|
|
36425
|
+
state: "missing",
|
|
36426
|
+
url: null,
|
|
36427
|
+
commonsImageId: null,
|
|
36428
|
+
readyForPublish: false,
|
|
36429
|
+
submitWillAttemptAutoHost: false,
|
|
36430
|
+
requiredAction: "Add a featured image by calling commons_host_image, then set featuredImage.url to its returned permanent URL."
|
|
36431
|
+
}
|
|
36432
|
+
},
|
|
35761
36433
|
errors,
|
|
35762
36434
|
warnings,
|
|
35763
36435
|
contract,
|
|
@@ -35838,7 +36510,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
35838
36510
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
35839
36511
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
35840
36512
|
const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
|
|
35841
|
-
const proposalId = `commons-proposal-${
|
|
36513
|
+
const proposalId = `commons-proposal-${randomUUID21()}`;
|
|
35842
36514
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
35843
36515
|
const entityId = existing?.id ?? normalized.entityId ?? allocateEntityId();
|
|
35844
36516
|
const proposalStatus = shouldApply ? "accepted" : "pending_review";
|
|
@@ -35893,7 +36565,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
35893
36565
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
35894
36566
|
`,
|
|
35895
36567
|
args: [
|
|
35896
|
-
`commons-ledger-${
|
|
36568
|
+
`commons-ledger-${randomUUID21()}`,
|
|
35897
36569
|
nextEntity.id,
|
|
35898
36570
|
proposalId,
|
|
35899
36571
|
user.id,
|
|
@@ -35999,7 +36671,7 @@ async function saveCommonsFilter(input, user) {
|
|
|
35999
36671
|
sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
|
|
36000
36672
|
args: [user.id, name]
|
|
36001
36673
|
});
|
|
36002
|
-
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${
|
|
36674
|
+
const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${randomUUID21()}`;
|
|
36003
36675
|
await getDb().execute({
|
|
36004
36676
|
sql: `
|
|
36005
36677
|
INSERT INTO commons_saved_filters (id, user_id, name, description, filter_json, created_at, updated_at)
|
|
@@ -37355,7 +38027,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
37355
38027
|
};
|
|
37356
38028
|
|
|
37357
38029
|
// src/api/commons-publication-repository.ts
|
|
37358
|
-
import { createHash as createHash18, randomUUID as
|
|
38030
|
+
import { createHash as createHash18, randomUUID as randomUUID22 } from "crypto";
|
|
37359
38031
|
|
|
37360
38032
|
// src/editorial-reading-room/article-page.ts
|
|
37361
38033
|
function escapeHtml6(value) {
|
|
@@ -37568,7 +38240,7 @@ async function claimCommonsPublication(input, user) {
|
|
|
37568
38240
|
}
|
|
37569
38241
|
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
37570
38242
|
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
37571
|
-
const id = `tcpub_${
|
|
38243
|
+
const id = `tcpub_${randomUUID22()}`;
|
|
37572
38244
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
37573
38245
|
try {
|
|
37574
38246
|
await getDb().execute({
|
|
@@ -37620,7 +38292,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
37620
38292
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
37621
38293
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
37622
38294
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
37623
|
-
const editionId = `tced_${
|
|
38295
|
+
const editionId = `tced_${randomUUID22()}`;
|
|
37624
38296
|
const revision = (latest?.revision ?? 0) + 1;
|
|
37625
38297
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
37626
38298
|
await getDb().batch([
|
|
@@ -37702,7 +38374,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
37702
38374
|
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
37703
38375
|
const rendered = renderEditorialReadingRoom(editionInput);
|
|
37704
38376
|
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
37705
|
-
const editionId = `tced_${
|
|
38377
|
+
const editionId = `tced_${randomUUID22()}`;
|
|
37706
38378
|
const revision = latest.revision + 1;
|
|
37707
38379
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
37708
38380
|
await getDb().batch([
|
|
@@ -38222,11 +38894,13 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
38222
38894
|
var HostImageSchema = z33.object({
|
|
38223
38895
|
sourceUrl: z33.string().url().optional(),
|
|
38224
38896
|
imageBase64: z33.string().min(16).optional(),
|
|
38897
|
+
artifactId: z33.string().min(1).max(500).optional(),
|
|
38898
|
+
sourceType: z33.enum(["uploaded", "ai_generated"]).optional(),
|
|
38225
38899
|
alt: z33.string().max(500).optional(),
|
|
38226
38900
|
license: z33.string().max(240).optional(),
|
|
38227
38901
|
attribution: z33.string().max(500).optional()
|
|
38228
|
-
}).strict().refine((body) =>
|
|
38229
|
-
message: "Provide
|
|
38902
|
+
}).strict().refine((body) => [body.sourceUrl, body.imageBase64, body.artifactId].filter(Boolean).length === 1, {
|
|
38903
|
+
message: "Provide exactly one of sourceUrl, imageBase64, or artifactId."
|
|
38230
38904
|
});
|
|
38231
38905
|
commonsApp.post("/images", auth2, async (c) => {
|
|
38232
38906
|
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
@@ -38236,7 +38910,7 @@ commonsApp.post("/images", auth2, async (c) => {
|
|
|
38236
38910
|
return c.json({ ok: true, data: hosted }, hosted.reused ? 200 : 201);
|
|
38237
38911
|
} catch (error) {
|
|
38238
38912
|
if (error instanceof CommonsImageError) {
|
|
38239
|
-
return c
|
|
38913
|
+
return imageErrorResponse(c, error);
|
|
38240
38914
|
}
|
|
38241
38915
|
throw error;
|
|
38242
38916
|
}
|
|
@@ -38300,7 +38974,14 @@ commonsApp.post("/entities/propose", auth2, async (c) => {
|
|
|
38300
38974
|
}
|
|
38301
38975
|
return c.json({ ok: true, data: result, billing: publicListingWriteBilling(billing) }, 201);
|
|
38302
38976
|
} catch (error) {
|
|
38303
|
-
|
|
38977
|
+
let chargeStatus = billing ? billing.duplicate ? "not_charged" : "refund_pending" : "not_charged";
|
|
38978
|
+
if (billing && !billing.duplicate) {
|
|
38979
|
+
const settlement = await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
|
|
38980
|
+
if (settlement) chargeStatus = "refunded";
|
|
38981
|
+
}
|
|
38982
|
+
if (error instanceof CommonsImageError) {
|
|
38983
|
+
return imageErrorResponse(c, error, chargeStatus);
|
|
38984
|
+
}
|
|
38304
38985
|
if (error instanceof CommonsRepositoryError) {
|
|
38305
38986
|
return c.json({ ok: false, error: error.code, message: error.message }, repositoryStatus(error));
|
|
38306
38987
|
}
|
|
@@ -38433,6 +39114,18 @@ function repositoryStatus(error) {
|
|
|
38433
39114
|
if (error.httpStatus === 409) return 409;
|
|
38434
39115
|
return 400;
|
|
38435
39116
|
}
|
|
39117
|
+
function imageErrorResponse(c, error, chargeStatus) {
|
|
39118
|
+
return c.json({
|
|
39119
|
+
ok: false,
|
|
39120
|
+
error: error.code,
|
|
39121
|
+
error_code: error.code,
|
|
39122
|
+
error_type: "invalid_request",
|
|
39123
|
+
retryable: error.retryable,
|
|
39124
|
+
...chargeStatus ? { charge_status: chargeStatus } : {},
|
|
39125
|
+
message: `${error.message} ${error.retryGuidance}`,
|
|
39126
|
+
details: { retry_guidance: error.retryGuidance }
|
|
39127
|
+
}, error.httpStatus);
|
|
39128
|
+
}
|
|
38436
39129
|
async function publicationOperation(c, operation, successStatus = 200) {
|
|
38437
39130
|
try {
|
|
38438
39131
|
return c.json({ ok: true, data: await operation() }, successStatus);
|
|
@@ -40172,10 +40865,10 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
40172
40865
|
}
|
|
40173
40866
|
|
|
40174
40867
|
// src/api/scheduled-run-view-links.ts
|
|
40175
|
-
import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as
|
|
40176
|
-
var
|
|
40868
|
+
import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as randomUUID23 } from "crypto";
|
|
40869
|
+
var schemaReady3 = false;
|
|
40177
40870
|
async function ensureScheduledRunViewLinksSchema() {
|
|
40178
|
-
if (
|
|
40871
|
+
if (schemaReady3) return;
|
|
40179
40872
|
const db = getDb();
|
|
40180
40873
|
await db.execute(`
|
|
40181
40874
|
CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
|
|
@@ -40223,7 +40916,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
40223
40916
|
CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
|
|
40224
40917
|
ON scheduled_run_view_links(expires_at, revoked_at)
|
|
40225
40918
|
`);
|
|
40226
|
-
|
|
40919
|
+
schemaReady3 = true;
|
|
40227
40920
|
}
|
|
40228
40921
|
function tokenHash2(token6) {
|
|
40229
40922
|
return createHash22("sha256").update(token6).digest("hex");
|
|
@@ -40248,7 +40941,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
40248
40941
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
40249
40942
|
const token6 = randomBytes5(32).toString("base64url");
|
|
40250
40943
|
const record = {
|
|
40251
|
-
shareId:
|
|
40944
|
+
shareId: randomUUID23(),
|
|
40252
40945
|
ownerId: input.ownerId,
|
|
40253
40946
|
runId: input.runId,
|
|
40254
40947
|
artifactId: input.artifactId,
|
|
@@ -41116,352 +41809,6 @@ import { Hono as Hono37 } from "hono";
|
|
|
41116
41809
|
|
|
41117
41810
|
// src/api/scheduler-integration-auth.ts
|
|
41118
41811
|
import { createHash as createHash28, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
41119
|
-
|
|
41120
|
-
// src/api/service-connections.ts
|
|
41121
|
-
import { randomUUID as randomUUID23 } from "crypto";
|
|
41122
|
-
var schemaReady3 = null;
|
|
41123
|
-
var schemaDb4 = null;
|
|
41124
|
-
function ensureServiceConnectionsSchema() {
|
|
41125
|
-
const currentDb = getDb();
|
|
41126
|
-
if (schemaReady3 && schemaDb4 === currentDb) return schemaReady3;
|
|
41127
|
-
schemaDb4 = currentDb;
|
|
41128
|
-
schemaReady3 = (async () => {
|
|
41129
|
-
const db = currentDb;
|
|
41130
|
-
await db.execute(`
|
|
41131
|
-
CREATE TABLE IF NOT EXISTS service_connections (
|
|
41132
|
-
id TEXT PRIMARY KEY,
|
|
41133
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
41134
|
-
provider_config_key TEXT NOT NULL,
|
|
41135
|
-
provider TEXT,
|
|
41136
|
-
transport TEXT NOT NULL,
|
|
41137
|
-
upstream_connection_id TEXT,
|
|
41138
|
-
label TEXT,
|
|
41139
|
-
user_label TEXT,
|
|
41140
|
-
provider_account_id TEXT,
|
|
41141
|
-
provider_account_email TEXT,
|
|
41142
|
-
provider_account_name TEXT,
|
|
41143
|
-
provider_identity_checked_at TEXT,
|
|
41144
|
-
provider_identity_source_updated_at TEXT,
|
|
41145
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
41146
|
-
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
41147
|
-
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
41148
|
-
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
41149
|
-
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
41150
|
-
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
41151
|
-
tool_revision TEXT,
|
|
41152
|
-
last_checked_at TEXT,
|
|
41153
|
-
last_successful_call_at TEXT,
|
|
41154
|
-
last_failure_at TEXT,
|
|
41155
|
-
last_failure_code TEXT,
|
|
41156
|
-
last_failure_retryable INTEGER,
|
|
41157
|
-
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
41158
|
-
vault_name TEXT,
|
|
41159
|
-
table_name TEXT,
|
|
41160
|
-
legacy_owner_identity TEXT,
|
|
41161
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
41162
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
41163
|
-
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
41164
|
-
)
|
|
41165
|
-
`);
|
|
41166
|
-
try {
|
|
41167
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN user_label TEXT`);
|
|
41168
|
-
} catch {
|
|
41169
|
-
}
|
|
41170
|
-
try {
|
|
41171
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT`);
|
|
41172
|
-
} catch {
|
|
41173
|
-
}
|
|
41174
|
-
try {
|
|
41175
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT`);
|
|
41176
|
-
} catch {
|
|
41177
|
-
}
|
|
41178
|
-
try {
|
|
41179
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT`);
|
|
41180
|
-
} catch {
|
|
41181
|
-
}
|
|
41182
|
-
try {
|
|
41183
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT`);
|
|
41184
|
-
} catch {
|
|
41185
|
-
}
|
|
41186
|
-
try {
|
|
41187
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT`);
|
|
41188
|
-
} catch {
|
|
41189
|
-
}
|
|
41190
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
|
|
41191
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
|
|
41192
|
-
await db.execute(`
|
|
41193
|
-
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
41194
|
-
id TEXT PRIMARY KEY,
|
|
41195
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
41196
|
-
operational_status TEXT NOT NULL,
|
|
41197
|
-
failure_code TEXT,
|
|
41198
|
-
retryable INTEGER,
|
|
41199
|
-
evidence_source TEXT NOT NULL,
|
|
41200
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
41201
|
-
)
|
|
41202
|
-
`);
|
|
41203
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
|
|
41204
|
-
await db.execute(`
|
|
41205
|
-
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
41206
|
-
id TEXT PRIMARY KEY,
|
|
41207
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
41208
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
41209
|
-
tool TEXT NOT NULL,
|
|
41210
|
-
request_id TEXT NOT NULL,
|
|
41211
|
-
status TEXT NOT NULL,
|
|
41212
|
-
error_code TEXT,
|
|
41213
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
41214
|
-
UNIQUE(user_id, request_id)
|
|
41215
|
-
)
|
|
41216
|
-
`);
|
|
41217
|
-
try {
|
|
41218
|
-
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
|
|
41219
|
-
} catch {
|
|
41220
|
-
}
|
|
41221
|
-
try {
|
|
41222
|
-
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
|
|
41223
|
-
} catch {
|
|
41224
|
-
}
|
|
41225
|
-
})().catch((error) => {
|
|
41226
|
-
schemaReady3 = null;
|
|
41227
|
-
schemaDb4 = null;
|
|
41228
|
-
throw error;
|
|
41229
|
-
});
|
|
41230
|
-
return schemaReady3;
|
|
41231
|
-
}
|
|
41232
|
-
function parseTools(value) {
|
|
41233
|
-
if (typeof value !== "string") return [];
|
|
41234
|
-
try {
|
|
41235
|
-
const parsed = JSON.parse(value);
|
|
41236
|
-
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
|
|
41237
|
-
} catch {
|
|
41238
|
-
return [];
|
|
41239
|
-
}
|
|
41240
|
-
}
|
|
41241
|
-
function rowToRecord(row) {
|
|
41242
|
-
return {
|
|
41243
|
-
id: String(row.id),
|
|
41244
|
-
userId: Number(row.user_id),
|
|
41245
|
-
providerConfigKey: String(row.provider_config_key),
|
|
41246
|
-
provider: typeof row.provider === "string" ? row.provider : null,
|
|
41247
|
-
transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
|
|
41248
|
-
upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
|
|
41249
|
-
label: typeof row.label === "string" ? row.label : null,
|
|
41250
|
-
userLabel: typeof row.user_label === "string" ? row.user_label : null,
|
|
41251
|
-
providerAccountId: typeof row.provider_account_id === "string" ? row.provider_account_id : null,
|
|
41252
|
-
providerAccountEmail: typeof row.provider_account_email === "string" ? row.provider_account_email : null,
|
|
41253
|
-
providerAccountName: typeof row.provider_account_name === "string" ? row.provider_account_name : null,
|
|
41254
|
-
providerIdentityCheckedAt: typeof row.provider_identity_checked_at === "string" ? row.provider_identity_checked_at : null,
|
|
41255
|
-
providerIdentitySourceUpdatedAt: typeof row.provider_identity_source_updated_at === "string" ? row.provider_identity_source_updated_at : null,
|
|
41256
|
-
lifecycleStatus: String(row.lifecycle_status),
|
|
41257
|
-
operationalStatus: String(row.operational_status),
|
|
41258
|
-
reconnectRequired: Number(row.reconnect_required) === 1,
|
|
41259
|
-
actionsEnabled: Number(row.actions_enabled) === 1,
|
|
41260
|
-
readTools: parseTools(row.read_tools_json),
|
|
41261
|
-
actionTools: parseTools(row.action_tools_json),
|
|
41262
|
-
toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
|
|
41263
|
-
lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
|
|
41264
|
-
lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
|
|
41265
|
-
lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
|
|
41266
|
-
lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
|
|
41267
|
-
lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
|
|
41268
|
-
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
|
41269
|
-
vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
|
|
41270
|
-
tableName: typeof row.table_name === "string" ? row.table_name : null,
|
|
41271
|
-
createdAt: String(row.created_at),
|
|
41272
|
-
updatedAt: String(row.updated_at)
|
|
41273
|
-
};
|
|
41274
|
-
}
|
|
41275
|
-
async function userIdForIdentity(identity) {
|
|
41276
|
-
const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
|
|
41277
|
-
if (!user) throw new Error("service_connection_user_not_found");
|
|
41278
|
-
return Number(user.id);
|
|
41279
|
-
}
|
|
41280
|
-
async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
41281
|
-
await ensureServiceConnectionsSchema();
|
|
41282
|
-
const db = getDb();
|
|
41283
|
-
const userId = await userIdForIdentity(identity);
|
|
41284
|
-
for (const connection of discovered) {
|
|
41285
|
-
await db.execute({
|
|
41286
|
-
sql: `
|
|
41287
|
-
INSERT INTO service_connections (
|
|
41288
|
-
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
41289
|
-
label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
|
|
41290
|
-
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
41291
|
-
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
41292
|
-
provider = excluded.provider,
|
|
41293
|
-
label = COALESCE(excluded.label, service_connections.label),
|
|
41294
|
-
lifecycle_status = excluded.lifecycle_status,
|
|
41295
|
-
reconnect_required = excluded.reconnect_required,
|
|
41296
|
-
updated_at = excluded.updated_at
|
|
41297
|
-
`,
|
|
41298
|
-
args: [
|
|
41299
|
-
randomUUID23(),
|
|
41300
|
-
userId,
|
|
41301
|
-
connection.providerConfigKey,
|
|
41302
|
-
connection.provider,
|
|
41303
|
-
connection.upstreamConnectionId,
|
|
41304
|
-
connection.label,
|
|
41305
|
-
connection.lifecycleStatus,
|
|
41306
|
-
connection.reconnectRequired ? 1 : 0,
|
|
41307
|
-
identity,
|
|
41308
|
-
connection.createdAt,
|
|
41309
|
-
connection.updatedAt
|
|
41310
|
-
]
|
|
41311
|
-
});
|
|
41312
|
-
}
|
|
41313
|
-
return listServiceConnections(identity, "nango");
|
|
41314
|
-
}
|
|
41315
|
-
async function listServiceConnections(identity, transport) {
|
|
41316
|
-
await ensureServiceConnectionsSchema();
|
|
41317
|
-
const userId = await userIdForIdentity(identity);
|
|
41318
|
-
const result = await getDb().execute({
|
|
41319
|
-
sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
|
|
41320
|
-
args: transport ? [userId, transport] : [userId]
|
|
41321
|
-
});
|
|
41322
|
-
return result.rows.map((row) => rowToRecord(row));
|
|
41323
|
-
}
|
|
41324
|
-
async function getOwnedServiceConnection(identity, connectionId) {
|
|
41325
|
-
await ensureServiceConnectionsSchema();
|
|
41326
|
-
const userId = await userIdForIdentity(identity);
|
|
41327
|
-
const result = await getDb().execute({
|
|
41328
|
-
sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
|
|
41329
|
-
args: [connectionId, userId]
|
|
41330
|
-
});
|
|
41331
|
-
const row = result.rows[0];
|
|
41332
|
-
return row ? rowToRecord(row) : null;
|
|
41333
|
-
}
|
|
41334
|
-
async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
|
|
41335
|
-
await ensureServiceConnectionsSchema();
|
|
41336
|
-
await getDb().execute({
|
|
41337
|
-
sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
41338
|
-
args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
|
|
41339
|
-
});
|
|
41340
|
-
}
|
|
41341
|
-
async function setServiceConnectionProviderIdentity(args) {
|
|
41342
|
-
await ensureServiceConnectionsSchema();
|
|
41343
|
-
await getDb().execute({
|
|
41344
|
-
sql: `
|
|
41345
|
-
UPDATE service_connections SET
|
|
41346
|
-
provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
|
|
41347
|
-
provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
|
|
41348
|
-
updated_at = datetime('now')
|
|
41349
|
-
WHERE id = ?
|
|
41350
|
-
`,
|
|
41351
|
-
args: [
|
|
41352
|
-
args.providerAccountId ?? null,
|
|
41353
|
-
args.providerAccountEmail?.toLowerCase() ?? null,
|
|
41354
|
-
args.providerAccountName ?? null,
|
|
41355
|
-
args.sourceUpdatedAt,
|
|
41356
|
-
args.connectionId
|
|
41357
|
-
]
|
|
41358
|
-
});
|
|
41359
|
-
}
|
|
41360
|
-
async function setServiceConnectionUserLabel(identity, connectionId, userLabel) {
|
|
41361
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
41362
|
-
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
41363
|
-
await getDb().execute({
|
|
41364
|
-
sql: `UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
41365
|
-
args: [userLabel, connectionId, connection.userId]
|
|
41366
|
-
});
|
|
41367
|
-
return userLabel;
|
|
41368
|
-
}
|
|
41369
|
-
async function recordServiceConnectionHealth(args) {
|
|
41370
|
-
await ensureServiceConnectionsSchema();
|
|
41371
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
41372
|
-
await getDb().execute({
|
|
41373
|
-
sql: `
|
|
41374
|
-
UPDATE service_connections SET
|
|
41375
|
-
operational_status = ?, last_checked_at = ?,
|
|
41376
|
-
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
41377
|
-
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
41378
|
-
last_failure_code = ?, last_failure_retryable = ?,
|
|
41379
|
-
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
41380
|
-
updated_at = datetime('now')
|
|
41381
|
-
WHERE id = ?
|
|
41382
|
-
`,
|
|
41383
|
-
args: [
|
|
41384
|
-
args.operationalStatus,
|
|
41385
|
-
now,
|
|
41386
|
-
args.operationalStatus,
|
|
41387
|
-
now,
|
|
41388
|
-
args.operationalStatus,
|
|
41389
|
-
now,
|
|
41390
|
-
args.failureCode ?? null,
|
|
41391
|
-
args.retryable == null ? null : args.retryable ? 1 : 0,
|
|
41392
|
-
args.operationalStatus,
|
|
41393
|
-
args.connectionId
|
|
41394
|
-
]
|
|
41395
|
-
});
|
|
41396
|
-
await getDb().execute({
|
|
41397
|
-
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
41398
|
-
args: [randomUUID23(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
41399
|
-
});
|
|
41400
|
-
}
|
|
41401
|
-
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
41402
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
41403
|
-
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
41404
|
-
await getDb().execute({
|
|
41405
|
-
sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
41406
|
-
args: [enabled ? 1 : 0, connectionId, connection.userId]
|
|
41407
|
-
});
|
|
41408
|
-
return enabled;
|
|
41409
|
-
}
|
|
41410
|
-
async function markServiceConnectionDisconnected(identity, connectionId) {
|
|
41411
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
41412
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
41413
|
-
await getDb().execute({
|
|
41414
|
-
sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
41415
|
-
args: [connectionId, connection.userId]
|
|
41416
|
-
});
|
|
41417
|
-
const remaining = await getDb().execute({
|
|
41418
|
-
sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
|
|
41419
|
-
args: [connection.userId]
|
|
41420
|
-
});
|
|
41421
|
-
return Number(remaining.rows[0]?.count ?? 0);
|
|
41422
|
-
}
|
|
41423
|
-
async function recordServiceConnectionAction(args) {
|
|
41424
|
-
await ensureServiceConnectionsSchema();
|
|
41425
|
-
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
41426
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
41427
|
-
await getDb().execute({
|
|
41428
|
-
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
41429
|
-
args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
|
|
41430
|
-
});
|
|
41431
|
-
}
|
|
41432
|
-
async function claimServiceConnectionAction(args) {
|
|
41433
|
-
await ensureServiceConnectionsSchema();
|
|
41434
|
-
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
41435
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
41436
|
-
const inserted = await getDb().execute({
|
|
41437
|
-
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
41438
|
-
args: [randomUUID23(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
41439
|
-
});
|
|
41440
|
-
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
41441
|
-
const existing = await getDb().execute({
|
|
41442
|
-
sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
41443
|
-
args: [connection.userId, args.requestId]
|
|
41444
|
-
});
|
|
41445
|
-
const row = existing.rows[0];
|
|
41446
|
-
const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
|
|
41447
|
-
let result = void 0;
|
|
41448
|
-
if (!conflict && typeof row?.result_json === "string") {
|
|
41449
|
-
try {
|
|
41450
|
-
result = JSON.parse(row.result_json);
|
|
41451
|
-
} catch {
|
|
41452
|
-
result = void 0;
|
|
41453
|
-
}
|
|
41454
|
-
}
|
|
41455
|
-
return {
|
|
41456
|
-
claimed: false,
|
|
41457
|
-
status: String(row?.status ?? "unknown"),
|
|
41458
|
-
errorCode: typeof row?.error_code === "string" ? row.error_code : null,
|
|
41459
|
-
conflict,
|
|
41460
|
-
...result !== void 0 ? { result } : {}
|
|
41461
|
-
};
|
|
41462
|
-
}
|
|
41463
|
-
|
|
41464
|
-
// src/api/scheduler-integration-auth.ts
|
|
41465
41812
|
var MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
41466
41813
|
var SchedulerIntegrationAuthError = class extends Error {
|
|
41467
41814
|
code;
|
|
@@ -49707,7 +50054,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
49707
50054
|
return c.json({ error: "Unauthorized" }, 401);
|
|
49708
50055
|
}
|
|
49709
50056
|
const startedAt = Date.now();
|
|
49710
|
-
const { drainQueue } = await import("./worker-
|
|
50057
|
+
const { drainQueue } = await import("./worker-JGWV2LID.js");
|
|
49711
50058
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
49712
50059
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
49713
50060
|
const unfinished = [];
|
|
@@ -49785,8 +50132,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
49785
50132
|
finishExtractJob: finishExtractJob2,
|
|
49786
50133
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
49787
50134
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
49788
|
-
} = await import("./site-extract-repository-
|
|
49789
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
50135
|
+
} = await import("./site-extract-repository-7EP6UX6I.js");
|
|
50136
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
|
|
49790
50137
|
const existing = await getExtractJob2(jobId2);
|
|
49791
50138
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
49792
50139
|
if (existing.status !== "failed") {
|
|
@@ -49813,7 +50160,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
49813
50160
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
49814
50161
|
let settlement = "already_settled_or_refunded";
|
|
49815
50162
|
if (job.billedMc == null && job.userId != null) {
|
|
49816
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
50163
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-7EP6UX6I.js");
|
|
49817
50164
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
49818
50165
|
const successful = await countSuccessfulPages2(jobId2);
|
|
49819
50166
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|