mcp-scraper 0.66.6 → 0.67.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +16 -1
  2. package/README.md +1 -1
  3. package/dist/bin/api-server.cjs +1122 -625
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +2 -2
  8. package/dist/bin/mcp-scraper-install.js +2 -2
  9. package/dist/bin/mcp-stdio-server.cjs +147 -17
  10. package/dist/bin/mcp-stdio-server.js +6 -6
  11. package/dist/bin/paa-harvest.cjs +20 -1
  12. package/dist/bin/paa-harvest.js +3 -3
  13. package/dist/{chunk-3EYALCYK.js → chunk-2L4C4DAZ.js} +1 -1
  14. package/dist/{chunk-CETDPNMZ.js → chunk-2LXTOAKS.js} +1 -1
  15. package/dist/{chunk-K7Y27ND3.js → chunk-62LEXS5O.js} +2 -2
  16. package/dist/{chunk-55B3T5L2.js → chunk-65FTMB7G.js} +92 -1
  17. package/dist/{chunk-3TKKJQB4.js → chunk-ES25GP6C.js} +1 -1
  18. package/dist/{chunk-NLEAYKOC.js → chunk-GT63KEUB.js} +1 -1
  19. package/dist/{chunk-KPE52NPT.js → chunk-IHDTONZV.js} +1 -1
  20. package/dist/{chunk-KXRWY7DZ.js → chunk-LCGTNWOT.js} +1 -1
  21. package/dist/{chunk-DBWWIK7O.js → chunk-LN6N3YLK.js} +1 -1
  22. package/dist/{chunk-ABYBPODM.js → chunk-UL4ZKAWZ.js} +1 -1
  23. package/dist/{chunk-VMWVEKZT.js → chunk-XOOKZYZG.js} +71 -18
  24. package/dist/{chunk-PMCT2YQM.js → chunk-Y46YNQMM.js} +1 -1
  25. package/dist/{db-4RVZ3NYZ.js → db-ZGZAKYBW.js} +1 -1
  26. package/dist/{extract-bundle-EA5URLXN.js → extract-bundle-QQIYJSBW.js} +3 -3
  27. package/dist/index.cjs +20 -1
  28. package/dist/index.js +3 -3
  29. package/dist/{lead-list-enrichment-repository-IBXNWM6H.js → lead-list-enrichment-repository-CXLVIV3R.js} +2 -2
  30. package/dist/{location-data-repository-WHHQUCUI.js → location-data-repository-G72M7DTN.js} +2 -2
  31. package/dist/{server-5WPS3LZD.js → server-MFVYRB6I.js} +818 -471
  32. package/dist/{site-extract-repository-NTM4MDQN.js → site-extract-repository-7EP6UX6I.js} +2 -2
  33. package/dist/{worker-UTBUWEXQ.js → worker-JGWV2LID.js} +5 -5
  34. package/package.json +1 -1
@@ -10,13 +10,13 @@ import {
10
10
  queuedCommonsDocumentCount,
11
11
  semanticCommonsEntityScores,
12
12
  typeFromMime
13
- } from "./chunk-DBWWIK7O.js";
13
+ } from "./chunk-LN6N3YLK.js";
14
14
  import {
15
15
  REQUIRED_LOCATION_STATE_CODES,
16
16
  importHostedCensusPlacesCsv,
17
17
  importHostedZipGroupsCsv,
18
18
  listActiveHostedLocationDatasets
19
- } from "./chunk-PMCT2YQM.js";
19
+ } from "./chunk-Y46YNQMM.js";
20
20
  import {
21
21
  cancelPendingLeadListEnrichmentRows,
22
22
  claimLeadListEnrichmentOutbox,
@@ -43,7 +43,7 @@ import {
43
43
  saveLeadListEnrichmentRowResult,
44
44
  setInitializingLeadListEnrichmentConcurrencyLock,
45
45
  stableCanonicalJson
46
- } from "./chunk-ABYBPODM.js";
46
+ } from "./chunk-UL4ZKAWZ.js";
47
47
  import {
48
48
  ANALYTICS_CONTENT_SORTS,
49
49
  AnalyticsRepositoryError,
@@ -108,7 +108,7 @@ import {
108
108
  harvestProblemResponse,
109
109
  publicErrorBoundary,
110
110
  serializeHarvestProblem
111
- } from "./chunk-CETDPNMZ.js";
111
+ } from "./chunk-2LXTOAKS.js";
112
112
  import "./chunk-OZJMVCDK.js";
113
113
  import {
114
114
  BACKUP_LAST_RESORT_ATTEMPTS,
@@ -127,7 +127,7 @@ import {
127
127
  resolveKernelProxyId,
128
128
  serpArrivalUrl,
129
129
  withRuntimeUserAgent
130
- } from "./chunk-K7Y27ND3.js";
130
+ } from "./chunk-62LEXS5O.js";
131
131
  import {
132
132
  ApiErrorResponseSchema,
133
133
  ArtifactTemplatePresetKeySchema,
@@ -199,7 +199,7 @@ import {
199
199
  resolveDeploymentProfile,
200
200
  resolveLocalSourcebookSchemaType,
201
201
  transcribeMediaUrl
202
- } from "./chunk-VMWVEKZT.js";
202
+ } from "./chunk-XOOKZYZG.js";
203
203
  import {
204
204
  auditImageUrls,
205
205
  auditImages,
@@ -210,7 +210,7 @@ import {
210
210
  sanitizeAttempts,
211
211
  sanitizeHarvestResult,
212
212
  sanitizeOutboundDiagnostics
213
- } from "./chunk-3EYALCYK.js";
213
+ } from "./chunk-2L4C4DAZ.js";
214
214
  import {
215
215
  browserServiceApiKey,
216
216
  browserServiceProfileName,
@@ -220,7 +220,7 @@ import {
220
220
  recordKernelSession,
221
221
  recordVendorUsage,
222
222
  runWithCostContext
223
- } from "./chunk-3TKKJQB4.js";
223
+ } from "./chunk-ES25GP6C.js";
224
224
  import {
225
225
  listWorkflowDefinitions,
226
226
  runWorkflow,
@@ -256,7 +256,7 @@ import {
256
256
  } from "./chunk-GGZEC22A.js";
257
257
  import {
258
258
  PACKAGE_VERSION
259
- } from "./chunk-NLEAYKOC.js";
259
+ } from "./chunk-GT63KEUB.js";
260
260
  import {
261
261
  SITE_EXTRACT_ARTIFACT_PREFIX,
262
262
  abandonExtractSettlement,
@@ -287,7 +287,7 @@ import {
287
287
  setExtractJobTotal,
288
288
  settleExtractJob,
289
289
  terminalExtractJobStatus
290
- } from "./chunk-KXRWY7DZ.js";
290
+ } from "./chunk-LCGTNWOT.js";
291
291
  import {
292
292
  createPrivateArtifact,
293
293
  privateArtifactOwnerId,
@@ -451,7 +451,7 @@ import {
451
451
  softDeleteInboxMessage,
452
452
  upsertPageSnapshot,
453
453
  verifyPassword
454
- } from "./chunk-55B3T5L2.js";
454
+ } from "./chunk-65FTMB7G.js";
455
455
 
456
456
  // src/blog/registry.ts
457
457
  var posts = [
@@ -6312,7 +6312,8 @@ function firstJsonLdImage(value) {
6312
6312
  }
6313
6313
  return null;
6314
6314
  }
6315
- function firstContentImage(html) {
6315
+ function contentImages(html) {
6316
+ const images = [];
6316
6317
  for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
6317
6318
  const tag = match[0];
6318
6319
  const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
@@ -6323,30 +6324,36 @@ function firstContentImage(html) {
6323
6324
  const width = Number(attr("width") ?? 0);
6324
6325
  const height = Number(attr("height") ?? 0);
6325
6326
  if (width > 0 && width < 200 || height > 0 && height < 120) continue;
6326
- return candidate;
6327
+ images.push(candidate);
6327
6328
  }
6328
- return null;
6329
+ return images;
6329
6330
  }
6330
- function resolveFeaturedImage(input) {
6331
+ function resolveFeaturedImageCandidates(input) {
6331
6332
  const replay = parseWaybackReplayUrl(input.pageUrl);
6332
6333
  const baseUrl = replay?.originalUrl ?? input.pageUrl;
6333
6334
  const candidates = [
6334
6335
  { value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
6335
6336
  { value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
6336
6337
  { value: firstJsonLdImage(input.schema), source: "json-ld" },
6337
- { value: firstContentImage(input.html), source: "content-image" }
6338
+ ...contentImages(input.html).map((value) => ({ value, source: "content-image" }))
6338
6339
  ];
6340
+ const seen = /* @__PURE__ */ new Set();
6341
+ const resolved = [];
6339
6342
  for (const candidate of candidates) {
6340
6343
  if (!candidate.value) continue;
6341
6344
  const url = safeHttpUrl(candidate.value, baseUrl);
6342
- if (!url) continue;
6343
- return {
6345
+ if (!url || seen.has(url)) continue;
6346
+ seen.add(url);
6347
+ resolved.push({
6344
6348
  url,
6345
6349
  archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
6346
6350
  source: candidate.source
6347
- };
6351
+ });
6348
6352
  }
6349
- return null;
6353
+ return resolved;
6354
+ }
6355
+ function resolveFeaturedImage(input) {
6356
+ return resolveFeaturedImageCandidates(input)[0] ?? null;
6350
6357
  }
6351
6358
 
6352
6359
  // src/api/site-extractor.ts
@@ -9545,7 +9552,7 @@ var siteExtractFn = inngest.createFunction(
9545
9552
  });
9546
9553
  }) : null;
9547
9554
  const finalized = await step.run("finalize", async () => {
9548
- const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
9555
+ const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
9549
9556
  const current = await getExtractJob(jobId2);
9550
9557
  if (!current) throw new Error("extract job disappeared before finalization");
9551
9558
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10505,7 +10512,7 @@ async function resolveUsZipsPath(requestedPath) {
10505
10512
  async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
10506
10513
  const mode = resolvedLocationDataMode({ zipSource: sourceMode });
10507
10514
  if (mode === "hosted") {
10508
- const { getHostedZipGroups } = await import("./location-data-repository-WHHQUCUI.js");
10515
+ const { getHostedZipGroups } = await import("./location-data-repository-G72M7DTN.js");
10509
10516
  const hosted = await getHostedZipGroups(stateAbbr);
10510
10517
  if (!hosted.dataset) {
10511
10518
  throw new DirectoryLocationError(
@@ -10568,7 +10575,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
10568
10575
  const sourceUrl = censusStateUrl(state.fips);
10569
10576
  const warnings = [];
10570
10577
  if (resolvedLocationDataMode(config) === "hosted") {
10571
- const { queryHostedLocationMarkets } = await import("./location-data-repository-WHHQUCUI.js");
10578
+ const { queryHostedLocationMarkets } = await import("./location-data-repository-G72M7DTN.js");
10572
10579
  const hosted = await queryHostedLocationMarkets({
10573
10580
  state: state.abbr,
10574
10581
  minPopulation: options.minPopulation,
@@ -12055,7 +12062,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
12055
12062
  async function settlePersistedLeadListEnrichment(jobId2) {
12056
12063
  const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
12057
12064
  const job = jobs.find((candidate) => candidate.id === jobId2);
12058
- const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
12065
+ const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
12059
12066
  const current = job ?? await getLeadListEnrichmentJob2(jobId2);
12060
12067
  if (!current) return null;
12061
12068
  const rows = await listLeadListEnrichmentRows(jobId2);
@@ -12111,7 +12118,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
12111
12118
  const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
12112
12119
  const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
12113
12120
  if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
12114
- const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
12121
+ const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
12115
12122
  await failLeadListEnrichmentJob2({
12116
12123
  jobId: job.id,
12117
12124
  errorCode: "lead_list_all_rows_failed",
@@ -16505,6 +16512,350 @@ function findPreferredEntitledSubscription(subscriptions, predicate) {
16505
16512
  return void 0;
16506
16513
  }
16507
16514
 
16515
+ // src/api/service-connections.ts
16516
+ import { randomUUID as randomUUID6 } from "crypto";
16517
+ var schemaReady = null;
16518
+ var schemaDb3 = null;
16519
+ function ensureServiceConnectionsSchema() {
16520
+ const currentDb = getDb();
16521
+ if (schemaReady && schemaDb3 === currentDb) return schemaReady;
16522
+ schemaDb3 = currentDb;
16523
+ schemaReady = (async () => {
16524
+ const db = currentDb;
16525
+ await db.execute(`
16526
+ CREATE TABLE IF NOT EXISTS service_connections (
16527
+ id TEXT PRIMARY KEY,
16528
+ user_id INTEGER NOT NULL REFERENCES users(id),
16529
+ provider_config_key TEXT NOT NULL,
16530
+ provider TEXT,
16531
+ transport TEXT NOT NULL,
16532
+ upstream_connection_id TEXT,
16533
+ label TEXT,
16534
+ user_label TEXT,
16535
+ provider_account_id TEXT,
16536
+ provider_account_email TEXT,
16537
+ provider_account_name TEXT,
16538
+ provider_identity_checked_at TEXT,
16539
+ provider_identity_source_updated_at TEXT,
16540
+ lifecycle_status TEXT NOT NULL DEFAULT 'pending',
16541
+ operational_status TEXT NOT NULL DEFAULT 'unknown',
16542
+ reconnect_required INTEGER NOT NULL DEFAULT 0,
16543
+ actions_enabled INTEGER NOT NULL DEFAULT 0,
16544
+ read_tools_json TEXT NOT NULL DEFAULT '[]',
16545
+ action_tools_json TEXT NOT NULL DEFAULT '[]',
16546
+ tool_revision TEXT,
16547
+ last_checked_at TEXT,
16548
+ last_successful_call_at TEXT,
16549
+ last_failure_at TEXT,
16550
+ last_failure_code TEXT,
16551
+ last_failure_retryable INTEGER,
16552
+ consecutive_failures INTEGER NOT NULL DEFAULT 0,
16553
+ vault_name TEXT,
16554
+ table_name TEXT,
16555
+ legacy_owner_identity TEXT,
16556
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
16557
+ updated_at TEXT NOT NULL DEFAULT (datetime('now')),
16558
+ UNIQUE(user_id, provider_config_key, upstream_connection_id)
16559
+ )
16560
+ `);
16561
+ try {
16562
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN user_label TEXT`);
16563
+ } catch {
16564
+ }
16565
+ try {
16566
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT`);
16567
+ } catch {
16568
+ }
16569
+ try {
16570
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT`);
16571
+ } catch {
16572
+ }
16573
+ try {
16574
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT`);
16575
+ } catch {
16576
+ }
16577
+ try {
16578
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT`);
16579
+ } catch {
16580
+ }
16581
+ try {
16582
+ await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT`);
16583
+ } catch {
16584
+ }
16585
+ await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
16586
+ await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
16587
+ await db.execute(`
16588
+ CREATE TABLE IF NOT EXISTS service_connection_health_events (
16589
+ id TEXT PRIMARY KEY,
16590
+ connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
16591
+ operational_status TEXT NOT NULL,
16592
+ failure_code TEXT,
16593
+ retryable INTEGER,
16594
+ evidence_source TEXT NOT NULL,
16595
+ created_at TEXT NOT NULL DEFAULT (datetime('now'))
16596
+ )
16597
+ `);
16598
+ await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
16599
+ await db.execute(`
16600
+ CREATE TABLE IF NOT EXISTS service_connection_action_audit (
16601
+ id TEXT PRIMARY KEY,
16602
+ connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
16603
+ user_id INTEGER NOT NULL REFERENCES users(id),
16604
+ tool TEXT NOT NULL,
16605
+ request_id TEXT NOT NULL,
16606
+ status TEXT NOT NULL,
16607
+ error_code TEXT,
16608
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
16609
+ UNIQUE(user_id, request_id)
16610
+ )
16611
+ `);
16612
+ try {
16613
+ await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
16614
+ } catch {
16615
+ }
16616
+ try {
16617
+ await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
16618
+ } catch {
16619
+ }
16620
+ })().catch((error) => {
16621
+ schemaReady = null;
16622
+ schemaDb3 = null;
16623
+ throw error;
16624
+ });
16625
+ return schemaReady;
16626
+ }
16627
+ function parseTools(value) {
16628
+ if (typeof value !== "string") return [];
16629
+ try {
16630
+ const parsed = JSON.parse(value);
16631
+ return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
16632
+ } catch {
16633
+ return [];
16634
+ }
16635
+ }
16636
+ function rowToRecord(row) {
16637
+ return {
16638
+ id: String(row.id),
16639
+ userId: Number(row.user_id),
16640
+ providerConfigKey: String(row.provider_config_key),
16641
+ provider: typeof row.provider === "string" ? row.provider : null,
16642
+ transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
16643
+ upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
16644
+ label: typeof row.label === "string" ? row.label : null,
16645
+ userLabel: typeof row.user_label === "string" ? row.user_label : null,
16646
+ providerAccountId: typeof row.provider_account_id === "string" ? row.provider_account_id : null,
16647
+ providerAccountEmail: typeof row.provider_account_email === "string" ? row.provider_account_email : null,
16648
+ providerAccountName: typeof row.provider_account_name === "string" ? row.provider_account_name : null,
16649
+ providerIdentityCheckedAt: typeof row.provider_identity_checked_at === "string" ? row.provider_identity_checked_at : null,
16650
+ providerIdentitySourceUpdatedAt: typeof row.provider_identity_source_updated_at === "string" ? row.provider_identity_source_updated_at : null,
16651
+ lifecycleStatus: String(row.lifecycle_status),
16652
+ operationalStatus: String(row.operational_status),
16653
+ reconnectRequired: Number(row.reconnect_required) === 1,
16654
+ actionsEnabled: Number(row.actions_enabled) === 1,
16655
+ readTools: parseTools(row.read_tools_json),
16656
+ actionTools: parseTools(row.action_tools_json),
16657
+ toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
16658
+ lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
16659
+ lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
16660
+ lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
16661
+ lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
16662
+ lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
16663
+ consecutiveFailures: Number(row.consecutive_failures ?? 0),
16664
+ vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
16665
+ tableName: typeof row.table_name === "string" ? row.table_name : null,
16666
+ createdAt: String(row.created_at),
16667
+ updatedAt: String(row.updated_at)
16668
+ };
16669
+ }
16670
+ async function userIdForIdentity(identity) {
16671
+ const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
16672
+ if (!user) throw new Error("service_connection_user_not_found");
16673
+ return Number(user.id);
16674
+ }
16675
+ async function reconcileDiscoveredNangoConnections(identity, discovered) {
16676
+ await ensureServiceConnectionsSchema();
16677
+ const db = getDb();
16678
+ const userId = await userIdForIdentity(identity);
16679
+ for (const connection of discovered) {
16680
+ await db.execute({
16681
+ sql: `
16682
+ INSERT INTO service_connections (
16683
+ id, user_id, provider_config_key, provider, transport, upstream_connection_id,
16684
+ label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
16685
+ ) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
16686
+ ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
16687
+ provider = excluded.provider,
16688
+ label = COALESCE(excluded.label, service_connections.label),
16689
+ lifecycle_status = excluded.lifecycle_status,
16690
+ reconnect_required = excluded.reconnect_required,
16691
+ updated_at = excluded.updated_at
16692
+ `,
16693
+ args: [
16694
+ randomUUID6(),
16695
+ userId,
16696
+ connection.providerConfigKey,
16697
+ connection.provider,
16698
+ connection.upstreamConnectionId,
16699
+ connection.label,
16700
+ connection.lifecycleStatus,
16701
+ connection.reconnectRequired ? 1 : 0,
16702
+ identity,
16703
+ connection.createdAt,
16704
+ connection.updatedAt
16705
+ ]
16706
+ });
16707
+ }
16708
+ return listServiceConnections(identity, "nango");
16709
+ }
16710
+ async function listServiceConnections(identity, transport) {
16711
+ await ensureServiceConnectionsSchema();
16712
+ const userId = await userIdForIdentity(identity);
16713
+ const result = await getDb().execute({
16714
+ sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
16715
+ args: transport ? [userId, transport] : [userId]
16716
+ });
16717
+ return result.rows.map((row) => rowToRecord(row));
16718
+ }
16719
+ async function getOwnedServiceConnection(identity, connectionId) {
16720
+ await ensureServiceConnectionsSchema();
16721
+ const userId = await userIdForIdentity(identity);
16722
+ const result = await getDb().execute({
16723
+ sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
16724
+ args: [connectionId, userId]
16725
+ });
16726
+ const row = result.rows[0];
16727
+ return row ? rowToRecord(row) : null;
16728
+ }
16729
+ async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
16730
+ await ensureServiceConnectionsSchema();
16731
+ await getDb().execute({
16732
+ sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
16733
+ args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
16734
+ });
16735
+ }
16736
+ async function setServiceConnectionProviderIdentity(args) {
16737
+ await ensureServiceConnectionsSchema();
16738
+ await getDb().execute({
16739
+ sql: `
16740
+ UPDATE service_connections SET
16741
+ provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
16742
+ provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
16743
+ updated_at = datetime('now')
16744
+ WHERE id = ?
16745
+ `,
16746
+ args: [
16747
+ args.providerAccountId ?? null,
16748
+ args.providerAccountEmail?.toLowerCase() ?? null,
16749
+ args.providerAccountName ?? null,
16750
+ args.sourceUpdatedAt,
16751
+ args.connectionId
16752
+ ]
16753
+ });
16754
+ }
16755
+ async function setServiceConnectionUserLabel(identity, connectionId, userLabel) {
16756
+ const connection = await getOwnedServiceConnection(identity, connectionId);
16757
+ if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
16758
+ await getDb().execute({
16759
+ sql: `UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
16760
+ args: [userLabel, connectionId, connection.userId]
16761
+ });
16762
+ return userLabel;
16763
+ }
16764
+ async function recordServiceConnectionHealth(args) {
16765
+ await ensureServiceConnectionsSchema();
16766
+ const now = (/* @__PURE__ */ new Date()).toISOString();
16767
+ await getDb().execute({
16768
+ sql: `
16769
+ UPDATE service_connections SET
16770
+ operational_status = ?, last_checked_at = ?,
16771
+ last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
16772
+ last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
16773
+ last_failure_code = ?, last_failure_retryable = ?,
16774
+ consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
16775
+ updated_at = datetime('now')
16776
+ WHERE id = ?
16777
+ `,
16778
+ args: [
16779
+ args.operationalStatus,
16780
+ now,
16781
+ args.operationalStatus,
16782
+ now,
16783
+ args.operationalStatus,
16784
+ now,
16785
+ args.failureCode ?? null,
16786
+ args.retryable == null ? null : args.retryable ? 1 : 0,
16787
+ args.operationalStatus,
16788
+ args.connectionId
16789
+ ]
16790
+ });
16791
+ await getDb().execute({
16792
+ sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
16793
+ args: [randomUUID6(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
16794
+ });
16795
+ }
16796
+ async function setServiceConnectionActions(identity, connectionId, enabled) {
16797
+ const connection = await getOwnedServiceConnection(identity, connectionId);
16798
+ if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
16799
+ await getDb().execute({
16800
+ sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
16801
+ args: [enabled ? 1 : 0, connectionId, connection.userId]
16802
+ });
16803
+ return enabled;
16804
+ }
16805
+ async function markServiceConnectionDisconnected(identity, connectionId) {
16806
+ const connection = await getOwnedServiceConnection(identity, connectionId);
16807
+ if (!connection) throw new Error("service_connection_not_found");
16808
+ await getDb().execute({
16809
+ sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
16810
+ args: [connectionId, connection.userId]
16811
+ });
16812
+ const remaining = await getDb().execute({
16813
+ sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
16814
+ args: [connection.userId]
16815
+ });
16816
+ return Number(remaining.rows[0]?.count ?? 0);
16817
+ }
16818
+ async function recordServiceConnectionAction(args) {
16819
+ await ensureServiceConnectionsSchema();
16820
+ const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
16821
+ if (!connection) throw new Error("service_connection_not_found");
16822
+ await getDb().execute({
16823
+ sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
16824
+ args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
16825
+ });
16826
+ }
16827
+ async function claimServiceConnectionAction(args) {
16828
+ await ensureServiceConnectionsSchema();
16829
+ const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
16830
+ if (!connection) throw new Error("service_connection_not_found");
16831
+ const inserted = await getDb().execute({
16832
+ sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
16833
+ args: [randomUUID6(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
16834
+ });
16835
+ if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
16836
+ const existing = await getDb().execute({
16837
+ sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
16838
+ args: [connection.userId, args.requestId]
16839
+ });
16840
+ const row = existing.rows[0];
16841
+ const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
16842
+ let result = void 0;
16843
+ if (!conflict && typeof row?.result_json === "string") {
16844
+ try {
16845
+ result = JSON.parse(row.result_json);
16846
+ } catch {
16847
+ result = void 0;
16848
+ }
16849
+ }
16850
+ return {
16851
+ claimed: false,
16852
+ status: String(row?.status ?? "unknown"),
16853
+ errorCode: typeof row?.error_code === "string" ? row.error_code : null,
16854
+ conflict,
16855
+ ...result !== void 0 ? { result } : {}
16856
+ };
16857
+ }
16858
+
16508
16859
  // src/api/connected-account-billing.ts
16509
16860
  var DEFAULT_CONNECTED_ACCOUNT_PRICE_ID = "price_1TtYPAS8aAcsk3TGHOgrDZiQ";
16510
16861
  function connectedBillingPeriodStart(now = /* @__PURE__ */ new Date()) {
@@ -16521,6 +16872,87 @@ async function chargedConnectionIds(userId, periodStart) {
16521
16872
  });
16522
16873
  return new Set(result.rows.map((row) => String(row.connection_id)));
16523
16874
  }
16875
+ async function inheritDisconnectedConnectionCharge(userId, connectionId, periodStart) {
16876
+ await ensureServiceConnectionsSchema();
16877
+ const normalizedUserId = Number(userId);
16878
+ const db = getDb();
16879
+ await db.batch([
16880
+ {
16881
+ sql: `
16882
+ INSERT OR IGNORE INTO connected_account_charge_transfers (
16883
+ user_id, period_start, from_connection_id, to_connection_id, provider_config_key
16884
+ )
16885
+ SELECT ?, ?, previous.id, replacement.id, replacement.provider_config_key
16886
+ FROM service_connections AS replacement
16887
+ JOIN service_connections AS previous
16888
+ ON previous.user_id = replacement.user_id
16889
+ AND previous.transport = 'nango'
16890
+ AND previous.provider_config_key = replacement.provider_config_key
16891
+ AND previous.lifecycle_status = 'disconnected'
16892
+ JOIN connected_account_charges AS charge
16893
+ ON charge.user_id = previous.user_id
16894
+ AND charge.connection_id = previous.id
16895
+ AND charge.period_start = ?
16896
+ WHERE replacement.id = ?
16897
+ AND replacement.user_id = ?
16898
+ AND replacement.transport = 'nango'
16899
+ AND replacement.lifecycle_status IN ('connected', 'needs_reauth')
16900
+ AND NOT EXISTS (
16901
+ SELECT 1 FROM connected_account_charges AS existing
16902
+ WHERE existing.user_id = replacement.user_id
16903
+ AND existing.connection_id = replacement.id
16904
+ AND existing.period_start = ?
16905
+ )
16906
+ ORDER BY previous.updated_at DESC, previous.id ASC
16907
+ LIMIT 1
16908
+ `,
16909
+ args: [normalizedUserId, periodStart, periodStart, connectionId, normalizedUserId, periodStart]
16910
+ },
16911
+ {
16912
+ sql: `
16913
+ UPDATE connected_account_charges
16914
+ SET connection_id = ?
16915
+ WHERE user_id = ?
16916
+ AND period_start = ?
16917
+ AND connection_id = (
16918
+ SELECT transfer.from_connection_id
16919
+ FROM connected_account_charge_transfers AS transfer
16920
+ JOIN connected_account_charges AS current_charge
16921
+ ON current_charge.user_id = transfer.user_id
16922
+ AND current_charge.period_start = transfer.period_start
16923
+ AND current_charge.connection_id = transfer.from_connection_id
16924
+ WHERE transfer.user_id = ?
16925
+ AND transfer.period_start = ?
16926
+ AND transfer.to_connection_id = ?
16927
+ ORDER BY transfer.transferred_at DESC, transfer.id DESC
16928
+ LIMIT 1
16929
+ )
16930
+ AND NOT EXISTS (
16931
+ SELECT 1 FROM connected_account_charges AS existing
16932
+ WHERE existing.user_id = ?
16933
+ AND existing.connection_id = ?
16934
+ AND existing.period_start = ?
16935
+ )
16936
+ `,
16937
+ args: [
16938
+ connectionId,
16939
+ normalizedUserId,
16940
+ periodStart,
16941
+ normalizedUserId,
16942
+ periodStart,
16943
+ connectionId,
16944
+ normalizedUserId,
16945
+ connectionId,
16946
+ periodStart
16947
+ ]
16948
+ }
16949
+ ], "write");
16950
+ const inherited = await db.execute({
16951
+ sql: "SELECT 1 FROM connected_account_charges WHERE user_id = ? AND connection_id = ? AND period_start = ? LIMIT 1",
16952
+ args: [normalizedUserId, connectionId, periodStart]
16953
+ });
16954
+ return inherited.rows.length > 0;
16955
+ }
16524
16956
  async function getConnectedAccountBillingState(userId) {
16525
16957
  const result = await getDb().execute({
16526
16958
  sql: "SELECT * FROM connected_account_billing WHERE user_id = ? LIMIT 1",
@@ -16661,6 +17093,7 @@ async function reconcileConnectedAccountBilling(user, connections) {
16661
17093
  const alreadyCharged = await chargedConnectionIds(user.id, periodStart);
16662
17094
  const unpaid = billableConnectionIds.filter((id) => !alreadyCharged.has(id));
16663
17095
  for (const connectionId of unpaid) {
17096
+ if (await inheritDisconnectedConnectionCharge(user.id, connectionId, periodStart)) continue;
16664
17097
  const feeMc = CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC;
16665
17098
  const debit = await debitMcIdempotent(
16666
17099
  user.id,
@@ -19752,7 +20185,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
19752
20185
  import { Hono as Hono9 } from "hono";
19753
20186
 
19754
20187
  // src/api/admin-credits.ts
19755
- import { randomUUID as randomUUID6 } from "crypto";
20188
+ import { randomUUID as randomUUID7 } from "crypto";
19756
20189
  var MIN_ADJUSTMENT_CREDITS = 1;
19757
20190
  var LARGE_ADJUSTMENT_CREDITS = 1e5;
19758
20191
  var MAX_ADJUSTMENT_CREDITS = 1e7;
@@ -19902,7 +20335,7 @@ async function applyAdjustment(input) {
19902
20335
  const credits = validateCredits(input.credits, input.confirmLarge === true);
19903
20336
  const reason = validateReason(input.reason);
19904
20337
  const actor = validateActor(input.actor);
19905
- const reference = input.reference?.trim() || `adj_${randomUUID6()}`;
20338
+ const reference = input.reference?.trim() || `adj_${randomUUID7()}`;
19906
20339
  if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
19907
20340
  const db = getDb();
19908
20341
  const existing = await db.execute({
@@ -22509,17 +22942,17 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
22509
22942
  });
22510
22943
 
22511
22944
  // src/api/directory-routes.ts
22512
- import { createHash as createHash5, randomUUID as randomUUID8 } from "crypto";
22945
+ import { createHash as createHash5, randomUUID as randomUUID9 } from "crypto";
22513
22946
  import { Hono as Hono16 } from "hono";
22514
22947
 
22515
22948
  // src/api/directory-workflow-dispatch.ts
22516
- import { randomUUID as randomUUID7 } from "crypto";
22949
+ import { randomUUID as randomUUID8 } from "crypto";
22517
22950
  function retryDelaySeconds(attempts) {
22518
22951
  return Math.min(60 * 60, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
22519
22952
  }
22520
22953
  async function dispatchPendingDirectoryWorkflows(limit = 25) {
22521
22954
  const rows = await claimDirectoryWorkflowOutbox({
22522
- workerId: `directory-dispatch-${process.pid}-${randomUUID7().slice(0, 8)}`,
22955
+ workerId: `directory-dispatch-${process.pid}-${randomUUID8().slice(0, 8)}`,
22523
22956
  limit
22524
22957
  });
22525
22958
  const result = { claimed: rows.length, dispatched: 0, failed: 0 };
@@ -22558,14 +22991,14 @@ function idempotencyKey(raw) {
22558
22991
  if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
22559
22992
  return { ok: true, key: trimmed };
22560
22993
  }
22561
- return { ok: true, key: `directory-${randomUUID8()}` };
22994
+ return { ok: true, key: `directory-${randomUUID9()}` };
22562
22995
  }
22563
22996
  function debitKeyFor(userId, responseKey) {
22564
22997
  const digest2 = createHash5("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
22565
22998
  return `directory-workflow:${userId}:${digest2}`;
22566
22999
  }
22567
23000
  function jobId() {
22568
- return `dir_${randomUUID8().replace(/-/g, "")}`;
23001
+ return `dir_${randomUUID9().replace(/-/g, "")}`;
22569
23002
  }
22570
23003
  function publicStatus(job, result) {
22571
23004
  if (job.status === "failed") return "failed";
@@ -22879,8 +23312,8 @@ import { homedir as homedir3 } from "os";
22879
23312
  import { join as join7 } from "path";
22880
23313
 
22881
23314
  // src/api/lead-list-import-repository.ts
22882
- import { randomUUID as randomUUID9 } from "crypto";
22883
- var schemaDb3 = null;
23315
+ import { randomUUID as randomUUID10 } from "crypto";
23316
+ var schemaDb4 = null;
22884
23317
  var schemaPromise3 = null;
22885
23318
  function parseJson3(value, fallback) {
22886
23319
  if (value == null) return fallback;
@@ -22945,15 +23378,15 @@ function bounded(value, field, min, max) {
22945
23378
  return normalized;
22946
23379
  }
22947
23380
  function newLeadListUploadId() {
22948
- return `upl_${randomUUID9().replace(/-/g, "")}`;
23381
+ return `upl_${randomUUID10().replace(/-/g, "")}`;
22949
23382
  }
22950
23383
  function newImportedLeadListId() {
22951
- return `lst_${randomUUID9().replace(/-/g, "")}`;
23384
+ return `lst_${randomUUID10().replace(/-/g, "")}`;
22952
23385
  }
22953
23386
  async function ensureLeadListImportRepositorySchema() {
22954
23387
  const db = getDb();
22955
- if (schemaDb3 !== db) {
22956
- schemaDb3 = db;
23388
+ if (schemaDb4 !== db) {
23389
+ schemaDb4 = db;
22957
23390
  schemaPromise3 = null;
22958
23391
  }
22959
23392
  if (!schemaPromise3) {
@@ -24090,17 +24523,17 @@ leadListInputApp.post("/imports", async (c) => {
24090
24523
  });
24091
24524
 
24092
24525
  // src/api/lead-list-enrichment-routes.ts
24093
- import { createHash as createHash8, randomUUID as randomUUID11 } from "crypto";
24526
+ import { createHash as createHash8, randomUUID as randomUUID12 } from "crypto";
24094
24527
  import { Hono as Hono18 } from "hono";
24095
24528
  import { z as z25 } from "zod";
24096
24529
 
24097
24530
  // src/api/lead-list-enrichment-dispatch.ts
24098
- import { randomUUID as randomUUID10 } from "crypto";
24531
+ import { randomUUID as randomUUID11 } from "crypto";
24099
24532
  function retryDelaySeconds2(attempts) {
24100
24533
  return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
24101
24534
  }
24102
24535
  async function dispatchPendingLeadListEnrichments(limit = 25) {
24103
- const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID10().slice(0, 8)}`, limit });
24536
+ const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID11().slice(0, 8)}`, limit });
24104
24537
  const result = { claimed: rows.length, dispatched: 0, failed: 0 };
24105
24538
  for (const row of rows) {
24106
24539
  if (!row.claimToken) continue;
@@ -24148,7 +24581,7 @@ function idempotencyKey2(raw) {
24148
24581
  return { ok: true, value };
24149
24582
  }
24150
24583
  function newJobId() {
24151
- return `lle_${randomUUID11().replace(/-/g, "")}`;
24584
+ return `lle_${randomUUID12().replace(/-/g, "")}`;
24152
24585
  }
24153
24586
  function debitKeyFor2(userId, key) {
24154
24587
  const digest2 = createHash8("sha256").update(String(userId)).update("\0").update(key).digest("hex");
@@ -26158,7 +26591,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
26158
26591
 
26159
26592
  // src/api/serp-intelligence-routes.ts
26160
26593
  import { Hono as Hono22 } from "hono";
26161
- import { createHash as createHash10, randomUUID as randomUUID12 } from "crypto";
26594
+ import { createHash as createHash10, randomUUID as randomUUID13 } from "crypto";
26162
26595
 
26163
26596
  // src/serp-intelligence/page-snapshot-extractor.ts
26164
26597
  import { createHash as createHash9 } from "crypto";
@@ -26944,7 +27377,7 @@ async function logRequestEventBestEffort(input) {
26944
27377
  }
26945
27378
  }
26946
27379
  function captureBillingKeys(userId, suppliedKey, body) {
26947
- const responseKey = suppliedKey?.trim() || randomUUID12();
27380
+ const responseKey = suppliedKey?.trim() || randomUUID13();
26948
27381
  const requestFingerprint3 = createHash10("sha256").update(JSON.stringify(body)).digest("hex");
26949
27382
  const keyDigest = createHash10("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
26950
27383
  return {
@@ -27274,7 +27707,7 @@ var MemoryMcpToolExecutor = class {
27274
27707
  };
27275
27708
 
27276
27709
  // src/api/image-source-artifacts.ts
27277
- import { randomUUID as randomUUID13 } from "crypto";
27710
+ import { randomUUID as randomUUID14 } from "crypto";
27278
27711
  var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
27279
27712
  var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
27280
27713
  var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
@@ -27305,7 +27738,7 @@ async function createImageSourceArtifact(args) {
27305
27738
  if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
27306
27739
  throw new Error("image_source_size_invalid");
27307
27740
  }
27308
- const id = randomUUID13().replaceAll("-", "");
27741
+ const id = randomUUID14().replaceAll("-", "");
27309
27742
  return createPrivateArtifact({
27310
27743
  policy: policy3(),
27311
27744
  ownerId: args.ownerId,
@@ -28050,11 +28483,11 @@ mcpApp.all("/", async (c) => {
28050
28483
  });
28051
28484
 
28052
28485
  // src/api/browser-agent-routes.ts
28053
- import { randomUUID as randomUUID17 } from "crypto";
28486
+ import { randomUUID as randomUUID18 } from "crypto";
28054
28487
  import { Hono as Hono24 } from "hono";
28055
28488
 
28056
28489
  // src/api/browser-agent-db.ts
28057
- import { randomUUID as randomUUID14 } from "crypto";
28490
+ import { randomUUID as randomUUID15 } from "crypto";
28058
28491
  var _ready = false;
28059
28492
  var _migrationPromise = null;
28060
28493
  async function migrateBrowserAgent() {
@@ -28185,7 +28618,7 @@ async function runBrowserAgentMigration() {
28185
28618
  }
28186
28619
  async function createExtensionRow(input) {
28187
28620
  const db = getDb();
28188
- const id = `bext_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
28621
+ const id = `bext_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28189
28622
  await db.execute({
28190
28623
  sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
28191
28624
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
@@ -28220,7 +28653,7 @@ async function deleteExtensionRow(userId, name) {
28220
28653
  }
28221
28654
  async function createAuthConnectionRow(input) {
28222
28655
  const db = getDb();
28223
- const connectionId = `authc_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
28656
+ const connectionId = `authc_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28224
28657
  await db.execute({
28225
28658
  sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
28226
28659
  VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
@@ -28305,7 +28738,7 @@ async function deleteProfileLabel(userId, profile) {
28305
28738
  }
28306
28739
  async function createSessionRow(input) {
28307
28740
  const db = getDb();
28308
- const id = `bas_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
28741
+ const id = `bas_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28309
28742
  await db.execute({
28310
28743
  sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
28311
28744
  VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
@@ -28383,7 +28816,7 @@ async function recordAction(input) {
28383
28816
  sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
28384
28817
  VALUES (?, ?, ?, ?, ?, ?)`,
28385
28818
  args: [
28386
- `baa_${randomUUID14().replace(/-/g, "").slice(0, 20)}`,
28819
+ `baa_${randomUUID15().replace(/-/g, "").slice(0, 20)}`,
28387
28820
  input.sessionId,
28388
28821
  input.type,
28389
28822
  input.params == null ? null : JSON.stringify(input.params),
@@ -29494,9 +29927,9 @@ async function replayList(runtimeSessionId) {
29494
29927
  }
29495
29928
 
29496
29929
  // src/api/serp-identity-db.ts
29497
- import { randomUUID as randomUUID15 } from "crypto";
29930
+ import { randomUUID as randomUUID16 } from "crypto";
29498
29931
  async function createSerpIdentityRow(input) {
29499
- const id = `serpi_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
29932
+ const id = `serpi_${randomUUID16().replace(/-/g, "").slice(0, 20)}`;
29500
29933
  await getDb().execute({
29501
29934
  sql: `INSERT INTO serp_identities
29502
29935
  (id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
@@ -29543,7 +29976,7 @@ async function deleteSerpIdentityRow(userId, name) {
29543
29976
  }
29544
29977
 
29545
29978
  // src/api/serp-identity-service.ts
29546
- import { createHash as createHash11, randomUUID as randomUUID16 } from "crypto";
29979
+ import { createHash as createHash11, randomUUID as randomUUID17 } from "crypto";
29547
29980
  import Kernel5 from "@onkernel/sdk";
29548
29981
  var MAX_SERP_IDENTITIES_PER_USER = 3;
29549
29982
  function kernelClient() {
@@ -29553,7 +29986,7 @@ function kernelClient() {
29553
29986
  }
29554
29987
  function backendName(userId, name, resource) {
29555
29988
  const digest2 = createHash11("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
29556
- const nonce = randomUUID16().replace(/-/g, "").slice(0, 8);
29989
+ const nonce = randomUUID17().replace(/-/g, "").slice(0, 8);
29557
29990
  return `mcp-serp-${resource}-${digest2}-${nonce}`;
29558
29991
  }
29559
29992
  function isNotFound2(error) {
@@ -30339,7 +30772,7 @@ function buildBrowserAgentRoutes() {
30339
30772
  }
30340
30773
  const existing = await getExtensionRow(user.id, name);
30341
30774
  if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
30342
- const backendName2 = `u${user.id}_${randomUUID17().replace(/-/g, "")}`;
30775
+ const backendName2 = `u${user.id}_${randomUUID18().replace(/-/g, "")}`;
30343
30776
  try {
30344
30777
  const imported = await importExtensionFromStore(storeUrl, backendName2);
30345
30778
  const row = await createExtensionRow({
@@ -30892,7 +31325,7 @@ async function resolveUser3(customerId, emailFallback) {
30892
31325
  // src/api/oauth-routes.ts
30893
31326
  import { Hono as Hono26 } from "hono";
30894
31327
  import { getCookie, setCookie } from "hono/cookie";
30895
- import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID18 } from "crypto";
31328
+ import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID19 } from "crypto";
30896
31329
  import { importPKCS8, exportJWK, SignJWT } from "jose";
30897
31330
  var ISSUER = "https://mcpscraper.dev";
30898
31331
  var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
@@ -31277,7 +31710,7 @@ function pkceMatches(verifier, challenge) {
31277
31710
  }
31278
31711
  async function mintAccessToken(identity, scope, plan, audience) {
31279
31712
  const { privateKey, kid } = await getKeys();
31280
- return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID18()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
31713
+ return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID19()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
31281
31714
  }
31282
31715
  function tokenErrorResponse(c, error, description, status) {
31283
31716
  return c.json({ error, error_description: description }, status);
@@ -33727,7 +34160,7 @@ function renderWorkshopFieldNotesBody() {
33727
34160
  }
33728
34161
 
33729
34162
  // src/api/workshop-registration-repository.ts
33730
- import { randomUUID as randomUUID19 } from "crypto";
34163
+ import { randomUUID as randomUUID20 } from "crypto";
33731
34164
  function rowToRegistration(row) {
33732
34165
  return {
33733
34166
  id: String(row.id),
@@ -33796,7 +34229,7 @@ async function claimWorkshopRegistration(input) {
33796
34229
  ) < ?
33797
34230
  `,
33798
34231
  args: [
33799
- randomUUID19(),
34232
+ randomUUID20(),
33800
34233
  input.eventSlug,
33801
34234
  input.email,
33802
34235
  input.firstName,
@@ -33813,7 +34246,7 @@ async function claimWorkshopRegistration(input) {
33813
34246
  id, event_slug, email, first_name, last_name, status
33814
34247
  ) VALUES (?, ?, ?, ?, ?, 'waitlisted')
33815
34248
  `,
33816
- args: [randomUUID19(), input.eventSlug, input.email, input.firstName, input.lastName || null]
34249
+ args: [randomUUID20(), input.eventSlug, input.email, input.firstName, input.lastName || null]
33817
34250
  });
33818
34251
  const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
33819
34252
  if (!waitlisted) throw new Error("workshop registration could not be claimed");
@@ -33828,7 +34261,7 @@ async function registerWorkshopInterest(input) {
33828
34261
  id, event_slug, email, first_name, last_name, status
33829
34262
  ) VALUES (?, ?, ?, ?, ?, 'interested')
33830
34263
  `,
33831
- args: [randomUUID19(), input.eventSlug, input.email, input.firstName, input.lastName || null]
34264
+ args: [randomUUID20(), input.eventSlug, input.email, input.firstName, input.lastName || null]
33832
34265
  });
33833
34266
  const registration = await getWorkshopRegistration(input.eventSlug, input.email);
33834
34267
  if (!registration) throw new Error("workshop interest could not be recorded");
@@ -34631,7 +35064,7 @@ import { Hono as Hono33 } from "hono";
34631
35064
  import { z as z33 } from "zod";
34632
35065
 
34633
35066
  // src/api/commons-repository.ts
34634
- import { createHash as createHash17, randomUUID as randomUUID20 } from "crypto";
35067
+ import { createHash as createHash17, randomUUID as randomUUID21 } from "crypto";
34635
35068
 
34636
35069
  // src/api/commons-image-store.ts
34637
35070
  import { createHash as createHash15 } from "crypto";
@@ -34640,20 +35073,29 @@ import { isIP } from "net";
34640
35073
  var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
34641
35074
  var COMMONS_IMAGE_MAX_REDIRECTS = 3;
34642
35075
  var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
35076
+ var COMMONS_IMAGE_PAGE_MAX_BYTES = 2 * 1024 * 1024;
35077
+ var COMMONS_IMAGE_MAX_PAGE_CANDIDATES = 12;
35078
+ var COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with imageBase64, an owner-authorized artifactId, a public HTTPS image URL, or a public HTTPS webpage containing a usable featured image. Replace featuredImage.url with the returned permanent URL, validate again, then retry commons_submit_entity with the same idempotency key.";
34643
35079
  var CommonsImageError = class extends Error {
34644
- constructor(code, message, httpStatus = 400) {
35080
+ constructor(code, message, httpStatus = 400, retryable = false, retryGuidance = COMMONS_IMAGE_RETRY_GUIDANCE) {
34645
35081
  super(message);
34646
35082
  this.code = code;
34647
35083
  this.httpStatus = httpStatus;
35084
+ this.retryable = retryable;
35085
+ this.retryGuidance = retryGuidance;
34648
35086
  this.name = "CommonsImageError";
34649
35087
  }
34650
35088
  code;
34651
35089
  httpStatus;
35090
+ retryable;
35091
+ retryGuidance;
34652
35092
  };
34653
35093
  var imageSchemaReady = false;
35094
+ var imageSchemaDb = null;
34654
35095
  async function ensureCommonsImageSchema() {
34655
- if (imageSchemaReady) return;
34656
- await getDb().execute(`
35096
+ const db = getDb();
35097
+ if (imageSchemaReady && imageSchemaDb === db) return;
35098
+ await db.execute(`
34657
35099
  CREATE TABLE IF NOT EXISTS commons_images (
34658
35100
  id TEXT PRIMARY KEY,
34659
35101
  user_id INTEGER NOT NULL,
@@ -34669,8 +35111,9 @@ async function ensureCommonsImageSchema() {
34669
35111
  created_at TEXT NOT NULL DEFAULT (datetime('now'))
34670
35112
  )
34671
35113
  `);
34672
- await getDb().execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
35114
+ await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
34673
35115
  imageSchemaReady = true;
35116
+ imageSchemaDb = db;
34674
35117
  }
34675
35118
  function sniffImageFormat(bytes) {
34676
35119
  if (bytes.length < 12) return null;
@@ -34700,44 +35143,85 @@ async function assertPublicHttpsUrl(rawUrl) {
34700
35143
  try {
34701
35144
  url = new URL(rawUrl);
34702
35145
  } catch {
34703
- throw new CommonsImageError("image_url_invalid", `"${rawUrl}" is not a valid URL.`);
35146
+ throw new CommonsImageError("image_url_invalid", "The image source is not a valid URL.");
34704
35147
  }
34705
35148
  if (url.protocol !== "https:") {
34706
35149
  throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
34707
35150
  }
35151
+ if (url.username || url.password) {
35152
+ throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
35153
+ }
34708
35154
  const hostname = url.hostname.replace(/^\[|\]$/g, "");
34709
35155
  const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
34710
35156
  if (!addresses.length) {
34711
- throw new CommonsImageError("image_host_unresolvable", `Could not resolve ${url.hostname}.`);
35157
+ throw new CommonsImageError("image_host_unresolvable", "The image source host could not be resolved.");
34712
35158
  }
34713
35159
  if (addresses.some((entry) => isBlockedAddress(entry.address))) {
34714
35160
  throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
34715
35161
  }
34716
35162
  return url;
34717
35163
  }
34718
- async function downloadImage(sourceUrl) {
35164
+ async function readResponseBytes(response, maxBytes) {
35165
+ const declaredLength = Number(response.headers.get("content-length") ?? "0");
35166
+ if (declaredLength > maxBytes) {
35167
+ throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
35168
+ }
35169
+ if (!response.body) return Buffer.alloc(0);
35170
+ const reader = response.body.getReader();
35171
+ const chunks = [];
35172
+ let total = 0;
35173
+ while (true) {
35174
+ const { done, value } = await reader.read();
35175
+ if (done) break;
35176
+ total += value.byteLength;
35177
+ if (total > maxBytes) {
35178
+ await reader.cancel().catch(() => void 0);
35179
+ throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
35180
+ }
35181
+ chunks.push(Buffer.from(value));
35182
+ }
35183
+ return Buffer.concat(chunks, total);
35184
+ }
35185
+ async function downloadPublicSource(sourceUrl) {
34719
35186
  let current = sourceUrl;
34720
35187
  for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
34721
35188
  const url = await assertPublicHttpsUrl(current);
34722
- const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
35189
+ let response;
35190
+ try {
35191
+ response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
35192
+ } catch (error) {
35193
+ if (error instanceof CommonsImageError) throw error;
35194
+ throw new CommonsImageError(
35195
+ "image_source_unavailable",
35196
+ "The image source could not be reached by Commons.",
35197
+ 422,
35198
+ true
35199
+ );
35200
+ }
34723
35201
  if (response.status >= 300 && response.status < 400) {
34724
35202
  const location2 = response.headers.get("location");
34725
35203
  if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
34726
- current = new URL(location2, url).toString();
35204
+ try {
35205
+ current = new URL(location2, url).toString();
35206
+ } catch {
35207
+ throw new CommonsImageError("image_redirect_invalid", "Image source redirected to an invalid destination.");
35208
+ }
34727
35209
  continue;
34728
35210
  }
34729
35211
  if (!response.ok) {
34730
- throw new CommonsImageError("image_source_unavailable", `Image source returned HTTP ${response.status}.`, 422);
34731
- }
34732
- const declaredLength = Number(response.headers.get("content-length") ?? "0");
34733
- if (declaredLength > COMMONS_IMAGE_MAX_BYTES) {
34734
- throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
34735
- }
34736
- const bytes = Buffer.from(await response.arrayBuffer());
34737
- if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
34738
- throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
35212
+ const retryable = response.status === 408 || response.status === 425 || response.status === 429 || response.status >= 500;
35213
+ throw new CommonsImageError(
35214
+ "image_source_unavailable",
35215
+ `The image source returned HTTP ${response.status} and could not be hosted.`,
35216
+ 422,
35217
+ retryable
35218
+ );
34739
35219
  }
34740
- return bytes;
35220
+ return {
35221
+ bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
35222
+ finalUrl: url.toString(),
35223
+ contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
35224
+ };
34741
35225
  }
34742
35226
  throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
34743
35227
  }
@@ -34750,21 +35234,161 @@ function decodeBase64Image(value) {
34750
35234
  }
34751
35235
  return bytes;
34752
35236
  }
35237
+ function safeProvenanceUrl(rawUrl) {
35238
+ try {
35239
+ const url = new URL(rawUrl);
35240
+ url.username = "";
35241
+ url.password = "";
35242
+ url.search = "";
35243
+ url.hash = "";
35244
+ return url.toString();
35245
+ } catch {
35246
+ return null;
35247
+ }
35248
+ }
35249
+ function looksLikeHtml(bytes, declaredType) {
35250
+ if (declaredType === "text/html" || declaredType === "application/xhtml+xml") return true;
35251
+ const prefix = bytes.subarray(0, 512).toString("utf8").trimStart().toLowerCase();
35252
+ return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
35253
+ }
35254
+ async function resolveCommonsImageSource(input) {
35255
+ const supplied = [input.sourceUrl, input.imageBase64, input.artifactId].filter((value) => Boolean(value)).length;
35256
+ if (supplied === 0) {
35257
+ throw new CommonsImageError("image_source_missing", "Provide exactly one of sourceUrl, imageBase64, or artifactId.");
35258
+ }
35259
+ if (supplied !== 1) {
35260
+ throw new CommonsImageError("image_source_ambiguous", "Provide only one image source: sourceUrl, imageBase64, or artifactId.");
35261
+ }
35262
+ if (input.sourceUrl && input.sourceType) {
35263
+ throw new CommonsImageError("image_source_type_invalid", "sourceType applies only to imageBase64 or artifactId, not sourceUrl.");
35264
+ }
35265
+ if (input.imageBase64) {
35266
+ return {
35267
+ bytes: decodeBase64Image(input.imageBase64),
35268
+ sourceUrl: null,
35269
+ sourcePageUrl: null,
35270
+ sourceKind: input.sourceType ?? "uploaded",
35271
+ selectionSource: "provided_bytes"
35272
+ };
35273
+ }
35274
+ if (input.artifactId) {
35275
+ const bytes = await readOwnedImageSourceArtifact({ artifactId: input.artifactId, ownerId: String(input.userId) });
35276
+ if (!bytes) {
35277
+ throw new CommonsImageError("image_artifact_not_found", "The image artifact was not found, expired, or is not owned by this account.", 422);
35278
+ }
35279
+ if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
35280
+ throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
35281
+ }
35282
+ return {
35283
+ bytes,
35284
+ sourceUrl: null,
35285
+ sourcePageUrl: null,
35286
+ sourceKind: input.sourceType ?? "artifact",
35287
+ selectionSource: "artifact"
35288
+ };
35289
+ }
35290
+ const requestedUrl = String(input.sourceUrl);
35291
+ const downloaded = await downloadPublicSource(requestedUrl);
35292
+ if (sniffImageFormat(downloaded.bytes)) {
35293
+ return {
35294
+ bytes: downloaded.bytes,
35295
+ sourceUrl: safeProvenanceUrl(downloaded.finalUrl),
35296
+ sourcePageUrl: null,
35297
+ sourceKind: "direct_url",
35298
+ selectionSource: "direct_url"
35299
+ };
35300
+ }
35301
+ if (!looksLikeHtml(downloaded.bytes, downloaded.contentType)) {
35302
+ throw new CommonsImageError(
35303
+ "image_format_unsupported",
35304
+ "The supplied URL returned neither a supported image nor an HTML webpage with a discoverable image.",
35305
+ 415
35306
+ );
35307
+ }
35308
+ if (downloaded.bytes.length > COMMONS_IMAGE_PAGE_MAX_BYTES) {
35309
+ throw new CommonsImageError("image_page_too_large", `Webpage exceeds the ${COMMONS_IMAGE_PAGE_MAX_BYTES} byte discovery limit.`, 413);
35310
+ }
35311
+ const html = downloaded.bytes.toString("utf8");
35312
+ const document2 = loadHtmlDocument(html);
35313
+ const candidates = resolveFeaturedImageCandidates({
35314
+ html,
35315
+ meta: document2.meta,
35316
+ schema: document2.schema,
35317
+ pageUrl: downloaded.finalUrl
35318
+ }).slice(0, COMMONS_IMAGE_MAX_PAGE_CANDIDATES);
35319
+ if (!candidates.length) {
35320
+ throw new CommonsImageError("page_image_not_found", "The webpage did not expose a usable featured or content image.", 422);
35321
+ }
35322
+ let sawRetryableFailure = false;
35323
+ for (const candidate of candidates) {
35324
+ try {
35325
+ const image = await downloadPublicSource(candidate.url);
35326
+ if (!sniffImageFormat(image.bytes)) continue;
35327
+ return {
35328
+ bytes: image.bytes,
35329
+ sourceUrl: safeProvenanceUrl(image.finalUrl),
35330
+ sourcePageUrl: safeProvenanceUrl(downloaded.finalUrl),
35331
+ sourceKind: "webpage",
35332
+ selectionSource: candidate.source
35333
+ };
35334
+ } catch (error) {
35335
+ if (error instanceof CommonsImageError && error.retryable) sawRetryableFailure = true;
35336
+ }
35337
+ }
35338
+ throw new CommonsImageError(
35339
+ "page_image_candidates_unusable",
35340
+ "The webpage exposed image candidates, but none returned supported public image bytes.",
35341
+ 422,
35342
+ sawRetryableFailure
35343
+ );
35344
+ }
34753
35345
  function commonsImageBlobStore() {
34754
35346
  const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
34755
35347
  return token6 ? new VercelBlobStore(token6) : getBlobStore();
34756
35348
  }
34757
- function isCommonsHostedImageUrl(url) {
34758
- return /^https:\/\/[^/]*(\.public\.blob\.vercel-storage\.com|blob\.vercel-storage\.com)\//i.test(url);
35349
+ function normalizeCommonsImageUrl(rawUrl) {
35350
+ try {
35351
+ const url = new URL(rawUrl.trim());
35352
+ if (url.protocol !== "https:") return null;
35353
+ url.hash = "";
35354
+ return url.toString();
35355
+ } catch {
35356
+ return null;
35357
+ }
35358
+ }
35359
+ async function findCommonsHostedImageByUrl(rawUrl) {
35360
+ const url = normalizeCommonsImageUrl(rawUrl);
35361
+ if (!url) return null;
35362
+ await ensureCommonsImageSchema();
35363
+ const result = await getDb().execute({
35364
+ sql: "SELECT id, url, content_type, bytes, digest, source_url FROM commons_images WHERE url = ? LIMIT 1",
35365
+ args: [url]
35366
+ });
35367
+ const row = result.rows[0];
35368
+ if (!row) return null;
35369
+ return {
35370
+ id: String(row.id),
35371
+ url: String(row.url),
35372
+ contentType: String(row.content_type),
35373
+ bytes: Number(row.bytes),
35374
+ digest: String(row.digest),
35375
+ sourceUrl: row.source_url ? String(row.source_url) : null,
35376
+ sourcePageUrl: null,
35377
+ sourceKind: "registered",
35378
+ selectionSource: "registered",
35379
+ reused: true
35380
+ };
34759
35381
  }
34760
35382
  var hostOverride = null;
34761
35383
  async function hostCommonsImage(input) {
34762
35384
  if (hostOverride) return hostOverride(input);
34763
35385
  await ensureCommonsImageSchema();
34764
- if (!input.sourceUrl && !input.imageBase64) {
34765
- throw new CommonsImageError("image_source_missing", "Provide either sourceUrl or imageBase64.");
35386
+ if (input.sourceUrl && !input.imageBase64 && !input.artifactId && !input.sourceType) {
35387
+ const registered6 = await findCommonsHostedImageByUrl(input.sourceUrl);
35388
+ if (registered6) return registered6;
34766
35389
  }
34767
- const bytes = input.imageBase64 ? decodeBase64Image(input.imageBase64) : await downloadImage(String(input.sourceUrl));
35390
+ const resolved = await resolveCommonsImageSource(input);
35391
+ const bytes = resolved.bytes;
34768
35392
  const contentType = sniffImageFormat(bytes);
34769
35393
  if (!contentType) {
34770
35394
  throw new CommonsImageError(
@@ -34786,7 +35410,10 @@ async function hostCommonsImage(input) {
34786
35410
  contentType: String(reusable.content_type),
34787
35411
  bytes: Number(reusable.bytes),
34788
35412
  digest: digest2,
34789
- sourceUrl: reusable.source_url ? String(reusable.source_url) : null,
35413
+ sourceUrl: resolved.sourceUrl,
35414
+ sourcePageUrl: resolved.sourcePageUrl,
35415
+ sourceKind: resolved.sourceKind,
35416
+ selectionSource: resolved.selectionSource,
34790
35417
  reused: true
34791
35418
  };
34792
35419
  }
@@ -34815,19 +35442,31 @@ async function hostCommonsImage(input) {
34815
35442
  stored.url,
34816
35443
  contentType,
34817
35444
  bytes.length,
34818
- input.sourceUrl ?? null,
35445
+ resolved.sourceUrl,
34819
35446
  input.alt ?? null,
34820
35447
  input.license ?? null,
34821
35448
  input.attribution ?? null,
34822
35449
  (/* @__PURE__ */ new Date()).toISOString()
34823
35450
  ]
34824
35451
  });
34825
- return { id, url: stored.url, contentType, bytes: bytes.length, digest: digest2, sourceUrl: input.sourceUrl ?? null, reused: false };
35452
+ return {
35453
+ id,
35454
+ url: stored.url,
35455
+ contentType,
35456
+ bytes: bytes.length,
35457
+ digest: digest2,
35458
+ sourceUrl: resolved.sourceUrl,
35459
+ sourcePageUrl: resolved.sourcePageUrl,
35460
+ sourceKind: resolved.sourceKind,
35461
+ selectionSource: resolved.selectionSource,
35462
+ reused: false
35463
+ };
34826
35464
  }
34827
35465
  async function hostEntityImages(input) {
34828
35466
  const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
34829
35467
  const featuredUrl = input.featuredImage?.url;
34830
- if (featuredUrl && !isCommonsHostedImageUrl(featuredUrl)) {
35468
+ const registeredFeatured = featuredUrl ? await findCommonsHostedImageByUrl(featuredUrl) : null;
35469
+ if (featuredUrl && !registeredFeatured) {
34831
35470
  const hosted = await hostCommonsImage({
34832
35471
  userId: input.userId,
34833
35472
  sourceUrl: featuredUrl,
@@ -34835,17 +35474,19 @@ async function hostEntityImages(input) {
34835
35474
  license: input.featuredImage?.license,
34836
35475
  attribution: input.featuredImage?.sourceUrl
34837
35476
  }).catch((error) => {
34838
- const message = error instanceof Error ? error.message : String(error);
35477
+ if (error instanceof CommonsImageError) throw error;
34839
35478
  throw new CommonsImageError(
34840
- error instanceof CommonsImageError ? error.code : "featured_image_unhostable",
34841
- `The featured image could not be hosted, so this entity would publish with a link that can break: ${message}`,
35479
+ "featured_image_unhostable",
35480
+ "The featured image could not be hosted, so the entity was not persisted with a link that could break.",
34842
35481
  422
34843
35482
  );
34844
35483
  });
34845
35484
  rewrite.featuredImageUrl = hosted.url;
35485
+ rewrite.mediaUrls.set(featuredUrl, hosted.url);
34846
35486
  }
34847
35487
  for (const item of input.media ?? []) {
34848
- if (!item.url || isCommonsHostedImageUrl(item.url)) continue;
35488
+ if (item.url && rewrite.mediaUrls.has(item.url)) continue;
35489
+ if (!item.url || await findCommonsHostedImageByUrl(item.url)) continue;
34849
35490
  try {
34850
35491
  const hosted = await hostCommonsImage({
34851
35492
  userId: input.userId,
@@ -35213,7 +35854,7 @@ var COMMONS_ENTITY_PROFILES = {
35213
35854
  ]
35214
35855
  }
35215
35856
  };
35216
- var schemaReady = false;
35857
+ var schemaReady2 = false;
35217
35858
  var COMMONS_SCHEMA_OBJECTS = [
35218
35859
  "schema_migrations",
35219
35860
  "commons_entities",
@@ -35240,9 +35881,9 @@ var COMMONS_SCHEMA_OBJECTS = [
35240
35881
  "commons_publication_editions_publication_published"
35241
35882
  ];
35242
35883
  async function ensureCommonsSchema() {
35243
- if (schemaReady) return;
35884
+ if (schemaReady2) return;
35244
35885
  if (await schemaObjectsPresent(COMMONS_SCHEMA_OBJECTS)) {
35245
- schemaReady = true;
35886
+ schemaReady2 = true;
35246
35887
  return;
35247
35888
  }
35248
35889
  const db = getDb();
@@ -35465,7 +36106,7 @@ async function ensureCommonsSchema() {
35465
36106
  args: [COMMONS_SCHEMA_VERSION]
35466
36107
  }
35467
36108
  ], "write");
35468
- schemaReady = true;
36109
+ schemaReady2 = true;
35469
36110
  }
35470
36111
  async function searchCommonsEntities(filters = {}, userId) {
35471
36112
  await ensureCommonsSchema();
@@ -35674,7 +36315,7 @@ async function prepareCommonsEntity(input, user) {
35674
36315
  normalizedEntityType: entityType,
35675
36316
  proposedSlug: slug4 || null,
35676
36317
  publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
35677
- writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
36318
+ writePath: "commons_prepare_entity -> compose the entity -> commons_host_image -> commons_validate_entity -> commons_submit_entity"
35678
36319
  },
35679
36320
  contract,
35680
36321
  billing: {
@@ -35696,8 +36337,8 @@ async function prepareCommonsEntity(input, user) {
35696
36337
  "Search likely duplicates before submitting a new entity. If one matches, edit it with entityId and baseRevision instead of creating another page.",
35697
36338
  "Use the returned profile as a contract, not a rigid empty template: do not include History, Reception, Pricing, Timeline, or similar sections unless source evidence supports them.",
35698
36339
  "Write neutral encyclopedia prose. Do not paste raw scrape Markdown, marketing layout fragments, or empty/filler headings.",
35699
- "Attach a featured image for auto-publish eligibility, preserve original URL/source byline/relCanonical when republishing, and put media assets in the media array.",
35700
- "Call commons_validate_entity with the composed payload before commons_submit_entity.",
36340
+ "For a featured image not already hosted by Commons, call commons_host_image and replace featuredImage.url with its returned permanent URL. Preserve original URL/source byline/relCanonical when republishing, and put media assets in the media array.",
36341
+ "Call commons_validate_entity with the composed, image-ready payload before commons_submit_entity.",
35701
36342
  "Validation is free. Submitting a new entity costs 10 Credits; submitting an edit to an existing entity costs 2 Credits."
35702
36343
  ]
35703
36344
  };
@@ -35712,6 +36353,8 @@ async function validateCommonsEntity(input, user) {
35712
36353
  const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
35713
36354
  const source = normalizeSource(input.source);
35714
36355
  const featuredImage = normalizeFeaturedImage(input.featuredImage);
36356
+ const normalizedFeaturedImageUrl = featuredImage?.url ? normalizeCommonsImageUrl(featuredImage.url) : null;
36357
+ const registeredFeaturedImage = normalizedFeaturedImageUrl ? await findCommonsHostedImageByUrl(normalizedFeaturedImageUrl) : null;
35715
36358
  const bodyMarkdown = safeCandidateBody(input.bodyMarkdown, input.contentSections);
35716
36359
  const headings = extractMarkdownHeadings(bodyMarkdown);
35717
36360
  const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
@@ -35749,15 +36392,44 @@ async function validateCommonsEntity(input, user) {
35749
36392
  if (input.reviewPolicy === "always_review") {
35750
36393
  warnings.push("reviewPolicy is always_review, so the proposal will not auto-publish even if the payload is otherwise valid.");
35751
36394
  }
36395
+ if (featuredImage?.url && !registeredFeaturedImage) {
36396
+ warnings.push("The featured image is not registered in permanent Commons image storage. Call commons_host_image, replace featuredImage.url with its returned URL, and validate again before submitting.");
36397
+ }
35752
36398
  if (user?.id == null) warnings.push("No authenticated user was supplied to validation; submit still requires API-key authenticated user context.");
35753
36399
  const valid = errors.length === 0;
35754
- const autoPublishEligible = valid && input.reviewPolicy !== "always_review";
36400
+ const imageReadyForPublish = Boolean(registeredFeaturedImage);
36401
+ const publishable = valid && imageReadyForPublish;
36402
+ const autoPublishEligible = publishable && input.reviewPolicy !== "always_review";
35755
36403
  const tagVocabulary = await commonsTagVocabulary();
35756
36404
  return {
35757
36405
  ok: true,
35758
36406
  valid,
35759
- publishable: valid,
36407
+ publishable,
35760
36408
  autoPublishEligible,
36409
+ imageDiagnostics: {
36410
+ featuredImage: registeredFeaturedImage ? {
36411
+ state: "commons_hosted",
36412
+ url: registeredFeaturedImage.url,
36413
+ commonsImageId: registeredFeaturedImage.id,
36414
+ readyForPublish: true,
36415
+ submitWillAttemptAutoHost: false,
36416
+ requiredAction: null
36417
+ } : featuredImage?.url ? {
36418
+ state: "external_unhosted",
36419
+ url: normalizedFeaturedImageUrl,
36420
+ commonsImageId: null,
36421
+ readyForPublish: false,
36422
+ submitWillAttemptAutoHost: true,
36423
+ requiredAction: "Call commons_host_image, replace featuredImage.url with its returned permanent URL, and validate again."
36424
+ } : {
36425
+ state: "missing",
36426
+ url: null,
36427
+ commonsImageId: null,
36428
+ readyForPublish: false,
36429
+ submitWillAttemptAutoHost: false,
36430
+ requiredAction: "Add a featured image by calling commons_host_image, then set featuredImage.url to its returned permanent URL."
36431
+ }
36432
+ },
35761
36433
  errors,
35762
36434
  warnings,
35763
36435
  contract,
@@ -35838,7 +36510,7 @@ async function submitCommonsEntity(input, user) {
35838
36510
  const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
35839
36511
  const safety = evaluatePublishSafety(normalized, existing);
35840
36512
  const shouldApply = safety.safe && normalized.reviewPolicy !== "always_review";
35841
- const proposalId = `commons-proposal-${randomUUID20()}`;
36513
+ const proposalId = `commons-proposal-${randomUUID21()}`;
35842
36514
  const now = (/* @__PURE__ */ new Date()).toISOString();
35843
36515
  const entityId = existing?.id ?? normalized.entityId ?? allocateEntityId();
35844
36516
  const proposalStatus = shouldApply ? "accepted" : "pending_review";
@@ -35893,7 +36565,7 @@ async function submitCommonsEntity(input, user) {
35893
36565
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
35894
36566
  `,
35895
36567
  args: [
35896
- `commons-ledger-${randomUUID20()}`,
36568
+ `commons-ledger-${randomUUID21()}`,
35897
36569
  nextEntity.id,
35898
36570
  proposalId,
35899
36571
  user.id,
@@ -35999,7 +36671,7 @@ async function saveCommonsFilter(input, user) {
35999
36671
  sql: "SELECT id FROM commons_saved_filters WHERE user_id = ? AND name = ? LIMIT 1",
36000
36672
  args: [user.id, name]
36001
36673
  });
36002
- const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${randomUUID20()}`;
36674
+ const id = existing.rows[0]?.id != null ? String(existing.rows[0].id) : `commons-filter-${randomUUID21()}`;
36003
36675
  await getDb().execute({
36004
36676
  sql: `
36005
36677
  INSERT INTO commons_saved_filters (id, user_id, name, description, filter_json, created_at, updated_at)
@@ -37355,7 +38027,7 @@ var CommonsRepositoryError = class extends Error {
37355
38027
  };
37356
38028
 
37357
38029
  // src/api/commons-publication-repository.ts
37358
- import { createHash as createHash18, randomUUID as randomUUID21 } from "crypto";
38030
+ import { createHash as createHash18, randomUUID as randomUUID22 } from "crypto";
37359
38031
 
37360
38032
  // src/editorial-reading-room/article-page.ts
37361
38033
  function escapeHtml6(value) {
@@ -37568,7 +38240,7 @@ async function claimCommonsPublication(input, user) {
37568
38240
  }
37569
38241
  const existingName = await getCommonsPublicationBySubdomain(subdomain);
37570
38242
  if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
37571
- const id = `tcpub_${randomUUID21()}`;
38243
+ const id = `tcpub_${randomUUID22()}`;
37572
38244
  const now = (/* @__PURE__ */ new Date()).toISOString();
37573
38245
  try {
37574
38246
  await getDb().execute({
@@ -37620,7 +38292,7 @@ async function publishCommonsEditorial(input, user) {
37620
38292
  const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
37621
38293
  const rendered = renderEditorialReadingRoom(editionInput);
37622
38294
  const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
37623
- const editionId = `tced_${randomUUID21()}`;
38295
+ const editionId = `tced_${randomUUID22()}`;
37624
38296
  const revision = (latest?.revision ?? 0) + 1;
37625
38297
  const now = (/* @__PURE__ */ new Date()).toISOString();
37626
38298
  await getDb().batch([
@@ -37702,7 +38374,7 @@ async function updateCommonsEditorialArticle(input, user) {
37702
38374
  const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
37703
38375
  const rendered = renderEditorialReadingRoom(editionInput);
37704
38376
  const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
37705
- const editionId = `tced_${randomUUID21()}`;
38377
+ const editionId = `tced_${randomUUID22()}`;
37706
38378
  const revision = latest.revision + 1;
37707
38379
  const now = (/* @__PURE__ */ new Date()).toISOString();
37708
38380
  await getDb().batch([
@@ -38222,11 +38894,13 @@ commonsApp.get("/publications/:subdomain", async (c) => {
38222
38894
  var HostImageSchema = z33.object({
38223
38895
  sourceUrl: z33.string().url().optional(),
38224
38896
  imageBase64: z33.string().min(16).optional(),
38897
+ artifactId: z33.string().min(1).max(500).optional(),
38898
+ sourceType: z33.enum(["uploaded", "ai_generated"]).optional(),
38225
38899
  alt: z33.string().max(500).optional(),
38226
38900
  license: z33.string().max(240).optional(),
38227
38901
  attribution: z33.string().max(500).optional()
38228
- }).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
38229
- message: "Provide either sourceUrl or imageBase64."
38902
+ }).strict().refine((body) => [body.sourceUrl, body.imageBase64, body.artifactId].filter(Boolean).length === 1, {
38903
+ message: "Provide exactly one of sourceUrl, imageBase64, or artifactId."
38230
38904
  });
38231
38905
  commonsApp.post("/images", auth2, async (c) => {
38232
38906
  const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
@@ -38236,7 +38910,7 @@ commonsApp.post("/images", auth2, async (c) => {
38236
38910
  return c.json({ ok: true, data: hosted }, hosted.reused ? 200 : 201);
38237
38911
  } catch (error) {
38238
38912
  if (error instanceof CommonsImageError) {
38239
- return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
38913
+ return imageErrorResponse(c, error);
38240
38914
  }
38241
38915
  throw error;
38242
38916
  }
@@ -38300,7 +38974,14 @@ commonsApp.post("/entities/propose", auth2, async (c) => {
38300
38974
  }
38301
38975
  return c.json({ ok: true, data: result, billing: publicListingWriteBilling(billing) }, 201);
38302
38976
  } catch (error) {
38303
- if (billing) await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
38977
+ let chargeStatus = billing ? billing.duplicate ? "not_charged" : "refund_pending" : "not_charged";
38978
+ if (billing && !billing.duplicate) {
38979
+ const settlement = await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
38980
+ if (settlement) chargeStatus = "refunded";
38981
+ }
38982
+ if (error instanceof CommonsImageError) {
38983
+ return imageErrorResponse(c, error, chargeStatus);
38984
+ }
38304
38985
  if (error instanceof CommonsRepositoryError) {
38305
38986
  return c.json({ ok: false, error: error.code, message: error.message }, repositoryStatus(error));
38306
38987
  }
@@ -38433,6 +39114,18 @@ function repositoryStatus(error) {
38433
39114
  if (error.httpStatus === 409) return 409;
38434
39115
  return 400;
38435
39116
  }
39117
+ function imageErrorResponse(c, error, chargeStatus) {
39118
+ return c.json({
39119
+ ok: false,
39120
+ error: error.code,
39121
+ error_code: error.code,
39122
+ error_type: "invalid_request",
39123
+ retryable: error.retryable,
39124
+ ...chargeStatus ? { charge_status: chargeStatus } : {},
39125
+ message: `${error.message} ${error.retryGuidance}`,
39126
+ details: { retry_guidance: error.retryGuidance }
39127
+ }, error.httpStatus);
39128
+ }
38436
39129
  async function publicationOperation(c, operation, successStatus = 200) {
38437
39130
  try {
38438
39131
  return c.json({ ok: true, data: await operation() }, successStatus);
@@ -40172,10 +40865,10 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
40172
40865
  }
40173
40866
 
40174
40867
  // src/api/scheduled-run-view-links.ts
40175
- import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as randomUUID22 } from "crypto";
40176
- var schemaReady2 = false;
40868
+ import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as randomUUID23 } from "crypto";
40869
+ var schemaReady3 = false;
40177
40870
  async function ensureScheduledRunViewLinksSchema() {
40178
- if (schemaReady2) return;
40871
+ if (schemaReady3) return;
40179
40872
  const db = getDb();
40180
40873
  await db.execute(`
40181
40874
  CREATE TABLE IF NOT EXISTS scheduled_run_view_links (
@@ -40223,7 +40916,7 @@ async function ensureScheduledRunViewLinksSchema() {
40223
40916
  CREATE INDEX IF NOT EXISTS scheduled_run_view_links_expiry
40224
40917
  ON scheduled_run_view_links(expires_at, revoked_at)
40225
40918
  `);
40226
- schemaReady2 = true;
40919
+ schemaReady3 = true;
40227
40920
  }
40228
40921
  function tokenHash2(token6) {
40229
40922
  return createHash22("sha256").update(token6).digest("hex");
@@ -40248,7 +40941,7 @@ async function createScheduledRunViewLink(input) {
40248
40941
  const now = input.now ?? /* @__PURE__ */ new Date();
40249
40942
  const token6 = randomBytes5(32).toString("base64url");
40250
40943
  const record = {
40251
- shareId: randomUUID22(),
40944
+ shareId: randomUUID23(),
40252
40945
  ownerId: input.ownerId,
40253
40946
  runId: input.runId,
40254
40947
  artifactId: input.artifactId,
@@ -41116,352 +41809,6 @@ import { Hono as Hono37 } from "hono";
41116
41809
 
41117
41810
  // src/api/scheduler-integration-auth.ts
41118
41811
  import { createHash as createHash28, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
41119
-
41120
- // src/api/service-connections.ts
41121
- import { randomUUID as randomUUID23 } from "crypto";
41122
- var schemaReady3 = null;
41123
- var schemaDb4 = null;
41124
- function ensureServiceConnectionsSchema() {
41125
- const currentDb = getDb();
41126
- if (schemaReady3 && schemaDb4 === currentDb) return schemaReady3;
41127
- schemaDb4 = currentDb;
41128
- schemaReady3 = (async () => {
41129
- const db = currentDb;
41130
- await db.execute(`
41131
- CREATE TABLE IF NOT EXISTS service_connections (
41132
- id TEXT PRIMARY KEY,
41133
- user_id INTEGER NOT NULL REFERENCES users(id),
41134
- provider_config_key TEXT NOT NULL,
41135
- provider TEXT,
41136
- transport TEXT NOT NULL,
41137
- upstream_connection_id TEXT,
41138
- label TEXT,
41139
- user_label TEXT,
41140
- provider_account_id TEXT,
41141
- provider_account_email TEXT,
41142
- provider_account_name TEXT,
41143
- provider_identity_checked_at TEXT,
41144
- provider_identity_source_updated_at TEXT,
41145
- lifecycle_status TEXT NOT NULL DEFAULT 'pending',
41146
- operational_status TEXT NOT NULL DEFAULT 'unknown',
41147
- reconnect_required INTEGER NOT NULL DEFAULT 0,
41148
- actions_enabled INTEGER NOT NULL DEFAULT 0,
41149
- read_tools_json TEXT NOT NULL DEFAULT '[]',
41150
- action_tools_json TEXT NOT NULL DEFAULT '[]',
41151
- tool_revision TEXT,
41152
- last_checked_at TEXT,
41153
- last_successful_call_at TEXT,
41154
- last_failure_at TEXT,
41155
- last_failure_code TEXT,
41156
- last_failure_retryable INTEGER,
41157
- consecutive_failures INTEGER NOT NULL DEFAULT 0,
41158
- vault_name TEXT,
41159
- table_name TEXT,
41160
- legacy_owner_identity TEXT,
41161
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
41162
- updated_at TEXT NOT NULL DEFAULT (datetime('now')),
41163
- UNIQUE(user_id, provider_config_key, upstream_connection_id)
41164
- )
41165
- `);
41166
- try {
41167
- await db.execute(`ALTER TABLE service_connections ADD COLUMN user_label TEXT`);
41168
- } catch {
41169
- }
41170
- try {
41171
- await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT`);
41172
- } catch {
41173
- }
41174
- try {
41175
- await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT`);
41176
- } catch {
41177
- }
41178
- try {
41179
- await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT`);
41180
- } catch {
41181
- }
41182
- try {
41183
- await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT`);
41184
- } catch {
41185
- }
41186
- try {
41187
- await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT`);
41188
- } catch {
41189
- }
41190
- await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
41191
- await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
41192
- await db.execute(`
41193
- CREATE TABLE IF NOT EXISTS service_connection_health_events (
41194
- id TEXT PRIMARY KEY,
41195
- connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
41196
- operational_status TEXT NOT NULL,
41197
- failure_code TEXT,
41198
- retryable INTEGER,
41199
- evidence_source TEXT NOT NULL,
41200
- created_at TEXT NOT NULL DEFAULT (datetime('now'))
41201
- )
41202
- `);
41203
- await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
41204
- await db.execute(`
41205
- CREATE TABLE IF NOT EXISTS service_connection_action_audit (
41206
- id TEXT PRIMARY KEY,
41207
- connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
41208
- user_id INTEGER NOT NULL REFERENCES users(id),
41209
- tool TEXT NOT NULL,
41210
- request_id TEXT NOT NULL,
41211
- status TEXT NOT NULL,
41212
- error_code TEXT,
41213
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
41214
- UNIQUE(user_id, request_id)
41215
- )
41216
- `);
41217
- try {
41218
- await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
41219
- } catch {
41220
- }
41221
- try {
41222
- await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
41223
- } catch {
41224
- }
41225
- })().catch((error) => {
41226
- schemaReady3 = null;
41227
- schemaDb4 = null;
41228
- throw error;
41229
- });
41230
- return schemaReady3;
41231
- }
41232
- function parseTools(value) {
41233
- if (typeof value !== "string") return [];
41234
- try {
41235
- const parsed = JSON.parse(value);
41236
- return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
41237
- } catch {
41238
- return [];
41239
- }
41240
- }
41241
- function rowToRecord(row) {
41242
- return {
41243
- id: String(row.id),
41244
- userId: Number(row.user_id),
41245
- providerConfigKey: String(row.provider_config_key),
41246
- provider: typeof row.provider === "string" ? row.provider : null,
41247
- transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
41248
- upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
41249
- label: typeof row.label === "string" ? row.label : null,
41250
- userLabel: typeof row.user_label === "string" ? row.user_label : null,
41251
- providerAccountId: typeof row.provider_account_id === "string" ? row.provider_account_id : null,
41252
- providerAccountEmail: typeof row.provider_account_email === "string" ? row.provider_account_email : null,
41253
- providerAccountName: typeof row.provider_account_name === "string" ? row.provider_account_name : null,
41254
- providerIdentityCheckedAt: typeof row.provider_identity_checked_at === "string" ? row.provider_identity_checked_at : null,
41255
- providerIdentitySourceUpdatedAt: typeof row.provider_identity_source_updated_at === "string" ? row.provider_identity_source_updated_at : null,
41256
- lifecycleStatus: String(row.lifecycle_status),
41257
- operationalStatus: String(row.operational_status),
41258
- reconnectRequired: Number(row.reconnect_required) === 1,
41259
- actionsEnabled: Number(row.actions_enabled) === 1,
41260
- readTools: parseTools(row.read_tools_json),
41261
- actionTools: parseTools(row.action_tools_json),
41262
- toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
41263
- lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
41264
- lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
41265
- lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
41266
- lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
41267
- lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
41268
- consecutiveFailures: Number(row.consecutive_failures ?? 0),
41269
- vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
41270
- tableName: typeof row.table_name === "string" ? row.table_name : null,
41271
- createdAt: String(row.created_at),
41272
- updatedAt: String(row.updated_at)
41273
- };
41274
- }
41275
- async function userIdForIdentity(identity) {
41276
- const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
41277
- if (!user) throw new Error("service_connection_user_not_found");
41278
- return Number(user.id);
41279
- }
41280
- async function reconcileDiscoveredNangoConnections(identity, discovered) {
41281
- await ensureServiceConnectionsSchema();
41282
- const db = getDb();
41283
- const userId = await userIdForIdentity(identity);
41284
- for (const connection of discovered) {
41285
- await db.execute({
41286
- sql: `
41287
- INSERT INTO service_connections (
41288
- id, user_id, provider_config_key, provider, transport, upstream_connection_id,
41289
- label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
41290
- ) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
41291
- ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
41292
- provider = excluded.provider,
41293
- label = COALESCE(excluded.label, service_connections.label),
41294
- lifecycle_status = excluded.lifecycle_status,
41295
- reconnect_required = excluded.reconnect_required,
41296
- updated_at = excluded.updated_at
41297
- `,
41298
- args: [
41299
- randomUUID23(),
41300
- userId,
41301
- connection.providerConfigKey,
41302
- connection.provider,
41303
- connection.upstreamConnectionId,
41304
- connection.label,
41305
- connection.lifecycleStatus,
41306
- connection.reconnectRequired ? 1 : 0,
41307
- identity,
41308
- connection.createdAt,
41309
- connection.updatedAt
41310
- ]
41311
- });
41312
- }
41313
- return listServiceConnections(identity, "nango");
41314
- }
41315
- async function listServiceConnections(identity, transport) {
41316
- await ensureServiceConnectionsSchema();
41317
- const userId = await userIdForIdentity(identity);
41318
- const result = await getDb().execute({
41319
- sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
41320
- args: transport ? [userId, transport] : [userId]
41321
- });
41322
- return result.rows.map((row) => rowToRecord(row));
41323
- }
41324
- async function getOwnedServiceConnection(identity, connectionId) {
41325
- await ensureServiceConnectionsSchema();
41326
- const userId = await userIdForIdentity(identity);
41327
- const result = await getDb().execute({
41328
- sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
41329
- args: [connectionId, userId]
41330
- });
41331
- const row = result.rows[0];
41332
- return row ? rowToRecord(row) : null;
41333
- }
41334
- async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
41335
- await ensureServiceConnectionsSchema();
41336
- await getDb().execute({
41337
- sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
41338
- args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
41339
- });
41340
- }
41341
- async function setServiceConnectionProviderIdentity(args) {
41342
- await ensureServiceConnectionsSchema();
41343
- await getDb().execute({
41344
- sql: `
41345
- UPDATE service_connections SET
41346
- provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
41347
- provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
41348
- updated_at = datetime('now')
41349
- WHERE id = ?
41350
- `,
41351
- args: [
41352
- args.providerAccountId ?? null,
41353
- args.providerAccountEmail?.toLowerCase() ?? null,
41354
- args.providerAccountName ?? null,
41355
- args.sourceUpdatedAt,
41356
- args.connectionId
41357
- ]
41358
- });
41359
- }
41360
- async function setServiceConnectionUserLabel(identity, connectionId, userLabel) {
41361
- const connection = await getOwnedServiceConnection(identity, connectionId);
41362
- if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
41363
- await getDb().execute({
41364
- sql: `UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
41365
- args: [userLabel, connectionId, connection.userId]
41366
- });
41367
- return userLabel;
41368
- }
41369
- async function recordServiceConnectionHealth(args) {
41370
- await ensureServiceConnectionsSchema();
41371
- const now = (/* @__PURE__ */ new Date()).toISOString();
41372
- await getDb().execute({
41373
- sql: `
41374
- UPDATE service_connections SET
41375
- operational_status = ?, last_checked_at = ?,
41376
- last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
41377
- last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
41378
- last_failure_code = ?, last_failure_retryable = ?,
41379
- consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
41380
- updated_at = datetime('now')
41381
- WHERE id = ?
41382
- `,
41383
- args: [
41384
- args.operationalStatus,
41385
- now,
41386
- args.operationalStatus,
41387
- now,
41388
- args.operationalStatus,
41389
- now,
41390
- args.failureCode ?? null,
41391
- args.retryable == null ? null : args.retryable ? 1 : 0,
41392
- args.operationalStatus,
41393
- args.connectionId
41394
- ]
41395
- });
41396
- await getDb().execute({
41397
- sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
41398
- args: [randomUUID23(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
41399
- });
41400
- }
41401
- async function setServiceConnectionActions(identity, connectionId, enabled) {
41402
- const connection = await getOwnedServiceConnection(identity, connectionId);
41403
- if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
41404
- await getDb().execute({
41405
- sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
41406
- args: [enabled ? 1 : 0, connectionId, connection.userId]
41407
- });
41408
- return enabled;
41409
- }
41410
- async function markServiceConnectionDisconnected(identity, connectionId) {
41411
- const connection = await getOwnedServiceConnection(identity, connectionId);
41412
- if (!connection) throw new Error("service_connection_not_found");
41413
- await getDb().execute({
41414
- sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
41415
- args: [connectionId, connection.userId]
41416
- });
41417
- const remaining = await getDb().execute({
41418
- sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
41419
- args: [connection.userId]
41420
- });
41421
- return Number(remaining.rows[0]?.count ?? 0);
41422
- }
41423
- async function recordServiceConnectionAction(args) {
41424
- await ensureServiceConnectionsSchema();
41425
- const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
41426
- if (!connection) throw new Error("service_connection_not_found");
41427
- await getDb().execute({
41428
- sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
41429
- args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
41430
- });
41431
- }
41432
- async function claimServiceConnectionAction(args) {
41433
- await ensureServiceConnectionsSchema();
41434
- const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
41435
- if (!connection) throw new Error("service_connection_not_found");
41436
- const inserted = await getDb().execute({
41437
- sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
41438
- args: [randomUUID23(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
41439
- });
41440
- if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
41441
- const existing = await getDb().execute({
41442
- sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
41443
- args: [connection.userId, args.requestId]
41444
- });
41445
- const row = existing.rows[0];
41446
- const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
41447
- let result = void 0;
41448
- if (!conflict && typeof row?.result_json === "string") {
41449
- try {
41450
- result = JSON.parse(row.result_json);
41451
- } catch {
41452
- result = void 0;
41453
- }
41454
- }
41455
- return {
41456
- claimed: false,
41457
- status: String(row?.status ?? "unknown"),
41458
- errorCode: typeof row?.error_code === "string" ? row.error_code : null,
41459
- conflict,
41460
- ...result !== void 0 ? { result } : {}
41461
- };
41462
- }
41463
-
41464
- // src/api/scheduler-integration-auth.ts
41465
41812
  var MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
41466
41813
  var SchedulerIntegrationAuthError = class extends Error {
41467
41814
  code;
@@ -49707,7 +50054,7 @@ app.get("/cron/tick", async (c) => {
49707
50054
  return c.json({ error: "Unauthorized" }, 401);
49708
50055
  }
49709
50056
  const startedAt = Date.now();
49710
- const { drainQueue } = await import("./worker-UTBUWEXQ.js");
50057
+ const { drainQueue } = await import("./worker-JGWV2LID.js");
49711
50058
  const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
49712
50059
  const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
49713
50060
  const unfinished = [];
@@ -49785,8 +50132,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
49785
50132
  finishExtractJob: finishExtractJob2,
49786
50133
  terminalExtractJobStatus: terminalExtractJobStatus2,
49787
50134
  extractJobLimitInfo: extractJobLimitInfo2
49788
- } = await import("./site-extract-repository-NTM4MDQN.js");
49789
- const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
50135
+ } = await import("./site-extract-repository-7EP6UX6I.js");
50136
+ const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
49790
50137
  const existing = await getExtractJob2(jobId2);
49791
50138
  if (!existing) return c.json({ error: "job not found" }, 404);
49792
50139
  if (existing.status !== "failed") {
@@ -49813,7 +50160,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
49813
50160
  if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
49814
50161
  let settlement = "already_settled_or_refunded";
49815
50162
  if (job.billedMc == null && job.userId != null) {
49816
- const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NTM4MDQN.js");
50163
+ const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-7EP6UX6I.js");
49817
50164
  const heldMc = Number(job.options.heldMc ?? 0);
49818
50165
  const successful = await countSuccessfulPages2(jobId2);
49819
50166
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);