mcp-scraper 0.40.1 → 0.40.2

package/dist/chunk-X2LKCX6H.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/site-extract-repository.ts"],"sourcesContent":["import { CREDIT_LOT_TTL, getDb, settleDebitMcIdempotent } from './db.js'\nimport { sanitizeVendorName } from '../errors.js'\nimport { LedgerOperation } from './rates.js'\nimport type { PageData } from './site-extractor.js'\n\nexport type ExtractJobStatus = 'pending' | 'running' | 'complete' | 'partial' | 'failed'\n\nexport interface ExtractJobArtifact {\n  key: string\n  url: string\n  bytes: number\n  contentType: string\n  filename?: string\n  sha256?: string\n  expiresAt?: string\n  downloadUrlExpiresAt?: string | null\n}\n\nexport interface ExtractJobRow {\n  id: string\n  userId: number | null\n  idempotencyKey: string | null\n  requestFingerprint: string | null\n  status: ExtractJobStatus\n  startUrl: string\n  options: Record<string, unknown>\n  totalUrls: number\n  doneUrls: number\n  attemptedUrls: number\n  successfulUrls: number\n  failedUrls: number\n  remainingUrls: number\n  artifacts: ExtractJobArtifact[] | null\n  error: string | null\n  billedMc: number | null\n  createdAt: string\n  updatedAt: string\n}\n\nexport function extractJobLimitInfo(job: Pick<ExtractJobRow, 'options' | 'totalUrls'>): {\n  requestedMaxPages: number\n  effectiveMaxPages: number\n  creditLimited: boolean\n  creditTruncated: boolean\n} {\n  const effectiveMaxPages = Math.max(1, Number(job.options.effectiveMaxPages ?? job.options.maxPages ?? 1))\n  const requestedMaxPages = Math.max(effectiveMaxPages, Number(job.options.requestedMaxPages ?? effectiveMaxPages))\n  const creditLimited = requestedMaxPages > effectiveMaxPages\n  return {\n    requestedMaxPages,\n    effectiveMaxPages,\n    creditLimited,\n    // If discovery exhausted below the funded cap, the smaller hold did not\n    // actually truncate this crawl. Hitting the cap is conservatively partial.\n    creditTruncated: creditLimited && job.totalUrls >= effectiveMaxPages,\n  }\n}\n\nexport function terminalExtractJobStatus(\n  progress: Pick<ExtractJobRow, 'successfulUrls' | 'failedUrls' | 'remainingUrls'>,\n  creditTruncated = false,\n): Extract<'complete' | 'partial' | 'failed', ExtractJobStatus> {\n  if (progress.successfulUrls === 0) return 'failed'\n  if (progress.failedUrls > 0 || progress.remainingUrls > 0 || creditTruncated) return 'partial'\n  return 'complete'\n}\n\nfunction rowToJob(r: Record<string, unknown>): ExtractJobRow {\n  const totalUrls = Number(r.total_urls ?? 0)\n  const legacyProgress = r.attempted_urls == null\n  const attemptedUrls = Number(legacyProgress ? (r.done_urls ?? 0) : r.attempted_urls)\n  const successfulUrls = Number(legacyProgress ? (r.done_urls ?? 0) : (r.successful_urls ?? 0))\n  const failedUrls = Number(r.failed_urls ?? Math.max(0, attemptedUrls - successfulUrls))\n  return {\n    id: String(r.id),\n    userId: r.user_id != null ? Number(r.user_id) : null,\n    idempotencyKey: r.idempotency_key != null ? String(r.idempotency_key) : null,\n    requestFingerprint: r.request_fingerprint != null ? String(r.request_fingerprint) : null,\n    status: (r.status != null ? String(r.status) : 'pending') as ExtractJobStatus,\n    startUrl: String(r.start_url ?? ''),\n    options: r.options ? JSON.parse(String(r.options)) : {},\n    totalUrls,\n    doneUrls: attemptedUrls,\n    attemptedUrls,\n    successfulUrls,\n    failedUrls,\n    remainingUrls: Math.max(0, totalUrls - attemptedUrls),\n    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,\n    error: r.error != null ? String(r.error) : null,\n    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,\n    createdAt: String(r.created_at ?? ''),\n    updatedAt: String(r.updated_at ?? ''),\n  }\n}\n\nexport async function createExtractJob(jobId: string, userId: number, startUrl: string, options: Record<string, unknown>): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)\n          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,\n    args: [jobId, userId, startUrl, JSON.stringify(options)],\n  })\n}\n\nexport async function getExtractJobByIdempotencyKey(userId: number, idempotencyKey: string): Promise<ExtractJobRow | null> {\n  const db = getDb()\n  const res = await db.execute({\n    sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1`,\n    args: [userId, idempotencyKey],\n  })\n  return res.rows[0] ? rowToJob(res.rows[0] as unknown as Record<string, unknown>) : null\n}\n\nexport async function createOrGetExtractJob(input: {\n  jobId: string\n  userId: number\n  startUrl: string\n  options: Record<string, unknown>\n  idempotencyKey: string\n  requestFingerprint: string\n}): Promise<{ job: ExtractJobRow; created: boolean; conflict: boolean }> {\n  const db = getDb()\n  const inserted = await db.execute({\n    sql: `INSERT OR IGNORE INTO site_extract_jobs\n            (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)\n          VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,\n    args: [\n      input.jobId,\n      input.userId,\n      input.startUrl,\n      JSON.stringify(input.options),\n      input.idempotencyKey,\n      input.requestFingerprint,\n    ],\n  })\n  const job = await getExtractJobByIdempotencyKey(input.userId, input.idempotencyKey)\n  if (!job) throw new Error('idempotent extract job was not persisted')\n  return {\n    job,\n    created: inserted.rowsAffected === 1,\n    conflict: job.requestFingerprint !== input.requestFingerprint,\n  }\n}\n\nexport async function getExtractJob(jobId: string): Promise<ExtractJobRow | null> {\n  const db = getDb()\n  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] })\n  return res.rows[0] ? rowToJob(res.rows[0] as unknown as Record<string, unknown>) : null\n}\n\nexport async function listExtractJobs(userId: number): Promise<ExtractJobRow[]> {\n  const db = getDb()\n  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50`, args: [userId] })\n  return res.rows.map(r => rowToJob(r as unknown as Record<string, unknown>))\n}\n\nexport async function listUnsettledExtractJobs(limit = 25): Promise<ExtractJobRow[]> {\n  const db = getDb()\n  const res = await db.execute({\n    sql: `SELECT * FROM site_extract_jobs\n          WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')\n            AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))\n          ORDER BY updated_at ASC LIMIT ?`,\n    args: [Math.max(1, Math.min(100, Math.round(limit)))],\n  })\n  return res.rows.map(row => rowToJob(row as unknown as Record<string, unknown>))\n}\n\nexport async function listFundedPendingExtractJobs(limit = 25, staleMinutes = 2): Promise<ExtractJobRow[]> {\n  const db = getDb()\n  const safeStaleMinutes = Math.max(1, Math.min(60, Math.round(staleMinutes)))\n  const res = await db.execute({\n    sql: `SELECT j.* FROM site_extract_jobs j\n          JOIN billing_debits d\n            ON d.idempotency_key = json_extract(j.options, '$.debitKey')\n           AND d.user_id = j.user_id\n           AND d.status = 'applied'\n          WHERE j.status = 'pending'\n            AND j.billed_mc IS NULL\n            AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))\n            AND j.updated_at <= datetime('now', ?)\n          ORDER BY j.updated_at ASC LIMIT ?`,\n    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))],\n  })\n  return res.rows.map(row => rowToJob(row as unknown as Record<string, unknown>))\n}\n\nexport async function markExtractJobDispatchAttempt(jobId: string): Promise<ExtractJobRow | null> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs\n          SET dispatch_attempts = dispatch_attempts + 1,\n              updated_at = datetime('now'),\n              next_dispatch_at = datetime('now', '+5 minutes'),\n              dispatch_error = NULL,\n              status = CASE\n                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'\n                ELSE status\n              END,\n              error = CASE\n                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')\n                  THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'\n                ELSE error\n              END\n          WHERE id = ? AND status = 'pending'`,\n    args: [jobId],\n  })\n  return getExtractJob(jobId)\n}\n\nexport async function recordExtractJobDispatchFailure(jobId: string, error: string): Promise<ExtractJobRow | null> {\n  const db = getDb()\n  const safeError = sanitizeVendorName(error).slice(0, 1000)\n  await db.execute({\n    sql: `UPDATE site_extract_jobs SET\n            dispatch_attempts = dispatch_attempts + 1,\n            dispatch_error = ?,\n            next_dispatch_at = datetime('now', '+5 minutes'),\n            updated_at = datetime('now'),\n            status = CASE\n              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'\n              ELSE status\n            END,\n            error = CASE\n              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')\n                THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'\n              ELSE error\n            END\n          WHERE id = ? AND status = 'pending'`,\n    args: [safeError, jobId],\n  })\n  return getExtractJob(jobId)\n}\n\nexport async function recordExtractSettlementFailure(jobId: string, error: string): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs SET\n            settlement_attempts = settlement_attempts + 1,\n            settlement_error = ?,\n            next_settlement_at = datetime('now', '+15 minutes'),\n            updated_at = datetime('now')\n          WHERE id = ? AND billed_mc IS NULL`,\n    args: [sanitizeVendorName(error).slice(0, 1000), jobId],\n  })\n}\n\n/** Running jobs update updated_at after every durable crawl batch. A job with\n * no heartbeat for this lease window is no longer executing safely and must\n * become terminal so the credit hold can be settled by reconciliation. */\nexport async function listStaleRunningExtractJobs(limit = 25, staleMinutes = 60): Promise<ExtractJobRow[]> {\n  const db = getDb()\n  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)))\n  const res = await db.execute({\n    sql: `SELECT j.* FROM site_extract_jobs j\n          LEFT JOIN billing_debits d\n            ON d.idempotency_key = json_extract(j.options, '$.debitKey')\n           AND d.user_id = j.user_id\n           AND d.status = 'applied'\n          WHERE j.status = 'running'\n            AND j.billed_mc IS NULL\n            AND (\n              json_extract(j.options, '$.debitKey') IS NULL\n              OR d.idempotency_key IS NOT NULL\n            )\n            AND j.updated_at <= datetime('now', ?)\n          ORDER BY j.updated_at ASC LIMIT ?`,\n    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))],\n  })\n  return res.rows.map(row => rowToJob(row as unknown as Record<string, unknown>))\n}\n\nexport async function failStaleRunningExtractJob(jobId: string, staleMinutes = 60): Promise<boolean> {\n  const db = getDb()\n  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)))\n  const result = await db.execute({\n    sql: `UPDATE site_extract_jobs\n          SET status = 'failed',\n              error = 'Background crawl stopped without a heartbeat; completed pages were preserved and the unused hold will be refunded.',\n              updated_at = datetime('now')\n          WHERE id = ?\n            AND status = 'running'\n            AND billed_mc IS NULL\n            AND updated_at <= datetime('now', ?)`,\n    args: [jobId, `-${safeStaleMinutes} minutes`],\n  })\n  return result.rowsAffected === 1\n}\n\n/** Claims a failed job before rebuilding its artifact so concurrent support\n * retries cannot assemble and publish two bundles for the same job. */\nexport async function claimFailedExtractJobForRefinalize(jobId: string): Promise<ExtractJobRow | null> {\n  const db = getDb()\n  const claimed = await db.execute({\n    sql: `UPDATE site_extract_jobs\n          SET status = 'running', updated_at = datetime('now')\n          WHERE id = ? AND status = 'failed'`,\n    args: [jobId],\n  })\n  if (claimed.rowsAffected !== 1) return null\n  return getExtractJob(jobId)\n}\n\nexport async function abandonExtractSettlement(jobId: string, error: string): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')\n          WHERE id = ? AND billed_mc IS NULL`,\n    args: [sanitizeVendorName(error).slice(0, 1000), jobId],\n  })\n}\n\nexport async function setExtractJobTotal(jobId: string, totalUrls: number): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs\n            SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')\n          WHERE id = ? AND status IN ('pending', 'running')`,\n    args: [totalUrls, jobId],\n  })\n}\n\nexport async function saveExtractPages(jobId: string, pages: PageData[]): Promise<void> {\n  if (pages.length === 0) return\n  const db = getDb()\n  await db.batch([\n    ...pages.map(p => {\n      const { discoveryLinks: _ephemeralDiscoveryLinks, ...storedPage } = p\n      return {\n      sql: `INSERT INTO site_extract_pages (job_id, url, page)\n            SELECT ?, ?, ?\n            WHERE EXISTS (\n              SELECT 1 FROM site_extract_jobs\n              WHERE id = ? AND status IN ('pending', 'running')\n            )\n            ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page\n            WHERE NOT CASE\n                    WHEN json_valid(site_extract_pages.page) = 0 THEN 0\n                    WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL\n                      THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'\n                    ELSE COALESCE(\n                      json_extract(site_extract_pages.page, '$.status') >= 200\n                      AND json_extract(site_extract_pages.page, '$.status') < 300\n                      AND json_extract(site_extract_pages.page, '$.wordCount') > 0,\n                      0\n                    )\n                  END\n               OR CASE\n                    WHEN json_valid(excluded.page) = 0 THEN 0\n                    WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL\n                      THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'\n                    ELSE COALESCE(\n                      json_extract(excluded.page, '$.status') >= 200\n                      AND json_extract(excluded.page, '$.status') < 300\n                      AND json_extract(excluded.page, '$.wordCount') > 0,\n                      0\n                    )\n                  END`,\n      args: [jobId, p.archivedUrl ?? p.url, JSON.stringify(storedPage), jobId],\n      }\n    }),\n    {\n    sql: `UPDATE site_extract_jobs SET\n            done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),\n            attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),\n            successful_urls = (\n              SELECT COUNT(*) FROM site_extract_pages\n              WHERE job_id = ? AND CASE\n                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL\n                  THEN json_extract(page, '$.extractionStatus') = 'successful'\n                ELSE json_extract(page, '$.status') >= 200\n                  AND json_extract(page, '$.status') < 300\n                  AND json_extract(page, '$.wordCount') > 0\n              END\n            ),\n            failed_urls = (\n              SELECT COUNT(*) FROM site_extract_pages\n              WHERE job_id = ? AND NOT CASE\n                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL\n                  THEN json_extract(page, '$.extractionStatus') = 'successful'\n                ELSE COALESCE(\n                  json_extract(page, '$.status') >= 200\n                  AND json_extract(page, '$.status') < 300\n                  AND json_extract(page, '$.wordCount') > 0,\n                  0\n                )\n              END\n            ),\n            updated_at = datetime('now')\n          WHERE id = ? AND status IN ('pending', 'running')`,\n    args: [jobId, jobId, jobId, jobId, jobId],\n    },\n  ], 'write')\n}\n\nexport async function getExtractedPages(jobId: string): Promise<PageData[]> {\n  const db = getDb()\n  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] })\n  return res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n}\n\n/** Projects only bounded image URLs in SQL so large background jobs do not\n * deserialize every page body merely to run the optional image audit. */\nexport async function getExtractedImageLinks(jobId: string, limit = 181): Promise<string[]> {\n  const db = getDb()\n  const safeLimit = Math.max(1, Math.min(5_001, Math.round(limit)))\n  const res = await db.execute({\n    sql: `SELECT DISTINCT CAST(images.value AS TEXT) AS url\n          FROM site_extract_pages p,\n               json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images\n          WHERE p.job_id = ?\n            AND images.type = 'text'\n            AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096\n          LIMIT ?`,\n    args: [jobId, safeLimit],\n  })\n  return res.rows.map(row => String(row.url))\n}\n\nexport async function countSuccessfulPages(jobId: string): Promise<number> {\n  const db = getDb()\n  const res = await db.execute({\n    sql: `SELECT COUNT(*) AS n FROM site_extract_pages\n            WHERE job_id = ? AND CASE\n              WHEN json_extract(page, '$.extractionStatus') IS NOT NULL\n                THEN json_extract(page, '$.extractionStatus') = 'successful'\n              ELSE COALESCE(\n                json_extract(page, '$.status') >= 200\n                AND json_extract(page, '$.status') < 300\n                AND json_extract(page, '$.wordCount') > 0,\n                0\n              )\n            END`,\n    args: [jobId],\n  })\n  return Number((res.rows[0] as unknown as Record<string, unknown>)?.n ?? 0)\n}\n\nexport async function getExtractedUrls(jobId: string): Promise<Set<string>> {\n  const db = getDb()\n  const res = await db.execute({ sql: `SELECT url FROM site_extract_pages WHERE job_id = ?`, args: [jobId] })\n  return new Set(res.rows.map(r => String((r as unknown as Record<string, unknown>).url)))\n}\n\nexport async function finishExtractJob(\n  jobId: string,\n  artifacts: ExtractJobRow['artifacts'],\n  status: Extract<'complete' | 'partial' | 'failed', ExtractJobStatus>,\n  error: string | null = null,\n  allowFailedRecovery = false,\n): Promise<boolean> {\n  const db = getDb()\n  const result = await db.execute({\n    sql: `UPDATE site_extract_jobs\n            SET status = ?, artifacts = ?, error = ?, updated_at = datetime('now')\n          WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,\n    args: [status, JSON.stringify(artifacts ?? []), error ? sanitizeVendorName(error).slice(0, 2000) : null, jobId, allowFailedRecovery ? 1 : 0],\n  })\n  return result.rowsAffected === 1\n}\n\nexport async function completeExtractJob(jobId: string, artifacts: ExtractJobRow['artifacts']): Promise<void> {\n  await finishExtractJob(jobId, artifacts, 'complete')\n}\n\nexport async function failExtractJob(jobId: string, error: string): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs\n            SET status = 'failed', error = ?, updated_at = datetime('now')\n          WHERE id = ? AND status IN ('pending', 'running')`,\n    args: [sanitizeVendorName(error).slice(0, 2000), jobId],\n  })\n}\n\nexport async function failUnfundedExtractJob(jobId: string, error: string): Promise<void> {\n  const db = getDb()\n  await db.execute({\n    sql: `UPDATE site_extract_jobs\n            SET status = 'failed', billed_mc = 0, error = ?, updated_at = datetime('now')\n          WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,\n    args: [sanitizeVendorName(error).slice(0, 2000), jobId],\n  })\n}\n\nexport async function settleExtractJob(jobId: string, userId: number, refundMc: number, netChargeMc: number, reference: string): Promise<void> {\n  const db = getDb()\n  const job = await getExtractJob(jobId)\n  if (!job || job.billedMc != null) return\n  const debitKey = typeof job.options.debitKey === 'string' ? job.options.debitKey : null\n  if (debitKey) {\n    const settlement = await settleDebitMcIdempotent(\n      userId,\n      debitKey,\n      Math.max(0, netChargeMc),\n      LedgerOperation.EXTRACT_SITE_REFUND,\n      reference,\n      'extract_site',\n    )\n    await db.execute({\n      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`,\n      args: [settlement.final_amount_mc, jobId],\n    })\n    return\n  }\n\n  // Backward compatibility for jobs created before idempotent holds were\n  // introduced. Claiming billed_mc=-1 and applying the refund happen in one\n  // transaction, so two finalizers cannot both credit the wallet.\n  const safeRefundMc = Math.max(0, refundMc)\n  const safeNetChargeMc = Math.max(0, netChargeMc)\n  await db.batch([\n    {\n      sql: `UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL`,\n      args: [jobId],\n    },\n    {\n      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)\n            SELECT ?, ?, ?, ?, datetime(j.created_at, ?)\n            FROM site_extract_jobs j\n            WHERE j.id = ?\n              AND j.billed_mc = -1\n              AND ? > 0\n              AND changes() = 1\n              AND datetime(j.created_at, ?) > datetime('now')`,\n      args: [\n        userId,\n        safeRefundMc,\n        safeRefundMc,\n        LedgerOperation.EXTRACT_SITE_REFUND,\n        CREDIT_LOT_TTL,\n        jobId,\n        safeRefundMc,\n        CREDIT_LOT_TTL,\n      ],\n    },\n    {\n      sql: `UPDATE users SET balance_mc = balance_mc + ?\n            WHERE id = ? AND ? > 0 AND changes() = 1`,\n      args: [safeRefundMc, userId, safeRefundMc],\n    },\n    {\n      sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)\n            SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,\n      args: [userId, safeRefundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference, safeRefundMc],\n    },\n    {\n      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')\n            WHERE id = ? AND billed_mc = -1`,\n      args: [safeNetChargeMc, jobId],\n    },\n  ], 'write')\n}\n"],"mappings":";;;;;;;;;;;;;AAuCO,SAAS,oBAAoB,KAKlC;AACA,QAAM,oBAAoB,KAAK,IAAI,GAAG,OAAO,IAAI,QAAQ,qBAAqB,IAAI,QAAQ,YAAY,CAAC,CAAC;AACxG,QAAM,oBAAoB,KAAK,IAAI,mBAAmB,OAAO,IAAI,QAAQ,qBAAqB,iBAAiB,CAAC;AAChH,QAAM,gBAAgB,oBAAoB;AAC1C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA,IAGA,iBAAiB,iBAAiB,IAAI,aAAa;AAAA,EACrD;AACF;AAEO,SAAS,yBACd,UACA,kBAAkB,OAC4C;AAC9D,MAAI,SAAS,mBAAmB,EAAG,QAAO;AAC1C,MAAI,SAAS,aAAa,KAAK,SAAS,gBAAgB,KAAK,gBAAiB,QAAO;AACrF,SAAO;AACT;AAEA,SAAS,SAAS,GAA2C;AAC3D,QAAM,YAAY,OAAO,EAAE,cAAc,CAAC;AAC1C,QAAM,iBAAiB,EAAE,kBAAkB;AAC3C,QAAM,gBAAgB,OAAO,iBAAkB,EAAE,aAAa,IAAK,EAAE,cAAc;AACnF,QAAM,iBAAiB,OAAO,iBAAkB,EAAE,aAAa,IAAM,EAAE,mBAAmB,CAAE;AAC5F,QAAM,aAAa,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,gBAAgB,cAAc,CAAC;AACtF,SAAO;AAAA,IACL,IAAI,OAAO,EAAE,EAAE;AAAA,IACf,QAAQ,EAAE,WAAW,OAAO,OAAO,EAAE,OAAO,IAAI;AAAA,IAChD,gBAAgB,EAAE,mBAAmB,OAAO,OAAO,EAAE,eAAe,IAAI;AAAA,IACxE,oBAAoB,EAAE,uBAAuB,OAAO,OAAO,EAAE,mBAAmB,IAAI;AAAA,IACpF,QAAS,EAAE,UAAU,OAAO,OAAO,EAAE,MAAM,IAAI;AAAA,IAC/C,UAAU,OAAO,EAAE,aAAa,EAAE;AAAA,IAClC,SAAS,EAAE,UAAU,KAAK,MAAM,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;AAAA,IACtD;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA,eAAe,KAAK,IAAI,GAAG,YAAY,aAAa;AAAA,IACpD,WAAW,EAAE,YAAY,KAAK,MAAM,OAAO,EAAE,SAAS,CAAC,IAAI;AAAA,IAC3D,OAAO,EAAE,SAAS,OAAO,OAAO,EAAE,KAAK,IAAI;AAAA,IAC3C,UAAU,EAAE,aAAa,OAAO,OAAO,EAAE,SAAS,IAAI;AAAA,IACtD,WAAW,OAAO,EAAE,cAAc,EAAE;AAAA,IACpC,WAAW,OAAO,EAAE,cAAc,EAAE;AAAA,EACtC;AACF;AAEA,eAAsB,iBAAiB,OAAe,QAAgB,UAAkB,SAAiD;AACvI,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,OAAO,QAAQ,UAAU,KAAK,UAAU,OAAO,CAAC;AAAA,EACzD,CAAC;AACH;AAEA,eAAsB,8BAA8B,QAAgB,gBAAuD;AACzH,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA,IACL,MAAM,CAAC,QAAQ,cAAc;AAAA,EAC/B,CAAC;AACD,SAAO,IAAI,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,CAAC,CAAuC,IAAI;AACrF;AAEA,eAAsB,sBAAsB,OAO6B;AACvE,QAAM,KAAK,MAAM;AACjB,QAAM,WAAW,MAAM,GAAG,QAAQ;AAAA,IAChC,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM;AAAA,MACN,KAAK,UAAU,MAAM,OAAO;AAAA,MAC5B,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,EACF,CAAC;AACD,QAAM,MAAM,MAAM,8BAA8B,MAAM,QAAQ,MAAM,cAAc;AAClF,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,0CAA0C;AACpE,SAAO;AAAA,IACL;AAAA,IACA,SAAS,SAAS,iBAAiB;AAAA,IACnC,UAAU,IAAI,uBAAuB,MAAM;AAAA,EAC7C;AACF;AAEA,eAAsB,cAAc,OAA8C;AAChF,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,gDAAgD,MAAM,CAAC,KAAK,EAAE,CAAC;AACnG,SAAO,IAAI,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,CAAC,CAAuC,IAAI;AACrF;AAEA,eAAsB,gBAAgB,QAA0C;AAC9E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,uFAAuF,MAAM,CAAC,MAAM,EAAE,CAAC;AAC3I,SAAO,IAAI,KAAK,IAAI,OAAK,SAAS,CAAuC,CAAC;AAC5E;AAEA,eAAsB,yBAAyB,QAAQ,IAA8B;AACnF,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA;AAAA;AAAA,IAIL,MAAM,CAAC,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC,CAAC,CAAC;AAAA,EACtD,CAAC;AACD,SAAO,IAAI,KAAK,IAAI,SAAO,SAAS,GAAyC,CAAC;AAChF;AAEA,eAAsB,6BAA6B,QAAQ,IAAI,eAAe,GAA6B;AACzG,QAAM,KAAK,MAAM;AACjB,QAAM,mBAAmB,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,MAAM,YAAY,CAAC,CAAC;AAC3E,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUL,MAAM,CAAC,IAAI,gBAAgB,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC,CAAC,CAAC;AAAA,EACtF,CAAC;AACD,SAAO,IAAI,KAAK,IAAI,SAAO,SAAS,GAAyC,CAAC;AAChF;AAEA,eAAsB,8BAA8B,OAA8C;AAChG,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAeL,MAAM,CAAC,KAAK;AAAA,EACd,CAAC;AACD,SAAO,cAAc,KAAK;AAC5B;AAEA,eAAsB,gCAAgC,OAAe,OAA8C;AACjH,QAAM,KAAK,MAAM;AACjB,QAAM,YAAY,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI;AACzD,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAeL,MAAM,CAAC,WAAW,KAAK;AAAA,EACzB,CAAC;AACD,SAAO,cAAc,KAAK;AAC5B;AAEA,eAAsB,+BAA+B,OAAe,OAA8B;AAChG,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAML,MAAM,CAAC,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,GAAG,KAAK;AAAA,EACxD,CAAC;AACH;AAKA,eAAsB,4BAA4B,QAAQ,IAAI,eAAe,IAA8B;AACzG,QAAM,KAAK,MAAM;AACjB,QAAM,mBAAmB,KAAK,IAAI,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,MAAM,YAAY,CAAC,CAAC;AACjF,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAaL,MAAM,CAAC,IAAI,gBAAgB,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC,CAAC,CAAC;AAAA,EACtF,CAAC;AACD,SAAO,IAAI,KAAK,IAAI,SAAO,SAAS,GAAyC,CAAC;AAChF;AAEA,eAAsB,2BAA2B,OAAe,eAAe,IAAsB;AACnG,QAAM,KAAK,MAAM;AACjB,QAAM,mBAAmB,KAAK,IAAI,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,MAAM,YAAY,CAAC,CAAC;AACjF,QAAM,SAAS,MAAM,GAAG,QAAQ;AAAA,IAC9B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQL,MAAM,CAAC,OAAO,IAAI,gBAAgB,UAAU;AAAA,EAC9C,CAAC;AACD,SAAO,OAAO,iBAAiB;AACjC;AAIA,eAAsB,mCAAmC,OAA8C;AACrG,QAAM,KAAK,MAAM;AACjB,QAAM,UAAU,MAAM,GAAG,QAAQ;AAAA,IAC/B,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,KAAK;AAAA,EACd,CAAC;AACD,MAAI,QAAQ,iBAAiB,EAAG,QAAO;AACvC,SAAO,cAAc,KAAK;AAC5B;AAEA,eAAsB,yBAAyB,OAAe,OAA8B;AAC1F,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,GAAG,KAAK;AAAA,EACxD,CAAC;AACH;AAEA,eAAsB,mBAAmB,OAAe,WAAkC;AACxF,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,WAAW,KAAK;AAAA,EACzB,CAAC;AACH;AAEA,eAAsB,iBAAiB,OAAe,OAAkC;AACtF,MAAI,MAAM,WAAW,EAAG;AACxB,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,MAAM;AAAA,IACb,GAAG,MAAM,IAAI,OAAK;AAChB,YAAM,EAAE,gBAAgB,0BAA0B,GAAG,WAAW,IAAI;AACpE,aAAO;AAAA,QACP,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QA6BL,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,KAAK,KAAK,UAAU,UAAU,GAAG,KAAK;AAAA,MACvE;AAAA,IACF,CAAC;AAAA,IACD;AAAA,MACA,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MA4BL,MAAM,CAAC,OAAO,OAAO,OAAO,OAAO,KAAK;AAAA,IACxC;AAAA,EACF,GAAG,OAAO;AACZ;AAEA,eAAsB,kBAAkB,OAAoC;AAC1E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,wDAAwD,MAAM,CAAC,KAAK,EAAE,CAAC;AAC3G,SAAO,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AACzG;AAIA,eAAsB,uBAAuB,OAAe,QAAQ,KAAwB;AAC1F,QAAM,KAAK,MAAM;AACjB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,MAAO,KAAK,MAAM,KAAK,CAAC,CAAC;AAChE,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOL,MAAM,CAAC,OAAO,SAAS;AAAA,EACzB,CAAC;AACD,SAAO,IAAI,KAAK,IAAI,SAAO,OAAO,IAAI,GAAG,CAAC;AAC5C;AAEA,eAAsB,qBAAqB,OAAgC;AACzE,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWL,MAAM,CAAC,KAAK;AAAA,EACd,CAAC;AACD,SAAO,OAAQ,IAAI,KAAK,CAAC,GAA0C,KAAK,CAAC;AAC3E;AAEA,eAAsB,iBAAiB,OAAqC;AAC1E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,uDAAuD,MAAM,CAAC,KAAK,EAAE,CAAC;AAC1G,SAAO,IAAI,IAAI,IAAI,KAAK,IAAI,OAAK,OAAQ,EAAyC,GAAG,CAAC,CAAC;AACzF;AAEA,eAAsB,iBACpB,OACA,WACA,QACA,QAAuB,MACvB,sBAAsB,OACJ;AAClB,QAAM,KAAK,MAAM;AACjB,QAAM,SAAS,MAAM,GAAG,QAAQ;AAAA,IAC9B,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,QAAQ,KAAK,UAAU,aAAa,CAAC,CAAC,GAAG,QAAQ,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,IAAI,MAAM,OAAO,sBAAsB,IAAI,CAAC;AAAA,EAC7I,CAAC;AACD,SAAO,OAAO,iBAAiB;AACjC;AAEA,eAAsB,mBAAmB,OAAe,WAAsD;AAC5G,QAAM,iBAAiB,OAAO,WAAW,UAAU;AACrD;AAEA,eAAsB,eAAe,OAAe,OAA8B;AAChF,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,GAAG,KAAK;AAAA,EACxD,CAAC;AACH;AAEA,eAAsB,uBAAuB,OAAe,OAA8B;AACxF,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,GAAG,KAAK;AAAA,EACxD,CAAC;AACH;AAEA,eAAsB,iBAAiB,OAAe,QAAgB,UAAkB,aAAqB,WAAkC;AAC7I,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,cAAc,KAAK;AACrC,MAAI,CAAC,OAAO,IAAI,YAAY,KAAM;AAClC,QAAM,WAAW,OAAO,IAAI,QAAQ,aAAa,WAAW,IAAI,QAAQ,WAAW;AACnF,MAAI,UAAU;AACZ,UAAM,aAAa,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,MACA,KAAK,IAAI,GAAG,WAAW;AAAA,MACvB,gBAAgB;AAAA,MAChB;AAAA,MACA;AAAA,IACF;AACA,UAAM,GAAG,QAAQ;AAAA,MACf,KAAK;AAAA,MACL,MAAM,CAAC,WAAW,iBAAiB,KAAK;AAAA,IAC1C,CAAC;AACD;AAAA,EACF;AAKA,QAAM,eAAe,KAAK,IAAI,GAAG,QAAQ;AACzC,QAAM,kBAAkB,KAAK,IAAI,GAAG,WAAW;AAC/C,QAAM,GAAG,MAAM;AAAA,IACb;AAAA,MACE,KAAK;AAAA,MACL,MAAM,CAAC,KAAK;AAAA,IACd;AAAA,IACA;AAAA,MACE,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQL,MAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA,gBAAgB;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE,KAAK;AAAA;AAAA,MAEL,MAAM,CAAC,cAAc,QAAQ,YAAY;AAAA,IAC3C;AAAA,IACA;AAAA,MACE,KAAK;AAAA;AAAA,MAEL,MAAM,CAAC,QAAQ,cAAc,gBAAgB,qBAAqB,WAAW,YAAY;AAAA,IAC3F;AAAA,IACA;AAAA,MACE,KAAK;AAAA;AAAA,MAEL,MAAM,CAAC,iBAAiB,KAAK;AAAA,IAC/B;AAAA,EACF,GAAG,OAAO;AACZ;","names":[]}

package/dist/chunk-XELSA2MS.js

@@ -0,0 +1,108 @@
+// src/install-terminal.ts
+var CODES = {
+  reset: "\x1B[0m",
+  cyan: "\x1B[36m",
+  lime: "\x1B[32m",
+  amber: "\x1B[33m",
+  red: "\x1B[31m",
+  muted: "\x1B[90m",
+  bold: "\x1B[1m"
+};
+function colorize(value, tone, enabled) {
+  if (!enabled) return value;
+  return `${CODES[tone]}${value}${CODES.reset}`;
+}
+function toolRow(label, tools, enabled) {
+  const padded = label.padEnd(9, " ");
+  return `  ${colorize(padded, "muted", enabled)} ${tools.join(colorize("  .  ", "muted", enabled))}`;
+}
+function renderInstallTerminal(options) {
+  const color = options.color ?? true;
+  const apiKeyValue = options.apiKeyConfigured ? "$MCP_SCRAPER_API_KEY" : "sk_live_your_key";
+  const ascii = String.raw`
+ __  __  ____ ____ 
+|  \/  |/ ___|  _ \
+| |\/| | |   | |_) |
+| |  | | |___|  __/
+|_|  |_|\____|_|
+
+ ____   ____ ____      _    ____  _____ ____
+/ ___| / ___|  _ \    / \  |  _ \| ____|  _ \
+\___ \| |   | |_) |  / _ \ | |_) |  _| | |_) |
+ ___) | |___|  _ <  / ___ \|  __/| |___|  _ <
+|____/ \____|_| \_\/_/   \_\_|   |_____|_| \_\
+`;
+  const claudeCommand = [
+    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\`,
+    "  mcp-scraper-cli agent install claude --apply"
+  ].join("\n");
+  const codexConfig = [
+    "[mcp_servers.mcp-scraper]",
+    'command = "npx"',
+    'args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',
+    `env = { MCP_SCRAPER_API_KEY = "${apiKeyValue}" }`
+  ].join("\n");
+  return [
+    colorize(`mcp-scraper v${options.version}`, "bold", color),
+    colorize("> mcp-scraper-install", "muted", color),
+    colorize(ascii, "amber", color),
+    `${colorize("MCP Scraper Agent", "cyan", color)}  .  v${options.version}  .  mcpscraper.dev`,
+    "1/1 install surfaces ready",
+    colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
+    "",
+    `${colorize("Tools", "cyan", color)}  ${colorize("(175 MCP tools)", "muted", color)}`,
+    toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
+    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
+    toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
+    toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download", "reddit_thread"], color),
+    toolRow("browser", ["browser_open", "browser_profile_connect", "browser_profile_list", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
+    toolRow("connect", ["list_service_connections", "describe_service_connection_tool", "import_service_connection_to_memory", "export_connected_service_data", "renew_connected_data_download", "read_service_connection", "call_service_connection_action"], color),
+    toolRow("account", ["credits_info", "reports", "MCP resources"], color),
+    toolRow("memory", ["memory-put", "memory-get", "memory-search", "list-vaults", "record-fact", "list-scheduled-actions"], color),
+    `${colorize("Workflows", "cyan", color)}  ${colorize("(MCP + CLI + API)", "muted", color)}`,
+    toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
+    toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
+    "",
+    colorize("Usage tips:", "amber", color),
+    "Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
+    "Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.",
+    "Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install",
+    "Hosted browser sessions use direct/no-proxy egress by default.",
+    "Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.",
+    "Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.",
+    "Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.",
+    "Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.",
+    "Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.",
+    "For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.",
+    "If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.",
+    "",
+    `${colorize("Ready.", "lime", color)} Install the combined MCP server with one command:`,
+    "",
+    colorize("Setup doctor", "amber", color),
+    "npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
+    "",
+    colorize("Hosted profile setup", "amber", color),
+    'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',
+    "Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.",
+    "",
+    colorize("Claude Code one-command setup", "amber", color),
+    claudeCommand,
+    "Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list",
+    "",
+    colorize("Codex config", "amber", color),
+    codexConfig,
+    "",
+    colorize("Claude Desktop Extension", "amber", color),
+    "Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
+    "",
+    colorize("Safety note:", "muted", color),
+    "mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.",
+    "Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",
+    ""
+  ].join("\n");
+}
+
+export {
+  renderInstallTerminal
+};
+//# sourceMappingURL=chunk-XELSA2MS.js.map

package/dist/chunk-XELSA2MS.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(175 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}