mcp-scraper 0.4.5 → 0.4.12

package/dist/index.cjs

@@ -315,11 +315,25 @@ function currentCostContext() {
 // src/api/cost-rates.ts
 var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
 var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
-var HEADLESS_OPS = /* @__PURE__ */ new Set(["harvest", "maps_search", "yt_channel", "fb_search", "fb_ad", "instagram"]);
+var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
+var FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
+var FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
+var DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
 }
+function vendorCostUsd(vendor, units) {
+  if (vendor === "fal_wizper") {
+    const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR;
+    return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC;
+  }
+  if (vendor === "deepinfra_qwen") return Math.max(0, units) / 1e3 * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
+  if (vendor === "openrouter") return Math.max(0, units);
+  if (vendor === "mcp_memory_video") return Math.max(0, units);
+  if (vendor === "mcp_memory_ai") return Math.max(0, units);
+  return 0;
+}
 
 // src/api/cost-telemetry.ts
 var _ready = false;
@@ -358,6 +372,10 @@ async function migrateCostTelemetry() {
     await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
   } catch {
   }
+  try {
+    await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`);
+  } catch {
+  }
   await db.execute(`
     CREATE TABLE IF NOT EXISTS vendor_usage_log (
       id           TEXT PRIMARY KEY,
@@ -376,6 +394,10 @@ async function migrateCostTelemetry() {
   await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
   await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
   await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
+  } catch {
+  }
   await db.execute(`
     CREATE TABLE IF NOT EXISTS cost_probe_runs (
       id                            TEXT PRIMARY KEY,
@@ -422,8 +444,8 @@ async function recordKernelSession(r) {
     const db = getDb();
     await db.execute({
       sql: `INSERT INTO kernel_session_log
-        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
       args: [
         (0, import_node_crypto2.randomUUID)(),
         r.kernelSessionId ?? null,
@@ -442,13 +464,41 @@ async function recordKernelSession(r) {
         durationMs,
         kernelCostUsd(durationMs, false),
         kernelCostUsd(durationMs, true),
-        r.error ?? null
+        r.error ?? null,
+        ctx?.subOp ?? null
       ]
     });
   } catch (err) {
     console.warn("[cost-telemetry] recordKernelSession failed:", err instanceof Error ? err.message : String(err));
   }
 }
+async function recordVendorUsage(r) {
+  try {
+    await migrateCostTelemetry();
+    const ctx = currentCostContext();
+    const db = getDb();
+    await db.execute({
+      sql: `INSERT INTO vendor_usage_log
+        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
+      args: [
+        (0, import_node_crypto2.randomUUID)(),
+        ctx?.op ?? null,
+        ctx?.probeRunId ?? null,
+        ctx?.userId ?? null,
+        r.vendor,
+        r.model ?? null,
+        r.units,
+        r.unitType,
+        vendorCostUsd(r.vendor, r.units),
+        r.error ?? null,
+        ctx?.subOp ?? null
+      ]
+    });
+  } catch (err) {
+    console.warn("[cost-telemetry] recordVendorUsage failed:", err instanceof Error ? err.message : String(err));
+  }
+}
 function boolToInt(v) {
   if (v === null || v === void 0) return null;
   return v ? 1 : 0;
@@ -3457,6 +3507,11 @@ Answer: ${answer.slice(0, 500)}` }
   });
   if (!res.ok) throw new Error(`LLM call failed (${res.status}): ${await res.text()}`);
   const data = await res.json();
+  if (baseUrl === OPENROUTER_URL) {
+    void recordVendorUsage({ vendor: "openrouter", model: QWEN_MODEL, units: data.usage?.cost ?? 0, unitType: "usd" });
+  } else {
+    void recordVendorUsage({ vendor: "deepinfra_qwen", model: QWEN_MODEL, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
+  }
   const raw = data.choices[0]?.message?.content?.trim() ?? "";
   const match = raw.match(/\{[\s\S]*\}/);
   if (!match) throw new Error(`No JSON in QWEN response: ${raw.slice(0, 200)}`);