@mackody/quickenrich-mcp 0.1.0

package/scripts/live-benchmark.mjs

@@ -0,0 +1,518 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { performance } from "node:perf_hooks";
+import { fileURLToPath } from "node:url";
+import {
+  apiBaseUrl,
+  apiKey,
+  authStatus,
+  callQuickEnrichTool,
+  createClient,
+  redact,
+} from "./lib/core.mjs";
+import {
+  makeDuplicateDomainBenchmark,
+  summarizeBenchmarkComparison,
+} from "./lib/benchmark.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = join(__dirname, "..");
+const DEFAULT_CREDIT_BENCH_DOMAINS = [
+  "danosseasoning.com",
+  "sagegoddess.com",
+];
+
+const creditMode = truthy(process.env.QUICKENRICH_BENCH_SPEND_CREDITS) || String(process.env.QUICKENRICH_BENCH_MODE || "").toLowerCase() === "credit";
+const configuredCreditDomains = parseDomains(process.env.QUICKENRICH_BENCH_DOMAINS);
+const domainInputs = creditMode ? configuredCreditDomains.length
+  ? configuredCreditDomains
+  : DEFAULT_CREDIT_BENCH_DOMAINS
+  : [];
+
+const settings = {
+  creditMode,
+  noCreditGuard: !creditMode,
+  requireCredits: creditMode && !falsey(process.env.QUICKENRICH_BENCH_REQUIRE_CREDITS),
+  domainInputs,
+  iterations: envInteger("QUICKENRICH_BENCH_ITERATIONS", 1, 5, creditMode ? 1 : 2),
+  uniqueCount: domainInputs.length || envInteger("QUICKENRICH_BENCH_UNIQUE", 1, 24, 6),
+  duplicateFactor: envInteger("QUICKENRICH_BENCH_DUPLICATE_FACTOR", 1, 8, 2),
+  concurrency: envInteger("QUICKENRICH_BENCH_CONCURRENCY", 1, 32, 8),
+  perPage: envInteger("QUICKENRICH_BENCH_PER_PAGE", 1, 20, 5),
+  title: process.env.QUICKENRICH_BENCH_TITLE || "Founder",
+  timeoutMs: envInteger("QUICKENRICH_BENCH_TIMEOUT_MS", 1000, 120_000, 30_000),
+  metaLane: !creditMode && truthy(process.env.QUICKENRICH_BENCH_META_LANE),
+};
+
+const status = authStatus();
+if (!status.configured) {
+  console.log(JSON.stringify({
+    ok: true,
+    skipped: true,
+    reason: "Quick Enrich API key is not configured. Set QUICKENRICH_API_KEY or run node scripts/quickenrich-token.mjs set.",
+  }, null, 2));
+  process.exit(0);
+}
+
+const mcp = startMcpClient();
+const rows = [];
+
+try {
+  await initializeMcp(mcp);
+  for (let iteration = 1; iteration <= settings.iterations; iteration += 1) {
+    rows.push(await measureScenario("raw_sequential", iteration, () => (
+      runRawSequential(makeFixture(iteration, "raw-seq").companies)
+    )));
+    rows.push(await measureScenario("raw_concurrent_deduped", iteration, () => (
+      runRawConcurrent(makeFixture(iteration, "raw-concurrent").uniqueCompanies)
+    )));
+    rows.push(await measureScenario("direct_core_batch", iteration, () => (
+      runDirectCoreBatch(makeFixture(iteration, "core-batch").companies)
+    )));
+    rows.push(await measureScenario("mcp_batch_stdio", iteration, () => (
+      runMcpBatch(mcp, makeFixture(iteration, "mcp-batch").companies)
+    )));
+    if (settings.metaLane) {
+      rows.push(await measureScenario("mcp_batch_stdio_meta", iteration, () => (
+        runMcpBatch(mcp, makeFixture(iteration, "mcp-meta").companies, { response: { mode: "meta" } })
+      )));
+    }
+  }
+
+  const averages = aggregateRows(rows);
+  const mcpComparison = summarizeBenchmarkComparison({
+    rows: averages,
+    baselineLabel: "raw_sequential",
+    candidateLabel: "mcp_batch_stdio",
+    optimizedLabel: "raw_concurrent_deduped",
+  });
+  const directCoreComparison = summarizeBenchmarkComparison({
+    rows: averages,
+    baselineLabel: "raw_sequential",
+    candidateLabel: "direct_core_batch",
+    optimizedLabel: "raw_concurrent_deduped",
+  });
+  const mcpMetaComparison = averages.some((row) => row.label === "mcp_batch_stdio_meta")
+    ? summarizeBenchmarkComparison({
+      rows: averages,
+      baselineLabel: "raw_sequential",
+      candidateLabel: "mcp_batch_stdio_meta",
+      optimizedLabel: "raw_concurrent_deduped",
+    })
+    : undefined;
+  const totalCreditsSpent = roundNumber(rows.reduce((sum, row) => sum + Number(row.creditsUsed || 0), 0));
+  if (settings.requireCredits && totalCreditsSpent <= 0) {
+    throw new Error("Credit benchmark did not spend any credits. Override domains with QUICKENRICH_BENCH_DOMAINS or set QUICKENRICH_BENCH_REQUIRE_CREDITS=0.");
+  }
+
+  console.log(JSON.stringify({
+    ok: true,
+    skipped: false,
+    apiBaseUrl: apiBaseUrl(),
+    benchmarkShape: {
+      iterations: settings.iterations,
+      inputCompanyCount: settings.uniqueCount * settings.duplicateFactor,
+      uniqueCompanyCount: settings.uniqueCount,
+      duplicateFactor: settings.duplicateFactor,
+      coalescedCompanyCount: settings.uniqueCount * (settings.duplicateFactor - 1),
+      title: settings.title,
+      perPage: settings.perPage,
+      concurrency: settings.concurrency,
+      metaLane: settings.metaLane || undefined,
+      mode: settings.creditMode ? "credit" : "no_credit",
+      domains: settings.creditMode ? settings.domainInputs : undefined,
+      noCreditGuard: settings.noCreditGuard,
+      requireCredits: settings.requireCredits || undefined,
+      totalCreditsSpent,
+    },
+    averagedRows: averages,
+    comparisons: {
+      mcpVsNaiveRawSequential: mcpComparison,
+      mcpMetaVsNaiveRawSequential: mcpMetaComparison,
+      directCoreVsNaiveRawSequential: directCoreComparison,
+      mcpCreditUsageVsNaiveRawSequential: creditComparison(averages, "raw_sequential", "mcp_batch_stdio"),
+      mcpMetaCreditUsageVsNaiveRawSequential: creditComparison(averages, "raw_sequential", "mcp_batch_stdio_meta"),
+      directCoreCreditUsageVsNaiveRawSequential: creditComparison(averages, "raw_sequential", "direct_core_batch"),
+    },
+    rows,
+  }, null, 2));
+} catch (error) {
+  console.error(JSON.stringify({
+    ok: false,
+    error: error.message || String(error),
+    data: error.result ? redact(error.result) : undefined,
+    rows,
+  }, null, 2));
+  process.exitCode = 1;
+} finally {
+  await mcp.close();
+}
+
+function makeFixture(iteration, lane) {
+  if (settings.domainInputs.length) {
+    const uniqueCompanies = settings.domainInputs.map((domain) => ({
+      company_url: domain,
+      title: settings.title,
+    }));
+    const companies = uniqueCompanies.flatMap((company) => (
+      Array.from({ length: settings.duplicateFactor }, () => ({ ...company }))
+    ));
+    return {
+      title: settings.title,
+      uniqueCount: uniqueCompanies.length,
+      duplicateFactor: settings.duplicateFactor,
+      inputCount: companies.length,
+      coalescedCount: companies.length - uniqueCompanies.length,
+      companies,
+      uniqueCompanies,
+    };
+  }
+  return makeDuplicateDomainBenchmark({
+    uniqueCount: settings.uniqueCount,
+    duplicateFactor: settings.duplicateFactor,
+    prefix: `qe-mcp-live-${Date.now()}-${iteration}-${lane}`,
+    title: settings.title,
+  });
+}
+
+async function measureScenario(label, iteration, run) {
+  const startedAt = performance.now();
+  const result = await run();
+  return {
+    label,
+    iteration,
+    durationMs: roundNumber(performance.now() - startedAt),
+    ...result,
+  };
+}
+
+async function runRawSequential(companies) {
+  const results = [];
+  for (const company of companies) {
+    results.push(await rawDomainSearch(company));
+  }
+  return combineCallResults(results, { requestCount: companies.length });
+}
+
+async function runRawConcurrent(companies) {
+  const results = await withConcurrency(companies, settings.concurrency, rawDomainSearch);
+  return combineCallResults(results, { requestCount: companies.length });
+}
+
+async function runDirectCoreBatch(companies) {
+  const result = await callQuickEnrichTool("quickenrich_batch_pull_leads", {
+    mode: "domain_search",
+    companies,
+    pages_per_company: 1,
+    per_company_limit: settings.perPage,
+    per_page: settings.perPage,
+    concurrency: settings.concurrency,
+    enrich_contacts: false,
+    response: { mode: "compact", maxOutputChars: 12_000 },
+  }, createClient());
+  assertBatchGuard(result, "direct_core_batch");
+  return {
+    ok: result.ok,
+    requestCount: result.requestCount,
+    creditsUsed: numeric(result.creditsUsed, 0),
+    remainingCredits: result.remainingCredits,
+    leadCount: numeric(result.leadCount, 0),
+    companyCount: result.companyCount,
+    coalescedCompanyCount: result.coalescedCompanyCount,
+  };
+}
+
+async function runMcpBatch(mcp, companies, options = {}) {
+  const responseOptions = options.response || { mode: "compact", maxOutputChars: 12_000 };
+  const response = await mcp.request("tools/call", {
+    name: "quickenrich_batch_pull_leads",
+    arguments: {
+      mode: "domain_search",
+      companies,
+      pages_per_company: 1,
+      per_company_limit: settings.perPage,
+      per_page: settings.perPage,
+      concurrency: settings.concurrency,
+      enrich_contacts: false,
+      response: responseOptions,
+    },
+  }, settings.timeoutMs + 10_000);
+  if (response.error) throw new Error(`MCP batch call failed: ${JSON.stringify(response.error)}`);
+  const payload = JSON.parse(response.result?.content?.[0]?.text || "{}");
+  assertBatchGuard(payload, "mcp_batch_stdio");
+  return {
+    ok: payloadScalar(payload, "ok"),
+    requestCount: payloadScalar(payload, "requestCount"),
+    creditsUsed: numeric(payloadScalar(payload, "creditsUsed"), 0),
+    remainingCredits: payloadScalar(payload, "remainingCredits"),
+    leadCount: numeric(payloadScalar(payload, "leadCount"), 0),
+    companyCount: payloadScalar(payload, "companyCount"),
+    coalescedCompanyCount: payloadScalar(payload, "coalescedCompanyCount"),
+  };
+}
+
+async function rawDomainSearch(company) {
+  const url = new URL("/api/employees/dataset-search", `${apiBaseUrl()}/`);
+  url.searchParams.set("company_url", company.company_url);
+  url.searchParams.set("title", company.title || settings.title);
+  url.searchParams.set("page", "1");
+  url.searchParams.set("per_page", String(settings.perPage));
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), settings.timeoutMs);
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${apiKey()}`,
+        "User-Agent": "quickenrich-mcp-live-benchmark/0.1.0",
+      },
+      signal: controller.signal,
+    });
+    const text = await response.text();
+    const payload = text ? JSON.parse(text) : {};
+    if (!response.ok) {
+      const error = new Error(`Raw Quick Enrich domain search failed with HTTP ${response.status}`);
+      error.result = { status: response.status, data: payload };
+      throw error;
+    }
+    const records = payloadRecords(payload);
+    const creditsUsed = numeric(payload?.meta?.credits_used, 0);
+    if (settings.noCreditGuard && creditsUsed > 0) {
+      throw new Error(`No-match raw benchmark unexpectedly used ${creditsUsed} credits for ${company.company_url}.`);
+    }
+    if (settings.noCreditGuard && records.length > 0) {
+      throw new Error(`No-match raw benchmark unexpectedly returned ${records.length} records for ${company.company_url}.`);
+    }
+    return {
+      requestCount: 1,
+      creditsUsed,
+      remainingCredits: numeric(payload?.meta?.remaining_credits),
+      leadCount: records.length,
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function initializeMcp(mcp) {
+  const init = await mcp.request("initialize", { protocolVersion: "2024-11-05" });
+  if (init.error) throw new Error(`MCP initialize failed: ${JSON.stringify(init.error)}`);
+  const list = await mcp.request("tools/list");
+  if (list.error) throw new Error(`MCP tools/list failed: ${JSON.stringify(list.error)}`);
+  const tools = list.result?.tools?.map((tool) => tool.name) || [];
+  if (!tools.includes("quickenrich_batch_pull_leads")) {
+    throw new Error("MCP server did not expose quickenrich_batch_pull_leads.");
+  }
+}
+
+function startMcpClient() {
+  const mcpConfig = JSON.parse(readFileSync(join(repoRoot, ".mcp.json"), "utf8"));
+  const server = mcpConfig.mcpServers?.["quick-enrich"];
+  if (!server?.command) throw new Error("Missing quick-enrich MCP server command.");
+
+  const proc = spawn(server.command, server.args || [], {
+    cwd: repoRoot,
+    env: {
+      ...process.env,
+      ...(server.env || {}),
+      CLAUDE_PLUGIN_ROOT: repoRoot,
+    },
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  let buffer = Buffer.alloc(0);
+  let nextId = 1;
+  let stderr = "";
+  const pending = new Map();
+
+  proc.stdout.on("data", (chunk) => {
+    buffer = Buffer.concat([buffer, chunk]);
+    parseMcpFrames();
+  });
+  proc.stderr.on("data", (chunk) => {
+    stderr += chunk.toString("utf8");
+    stderr = stderr.slice(-8000);
+  });
+  proc.on("exit", (code, signal) => {
+    const error = new Error(`MCP process exited before benchmark completed: code=${code} signal=${signal} stderr=${stderr.trim()}`);
+    for (const { reject, timeout } of pending.values()) {
+      clearTimeout(timeout);
+      reject(error);
+    }
+    pending.clear();
+  });
+
+  function request(method, params = {}, timeoutMs = settings.timeoutMs) {
+    const id = nextId;
+    nextId += 1;
+    const message = JSON.stringify({ jsonrpc: "2.0", id, method, params });
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        pending.delete(id);
+        reject(new Error(`Timed out waiting for MCP response id=${id} method=${method}`));
+      }, timeoutMs);
+      pending.set(id, { resolve, reject, timeout });
+      proc.stdin.write(`Content-Length: ${Buffer.byteLength(message)}\r\n\r\n${message}`);
+    });
+  }
+
+  function parseMcpFrames() {
+    while (true) {
+      const headerEnd = buffer.indexOf("\r\n\r\n");
+      if (headerEnd === -1) return;
+      const header = buffer.slice(0, headerEnd).toString("utf8");
+      const match = header.match(/Content-Length:\s*(\d+)/i);
+      if (!match) throw new Error(`Invalid MCP header: ${header}`);
+      const length = Number(match[1]);
+      const bodyStart = headerEnd + 4;
+      const bodyEnd = bodyStart + length;
+      if (buffer.length < bodyEnd) return;
+      const body = buffer.slice(bodyStart, bodyEnd).toString("utf8");
+      buffer = buffer.slice(bodyEnd);
+      const response = JSON.parse(body);
+      const waiter = pending.get(response.id);
+      if (!waiter) continue;
+      pending.delete(response.id);
+      clearTimeout(waiter.timeout);
+      waiter.resolve(response);
+    }
+  }
+
+  async function close() {
+    if (proc.exitCode !== null || proc.killed) return;
+    proc.kill();
+    await new Promise((resolve) => proc.once("exit", resolve));
+  }
+
+  return { request, close };
+}
+
+function aggregateRows(rows) {
+  const labels = [...new Set(rows.map((row) => row.label))];
+  return labels.map((label) => {
+    const group = rows.filter((row) => row.label === label);
+    const durations = group.map((row) => Number(row.durationMs)).filter(Number.isFinite);
+    return {
+      label,
+      iterations: group.length,
+      durationMs: average(durations),
+      minDurationMs: roundNumber(Math.min(...durations)),
+      maxDurationMs: roundNumber(Math.max(...durations)),
+      requestCount: average(group.map((row) => row.requestCount)),
+      totalRequests: roundNumber(group.reduce((sum, row) => sum + Number(row.requestCount || 0), 0)),
+      creditsUsed: roundNumber(group.reduce((sum, row) => sum + Number(row.creditsUsed || 0), 0)),
+      leadCount: roundNumber(group.reduce((sum, row) => sum + Number(row.leadCount || 0), 0)),
+      coalescedCompanyCount: average(group.map((row) => row.coalescedCompanyCount).filter((value) => value !== undefined)),
+    };
+  });
+}
+
+function combineCallResults(results, defaults = {}) {
+  return {
+    requestCount: defaults.requestCount ?? results.reduce((sum, result) => sum + Number(result.requestCount || 0), 0),
+    creditsUsed: results.reduce((sum, result) => sum + Number(result.creditsUsed || 0), 0),
+    remainingCredits: minDefined(results.map((result) => result.remainingCredits)),
+    leadCount: results.reduce((sum, result) => sum + Number(result.leadCount || 0), 0),
+  };
+}
+
+function assertBatchGuard(payload, label) {
+  if (payloadScalar(payload, "ok") !== true) throw new Error(`${label} returned ok=false: ${JSON.stringify(redact(payload))}`);
+  const creditsUsed = numeric(payloadScalar(payload, "creditsUsed"), 0);
+  if (settings.noCreditGuard && creditsUsed > 0) throw new Error(`${label} unexpectedly used ${creditsUsed} credits.`);
+  const leadCount = numeric(payloadScalar(payload, "leadCount"), 0);
+  if (settings.noCreditGuard && leadCount > 0) throw new Error(`${label} unexpectedly returned ${leadCount} benchmark records.`);
+}
+
+function payloadScalar(payload, key) {
+  if (!payload || typeof payload !== "object") return undefined;
+  if (Object.hasOwn(payload, key)) return payload[key];
+  return payload.scalars?.[key];
+}
+
+async function withConcurrency(items, concurrency, worker) {
+  const output = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length || 1) }, async () => {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      output[index] = await worker(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return output;
+}
+
+function payloadRecords(payload) {
+  if (Array.isArray(payload?.data)) return payload.data;
+  if (payload?.data && typeof payload.data === "object") return [payload.data];
+  if (Array.isArray(payload)) return payload;
+  return [];
+}
+
+function parseDomains(value) {
+  return String(value || "")
+    .split(/[\s,]+/)
+    .map((entry) => entry.trim().toLowerCase())
+    .filter(Boolean)
+    .map((entry) => entry.replace(/^https?:\/\//, "").replace(/^www\./, "").replace(/\/.*$/, ""))
+    .filter((entry, index, all) => all.indexOf(entry) === index);
+}
+
+function creditComparison(rows, baselineLabel, candidateLabel) {
+  const baseline = rows.find((row) => row.label === baselineLabel);
+  const candidate = rows.find((row) => row.label === candidateLabel);
+  if (!baseline || !candidate) return undefined;
+  const baselineCredits = numeric(baseline.creditsUsed, 0);
+  const candidateCredits = numeric(candidate.creditsUsed, 0);
+  const saved = baselineCredits - candidateCredits;
+  return {
+    baselineLabel,
+    candidateLabel,
+    baselineCredits,
+    candidateCredits,
+    creditsSaved: roundNumber(saved),
+    creditReductionPct: baselineCredits > 0 ? roundNumber((saved / baselineCredits) * 100) : undefined,
+  };
+}
+
+function envInteger(name, min, max, fallback) {
+  const value = Number.parseInt(process.env[name] || "", 10);
+  if (!Number.isFinite(value)) return fallback;
+  return Math.min(max, Math.max(min, value));
+}
+
+function truthy(value) {
+  return /^(1|true|yes|on|credit|spend)$/i.test(String(value || ""));
+}
+
+function falsey(value) {
+  return /^(0|false|no|off)$/i.test(String(value || ""));
+}
+
+function numeric(value, fallback = undefined) {
+  const number = Number(value);
+  return Number.isFinite(number) ? number : fallback;
+}
+
+function minDefined(values) {
+  const numbers = values.map((value) => Number(value)).filter(Number.isFinite);
+  return numbers.length ? Math.min(...numbers) : undefined;
+}
+
+function average(values) {
+  const numbers = values.map((value) => Number(value)).filter(Number.isFinite);
+  if (!numbers.length) return undefined;
+  return roundNumber(numbers.reduce((sum, value) => sum + value, 0) / numbers.length);
+}
+
+function roundNumber(value) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) return undefined;
+  return Math.round(number * 100) / 100;
+}

package/scripts/live-smoke.mjs

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+
+import { authStatus, callQuickEnrichTool, createClient } from "./lib/core.mjs";
+
+const status = authStatus();
+if (!status.configured) {
+  console.log(JSON.stringify({
+    ok: true,
+    skipped: true,
+    reason: "Quick Enrich API key is not configured. Set QUICKENRICH_API_KEY or run node scripts/quickenrich-token.mjs set.",
+  }, null, 2));
+  process.exit(0);
+}
+
+const client = createClient();
+let publicLookup;
+try {
+  publicLookup = await callQuickEnrichTool("quickenrich_lookup_values", {
+    lookup: "country_codes",
+    response: { mode: "meta" },
+  }, client);
+} catch (error) {
+  publicLookup = {
+    ok: false,
+    nonBlocking: true,
+    error: error.message || String(error),
+    data: error.result,
+  };
+}
+
+const authProbeEmail = `qe-mcp-smoke-${Date.now()}-${Math.random().toString(16).slice(2)}@gmail.com`;
+const authProbe = await callQuickEnrichTool("quickenrich_reverse_email", {
+  email: authProbeEmail,
+  response: { mode: "compact", maxItems: 3, maxOutputChars: 4000 },
+}, client);
+
+if (!authProbe?.ok) {
+  console.error(JSON.stringify({ ok: false, publicLookup, authProbe }, null, 2));
+  process.exit(1);
+}
+
+const creditsUsed = Number(authProbe.meta?.credits_used || 0);
+if (Number.isFinite(creditsUsed) && creditsUsed > 0) {
+  console.error(JSON.stringify({ ok: false, reason: "Auth probe unexpectedly used credits.", creditsUsed, authProbe }, null, 2));
+  process.exit(1);
+}
+
+console.log(JSON.stringify({
+  ok: true,
+  skipped: false,
+  checks: ["reverse_email:no_match_auth_probe"],
+  diagnostics: ["lookup_values:country_codes"],
+  publicLookup,
+  authProbe: {
+    ok: authProbe.ok,
+    found: authProbe.found,
+    meta: authProbe.meta,
+    rateLimit: authProbe.rateLimit,
+    rateLimitPerMinute: authProbe.rateLimitPerMinute,
+  },
+}, null, 2));