flashrev-ai-enrich 1.0.0

package/src/customer-api.js

@@ -0,0 +1,101 @@
+/**
+ * customer_api is a special capability that does NOT call the FlashRev backend.
+ * The CLI fetches the user-provided URL locally and returns the parsed response.
+ *
+ * Input fields (provided via --input or mapped from CSV columns via inputMapping):
+ *   url       string  required, target URL
+ *   method    string  optional, default GET
+ *   headers   object  optional, HTTP headers
+ *   body      any     optional, request body (object → JSON.stringify, string passed as-is)
+ *   params    object  optional, query string params
+ *   timeout   number  optional, milliseconds, default 30000
+ *
+ * Returns: parsed response JSON (or { text } wrapper for non-JSON text).
+ */
+export async function callCustomerApi(input = {}) {
+  const url = input.url || input.endpoint;
+  if (!url) {
+    const error = new Error("customer_api requires `url` in --input");
+    error.status = 412;
+    throw error;
+  }
+
+  const method = String(input.method || "GET").toUpperCase();
+  const headers = { ...(input.headers || {}) };
+  const params = input.params;
+  const body = input.body;
+  const timeoutMs = Number(input.timeout) || 30_000;
+
+  // append query string
+  const finalUrl = params && typeof params === "object" && Object.keys(params).length
+    ? appendQuery(url, params)
+    : url;
+
+  // serialize body
+  let serializedBody;
+  if (body !== undefined && body !== null && !isBodylessMethod(method)) {
+    if (typeof body === "string") {
+      serializedBody = body;
+    } else {
+      serializedBody = JSON.stringify(body);
+      if (!hasContentType(headers)) headers["Content-Type"] = "application/json";
+    }
+  }
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const response = await fetch(finalUrl, {
+      method,
+      headers,
+      body: serializedBody,
+      signal: controller.signal
+    });
+    const text = await response.text();
+    const data = parseMaybeJson(text);
+    if (!response.ok) {
+      const error = new Error(`customer_api HTTP ${response.status}`);
+      error.status = response.status;
+      error.response = data;
+      throw error;
+    }
+    return data;
+  } catch (e) {
+    if (e.name === "AbortError") {
+      const error = new Error(`customer_api timeout after ${timeoutMs}ms`);
+      error.status = 504;
+      throw error;
+    }
+    throw e;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function appendQuery(url, params) {
+  const search = new URLSearchParams();
+  for (const [key, value] of Object.entries(params)) {
+    if (value === undefined || value === null) continue;
+    search.set(key, String(value));
+  }
+  const sep = url.includes("?") ? "&" : "?";
+  return `${url}${sep}${search.toString()}`;
+}
+
+function isBodylessMethod(method) {
+  return method === "GET" || method === "HEAD";
+}
+
+function hasContentType(headers) {
+  return Object.keys(headers).some((key) => key.toLowerCase() === "content-type");
+}
+
+function parseMaybeJson(text) {
+  if (!text) return {};
+  try {
+    return JSON.parse(text);
+  } catch {
+    return { text };
+  }
+}

package/src/estimate.js

@@ -0,0 +1,64 @@
+import { billingForCapability, unitPriceFor } from "./billing.js";
+
+/**
+ * Estimation shown by dry-run.
+ *
+ * unitPriceToken prefers the real price synced from /configs onto the capability;
+ * falls back to billing.js when remote fetch failed.
+ *
+ * unlock_contact capabilities may charge less due to backend dedup (same person_id
+ * unlocked again is free). The estimate does not predict this — it returns the
+ * theoretical maximum cost.
+ */
+export function estimateJob(config, job, rows) {
+  const rowCount = rows.length;
+  const billing = billingForCapability(job.capability);
+  const unitCost = unitPriceFor(job.capability);
+  const effectiveConcurrency = effectiveConcurrencyOf(config, job);
+  const avgLatency = avgLatencyOf(job.capability);
+  const estimatedSeconds = Math.ceil((rowCount / Math.max(effectiveConcurrency, 1)) * avgLatency);
+
+  return {
+    capability: job.capability.id,
+    funcName: job.capability.funcName || job.capability.id,
+    displayName: job.capability.displayName,
+    rowCount,
+    unitCost,
+    unit: billing.unit,
+    billingAction: billing.action,
+    estimatedApiCalls: rowCount,
+    estimatedTokens: rowCount * unitCost,
+    effectiveConcurrency,
+    estimatedSeconds,
+    estimateMethod: billing.source === "remote" ? "remote_unit_price" : "local_fallback",
+    note: noteFor(job.capability)
+  };
+}
+
+function effectiveConcurrencyOf(config, job) {
+  const cliMax = Number(config?.defaults?.concurrency || 3);
+  const capMax = Number(job?.capability?.concurrency || cliMax);
+  return Math.max(1, Math.min(cliMax, capMax));
+}
+
+/**
+ * Approximate per-row latency in seconds. Used to display estimatedSeconds in dry-run.
+ * Bucketed by featId.
+ */
+function avgLatencyOf(capability) {
+  const featId = capability?.featId || "";
+  if (featId.startsWith("run_llm")) return 30;            // LLM ~30s
+  if (featId === "enrich_flashagent") return 8;           // scrape/search ~8s
+  if (featId === "verify_phone_number") return 2;
+  if (featId === "verify_email_address") return 2;
+  if (featId === "unlock_contact") return 3;
+  if (capability?.concurrency === 1) return 10;           // LinkedIn realtime scrape (serial)
+  return 2;
+}
+
+function noteFor(capability) {
+  if (capability?.featId === "unlock_contact") {
+    return "Rows already unlocked may be served from cache (cost=0). Actual cost ≤ estimate.";
+  }
+  return undefined;
+}

package/src/flashrev-client.js

@@ -0,0 +1,338 @@
+import { callCustomerApi } from "./customer-api.js";
+import { firstDefined, getPath, redactSecrets } from "./utils.js";
+
+/**
+ * FlashRev open-api client.
+ *
+ * Wire path:
+ *   CLI → X-API-Key header → FlashRev API gateway (open-ai-api.flashlabs.ai)
+ *
+ * Four endpoints:
+ *   /flashrev/api/v2/oauth/me                              token balance
+ *   /flashrev/api/v2/commodity/token/transaction/list      consumption history
+ *   /flashrev/api/v1/enrich/configs                        capability registry (incl. unitPriceToken)
+ *   /flashrev/api/v1/enrich/run                            per-row enrichment (sync)
+ */
+export class FlashRevEnrichClient {
+  constructor(config) {
+    this.config = config;
+  }
+
+  /** Query token balance. */
+  async tokens() {
+    return normalizeTokens(await this.requestEndpoint("tokens", {}));
+  }
+
+  /**
+   * Query token consumption history.
+   *
+   * The backend MVP does not support startTime/endTime — the CLI filters locally by date.
+   * When --from/--to are given we may need to paginate to cover the desired window.
+   * Paging stops on any of:
+   *   - reached the requested limit
+   *   - current page already entirely older than --from (no point paging further)
+   *   - hit MAX_PAGES (safety bound)
+   *   - backend returned an empty page
+   */
+  async tokenHistory({ from, to, limit, featIds } = {}) {
+    const MAX_PAGES = 20;       // at most 20 × 100 = 2000 records
+    const pageSize = 100;
+    const fromTs = from ? Date.parse(`${from}T00:00:00`) : null;
+    const limitNum = limit ? Number(limit) : Infinity;
+
+    const collected = [];
+    for (let page = 1; page <= MAX_PAGES; page++) {
+      const payload = { page, pageSize, transactionType: 2 };
+      if (Array.isArray(featIds) && featIds.length) payload.featIds = featIds;
+      const raw = await this.requestEndpoint("tokenHistory", payload);
+      const batch = normalizeTokenHistory(raw);
+      if (!batch.length) break;
+
+      const filtered = filterByDateRange(batch, from, to);
+      collected.push(...filtered);
+
+      if (collected.length >= limitNum) break;
+
+      // Early stop: if the oldest item on this page is already older than --from, paging further is wasted.
+      if (fromTs != null) {
+        const oldest = batch[batch.length - 1];
+        const oldestTs = oldest?.date ? Date.parse(String(oldest.date).replace(" ", "T")) : null;
+        if (oldestTs != null && oldestTs < fromTs) break;
+      }
+      // No filter at all → keep original behavior: first page only.
+      if (!from && !to && !limit) break;
+      if (batch.length < pageSize) break;   // last page
+    }
+
+    return limit ? collected.slice(0, limitNum) : collected;
+  }
+
+  /** Fetch the capability registry (~34 entries). */
+  async listConfigs() {
+    return normalizeConfigs(await this.requestEndpoint("configs"));
+  }
+
+  /**
+   * Per-row enrichment. When capability.funcName === "customer_api" the CLI handles it locally.
+   * Everything else goes through POST /api/v1/enrich/run with { func_name, input }.
+   */
+  async enrichRow({ capability, input, outputs = [], rowIndex } = {}) {
+    const funcName = capability.funcName || capability.id;
+
+    if (funcName === "customer_api") {
+      const data = await callCustomerApi(input);
+      return normalizeEnrichResponse({ data }, outputs, { cost: { tokens: 0, cached: false } });
+    }
+
+    const body = {
+      func_name: funcName,
+      input: input || {},
+      row_id: rowIndex == null ? undefined : `row-${rowIndex}`
+    };
+    const raw = await this.requestEndpoint("enrich", body);
+    return normalizeEnrichResponse(raw, outputs);
+  }
+
+  // ──────────────────────────────────────────────────────────
+  // Generic HTTP layer
+  // ──────────────────────────────────────────────────────────
+
+  async requestEndpoint(name, payload = undefined) {
+    const endpoint = this.config.flashrev.endpoints[name];
+    if (!endpoint) throw new Error(`Missing FlashRev endpoint config for ${name}.`);
+    return this.request(endpoint, payload, name);
+  }
+
+  async request(endpoint, payload = undefined, label = "request") {
+    const apiKey = this.config.__apiKey;
+    if (!apiKey) {
+      const envName = this.config.flashrev.apiKeyEnv || "FLASHREV_API_KEY";
+      const error = new Error(`Missing FlashRev API key. Set ${envName} or flashrev.apiKey in config.`);
+      error.hint = "Users can get an API key from FlashRev, then export it before running this CLI.";
+      throw error;
+    }
+
+    const method = String(endpoint.method || "POST").toUpperCase();
+    const isBodyless = method === "GET" || method === "HEAD";
+    const url = new URL(endpoint.path, this.config.flashrev.baseUrl);
+    if (isBodyless && payload && typeof payload === "object") {
+      for (const [key, value] of Object.entries(payload)) {
+        if (value === undefined || value === null) continue;
+        url.searchParams.set(key, String(value));
+      }
+    }
+
+    const headers = { Accept: "application/json", "Content-Type": "application/json" };
+    applyAuth(headers, this.config.flashrev.auth, apiKey);
+
+    const response = await fetch(url, {
+      method,
+      headers,
+      body: isBodyless ? undefined : JSON.stringify(payload || {})
+    });
+
+    const text = await response.text();
+    const parsed = parseMaybeJson(text);
+
+    if (!response.ok) {
+      const detail = parsed?.msg || parsed?.message || parsed?.error || "";
+      const error = new Error(`FlashRev ${label} failed: HTTP ${response.status}${detail ? ` - ${detail}` : ""}`);
+      error.status = response.status;
+      error.response = redactSecrets(parsed);
+      if (response.status === 401 || response.status === 403) {
+        error.hint = "Check the FlashRev API key and account permissions.";
+      }
+      if (response.status === 402) {
+        error.hint = "The FlashRev account does not have enough tokens for this operation.";
+      }
+      if (response.status === 429) {
+        error.hint = "Rate limited. Retry later or lower --concurrency.";
+      }
+      throw error;
+    }
+
+    // Business-level error: HTTP 200 but inner code != 200.
+    if (parsed && typeof parsed === "object" && parsed.code != null && parsed.code !== 200) {
+      const error = new Error(`FlashRev ${label} failed: code=${parsed.code} ${parsed.msg || ""}`);
+      error.status = Number(parsed.code) || 500;
+      error.response = redactSecrets(parsed);
+      throw error;
+    }
+
+    return parsed;
+  }
+}
+
+// ──────────────────────────────────────────────────────────
+// Response normalization
+// ──────────────────────────────────────────────────────────
+
+/**
+ * Token balance returned by /api/v2/oauth/me. The token fields live under data.limit:
+ *   tokenTotal           total quota
+ *   tokenCost            consumed amount
+ *   tokenCategoryRemain  remaining by bucket (SUBSCRIPTION / GIFT / ADDON)
+ *   newCreditFlag = "Y"  account is on the token-credit model
+ * remaining = tokenTotal - tokenCost.
+ */
+export function normalizeTokens(raw = {}) {
+  const data = raw?.data ?? raw;
+  const limit = data?.limit || {};
+  const total = Number(limit.tokenTotal ?? 0);
+  const used = Number(limit.tokenCost ?? 0);
+  const remaining = Math.max(0, total - used);
+  return {
+    remaining,
+    used,
+    total,
+    plan: data?.vip?.packageName || firstByPaths(data, ["packageName", "plan"]) || "",
+    tokenCategoryRemain: limit.tokenCategoryRemain || null,
+    newCreditFlag: data?.newCreditFlag || null,
+    raw
+  };
+}
+
+/** Consumption history returned by /commodity/token/transaction/list. */
+export function normalizeTokenHistory(raw = {}) {
+  const entries =
+    firstByPaths(raw, [
+      "data.list",
+      "data.records",
+      "list",
+      "records",
+      "items",
+      "data.items",
+      "data.history",
+      "history"
+    ]) || [];
+  return entries.map((entry) => ({
+    date: firstDefined(
+      entry.createdTime,
+      entry.createdAt,
+      entry.created_at,
+      entry.date,
+      entry.time,
+      ""
+    ),
+    action: firstDefined(entry.featName, entry.action, entry.featId, entry.feat_id, entry.name, ""),
+    featId: firstDefined(entry.featId, entry.feat_id, ""),
+    tokens: Number(firstDefined(entry.tokenAmount, entry.tokens, entry.amount, entry.cost, 0)),
+    unit: firstDefined(entry.unit, ""),
+    status: firstDefined(entry.status, ""),
+    raw: entry
+  }));
+}
+
+/**
+ * Capability registry returned by /enrich/configs. Public fields:
+ *   funcName, displayName, featId, unitPriceToken, concurrency,
+ *   inputColumn[], outputColumn[], rules[][]
+ */
+export function normalizeConfigs(raw = {}) {
+  const list = firstByPaths(raw, ["data.list", "data", "list"]) || [];
+  if (!Array.isArray(list)) return [];
+  return list;
+}
+
+/**
+ * Per-row result returned by /enrich/run.
+ * Backend EnrichRunResult: { code, msg, data, cost: { tokens, cached } }
+ */
+export function normalizeEnrichResponse(raw = {}, outputs = [], overrides = {}) {
+  const data = extractEnrichData(raw);
+  const cost = overrides.cost ?? raw.cost ?? raw.data?.cost ?? { tokens: 0, cached: false };
+  const values = {};
+  if (!outputs.length && data && typeof data === "object" && !Array.isArray(data)) {
+    return { values: flattenDynamicData(data), cost, raw };
+  }
+  for (const output of outputs) {
+    values[output.outputColumn] = firstByPaths(data, [
+      output.responseField,
+      `data.${output.responseField}`,
+      `result.${output.responseField}`
+    ]);
+  }
+  return { values, cost, raw };
+}
+
+export function buildEnrichPayload(args) {
+  // Backward-compat helper for tests: emit the new request body shape.
+  return {
+    func_name: args?.capability?.funcName || args?.capability?.id,
+    input: args?.input || {},
+    row_id: args?.rowIndex == null ? undefined : `row-${args.rowIndex}`
+  };
+}
+
+// ──────────────────────────────────────────────────────────
+// Utilities
+// ──────────────────────────────────────────────────────────
+
+function extractEnrichData(raw = {}) {
+  return firstDefined(raw.values, raw.data?.values, raw.data, raw.result, raw.results, raw.output, raw);
+}
+
+function flattenDynamicData(data) {
+  const blocked = new Set(["raw", "metadata", "usage", "debug", "sources", "cost"]);
+  return Object.fromEntries(
+    Object.entries(data).filter(([key, value]) => {
+      if (blocked.has(key)) return false;
+      return value === null || ["string", "number", "boolean"].includes(typeof value) || Array.isArray(value);
+    })
+  );
+}
+
+function firstByPaths(raw, paths = []) {
+  for (const path of paths) {
+    const value = getPath(raw, path);
+    if (value !== undefined && value !== null && value !== "") return value;
+  }
+  return undefined;
+}
+
+function applyAuth(headers, auth, apiKey) {
+  const type = auth?.type || "bearer";
+  const header = auth?.header || "Authorization";
+  if (type === "header") {
+    headers[header] = apiKey;
+    return;
+  }
+  headers[header] = `${auth?.prefix ?? "Bearer "}${apiKey}`;
+}
+
+function parseMaybeJson(text) {
+  if (!text) return {};
+  try {
+    return JSON.parse(text);
+  } catch {
+    return { text };
+  }
+}
+
+function filterByDateRange(history, from, to) {
+  // --from / --to are interpreted as local-timezone dates (YYYY-MM-DD).
+  // Backend createdTime like "2026-05-29 06:59:56" is also local time.
+  // Use local Date constructors to avoid UTC offset filtering out same-day records.
+  const fromTs = from ? localDayStart(from) : null;
+  const toTs = to ? localDayEnd(to) : null;
+  return history.filter((item) => {
+    if (!item.date) return false;
+    const ts = Date.parse(String(item.date).replace(" ", "T"));
+    if (Number.isNaN(ts)) return false;
+    if (fromTs != null && ts < fromTs) return false;
+    if (toTs != null && ts > toTs) return false;
+    return true;
+  });
+}
+
+function localDayStart(ymd) {
+  const m = /^(\d{4})-(\d{2})-(\d{2})/.exec(ymd);
+  if (!m) return Date.parse(ymd);
+  return new Date(Number(m[1]), Number(m[2]) - 1, Number(m[3]), 0, 0, 0, 0).getTime();
+}
+
+function localDayEnd(ymd) {
+  const m = /^(\d{4})-(\d{2})-(\d{2})/.exec(ymd);
+  if (!m) return Date.parse(ymd);
+  return new Date(Number(m[1]), Number(m[2]) - 1, Number(m[3]), 23, 59, 59, 999).getTime();
+}

package/src/job.js

@@ -0,0 +1,126 @@
+import { getCapability } from "./capabilities.js";
+import { routePromptToJob } from "./prompt-router.js";
+import { readJson, parseKeyValueList } from "./utils.js";
+
+export async function resolveJob(flags, cliMaps, cliOutputs, headers = [], { client, capabilities } = {}) {
+  const jobPath = flags.job;
+  const fileJob = jobPath ? await readJson(jobPath) : {};
+
+  const explicitCapabilityId = flags.capability || fileJob.capability;
+
+  // Explicit --capability wins. If --prompt is also set, ignore it with a stderr warning.
+  if (explicitCapabilityId && flags.prompt) {
+    process.stderr.write(
+      `warning: --prompt ignored because --capability=${explicitCapabilityId} is set explicitly.\n`
+    );
+  }
+
+  // Prompt-only path: ask run_llm to route the natural-language request to a funcName.
+  if (!explicitCapabilityId && flags.prompt) {
+    if (!client || !Array.isArray(capabilities)) {
+      throw new Error("Internal: --prompt routing requires the FlashRev client and capability registry.");
+    }
+    process.stderr.write("Routing prompt via run_llm (1 token)...\n");
+    const routed = await routePromptToJob({ client, prompt: flags.prompt, csvHeaders: headers, capabilities });
+    const capability = getCapability(routed.funcName);
+    const inputMapping = { ...routed.inputMapping, ...cliMaps };
+    const outputMapping = { ...routed.outputMapping, ...cliOutputs };
+    let outputs;
+    try {
+      outputs = resolveOutputs(capability, outputMapping, flags);
+    } catch (e) {
+      // Surface the LLM's routing reasoning so the user knows why they ended up here.
+      const reasoningHint = routed.reasoning ? ` LLM reasoning: ${routed.reasoning}` : "";
+      throw new Error(
+        `Prompt routing produced an unusable job for ${routed.funcName}: ${e.message}${reasoningHint} Retry with --capability.`
+      );
+    }
+    return {
+      capability,
+      prompt: flags.prompt,
+      inputMapping,
+      outputMapping,
+      outputs,
+      routedFromPrompt: true,
+      routerReasoning: routed.reasoning,
+      routerTokensCharged: routed.tokensCharged
+    };
+  }
+
+  if (!explicitCapabilityId) {
+    throw new Error("Missing enrichment intent. Provide --capability, --prompt, or --job.");
+  }
+
+  const capability = getCapability(explicitCapabilityId);
+  const prompt = fileJob.prompt || "";
+  const inputMapping = {
+    ...(fileJob.inputMapping || fileJob.inputs || {}),
+    ...cliMaps
+  };
+  const outputMapping = {
+    ...(fileJob.outputMapping || fileJob.outputs || {}),
+    ...cliOutputs
+  };
+  const outputs = resolveOutputs(capability, outputMapping, flags);
+  return { capability, prompt, inputMapping, outputMapping, outputs };
+}
+
+export function mappingsFromFlags(mapValues, outputValues) {
+  return {
+    inputMapping: parseKeyValueList(mapValues, "--map"),
+    outputMapping: parseKeyValueList(outputValues, "--output")
+  };
+}
+
+export function resolveOutputs(capability, outputMapping, flags = {}) {
+  const entries = Object.entries(outputMapping || {});
+  if (entries.length) {
+    return entries.map(([outputColumn, responseField]) => ({ outputColumn, responseField }));
+  }
+
+  const requested = flags.outputFields
+    ? String(flags.outputFields)
+        .split(",")
+        .map((field) => field.trim())
+        .filter(Boolean)
+    : [];
+
+  if (requested.length) {
+    return requested.map((field) => ({ outputColumn: field, responseField: field }));
+  }
+
+  if (capability.dynamicOutput) {
+    throw new Error(
+      `${capability.id} has dynamic outputs. Provide --output field=field or --output-fields field1,field2.`
+    );
+  }
+
+  return capability.outputFields.map((field) => ({
+    outputColumn: field.key,
+    responseField: field.key
+  }));
+}
+
+export function validateJobForHeaders(job, headers) {
+  const headerSet = new Set(headers);
+  const missingColumns = Object.entries(job.inputMapping)
+    .filter(([, column]) => !headerSet.has(column))
+    .map(([input, column]) => `${input}=${column}`);
+  if (missingColumns.length) {
+    throw new Error(`Input mapping references missing CSV columns: ${missingColumns.join(", ")}`);
+  }
+
+  const satisfiable =
+    job.capability.rules.length === 0 ||
+    job.capability.rules.some((rule) => rule.every((field) => job.inputMapping[field]));
+  if (!satisfiable) {
+    const rules = job.capability.rules.map((rule) => rule.join(" + ")).join(" OR ");
+    throw new Error(`Input mapping does not satisfy ${job.capability.id}. Need one of: ${rules}`);
+  }
+}
+
+export function buildInputForRow(row, inputMapping) {
+  const input = {};
+  for (const [inputField, column] of Object.entries(inputMapping)) input[inputField] = row[column] ?? "";
+  return input;
+}