deepline 0.0.1 → 0.1.1

package/dist/repo/sdk/src/cli/dataset-stats.ts

@@ -0,0 +1,341 @@
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { csvStringFromRows } from './utils.js';
+
+export type DatasetStats = {
+  total_rows: number;
+  columnStats: Record<
+    string,
+    {
+      non_empty: string;
+      unique: number;
+      sample_value?: unknown;
+      sample_type?: string;
+      top_values?: Record<string, string>;
+    }
+  >;
+  miss_reasons?: Record<string, Record<string, number>>;
+  miss_details?: Record<
+    string,
+    Record<
+      string,
+      {
+        count: number;
+        examples: Array<{ row: number; detail: string }>;
+      }
+    >
+  >;
+};
+
+export type CanonicalRowsInfo = {
+  rows: Array<Record<string, unknown>>;
+  totalRows: number;
+  columns: string[];
+  complete: boolean;
+  source: string | null;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+
+function isSerializedDataset(value: unknown): value is {
+  kind: 'dataset';
+  count: number;
+  preview: unknown[];
+  columns?: unknown[];
+  tableNamespace?: unknown;
+} {
+  return (
+    isRecord(value) &&
+    value.kind === 'dataset' &&
+    typeof value.count === 'number' &&
+    Array.isArray(value.preview)
+  );
+}
+
+function rowArray(value: unknown): Array<Record<string, unknown>> | null {
+  if (!Array.isArray(value)) {
+    return null;
+  }
+  const rows = value.filter(
+    (row): row is Record<string, unknown> => isRecord(row),
+  );
+  return rows.length === value.length ? rows : null;
+}
+
+function readNumber(value: unknown): number | null {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0
+    ? Math.trunc(value)
+    : null;
+}
+
+function inferColumns(rows: Array<Record<string, unknown>>): string[] {
+  const columns: string[] = [];
+  const seen = new Set<string>();
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (key.startsWith('_')) {
+        continue;
+      }
+      if (!seen.has(key)) {
+        seen.add(key);
+        columns.push(key);
+      }
+    }
+  }
+  return columns;
+}
+
+export function extractCanonicalRowsInfo(statusOrResult: unknown): CanonicalRowsInfo | null {
+  const root = isRecord(statusOrResult) ? statusOrResult : null;
+  const result = isRecord(root?.result) ? root.result : root;
+  if (!result) {
+    return null;
+  }
+  const metadata = isRecord(result._metadata) ? result._metadata : null;
+  const totalFromMetadata = metadata?.totalRows ?? metadata?.rowCount ?? metadata?.count;
+
+  const candidates: Array<{ source: string; value: unknown; total?: unknown }> = [
+    { source: 'result.contacts', value: result.contacts, total: totalFromMetadata ?? result.totalRows ?? result.rowCount ?? result.count },
+    { source: 'result.previewRows', value: result.previewRows, total: totalFromMetadata ?? result.totalRows ?? result.rowCount ?? result.count },
+    { source: 'result.rows', value: result.rows, total: totalFromMetadata ?? result.totalRows ?? result.rowCount ?? result.count },
+    { source: 'result.results', value: result.results, total: totalFromMetadata ?? result.totalRows ?? result.rowCount ?? result.count },
+  ];
+  if (isRecord(result.output)) {
+    const outputMetadata = isRecord(result.output._metadata)
+      ? result.output._metadata
+      : null;
+    const outputTotalFromMetadata =
+      outputMetadata?.totalRows ?? outputMetadata?.rowCount ?? outputMetadata?.count;
+    candidates.push(
+      { source: 'result.output.contacts', value: result.output.contacts, total: outputTotalFromMetadata ?? result.output.totalRows ?? result.output.rowCount ?? result.output.count },
+      { source: 'result.output.previewRows', value: result.output.previewRows, total: outputTotalFromMetadata ?? result.output.totalRows ?? result.output.rowCount ?? result.output.count },
+      { source: 'result.output.rows', value: result.output.rows, total: outputTotalFromMetadata ?? result.output.totalRows ?? result.output.rowCount ?? result.output.count },
+      { source: 'result.output.results', value: result.output.results, total: outputTotalFromMetadata ?? result.output.totalRows ?? result.output.rowCount ?? result.output.count },
+    );
+  }
+
+  for (const candidate of candidates) {
+    if (isSerializedDataset(candidate.value)) {
+      const rows = rowArray(candidate.value.preview) ?? [];
+      const totalRows = readNumber(candidate.value.count) ?? rows.length;
+      return {
+        rows,
+        totalRows,
+        columns:
+          Array.isArray(candidate.value.columns) &&
+          candidate.value.columns.every((column) => typeof column === 'string')
+            ? (candidate.value.columns as string[])
+            : inferColumns(rows),
+        complete: rows.length === totalRows,
+        source: candidate.source,
+      };
+    }
+
+    const rows = rowArray(candidate.value);
+    if (!rows) {
+      continue;
+    }
+    const totalRows = readNumber(candidate.total) ?? rows.length;
+    return {
+      rows,
+      totalRows,
+      columns: inferColumns(rows),
+      complete: rows.length === totalRows,
+      source: candidate.source,
+    };
+  }
+
+  return null;
+}
+
+function percentText(numerator: number, denominator: number): string {
+  return denominator > 0
+    ? `${numerator}/${denominator} (${Math.round((100 * numerator) / denominator)}%)`
+    : '0/0 (0%)';
+}
+
+function countPercentText(count: number, denominator: number): string {
+  return denominator > 0
+    ? `${count} (${Math.round((100 * count) / denominator)}%)`
+    : '0 (0%)';
+}
+
+function parseJsonLike(value: unknown): unknown {
+  if (typeof value !== 'string') {
+    return value;
+  }
+  const text = value.trim();
+  if (!text || !['{', '['].includes(text[0] ?? '')) {
+    return value;
+  }
+  try {
+    return JSON.parse(text);
+  } catch {
+    return value;
+  }
+}
+
+function sampleType(value: unknown): string {
+  const parsed = parseJsonLike(value);
+  if (parsed === null || parsed === undefined) return 'null';
+  if (typeof parsed === 'boolean') return 'boolean';
+  if (Number.isInteger(parsed)) return 'integer';
+  if (typeof parsed === 'number') return 'number';
+  if (Array.isArray(parsed)) return 'array';
+  if (typeof parsed === 'object') return 'object';
+  return 'string';
+}
+
+function compactScalar(value: unknown, maxLength = 200): string {
+  const text = String(value).replace(/\s+/g, ' ').trim();
+  return text.length <= maxLength ? text : `${text.slice(0, maxLength - 3)}...`;
+}
+
+function summarizeSampleValue(value: unknown, depth = 0): unknown {
+  const parsed = parseJsonLike(value);
+  if (parsed === null || parsed === undefined) return '';
+  if (typeof parsed === 'string') return compactScalar(parsed);
+  if (typeof parsed === 'number' || typeof parsed === 'boolean') return parsed;
+  if (depth >= 3) {
+    if (Array.isArray(parsed)) return [];
+    if (isRecord(parsed)) return {};
+    return compactScalar(parsed);
+  }
+  if (Array.isArray(parsed)) {
+    return parsed.slice(0, 3).map((item) => summarizeSampleValue(item, depth + 1));
+  }
+  if (isRecord(parsed)) {
+    const out: Record<string, unknown> = {};
+    for (const [key, nested] of Object.entries(parsed)) {
+      if (['__dl', 'meta', 'metadata'].includes(key)) {
+        continue;
+      }
+      if (nested === null || nested === undefined || nested === '') {
+        continue;
+      }
+      out[key] = summarizeSampleValue(nested, depth + 1);
+      if (Object.keys(out).length >= 6) {
+        break;
+      }
+    }
+    return out;
+  }
+  return compactScalar(parsed);
+}
+
+function compactCell(value: unknown): string {
+  const parsed = parseJsonLike(value);
+  if (parsed === null || parsed === undefined) return '';
+  if (typeof parsed === 'string') return compactScalar(parsed, 120);
+  if (typeof parsed === 'number' || typeof parsed === 'boolean') return String(parsed);
+  if (Array.isArray(parsed)) {
+    if (parsed.length === 0) return '';
+    if (parsed.slice(0, 3).every((item) => ['string', 'number', 'boolean'].includes(typeof item))) {
+      return compactScalar(parsed.slice(0, 3).join(' | '), 120);
+    }
+    return `[${parsed.length} items]`;
+  }
+  if (isRecord(parsed)) {
+    for (const key of ['matched_result', 'output']) {
+      if (parsed[key] !== null && parsed[key] !== undefined && parsed[key] !== '') {
+        return compactCell(parsed[key]);
+      }
+    }
+    const preferred = ['email', 'status', 'name', 'full_name', 'title', 'domain', 'linkedin_url'];
+    const parts: string[] = [];
+    for (const key of preferred) {
+      if (parsed[key] !== null && parsed[key] !== undefined && parsed[key] !== '') {
+        parts.push(`${key}=${compactCell(parsed[key])}`);
+      }
+      if (parts.length >= 4) break;
+    }
+    return parts.length > 0 ? compactScalar(parts.join(' | '), 160) : '{}';
+  }
+  return compactScalar(parsed, 120);
+}
+
+export function buildDatasetStats(
+  rows: Array<Record<string, unknown>>,
+  totalRows = rows.length,
+  columns = inferColumns(rows),
+): DatasetStats {
+  const columnStats: DatasetStats['columnStats'] = {};
+  for (const column of columns) {
+    let nonEmpty = 0;
+    let empty = 0;
+    let sampleValue: unknown;
+    let sampleValueType: string | null = null;
+    const valueCounts = new Map<string, number>();
+
+    for (const row of rows) {
+      const raw = row[column];
+      const value = compactCell(raw);
+      if (value) {
+        nonEmpty += 1;
+        valueCounts.set(value, (valueCounts.get(value) ?? 0) + 1);
+        if (sampleValue === undefined) {
+          sampleValue = summarizeSampleValue(raw);
+          sampleValueType = sampleType(raw);
+        }
+      } else {
+        empty += 1;
+      }
+    }
+
+    const denominator = nonEmpty + empty;
+    const stat: DatasetStats['columnStats'][string] = {
+      non_empty: percentText(nonEmpty, denominator),
+      unique: valueCounts.size,
+    };
+    if (sampleValue !== undefined && sampleValueType) {
+      stat.sample_value = sampleValue;
+      stat.sample_type = sampleValueType;
+    }
+    if (valueCounts.size > 0 && valueCounts.size < nonEmpty) {
+      const top = [...valueCounts.entries()]
+        .sort((left, right) => right[1] - left[1])
+        .slice(0, 3);
+      const topKeys = new Set(top.map(([key]) => key));
+      const otherCount = [...valueCounts.entries()]
+        .filter(([key]) => !topKeys.has(key))
+        .reduce((sum, [, count]) => sum + count, 0);
+      stat.top_values = Object.fromEntries(
+        top.map(([key, count]) => [key, countPercentText(count, denominator)]),
+      );
+      if (otherCount > 0) {
+        stat.top_values['(other)'] = countPercentText(otherCount, denominator);
+      }
+      if (empty > 0) {
+        stat.top_values['(null)'] = countPercentText(empty, denominator);
+      }
+    } else if (empty > 0 && nonEmpty > 0) {
+      stat.top_values = { '(null)': countPercentText(empty, denominator) };
+    }
+    columnStats[column] = stat;
+  }
+
+  return {
+    total_rows: totalRows,
+    columnStats,
+  };
+}
+
+export function writeCanonicalRowsCsv(
+  rowsInfo: CanonicalRowsInfo,
+  outPath: string,
+): string {
+  if (!rowsInfo.complete) {
+    throw new Error(
+      `Run output only includes ${rowsInfo.rows.length} preview row(s) of ${rowsInfo.totalRows}; cannot export a complete CSV from this status payload yet.`,
+    );
+  }
+  const resolved = resolve(outPath);
+  writeFileSync(
+    resolved,
+    csvStringFromRows(rowsInfo.rows, rowsInfo.columns),
+    'utf-8',
+  );
+  return resolved;
+}

package/dist/repo/sdk/src/cli/index.ts

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import { DeeplineClient } from '../client.js';
+import { autoDetectBaseUrl } from '../config.js';
+import { enforceSdkCompatibility } from '../compat.js';
+import { SDK_VERSION } from '../version.js';
+import { registerAuthCommands } from './commands/auth.js';
+import { registerBillingCommands } from './commands/billing.js';
+import { registerCsvCommands } from './commands/csv.js';
+import { registerDbCommands } from './commands/db.js';
+import { registerFeedbackCommands } from './commands/feedback.js';
+import { registerOrgCommands } from './commands/org.js';
+import { registerPlayCommands } from './commands/play.js';
+import { registerToolsCommands } from './commands/tools.js';
+import { createCliProgress } from './progress.js';
+import { recordCliTrace, traceCliSpan } from './trace.js';
+import { printJsonError } from './utils.js';
+
+function shouldPrintStartupPhase(): boolean {
+  if (process.argv.includes('--json')) {
+    return false;
+  }
+  const args = process.argv.slice(2);
+  const command = args[0];
+  const subcommand = args[1];
+  return (command === 'play' || command === 'plays') && subcommand === 'run';
+}
+
+async function main(): Promise<void> {
+  const mainStartedAt = Date.now();
+  recordCliTrace({
+    phase: 'cli.main_start',
+    argv: process.argv.slice(2),
+  });
+  const printStartupPhase = shouldPrintStartupPhase();
+  const progress = printStartupPhase ? createCliProgress(true) : null;
+  if (printStartupPhase) {
+    progress?.phase('loading deepline cli');
+  }
+  const program = new Command();
+  program
+    .name('deepline')
+    .description('Deepline CLI (TypeScript SDK)')
+    .version(SDK_VERSION, '-v, --version', 'Show version')
+    .showHelpAfterError()
+    .showSuggestionAfterError(true)
+    .addHelpText(
+      'after',
+      `
+Common commands:
+  deepline health
+  deepline auth status --json
+  deepline plays search email --json
+  deepline plays describe person-linkedin-to-email --json
+  deepline plays run my.play.ts --input '{"domain":"stripe.com"}' --watch
+  deepline tools call hunter_email_verifier --input '{"email":"a@b.com"}'
+
+Output:
+  Structured commands print human-readable output in a terminal and JSON when stdout is piped.
+  Use --json to force JSON in an interactive terminal.
+`,
+    );
+
+  program.hook('preAction', async (_thisCommand, actionCommand) => {
+    if (actionCommand.name() === 'version') {
+      return;
+    }
+    if (printStartupPhase) {
+      progress?.phase('checking sdk compatibility');
+    }
+    const baseUrl = autoDetectBaseUrl().replace(/\/$/, '');
+    await traceCliSpan(
+      'cli.sdk_compatibility',
+      { baseUrl },
+      () => enforceSdkCompatibility(baseUrl),
+    );
+  });
+
+  registerAuthCommands(program);
+  registerToolsCommands(program);
+  registerPlayCommands(program);
+  registerBillingCommands(program);
+  registerOrgCommands(program);
+  registerCsvCommands(program);
+  registerDbCommands(program);
+  registerFeedbackCommands(program);
+
+  program
+    .command('health')
+    .description('Check server health.')
+    .action(async () => {
+      try {
+        const client = new DeeplineClient();
+        const data = await client.health();
+        process.stdout.write(`${JSON.stringify(data, null, 2)}\n`);
+      } catch (error) {
+        throw new Error(
+          `Cannot reach Deepline API: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    });
+
+  program
+    .command('version')
+    .description('Show version.')
+    .action(() => {
+      process.stdout.write(`deepline ${SDK_VERSION}\n`);
+    });
+
+  try {
+    await program.parseAsync(process.argv);
+    recordCliTrace({
+      phase: 'cli.main_total',
+      ms: Date.now() - mainStartedAt,
+      ok: true,
+    });
+  } catch (error) {
+    recordCliTrace({
+      phase: 'cli.main_total',
+      ms: Date.now() - mainStartedAt,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    if (process.argv.includes('--json')) {
+      printJsonError(error);
+    } else if (error instanceof Error) {
+      console.error(`Error: ${error.message}`);
+    } else {
+      console.error(`Error: ${String(error)}`);
+    }
+    process.exitCode = 1;
+  }
+
+  process.exit(process.exitCode ?? 0);
+}
+
+main();

package/dist/repo/sdk/src/cli/progress.ts

@@ -0,0 +1,135 @@
+import { Worker } from 'node:worker_threads';
+
+export class CliProgress {
+  private lastMessage: string | null = null;
+  private readonly interactive = Boolean(process.stderr.isTTY);
+  private readonly grey = '\x1b[90m';
+  private readonly cyan = '\x1b[36m';
+  private readonly dim = '\x1b[2m';
+  private readonly reset = '\x1b[0m';
+  private worker: Worker | null = null;
+
+  constructor(private readonly enabled: boolean) {}
+
+  phase(message: string): void {
+    if (!this.enabled || this.lastMessage === message) {
+      return;
+    }
+    if (this.interactive && this.lastMessage) {
+      this.settle('✓');
+    }
+    this.lastMessage = message;
+    if (!this.interactive) {
+      process.stderr.write(`${message}\n`);
+      return;
+    }
+    this.ensureWorker();
+    this.worker?.postMessage({ type: 'phase', message });
+  }
+
+  complete(): void {
+    this.settle('✓');
+  }
+
+  fail(): void {
+    this.settle('✗');
+  }
+
+  writeLogLine(line: string): void {
+    if (!this.enabled) {
+      process.stderr.write(`${line}\n`);
+      return;
+    }
+    const message = this.interactive
+      ? `${this.cyan}[logs]${this.reset} ${this.dim}${line}${this.reset}`
+      : `[logs] ${line}`;
+    if (!this.interactive) {
+      process.stderr.write(`${message}\n`);
+      return;
+    }
+    const activeMessage = this.lastMessage;
+    this.worker?.terminate().catch(() => undefined);
+    this.worker = null;
+    process.stderr.write(`\r\x1b[2K${message}\n`);
+    if (activeMessage) {
+      this.startWorker().postMessage({ type: 'phase', message: activeMessage });
+    }
+  }
+
+  private settle(mark: '✓' | '✗'): void {
+    if (!this.enabled || !this.lastMessage) {
+      return;
+    }
+    if (this.interactive) {
+      this.worker?.terminate().catch(() => undefined);
+      this.worker = null;
+      process.stderr.write(
+        `\r\x1b[2K${this.grey}${mark} ${this.lastMessage}${this.reset}\n`,
+      );
+    }
+    this.lastMessage = null;
+  }
+
+  private ensureWorker(): void {
+    this.startWorker();
+  }
+
+  private startWorker(): Worker {
+    if (this.worker) {
+      return this.worker;
+    }
+    this.worker = new Worker(
+      `
+        const { parentPort } = require('node:worker_threads');
+        const { writeSync } = require('node:fs');
+        const frames = ['—', '\\\\', '|', '/'];
+        const grey = '\\x1b[90m';
+        const reset = '\\x1b[0m';
+        let message = '';
+        let frameIndex = 0;
+        let timer = null;
+        function render() {
+          if (!message) return;
+          const frame = frames[frameIndex % frames.length] || '—';
+          frameIndex += 1;
+          writeSync(2, '\\r\\x1b[2K' + grey + frame + ' ' + message + reset);
+        }
+        parentPort.on('message', (event) => {
+          if (event.type === 'phase') {
+            message = String(event.message || '');
+            frameIndex = 0;
+            render();
+            if (!timer) {
+              timer = setInterval(render, 120);
+              timer.unref && timer.unref();
+            }
+          }
+          if (event.type === 'stop') {
+            if (timer) clearInterval(timer);
+            timer = null;
+            writeSync(2, '\\r\\x1b[2K');
+          }
+        });
+      `,
+      { eval: true },
+    );
+    this.worker.unref();
+    return this.worker;
+  }
+}
+
+let activeProgress: CliProgress | null = null;
+
+export function setActiveCliProgress(progress: CliProgress | null): void {
+  activeProgress = progress;
+}
+
+export function getActiveCliProgress(): CliProgress | null {
+  return activeProgress;
+}
+
+export function createCliProgress(enabled: boolean): CliProgress {
+  const progress = new CliProgress(enabled);
+  setActiveCliProgress(progress);
+  return progress;
+}

package/dist/repo/sdk/src/cli/trace.ts

@@ -0,0 +1,61 @@
+type CliTraceFields = Record<string, unknown>;
+
+type CliTraceEvent = CliTraceFields & {
+  phase: string;
+  ms?: number;
+  ok?: boolean;
+};
+
+const cliTraceStartedAt = Date.now();
+
+function isTruthyEnv(value: string | undefined): boolean {
+  return value === '1' || value === 'true' || value === 'yes';
+}
+
+export function isCliTraceEnabled(): boolean {
+  return isTruthyEnv(process.env.DEEPLINE_CLI_TRACE);
+}
+
+export function recordCliTrace(event: CliTraceEvent): void {
+  if (!isCliTraceEnabled()) {
+    return;
+  }
+  const now = Date.now();
+  const payload = {
+    ts: now,
+    source: 'cli',
+    sinceStartMs: now - cliTraceStartedAt,
+    ...event,
+  };
+  process.stderr.write(`[cli-trace] ${JSON.stringify(payload)}\n`);
+}
+
+export async function traceCliSpan<T>(
+  phase: string,
+  fields: CliTraceFields,
+  run: () => Promise<T>,
+): Promise<T> {
+  if (!isCliTraceEnabled()) {
+    return run();
+  }
+  const startedAt = Date.now();
+  try {
+    const result = await run();
+    recordCliTrace({
+      phase,
+      ms: Date.now() - startedAt,
+      ok: true,
+      ...fields,
+    });
+    return result;
+  } catch (error) {
+    recordCliTrace({
+      phase,
+      ms: Date.now() - startedAt,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+      ...fields,
+    });
+    throw error;
+  }
+}