deepline 0.0.1 → 0.1.1

package/dist/repo/sdk/src/tool-output.ts

@@ -0,0 +1,489 @@
+/**
+ * Tool output processing utilities.
+ *
+ * Tools return data in varied shapes — some return flat objects, others
+ * wrap results in `{ result: { data: [...] } }` envelopes, and list
+ * responses can be nested at different depths. This module provides
+ * utilities to normalize, extract, and persist tool outputs.
+ *
+ * ## Key functions
+ *
+ * - {@link createToolCallResult} — Wrap a tool response with ergonomic accessors.
+ *
+ * - {@link tryConvertToList} — Extract a list of records from any tool response.
+ *   Uses configured `listExtractorPaths` first, then falls back to auto-detection.
+ *
+ * - {@link writeCsvOutputFile} — Write records to a CSV file with automatic
+ *   column ordering, cell escaping, and preview generation.
+ *
+ * - {@link writeJsonOutputFile} — Write any payload to a pretty-printed JSON file.
+ *
+ * - {@link extractSummaryFields} — Pull out scalar fields (string, number, boolean)
+ *   for a quick summary of a tool response.
+ *
+ * All file outputs go to `~/.local/share/deepline/data/` with timestamped filenames.
+ *
+ * @module
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * Result of converting a tool response to a list of records.
+ *
+ * @example
+ * ```typescript
+ * const conversion = tryConvertToList(toolResponse, {
+ *   listExtractorPaths: ['people', 'result.data'],
+ * });
+ * if (conversion) {
+ *   console.log(`Found ${conversion.rows.length} rows via ${conversion.strategy}`);
+ *   console.log(`Source path: ${conversion.sourcePath}`);
+ * }
+ * ```
+ */
+export type ListConversionResult = {
+  /** Normalized array of record objects. Scalars are wrapped as `{ value: <scalar> }`. */
+  rows: Array<Record<string, unknown>>;
+  /**
+   * How the list was found:
+   * - `'configured_paths'` — matched one of the `listExtractorPaths`
+   * - `'auto_detected'` — found via recursive DFS (longest array wins)
+   */
+  strategy: 'configured_paths' | 'auto_detected';
+  /** Dotted path to where the list was found (e.g. `"result.data"`, `"people"`). */
+  sourcePath: string | null;
+};
+
+type Scalar = string | number | boolean | null;
+
+/** Ergonomic wrapper returned by high-level SDK tool execution. */
+export type ToolCallResult = {
+  /** Raw tool response. Use this when a provider-specific shape matters. */
+  readonly value: unknown;
+  /** Best-effort email extraction from common response shapes. */
+  getEmail(): string | null;
+  /** Best-effort phone extraction from common response shapes. */
+  getPhone(): string | null;
+  /** Best-effort list extraction. Returns rows only; use `tryConvertToList` for metadata. */
+  tryList(options?: { listExtractorPaths?: string[] }): Array<Record<string, unknown>> | null;
+};
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+const EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+const PHONE_KEY_PATTERN = /(^|[_-])(phone|mobile|cell|telephone|tel)([_-]|$)|phone|mobile|telephone/i;
+
+function normalizeScalarString(value: unknown): string | null {
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : null;
+  }
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return String(value);
+  }
+  return null;
+}
+
+function looksLikeEmail(value: unknown): string | null {
+  const candidate = normalizeScalarString(value);
+  if (!candidate || !EMAIL_PATTERN.test(candidate)) return null;
+  return candidate;
+}
+
+function looksLikePhone(value: unknown): string | null {
+  const candidate = normalizeScalarString(value);
+  if (!candidate) return null;
+  const digits = candidate.replace(/\D/g, '');
+  if (digits.length < 7 || digits.length > 16) return null;
+  return candidate;
+}
+
+function findEmail(value: unknown, depth = 0): string | null {
+  if (depth > 6) return null;
+
+  const direct = looksLikeEmail(value);
+  if (direct) return direct;
+
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const nested = findEmail(entry, depth + 1);
+      if (nested) return nested;
+    }
+    return null;
+  }
+
+  if (!isPlainObject(value)) return null;
+
+  for (const [key, child] of Object.entries(value)) {
+    if (/email/i.test(key)) {
+      const keyed = looksLikeEmail(child);
+      if (keyed) return keyed;
+    }
+  }
+
+  for (const child of Object.values(value)) {
+    const nested = findEmail(child, depth + 1);
+    if (nested) return nested;
+  }
+
+  return null;
+}
+
+function findPhone(value: unknown, depth = 0): string | null {
+  if (depth > 6) return null;
+
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const nested = findPhone(entry, depth + 1);
+      if (nested) return nested;
+    }
+    return null;
+  }
+
+  if (!isPlainObject(value)) return null;
+
+  for (const [key, child] of Object.entries(value)) {
+    if (PHONE_KEY_PATTERN.test(key)) {
+      const keyed = looksLikePhone(child);
+      if (keyed) return keyed;
+    }
+  }
+
+  for (const child of Object.values(value)) {
+    const nested = findPhone(child, depth + 1);
+    if (nested) return nested;
+  }
+
+  return null;
+}
+
+class DeeplineToolCallResult implements ToolCallResult {
+  constructor(readonly value: unknown) {}
+
+  getEmail(): string | null {
+    return findEmail(this.value);
+  }
+
+  getPhone(): string | null {
+    return findPhone(this.value);
+  }
+
+  tryList(options?: { listExtractorPaths?: string[] }): Array<Record<string, unknown>> | null {
+    return tryConvertToList(this.value, options)?.rows ?? null;
+  }
+}
+
+/** Wrap a raw tool response with the high-level SDK result API. */
+export function createToolCallResult(value: unknown): ToolCallResult {
+  return new DeeplineToolCallResult(value);
+}
+
+/**
+ * Traverse a nested object by a dotted path string.
+ *
+ * @param root - Object to traverse
+ * @param dottedPath - Path like `"result.data.items"`
+ * @returns Value at the path, or `null` if not found
+ *
+ * @example
+ * ```typescript
+ * getByDottedPath({ a: { b: { c: 42 } } }, 'a.b.c') // 42
+ * getByDottedPath({ a: 1 }, 'a.b.c')                 // null
+ * ```
+ */
+function getByDottedPath(root: unknown, dottedPath: string): unknown {
+  let current = root;
+  for (const segment of String(dottedPath || '').split('.').filter(Boolean)) {
+    if (!isPlainObject(current) || !(segment in current)) {
+      return null;
+    }
+    current = current[segment];
+  }
+  return current;
+}
+
+/**
+ * Normalize an array value to an array of record objects.
+ * Non-object entries are wrapped as `{ value: <entry> }`.
+ */
+function normalizeRows(value: unknown): Array<Record<string, unknown>> | null {
+  if (!Array.isArray(value)) return null;
+  return value.map((entry) => {
+    if (isPlainObject(entry)) return entry;
+    return { value: entry };
+  });
+}
+
+/**
+ * Generate candidate root objects to search for lists.
+ * Tries: raw payload → payload.result → payload.result.data.
+ */
+function candidateRoots(payload: unknown): Array<{ path: string | null; value: unknown }> {
+  const roots: Array<{ path: string | null; value: unknown }> = [{ path: null, value: payload }];
+  if (isPlainObject(payload) && isPlainObject(payload.result)) {
+    roots.push({ path: 'result', value: payload.result });
+    if (isPlainObject(payload.result.data)) {
+      roots.push({ path: 'result.data', value: payload.result.data });
+    }
+  }
+  return roots;
+}
+
+/**
+ * Recursively search for the largest array of objects in a nested structure.
+ * Depth-limited to 5 levels. Prefers arrays with real object entries
+ * (not just `{ value: ... }` wrappers).
+ */
+function findBestArrayCandidate(
+  value: unknown,
+  pathPrefix = '',
+  depth = 0,
+): { path: string; rows: Array<Record<string, unknown>> } | null {
+  if (depth > 5) return null;
+
+  const directRows = normalizeRows(value);
+  const hasObjectRow = directRows?.some((row) => Object.keys(row).some((key) => key !== 'value')) ?? false;
+  let best: { path: string; rows: Array<Record<string, unknown>> } | null =
+    directRows && directRows.length > 0 && hasObjectRow
+      ? { path: pathPrefix, rows: directRows }
+      : null;
+
+  if (!isPlainObject(value)) {
+    return best;
+  }
+
+  for (const [key, child] of Object.entries(value)) {
+    const childPath = pathPrefix ? `${pathPrefix}.${key}` : key;
+    const candidate = findBestArrayCandidate(child, childPath, depth + 1);
+    if (!candidate) continue;
+    if (!best || candidate.rows.length > best.rows.length) {
+      best = candidate;
+    }
+  }
+
+  return best;
+}
+
+/**
+ * Extract a list of records from a tool response.
+ *
+ * Handles the common problem of tools returning data in varied shapes.
+ * First tries configured `listExtractorPaths` (from tool metadata), then
+ * falls back to automatic detection via recursive DFS.
+ *
+ * ## Extraction strategy
+ *
+ * 1. **Configured paths** — If `listExtractorPaths` is provided, each path is
+ *    tried against multiple candidate roots (raw payload, `.result`, `.result.data`).
+ *    First match wins.
+ *
+ * 2. **Auto-detection** — If no configured path matches, recursively searches
+ *    the response for the largest array of objects (up to depth 5).
+ *
+ * @param payload - Raw tool response
+ * @param options - Optional extraction configuration
+ * @returns Extracted list with metadata, or `null` if no list found
+ *
+ * @example Using configured paths (from tool metadata)
+ * ```typescript
+ * const meta = await client.getTool('apollo_people_search');
+ * const result = await client.executeTool('apollo_people_search', { query: 'cto' });
+ *
+ * const list = tryConvertToList(result, {
+ *   listExtractorPaths: meta.listExtractorPaths,
+ * });
+ * if (list) {
+ *   console.log(`${list.rows.length} people found via ${list.strategy}`);
+ *   // Write to CSV
+ *   const csv = writeCsvOutputFile(list.rows, 'apollo-people');
+ *   console.log(`Saved to ${csv.path}`);
+ * }
+ * ```
+ *
+ * @example Auto-detection (no configured paths)
+ * ```typescript
+ * const result = await client.executeTool('some_tool', { query: 'test' });
+ * const list = tryConvertToList(result);
+ * // Finds the largest array of objects anywhere in the response
+ * ```
+ */
+export function tryConvertToList(
+  payload: unknown,
+  options?: { listExtractorPaths?: string[] },
+): ListConversionResult | null {
+  const listExtractorPaths = Array.isArray(options?.listExtractorPaths)
+    ? options?.listExtractorPaths.filter((entry): entry is string => typeof entry === 'string' && entry.trim().length > 0)
+    : [];
+
+  if (listExtractorPaths.length > 0) {
+    for (const root of candidateRoots(payload)) {
+      for (const extractorPath of listExtractorPaths) {
+        const resolved = getByDottedPath(root.value, extractorPath);
+        const rows = normalizeRows(resolved);
+        if (rows && rows.length > 0) {
+          const sourcePath = root.path ? `${root.path}.${extractorPath}` : extractorPath;
+          return { rows, strategy: 'configured_paths', sourcePath };
+        }
+      }
+    }
+  }
+
+  for (const root of candidateRoots(payload)) {
+    const candidate = findBestArrayCandidate(root.value, root.path ?? '');
+    if (!candidate || candidate.rows.length === 0) continue;
+    return {
+      rows: candidate.rows,
+      strategy: 'auto_detected',
+      sourcePath: candidate.path || root.path,
+    };
+  }
+
+  return null;
+}
+
+/** Ensure the shared output directory exists. Returns its path. */
+function ensureOutputDir(): string {
+  const outputDir = join(homedir(), '.local', 'share', 'deepline', 'data');
+  mkdirSync(outputDir, { recursive: true });
+  return outputDir;
+}
+
+/**
+ * Write a JSON payload to a timestamped file.
+ *
+ * Output location: `~/.local/share/deepline/data/{stem}_{timestamp}.json`
+ *
+ * @param payload - Any JSON-serializable value
+ * @param stem - Filename prefix (e.g. tool ID or play name)
+ * @returns Absolute path to the written file
+ *
+ * @example
+ * ```typescript
+ * const result = await client.executeTool('test_company_search', { domain: 'stripe.com' });
+ * const path = writeJsonOutputFile(result, 'test_company_search');
+ * console.log(`Saved to ${path}`);
+ * // ~/.local/share/deepline/data/test_company_search_1713456789000.json
+ * ```
+ */
+export function writeJsonOutputFile(payload: unknown, stem: string): string {
+  const outputDir = ensureOutputDir();
+  const outputPath = join(outputDir, `${stem}_${Date.now()}.json`);
+  writeFileSync(outputPath, JSON.stringify(payload, null, 2), 'utf-8');
+  return outputPath;
+}
+
+/**
+ * Write an array of records to a CSV file.
+ *
+ * Columns are ordered by first appearance across all rows. Cells containing
+ * commas, quotes, or newlines are properly escaped. Objects and arrays are
+ * JSON-serialized.
+ *
+ * Output location: `~/.local/share/deepline/data/{stem}_{timestamp}.csv`
+ *
+ * @param rows - Array of record objects
+ * @param stem - Filename prefix
+ * @returns File metadata including path, row count, columns, and a 5×5 preview
+ *
+ * @example
+ * ```typescript
+ * const list = tryConvertToList(toolResponse);
+ * if (list) {
+ *   const csv = writeCsvOutputFile(list.rows, 'search-results');
+ *   console.log(`Wrote ${csv.rowCount} rows, ${csv.columns.length} columns`);
+ *   console.log(`File: ${csv.path}`);
+ *   console.log(`Preview:\n${csv.preview}`);
+ * }
+ * ```
+ */
+export function writeCsvOutputFile(
+  rows: Array<Record<string, unknown>>,
+  stem: string,
+): { path: string; rowCount: number; columns: string[]; preview: string } {
+  const outputDir = ensureOutputDir();
+  const outputPath = join(outputDir, `${stem}_${Date.now()}.csv`);
+  const seen = new Set<string>();
+  const columns: string[] = [];
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!seen.has(key)) {
+        seen.add(key);
+        columns.push(key);
+      }
+    }
+  }
+
+  const escapeCell = (value: unknown): string => {
+    const normalized = value == null
+      ? ''
+      : typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
+        ? String(value)
+        : JSON.stringify(value);
+    if (/[",\n]/.test(normalized)) {
+      return `"${normalized.replace(/"/g, '""')}"`;
+    }
+    return normalized;
+  };
+
+  const lines: string[] = [];
+  lines.push(columns.map(escapeCell).join(','));
+  for (const row of rows) {
+    lines.push(columns.map((column) => escapeCell(row[column])).join(','));
+  }
+  writeFileSync(outputPath, `${lines.join('\n')}\n`, 'utf-8');
+
+  const previewRows = rows.slice(0, 5);
+  const previewColumns = columns.slice(0, 5);
+  const preview = [
+    previewColumns.join(','),
+    ...previewRows.map((row) =>
+      previewColumns.map((column) => escapeCell(row[column])).join(',')),
+  ].join('\n');
+
+  return {
+    path: outputPath,
+    rowCount: rows.length,
+    columns,
+    preview,
+  };
+}
+
+/**
+ * Extract scalar (non-nested) fields from a tool response for summary display.
+ *
+ * Searches through candidate roots (raw → `.result` → `.result.data`) and
+ * returns the first set of scalar fields found. Useful for displaying a
+ * quick summary of single-record responses.
+ *
+ * @param payload - Raw tool response
+ * @returns Object containing only scalar fields (string, number, boolean, null)
+ *
+ * @example
+ * ```typescript
+ * const result = await client.executeTool('test_company_search', { domain: 'stripe.com' });
+ * const summary = extractSummaryFields(result);
+ * // { name: "Stripe", industry: "Financial Services", employeeCount: 8000 }
+ * // (nested objects and arrays are excluded)
+ * ```
+ */
+export function extractSummaryFields(payload: unknown): Record<string, Scalar> {
+  const candidates = candidateRoots(payload);
+  for (const candidate of candidates) {
+    if (!isPlainObject(candidate.value)) continue;
+    const summaryEntries = Object.entries(candidate.value).filter(([, value]) => {
+      return (
+        value == null ||
+        typeof value === 'string' ||
+        typeof value === 'number' ||
+        typeof value === 'boolean'
+      );
+    });
+    if (summaryEntries.length === 0) continue;
+    return Object.fromEntries(summaryEntries) as Record<string, Scalar>;
+  }
+  return {};
+}