@cephalization/phoenix-insight 0.1.0

package/src/snapshot/experiments.ts

@@ -0,0 +1,246 @@
+import type { PhoenixClient } from "@arizeai/phoenix-client";
+import type { ExecutionMode } from "../modes/types.js";
+import { withErrorHandling, extractData } from "./client.js";
+
+interface Dataset {
+  id: string;
+  name: string;
+}
+
+interface Experiment {
+  id: string;
+  dataset_id: string;
+  dataset_version_id: string;
+  repetitions: number;
+  metadata: Record<string, unknown>;
+  project_name: string | null;
+  created_at: string;
+  updated_at: string;
+  example_count: number;
+  successful_run_count: number;
+  failed_run_count: number;
+  missing_run_count: number;
+}
+
+interface ExperimentRun {
+  id: string;
+  experiment_id: string;
+  dataset_example_id: string;
+  start_time: string;
+  end_time: string;
+  output: unknown;
+  error?: string | null;
+  trace_id?: string | null;
+  repetition_number?: number;
+}
+
+interface FetchExperimentsOptions {
+  /**
+   * Maximum number of experiments to fetch per dataset
+   */
+  limit?: number;
+  /**
+   * Include experiment runs in the snapshot
+   */
+  includeRuns?: boolean;
+}
+
+/**
+ * Converts an array to JSONL format
+ */
+function toJSONL(items: unknown[]): string {
+  if (items.length === 0) {
+    return "";
+  }
+  return items.map((item) => JSON.stringify(item)).join("\n");
+}
+
+/**
+ * Fetches all experiments and their runs from Phoenix
+ * Note: Experiments are fetched per dataset since there's no direct "all experiments" endpoint
+ */
+export async function fetchExperiments(
+  client: PhoenixClient,
+  mode: ExecutionMode,
+  options: FetchExperimentsOptions = {}
+): Promise<void> {
+  const { limit = 100, includeRuns = true } = options;
+
+  // First, we need to get all datasets to fetch their experiments
+  const datasetsResponse = await withErrorHandling(
+    () => client.GET("/v1/datasets", { params: { query: { limit: 1000 } } }),
+    "fetching datasets for experiments"
+  );
+
+  const datasetsData = extractData(datasetsResponse);
+  const datasets: Dataset[] = datasetsData.data;
+
+  // Collect all experiments from all datasets
+  const allExperiments: Array<Experiment & { datasetName: string }> = [];
+
+  for (const dataset of datasets) {
+    try {
+      // Fetch experiments for this dataset with pagination
+      const experiments: Experiment[] = [];
+      let cursor: string | null = null;
+
+      do {
+        const response = await withErrorHandling(
+          () =>
+            client.GET("/v1/datasets/{dataset_id}/experiments", {
+              params: {
+                path: {
+                  dataset_id: dataset.id,
+                },
+                query: {
+                  cursor,
+                  limit: 50,
+                },
+              },
+            }),
+          `fetching experiments for dataset ${dataset.name}`
+        );
+
+        const data = extractData(response);
+        experiments.push(...(data.data || []));
+        cursor = data.next_cursor || null;
+
+        // Stop if we've reached the overall limit
+        if (allExperiments.length + experiments.length >= limit) {
+          const remaining = limit - allExperiments.length;
+          experiments.splice(remaining);
+          cursor = null;
+        }
+      } while (cursor != null);
+
+      // Add dataset name to each experiment for context
+      const experimentsWithDatasetName = experiments.map((exp) => ({
+        ...exp,
+        datasetName: dataset.name,
+      }));
+
+      allExperiments.push(...experimentsWithDatasetName);
+
+      // Apply limit if specified
+      if (allExperiments.length >= limit) {
+        break;
+      }
+    } catch (error) {
+      // If fetching experiments for a dataset fails, log and continue
+      console.warn(
+        `Failed to fetch experiments for dataset ${dataset.name}:`,
+        error
+      );
+    }
+  }
+
+  // Write experiments index
+  await mode.writeFile(
+    "/phoenix/experiments/index.jsonl",
+    toJSONL(allExperiments)
+  );
+
+  // Fetch runs for each experiment if requested
+  if (includeRuns) {
+    for (const experiment of allExperiments) {
+      try {
+        // Write experiment metadata
+        await mode.writeFile(
+          `/phoenix/experiments/${experiment.id}/metadata.json`,
+          JSON.stringify(
+            {
+              id: experiment.id,
+              dataset_id: experiment.dataset_id,
+              dataset_name: experiment.datasetName,
+              dataset_version_id: experiment.dataset_version_id,
+              repetitions: experiment.repetitions,
+              metadata: experiment.metadata,
+              project_name: experiment.project_name,
+              created_at: experiment.created_at,
+              updated_at: experiment.updated_at,
+              example_count: experiment.example_count,
+              successful_run_count: experiment.successful_run_count,
+              failed_run_count: experiment.failed_run_count,
+              missing_run_count: experiment.missing_run_count,
+              snapshot_timestamp: new Date().toISOString(),
+            },
+            null,
+            2
+          )
+        );
+
+        // Fetch runs for this experiment with pagination
+        const runs: ExperimentRun[] = [];
+        let cursor: string | null = null;
+
+        do {
+          const runsResponse = await withErrorHandling(
+            () =>
+              client.GET("/v1/experiments/{experiment_id}/runs", {
+                params: {
+                  path: {
+                    experiment_id: experiment.id,
+                  },
+                  query: {
+                    cursor,
+                    limit: 100,
+                  },
+                },
+              }),
+            `fetching runs for experiment ${experiment.id}`
+          );
+
+          const runsData = extractData(runsResponse);
+          runs.push(...(runsData.data || []));
+          cursor = runsData.next_cursor || null;
+        } while (cursor != null);
+
+        // Write runs as JSONL
+        await mode.writeFile(
+          `/phoenix/experiments/${experiment.id}/runs.jsonl`,
+          toJSONL(runs)
+        );
+
+        // Write experiment summary with run stats
+        await mode.writeFile(
+          `/phoenix/experiments/${experiment.id}/summary.json`,
+          JSON.stringify(
+            {
+              experiment_id: experiment.id,
+              dataset_name: experiment.datasetName,
+              project_name: experiment.project_name,
+              total_runs: runs.length,
+              successful_runs: experiment.successful_run_count,
+              failed_runs: experiment.failed_run_count,
+              missing_runs: experiment.missing_run_count,
+              created_at: experiment.created_at,
+              updated_at: experiment.updated_at,
+            },
+            null,
+            2
+          )
+        );
+      } catch (error) {
+        // If fetching runs for an experiment fails, log and continue
+        console.warn(
+          `Failed to fetch runs for experiment ${experiment.id}:`,
+          error
+        );
+
+        // Still create the experiment metadata without runs
+        await mode.writeFile(
+          `/phoenix/experiments/${experiment.id}/metadata.json`,
+          JSON.stringify(
+            {
+              ...experiment,
+              error: "Failed to fetch runs",
+              snapshot_timestamp: new Date().toISOString(),
+            },
+            null,
+            2
+          )
+        );
+      }
+    }
+  }
+}

package/src/snapshot/index.ts

@@ -0,0 +1,403 @@
+// Export all snapshot modules
+export {
+  createPhoenixClient,
+  PhoenixClientError,
+  type PhoenixClientConfig,
+} from "./client.js";
+export { fetchProjects } from "./projects.js";
+export { snapshotSpans, type SnapshotSpansOptions } from "./spans.js";
+export { fetchDatasets } from "./datasets.js";
+export { fetchExperiments } from "./experiments.js";
+export { fetchPrompts } from "./prompts.js";
+export { generateContext } from "./context.js";
+
+// Import necessary types and modules for orchestration
+import type { ExecutionMode } from "../modes/types.js";
+import type { PhoenixClient } from "@arizeai/phoenix-client";
+import {
+  createPhoenixClient,
+  PhoenixClientError,
+  type PhoenixClientConfig,
+} from "./client.js";
+import { fetchProjects } from "./projects.js";
+import { snapshotSpans, type SnapshotSpansOptions } from "./spans.js";
+import { fetchDatasets } from "./datasets.js";
+import { fetchExperiments } from "./experiments.js";
+import { fetchPrompts } from "./prompts.js";
+import { generateContext } from "./context.js";
+import { SnapshotProgress } from "../progress.js";
+
+export interface SnapshotOptions {
+  /**
+   * Phoenix server base URL
+   */
+  baseURL: string;
+  /**
+   * Optional API key for authentication
+   */
+  apiKey?: string;
+  /**
+   * Maximum number of spans per project
+   */
+  spansPerProject?: number;
+  /**
+   * Time range filter for spans (ISO 8601 format)
+   */
+  startTime?: string;
+  endTime?: string;
+  /**
+   * Whether to show progress indicators
+   */
+  showProgress?: boolean;
+}
+
+export interface SnapshotMetadata {
+  created_at: string;
+  phoenix_url: string;
+  cursors: {
+    spans?: Record<string, { last_end_time?: string; cursor?: string }>;
+    datasets?: { last_fetch: string };
+    experiments?: { last_fetch: string };
+    prompts?: { last_fetch: string };
+  };
+  limits: {
+    spans_per_project: number;
+  };
+}
+
+/**
+ * Orchestrates all data fetchers to create a complete Phoenix snapshot
+ * @param mode - The execution mode (sandbox or local)
+ * @param options - Snapshot options including server URL and limits
+ */
+export async function createSnapshot(
+  mode: ExecutionMode,
+  options: SnapshotOptions
+): Promise<void> {
+  const {
+    baseURL,
+    apiKey,
+    spansPerProject = 1000,
+    startTime,
+    endTime,
+    showProgress = false,
+  } = options;
+
+  // Create progress indicator
+  const progress = new SnapshotProgress(showProgress);
+  progress.start("Creating Phoenix data snapshot");
+
+  // Create Phoenix client
+  const clientConfig: PhoenixClientConfig = {
+    baseURL,
+    apiKey,
+  };
+  const client = createPhoenixClient(clientConfig);
+
+  try {
+    // 1. Fetch projects first (required for spans)
+    progress.update("Fetching projects");
+    try {
+      await fetchProjects(client, mode);
+    } catch (error) {
+      progress.fail("Failed to fetch projects");
+      throw new PhoenixClientError(
+        `Failed to fetch projects: ${error instanceof Error ? error.message : String(error)}`,
+        error instanceof PhoenixClientError ? error.code : "UNKNOWN_ERROR",
+        error
+      );
+    }
+
+    // 2. Fetch spans and other data in parallel
+    progress.update(
+      "Fetching all data",
+      "spans, datasets, experiments, prompts"
+    );
+    const spansOptions: SnapshotSpansOptions = {
+      spansPerProject,
+      startTime,
+      endTime,
+    };
+
+    // Fetch all data types in parallel for better performance
+    const results = await Promise.allSettled([
+      snapshotSpans(client, mode, spansOptions),
+      fetchDatasets(client, mode),
+      fetchExperiments(client, mode),
+      fetchPrompts(client, mode),
+    ]);
+
+    // Check for failures and collect errors
+    const errors: Array<{ type: string; error: unknown }> = [];
+    const dataTypes = ["spans", "datasets", "experiments", "prompts"];
+
+    results.forEach((result, index) => {
+      if (result.status === "rejected") {
+        errors.push({
+          type: dataTypes[index] || "unknown",
+          error: result.reason,
+        });
+      }
+    });
+
+    if (errors.length > 0) {
+      // Log individual errors
+      errors.forEach(({ type, error }) => {
+        console.error(
+          `Warning: Failed to fetch ${type}:`,
+          error instanceof Error ? error.message : String(error)
+        );
+      });
+
+      // If spans failed, that's critical - throw error
+      if (errors.some((e) => e.type === "spans")) {
+        progress.fail("Failed to fetch spans");
+        throw new PhoenixClientError(
+          `Failed to fetch spans: ${errors.find((e) => e.type === "spans")?.error}`,
+          "UNKNOWN_ERROR",
+          errors
+        );
+      }
+
+      // If all other data failed, throw error. If partial success, continue with warning
+      if (errors.length === 4) {
+        progress.fail("Failed to fetch all data");
+        throw new PhoenixClientError(
+          "Failed to fetch all data types",
+          "UNKNOWN_ERROR",
+          errors
+        );
+      }
+    }
+
+    // 4. Generate context file
+    progress.update("Generating context");
+    await generateContext(mode, {
+      phoenixUrl: baseURL,
+      snapshotTime: new Date(),
+      spansPerProject,
+    });
+
+    // 5. Write metadata file
+    progress.update("Writing metadata");
+    const metadata: SnapshotMetadata = {
+      created_at: new Date().toISOString(),
+      phoenix_url: baseURL,
+      cursors: {
+        spans: {}, // TODO: Track span cursors when span fetching supports it
+        datasets: { last_fetch: new Date().toISOString() },
+        experiments: { last_fetch: new Date().toISOString() },
+        prompts: { last_fetch: new Date().toISOString() },
+      },
+      limits: {
+        spans_per_project: spansPerProject,
+      },
+    };
+
+    await mode.writeFile(
+      "/_meta/snapshot.json",
+      JSON.stringify(metadata, null, 2)
+    );
+
+    progress.succeed("✅ Snapshot created successfully!");
+  } catch (error) {
+    // Stop progress if not already stopped
+    progress.stop();
+
+    // Enhance error with context before rethrowing
+    if (error instanceof PhoenixClientError) {
+      throw error; // Already has good context
+    }
+
+    throw new PhoenixClientError(
+      `Failed to create snapshot: ${error instanceof Error ? error.message : String(error)}`,
+      "UNKNOWN_ERROR",
+      error
+    );
+  }
+}
+
+/**
+ * Loads existing snapshot metadata if available
+ * @param mode - The execution mode (sandbox or local)
+ * @returns The snapshot metadata or null if not found
+ */
+export async function loadSnapshotMetadata(
+  mode: ExecutionMode
+): Promise<SnapshotMetadata | null> {
+  try {
+    const result = await mode.exec(
+      "cat /phoenix/_meta/snapshot.json 2>/dev/null"
+    );
+    if (result.exitCode === 0) {
+      return JSON.parse(result.stdout);
+    }
+  } catch (error) {
+    // File doesn't exist or parse error
+  }
+  return null;
+}
+
+/**
+ * Creates an incremental snapshot, fetching only new/updated data
+ * @param mode - The execution mode (sandbox or local)
+ * @param options - Snapshot options including server URL and limits
+ */
+export async function createIncrementalSnapshot(
+  mode: ExecutionMode,
+  options: SnapshotOptions
+): Promise<void> {
+  // Load existing metadata to get cursors
+  const existingMetadata = await loadSnapshotMetadata(mode);
+
+  if (!existingMetadata) {
+    // No existing snapshot, create a full one
+    await createSnapshot(mode, options);
+    return;
+  }
+
+  const {
+    baseURL,
+    apiKey,
+    spansPerProject = 1000,
+    showProgress = false,
+  } = options;
+
+  // Create progress indicator
+  const progress = new SnapshotProgress(showProgress);
+  progress.start("Updating Phoenix data snapshot");
+
+  // Create Phoenix client
+  const clientConfig: PhoenixClientConfig = {
+    baseURL,
+    apiKey,
+  };
+  const client = createPhoenixClient(clientConfig);
+
+  try {
+    // Show time since last snapshot
+    const lastSnapshotDate = new Date(existingMetadata.created_at);
+    const timeSince = formatTimeSince(lastSnapshotDate);
+    progress.update("Checking for updates", `last snapshot ${timeSince} ago`);
+
+    // For incremental updates, we'll need to:
+    // 1. Fetch projects (always fetch all as they're small)
+    progress.update("Updating projects");
+    await fetchProjects(client, mode);
+
+    // 2. Fetch spans and other data in parallel for better performance
+    progress.update("Fetching updates", "new spans and refreshing other data");
+
+    const spansOptions: SnapshotSpansOptions = {
+      spansPerProject,
+      // Use the last end time from previous snapshot as start time
+      startTime: existingMetadata.cursors.spans
+        ? Object.values(existingMetadata.cursors.spans)
+            .map((cursor) => cursor.last_end_time)
+            .filter(Boolean)
+            .sort()
+            .pop()
+        : undefined,
+    };
+
+    // For datasets/experiments/prompts, check if they've been updated
+    const datasetsLastFetch = existingMetadata.cursors.datasets?.last_fetch;
+    const experimentsLastFetch =
+      existingMetadata.cursors.experiments?.last_fetch;
+    const promptsLastFetch = existingMetadata.cursors.prompts?.last_fetch;
+
+    // Fetch all data types in parallel
+    // For now, we'll refetch all as the API doesn't support filtering by updated_at
+    // In a future enhancement, we could check individual items for updates
+    const updateResults = await Promise.allSettled([
+      snapshotSpans(client, mode, spansOptions),
+      fetchDatasets(client, mode),
+      fetchExperiments(client, mode),
+      fetchPrompts(client, mode),
+    ]);
+
+    // Check for critical errors
+    const updateErrors: Array<{ type: string; error: unknown }> = [];
+    const updateDataTypes = ["spans", "datasets", "experiments", "prompts"];
+
+    updateResults.forEach((result, index) => {
+      if (result.status === "rejected") {
+        updateErrors.push({
+          type: updateDataTypes[index] || "unknown",
+          error: result.reason,
+        });
+      }
+    });
+
+    if (updateErrors.length > 0) {
+      // Log individual errors
+      updateErrors.forEach(({ type, error }) => {
+        console.error(
+          `Warning: Failed to update ${type}:`,
+          error instanceof Error ? error.message : String(error)
+        );
+      });
+    }
+
+    // 4. Regenerate context with updated data
+    progress.update("Regenerating context");
+    await generateContext(mode, {
+      phoenixUrl: baseURL,
+      snapshotTime: new Date(),
+      spansPerProject,
+    });
+
+    // 5. Update metadata
+    progress.update("Updating metadata");
+    const updatedSpansCursors = existingMetadata.cursors.spans || {};
+    const metadata: SnapshotMetadata = {
+      created_at: new Date().toISOString(),
+      phoenix_url: baseURL,
+      cursors: {
+        spans: updatedSpansCursors,
+        datasets: { last_fetch: new Date().toISOString() },
+        experiments: { last_fetch: new Date().toISOString() },
+        prompts: { last_fetch: new Date().toISOString() },
+      },
+      limits: {
+        spans_per_project: spansPerProject,
+      },
+    };
+
+    await mode.writeFile(
+      "/_meta/snapshot.json",
+      JSON.stringify(metadata, null, 2)
+    );
+
+    progress.succeed("✅ Incremental update complete!");
+  } catch (error) {
+    // Stop progress if not already stopped
+    progress.stop();
+
+    // Enhance error with context before rethrowing
+    if (error instanceof PhoenixClientError) {
+      throw error; // Already has good context
+    }
+
+    throw new PhoenixClientError(
+      `Failed to create incremental snapshot: ${error instanceof Error ? error.message : String(error)}`,
+      "UNKNOWN_ERROR",
+      error
+    );
+  }
+}
+
+/**
+ * Format time since a date in human-readable format
+ */
+function formatTimeSince(date: Date): string {
+  const seconds = Math.floor((new Date().getTime() - date.getTime()) / 1000);
+
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  const days = Math.floor(hours / 24);
+  return `${days}d`;
+}

package/src/snapshot/projects.ts

@@ -0,0 +1,58 @@
+import type { PhoenixClient } from "@arizeai/phoenix-client";
+import type { ExecutionMode } from "../modes/types.js";
+import { withErrorHandling } from "./client.js";
+
+/**
+ * Converts an array of items to JSONL format (one JSON object per line)
+ */
+function toJSONL(items: unknown[]): string {
+  return items.map((item) => JSON.stringify(item)).join("\n");
+}
+
+/**
+ * Fetches all projects and writes them to the filesystem
+ * @param client - The Phoenix client instance
+ * @param mode - The execution mode (sandbox or local)
+ */
+export async function fetchProjects(
+  client: PhoenixClient,
+  mode: ExecutionMode
+): Promise<void> {
+  // Fetch all projects with error handling
+  const projectsData = await withErrorHandling(async () => {
+    const response = await client.GET("/v1/projects", {
+      params: {
+        query: {
+          include_experiment_projects: false,
+        },
+      },
+    });
+
+    if (!response.data) {
+      throw new Error("No data returned from projects endpoint");
+    }
+
+    return response.data;
+  }, "fetching projects");
+
+  // Extract projects from the response
+  const projects = projectsData.data || [];
+
+  // Write projects list as JSONL to /phoenix/projects/index.jsonl
+  const projectsPath = "/phoenix/projects/index.jsonl";
+  await mode.writeFile(projectsPath, toJSONL(projects));
+
+  // For each project, create a metadata.json file
+  for (const project of projects) {
+    const projectDir = `/phoenix/projects/${project.name}`;
+    const metadataPath = `${projectDir}/metadata.json`;
+
+    // Write project metadata
+    await mode.writeFile(metadataPath, JSON.stringify(project, null, 2));
+
+    // Create empty spans directory (will be populated by snapshot-spans task)
+    const spansDir = `${projectDir}/spans`;
+    // Create directory by writing a placeholder that will be overwritten later
+    await mode.writeFile(`${spansDir}/.gitkeep`, "");
+  }
+}