@arizeai/phoenix-client 5.8.2 → 6.0.0

package/src/experiments/logging.ts

@@ -0,0 +1,169 @@
+import type { Logger } from "../logger";
+import type { ExperimentEvaluationRun } from "../types/experiments";
+
+/**
+ * Progress line prefixes used in experiment run output.
+ * e.g. "[start] Tasks (3 examples × 1 repetition)"
+ */
+export const PROGRESS_PREFIX = {
+  start: "[start] ",
+  progress: "[progress] ",
+  completed: "[completed] ",
+} as const;
+
+/**
+ * Log a task summary table.
+ * Outputs a header + a single-row table keyed as "summary".
+ */
+export function logTaskSummary(
+  logger: Logger,
+  {
+    nExamples,
+    repetitions,
+    nRuns,
+    nErrors,
+  }: {
+    nExamples: number;
+    repetitions: number;
+    nRuns: number;
+    nErrors: number;
+  }
+): void {
+  logger.info("");
+  logger.info("Task Summary");
+  const row: Record<string, unknown> = { examples: nExamples };
+  if (repetitions > 1) row.repetitions = repetitions;
+  row.runs = nRuns;
+  if (nErrors > 0) row.errors = nErrors;
+  logger.table({ summary: row });
+}
+
+/**
+ * Log an evaluation summary table, keyed by evaluator name.
+ * Aggregates scores and labels per evaluator.
+ */
+export function logEvalSummary(
+  logger: Logger,
+  evalRuns: ExperimentEvaluationRun[]
+): void {
+  logger.info("");
+  logger.info("Evaluation Summary");
+
+  const byEvaluator = new Map<string, ExperimentEvaluationRun[]>();
+  for (const ev of evalRuns) {
+    const list = byEvaluator.get(ev.name) ?? [];
+    list.push(ev);
+    byEvaluator.set(ev.name, list);
+  }
+
+  const tableObj: Record<string, Record<string, unknown>> = {};
+  for (const [name, evs] of byEvaluator.entries()) {
+    const scores = evs.flatMap((ev) =>
+      ev.result?.score != null ? [ev.result.score] : []
+    );
+    const labels = evs.flatMap((ev) =>
+      ev.result?.label != null ? [ev.result.label] : []
+    );
+    const errors = evs.filter((ev) => ev.error != null);
+
+    const row: Record<string, unknown> = { runs: evs.length };
+    if (errors.length > 0) row.errors = errors.length;
+    if (scores.length > 0) {
+      row.scores = scores.length;
+      row["avg score"] = Number(
+        (scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)
+      );
+    }
+    if (labels.length > 0) {
+      row.labels = labels.length;
+      const counts: Record<string, number> = {};
+      for (const label of labels) counts[label] = (counts[label] ?? 0) + 1;
+      const topLabels = Object.entries(counts)
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 2);
+      if (topLabels[0])
+        row["label 1"] = `${topLabels[0][0]} (${topLabels[0][1]})`;
+      if (topLabels[1])
+        row["label 2"] = `${topLabels[1][0]} (${topLabels[1][1]})`;
+    }
+    tableObj[name] = row;
+  }
+
+  logger.table(tableObj);
+}
+
+/**
+ * Log a padded "Links" block.
+ * Labels are padded to align URLs. No-ops when links is empty.
+ */
+export function logLinks(
+  logger: Logger,
+  links: Array<{ label: string; url: string }>
+): void {
+  if (links.length === 0) return;
+  const maxLen = Math.max(...links.map((link) => link.label.length));
+  logger.info("");
+  logger.info("Links");
+  for (const { label, url } of links) {
+    logger.info(`  ${label.padEnd(maxLen)}  ${url}`);
+  }
+  logger.info("");
+}
+
+/**
+ * Log experiment resume summary table.
+ * Outputs a header + a single-row table keyed as "summary".
+ */
+export function logExperimentResumeSummary(
+  logger: Logger,
+  {
+    experimentId,
+    processed,
+    completed,
+    failed,
+  }: {
+    experimentId: string;
+    processed: number;
+    completed: number;
+    failed: number;
+  }
+): void {
+  logger.info("Experiment Resume Summary");
+  logger.table({
+    summary: {
+      "experiment id": experimentId,
+      processed,
+      completed,
+      failed,
+    },
+  });
+}
+
+/**
+ * Log evaluation resume summary table.
+ * Outputs a header + a single-row table keyed as "summary".
+ */
+export function logEvalResumeSummary(
+  logger: Logger,
+  {
+    experimentId,
+    processed,
+    completed,
+    failed,
+  }: {
+    experimentId: string;
+    processed: number;
+    completed: number;
+    failed: number;
+  }
+): void {
+  logger.info("Evaluation Resume Summary");
+  logger.table({
+    summary: {
+      "experiment id": experimentId,
+      processed,
+      completed,
+      failed,
+    },
+  });
+}

package/src/experiments/resumeEvaluation.ts

@@ -3,18 +3,19 @@ import {
   OpenInferenceSpanKind,
   SemanticConventions,
 } from "@arizeai/openinference-semantic-conventions";
+import type { NodeTracerProvider, Tracer } from "@arizeai/phoenix-otel";
 import {
   type DiagLogLevel,
-  NodeTracerProvider,
   objectAsAttributes,
   register,
   SpanStatusCode,
-  Tracer,
 } from "@arizeai/phoenix-otel";
+import invariant from "tiny-invariant";
 
-import { components } from "../__generated__/api/v1";
+import type { components } from "../__generated__/api/v1";
 import { createClient, type PhoenixClient } from "../client";
-import { ClientFn } from "../types/core";
+import { createLogger, type Logger } from "../logger";
+import type { ClientFn } from "../types/core";
 import type {
   EvaluationResult,
   Evaluator,
@@ -22,15 +23,12 @@ import type {
   IncompleteEvaluation,
   TaskOutput,
 } from "../types/experiments";
-import { type Logger } from "../types/logger";
 import { Channel, ChannelError } from "../utils/channel";
 import { ensureString } from "../utils/ensureString";
 import { toObjectHeaders } from "../utils/toObjectHeaders";
-
 import { getExperimentInfo } from "./getExperimentInfo.js";
 import { getExperimentEvaluators } from "./helpers";
-
-import invariant from "tiny-invariant";
+import { logEvalResumeSummary, PROGRESS_PREFIX } from "./logging";
 
 /**
  * Error thrown when evaluation is aborted due to a failure in stopOnFirstError mode.
@@ -71,7 +69,7 @@ export type ResumeEvaluationParams = ClientFn & {
     | readonly ExperimentEvaluatorLike[];
   /**
    * The logger to use
-   * @default console
+   * @default createLogger()
    */
   readonly logger?: Logger;
   /**
@@ -227,29 +225,6 @@ function setupEvaluationTracer({
   return { provider, tracer };
 }
 
-/**
- * Prints evaluation summary to logger
- */
-function printEvaluationSummary({
-  logger,
-  experimentId,
-  totalProcessed,
-  totalCompleted,
-}: {
-  logger: Logger;
-  experimentId: string;
-  totalProcessed: number;
-  totalCompleted: number;
-}): void {
-  logger.info("\n" + "=".repeat(70));
-  logger.info("📊 Evaluation Resume Summary");
-  logger.info("=".repeat(70));
-  logger.info(`Experiment ID: ${experimentId}`);
-  logger.info(`Runs processed: ${totalProcessed}`);
-  logger.info(`Evaluations completed: ${totalCompleted}`);
-  logger.info("=".repeat(70));
-}
-
 /**
  * Resume incomplete evaluations for an experiment.
  *
@@ -314,7 +289,7 @@ export async function resumeEvaluation({
   client: _client,
   experimentId,
   evaluators: _evaluators,
-  logger = console,
+  logger = createLogger(),
   concurrency = 5,
   setGlobalTracerProvider = true,
   useBatchSpanProcessor = true,
@@ -332,7 +307,7 @@ export async function resumeEvaluation({
   invariant(evaluators.length > 0, "Must specify at least one evaluator");
 
   // Get experiment info
-  logger.info(`🔍 Checking for incomplete evaluations...`);
+  logger.info(`${PROGRESS_PREFIX.start}Checking for incomplete evaluations.`);
   const experiment = await getExperimentInfo({ client, experimentId });
 
   // Initialize tracer (only if experiment has a project_name)
@@ -380,7 +355,7 @@ export async function resumeEvaluation({
       do {
         // Stop fetching if abort signal received
         if (signal.aborted) {
-          logger.info("🛑 Stopping fetch due to error in evaluation");
+          logger.debug(`${PROGRESS_PREFIX.progress}Stopping fetch.`);
           break;
         }
 
@@ -438,14 +413,14 @@ export async function resumeEvaluation({
         if (batchIncomplete.length === 0) {
           if (totalProcessed === 0) {
             logger.info(
-              "✅ No incomplete evaluations found. All evaluations are complete."
+              `${PROGRESS_PREFIX.completed}No incomplete evaluations found.`
             );
           }
           break;
         }
 
         if (totalProcessed === 0) {
-          logger.info("🧠 Resuming evaluations...");
+          logger.info(`${PROGRESS_PREFIX.start}Resuming evaluations.`);
         }
 
         // Build evaluation tasks and send to channel
@@ -475,8 +450,8 @@ export async function resumeEvaluation({
           }
         }
 
-        logger.info(
-          `Fetched batch of ${batchCount} evaluation tasks (channel buffer: ${evalChannel.length})`
+        logger.debug(
+          `${PROGRESS_PREFIX.progress}Fetched batch of ${batchCount} evaluation tasks.`
         );
       } while (cursor !== null && !signal.aborted);
     } catch (error) {
@@ -525,7 +500,7 @@ export async function resumeEvaluation({
 
         // If stopOnFirstError is enabled, abort and re-throw
         if (stopOnFirstError) {
-          logger.error("🛑 Stopping on first error");
+          logger.warn("Stopping on first error");
           abortController.abort();
           throw error;
         }
@@ -551,7 +526,7 @@ export async function resumeEvaluation({
     // Always surface producer/infrastructure errors
     if (error instanceof EvaluationFetchError) {
       // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
-      logger.error(`❌ Critical: Failed to fetch evaluations from server`);
+      logger.error(`Critical: Failed to fetch evaluations from server`);
       executionError = err;
     } else if (error instanceof ChannelError && signal.aborted) {
       // Channel closed due to intentional abort - wrap in semantic error
@@ -565,7 +540,7 @@ export async function resumeEvaluation({
     } else {
       // Unexpected error (not from worker, not from producer fetch)
       // This could be a bug in our code or infrastructure failure
-      logger.error(`❌ Unexpected error during evaluation: ${err.message}`);
+      logger.error(`Unexpected error during evaluation: ${err.message}`);
       executionError = err;
     }
   } finally {
@@ -578,21 +553,18 @@ export async function resumeEvaluation({
 
   // Only show completion message if we didn't stop on error
   if (!executionError) {
-    logger.info(`✅ Evaluations completed.`);
+    logger.info(`${PROGRESS_PREFIX.completed}Evaluations completed.`);
   }
 
   if (totalFailed > 0 && !executionError) {
-    logger.info(
-      `⚠️  Warning: ${totalFailed} out of ${totalProcessed} evaluations failed.`
-    );
+    logger.warn(`${totalFailed} out of ${totalProcessed} evaluations failed.`);
   }
 
-  // Print summary
-  printEvaluationSummary({
-    logger,
+  logEvalResumeSummary(logger, {
     experimentId: experiment.id,
-    totalProcessed,
-    totalCompleted,
+    processed: totalProcessed,
+    completed: totalCompleted,
+    failed: totalFailed,
   });
 
   // Flush spans (if tracer was initialized)

package/src/experiments/resumeExperiment.ts

@@ -3,35 +3,37 @@ import {
   OpenInferenceSpanKind,
   SemanticConventions,
 } from "@arizeai/openinference-semantic-conventions";
+import type { NodeTracerProvider, Tracer } from "@arizeai/phoenix-otel";
 import {
   type DiagLogLevel,
-  NodeTracerProvider,
   objectAsAttributes,
   register,
   SpanStatusCode,
-  Tracer,
 } from "@arizeai/phoenix-otel";
+import invariant from "tiny-invariant";
 
-import { components } from "../__generated__/api/v1";
+import type { components } from "../__generated__/api/v1";
 import { createClient, type PhoenixClient } from "../client";
-import { ClientFn } from "../types/core";
-import { ExampleWithId } from "../types/datasets";
+import { createLogger, type Logger } from "../logger";
+import type { ClientFn } from "../types/core";
+import type { ExampleWithId } from "../types/datasets";
 import type {
   ExperimentEvaluatorLike,
   ExperimentTask,
 } from "../types/experiments";
-import { type Logger } from "../types/logger";
 import { Channel, ChannelError } from "../utils/channel";
 import { ensureString } from "../utils/ensureString";
 import { isHttpErrorWithStatus } from "../utils/isHttpError";
 import { toObjectHeaders } from "../utils/toObjectHeaders";
 import { getDatasetExperimentsUrl, getExperimentUrl } from "../utils/urlUtils";
-
 import { getExperimentInfo } from "./getExperimentInfo.js";
+import {
+  logExperimentResumeSummary,
+  logLinks,
+  PROGRESS_PREFIX,
+} from "./logging";
 import { resumeEvaluation } from "./resumeEvaluation";
 
-import invariant from "tiny-invariant";
-
 /**
  * Error thrown when task is aborted due to a failure in stopOnFirstError mode.
  * This provides semantic context that the abort was intentional, not an infrastructure failure.
@@ -74,7 +76,7 @@ export type ResumeExperimentParams = ClientFn & {
   readonly evaluators?: readonly ExperimentEvaluatorLike[];
   /**
    * The logger to use
-   * @default console
+   * @default createLogger()
    */
   readonly logger?: Logger;
   /**
@@ -206,29 +208,6 @@ function setupTracer({
   return { provider, tracer };
 }
 
-/**
- * Prints experiment summary to logger
- */
-function printExperimentSummary({
-  logger,
-  experimentId,
-  totalProcessed,
-  totalCompleted,
-}: {
-  logger: Logger;
-  experimentId: string;
-  totalProcessed: number;
-  totalCompleted: number;
-}): void {
-  logger.info("\n" + "=".repeat(70));
-  logger.info("📊 Experiment Resume Summary");
-  logger.info("=".repeat(70));
-  logger.info(`Experiment ID: ${experimentId}`);
-  logger.info(`Incomplete runs processed: ${totalProcessed}`);
-  logger.info(`Successfully completed: ${totalCompleted}`);
-  logger.info("=".repeat(70));
-}
-
 /**
  * Resume an incomplete experiment by running only the missing or failed runs.
  *
@@ -286,7 +265,7 @@ export async function resumeExperiment({
   experimentId,
   task,
   evaluators,
-  logger = console,
+  logger = createLogger(),
   concurrency = 5,
   setGlobalTracerProvider = true,
   useBatchSpanProcessor = true,
@@ -297,7 +276,7 @@ export async function resumeExperiment({
   const pageSize = DEFAULT_PAGE_SIZE;
 
   // Get experiment info
-  logger.info(`🔍 Fetching experiment info...`);
+  logger.info(`${PROGRESS_PREFIX.start}Fetching experiment info.`);
   const experiment = await getExperimentInfo({ client, experimentId });
 
   // Check if there are incomplete runs
@@ -305,12 +284,14 @@ export async function resumeExperiment({
   const incompleteCount = totalExpected - experiment.successfulRunCount;
 
   if (incompleteCount === 0) {
-    logger.info("✅ No incomplete runs found. Experiment is already complete.");
+    logger.info(
+      `${PROGRESS_PREFIX.completed}No incomplete runs found. Experiment is already complete.`
+    );
     return;
   }
 
   logger.info(
-    `🧪 Resuming experiment with ${incompleteCount} incomplete runs...`
+    `${PROGRESS_PREFIX.start}Resuming experiment with ${incompleteCount} incomplete runs.`
   );
 
   // Get base URL for tracing and URL generation
@@ -346,9 +327,6 @@ export async function resumeExperiment({
     experimentId: experiment.id,
   });
 
-  logger.info(`📺 View dataset experiments: ${datasetExperimentsUrl}`);
-  logger.info(`🔗 View this experiment: ${experimentUrl}`);
-
   // Create a CSP-style bounded buffer for task distribution
   const taskChannel = new Channel<TaskItem>(
     pageSize * CHANNEL_CAPACITY_MULTIPLIER
@@ -370,7 +348,7 @@ export async function resumeExperiment({
       do {
         // Stop fetching if abort signal received
         if (signal.aborted) {
-          logger.info("🛑 Stopping fetch due to error in task");
+          logger.debug(`${PROGRESS_PREFIX.progress}Stopping fetch.`);
           break;
         }
 
@@ -439,8 +417,8 @@ export async function resumeExperiment({
           }
         }
 
-        logger.info(
-          `Fetched batch of ${batchCount} incomplete runs (channel buffer: ${taskChannel.length})`
+        logger.debug(
+          `${PROGRESS_PREFIX.progress}Fetched batch of ${batchCount} incomplete runs.`
         );
       } while (cursor !== null && !signal.aborted);
     } catch (error) {
@@ -489,7 +467,7 @@ export async function resumeExperiment({
 
         // If stopOnFirstError is enabled, abort and re-throw
         if (stopOnFirstError) {
-          logger.error("🛑 Stopping on first error");
+          logger.warn("Stopping on first error");
           abortController.abort();
           throw error;
         }
@@ -515,7 +493,7 @@ export async function resumeExperiment({
     // Always surface producer/infrastructure errors
     if (error instanceof TaskFetchError) {
       // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
-      logger.error(`❌ Critical: Failed to fetch incomplete runs from server`);
+      logger.error(`Critical: Failed to fetch incomplete runs from server`);
       executionError = err;
     } else if (error instanceof ChannelError && signal.aborted) {
       // Channel closed due to intentional abort - wrap in semantic error
@@ -529,7 +507,7 @@ export async function resumeExperiment({
     } else {
       // Unexpected error (not from worker, not from producer fetch)
       // This could be a bug in our code or infrastructure failure
-      logger.error(`❌ Unexpected error during task execution: ${err.message}`);
+      logger.error(`Unexpected error during task execution: ${err.message}`);
       executionError = err;
     }
   } finally {
@@ -542,19 +520,17 @@ export async function resumeExperiment({
 
   // Only show completion message if we didn't stop on error
   if (!executionError) {
-    logger.info(`✅ Task runs completed.`);
+    logger.info(`${PROGRESS_PREFIX.completed}Task runs completed.`);
   }
 
   if (totalFailed > 0 && !executionError) {
-    logger.info(
-      `⚠️  Warning: ${totalFailed} out of ${totalProcessed} runs failed.`
-    );
+    logger.warn(`${totalFailed} out of ${totalProcessed} runs failed.`);
   }
 
   // Run evaluators if provided (only on runs missing evaluations)
   // Skip evaluators if we stopped on error
   if (evaluators && evaluators.length > 0 && !executionError) {
-    logger.info(`\n🔬 Running evaluators...`);
+    logger.info(`${PROGRESS_PREFIX.start}Running evaluators.`);
     await resumeEvaluation({
       experimentId,
       evaluators: [...evaluators],
@@ -568,13 +544,16 @@ export async function resumeExperiment({
     });
   }
 
-  // Print summary
-  printExperimentSummary({
-    logger,
+  logExperimentResumeSummary(logger, {
     experimentId: experiment.id,
-    totalProcessed,
-    totalCompleted,
+    processed: totalProcessed,
+    completed: totalCompleted,
+    failed: totalFailed,
   });
+  logLinks(logger, [
+    { label: "Experiments", url: datasetExperimentsUrl },
+    { label: "Experiment", url: experimentUrl },
+  ]);
 
   // Flush spans (if tracer was initialized)
   if (provider) {