@orq-ai/evaluatorq 1.0.0-6

package/src/lib/effects.ts

@@ -0,0 +1,174 @@
+import { Effect, pipe } from "effect";
+
+import { ProgressService } from "./progress.js";
+import type {
+  DataPoint,
+  DataPointResult,
+  Job,
+  JobResult,
+  Scorer,
+} from "./types.js";
+
+export function processDataPointEffect(
+  dataPromise: Promise<DataPoint>,
+  rowIndex: number,
+  jobs: Job[],
+  evaluators: { name: string; scorer: Scorer }[],
+  parallelism: number,
+): Effect.Effect<DataPointResult[], Error, ProgressService> {
+  return pipe(
+    Effect.tryPromise({
+      try: () => dataPromise,
+      catch: (error) => error as Error,
+    }),
+    Effect.flatMap((dataPoint) =>
+      Effect.gen(function* (_) {
+        const progress = yield* _(ProgressService);
+
+        // Update progress for this data point
+        yield* _(
+          progress.updateProgress({
+            currentDataPoint: rowIndex + 1,
+            phase: "processing",
+          }),
+        );
+
+        // Process jobs
+        const jobResults = yield* _(
+          Effect.forEach(
+            jobs,
+            (job) => processJobEffect(job, dataPoint, rowIndex, evaluators),
+            { concurrency: parallelism },
+          ),
+        );
+
+        return [
+          {
+            dataPoint,
+            jobResults,
+          },
+        ];
+      }),
+    ),
+    Effect.catchAll((error) =>
+      Effect.succeed([
+        {
+          dataPoint: { inputs: {} }, // Placeholder since we couldn't get the actual data
+          error,
+        },
+      ]),
+    ),
+  );
+}
+
+export function processJobEffect(
+  job: Job,
+  dataPoint: DataPoint,
+  rowIndex: number,
+  evaluators: { name: string; scorer: Scorer }[],
+): Effect.Effect<JobResult, Error, ProgressService> {
+  return Effect.gen(function* (_) {
+    const progress = yield* _(ProgressService);
+
+    // Update progress with current job
+    const jobResult = yield* _(
+      pipe(
+        Effect.Do,
+        Effect.bind("jobName", () =>
+          Effect.sync(() => {
+            // Try to get job name from a test run or use a placeholder
+            return "job";
+          }),
+        ),
+        Effect.tap(({ jobName }) =>
+          progress.updateProgress({
+            currentJob: jobName,
+            phase: "processing",
+          }),
+        ),
+        Effect.bind("result", () =>
+          Effect.tryPromise({
+            try: () => job(dataPoint, rowIndex),
+            catch: (error) => error as Error,
+          }),
+        ),
+        Effect.tap(({ result }) =>
+          progress.updateProgress({
+            currentJob: result.name,
+          }),
+        ),
+        Effect.map(({ result }) => result),
+      ),
+    );
+
+    // Process evaluators if any
+    if (evaluators.length > 0) {
+      // Update phase to evaluating
+      yield* _(progress.updateProgress({ phase: "evaluating" }));
+
+      const evaluatorScores = yield* _(
+        Effect.forEach(
+          evaluators,
+          (evaluator) =>
+            Effect.gen(function* (_) {
+              // Update current evaluator
+              yield* _(
+                progress.updateProgress({
+                  currentEvaluator: evaluator.name,
+                }),
+              );
+
+              const score = yield* _(
+                pipe(
+                  Effect.tryPromise({
+                    try: async () => {
+                      const result = await evaluator.scorer({
+                        data: dataPoint,
+                        output: jobResult.output,
+                      });
+                      return result;
+                    },
+                    catch: (error) => error as Error,
+                  }),
+                  Effect.map((score) => ({
+                    evaluatorName: evaluator.name,
+                    score: score as string | number | boolean,
+                  })),
+                  Effect.catchAll((error) =>
+                    Effect.succeed({
+                      evaluatorName: evaluator.name,
+                      score: "" as string,
+                      error: error as Error,
+                    }),
+                  ),
+                ),
+              );
+
+              return score;
+            }),
+          { concurrency: "unbounded" },
+        ),
+      );
+
+      return {
+        jobName: jobResult.name,
+        output: jobResult.output,
+        evaluatorScores,
+      };
+    }
+
+    return {
+      jobName: jobResult.name,
+      output: jobResult.output,
+      evaluatorScores: [],
+    };
+  }).pipe(
+    Effect.catchAll((error) =>
+      Effect.succeed({
+        jobName: "Unknown", // We don't know the job name if it threw before returning
+        output: null,
+        error,
+      }),
+    ),
+  );
+}

package/src/lib/evaluatorq.ts

@@ -0,0 +1,260 @@
+import { Effect, pipe } from "effect";
+
+import type { Orq } from "@orq-ai/node";
+
+import { processDataPointEffect } from "./effects.js";
+import {
+  ProgressService,
+  ProgressServiceLive,
+  withProgress,
+} from "./progress.js";
+import { displayResultsTableEffect } from "./table-display.js";
+import type {
+  DataPoint,
+  EvaluatorParams,
+  EvaluatorqResult,
+  Job,
+} from "./types.js";
+
+async function setupOrqClient(apiKey: string) {
+  try {
+    const client = await import("@orq-ai/node");
+
+    return new client.Orq({ apiKey, serverURL: "https://my.staging.orq.ai" });
+  } catch (error: unknown) {
+    const err = error as Error & { code?: string };
+    if (
+      err.code === "MODULE_NOT_FOUND" ||
+      err.code === "ERR_MODULE_NOT_FOUND" ||
+      err.message?.includes("Cannot find module")
+    ) {
+      throw new Error(
+        "The @orq-ai/node package is not installed. To use dataset features, please install it:\n" +
+          "  npm install @orq-ai/node\n" +
+          "  # or\n" +
+          "  yarn add @orq-ai/node\n" +
+          "  # or\n" +
+          "  bun add @orq-ai/node",
+      );
+    }
+    throw new Error(`Failed to setup ORQ client: ${err.message || err}`);
+  }
+}
+
+async function fetchDatasetAsDataPoints(
+  orqClient: Orq,
+  datasetId: string,
+): Promise<Promise<DataPoint>[]> {
+  try {
+    const response = await orqClient.datasets.listDatapoints({ datasetId });
+
+    return response.data.map((datapoint) =>
+      Promise.resolve({
+        inputs: datapoint.inputs || {},
+        expectedOutput: datapoint.expectedOutput,
+      } as DataPoint),
+    );
+  } catch (error) {
+    throw new Error(
+      `Failed to fetch dataset ${datasetId}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+/**
+ * @param _name - The name of the evaluation run.
+ * @param params - The parameters for the evaluation run.
+ * @returns The results of the evaluation run.
+ */
+export async function evaluatorq(
+  _name: string,
+  params: EvaluatorParams,
+): Promise<EvaluatorqResult> {
+  const { data, evaluators = [], jobs, parallelism = 1, print = true } = params;
+
+  let orqClient: Orq | undefined;
+  const orqApiKey = process.env.ORQ_API_KEY;
+
+  if (orqApiKey) {
+    orqClient = await setupOrqClient(orqApiKey);
+  }
+
+  let dataPromises: Promise<DataPoint>[];
+
+  // Handle datasetId case
+  if ("datasetId" in data) {
+    if (!orqApiKey || !orqClient) {
+      throw new Error(
+        "ORQ_API_KEY environment variable must be set to fetch datapoints from Orq platform.",
+      );
+    }
+    dataPromises = await fetchDatasetAsDataPoints(orqClient, data.datasetId);
+  } else {
+    dataPromises = data as Promise<DataPoint>[];
+  }
+
+  // Create Effect for processing all data points
+  const program = pipe(
+    Effect.gen(function* (_) {
+      const progress = yield* _(ProgressService);
+
+      // Initialize progress
+      yield* _(
+        progress.updateProgress({
+          totalDataPoints: dataPromises.length,
+          currentDataPoint: 0,
+          phase: "initializing",
+        }),
+      );
+
+      // Process data points
+      const results = yield* _(
+        Effect.forEach(
+          dataPromises.map((dataPromise, index) => ({ dataPromise, index })),
+          ({ dataPromise, index }) =>
+            processDataPointEffect(
+              dataPromise,
+              index,
+              jobs,
+              evaluators,
+              parallelism,
+            ),
+          { concurrency: parallelism },
+        ),
+      );
+
+      return results.flat();
+    }),
+    // Conditionally add table display
+    print
+      ? Effect.tap((results) => displayResultsTableEffect(results))
+      : Effect.tap(() => Effect.void),
+    // Provide the progress service
+    Effect.provide(ProgressServiceLive),
+    // Wrap with progress tracking
+    (effect) => withProgress(effect, print),
+  );
+
+  // Run the Effect and convert back to Promise
+  return Effect.runPromise(program);
+}
+
+// Create an Effect that runs evaluation and optionally displays results
+export const evaluatorqEffect = (
+  _name: string,
+  params: EvaluatorParams,
+): Effect.Effect<EvaluatorqResult, Error, never> => {
+  const { data, evaluators = [], jobs, parallelism = 1, print = true } = params;
+
+  // Handle datasetId case
+  if ("datasetId" in data) {
+    return Effect.gen(function* (_) {
+      const apiKey = process.env.ORQ_API_KEY;
+      if (!apiKey) {
+        return yield* _(
+          Effect.fail(
+            new Error(
+              "ORQ_API_KEY environment variable must be set to fetch datasets from Orq platform.",
+            ),
+          ),
+        );
+      }
+
+      const orqClient = yield* _(
+        Effect.tryPromise({
+          try: () => setupOrqClient(apiKey),
+          catch: (error) =>
+            new Error(
+              `Failed to setup Orq client: ${error instanceof Error ? error.message : String(error)}`,
+            ),
+        }),
+      );
+
+      if (!orqClient) {
+        return yield* _(Effect.fail(new Error("Failed to setup Orq client")));
+      }
+
+      const dataPromises = yield* _(
+        Effect.tryPromise({
+          try: () => fetchDatasetAsDataPoints(orqClient, data.datasetId),
+          catch: (error) =>
+            error instanceof Error
+              ? error
+              : new Error(`Failed to fetch dataset: ${String(error)}`),
+        }),
+      );
+
+      return yield* _(
+        runEvaluationEffect(dataPromises, evaluators, jobs, parallelism, print),
+      );
+    });
+  }
+
+  const dataPromises = data as Promise<DataPoint>[];
+  return runEvaluationEffect(
+    dataPromises,
+    evaluators,
+    jobs,
+    parallelism,
+    print,
+  );
+};
+
+// Extract common evaluation logic
+const runEvaluationEffect = (
+  dataPromises: Promise<DataPoint>[],
+  evaluators: EvaluatorParams["evaluators"] = [],
+  jobs: Job[],
+  parallelism: number,
+  print: boolean,
+): Effect.Effect<EvaluatorqResult, Error, never> =>
+  pipe(
+    Effect.gen(function* (_) {
+      const progress = yield* _(ProgressService);
+
+      // Initialize progress
+      yield* _(
+        progress.updateProgress({
+          totalDataPoints: dataPromises.length,
+          currentDataPoint: 0,
+          phase: "initializing",
+        }),
+      );
+
+      // Process data points
+      const results = yield* _(
+        Effect.forEach(
+          dataPromises.map((dataPromise, index) => ({ dataPromise, index })),
+          ({ dataPromise, index }) =>
+            processDataPointEffect(
+              dataPromise,
+              index,
+              jobs,
+              evaluators,
+              parallelism,
+            ),
+          { concurrency: parallelism },
+        ),
+      );
+
+      return results.flat();
+    }),
+    // Conditionally add table display
+    print
+      ? Effect.tap((results) => displayResultsTableEffect(results))
+      : Effect.tap(() => Effect.void),
+    // Provide the progress service
+    Effect.provide(ProgressServiceLive),
+    // Wrap with progress tracking
+    (effect) => withProgress(effect, print),
+  );
+
+// Composable evaluatorq with display
+export const evaluatorqWithTableEffect = (
+  name: string,
+  params: EvaluatorParams,
+): Effect.Effect<EvaluatorqResult, Error, never> =>
+  pipe(
+    evaluatorqEffect(name, params),
+    Effect.tap((results) => displayResultsTableEffect(results)),
+  );

package/src/lib/progress.ts

@@ -0,0 +1,170 @@
+import chalk from "chalk";
+import { Context, Effect, Layer } from "effect";
+import ora, { type Ora } from "ora";
+
+// Progress state types
+export interface ProgressState {
+  totalDataPoints: number;
+  currentDataPoint: number;
+  currentJob?: string;
+  currentEvaluator?: string;
+  phase: "initializing" | "processing" | "evaluating" | "completed";
+}
+
+// Progress service interface
+export interface ProgressService {
+  readonly updateProgress: (
+    update: Partial<ProgressState>,
+  ) => Effect.Effect<void>;
+  readonly startSpinner: () => Effect.Effect<void>;
+  readonly stopSpinner: () => Effect.Effect<void>;
+  readonly showMessage: (message: string) => Effect.Effect<void>;
+}
+
+// Context tag for the progress service
+export const ProgressService =
+  Context.GenericTag<ProgressService>("ProgressService");
+
+// Spinner instance
+let spinner: Ora | null = null;
+
+// Create the progress service implementation
+const makeProgressService = (): ProgressService => {
+  let state: ProgressState = {
+    totalDataPoints: 0,
+    currentDataPoint: 0,
+    phase: "initializing",
+  };
+
+  const formatProgressText = (): string => {
+    const percentage =
+      state.totalDataPoints > 0
+        ? Math.round((state.currentDataPoint / state.totalDataPoints) * 100)
+        : 0;
+
+    let text = "";
+
+    switch (state.phase) {
+      case "initializing":
+        text = chalk.cyan("Initializing evaluation...");
+        break;
+      case "processing":
+        text = chalk.cyan(
+          `Processing data point ${state.currentDataPoint}/${state.totalDataPoints} (${percentage}%)`,
+        );
+        if (state.currentJob) {
+          text += chalk.gray(
+            ` - Running job: ${chalk.white(state.currentJob)}`,
+          );
+        }
+        break;
+      case "evaluating":
+        text = chalk.cyan(
+          `Evaluating results ${state.currentDataPoint}/${state.totalDataPoints} (${percentage}%)`,
+        );
+        if (state.currentEvaluator) {
+          text += chalk.gray(
+            ` - Running evaluator: ${chalk.white(state.currentEvaluator)}`,
+          );
+        }
+        break;
+      case "completed":
+        text = chalk.green("✓ Evaluation completed");
+        break;
+    }
+
+    return text;
+  };
+
+  return {
+    updateProgress: (update) =>
+      Effect.sync(() => {
+        state = { ...state, ...update };
+        if (spinner) {
+          spinner.text = formatProgressText();
+        }
+      }),
+
+    startSpinner: () =>
+      Effect.sync(() => {
+        if (!spinner) {
+          // Reserve space first by printing empty lines
+          process.stdout.write("\n\n\n");
+          // Move cursor back up to where we want the spinner
+          process.stdout.write("\x1b[3A");
+
+          spinner = ora({
+            text: formatProgressText(),
+            spinner: "dots",
+            color: "cyan",
+          });
+          spinner.start();
+        }
+      }),
+
+    stopSpinner: () =>
+      Effect.sync(() => {
+        if (spinner) {
+          if (state.phase === "completed") {
+            spinner.succeed(chalk.green("✓ Evaluation completed successfully"));
+            // Just one newline since table display adds its own
+            process.stdout.write("\n");
+          } else {
+            spinner.stop();
+            // Just one newline since table display adds its own
+            process.stdout.write("\n");
+          }
+          spinner = null;
+        }
+      }),
+
+    showMessage: (message) =>
+      Effect.sync(() => {
+        if (spinner) {
+          spinner.info(message);
+        } else {
+          console.log(message);
+        }
+      }),
+  };
+};
+
+// Create a layer for the progress service
+export const ProgressServiceLive = Layer.succeed(
+  ProgressService,
+  makeProgressService(),
+);
+
+// Helper function to run with progress tracking
+export const withProgress = <R, E, A>(
+  effect: Effect.Effect<A, E, R>,
+  showProgress: boolean = true,
+): Effect.Effect<A, E, R> => {
+  if (!showProgress) {
+    return effect;
+  }
+
+  return Effect.gen(function* (_) {
+    const progress = yield* _(ProgressService);
+
+    // Start spinner
+    yield* _(progress.startSpinner());
+
+    try {
+      // Run the effect
+      const result = yield* _(effect);
+
+      // Update to completed state
+      yield* _(progress.updateProgress({ phase: "completed" }));
+
+      // Stop spinner with success
+      yield* _(progress.stopSpinner());
+
+      return result;
+    } catch (error) {
+      // Stop spinner on error
+      yield* _(progress.stopSpinner());
+      throw error;
+    }
+  }).pipe(Effect.provide(ProgressServiceLive));
+};