@orq-ai/evaluatorq 1.0.0-9 → 1.0.7

package/README.md

@@ -32,33 +32,36 @@ npm install @orq-ai/node
 ### Basic Usage
 
 ```typescript
-import { evaluatorq } from "@orq-ai/evaluatorq";
+import { evaluatorq, job } from "@orq-ai/evaluatorq";
+
+const textAnalyzer = job("text-analyzer", async (data) => {
+  const text = data.inputs.text;
+  const analysis = {
+    length: text.length,
+    wordCount: text.split(" ").length,
+    uppercase: text.toUpperCase(),
+  };
+  
+  return analysis;
+});
 
 await evaluatorq("text-analysis", {
   data: [
     { inputs: { text: "Hello world" } },
     { inputs: { text: "Testing evaluation" } },
   ],
-  jobs: [
-    async (data) => {
-      const text = data.inputs.text;
-      const analysis = {
-        length: text.length,
-        wordCount: text.split(" ").length,
-        uppercase: text.toUpperCase(),
-      };
-      
-      return {
-        name: "text-analyzer",
-        output: analysis,
-      };
-    },
-  ],
+  jobs: [textAnalyzer],
   evaluators: [
     {
       name: "length-check",
       scorer: async ({ output }) => {
-        return output.length > 10 ? 1 : 0;
+        const passesCheck = output.length > 10;
+        return {
+          value: passesCheck ? 1 : 0,
+          explanation: passesCheck
+            ? "Output length is sufficient"
+            : `Output too short (${output.length} chars, need >10)`,
+        };
       },
     },
   ],
@@ -68,28 +71,33 @@ await evaluatorq("text-analysis", {
 ### Using Orq Platform Datasets
 
 ```typescript
-import { evaluatorq } from "@orq-ai/evaluatorq";
+import { evaluatorq, job } from "@orq-ai/evaluatorq";
+
+const processor = job("processor", async (data) => {
+  // Process each data point from the dataset
+  return processData(data);
+});
 
 // Requires ORQ_API_KEY environment variable
 await evaluatorq("dataset-evaluation", {
   data: {
     datasetId: "your-dataset-id", // From Orq platform
   },
-  jobs: [
-    async (data) => {
-      // Process each data point from the dataset
-      return {
-        name: "processor",
-        output: processData(data),
-      };
-    },
-  ],
+  jobs: [processor],
   evaluators: [
     {
       name: "accuracy",
       scorer: async ({ data, output }) => {
         // Compare output with expected results
-        return calculateScore(output, data.expectedOutput);
+        const score = calculateScore(output, data.expectedOutput);
+        return {
+          value: score,
+          explanation: score > 0.8
+            ? "High accuracy match"
+            : score > 0.5
+              ? "Partial match"
+              : "Low accuracy match",
+        };
       },
     },
   ],
@@ -103,22 +111,15 @@ await evaluatorq("dataset-evaluation", {
 Run multiple jobs in parallel for each data point:
 
 ```typescript
+import { job } from "@orq-ai/evaluatorq";
+
+const preprocessor = job("preprocessor", async (data) => preprocess(data));
+const analyzer = job("analyzer", async (data) => analyze(data));
+const transformer = job("transformer", async (data) => transform(data));
+
 await evaluatorq("multi-job-eval", {
   data: [...],
-  jobs: [
-    async (data) => ({
-      name: "preprocessor",
-      output: preprocess(data),
-    }),
-    async (data) => ({
-      name: "analyzer",
-      output: analyze(data),
-    }),
-    async (data) => ({
-      name: "transformer",
-      output: transform(data),
-    }),
-  ],
+  jobs: [preprocessor, analyzer, transformer],
   evaluators: [...],
 });
 ```
@@ -126,19 +127,18 @@ await evaluatorq("multi-job-eval", {
 #### Custom Error Handling
 
 ```typescript
+import { job } from "@orq-ai/evaluatorq";
+
+const riskyJob = job("risky-job", async (data) => {
+  // Errors are captured and included in the evaluation results
+  // The job name is preserved even when errors occur
+  const result = await riskyOperation(data);
+  return result;
+});
+
 await evaluatorq("error-handling", {
   data: [...],
-  jobs: [
-    async (data) => {
-      try {
-        const result = await riskyOperation(data);
-        return { name: "risky-job", output: result };
-      } catch (error) {
-        // Errors are captured and included in the evaluation results
-        throw new Error(`Failed to process: ${error.message}`);
-      }
-    },
-  ],
+  jobs: [riskyJob],
   evaluators: [...],
 });
 ```
@@ -162,7 +162,55 @@ await evaluatorq("async-eval", {
 
 ### Environment Variables
 
-- `ORQ_API_KEY`: API key for Orq platform integration (required for dataset access)
+- `ORQ_API_KEY`: API key for Orq platform integration (required for dataset access and sending results)
+
+## 📊 Orq Platform Integration
+
+### Automatic Result Sending
+
+When the `ORQ_API_KEY` environment variable is set, evaluatorq automatically sends evaluation results to the Orq platform for visualization and analysis.
+
+```typescript
+import { evaluatorq, job } from "@orq-ai/evaluatorq";
+
+// Results are automatically sent when ORQ_API_KEY environment variable is present
+await evaluatorq("my-evaluation", {
+  data: [...],
+  jobs: [...],
+  evaluators: [...],
+  sendResults: true, // Enabled by default when ORQ_API_KEY environment variable is set
+});
+```
+
+#### Configuration Options
+
+- `sendResults`: Boolean flag to control result sending (defaults to `true` when `ORQ_API_KEY` is set)
+
+#### What Gets Sent
+
+When enabled, the following information is sent to Orq:
+- Evaluation name
+- Dataset ID (when using Orq datasets)
+- Job results with outputs and errors
+- Evaluator scores with values and explanations
+- Execution timing information
+
+Note: Evaluator explanations are included in the data sent to Orq but are not displayed in the terminal output to keep the console clean.
+
+#### Result Visualization
+
+After successful submission, you'll see a console message with a link to view your results:
+
+```
+📊 View your evaluation results at: <url to the evaluation>
+```
+
+The Orq platform provides:
+- Interactive result tables
+- Score statistics
+- Performance metrics
+- Historical comparisons
+
 
 ## 📚 API Reference
 
@@ -185,27 +233,54 @@ Promise that resolves when evaluation is complete.
 ### Types
 
 ```typescript
+type Output = string | number | boolean | Record<string, unknown> | null;
+
 interface DataPoint {
-  inputs: Record<string, any>;
-  expectedOutput?: any;
-  metadata?: Record<string, any>;
+  inputs: Record<string, unknown>;
+  expectedOutput?: Output;
 }
 
 interface JobResult {
-  name: string;
-  output: any;
+  jobName: string;
+  output: Output;
+  error?: Error;
+  evaluatorScores?: EvaluatorScore[];
 }
 
-interface Evaluator {
-  name: string;
-  scorer: (context: EvaluatorContext) => Promise<number>;
+interface EvaluatorScore {
+  evaluatorName: string;
+  score: EvaluationResult<number | boolean | string>;
+  error?: Error;
 }
 
-interface EvaluatorContext {
+type Job = (
+  data: DataPoint,
+  row: number,
+) => Promise<{
+  name: string;
+  output: Output;
+}>;
+
+// Helper function for creating jobs with preserved names on errors
+function job(
+  name: string,
+  fn: (data: DataPoint, row: number) => Promise<Output> | Output,
+): Job;
+
+type ScorerParameter = {
   data: DataPoint;
-  output: any;
-  row: number;
-}
+  output: Output;
+};
+
+type EvaluationResult<T> = {
+  value: T;
+  explanation?: string;
+};
+
+type Scorer =
+  | ((params: ScorerParameter) => Promise<EvaluationResult<string>>)
+  | ((params: ScorerParameter) => Promise<EvaluationResult<number>>)
+  | ((params: ScorerParameter) => Promise<EvaluationResult<boolean>>);
 ```
 
 ## 🛠️ Development
@@ -216,11 +291,4 @@ bunx nx build evaluatorq
 
 # Run type checking
 bunx nx typecheck evaluatorq
-
-# Run tests
-bunx nx test evaluatorq
 ```
-
-## 📄 License
-
-This is free and unencumbered software released into the public domain. See [UNLICENSE](https://unlicense.org) for details.

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./lib/evaluatorq.js";
+export { job } from "./lib/job-helper.js";
+export { sendResultsToOrqEffect } from "./lib/send-results.js";
 export { displayResultsTableEffect } from "./lib/table-display.js";
 export * from "./lib/types.js";
-export * from "./lib/visualizer/index.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,cAAc,gBAAgB,CAAC;AAC/B,cAAc,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,cAAc,gBAAgB,CAAC"}

package/dist/index.js

@@ -1,4 +1,5 @@
 export * from "./lib/evaluatorq.js";
+export { job } from "./lib/job-helper.js";
+export { sendResultsToOrqEffect } from "./lib/send-results.js";
 export { displayResultsTableEffect } from "./lib/table-display.js";
 export * from "./lib/types.js";
-export * from "./lib/visualizer/index.js";

package/dist/lib/effects.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"effects.d.ts","sourceRoot":"","sources":["../../src/lib/effects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,GAAG,EACH,SAAS,EACT,MAAM,EACP,MAAM,YAAY,CAAC;AAEpB,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,EAC/B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EAAE,EACX,UAAU,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,EAC9C,WAAW,EAAE,MAAM,GAClB,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,eAAe,CAAC,CA4C1D;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,CAAC,CAyGlD"}
+{"version":3,"file":"effects.d.ts","sourceRoot":"","sources":["../../src/lib/effects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,GAAG,EACH,SAAS,EACT,MAAM,EACP,MAAM,YAAY,CAAC;AAEpB,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,EAC/B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EAAE,EACX,UAAU,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,EAC9C,WAAW,EAAE,MAAM,GAClB,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,eAAe,CAAC,CA4C1D;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,CAAC,CA4GlD"}

package/dist/lib/effects.js

@@ -62,10 +62,10 @@ export function processJobEffect(job, dataPoint, rowIndex, evaluators) {
                     catch: (error) => error,
                 }), Effect.map((score) => ({
                     evaluatorName: evaluator.name,
-                    score: score,
+                    score,
                 })), Effect.catchAll((error) => Effect.succeed({
                     evaluatorName: evaluator.name,
-                    score: "",
+                    score: { value: "" },
                     error: error,
                 }))));
                 return score;
@@ -81,9 +81,14 @@ export function processJobEffect(job, dataPoint, rowIndex, evaluators) {
             output: jobResult.output,
             evaluatorScores: [],
         };
-    }).pipe(Effect.catchAll((error) => Effect.succeed({
-        jobName: "Unknown", // We don't know the job name if it threw before returning
-        output: null,
-        error,
-    })));
+    }).pipe(Effect.catchAll((error) => {
+        // Check if the error has a jobName property (set by our job helper)
+        const errorWithJobName = error;
+        const jobName = errorWithJobName.jobName || "Unknown";
+        return Effect.succeed({
+            jobName,
+            output: null,
+            error,
+        });
+    }));
 }

package/dist/lib/evaluatorq.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluatorq.d.ts","sourceRoot":"","sources":["../../src/lib/evaluatorq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAWtC,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA+CpB;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAoE3B;AAGD,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAuD9C,CAAC;AAoDF,eAAO,MAAM,yBAAyB,GACpC,MAAM,MAAM,EACZ,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAI5C,CAAC"}
+{"version":3,"file":"evaluatorq.d.ts","sourceRoot":"","sources":["../../src/lib/evaluatorq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAYtC,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA+CpB;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAmG3B;AAGD,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAmF9C,CAAC;AAgFF,eAAO,MAAM,yBAAyB,GACpC,MAAM,MAAM,EACZ,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAI5C,CAAC"}

package/dist/lib/evaluatorq.js

@@ -1,11 +1,12 @@
 import { Effect, pipe } from "effect";
 import { processDataPointEffect } from "./effects.js";
 import { ProgressService, ProgressServiceLive, withProgress, } from "./progress.js";
+import { sendResultsToOrqEffect } from "./send-results.js";
 import { displayResultsTableEffect } from "./table-display.js";
 async function setupOrqClient(apiKey) {
     try {
         const client = await import("@orq-ai/node");
-        return new client.Orq({ apiKey, serverURL: "https://my.staging.orq.ai" });
+        return new client.Orq({ apiKey });
     }
     catch (error) {
         const err = error;
@@ -40,18 +41,23 @@ async function fetchDatasetAsDataPoints(orqClient, datasetId) {
  * @returns The results of the evaluation run.
  */
 export async function evaluatorq(_name, params) {
-    const { data, evaluators = [], jobs, parallelism = 1, print = true } = params;
+    const { data, evaluators = [], jobs, parallelism = 1, print = true, sendResults, description, } = params;
     let orqClient;
     const orqApiKey = process.env.ORQ_API_KEY;
     if (orqApiKey) {
         orqClient = await setupOrqClient(orqApiKey);
     }
+    // Default sendResults to true when API key is available
+    const shouldSendResults = sendResults !== undefined ? sendResults : Boolean(orqApiKey);
+    const startTime = new Date();
     let dataPromises;
+    let datasetId;
     // Handle datasetId case
     if ("datasetId" in data) {
         if (!orqApiKey || !orqClient) {
             throw new Error("ORQ_API_KEY environment variable must be set to fetch datapoints from Orq platform.");
         }
+        datasetId = data.datasetId;
         dataPromises = await fetchDatasetAsDataPoints(orqClient, data.datasetId);
     }
     else {
@@ -67,13 +73,19 @@ export async function evaluatorq(_name, params) {
             phase: "initializing",
         }));
         // Process data points
-        const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise, index, jobs, evaluators, parallelism), { concurrency: parallelism }));
+        const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise instanceof Promise
+            ? dataPromise
+            : Promise.resolve(dataPromise), index, jobs, evaluators, parallelism), { concurrency: parallelism }));
         return results.flat();
     }), 
     // Conditionally add table display
     print
         ? Effect.tap((results) => displayResultsTableEffect(results))
         : Effect.tap(() => Effect.void), 
+    // Conditionally send results to Orq
+    shouldSendResults && orqApiKey
+        ? Effect.tap((results) => sendResultsToOrqEffect(orqApiKey, _name, description, datasetId, results, startTime, new Date()))
+        : Effect.tap(() => Effect.void), 
     // Provide the progress service
     Effect.provide(ProgressServiceLive), 
     // Wrap with progress tracking
@@ -83,7 +95,8 @@ export async function evaluatorq(_name, params) {
 }
 // Create an Effect that runs evaluation and optionally displays results
 export const evaluatorqEffect = (_name, params) => {
-    const { data, evaluators = [], jobs, parallelism = 1, print = true } = params;
+    const { data, evaluators = [], jobs, parallelism = 1, print = true, sendResults, description, } = params;
+    const startTime = new Date();
     // Handle datasetId case
     if ("datasetId" in data) {
         return Effect.gen(function* (_) {
@@ -104,32 +117,43 @@ export const evaluatorqEffect = (_name, params) => {
                     ? error
                     : new Error(`Failed to fetch dataset: ${String(error)}`),
             }));
-            return yield* _(runEvaluationEffect(dataPromises, evaluators, jobs, parallelism, print));
+            return yield* _(runEvaluationEffect(dataPromises, evaluators, jobs, parallelism, print, sendResults, description, _name, data.datasetId, apiKey, startTime));
         });
     }
     const dataPromises = data;
-    return runEvaluationEffect(dataPromises, evaluators, jobs, parallelism, print);
+    return runEvaluationEffect(dataPromises, evaluators, jobs, parallelism, print, sendResults, description, _name, undefined, undefined, startTime);
 };
 // Extract common evaluation logic
-const runEvaluationEffect = (dataPromises, evaluators = [], jobs, parallelism, print) => pipe(Effect.gen(function* (_) {
-    const progress = yield* _(ProgressService);
-    // Initialize progress
-    yield* _(progress.updateProgress({
-        totalDataPoints: dataPromises.length,
-        currentDataPoint: 0,
-        phase: "initializing",
-    }));
-    // Process data points
-    const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise, index, jobs, evaluators, parallelism), { concurrency: parallelism }));
-    return results.flat();
-}), 
-// Conditionally add table display
-print
-    ? Effect.tap((results) => displayResultsTableEffect(results))
-    : Effect.tap(() => Effect.void), 
-// Provide the progress service
-Effect.provide(ProgressServiceLive), 
-// Wrap with progress tracking
-(effect) => withProgress(effect, print));
+const runEvaluationEffect = (dataPromises, evaluators = [], jobs, parallelism, print, sendResults, description, evaluationName, datasetId, apiKey, startTime) => {
+    // Default sendResults to true when API key is available
+    const orqApiKey = apiKey || process.env.ORQ_API_KEY;
+    const shouldSendResults = sendResults !== undefined ? sendResults : Boolean(orqApiKey);
+    return pipe(Effect.gen(function* (_) {
+        const progress = yield* _(ProgressService);
+        // Initialize progress
+        yield* _(progress.updateProgress({
+            totalDataPoints: dataPromises.length,
+            currentDataPoint: 0,
+            phase: "initializing",
+        }));
+        // Process data points
+        const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise instanceof Promise
+            ? dataPromise
+            : Promise.resolve(dataPromise), index, jobs, evaluators, parallelism), { concurrency: parallelism }));
+        return results.flat();
+    }), 
+    // Conditionally add table display
+    print
+        ? Effect.tap((results) => displayResultsTableEffect(results))
+        : Effect.tap(() => Effect.void), 
+    // Conditionally send results to Orq
+    shouldSendResults && orqApiKey
+        ? Effect.tap((results) => sendResultsToOrqEffect(orqApiKey, evaluationName, description, datasetId, results, startTime, new Date()))
+        : Effect.tap(() => Effect.void), 
+    // Provide the progress service
+    Effect.provide(ProgressServiceLive), 
+    // Wrap with progress tracking
+    (effect) => withProgress(effect, print));
+};
 // Composable evaluatorq with display
 export const evaluatorqWithTableEffect = (name, params) => pipe(evaluatorqEffect(name, params), Effect.tap((results) => displayResultsTableEffect(results)));

package/dist/lib/job-helper.d.ts

@@ -0,0 +1,17 @@
+import type { DataPoint, Job, Output } from "./types.js";
+/**
+ * Helper function to create a named job that ensures the job name is preserved
+ * even when errors occur during execution.
+ *
+ * @param name - The name of the job
+ * @param fn - The job function that returns the output
+ * @returns A Job function that always includes the job name
+ *
+ * @example
+ * const myJob = job("myJobName", async (data) => {
+ *   // Your job logic here
+ *   return "output";
+ * });
+ */
+export declare function job(name: string, fn: (data: DataPoint, row: number) => Promise<Output> | Output): Job;
+//# sourceMappingURL=job-helper.d.ts.map

package/dist/lib/job-helper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-helper.d.ts","sourceRoot":"","sources":["../../src/lib/job-helper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CACjB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,GAC7D,GAAG,CAoBL"}

package/dist/lib/job-helper.js

@@ -0,0 +1,33 @@
+/**
+ * Helper function to create a named job that ensures the job name is preserved
+ * even when errors occur during execution.
+ *
+ * @param name - The name of the job
+ * @param fn - The job function that returns the output
+ * @returns A Job function that always includes the job name
+ *
+ * @example
+ * const myJob = job("myJobName", async (data) => {
+ *   // Your job logic here
+ *   return "output";
+ * });
+ */
+export function job(name, fn) {
+    return async (data, row) => {
+        try {
+            const output = await fn(data, row);
+            return {
+                name,
+                output,
+            };
+        }
+        catch (error) {
+            // Re-throw the error with the job name attached
+            // The error will be caught by the evaluatorq framework
+            // but the name will be preserved
+            throw Object.assign(error instanceof Error ? error : new Error(String(error)), {
+                jobName: name,
+            });
+        }
+    };
+}

package/dist/lib/send-results.d.ts

@@ -0,0 +1,32 @@
+import { Effect } from "effect";
+import type { DataPoint, EvaluatorqResult, Output } from "./types.js";
+export interface SerializedEvaluatorScore {
+    evaluatorName: string;
+    score: {
+        value: number | boolean | string;
+        explanation?: string;
+    };
+    error?: string;
+}
+export interface SerializedJobResult {
+    jobName: string;
+    output: Output;
+    error?: string;
+    evaluatorScores?: SerializedEvaluatorScore[];
+}
+export interface SerializedDataPointResult {
+    dataPoint: DataPoint;
+    error?: string;
+    jobResults?: SerializedJobResult[];
+}
+export interface SendResultsPayload {
+    _name: string;
+    _description?: string;
+    _createdAt: string;
+    _endedAt: string;
+    _evaluationDuration: number;
+    datasetId?: string;
+    results: SerializedDataPointResult[];
+}
+export declare const sendResultsToOrqEffect: (apiKey: string, evaluationName: string, evaluationDescription: string | undefined, datasetId: string | undefined, results: EvaluatorqResult, startTime: Date, endTime: Date) => Effect.Effect<void, never, never>;
+//# sourceMappingURL=send-results.d.ts.map

package/dist/lib/send-results.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"send-results.d.ts","sourceRoot":"","sources":["../../src/lib/send-results.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGtE,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;QACjC,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,wBAAwB,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,yBAAyB;IACxC,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAGD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,yBAAyB,EAAE,CAAC;CACtC;AAWD,eAAO,MAAM,sBAAsB,GACjC,QAAQ,MAAM,EACd,gBAAgB,MAAM,EACtB,uBAAuB,MAAM,GAAG,SAAS,EACzC,WAAW,MAAM,GAAG,SAAS,EAC7B,SAAS,gBAAgB,EACzB,WAAW,IAAI,EACf,SAAS,IAAI,KACZ,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAmG/B,CAAC"}

package/dist/lib/send-results.js

@@ -0,0 +1,70 @@
+import { Effect } from "effect";
+export const sendResultsToOrqEffect = (apiKey, evaluationName, evaluationDescription, datasetId, results, startTime, endTime) => Effect.gen(function* (_) {
+    // Convert Error objects to strings for JSON serialization
+    const serializedResults = results.map((result) => ({
+        dataPoint: result.dataPoint,
+        error: result.error ? String(result.error) : undefined,
+        jobResults: result.jobResults?.map((jobResult) => ({
+            jobName: jobResult.jobName,
+            output: jobResult.output,
+            error: jobResult.error ? String(jobResult.error) : undefined,
+            evaluatorScores: jobResult.evaluatorScores?.map((score) => ({
+                evaluatorName: score.evaluatorName,
+                score: score.score,
+                error: score.error ? String(score.error) : undefined,
+            })),
+        })),
+    }));
+    const payload = {
+        _name: evaluationName,
+        _description: evaluationDescription,
+        _createdAt: startTime.toISOString(),
+        _endedAt: endTime.toISOString(),
+        _evaluationDuration: endTime.getTime() - startTime.getTime(),
+        ...(datasetId && { datasetId }),
+        results: serializedResults,
+    };
+    // Use tryPromise but catch and log errors instead of propagating them
+    yield* _(Effect.tryPromise({
+        try: async () => {
+            const baseUrl = process.env.ORQ_BASE_URL || "https://api.orq.ai";
+            const response = await fetch(`${baseUrl}/v2/spreadsheets/evaluations/receive`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${apiKey}`,
+                },
+                body: JSON.stringify(payload),
+            });
+            if (!response.ok) {
+                const errorText = await response
+                    .text()
+                    .catch(() => "Unknown error");
+                // Log warning instead of throwing
+                console.warn(`\n⚠️  Warning: Could not send results to Orq platform (${response.status} ${response.statusText})`);
+                // Only show detailed error in verbose mode or specific error cases
+                if (process.env.ORQ_DEBUG === "true" || response.status >= 500) {
+                    console.warn(`   Details: ${errorText}`);
+                }
+                return; // Return early but don't throw
+            }
+            const result = (await response.json());
+            console.log(`\n✅ Results sent to Orq: ${result.experiment_name} (${result.rows_created} rows created)`);
+            // Display the experiment URL if available
+            if (result.experiment_url) {
+                console.log(`   📊 View your evaluation at: ${result.experiment_url}`);
+            }
+        },
+        catch: (error) => {
+            // Log warning for network or other errors
+            console.warn(`\n⚠️  Warning: Could not send results to Orq platform`);
+            if (process.env.ORQ_DEBUG === "true") {
+                console.warn(`   Details: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            // Return undefined to indicate handled error
+            return undefined;
+        },
+    }), 
+    // Catch any Effect errors and convert to success
+    Effect.catchAll(() => Effect.succeed(undefined)));
+});

package/dist/lib/table-display.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table-display.d.ts","sourceRoot":"","sources":["../../src/lib/table-display.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AA0TnD,eAAO,MAAM,yBAAyB,GACpC,SAAS,gBAAgB,KACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CA+B/B,CAAC"}
+{"version":3,"file":"table-display.d.ts","sourceRoot":"","sources":["../../src/lib/table-display.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAuVnD,eAAO,MAAM,yBAAyB,GACpC,SAAS,gBAAgB,KACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAqC/B,CAAC"}