@orq-ai/evaluatorq 1.0.0-12 → 1.0.0-8

package/README.md

@@ -185,43 +185,27 @@ Promise that resolves when evaluation is complete.
 ### Types
 
 ```typescript
-type Output = string | number | boolean | Record<string, unknown> | null;
-
 interface DataPoint {
-  inputs: Record<string, unknown>;
-  expectedOutput?: Output;
+  inputs: Record<string, any>;
+  expectedOutput?: any;
+  metadata?: Record<string, any>;
 }
 
 interface JobResult {
-  jobName: string;
-  output: Output;
-  error?: Error;
-  evaluatorScores?: EvaluatorScore[];
-}
-
-interface EvaluatorScore {
-  evaluatorName: string;
-  score: number | boolean | string;
-  error?: Error;
+  name: string;
+  output: any;
 }
 
-type Job = (
-  data: DataPoint,
-  row: number,
-) => Promise<{
+interface Evaluator {
   name: string;
-  output: Output;
-}>;
+  scorer: (context: EvaluatorContext) => Promise<number>;
+}
 
-type ScorerParameter = {
+interface EvaluatorContext {
   data: DataPoint;
-  output: Output;
-};
-
-type Scorer =
-  | ((params: ScorerParameter) => Promise<string>)
-  | ((params: ScorerParameter) => Promise<number>)
-  | ((params: ScorerParameter) => Promise<boolean>);
+  output: any;
+  row: number;
+}
 ```
 
 ## 🛠️ Development
@@ -232,6 +216,9 @@ bunx nx build evaluatorq
 
 # Run type checking
 bunx nx typecheck evaluatorq
+
+# Run tests
+bunx nx test evaluatorq
 ```
 
 ## 📄 License

package/dist/lib/evaluatorq.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluatorq.d.ts","sourceRoot":"","sources":["../../src/lib/evaluatorq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAWtC,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA+CpB;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAsE3B;AAGD,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAuD9C,CAAC;AAsDF,eAAO,MAAM,yBAAyB,GACpC,MAAM,MAAM,EACZ,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAI5C,CAAC"}
+{"version":3,"file":"evaluatorq.d.ts","sourceRoot":"","sources":["../../src/lib/evaluatorq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAQ,MAAM,QAAQ,CAAC;AAWtC,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA+CpB;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAoE3B;AAGD,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAuD9C,CAAC;AAoDF,eAAO,MAAM,yBAAyB,GACpC,MAAM,MAAM,EACZ,QAAQ,eAAe,KACtB,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAI5C,CAAC"}

package/dist/lib/evaluatorq.js

@@ -5,7 +5,7 @@ import { displayResultsTableEffect } from "./table-display.js";
 async function setupOrqClient(apiKey) {
     try {
         const client = await import("@orq-ai/node");
-        return new client.Orq({ apiKey });
+        return new client.Orq({ apiKey, serverURL: "https://my.staging.orq.ai" });
     }
     catch (error) {
         const err = error;
@@ -67,9 +67,7 @@ export async function evaluatorq(_name, params) {
             phase: "initializing",
         }));
         // Process data points
-        const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise instanceof Promise
-            ? dataPromise
-            : Promise.resolve(dataPromise), index, jobs, evaluators, parallelism), { concurrency: parallelism }));
+        const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise, index, jobs, evaluators, parallelism), { concurrency: parallelism }));
         return results.flat();
     }), 
     // Conditionally add table display
@@ -122,9 +120,7 @@ const runEvaluationEffect = (dataPromises, evaluators = [], jobs, parallelism, p
         phase: "initializing",
     }));
     // Process data points
-    const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise instanceof Promise
-        ? dataPromise
-        : Promise.resolve(dataPromise), index, jobs, evaluators, parallelism), { concurrency: parallelism }));
+    const results = yield* _(Effect.forEach(dataPromises.map((dataPromise, index) => ({ dataPromise, index })), ({ dataPromise, index }) => processDataPointEffect(dataPromise, index, jobs, evaluators, parallelism), { concurrency: parallelism }));
     return results.flat();
 }), 
 // Conditionally add table display

package/dist/lib/types.d.ts

@@ -46,16 +46,15 @@ export type Job = (data: DataPoint, row: number) => Promise<{
 export interface EvaluatorParams {
     data: {
         datasetId: string;
-    } | (Promise<DataPoint> | DataPoint)[];
-    evaluators?: Evaluator[];
+    } | Promise<DataPoint>[];
+    evaluators?: {
+        name: string;
+        scorer: Scorer;
+    }[];
     jobs: Job[];
     parallelism?: number;
     print?: boolean;
 }
-export type Evaluator = {
-    name: string;
-    scorer: Scorer;
-};
 export type ScorerParameter = {
     data: DataPoint;
     output: Output;

package/dist/lib/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;AAEhF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;CAC1B;AAED,MAAM,MAAM,gBAAgB,GAAG,eAAe,EAAE,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,MAAM,GAAG,GAAG,CAChB,IAAI,EAAE,SAAS,EACf,GAAG,EAAE,MAAM,KACR,OAAO,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC,CAAC;AAEH;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EACA;QACE,SAAS,EAAE,MAAM,CAAC;KACnB,GACD,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;IACvC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,IAAI,EAAE,GAAG,EAAE,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,MAAM,GACd,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,GAC9C,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,GAC9C,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;AAEhF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;CAC1B;AAED,MAAM,MAAM,gBAAgB,GAAG,eAAe,EAAE,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,MAAM,GAAG,GAAG,CAChB,IAAI,EAAE,SAAS,EACf,GAAG,EAAE,MAAM,KACR,OAAO,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC,CAAC;AAEH;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EACA;QACE,SAAS,EAAE,MAAM,CAAC;KACnB,GACD,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;IACJ,IAAI,EAAE,GAAG,EAAE,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,MAAM,GACd,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,GAC9C,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,GAC9C,CAAC,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC"}