@xandreed/evals-langsmith 0.14.0 → 0.15.0

package/README.md

@@ -1,5 +1,5 @@
 # @xandreed/evals-langsmith
 
-Efferent composable agent framework, 0.14.0. Node 24.
+Efferent composable agent framework, 0.15.0. Node 24.
 
 Documentation: https://github.com/xandreeddev/efferent

package/dist/index.d.ts

@@ -1 +1 @@
-export * from "./langsmith-export.adapter.js";
+export { EvaluationExportLangSmithLive, langsmithExporter, type LangSmithExportOptions, } from "./langsmith-export.adapter.js";

package/dist/index.js

@@ -1 +1,4 @@
-export * from "./langsmith-export.adapter.js";
+export {
+  EvaluationExportLangSmithLive,
+  langsmithExporter
+} from "./langsmith-export.adapter.js";

package/dist/langsmith-export.adapter.d.ts

@@ -1,10 +1,14 @@
 import { Layer } from "effect";
 import { EvaluationExport, type ExporterRegistration } from "@xandreed/evals";
+/** @public */
 export interface LangSmithExportOptions {
     readonly apiUrl: string;
     readonly apiKey: string;
     readonly project: string;
     readonly webUrl?: string;
+    /** The bound on one whole export, the flush included; 300 000 when absent. */
+    readonly timeoutMs?: number;
 }
+/** @public */
 export declare const EvaluationExportLangSmithLive: (options: LangSmithExportOptions) => Layer.Layer<EvaluationExport, never, never>;
 export declare const langsmithExporter: (options: LangSmithExportOptions) => ExporterRegistration;

package/dist/langsmith-export.adapter.js

@@ -1,75 +1,90 @@
 import { createHash } from "node:crypto";
 import { Client } from "langsmith";
-import { Effect, Layer, Schema } from "effect";
+import { Cause, Clock, Effect, Layer, Predicate, Schema } from "effect";
 import {
   EvaluationError,
   EvaluationExport,
-  EvaluationRun
+  EvaluationRun,
+  exportableScores,
+  reasonOf
 } from "@xandreed/evals";
-const remoteId = (id) => {
+const EXPORT_TIMEOUT_MS = 300000, REQUEST_TIMEOUT_MS = 30000, remoteId = (id) => {
   const hex = createHash("sha256").update(id).digest("hex");
   return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
-}, exportService = (options) => ({
-  exportRun: (run) => Effect.gen(function* () {
-    const encoded = yield* Schema.encodeEffect(EvaluationRun)(run).pipe(Effect.mapError((error) => new EvaluationError({ code: "invalid", message: String(error) }))), mappings = Object.fromEntries(run.trials.map((trial) => [trial.id, remoteId(trial.id)]));
-    yield* Effect.tryPromise({
-      try: async () => {
-        const client = new Client({
-          apiUrl: options.apiUrl,
-          apiKey: options.apiKey,
-          autoBatchTracing: !1
-        }), project = await client.createProject({
-          projectName: options.project,
-          upsert: !0,
-          metadata: { application: run.application }
-        });
-        await Promise.all(encoded.trials.map((trial) => client.createRun({
-          id: mappings[trial.id],
-          name: `${trial.suiteId}/${trial.task.id}`,
-          run_type: "chain",
-          project_name: options.project,
-          start_time: trial.startedAt,
-          end_time: trial.endedAt,
-          inputs: { input: trial.task.input },
-          outputs: { output: trial.output, outcome: trial.outcome },
-          ...trial.status === "completed" ? {} : { error: trial.reason },
-          extra: {
-            metadata: {
-              evaluation: trial,
-              fingerprints: run.fingerprints,
-              runId: run.id
-            }
+}, providerError = (error) => new EvaluationError({
+  code: "provider",
+  message: `LangSmith export: ${reasonOf(Cause.die(error))}`,
+  cause: error
+}), sdk = (request) => Effect.tryPromise({ try: request, catch: (error) => error }), isConflict = (error) => Predicate.hasProperty(error, "name") && error.name === "LangSmithConflictError", upsert = (create, update) => sdk(create).pipe(Effect.catchIf(isConflict, () => sdk(update)), Effect.mapError(providerError), Effect.asVoid), bounded = (timeoutMs) => (effect) => Effect.timeoutOrElse(effect, {
+  duration: timeoutMs,
+  orElse: () => Effect.fail(new EvaluationError({ code: "provider", message: `LangSmith export: not finished within ${timeoutMs} ms` }))
+}), exportService = (options) => ({
+  exportRun: Effect.fn("eval.export", { attributes: { "eval.exporter": "langsmith" } })(function* (run) {
+    const encoded = yield* Schema.encodeEffect(EvaluationRun)(run).pipe(Effect.mapError((error) => new EvaluationError({
+      code: "invalid",
+      message: `Run ${run.id} does not encode: ${error.message}`,
+      cause: error
+    }))), mappings = Object.fromEntries(encoded.trials.map((trial) => [trial.id, remoteId(trial.id)])), client = new Client({
+      apiUrl: options.apiUrl,
+      apiKey: options.apiKey,
+      autoBatchTracing: !1,
+      timeout_ms: REQUEST_TIMEOUT_MS
+    }), project = yield* sdk(() => client.createProject({
+      projectName: options.project,
+      upsert: !0,
+      metadata: { application: encoded.application }
+    })).pipe(Effect.mapError(providerError));
+    yield* Effect.forEach(encoded.trials, (trial) => {
+      const fields = {
+        name: `${trial.suiteId}/${trial.task.id}`,
+        start_time: trial.startedAt,
+        end_time: trial.endedAt,
+        inputs: { input: trial.task.input },
+        outputs: { output: trial.output, outcome: trial.outcome },
+        ...trial.status === "completed" ? {} : { error: trial.reason },
+        extra: {
+          metadata: {
+            evaluation: trial,
+            fingerprints: encoded.fingerprints,
+            runId: encoded.id
           }
-        })));
-        await client.flush();
-        await Promise.all(run.trials.flatMap((trial) => trial.grades.filter((grade) => grade.status === "scored").flatMap((grade) => grade.metrics.filter((metric) => metric.kind === "boolean" || metric.kind === "score" || metric.kind === "probability").map((metric) => client.createFeedback({
-          runId: mappings[trial.id],
-          sessionId: project.id,
-          key: `${grade.grader}@${grade.version}/${grade.scope}/${metric.name}`,
-          score: metric.kind === "boolean" ? Number(metric.value) : metric.value,
-          comment: grade.reason,
-          feedbackId: remoteId(`${trial.id}/${grade.grader}/${grade.version}/${grade.scope}/${metric.name}`),
-          feedbackSourceType: "api",
-          sourceInfo: {
-            context: grade.context,
-            usage: grade.usage
-          }
-        })))));
-      },
-      catch: (error) => new EvaluationError({
-        code: "provider",
-        message: `LangSmith export: ${String(error)}`
-      })
-    });
+        }
+      };
+      return upsert(() => client.createRun({
+        id: mappings[trial.id],
+        run_type: "chain",
+        project_name: options.project,
+        ...fields
+      }), () => client.updateRun(mappings[trial.id], fields));
+    }, { concurrency: 8, discard: !0 });
+    yield* sdk(() => client.flush()).pipe(Effect.mapError(providerError));
+    yield* Effect.forEach(exportableScores(encoded).map((score) => ({
+      runId: mappings[score.trial],
+      key: score.key,
+      score: score.value,
+      comment: score.comment,
+      feedbackId: remoteId(score.seed),
+      sourceInfo: {
+        context: score.context,
+        usage: score.usage
+      }
+    })), (feedback) => upsert(() => client.createFeedback({
+      ...feedback,
+      sessionId: project.id,
+      feedbackSourceType: "api"
+    }), () => client.updateFeedback(feedback.feedbackId, {
+      score: feedback.score,
+      comment: feedback.comment
+    })), { concurrency: 8, discard: !0 });
     return {
       provider: "langsmith",
       runId: run.id,
       url: options.webUrl ?? options.apiUrl,
       exported: run.trials.length,
-      at: Date.now(),
+      at: yield* Clock.currentTimeMillis,
       mappings
     };
-  })
+  }, (effect, run) => Effect.annotateCurrentSpan("eval.run.id", run.id).pipe(Effect.andThen(effect)), bounded(options.timeoutMs ?? EXPORT_TIMEOUT_MS))
 });
 export const EvaluationExportLangSmithLive = (options) => Layer.succeed(EvaluationExport, exportService(options)), langsmithExporter = (options) => ({
   id: "langsmith",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xandreed/evals-langsmith",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "engines": {
     "node": ">=24"
   },
@@ -13,12 +13,14 @@
     }
   },
   "dependencies": {
-    "@xandreed/evals": "0.14.0",
-    "effect": "4.0.7",
+    "@xandreed/evals": "^0.15.0",
     "langsmith": "0.10.8"
   },
   "private": false,
   "types": "./dist/index.d.ts",
+  "peerDependencies": {
+    "effect": "^4.0.7"
+  },
   "files": [
     "dist",
     "skills",

package/dist/export.test.d.ts

@@ -1 +0,0 @@
-export {};