orchajs 0.22.0 → 0.23.0

package/dist/amarsia/analytics.js

@@ -270,6 +270,7 @@ function projectRunFact(manifest, sessionId, run, projection) {
     const failedActions = runEvents.filter((event) => event.type === "action.failed");
     const evaluations = runEvents.filter((event) => event.type === "evaluation.completed" ||
         event.type === "evaluation.failed");
+    const evaluationMetrics = evaluations.flatMap((event) => Array.isArray(event.data.metrics) ? event.data.metrics : []);
     const error = asRecord(terminal?.data.error);
     const lastAssistant = assistants.at(-1);
     const status = terminal?.type === "run.completed"
@@ -324,6 +325,12 @@ function projectRunFact(manifest, sessionId, run, projection) {
         evaluation_count: evaluations.length,
         evaluation_pass_count: evaluations.filter((event) => event.type === "evaluation.completed" &&
             event.data.status === "passed").length,
+        evaluation_failure_count: evaluations.filter((event) => event.type === "evaluation.completed" &&
+            event.data.status === "failed").length,
+        evaluation_error_count: evaluations.filter((event) => event.type === "evaluation.failed").length,
+        evaluation_metric_count: evaluationMetrics.length,
+        evaluation_metric_pass_count: evaluationMetrics.filter((metric) => asRecord(metric)?.passed === true).length,
+        evaluations: projectEvaluationLogs(evaluations),
         ...(typeof error?.code === "string" ? { error_code: error.code } : {}),
         ...(typeof error?.message === "string"
             ? { error_message: sanitizeErrorMessage(error.message) }
@@ -405,6 +412,59 @@ function projectActionLogs(events) {
         };
     });
 }
+function projectEvaluationLogs(events) {
+    return events.slice(0, 50).map((event) => {
+        const usage = asRecord(event.data.usage);
+        const error = asRecord(event.data.error);
+        const metrics = Array.isArray(event.data.metrics)
+            ? event.data.metrics
+                .map(asRecord)
+                .filter((metric) => metric !== undefined)
+                .slice(0, 50)
+                .map((metric) => ({
+                name: typeof metric.name === "string" ? metric.name : "unknown_metric",
+                score: numberValue(metric.score),
+                threshold: numberValue(metric.threshold),
+                passed: metric.passed === true,
+                reasoning: typeof metric.reasoning === "string"
+                    ? sanitizeErrorMessage(metric.reasoning)
+                    : "",
+                evidence: Array.isArray(metric.evidence)
+                    ? metric.evidence
+                        .filter((item) => typeof item === "string")
+                        .slice(0, 20)
+                        .map(sanitizeErrorMessage)
+                    : [],
+            }))
+            : [];
+        return {
+            name: typeof event.data.name === "string"
+                ? event.data.name
+                : "unknown_evaluation",
+            status: event.type === "evaluation.failed"
+                ? "error"
+                : event.data.status === "passed"
+                    ? "passed"
+                    : "failed",
+            ...(typeof event.data.provider === "string"
+                ? { provider: event.data.provider }
+                : {}),
+            ...(typeof event.data.model === "string"
+                ? { model: event.data.model }
+                : {}),
+            duration_ms: numberValue(event.data.durationMs),
+            input_tokens: numberValue(usage?.inputTokens),
+            output_tokens: numberValue(usage?.outputTokens),
+            reasoning_tokens: numberValue(usage?.reasoningTokens),
+            cache_read_tokens: numberValue(usage?.cacheReadTokens),
+            cache_write_tokens: numberValue(usage?.cacheWriteTokens),
+            metrics,
+            ...(typeof error?.message === "string"
+                ? { error_message: sanitizeErrorMessage(error.message) }
+                : {}),
+        };
+    });
+}
 function takeBatch(queue) {
     const batch = [];
     let bytes = utf8ByteLength('{"schema_version":1,"runs":[]}');

package/dist/types.d.ts

@@ -97,6 +97,11 @@ export interface AmarsiaTelemetryRunV1 {
     actions: AmarsiaTelemetryActionV1[];
     evaluation_count: number;
     evaluation_pass_count: number;
+    evaluation_failure_count: number;
+    evaluation_error_count: number;
+    evaluation_metric_count: number;
+    evaluation_metric_pass_count: number;
+    evaluations: AmarsiaTelemetryEvaluationV1[];
     error_code?: string;
     error_message?: string;
     error_retryable?: boolean;
@@ -111,6 +116,28 @@ export interface AmarsiaTelemetryActionV1 {
     error_code?: string;
     error_message?: string;
 }
+export interface AmarsiaTelemetryEvaluationMetricV1 {
+    name: string;
+    score: number;
+    threshold: number;
+    passed: boolean;
+    reasoning: string;
+    evidence: string[];
+}
+export interface AmarsiaTelemetryEvaluationV1 {
+    name: string;
+    status: "passed" | "failed" | "error";
+    provider?: string;
+    model?: string;
+    duration_ms: number;
+    input_tokens: number;
+    output_tokens: number;
+    reasoning_tokens: number;
+    cache_read_tokens: number;
+    cache_write_tokens: number;
+    metrics: AmarsiaTelemetryEvaluationMetricV1[];
+    error_message?: string;
+}
 export interface AmarsiaTelemetryBatchV1 {
     schema_version: 1;
     runs: AmarsiaTelemetryRunV1[];

package/dist-cjs/amarsia/analytics.js

@@ -289,6 +289,9 @@ function projectRunFact(manifest, sessionId, run, projection) {
   const evaluations = runEvents.filter(
     (event) => event.type === "evaluation.completed" || event.type === "evaluation.failed"
   );
+  const evaluationMetrics = evaluations.flatMap(
+    (event) => Array.isArray(event.data.metrics) ? event.data.metrics : []
+  );
   const error = asRecord(terminal?.data.error);
   const lastAssistant = assistants.at(-1);
   const status = terminal?.type === "run.completed" ? "completed" : terminal?.type === "run.failed" ? "failed" : terminal?.type === "run.paused" ? "paused" : "running";
@@ -331,6 +334,17 @@ function projectRunFact(manifest, sessionId, run, projection) {
     evaluation_pass_count: evaluations.filter(
       (event) => event.type === "evaluation.completed" && event.data.status === "passed"
     ).length,
+    evaluation_failure_count: evaluations.filter(
+      (event) => event.type === "evaluation.completed" && event.data.status === "failed"
+    ).length,
+    evaluation_error_count: evaluations.filter(
+      (event) => event.type === "evaluation.failed"
+    ).length,
+    evaluation_metric_count: evaluationMetrics.length,
+    evaluation_metric_pass_count: evaluationMetrics.filter(
+      (metric) => asRecord(metric)?.passed === true
+    ).length,
+    evaluations: projectEvaluationLogs(evaluations),
     ...typeof error?.code === "string" ? { error_code: error.code } : {},
     ...typeof error?.message === "string" ? { error_message: sanitizeErrorMessage(error.message) } : {},
     ...typeof error?.retryable === "boolean" ? { error_retryable: error.retryable } : {}
@@ -396,6 +410,38 @@ function projectActionLogs(events) {
     };
   });
 }
+function projectEvaluationLogs(events) {
+  return events.slice(0, 50).map((event) => {
+    const usage = asRecord(event.data.usage);
+    const error = asRecord(event.data.error);
+    const metrics = Array.isArray(event.data.metrics) ? event.data.metrics.map(asRecord).filter(
+      (metric) => metric !== void 0
+    ).slice(0, 50).map((metric) => ({
+      name: typeof metric.name === "string" ? metric.name : "unknown_metric",
+      score: numberValue(metric.score),
+      threshold: numberValue(metric.threshold),
+      passed: metric.passed === true,
+      reasoning: typeof metric.reasoning === "string" ? sanitizeErrorMessage(metric.reasoning) : "",
+      evidence: Array.isArray(metric.evidence) ? metric.evidence.filter(
+        (item) => typeof item === "string"
+      ).slice(0, 20).map(sanitizeErrorMessage) : []
+    })) : [];
+    return {
+      name: typeof event.data.name === "string" ? event.data.name : "unknown_evaluation",
+      status: event.type === "evaluation.failed" ? "error" : event.data.status === "passed" ? "passed" : "failed",
+      ...typeof event.data.provider === "string" ? { provider: event.data.provider } : {},
+      ...typeof event.data.model === "string" ? { model: event.data.model } : {},
+      duration_ms: numberValue(event.data.durationMs),
+      input_tokens: numberValue(usage?.inputTokens),
+      output_tokens: numberValue(usage?.outputTokens),
+      reasoning_tokens: numberValue(usage?.reasoningTokens),
+      cache_read_tokens: numberValue(usage?.cacheReadTokens),
+      cache_write_tokens: numberValue(usage?.cacheWriteTokens),
+      metrics,
+      ...typeof error?.message === "string" ? { error_message: sanitizeErrorMessage(error.message) } : {}
+    };
+  });
+}
 function takeBatch(queue) {
   const batch = [];
   let bytes = utf8ByteLength('{"schema_version":1,"runs":[]}');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orchajs",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "description": "A filesystem-convention, stateful agent framework for JavaScript.",
   "license": "MIT",
   "repository": {