orchajs 0.22.0 → 0.23.1

package/dist/amarsia/analytics.js

@@ -270,6 +270,7 @@ function projectRunFact(manifest, sessionId, run, projection) {
     const failedActions = runEvents.filter((event) => event.type === "action.failed");
     const evaluations = runEvents.filter((event) => event.type === "evaluation.completed" ||
         event.type === "evaluation.failed");
+    const evaluationMetrics = evaluations.flatMap((event) => Array.isArray(event.data.metrics) ? event.data.metrics : []);
     const error = asRecord(terminal?.data.error);
     const lastAssistant = assistants.at(-1);
     const status = terminal?.type === "run.completed"
@@ -324,6 +325,12 @@ function projectRunFact(manifest, sessionId, run, projection) {
         evaluation_count: evaluations.length,
         evaluation_pass_count: evaluations.filter((event) => event.type === "evaluation.completed" &&
             event.data.status === "passed").length,
+        evaluation_failure_count: evaluations.filter((event) => event.type === "evaluation.completed" &&
+            event.data.status === "failed").length,
+        evaluation_error_count: evaluations.filter((event) => event.type === "evaluation.failed").length,
+        evaluation_metric_count: evaluationMetrics.length,
+        evaluation_metric_pass_count: evaluationMetrics.filter((metric) => asRecord(metric)?.passed === true).length,
+        evaluations: projectEvaluationLogs(evaluations),
         ...(typeof error?.code === "string" ? { error_code: error.code } : {}),
         ...(typeof error?.message === "string"
             ? { error_message: sanitizeErrorMessage(error.message) }
@@ -405,6 +412,59 @@ function projectActionLogs(events) {
         };
     });
 }
+function projectEvaluationLogs(events) {
+    return events.slice(0, 50).map((event) => {
+        const usage = asRecord(event.data.usage);
+        const error = asRecord(event.data.error);
+        const metrics = Array.isArray(event.data.metrics)
+            ? event.data.metrics
+                .map(asRecord)
+                .filter((metric) => metric !== undefined)
+                .slice(0, 50)
+                .map((metric) => ({
+                name: typeof metric.name === "string" ? metric.name : "unknown_metric",
+                score: numberValue(metric.score),
+                threshold: numberValue(metric.threshold),
+                passed: metric.passed === true,
+                reasoning: typeof metric.reasoning === "string"
+                    ? sanitizeErrorMessage(metric.reasoning)
+                    : "",
+                evidence: Array.isArray(metric.evidence)
+                    ? metric.evidence
+                        .filter((item) => typeof item === "string")
+                        .slice(0, 20)
+                        .map(sanitizeErrorMessage)
+                    : [],
+            }))
+            : [];
+        return {
+            name: typeof event.data.name === "string"
+                ? event.data.name
+                : "unknown_evaluation",
+            status: event.type === "evaluation.failed"
+                ? "error"
+                : event.data.status === "passed"
+                    ? "passed"
+                    : "failed",
+            ...(typeof event.data.provider === "string"
+                ? { provider: event.data.provider }
+                : {}),
+            ...(typeof event.data.model === "string"
+                ? { model: event.data.model }
+                : {}),
+            duration_ms: numberValue(event.data.durationMs),
+            input_tokens: numberValue(usage?.inputTokens),
+            output_tokens: numberValue(usage?.outputTokens),
+            reasoning_tokens: numberValue(usage?.reasoningTokens),
+            cache_read_tokens: numberValue(usage?.cacheReadTokens),
+            cache_write_tokens: numberValue(usage?.cacheWriteTokens),
+            metrics,
+            ...(typeof error?.message === "string"
+                ? { error_message: sanitizeErrorMessage(error.message) }
+                : {}),
+        };
+    });
+}
 function takeBatch(queue) {
     const batch = [];
     let bytes = utf8ByteLength('{"schema_version":1,"runs":[]}');

package/dist/compiler/compile.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { existsSync, readFileSync, readdirSync, statSync, } from "node:fs";
 import { resolve } from "node:path";
 import { buildSync } from "esbuild";
 import { isBuiltInProvider } from "../providers/catalog.js";
@@ -163,27 +163,24 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
         return {};
     }
     const actions = {};
-    for (const entry of readdirSync(actionsDirectory, { withFileTypes: true })) {
-        if (!entry.isDirectory()) {
-            continue;
-        }
-        const configurationPath = resolve(actionsDirectory, entry.name, "index.json");
+    for (const entry of discoverDirectories(actionsDirectory)) {
+        const configurationPath = resolve(actionsDirectory, entry, "index.json");
         let configuration;
         try {
             configuration = JSON.parse(readFileSync(configurationPath, "utf8"));
         }
         catch (error) {
-            throw new Error(`Action "${agentName}/${entry.name}" is missing a valid index.json.`, { cause: error });
+            throw new Error(`Action "${agentName}/${entry}" is missing a valid index.json.`, { cause: error });
         }
-        validateEnabled(`Action "${agentName}/${entry.name}"`, configuration?.enabled);
+        validateEnabled(`Action "${agentName}/${entry}"`, configuration?.enabled);
         if (configuration?.enabled === false) {
             continue;
         }
         if (!configuration.name?.trim() || !configuration.description?.trim()) {
-            throw new Error(`Action "${agentName}/${entry.name}" requires name and description.`);
+            throw new Error(`Action "${agentName}/${entry}" requires name and description.`);
         }
         if (!/^[a-zA-Z0-9_-]{1,64}$/.test(configuration.name)) {
-            throw new Error(`Action "${agentName}/${entry.name}" has an invalid model-facing name.`);
+            throw new Error(`Action "${agentName}/${entry}" has an invalid model-facing name.`);
         }
         if ([
             "load_skill",
@@ -191,50 +188,50 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
             "resume_agent",
             "inspect_agent",
         ].includes(configuration.name)) {
-            throw new Error(`Action "${agentName}/${entry.name}" uses reserved name "${configuration.name}".`);
+            throw new Error(`Action "${agentName}/${entry}" uses reserved name "${configuration.name}".`);
         }
         const execution = configuration.execution ?? "local";
         if (!["client", "local"].includes(execution)) {
-            throw new Error(`Action "${agentName}/${entry.name}" has invalid execution "${configuration.execution}".`);
+            throw new Error(`Action "${agentName}/${entry}" has invalid execution "${configuration.execution}".`);
         }
         if (!configuration.parameters ||
             typeof configuration.parameters !== "object" ||
             Array.isArray(configuration.parameters)) {
-            throw new Error(`Action "${agentName}/${entry.name}" requires a parameters JSON Schema.`);
+            throw new Error(`Action "${agentName}/${entry}" requires a parameters JSON Schema.`);
         }
         if (configuration.outputSchema !== undefined &&
             (typeof configuration.outputSchema !== "object" ||
                 configuration.outputSchema === null ||
                 Array.isArray(configuration.outputSchema))) {
-            throw new Error(`Action "${agentName}/${entry.name}" has an invalid outputSchema.`);
+            throw new Error(`Action "${agentName}/${entry}" has an invalid outputSchema.`);
         }
-        validateNumber(`${agentName}/${entry.name}`, "timeoutMs", configuration.timeoutMs, { integer: true, minimum: 1, maximum: 120_000 });
+        validateNumber(`${agentName}/${entry}`, "timeoutMs", configuration.timeoutMs, { integer: true, minimum: 1, maximum: 120_000 });
         if (configuration.permissions?.env !== undefined &&
             !Array.isArray(configuration.permissions.env)) {
-            throw new Error(`Action "${agentName}/${entry.name}" permissions.env must be an array.`);
+            throw new Error(`Action "${agentName}/${entry}" permissions.env must be an array.`);
         }
         if (configuration.permissions?.network !== undefined &&
             !Array.isArray(configuration.permissions.network)) {
-            throw new Error(`Action "${agentName}/${entry.name}" permissions.network must be an array.`);
+            throw new Error(`Action "${agentName}/${entry}" permissions.network must be an array.`);
         }
         for (const permission of [
             ...(configuration.permissions?.env ?? []),
             ...(configuration.permissions?.network ?? []),
         ]) {
             if (typeof permission !== "string" || !permission.trim()) {
-                throw new Error(`Action "${agentName}/${entry.name}" has an invalid permission.`);
+                throw new Error(`Action "${agentName}/${entry}" has an invalid permission.`);
             }
         }
         if (actions[configuration.name]) {
             throw new Error(`Agent "${agentName}" has duplicate action name "${configuration.name}".`);
         }
         const compiledSource = execution === "local"
-            ? compileLocalAction(agentName, entry.name, actionsDirectory, actionRuntime, target)
+            ? compileLocalAction(agentName, entry, actionsDirectory, actionRuntime, target)
             : undefined;
         actions[configuration.name] = Object.freeze({
             ...configuration,
             execution,
-            directoryName: entry.name,
+            directoryName: entry,
             ...compiledSource,
         });
     }
@@ -321,7 +318,10 @@ function compileEvaluations(agentName, sourceDirectory) {
 }
 function discoverDirectories(directory) {
     return readdirSync(directory, { withFileTypes: true })
-        .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
+        .filter((entry) => !entry.name.startsWith(".") &&
+        (entry.isDirectory() ||
+            (entry.isSymbolicLink() &&
+                statSync(resolve(directory, entry.name)).isDirectory())))
         .map((entry) => entry.name)
         .sort();
 }

package/dist/playground/source.js

@@ -1,4 +1,4 @@
-import { readFile, readdir } from "node:fs/promises";
+import { readFile, readdir, stat } from "node:fs/promises";
 import { resolve } from "node:path";
 import { getOrchaRuntimeContext } from "../runtime/context.js";
 export async function readPlaygroundAgentSource(orcha, agentKey) {
@@ -57,10 +57,17 @@ async function readCollectionItems(agentDirectory, collection, manifests, includ
         manifest.directoryName,
         { key, manifest },
     ]));
-    return Promise.all(entries
-        .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
-        .sort((left, right) => left.name.localeCompare(right.name))
-        .map(async (entry) => {
+    const directories = (await Promise.all(entries
+        .filter((entry) => !entry.name.startsWith("."))
+        .map(async (entry) => ({
+        entry,
+        isDirectory: entry.isDirectory() ||
+            (entry.isSymbolicLink() &&
+                (await stat(resolve(root, entry.name))).isDirectory()),
+    })))).filter(({ isDirectory }) => isDirectory);
+    return Promise.all(directories
+        .sort((left, right) => left.entry.name.localeCompare(right.entry.name))
+        .map(async ({ entry }) => {
         const directory = resolve(root, entry.name);
         const rawConfiguration = await readRequired(resolve(directory, "index.json"));
         const configuration = parseObject(rawConfiguration, `${collection}/${entry.name}/index.json`);

package/dist/runtime/agent.js

@@ -1740,7 +1740,16 @@ function normalizeDelegatedInput(input, projectRoot) {
     if (!content) {
         throw new Error("run_agent input must contain at least one valid content item.");
     }
-    return content;
+    if (content.some((item) => item.type === "text")) {
+        return content;
+    }
+    return [
+        {
+            type: "text",
+            text: "Process the attached content according to your agent instructions.",
+        },
+        ...content,
+    ];
 }
 function agentContentInputSchema() {
     const textItem = {

package/dist/types.d.ts

@@ -97,6 +97,11 @@ export interface AmarsiaTelemetryRunV1 {
     actions: AmarsiaTelemetryActionV1[];
     evaluation_count: number;
     evaluation_pass_count: number;
+    evaluation_failure_count: number;
+    evaluation_error_count: number;
+    evaluation_metric_count: number;
+    evaluation_metric_pass_count: number;
+    evaluations: AmarsiaTelemetryEvaluationV1[];
     error_code?: string;
     error_message?: string;
     error_retryable?: boolean;
@@ -111,6 +116,28 @@ export interface AmarsiaTelemetryActionV1 {
     error_code?: string;
     error_message?: string;
 }
+export interface AmarsiaTelemetryEvaluationMetricV1 {
+    name: string;
+    score: number;
+    threshold: number;
+    passed: boolean;
+    reasoning: string;
+    evidence: string[];
+}
+export interface AmarsiaTelemetryEvaluationV1 {
+    name: string;
+    status: "passed" | "failed" | "error";
+    provider?: string;
+    model?: string;
+    duration_ms: number;
+    input_tokens: number;
+    output_tokens: number;
+    reasoning_tokens: number;
+    cache_read_tokens: number;
+    cache_write_tokens: number;
+    metrics: AmarsiaTelemetryEvaluationMetricV1[];
+    error_message?: string;
+}
 export interface AmarsiaTelemetryBatchV1 {
     schema_version: 1;
     runs: AmarsiaTelemetryRunV1[];

package/dist-cjs/amarsia/analytics.js

@@ -289,6 +289,9 @@ function projectRunFact(manifest, sessionId, run, projection) {
   const evaluations = runEvents.filter(
     (event) => event.type === "evaluation.completed" || event.type === "evaluation.failed"
   );
+  const evaluationMetrics = evaluations.flatMap(
+    (event) => Array.isArray(event.data.metrics) ? event.data.metrics : []
+  );
   const error = asRecord(terminal?.data.error);
   const lastAssistant = assistants.at(-1);
   const status = terminal?.type === "run.completed" ? "completed" : terminal?.type === "run.failed" ? "failed" : terminal?.type === "run.paused" ? "paused" : "running";
@@ -331,6 +334,17 @@ function projectRunFact(manifest, sessionId, run, projection) {
     evaluation_pass_count: evaluations.filter(
       (event) => event.type === "evaluation.completed" && event.data.status === "passed"
     ).length,
+    evaluation_failure_count: evaluations.filter(
+      (event) => event.type === "evaluation.completed" && event.data.status === "failed"
+    ).length,
+    evaluation_error_count: evaluations.filter(
+      (event) => event.type === "evaluation.failed"
+    ).length,
+    evaluation_metric_count: evaluationMetrics.length,
+    evaluation_metric_pass_count: evaluationMetrics.filter(
+      (metric) => asRecord(metric)?.passed === true
+    ).length,
+    evaluations: projectEvaluationLogs(evaluations),
     ...typeof error?.code === "string" ? { error_code: error.code } : {},
     ...typeof error?.message === "string" ? { error_message: sanitizeErrorMessage(error.message) } : {},
     ...typeof error?.retryable === "boolean" ? { error_retryable: error.retryable } : {}
@@ -396,6 +410,38 @@ function projectActionLogs(events) {
     };
   });
 }
+function projectEvaluationLogs(events) {
+  return events.slice(0, 50).map((event) => {
+    const usage = asRecord(event.data.usage);
+    const error = asRecord(event.data.error);
+    const metrics = Array.isArray(event.data.metrics) ? event.data.metrics.map(asRecord).filter(
+      (metric) => metric !== void 0
+    ).slice(0, 50).map((metric) => ({
+      name: typeof metric.name === "string" ? metric.name : "unknown_metric",
+      score: numberValue(metric.score),
+      threshold: numberValue(metric.threshold),
+      passed: metric.passed === true,
+      reasoning: typeof metric.reasoning === "string" ? sanitizeErrorMessage(metric.reasoning) : "",
+      evidence: Array.isArray(metric.evidence) ? metric.evidence.filter(
+        (item) => typeof item === "string"
+      ).slice(0, 20).map(sanitizeErrorMessage) : []
+    })) : [];
+    return {
+      name: typeof event.data.name === "string" ? event.data.name : "unknown_evaluation",
+      status: event.type === "evaluation.failed" ? "error" : event.data.status === "passed" ? "passed" : "failed",
+      ...typeof event.data.provider === "string" ? { provider: event.data.provider } : {},
+      ...typeof event.data.model === "string" ? { model: event.data.model } : {},
+      duration_ms: numberValue(event.data.durationMs),
+      input_tokens: numberValue(usage?.inputTokens),
+      output_tokens: numberValue(usage?.outputTokens),
+      reasoning_tokens: numberValue(usage?.reasoningTokens),
+      cache_read_tokens: numberValue(usage?.cacheReadTokens),
+      cache_write_tokens: numberValue(usage?.cacheWriteTokens),
+      metrics,
+      ...typeof error?.message === "string" ? { error_message: sanitizeErrorMessage(error.message) } : {}
+    };
+  });
+}
 function takeBatch(queue) {
   const batch = [];
   let bytes = utf8ByteLength('{"schema_version":1,"runs":[]}');

package/dist-cjs/compiler/compile.js

@@ -215,13 +215,10 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
     return {};
   }
   const actions = {};
-  for (const entry of (0, import_fs.readdirSync)(actionsDirectory, { withFileTypes: true })) {
-    if (!entry.isDirectory()) {
-      continue;
-    }
+  for (const entry of discoverDirectories(actionsDirectory)) {
     const configurationPath = (0, import_path.resolve)(
       actionsDirectory,
-      entry.name,
+      entry,
       "index.json"
     );
     let configuration;
@@ -231,12 +228,12 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
       );
     } catch (error) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" is missing a valid index.json.`,
+        `Action "${agentName}/${entry}" is missing a valid index.json.`,
         { cause: error }
       );
     }
     validateEnabled(
-      `Action "${agentName}/${entry.name}"`,
+      `Action "${agentName}/${entry}"`,
       configuration?.enabled
     );
     if (configuration?.enabled === false) {
@@ -244,12 +241,12 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
     }
     if (!configuration.name?.trim() || !configuration.description?.trim()) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" requires name and description.`
+        `Action "${agentName}/${entry}" requires name and description.`
       );
     }
     if (!/^[a-zA-Z0-9_-]{1,64}$/.test(configuration.name)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" has an invalid model-facing name.`
+        `Action "${agentName}/${entry}" has an invalid model-facing name.`
       );
     }
     if ([
@@ -259,39 +256,39 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
       "inspect_agent"
     ].includes(configuration.name)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" uses reserved name "${configuration.name}".`
+        `Action "${agentName}/${entry}" uses reserved name "${configuration.name}".`
       );
     }
     const execution = configuration.execution ?? "local";
     if (!["client", "local"].includes(execution)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" has invalid execution "${configuration.execution}".`
+        `Action "${agentName}/${entry}" has invalid execution "${configuration.execution}".`
       );
     }
     if (!configuration.parameters || typeof configuration.parameters !== "object" || Array.isArray(configuration.parameters)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" requires a parameters JSON Schema.`
+        `Action "${agentName}/${entry}" requires a parameters JSON Schema.`
       );
     }
     if (configuration.outputSchema !== void 0 && (typeof configuration.outputSchema !== "object" || configuration.outputSchema === null || Array.isArray(configuration.outputSchema))) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" has an invalid outputSchema.`
+        `Action "${agentName}/${entry}" has an invalid outputSchema.`
       );
     }
     validateNumber(
-      `${agentName}/${entry.name}`,
+      `${agentName}/${entry}`,
       "timeoutMs",
       configuration.timeoutMs,
       { integer: true, minimum: 1, maximum: 12e4 }
     );
     if (configuration.permissions?.env !== void 0 && !Array.isArray(configuration.permissions.env)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" permissions.env must be an array.`
+        `Action "${agentName}/${entry}" permissions.env must be an array.`
       );
     }
     if (configuration.permissions?.network !== void 0 && !Array.isArray(configuration.permissions.network)) {
       throw new Error(
-        `Action "${agentName}/${entry.name}" permissions.network must be an array.`
+        `Action "${agentName}/${entry}" permissions.network must be an array.`
       );
     }
     for (const permission of [
@@ -300,7 +297,7 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
     ]) {
       if (typeof permission !== "string" || !permission.trim()) {
         throw new Error(
-          `Action "${agentName}/${entry.name}" has an invalid permission.`
+          `Action "${agentName}/${entry}" has an invalid permission.`
         );
       }
     }
@@ -311,7 +308,7 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
     }
     const compiledSource = execution === "local" ? compileLocalAction(
       agentName,
-      entry.name,
+      entry,
       actionsDirectory,
       actionRuntime,
       target
@@ -319,7 +316,7 @@ function compileActions(agentName, sourceDirectory, actionRuntime, target) {
     actions[configuration.name] = Object.freeze({
       ...configuration,
       execution,
-      directoryName: entry.name,
+      directoryName: entry,
       ...compiledSource
     });
   }
@@ -426,7 +423,9 @@ function compileEvaluations(agentName, sourceDirectory) {
   return Object.freeze(evaluations);
 }
 function discoverDirectories(directory) {
-  return (0, import_fs.readdirSync)(directory, { withFileTypes: true }).filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name).sort();
+  return (0, import_fs.readdirSync)(directory, { withFileTypes: true }).filter(
+    (entry) => !entry.name.startsWith(".") && (entry.isDirectory() || entry.isSymbolicLink() && (0, import_fs.statSync)((0, import_path.resolve)(directory, entry.name)).isDirectory())
+  ).map((entry) => entry.name).sort();
 }
 function rejectLegacyRegistry(agentName, directory, collection) {
   if ((0, import_fs.existsSync)((0, import_path.resolve)(directory, "index.js"))) {

package/dist-cjs/playground/source.js

@@ -102,8 +102,16 @@ async function readCollectionItems(agentDirectory, collection, manifests, includ
       { key, manifest }
     ])
   );
+  const directories = (await Promise.all(
+    entries.filter((entry) => !entry.name.startsWith(".")).map(async (entry) => ({
+      entry,
+      isDirectory: entry.isDirectory() || entry.isSymbolicLink() && (await (0, import_promises.stat)((0, import_path.resolve)(root, entry.name))).isDirectory()
+    }))
+  )).filter(({ isDirectory }) => isDirectory);
   return Promise.all(
-    entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).sort((left, right) => left.name.localeCompare(right.name)).map(async (entry) => {
+    directories.sort(
+      (left, right) => left.entry.name.localeCompare(right.entry.name)
+    ).map(async ({ entry }) => {
       const directory = (0, import_path.resolve)(root, entry.name);
       const rawConfiguration = await readRequired(
         (0, import_path.resolve)(directory, "index.json")

package/dist-cjs/runtime/agent.js

@@ -2024,7 +2024,16 @@ function normalizeDelegatedInput(input, projectRoot) {
       "run_agent input must contain at least one valid content item."
     );
   }
-  return content;
+  if (content.some((item) => item.type === "text")) {
+    return content;
+  }
+  return [
+    {
+      type: "text",
+      text: "Process the attached content according to your agent instructions."
+    },
+    ...content
+  ];
 }
 function agentContentInputSchema() {
   const textItem = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orchajs",
-  "version": "0.22.0",
+  "version": "0.23.1",
   "description": "A filesystem-convention, stateful agent framework for JavaScript.",
   "license": "MIT",
   "repository": {