@zackbart/connecta 0.10.1 → 0.10.2

package/src/execute.ts

@@ -34,6 +34,145 @@ import type {
 const EXECUTE_MAX_HOST_CALLS = 20;
 export const EXECUTE_MAX_BATCH_CALLS = 10;
 const EXECUTE_HOST_CALL_TIMEOUT_MS = 15_000;
+const diagnosticsEncoder = new TextEncoder();
+
+type ExecuteDiagnosticOperation = "search" | "describe" | "call" | "batch";
+
+interface ExecuteOperationDiagnostics {
+  operation: ExecuteDiagnosticOperation;
+  count: number;
+  failures: number;
+  durationMs: number;
+  resultBytes: number;
+  catalogMs: number;
+  connectorMs: number;
+  calls?: number;
+}
+
+type MutableOperationDiagnostics = ExecuteOperationDiagnostics;
+
+class ExecuteDiagnostics {
+  private readonly started = Date.now();
+  private readonly operations = new Map<
+    ExecuteDiagnosticOperation,
+    MutableOperationDiagnostics
+  >();
+  admissionMs = 0;
+  setupMs = 0;
+  executorWallMs = 0;
+
+  private stats(operation: ExecuteDiagnosticOperation) {
+    let stats = this.operations.get(operation);
+    if (!stats) {
+      stats = {
+        operation,
+        count: 0,
+        failures: 0,
+        durationMs: 0,
+        resultBytes: 0,
+        catalogMs: 0,
+        connectorMs: 0,
+      };
+      this.operations.set(operation, stats);
+    }
+    return stats;
+  }
+
+  recordCatalog(
+    operation: "search" | "describe",
+    durationMs: number,
+    ok: boolean,
+    result?: unknown,
+  ): void {
+    const stats = this.stats(operation);
+    stats.count++;
+    stats.failures += ok ? 0 : 1;
+    stats.durationMs += durationMs;
+    stats.catalogMs += durationMs;
+    if (ok) stats.resultBytes += serializedDiagnosticBytes(result);
+  }
+
+  recordCall(
+    operation: "call" | "batch",
+    outcome: {
+      ok: boolean;
+      durationMs: number;
+      timing: { catalogMs: number; connectorMs: number };
+      value?: unknown;
+    },
+  ): void {
+    const stats = this.stats(operation);
+    if (operation === "call") {
+      stats.count++;
+      if (outcome.ok) {
+        stats.resultBytes += serializedDiagnosticBytes(outcome.value);
+      }
+    } else {
+      stats.calls = (stats.calls ?? 0) + 1;
+    }
+    stats.failures += outcome.ok ? 0 : 1;
+    if (operation === "call") stats.durationMs += outcome.durationMs;
+    stats.catalogMs += outcome.timing.catalogMs;
+    stats.connectorMs += outcome.timing.connectorMs;
+  }
+
+  recordBatch(
+    durationMs: number,
+    ok: boolean,
+    calls: number,
+    result?: unknown,
+  ): void {
+    const stats = this.stats("batch");
+    stats.count++;
+    // Calls normally accrue while each child runs. Invalid batch input never
+    // starts children, so retain the attempted cardinality here.
+    if (!ok) stats.calls = Math.max(stats.calls ?? 0, calls);
+    stats.durationMs += durationMs;
+    if (!ok) stats.failures++;
+    else stats.resultBytes += serializedDiagnosticBytes(result);
+  }
+
+  finish(): {
+    timing: {
+      totalMs: number;
+      admissionMs: number;
+      setupMs: number;
+      executorWallMs: number;
+      catalogMs: number;
+      connectorMs: number;
+    };
+    operations: ExecuteOperationDiagnostics[];
+  } {
+    const operations = [...this.operations.values()];
+    return {
+      timing: {
+        totalMs: Date.now() - this.started,
+        admissionMs: this.admissionMs,
+        setupMs: this.setupMs,
+        executorWallMs: this.executorWallMs,
+        catalogMs: operations.reduce((sum, item) => sum + item.catalogMs, 0),
+        connectorMs: operations.reduce(
+          (sum, item) => sum + item.connectorMs,
+          0,
+        ),
+      },
+      operations,
+    };
+  }
+}
+
+function serializedDiagnosticBytes(value: unknown): number {
+  try {
+    const text = JSON.stringify(value);
+    return text === undefined
+      ? 0
+      : diagnosticsEncoder.encode(text).byteLength;
+  } catch {
+    // The executor's normal result guard owns the error. Diagnostics must
+    // never turn measurement into a second failure path.
+    return 0;
+  }
+}
 
 // deno-fmt-ignore
 const RESERVED = new Set([
@@ -154,6 +293,7 @@ export async function buildSandboxProviders(
     hostCallTimeoutMs?: number;
     discoveryConcurrency?: number;
     onInvocationFailure?: (failure: InvocationFailure) => void;
+    diagnostics?: ExecuteDiagnostics;
   } = {},
 ): Promise<ExecutorProvider[]> {
   // All host calls made by one execute_code invocation share a downstream
@@ -231,12 +371,17 @@ export async function buildSandboxProviders(
       throw err;
     }
   };
-  const callAddress = async (address: unknown, args: unknown) => {
+  const callAddress = async (
+    address: unknown,
+    args: unknown,
+    diagnosticOperation: "call" | "batch" = "call",
+  ) => {
     const outcome = await invocation.invoke(
       String(address),
       args ?? {},
       invocationContext(),
     );
+    limits.diagnostics?.recordCall(diagnosticOperation, outcome);
     if (!outcome.ok) {
       const failure = new InvocationFailure(outcome.error);
       limits.onInvocationFailure?.(failure);
@@ -256,6 +401,7 @@ export async function buildSandboxProviders(
       args ?? {},
       invocationContext(),
     );
+    limits.diagnostics?.recordCall("call", outcome);
     if (!outcome.ok) {
       const failure = new InvocationFailure(outcome.error);
       limits.onInvocationFailure?.(failure);
@@ -270,83 +416,143 @@ export async function buildSandboxProviders(
       prelude: lazyNamespacePrelude(namespaces),
       fns: {
         __callNamespace: callNamespace,
-        call: callAddress,
+        call: (address: unknown, args: unknown) =>
+          callAddress(address, args),
         batch: async (calls: unknown) => {
-          if (!Array.isArray(calls)) throw new Error("calls must be an array");
-          if (calls.length > EXECUTE_MAX_BATCH_CALLS) {
-            throw new Error(
-              `connecta.batch accepts at most ${EXECUTE_MAX_BATCH_CALLS} calls`,
+          const started = Date.now();
+          const callCount = Array.isArray(calls) ? calls.length : 0;
+          try {
+            if (!Array.isArray(calls)) throw new Error("calls must be an array");
+            if (calls.length > EXECUTE_MAX_BATCH_CALLS) {
+              throw new Error(
+                `connecta.batch accepts at most ${EXECUTE_MAX_BATCH_CALLS} calls`,
+              );
+            }
+            const result = await Promise.all(
+              calls.map(async (call) => {
+                const item = call as { address?: unknown; args?: unknown };
+                try {
+                  return {
+                    address: String(item.address),
+                    ok: true,
+                    data: await callAddress(
+                      item.address,
+                      item.args,
+                      "batch",
+                    ),
+                  };
+                } catch (err) {
+                  // Same failure shape batch_call reports: the message a program
+                  // can log, plus the typed details it must classify by. A
+                  // thrown host error crosses the sandbox bridge as a bare
+                  // message string in every executor, so this is the one place a
+                  // program can tell a policy refusal from a transient failure.
+                  const details =
+                    err instanceof InvocationFailure
+                      ? err.details
+                      : classifyCallError(err, "batch_call_failed");
+                  return {
+                    address: String(item.address),
+                    ok: false,
+                    error: details.message,
+                    errorDetails: details,
+                  };
+                }
+              }),
+            );
+            limits.diagnostics?.recordBatch(
+              Date.now() - started,
+              true,
+              callCount,
+              result,
+            );
+            return result;
+          } catch (err) {
+            limits.diagnostics?.recordBatch(
+              Date.now() - started,
+              false,
+              callCount,
             );
+            throw err;
           }
-          return await Promise.all(
-            calls.map(async (call) => {
-              const item = call as { address?: unknown; args?: unknown };
-              try {
-                return {
-                  address: String(item.address),
-                  ok: true,
-                  data: await callAddress(item.address, item.args),
-                };
-              } catch (err) {
-                // Same failure shape batch_call reports: the message a program
-                // can log, plus the typed details it must classify by. A
-                // thrown host error crosses the sandbox bridge as a bare
-                // message string in every executor, so this is the one place a
-                // program can tell a policy refusal from a transient failure.
-                const details =
-                  err instanceof InvocationFailure
-                    ? err.details
-                    : classifyCallError(err, "batch_call_failed");
-                return {
-                  address: String(item.address),
-                  ok: false,
-                  error: details.message,
-                  errorDetails: details,
-                };
-              }
-            }),
-          );
         },
-        search: async (raw: unknown) =>
-          typedDiscovery(async () => {
-            const args = (raw ?? {}) as {
-              query?: string;
-              connector?: string;
-              limit?: number;
-              offset?: number;
-              fullDescriptions?: boolean;
-              includeSchemas?: "compact" | "json";
-              includeSchemaKeys?: boolean;
-            };
-            const result = flatSearchResult(
-              await catalog.search({
-                ...args,
-                // Key metadata rides along with schemas by default, since that
-                // is the whole point of it in code mode. It stays opt-out
-                // because it counts against the same discovery-byte ceiling.
-                includeSchemaKeys: args.includeSchemaKeys !== false,
-              }),
-            );
-            boundedDiscoveryText(
+        search: async (raw: unknown) => {
+          const started = Date.now();
+          try {
+            const result = await typedDiscovery(async () => {
+              const args = (raw ?? {}) as {
+                query?: string;
+                connector?: string;
+                safety?: "readOnly" | "approvalRequired" | "all";
+                limit?: number;
+                offset?: number;
+                fullDescriptions?: boolean;
+                includeSchemas?: "compact" | "json";
+                includeSchemaKeys?: boolean;
+              };
+              const result = flatSearchResult(
+                await catalog.search({
+                  ...args,
+                  // Key metadata rides along with schemas by default, since that
+                  // is the whole point of it in code mode. It stays opt-out
+                  // because it counts against the same discovery-byte ceiling.
+                  includeSchemaKeys: args.includeSchemaKeys !== false,
+                }),
+              );
+              boundedDiscoveryText(
+                result,
+                "Request a smaller limit, omit fullDescriptions, use compact schemas, or pass includeSchemaKeys: false.",
+              );
+              return result;
+            });
+            limits.diagnostics?.recordCatalog(
+              "search",
+              Date.now() - started,
+              true,
               result,
-              "Request a smaller limit, omit fullDescriptions, use compact schemas, or pass includeSchemaKeys: false.",
             );
             return result;
-          }),
-        describe: async (raw: unknown) =>
-          typedDiscovery(async () => {
-            const args = (raw ?? {}) as {
-              addresses?: unknown;
-              format?: "compact" | "json";
-              fullDescriptions?: boolean;
-            };
-            const result = { tools: await catalog.describe(args) };
-            boundedDiscoveryText(
+          } catch (err) {
+            limits.diagnostics?.recordCatalog(
+              "search",
+              Date.now() - started,
+              false,
+            );
+            throw err;
+          }
+        },
+        describe: async (raw: unknown) => {
+          const started = Date.now();
+          try {
+            const result = await typedDiscovery(async () => {
+              const args = (raw ?? {}) as {
+                addresses?: unknown;
+                format?: "compact" | "json";
+                fullDescriptions?: boolean;
+              };
+              const result = { tools: await catalog.describe(args) };
+              boundedDiscoveryText(
+                result,
+                'Split the address list or use format: "compact".',
+              );
+              return result;
+            });
+            limits.diagnostics?.recordCatalog(
+              "describe",
+              Date.now() - started,
+              true,
               result,
-              'Split the address list or use format: "compact".',
             );
             return result;
-          }),
+          } catch (err) {
+            limits.diagnostics?.recordCatalog(
+              "describe",
+              Date.now() - started,
+              false,
+            );
+            throw err;
+          }
+        },
       },
     },
   ];
@@ -362,7 +568,10 @@ export function createExecuteTool(
   config: { discoveryConcurrency?: number } = {},
 ) {
   return async (
-    { code }: { code: string },
+    { code, diagnostics: diagnosticsRequested }: {
+      code: string;
+      diagnostics?: boolean;
+    },
     options: { signal?: AbortSignal } = {},
   ): Promise<ToolResult> => {
     const controller = new AbortController();
@@ -373,42 +582,64 @@ export function createExecuteTool(
     }
     let lease;
     let outcome;
+    const diagnostics = diagnosticsRequested ? new ExecuteDiagnostics() : undefined;
     const invocationFailures: InvocationFailure[] = [];
     try {
       // Admission comes before provider construction: queued calls retain no
       // catalogs, request scopes, or one-closure-per-tool provider arrays.
       if (isAdmittingExecutor(executor)) {
-        lease = await executor.acquire({ signal: controller.signal });
+        const admissionStarted = Date.now();
+        try {
+          lease = await executor.acquire({ signal: controller.signal });
+        } finally {
+          if (diagnostics) {
+            diagnostics.admissionMs = Date.now() - admissionStarted;
+          }
+        }
         if ((lease.waitMs ?? 0) > 0) {
           logger.debug("[connecta] execute_code admitted after queue wait", {
             waitMs: lease.waitMs,
           });
         }
       }
-      const providers = await buildSandboxProviders(
-        registry,
-        baseUrl,
-        logger,
-        activity,
-        {
-          signal: controller.signal,
-          onInvocationFailure: (failure) => {
-            invocationFailures.push(failure);
+      const setupStarted = Date.now();
+      let providers: ExecutorProvider[];
+      try {
+        providers = await buildSandboxProviders(
+          registry,
+          baseUrl,
+          logger,
+          activity,
+          {
+            signal: controller.signal,
+            onInvocationFailure: (failure) => {
+              invocationFailures.push(failure);
+            },
+            ...(diagnostics ? { diagnostics } : {}),
+            ...(config.discoveryConcurrency !== undefined
+              ? { discoveryConcurrency: config.discoveryConcurrency }
+              : {}),
           },
-          ...(config.discoveryConcurrency !== undefined
-            ? { discoveryConcurrency: config.discoveryConcurrency }
-            : {}),
-        },
-      );
+        );
+      } finally {
+        if (diagnostics) diagnostics.setupMs = Date.now() - setupStarted;
+      }
       if (controller.signal.aborted) {
         throw new ExecutorAdmissionError(
           "executor_cancelled",
           "Execution was cancelled during sandbox setup.",
         );
       }
-      outcome = lease
-        ? await lease.execute(code, providers)
-        : await executor.execute(code, providers);
+      const executorStarted = Date.now();
+      try {
+        outcome = lease
+          ? await lease.execute(code, providers)
+          : await executor.execute(code, providers);
+      } finally {
+        if (diagnostics) {
+          diagnostics.executorWallMs = Date.now() - executorStarted;
+        }
+      }
     } catch (err) {
       if (err instanceof ExecutorAdmissionError) {
         if (err.code === "executor_overloaded") {
@@ -426,6 +657,19 @@ export function createExecuteTool(
               ? { retryAfterMs: err.retryAfterMs }
               : {}),
           },
+          ...(diagnostics ? { diagnostics: diagnostics.finish() } : {}),
+        });
+        result.isError = true;
+        return result;
+      }
+      if (diagnostics) {
+        const result = jsonResult({
+          error: {
+            code: "executor_failed",
+            message: `Executor failed: ${msg(err)}`,
+            retryable: false,
+          },
+          diagnostics: diagnostics.finish(),
         });
         result.isError = true;
         return result;
@@ -476,13 +720,26 @@ export function createExecuteTool(
         const result = jsonResult({
           error: invocationFailure.details,
           ...(logs ? { logs } : {}),
+          ...(diagnostics ? { diagnostics: diagnostics.finish() } : {}),
         });
         result.isError = true;
         return result;
       }
-      return errorResult(
-        `Error: ${outcome.error}${logs ? `\n\nLogs:\n${logs}` : ""}`,
-      );
+      const message = `Error: ${outcome.error}`;
+      if (diagnostics) {
+        const result = jsonResult({
+          error: {
+            code: "executor_failed",
+            message,
+            retryable: false,
+          },
+          ...(logs ? { logs } : {}),
+          diagnostics: diagnostics.finish(),
+        });
+        result.isError = true;
+        return result;
+      }
+      return errorResult(`${message}${logs ? `\n\nLogs:\n${logs}` : ""}`);
     }
     // A result crossing back as a host BigInt (or otherwise unserializable
     // value) makes JSON.stringify throw — keep that inside the structured
@@ -491,13 +748,26 @@ export function createExecuteTool(
     try {
       result = guardExecuteResultValue(outcome.result);
     } catch (err) {
-      return errorResult(
-        `Error: result is not JSON-serializable: ${msg(err)}${logs ? `\n\nLogs:\n${logs}` : ""}`,
-      );
+      const message = `Error: result is not JSON-serializable: ${msg(err)}`;
+      if (diagnostics) {
+        const response = jsonResult({
+          error: {
+            code: "executor_failed",
+            message,
+            retryable: false,
+          },
+          ...(logs ? { logs } : {}),
+          diagnostics: diagnostics.finish(),
+        });
+        response.isError = true;
+        return response;
+      }
+      return errorResult(`${message}${logs ? `\n\nLogs:\n${logs}` : ""}`);
     }
     return jsonResult({
       result,
       ...(logs ? { logs } : {}),
+      ...(diagnostics ? { diagnostics: diagnostics.finish() } : {}),
     });
   };
 }
@@ -529,13 +799,13 @@ const executeDescription = (
 Write an async arrow function. It runs with NO network, filesystem, timers, or imports — the only capabilities are:
 - One global per connector: every address <connectorId>.<toolName> from search_tools is callable as <connectorId>.<toolName>(args) with a single args object matching the schema from ${EXECUTE_SCHEMA_SOURCE[surface]}. Names are sanitized to JS identifiers: characters outside [A-Za-z0-9_$] become "_" (e.g. my-service.get.thing → my_service.get_thing), leading digits get "_" prefixed, reserved words get "_" appended.
 - connecta.call(address, args) and connecta.batch(calls) — call raw addresses.
-- connecta.search(args) and connecta.describe(args) — load and inspect request-local catalogs on demand. Matches carrying schemas also list inputKeys, requiredInputKeys, and outputKeys — the same names the schema shows, ready to check against before building args. They are absent when a schema is not a plain object shape, so read the schema itself rather than assuming a missing list means no fields.
+- connecta.search(args) and connecta.describe(args) — load and inspect request-local catalogs on demand. Use safety: "readOnly" to avoid advertising calls this sandbox cannot execute; the filter changes results, not authority. Matches carrying schemas also list inputKeys, requiredInputKeys, and outputKeys — the same names the schema shows, ready to check against before building args. They are absent when a schema is not a plain object shape, so read the schema itself rather than assuming a missing list means no fields.
 - console.log(...) — captured and returned alongside the result.
 
 Tool calls return plain values (MCP text content is JSON-parsed when possible) and throw on downstream errors — use try/catch to handle them. A thrown error carries only a message; connecta.batch reports each call as { address, ok: true, data } or { address, ok: false, error, errorDetails: { code, retryable } }, so use it when the program must tell a policy refusal from a transient failure. Never retry a failure whose retryable is false, and never retry a rate_limited one immediately — the sandbox has no timers. Return a JSON-serializable value; large results are truncated, so reduce data in code instead of returning raw payloads.
 
 Plain JavaScript only — no TypeScript syntax. For unknown-address dependent work, use one execute_code call: search inside it, read the compact schemas, and continue to the dependent calls; do not return search results for a second execute_code call. Compact schemas are TypeScript-like strings, not JSON Schema objects: write the property names they display, never a positional guess or an invented alias.
-Dependent example (only when the second call requires a value returned by the first): async () => { const { tools } = await connecta.search({ query: "pipeline run job logs", includeSchemas: "compact" }); const pick = (suffix) => { const match = tools.find((tool) => tool.address.endsWith(suffix)); if (!match) throw new Error("no tool matching " + suffix); return match.address; }; const run = await connecta.call(pick(".get_run"), { runId: 42 }); const logs = await connecta.call(pick(".get_job_logs"), { jobId: run.failedJobId }); return [run, logs]; }`;
+Dependent example (only when the second call requires a value returned by the first): async () => { const { tools } = await connecta.search({ query: "pipeline run job logs", safety: "readOnly", includeSchemas: "compact" }); const pick = (suffix) => { const match = tools.find((tool) => tool.address.endsWith(suffix)); if (!match) throw new Error("no tool matching " + suffix); return match.address; }; const run = await connecta.call(pick(".get_run"), { runId: 42 }); const logs = await connecta.call(pick(".get_job_logs"), { jobId: run.failedJobId }); return [run, logs]; }`;
 
 /** Register the execute_code meta-tool. Only called when an executor is configured. */
 export function registerExecuteTool(
@@ -570,6 +840,12 @@ export function registerExecuteTool(
         code: z
           .string()
           .describe("A JavaScript async arrow function to execute."),
+        diagnostics: z
+          .boolean()
+          .optional()
+          .describe(
+            "Add request-local, payload-free timing and result-size summaries.",
+          ),
       }),
       // The sandbox exposes only tools that are explicitly read-only, and the
       // executor grants no network, filesystem, env, or timer capabilities.
@@ -591,7 +867,7 @@ export function registerExecuteTool(
         return { signal, forward };
       });
       try {
-        return await handler(args as { code: string }, {
+        return await handler(args as { code: string; diagnostics?: boolean }, {
           signal: controller.signal,
         });
       } finally {

package/src/invocation.ts

@@ -18,6 +18,7 @@ import {
 } from "./errors.js";
 import { unwrapMcpResult } from "./mcp-result.js";
 import type { RegistryView } from "./registry.js";
+import { isExplicitlyReadOnly } from "./tool-safety.js";
 import type { ToolDef } from "./types.js";
 
 /**
@@ -55,14 +56,6 @@ export function retryBackoffMs(
   return retryAfterMs <= MAX_RETRY_BACKOFF_MS ? retryAfterMs : undefined;
 }
 
-/** The one fail-closed admission predicate for every invocation adapter. */
-function isExplicitlyReadOnly(definition: ToolDef): boolean {
-  return (
-    definition.annotations?.readOnlyHint === true &&
-    definition.annotations?.destructiveHint !== true
-  );
-}
-
 function retrySafe(definition: ToolDef): boolean {
   return (
     definition.annotations?.readOnlyHint === true ||