@devosurf/tesser-testing 0.1.0-alpha.0

package/src/invoke-action.ts

@@ -0,0 +1,105 @@
+// Direct action invocation for connector tests (and the connector-authoring skill):
+// build a real ActionCtx over a fake/recorded fetch and run one action with full
+// input/output validation — the same path the runtime takes.
+
+import type { ActionCtx, ConnectorInstance } from "@devosurf/tesser-sdk/connector";
+import {
+  actionAtPath,
+  createHttpClient,
+  runAction,
+  type TesserHttpConfig,
+} from "@devosurf/tesser-sdk/internal";
+
+export interface InvokeActionOptions {
+  /** Fake fetch (or a cassette-backed one). Defaults to real fetch — live tests only. */
+  fetchImpl?: typeof fetch;
+  /** Credential fields for ctx.auth + custom signers. Defaults to a dummy api key. */
+  auth?: { kind?: "oauth2" | "apiKey" | "basic" | "custom"; mode?: string; fields?: Record<string, string> };
+  /** Override the connector/provider base URL. */
+  baseUrl?: string;
+  idempotencyKey?: string;
+  /** Apply the credential to outbound requests; defaults to Authorization: Bearer. */
+  applyAuth?: TesserHttpConfig["applyAuth"];
+}
+
+export async function invokeAction(
+  connector: ConnectorInstance<any, any>,
+  path: string,
+  input: unknown,
+  opts: InvokeActionOptions = {},
+): Promise<unknown> {
+  const segments = path.split(".");
+  const def = actionAtPath(connector, segments);
+  if (!def) {
+    throw new TypeError(`invokeAction: connector "${connector.id}" has no action "${path}"`);
+  }
+  const spec = connector.__connector;
+  const baseUrl =
+    opts.baseUrl ??
+    spec.baseUrl ??
+    (typeof spec.provider === "object" ? spec.provider.baseUrl : undefined);
+  const fields = opts.auth?.fields ?? { api_key: "test-key" };
+  const http = createHttpClient({
+    ...(baseUrl !== undefined ? { baseUrl } : {}),
+    ...(opts.fetchImpl !== undefined ? { fetchImpl: opts.fetchImpl } : {}),
+    ...(spec.defaultHeaders !== undefined ? { defaultHeaders: spec.defaultHeaders } : {}),
+    applyAuth:
+      opts.applyAuth ??
+      (({ headers }) => {
+        const token = fields["access_token"] ?? fields["api_key"] ?? fields["token"];
+        if (token !== undefined) headers.set("authorization", `Bearer ${token}`);
+      }),
+    ...(def.classifyError !== undefined ? { classifyError: def.classifyError } : {}),
+  });
+  const ctx: ActionCtx = {
+    http,
+    auth: { kind: opts.auth?.kind ?? "apiKey", fields, ...(opts.auth?.mode !== undefined ? { mode: opts.auth.mode } : {}) },
+    logger: { info() {}, warn() {}, error() {} },
+    ...(opts.idempotencyKey !== undefined ? { idempotencyKey: opts.idempotencyKey } : {}),
+  };
+  return runAction(def, ctx, input, `${connector.id}.${path}`);
+}
+
+export interface FakeRequest {
+  url: URL;
+  method: string;
+  headers: Headers;
+  body: string;
+}
+// Not `unknown` — a union with unknown collapses and kills contextual typing of handlers.
+type FakeRouteResult = Response | string | number | boolean | null | object;
+export type FakeRoute = FakeRouteResult | ((req: FakeRequest) => FakeRouteResult | Promise<FakeRouteResult>);
+
+/** A fetch stub from a route table: "METHOD /path" (with :param segments) → response. */
+export function fakeFetch(
+  routes: Record<string, FakeRoute>,
+): typeof fetch & { requests: Array<{ method: string; url: string; body: string; headers: Headers }> } {
+  const requests: Array<{ method: string; url: string; body: string; headers: Headers }> = [];
+  const impl = (async (input: unknown, init?: { method?: string; headers?: unknown; body?: unknown }) => {
+    const url = new URL(String(input instanceof Request ? input.url : input));
+    const method = (init?.method ?? "GET").toUpperCase();
+    const body = typeof init?.body === "string" ? init.body : "";
+    const headers = new Headers(init?.headers as never);
+    requests.push({ method, url: url.toString(), body, headers });
+    for (const [route, handler] of Object.entries(routes)) {
+      const [m, p] = route.split(" ", 2) as [string, string];
+      const matches =
+        m.toUpperCase() === method &&
+        (p === url.pathname || new RegExp(`^${p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/:\w+/g, "[^/]+")}$`).test(url.pathname));
+      if (matches) {
+        const result = typeof handler === "function" ? await handler({ url, method, headers, body }) : handler;
+        if (result instanceof Response) return result;
+        return new Response(JSON.stringify(result), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        });
+      }
+    }
+    return new Response(JSON.stringify({ error: `no fake route for ${method} ${url.pathname}` }), {
+      status: 404,
+      headers: { "content-type": "application/json" },
+    });
+  }) as never as typeof fetch & { requests: typeof requests };
+  (impl as { requests: typeof requests }).requests = requests;
+  return impl;
+}

package/src/sample.ts

@@ -0,0 +1,205 @@
+// Schema-driven sample + edge-case generation (ADR-0008). Works through JSON Schema:
+// any validator that can describe itself as JSON Schema participates; with zod we use
+// its built-in conversion. Everything degrades gracefully to "ask for an explicit mock".
+
+import type { Schema } from "@devosurf/tesser-sdk";
+
+type JsonSchema = {
+  type?: string | string[];
+  properties?: Record<string, JsonSchema>;
+  required?: string[];
+  items?: JsonSchema | JsonSchema[];
+  prefixItems?: JsonSchema[];
+  enum?: unknown[];
+  const?: unknown;
+  anyOf?: JsonSchema[];
+  oneOf?: JsonSchema[];
+  allOf?: JsonSchema[];
+  nullable?: boolean;
+  default?: unknown;
+  minimum?: number;
+  maximum?: number;
+  exclusiveMinimum?: number;
+  exclusiveMaximum?: number;
+  minLength?: number;
+  maxLength?: number;
+  minItems?: number;
+  format?: string;
+  pattern?: string;
+  additionalProperties?: JsonSchema | boolean;
+  [k: string]: unknown;
+};
+
+/** Best-effort JSON Schema for a Standard Schema. Currently: zod (vendor "zod") via its
+ *  own toJSONSchema; otherwise undefined. */
+export function toJsonSchema(schema: Schema<unknown>): JsonSchema | undefined {
+  const std = (schema as { "~standard"?: { vendor?: string } })["~standard"];
+  if (!std) return undefined;
+  try {
+    if (std.vendor === "zod") {
+      const z = schema as unknown as {
+        constructor?: unknown;
+      };
+      // zod v4 exposes conversion as a static on the module; reach it through the
+      // schema's own registry-free helper if present.
+      const anySchema = z as { toJSONSchema?: () => unknown };
+      if (typeof anySchema.toJSONSchema === "function") {
+        return anySchema.toJSONSchema() as JsonSchema;
+      }
+      // Fall back to importing zod lazily — present whenever a zod schema is in play.
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      return undefined;
+    }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+/** Async variant that can lazily import zod for conversion. */
+export async function toJsonSchemaAsync(schema: Schema<unknown>): Promise<JsonSchema | undefined> {
+  const direct = toJsonSchema(schema);
+  if (direct) return direct;
+  const std = (schema as { "~standard"?: { vendor?: string } })["~standard"];
+  if (std?.vendor === "zod") {
+    try {
+      const zod = (await import("zod")) as unknown as {
+        toJSONSchema?: (s: unknown, opts?: unknown) => unknown;
+        z?: { toJSONSchema?: (s: unknown, opts?: unknown) => unknown };
+      };
+      const convert = zod.toJSONSchema ?? zod.z?.toJSONSchema;
+      if (convert) return convert(schema, { unrepresentable: "any" }) as JsonSchema;
+    } catch {
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+const SAMPLE_STRINGS: Record<string, string> = {
+  email: "sample@example.com",
+  uri: "https://example.com/sample",
+  url: "https://example.com/sample",
+  uuid: "00000000-0000-4000-8000-000000000000",
+  "date-time": "2026-01-01T00:00:00.000Z",
+  date: "2026-01-01",
+  time: "00:00:00",
+};
+
+export function sampleFromJsonSchema(js: JsonSchema, depth = 0): unknown {
+  if (depth > 8) return null;
+  if (js.default !== undefined) return js.default;
+  if (js.const !== undefined) return js.const;
+  if (js.enum && js.enum.length > 0) return js.enum[0];
+  const variants = js.anyOf ?? js.oneOf;
+  if (variants && variants.length > 0) {
+    // Prefer a non-null variant for a useful sample.
+    const pick = variants.find((v) => v.type !== "null") ?? variants[0];
+    return sampleFromJsonSchema(pick as JsonSchema, depth + 1);
+  }
+  if (js.allOf && js.allOf.length > 0) {
+    return js.allOf.reduce<Record<string, unknown>>((acc, part) => {
+      const piece = sampleFromJsonSchema(part, depth + 1);
+      return typeof piece === "object" && piece !== null
+        ? { ...acc, ...(piece as Record<string, unknown>) }
+        : acc;
+    }, {});
+  }
+  const type = Array.isArray(js.type) ? js.type[0] : js.type;
+  switch (type) {
+    case "string": {
+      if (js.format && SAMPLE_STRINGS[js.format]) return SAMPLE_STRINGS[js.format];
+      const min = js.minLength ?? 0;
+      const base = "sample";
+      return base.length >= min ? base : base + "x".repeat(min - base.length);
+    }
+    case "number":
+    case "integer": {
+      let n = 1;
+      if (js.minimum !== undefined) n = js.minimum;
+      if (js.exclusiveMinimum !== undefined) n = js.exclusiveMinimum + 1;
+      if (js.maximum !== undefined && n > js.maximum) n = js.maximum;
+      return type === "integer" ? Math.round(n) : n;
+    }
+    case "boolean":
+      return true;
+    case "null":
+      return null;
+    case "array": {
+      const itemSchema = Array.isArray(js.items) ? js.items[0] : js.items;
+      const count = Math.max(js.minItems ?? 1, 1);
+      if (!itemSchema) return [];
+      return Array.from({ length: count }, () => sampleFromJsonSchema(itemSchema, depth + 1));
+    }
+    case "object":
+    default: {
+      if (js.properties) {
+        const out: Record<string, unknown> = {};
+        const required = new Set(js.required ?? Object.keys(js.properties));
+        for (const [key, prop] of Object.entries(js.properties)) {
+          if (required.has(key) || prop.default !== undefined) {
+            out[key] = sampleFromJsonSchema(prop, depth + 1);
+          }
+        }
+        return out;
+      }
+      if (type === "object" || type === undefined) return {};
+      return null;
+    }
+  }
+}
+
+/** A sample value satisfying the schema, or undefined when it cannot be derived. */
+export async function sampleFromSchema(schema: Schema<unknown>): Promise<unknown | undefined> {
+  const js = await toJsonSchemaAsync(schema);
+  if (!js) return undefined;
+  return sampleFromJsonSchema(js);
+}
+
+export interface EdgeCase {
+  name: string;
+  value: unknown;
+  /** Whether the schema should accept it — invalid cases assert the unhappy path. */
+  expectValid: boolean;
+}
+
+/** Boundary / malformed inputs from a typed contract (ADR-0008, built incrementally). */
+export async function edgeCasesFromSchema(schema: Schema<unknown>): Promise<EdgeCase[]> {
+  const js = await toJsonSchemaAsync(schema);
+  if (!js) return [];
+  const valid = sampleFromJsonSchema(js);
+  const cases: EdgeCase[] = [{ name: "minimal valid sample", value: valid, expectValid: true }];
+  cases.push({ name: "null payload", value: null, expectValid: (js.type ?? "object") === "null" });
+  cases.push({
+    name: "wrong primitive (number)",
+    value: 42,
+    expectValid: js.type === "number" || js.type === "integer",
+  });
+
+  if (js.type === "object" && js.properties && typeof valid === "object" && valid !== null) {
+    for (const key of js.required ?? []) {
+      const clone = { ...(valid as Record<string, unknown>) };
+      delete clone[key];
+      cases.push({ name: `missing required "${key}"`, value: clone, expectValid: false });
+    }
+    for (const [key, prop] of Object.entries(js.properties)) {
+      const t = Array.isArray(prop.type) ? prop.type[0] : prop.type;
+      if (t === "string" && (js.required ?? []).includes(key)) {
+        const minOk = (prop.minLength ?? 0) === 0;
+        cases.push({
+          name: `empty string "${key}"`,
+          value: { ...(valid as Record<string, unknown>), [key]: "" },
+          expectValid: minOk && prop.format === undefined && prop.pattern === undefined,
+        });
+      }
+      if ((t === "number" || t === "integer") && prop.minimum !== undefined) {
+        cases.push({
+          name: `below minimum "${key}"`,
+          value: { ...(valid as Record<string, unknown>), [key]: prop.minimum - 1 },
+          expectValid: false,
+        });
+      }
+    }
+  }
+  return cases;
+}

package/src/smoke.ts

@@ -0,0 +1,78 @@
+// The auto-generated default smoke test (ADR-0008): fire the trigger with a sample
+// payload, auto-mock connectors, assert completion + output schema. Zero-config green/red
+// the moment an automation is written — `tesser test` synthesizes one per automation
+// that lacks a colocated test.
+
+import type { AutomationDef, OperatorDef } from "@devosurf/tesser-sdk";
+import { executeAutomation, type RunOptions, type TestRunResult } from "./engine.js";
+import { sampleFromSchema } from "./sample.js";
+
+export interface SmokeOutcome {
+  automation: string;
+  passed: boolean;
+  result: TestRunResult;
+  /** Machine-actionable reason when red (ADR-0008). */
+  reason?: string;
+}
+
+export async function smokeTest(def: AutomationDef<any, any, any, any, any, any, any>): Promise<SmokeOutcome> {
+  try {
+    const models = await smokeModelScripts(def);
+    const result = await executeAutomation(def, Object.keys(models).length > 0 ? { models } : {});
+    if (result.status === "completed") {
+      return { automation: def.id, passed: true, result };
+    }
+    return {
+      automation: def.id,
+      passed: false,
+      result,
+      reason:
+        result.error?.step !== undefined
+          ? `step "${result.error.step}" failed: ${result.error.message}`
+          : (result.error?.message ?? "run failed"),
+    };
+  } catch (err) {
+    // TestConfigError (e.g. an un-derivable sample, a signal wait with no timeout) is a
+    // legitimate smoke failure: the agent gets told exactly what to provide.
+    return {
+      automation: def.id,
+      passed: false,
+      result: {
+        status: "failed",
+        steps: {},
+        journal: [],
+        calls: {},
+        emitted: [],
+        slept: [],
+        undone: [],
+        logs: [],
+        error: { name: (err as Error).name, message: (err as Error).message, retryable: false, terminal: true },
+        failure: () => ({
+          automation: def.id,
+          status: "failed",
+          error: { name: (err as Error).name, message: (err as Error).message, retryable: false, terminal: true },
+          steps: [],
+          connectorCalls: [],
+        }),
+      },
+      reason: (err as Error).message,
+    };
+  }
+}
+
+async function smokeModelScripts(
+  def: AutomationDef<any, any, any, any, any, any, any>,
+): Promise<NonNullable<RunOptions["models"]>> {
+  const models: NonNullable<RunOptions["models"]> = {};
+  const operators = (def.operators ?? {}) as Record<string, OperatorDef>;
+  for (const [operatorKey, op] of Object.entries(operators)) {
+    const sample = await sampleFromSchema(op.output).catch(() => undefined);
+    (models[operatorKey] ??= {})[op.model] = {
+      ...(sample !== undefined ? { output: sample as never } : { content: "{}" }),
+      usage: { inputTokens: 1, outputTokens: 1 },
+      provider: "tesser-smoke",
+      model: op.model,
+    };
+  }
+  return models;
+}

package/src/spy.ts

@@ -0,0 +1,62 @@
+// A minimal mock-function record compatible with vitest/jest `expect(...).toHaveBeenCalled*`
+// matchers (they accept anything exposing _isMockFunction + .mock.calls). Hand-rolled so
+// @devosurf/tesser-testing does not depend on a test runner.
+
+export interface CapturedCall {
+  args: unknown[];
+  /** Step that was active when the call happened. */
+  step?: string;
+  action?: string;
+  result?: unknown;
+  error?: string;
+}
+
+export interface Spy {
+  (...args: unknown[]): void;
+  _isMockFunction: true;
+  getMockName(): string;
+  mock: {
+    calls: unknown[][];
+    results: Array<{ type: "return" | "throw"; value: unknown }>;
+    instances: unknown[];
+    contexts: unknown[];
+    invocationCallOrder: number[];
+    lastCall: unknown[] | undefined;
+  };
+  /** Rich capture (step/action attribution) beyond the matcher contract. */
+  captured: CapturedCall[];
+}
+
+let order = 1;
+
+export function createSpy(name: string): Spy {
+  const mock: Spy["mock"] = {
+    calls: [],
+    results: [],
+    instances: [],
+    contexts: [],
+    invocationCallOrder: [],
+    lastCall: undefined,
+  };
+  const captured: CapturedCall[] = [];
+  const fn = ((...args: unknown[]) => {
+    mock.calls.push(args);
+    mock.invocationCallOrder.push(order++);
+    mock.lastCall = args;
+  }) as Spy;
+  fn._isMockFunction = true;
+  fn.getMockName = () => name;
+  fn.mock = mock;
+  fn.captured = captured;
+  return fn;
+}
+
+export function recordCall(spy: Spy, call: CapturedCall): void {
+  spy(...call.args);
+  spy.mock.results.push(
+    call.error !== undefined
+      ? { type: "throw", value: call.error }
+      : { type: "return", value: call.result },
+  );
+  spy.captured.push(call);
+}