failproofai 1.0.7-beta.1 → 1.0.7-beta.2

package/.next/standalone/sdk/typescript/integration/runtime-parity.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  describeTrace,
+  runAgentAsync,
+  scenarios,
+  traceViolations,
+  type Event,
+  type Format,
+  type RunResult,
+} from "./harness.js";
+
+/**
+ * Runtime parity: every scenario of every framework fixture, run under another
+ * runtime and under Node, must record the same trace.
+ *
+ * The framework files (`ai.test.ts`, …) say what a correct trace IS; this says
+ * only that Bun or Deno produce the one Node does. That keeps a runtime suite
+ * from restating — and drifting from — a hundred and eighty expectations, and
+ * it still catches every way a runtime differs that matters here: the adapter
+ * patched a copy the app does not run (no events), a hook never fired (a
+ * missing pair), the exit flush never ran (a short trace), a warning Node does
+ * not print.
+ */
+
+export const FRAMEWORK_FIXTURES = [
+  "ai-4",
+  "ai-5",
+  "ai-6",
+  "ai-7",
+  "langchain-0.3",
+  "langchain-1",
+  "mastra-0",
+  "mastra-1",
+  "llamaindex-0.11",
+  "llamaindex-0.12",
+] as const;
+
+/**
+ * The fields of an event that are a property of the RUN, not of the moment:
+ * no timestamps, durations, generated ids or session ids.
+ */
+const STABLE = [
+  "type",
+  "agent_id",
+  "parent_id",
+  "tool_name",
+  "hook_name",
+  "trigger_event",
+  "model",
+  "input_tokens",
+  "output_tokens",
+  "stop_reason",
+  "tool_call_id",
+  "outcome",
+  "framework",
+] as const;
+
+/** A generated id (a tool call with no model-assigned id gets a UUID). */
+const GENERATED = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/**
+ * Each session's events in order, sessions sorted. Two sessions run
+ * concurrently (`shared-query`) interleave by scheduling, which is a property
+ * of the event loop and not of the SDK; the order WITHIN a session is not.
+ */
+export const digest = (events: Event[], unordered = false): string[][] => {
+  const sessions = new Map<string, Event[]>();
+  for (const event of events) sessions.set(event.session_id, [...(sessions.get(event.session_id) ?? []), event]);
+  return [...sessions.values()]
+    .map((session) => (unordered ? digestSession(session).sort() : digestSession(session)))
+    .sort((a, b) => a.join().localeCompare(b.join()));
+};
+
+/**
+ * Scenarios that run several agents at once INSIDE ONE session (ten
+ * `generate`s on one agent, say). Their events interleave by scheduling within
+ * the session too, so for these the comparison is the same MULTISET of events
+ * per session rather than the same sequence — still exact about what was
+ * recorded, silent only about an order no runtime (Node included) repeats.
+ */
+const UNORDERED = /^concurrent/;
+
+/**
+ * Scenarios not compared at all, with the reason. Each is one its own
+ * framework suite already skips as a documented limit, where the trace is not
+ * a stable property of the run on any runtime.
+ */
+const SKIPPED: Record<string, string> = {
+  "mastra-0:tripwire-output":
+    "Mastra 0.24 gives no end signal for an output-processor tripwire on a stream, so the run is left open (skipped in mastra-coverage.test.ts)",
+};
+
+const digestSession = (events: Event[]): string[] =>
+  events.map((event) =>
+    JSON.stringify(
+      Object.fromEntries(
+        STABLE.filter((key) => key in event).map((key) => {
+          const value = event[key];
+          return [key, typeof value === "string" && GENERATED.test(value) ? "<generated>" : value];
+        }),
+      ),
+    ),
+  );
+
+/** The SDK's own diagnostics, without the parts that name paths or numbers. */
+export const sdkLines = (stderr: string): string[] =>
+  stderr
+    .split("\n")
+    .filter((line) => line.includes("[failproofai-sdk]"))
+    .map((line) => line.replace(/\/[^\s"')]+/g, "<path>").replace(/\d+/g, "N"));
+
+/**
+ * Labels in an agent's `switch` that are not scenarios: each is one half of a
+ * scenario that spawns it with arguments of its own, and is exercised through
+ * that scenario — under the runtime being tested, since the parent re-executes
+ * `process.execPath`. Run bare, they only exercise argument handling (and Deno,
+ * whose `writeFileSync(undefined)` does not throw, writes a file named
+ * `undefined` into the fixture).
+ */
+const HELPERS: Record<string, string> = {
+  "remote-resume-pause": "the pausing process of remote-resume, which passes it a checkpoint path",
+};
+
+export interface Divergence {
+  /** Why this scenario is allowed to differ, stated as the observed behaviour. */
+  reason: string;
+}
+
+/**
+ * One `describe` per fixture and module system; one concurrent case per
+ * scenario. `known` lists scenarios whose difference is understood and is NOT
+ * the SDK's (each is asserted to still differ, so a fix upstream shows up),
+ * keyed `fixture:format:scenario` with `*` for either of the first two.
+ */
+export function parity(
+  label: string,
+  pairs: ReadonlyArray<readonly [node: Format, other: Format]>,
+  known: Record<string, Divergence> = {},
+): void {
+  describe.each(FRAMEWORK_FIXTURES)(`%s under ${label}`, (fixture) => {
+    describe.each(pairs)("%s vs %s", (nodeFormat, otherFormat) => {
+      it.concurrent.each(
+        scenarios(fixture).filter((name) => !(name in HELPERS) && !(`${fixture}:${name}` in SKIPPED)),
+      )("%s", async (scenario) => {
+        const [node, other] = await Promise.all([
+          runAgentAsync(fixture, nodeFormat, scenario),
+          runAgentAsync(fixture, otherFormat, scenario),
+        ]);
+        const key = `${fixture}:${otherFormat}:${scenario}`;
+        const unordered = UNORDERED.test(scenario);
+        const matches = same(node, other, unordered);
+        const divergence =
+          known[key] ??
+          known[`${fixture}:*:${scenario}`] ??
+          known[`*:${otherFormat}:${scenario}`] ??
+          known[`*:*:${scenario}`];
+        if (divergence !== undefined) {
+          expect(matches, `${key} is listed as a known divergence (${divergence.reason}) but now matches Node`).toBe(
+            false,
+          );
+          return;
+        }
+        const context = `NODE ${nodeFormat}\n${describeTrace(node)}\n\n${otherFormat.toUpperCase()}\n${describeTrace(other)}`;
+        expect(other.status, context).toBe(node.status);
+        expect(digest(other.events, unordered), context).toEqual(digest(node.events, unordered));
+        expect(traceViolations(other.events), context).toEqual(traceViolations(node.events));
+        expect(sdkLines(other.stderr), context).toEqual(sdkLines(node.stderr));
+      });
+    });
+  });
+}
+
+function same(a: RunResult, b: RunResult, unordered: boolean): boolean {
+  return (
+    a.status === b.status &&
+    JSON.stringify(digest(a.events, unordered)) === JSON.stringify(digest(b.events, unordered)) &&
+    JSON.stringify(sdkLines(a.stderr)) === JSON.stringify(sdkLines(b.stderr))
+  );
+}

package/.next/standalone/sdk/typescript/integration/runtimes.bun.test.ts

@@ -0,0 +1,15 @@
+import { parity } from "./runtime-parity.js";
+
+/**
+ * Bun: every scenario of every framework fixture, as an ES module and as
+ * CommonJS, against Node's trace for the same agent.
+ *
+ * Bun implements `node:module`'s `createRequire`, `require.cache`,
+ * `require.main`, `AsyncLocalStorage` and `process.on("exit")`, which is
+ * everything the SDK's copy resolution and exit flush rely on — this proves it
+ * on real frameworks instead of assuming it.
+ */
+parity("bun", [
+  ["esm", "bun-esm"],
+  ["cjs", "bun-cjs"],
+]);

package/.next/standalone/sdk/typescript/integration/runtimes.core.test.ts

@@ -0,0 +1,113 @@
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import {
+  BUN_FORMATS,
+  DENO_FORMATS,
+  FIXTURES,
+  FORMATS,
+  count,
+  describeTrace,
+  runAgentAsync,
+  runProcess,
+  runtimeCommand,
+  traceViolations,
+  type Event,
+} from "./harness.js";
+
+/**
+ * The SDK with no framework, under every runtime it can be imported in with a
+ * filesystem — Node, Bun, Deno — as an ES module and as CommonJS.
+ *
+ * Agent: `fixtures/runtimes/agent.ts`.
+ */
+
+const ALL = [...FORMATS, ...BUN_FORMATS, ...DENO_FORMATS];
+
+/** All 15 event types, each exactly as often as `core()` emits it. */
+const CORE_COUNTS: Record<string, number> = {
+  agent_start: 2,
+  agent_end: 2,
+  model_request: 2,
+  model_response: 2,
+  tool_use: 2,
+  tool_result: 2,
+  hook_triggered: 1,
+  hook_completed: 1,
+  agent_pause: 1,
+  agent_resume: 1,
+  human_wait: 1,
+  human_input: 1,
+  human_pause: 1,
+  human_interrupt: 1,
+  error: 1,
+};
+
+const shape = (events: Event[]) => events.map((e) => `${e.agent_id} ${e.type}`);
+
+/**
+ * `traceViolations`, less the one rule this program breaks by design: a
+ * hand-emitted `model_response` carries no `duration_ms` (the SDK measures only
+ * the pairs it can see open — tools, hooks, pauses, human input — exactly like
+ * the Python SDK). The adapters are what supply model timings.
+ */
+const violations = (events: Event[]) =>
+  traceViolations(events).filter((problem) => !problem.endsWith("has no duration_ms"));
+
+describe.each(ALL)("as %s", (format) => {
+  const run = (scenario: string) => runAgentAsync("runtimes", format, scenario);
+
+  it("records every scope and all 15 event methods, and flush() writes them", async () => {
+    const result = await run("core");
+    expect(result.status, describeTrace(result)).toBe(0);
+    expect(violations(result.events), describeTrace(result)).toEqual([]);
+    const counts = Object.fromEntries(Object.keys(CORE_COUNTS).map((type) => [type, count(result.events, type)]));
+    expect(counts, describeTrace(result)).toEqual(CORE_COUNTS);
+    expect(result.events).toHaveLength(21);
+    for (const event of result.events) expect(event.session_id).toBe("core-session");
+    const helper = result.events.find((e) => e.type === "agent_start" && e.agent_id === "helper")!;
+    expect(helper.parent_id).toBe("planner");
+    expect(result.stderr, describeTrace(result)).not.toContain("[failproofai-sdk]");
+  });
+});
+
+describe("Deno with npm: specifiers", () => {
+  const GEN = [
+    "weather-agent agent_start",
+    "weather-agent model_request",
+    "weather-agent model_response",
+    "weather-agent tool_use",
+    "weather-agent tool_result",
+    "weather-agent model_request",
+    "weather-agent model_response",
+    "weather-agent agent_end",
+  ];
+  const run = (scenario: string) =>
+    runProcess([...runtimeCommand("deno"), "deno-npm.ts", scenario], {
+      cwd: join(FIXTURES, "runtimes"),
+      label: "deno-npm",
+    });
+
+  // wrapModel() sees model calls only — the tools run outside the model — so
+  // its trace is GEN without the tool pair, exactly as `ai.test.ts` expects.
+  const WRAPPED = GEN.filter((line) => !line.includes(" tool_"));
+
+  it.each([
+    ["telemetry", GEN],
+    ["wrap", WRAPPED],
+  ])("records an ai tool loop through %s()", async (scenario, expected) => {
+    const result = await run(scenario);
+    expect(result.status, describeTrace(result)).toBe(0);
+    expect(result.stdout).toContain("It is 20C in Paris.");
+    expect(shape(result.events), describeTrace(result)).toEqual(expected);
+    expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+    expect(
+      result.events.filter((e) => e.type === "model_response").map((e) => [e.input_tokens, e.output_tokens]),
+    ).toEqual([
+      [11, 7],
+      [23, 9],
+    ]);
+    expect(result.stderr, describeTrace(result)).not.toContain("[failproofai-sdk]");
+  });
+});

package/.next/standalone/sdk/typescript/integration/runtimes.deno.test.ts

@@ -0,0 +1,19 @@
+import { parity } from "./runtime-parity.js";
+
+/**
+ * Deno (`deno run --allow-all`), running each fixture's transpiled agent
+ * straight out of the fixture's `node_modules` — Deno 2's npm compatibility,
+ * with the frameworks resolved exactly as Node resolves them.
+ *
+ * This file found the SDK's one Deno bug: Deno answers `require.main === null`
+ * (not `undefined`) for an ES-module entry, which the SDK read as "CommonJS",
+ * so `instrument()` patched the CommonJS copy of LangChain, Mastra and
+ * LlamaIndex while the app ran the ES-module copy — success reported, nothing
+ * recorded. See `isCommonJsMain` in `src/node-require.ts`.
+ *
+ * `npm:` specifiers and the core SDK under Deno are in `runtimes.core.test.ts`.
+ */
+parity("deno", [
+  ["esm", "deno-esm"],
+  ["cjs", "deno-cjs"],
+]);

package/.next/standalone/sdk/typescript/integration/types.test.ts

@@ -0,0 +1,141 @@
+import { spawnSync } from "node:child_process";
+import { closeSync, copyFileSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { describe, expect, inject, it } from "vitest";
+
+import { FIXTURES } from "./harness.js";
+
+/**
+ * The packed tarball's type declarations, as every kind of TypeScript project
+ * sees them.
+ *
+ * The SDK's own tsconfig is `nodenext` ESM, the one setup in which a dual
+ * package whose `exports` hand ESM declarations to `require` looks fine. Every
+ * other setup a customer can carry was broken by exactly that, and silently
+ * from here:
+ *
+ *   * a CommonJS project on `module: node16` (or `nodenext` before TS 5.8) got
+ *     TS1479 — "the referenced file is an ECMAScript module" — on every import;
+ *   * `moduleResolution: node` (node10, still what `module: commonjs` implies,
+ *     e.g. every NestJS project) resolved the root and no subpath at all.
+ *
+ * So `agent.ts` — importing the root and every public subpath, using the names
+ * in ways a wrong or `any` type would fail — is compiled under each consumer
+ * mode, on the newest TypeScript and on the oldest one this package supports,
+ * with `skipLibCheck: false` so the declarations themselves are checked too.
+ * `@arethetypeswrong/cli` then runs over the same tarball: it checks JavaScript
+ * and declarations AGREE on module format, which no compile of ours can see.
+ *
+ * Minimum TypeScript: 5.4. It is the oldest release in which every mode below
+ * exists (`module: preserve` arrived in 5.4). The declarations themselves need
+ * only a type for `Symbol.dispose` (the `using` scopes), which `@types/node`
+ * supplies, as does `lib: esnext.disposable` from TS 5.2 — measured: 5.2
+ * passes every mode it has; 5.0 passes them only with `skipLibCheck`, because
+ * `@types/node` 22 itself no longer compiles there.
+ */
+
+const FIXTURE = "types";
+const DIR = join(FIXTURES, FIXTURE);
+
+interface Mode {
+  /** The consumer file's extension, which fixes its module format under node16/nodenext. */
+  ext: "ts" | "mts" | "cts";
+  module: string;
+  moduleResolution: string;
+  /** Written as the mode directory's package.json `type`; absent = CommonJS. */
+  type?: "module";
+}
+
+const MODES: Record<string, Mode> = {
+  "ESM nodenext": { ext: "ts", module: "nodenext", moduleResolution: "nodenext", type: "module" },
+  "CJS .cts nodenext": { ext: "cts", module: "nodenext", moduleResolution: "nodenext" },
+  "CJS node16": { ext: "ts", module: "node16", moduleResolution: "node16" },
+  "CJS commonjs + node10": { ext: "ts", module: "commonjs", moduleResolution: "node" },
+  "esnext + bundler": { ext: "ts", module: "esnext", moduleResolution: "bundler" },
+  "preserve + bundler": { ext: "ts", module: "preserve", moduleResolution: "bundler" },
+};
+
+/** The installed compiler packages, and the release each must be. */
+const COMPILERS = { typescript: "5.9.3", "typescript-min": "5.4.5" } as const;
+
+const ENTRYPOINTS = [".", "./ai", "./mastra", "./langchain", "./llamaindex", "./next", "./evaluator", "./sandbox-worker"];
+
+function writeMode(name: string, mode: Mode): string {
+  const dir = join(DIR, ".run", "types", name.replace(/[^a-z0-9]+/gi, "-").toLowerCase());
+  rmSync(dir, { recursive: true, force: true });
+  mkdirSync(dir, { recursive: true });
+  const file = `agent.${mode.ext}`;
+  copyFileSync(join(DIR, "agent.ts"), join(dir, file));
+  // Every mode gets a package.json, so the fixture's own cannot decide the
+  // format of a mode that expects the other one.
+  writeFileSync(join(dir, "package.json"), JSON.stringify(mode.type ? { type: mode.type } : {}));
+  writeFileSync(
+    join(dir, "tsconfig.json"),
+    JSON.stringify({
+      compilerOptions: {
+        target: "ES2022",
+        module: mode.module,
+        moduleResolution: mode.moduleResolution,
+        strict: true,
+        noEmit: true,
+        skipLibCheck: false,
+        types: ["node"],
+      },
+      files: [file],
+    }),
+  );
+  return dir;
+}
+
+function tsc(compiler: string, project: string): string {
+  const bin = join(DIR, "node_modules", compiler, "bin", "tsc");
+  const result = spawnSync(process.execPath, [bin, "-p", project], { cwd: project, encoding: "utf8" });
+  return (result.stdout + result.stderr).trim();
+}
+
+describe.each(Object.entries(COMPILERS))("TypeScript %s", (compiler, release) => {
+  it(`is ${release}`, () => {
+    const manifest = JSON.parse(readFileSync(join(DIR, "node_modules", compiler, "package.json"), "utf8"));
+    expect(manifest.version).toBe(release);
+  });
+
+  it.each(Object.entries(MODES))("typechecks every entry point as a %s project", (name, mode) => {
+    expect(tsc(compiler, writeMode(`${compiler}-${name}`, mode))).toBe("");
+  });
+});
+
+describe("@arethetypeswrong/cli", () => {
+  it("finds no problem in any entry point under any resolution mode", () => {
+    const attw = join(DIR, "node_modules", "@arethetypeswrong", "cli", "dist", "index.js");
+    // Into a file, not a pipe: attw calls process.exit() straight after
+    // writing, which truncates a large report written to a pipe.
+    const out = join(DIR, ".run", "attw.json");
+    mkdirSync(dirname(out), { recursive: true });
+    const fd = openSync(out, "w");
+    let result;
+    try {
+      result = spawnSync(process.execPath, [attw, inject("tarball"), "--format", "json"], {
+        cwd: DIR,
+        encoding: "utf8",
+        stdio: ["ignore", fd, "pipe"],
+      });
+    } finally {
+      closeSync(fd);
+    }
+    const report = JSON.parse(readFileSync(out, "utf8")) as {
+      analysis: {
+        entrypoints: Record<string, { resolutions: Record<string, unknown> }>;
+        problems: unknown[];
+      };
+    };
+    expect(Object.keys(report.analysis.entrypoints).filter((e) => e !== "./package.json").sort()).toEqual(
+      [...ENTRYPOINTS].sort(),
+    );
+    for (const [entry, { resolutions }] of Object.entries(report.analysis.entrypoints)) {
+      expect(Object.keys(resolutions).sort(), entry).toEqual(["bundler", "node10", "node16-cjs", "node16-esm"]);
+    }
+    expect(report.analysis.problems).toEqual([]);
+    expect(result.status, result.stderr).toBe(0);
+  });
+});