@codemation/agent-skills 0.1.9 → 0.2.0

package/skills/codemation-workflow-dsl/references/complete-example.md

@@ -0,0 +1,263 @@
+Load this when you need to see a complete workflow that exercises most authoring features end-to-end.
+
+## The dense example (manual trigger — full fluent sugar)
+
+The fluent `.map`/`.if`/`.switch`/`.split`/`.agent`/`.node` helpers are only available after `.manualTrigger(...)`. The example below is a manual-trigger workflow so it can demonstrate all of them. For cron / webhook variants, see the snippet at the bottom.
+
+```ts
+// src/workflows/dailyCsvDigest.ts
+//
+// Theme: a manual-triggered "daily CSV digest". Caller passes { date: "YYYY-MM-DD" }.
+// The flow fetches that day's sales CSV from a reporting API, parses each row,
+// classifies rows with an LLM agent, and sends a per-row digest email.
+//
+// Register in codemation.config.ts:
+//   import dailyCsvDigest from "./src/workflows/dailyCsvDigest";
+//   workflows: [dailyCsvDigest]
+
+import { z } from "zod";
+import { callableTool, itemExpr } from "@codemation/core";
+import { HttpRequest } from "@codemation/core-nodes";
+import { workflow } from "@codemation/host";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+type TriggerInput = { date: string }; // e.g. "2025-05-14"
+
+type FetchMeta = {
+  url: string;
+  ok: boolean;
+  status: number;
+  binarySlot: string;
+};
+
+type CsvRow = {
+  region: string;
+  product: string;
+  revenue: number;
+  anomaly: boolean;
+};
+
+type ClassifiedRow = CsvRow & {
+  classification: "normal" | "warning" | "critical";
+  rationale: string;
+};
+
+// ---------------------------------------------------------------------------
+// Inline callable tool — classify a single row
+// ---------------------------------------------------------------------------
+
+const classifyRowTool = callableTool({
+  name: "classify_row",
+  description: "Classify a revenue row as normal, warning, or critical.",
+  inputSchema: z.object({
+    region: z.string(),
+    product: z.string(),
+    revenue: z.number(),
+    anomaly: z.boolean(),
+  }),
+  outputSchema: z.object({
+    classification: z.enum(["normal", "warning", "critical"]),
+    rationale: z.string(),
+  }),
+  execute: async ({ input }) => {
+    // Fallback executor if the agent doesn't call the tool — keeps the workflow deterministic in tests.
+    const classification =
+      input.anomaly || input.revenue < 0 ? "critical" : input.revenue < 1000 ? "warning" : "normal";
+    return { classification, rationale: `Revenue ${input.revenue}, anomaly=${input.anomaly}` };
+  },
+});
+
+// ---------------------------------------------------------------------------
+// Workflow
+// ---------------------------------------------------------------------------
+
+export default workflow("wf.daily-csv-digest")
+  .name("Daily CSV Digest")
+  // Manual trigger seeded with a default date — callers can override at run time.
+  .manualTrigger<TriggerInput>("Start", { date: "2025-05-14" })
+
+  // ── Step 1: build the fetch URL ────────────────────────────────────────────
+  // async .map — use when you need await (date math here is sync, but the API call below is async).
+  .map("Build fetch URL", async (item, _ctx) => ({
+    date: item.json.date,
+    reportUrl: `https://reports.internal/sales/${item.json.date}.csv`,
+  }))
+
+  // HttpRequest with responseFormat:"binary" stores the body in ctx.binary automatically.
+  // Explicit id "fetch-report" keeps the credential binding stable across label renames.
+  .then(
+    new HttpRequest("Fetch report CSV", {
+      id: "fetch-report",
+      urlField: "reportUrl",
+      responseFormat: "binary",
+      responseBinarySlot: "csvFile",
+      credentialSlot: "reportApi",
+    }),
+  )
+
+  // ── Step 2: gate on HTTP success ───────────────────────────────────────────
+  // .if predicate receives (item, ctx). Use for fast boolean branches; .switch is overkill for two outcomes.
+  .if((item: { json: FetchMeta }, _ctx) => item.json.ok, {
+    true: (branch) =>
+      branch
+        // ── Step 3: parse CSV from binary ──────────────────────────────────────
+        // async .map — needs await to read from binary storage.
+        .map("Parse CSV rows", async (item: { json: FetchMeta }, ctx) => {
+          const stream = await ctx.binary.openReadStream(item.json.binarySlot);
+          const text = await streamToText(stream);
+          const rows = parseCsv(text);
+          return { rows, fetchedAt: item.json.url };
+        })
+
+        // .split emits one item per CSV row so downstream steps run per-row.
+        .split("Split rows", (item: { json: { rows: CsvRow[] } }) => item.json.rows)
+
+        // ── Step 4: classify each row with an agent ──────────────────────────
+        // itemExpr defers message construction to per-item runtime — required when content depends on the current item.
+        .agent("Classify row", {
+          model: "openai:gpt-4o-mini",
+          messages: itemExpr(({ item }: { item: { json: CsvRow } }) => [
+            { role: "system" as const, content: "You are a revenue analyst. Use classify_row." },
+            { role: "user" as const, content: JSON.stringify(item.json) },
+          ]),
+          tools: [classifyRowTool],
+          outputSchema: z.object({
+            classification: z.enum(["normal", "warning", "critical"]),
+            rationale: z.string(),
+          }),
+        })
+
+        // ── Step 5: merge agent output with the original row via ctx.data ──────
+        // ctx.data is keyed by node id (the slug of the node label).
+        // "Split rows" slugs to "split-rows"; we read its emitted item back here.
+        // sync .map — pure object merge, no I/O.
+        .map("Enrich classification", (item: { json: { classification: string; rationale: string } }, ctx) => {
+          const originalRow = ctx.data["split-rows"]?.items?.[0]?.json as CsvRow | undefined;
+          return {
+            ...originalRow,
+            classification: item.json.classification as ClassifiedRow["classification"],
+            rationale: item.json.rationale,
+          } satisfies Partial<ClassifiedRow>;
+        })
+
+        // ── Step 6: send digest email via a registered node ───────────────────
+        // .node(name, config, options) — explicit id keeps credential binding stable.
+        // SendEmailNodeConfig is illustrative; replace with the email node available in your project.
+        .node(
+          "Send digest email",
+          new SendEmailNodeConfig({
+            // itemExpr on a config field — engine resolves once per item at execution time.
+            subject: itemExpr(
+              ({ item }: { item: { json: Partial<ClassifiedRow> } }) =>
+                `[${item.json.classification?.toUpperCase()}] ${item.json.region} – ${item.json.product}`,
+            ),
+            to: "ops-team@example.com",
+            body: itemExpr(
+              ({ item }: { item: { json: Partial<ClassifiedRow> } }) =>
+                `Region: ${item.json.region}\nRevenue: ${item.json.revenue}\nRationale: ${item.json.rationale}`,
+            ),
+          }),
+          { id: "send-digest-email" },
+        ),
+
+    false: (branch) =>
+      branch.map("Log fetch failure", (item: { json: FetchMeta }, _ctx) => ({
+        error: `Fetch failed: HTTP ${item.json.status}`,
+        url: item.json.url,
+      })),
+  })
+
+  // .build() validates non-empty + unique node ids (including agent connection children).
+  // Throws WorkflowDefinitionError on violation.
+  .build();
+
+// ---------------------------------------------------------------------------
+// Helpers (inline for brevity — promote to lib/ if reused)
+// ---------------------------------------------------------------------------
+
+async function streamToText(stream: AsyncIterable<Uint8Array>): Promise<string> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) chunks.push(Buffer.from(chunk));
+  return Buffer.concat(chunks).toString("utf-8");
+}
+
+function parseCsv(text: string): CsvRow[] {
+  const [header, ...lines] = text.trim().split("\n");
+  const cols = header!.split(",");
+  return lines.map((line) => {
+    const vals = line.split(",");
+    return {
+      region: vals[cols.indexOf("region")] ?? "",
+      product: vals[cols.indexOf("product")] ?? "",
+      revenue: Number(vals[cols.indexOf("revenue")] ?? 0),
+      anomaly: vals[cols.indexOf("anomaly")] === "true",
+    };
+  });
+}
+```
+
+## What this exercises
+
+- **Manual trigger with typed default item** → `workflow("...").manualTrigger<TriggerInput>("Start", {...})`
+- **sync `.map`** → "Enrich classification" — pure object merge, no `await`
+- **async `.map`** → "Build fetch URL" and "Parse CSV rows" — uses `await` for binary read
+- **`.if` per-item predicate** → `(item, _ctx) => item.json.ok` with branch factories
+- **`HttpRequest` with explicit `id:`** → `id: "fetch-report"` (credential binding stability)
+- **`.split`** → fan-out one batch into many items
+- **`.agent(...)` with `messages`, `model`, `tools`, `outputSchema`** → typed structured output
+- **`callableTool` with Zod schemas and `execute({ input })`** → inline tool definition
+- **`itemExpr(...)`** → on agent messages (per-item content) and on `.node` config fields (per-item subject/body)
+- **`.node(name, config, options)` with explicit id** → stable credential binding
+- **`ctx.data["<slug>"]`** → reading earlier node output without threading it through every step
+- **`ctx.binary.openReadStream(slot)`** → reading bytes from a binary slot attached upstream
+- **`.build()`** → final validation pass
+
+## Cron / webhook variant (alternative trigger)
+
+When the trigger isn't manual, the fluent `.map`/`.if`/`.agent` sugar isn't available — you use the lower-level builder and `.then(new SomeNodeConfig(...))`. Shape:
+
+```ts
+import { Callback, CronTrigger, createWorkflowBuilder, HttpRequest } from "@codemation/core-nodes";
+
+export default createWorkflowBuilder({
+  id: "wf.daily-csv-digest.cron",
+  name: "Daily CSV Digest (cron)",
+})
+  .trigger(new CronTrigger("Daily 06:00", { schedule: "0 6 * * *", timezone: "UTC" }))
+  // Cron fires one item per tick: { firedAt, scheduledFor }. Wrap downstream logic in Callback configs:
+  .then(
+    new Callback("Build fetch URL", (items, _ctx) => {
+      return items.map((item) => {
+        const date = new Date((item.json as { scheduledFor: string }).scheduledFor).toISOString().slice(0, 10);
+        return { date, reportUrl: `https://reports.internal/sales/${date}.csv` };
+      });
+    }),
+  )
+  .then(
+    new HttpRequest("Fetch report CSV", {
+      id: "fetch-report",
+      urlField: "reportUrl",
+      responseFormat: "binary",
+      responseBinarySlot: "csvFile",
+      credentialSlot: "reportApi",
+    }),
+  )
+  // For branching, use `new If(...)`. For per-item agent calls, use `new AIAgent({...})`.
+  // For row fan-out, use `new Split(...)`. The execution semantics match the fluent helpers
+  // — only the surface syntax differs.
+  .build();
+```
+
+If you need both cron + the fluent sugar in the same workflow, you can wrap the cursor manually:
+
+```ts
+import { WorkflowChain } from "@codemation/core-nodes";
+
+const cursor = createWorkflowBuilder({ id, name }).trigger(new CronTrigger("Tick", { schedule: "..." }));
+export default new WorkflowChain(cursor).map("First step", (item) => ({ ...item.json })).build();
+```
+
+This is uncommon in production code; reach for it only when the fluent helpers genuinely help readability.

package/skills/codemation-workflow-dsl/references/workflow-testing.md

@@ -0,0 +1,194 @@
+# Workflow Testing
+
+## Use this reference when
+
+You are authoring or reviewing a workflow that needs **end-to-end tests**: validate agent behavior, regression-test branching, score LLM outputs over time, or assert that a workflow produces the expected output for a known set of inputs.
+
+This is **not** for unit-testing individual nodes — use `WorkflowTestKit` from `@codemation/core/testing` for that.
+
+## Three building blocks
+
+1. **`TestTrigger`** — drops on the canvas alongside live triggers (Webhook / Cron / Gmail / etc.). Authored callback yields one item per test case.
+2. **`IsTestRun`** — per-item router with `true` / `false` ports. Branches based on whether the run was started by the test orchestrator.
+3. **`Assertion`** — generic per-item assertion node; returns one or more `AssertionResult`s per input item, one persisted `TestAssertion` row per result.
+
+## Typical workflow shape
+
+```
+[GmailTrigger: new email] ──┐
+                            │
+[TestTrigger: 10 fixtures]──┴─→ [ClassifyAgent]
+                                      │
+                                [IsTestRun?]
+                                  │   │
+                              true│   │false
+                                  ↓   ↓
+                          [Assertion]  [SendReply] (real side effect — skipped in tests)
+```
+
+## Authoring a TestTrigger
+
+```ts
+import { TestTrigger } from "@codemation/core-nodes";
+import { gmailCredentialType, type GmailSession } from "@codemation/core-nodes-gmail";
+
+export const fixtureMailsTrigger = new TestTrigger<{ subject: string; body: string }>({
+  name: "Email fixtures",
+  credentialRequirements: [
+    { slotKey: "gmail", label: "Gmail", acceptedTypes: [gmailCredentialType.definition.typeId] },
+  ],
+  async *generateItems(ctx) {
+    const gmail = await ctx.getCredential<GmailSession>("gmail");
+    const messages = await gmail.listMessages({ labelIds: ["Label_test_mails"] });
+    for (const message of messages) {
+      if (ctx.signal.aborted) break;
+      yield { json: { subject: message.subject, body: message.body } };
+    }
+  },
+  concurrency: 8, // optional; default 4
+  caseLabel: (item) => item.json.subject, // optional; rows fall back to runId
+});
+```
+
+Notes:
+
+- `triggerKind: "test"` is set automatically — `TriggerRuntimeService` skips it during live activation.
+- `ctx.signal` is an `AbortSignal` raised when the suite is cancelled; long pulls should bail out.
+- For hardcoded fixtures, just `yield { json: { ... } }` — no need to use credentials.
+- Set `caseLabel` so the Tests-tab tree-table shows something readable instead of opaque runIds.
+
+## Branching in the workflow
+
+```ts
+import { IsTestRun } from "@codemation/core-nodes";
+
+const isTestRun = new IsTestRun("Skip side effects in tests");
+```
+
+Or read `ctx.testContext` directly from a custom node:
+
+```ts
+async execute({ item, ctx }) {
+  if (ctx.testContext) {
+    return { json: { result: "synthetic-test-output" } };
+  }
+  return { json: await this.realApi.send(item.json) };
+}
+```
+
+## Authoring assertions
+
+```ts
+import { Assertion } from "@codemation/core-nodes";
+
+const checkClassification = new Assertion<{ label: string; confidence: number }>({
+  name: "Classification checks",
+  assertions: (item) => [
+    {
+      // Boolean-style: 1 = pass, 0 = fail. Default threshold (0.5) handles this.
+      name: "label is spam",
+      score: item.json.label === "spam" ? 1 : 0,
+      expected: "spam",
+      actual: item.json.label,
+    },
+    {
+      // Continuous-score: declare the threshold explicitly.
+      name: "confidence ≥ 0.8",
+      score: item.json.confidence,
+      passThreshold: 0.8,
+      expected: "≥ 0.8",
+      actual: item.json.confidence,
+    },
+  ],
+});
+```
+
+The `AssertionResult` shape (stable; persister + chart UIs key off these fields):
+
+```ts
+interface AssertionResult {
+  readonly name: string;
+  /** 0..1 score. Source of truth for pass/fail (compared against `passThreshold`). */
+  readonly score: number;
+  /** 0..1 threshold for "passed". When omitted, consumers default to 0.5. */
+  readonly passThreshold?: number;
+  /** True when evaluating the assertion threw — treated as fail regardless of `score`. */
+  readonly errored?: true;
+  readonly expected?: JsonValue;
+  readonly actual?: JsonValue;
+  readonly message?: string;
+  readonly details?: Readonly<Record<string, JsonValue>>;
+}
+```
+
+Pass/fail derivation (canonical, in `@codemation/core`):
+
+```ts
+import { deriveAssertionPassed } from "@codemation/core";
+// errored ? false : score >= (passThreshold ?? 0.5)
+```
+
+`errored: true` is for the assertion code itself crashing (judge agent crashed, JSON parse failed) — use it to separate "broken evaluator" from "wrong workflow output" in dashboards:
+
+```ts
+assertions: async (item, ctx) => {
+  try {
+    const j = await runJudge(item, ctx);
+    return [{ name: "polite reply", score: j.score, passThreshold: 0.7, message: j.reason }];
+  } catch (err) {
+    return [{ name: "polite reply", score: 0, errored: true, message: String(err) }];
+  }
+};
+```
+
+## Judge-by-Agent
+
+A judge-by-agent is just an AI agent step feeding into an Assertion callback. Run an agent that returns a structured judgment, then map its output to an `AssertionResult` (`score: 0..1`, set `passThreshold`).
+
+## Running tests
+
+- **From the UI**: open the workflow → **Tests** tab. Pick a TestTrigger from the dropdown (the picker lists every `triggerKind === "test"` node), click **Run tests**. Use the metric selector on the trend chart to plot pass-rate, per-assertion average scores, or case counts. Click two historical runs to compare them side-by-side.
+- **From code**: instantiate `TestSuiteOrchestrator` from `@codemation/core/bootstrap`, call `runSuite({ workflow, triggerNodeId })`.
+- **From HTTP**: `POST /api/workflows/:workflowId/test-suite-runs` with `{ triggerNodeId, concurrency? }`.
+
+## Status
+
+### Per case (`Run.testCaseStatus`)
+
+| Status      | Meaning                                                                               |
+| ----------- | ------------------------------------------------------------------------------------- |
+| `running`   | Workflow run dispatched, not yet finished.                                            |
+| `succeeded` | Workflow completed AND every assertion passed.                                        |
+| `failed`    | Assertion-rollup downgrade OR the workflow itself reported failure.                   |
+| `errored`   | Workflow run threw before reaching a terminal state (engine error, not an assertion). |
+| `cancelled` | Suite's `AbortSignal` fired before this case completed.                               |
+
+### Suite
+
+| Status      | Meaning                                                             |
+| ----------- | ------------------------------------------------------------------- |
+| `succeeded` | All cases passed (or zero cases yielded).                           |
+| `failed`    | Every case failed.                                                  |
+| `partial`   | Some passed, some failed — **the normal "1 of 10 failed" outcome**. |
+| `cancelled` | Suite was aborted before all cases finished.                        |
+| `errored`   | The `generateItems` callback itself threw.                          |
+
+The suite counters and status are re-derived from the final per-case statuses, so an "all workflows completed cleanly but assertions caught regressions" suite reports `partial` rather than `succeeded`.
+
+## Best practices
+
+- **Don't `throw` from `execute` to fail a case.** Throwing skips downstream nodes — including the Assertion node — so you lose all assertion data and only get a run-level error. Instead, let the workflow complete and assert on the (wrong) output. The assertion-rollup downgrades the case to `failed`.
+- Use `score: 1`/`score: 0` for boolean checks (equality, contains, regex). The default `passThreshold = 0.5` handles them.
+- Use `passThreshold` for continuous metrics (confidence, judge ratings, similarity).
+- Reserve `errored: true` for assertion-code crashes, not low scores.
+- Keep TestTriggers as source-controlled fixtures so historical chart comparisons are apples-to-apples.
+
+## What's deferred (Phase 2)
+
+- **Test-input snapshots** — Phase 1 fetches inputs live every run (rolling-input). Snapshotting will land in Phase 2 for stable judge-score charts.
+- **Declarative assertion shorthands** — `StringEqualsAssertion`, `JudgeByAgentAssertion`, etc. compose on top of the generic `Assertion` shipping today.
+- **CLI / cron / GitHub PR integration** — currently triggered manually via UI or HTTP only.
+
+## Read more
+
+- Top-level walkthrough: [`docs/workflow-testing.md`](../../../../docs/workflow-testing.md)