@blucca/n8n-check 0.0.0-stage → 0.1.7

package/examples/stop-after/observed-results.json

@@ -0,0 +1,162 @@
+{
+  "description": "Synthetic full Sheets-to-Slack exports; only the fixture and business transformation run. Real n8n, loopback-only network.",
+  "files": {
+    "fixed.json": "a20e75366252c5cd165821eeca8b54671b69df51242fe6e6c3f847bc7d651ec6",
+    "broken.json": "899bb0f39ce8f494b9f44e9b4b79874f93065b4bc379fab970aad90ee907ed14",
+    "case.json": "81fe5dcca0b261ca97fb7cff1d27ce5a177ffbb61aeec7c75cb1c63fb28f0ded"
+  },
+  "runs": {
+    "fixed": {
+      "runnerVersion": "0.1.5",
+      "n8nVersion": "2.41.7",
+      "nodeVersion": "v26.10.0",
+      "startedAt": "2026-10-06T22:13:13.515Z",
+      "status": "passed",
+      "exitCode": 0,
+      "checks": [
+        {
+          "name": "n8n execution succeeds",
+          "passed": true,
+          "actual": {
+            "cliExitCode": 0,
+            "lastNode": "Prepare notification",
+            "executionFound": true
+          }
+        },
+        {
+          "name": "Prepare notification: output 0",
+          "passed": true,
+          "expected": [
+            {
+              "invoiceId": "INV-42",
+              "text": "Invoice INV-42: $650.00"
+            },
+            {
+              "invoiceId": "INV-43",
+              "text": "Invoice INV-43: $120.50"
+            }
+          ],
+          "actual": [
+            {
+              "invoiceId": "INV-42",
+              "text": "Invoice INV-42: $650.00"
+            },
+            {
+              "invoiceId": "INV-43",
+              "text": "Invoice INV-43: $120.50"
+            }
+          ]
+        },
+        {
+          "name": "mock: all requests match routes",
+          "passed": true,
+          "actual": []
+        },
+        {
+          "name": "mock: request limits and server",
+          "passed": true,
+          "actual": []
+        }
+      ],
+      "network": {
+        "mode": "loopback-only",
+        "enforced": true,
+        "interfaces": [
+          "lo"
+        ]
+      },
+      "preparation": {
+        "inputNode": "Read invoices",
+        "changes": [
+          {
+            "node": "Read invoices",
+            "change": "replace with fixture JSON items"
+          },
+          {
+            "node": "Prepare notification",
+            "change": "stop this branch after the node; remove outgoing connections; retain independently reachable branches"
+          }
+        ],
+        "omittedNodes": [
+          "Start",
+          "Notify team"
+        ]
+      },
+      "requests": []
+    },
+    "broken": {
+      "runnerVersion": "0.1.5",
+      "n8nVersion": "2.41.7",
+      "nodeVersion": "v26.10.0",
+      "startedAt": "2026-10-06T22:13:22.755Z",
+      "status": "failed",
+      "exitCode": 1,
+      "checks": [
+        {
+          "name": "n8n execution succeeds",
+          "passed": true,
+          "actual": {
+            "cliExitCode": 0,
+            "lastNode": "Prepare notification",
+            "executionFound": true
+          }
+        },
+        {
+          "name": "Prepare notification: output 0",
+          "passed": false,
+          "expected": [
+            {
+              "invoiceId": "INV-42",
+              "text": "Invoice INV-42: $650.00"
+            },
+            {
+              "invoiceId": "INV-43",
+              "text": "Invoice INV-43: $120.50"
+            }
+          ],
+          "actual": [
+            {
+              "invoiceId": "INV-42",
+              "text": "Invoice INV-42: $650.00"
+            }
+          ]
+        },
+        {
+          "name": "mock: all requests match routes",
+          "passed": true,
+          "actual": []
+        },
+        {
+          "name": "mock: request limits and server",
+          "passed": true,
+          "actual": []
+        }
+      ],
+      "network": {
+        "mode": "loopback-only",
+        "enforced": true,
+        "interfaces": [
+          "lo"
+        ]
+      },
+      "preparation": {
+        "inputNode": "Read invoices",
+        "changes": [
+          {
+            "node": "Read invoices",
+            "change": "replace with fixture JSON items"
+          },
+          {
+            "node": "Prepare notification",
+            "change": "stop this branch after the node; remove outgoing connections; retain independently reachable branches"
+          }
+        ],
+        "omittedNodes": [
+          "Start",
+          "Notify team"
+        ]
+      },
+      "requests": []
+    }
+  }
+}

package/examples/vapi-tool-contract/README.md

@@ -0,0 +1,67 @@
+# Vapi tool calls: preserve the caller, surface the backend failure
+
+**Two tool calls in one request. Two different customers. Every response keeps its own `toolCallId`.**
+
+This synthetic backend check looks up customer orders through a real n8n HTTP Request node. It checks outgoing customer/ID pairs, response serialization, and the complete Vapi `results` envelope.
+
+| Scenario | Backend responses | Expected tool response |
+| --- | --- | --- |
+| `success` | Two HTTP 200 responses with different orders | Two distinct IDs and the correct serialized order data |
+| `backend-503` | Customer A succeeds; customer B returns 503 | A retains its result; B receives an explicit `error` string |
+| `empty-orders` | Customer B returns `{"orders":[]}` | B receives the successful string `"{\"orders\":[]}"` |
+| `wrong-id.json` with the unchanged success case | Both backend requests succeed | **FAIL**: the second result incorrectly repeats the first tool-call ID |
+
+## The workflow
+
+`Vapi webhook → Split tool calls → Lookup orders → Build Vapi response → Respond to Vapi`
+
+- [`workflow.json`](workflow.json) is an importable synthetic workflow with a Webhook input and Respond to Webhook configured for HTTP 200.
+- Each case injects the Webhook node's `body.message` JSON and stops after **Build Vapi response**. The real Split, HTTP Request, and response-building nodes execute; the fixture replaces webhook delivery and the response node sits beyond this test boundary.
+- HTTP Request uses **Include Response Headers and Status** and **Never Error** to preserve the backend's status for response mapping. Local HTTP mocks return the declared 200/503 responses.
+- Response construction follows n8n item pairing back to each split tool call. Each route checks exactly one request and its exact customer/`toolCallId` body.
+- Success uses `JSON.stringify` for a single-line `result`. HTTP 503 maps to `error`, preserving the sibling success. Empty order data stays an explicit successful result.
+
+## Run from your browser
+
+1. [Fork n8n-check](https://github.com/blucca/n8n-check/fork), then enable **Actions** in your fork.
+2. Open **Try a Vapi tool-call contract → Run workflow**.
+3. Choose `backend-503` with `workflow` to check a successful result beside an explicit tool error. Choose `success` with `wrong-id` to see the regression turn the job red.
+4. Read the check summary; download `vapi-tool-contract-results` for JSON/JUnit.
+
+This Action runs the selected case with the released **v0.1.7** runner and **n8n 2.41.7**, using loopback-only Docker networking and synthetic data. GitHub Actions usage follows your account plan. [Action source](../../.github/workflows/try-vapi-tool-contract.yml).
+
+## Run with real n8n
+
+From the repository root, with Node.js 24+ and n8n installed:
+
+```sh
+# Optional: select an existing installation.
+# export N8N_BINARY=/absolute/path/to/n8n
+for scenario in success backend-503 empty-orders; do
+  node bin/n8n-check.mjs examples/vapi-tool-contract/workflow.json \
+    "examples/vapi-tool-contract/$scenario.case.json" \
+    --allow-network --out "temp/vapi-tool-contract/$scenario" || exit $?
+done
+
+# Negative control: expect exit 1 and a failed Build Vapi response assertion.
+node bin/n8n-check.mjs examples/vapi-tool-contract/wrong-id.json \
+  examples/vapi-tool-contract/success.case.json \
+  --allow-network --out temp/vapi-tool-contract/wrong-id
+```
+
+The bundled source and fixtures use synthetic values. The local command enables host networking; the runner redirects both HTTP routes to its loopback mock. Use the repository's [Docker route](../../README.md#try-a-failure-then-its-fix) for an isolated runtime. Run local CLI cases sequentially: n8n's task broker uses a shared default port.
+
+Recorded execution: **n8n 2.41.7, n8n-check 0.1.6**, three passing cases, each with eight checks and two HTTP requests. The wrong-ID control retains successful execution and both correct backend requests while failing the output assertion. [`observed-results.json`](observed-results.json) contains compact check results, mock traces, and the actual emitted envelopes.
+
+**[Completed GitHub Actions run](https://github.com/blucca/n8n-check/actions/runs/37558788945):** `backend-503`, 8/8 checks, two HTTP requests, loopback-only networking. The result preserves customer A’s order and returns customer B’s tool error.
+
+## Verified Vapi contract and scope
+
+Official references, read 2026-10-07:
+
+- [Custom tools — request and response format](https://docs.vapi.ai/tools/custom-tools): `message.toolCallList[].function.name` and object-valued `function.arguments`; `results[]` associates each response with `toolCallId`; `result` and `error` are single-line strings. Each entry uses one of those fields. Multiple results may appear in any order. The server returns HTTP 200 for successful and tool-error envelopes.
+- [Server URL events — tool calls](https://docs.vapi.ai/server-url/events): documents tool-call delivery and response correlation. Its introductory flattened `name`/`parameters` example is a separate input shape; this fixture follows the custom-tools function/arguments example above.
+
+This pack covers one tool, `lookupOrders`, a non-empty batch with unique call IDs, object-valued arguments, synthetic customer IDs, and backend HTTP responses containing JSON. The assertions preserve this sample's response order as well as the Vapi ID mapping. HTTP error handling here covers returned statuses; transport timeouts/DNS errors, malformed tool arguments, webhook authentication/delivery, and actual HTTP 200 response delivery require additional integration cases. The sample's Webhook response configuration is supplied for importing and subsequent endpoint testing.
+
+Voice/audio, Twilio delivery, ElevenLabs speech, model decisions, live Vapi acceptance, and production credentials are separate live-agent acceptance work. A buyer's export, tool schema, example call trace, and agreed behavior provide the next test inputs.

package/examples/vapi-tool-contract/backend-503.case.json

@@ -0,0 +1,113 @@
+{
+  "version": 1,
+  "name": "Vapi tool contract: backend-503",
+  "input": {
+    "node": "Vapi webhook",
+    "items": [
+      {
+        "body": {
+          "message": {
+            "type": "tool-calls",
+            "toolCallList": [
+              {
+                "id": "call-order-A",
+                "type": "function",
+                "function": {
+                  "name": "lookupOrders",
+                  "arguments": {
+                    "customerId": "customer-A"
+                  }
+                }
+              },
+              {
+                "id": "call-order-B",
+                "type": "function",
+                "function": {
+                  "name": "lookupOrders",
+                  "arguments": {
+                    "customerId": "customer-B"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    ]
+  },
+  "stopAfter": "Build Vapi response",
+  "mocks": [
+    {
+      "node": "Lookup orders",
+      "url": "/lookup/{{ $json.customerId }}",
+      "routes": [
+        {
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "responses": [
+            {
+              "status": 200,
+              "json": {
+                "orders": [
+                  {
+                    "id": "order-A-101",
+                    "status": "shipped"
+                  }
+                ]
+              }
+            }
+          ],
+          "expect": {
+            "count": 1,
+            "bodies": [
+              {
+                "customerId": "customer-A",
+                "toolCallId": "call-order-A"
+              }
+            ]
+          }
+        },
+        {
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "responses": [
+            {
+              "status": 503,
+              "json": {
+                "message": "upstream overloaded"
+              }
+            }
+          ],
+          "expect": {
+            "count": 1,
+            "bodies": [
+              {
+                "customerId": "customer-B",
+                "toolCallId": "call-order-B"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "assertions": [
+    {
+      "node": "Build Vapi response",
+      "equals": [
+        {
+          "results": [
+            {
+              "toolCallId": "call-order-A",
+              "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+            },
+            {
+              "toolCallId": "call-order-B",
+              "error": "Order service temporarily unavailable. Please try again later."
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

package/examples/vapi-tool-contract/empty-orders.case.json

@@ -0,0 +1,113 @@
+{
+  "version": 1,
+  "name": "Vapi tool contract: empty-orders",
+  "input": {
+    "node": "Vapi webhook",
+    "items": [
+      {
+        "body": {
+          "message": {
+            "type": "tool-calls",
+            "toolCallList": [
+              {
+                "id": "call-order-A",
+                "type": "function",
+                "function": {
+                  "name": "lookupOrders",
+                  "arguments": {
+                    "customerId": "customer-A"
+                  }
+                }
+              },
+              {
+                "id": "call-order-B",
+                "type": "function",
+                "function": {
+                  "name": "lookupOrders",
+                  "arguments": {
+                    "customerId": "customer-B"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    ]
+  },
+  "stopAfter": "Build Vapi response",
+  "mocks": [
+    {
+      "node": "Lookup orders",
+      "url": "/lookup/{{ $json.customerId }}",
+      "routes": [
+        {
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "responses": [
+            {
+              "status": 200,
+              "json": {
+                "orders": [
+                  {
+                    "id": "order-A-101",
+                    "status": "shipped"
+                  }
+                ]
+              }
+            }
+          ],
+          "expect": {
+            "count": 1,
+            "bodies": [
+              {
+                "customerId": "customer-A",
+                "toolCallId": "call-order-A"
+              }
+            ]
+          }
+        },
+        {
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "responses": [
+            {
+              "status": 200,
+              "json": {
+                "orders": []
+              }
+            }
+          ],
+          "expect": {
+            "count": 1,
+            "bodies": [
+              {
+                "customerId": "customer-B",
+                "toolCallId": "call-order-B"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "assertions": [
+    {
+      "node": "Build Vapi response",
+      "equals": [
+        {
+          "results": [
+            {
+              "toolCallId": "call-order-A",
+              "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+            },
+            {
+              "toolCallId": "call-order-B",
+              "result": "{\"orders\":[]}"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

package/examples/vapi-tool-contract/observed-results.json

@@ -0,0 +1,234 @@
+{
+  "n8nVersion": "2.41.7",
+  "runnerVersion": "0.1.6",
+  "networkMode": "host-network",
+  "boundary": "Fixture at Vapi webhook; real Split tool calls, Lookup orders HTTP Request and Build Vapi response; stop before Respond to Vapi. Local synthetic HTTP mocks.",
+  "runs": [
+    {
+      "scenario": "success",
+      "workflow": "workflow.json",
+      "case": "success.case.json",
+      "startedAt": "2026-10-07T01:44:22.166Z",
+      "status": "passed",
+      "exitCode": 0,
+      "checksPassed": 8,
+      "checksTotal": 8,
+      "failedChecks": [],
+      "requests": [
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "body": {
+            "customerId": "customer-A",
+            "toolCallId": "call-order-A"
+          },
+          "status": 200
+        },
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "body": {
+            "customerId": "customer-B",
+            "toolCallId": "call-order-B"
+          },
+          "status": 200
+        }
+      ],
+      "actualResponse": {
+        "results": [
+          {
+            "toolCallId": "call-order-A",
+            "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+          },
+          {
+            "toolCallId": "call-order-B",
+            "result": "{\"orders\":[{\"id\":\"order-B-202\",\"status\":\"processing\"}]}"
+          }
+        ]
+      }
+    },
+    {
+      "scenario": "backend-503",
+      "workflow": "workflow.json",
+      "case": "backend-503.case.json",
+      "startedAt": "2026-10-07T01:44:47.031Z",
+      "status": "passed",
+      "exitCode": 0,
+      "checksPassed": 8,
+      "checksTotal": 8,
+      "failedChecks": [],
+      "requests": [
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "body": {
+            "customerId": "customer-A",
+            "toolCallId": "call-order-A"
+          },
+          "status": 200
+        },
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "body": {
+            "customerId": "customer-B",
+            "toolCallId": "call-order-B"
+          },
+          "status": 503
+        }
+      ],
+      "actualResponse": {
+        "results": [
+          {
+            "toolCallId": "call-order-A",
+            "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+          },
+          {
+            "toolCallId": "call-order-B",
+            "error": "Order service temporarily unavailable. Please try again later."
+          }
+        ]
+      }
+    },
+    {
+      "scenario": "empty-orders",
+      "workflow": "workflow.json",
+      "case": "empty-orders.case.json",
+      "startedAt": "2026-10-07T01:44:29.107Z",
+      "status": "passed",
+      "exitCode": 0,
+      "checksPassed": 8,
+      "checksTotal": 8,
+      "failedChecks": [],
+      "requests": [
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "body": {
+            "customerId": "customer-A",
+            "toolCallId": "call-order-A"
+          },
+          "status": 200
+        },
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "body": {
+            "customerId": "customer-B",
+            "toolCallId": "call-order-B"
+          },
+          "status": 200
+        }
+      ],
+      "actualResponse": {
+        "results": [
+          {
+            "toolCallId": "call-order-A",
+            "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+          },
+          {
+            "toolCallId": "call-order-B",
+            "result": "{\"orders\":[]}"
+          }
+        ]
+      }
+    },
+    {
+      "scenario": "wrong-id",
+      "workflow": "wrong-id.json",
+      "case": "success.case.json",
+      "startedAt": "2026-10-07T01:44:56.484Z",
+      "status": "failed",
+      "exitCode": 1,
+      "checksPassed": 7,
+      "checksTotal": 8,
+      "failedChecks": [
+        {
+          "name": "Build Vapi response: output 0",
+          "passed": false,
+          "expected": [
+            {
+              "results": [
+                {
+                  "toolCallId": "call-order-A",
+                  "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+                },
+                {
+                  "toolCallId": "call-order-B",
+                  "result": "{\"orders\":[{\"id\":\"order-B-202\",\"status\":\"processing\"}]}"
+                }
+              ]
+            }
+          ],
+          "actual": [
+            {
+              "results": [
+                {
+                  "toolCallId": "call-order-A",
+                  "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+                },
+                {
+                  "toolCallId": "call-order-A",
+                  "result": "{\"orders\":[{\"id\":\"order-B-202\",\"status\":\"processing\"}]}"
+                }
+              ]
+            }
+          ]
+        }
+      ],
+      "requests": [
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-A",
+          "body": {
+            "customerId": "customer-A",
+            "toolCallId": "call-order-A"
+          },
+          "status": 200
+        },
+        {
+          "node": "Lookup orders",
+          "method": "POST",
+          "path": "/lookup/customer-B",
+          "body": {
+            "customerId": "customer-B",
+            "toolCallId": "call-order-B"
+          },
+          "status": 200
+        }
+      ],
+      "actualResponse": {
+        "results": [
+          {
+            "toolCallId": "call-order-A",
+            "result": "{\"orders\":[{\"id\":\"order-A-101\",\"status\":\"shipped\"}]}"
+          },
+          {
+            "toolCallId": "call-order-A",
+            "result": "{\"orders\":[{\"id\":\"order-B-202\",\"status\":\"processing\"}]}"
+          }
+        ]
+      }
+    }
+  ],
+  "ci": {
+    "url": "https://github.com/blucca/n8n-check/actions/runs/37558788945",
+    "sourceCommit": "fe78330",
+    "scenario": "backend-503",
+    "workflow": "workflow.json",
+    "n8nVersion": "2.41.7",
+    "runnerVersion": "0.1.6",
+    "networkMode": "loopback-only",
+    "checksPassed": 8,
+    "checksTotal": 8,
+    "requests": 2,
+    "exitCode": 0
+  }
+}