@kirrosh/zond 0.8.0 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kirrosh/zond",
-  "version": "0.8.0",
+  "version": "0.9.1",
   "description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
   "license": "MIT",
   "module": "index.ts",
@@ -26,11 +26,13 @@
   "scripts": {
     "zond": "bun run src/cli/index.ts",
     "test": "bun run test:unit && bun run test:mocked",
-    "test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/chat.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/doctor.test.ts tests/cli/init.test.ts tests/cli/runs.test.ts tests/cli/safe-run.test.ts tests/cli/update.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/agent/agent-loop.test.ts tests/agent/context-manager.test.ts tests/agent/system-prompt.test.ts tests/reporter/",
+    "test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/chat.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/doctor.test.ts tests/cli/init.test.ts tests/cli/runs.test.ts tests/cli/safe-run.test.ts tests/cli/update.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/agent/agent-loop.test.ts tests/agent/context-manager.test.ts tests/agent/system-prompt.test.ts tests/reporter/ tests/version-sync.test.ts",
     "test:mocked": "bun run scripts/run-mocked-tests.ts",
     "test:ai": "bun test tests/ai/",
     "check": "tsc --noEmit --project tsconfig.json",
-    "build": "bun build --compile src/cli/index.ts --outfile zond"
+    "build": "bun build --compile src/cli/index.ts --outfile zond",
+    "version:sync": "bun run scripts/sync-version.ts",
+    "postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json .claude-plugin/marketplace.json"
   },
   "devDependencies": {
     "@types/bun": "latest"

package/src/core/diagnostics/failure-hints.ts

@@ -44,6 +44,16 @@ export function envCategory(hint: string | undefined): string | null {
   return null;
 }
 
+export function schemaHint(
+  failureType: string,
+  responseStatus: number | null | undefined,
+): string | null {
+  if (failureType === "assertion_failed" || responseStatus === 400 || responseStatus === 422) {
+    return "Use describe_endpoint(specPath, method, path) to verify expected request/response schema";
+  }
+  return null;
+}
+
 export function computeSharedEnvIssue(
   failures: Array<{ hint?: string }>,
   envFilePath?: string,

package/src/core/generator/guide-builder.ts

@@ -72,269 +72,15 @@ export interface GuideOptions {
 export function buildGenerationGuide(opts: GuideOptions): string {
   const hasAuth = opts.securitySchemes.length > 0;
 
+  const securitySummary = hasAuth
+    ? `Security: ${opts.securitySchemes.map(s => `${s.name} (${s.type}${s.scheme ? `/${s.scheme}` : ""})`).join(", ")}`
+    : "Security: none";
+
   return `# Test Generation Guide for ${opts.title}
 ${opts.coverageHeader ? `\n${opts.coverageHeader}\n` : ""}
 ## API Specification (${opts.endpointCount} endpoints)
 ${opts.baseUrl ? `Base URL: ${opts.baseUrl}` : "Base URL: use {{base_url}} environment variable"}
+${securitySummary}
 
-${opts.apiContext}
-
-${hasAuth ? `---
-
-## Environment Setup (Required for Authentication)
-
-This API uses authentication. Before running tests, set up your credentials:
-
-### Edit the env file
-After \`setup_api\`, the collection directory contains \`.env.yaml\`. Edit it to add your credentials:
-\`\`\`yaml
-base_url: "https://api.example.com"
-api_key: "your-actual-api-key-here"
-auth_token: "your-token-here"
-\`\`\`
-
-### How it works
-- Tests **automatically** load the \`"default"\` environment — no need to pass \`envName\` to \`run_tests\`
-- If the env file is in the collection root and tests are in a \`tests/\` subdirectory, the file is still found automatically
-- Use \`{{api_key}}\`, \`{{auth_token}}\`, \`{{base_url}}\` etc. in test headers/bodies
-- **Never hardcode credentials** in YAML files — always use \`{{variable}}\` references
-
-` : ""}---
-
-## YAML Test Suite Format Reference
-
-\`\`\`yaml
-name: "Suite Name"
-description: "What this suite tests"  # optional
-tags: [smoke, crud]                   # optional — used for filtering with --tag
-base_url: "{{base_url}}"
-headers:                          # optional suite-level headers
-  Authorization: "Bearer {{auth_token}}"
-  Content-Type: "application/json"
-config:                           # optional
-  timeout: 30000
-  retries: 0
-  follow_redirects: true
-tests:
-  - name: "Test step name"
-    POST: "/path/{{variable}}"    # exactly ONE method key: GET, POST, PUT, PATCH, DELETE
-    json:                         # request body (object)
-      field: "value"
-    query:                        # query parameters
-      limit: "10"
-    headers:                      # step-level headers (override suite)
-      X-Custom: "value"
-    expect:
-      status: 200                 # expected HTTP status: integer OR array [200, 204]
-      body:                       # field-level assertions
-        id: { type: "integer", capture: "item_id" }
-        name: { equals: "expected" }
-        email: { contains: "@", type: "string" }
-        count: { gt: 0, lt: 100 }
-        items: { exists: true }       # exists must be boolean, NEVER string
-        pattern: { matches: "^[A-Z]+" }
-      headers:
-        Content-Type: "application/json"
-      duration: 5000              # max response time in ms
-\`\`\`
-
-### Assertion Rules
-- \`capture: "var_name"\` — SAVES the value into a variable (use in later steps as {{var_name}})
-- \`equals: value\` — exact match COMPARISON (NEVER use equals to save a value!)
-- \`type: "string"|"number"|"integer"|"boolean"|"array"|"object"\`
-- \`contains: "substring"\` — string substring match
-- \`matches: "regex"\` — regex pattern match
-- \`gt: N\` / \`lt: N\` — numeric comparison
-- \`exists: true|false\` — field presence check (MUST be boolean, not string)
-
-### Nested Body Assertions
-Both forms are equivalent and supported:
-
-**Dot-notation (flat):**
-\`\`\`yaml
-body:
-  "category.name": { equals: "Dogs" }
-  "address.city": { type: "string" }
-\`\`\`
-
-**Nested YAML (auto-flattened):**
-\`\`\`yaml
-body:
-  category:
-    name: { equals: "Dogs" }
-  address:
-    city: { type: "string" }
-\`\`\`
-
-### Root Body Assertions (\`_body\`)
-Use \`_body\` to assert on the response body itself (not a field inside it):
-
-\`\`\`yaml
-body:
-  _body: { type: "array" }           # check that response body IS an array
-  _body: { type: "object" }          # check that response body IS an object
-  _body: { exists: true }            # check that body is not null/undefined
-\`\`\`
-
-### Built-in Generators
-Use in string values: \`{{$randomInt}}\`, \`{{$uuid}}\`, \`{{$timestamp}}\`, \`{{$randomEmail}}\`, \`{{$randomString}}\`, \`{{$randomName}}\`
-These are the ONLY generators — do NOT invent others.
-
-### Variable Interpolation
-- \`{{variable}}\` in paths, bodies, headers, query params
-- Captured values from previous steps are available in subsequent steps
-- Environment variables from .env.yaml files: \`{{base_url}}\`, \`{{auth_username}}\`, etc.
-
----
-
-## Step-by-Step Generation Algorithm
-
-### Step 0: Register the API (REQUIRED FIRST)
-**Always call \`setup_api\` before generating any tests.** This registers the collection in the database so WebUI, coverage tracking, and env loading all work.
-\`\`\`
-setup_api(name: "myapi", specPath: "/path/to/openapi.json", dir: "/path/to/project/apis/myapi")
-\`\`\`
-If you skip this step, WebUI will show "No API collections registered yet" and env variables won't auto-load.
-
-${hasAuth ? `**Then set credentials immediately after setup_api** — edit the \`.env.yaml\` file in the API directory to add your credentials:
-\`\`\`yaml
-api_key: "<actual-key>"
-base_url: "https://..."
-\`\`\`
-Never put actual key values directly in YAML test files.
-
-` : ""}\
-### Step 1: Analyze the API
-- Identify authentication method (${hasAuth ? opts.securitySchemes.map(s => `${s.name}: ${s.type}${s.scheme ? `/${s.scheme}` : ""}`).join(", ") : "none detected"})
-- Group endpoints by resource (e.g., /users/*, /pets/*, /orders/*)
-- Identify CRUD patterns: POST (create) → GET (read) → PUT/PATCH (update) → DELETE
-- Note required fields in request bodies
-
-### Step 2: Plan Test Suites
-Before generating, check coverage with \`coverage_analysis\` to avoid duplicating existing tests. Use \`generate_and_save(testsDir=...)\` for incremental generation of uncovered endpoints only.
-
-> **Coverage note**: coverage is a static scan of YAML files — an endpoint is "covered" if a test file contains a matching METHOD + path line, regardless of whether tests pass or actually run.
-
-Create separate files for each concern:
-${hasAuth ? `- \`${opts.outputDir}auth.yaml\` — Authentication flow\n` : ""}\
-- \`${opts.outputDir}{resource}-crud.yaml\` — CRUD lifecycle per resource
-- \`${opts.outputDir}{resource}-validation.yaml\` — Error cases per resource
-
-### Step 3: Generate Each Suite
-
-${hasAuth ? `**Auth suite** (\`auth.yaml\`):
-1. Login with valid credentials → capture token
-2. Access protected endpoint with token → 200
-3. Login with invalid credentials → 401/403
-4. Access protected endpoint without token → 401
-
-` : ""}\
-**CRUD lifecycle** (\`{resource}-crud.yaml\`):
-1. Create resource (POST) → 201, **always verify key fields in response body** (at minimum: id, name/title)
-2. Read created resource (GET /resource/{{id}}) → 200, verify fields match what was sent
-3. List resources (GET /resource) → 200, verify \`_body: { type: "array" }\` AND \`_body.length: { gt: 0 }\`
-4. Update resource (PUT/PATCH /resource/{{id}}) → 200
-5. Read updated resource → verify changes applied
-6. Delete resource (DELETE /resource/{{id}}) → 200/204
-7. Verify deleted (GET /resource/{{id}}) → 404
-8. For bulk create endpoints (createWithArray/List): create → then GET each to verify they exist
-
-**Validation suite** (\`{resource}-validation.yaml\`):
-1. Create with missing required fields → 400/422, verify \`message: { exists: true }\` in error body
-2. Create with invalid field types → 400/422
-3. Get non-existent resource (e.g. id=999999) → 404
-4. Delete non-existent resource → 404
-5. For error responses: always assert error body has meaningful content, not just status code
-
-### Step 4: Save, Run, Debug
-1. Use \`save_test_suite\` to save each file — it validates YAML before writing
-2. Use \`run_tests\` to execute — review pass/fail summary
-3. If failures: use \`query_db\` with \`action: "diagnose_failure"\` and the runId to see full request/response details
-4. Fix issues and re-save with \`overwrite: true\`
-
----
-
-## Tag Conventions
-
-Use standard tags to enable safe filtering:
-
-| Tag | HTTP Methods | Safe for |
-|-----|-------------|---------|
-| \`smoke\` | GET only | Production (read-only, zero risk) |
-| \`crud\` | POST/PUT/PATCH | Staging only (state-changing) |
-| \`destructive\` | DELETE | Explicit opt-in, run last |
-| \`auth\` | Any (auth flows) | Run first to capture tokens |
-
-Example: \`zond run --tag smoke --safe\` → reads-only, safe against production.
-
----
-
-## Practical Tips
-
-- **int64 IDs**: For APIs returning large auto-generated IDs (int64), prefer setting fixed IDs in request bodies rather than capturing auto-generated ones, as JSON number precision may cause mismatches.
-- **Nested assertions**: Use dot-notation or nested YAML — both work identically.
-- **Root body type**: Use \`_body: { type: "array" }\` to verify the response body type itself.
-- **List endpoints**: Always check both type AND non-emptiness: \`_body: { type: "array" }\` + \`_body.length: { gt: 0 }\`
-- **Create responses**: Always verify at least the key identifying fields (id, name) in the response body — don't just check status.
-- **Error responses**: Assert that error bodies contain useful info (\`message: { exists: true }\`), not just status codes.
-- **Bulk operations**: After bulk create (createWithArray, createWithList), add GET steps to verify resources were actually created.
-- **204 No Content**: When an endpoint returns 204, omit \`body:\` assertions entirely — an empty response IS the correct behavior. Adding body assertions on 204 will always fail.
-- **Cleanup pattern**: Always delete test data in the same suite. Use a create → read → delete lifecycle so tests are idempotent:
-  \`\`\`yaml
-  tests:
-    - name: Create test resource
-      POST: /users
-      json: { name: "zond-test-{{$randomString}}" }
-      expect:
-        status: 201
-        body:
-          id: { capture: user_id }
-    - name: Read created resource
-      GET: /users/{{user_id}}
-      expect:
-        status: 200
-    - name: Cleanup - delete test resource
-      DELETE: /users/{{user_id}}
-      expect:
-        status: 204
-  \`\`\`
-- **Identifiable test data**: Prefix test data with \`zond-test-\` or use \`{{$uuid}}\` / \`zond-test-{{$randomString}}\` so you can identify and clean up leftover test data if needed.
-
----
-
-## Common Mistakes to Avoid
-
-1. **equals vs capture**: \`capture\` SAVES a value, \`equals\` COMPARES. To extract a token: \`{ capture: "token" }\` NOT \`{ equals: "{{token}}" }\`
-2. **exists must be boolean**: \`exists: true\` NOT \`exists: "true"\`
-3. **Status must be integer or array**: \`status: 200\` or \`status: [200, 204]\` NOT \`status: "200"\`
-4. **One method per step**: Each test step has exactly ONE of GET/POST/PUT/PATCH/DELETE
-5. **Don't hardcode base URL**: Use \`{{base_url}}\` — set it in environment or suite base_url
-6. **Auth credentials**: Use environment variables \`{{auth_username}}\`, \`{{auth_password}}\` — NOT generators
-7. **String query params**: Query parameter values must be strings: \`limit: "10"\` not \`limit: 10\`
-8. **Hardcoded credentials**: NEVER put actual API keys/tokens in YAML — use \`{{api_key}}\` from env instead
-9. **Body assertions on 204**: Don't add \`body:\` checks for DELETE or other endpoints that return 204 No Content — the body is empty by design.
-
----
-
-## Tools to Use
-
-| Tool | When |
-|------|------|
-| \`setup_api\` | Register a new API (creates dirs, reads spec, sets up env) |
-| \`generate_and_save\` | Get test generation guide (with auto-chunking for large APIs) |
-| \`save_test_suite\` | Save generated YAML (validates before writing) |
-| \`save_test_suites\` | Save multiple YAML files in one call |
-| \`run_tests\` | Execute saved test suites |
-| \`query_db\` | Query runs, collections, results, diagnose failures |
-| \`coverage_analysis\` | Find untested endpoints for incremental generation |
-| \`describe_endpoint\` | Get full details for one endpoint when debugging |
-| \`ci_init\` | Generate CI/CD workflow (GitHub Actions / GitLab CI) to run tests on push |
-
-## Workflow After Tests Pass
-
-After tests are saved and running successfully, ask the user if they want to set up CI/CD:
-1. Use \`ci_init\` to generate a CI workflow (auto-detects platform or use platform param)
-2. Help them commit and push to their repository
-3. Tests will run automatically on push, PR, and on schedule
-`;
+${opts.apiContext}`;
 }

package/src/mcp/tools/query-db.ts

@@ -4,7 +4,7 @@ import { getDb } from "../../db/schema.ts";
 import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
 import { join } from "node:path";
 import { TOOL_DESCRIPTIONS } from "../descriptions.js";
-import { statusHint, classifyFailure, envHint, envCategory } from "../../core/diagnostics/failure-hints.ts";
+import { statusHint, classifyFailure, envHint, envCategory, schemaHint } from "../../core/diagnostics/failure-hints.ts";
 
 function parseBodySafe(raw: string | null | undefined): unknown {
   if (!raw) return undefined;
@@ -124,6 +124,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
               // env issues take priority over generic status hints
               const hint = envHint(r.request_url, r.error_message, envFilePath) ?? statusHint(r.response_status);
               const failure_type = classifyFailure(r.status, r.response_status);
+              const sHint = schemaHint(failure_type, r.response_status);
               return {
                 suite_name: r.suite_name,
                 test_name: r.test_name,
@@ -134,6 +135,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
                 request_url: r.request_url,
                 response_status: r.response_status,
                 ...(hint ? { hint } : {}),
+                ...(sHint ? { schema_hint: sHint } : {}),
                 response_body: parseBodySafe(r.response_body),
                 response_headers: r.response_headers
                   ? JSON.parse(r.response_headers)

package/src/mcp/tools/run-tests.ts

@@ -48,6 +48,12 @@ export function registerRunTestsTool(server: McpServer, dbPath?: string) {
     const hints: string[] = [];
     if (failedSteps.length > 0) {
       hints.push("Use query_db(action: 'diagnose_failure', runId: " + runId + ") for detailed failure analysis");
+      const hasAssertionFailures = failedSteps.some(s => s.assertions.length > 0);
+      if (hasAssertionFailures) {
+        hints.push(
+          "Some tests have assertion failures — use describe_endpoint(specPath, method, path) to verify expected schemas"
+        );
+      }
     }
     hints.push("Use manage_server(action: 'start') to launch the Web UI and view results visually in a browser at http://localhost:8080");
     hints.push("Ask the user if they want to set up CI/CD to run these tests automatically on push. If yes, use ci_init to generate a workflow and help them push to GitHub/GitLab.");