@cliwatch/cli-bench 0.6.3 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliwatch/cli-bench",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "description": "LLM CLI agent testing framework — benchmark how well AI models use your CLI tool",
   "keywords": ["cli", "benchmark", "llm", "testing", "ai-agent", "cliwatch", "evaluation"],
   "license": "MIT",
@@ -14,7 +14,6 @@
   },
   "files": [
     "dist",
-    "task_suites",
     "LICENSE",
     "CHANGELOG.md"
   ],
@@ -35,6 +34,7 @@
     "test": "vitest"
   },
   "dependencies": {
+    "@ai-sdk/google": "^3.0.0",
     "@hey-api/client-fetch": "^0.13.1",
     "ai": "^6.0.18",
     "yaml": "^2.7.0",

package/task_suites/curl.yaml

@@ -1,138 +0,0 @@
-cli: curl
-version_command: "curl --version | head -1"
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: simple-get
-    intent: "Fetch the contents of https://httpbin.org/get"
-    difficulty: easy
-    category: query
-    max_turns: 2
-    assert:
-      - ran: "curl.*httpbin.org/get"
-      - exit_code: 0
-      - output_contains: "origin"
-
-  - id: head-request
-    intent: "Send a HEAD request to https://httpbin.org/get and show the response headers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-I|--head"
-      - output_contains: "HTTP"
-
-  - id: download-file
-    intent: "Download https://httpbin.org/robots.txt and save it as robots.txt"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-o|--output"
-      - file_exists: "robots.txt"
-
-  - id: follow-redirects
-    intent: "Fetch https://httpbin.org/redirect/2 and follow all redirects"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-L|--location"
-      - exit_code: 0
-
-  # -- Medium ------------------------------------------------------------------
-  - id: post-json
-    intent: "Send a POST request to https://httpbin.org/post with JSON body {\"name\": \"bench\", \"version\": 1} and set the Content-Type header to application/json"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-X POST|--request POST|-d|--data"
-      - ran: "Content-Type.*application/json"
-      - output_contains: "bench"
-
-  - id: custom-headers
-    intent: "Send a GET request to https://httpbin.org/headers with custom headers X-Request-ID: abc123 and Accept: application/xml"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-H|--header"
-      - ran: "X-Request-ID"
-      - output_contains: "abc123"
-
-  - id: basic-auth
-    intent: "Send a GET request to https://httpbin.org/basic-auth/user/passwd using basic authentication with username 'user' and password 'passwd'"
-    difficulty: medium
-    category: auth
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-u|--user|user:passwd"
-      - output_contains: "authenticated"
-
-  - id: verbose-timing
-    intent: "Fetch https://httpbin.org/get and show the total time taken for the request using curl's write-out feature"
-    difficulty: medium
-    category: output
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-w|--write-out|time_total"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: put-with-file
-    intent: "Upload the file /tmp/bench-workspace/data.json to https://httpbin.org/put using a PUT request with Content-Type application/json"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"key\": \"value\"}' > /tmp/bench-workspace/data.json"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-X PUT|--request PUT|-T"
-      - ran: "data.json"
-      - output_contains: "key"
-
-  - id: retry-with-timeout
-    intent: "Fetch https://httpbin.org/delay/1 with a 5 second timeout, retry 3 times on failure, and save the response to response.json"
-    difficulty: hard
-    category: crud
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "--retry.*3"
-      - ran: "--max-time|--connect-timeout|-m"
-      - ran: "-o|--output"
-
-  - id: multipart-upload
-    intent: "Send a multipart/form-data POST to https://httpbin.org/post with a field 'username' set to 'admin' and a file field 'config' uploading /tmp/bench-workspace/app.conf"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo 'port=8080' > /tmp/bench-workspace/app.conf"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-F|--form"
-      - ran: "username.*admin"
-      - ran: "config.*@.*app.conf"
-      - output_contains: "admin"
-
-  - id: conditional-request
-    intent: "Fetch https://httpbin.org/cache and use conditional headers: set If-None-Match to '12345' and If-Modified-Since to 'Thu, 01 Jan 2025 00:00:00 GMT'. Show the response status code."
-    difficulty: hard
-    category: query
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "If-None-Match"
-      - ran: "If-Modified-Since"
-      - ran: "-w|--write-out|-I|--head|-v|--verbose"

package/task_suites/docker.yaml

@@ -1,163 +0,0 @@
-cli: docker
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: list-containers
-    intent: "List all running containers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker ps"
-      - exit_code: 0
-
-  - id: list-images
-    intent: "List all local Docker images"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker image"
-      - exit_code: 0
-
-  - id: pull-image
-    intent: "Pull the latest nginx image from Docker Hub"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "docker pull.*nginx"
-      - verify:
-          run: "docker images nginx --format '{{.Repository}}'"
-          output_contains: "nginx"
-
-  - id: view-logs
-    intent: "Show the last 100 lines of logs from the container 'api' and follow new output"
-    difficulty: easy
-    category: query
-    setup:
-      - "docker run -d --name api alpine sh -c 'for i in $(seq 1 200); do echo line-$i; done; sleep 3600'"
-    max_turns: 3
-    assert:
-      - ran: "docker logs"
-      - ran: "--tail"
-
-  - id: stop-container
-    intent: "Stop the container named 'web-server'"
-    difficulty: easy
-    category: crud
-    setup:
-      - "docker run -d --name web-server alpine sleep 3600"
-    max_turns: 3
-    assert:
-      - ran: "docker stop.*web-server"
-      - verify:
-          run: "docker ps --filter name=web-server --format '{{.Names}}'"
-          output_equals: ""
-
-  # -- Medium ------------------------------------------------------------------
-  - id: run-detached
-    intent: "Run an nginx container in the background, mapping host port 8080 to container port 80, named 'web'"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker run"
-      - ran: "-d"
-      - ran: "--name.*web"
-      - verify:
-          run: "docker ps --filter name=web --format '{{.Names}}'"
-          output_contains: "web"
-
-  - id: build-with-tag
-    intent: "Build a Docker image from ./Dockerfile and tag it as myapp:v2"
-    difficulty: medium
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest\nRUN echo hello' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 5
-    assert:
-      - ran: "docker build"
-      - ran: "myapp:v2"
-      - verify:
-          run: "docker images myapp:v2 --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:v2"
-
-  - id: exec-into-container
-    intent: "Run the command 'cat /etc/os-release' inside the running container 'web'"
-    difficulty: medium
-    category: crud
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker exec.*web.*cat /etc/os-release"
-      - exit_code: 0
-
-  - id: inspect-json
-    intent: "Get the IP address of the container 'web' using docker inspect with a format template"
-    difficulty: medium
-    category: output
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker inspect"
-      - ran: "--format"
-
-  - id: prune-all
-    intent: "Remove all stopped containers, unused networks, dangling images, and build cache without prompting for confirmation"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker system prune"
-      - ran: "--force|-f"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: run-complex
-    intent: "Run a postgres:16 container named 'db' in the background with environment variables POSTGRES_USER=admin and POSTGRES_PASSWORD=secret, mount a volume 'pgdata' to /var/lib/postgresql/data, and connect it to the network 'backend'"
-    difficulty: hard
-    category: crud
-    setup:
-      - "docker network create backend || true"
-    max_turns: 7
-    assert:
-      - ran: "docker run"
-      - ran: "--name.*db"
-      - ran: "POSTGRES_USER=admin"
-      - ran: "POSTGRES_PASSWORD=secret"
-      - verify:
-          run: "docker ps --filter name=db --format '{{.Names}}'"
-          output_contains: "db"
-
-  - id: compose-up
-    intent: "Start all services defined in docker-compose.yml in detached mode and rebuild any changed images"
-    difficulty: hard
-    category: workflow
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'services:\n  web:\n    image: alpine\n    command: sleep 3600\n' > /tmp/bench-workspace/docker-compose.yml"
-    max_turns: 7
-    assert:
-      - ran: "docker compose.*up"
-      - ran: "-d|--detach"
-      - ran: "--build"
-
-  - id: multi-stage-debug
-    intent: "Build only the 'builder' stage from /tmp/bench-workspace/Dockerfile, tag it as 'myapp:debug', and don't use cache"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest AS builder\nRUN echo building\nFROM alpine:latest\nCOPY --from=builder / /\n' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 7
-    assert:
-      - ran: "docker build"
-      - ran: "--target.*builder"
-      - ran: "myapp:debug"
-      - ran: "--no-cache"
-      - verify:
-          run: "docker images myapp:debug --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:debug"

package/task_suites/gh.yaml

@@ -1,118 +0,0 @@
-cli: gh
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: list-repos
-    intent: "List all public repositories for the GitHub user 'octocat'"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "gh repo list.*octocat"
-      - ran: "--visibility"
-
-  - id: view-issue
-    intent: "View the details of issue #42 in the repository cli/cli"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "gh issue view.*42"
-      - ran: "--repo.*cli/cli"
-
-  - id: list-prs-json
-    intent: "List open pull requests in the current repo and output as JSON with fields number, title, and author"
-    difficulty: easy
-    category: output
-    max_turns: 3
-    assert:
-      - ran: "gh pr list"
-      - ran: "--json"
-
-  - id: whoami
-    intent: "Show which GitHub account you're currently authenticated as"
-    difficulty: easy
-    category: auth
-    max_turns: 3
-    assert:
-      - ran: "gh auth status"
-
-  - id: set-config
-    intent: "Set the default git protocol to SSH for GitHub CLI"
-    difficulty: easy
-    category: config
-    max_turns: 3
-    assert:
-      - ran: "gh config set.*git_protocol.*ssh"
-
-  # -- Medium ------------------------------------------------------------------
-  - id: create-issue-with-labels
-    intent: "Create a new issue titled 'Login page broken' with labels 'bug' and 'urgent' in the repository myorg/myapp"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "gh issue create"
-      - ran: "--title"
-      - ran: "--label"
-      - ran: "--repo.*myorg/myapp"
-
-  - id: create-release
-    intent: "Create a new release tagged v2.0.0 with title 'Major Release' and auto-generate the release notes"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "gh release create.*v2.0.0"
-      - ran: "--title"
-      - ran: "--generate-notes"
-
-  - id: search-issues
-    intent: "Search for open issues labeled 'good first issue' in the rust-lang/rust repository"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "gh search issues"
-      - ran: "--repo.*rust-lang/rust"
-      - ran: "--label"
-
-  - id: pr-diff
-    intent: "Show the diff for pull request #500 in the current repository"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "gh pr diff.*500"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: review-pr-workflow
-    intent: "Check out pull request #123, then approve it with the comment 'Looks good, tests pass'"
-    difficulty: hard
-    category: workflow
-    max_turns: 7
-    assert:
-      - ran: "gh pr checkout.*123"
-      - ran: "gh pr review.*123.*--approve"
-
-  - id: api-graphql
-    intent: "Use the GitHub API to fetch the description of the repository cli/cli using a GraphQL query"
-    difficulty: hard
-    category: query
-    max_turns: 7
-    assert:
-      - ran: "gh api graphql"
-      - ran: "repository"
-
-  - id: create-repo-full
-    intent: "Create a new private repository called 'my-project' with a README, a .gitignore for Python, and the MIT license"
-    difficulty: hard
-    category: crud
-    max_turns: 7
-    assert:
-      - ran: "gh repo create.*my-project"
-      - ran: "--private"
-      - ran: "--add-readme"
-      - ran: "--gitignore"
-      - ran: "--license"
-      - not_ran: "--public"

package/task_suites/jq.yaml

@@ -1,172 +0,0 @@
-cli: jq
-version_command: "jq --version"
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: pretty-print
-    intent: "Pretty-print the JSON file /tmp/bench-workspace/data.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\",\"age\":30,\"active\":true}' > /tmp/bench-workspace/data.json"
-    max_turns: 2
-    assert:
-      - ran: "jq.*data.json"
-      - exit_code: 0
-      - output_contains: "alice"
-
-  - id: extract-field
-    intent: "Extract the 'name' field from /tmp/bench-workspace/data.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\",\"age\":30}' > /tmp/bench-workspace/data.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*\\.name.*data.json"
-      - output_contains: "alice"
-
-  - id: raw-output
-    intent: "Extract the 'name' field from /tmp/bench-workspace/data.json as a raw string without quotes"
-    difficulty: easy
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\"}' > /tmp/bench-workspace/data.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*-r|--raw-output"
-      - ran: "\\.name"
-
-  - id: array-length
-    intent: "Count the number of items in the JSON array in /tmp/bench-workspace/list.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[1,2,3,4,5]' > /tmp/bench-workspace/list.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*length.*list.json"
-      - output_contains: "5"
-
-  # -- Medium ------------------------------------------------------------------
-  - id: filter-array
-    intent: "From /tmp/bench-workspace/users.json, select only users where age is greater than 25"
-    difficulty: medium
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20},{\"name\":\"carol\",\"age\":35}]' > /tmp/bench-workspace/users.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*select.*age.*25.*users.json"
-      - output_contains: "alice"
-      - output_contains: "carol"
-
-  - id: map-transform
-    intent: "From /tmp/bench-workspace/users.json, extract just the names of all users as a JSON array"
-    difficulty: medium
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20}]' > /tmp/bench-workspace/users.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*\\.name.*users.json|jq.*map.*users.json"
-      - output_contains: "alice"
-      - output_contains: "bob"
-
-  - id: nested-access
-    intent: "From /tmp/bench-workspace/config.json, extract the host field from inside the database object"
-    difficulty: medium
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"database\":{\"host\":\"localhost\",\"port\":5432},\"cache\":{\"host\":\"redis\"}}' > /tmp/bench-workspace/config.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*\\.database\\.host.*config.json"
-      - output_contains: "localhost"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: group-and-count
-    intent: "From /tmp/bench-workspace/events.json, group the events by their 'type' field and count how many of each type there are"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"type\":\"click\",\"page\":\"/home\"},{\"type\":\"view\",\"page\":\"/about\"},{\"type\":\"click\",\"page\":\"/pricing\"},{\"type\":\"view\",\"page\":\"/home\"},{\"type\":\"click\",\"page\":\"/home\"}]' > /tmp/bench-workspace/events.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*group_by.*events.json"
-      - output_contains: "click"
-      - output_contains: "view"
-
-  - id: merge-objects
-    intent: "Merge /tmp/bench-workspace/defaults.json and /tmp/bench-workspace/overrides.json, where overrides take precedence"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"color\":\"red\",\"size\":10,\"debug\":false}' > /tmp/bench-workspace/defaults.json"
-      - "echo '{\"color\":\"blue\",\"verbose\":true}' > /tmp/bench-workspace/overrides.json"
-    max_turns: 7
-    assert:
-      - ran: "jq"
-      - output_contains: "blue"
-      - output_contains: "verbose"
-
-  - id: csv-output
-    intent: "Convert /tmp/bench-workspace/users.json into CSV format with columns name and age"
-    difficulty: hard
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20}]' > /tmp/bench-workspace/users.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*-r|--raw-output"
-      - ran: "@csv|@tsv|join"
-      - output_contains: "alice"
-      - output_contains: "bob"
-
-  - id: recursive-descent
-    intent: "From /tmp/bench-workspace/deep.json, find ALL values of any key named 'id' anywhere in the nested structure, no matter how deep"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"id\":1,\"children\":[{\"id\":2,\"meta\":{\"id\":3}},{\"id\":4}]}' > /tmp/bench-workspace/deep.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*\\.\\..*id|jq.*recurse.*id"
-      - output_contains: "1"
-      - output_contains: "3"
-
-  - id: conditional-update
-    intent: "In /tmp/bench-workspace/items.json, update the price to 0 for all items where the stock is 0, and output the modified JSON"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"A\",\"price\":10,\"stock\":5},{\"name\":\"B\",\"price\":20,\"stock\":0},{\"name\":\"C\",\"price\":15,\"stock\":0}]' > /tmp/bench-workspace/items.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*if.*stock.*then|jq.*select.*stock|jq.*map.*stock"
-      - output_contains: "\"price\":0"
-
-  - id: pivot-table
-    intent: "From /tmp/bench-workspace/sales.json, create an object where each key is a unique region and each value is the total revenue for that region"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"region\":\"US\",\"revenue\":100},{\"region\":\"EU\",\"revenue\":200},{\"region\":\"US\",\"revenue\":150},{\"region\":\"EU\",\"revenue\":50}]' > /tmp/bench-workspace/sales.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*group_by.*region"
-      - output_contains: "US"
-      - output_contains: "250"

package/task_suites/kubectl.yaml

@@ -1,74 +0,0 @@
-cli: kubectl
-version_command: "kubectl version --client --short 2>/dev/null || kubectl version --client -o yaml | head -5"
-
-tasks:
-  - id: get-help
-    intent: "Show help information for kubectl"
-    max_turns: 2
-    assert:
-      - ran: "kubectl.*--help"
-      - output_contains: "kubectl"
-
-  - id: create-namespace-dry-run
-    intent: "Create a namespace called 'test-ns' but only show what would be created, don't actually create it"
-    assert:
-      - output_contains: "test-ns"
-      - ran: "kubectl create"
-      - ran: "dry-run"
-
-  - id: create-deployment-dry-run
-    intent: "Create a deployment named 'nginx-app' using the nginx image, show the YAML output without creating it"
-    assert:
-      - output_contains: "nginx-app"
-      - output_contains: "nginx"
-      - ran: "kubectl create deploy"
-
-  - id: create-configmap-literal
-    intent: "Create a configmap named 'app-config' with database_url=localhost:5432 and debug=true, just show what it would look like"
-    assert:
-      - output_contains: "app-config"
-      - output_contains: "database_url"
-      - output_contains: "debug"
-
-  - id: create-job-with-command
-    intent: "Create a job named 'backup-job' using busybox that runs 'echo backup completed', show the YAML"
-    assert:
-      - output_contains: "backup-job"
-      - output_contains: "busybox"
-      - ran: "kubectl create job"
-
-  - id: create-cronjob-schedule
-    intent: "Create a cronjob named 'daily-cleanup' that runs every day at 2 AM using busybox with command 'rm -rf /tmp/*'"
-    assert:
-      - output_contains: "daily-cleanup"
-      - output_contains: "busybox"
-      - ran: "kubectl create cronjob"
-
-  - id: create-deployment-replicas
-    intent: "Create a deployment named 'web-app' using nginx image with 3 replicas"
-    assert:
-      - output_contains: "web-app"
-      - output_contains: "replicas"
-      - ran: "kubectl create deploy"
-
-  - id: create-configmap-from-file
-    intent: "Create a configmap named 'nginx-config' from the file at /tmp/bench-workspace/nginx.conf"
-    setup:
-      - "mkdir -p /tmp/bench-workspace && echo 'server { listen 80; }' > /tmp/bench-workspace/nginx.conf"
-    assert:
-      - output_contains: "nginx-config"
-      - ran: "kubectl create configmap"
-
-  - id: create-clusterrole-permissions
-    intent: "Create a cluster role named 'pod-reader' that allows get, list, and watch on pods"
-    assert:
-      - output_contains: "pod-reader"
-      - ran: "kubectl create clusterrole"
-
-  - id: create-ingress-complex
-    intent: "Create an ingress named 'web-ingress' routing foo.com/api to api-service:8080 and foo.com/web to web-service:80"
-    difficulty: hard
-    assert:
-      - output_contains: "web-ingress"
-      - output_contains: "foo.com"
-      - ran: "kubectl create ingress"