@cliwatch/cli-bench 0.6.3 → 0.7.1

package/task_suites/gh.yaml

@@ -1,118 +0,0 @@
-cli: gh
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: list-repos
-    intent: "List all public repositories for the GitHub user 'octocat'"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "gh repo list.*octocat"
-      - ran: "--visibility"
-
-  - id: view-issue
-    intent: "View the details of issue #42 in the repository cli/cli"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "gh issue view.*42"
-      - ran: "--repo.*cli/cli"
-
-  - id: list-prs-json
-    intent: "List open pull requests in the current repo and output as JSON with fields number, title, and author"
-    difficulty: easy
-    category: output
-    max_turns: 3
-    assert:
-      - ran: "gh pr list"
-      - ran: "--json"
-
-  - id: whoami
-    intent: "Show which GitHub account you're currently authenticated as"
-    difficulty: easy
-    category: auth
-    max_turns: 3
-    assert:
-      - ran: "gh auth status"
-
-  - id: set-config
-    intent: "Set the default git protocol to SSH for GitHub CLI"
-    difficulty: easy
-    category: config
-    max_turns: 3
-    assert:
-      - ran: "gh config set.*git_protocol.*ssh"
-
-  # -- Medium ------------------------------------------------------------------
-  - id: create-issue-with-labels
-    intent: "Create a new issue titled 'Login page broken' with labels 'bug' and 'urgent' in the repository myorg/myapp"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "gh issue create"
-      - ran: "--title"
-      - ran: "--label"
-      - ran: "--repo.*myorg/myapp"
-
-  - id: create-release
-    intent: "Create a new release tagged v2.0.0 with title 'Major Release' and auto-generate the release notes"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "gh release create.*v2.0.0"
-      - ran: "--title"
-      - ran: "--generate-notes"
-
-  - id: search-issues
-    intent: "Search for open issues labeled 'good first issue' in the rust-lang/rust repository"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "gh search issues"
-      - ran: "--repo.*rust-lang/rust"
-      - ran: "--label"
-
-  - id: pr-diff
-    intent: "Show the diff for pull request #500 in the current repository"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "gh pr diff.*500"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: review-pr-workflow
-    intent: "Check out pull request #123, then approve it with the comment 'Looks good, tests pass'"
-    difficulty: hard
-    category: workflow
-    max_turns: 7
-    assert:
-      - ran: "gh pr checkout.*123"
-      - ran: "gh pr review.*123.*--approve"
-
-  - id: api-graphql
-    intent: "Use the GitHub API to fetch the description of the repository cli/cli using a GraphQL query"
-    difficulty: hard
-    category: query
-    max_turns: 7
-    assert:
-      - ran: "gh api graphql"
-      - ran: "repository"
-
-  - id: create-repo-full
-    intent: "Create a new private repository called 'my-project' with a README, a .gitignore for Python, and the MIT license"
-    difficulty: hard
-    category: crud
-    max_turns: 7
-    assert:
-      - ran: "gh repo create.*my-project"
-      - ran: "--private"
-      - ran: "--add-readme"
-      - ran: "--gitignore"
-      - ran: "--license"
-      - not_ran: "--public"

package/task_suites/jq.yaml

@@ -1,172 +0,0 @@
-cli: jq
-version_command: "jq --version"
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: pretty-print
-    intent: "Pretty-print the JSON file /tmp/bench-workspace/data.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\",\"age\":30,\"active\":true}' > /tmp/bench-workspace/data.json"
-    max_turns: 2
-    assert:
-      - ran: "jq.*data.json"
-      - exit_code: 0
-      - output_contains: "alice"
-
-  - id: extract-field
-    intent: "Extract the 'name' field from /tmp/bench-workspace/data.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\",\"age\":30}' > /tmp/bench-workspace/data.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*\\.name.*data.json"
-      - output_contains: "alice"
-
-  - id: raw-output
-    intent: "Extract the 'name' field from /tmp/bench-workspace/data.json as a raw string without quotes"
-    difficulty: easy
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"name\":\"alice\"}' > /tmp/bench-workspace/data.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*-r|--raw-output"
-      - ran: "\\.name"
-
-  - id: array-length
-    intent: "Count the number of items in the JSON array in /tmp/bench-workspace/list.json"
-    difficulty: easy
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[1,2,3,4,5]' > /tmp/bench-workspace/list.json"
-    max_turns: 3
-    assert:
-      - ran: "jq.*length.*list.json"
-      - output_contains: "5"
-
-  # -- Medium ------------------------------------------------------------------
-  - id: filter-array
-    intent: "From /tmp/bench-workspace/users.json, select only users where age is greater than 25"
-    difficulty: medium
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20},{\"name\":\"carol\",\"age\":35}]' > /tmp/bench-workspace/users.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*select.*age.*25.*users.json"
-      - output_contains: "alice"
-      - output_contains: "carol"
-
-  - id: map-transform
-    intent: "From /tmp/bench-workspace/users.json, extract just the names of all users as a JSON array"
-    difficulty: medium
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20}]' > /tmp/bench-workspace/users.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*\\.name.*users.json|jq.*map.*users.json"
-      - output_contains: "alice"
-      - output_contains: "bob"
-
-  - id: nested-access
-    intent: "From /tmp/bench-workspace/config.json, extract the host field from inside the database object"
-    difficulty: medium
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"database\":{\"host\":\"localhost\",\"port\":5432},\"cache\":{\"host\":\"redis\"}}' > /tmp/bench-workspace/config.json"
-    max_turns: 5
-    assert:
-      - ran: "jq.*\\.database\\.host.*config.json"
-      - output_contains: "localhost"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: group-and-count
-    intent: "From /tmp/bench-workspace/events.json, group the events by their 'type' field and count how many of each type there are"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"type\":\"click\",\"page\":\"/home\"},{\"type\":\"view\",\"page\":\"/about\"},{\"type\":\"click\",\"page\":\"/pricing\"},{\"type\":\"view\",\"page\":\"/home\"},{\"type\":\"click\",\"page\":\"/home\"}]' > /tmp/bench-workspace/events.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*group_by.*events.json"
-      - output_contains: "click"
-      - output_contains: "view"
-
-  - id: merge-objects
-    intent: "Merge /tmp/bench-workspace/defaults.json and /tmp/bench-workspace/overrides.json, where overrides take precedence"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"color\":\"red\",\"size\":10,\"debug\":false}' > /tmp/bench-workspace/defaults.json"
-      - "echo '{\"color\":\"blue\",\"verbose\":true}' > /tmp/bench-workspace/overrides.json"
-    max_turns: 7
-    assert:
-      - ran: "jq"
-      - output_contains: "blue"
-      - output_contains: "verbose"
-
-  - id: csv-output
-    intent: "Convert /tmp/bench-workspace/users.json into CSV format with columns name and age"
-    difficulty: hard
-    category: output
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"alice\",\"age\":30},{\"name\":\"bob\",\"age\":20}]' > /tmp/bench-workspace/users.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*-r|--raw-output"
-      - ran: "@csv|@tsv|join"
-      - output_contains: "alice"
-      - output_contains: "bob"
-
-  - id: recursive-descent
-    intent: "From /tmp/bench-workspace/deep.json, find ALL values of any key named 'id' anywhere in the nested structure, no matter how deep"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"id\":1,\"children\":[{\"id\":2,\"meta\":{\"id\":3}},{\"id\":4}]}' > /tmp/bench-workspace/deep.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*\\.\\..*id|jq.*recurse.*id"
-      - output_contains: "1"
-      - output_contains: "3"
-
-  - id: conditional-update
-    intent: "In /tmp/bench-workspace/items.json, update the price to 0 for all items where the stock is 0, and output the modified JSON"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"name\":\"A\",\"price\":10,\"stock\":5},{\"name\":\"B\",\"price\":20,\"stock\":0},{\"name\":\"C\",\"price\":15,\"stock\":0}]' > /tmp/bench-workspace/items.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*if.*stock.*then|jq.*select.*stock|jq.*map.*stock"
-      - output_contains: "\"price\":0"
-
-  - id: pivot-table
-    intent: "From /tmp/bench-workspace/sales.json, create an object where each key is a unique region and each value is the total revenue for that region"
-    difficulty: hard
-    category: query
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '[{\"region\":\"US\",\"revenue\":100},{\"region\":\"EU\",\"revenue\":200},{\"region\":\"US\",\"revenue\":150},{\"region\":\"EU\",\"revenue\":50}]' > /tmp/bench-workspace/sales.json"
-    max_turns: 7
-    assert:
-      - ran: "jq.*group_by.*region"
-      - output_contains: "US"
-      - output_contains: "250"

package/task_suites/kubectl.yaml

@@ -1,74 +0,0 @@
-cli: kubectl
-version_command: "kubectl version --client --short 2>/dev/null || kubectl version --client -o yaml | head -5"
-
-tasks:
-  - id: get-help
-    intent: "Show help information for kubectl"
-    max_turns: 2
-    assert:
-      - ran: "kubectl.*--help"
-      - output_contains: "kubectl"
-
-  - id: create-namespace-dry-run
-    intent: "Create a namespace called 'test-ns' but only show what would be created, don't actually create it"
-    assert:
-      - output_contains: "test-ns"
-      - ran: "kubectl create"
-      - ran: "dry-run"
-
-  - id: create-deployment-dry-run
-    intent: "Create a deployment named 'nginx-app' using the nginx image, show the YAML output without creating it"
-    assert:
-      - output_contains: "nginx-app"
-      - output_contains: "nginx"
-      - ran: "kubectl create deploy"
-
-  - id: create-configmap-literal
-    intent: "Create a configmap named 'app-config' with database_url=localhost:5432 and debug=true, just show what it would look like"
-    assert:
-      - output_contains: "app-config"
-      - output_contains: "database_url"
-      - output_contains: "debug"
-
-  - id: create-job-with-command
-    intent: "Create a job named 'backup-job' using busybox that runs 'echo backup completed', show the YAML"
-    assert:
-      - output_contains: "backup-job"
-      - output_contains: "busybox"
-      - ran: "kubectl create job"
-
-  - id: create-cronjob-schedule
-    intent: "Create a cronjob named 'daily-cleanup' that runs every day at 2 AM using busybox with command 'rm -rf /tmp/*'"
-    assert:
-      - output_contains: "daily-cleanup"
-      - output_contains: "busybox"
-      - ran: "kubectl create cronjob"
-
-  - id: create-deployment-replicas
-    intent: "Create a deployment named 'web-app' using nginx image with 3 replicas"
-    assert:
-      - output_contains: "web-app"
-      - output_contains: "replicas"
-      - ran: "kubectl create deploy"
-
-  - id: create-configmap-from-file
-    intent: "Create a configmap named 'nginx-config' from the file at /tmp/bench-workspace/nginx.conf"
-    setup:
-      - "mkdir -p /tmp/bench-workspace && echo 'server { listen 80; }' > /tmp/bench-workspace/nginx.conf"
-    assert:
-      - output_contains: "nginx-config"
-      - ran: "kubectl create configmap"
-
-  - id: create-clusterrole-permissions
-    intent: "Create a cluster role named 'pod-reader' that allows get, list, and watch on pods"
-    assert:
-      - output_contains: "pod-reader"
-      - ran: "kubectl create clusterrole"
-
-  - id: create-ingress-complex
-    intent: "Create an ingress named 'web-ingress' routing foo.com/api to api-service:8080 and foo.com/web to web-service:80"
-    difficulty: hard
-    assert:
-      - output_contains: "web-ingress"
-      - output_contains: "foo.com"
-      - ran: "kubectl create ingress"