@cognite/cli 1.9.0-alpha.3 → 1.10.0

package/README.md

@@ -12,10 +12,6 @@ npx @cognite/cli apps create
 
 This prompts for your app name, org, project, and cluster, then generates a fully configured React + TypeScript project.
 
-## Feature Flags
-
-Set environment variable `COGNITE_ALPHA_ENABLE_SESSION_AUTH` to true in order to enable session authentication enabling auth login and logout commands which will let you skip browser login by securely persisting access tokens on your machine.
-
 ## Authentication
 
 New apps created with `npx @cognite/cli apps create` depend on [`@cognite/app-sdk`](https://www.npmjs.com/package/@cognite/app-sdk) — **not `@cognite/cli`** — for auth and host integration. `@cognite/cli` is the CLI used to scaffold, develop, and deploy the app; the generated app itself talks to the Fusion app host via `@cognite/app-sdk`'s Comlink handshake. The template wires this up for you.
@@ -28,6 +24,14 @@ Deploy interactively via browser OAuth:
 npx @cognite/cli apps deploy --interactive
 ```
 
+For local non-interactive deploys, sign in once to persist a refreshable
+session — `apps deploy` picks it up automatically:
+
+```bash
+npx @cognite/cli auth login
+npx @cognite/cli apps deploy
+```
+
 For CI, set your client secret as an environment variable and run:
 
 ```bash
@@ -69,6 +73,7 @@ Browse available skills at [cognitedata/builder-skills](https://github.com/cogni
 ## Requirements
 
 - Node.js ≥ 20
+- npm ≥ 11.10.0 to install/build generated apps — enforced via `engines.npm` + `engine-strict=true` in the generated app's `.npmrc`. Not required for running `@cognite/cli` itself (e.g. via `npx`).
 - React ≥ 18 (optional peer dependency — only needed for auth components)
 
 ## Telemetry
@@ -144,6 +149,7 @@ MOCK_SCENARIO=403 pnpm mock:server
 | `republish` | `ensureApp` recovers (409), then `uploadVersion` fails (409 — version already published) |
 
 The mock server and MSW handlers live in `cli/testing/msw/`. The same handlers are used by the Vitest integration tests in `src/deploy/apphosting-deployer.msw.test.ts`.
+
 ## Maintenance
 
 ### Updating the spec-kit vendor snapshot

package/_templates/agents/create/AGENTS_TEMPLATE.md

@@ -0,0 +1,179 @@
+# Agent Project — AI Development Guide
+
+This file is for **AI coding assistants** (Cursor, Claude Code, etc.) working in this
+repository. Humans should start with `README.md`.
+
+## Project layout
+
+| Path | Purpose |
+|------|---------|
+| `<name>.agent.yaml` | Agent definition — read this file for current field values |
+| `eval/eval.yaml` | Eval cases and optional `include:` fragments under `eval/` |
+| `README.md` | Human-oriented quick start and workflow |
+
+## Reference documentation
+
+For full field schemas, tool types, scorer details, and CLI flags beyond what this
+file covers, see the Cognite CLI agents documentation:
+
+- [Cognite CLI — Agents](https://docs.cognite.com/dev/sdks/cognite-cli/agents)
+- [Eval cases JSON Schema](https://docs.cognite.com/assets/schemas/agent-eval-cases.schema.json)
+
+## CLI validation
+
+The CLI validates both `<name>.agent.yaml` and `eval/eval.yaml` at runtime. Schema errors
+surface before commands execute — you do not need to validate YAML manually:
+
+- **`<name>.agent.yaml`** is parsed and checked when you run `push`, `status`, `eval`, or
+  other commands that read the project.
+- **`eval/eval.yaml`** (and any `include:` fragments) is validated at the start of
+  `cognite agents eval` before cases run.
+
+Use the scaffolded templates and examples as your reference for valid field shapes.
+
+## Design principle: discover via tools
+
+Agents should **retrieve** answers with declared tools, not rely on eval YAML to inject
+facts into the chat. Evals should validate that behavior.
+
+- Use `dataModels`, `instanceSpaces`, and `appContext` to scope **where** the agent
+  operates (plant, site, spaces) — not to paste answer text the agent should have found.
+- **`faithfulness` `groundTruth`** is supplied to the **LLM judge only**. It does not replace
+  tool use when you are testing retrieval.
+
+**Do not** put expected answer facts in `appContext` to make a discovery test pass:
+
+```yaml
+# Wrong — agent sees the answer without using tools
+appContext: "Pump P-101 was shut down on 2024-03-15."
+turns:
+  - input: "When was P-101 shut down?"
+    scorers:
+      - type: faithfulness
+        groundTruth: "Pump P-101 was shut down on 2024-03-15."
+```
+
+```yaml
+# Right — judge checks grounding; agent must still retrieve in production
+turns:
+  - input: "When was P-101 shut down?"
+    scorers:
+      - type: faithfulness
+        groundTruth: |
+          Pump P-101 was shut down on 2024-03-15 due to bearing wear.
+```
+
+## Two-layer prompts
+
+- **`instructions`** in `<name>.agent.yaml` is the builder-editable system guidance for this agent.
+- The platform may apply additional system behavior that is not editable in this file.
+
+## Workflow
+
+`create` → edit `<name>.agent.yaml` → `cognite agents push` → `cognite agents open` (manual test)
+→ `cognite agents eval` → `cognite agents publish` when ready.
+
+Other commands: `pull`, `list`, `status`, `unpublish`. See `README.md` for examples.
+
+**Eval runs against deployed config.** Push first, or run `cognite agents eval --upsert`.
+To evaluate a deployed agent without a local `<name>.agent.yaml` (e.g. Cognite system agents
+you cannot pull), use `cognite agents eval --system-agent` or
+`cognite agents eval --external-id <id>` — only `eval/eval.yaml` is required on disk.
+`--system-agent` and `--external-id` cannot be used together.
+Auth: `cognite auth login`; optional env vars are documented in the CLI package
+`env.example` (do not commit secrets).
+
+## Eval essentials
+
+- Cases live in `eval/eval.yaml`; split suites with `include:` (paths relative to `eval/`).
+- Filter runs: `--tag`, `--case` (repeatable), `--fail-fast`, `-v`.
+- Use `--system-agent` to evaluate the Cognite-managed system agent without `<name>.agent.yaml`,
+  or `--external-id <id>` for any other deployed agent. These flags skip `<name>.agent.yaml`
+  entirely — only `eval/eval.yaml` is required on disk.
+- Multi-turn cases thread the conversation cursor; scorers run on the turns that declare them.
+
+| Scorer | Use when | Required / notable fields |
+|--------|----------|---------------------------|
+| `correctness` | General answer quality | `criteria` |
+| `faithfulness` | Output must match judge-supplied facts | `groundTruth` |
+| `toolSelection` | Whether the agent picked the right tools (names only) | Optional `tools`; use runtime names (see below) |
+| `toolInvocation` | Whether tool arguments match the question | Optional `tools`; put parameter details in each tool `description` (see below) |
+
+Phoenix metric docs: [Correctness](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/correctness),
+[Faithfulness](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/faithfulness),
+[Tool Selection](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/tool-selection),
+[Tool Invocation](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/tool-invocation).
+
+### Tool scorers: `tools` field and gotchas
+
+`toolSelection` and `toolInvocation` need a resolved tool list for the LLM judge.
+The judge does a **literal name membership check** — if the names don't match what
+the agent actually called, the scorer will report `incorrect` even when tool use
+was appropriate.
+
+**Eval `tools` are NOT `<name>.agent.yaml` tools.** `<name>.agent.yaml` declares tools for the
+agent runtime (config-time names like `query`). Eval `tools` declare what the
+*judge* should expect the agent to *call* at runtime (e.g. `find_assets`, `execute`).
+These are often different names.
+
+**Three categories of tools:**
+
+- **Custom tools** (user-defined): config name = runtime name. Tool scorers work
+  with tools from `<name>.agent.yaml` — no eval `tools` needed.
+- **Cognite platform tools** (`query`, `ask_document`, etc.): the runtime decomposes
+  these into sub-tools (`find_assets`, `list_views`) whose names differ from config.
+  Declare runtime tool names in eval `tools`, or use `correctness`/`faithfulness`
+  to evaluate outcomes instead.
+- **System tools** (sandbox `execute`, `commandTool`): never in any config. Declare
+  them in eval `tools`, or use `correctness`.
+
+**Discovering runtime tool names:** Run eval once with `-v` or `--report-dir` and
+look at the `toolsCalled:` output to see what the agent actually invoked.
+
+**Tool list cascade** (most specific wins, whole-array replace — not merge-by-name):
+
+```
+scorer.tools → case.tools → included-file.tools → root eval.yaml tools → <name>.agent.yaml tools
+```
+
+With `--external-id` or `--system-agent`, `<name>.agent.yaml` is skipped entirely —
+declare eval `tools` at root, file, case, or scorer level.
+
+**Empty resolution skips the scorer.** If no tools resolve after the full cascade,
+the scorer is skipped with a stderr warning (not a misleading "incorrect" verdict).
+Other scorers on the same turn still run. A case whose *only* scorers were all
+skipped fails the verdict (zero scores).
+
+**`toolInvocation` description quality:** The judge evaluates arguments against
+whatever you put in each tool's `description`. Include parameter details for
+accurate invocation scoring:
+
+```yaml
+tools:
+  - name: find_assets
+    description: |
+      Find assets in the knowledge graph.
+      Parameters: filter (string, required), limit (integer, optional).
+```
+
+Without parameter details, the judge can only check whether arguments "seem
+reasonable" for the user's question — it cannot verify schema compliance.
+
+## Eval YAML inheritance
+
+Fields resolve at three levels for session context and case-level tools: root
+`eval/eval.yaml` → included file → individual case. Scorer-level `tools` on a turn
+override case/file/root (and `<name>.agent.yaml` fallback) for that scorer only.
+
+- **`tags`** — union (file-level tags merge into each case's tags, deduplicated).
+- **`dataModels`, `instanceSpaces`, `appContext`** — per-field replace (most specific wins; no deep merge).
+- **`tools`** — whole-array replace at each level (not merge-by-name). Used by
+  `toolSelection` / `toolInvocation`; does not change what tools the agent can call —
+  only what the judge considers when scoring.
+
+## Conventions for AI assistants
+
+- Minimize scope when editing YAML; match existing case `id` style and tags.
+- Do not commit credentials; use environment variables.
+- For detailed eval authoring or debugging workflows, use project skills under
+  `.cursor/skills` or `.claude/skills` when they are present.

package/_templates/agents/create/EVAL_CASES_TEMPLATE.yaml

@@ -1,77 +1,41 @@
 # Eval cases for {{displayName}}
-#
-# Each case is a conversation with the agent plus one or more "scorers" that
-# judge the responses. An LLM judge grades every scored turn. Run the suite with:
-#
-#   cognite agents eval
-#
-# Supported scorer types:
-#   - correctness:   compares the answer to a reference description you provide
-#                    (`reference`). Use it when you can describe a good answer.
-#   - faithfulness:  checks the answer is grounded in the supplied `context` and
-#                    does not hallucinate. Use it for retrieval / grounded answers.
-#   - toolSelection: checks whether the agent picked an appropriate tool (or
-#                    correctly used none) for the question, based on the tools
-#                    configured in agent.yaml. Requires no extra fields, but only
-#                    makes sense once you've added tools (see README.md's
-#                    "Adding tools" section).
-#   - toolInvocation: checks whether the agent invoked tools with correct
-#                    arguments and formatting. Complements toolSelection — use
-#                    both when you need full tool-calling coverage. Requires no
-#                    extra fields.
-#
-# These are placeholders — edit the inputs, references, and context to match what
-# your agent actually does, then add more cases over time.
+# Run with: cognite agents eval
+# Add more cases and scorer types as you add tools. See README.md.
 
 # To split cases across multiple files as your suite grows, add:
 #   include:
 #     - cases/maintenance.yaml
 
+# Optional: tools for toolSelection / toolInvocation scorers (judge rubric — NOT the same
+# as <name>.agent.yaml tools). <name>.agent.yaml configures the runtime (e.g. query); eval tools use
+# names the agent actually calls (e.g. find_assets, execute). Omit when <name>.agent.yaml names
+# match runtime calls; required when using --external-id / --system-agent.
+# tools:
+#   - name: find_assets
+#     description: |
+#       Find assets in the knowledge graph.
+#       Parameters: filter (string, required), limit (integer, optional).
+
 cases:
-  # Single-turn case scored for correctness.
+  # Single-turn case: the agent should explain what it can help with.
   - id: greeting
     turns:
       - input: "Hi, what can you help me with?"
         scorers:
           - type: correctness
-            # Describe what a good answer looks like; the judge compares against this.
-            reference: >-
+            criteria: >-
               A friendly greeting that briefly explains what this agent can help
               the user with.
 
-  # Single-turn case scored for faithfulness against supplied context.
-  - id: grounded-answer
-    turns:
-      - input: "Where is the main compressor located?"
-        scorers:
-          - type: faithfulness
-            # The answer must be grounded in this context and not invent facts.
-            context: >-
-              The main compressor (unit C-101) is installed on Deck 2 of the
-              North platform, next to the gas separation train.
-
-  # Single-turn case scored for tool selection. Assumes you've added the
-  # `find_assets` tool from README.md's "Adding tools" example — the judge
-  # checks whether the agent picked an appropriate tool (or none) based on
-  # the tools listed in agent.yaml.
-  - id: find-assets-tool
-    turns:
-      - input: "Find assets related to compressors in the knowledge graph."
-        scorers:
-          - type: toolSelection
-          - type: toolInvocation
-
   # Multi-turn case: the agent should carry context across turns.
-  - id: assets-followup
+  - id: follow-up
     turns:
-      - input: "List the assets in the cooling system."
+      - input: "What can you help me with?"
         scorers:
           - type: correctness
-            reference: "Lists the assets that belong to the cooling system."
-      - input: "Now show only the ones that are currently active."
+            criteria: "Describes the agent's capabilities."
+      - input: "Can you elaborate on the first thing you mentioned?"
         scorers:
           - type: correctness
-            # Relies on the previous turn — the judge sees the earlier turns as context.
-            reference: >-
-              Narrows the previously listed cooling-system assets down to only
-              the active ones.
+            criteria: >-
+              Expands on a capability mentioned in the previous response.

package/_templates/agents/create/README_TEMPLATE.md

@@ -1,6 +1,6 @@
 # {{displayName}}
 
-> Edit `agent.yaml` to configure your agent — tools, model, instructions.
+> Edit `<name>.agent.yaml` to configure your agent — tools, model, instructions.
 
 ## Quick start
 
@@ -19,24 +19,28 @@ cognite agents publish
 
 | Path | Purpose |
 |------|---------|
-| `agent.yaml` | Agent definition (externalId, tools, model, instructions) |
+| `<name>.agent.yaml` | Agent definition (externalId, tools, model, instructions), named after this folder |
 | `eval/eval.yaml` | Example eval cases — run with `cognite agents eval` |
 | `README.md` | This file |
 
 ## Evaluating the agent
 
 `eval/eval.yaml` contains starter test cases (single-turn and multi-turn) that
-grade the agent's responses with an LLM judge. After pushing the agent, run:
+grade the agent's responses with an LLM judge. The default cases are designed to
+pass on a freshly pushed agent — push first, then run:
 
 ```bash
 cognite agents eval
 ```
 
 Edit the cases to match what your agent does, and add more over time.
+Add tools in `<name>.agent.yaml` before adding tool-scorer eval cases (`toolSelection`, `toolInvocation`).
 
 ## Adding tools
 
-Edit the `tools` array in `agent.yaml`. Available tool types:
+Edit the `tools` array in `<name>.agent.yaml`. Common tool types (illustrative — may not
+reflect all available types; check the Cognite documentation or the agent builder
+UI in Fusion for the latest list):
 
 - `analyzeData` — analyze tabular or structured data
 - `analyzeImage` — analyze images and P&ID diagrams
@@ -74,5 +78,8 @@ tools:
 
 ## Deployment with Toolkit
 
-The generated `agent.yaml` is compatible with [Cognite Toolkit](https://docs.cognite.com/cdf/deploy/toolkit/).
-Place it in your Toolkit module under `agents/` and deploy with `cdf deploy`.
+The generated `<name>.agent.yaml` is compatible with [Cognite Toolkit](https://docs.cognite.com/cdf/deploy/toolkit/).
+Place it in your Toolkit module under `agents/` and deploy with `cdf deploy`. Toolkit
+treats the `<name>` part as a label and reads the `externalId` from inside the file,
+so you can rename the file freely — the CLI keeps working as long as the folder holds
+one definition.

package/_templates/agents/create/skills/check-agent/SKILL.md

@@ -0,0 +1,44 @@
+---
+name: check-agent
+description: >-
+  Review <name>.agent.yaml and eval setup locally — description, instructions, tool alignment,
+  and common eval mismatches. Use when reviewing an agent, linting config, or asking
+  if anything looks wrong before push or publish.
+---
+
+# Check agent
+
+Produce a short markdown report from **local files only**. Do not call CDF APIs unless the user explicitly asks; for deployed state, suggest `cognite agents status`.
+
+## Workflow
+
+1. Read `<name>.agent.yaml` — `name`, `description`, `instructions`, `tools`, `exampleQuestions`, labels (e.g. published).
+2. Read `eval/eval.yaml` and any `include:` files under `eval/`.
+3. Read `AGENTS.md` when you need eval or prompt conventions.
+4. Run checks and label each finding **error**, **warn**, or **info** with a concrete fix:
+
+| Check | Severity | Rule |
+|-------|----------|------|
+| `description` missing or empty | warn | Add a clear user-facing description. |
+| `instructions` empty or obvious scaffold placeholder | warn | Replace with real guidance (see write-instructions skill). |
+| Tool named in `instructions` but not in `tools[]` | error | Add the tool or remove the mention. |
+| Tool in `tools[]` never mentioned in `instructions` | info | Document when/how to use it. |
+| `toolSelection` / `toolInvocation` in eval but no tools resolve (no eval `tools` at any cascade level and no `<name>.agent.yaml` `tools`) | error | Add a `tools` field at root, file, case, or scorer level in eval yaml, or add tools to `<name>.agent.yaml`. With `--external-id`/`--system-agent`, `<name>.agent.yaml` is not loaded. |
+| `labels` includes published but `exampleQuestions` empty | warn | Add starter questions for Fusion. |
+| Eval case ids duplicated | error | Rename ids in `eval/eval.yaml`. |
+
+5. Optionally skim `README.md` for project-specific tool setup the checks above cannot see.
+
+Keep the report scannable: bullet list grouped by severity, then suggested command order (`check-agent` → edit → `push` → `eval`).
+
+## Anti-patterns
+
+- Do not block on network or credentials for a default review.
+- Do not rewrite `instructions` in full during a review — flag issues and offer the write-instructions skill if the user wants a rewrite.
+- Do not invent tool config schemas — point to `README.md` and `<name>.agent.yaml` examples.
+
+## Cross-references
+
+- To improve `instructions` → **write-instructions** skill, then re-run this check.
+- To add or fix eval coverage → **write-eval-case** skill.
+- After a failed eval → **debug-eval** skill.

package/_templates/agents/create/skills/debug-eval/SKILL.md

@@ -0,0 +1,36 @@
+---
+name: debug-eval
+description: >-
+  Diagnose Cognite agent eval failures from CLI output — config drift, deployment,
+  invoke errors, and scorer rationales. Use when eval failed, a scorer failed, or
+  the user pastes eval run output.
+---
+
+# Debug eval
+
+Work through failures systematically using the user’s eval output (paste or summary).
+
+## Workflow
+
+1. Identify the failing **case id**, **turn**, and **scorer type** (if any). Note exit code and whether the whole suite or one case failed.
+2. **Config drift** — local `<name>.agent.yaml` / `eval/eval.yaml` may differ from what CDF runs. Suggest `cognite agents push` or `cognite agents eval --upsert` so eval matches the project on disk.
+3. **Not deployed** — if push was never done or status is stale, suggest `cognite agents push` and `cognite agents status` (run `cognite agents status --help` for flags).
+4. **Invoke / runtime errors** — auth, routing, or agent errors before scoring. Suggest `cognite auth login`, confirm project/cluster flags, and re-run with verbosity from `cognite agents eval --help` (e.g. `-v`).
+5. **Scorer failures** — read the judge rationale. For `correctness`, compare the response to `criteria`. For `faithfulness`, check `groundTruth` vs actual tool-retrieved facts. For tool scorers, check where the tool list resolved from (scorer → case → file → root `eval/eval.yaml` → `<name>.agent.yaml`); confirm runtime tool names match what the agent called (`toolsCalled:` output). A "Skipping … no tools available" warning means no tools resolved at all — add a `tools` field at the appropriate cascade level.
+   - **Common gotcha:** `<name>.agent.yaml` declares config-time names (e.g. `query`) but the agent calls runtime sub-tools (e.g. `find_assets`). Override with runtime names in eval `tools`.
+   - **`--external-id` / `--system-agent`:** `<name>.agent.yaml` tools are not loaded; tools must come from eval yaml.
+6. **Narrow the run** — suggest isolating with `--case` (repeatable), `--tag`, or `--fail-fast` per `cognite agents eval --help`.
+7. For scorer semantics or field meaning, read **AGENTS.md** rather than guessing.
+
+Summarize likely root cause, concrete next step, and whether the fix is config, deployment, auth, instructions, or the eval case itself.
+
+## Anti-patterns
+
+- Do not assume eval uses local YAML without push/`--upsert` unless the user confirmed deployment state.
+- Do not change `criteria` or `groundTruth` to match a bad hallucination — fix retrieval, instructions, or the case intent.
+- Do not list every CLI flag from memory — delegate to `--help`.
+
+## Cross-references
+
+- If the case design or scorers are wrong → **write-eval-case** skill.
+- If the agent behavior but instructions are weak → **write-instructions** skill, then **check-agent** for alignment.

package/_templates/agents/create/skills/write-eval-case/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: write-eval-case
+description: >-
+  Guide authoring eval cases in eval/eval.yaml — scorer choice, multi-turn patterns,
+  and YAML structure. Use when adding tests, extending the eval suite, or validating
+  agent behavior for a Cognite agent project.
+---
+
+# Write eval case
+
+Help the user add or edit cases in `eval/eval.yaml` (or an `include:` fragment under `eval/`).
+
+## Workflow
+
+1. Read `<name>.agent.yaml` — note declared tools, model, and what the agent is meant to do.
+2. Read `AGENTS.md` — use the eval essentials table for scorer types and required fields, plus inheritance rules for `tags`, `dataModels`, `instanceSpaces`, and `appContext`.
+3. Ask what behavior to test: single-turn vs multi-turn, happy path vs edge case, and whether tool use should be exercised.
+4. Choose scorers using the AGENTS.md table:
+   - `correctness` when a good answer can be described in `criteria`.
+   - `faithfulness` when grounding against judge-only `groundTruth` matters (retrieval / facts).
+   - `toolSelection` / `toolInvocation` when tools are available via the cascade: scorer `tools` → case `tools` → file `tools` → root `eval/eval.yaml` `tools` → `<name>.agent.yaml` `tools`. Use runtime names (e.g. `find_assets`, `execute`), not config-time names (e.g. `query`). For `toolInvocation`, include parameter details in each tool's `description`.
+5. Draft YAML: unique `id`, optional `tags`, one or more `turns` with `input` and `scorers` on the turns you want judged.
+6. Validate locally: every case has at least one scored turn; no duplicate `id` values across the suite; tool scorers have tools resolvable via the cascade (eval yaml or `<name>.agent.yaml`).
+7. Remind the user that eval runs against **deployed** config — run `cognite agents push` first, or use `cognite agents eval --upsert`. For run filters and flags, run `cognite agents eval --help`.
+
+Match existing `id` and tag style in the file. Prefer small, focused cases over one huge conversation.
+
+## Anti-patterns
+
+- Do not put answer facts in `appContext` so the agent passes without using tools — see AGENTS.md “discover via tools”.
+- Do not add `toolSelection` or `toolInvocation` when no tools resolve from the cascade (eval yaml levels + `<name>.agent.yaml`). If tools are empty, these scorers are skipped with a warning — not scored as "incorrect."
+- Do not use `<name>.agent.yaml` config-time names (e.g. `query`) in eval `tools` when the agent calls different runtime sub-tools (e.g. `find_assets`). Run eval with `-v` to discover actual tool names.
+- Do not duplicate case `id` values or leave turns unscored when the user expects a pass/fail result.
+- Do not inline full scorer schemas in chat — point to `AGENTS.md` and the scaffolded examples in `eval/eval.yaml`.
+
+## Cross-references
+
+- If the agent cannot pass because instructions are vague or wrong → use the **write-instructions** skill, then re-run eval.
+- If a run fails and the case looks correct → use the **debug-eval** skill.

package/_templates/agents/create/skills/write-instructions/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: write-instructions
+description: >-
+  Guide writing or improving agent instructions in <name>.agent.yaml — purpose, constraints,
+  tool usage, and response style. Use when improving the system prompt, clarifying
+  behavior, or aligning instructions with declared tools.
+---
+
+# Write instructions
+
+Edit the `instructions` field in `<name>.agent.yaml`. Platform system behavior may apply on top; you only control the builder-editable layer.
+
+## Workflow
+
+1. Read `<name>.agent.yaml` — current `instructions`, `description`, `tools`, and any `appContext`-related fields you should stay consistent with.
+2. Read `AGENTS.md` — **Two-layer prompts** (your instructions vs platform behavior) and **discover via tools** (agents should retrieve with tools, not rely on hidden cheat sheets).
+3. Clarify with the user: primary audience, tone, allowed/refused actions, and which tools must be used for which tasks.
+4. Structure the prompt clearly:
+   - **Purpose** — what the agent is for in one short block.
+   - **Working principles** — how to reason, when to use tools, how to handle uncertainty.
+   - **Constraints** — safety, data scope, “do not” rules, org/project facts that must not be guessed.
+   - **Response style** — format, brevity, widgets or tables if the project uses them.
+5. For every entry in `tools[]`, add explicit guidance: when to call it, what inputs mean, and what to do if the tool fails.
+6. Remove contradictions, vague placeholders, and duplicate rules. Prefer specific, testable guidance evals can target.
+7. After edits, suggest `cognite agents push` and a targeted `cognite agents eval` (see `cognite agents eval --help`).
+
+Propose changes as a unified diff or full `instructions` block the user can paste into `<name>.agent.yaml`.
+
+## Anti-patterns
+
+- Do not promise capabilities no tool or platform feature provides.
+- Do not embed secrets, tokens, or user credentials in instructions.
+- Do not tell the agent to ignore tools when evals expect tool use (or the reverse).
+- Do not paste entire scorer or CLI reference docs — link to `AGENTS.md` and `--help`.
+
+## Cross-references
+
+- After rewriting → run **check-agent** to verify tool/instruction alignment and eval consistency.
+- If evals fail on grounding or retrieval → **debug-eval** skill; if cases need updates → **write-eval-case** skill.

package/_templates/app/new/root/package.json.ejs.t

@@ -26,9 +26,9 @@ to: '<%= useCurrentDir ? "" : ((directoryName || name) + "/") %>package.json'
     "setup-https": "npx @cognite/cli@latest apps setup-https"
   },
   "dependencies": {
-    "@cognite/aura": "^0.3.1",
+    "@cognite/aura": "^0.3.5",
     "@cognite/sdk": "^10.10.0",
-    "@cognite/app-sdk": "^0.8.0",
+    "@cognite/app-sdk": "^0.9.0",
     "@tabler/icons-react": "^3.35.0",
     "@tanstack/react-query": "^5.90.10",
     "clsx": "^2.1.1",

package/_templates/app/new/src/App.test.tsx.ejs.t

@@ -12,33 +12,37 @@ import App from './App';
 
 type AppDeps = NonNullable<ComponentProps<typeof App>['deps']>;
 
-function makeApi(): HostAppAPI {
+type AppApi = Pick<HostAppAPI, 'syncInternalState'>;
+
+function makeApi(): AppApi {
   return {
-    getProject: vi.fn<HostAppAPI['getProject']>(() => Promise.resolve('<%= project %>')),
-    getBaseUrl: vi.fn<HostAppAPI['getBaseUrl']>(() => Promise.resolve('https://cognite.test')),
-    getAccessToken: vi.fn<HostAppAPI['getAccessToken']>(() => Promise.resolve('test-token')),
-    getAppId: vi.fn<HostAppAPI['getAppId']>(() => Promise.resolve('test-app-id')),
     syncInternalState: vi.fn<HostAppAPI['syncInternalState']>(() => Promise.resolve(true)),
-    navigateInternal: vi.fn<HostAppAPI['navigateInternal']>(() => Promise.resolve(true)),
-    navigateExternal: vi.fn<HostAppAPI['navigateExternal']>(() => Promise.resolve(true)),
-    registerAgentServer: vi.fn<HostAppAPI['registerAgentServer']>(() => Promise.resolve()),
-    unregisterAgentServer: vi.fn<HostAppAPI['unregisterAgentServer']>(() => Promise.resolve()),
-    sendAgentLayoutMode: vi.fn<HostAppAPI['sendAgentLayoutMode']>(() => Promise.resolve()),
-    sendAgentMessage: vi.fn<HostAppAPI['sendAgentMessage']>(() => Promise.resolve()),
-    sendAgentTheme: vi.fn<HostAppAPI['sendAgentTheme']>(() => Promise.resolve()),
   };
 }
 
-function makeLoadingDeps(): AppDeps {
+function makeConnectedFn(api: AppApi = makeApi()) {
+  return vi.fn(() => Promise.resolve({ api }));
+}
+
+function makeDeps(): AppDeps {
   return {
-    connectToHostApp: vi.fn<AppDeps['connectToHostApp']>(() => new Promise<ConnectToHostAppResult>(() => undefined)),
+    connectToHostApp: vi.fn<AppDeps['connectToHostApp']>(() =>
+      Promise.resolve({
+        api: {
+          getProject: vi.fn<HostAppAPI['getProject']>(() => Promise.resolve('<%= project %>')),
+          getBaseUrl: vi.fn<HostAppAPI['getBaseUrl']>(() => Promise.resolve('https://cognite.test')),
+          getAccessToken: vi.fn<HostAppAPI['getAccessToken']>(() => Promise.resolve('test-token')),
+          getAppId: vi.fn<HostAppAPI['getAppId']>(() => Promise.resolve('test-app-id')),
+        } as Partial<HostAppAPI> as HostAppAPI,
+      })
+    ),
     createClient: vi.fn<AppDeps['createClient']>((config) => new CogniteClient(config)),
   };
 }
 
-function makeConnectedDeps(api = makeApi()): AppDeps {
+function makeLoadingDeps(): AppDeps {
   return {
-    connectToHostApp: vi.fn<AppDeps['connectToHostApp']>(() => Promise.resolve({ api })),
+    connectToHostApp: vi.fn<AppDeps['connectToHostApp']>(() => new Promise<ConnectToHostAppResult>(() => undefined)),
     createClient: vi.fn<AppDeps['createClient']>((config) => new CogniteClient(config)),
   };
 }
@@ -49,12 +53,12 @@ describe('App', () => {
   });
 
   it('renders loading state', () => {
-    render(<App deps={makeLoadingDeps()} />);
+    render(<App deps={makeLoadingDeps()} connectToHostApp={() => new Promise<never>(() => undefined)} />);
     expect(screen.getByText('Loading project...')).toBeInTheDocument();
   });
 
   it('renders splash with deployment targets and checklist copy', async () => {
-    render(<App deps={makeConnectedDeps()} />);
+    render(<App deps={makeDeps()} connectToHostApp={makeConnectedFn()} />);
     await waitFor(() => expect(screen.getByText('Welcome to Flows custom apps')).toBeInTheDocument());
     expect(screen.getByText('App deployment checklist')).toBeInTheDocument();
     expect(screen.getByText('Plan')).toBeInTheDocument();
@@ -73,7 +77,7 @@ describe('App', () => {
 
   it('syncs internal state when the open step changes', async () => {
     const api = makeApi();
-    render(<App deps={makeConnectedDeps(api)} />);
+    render(<App deps={makeDeps()} connectToHostApp={makeConnectedFn(api)} />);
     await waitFor(() => expect(screen.getByText('App deployment checklist')).toBeInTheDocument());
 
     await userEvent.click(screen.getByText('Explore'));
@@ -85,13 +89,10 @@ describe('App', () => {
 
   it('restores the open step from initial state', async () => {
     const api = makeApi();
-    const deps: AppDeps = {
-      connectToHostApp: vi.fn<AppDeps['connectToHostApp']>(() =>
-        Promise.resolve({ api, initialState: JSON.stringify({ openStep: 'Deploy' }) })
-      ),
-      createClient: vi.fn<AppDeps['createClient']>((config) => new CogniteClient(config)),
-    };
-    render(<App deps={deps} />);
+    render(<App
+      deps={makeDeps()}
+      connectToHostApp={() => Promise.resolve({ api, initialState: JSON.stringify({ openStep: 'Deploy' }) })}
+    />);
     await waitFor(() => expect(screen.getByText('App deployment checklist')).toBeInTheDocument());
 
     await waitFor(() =>