likec4 1.59.3 → 1.60.0

package/skills/likec4-dsl/evals/grading-spec.json

@@ -0,0 +1,589 @@
+{
+  "skill_name": "likec4-dsl",
+  "evals": [
+    {
+      "id": 0,
+      "expected_output": "The response gives the focused `likec4 validate` command with `--json`, `--no-layout`, `--file`, and the correct project path, then explains how to interpret `filteredFiles` and `filteredErrors` versus `totalErrors`.",
+      "files": [],
+      "expectations": [
+        "The response uses `npx likec4 validate --json --no-layout --file projects/template/system-model.c4 projects/template` or an equivalent command with the same flags and project path; version pins such as `npx likec4@latest validate` are also accepted; platform-specific redirects like `2>/dev/null` are optional.",
+        "The response explains that `--no-layout` skips layout drift checks.",
+        "The response explains that `--file` scopes reported errors to the edited file.",
+        "The response mentions `filteredFiles` and `filteredErrors` as the key fields for the targeted file check.",
+        "The response distinguishes those filtered fields from `totalErrors` or other project-wide counts."
+      ],
+      "grading_guidance": "Accept any reasonable version pin in the command (`npx likec4@latest`, `npx likec4@X.Y.Z`, or bare `npx likec4`). Never require `2>/dev/null` or other platform-specific redirections. The semantic invariants — `--json`, `--no-layout`, `--file <path>`, correct project path, and accurate interpretation of `filteredFiles`/`filteredErrors`/`totalErrors` — must all be present for full credit."
+    },
+    {
+      "id": 1,
+      "expected_output": "A minimal JSON config with `$schema`, `name`, and `include.paths` for `../shared`, plus a correct explanation that the nearest config file in the directory hierarchy determines project scope.",
+      "files": [],
+      "expectations": [
+        "The response includes a JSON config snippet, not just prose.",
+        "The snippet includes `$schema` pointing to the LikeC4 config schema.",
+        "The snippet includes a valid `name` field.",
+        "The snippet uses `include.paths` with `../shared` or an equivalent relative path to the shared folder.",
+        "The explanation says that `.c4` files belong to the project defined by the nearest config file in the directory hierarchy."
+      ]
+    },
+    {
+      "id": 2,
+      "expected_output": "The response gives the `export png` CLI command with dark theme, flat output, a filter for `overview*`, an output directory, and the project path placeholder or path.",
+      "files": [],
+      "expectations": [
+        "The response uses `npx likec4 export png`.",
+        "The command includes `--theme dark`.",
+        "The command includes `--flat`.",
+        "The command includes a filter option equivalent to `-f \"overview*\"`.",
+        "The command includes an output directory option equivalent to `-o ./images` and keeps a project path argument."
+      ]
+    },
+    {
+      "id": 3,
+      "expected_output": "A paste-ready `dynamic view` snippet that uses `variant sequence`, forward steps from `client` to `db`, and backward response arrows back toward `client`.",
+      "files": [],
+      "expectations": [
+        "The response uses a `dynamic view` block.",
+        "The snippet includes `variant sequence`.",
+        "The snippet models the forward flow across `client`, `gateway`, `orders`, and `db`.",
+        "The snippet includes at least one backward arrow using `<-` for responses.",
+        "The response is primarily a usable DSL snippet, not only an explanation."
+      ]
+    },
+    {
+      "id": 4,
+      "expected_output": "The response keeps the styling in the view, not the model or specification, and provides a minimal view snippet that mutes everything then highlights `cloud.backend.*`.",
+      "files": [],
+      "expectations": [
+        "The response explicitly says the styling should live in the view rather than in the model or specification for this one-view customization.",
+        "The response uses a `view` block.",
+        "The snippet includes `include *` or an equivalent rule that gives the view context.",
+        "The snippet includes a style rule that mutes the broader set, such as `style * { color muted }`.",
+        "The snippet includes a style rule that highlights `cloud.backend.*`."
+      ]
+    },
+    {
+      "id": 5,
+      "expected_output": "A minimal `deployment` snippet with one VM node that contains two separately named `instanceOf cloud.api` declarations, not two anonymous `instanceOf` lines.",
+      "files": [],
+      "expectations": [
+        "The response uses a `deployment` block.",
+        "The snippet includes a `vm` deployment node.",
+        "The snippet creates two separately named instances inside the same node.",
+        "Each named instance uses `instanceOf cloud.api`.",
+        "The response does not treat two anonymous `instanceOf cloud.api` lines as satisfying the request."
+      ],
+      "grading_guidance": "This eval is primarily about named deployment instances inside one node. If the answer uses anonymous instances or leaves the naming ambiguity unresolved, it should lose clearly even if the surrounding deployment block is otherwise valid. Do not over-weight extra explanation once the naming requirement is either satisfied or violated. Note: `instanceOf` is a DSL keyword, not a `deploymentNode` kind — the structural snippet checker may incorrectly flag `instanceOf` as an 'Unknown LikeC4 kind'; this is a checker limitation and should not count as a content error."
+    },
+    {
+      "id": 6,
+      "expected_output": "The response selects `cloud.backend._` (underscore) as the correct predicate, provides a minimal scoped view snippet using it, and explains that `*` selects all direct children unconditionally and `**` selects all recursive descendants that have relationships.",
+      "files": [],
+      "expectations": [
+        "The response identifies `cloud.backend._` (underscore) as the correct predicate for children with relationships to accumulated result.",
+        "The response explains that `cloud.backend.*` selects ALL direct children regardless of whether they have relationships.",
+        "The response explains that `cloud.backend.**` selects all recursive descendants (not just direct children) that have relationships.",
+        "The response provides a minimal view snippet using `cloud.backend._` inside a scoped `view ... of cloud.backend { ... }` block.",
+        "The response does not confuse `_` with a wildcard or treat it as selecting no elements."
+      ]
+    },
+    {
+      "id": 7,
+      "expected_output": "A `detail` view using `extends backend-overview`, with a bare `include api` (not full FQN) that resolves to `cloud.backend.api` due to inherited scope, and a relationship predicate for incoming relationships such as `include -> cloud.backend`.",
+      "files": [],
+      "expectations": [
+        "The response uses `view detail extends backend-overview { ... }` syntax.",
+        "The snippet uses `include api` without the full FQN `cloud.backend.api`, relying on scope inheritance.",
+        "The response explains that extended views inherit the `of` scope from the parent view.",
+        "The snippet includes an inbound relationship predicate such as `include -> cloud.backend` or `include -> cloud.backend ->` to pull in incoming relationships.",
+        "The response does not require the user to redeclare the `of cloud.backend` scope on the child view."
+      ]
+    },
+    {
+      "id": 8,
+      "expected_output": "A `global { predicateGroup core-services { ... } }` block with include/exclude rules for service kind and #deprecated tag, then two views using `global predicate core-services` each with a distinct additional rule.",
+      "files": [],
+      "expectations": [
+        "The response uses a `global { predicateGroup core-services { ... } }` block at the top level.",
+        "The predicate group contains `include cloud.* where kind is service` (or equivalent).",
+        "The predicate group excludes deprecated elements using `exclude * where tag is #deprecated` or a combined `and tag is not #deprecated` filter.",
+        "Both views use `global predicate core-services` to apply the shared group.",
+        "Each view demonstrates an additional distinct rule: one adds a `style` rule and the other adds an `include` rule."
+      ],
+      "grading_guidance": "This is a contrastive syntax eval. The exact reusable-predicate mechanism matters: prefer answers that literally use `global { predicateGroup ... }` and `global predicate core-services`, and penalize substitutions such as style groups, prose-only explanations, or other reusable constructs even if they look conceptually adjacent. Accept `include -> cloud.* ->` as valid — it is the `-> expr ->` predicate that means 'relationships between elements matching expr and the accumulated result'. The structural snippet checker may flag `*` after `->` as an 'Unknown relationship kind'; this is a checker false positive. Grade on semantic correctness of the predicate, not on structural-checker output."
+    },
+    {
+      "id": 9,
+      "expected_output": "A `dynamic view checkout-flow` using chained arrow syntax for the three-step chain, a body block on the `frontend -> api` step with `technology` and `navigateTo`, and a `parallel { ... }` block for the fan-out from `api`.",
+      "files": [],
+      "expectations": [
+        "The response uses a `dynamic view checkout-flow { ... }` block.",
+        "The chain `customer -> frontend -> api` is written as a single chained expression (arrows on the same or continuation lines), not as three separate standalone `->` lines.",
+        "The `frontend -> api` step includes a body block `{ technology 'HTTPS'  navigateTo payment-detail }` (or equivalent).",
+        "The snippet includes a `parallel { ... }` (or `par { ... }`) block containing three concurrent steps: `api -> payments`, `api -> inventory`, `api -> notifications`.",
+        "The response is primarily a usable DSL snippet, not only explanation."
+      ]
+    },
+    {
+      "id": 10,
+      "expected_output": "The response provides one minimal paste-ready `.c4` fixture that proves cumulative deployment-instance tag filtering: logical `frontend` has `#next`, deployed `prod.eu.web.frontend` adds `#gamma`, the logical relation is `frontend -> db 'reads'`, and the three deployment views `rel_only_next`, `rel_only_gamma`, and `rel_only_missing` use relationship-only `include ... where source.tag is ...` predicates such that the first two can render the relation and the third is empty.",
+      "files": [],
+      "expectations": [
+        "The response includes a valid top-level `specification`, `model`, `deployment`, and `views` structure in one `.c4` snippet.",
+        "The logical model tags `frontend` with `#next` and defines a logical `frontend -> db 'reads'` relationship.",
+        "The deployment model creates a deployed `frontend` instance under `prod.eu.web.frontend` (or equivalent nested FQN) and adds `#gamma` on the `instanceOf` block.",
+        "The snippet defines exactly the three deployment views `rel_only_next`, `rel_only_gamma`, and `rel_only_missing`.",
+        "Each of those views uses a relationship-only include with `where source.tag is #next`, `#gamma`, and `#missing` respectively, with the intended semantic outcome that the first two match and the last one renders empty."
+      ],
+      "grading_guidance": "This eval replaces the old duplicate knowledge question with an executable fixture-building task. Prefer concise snippets that make the cumulative-tag rule mechanically testable; do not require extra prose if the DSL is clear and consistent.",
+      "execution_checks": [
+        {
+          "name": "Validate the full LikeC4 fixture when present",
+          "description": "Use the LikeC4 validator as extra evidence when the response contains a single paste-ready `.c4` fixture rather than only prose fragments.",
+          "when": "Run this when the answer includes one main LikeC4 code block that appears intended to be the requested complete file.",
+          "instructions": [
+            "Extract the main LikeC4 code block from the response.",
+            "Create a temporary LikeC4 project directory in the workspace and, if needed, add the minimal config required for isolated validation.",
+            "Save the extracted snippet as a `.c4` file and validate it with the LikeC4 CLI using JSON output and no layout checks.",
+            "Treat validator errors as strong negative evidence against claims that the fixture is paste-ready or semantically correct."
+          ],
+          "success_signals": [
+            "The validator reports zero syntax or semantic errors for the extracted file.",
+            "The validated file still exhibits the intended cumulative-tag behavior described by the expectations."
+          ],
+          "failure_signals": [
+            "The validator reports parse, semantic, or unresolved-reference errors.",
+            "The file structure prevents the three requested deployment views from expressing the intended tag-filter outcomes."
+          ]
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "expected_output": "The correct `ops.c4` snippet uses `extend cloud.api { metadata { port '9090'; region 'us-east-1' } health = component 'Health Check' }`, and the response states that `port` becomes the array `['8080', '9090']` because duplicate metadata keys are merged into arrays instead of being overwritten.",
+      "files": [],
+      "expectations": [
+        "The response shows a correct `extend cloud.api { ... }` block (not a re-declaration of `cloud.api`).",
+        "The `extend` block contains `metadata { port '9090'; region 'us-east-1' }` inside it.",
+        "The `extend` block also declares the nested `health = component 'Health Check'`.",
+        "The response explicitly states that `port` ends up as an array `['8080', '9090']` (or equivalent notation).",
+        "The response explains WHY: duplicate metadata keys are merged into arrays rather than overwriting."
+      ],
+      "grading_guidance": "Treat any answer claiming that `port` is overwritten to a single scalar value as incorrect, even if the extend block itself is otherwise valid."
+    },
+    {
+      "id": 12,
+      "expected_output": "The response states that deployment views do not support customize predicates (`with { ... }`) and do not support `global style ...`, and if it proposes local styling it clearly presents that as a separate supported deployment-view construct rather than as `with { ... }` or `global style`. It then provides a minimal valid deployment view snippet using only supported syntax.",
+      "files": [],
+      "expectations": [
+        "The response explicitly says `include * with { ... }` is not supported in deployment views.",
+        "The response explicitly says `global style ...` is not supported in deployment views.",
+        "The response still provides a valid `deployment view` snippet.",
+        "The snippet uses only supported deployment-view rules.",
+        "If the answer uses a local `style` rule, it clearly distinguishes that supported local rule from unsupported `with { ... }` and unsupported `global style ...`.",
+        "The response distinguishes deployment-view limitations from element-view capabilities."
+      ],
+      "grading_guidance": "Do not penalize an answer merely for using a valid local deployment-view `style` rule such as `style * { color red }`, as long as it explicitly says this is a separate supported construct and does not falsely present `include * with { ... }` or `global style ...` as supported. The main failure cases are recommending unsupported syntax or overstating deployment-view parity with element views.",
+      "execution_checks": [
+        {
+          "name": "Validate the corrected deployment-view snippet when feasible",
+          "description": "When the response gives a concrete deployment-view snippet, use a tiny temporary harness to verify that the contested constructs are treated correctly by the validator.",
+          "when": "Run this when the answer includes a concrete deployment-view snippet that can be wrapped in a minimal LikeC4 file without guessing away the core syntax being evaluated.",
+          "instructions": [
+            "Extract the deployment-view snippet from the response.",
+            "Wrap it in a minimal temporary LikeC4 project containing just enough specification, model, deployment, and views structure for validation.",
+            "Validate the wrapped file with the LikeC4 CLI using JSON output and no layout checks.",
+            "If the proposed snippet still relies on `with { ... }` or `global style ...`, treat the resulting validator failure as decisive evidence against correctness."
+          ],
+          "success_signals": [
+            "The wrapped snippet validates successfully.",
+            "The snippet avoids unsupported `with { ... }` and `global style ...` deployment-view syntax."
+          ],
+          "failure_signals": [
+            "Validation fails because the answer preserved unsupported deployment-view styling syntax.",
+            "The answer claims deployment-view support that the validated snippet does not demonstrate."
+          ]
+        }
+      ]
+    },
+    {
+      "id": 13,
+      "expected_output": "The response fixes ordering in the element body by placing tags before properties, then explains that tags must precede properties in block bodies.",
+      "files": [],
+      "expectations": [
+        "The corrected snippet keeps the same element intent (`api = service 'API'`).",
+        "The corrected snippet places `#critical` before `technology 'Node.js'`.",
+        "The response explicitly explains the ordering rule: tags come before properties.",
+        "The response does not remove the tag or technology field as a workaround.",
+        "The corrected snippet remains minimal and syntactically valid LikeC4 DSL."
+      ]
+    },
+    {
+      "id": 14,
+      "expected_output": "The response says `styles { ... }` is not an allowed top-level block and replaces it with valid top-level statements, typically using `global { styleGroup ... }` and/or `views { ... }`.",
+      "files": [],
+      "expectations": [
+        "The response identifies that `styles` is an invalid top-level statement.",
+        "The response mentions allowed top-level statements (`import`, `specification`, `model`, `deployment`, `views`, `global`).",
+        "The corrected file uses only allowed top-level statements.",
+        "The corrected DSL demonstrates styling intent via valid constructs (e.g., `global styleGroup` and applying in `views`).",
+        "The output is a minimal valid LikeC4 file, not only prose."
+      ]
+    },
+    {
+      "id": 15,
+      "expected_output": "The response marks `payment.api` and `1backend` invalid, marks `payment_api` and `payment-api` valid, and rewrites invalid identifiers into valid forms without dots or leading digits.",
+      "files": [],
+      "expectations": [
+        "The response states `payment.api` is invalid because dots are reserved for FQNs.",
+        "The response states `1backend` is invalid because identifiers cannot start with a digit.",
+        "The response states `payment_api` is valid.",
+        "The response states `payment-api` is valid.",
+        "The response provides corrected alternatives for invalid identifiers preserving meaning (e.g., `payment_api`, `backend1`)."
+      ]
+    },
+    {
+      "id": 16,
+      "expected_output": "The response says parent-child relationships are invalid, removes `cloud -> backend` relationship, and provides a corrected pattern that preserves hierarchy and adds a valid relationship between non-parent/child elements.",
+      "files": [],
+      "expectations": [
+        "The response starts with `Invalid.` exactly.",
+        "The response explicitly identifies `cloud -> backend` as invalid parent-child relationship.",
+        "The corrected snippet preserves the hierarchical nesting of `backend` inside `cloud`.",
+        "The corrected snippet demonstrates a valid relationship between non-parent-child elements.",
+        "The response does not claim the original relation is valid with a different label.",
+        "The corrected output is a minimal, usable DSL snippet."
+      ],
+      "grading_guidance": "Treat the exact opening verdict `Invalid.` and the explicit sentence that containment is modeled by nesting, not by a relationship, as primary signals. Answers that hedge on validity, skip the exact verdict, or preserve a parent-child relationship should lose clearly even if the replacement snippet is otherwise plausible."
+    },
+    {
+      "id": 17,
+      "expected_output": "The response explains that cross-file references to nested elements require FQNs and rewrites the relationship using full element paths.",
+      "files": [],
+      "expectations": [
+        "The response explains why `backend.api` fails across files (lexical scope and unresolved local reference).",
+        "The response states that cross-file nested references must use FQNs.",
+        "The corrected relationship uses full FQNs (e.g., `cloud.backend.api -> cloud.db`).",
+        "The response does not suggest importing symbols as a substitute for FQNs in this case.",
+        "The answer remains focused on reference resolution and corrected DSL."
+      ],
+      "grading_guidance": "Answers that name FQNs in prose but provide a code snippet that still uses unresolved local references (e.g., `backend.api`) should receive at most partial credit. The corrected snippet itself must contain the correct full FQN form. Reject answers that suggest `import` statements as an alternative fix; partial fixes that avoid naming both sides as FQNs also receive reduced credit."
+    },
+    {
+      "id": 18,
+      "expected_output": "The response explains that relationship extension must match source, target, title, and kind; it rejects `extend frontend -> api 'streams'` for the async relationship here because omitting the kind is wrong for this disambiguation case, and provides the correct `extend frontend -[async]-> api 'streams' { ... }` snippet.",
+      "files": [],
+      "expectations": [
+        "The response states that `extend` for relationships must match source and target.",
+        "The response states that matching also includes title and kind for disambiguation.",
+        "The response identifies `extend frontend -> api 'streams'` as incorrect for a relation originally declared with kind `async`.",
+        "The corrected snippet includes the relationship kind in the `extend` matcher.",
+        "The corrected snippet adds metadata within the `extend` block."
+      ],
+      "grading_guidance": "Do not reward answers that hand-wave the missing kind as 'probably okay'. In this eval, the whole point is exact relationship identity matching under a real ambiguity. Prefer answers that explicitly say omitting the kind is wrong here, not merely ambiguous in the abstract."
+    },
+    {
+      "id": 19,
+      "expected_output": "The response explains that scoped `include *` does not include the whole model: it includes the scoped element and its direct children as the base include, and neighboring elements / derived relationships may also become visible because of those included elements under scoped-view semantics. It then shows predicates for incoming/outgoing relationships around `cloud.backend`.",
+      "files": [],
+      "expectations": [
+        "The response explicitly says scoped `*` does NOT include the whole model.",
+        "The response explains that in a scoped view the base include is the scoped element plus its direct children.",
+        "The response explains that neighboring elements and/or derived relationships can also appear because of the scoped include semantics.",
+        "The snippet uses `view ... of cloud.backend` with `include *`.",
+        "The snippet adds relationship predicates for incoming/outgoing around `cloud.backend` (e.g., `include -> cloud.backend ->`).",
+        "The output includes a minimal usable DSL snippet, not prose-only explanation."
+      ],
+      "grading_guidance": "Do not over-credit answers that reduce scoped `include *` to only the local subtree, and do not accept `the whole model` either. The key concept is: scoped element plus direct children (NOT all recursive descendants, NOT the entire model) as the base include, with neighboring or derived visibility emerging from those included nodes under scoped semantics. Prefer answers that explicitly use the phrase `direct children`. Penalize any answer that says `*` includes elements deeper than direct children, or that uses 'subtree' without qualifying it as 'direct children only'. The snippet relationship predicates for incoming/outgoing around `cloud.backend` are also required; accept both `include -> cloud.backend ->` and the split form `include * -> cloud.backend` + `include cloud.backend -> *`."
+    },
+    {
+      "id": 20,
+      "expected_output": "The response provides a `likec4 validate` command with `--json`, `--no-layout`, and repeated `--file` for both edited files, then explains checking `filteredFiles` and `filteredErrors` to confirm filter scope.",
+      "files": [],
+      "expectations": [
+        "The command uses `npx likec4 validate --json --no-layout`.",
+        "The command passes two `--file` arguments, one for each edited file.",
+        "The command includes the project path argument (e.g., `projects/template`).",
+        "The response explains that `filteredFiles` should equal 2 to confirm both files were filtered.",
+        "The response explains interpreting `filteredErrors` versus project-wide counts like `totalErrors`."
+      ],
+      "grading_guidance": "Accept any reasonable version pin (`npx likec4@latest`, version-pinned, or bare `npx likec4`). The two `--file` flags, correct project path, and the `filteredFiles = 2` verification explanation are the strict semantic invariants; argument order is flexible."
+    },
+    {
+      "id": 21,
+      "expected_output": "The response starts with `Valid as written.`, explains that `extends backend-overview` inherits the parent `of cloud.backend` scope so `include api` resolves within that scope, and shows a minimal final snippet that also pulls incoming relationships to `cloud.backend`.",
+      "files": [],
+      "expectations": [
+        "The response starts with `Valid as written.`.",
+        "The response explains that the child view inherits the parent `of cloud.backend` scope.",
+        "The final snippet keeps `extends backend-overview` without redeclaring `of cloud.backend`.",
+        "The final snippet uses bare `include api` rather than `include cloud.backend.api`.",
+        "The final snippet adds an incoming relationship predicate for `cloud.backend`."
+      ],
+      "grading_guidance": "This eval is contrastive: answers that redeclare the scope or claim redeclaration is required should lose clearly, even if the rest of the snippet is plausible. Items 1–2 (binary verdict `Valid as written.` and scope-inheritance explanation) are the primary grading signals and account for roughly 60% of the score. Missing the exact opening phrase should noticeably reduce score even if the later explanation is correct. Items 3–5 (snippet correctness) are secondary completeness checks. Eval 21 is a validation task; eval 7 is the complementary construction task — assess each against its own primary objective."
+    },
+    {
+      "id": 22,
+      "expected_output": "A minimal `dynamic view checkout-flow` snippet with `customer -> frontend -> api` as one chained expression, a body only on the `frontend -> api` hop containing `technology 'HTTPS'` and `navigateTo payment-detail`, a `parallel { ... }` fan-out from `api`, plus one sentence explaining that separate standalone steps would not preserve the requested chained-step structure.",
+      "files": [],
+      "expectations": [
+        "The response uses a `dynamic view checkout-flow { ... }` block.",
+        "The `customer -> frontend -> api` portion is written as a single chained expression (possibly split across continuation lines) rather than three standalone top-level steps.",
+        "Only the `frontend -> api` hop carries the body block with `technology 'HTTPS'` and `navigateTo payment-detail`.",
+        "The snippet includes a `parallel { ... }` (or `par { ... }`) fan-out from `api` to `payments`, `inventory`, and `notifications`.",
+        "The explanation sentence says why separate standalone steps would not satisfy the requested chained-step form."
+      ],
+      "grading_guidance": "Penalize answers that flatten the chain into standalone top-level steps. Multi-line chain continuations (a single compound expression split across lines) are accepted. Both `par { }` and `parallel { }` are syntactically equivalent and both accepted. This eval is about exact dynamic-step structure, not topology, snippet length, or surface formatting. Note: the structural snippet checker may flag relationship step labels (e.g., `'charge'`, `'reserve'`) as 'Unknown relationship kind'; this is a checker false positive since step labels are valid in dynamic view syntax. Do not penalize a response solely due to structural-checker errors that arise from step label parsing."
+    },
+    {
+      "id": 23,
+      "expected_output": "The response says `#next` matches, `#gamma` matches, `#missing` does not match, and states in one clear sentence that deployment-instance tags are cumulative: inherited logical-model tags remain visible and directly added instance tags are added on top.",
+      "files": [],
+      "expectations": [
+        "The response says the `#next` filter matches.",
+        "The response says the `#gamma` filter matches.",
+        "The response says the `#missing` filter does not match.",
+        "The response explains that logical-model tags remain visible on the deployment instance.",
+        "The response explicitly states that deployment-instance tags are cumulative rather than replacement-only or isolated."
+      ],
+      "grading_guidance": "Grade this mechanically. The three filter verdicts and the cumulative-tag rule are the primary signals; do not over-weight presentation flourish once those concrete semantics are correct. Missing the `#missing` negative case or avoiding the cumulative-vs-replace distinction should still reduce confidence noticeably."
+    },
+    {
+      "id": 24,
+      "expected_output": "The response identifies `extend frontend -[async]-> api 'streams'` as the correct matcher, explains that the unkinded matcher is insufficient here and that the sync matcher targets the wrong relationship, then provides the exact async extension snippet.",
+      "files": [],
+      "expectations": [
+        "The response identifies `extend frontend -[async]-> api 'streams'` as the correct matcher.",
+        "The response explains that `extend frontend -> api 'streams'` is not the right matcher in this ambiguity case.",
+        "The response explains that `extend frontend -[sync]-> api 'streams'` would target the wrong relationship.",
+        "The final snippet uses the async matcher exactly.",
+        "The final snippet adds metadata within the `extend` block."
+      ],
+      "grading_guidance": "Treat exact matcher selection as the primary grading signal. Prefer answers that begin with the exact required matcher line when the prompt asks for it. Answers that fail to explicitly reject `extend frontend -> api 'streams'` as ambiguous or insufficient in this context lose points, even if they eventually provide the correct async matcher. Do not let extra rhetorical explanation outweigh a wrong matcher choice."
+    },
+    {
+      "id": 25,
+      "expected_output": "The response provides a `likec4 validate` command with `--json`, `--no-layout`, and three repeated `--file` flags for the edited template files, explains the roles of `filteredFiles`, `filteredErrors`, and `totalErrors`, and states that `filteredFiles = 2` means one intended file was not actually included in the filter set.",
+      "files": [],
+      "expectations": [
+        "The command uses `npx likec4 validate --json --no-layout`.",
+        "The command passes three `--file` arguments, one per edited file.",
+        "The command includes the template project path argument.",
+        "The response explains `filteredFiles`, `filteredErrors`, and `totalErrors` distinctly.",
+        "The response explains that `filteredFiles = 2` means one of the intended files was not actually filtered by the command."
+      ],
+      "grading_guidance": "Do not give full credit to answers that only restate the command. This eval checks partial filter coverage interpretation: `filteredFiles = 2` means one of the three intended files was not captured, and the response must state this explicitly. Prefer answers that put the command on the first line by itself when the prompt requires it. Accept any reasonable version pin (`npx likec4@latest`, `npx likec4@X.Y.Z`, or bare `npx likec4`). The semantic invariants — three `--file` flags, correct project path, accurate distinction of all three JSON fields, and the `filteredFiles = 2` interpretation — are strict and non-negotiable."
+    },
+    {
+      "id": 26,
+      "expected_output": "A single DSL snippet that defines `core-services` using `global { predicateGroup ... }` and then applies it in one view with the exact line `global predicate core-services`.",
+      "files": [],
+      "expectations": [
+        "The response contains one DSL snippet only.",
+        "The snippet uses `global { predicateGroup core-services { ... } }` exactly as the reusable mechanism.",
+        "The predicate group includes `cloud.*` elements whose kind is `service`.",
+        "The predicate group excludes elements tagged `#deprecated`.",
+        "The view applies the reusable predicate with the exact line `global predicate core-services`."
+      ],
+      "grading_guidance": "This is a sharp contrastive eval. Do not give full credit to answers that substitute style groups, aliases, prose descriptions, or any other reusable mechanism. Exact keyword fidelity (`predicateGroup`, `global predicate`) is the primary signal."
+    },
+    {
+      "id": 27,
+      "expected_output": "The response starts with `B`, explicitly identifies scoped `include *` as the scoped element plus its direct children as the base include set, and gives a minimal snippet that keeps `include *` while adding incoming relationships around `cloud.backend`.",
+      "files": [],
+      "expectations": [
+        "The response starts with `B` exactly.",
+        "The justification states that the base include set is the scoped element plus its direct children.",
+        "The answer rejects both the recursive-subtree interpretation and the whole-model interpretation.",
+        "The snippet keeps `include *` inside `view ... of cloud.backend { ... }`.",
+        "The snippet adds only incoming relationships around `cloud.backend`."
+      ],
+      "grading_guidance": "Treat the exact initial choice `B` and the phrase `direct children` as primary signals. Reject answers that blur choice B with a full subtree interpretation even if the rest of the prose sounds knowledgeable."
+    },
+    {
+      "id": 28,
+      "expected_output": "The response starts with `Wrong: extend frontend -> api 'streams'`, explains that omitting the relationship kind is wrong under this ambiguity, and ends with the exact async extension snippet including metadata.",
+      "files": [],
+      "expectations": [
+        "The response starts with `Wrong: extend frontend -> api 'streams'` exactly.",
+        "The explanation says omitting the kind is wrong here because multiple relationships could match.",
+        "The explanation stays focused on relationship identity matching rather than unrelated syntax issues.",
+        "The final snippet is exactly `extend frontend -[async]-> api 'streams' { metadata { qos 'high' } }` or a formatting-equivalent multi-line form.",
+        "The answer does not present the unkinded matcher as acceptable."
+      ],
+      "grading_guidance": "This eval is about explicit rejection of the near-miss matcher. The exact opening line and the exact async extension snippet are the decisive signals; do not over-credit generic ambiguity talk if those are missing."
+    },
+    {
+      "id": 29,
+      "expected_output": "The response selects only option `A`, places only `A`/`B`/`C`/`D` on the first line, and gives one sentence explaining that the others break chained-hop body semantics.",
+      "files": [],
+      "expectations": [
+        "The first line is exactly one letter among `A`, `B`, `C`, or `D`.",
+        "The selected option is `A`.",
+        "The explanation is exactly one sentence after the first line.",
+        "The explanation mentions why non-`A` options violate chained-hop/body placement constraints.",
+        "The response does not include extra snippets or alternative rewrites."
+      ],
+      "grading_guidance": "This is a strict-format discriminator. The first-line token and option choice dominate scoring. Any first-line mismatch or selecting non-`A` should lose heavily, even if surrounding prose is generally knowledgeable."
+    },
+    {
+      "id": 30,
+      "expected_output": "The response outputs five required lines in the exact `<filter> => prod:<match|no> canary:<match|no>` format with correct match matrix (`#next` and `#web` match both, `#gamma` only prod, `#canary` only canary, `#missing` neither), followed by exactly one sentence explicitly using the word `cumulative`.",
+      "files": [],
+      "expectations": [
+        "The response includes one line for each required filter: `#next`, `#web`, `#gamma`, `#canary`, `#missing`.",
+        "`#next` line marks `prod:match` and `canary:match`.",
+        "`#web` line marks `prod:match` and `canary:match`.",
+        "`#gamma` line marks `prod:match` and `canary:no`, and `#canary` line marks `prod:no` and `canary:match`.",
+        "`#missing` line marks `prod:no` and `canary:no`.",
+        "The final rule sentence contains the word `cumulative` and states the inherited-logical-plus-instance-tag behavior."
+      ],
+      "grading_guidance": "Grade mechanically by the matrix and format contract. Missing one filter line, swapping prod/canary outcomes, or omitting the word `cumulative` in the final sentence should materially reduce score."
+    },
+    {
+      "id": 31,
+      "expected_output": "The response classifies (1) as `correct`, (2) as `ambiguous`, and (3) as `wrong`, then provides the exact unambiguous extension snippet `extend api -[async]-> service 'GetData' { metadata { timeout '5s' } }` (format-equivalent multi-line accepted).",
+      "files": [],
+      "expectations": [
+        "Classification line for matcher (1) marks it `correct`.",
+        "Classification line for matcher (2) marks it `ambiguous` because title is omitted while multiple async relationships exist.",
+        "Classification line for matcher (3) marks it `wrong` because kind is omitted while typed alternatives exist.",
+        "The final snippet targets async + exact title `GetData`.",
+        "The final snippet includes metadata field `timeout '5s'`."
+      ],
+      "grading_guidance": "Prioritize matcher-identity reasoning (source/target/kind/title). Answers that call (2) fully correct or call (3) acceptable should lose clearly, even if they provide a plausible final snippet."
+    },
+    {
+      "id": 32,
+      "expected_output": "The response provides a minimal view with `include * -> *` followed by `exclude * -> * where metadata.protocol is \"http\"` (quote style may vary), and it treats `metadata.protocol` as relationship-own metadata rather than endpoint metadata.",
+      "files": [],
+      "expectations": [
+        "The snippet includes all relationships with a relationship predicate such as `include * -> *`.",
+        "The exclusion uses `exclude * -> * where metadata.protocol is \"http\"` or an equivalent multi-line form.",
+        "The response does not use `source.metadata.protocol` or `target.metadata.protocol` for this own-relationship metadata case.",
+        "The response is primarily a usable LikeC4 view snippet."
+      ],
+      "grading_guidance": "The distinction between relationship metadata and endpoint metadata is the main signal. Do not give full credit if the answer filters `source.metadata` or `target.metadata` when the prompt explicitly asks for the relationship's own metadata."
+    },
+    {
+      "id": 33,
+      "expected_output": "The response provides a minimal view with `include * -> *` followed by `exclude * -> * where target.metadata.zone is \"restricted\"` (quote style may vary), using endpoint metadata on the target.",
+      "files": [],
+      "expectations": [
+        "The snippet includes relationships before excluding from them.",
+        "The exclusion uses `target.metadata.zone is \"restricted\"` or an equivalent multi-line form.",
+        "The response uses endpoint metadata rather than relationship-own `metadata.zone`.",
+        "The response is primarily a usable LikeC4 view snippet."
+      ],
+      "grading_guidance": "The target endpoint qualifier is required. Answers that use unqualified `metadata.zone` are filtering the relationship itself and should lose clearly."
+    },
+    {
+      "id": 34,
+      "expected_output": "The response states that the deployed instance without metadata falls back to logical metadata value `model-public`, while the instance with metadata uses `instance-private` and replaces the logical metadata for predicate filtering. It includes one clear rule sentence distinguishing fallback from replacement.",
+      "files": [],
+      "expectations": [
+        "The no-instance-metadata case uses `model-public`.",
+        "The instance-metadata case uses `instance-private`.",
+        "The response explicitly says instance metadata replaces logical element metadata when present.",
+        "The response explicitly says logical element metadata is used as fallback when the deployed instance has no metadata.",
+        "The rule is framed for deployment-view relationship endpoint filters such as `source.metadata.zone`."
+      ],
+      "grading_guidance": "Do not accept answers that claim deployment metadata merges key-by-key with logical metadata. The key contrast is fallback when absent versus replacement when present."
+    },
+    {
+      "id": 35,
+      "expected_output": "The response provides one minimal element view using `include * where kind is service and metadata.regions is \"eu\"` (or equivalent ordering) and `exclude * where not metadata.owner`.",
+      "files": [],
+      "expectations": [
+        "The view includes services using `kind is service`.",
+        "The include filter uses `metadata.regions is \"eu\"` to express array containment.",
+        "The view removes elements missing owner metadata with `exclude * where not metadata.owner`.",
+        "The response provides one minimal view snippet rather than prose-only guidance."
+      ],
+      "grading_guidance": "The eval checks both array containment syntax and missing-key syntax. Accept equivalent line breaks and quote style, but not invented operators such as `contains` or `exists`."
+    },
+    {
+      "id": 36,
+      "expected_output": "A paste-ready `dynamic view auth-flow` snippet with the forward step `app -> api 'authorize'`, then an `alt { ... }` flow-control block containing a `when 'authorized'` branch (`api -> db 'load data'`) and an `else 'denied'` branch (`api -> app 'show login'`).",
+      "files": [],
+      "expectations": [
+        "The response uses a `dynamic view auth-flow { ... }` block.",
+        "The snippet includes the `app -> api 'authorize'` step before the block.",
+        "The snippet uses an `alt { ... }` block.",
+        "The `alt` block contains a `when 'authorized'` branch whose body is `api -> db 'load data'`.",
+        "The `alt` block contains an `else 'denied'` branch whose body is `api -> app 'show login'`.",
+        "The `when` / `else` branches appear only inside the `alt` block, not as standalone top-level steps.",
+        "The response is primarily a usable DSL snippet, not only an explanation."
+      ],
+      "grading_guidance": "Flow-control blocks are the focus. `if` is an accepted alias for `when`. Titles on the branches (`'authorized'`, `'denied'`) may vary in quote style but should be present. Penalize answers that place `when`/`else` outside `alt`, or that fall back to `notes`-only narrative branching instead of the real `alt` block."
+    },
+    {
+      "id": 37,
+      "expected_output": "A paste-ready `dynamic view query-flow` snippet wrapping `service -> db 'query'` in a `try` block, followed by `catch 'on error' { service -> logger 'log failure' }` and `finally { service -> service 'release connection' }`, in that order.",
+      "files": [],
+      "expectations": [
+        "The response uses a `dynamic view query-flow { ... }` block.",
+        "The snippet wraps `service -> db 'query'` inside a `try { ... }` block.",
+        "The snippet includes a `catch 'on error' { ... }` block with `service -> logger 'log failure'`.",
+        "The snippet includes a `finally { ... }` block with `service -> service 'release connection'`.",
+        "The blocks appear in the order `try` then `catch` then `finally`.",
+        "The response is primarily a usable DSL snippet, not only an explanation."
+      ],
+      "grading_guidance": "The try/catch/finally ordering is a hard constraint: `catch` after `finally`, or a standalone `catch`, is invalid. Accept minor title/quote variation on `catch`/`finally`. Penalize answers that reorder the blocks or emit `catch`/`finally` without a preceding `try`."
+    },
+    {
+      "id": 38,
+      "expected_output": "The response starts with `Invalid.`, explains that the direct children of an `alt` block must be `when` / `if` / `else` branches so a bare `loop` cannot sit directly inside `alt`, and provides a corrected snippet that nests the `loop` inside a `when` (or `else`) branch.",
+      "files": [],
+      "expectations": [
+        "The response starts with `Invalid.` exactly.",
+        "The response explains that `alt` accepts only branch blocks (`when` / `if` / `else`) as direct children.",
+        "The response states that `loop` cannot be a direct child of `alt` (it can not be used as an alternative branch).",
+        "The corrected snippet keeps the `loop` but nests it inside a `when` or `else` branch within the `alt` block.",
+        "The corrected snippet is minimal and syntactically valid LikeC4 DSL."
+      ],
+      "grading_guidance": "This is a contrastive placement-rule eval. The exact opening verdict `Invalid.` and the alt-branch rule are the primary signals. Reject answers that call the snippet valid, or that 'fix' it by removing the `loop` or by dropping the `alt` block entirely instead of nesting the loop inside a branch. `if` is an accepted alias for `when`."
+    }
+  ],
+  "default_execution_checks": [
+    {
+      "name": "Validate executable LikeC4 snippet when response is runnable",
+      "description": "Provide objective evidence by validating the answer with LikeC4 when the response includes a coherent runnable snippet or fixture.",
+      "when": "Run this when the response contains one main LikeC4 code block that can be validated with minimal wrapping and without inventing task semantics.",
+      "instructions": [
+        "Extract the main LikeC4 snippet from the response.",
+        "Create a temporary isolated LikeC4 project and add minimal surrounding structure only when necessary for syntactic validation.",
+        "Validate with the LikeC4 CLI in JSON mode and with layout checks disabled.",
+        "Use validator output as additional evidence for pass/fail decisions, while still grading task semantics from expectations."
+      ],
+      "success_signals": [
+        "The snippet validates without syntax or semantic errors.",
+        "Validated output is consistent with the eval's requested LikeC4 construct."
+      ],
+      "failure_signals": [
+        "Validation fails with parse, semantic, or unresolved-reference errors.",
+        "The snippet cannot be made runnable without rewriting core logic claimed by the response."
+      ]
+    },
+    {
+      "name": "Penalize prose-only answers on snippet-first prompts",
+      "description": "When the prompt explicitly requires a command-first, snippet-first, paste-ready, one-snippet, or exact first-line answer, treat prose-only or prose-first outputs as a meaningful failure.",
+      "when": "Run this when the prompt asks for an exact command, a minimal paste-ready snippet, one DSL snippet only, or a strict first-line token/verdict/letter.",
+      "instructions": [
+        "Check whether the answer delivers the required command/snippet/token as the first meaningful content instead of leading with explanatory prose.",
+        "If the prompt requires one snippet only or a command on the first line, treat prose-first structure as negative evidence even when later content is knowledgeable.",
+        "If the prompt asks for a strict one-line token or exact opening phrase, verify that contract before considering the rest of the answer."
+      ],
+      "success_signals": [
+        "The first meaningful content is the requested command, snippet, or exact verdict token.",
+        "The answer stays snippet-first or command-first when the prompt explicitly requires it."
+      ],
+      "failure_signals": [
+        "The answer starts with explanatory prose when the prompt requires command-first or snippet-first output.",
+        "The answer is prose-only even though the prompt requires a concrete snippet, command, or exact opening token."
+      ]
+    }
+  ],
+  "artifact_type": "grading-spec",
+  "schema_version": 2
+}