@weavetab/skills 2.5.0-beta.5 → 2.5.1

package/evals/scenarios.json

@@ -1,239 +1,239 @@
-{
-  "version": "2.5.0-beta.5",
-  "scenarios": [
-    {
-      "id": "spa_stale_ref_recovery",
-      "description": "Agent encounters detached DOM node during Next.js hydration",
-      "intent": "Button ref w:14 became detached after client-side state update",
-      "expectedDomain": "resilience",
-      "expectedGuide": "stale-ref-recovery.md",
-      "expectedTools": [
-        "browser_find",
-        "browser_map"
-      ],
-      "bannedTools": [
-        "scrape_page",
-        "browser_evaluate"
-      ],
-      "assertion": "Must rely on 4-tier auto-recovery or targeted browser_find rather than repeating identical stale ref"
-    },
-    {
-      "id": "secret_containment_verification",
-      "description": "Agent enters user master password or OAuth secret",
-      "intent": "Enter user account password and security pin on auth page",
-      "expectedDomain": "security",
-      "expectedGuide": "secret-containment-protocol.md",
-      "expectedTools": [
-        "browser_type_secret"
-      ],
-      "bannedTools": [
-        "scrape_page",
-        "browser_evaluate",
-        "browser_ask"
-      ],
-      "assertion": "Must strictly use browser_type_secret to guarantee zero-bleed in logs and transcripts"
-    },
-    {
-      "id": "electron_app_switch",
-      "description": "Agent detects and sends message inside Slack desktop client",
-      "intent": "Find running Slack desktop app and post deployment confirmation",
-      "expectedDomain": "desktop",
-      "expectedGuide": "desktop-targeting.md",
-      "expectedTools": [
-        "browser_detect",
-        "browser_type"
-      ],
-      "bannedTools": [
-        "click_element"
-      ],
-      "assertion": "Must discover running Electron app and pass app: 'slack' parameter"
-    },
-    {
-      "id": "checkpoint_rollback_on_error",
-      "description": "Agent snapshots session before checkout and rolls back after card decline",
-      "intent": "Save browser checkpoint before payment submit; restore on failure",
-      "expectedDomain": "resilience",
-      "expectedGuide": "session-time-travel.md",
-      "expectedTools": [
-        "browser_checkpoint"
-      ],
-      "bannedTools": [
-        "browser_evaluate"
-      ],
-      "assertion": "Must execute browser_checkpoint with action save and restore to preserve cookies and storage"
-    },
-    {
-      "id": "token_conservation_dom_map",
-      "description": "Agent needs to inspect interactive buttons without burning 50k tokens",
-      "intent": "Inspect interactive controls on large documentation portal",
-      "expectedDomain": "browser",
-      "expectedGuide": "dom-strategy.md",
-      "expectedTools": [
-        "browser_map",
-        "browser_find"
-      ],
-      "bannedTools": [
-        "browser_ask"
-      ],
-      "assertion": "Must pass lite: true or use targeted browser_find for keyword jumps"
-    },
-    {
-      "id": "office_document_parsing",
-      "description": "Agent downloads and parses quarterly revenue spreadsheet",
-      "intent": "Read quarterly_revenue.xlsx sheets and extract department budgets",
-      "expectedDomain": "documents",
-      "expectedGuide": "office-document-ingestion.md",
-      "expectedTools": [
-        "browser_office"
-      ],
-      "bannedTools": [
-        "browser_evaluate"
-      ],
-      "assertion": "Must use browser_office to ingest Excel spreadsheet data into structured JSON"
-    },
-    {
-      "id": "network_mock_and_intercept",
-      "description": "Agent mocks a failing billing endpoint to verify error state UI",
-      "intent": "Intercept /api/v1/billing and return HTTP 500 error",
-      "expectedDomain": "network-and-perf",
-      "expectedGuide": "network-intercept-and-mock.md",
-      "expectedTools": [
-        "browser_network_intercept"
-      ],
-      "bannedTools": [
-        "scrape_page"
-      ],
-      "assertion": "Must route request stubbing through browser_network_intercept"
-    },
-    {
-      "id": "macro_compilation_workflow",
-      "description": "Agent compiles multi-step report generation into deterministic pipeline",
-      "intent": "Compile recent search and export actions into parameterized macro",
-      "expectedDomain": "memory",
-      "expectedGuide": "session-trail.md",
-      "expectedTools": [
-        "browser_burst"
-      ],
-      "bannedTools": [
-        "browser_evaluate"
-      ],
-      "assertion": "Must compile trail using browser_burst with compile_from_history: true"
-    },
-    {
-      "id": "form_atomic_batch_fill",
-      "description": "Agent fills out 5-field customer registration form in one turn",
-      "intent": "Fill first name, last name, email, and agree to terms checkbox",
-      "expectedDomain": "browser",
-      "expectedGuide": "form-automation.md",
-      "expectedTools": [
-        "browser_fill"
-      ],
-      "bannedTools": [
-        "fill_field"
-      ],
-      "assertion": "Must pass array of fields to browser_fill rather than executing individual typing turns"
-    },
-    {
-      "id": "pointer_trajectory_drag_and_drop",
-      "description": "Agent drags kanban card from In Progress to Done",
-      "intent": "Move Kanban card element from coordinates (200,300) to (600,300)",
-      "expectedDomain": "browser",
-      "expectedGuide": "interaction-pipeline.md",
-      "expectedTools": [
-        "browser_pointer"
-      ],
-      "bannedTools": [
-        "click_element"
-      ],
-      "assertion": "Must dispatch down, move, and up actions via browser_pointer"
-    },
-    {
-      "id": "tab_switch_and_close",
-      "description": "Agent opens OAuth popup, completes flow, and switches back to main tab",
-      "intent": "Enumerate active page targets, switch focus to popup, and close on finish",
-      "expectedDomain": "browser",
-      "expectedGuide": "tab-and-window.md",
-      "expectedTools": [
-        "browser_tabs"
-      ],
-      "bannedTools": [
-        "browser_evaluate"
-      ],
-      "assertion": "Must use browser_tabs actions (list, switch, close)"
-    },
-    {
-      "id": "core_web_vitals_audit",
-      "description": "Agent measures LCP and CLS metrics on landing page",
-      "intent": "Measure Core Web Vitals and verify page render performance",
-      "expectedDomain": "network-and-perf",
-      "expectedGuide": "performance-and-vitals.md",
-      "expectedTools": [
-        "browser_performance",
-        "browser_wait"
-      ],
-      "bannedTools": [
-        "browser_evaluate"
-      ],
-      "assertion": "Must query browser_performance for vitals metrics"
-    },
-    {
-      "id": "device_mobile_emulation",
-      "description": "Agent emulates iPhone 15 viewport and throttled 3G connectivity",
-      "intent": "Audit mobile layout on iPhone 15 with slow network profile",
-      "expectedDomain": "network-and-perf",
-      "expectedGuide": "device-and-environment.md",
-      "expectedTools": [
-        "browser_emulate"
-      ],
-      "bannedTools": [
-        "scrape_page"
-      ],
-      "assertion": "Must configure viewport and network throttling via browser_emulate"
-    },
-    {
-      "id": "plugin_dynamic_load",
-      "description": "Agent uses tools registered by an installed Weavetab plugin",
-      "intent": "Use plugin-provided tools that are automatically loaded at server startup",
-      "expectedDomain": "plugins",
-      "expectedGuide": "plugin-runtime.md",
-      "expectedTools": [
-        "browser_snapshot"
-      ],
-      "bannedTools": [
-        "browser_ask",
-        "list_plugins",
-        "load_plugin"
-      ],
-      "assertion": "Must NOT call list_plugins or load_plugin — plugin tools are available directly in the tool list"
-    },
-    {
-      "id": "github_repository_triage",
-      "description": "Agent inspects open pull requests on Weavetab/MCP repository",
-      "intent": "Examine PR diff and review comments without loading web browser UI",
-      "expectedDomain": "developer",
-      "expectedGuide": "github-forensics.md",
-      "expectedTools": [
-        "browser_github"
-      ],
-      "bannedTools": [
-        "browser_navigate",
-        "browser_map"
-      ],
-      "assertion": "Must use native browser_github tool to save browser turns"
-    },
-    {
-      "id": "loop_breaker_reset",
-      "description": "Agent breaks out of repetitive click strike trap",
-      "intent": "Diagnose silent validation error and reset server loop strike counter",
-      "expectedDomain": "resilience",
-      "expectedGuide": "loop-breaker.md",
-      "expectedTools": [
-        "browser_map"
-      ],
-      "bannedTools": [
-        "browser_ask"
-      ],
-      "assertion": "Must reset loop strikes via browser_map({ force: true }) after diagnosing issue"
-    }
-  ]
-}
+{
+  "version": "2.5.1",
+  "scenarios": [
+    {
+      "id": "spa_stale_ref_recovery",
+      "description": "Agent encounters detached DOM node during Next.js hydration",
+      "intent": "Button ref w:14 became detached after client-side state update",
+      "expectedDomain": "resilience",
+      "expectedGuide": "stale-ref-recovery.md",
+      "expectedTools": [
+        "browser_find",
+        "browser_map"
+      ],
+      "bannedTools": [
+        "scrape_page",
+        "browser_evaluate"
+      ],
+      "assertion": "Must rely on multi-tier auto-recovery or targeted browser_find rather than repeating identical stale ref"
+    },
+    {
+      "id": "secret_containment_verification",
+      "description": "Agent enters user master password or OAuth secret",
+      "intent": "Enter user account password and security pin on auth page",
+      "expectedDomain": "security",
+      "expectedGuide": "secret-containment-protocol.md",
+      "expectedTools": [
+        "browser_type_secret"
+      ],
+      "bannedTools": [
+        "scrape_page",
+        "browser_evaluate",
+        "browser_ask"
+      ],
+      "assertion": "Must strictly use browser_type_secret to guarantee zero-bleed in logs and transcripts"
+    },
+    {
+      "id": "electron_app_switch",
+      "description": "Agent detects and sends message inside Slack desktop client",
+      "intent": "Find running Slack desktop app and post deployment confirmation",
+      "expectedDomain": "desktop",
+      "expectedGuide": "desktop-targeting.md",
+      "expectedTools": [
+        "browser_detect",
+        "browser_type"
+      ],
+      "bannedTools": [
+        "click_element"
+      ],
+      "assertion": "Must discover running Electron app and pass app: 'slack' parameter"
+    },
+    {
+      "id": "checkpoint_rollback_on_error",
+      "description": "Agent snapshots session before checkout and rolls back after card decline",
+      "intent": "Save browser checkpoint before payment submit; restore on failure",
+      "expectedDomain": "resilience",
+      "expectedGuide": "session-time-travel.md",
+      "expectedTools": [
+        "browser_checkpoint"
+      ],
+      "bannedTools": [
+        "browser_evaluate"
+      ],
+      "assertion": "Must execute browser_checkpoint with action save and restore to preserve cookies and storage"
+    },
+    {
+      "id": "token_conservation_dom_map",
+      "description": "Agent needs to inspect interactive buttons without burning 50k tokens",
+      "intent": "Inspect interactive controls on large documentation portal",
+      "expectedDomain": "browser",
+      "expectedGuide": "dom-strategy.md",
+      "expectedTools": [
+        "browser_map",
+        "browser_find"
+      ],
+      "bannedTools": [
+        "browser_ask"
+      ],
+      "assertion": "Must pass lite: true or use targeted browser_find for keyword jumps"
+    },
+    {
+      "id": "office_document_parsing",
+      "description": "Agent downloads and parses quarterly revenue spreadsheet",
+      "intent": "Read quarterly_revenue.xlsx sheets and extract department budgets",
+      "expectedDomain": "documents",
+      "expectedGuide": "office-document-ingestion.md",
+      "expectedTools": [
+        "browser_office"
+      ],
+      "bannedTools": [
+        "browser_evaluate"
+      ],
+      "assertion": "Must use browser_office to ingest Excel spreadsheet data into structured JSON"
+    },
+    {
+      "id": "network_mock_and_intercept",
+      "description": "Agent mocks a failing billing endpoint to verify error state UI",
+      "intent": "Intercept /api/v1/billing and return HTTP 500 error",
+      "expectedDomain": "network-and-perf",
+      "expectedGuide": "network-intercept-and-mock.md",
+      "expectedTools": [
+        "browser_network_intercept"
+      ],
+      "bannedTools": [
+        "scrape_page"
+      ],
+      "assertion": "Must route request stubbing through browser_network_intercept"
+    },
+    {
+      "id": "macro_compilation_workflow",
+      "description": "Agent compiles multi-step report generation into deterministic pipeline",
+      "intent": "Compile recent search and export actions into parameterized macro",
+      "expectedDomain": "memory",
+      "expectedGuide": "session-trail.md",
+      "expectedTools": [
+        "browser_burst"
+      ],
+      "bannedTools": [
+        "browser_evaluate"
+      ],
+      "assertion": "Must compile trail using browser_burst with compile_from_history: true"
+    },
+    {
+      "id": "form_atomic_batch_fill",
+      "description": "Agent fills out 5-field customer registration form in one turn",
+      "intent": "Fill first name, last name, email, and agree to terms checkbox",
+      "expectedDomain": "browser",
+      "expectedGuide": "form-automation.md",
+      "expectedTools": [
+        "browser_fill"
+      ],
+      "bannedTools": [
+        "fill_field"
+      ],
+      "assertion": "Must pass array of fields to browser_fill rather than executing individual typing turns"
+    },
+    {
+      "id": "pointer_trajectory_drag_and_drop",
+      "description": "Agent drags kanban card from In Progress to Done",
+      "intent": "Move Kanban card element from coordinates (200,300) to (600,300)",
+      "expectedDomain": "browser",
+      "expectedGuide": "interaction-pipeline.md",
+      "expectedTools": [
+        "browser_pointer"
+      ],
+      "bannedTools": [
+        "click_element"
+      ],
+      "assertion": "Must use the drag_and_drop action via browser_pointer (not manual down/move/up)"
+    },
+    {
+      "id": "tab_switch_and_close",
+      "description": "Agent opens OAuth popup, completes flow, and switches back to main tab",
+      "intent": "Enumerate active page targets, switch focus to popup, and close on finish",
+      "expectedDomain": "browser",
+      "expectedGuide": "tab-and-window.md",
+      "expectedTools": [
+        "browser_tabs"
+      ],
+      "bannedTools": [
+        "browser_evaluate"
+      ],
+      "assertion": "Must use browser_tabs actions (list, switch, close)"
+    },
+    {
+      "id": "core_web_vitals_audit",
+      "description": "Agent measures LCP and CLS metrics on landing page",
+      "intent": "Measure Core Web Vitals and verify page render performance",
+      "expectedDomain": "network-and-perf",
+      "expectedGuide": "performance-and-vitals.md",
+      "expectedTools": [
+        "browser_performance",
+        "browser_wait"
+      ],
+      "bannedTools": [
+        "browser_evaluate"
+      ],
+      "assertion": "Must query browser_performance for vitals metrics"
+    },
+    {
+      "id": "device_mobile_emulation",
+      "description": "Agent emulates iPhone 15 viewport and throttled 3G connectivity",
+      "intent": "Audit mobile layout on iPhone 15 with slow network profile",
+      "expectedDomain": "network-and-perf",
+      "expectedGuide": "device-and-environment.md",
+      "expectedTools": [
+        "browser_emulate"
+      ],
+      "bannedTools": [
+        "scrape_page"
+      ],
+      "assertion": "Must configure viewport and network throttling via browser_emulate"
+    },
+    {
+      "id": "plugin_dynamic_load",
+      "description": "Agent uses tools registered by an installed Weavetab plugin",
+      "intent": "Use plugin-provided tools that are automatically loaded at server startup",
+      "expectedDomain": "plugins",
+      "expectedGuide": "plugin-runtime.md",
+      "expectedTools": [
+        "browser_snapshot"
+      ],
+      "bannedTools": [
+        "browser_ask",
+        "list_plugins",
+        "load_plugin"
+      ],
+      "assertion": "Must NOT call list_plugins or load_plugin — plugin tools are available directly in the tool list"
+    },
+    {
+      "id": "github_repository_triage",
+      "description": "Agent inspects open pull requests on Weavetab/MCP repository",
+      "intent": "Examine PR diff and review comments without loading web browser UI",
+      "expectedDomain": "developer",
+      "expectedGuide": "github-forensics.md",
+      "expectedTools": [
+        "github"
+      ],
+      "bannedTools": [
+        "browser_navigate",
+        "browser_map"
+      ],
+      "assertion": "Must use the github plugin tool to save browser turns"
+    },
+    {
+      "id": "loop_breaker_reset",
+      "description": "Agent breaks out of repetitive click strike trap",
+      "intent": "Diagnose silent validation error and reset server loop strike counter",
+      "expectedDomain": "resilience",
+      "expectedGuide": "loop-breaker.md",
+      "expectedTools": [
+        "browser_map"
+      ],
+      "bannedTools": [
+        "browser_ask"
+      ],
+      "assertion": "Must reset loop strikes via browser_map({ force: true }) after diagnosing issue"
+    }
+  ]
+}