@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/debugging-instruments/evals/evals.json

@@ -2,45 +2,42 @@
   "skill_name": "debugging-instruments",
   "evals": [
     {
-      "id": 0,
-      "prompt": "I need a concise profiling plan for an iOS app that has a slow feed refresh, battery-drain complaints, and a CI job that should collect a Time Profiler trace with xctrace. Include the Instruments templates or instruments to use, and show both the basic xctrace launch command and the command shape if we add Allocations to the same Time Profiler recording.",
-      "expected_output": "A source-grounded Instruments plan that chooses current template names, profiles on device, uses Power Profiler for battery work, shows xctrace launch syntax with the required -- separator, and uses --instrument for adding Allocations to a Time Profiler recording.",
+      "id": 1,
+      "prompt": "Our iOS 26 recipe app gets slow when the ingredient list filters, and a few users say it drains their battery. Give me a profiling plan, including how our CI job can record a Time Profiler trace that also captures allocations.",
+      "expected_output": "A profiling plan that picks Time Profiler for the slow interaction and Power Profiler for battery, records on a physical device, and shows an xctrace command with one template plus an added instrument.",
       "files": [],
-      "expectations": [
-        "Recommends Time Profiler for CPU or slow interaction triage.",
-        "Recommends Power Profiler, not Energy Log, for battery or energy-impact analysis.",
-        "Mentions profiling on a physical device for realistic measurements.",
-        "Shows xcrun xctrace record using --launch -- before the app command or bundle path.",
-        "Uses --instrument for adding an extra instrument to a template rather than repeating --template for multiple instruments.",
-        "Does not claim that Energy Log is the current Instruments template name."
+      "assertions": [
+        "Recommends Time Profiler for the CPU-bound slow filtering interaction.",
+        "Recommends Power Profiler for the battery complaint and does not present Energy Log as the current template name.",
+        "Says to profile on a physical device, not in Simulator.",
+        "Shows xcrun xctrace record with the --launch -- separator before the app path.",
+        "Adds Allocations with --instrument rather than passing a second --template."
       ]
     },
     {
-      "id": 1,
-      "prompt": "Review this debugging plan: use the Leaks template to prove every retain cycle, run leaks --atExit -- ./MyApp.app/MyApp for our iOS build, call any 250 ms main-thread pause a severe hang, and enable Thread Sanitizer on a real iPhone test run. Correct the plan.",
-      "expected_output": "A correction-focused review that narrows leak, hang, Memory Graph, and Thread Sanitizer guidance using Apple-documented behavior.",
+      "id": 2,
+      "prompt": "Review this plan from a teammate: 'Run the Leaks template once; if it finds nothing we have no retain cycles. For the device build run leaks --atExit. Hangs under 250 ms are fine because Xcode does not report them. Turn on Thread Sanitizer on the iPhone. Then wire up MetricKit fully.'",
+      "expected_output": "A correction of each point: Leaks scope, the memgraph workflow, hang perception versus tool threshold, where TSan runs, and a short MetricKit handoff.",
       "files": [],
-      "expectations": [
-        "Explains that Leaks detects leaked allocations or isolated cycles but not every still-reachable retain cycle.",
-        "Recommends Memory Graph Debugger and Allocations generation marks for iOS retain-cycle or growth triage.",
-        "Uses exported .memgraph files with leaks for CLI memory-graph inspection instead of presenting --atExit as the default iOS app workflow.",
-        "States that a few hundred milliseconds can feel unresponsive and that Apple tools typically report main-run-loop busy periods over 250 ms.",
-        "States that Thread Sanitizer for iOS-family apps runs in Simulator, while Apple documents device support only for 64-bit macOS apps.",
-        "Does not expand into MetricKit payload handling beyond a concise handoff to the metrickit skill."
+      "assertions": [
+        "Explains that Leaks does not prove the absence of cycles that are still reachable, and recommends the Memory Graph Debugger plus Allocations generation marks.",
+        "Recommends exporting a .memgraph and inspecting it with leaks instead of leaks --atExit as the default iOS workflow.",
+        "States that a few hundred milliseconds can already feel unresponsive even though tools report above 250 ms.",
+        "States that Thread Sanitizer runs in Simulator for iOS-family apps and that device support is documented only for 64-bit macOS apps.",
+        "Keeps MetricKit to a brief pointer to the metrickit-diagnostics skill."
       ]
     },
     {
-      "id": 2,
-      "prompt": "My SwiftUI screen stutters while scrolling and I also want production hang telemetry in MetricKit plus a new Swift Testing benchmark. What should this debugging skill handle directly, and what should move to adjacent skills?",
-      "expected_output": "A boundary-aware answer that keeps Instruments and LLDB guidance in debugging-instruments while routing SwiftUI remediation, MetricKit ingestion, and Swift Testing test design to sibling skills.",
+      "id": 3,
+      "prompt": "Scrolling our SwiftUI photo feed stutters on an iPhone 15. How should I capture what is happening, how do I fix the views, how do I collect hang reports from production, and how should I structure benchmark tests for this?",
+      "expected_output": "Capture guidance stays in scope; SwiftUI fixes, production hang ingestion and benchmark organisation are routed to their skills in a sentence each.",
       "files": [],
-      "expectations": [
-        "Keeps Instruments capture guidance in scope, such as SwiftUI or Animation Hitches with Time Profiler on device.",
-        "Routes SwiftUI code-level remediation patterns to the swiftui-performance skill.",
-        "Routes MetricKit subscriber setup, MXDiagnosticPayload processing, or MXHangDiagnostic ingestion details to the metrickit skill.",
-        "Routes Swift Testing or XCTest benchmark organization details to the swift-testing skill.",
-        "Mentions OSSignposter or signposts as useful instrumentation for correlating app intervals with Instruments.",
-        "Does not collapse the sibling skills by giving a full MetricKit ingestion implementation or full SwiftUI refactor recipe."
+      "assertions": [
+        "Recommends recording the SwiftUI and Animation Hitches instruments together with Time Profiler on the device.",
+        "Mentions OSSignposter or signposts to correlate app phases with the trace.",
+        "Routes SwiftUI view remediation to swiftui-performance without writing a full refactor.",
+        "Routes MXDiagnosticPayload and MXHangDiagnostic handling to metrickit-diagnostics without a full implementation.",
+        "Routes benchmark test organisation to swift-testing without a full implementation."
       ]
     }
   ]