@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/energykit/evals/evals.json

@@ -2,43 +2,44 @@
   "skill_name": "energykit",
   "evals": [
     {
-      "id": 1,
-      "name": "ev-guidance-load-events",
-      "prompt": "Review this EnergyKit EV charging integration. It enables the capability, fetches ElectricityGuidance for an EnergyVenue, builds a charging schedule, then submits one ElectricVehicleLoadEvent at the end of the session with `GuidanceState(wasFollowingGuidance: true, guidanceToken: UUID())`. Give corrected Swift-level guidance without turning this into a HomeKit or BackgroundTasks tutorial.",
-      "expected_output": "A concise review that preserves the EnergyKit boundary, uses the real guidance token, and explains the load event lifecycle and cadence for EV charging.",
+      "id": 0,
+      "name": "ev-charging-guidance-events",
+      "prompt": "We make a home EV charger app for iOS 26. Show how to fetch cleaner-grid charging windows and report the charging session back to EnergyKit. Our draft generates a UUID() as the guidance token and sends one event when charging finishes. Also, should we build HomeKit scenes and a BGTaskScheduler job for this?",
+      "expected_output": "Starts with the EnergyKit capability and com.apple.developer.energykit entitlement, discovers the venue, queries guidance with .shift, stores the real guidanceToken and attaches it to load events, rejects the fabricated UUID, reports begin, periodic active samples about every 15 minutes plus extra samples on user actions, pauses, new guidance or sharp power changes, and end, and keeps HomeKit automation and BackgroundTasks out of scope.",
       "files": [],
-      "assertions": [
-        "Lists the EnergyKit capability or `com.apple.developer.energykit` entitlement as a setup prerequisite, not only as an error-handling detail.",
-        "Rejects fabricating a guidance token and says the event must use the `guidanceToken` returned by EnergyKit guidance on the device/app that requested it.",
-        "Requires a session lifecycle with `.begin`, `.active`, and `.end` events rather than only one final event.",
-        "Mentions EV sampling about every 15 minutes when charging is steady and additional events for significant changes such as user actions, pauses, new guidance, or rapid power changes.",
-        "Keeps the answer focused on EnergyKit rather than expanding into HomeKit automation or BackgroundTasks implementation details."
+      "expectations": [
+        "Treats the EnergyKit entitlement as a setup prerequisite",
+        "Rejects a fabricated UUID token and uses the guidanceToken EnergyKit returned",
+        "Reports a begin, active and end lifecycle",
+        "Samples roughly every 15 minutes and adds events for significant changes",
+        "Does not expand into HomeKit automation or BackgroundTasks implementation"
       ]
     },
     {
-      "id": 2,
-      "name": "insight-availability-granularity",
-      "prompt": "Fix this EnergyKit insight snippet for an app that supports iOS 26.0: it requests `.daily` insights for the last 7 days and immediately reads `record.dataByGridCleanliness?.cleaner`. Include the right availability and range guidance.",
-      "expected_output": "Corrects the query granularity/range mismatch and guards grid cleanliness insight access for iOS/iPadOS 26.1+.",
+      "id": 1,
+      "name": "insight-history-granularity",
+      "prompt": "Our energy dashboard supports iOS 26.0 and later. We want a seven-day chart split by cleaner versus less clean grid power, and a monthly total. Which EnergyKit insight queries should we build?",
+      "expected_output": "Uses .hourly granularity for the seven-day range and .daily only for ranges of at least a calendar month, requests .cleanliness only for the split chart and leaves options empty for totals, guards dataByGridCleanliness with a 26.1 availability check, and does not claim that every insight API requires 26.1.",
       "files": [],
-      "assertions": [
-        "States that `dataByGridCleanliness` / grid cleanliness breakdowns require iOS/iPadOS 26.1+ and should be guarded or omitted on 26.0.",
-        "Does not claim all EnergyKit insight APIs are 26.1-only.",
-        "Pairs granularity to the requested range, using `.hourly` for a seven-day or calendar-week view and `.daily` only for at least a calendar month.",
-        "Mentions that empty `ElectricityInsightQuery.Options` returns totals without cleanliness or tariff breakdown."
+      "expectations": [
+        "Guards grid-cleanliness data with an iOS 26.1 availability check",
+        "Does not say all insight APIs are 26.1 only",
+        "Uses .hourly for seven days and .daily only for a month or longer",
+        "Explains that empty options return totals only"
       ]
     },
     {
-      "id": 3,
-      "name": "energykit-boundary-review",
-      "prompt": "A teammate wrote: 'EnergyKit works anywhere, no entitlement is needed, and if there are no insights we can infer usage from MetricKit power metrics.' Review this plan for a smart thermostat app and identify the minimal corrections.",
-      "expected_output": "Identifies EnergyKit entitlement, region/venue limitations, submitted load-event dependency for insights, HVAC stage-change event guidance, and avoids MetricKit substitution.",
+      "id": 2,
+      "name": "energykit-scope-review",
+      "prompt": "Review this heat-pump app design: no EnergyKit capability yet, it assumes forecasts exist in every country, it uses a hardcoded venue UUID, it reads MetricKit power data to fill the insights screen, and it logs one HVAC row per day.",
+      "expected_output": "Adds the entitlement and handles permissionDenied, handles unsupportedRegion and guidanceUnavailable, discovers venues instead of hardcoding one, explains that insights come only from submitted EnergyKit load events so MetricKit is not a substitute, and reports separate HVAC events at start, each stage change and stop.",
       "files": [],
-      "assertions": [
-        "Requires the `com.apple.developer.energykit` entitlement and handles `EnergyKitError.permissionDenied` rather than saying no entitlement is needed.",
-        "Handles unsupported regions and unavailable or restricted venues instead of claiming EnergyKit works anywhere.",
-        "States that EnergyKit insights depend on submitted EnergyKit load events and cannot be replaced by MetricKit power metrics.",
-        "For HVAC, recommends separate load events for heating/cooling stage transitions rather than only fixed timer samples or a final summary."
+      "expectations": [
+        "Requires the entitlement and handling of permissionDenied",
+        "Handles unsupported regions and missing guidance",
+        "Discovers venues rather than hardcoding a UUID",
+        "States insights depend on submitted load events and MetricKit is not a replacement",
+        "Reports distinct HVAC events for stage transitions"
       ]
     }
   ]