@mmerterden/multi-agent-pipeline 20.6.0 → 20.8.0

package/pipeline/skills/shared/external/homekit-matter/evals/evals.json

@@ -3,42 +3,47 @@
   "evals": [
     {
       "id": 0,
-      "prompt": "I'm building a HomeKit app that should create a nightly Good Night automation at 10:30 PM. Sketch the Swift guidance for creating the action set, adding a timer trigger that repeats daily, attaching the scene, and enabling the trigger. Call out the HomeKit setup prerequisites and any timing gotchas.",
-      "expected_output": "HomeKit automation guidance that uses a single HMHomeManager, creates an HMActionSet, creates an HMTimerTrigger with a first fire date at 22:30 and a daily recurrence interval, attaches the action set, enables the trigger, and lists HomeKit capability/usage-description prerequisites.",
+      "name": "nightly-scene-timer",
+      "prompt": "Our iOS 26 home app needs a Lights Out scene that switches off the hallway and kitchen lamps and locks the front door, running automatically every night at 22:30. Show how to set this up with HomeKit, including project setup and when the app may start touching the user's homes.",
+      "expected_output": "Uses one shared HMHomeManager and waits for homeManagerDidUpdateHomes before reading homes. Builds an HMActionSet with HMCharacteristicWriteAction entries, creates an HMTimerTrigger whose first fire date is the next 22:30 and whose recurrence is DateComponents(day: 1), then adds the trigger to the home, attaches the action set and enables it. Mentions the HomeKit capability and NSHomeKitUsageDescription. Stays in HomeKit.",
       "files": [],
       "expectations": [
-        "Uses HomeKit/HMHomeManager, HMHome, HMActionSet, and HMTimerTrigger rather than routing the task to MatterSupport or AccessorySetupKit.",
-        "Computes the first fire date from hour/minute components such as 22:30.",
-        "Uses a recurrence interval such as DateComponents(day: 1) for daily repeat instead of reusing hour/minute wall-clock components as the recurrence.",
-        "Adds the trigger to the home, adds the action set to the trigger, and enables the trigger.",
-        "Mentions the HomeKit capability and NSHomeKitUsageDescription, and does not access homes before homeManagerDidUpdateHomes."
+        "Keeps a single HMHomeManager and does not read homes until homeManagerDidUpdateHomes(_:) runs",
+        "Builds the scene as an HMActionSet containing HMCharacteristicWriteAction values",
+        "Computes the first 22:30 fire date and uses DateComponents(day: 1) as the recurrence, not hour and minute components",
+        "Orders the calls as addTrigger, then addActionSet on the trigger, then enable(true)",
+        "Lists the HomeKit capability and NSHomeKitUsageDescription",
+        "Does not route the task to MatterSupport or AccessorySetupKit"
       ]
     },
     {
       "id": 1,
-      "prompt": "We need to add Matter lights into our own smart-home ecosystem from an iOS app. Write the setup checklist and Swift-oriented commissioning outline, including discovery configuration, the extension handler callbacks, optional setup-code handling, and what the app should do after perform() succeeds.",
-      "expected_output": "MatterSupport commissioning guidance that adds the MatterSupport extension, declares Matter Bonjour services, uses MatterAddDeviceRequest with topology and isSupported, explains conditional setup-payload entitlement/MTRSetupPayload usage, and outlines the extension handler callbacks for credential validation, rooms, commissioning, configuration, and optional network selection.",
+      "name": "matter-into-own-fabric",
+      "prompt": "We sell Matter smart plugs and run our own cloud ecosystem. Customers should onboard plugs from our iOS app straight into our fabric, sometimes by scanning a code in our own camera screen. What targets, Info.plist keys, entitlements and code do we need?",
+      "expected_output": "Describes a MatterSupport extension target whose principal class subclasses MatterAddDeviceExtensionRequestHandler, the three Bonjour service types, checking MatterAddDeviceRequest.isSupported and building a Topology before calling perform(), and the setup-payload entitlement only because the app passes an MTRSetupPayload it scanned itself. Implements validateDeviceCredential(_:), rooms(in:), configureDevice(named:in:) and commissionDevice(in:onboardingPayload:commissioningID:), optionally Wi-Fi or Thread network selection.",
       "files": [],
       "expectations": [
-        "Adds or verifies a MatterSupport extension target with a MatterAddDeviceExtensionRequestHandler principal handler.",
-        "Includes NSBonjourServices entries for _matter._tcp, _matterc._udp, and _matterd._udp.",
-        "Checks MatterAddDeviceRequest.isSupported and builds a topology with ecosystem name and homes before perform().",
-        "Limits com.apple.developer.matter.allow-setup-payload to callers that provide setup codes programmatically and mentions MTRSetupPayload from the Matter framework for that case.",
-        "Names extension callbacks such as validateDeviceCredential(_:), rooms(in:), commissionDevice(in:onboardingPayload:commissioningID:), configureDevice(named:in:), and optional Wi-Fi/Thread network selection.",
-        "Does not present MatterSupport commissioning as ordinary HMHome accessory setup or a lower-level AccessorySetupKit picker flow."
+        "Adds a MatterSupport extension target with a MatterAddDeviceExtensionRequestHandler subclass as principal class",
+        "Lists _matter._tcp, _matterc._udp and _matterd._udp under NSBonjourServices",
+        "Checks MatterAddDeviceRequest.isSupported and passes a Topology before perform()",
+        "Ties com.apple.developer.matter.allow-setup-payload to passing an MTRSetupPayload from the app",
+        "Implements validateDeviceCredential(_:), rooms(in:), configureDevice(named:in:) and commissionDevice(in:onboardingPayload:commissioningID:)",
+        "Mentions optional selectWiFiNetwork or selectThreadNetwork overrides",
+        "Does not describe the flow as creating an HMHome accessory or as an AccessorySetupKit picker"
       ]
     },
     {
       "id": 2,
-      "prompt": "A product proposal says one Apple accessory guide should cover Matter light onboarding, HomeKit rooms and automations, BLE discovery for a setup-only accessory, GATT control after selection, and joining a temporary Wi-Fi setup network. Write a routing note that assigns each part to the right framework domain and explains what belongs in the HomeKit skill.",
-      "expected_output": "A boundary-aware routing note that assigns HomeKit/MatterSupport to homes, rooms, HMAccessory characteristics, action sets, triggers, and Matter onboarding; AccessorySetupKit to Bluetooth/Wi-Fi discovery, picker authorization, session events, and migration; CoreBluetooth/GATT to BLE communication after selection; and NetworkExtension to Wi-Fi setup-network joins.",
+      "name": "framework-routing",
+      "prompt": "Write a short routing note for our team: we build a thermostat app that pairs a Bluetooth sensor, sometimes joins the sensor's setup Wi-Fi, and also creates HomeKit scenes. Which Apple framework owns which part?",
+      "expected_output": "HomeKit and MatterSupport own homes, rooms, characteristics, scenes, triggers, automations and Matter onboarding. AccessorySetupKit owns BLE and Wi-Fi discovery descriptors, picker authorization, ASAccessorySession events and migration. CoreBluetooth handles GATT traffic with the picked sensor. NetworkExtension handles joining the temporary setup Wi-Fi. The note keeps these separate.",
       "files": [],
       "expectations": [
-        "Identifies HomeKit/MatterSupport as the owner for homes, rooms, HMAccessory services/characteristics, action sets, triggers, automations, and Matter commissioning.",
-        "Limits AccessorySetupKit to Bluetooth/Wi-Fi discovery descriptors, privacy-preserving picker authorization, or ASAccessorySession setup events.",
-        "Routes post-selection BLE service and characteristic communication to CoreBluetooth/GATT.",
-        "Routes joining or configuring a temporary Wi-Fi setup network to NetworkExtension.",
-        "Does not collapse lower-level accessory discovery, GATT communication, or Wi-Fi network joins into HomeKit automation guidance."
+        "Assigns homes, rooms, characteristics, action sets, triggers and Matter onboarding to HomeKit and MatterSupport",
+        "Assigns discovery descriptors, picker authorization, ASAccessorySession events and migration to AccessorySetupKit",
+        "Assigns post-selection Bluetooth communication to CoreBluetooth and GATT",
+        "Assigns joining a temporary setup Wi-Fi network to NetworkExtension",
+        "Does not fold AccessorySetupKit, CoreBluetooth or NetworkExtension work into HomeKit"
       ]
     }
   ]