@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/background-processing/evals/evals.json

@@ -2,43 +2,43 @@
   "skill_name": "background-processing",
   "evals": [
     {
-      "id": 1,
+      "id": 0,
       "name": "bg-task-registration-review",
-      "prompt": "Review this iOS background task setup for a SwiftUI app. It registers a BGAppRefreshTask from a view's .task modifier, submits a BGProcessingTask without adding BGTaskSchedulerPermittedIdentifiers, and does not set an expirationHandler. Give a concise fix plan and corrected Swift/Info.plist snippets.",
-      "expected_output": "Identifies launch-time registration, Info.plist permitted identifiers, required UIBackgroundModes, expiration handling, setTaskCompleted on all paths, and rescheduling patterns.",
+      "prompt": "Our SwiftUI news app registers its BGAppRefreshTask handler inside the root view's .task modifier. The identifier is not in Info.plist yet, the handler only calls setTaskCompleted when the fetch succeeds, and we schedule the next refresh from the Settings screen. Refreshes never seem to happen. What is wrong?",
+      "expected_output": "Moves registration to App.init or didFinishLaunching, adds the identifier to BGTaskSchedulerPermittedIdentifiers with the fetch background mode, completes the task on success, failure and expiry with an expirationHandler that cancels work, and reschedules from inside the handler.",
       "files": [],
       "assertions": [
-        "Says BGTaskScheduler handlers must be registered before app launch completes, not from a view task.",
-        "Names BGTaskSchedulerPermittedIdentifiers and the fetch/processing UIBackgroundModes requirements.",
-        "Requires expirationHandler cancellation and setTaskCompleted(success:) on every path.",
-        "Shows or describes scheduling the next request from inside the task handler."
+        "Registration happens during launch (App initializer or application(_:didFinishLaunchingWithOptions:)), never from a view's .task",
+        "The identifier is listed in BGTaskSchedulerPermittedIdentifiers and UIBackgroundModes includes fetch (and processing where used)",
+        "An expirationHandler cancels in-flight work and setTaskCompleted(success:) is called on every path",
+        "The next refresh is scheduled inside the handler itself"
       ]
     },
     {
-      "id": 2,
+      "id": 1,
       "name": "continued-processing-ios26",
-      "prompt": "I need an iOS 26 photo export that starts from a user tap and keeps running if the user backgrounds the app. It may use GPU acceleration. What BackgroundTasks API should I use, and what are the identifier, progress, cancellation, and entitlement gotchas?",
-      "expected_output": "Recommends BGContinuedProcessingTaskRequest/BGContinuedProcessingTask and covers foreground user action, wildcard permitted identifier base with unique job identifiers, ProgressReporting, updateTitle/subtitle, expiration cancellation, supportedResources, and the Background GPU Access entitlement.",
+      "prompt": "On iOS 26 our video editor should keep rendering an export after the user taps Export and switches apps. Rendering uses the GPU when possible and several exports can run in a row. How should this be built?",
+      "expected_output": "Uses BGContinuedProcessingTaskRequest and BGContinuedProcessingTask, gives every export its own identifier under a permitted wildcard, reports progress through task.progress with cancellation handling, and requests GPU only after checking supportedResources, with the Background GPU Access entitlement.",
       "files": [],
       "assertions": [
-        "Recommends BGContinuedProcessingTaskRequest and BGContinuedProcessingTask for iOS 26 foreground-started work.",
-        "Explains that submitted identifiers should be unique per job under a bundle-prefixed wildcard permitted identifier.",
-        "Mentions ProgressReporting or task.progress and cancellation/expiration handling.",
-        "Mentions checking BGTaskScheduler.supportedResources before requesting GPU and requiring the Background GPU Access entitlement."
+        "Chooses BGContinuedProcessingTaskRequest submitted from the foreground and handles a BGContinuedProcessingTask",
+        "Each export gets a unique identifier under a wildcard entry that starts with the bundle prefix",
+        "Updates task.progress (ProgressReporting) as work advances and stops cleanly on cancellation or expiry",
+        "Checks BGTaskScheduler.supportedResources for .gpu and adds the Background GPU Access entitlement before setting requiredResources"
       ]
     },
     {
-      "id": 3,
+      "id": 2,
       "name": "background-push-boundary",
-      "prompt": "Our server wants to wake the app with silent pushes every few minutes to keep chat data fresh, and one engineer suggested using apns-priority 10 so they arrive instantly. Review the plan and say what should change. Do not redesign our notification UI.",
-      "expected_output": "Keeps scope to background push refresh behavior, rejects frequent silent pushes and priority 10, requires content-available, remote-notification mode, apns-push-type background, apns-priority 5, and prompt fetch completion.",
+      "prompt": "Our backend wants to send a silent push every five minutes so the app always has fresh prices when opened. The payload is just {\"aps\":{}} with a price key, sent at priority 10. Also, can you redesign how our price alert banners look?",
+      "expected_output": "Adds content-available: 1 and the Remote notifications background mode, sends apns-push-type background with priority 5, explains that silent pushes are throttled and not guaranteed so every five minutes will not work, and leaves banner design out of scope.",
       "files": [],
       "assertions": [
-        "Requires content-available: 1 and the Remote notifications background mode.",
-        "Requires APNs headers apns-push-type: background and apns-priority: 5.",
-        "Warns that background pushes are low priority, throttled, not guaranteed, and should not be sent every few minutes.",
-        "Stays focused on background refresh behavior rather than redesigning visible notification UI."
+        "Payload gets content-available: 1 and the app enables the Remote notifications background mode",
+        "Request uses the background push type header with priority 5 instead of 10",
+        "Warns that pushes every few minutes will be throttled and delivery is never guaranteed",
+        "Does not redesign visible notification UI"
       ]
     }
   ]
-}
+}