@mmerterden/multi-agent-pipeline 20.6.0 → 20.8.0

package/pipeline/skills/shared/external/permissionkit/evals/evals.json

@@ -2,45 +2,42 @@
   "skill_name": "permissionkit",
   "evals": [
     {
-      "id": 0,
-      "name": "availability-imessage-review",
-      "prompt": "Review this PermissionKit setup for an app that supports iOS 26.0 and iPadOS 26.0: the team uses PermissionButton and AskCenter everywhere without availability checks, says PermissionKit can route child approval through our own in-app DM system, and wants to add a made-up PermissionKit entitlement. Write the corrected implementation guidance.",
-      "expected_output": "A concise PermissionKit setup review that distinguishes 26.0/26.1/26.2 API availability, preserves the iMessage-only communication-experience limitation, and avoids invented entitlement requirements.",
+      "id": 1,
+      "prompt": "Our kids' trivia app has in-game chat. Review this plan: target iOS 26+, add a com.apple.developer.permissionkit entitlement, and when a child wants to chat with a new player, send the parent request through our own in-app inbox using PermissionKit.",
+      "expected_output": "A review that states the exact availability tiers, explains that requests only travel through iMessage, rejects the invented entitlement and keeps the answer on PermissionKit itself.",
       "files": [],
-      "assertions": [
-        "States that core PermissionKit topic, handle, question, response, choice, and CommunicationLimits APIs are available starting in iOS/iPadOS 26.0.",
-        "States that AskError starts in iOS/iPadOS 26.1 and AskCenter, AskCenter ask/responses APIs, PermissionButton, and SignificantAppUpdateTopic require iOS/iPadOS 26.2 or appropriate availability checks.",
-        "Says PermissionKit communication experiences are available only through iMessage and should not be presented as an arbitrary in-app DM routing framework.",
-        "Rejects inventing a PermissionKit entitlement key and says to verify current Apple docs/Xcode capabilities before adding signing requirements.",
-        "Keeps the answer focused on PermissionKit rather than turning it into a general chat moderation or family-account architecture."
+      "expectations": [
+        "States the tiers precisely: 26.0 for the core topic, handle, question, response and CommunicationLimits types, 26.1 for AskError, 26.2 for AskCenter, PermissionButton and SignificantAppUpdateTopic",
+        "Recommends #available or @available checks for the 26.2 calls instead of a blanket iOS 26+ claim",
+        "Explains that the parent request goes through iMessage and cannot be routed through the app's own inbox or DM system",
+        "Rejects the made-up entitlement key and says to confirm any signing requirement in Apple documentation and Xcode capabilities",
+        "Stays on PermissionKit and does not drift into designing a general moderation or chat-safety system"
       ]
     },
     {
-      "id": 1,
-      "name": "known-handle-response-flow",
-      "prompt": "Fix this PermissionKit flow: before showing an Ask button we call `isKnownHandle(_:)` and treat `false` as proof that communication limits are enabled, then call `AskCenter.shared.ask` without handling `communicationLimitsNotEnabled`, and our UI waits forever for a PermissionResponse after every tap. Give corrected Swift-level guidance and pseudocode.",
-      "expected_output": "A corrected PermissionKit communication flow that separates known-handle lookup from enabled-limits handling, catches AskError cases, tracks pending questions, and accounts for child cancellation.",
+      "id": 2,
+      "prompt": "Here is our flow: if CommunicationLimits.current.isKnownHandle(handle) is false we assume limits are on and call CommunicationLimits.current.ask(question, in: self), then read CommunicationLimits.current.updates for the answer and show a spinner until it comes. What should change?",
+      "expected_output": "Corrections covering what known-handle checks mean, the bundle identifier guard, the limits-off error, the current AskCenter APIs and explicit pending, retry, cancel and expiry states.",
       "files": [],
-      "assertions": [
-        "Explains that isKnownHandle(_:) and knownHandles(in:) classify system-known handles and do not prove communication limits are enabled.",
-        "Mentions that knownHandles(in:) requires a non-nil, nonempty app bundle identifier.",
-        "Handles AskError.communicationLimitsNotEnabled from AskCenter.shared.ask as the normal fallback when communication limits are not active.",
-        "Uses AskCenter.shared.responses(for: CommunicationTopic.self) to observe parent decisions rather than deprecated CommunicationLimits ask/update APIs.",
-        "Models pending, retry, cancellation, or expiration because child cancellation of the iMessage send flow may produce no PermissionResponse."
+      "expectations": [
+        "Explains that isKnownHandle and knownHandles(in:) only classify handles and do not prove communication limits are enabled",
+        "Mentions that knownHandles(in:) needs a nonempty bundle identifier and shows a guard for it",
+        "Handles AskError.communicationLimitsNotEnabled as the signal to allow normal communication",
+        "Replaces the deprecated CommunicationLimits ask and updates APIs with AskCenter.shared.ask(_:in:) and AskCenter.shared.responses(for: CommunicationTopic.self)",
+        "Models pending state by question id with retry or expiry, noting that a cancelled send produces no response so the spinner cannot wait forever"
       ]
     },
     {
-      "id": 2,
-      "name": "significant-update-boundary",
-      "prompt": "A children’s app wants to use `SignificantAppUpdateTopic(description: \"We improved the app\")` on devices running iOS 26.0, skip response observation, and use the same flow for every terms-of-service wording tweak. Review the plan and provide the minimal corrected PermissionKit guidance.",
-      "expected_output": "A significant-app-update PermissionKit review that guards iOS/iPadOS 26.2 APIs, requires concrete descriptions, explains the developer/regulatory significance decision, and observes responses.",
+      "id": 3,
+      "prompt": "We plan to ask parents to approve every app update with SignificantAppUpdateTopic(description: \"Improvements and bug fixes\"). Is that right, and how do we handle the answer?",
+      "expected_output": "Guidance that keeps the 26.2 availability, leaves the significance decision to the developer under applicable regulations, rewrites the description as a concrete change and observes responses including the no-answer case, without giving jurisdiction-specific legal advice.",
       "files": [],
-      "assertions": [
-        "States that SignificantAppUpdateTopic and the matching AskCenter ask/responses flow require iOS/iPadOS 26.2+ availability.",
-        "Says the app developer determines whether an update is significant based on applicable regulations rather than using the topic for every small wording tweak.",
-        "Replaces vague descriptions like \"We improved the app\" with concise, understandable text describing concrete user-visible changes.",
-        "Observes AskCenter.shared.responses(for: SignificantAppUpdateTopic.self) and handles approval, denial, and missing response states.",
-        "Keeps legal/regulatory discussion at the boundary and does not provide jurisdiction-specific legal advice."
+      "expectations": [
+        "Notes that SignificantAppUpdateTopic, AskCenter and PermissionButton need 26.2 and shows an availability check",
+        "Says the developer decides which updates are significant under the regulations that apply, not every copy or bug-fix release",
+        "Replaces the vague description with one naming the concrete change the parent approves",
+        "Observes AskCenter.shared.responses(for: SignificantAppUpdateTopic.self) and handles approval, denial and the case where no response arrives",
+        "Recommends qualified legal counsel for interpretation and does not give jurisdiction-specific legal advice"
       ]
     }
   ]