@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/avkit/evals/evals.json

@@ -3,52 +3,52 @@
   "evals": [
     {
       "id": 1,
-      "prompt": "Audit this proposed iOS video player setup: it calls AVAudioSession.setCategory(.playback, mode: .moviePlayback) and setActive(true) in app launch, adds no background modes, and expects Picture in Picture and AirPlay to work automatically. Write a concise review with corrected Swift and Info.plist guidance.",
-      "expected_output": "A review that corrects the audio session and background mode setup for an iOS AVKit playback app, recommends deferring setActive until playback begins, and avoids unsupported claims.",
+      "prompt": "Audit our video app setup: we never set an audio session category, we call setActive(true) in application(_:didFinishLaunchingWithOptions:), there is no background mode, and PiP never shows up. Someone suggested we just need a Picture in Picture entitlement. What do we change?",
+      "expected_output": "Adds the Audio, AirPlay, and Picture in Picture background mode (audio in UIBackgroundModes), sets the .playback category with .moviePlayback mode, defers setActive(true) until playback starts, does not claim PiP fails silently in a way that cannot be diagnosed, and states that standard-player PiP needs no entitlement.",
       "files": [],
       "assertions": [
-        "Identifies that Background Modes > Audio, AirPlay, and Picture in Picture must be enabled for background audio, AirPlay streaming, and PiP support.",
-        "Recommends setting AVAudioSession category to .playback with .moviePlayback for movie playback.",
-        "Recommends deferring AVAudioSession.setActive(true) until playback begins rather than activating at app launch.",
-        "Does not claim PiP failure is silent or require a PiP entitlement for standard playback."
+        "Requires the audio background mode (Audio, AirPlay, and Picture in Picture)",
+        "Sets category .playback with mode .moviePlayback",
+        "Moves setActive(true) to the moment playback begins",
+        "Does not claim that PiP failure is silent or undiagnosable",
+        "States that no PiP entitlement is required for standard playback"
       ]
     },
     {
       "id": 2,
-      "prompt": "Implement a custom iOS AVPlayerLayer view that offers a Picture in Picture button. Include the important checks and delegate handling, but keep it as snippets rather than a full app.",
-      "expected_output": "Swift snippets for custom AVPlayerLayer-based PiP that check device support and current PiP possibility, start PiP only from user interaction, configure the delegate, and restore UI correctly.",
+      "prompt": "We render video through our own AVPlayerLayer-based view with custom controls. How do we add Picture-in-Picture, and should we start it automatically when the video loads?",
+      "expected_output": "Creates an AVPictureInPictureController from the AVPlayerLayer or a ContentSource without subclassing anything, checks isPictureInPictureSupported before offering PiP, checks isPictureInPicturePossible before starting, starts only from a user action, and implements the restore delegate method calling completionHandler(true).",
       "files": [],
       "assertions": [
-        "Uses AVPictureInPictureController with an AVPlayerLayer or ContentSource instead of subclassing AVPlayerViewController.",
-        "Checks AVPictureInPictureController.isPictureInPictureSupported() before exposing or configuring PiP UI.",
-        "Checks the controller's isPictureInPicturePossible before calling startPictureInPicture().",
-        "Starts custom Picture in Picture only from an explicit user interaction such as a PiP button action.",
-        "Implements restoreUserInterfaceForPictureInPictureStopWithCompletionHandler and calls its completion handler."
+        "Uses AVPictureInPictureController with the player layer or a ContentSource, without subclassing",
+        "Checks AVPictureInPictureController.isPictureInPictureSupported()",
+        "Checks isPictureInPicturePossible before startPictureInPicture()",
+        "Starts PiP only from explicit user interaction, not automatically on load",
+        "Implements the restore delegate method and calls completionHandler(true)"
       ]
     },
     {
       "id": 3,
-      "prompt": "Review an iOS AVKit implementation that uses playerVC.isSkipForwardEnabled, playerVC.isSkipBackwardEnabled, playerVC.skippingBehavior = .skipItem, playerVC.allowedSubtitleOptionLanguages, playerVC.requiresFullSubtitles, and playerItem.interstitialTimeRanges = [...] to support skip buttons, subtitle restrictions, and ad breaks. Explain what should change for iOS 26.",
-      "expected_output": "A review that flags tvOS-only skipping and subtitle-restriction APIs, recommends iOS-safe seeking or standard controls, and uses stream-defined interstitial ranges or AVPlayerInterstitialEventController instead of direct interstitialTimeRanges assignment as the iOS recipe.",
+      "prompt": "Review this iOS 26 player code: it sets isSkipForwardEnabled and skippingBehavior on AVPlayerViewController, restricts subtitles with allowedSubtitleOptionLanguages, and schedules ad breaks by assigning playerItem.interstitialTimeRanges.",
+      "expected_output": "Flags the skipping APIs and subtitle restrictions as tvOS-only, recommends AVPlayer.seek or the standard controls for seeking on iOS and legible media selection for subtitles, and explains that interstitials come from the stream or from AVPlayerInterstitialEventController rather than direct assignment.",
       "files": [],
       "assertions": [
-        "Flags isSkipForwardEnabled, isSkipBackwardEnabled, and skippingBehavior as tvOS-focused APIs that should not be used for iOS transport controls.",
-        "Flags allowedSubtitleOptionLanguages and requiresFullSubtitles as tvOS-only subtitle controls and recommends AVAsset/AVPlayerItem media selection for iOS defaults.",
-        "Recommends AVPlayer.seek(...) or standard AVPlayerViewController transport controls for iOS seeking behavior.",
-        "States that iOS interstitial ranges must be stream-defined or scheduled with AVPlayerInterstitialEventController.",
-        "Does not present playerItem.interstitialTimeRanges assignment as the primary iOS scheduling solution."
+        "Flags isSkipForwardEnabled and skippingBehavior as tvOS-only",
+        "Flags allowedSubtitleOptionLanguages as tvOS-only",
+        "Recommends AVPlayer.seek or the standard transport controls on iOS",
+        "Explains that interstitials are stream-defined or scheduled with AVPlayerInterstitialEventController",
+        "Rejects assigning interstitialTimeRanges directly"
       ]
     },
     {
       "id": 4,
-      "prompt": "Review this SwiftUI AVKit snippet for iOS: VideoPlayer(player: player) { Button(\"Buy\") { purchase() } } plus player.usesExternalPlaybackWhileExternalScreenIsActive = true. Explain whether the overlay can be interactive and when to set the external playback properties.",
-      "expected_output": "A review that accurately explains VideoPlayer overlay event handling and keeps AirPlay/external-screen guidance narrow.",
+      "prompt": "In SwiftUI we put a Like button in the VideoPlayer overlay closure and it sometimes does not respond. Separately, our AirPlay button disappeared after a teammate set usesExternalPlaybackWhileExternalScreenIsActive to false and allowsExternalPlayback to false. Explain both.",
+      "expected_output": "Explains that the VideoPlayer overlay sits above the video but below the system controls and receives only events the controls do not handle, that allowsExternalPlayback defaults to true and must stay enabled for AirPlay, and that usesExternalPlaybackWhileExternalScreenIsActive only controls automatic switching during external-screen mode.",
       "files": [],
       "assertions": [
-        "States that VideoPlayer's videoOverlay is fully interactive but sits below system playback controls and receives only unhandled events.",
-        "Does not describe VideoPlayer videoOverlay as non-interactive.",
-        "States that AVPlayer.allowsExternalPlayback defaults to true and should usually be left enabled for AirPlay.",
-        "Explains that usesExternalPlaybackWhileExternalScreenIsActive is only for automatic external-screen mode, not a blanket AirPlay requirement."
+        "States that the overlay is interactive but below the system controls and only gets unhandled events",
+        "States that allowsExternalPlayback defaults to true and should stay enabled for AirPlay",
+        "Explains that usesExternalPlaybackWhileExternalScreenIsActive only concerns automatic external-screen playback"
       ]
     }
   ]