@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.1

package/pipeline/skills/shared/external/cryptokit/evals/evals.json

@@ -4,40 +4,42 @@
     {
       "id": 1,
       "name": "hpke-recipient-encryption",
-      "prompt": "I need to encrypt a payload in an iOS app for a recipient public key. Please outline the CryptoKit approach and include the Swift details that prevent the common ECDH/HKDF/AES-GCM mistakes.",
-      "expected_output": "Recommends HPKE on iOS 17+, shows stateful Sender/Recipient usage, sends encapsulatedKey with ciphertext, uses AAD correctly, and avoids a manual ECDH+HKDF+AEAD protocol unless HPKE is unavailable.",
+      "prompt": "Our iOS 17+ app needs to encrypt a series of diagnostic bundles to our server's Curve25519 public key. A teammate wrote their own ECDH plus HKDF plus AES-GCM scheme and declares the sender with let. The bundle ID should be tamper-proof but readable. What should we do instead?",
+      "expected_output": "Recommends HPKE over the hand-rolled scheme on iOS 17+, declares HPKE.Sender and HPKE.Recipient as var because they are stateful, sends encapsulatedKey with the ciphertext, protects the bundle ID as authenticated data, and opens messages in the order they were sealed.",
       "files": [],
       "assertions": [
-        "Recommends HPKE instead of hand-rolled ECDH+HKDF+AEAD for recipient public-key encryption when iOS 17+ is available.",
-        "Shows or states that HPKE.Sender and HPKE.Recipient are stateful and must be var when sealing or opening.",
-        "States that sender.encapsulatedKey must be transmitted alongside the ciphertext.",
-        "Mentions AAD/metadata authentication and same-order open semantics for multi-message HPKE."
+        "Prefers HPKE to a custom ECDH + HKDF + AEAD construction on iOS 17+",
+        "Declares HPKE.Sender and HPKE.Recipient with var",
+        "Transmits encapsulatedKey together with the ciphertext",
+        "Uses authenticated data for the readable bundle ID",
+        "Opens messages in the same order they were sealed"
       ]
     },
     {
       "id": 2,
       "name": "post-quantum-workflow",
-      "prompt": "We are targeting iOS 26 and want quantum-secure CryptoKit for a document sharing flow. What APIs should we use for key exchange and signatures, and what availability or Secure Enclave caveats matter?",
-      "expected_output": "Covers X-Wing HPKE, ML-KEM, ML-DSA, iOS 26 availability, Secure Enclave ML-KEM/ML-DSA variants on supported hardware, and key/signature size tradeoffs.",
+      "prompt": "Security wants us to be quantum-ready. We support iOS 18 and later. Which CryptoKit APIs apply for key exchange, encryption to a recipient and signing, and can we keep all of it in the Secure Enclave?",
+      "expected_output": "Uses the X-Wing HPKE ciphersuite XWingMLKEM768X25519_SHA256_AES_GCM_256, MLKEM768 or MLKEM1024 for key encapsulation, MLDSA65 or MLDSA87 for signatures, guards all of it with iOS 26 availability checks, and explains that classical Secure Enclave support is P256 only while iOS 26 adds Secure Enclave ML-KEM and ML-DSA on supported hardware.",
       "files": [],
       "assertions": [
-        "Names XWingMLKEM768X25519 or the .XWingMLKEM768X25519_SHA256_AES_GCM_256 HPKE ciphersuite for hybrid quantum-secure public-key encryption.",
-        "Names MLKEM768 or MLKEM1024 for key encapsulation and MLDSA65 or MLDSA87 for signatures.",
-        "States that SHA-3/post-quantum CryptoKit APIs require iOS 26+ availability checks unless the deployment target is iOS 26+.",
-        "Correctly distinguishes classical Secure Enclave P256 support from iOS 26 SecureEnclave.MLKEM and SecureEnclave.MLDSA variants."
+        "Names the X-Wing HPKE ciphersuite for hybrid recipient encryption",
+        "Names MLKEM768 and MLKEM1024 for key encapsulation",
+        "Names MLDSA65 and MLDSA87 for signatures",
+        "Guards the post-quantum APIs with iOS 26 availability checks for an iOS 18 target",
+        "Separates classical Secure Enclave P256 support from iOS 26 Secure Enclave ML-KEM and ML-DSA"
       ]
     },
     {
       "id": 3,
       "name": "storage-boundary",
-      "prompt": "Please review this design: generate a CryptoKit SymmetricKey, save its bytes in UserDefaults, then use AES-GCM for local file encryption. Should the CryptoKit skill own the whole fix?",
-      "expected_output": "Flags UserDefaults key storage as wrong, keeps CryptoKit focused on AES-GCM/nonce/AAD/key material handling, and routes durable secret storage/access-control policy to Keychain or the swift-security domain.",
+      "prompt": "We encrypt journal entries with AES-GCM and keep the SymmetricKey bytes in UserDefaults so the app can reload them. We also want Face ID to guard the key and a policy for rotating it when the user signs out. Review this and tell us what belongs in CryptoKit.",
+      "expected_output": "Rejects storing key bytes in UserDefaults or files, keeps CryptoKit focused on AES-GCM, nonces, AAD and key material handling, routes Keychain queries, biometric access control and credential lifecycle to Keychain and swift-security guidance, and mentions export compliance.",
       "files": [],
       "assertions": [
-        "Rejects storing symmetric key material in UserDefaults or normal files.",
-        "Keeps CryptoKit guidance focused on AES-GCM, nonce reuse avoidance, AAD, and key material handling.",
-        "Routes durable key storage, Keychain queries, biometric/passcode access control, and broader credential lifecycle to Keychain/swift-security guidance.",
-        "Mentions encryption export compliance or App Store encryption declaration when app-level encryption is relevant."
+        "Rejects UserDefaults and plain files for key storage",
+        "Keeps the CryptoKit answer to AES-GCM, random nonces, AAD and key material",
+        "Routes Keychain queries, access control and credential lifecycle to the Keychain or swift-security guidance",
+        "Mentions export compliance (ITSAppUsesNonExemptEncryption)"
       ]
     }
   ]