@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/authentication/evals/evals.json

@@ -3,45 +3,45 @@
   "evals": [
     {
       "id": 0,
-      "name": "passkey-registration-and-signin",
-      "prompt": "We're adding passwordless sign-in to our iOS app for accounts on example.com. The server can mint WebAuthn challenges, and product wants passkeys first, a physical security key fallback, and inline suggestions in the username field. What should the implementation checklist and core AuthenticationServices flow look like?",
-      "expected_output": "A passkey implementation checklist that covers associated domains, platform passkey registration/assertion, AutoFill-assisted requests, physical security key boundaries, and server-side verification.",
+      "name": "passkey-enrollment-and-login",
+      "prompt": "Our recipe app wants passwordless accounts on iOS 26. Show how to add passkey enrollment and passkey login, including suggestions in the username field, and mention hardware keys for staff accounts.",
+      "expected_output": "Configures Associated Domains with a webcredentials entry and association file, uses ASAuthorizationPlatformPublicKeyCredentialProvider with separate registration and assertion requests built from server challenges, sends results to the server for verification before creating a session, sets textContentType .username and calls performAutoFillAssistedRequests, and uses the security-key provider only as a separate path for physical keys.",
       "files": [],
       "expectations": [
-        "Names the Associated Domains requirement with the `webcredentials:` service for the relying-party domain.",
-        "Uses `ASAuthorizationPlatformPublicKeyCredentialProvider(relyingPartyIdentifier:)` for platform passkeys and distinguishes registration from assertion requests.",
-        "States that registration and assertion challenges come from the relying-party server and that passkey results must be verified server-side before issuing a session.",
-        "Mentions `performAutoFillAssistedRequests()` with a username text field using `.username` for inline passkey suggestions.",
-        "Uses `ASAuthorizationSecurityKeyPublicKeyCredentialProvider` only for physical security key fallback, not as the default platform passkey provider."
+        "Requires a webcredentials entry under Associated Domains",
+        "Uses the platform provider with distinct registration and assertion requests",
+        "Takes challenges from the server and verifies results server-side before issuing a session",
+        "Uses performAutoFillAssistedRequests with a .username field for inline suggestions",
+        "Treats the security-key provider as a separate branch for physical keys only"
       ]
     },
     {
       "id": 1,
-      "name": "sign-in-with-apple-oauth-review",
-      "prompt": "Review this iOS login plan before implementation: Sign in with Apple for new accounts, saved password suggestions for returning users, GitHub OAuth, cached account profile data, local Face ID re-auth before account deletion, and token storage. What are the must-have AuthenticationServices and LocalAuthentication checks?",
-      "expected_output": "A review checklist that covers Sign in with Apple setup, credential handling, existing account flows, OAuth session presentation, biometric re-auth boundaries, and secure token storage.",
+      "name": "apple-and-github-login-review",
+      "prompt": "Review our login code: Sign in with Apple reads credential.email! on every sign-in, we never check whether the Apple ID is still linked, the identity token is decoded in the app to get the user, GitHub login runs inside a WKWebView, the session token is in UserDefaults, and Face ID success unlocks the stored token.",
+      "expected_output": "Flags the missing capability or presentation provider if absent, caches email and name only on first authorization without force unwraps, adds getCredentialState at launch plus credentialRevokedNotification, moves identity token and authorization code validation to the server, switches GitHub to ASWebAuthenticationSession with a real presentation anchor, stores tokens in the Keychain, and treats LAContext as local re-authentication rather than a secret release.",
       "files": [],
       "expectations": [
-        "Requires the Sign in with Apple capability plus `ASAuthorizationControllerPresentationContextProviding` for authorization UI.",
-        "Explains that `email` and `fullName` must be cached during first authorization and handled as optional later.",
-        "Requires Apple ID credential state checks with `getCredentialState(forUserID:completion:)` and handling credential revocation notifications.",
-        "Sends `identityToken` and `authorizationCode` to the server for validation instead of trusting them only on device.",
-        "Uses `ASWebAuthenticationSession` for GitHub OAuth, avoids `WKWebView`, and returns a real active presentation anchor.",
-        "Stores secrets in Keychain and treats `LAContext.evaluatePolicy` as local re-authentication rather than standalone proof for releasing stored secrets."
+        "Notes that email and fullName arrive only on the first authorization and must be cached then",
+        "Adds getCredentialState at launch and observes credentialRevokedNotification",
+        "Validates the identity token and authorization code on the server",
+        "Replaces WKWebView with ASWebAuthenticationSession and a real presentation anchor",
+        "Moves tokens to the Keychain",
+        "States that LAContext success alone must not release stored secrets"
       ]
     },
     {
       "id": 2,
-      "name": "security-boundary-routing",
-      "prompt": "A team asks the authentication skill to design the whole account security layer: Sign in with Apple, passkeys, refresh-token storage, CryptoKit encryption, certificate pinning, biometric unlock, and OWASP MASVS mapping. Give a concise scope review that says what this skill should cover and what belongs in sibling security work.",
-      "expected_output": "A boundary-aware scope review that keeps AuthenticationServices auth flows in scope while routing deep Keychain, CryptoKit, certificate trust, and compliance work to the security skill.",
+      "name": "auth-scope-routing",
+      "prompt": "While adding Sign in with Apple, can you also design our Keychain migration, pick SecAccessControl flags, add certificate pinning, and map us against OWASP MASVS? Also write our WebAuthn relying-party server.",
+      "expected_output": "Keeps the sign-in flow in scope, routes Keychain migration, access-control policy, CryptoKit, Secure Enclave, pinning and MASVS to the swift-security skill, restates the minimum rule that tokens go in the Keychain and evaluatePolicy does not release secrets, and does not claim to own the backend WebAuthn relying-party implementation beyond server challenges and verification.",
       "files": [],
       "expectations": [
-        "Keeps Sign in with Apple, passkeys, Password AutoFill, ASAuthorizationController flows, ASWebAuthenticationSession OAuth, and local biometric re-authentication in the authentication skill scope.",
-        "Routes detailed Keychain architecture, CryptoKit encryption, Secure Enclave, certificate pinning, and OWASP MASVS mapping to the sibling security skill instead of expanding the authentication answer into those domains.",
-        "Still states that auth tokens and secrets should be stored in Keychain, not UserDefaults, files, or Core Data.",
-        "Warns that `LAContext.evaluatePolicy` alone should not release stored secrets and that Keychain access control is the right handoff for protected token/key reads.",
-        "Does not claim that the authentication skill owns backend WebAuthn relying-party implementation beyond requiring server challenges and server verification."
+        "Implements or discusses the Sign in with Apple flow itself",
+        "Routes Keychain architecture, access-control flags, pinning and MASVS to swift-security",
+        "Keeps the rule that tokens belong in the Keychain",
+        "Says evaluatePolicy alone does not release protected secrets",
+        "Does not present itself as the owner of a backend WebAuthn relying-party implementation"
       ]
     }
   ]

package/pipeline/skills/shared/external/authentication/references/keychain-biometric.md

@@ -1,238 +1,193 @@
-# Keychain Token Storage & Biometric Authentication
+# Keychain and Biometrics
 
-Self-contained reference for storing authentication tokens in Keychain and
-protecting them with Keychain-bound biometric authentication (Face ID /
-Touch ID). Covers the patterns most commonly needed alongside Sign in with
-Apple, passkey, and OAuth flows.
+The minimum storage and biometric patterns that sign-in code needs. This is a
+quick reference, not a security architecture: Keychain migration,
+access-control policy design, CryptoKit, Secure Enclave, pinning, sharing
+items across apps, hardening and MASVS/MASTG mapping belong to `swift-security`.
+Back to the [skill overview](../SKILL.md).
 
-This is an authentication-adjacent quick reference, not a full security
-architecture guide. Route Keychain migration, access-control policy design,
-CryptoKit encryption, Secure Enclave keys, certificate pinning, keychain
-sharing, storage-hardening strategy, and OWASP MASVS/MASTG mapping to
-`swift-security`.
+## Where secrets go
 
-## Contents
+Session tokens, refresh tokens, passwords and API keys belong in the Keychain.
+`UserDefaults`, loose files and Core Data are all wrong for them.
 
-- [Storing Tokens in Keychain](#storing-tokens-in-keychain)
-- [Reading Tokens from Keychain](#reading-tokens-from-keychain)
-- [Deleting Tokens from Keychain](#deleting-tokens-from-keychain)
-- [Biometric Authentication with LAContext](#biometric-authentication-with-lacontext)
-- [Biometric-Protected Keychain Items](#biometric-protected-keychain-items)
-- [Keychain Error Handling](#keychain-error-handling)
-- [References](#references)
+## A small token vault
 
-## Storing Tokens in Keychain
+```swift
+import Foundation
+import Security
 
-The Keychain is the ONLY correct place to store tokens, passwords, API keys, or
-secrets. Never store these in UserDefaults, files, or Core Data.
+struct TokenVault {
+    let service: String
 
-```swift
-func saveToKeychain(account: String, data: Data, service: String) throws {
-    let query: [String: Any] = [
-        kSecClass as String: kSecClassGenericPassword,
-        kSecAttrAccount as String: account,
-        kSecAttrService as String: service,
-        kSecValueData as String: data,
-        kSecAttrAccessible as String: kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
-    ]
+    private func query(_ account: String) -> [String: Any] {
+        [kSecClass as String: kSecClassGenericPassword,
+         kSecAttrService as String: service,
+         kSecAttrAccount as String: account]
+    }
 
-    let status = SecItemAdd(query as CFDictionary, nil)
-
-    if status == errSecDuplicateItem {
-        let updateQuery: [String: Any] = [
-            kSecClass as String: kSecClassGenericPassword,
-            kSecAttrAccount as String: account,
-            kSecAttrService as String: service
-        ]
-        let updates: [String: Any] = [kSecValueData as String: data]
-        let updateStatus = SecItemUpdate(updateQuery as CFDictionary, updates as CFDictionary)
-        guard updateStatus == errSecSuccess else {
-            throw KeychainError.updateFailed(updateStatus)
+    func store(_ secret: Data, account: String) throws {
+        var insert = query(account)
+        insert[kSecValueData as String] = secret
+        insert[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
+
+        switch SecItemAdd(insert as CFDictionary, nil) {
+        case errSecSuccess:
+            return
+        case errSecDuplicateItem:
+            let changes = [kSecValueData as String: secret] as CFDictionary
+            let status = SecItemUpdate(query(account) as CFDictionary, changes)
+            if status != errSecSuccess { throw VaultError.updateFailed(status) }
+        case let status:
+            throw VaultError.saveFailed(status)
         }
-    } else if status != errSecSuccess {
-        throw KeychainError.saveFailed(status)
     }
-}
-```
-
-## Reading Tokens from Keychain
 
-```swift
-func readFromKeychain(account: String, service: String) throws -> Data {
-    let query: [String: Any] = [
-        kSecClass as String: kSecClassGenericPassword,
-        kSecAttrAccount as String: account,
-        kSecAttrService as String: service,
-        kSecReturnData as String: true,
-        kSecMatchLimit as String: kSecMatchLimitOne
-    ]
-
-    var result: AnyObject?
-    let status = SecItemCopyMatching(query as CFDictionary, &result)
-
-    guard status == errSecSuccess, let data = result as? Data else {
-        throw KeychainError.readFailed(status)
+    func fetch(account: String) throws -> Data {
+        var lookup = query(account)
+        lookup[kSecReturnData as String] = kCFBooleanTrue
+        lookup[kSecMatchLimit as String] = kSecMatchLimitOne
+        var item: AnyObject?
+        let status = SecItemCopyMatching(lookup as CFDictionary, &item)
+        guard status == errSecSuccess else { throw VaultError.readFailed(status) }
+        guard let secret = item as? Data else { throw VaultError.readFailed(errSecDecode) }
+        return secret
     }
-    return data
-}
-```
-
-## Deleting Tokens from Keychain
 
-```swift
-func deleteFromKeychain(account: String, service: String) throws {
-    let query: [String: Any] = [
-        kSecClass as String: kSecClassGenericPassword,
-        kSecAttrAccount as String: account,
-        kSecAttrService as String: service
-    ]
-
-    let status = SecItemDelete(query as CFDictionary)
-    guard status == errSecSuccess || status == errSecItemNotFound else {
-        throw KeychainError.deleteFailed(status)
+    func remove(account: String) throws {
+        let status = SecItemDelete(query(account) as CFDictionary)
+        // A missing item already counts as removed.
+        if status != errSecSuccess && status != errSecItemNotFound {
+            throw VaultError.deleteFailed(status)
+        }
     }
 }
 ```
 
-Use a `ThisDeviceOnly` accessibility class for sensitive app credentials unless
-the product explicitly requires restore or sharing behavior. Choose broader
-Keychain accessibility, migration, or sharing strategy in `swift-security`, not
-in this authentication skill.
+Every operation goes through the same `query(_:)`, so add, update, read and
+delete always match the same item.
+
+### Accessibility
 
-Do not use `kSecAttrAccessibleAlways` for app credentials; Apple marks it as not
-recommended for application use because items remain accessible regardless of
-lock state.
+Pick a `ThisDeviceOnly` class for sensitive credentials unless backup restore
+or sharing is an explicit requirement; anything broader is a `swift-security`
+decision. Do not use `kSecAttrAccessibleAlways`. Apple lists it as not
+recommended because the item stays readable whatever the lock state.
 
+## Biometrics with `LAContext`
 
-## Biometric Authentication with LAContext
+An `LAContext` prompt is a local gate for the interface. It does not prove that
+a secret may be released. For high-value secrets, attach access control to the
+Keychain item and let `SecItemCopyMatching` run the authentication.
 
-Use `LAContext` from LocalAuthentication for Face ID / Touch ID prompts before
-showing sensitive screens or performing protected actions. This is a local
-interaction gate, not proof that a stored secret is safe to release. For tokens,
-private keys, or high-value credentials, bind access to the Keychain item with
-`SecAccessControl` and let `SecItemCopyMatching` perform the authentication.
+Falling back to the device passcode when biometrics are not available:
 
 ```swift
 import LocalAuthentication
 
-func authenticateWithBiometrics() async throws -> Bool {
+func unlockSettings(reason: String) async throws -> Bool {
     let context = LAContext()
-    var error: NSError?
-
-    guard context.canEvaluatePolicy(
-        .deviceOwnerAuthenticationWithBiometrics, error: &error
-    ) else {
-        // Biometrics not available -- fall back to passcode
-        if context.canEvaluatePolicy(.deviceOwnerAuthentication, error: &error) {
-            return try await context.evaluatePolicy(
-                .deviceOwnerAuthentication,
-                localizedReason: "Authenticate to access your account"
-            )
+    let policy: LAPolicy
+    if context.canEvaluatePolicy(.deviceOwnerAuthenticationWithBiometrics, error: nil) {
+        policy = .deviceOwnerAuthenticationWithBiometrics
+    } else {
+        var passcodeProblem: NSError?
+        guard context.canEvaluatePolicy(.deviceOwnerAuthentication, error: &passcodeProblem) else {
+            throw passcodeProblem ?? LAError(.passcodeNotSet)
         }
-        throw AuthError.biometricsUnavailable
+        policy = .deviceOwnerAuthentication   // biometrics or passcode
     }
-
-    return try await context.evaluatePolicy(
-        .deviceOwnerAuthenticationWithBiometrics,
-        localizedReason: "Authenticate to access your account"
-    )
+    return try await context.evaluatePolicy(policy, localizedReason: reason)
 }
 ```
 
-### Info.plist Requirement
-
-You MUST include `NSFaceIDUsageDescription` in Info.plist:
+### Info.plist
 
-```xml
-<key>NSFaceIDUsageDescription</key>
-<string>Authenticate to access your secure data</string>
-```
-
-Missing this key causes a crash on Face ID devices.
+Add `NSFaceIDUsageDescription` with a sentence explaining why the app uses
+Face ID. Face ID devices crash the app if it is missing.
 
-### LAContext Configuration
+### Tuning the context
 
 ```swift
 let context = LAContext()
-context.localizedFallbackTitle = "Use Passcode"
-context.touchIDAuthenticationAllowableReuseDuration = 30
-let currentState = context.evaluatedPolicyDomainState // Compare to detect enrollment changes
+context.localizedFallbackTitle = "Enter Passcode"
+context.touchIDAuthenticationAllowableReuseDuration = 60   // seconds
+let enrollmentHash = context.domainState.biometry.stateHash   // store it; a different value later means enrollment changed
 ```
 
-## Biometric-Protected Keychain Items
+`evaluatedPolicyDomainState` served the same purpose but is deprecated since
+iOS 18 in favor of `domainState.biometry.stateHash`.
 
-Protect keychain items so they require biometric authentication to read:
+## Items that require biometrics to read
 
 ```swift
-let access = SecAccessControlCreateWithFlags(
-    nil,
-    kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly,
-    .biometryCurrentSet,
-    nil
-)!
-
-let query: [String: Any] = [
-    kSecClass as String: kSecClassGenericPassword,
-    kSecAttrAccount as String: "auth-token",
-    kSecValueData as String: tokenData,
-    kSecAttrAccessControl as String: access
-]
-```
-
-Read the protected item with an authentication context so Keychain, not app
-logic, controls release of the secret:
+func sealBehindBiometrics(_ payload: Data, in item: (service: String, account: String)) throws {
+    guard let control = SecAccessControlCreateWithFlags(
+        nil,
+        kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly,
+        .biometryCurrentSet,
+        nil
+    ) else { throw VaultError.saveFailed(errSecParam) }
+
+    let attributes: [String: Any] = [
+        kSecClass as String: kSecClassGenericPassword,
+        kSecAttrService as String: item.service,
+        kSecAttrAccount as String: item.account,
+        kSecValueData as String: payload,
+        kSecAttrAccessControl as String: control,
+    ]
+    let added = SecItemAdd(attributes as CFDictionary, nil)
+    if added != errSecSuccess { throw VaultError.saveFailed(added) }
+}
 
-```swift
-let context = LAContext()
-context.localizedReason = "Authenticate to access your account"
-
-let query: [String: Any] = [
-    kSecClass as String: kSecClassGenericPassword,
-    kSecAttrAccount as String: "auth-token",
-    kSecReturnData as String: true,
-    kSecMatchLimit as String: kSecMatchLimitOne,
-    kSecUseAuthenticationContext as String: context
-]
-
-var result: AnyObject?
-let status = SecItemCopyMatching(query as CFDictionary, &result)
-guard status == errSecSuccess, let tokenData = result as? Data else {
-    throw KeychainError.readFailed(status)
+func openBiometricItem(_ item: (service: String, account: String)) throws -> Data {  // prompts via Keychain
+    let context = LAContext()
+    context.localizedReason = "Unlock your saved card"
+    let lookup: [String: Any] = [
+        kSecClass as String: kSecClassGenericPassword,
+        kSecAttrService as String: item.service,
+        kSecAttrAccount as String: item.account,
+        kSecReturnData as String: true,
+        kSecUseAuthenticationContext as String: context,
+    ]
+    var item: AnyObject?
+    let status = SecItemCopyMatching(lookup as CFDictionary, &item)
+    guard status == errSecSuccess, let data = item as? Data else { throw VaultError.readFailed(status) }
+    return data
 }
 ```
 
-Use `.biometryCurrentSet` when an auth token must be invalidated after biometric
-enrollment changes. Route broader SecAccessControl flag selection and policy
-tradeoffs to `swift-security`.
-
+Here the Keychain decides whether the data is released. `.biometryCurrentSet`
+makes the item unreadable once enrolled fingers or faces change; choosing
+between that and other flags is a `swift-security` question.
 
-## Keychain Error Handling
+## Errors
 
 ```swift
-enum KeychainError: Error {
+enum VaultError: Error, CustomStringConvertible {
     case saveFailed(OSStatus)
     case updateFailed(OSStatus)
     case readFailed(OSStatus)
     case deleteFailed(OSStatus)
 
-    var localizedDescription: String {
+    var description: String {
+        let status: OSStatus
         switch self {
-        case .saveFailed(let status),
-             .updateFailed(let status),
-             .readFailed(let status),
-             .deleteFailed(let status):
-            return SecCopyErrorMessageString(status, nil) as String? ?? "Unknown error"
+        case .saveFailed(let s), .updateFailed(let s), .readFailed(let s), .deleteFailed(let s):
+            status = s
         }
+        let message = SecCopyErrorMessageString(status, nil).map { $0 as String }
+        return message ?? "OSStatus \(status)"
     }
 }
 ```
 
-## References
+## Apple documentation
 
-- [Restricting keychain item accessibility](https://sosumi.ai/documentation/security/restricting-keychain-item-accessibility)
-- [Accessing Keychain Items with Face ID or Touch ID](https://sosumi.ai/documentation/localauthentication/accessing-keychain-items-with-face-id-or-touch-id)
-- [LAContext](https://sosumi.ai/documentation/localauthentication/lacontext)
-- [NSFaceIDUsageDescription](https://sosumi.ai/documentation/bundleresources/information-property-list/nsfaceidusagedescription)
-- [kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly](https://sosumi.ai/documentation/security/ksecattraccessiblewhenpasscodesetthisdeviceonly)
-- [SecAccessControlCreateFlags.userPresence](https://sosumi.ai/documentation/security/secaccesscontrolcreateflags/userpresence)
-- [SecAccessControlCreateFlags.biometryCurrentSet](https://sosumi.ai/documentation/security/secaccesscontrolcreateflags/biometrycurrentset)
+- [LAContext](https://developer.apple.com/documentation/localauthentication/lacontext)
+- [biometryCurrentSet](https://developer.apple.com/documentation/security/secaccesscontrolcreateflags/biometrycurrentset)
+  and [userPresence](https://developer.apple.com/documentation/security/secaccesscontrolcreateflags/userpresence) access-control flags
+- [kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly](https://developer.apple.com/documentation/security/ksecattraccessiblewhenpasscodesetthisdeviceonly)
+- Security framework article on limiting when keychain items are accessible
+- LocalAuthentication article on reading keychain items behind Face ID or Touch ID
+- The `NSFaceIDUsageDescription` Info.plist key reference