@hasna/recordings 0.2.13 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile.package +4 -5
- package/README.md +113 -8
- package/bun.lock +2 -2
- package/dist/__tests__/helpers/installer-guard-execution.d.ts +22 -0
- package/dist/__tests__/helpers/installer-guard-execution.d.ts.map +1 -0
- package/dist/__tests__/helpers/installer-preflight.d.ts +22 -0
- package/dist/__tests__/helpers/installer-preflight.d.ts.map +1 -0
- package/dist/__tests__/helpers/native-fs-guard.d.ts.map +1 -1
- package/dist/__tests__/helpers/source-assertions.d.ts +169 -0
- package/dist/__tests__/helpers/source-assertions.d.ts.map +1 -0
- package/dist/__tests__/preload.d.ts +1 -4
- package/dist/__tests__/preload.d.ts.map +1 -1
- package/dist/cli/desktop-snapshot.d.ts +20 -0
- package/dist/cli/desktop-snapshot.d.ts.map +1 -0
- package/dist/cli/index.js +2078 -277
- package/dist/cli/macos-permissions.d.ts +72 -0
- package/dist/cli/macos-permissions.d.ts.map +1 -1
- package/dist/cli/macos-shortcut.d.ts +139 -0
- package/dist/cli/macos-shortcut.d.ts.map +1 -0
- package/dist/cli/trigger-probe.d.ts +172 -0
- package/dist/cli/trigger-probe.d.ts.map +1 -0
- package/dist/db/database.d.ts +9 -0
- package/dist/db/database.d.ts.map +1 -1
- package/dist/http/client.d.ts +10 -7
- package/dist/http/client.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +634 -49
- package/dist/lib/capture-probe.d.ts +216 -0
- package/dist/lib/capture-probe.d.ts.map +1 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/macos-bundle.d.ts +50 -0
- package/dist/lib/macos-bundle.d.ts.map +1 -0
- package/dist/lib/persistence-probe.d.ts +208 -0
- package/dist/lib/persistence-probe.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts.map +1 -1
- package/dist/lib/retired-deployment-modes.d.ts +24 -0
- package/dist/lib/retired-deployment-modes.d.ts.map +1 -0
- package/dist/lib/transcriber.d.ts +60 -0
- package/dist/lib/transcriber.d.ts.map +1 -1
- package/dist/mcp/index.js +140 -56
- package/dist/server/cloud-config.d.ts +25 -3
- package/dist/server/cloud-config.d.ts.map +1 -1
- package/dist/server/cloud-readiness.d.ts.map +1 -1
- package/dist/server/cloud.d.ts +8 -7
- package/dist/server/cloud.d.ts.map +1 -1
- package/dist/server/index.js +187 -79
- package/dist/server/migrate-command.d.ts.map +1 -1
- package/dist/server/serve.d.ts.map +1 -1
- package/dist/storage.d.ts +2 -2
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +108 -49
- package/dist/store.d.ts +1 -1
- package/dist/store.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/package.json +25 -6
- package/scripts/build_native_fs_guard.sh +15 -2
- package/scripts/ci-linux-suite.ts +403 -0
- package/scripts/ci-native-build.ts +402 -0
- package/scripts/enforce_identity_migration.sh +110 -0
- package/scripts/generate-sdk.ts +1 -1
- package/scripts/install_macos_app.sh +138 -24
- package/scripts/macos_artifact.ts +205 -33
- package/scripts/native/realtime-settle-probe/README.md +39 -0
- package/scripts/native/realtime-settle-probe/main.swift +135 -0
- package/scripts/policy/local-only-approved-targets.txt +21 -0
- package/scripts/read_local_only_targets.sh +203 -0
- package/scripts/scan-artifact.ts +40 -0
- package/scripts/set-version.ts +145 -0
- package/scripts/vacuity-manifest-gen.ts +96 -0
- package/scripts/vacuity-manifests/enhancement-screen-mirror.tsv +44 -0
- package/scripts/vacuity-manifests/install-chain.tsv +13 -0
- package/scripts/vacuity-manifests/must-survive-controls.tsv +10 -0
- package/scripts/vacuity-manifests/reorder.tsv +4 -0
- package/scripts/vacuity-manifests/source-side.tsv +15 -0
- package/scripts/vacuity-manifests/variable-operands.tsv +16 -0
- package/scripts/vacuity-manifests/version-sites.tsv +61 -0
- package/scripts/vacuity-mutation-battery.ts +266 -0
- package/src/native/Recordings/App/MenuBarStatusView.swift +14 -7
- package/src/native/Recordings/App/RuntimeSmoke.swift +5 -1
- package/src/native/Recordings/App/SidebarView.swift +1 -1
- package/src/native/Recordings/Package.swift +24 -5
- package/src/native/Recordings/RecordingsLib/EnhancementScreen.swift +91 -0
- package/src/native/Recordings/RecordingsLib/FnKeyMonitor.swift +27 -4
- package/src/native/Recordings/RecordingsLib/Info.plist +2 -2
- package/src/native/Recordings/RecordingsLib/MenuBarPresentation.swift +56 -3
- package/src/native/Recordings/RecordingsLib/NativePCMRecorder.swift +13 -0
- package/src/native/Recordings/RecordingsLib/PasteDeliveryVerification.swift +460 -0
- package/src/native/Recordings/RecordingsLib/RealtimeTranscriptionClient.swift +41 -7
- package/src/native/Recordings/RecordingsLib/RecordingAttemptAlert.swift +36 -0
- package/src/native/Recordings/RecordingsLib/RecordingEngine.swift +1071 -75
- package/src/native/Recordings/RecordingsLib/SettingsView.swift +47 -3
- package/src/native/Recordings/RecordingsLib/VoiceShortcuts.swift +11 -6
- package/src/native/Recordings/RecordingsTests/CLIRunnerTests.swift +3 -3
- package/src/native/Recordings/RecordingsTests/EnhancementScreenTests.swift +191 -0
- package/src/native/Recordings/RecordingsTests/MenuBarPresentationTests.swift +144 -4
- package/src/native/Recordings/RecordingsTests/PasteDeliveryVerificationTests.swift +463 -0
- package/src/native/Recordings/RecordingsTests/PasteTargetTests.swift +33 -20
- package/src/native/Recordings/RecordingsTests/ProjectStoreTests.swift +1 -1
- package/src/native/Recordings/RecordingsTests/RealtimeSettleBenchmark.swift +200 -0
- package/src/native/Recordings/RecordingsTests/RealtimeSettleBudgetTests.swift +80 -0
- package/src/native/Recordings/RecordingsTests/RealtimeTranscriptionTests.swift +22 -4
- package/src/native/Recordings/RecordingsTests/RecordingEngineDeliveryTests.swift +12 -6
- package/src/native/Recordings/RecordingsTests/RecordingEngineHomeIsolationTests.swift +67 -0
- package/src/native/Recordings/RecordingsTests/RecordingStartGateTests.swift +12 -3
- package/src/native/Recordings/RecordingsTests/RecordingStartTimingTests.swift +207 -11
- package/src/native/Recordings/RecordingsTests/SpeechIntentTests.swift +17 -6
- package/src/native/Recordings/RecordingsTests/TestHomeDirectory.swift +23 -0
- package/src/native/Recordings/Updater/Broker/ApplicationProcessQuiescence.swift +7 -1
- package/src/native/Recordings/Updater/Broker/AtomicActivation.swift +10 -10
- package/src/native/Recordings/Updater/Broker/BrokerMain.swift +5 -1
- package/src/native/Recordings/Updater/Broker/HostOSProductVersion.swift +1 -0
- package/src/native/Recordings/Updater/Broker/PeerIdentity.swift +59 -3
- package/src/native/Recordings/Updater/Protocol/UpdateProtocol.swift +21 -2
- package/src/native/Recordings/Updater/ProtocolTests/ExpectationIntegrityTests.swift +60 -0
- package/src/native/Recordings/Updater/ProtocolTests/UpdateXPCInterfaceTests.swift +113 -0
- package/src/native/Recordings/Updater/VerifierLauncher/RecordingsVerifierLauncher.c +106 -5
- package/src/native/Recordings/build.sh +75 -9
- package/scripts/native/prebuilds/darwin-universal/recordings_fs_guard.node +0 -0
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
import AppKit
|
|
2
|
+
@preconcurrency import ApplicationServices
|
|
3
|
+
import CoreGraphics
|
|
4
|
+
import Foundation
|
|
5
|
+
|
|
6
|
+
// MARK: - Why a paste cannot be proven
|
|
7
|
+
|
|
8
|
+
/// `CGEvent.post` returns `Void`. macOS never tells the sender whether a synthetic keystroke
|
|
9
|
+
/// reached a consumer, so "we posted Cmd-V" is not evidence that text landed anywhere. The
|
|
10
|
+
/// only evidence available to this process is reading the target app's focused field back
|
|
11
|
+
/// over the Accessibility API and observing that it gained the pasted text.
|
|
12
|
+
///
|
|
13
|
+
/// Every value in this file exists to keep three separable facts separable:
|
|
14
|
+
/// 1. the pasteboard write succeeded and we still own the payload,
|
|
15
|
+
/// 2. the key events were constructed and posted,
|
|
16
|
+
/// 3. the text was observed to land.
|
|
17
|
+
/// Only (3) is delivery. (1) and (2) are preconditions and are reported as such.
|
|
18
|
+
|
|
19
|
+
/// Why one Accessibility read of the focused field produced no comparable text.
|
|
20
|
+
enum FocusedTextReadFailure: String, Equatable, Sendable {
|
|
21
|
+
/// The target app exposes no focused element (AX disabled, no key window, app busy).
|
|
22
|
+
case elementUnavailable = "focused_element_unavailable"
|
|
23
|
+
/// The focused element has no readable `kAXValue` string. Normal for Chrome/Electron web
|
|
24
|
+
/// inputs that publish no value, terminals, and canvas-drawn editors.
|
|
25
|
+
case valueUnreadable = "focused_value_unreadable"
|
|
26
|
+
/// The field holds more text than we are willing to copy and compare on the paste path.
|
|
27
|
+
case valueTooLarge = "focused_value_too_large"
|
|
28
|
+
/// Focus moved to a different element between the baseline and the read-back, so the two
|
|
29
|
+
/// reads describe different fields and cannot be compared.
|
|
30
|
+
case elementChanged = "focused_element_changed"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// Why a paste could not be *confirmed*. Every case means "we do not know that the text
|
|
34
|
+
/// landed" — never "it landed".
|
|
35
|
+
enum PasteDeliveryUnverifiedReason: Equatable, Sendable {
|
|
36
|
+
/// No read-back was wired up (test seams, and the default so a caller that forgets to
|
|
37
|
+
/// verify cannot accidentally inherit a success).
|
|
38
|
+
case readBackNotAttempted
|
|
39
|
+
case emptyPayload
|
|
40
|
+
case baselineUnreadable(FocusedTextReadFailure)
|
|
41
|
+
case readBackUnreadable(FocusedTextReadFailure)
|
|
42
|
+
/// The field changed but not in a way that accounts for our text: the app may have
|
|
43
|
+
/// transformed, truncated, or rejected it, or the change may be unrelated typing.
|
|
44
|
+
case changedWithoutMatch
|
|
45
|
+
|
|
46
|
+
var logToken: String {
|
|
47
|
+
switch self {
|
|
48
|
+
case .readBackNotAttempted: "read_back_not_attempted"
|
|
49
|
+
case .emptyPayload: "empty_payload"
|
|
50
|
+
case .baselineUnreadable(let failure): "baseline_unreadable:\(failure.rawValue)"
|
|
51
|
+
case .readBackUnreadable(let failure): "read_back_unreadable:\(failure.rawValue)"
|
|
52
|
+
case .changedWithoutMatch: "changed_without_match"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// What the post-paste read of the focused field actually showed.
|
|
58
|
+
enum PasteDeliveryEvidence: Equatable, Sendable {
|
|
59
|
+
/// The focused field gained an occurrence of the pasted text. This is the only kind of
|
|
60
|
+
/// evidence that justifies telling the user the paste worked.
|
|
61
|
+
case confirmedByFocusedValue
|
|
62
|
+
/// The field's selection reads back as exactly the pasted text and did not before the
|
|
63
|
+
/// paste — the shape a paste over a selection leaves in some editors.
|
|
64
|
+
case confirmedBySelectedText
|
|
65
|
+
/// The field was readable before and after and did not change: the keystroke did not
|
|
66
|
+
/// reach it. A paste that landed somewhere else is not a paste that landed here.
|
|
67
|
+
case notObservedFocusedValueUnchanged
|
|
68
|
+
case unverified(PasteDeliveryUnverifiedReason)
|
|
69
|
+
|
|
70
|
+
var isConfirmed: Bool {
|
|
71
|
+
switch self {
|
|
72
|
+
case .confirmedByFocusedValue, .confirmedBySelectedText: true
|
|
73
|
+
case .notObservedFocusedValueUnchanged, .unverified: false
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
var logToken: String {
|
|
78
|
+
switch self {
|
|
79
|
+
case .confirmedByFocusedValue: "confirmed_focused_value"
|
|
80
|
+
case .confirmedBySelectedText: "confirmed_selected_text"
|
|
81
|
+
case .notObservedFocusedValueUnchanged: "not_observed_focused_value_unchanged"
|
|
82
|
+
case .unverified(let reason): "unverified:\(reason.logToken)"
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// MARK: - Secure input
|
|
88
|
+
|
|
89
|
+
/// The process holding secure event input, when one does. `bundleIdentifier` is best effort:
|
|
90
|
+
/// the pid may belong to a process we cannot resolve or one that has already exited.
|
|
91
|
+
struct SecureInputHolder: Equatable, Sendable {
|
|
92
|
+
let pid: pid_t?
|
|
93
|
+
let bundleIdentifier: String?
|
|
94
|
+
|
|
95
|
+
var logToken: String {
|
|
96
|
+
"pid=\(pid.map(String.init) ?? "unknown"),app=\(bundleIdentifier ?? "unknown")"
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Secure event input is a window-server mode a password field turns on. While it is on,
|
|
101
|
+
/// every synthetic key event is dropped for every consumer — a posted Cmd-V cannot paste,
|
|
102
|
+
/// and nothing in the posting API says so.
|
|
103
|
+
enum SecureInputState: Equatable, Sendable {
|
|
104
|
+
case inactive
|
|
105
|
+
case active(SecureInputHolder)
|
|
106
|
+
/// The window session could not be interrogated (no GUI session at all, or the session
|
|
107
|
+
/// dictionary did not carry the keys we know how to read). Reported, never assumed off.
|
|
108
|
+
case unknown
|
|
109
|
+
|
|
110
|
+
var isActive: Bool {
|
|
111
|
+
switch self {
|
|
112
|
+
case .active: true
|
|
113
|
+
case .inactive, .unknown: false
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
var logToken: String {
|
|
118
|
+
switch self {
|
|
119
|
+
case .inactive: "inactive"
|
|
120
|
+
case .active(let holder): "active(\(holder.logToken))"
|
|
121
|
+
case .unknown: "unknown"
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
enum SecureInputProbe {
|
|
127
|
+
/// Window-session dictionary keys. CoreGraphics publishes these as C string macros that
|
|
128
|
+
/// Swift does not import, so the key strings are named here once rather than spelled at
|
|
129
|
+
/// the call site. `secureInputPIDKey` is only present while some process holds secure
|
|
130
|
+
/// input, which is why its absence is read together with a session marker key: absent
|
|
131
|
+
/// key plus a recognisable session means "off", absent key plus no session means
|
|
132
|
+
/// "unknown".
|
|
133
|
+
static let secureInputPIDKey = "kCGSSessionSecureInputPID"
|
|
134
|
+
static let onConsoleKey = "kCGSSessionOnConsoleKey"
|
|
135
|
+
static let userIDKey = "kCGSSessionUserIDKey"
|
|
136
|
+
|
|
137
|
+
@MainActor
|
|
138
|
+
static func current() -> SecureInputState {
|
|
139
|
+
guard let rawSession = CGSessionCopyCurrentDictionary() else {
|
|
140
|
+
return state(sessionAvailable: false, sessionMarkerPresent: false, secureInputPID: nil)
|
|
141
|
+
}
|
|
142
|
+
let session = rawSession as NSDictionary
|
|
143
|
+
return state(
|
|
144
|
+
sessionAvailable: true,
|
|
145
|
+
sessionMarkerPresent: session[onConsoleKey] != nil || session[userIDKey] != nil,
|
|
146
|
+
secureInputPID: (session[secureInputPIDKey] as? NSNumber)?.intValue,
|
|
147
|
+
resolveBundleIdentifier: { NSRunningApplication(processIdentifier: $0)?.bundleIdentifier }
|
|
148
|
+
)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/// Pure decision table, so the three outcomes can be tested without a window session.
|
|
152
|
+
static func state(
|
|
153
|
+
sessionAvailable: Bool,
|
|
154
|
+
sessionMarkerPresent: Bool,
|
|
155
|
+
secureInputPID: Int?,
|
|
156
|
+
resolveBundleIdentifier: (pid_t) -> String? = { _ in nil }
|
|
157
|
+
) -> SecureInputState {
|
|
158
|
+
guard sessionAvailable, sessionMarkerPresent else { return .unknown }
|
|
159
|
+
guard let secureInputPID, secureInputPID > 0 else { return .inactive }
|
|
160
|
+
let pid = pid_t(secureInputPID)
|
|
161
|
+
return .active(SecureInputHolder(pid: pid, bundleIdentifier: resolveBundleIdentifier(pid)))
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// MARK: - Reading the focused field
|
|
166
|
+
|
|
167
|
+
struct FocusedTextSnapshot: Equatable, Sendable {
|
|
168
|
+
let value: String
|
|
169
|
+
let selectedText: String?
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
enum FocusedTextRead: Equatable, Sendable {
|
|
173
|
+
case read(FocusedTextSnapshot)
|
|
174
|
+
case unreadable(FocusedTextReadFailure)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/// Holds the focused element captured before a paste so the read-back compares the same
|
|
178
|
+
/// element rather than whatever happens to be focused afterwards. AX calls are Mach IPC and
|
|
179
|
+
/// thread safe; the stored elements are never mutated.
|
|
180
|
+
///
|
|
181
|
+
/// CAPABILITY DISCLOSURE — read this before extending anything in here.
|
|
182
|
+
///
|
|
183
|
+
/// This type is the reason a dictation app can see text the user did not dictate. It reads the
|
|
184
|
+
/// **full value** of the focused field in the target application, plus its current selection,
|
|
185
|
+
/// twice around each paste. That is a meaningful widening of what the app can observe and it is
|
|
186
|
+
/// deliberate: `CGEvent.post` returns `Void`, so nothing else can distinguish a paste that
|
|
187
|
+
/// landed from one the window server discarded, and the alternative is the app claiming a paste
|
|
188
|
+
/// it never proved.
|
|
189
|
+
///
|
|
190
|
+
/// The constraints that make it acceptable are load-bearing, not incidental:
|
|
191
|
+
///
|
|
192
|
+
/// - the read-back text is **never logged and never persisted** — only the verdict is;
|
|
193
|
+
/// - values above `maximumComparableCharacterCount` are reported unverifiable instead of copied;
|
|
194
|
+
/// - no new permission is requested: this rides the Accessibility grant the keystroke already
|
|
195
|
+
/// needs, which is precisely why the capability has to be documented rather than inferred.
|
|
196
|
+
///
|
|
197
|
+
/// A change that logs, stores, transmits or forwards a `FocusedTextSnapshot` value breaks that
|
|
198
|
+
/// contract. `README.md` ("What the app reads to confirm a paste") states this to users; keep
|
|
199
|
+
/// the two in step.
|
|
200
|
+
final class FocusedTextProbe: @unchecked Sendable {
|
|
201
|
+
/// Cap on the field text copied into this process for comparison. A large document is
|
|
202
|
+
/// reported unverifiable rather than copied and scanned on the paste path.
|
|
203
|
+
static let maximumComparableCharacterCount = 20_000
|
|
204
|
+
/// Tighter than `AccessibilitySelectionToken.captureMessagingTimeout`: this read happens
|
|
205
|
+
/// twice around the keystroke, so a slow target app must not delay the paste or the
|
|
206
|
+
/// verdict. A timeout surfaces as `elementUnavailable`/`valueUnreadable`, never success.
|
|
207
|
+
static let messagingTimeout: Float = 0.12
|
|
208
|
+
|
|
209
|
+
private let applicationElement: AXUIElement
|
|
210
|
+
private let focusedElement: AXUIElement?
|
|
211
|
+
let baseline: FocusedTextRead
|
|
212
|
+
|
|
213
|
+
private init(
|
|
214
|
+
applicationElement: AXUIElement,
|
|
215
|
+
focusedElement: AXUIElement?,
|
|
216
|
+
baseline: FocusedTextRead
|
|
217
|
+
) {
|
|
218
|
+
self.applicationElement = applicationElement
|
|
219
|
+
self.focusedElement = focusedElement
|
|
220
|
+
self.baseline = baseline
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
static func capture(pid: pid_t) -> FocusedTextProbe {
|
|
224
|
+
let application = AXUIElementCreateApplication(pid)
|
|
225
|
+
AXUIElementSetMessagingTimeout(application, messagingTimeout)
|
|
226
|
+
guard let focused = resolveFocusedElement(of: application) else {
|
|
227
|
+
return FocusedTextProbe(
|
|
228
|
+
applicationElement: application,
|
|
229
|
+
focusedElement: nil,
|
|
230
|
+
baseline: .unreadable(.elementUnavailable)
|
|
231
|
+
)
|
|
232
|
+
}
|
|
233
|
+
AXUIElementSetMessagingTimeout(focused, messagingTimeout)
|
|
234
|
+
return FocusedTextProbe(
|
|
235
|
+
applicationElement: application,
|
|
236
|
+
focusedElement: focused,
|
|
237
|
+
baseline: read(focused)
|
|
238
|
+
)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// Re-reads the element the baseline came from, refusing to compare across a focus move.
|
|
242
|
+
func readBack() -> FocusedTextRead {
|
|
243
|
+
guard let focusedElement else { return .unreadable(.elementUnavailable) }
|
|
244
|
+
guard let current = Self.resolveFocusedElement(of: applicationElement) else {
|
|
245
|
+
return .unreadable(.elementUnavailable)
|
|
246
|
+
}
|
|
247
|
+
guard CFEqual(current, focusedElement) else { return .unreadable(.elementChanged) }
|
|
248
|
+
return Self.read(focusedElement)
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
private static func resolveFocusedElement(of application: AXUIElement) -> AXUIElement? {
|
|
252
|
+
var focusedElementRef: CFTypeRef?
|
|
253
|
+
guard AXUIElementCopyAttributeValue(
|
|
254
|
+
application,
|
|
255
|
+
kAXFocusedUIElementAttribute as CFString,
|
|
256
|
+
&focusedElementRef
|
|
257
|
+
) == .success,
|
|
258
|
+
let focusedElementRef,
|
|
259
|
+
CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() else { return nil }
|
|
260
|
+
return (focusedElementRef as! AXUIElement)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
private static func read(_ element: AXUIElement) -> FocusedTextRead {
|
|
264
|
+
guard let value = stringAttribute(kAXValueAttribute as CFString, on: element) else {
|
|
265
|
+
return .unreadable(.valueUnreadable)
|
|
266
|
+
}
|
|
267
|
+
guard value.count <= maximumComparableCharacterCount else {
|
|
268
|
+
return .unreadable(.valueTooLarge)
|
|
269
|
+
}
|
|
270
|
+
let selectedText = stringAttribute(kAXSelectedTextAttribute as CFString, on: element)
|
|
271
|
+
return .read(FocusedTextSnapshot(
|
|
272
|
+
value: value,
|
|
273
|
+
selectedText: selectedText.flatMap {
|
|
274
|
+
$0.count <= maximumComparableCharacterCount ? $0 : nil
|
|
275
|
+
}
|
|
276
|
+
))
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private static func stringAttribute(_ attribute: CFString, on element: AXUIElement) -> String? {
|
|
280
|
+
var valueRef: CFTypeRef?
|
|
281
|
+
guard AXUIElementCopyAttributeValue(element, attribute, &valueRef) == .success else {
|
|
282
|
+
return nil
|
|
283
|
+
}
|
|
284
|
+
return valueRef as? String
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// MARK: - Verdict
|
|
289
|
+
|
|
290
|
+
enum PasteDeliveryVerifier {
|
|
291
|
+
/// Decides what two reads of the focused field prove. Ordered so that the positive
|
|
292
|
+
/// verdicts require an observed *gain* of the pasted text: text that was already in the
|
|
293
|
+
/// field before the paste can never be counted as this paste's delivery.
|
|
294
|
+
///
|
|
295
|
+
/// Known false negative, in the safe direction and deliberately not chased: pasting text
|
|
296
|
+
/// identical to the selection it replaces (baseline `hello WORLD`, selection `WORLD`,
|
|
297
|
+
/// transcript `WORLD`) leaves both the value and the occurrence count unchanged, so a
|
|
298
|
+
/// successful delivery classifies as `.notObservedFocusedValueUnchanged`. Distinguishing it
|
|
299
|
+
/// would mean treating an unchanged field as a possible success, which is the direction that
|
|
300
|
+
/// manufactures false confirmations. Under-claiming is the correct failure mode here, and
|
|
301
|
+
/// the case name says "not observed" rather than "failed" so the log does not overstate it.
|
|
302
|
+
static func classify(
|
|
303
|
+
pastedText: String,
|
|
304
|
+
baseline: FocusedTextRead,
|
|
305
|
+
readBack: FocusedTextRead
|
|
306
|
+
) -> PasteDeliveryEvidence {
|
|
307
|
+
guard !pastedText.isEmpty else { return .unverified(.emptyPayload) }
|
|
308
|
+
|
|
309
|
+
switch (baseline, readBack) {
|
|
310
|
+
case (.unreadable(let failure), _):
|
|
311
|
+
return .unverified(.baselineUnreadable(failure))
|
|
312
|
+
case (_, .unreadable(let failure)):
|
|
313
|
+
return .unverified(.readBackUnreadable(failure))
|
|
314
|
+
case (.read(let before), .read(let after)):
|
|
315
|
+
let occurrencesBefore = occurrences(of: pastedText, in: before.value)
|
|
316
|
+
let occurrencesAfter = occurrences(of: pastedText, in: after.value)
|
|
317
|
+
if occurrencesAfter > occurrencesBefore { return .confirmedByFocusedValue }
|
|
318
|
+
if selectionConfirms(before: before, after: after, pastedText: pastedText) {
|
|
319
|
+
return .confirmedBySelectedText
|
|
320
|
+
}
|
|
321
|
+
if normalized(after.value) == normalized(before.value) {
|
|
322
|
+
return .notObservedFocusedValueUnchanged
|
|
323
|
+
}
|
|
324
|
+
return .unverified(.changedWithoutMatch)
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/// A selection that already read as the pasted text before the paste is indistinguishable
|
|
329
|
+
/// from a paste that never happened, so it does not count.
|
|
330
|
+
private static func selectionConfirms(
|
|
331
|
+
before: FocusedTextSnapshot,
|
|
332
|
+
after: FocusedTextSnapshot,
|
|
333
|
+
pastedText: String
|
|
334
|
+
) -> Bool {
|
|
335
|
+
guard let selected = after.selectedText, !selected.isEmpty else { return false }
|
|
336
|
+
guard normalized(selected) == normalized(pastedText) else { return false }
|
|
337
|
+
return normalized(before.selectedText ?? "") != normalized(pastedText)
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/// Line endings are the one transformation apps routinely apply to pasted text; anything
|
|
341
|
+
/// beyond that is reported as `changedWithoutMatch` rather than guessed at.
|
|
342
|
+
static func normalized(_ text: String) -> String {
|
|
343
|
+
text
|
|
344
|
+
.replacingOccurrences(of: "\r\n", with: "\n")
|
|
345
|
+
.replacingOccurrences(of: "\r", with: "\n")
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
static func occurrences(of needle: String, in haystack: String) -> Int {
|
|
349
|
+
let normalizedNeedle = normalized(needle)
|
|
350
|
+
guard !normalizedNeedle.isEmpty else { return 0 }
|
|
351
|
+
let normalizedHaystack = normalized(haystack)
|
|
352
|
+
var count = 0
|
|
353
|
+
var searchStart = normalizedHaystack.startIndex
|
|
354
|
+
while searchStart < normalizedHaystack.endIndex,
|
|
355
|
+
let found = normalizedHaystack.range(
|
|
356
|
+
of: normalizedNeedle,
|
|
357
|
+
range: searchStart..<normalizedHaystack.endIndex
|
|
358
|
+
) {
|
|
359
|
+
count += 1
|
|
360
|
+
searchStart = found.upperBound
|
|
361
|
+
}
|
|
362
|
+
return count
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// MARK: - What the keystroke attempt itself did
|
|
367
|
+
|
|
368
|
+
/// What a `PastePoster` is allowed to report.
|
|
369
|
+
///
|
|
370
|
+
/// Deliberately narrower than `PasteAttempt`: a poster runs only after the target and the
|
|
371
|
+
/// clipboard have already been proved, so "the paste failed before the keystroke step" is not
|
|
372
|
+
/// something it can observe. Making that unrepresentable is not tidiness — `PasteAttempt` was
|
|
373
|
+
/// originally the poster's return type, and the coordinator's `switch` over it did not compile
|
|
374
|
+
/// (`switch must be exhaustive`) precisely because the type offered a case the poster could
|
|
375
|
+
/// never produce and no honest outcome could be mapped to.
|
|
376
|
+
enum PasteKeystrokeAttempt: Equatable, Sendable {
|
|
377
|
+
/// Two `CGEvent`s were constructed and handed to the window server. Claims nothing about
|
|
378
|
+
/// delivery — `CGEvent.post` returns `Void`.
|
|
379
|
+
case posted
|
|
380
|
+
case constructionFailed
|
|
381
|
+
/// Secure event input is held, so nothing was posted. Carries the holder so the log and the
|
|
382
|
+
/// status line can name it.
|
|
383
|
+
case refusedSecureInput(SecureInputHolder)
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/// Outcome of the posting step alone, as reported in the log. `posted` claims nothing about
|
|
387
|
+
/// delivery — it means two `CGEvent`s were constructed and handed to the window server.
|
|
388
|
+
///
|
|
389
|
+
/// Wider than `PasteKeystrokeAttempt` by exactly one case, because this type is also derived
|
|
390
|
+
/// from a finished outcome via `forOutcome(_:)`, where "never got as far as a keystroke" is a
|
|
391
|
+
/// real thing to report.
|
|
392
|
+
enum PasteAttempt: Equatable, Sendable {
|
|
393
|
+
case posted
|
|
394
|
+
case constructionFailed
|
|
395
|
+
case refusedSecureInput(SecureInputHolder)
|
|
396
|
+
/// The paste failed before the keystroke step (no target, clipboard write failed, payload
|
|
397
|
+
/// ownership lost). Reported so the log never implies a keystroke that never happened.
|
|
398
|
+
case notAttempted
|
|
399
|
+
|
|
400
|
+
var logToken: String {
|
|
401
|
+
switch self {
|
|
402
|
+
case .posted: "constructed_and_posted"
|
|
403
|
+
case .constructionFailed: "construction_failed"
|
|
404
|
+
case .refusedSecureInput: "not_posted_secure_input"
|
|
405
|
+
case .notAttempted: "not_attempted"
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/// Widen what the poster reported into the log vocabulary. One direction only, so the two
|
|
410
|
+
/// types cannot drift into two descriptions of one event.
|
|
411
|
+
init(_ keystroke: PasteKeystrokeAttempt) {
|
|
412
|
+
switch keystroke {
|
|
413
|
+
case .posted: self = .posted
|
|
414
|
+
case .constructionFailed: self = .constructionFailed
|
|
415
|
+
case .refusedSecureInput(let holder): self = .refusedSecureInput(holder)
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/// What the posting step did, read back off the transaction outcome so the log line and
|
|
420
|
+
/// the outcome can never disagree.
|
|
421
|
+
static func forOutcome(_ outcome: PasteDeliveryOutcome) -> PasteAttempt {
|
|
422
|
+
switch outcome {
|
|
423
|
+
case .pasted, .deliveryNotObserved, .deliveredUnverified: .posted
|
|
424
|
+
case .eventPostFailed: .constructionFailed
|
|
425
|
+
case .secureInputActive(let holder): .refusedSecureInput(holder)
|
|
426
|
+
case .targetUnavailable, .clipboardOwnershipLost, .clipboardWriteFailed: .notAttempted
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// MARK: - One log line per delivery
|
|
432
|
+
|
|
433
|
+
/// The record a human reads after trying a dictation. Each field answers a different
|
|
434
|
+
/// question, so a failure cannot hide behind a neighbouring success:
|
|
435
|
+
/// `clipboard=` did the payload reach the pasteboard and stay ours
|
|
436
|
+
/// `clipboard_change_count=` did the pasteboard actually advance for our write
|
|
437
|
+
/// `events=` were the key events constructed and posted
|
|
438
|
+
/// `secure_input=` could any synthetic event have been delivered at all
|
|
439
|
+
/// `delivery=` was the text observed to land in the focused field
|
|
440
|
+
struct PasteDeliveryReport: Equatable, Sendable {
|
|
441
|
+
let targetBundleIdentifier: String?
|
|
442
|
+
let characterCount: Int
|
|
443
|
+
let clipboardWriteVerified: Bool
|
|
444
|
+
let clipboardChangeCountAdvanced: Bool
|
|
445
|
+
let attempt: PasteAttempt
|
|
446
|
+
let secureInput: SecureInputState?
|
|
447
|
+
let evidence: PasteDeliveryEvidence
|
|
448
|
+
let readBackAttempts: Int
|
|
449
|
+
|
|
450
|
+
var logLine: String {
|
|
451
|
+
"paste_delivery target=\(targetBundleIdentifier ?? "?")"
|
|
452
|
+
+ " chars=\(characterCount)"
|
|
453
|
+
+ " clipboard=\(clipboardWriteVerified ? "verified" : "unverified")"
|
|
454
|
+
+ " clipboard_change_count=\(clipboardChangeCountAdvanced ? "advanced" : "unchanged")"
|
|
455
|
+
+ " events=\(attempt.logToken)"
|
|
456
|
+
+ " secure_input=\(secureInput?.logToken ?? "not_probed")"
|
|
457
|
+
+ " delivery=\(evidence.logToken)"
|
|
458
|
+
+ " read_back_attempts=\(readBackAttempts)"
|
|
459
|
+
}
|
|
460
|
+
}
|
|
@@ -294,7 +294,10 @@ public final class RealtimeTranscriptionClient: ObservableObject, @unchecked Sen
|
|
|
294
294
|
let sessionConfig = Self.transcriptionSessionUpdateEvent(transcription: transcription)
|
|
295
295
|
|
|
296
296
|
do {
|
|
297
|
-
try await sendEvent(
|
|
297
|
+
try await sendEvent(
|
|
298
|
+
sessionConfig,
|
|
299
|
+
timeoutMilliseconds: Self.configureSendTimeoutMilliseconds
|
|
300
|
+
)
|
|
298
301
|
isConfigured = true
|
|
299
302
|
flushPendingAudio()
|
|
300
303
|
} catch {
|
|
@@ -369,8 +372,11 @@ public final class RealtimeTranscriptionClient: ObservableObject, @unchecked Sen
|
|
|
369
372
|
}
|
|
370
373
|
|
|
371
374
|
/// Commit buffered input, wait briefly for a final completed event, then close.
|
|
375
|
+
/// The settlement budget is the caller's decision (`RecordingEngine` scales it with
|
|
376
|
+
/// captured audio length); there is deliberately no default so a new call site cannot
|
|
377
|
+
/// silently reintroduce the fixed 700 ms budget this parameter replaced.
|
|
372
378
|
public func finish(
|
|
373
|
-
timeoutMilliseconds: UInt64
|
|
379
|
+
timeoutMilliseconds: UInt64,
|
|
374
380
|
pipelineID: String? = nil,
|
|
375
381
|
pipelineStartedUptimeMilliseconds: UInt64? = nil
|
|
376
382
|
) async -> RealtimeFinishResult {
|
|
@@ -611,14 +617,42 @@ public final class RealtimeTranscriptionClient: ObservableObject, @unchecked Sen
|
|
|
611
617
|
}
|
|
612
618
|
}
|
|
613
619
|
|
|
614
|
-
|
|
615
|
-
|
|
620
|
+
/// Deadline for the session-configure send. This is the first frame on a
|
|
621
|
+
/// just-resumed socket, so it absorbs DNS + TCP + TLS + WebSocket upgrade on a
|
|
622
|
+
/// cold connection — measured 0.77-1.0 s from a fresh process against the live
|
|
623
|
+
/// endpoint. The old shared 500 ms deadline made a cold morning connection fail
|
|
624
|
+
/// configuration outright, poisoning the whole session and silently demoting
|
|
625
|
+
/// every recording to the batch path. The user is speaking while this runs
|
|
626
|
+
/// (audio buffers locally either way), so the larger budget costs nothing on
|
|
627
|
+
/// the delivery path.
|
|
628
|
+
nonisolated static let configureSendTimeoutMilliseconds: UInt64 = 3_000
|
|
629
|
+
/// Deadline for every send after configuration, riding the established
|
|
630
|
+
/// connection. A single send past this deadline records a transport failure,
|
|
631
|
+
/// which permanently vetoes the realtime fast path for the whole recording —
|
|
632
|
+
/// so the deadline must tolerate ordinary jitter, not just the median. At the
|
|
633
|
+
/// old 500 ms value, 3 of 5 two-minute streaming sessions against the live
|
|
634
|
+
/// endpoint were poisoned by one slow frame and demoted to the batch path.
|
|
635
|
+
/// True connection stalls are still detected structurally: the bounded
|
|
636
|
+
/// outbound queue (262,144 bytes of raw 24 kHz/16-bit PCM ≈ 5.5 s of audio,
|
|
637
|
+
/// 262,144 ÷ 48,000 bytes/s) overflows and fails the transport
|
|
638
|
+
/// if sends stop draining, and the settle budget bounds release latency
|
|
639
|
+
/// regardless of this deadline.
|
|
640
|
+
nonisolated static let outboundSendTimeoutMilliseconds: UInt64 = 2_500
|
|
641
|
+
|
|
642
|
+
private func sendEvent(
|
|
643
|
+
_ obj: [String: Any],
|
|
644
|
+
timeoutMilliseconds: UInt64 = RealtimeTranscriptionClient.outboundSendTimeoutMilliseconds
|
|
645
|
+
) async throws {
|
|
646
|
+
try await sendEncodedEvent(encodeJSON(obj), timeoutMilliseconds: timeoutMilliseconds)
|
|
616
647
|
}
|
|
617
648
|
|
|
618
|
-
private func sendEncodedEvent(
|
|
649
|
+
private func sendEncodedEvent(
|
|
650
|
+
_ text: String,
|
|
651
|
+
timeoutMilliseconds: UInt64 = RealtimeTranscriptionClient.outboundSendTimeoutMilliseconds
|
|
652
|
+
) async throws {
|
|
619
653
|
guard let ws else { throw URLError(.networkConnectionLost) }
|
|
620
654
|
try await Self.runOutboundOperationWithDeadline(
|
|
621
|
-
timeoutMilliseconds:
|
|
655
|
+
timeoutMilliseconds: timeoutMilliseconds,
|
|
622
656
|
operation: { try await ws.send(.string(text)) },
|
|
623
657
|
nowMilliseconds: Self.monotonicMilliseconds,
|
|
624
658
|
sleepMilliseconds: { milliseconds in
|
|
@@ -826,7 +860,7 @@ extension RealtimeTranscriptionClient {
|
|
|
826
860
|
_ operation: @escaping @MainActor @Sendable () async throws -> Void
|
|
827
861
|
) async -> Bool {
|
|
828
862
|
await enqueueOutboundOperationTestHelper(
|
|
829
|
-
timeoutMilliseconds:
|
|
863
|
+
timeoutMilliseconds: Self.outboundSendTimeoutMilliseconds,
|
|
830
864
|
operation: operation,
|
|
831
865
|
nowMilliseconds: Self.monotonicMilliseconds,
|
|
832
866
|
sleepMilliseconds: { milliseconds in
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// A recording attempt that produced nothing the user can see anywhere else.
|
|
4
|
+
///
|
|
5
|
+
/// Recordings is an `LSUIElement` app: its only always-on surface is the menu-bar glyph.
|
|
6
|
+
/// `statusMessage` and the Record pane both sit behind a click, so an outcome delivered only
|
|
7
|
+
/// there is invisible to someone who pressed the key while looking at their editor — which is
|
|
8
|
+
/// how a swallowed attempt reads as "the app is still broken" no matter what the log says.
|
|
9
|
+
///
|
|
10
|
+
/// This is a message vocabulary, not a surface. The engine discloses these through
|
|
11
|
+
/// `setBlockedReason(_:for: .pressConsumed)` — the single published `blockedReason` field, which
|
|
12
|
+
/// `MenuBarPresentation` already renders with its own icon and its own VoiceOver label, and which
|
|
13
|
+
/// `startRecording` already clears. A second published field for the same idea would undo the
|
|
14
|
+
/// collapse that field exists to be.
|
|
15
|
+
public enum RecordingAttemptAlert: Equatable, Sendable {
|
|
16
|
+
/// The trigger was released before the microphone delivered its first sample, so no audio
|
|
17
|
+
/// ever existed to transcribe.
|
|
18
|
+
case releasedBeforeAudio
|
|
19
|
+
/// Capture ran to completion but the microphone produced no audio at all.
|
|
20
|
+
case noAudioCaptured
|
|
21
|
+
|
|
22
|
+
/// Shown on the menu-bar status line and in the Record pane. Kept short enough to fit the
|
|
23
|
+
/// 260 pt popover without wrapping past two lines, and phrased as the corrective action
|
|
24
|
+
/// rather than a diagnosis.
|
|
25
|
+
///
|
|
26
|
+
/// Deliberately carries no millisecond figure. The hold a user has to manage is the cold
|
|
27
|
+
/// audio engine plus one input buffer period, and the buffer period is a property of
|
|
28
|
+
/// whichever device is selected — so any number printed here would be wrong on some
|
|
29
|
+
/// machines.
|
|
30
|
+
public var message: String {
|
|
31
|
+
switch self {
|
|
32
|
+
case .releasedBeforeAudio: "Released too soon — hold the key a moment longer"
|
|
33
|
+
case .noAudioCaptured: "No audio captured"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|