@agent-native/core 0.114.6 → 0.114.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +15 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/cli/mcp-config-writers.ts +6 -4
  5. package/corpus/core/src/cli/mcp.ts +124 -0
  6. package/corpus/core/src/cli/skills-content/help.ts +4 -3
  7. package/corpus/core/src/cli/skills-content/index.ts +1 -0
  8. package/corpus/core/src/cli/skills-content/rewind-skill.ts +52 -0
  9. package/corpus/core/src/cli/skills.ts +115 -13
  10. package/corpus/core/src/mcp/screen-memory-stdio.ts +1360 -104
  11. package/corpus/core/src/mcp-client/index.ts +9 -0
  12. package/corpus/core/src/mcp-client/screen-memory-local.ts +847 -13
  13. package/corpus/core/src/server/agent-chat-plugin.ts +16 -9
  14. package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +19 -7
  15. package/corpus/templates/analytics/app/hooks/use-chart-tooltip-portal.ts +60 -0
  16. package/corpus/templates/analytics/app/routes/chart.tsx +1 -1
  17. package/corpus/templates/analytics/changelog/2026-07-20-dashboard-requests-no-longer-stall-without-progress.md +6 -0
  18. package/corpus/templates/analytics/server/plugins/agent-chat.ts +6 -0
  19. package/corpus/templates/clips/actions/apply-rewind-extension.ts +195 -0
  20. package/corpus/templates/clips/actions/delete-agent-recording-if-unpromoted.ts +64 -0
  21. package/corpus/templates/clips/actions/get-rewind-extension-request.ts +22 -0
  22. package/corpus/templates/clips/actions/get-screen-memory-status.ts +18 -2
  23. package/corpus/templates/clips/actions/list-rewind-extension-requests.ts +52 -0
  24. package/corpus/templates/clips/actions/make-recording-private-for-rewind.ts +55 -0
  25. package/corpus/templates/clips/actions/query-screen-memory-context.ts +4 -4
  26. package/corpus/templates/clips/actions/request-rewind-extension.ts +81 -0
  27. package/corpus/templates/clips/actions/update-rewind-extension-request.ts +47 -0
  28. package/corpus/templates/clips/app/components/editor/editor-layout.tsx +25 -0
  29. package/corpus/templates/clips/app/components/editor/editor-toolbar.tsx +36 -0
  30. package/corpus/templates/clips/app/components/editor/rewind-extension-dialog.tsx +338 -0
  31. package/corpus/templates/clips/app/components/editor/stitch-manager.tsx +1 -16
  32. package/corpus/templates/clips/app/components/editor/timeline.tsx +26 -0
  33. package/corpus/templates/clips/app/i18n/en-US.ts +18 -0
  34. package/corpus/templates/clips/app/lib/ffmpeg-export.ts +19 -12
  35. package/corpus/templates/clips/app/lib/rewind-visibility.ts +31 -0
  36. package/corpus/templates/clips/app/lib/timestamp-mapping.ts +7 -0
  37. package/corpus/templates/clips/app/lib/upload-file-client.ts +17 -0
  38. package/corpus/templates/clips/changelog/2026-07-14-rewind-keeps-recent-screen-context-local-and-turns-it-into-clips.md +6 -0
  39. package/corpus/templates/clips/changelog/2026-07-17-active-clips-keep-rewind-stable-and-settings-clear.md +12 -0
  40. package/corpus/templates/clips/changelog/2026-07-17-clicking-the-clips-icon-during-a-recording-now-opens-the-app.md +6 -0
  41. package/corpus/templates/clips/changelog/2026-07-17-default-mic-now-uses-the-mac-default-and-transcription-shares-one-capture.md +6 -0
  42. package/corpus/templates/clips/changelog/2026-07-17-full-screen-clips-finalize-even-when-screencapturekit-stop-hangs.md +6 -0
  43. package/corpus/templates/clips/changelog/2026-07-17-normal-full-screen-clips-no-longer-stop-immediately-after-th.md +6 -0
  44. package/corpus/templates/clips/changelog/2026-07-17-rewind-can-hand-one-bounded-private-clip-to-your-agent.md +12 -0
  45. package/corpus/templates/clips/changelog/2026-07-17-rewind-home-can-copy-a-local-agent-prompt.md +6 -0
  46. package/corpus/templates/clips/changelog/2026-07-17-rewind-now-has-a-calm-home-status-focused-first-run-consent-.md +6 -0
  47. package/corpus/templates/clips/changelog/2026-07-17-rewind-settings-are-smaller-clearer-and-agent-first.md +6 -0
  48. package/corpus/templates/clips/changelog/2026-07-17-the-full-clips-interface-stays-available-while-recording.md +6 -0
  49. package/corpus/templates/clips/changelog/2026-07-18-rewind-organizes-recent-work-into-local-searchable-chapters.md +8 -0
  50. package/corpus/templates/clips/changelog/2026-07-18-searchable-rewind-chapters.md +6 -0
  51. package/corpus/templates/clips/changelog/2026-07-19-rewind-backed-recordings-now-start-immediately-after-the-cou.md +6 -0
  52. package/corpus/templates/clips/changelog/2026-07-20-rewind-can-set-up-your-local-agent-once-so-future-requests-w.md +6 -0
  53. package/corpus/templates/clips/desktop/package.json +1 -0
  54. package/corpus/templates/clips/desktop/scripts/sign-macos-local.ts +12 -1
  55. package/corpus/templates/clips/desktop/src/app.tsx +2498 -239
  56. package/corpus/templates/clips/desktop/src/components/ReadinessPanel.tsx +21 -1
  57. package/corpus/templates/clips/desktop/src/components/Switch.tsx +3 -0
  58. package/corpus/templates/clips/desktop/src/hooks/useMeetingTranscription.ts +156 -24
  59. package/corpus/templates/clips/desktop/src/lib/native-recording-warm.ts +6 -7
  60. package/corpus/templates/clips/desktop/src/lib/recorder.ts +1763 -1160
  61. package/corpus/templates/clips/desktop/src/lib/rewind-agent-prompt.ts +9 -0
  62. package/corpus/templates/clips/desktop/src/lib/rewind-recording-start.ts +20 -0
  63. package/corpus/templates/clips/desktop/src/main.tsx +3 -0
  64. package/corpus/templates/clips/desktop/src/overlays/countdown.tsx +14 -14
  65. package/corpus/templates/clips/desktop/src/overlays/onboarding.tsx +4 -4
  66. package/corpus/templates/clips/desktop/src/overlays/preparing.tsx +13 -0
  67. package/corpus/templates/clips/desktop/src/overlays/toolbar.tsx +27 -2
  68. package/corpus/templates/clips/desktop/src/shared/config.ts +16 -0
  69. package/corpus/templates/clips/desktop/src/styles.css +956 -134
  70. package/corpus/templates/clips/desktop/src-tauri/Cargo.toml +5 -0
  71. package/corpus/templates/clips/desktop/src-tauri/build.rs +69 -1
  72. package/corpus/templates/clips/desktop/src-tauri/src/accessibility.rs +386 -12
  73. package/corpus/templates/clips/desktop/src-tauri/src/capture_audio_bus.rs +382 -0
  74. package/corpus/templates/clips/desktop/src-tauri/src/capture_graph.rs +777 -0
  75. package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +249 -22
  76. package/corpus/templates/clips/desktop/src-tauri/src/config.rs +148 -1
  77. package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +58 -4
  78. package/corpus/templates/clips/desktop/src-tauri/src/logfile.rs +13 -1
  79. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +1505 -51
  80. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +124 -0
  81. package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +1006 -95
  82. package/corpus/templates/clips/desktop/src-tauri/src/rewind_capture_suspension.rs +165 -0
  83. package/corpus/templates/clips/desktop/src-tauri/src/rewind_chapters.rs +2434 -0
  84. package/corpus/templates/clips/desktop/src-tauri/src/rewind_clip.rs +1650 -0
  85. package/corpus/templates/clips/desktop/src-tauri/src/rewind_egress.rs +643 -0
  86. package/corpus/templates/clips/desktop/src-tauri/src/rewind_local_ask.rs +803 -0
  87. package/corpus/templates/clips/desktop/src-tauri/src/rewind_meeting_history.rs +595 -0
  88. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory.rs +2900 -91
  89. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory_ocr.rs +360 -0
  90. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory_transcript.rs +408 -0
  91. package/corpus/templates/clips/desktop/src-tauri/src/shortcuts.rs +114 -33
  92. package/corpus/templates/clips/desktop/src-tauri/src/state.rs +2 -2
  93. package/corpus/templates/clips/desktop/src-tauri/src/system_audio.rs +13 -3
  94. package/corpus/templates/clips/desktop/src-tauri/src/tray.rs +4 -1
  95. package/corpus/templates/clips/desktop/src-tauri/src/util.rs +7 -13
  96. package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +198 -11
  97. package/corpus/templates/clips/desktop/src-tauri/tauri.alpha.conf.json +17 -0
  98. package/corpus/toolkit/CHANGELOG.md +6 -0
  99. package/corpus/toolkit/package.json +1 -1
  100. package/dist/cli/mcp-config-writers.d.ts +2 -2
  101. package/dist/cli/mcp-config-writers.d.ts.map +1 -1
  102. package/dist/cli/mcp-config-writers.js +6 -6
  103. package/dist/cli/mcp-config-writers.js.map +1 -1
  104. package/dist/cli/mcp.d.ts +14 -0
  105. package/dist/cli/mcp.d.ts.map +1 -1
  106. package/dist/cli/mcp.js +79 -0
  107. package/dist/cli/mcp.js.map +1 -1
  108. package/dist/cli/skills-content/help.d.ts +1 -1
  109. package/dist/cli/skills-content/help.d.ts.map +1 -1
  110. package/dist/cli/skills-content/help.js +4 -3
  111. package/dist/cli/skills-content/help.js.map +1 -1
  112. package/dist/cli/skills-content/index.d.ts +1 -0
  113. package/dist/cli/skills-content/index.d.ts.map +1 -1
  114. package/dist/cli/skills-content/index.js +1 -0
  115. package/dist/cli/skills-content/index.js.map +1 -1
  116. package/dist/cli/skills-content/rewind-skill.d.ts +2 -0
  117. package/dist/cli/skills-content/rewind-skill.d.ts.map +1 -0
  118. package/dist/cli/skills-content/rewind-skill.js +53 -0
  119. package/dist/cli/skills-content/rewind-skill.js.map +1 -0
  120. package/dist/cli/skills.d.ts +6 -0
  121. package/dist/cli/skills.d.ts.map +1 -1
  122. package/dist/cli/skills.js +90 -14
  123. package/dist/cli/skills.js.map +1 -1
  124. package/dist/collab/struct-routes.d.ts +1 -1
  125. package/dist/mcp/screen-memory-stdio.d.ts +365 -0
  126. package/dist/mcp/screen-memory-stdio.d.ts.map +1 -1
  127. package/dist/mcp/screen-memory-stdio.js +1014 -91
  128. package/dist/mcp/screen-memory-stdio.js.map +1 -1
  129. package/dist/mcp-client/index.d.ts +1 -1
  130. package/dist/mcp-client/index.d.ts.map +1 -1
  131. package/dist/mcp-client/index.js +1 -1
  132. package/dist/mcp-client/index.js.map +1 -1
  133. package/dist/mcp-client/screen-memory-local.d.ts +90 -0
  134. package/dist/mcp-client/screen-memory-local.d.ts.map +1 -1
  135. package/dist/mcp-client/screen-memory-local.js +590 -12
  136. package/dist/mcp-client/screen-memory-local.js.map +1 -1
  137. package/dist/observability/routes.d.ts +1 -1
  138. package/dist/progress/routes.d.ts +1 -1
  139. package/dist/provider-api/actions/provider-api.d.ts +4 -4
  140. package/dist/resources/handlers.d.ts +1 -1
  141. package/dist/server/agent-chat-plugin.d.ts +5 -0
  142. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  143. package/dist/server/agent-chat-plugin.js +10 -9
  144. package/dist/server/agent-chat-plugin.js.map +1 -1
  145. package/package.json +2 -2
  146. package/src/cli/mcp-config-writers.ts +6 -4
  147. package/src/cli/mcp.ts +124 -0
  148. package/src/cli/skills-content/help.ts +4 -3
  149. package/src/cli/skills-content/index.ts +1 -0
  150. package/src/cli/skills-content/rewind-skill.ts +52 -0
  151. package/src/cli/skills.ts +115 -13
  152. package/src/mcp/screen-memory-stdio.ts +1360 -104
  153. package/src/mcp-client/index.ts +9 -0
  154. package/src/mcp-client/screen-memory-local.ts +847 -13
  155. package/src/server/agent-chat-plugin.ts +16 -9
@@ -0,0 +1,803 @@
1
+ //! First-party, fully local Rewind search and replay.
2
+ //!
3
+ //! This module reads local indexes inside Clips, returns bounded text-only evidence to the
4
+ //! renderer, and never performs a network request. Raw archive paths stay on
5
+ //! the Rust side of the command boundary.
6
+
7
+ use crate::capture_graph::{CaptureGraphState, CaptureSource};
8
+ use crate::config::RewindCaptureMode;
9
+ use crate::native_screen::{self, NativeAudioSelection, NativeMediaSlice};
10
+ use crate::screen_memory::{self, ScreenMemoryEvent, ScreenMemorySegmentMetadata};
11
+ use crate::screen_memory_ocr::ScreenMemoryOcrRow;
12
+ use crate::screen_memory_transcript::ScreenMemoryTranscriptRow;
13
+ use chrono::{DateTime, Utc};
14
+ use serde::{Deserialize, Serialize};
15
+ use std::fs::File;
16
+ use std::io::{BufRead, BufReader};
17
+ use std::path::{Path, PathBuf};
18
+ use std::process::{Command, Stdio};
19
+ use std::time::{Duration, SystemTime};
20
+ use tauri::{AppHandle, Manager};
21
+
22
+ const MAX_QUERY_BYTES: usize = 500;
23
+ const DEFAULT_EVIDENCE_LIMIT: usize = 12;
24
+ const MAX_EVIDENCE_LIMIT: usize = 20;
25
+ const MAX_EXCERPT_BYTES: usize = 600;
26
+ const REPLAY_BEFORE_MS: u64 = 5_000;
27
+ const REPLAY_AFTER_MS: u64 = 10_000;
28
+ const MAX_REPLAY_MS: u64 = 30_000;
29
+ const PREVIEW_MAX_FILES: usize = 8;
30
+ const PREVIEW_MAX_AGE: Duration = Duration::from_secs(60 * 60);
31
+
32
+ #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
33
+ #[serde(rename_all = "kebab-case")]
34
+ enum LocalEvidenceSource {
35
+ AppContext,
36
+ Transcript,
37
+ Ocr,
38
+ }
39
+
40
+ #[derive(Clone, Debug, Serialize)]
41
+ #[serde(rename_all = "camelCase")]
42
+ pub(crate) struct RewindLocalEvidence {
43
+ id: String,
44
+ source_type: LocalEvidenceSource,
45
+ captured_at: String,
46
+ excerpt: String,
47
+ confidence: Option<f32>,
48
+ segment_id: String,
49
+ offset_ms: u64,
50
+ }
51
+
52
+ #[derive(Clone, Debug, Serialize)]
53
+ #[serde(rename_all = "camelCase")]
54
+ pub(crate) struct RewindLocalCoverageGap {
55
+ kind: String,
56
+ source: String,
57
+ started_at: Option<String>,
58
+ ended_at: Option<String>,
59
+ detail: String,
60
+ }
61
+
62
+ #[derive(Clone, Debug, Serialize)]
63
+ #[serde(rename_all = "camelCase")]
64
+ pub(crate) struct RewindLocalCoverage {
65
+ segments_considered: usize,
66
+ transcript_indexes_ready: usize,
67
+ ocr_indexes_ready: usize,
68
+ gaps: Vec<RewindLocalCoverageGap>,
69
+ }
70
+
71
+ #[derive(Clone, Debug, Serialize)]
72
+ #[serde(rename_all = "camelCase")]
73
+ pub(crate) struct RewindLocalAskResult {
74
+ query: String,
75
+ answer_summary: String,
76
+ evidence: Vec<RewindLocalEvidence>,
77
+ coverage: RewindLocalCoverage,
78
+ confidence: String,
79
+ truncated: bool,
80
+ }
81
+
82
+ #[derive(Clone, Debug, Serialize)]
83
+ #[serde(rename_all = "camelCase")]
84
+ pub(crate) struct RewindReplayResult {
85
+ opened: bool,
86
+ segment_id: String,
87
+ offset_ms: u64,
88
+ preview_started_ms: u64,
89
+ preview_ended_ms: u64,
90
+ }
91
+
92
+ #[derive(Clone, Debug)]
93
+ struct ScoredEvidence {
94
+ score: usize,
95
+ evidence: RewindLocalEvidence,
96
+ }
97
+
98
+ #[derive(Debug, Deserialize)]
99
+ struct IndexStatus {
100
+ state: String,
101
+ }
102
+
103
+ fn bounded_text(value: &str, max_bytes: usize) -> String {
104
+ if value.len() <= max_bytes {
105
+ return value.to_owned();
106
+ }
107
+ let mut end = max_bytes;
108
+ while !value.is_char_boundary(end) {
109
+ end -= 1;
110
+ }
111
+ format!("{}…", value[..end].trim_end())
112
+ }
113
+
114
+ fn normalized_terms(query: &str) -> Vec<String> {
115
+ let stop_words = [
116
+ "a", "about", "an", "and", "did", "do", "for", "from", "i", "in", "is", "it", "me", "my",
117
+ "of", "on", "the", "to", "was", "what", "when", "where", "with",
118
+ ];
119
+ let mut terms = query
120
+ .split(|character: char| !character.is_alphanumeric())
121
+ .map(str::trim)
122
+ .filter(|term| term.len() >= 2)
123
+ .map(str::to_lowercase)
124
+ .filter(|term| !stop_words.contains(&term.as_str()))
125
+ .collect::<Vec<_>>();
126
+ terms.sort();
127
+ terms.dedup();
128
+ if terms.is_empty() {
129
+ let fallback = query.trim().to_lowercase();
130
+ if !fallback.is_empty() {
131
+ terms.push(fallback);
132
+ }
133
+ }
134
+ terms
135
+ }
136
+
137
+ fn match_score(text: &str, terms: &[String]) -> usize {
138
+ let haystack = text.to_lowercase();
139
+ let counts = terms
140
+ .iter()
141
+ .map(|term| haystack.matches(term).count())
142
+ .collect::<Vec<_>>();
143
+ let matched_terms = counts.iter().filter(|count| **count > 0).count();
144
+ let minimum_terms = if terms.len() <= 1 {
145
+ 1
146
+ } else {
147
+ terms.len().div_ceil(2)
148
+ };
149
+ if matched_terms < minimum_terms {
150
+ return 0;
151
+ }
152
+ matched_terms * 1_000 + counts.into_iter().sum::<usize>()
153
+ }
154
+
155
+ fn matching_excerpt(text: &str, terms: &[String], max_bytes: usize) -> String {
156
+ let lines = text.lines().collect::<Vec<_>>();
157
+ let Some((best_index, _)) = lines
158
+ .iter()
159
+ .enumerate()
160
+ .map(|(index, line)| (index, match_score(line, terms)))
161
+ .max_by_key(|(_, score)| *score)
162
+ .filter(|(_, score)| *score > 0)
163
+ else {
164
+ return bounded_text(text.trim(), max_bytes);
165
+ };
166
+ let start = best_index.saturating_sub(1);
167
+ let end = (best_index + 2).min(lines.len());
168
+ bounded_text(lines[start..end].join("\n").trim(), max_bytes)
169
+ }
170
+
171
+ fn parse_time(value: &str) -> Option<DateTime<Utc>> {
172
+ DateTime::parse_from_rfc3339(value)
173
+ .ok()
174
+ .map(|value| value.with_timezone(&Utc))
175
+ }
176
+
177
+ fn event_segment<'a>(
178
+ event: &ScreenMemoryEvent,
179
+ segments: &'a [ScreenMemorySegmentMetadata],
180
+ ) -> Option<(&'a ScreenMemorySegmentMetadata, u64)> {
181
+ let captured = parse_time(&event.captured_at)?;
182
+ segments.iter().find_map(|segment| {
183
+ let started = parse_time(&segment.started_at)?;
184
+ let ended = parse_time(&segment.ended_at)?;
185
+ if captured < started || captured > ended {
186
+ return None;
187
+ }
188
+ let offset = captured
189
+ .signed_duration_since(started)
190
+ .num_milliseconds()
191
+ .max(0) as u64;
192
+ Some((segment, offset.min(segment.duration_ms as u64)))
193
+ })
194
+ }
195
+
196
+ fn read_jsonl<T: for<'de> Deserialize<'de>>(path: &Path) -> Vec<T> {
197
+ let Ok(file) = File::open(path) else {
198
+ return Vec::new();
199
+ };
200
+ BufReader::new(file)
201
+ .lines()
202
+ .map_while(Result::ok)
203
+ .filter_map(|line| serde_json::from_str(&line).ok())
204
+ .collect()
205
+ }
206
+
207
+ fn index_ready(path: &Path) -> bool {
208
+ std::fs::read(path)
209
+ .ok()
210
+ .and_then(|bytes| serde_json::from_slice::<IndexStatus>(&bytes).ok())
211
+ .is_some_and(|status| status.state == "ready")
212
+ }
213
+
214
+ fn adjacent_path(segment: &ScreenMemorySegmentMetadata, suffix: &str) -> Option<PathBuf> {
215
+ segment
216
+ .path
217
+ .parent()
218
+ .map(|directory| directory.join(format!("{}.{suffix}", segment.id)))
219
+ }
220
+
221
+ fn evidence_id(source: LocalEvidenceSource, segment_id: &str, offset_ms: u64) -> String {
222
+ let source = match source {
223
+ LocalEvidenceSource::AppContext => "app",
224
+ LocalEvidenceSource::Transcript => "transcript",
225
+ LocalEvidenceSource::Ocr => "ocr",
226
+ };
227
+ format!("{source}:{segment_id}:{offset_ms}")
228
+ }
229
+
230
+ fn push_match(
231
+ matches: &mut Vec<ScoredEvidence>,
232
+ terms: &[String],
233
+ source_type: LocalEvidenceSource,
234
+ captured_at: String,
235
+ text: String,
236
+ confidence: Option<f32>,
237
+ segment_id: String,
238
+ offset_ms: u64,
239
+ ) {
240
+ let score = match_score(&text, terms);
241
+ if score == 0 {
242
+ return;
243
+ }
244
+ matches.push(ScoredEvidence {
245
+ score,
246
+ evidence: RewindLocalEvidence {
247
+ id: evidence_id(source_type, &segment_id, offset_ms),
248
+ source_type,
249
+ captured_at,
250
+ excerpt: matching_excerpt(&text, terms, MAX_EXCERPT_BYTES),
251
+ confidence,
252
+ segment_id,
253
+ offset_ms,
254
+ },
255
+ });
256
+ }
257
+
258
+ fn local_query(
259
+ directory: &Path,
260
+ segments: Vec<ScreenMemorySegmentMetadata>,
261
+ query: &str,
262
+ limit: usize,
263
+ mut gaps: Vec<RewindLocalCoverageGap>,
264
+ ) -> RewindLocalAskResult {
265
+ let terms = normalized_terms(query);
266
+ let valid_segments = segments
267
+ .into_iter()
268
+ .filter(|segment| {
269
+ segment.path.exists()
270
+ && !segment.corrupt
271
+ && segment.error.is_none()
272
+ && !segment.exclusion_tainted
273
+ })
274
+ .collect::<Vec<_>>();
275
+ let mut matches = Vec::new();
276
+
277
+ for event in read_jsonl::<ScreenMemoryEvent>(&directory.join("events.jsonl")) {
278
+ if event.source == "coverage-gap" {
279
+ continue;
280
+ }
281
+ let Some((segment, offset_ms)) = event_segment(&event, &valid_segments) else {
282
+ continue;
283
+ };
284
+ let text = [
285
+ event.app_name.as_deref(),
286
+ event.window_title.as_deref(),
287
+ event.bundle_id.as_deref(),
288
+ ]
289
+ .into_iter()
290
+ .flatten()
291
+ .collect::<Vec<_>>()
292
+ .join(" — ");
293
+ push_match(
294
+ &mut matches,
295
+ &terms,
296
+ LocalEvidenceSource::AppContext,
297
+ event.captured_at,
298
+ text,
299
+ None,
300
+ segment.id.clone(),
301
+ offset_ms,
302
+ );
303
+ }
304
+
305
+ let mut transcript_ready = 0;
306
+ let mut ocr_ready = 0;
307
+ for segment in &valid_segments {
308
+ let transcript_status = adjacent_path(segment, "transcript-status.json");
309
+ let transcript_rows = adjacent_path(segment, "transcript.jsonl");
310
+ if transcript_status.as_deref().is_some_and(index_ready) {
311
+ transcript_ready += 1;
312
+ if let Some(path) = transcript_rows {
313
+ for row in read_jsonl::<ScreenMemoryTranscriptRow>(&path) {
314
+ push_match(
315
+ &mut matches,
316
+ &terms,
317
+ LocalEvidenceSource::Transcript,
318
+ row.captured_at,
319
+ row.text,
320
+ None,
321
+ segment.id.clone(),
322
+ row.start_ms.min(segment.duration_ms as u64),
323
+ );
324
+ }
325
+ }
326
+ } else {
327
+ gaps.push(RewindLocalCoverageGap {
328
+ kind: "index".into(),
329
+ source: "transcript".into(),
330
+ started_at: Some(segment.started_at.clone()),
331
+ ended_at: Some(segment.ended_at.clone()),
332
+ detail: "Local transcript is not ready for this segment.".into(),
333
+ });
334
+ }
335
+
336
+ let ocr_status = adjacent_path(segment, "ocr-status.json");
337
+ let ocr_rows = adjacent_path(segment, "ocr.jsonl");
338
+ if ocr_status.as_deref().is_some_and(index_ready) {
339
+ ocr_ready += 1;
340
+ if let Some(path) = ocr_rows {
341
+ for row in read_jsonl::<ScreenMemoryOcrRow>(&path) {
342
+ push_match(
343
+ &mut matches,
344
+ &terms,
345
+ LocalEvidenceSource::Ocr,
346
+ row.captured_at,
347
+ row.ocr_text,
348
+ Some(row.confidence),
349
+ segment.id.clone(),
350
+ row.offset_ms.max(0) as u64,
351
+ );
352
+ }
353
+ }
354
+ } else {
355
+ gaps.push(RewindLocalCoverageGap {
356
+ kind: "index".into(),
357
+ source: "ocr".into(),
358
+ started_at: Some(segment.started_at.clone()),
359
+ ended_at: Some(segment.ended_at.clone()),
360
+ detail: "Local visual index is not ready for this segment.".into(),
361
+ });
362
+ }
363
+ }
364
+
365
+ matches.sort_by(|left, right| {
366
+ right
367
+ .score
368
+ .cmp(&left.score)
369
+ .then_with(|| right.evidence.captured_at.cmp(&left.evidence.captured_at))
370
+ });
371
+ let total_matches = matches.len();
372
+ let limit = limit.clamp(1, MAX_EVIDENCE_LIMIT);
373
+ let evidence = matches
374
+ .into_iter()
375
+ .take(limit)
376
+ .map(|item| item.evidence)
377
+ .collect::<Vec<_>>();
378
+ let truncated = total_matches > evidence.len();
379
+ let answer_summary = match evidence.first() {
380
+ Some(_) => format!(
381
+ "Found {total_matches} local match{} in Rewind. The strongest match is shown first; this is search evidence, not a generated conclusion.",
382
+ if total_matches == 1 { "" } else { "es" }
383
+ ),
384
+ None => "No matching local evidence was found in the indexed Rewind coverage. This does not prove the event did not happen; coverage or indexes may be incomplete.".into(),
385
+ };
386
+ let confidence = if evidence.is_empty() {
387
+ "No matching evidence".into()
388
+ } else if gaps.is_empty() && !truncated {
389
+ "Direct local matches; complete for the indexed segments shown".into()
390
+ } else {
391
+ "Direct local matches; incomplete coverage or indexing".into()
392
+ };
393
+ RewindLocalAskResult {
394
+ query: query.to_owned(),
395
+ answer_summary,
396
+ evidence,
397
+ coverage: RewindLocalCoverage {
398
+ segments_considered: valid_segments.len(),
399
+ transcript_indexes_ready: transcript_ready,
400
+ ocr_indexes_ready: ocr_ready,
401
+ gaps,
402
+ },
403
+ confidence,
404
+ truncated,
405
+ }
406
+ }
407
+
408
+ fn graph_coverage_gaps(app: &AppHandle) -> Vec<RewindLocalCoverageGap> {
409
+ let graph_state = app.state::<CaptureGraphState>();
410
+ let Ok(graph) = graph_state.0.lock() else {
411
+ return vec![RewindLocalCoverageGap {
412
+ kind: "capture".into(),
413
+ source: "screen".into(),
414
+ started_at: None,
415
+ ended_at: None,
416
+ detail: "Capture coverage state is unavailable.".into(),
417
+ }];
418
+ };
419
+ let Ok(status) = graph.status_at(std::time::Instant::now()) else {
420
+ return Vec::new();
421
+ };
422
+ status
423
+ .coverage_gaps
424
+ .into_iter()
425
+ .map(|gap| RewindLocalCoverageGap {
426
+ kind: "capture".into(),
427
+ source: match gap.source {
428
+ CaptureSource::Screen => "screen",
429
+ CaptureSource::SystemAudio => "system-audio",
430
+ CaptureSource::Microphone => "microphone",
431
+ CaptureSource::Camera => "camera",
432
+ }
433
+ .into(),
434
+ started_at: Some(gap.interval.started_at),
435
+ ended_at: Some(gap.interval.ended_at),
436
+ detail: format!("Capture gap: {:?}.", gap.reason),
437
+ })
438
+ .collect()
439
+ }
440
+
441
+ #[tauri::command]
442
+ pub(crate) fn rewind_local_ask(
443
+ app: AppHandle,
444
+ query: String,
445
+ limit: Option<usize>,
446
+ ) -> Result<RewindLocalAskResult, String> {
447
+ let query = query.trim();
448
+ if query.is_empty() {
449
+ return Err("Ask Rewind needs a question or search phrase.".into());
450
+ }
451
+ if query.len() > MAX_QUERY_BYTES {
452
+ return Err(format!(
453
+ "Ask Rewind queries must be at most {MAX_QUERY_BYTES} bytes."
454
+ ));
455
+ }
456
+ let segments = screen_memory::finalized_segments(&app)?;
457
+ let directory = segments
458
+ .first()
459
+ .and_then(|segment| segment.path.parent().map(Path::to_path_buf))
460
+ .or_else(|| {
461
+ app.path()
462
+ .app_data_dir()
463
+ .ok()
464
+ .map(|path| path.join("screen-memory"))
465
+ })
466
+ .ok_or_else(|| "Rewind storage directory is unavailable.".to_string())?;
467
+ Ok(local_query(
468
+ &directory,
469
+ segments,
470
+ query,
471
+ limit.unwrap_or(DEFAULT_EVIDENCE_LIMIT),
472
+ graph_coverage_gaps(&app),
473
+ ))
474
+ }
475
+
476
+ fn replay_slice(duration_ms: u64, offset_ms: u64) -> Result<(u64, u64), String> {
477
+ if duration_ms == 0 || offset_ms >= duration_ms {
478
+ return Err("Replay moment is outside the retained segment.".into());
479
+ }
480
+ let started_ms = offset_ms.saturating_sub(REPLAY_BEFORE_MS);
481
+ let ended_ms = offset_ms.saturating_add(REPLAY_AFTER_MS).min(duration_ms);
482
+ if ended_ms <= started_ms || ended_ms - started_ms > MAX_REPLAY_MS {
483
+ return Err("Replay moment could not be bounded safely.".into());
484
+ }
485
+ Ok((started_ms, ended_ms))
486
+ }
487
+
488
+ fn cleanup_previews(directory: &Path) {
489
+ let now = SystemTime::now();
490
+ let mut files = std::fs::read_dir(directory)
491
+ .into_iter()
492
+ .flatten()
493
+ .flatten()
494
+ .filter_map(|entry| {
495
+ let metadata = entry.metadata().ok()?;
496
+ if !metadata.is_file() {
497
+ return None;
498
+ }
499
+ Some((
500
+ entry.path(),
501
+ metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
502
+ ))
503
+ })
504
+ .collect::<Vec<_>>();
505
+ for (path, modified) in &files {
506
+ if now.duration_since(*modified).unwrap_or_default() > PREVIEW_MAX_AGE {
507
+ let _ = std::fs::remove_file(path);
508
+ }
509
+ }
510
+ files.retain(|(path, _)| path.exists());
511
+ files.sort_by_key(|(_, modified)| std::cmp::Reverse(*modified));
512
+ for (path, _) in files.into_iter().skip(PREVIEW_MAX_FILES.saturating_sub(1)) {
513
+ let _ = std::fs::remove_file(path);
514
+ }
515
+ }
516
+
517
+ fn slice_overlaps_graph_gap(
518
+ app: &AppHandle,
519
+ segment: &ScreenMemorySegmentMetadata,
520
+ started_ms: u64,
521
+ ended_ms: u64,
522
+ ) -> bool {
523
+ let selected_start = segment.graph_started_elapsed_ms.saturating_add(started_ms);
524
+ let selected_end = segment.graph_started_elapsed_ms.saturating_add(ended_ms);
525
+ app.state::<CaptureGraphState>()
526
+ .0
527
+ .lock()
528
+ .ok()
529
+ .and_then(|graph| graph.status_at(std::time::Instant::now()).ok())
530
+ .is_some_and(|status| {
531
+ status.coverage_gaps.iter().any(|gap| {
532
+ gap.source == CaptureSource::Screen
533
+ && gap.interval.started_elapsed_ms < selected_end
534
+ && gap.interval.ended_elapsed_ms > selected_start
535
+ })
536
+ })
537
+ }
538
+
539
+ fn validate_replay_candidate(
540
+ segment: &ScreenMemorySegmentMetadata,
541
+ overlaps_gap: bool,
542
+ ) -> Result<(), String> {
543
+ if segment.corrupt || segment.error.is_some() || segment.exclusion_tainted {
544
+ return Err("Replay evidence is corrupt, incomplete, or privacy-tainted.".into());
545
+ }
546
+ if segment.graph_ended_elapsed_ms <= segment.graph_started_elapsed_ms {
547
+ return Err("Replay evidence has no trustworthy capture coverage mapping.".into());
548
+ }
549
+ if overlaps_gap {
550
+ return Err("Replay evidence overlaps a recorded capture gap.".into());
551
+ }
552
+ Ok(())
553
+ }
554
+
555
+ #[tauri::command]
556
+ pub(crate) async fn rewind_replay_moment(
557
+ app: AppHandle,
558
+ segment_id: String,
559
+ offset_ms: u64,
560
+ ) -> Result<RewindReplayResult, String> {
561
+ if segment_id.trim().is_empty()
562
+ || segment_id.len() > 255
563
+ || !segment_id
564
+ .chars()
565
+ .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_'))
566
+ {
567
+ return Err("Replay segment ID is invalid.".into());
568
+ }
569
+ let segment = screen_memory::finalized_segments(&app)?
570
+ .into_iter()
571
+ .find(|segment| segment.id == segment_id)
572
+ .ok_or_else(|| "Replay segment is missing from retained Rewind coverage.".to_string())?;
573
+ let duration_ms = u64::try_from(segment.duration_ms)
574
+ .map_err(|_| "Replay segment duration is invalid.".to_string())?;
575
+ let (started_ms, ended_ms) = replay_slice(duration_ms, offset_ms)?;
576
+ validate_replay_candidate(
577
+ &segment,
578
+ slice_overlaps_graph_gap(&app, &segment, started_ms, ended_ms),
579
+ )?;
580
+
581
+ let preview_dir = app
582
+ .path()
583
+ .app_local_data_dir()
584
+ .map_err(|error| format!("local preview directory unavailable: {error}"))?
585
+ .join("rewind-previews");
586
+ std::fs::create_dir_all(&preview_dir)
587
+ .map_err(|error| format!("local preview directory unavailable: {error}"))?;
588
+ cleanup_previews(&preview_dir);
589
+ let output = preview_dir.join(format!(
590
+ "moment-{}-{}-{}.mp4",
591
+ segment.id,
592
+ offset_ms,
593
+ Utc::now().timestamp_millis()
594
+ ));
595
+ let slice = NativeMediaSlice {
596
+ path: segment.path.clone(),
597
+ system_audio_path: segment.system_audio_path.clone(),
598
+ microphone_path: segment.microphone_path.clone(),
599
+ start_ms: started_ms,
600
+ end_ms: ended_ms,
601
+ };
602
+ let audio = if segment.capture_mode == RewindCaptureMode::VisualsAudio {
603
+ NativeAudioSelection::MixBoth
604
+ } else {
605
+ NativeAudioSelection::None
606
+ };
607
+ let materialized = output.clone();
608
+ tauri::async_runtime::spawn_blocking(move || {
609
+ native_screen::materialize_mp4_slices_exact(&[slice], &materialized, audio)
610
+ })
611
+ .await
612
+ .map_err(|error| format!("local replay worker failed: {error}"))??;
613
+
614
+ #[cfg(target_os = "macos")]
615
+ Command::new("/usr/bin/open")
616
+ .arg(&output)
617
+ .stdin(Stdio::null())
618
+ .stdout(Stdio::null())
619
+ .stderr(Stdio::null())
620
+ .spawn()
621
+ .map_err(|error| format!("could not open local replay: {error}"))?;
622
+ #[cfg(not(target_os = "macos"))]
623
+ return Err("Local Rewind replay is currently available on macOS.".into());
624
+
625
+ Ok(RewindReplayResult {
626
+ opened: true,
627
+ segment_id,
628
+ offset_ms,
629
+ preview_started_ms: started_ms,
630
+ preview_ended_ms: ended_ms,
631
+ })
632
+ }
633
+
634
+ #[cfg(test)]
635
+ mod tests {
636
+ use super::*;
637
+ use std::fs;
638
+ use std::sync::atomic::{AtomicU64, Ordering};
639
+
640
+ static TEST_COUNTER: AtomicU64 = AtomicU64::new(0);
641
+
642
+ fn test_dir(label: &str) -> PathBuf {
643
+ let path = std::env::temp_dir().join(format!(
644
+ "clips-rewind-local-{label}-{}-{}",
645
+ std::process::id(),
646
+ TEST_COUNTER.fetch_add(1, Ordering::Relaxed)
647
+ ));
648
+ fs::create_dir_all(&path).unwrap();
649
+ path
650
+ }
651
+
652
+ fn segment(directory: &Path, id: &str, tainted: bool) -> ScreenMemorySegmentMetadata {
653
+ let path = directory.join(format!("{id}.mp4"));
654
+ fs::write(&path, b"media").unwrap();
655
+ ScreenMemorySegmentMetadata {
656
+ id: id.into(),
657
+ path,
658
+ file_name: format!("{id}.mp4"),
659
+ mime_type: "video/mp4".into(),
660
+ started_at: "2026-07-14T12:00:00Z".into(),
661
+ ended_at: "2026-07-14T12:05:00Z".into(),
662
+ duration_ms: 300_000,
663
+ width: Some(1920),
664
+ height: Some(1080),
665
+ bytes: 5,
666
+ system_audio_path: None,
667
+ microphone_path: None,
668
+ corrupt: false,
669
+ error: None,
670
+ capture_mode: RewindCaptureMode::Visuals,
671
+ exclusion_tainted: tainted,
672
+ graph_epoch_id: Some("test-epoch".into()),
673
+ graph_started_elapsed_ms: 1_000,
674
+ graph_ended_elapsed_ms: 301_000,
675
+ }
676
+ }
677
+
678
+ #[test]
679
+ fn query_matches_all_local_sources_and_stays_bounded() {
680
+ let directory = test_dir("query");
681
+ let segment = segment(&directory, "segment-one", false);
682
+ fs::write(
683
+ directory.join("events.jsonl"),
684
+ r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Zoom","windowTitle":"Lilian audience review","bundleId":"us.zoom.xos","source":"accessibility"}
685
+ "#,
686
+ )
687
+ .unwrap();
688
+ fs::write(
689
+ directory.join("segment-one.transcript-status.json"),
690
+ r#"{"state":"ready"}"#,
691
+ )
692
+ .unwrap();
693
+ fs::write(
694
+ directory.join("segment-one.transcript.jsonl"),
695
+ r#"{"schemaVersion":1,"segmentId":"segment-one","source":"microphone","capturedAt":"2026-07-14T12:00:20Z","startMs":20000,"endMs":22000,"text":"Lilian explained the audience"}
696
+ "#,
697
+ )
698
+ .unwrap();
699
+ fs::write(
700
+ directory.join("segment-one.ocr-status.json"),
701
+ r#"{"state":"ready"}"#,
702
+ )
703
+ .unwrap();
704
+ fs::write(
705
+ directory.join("segment-one.ocr.jsonl"),
706
+ r#"{"schemaVersion":1,"segmentId":"segment-one","capturedAt":"2026-07-14T12:00:30Z","offsetMs":30000,"source":"ocr","ocrText":"Audience segments by Lilian","confidence":0.91,"frameWidth":1920,"frameHeight":1080}
707
+ "#,
708
+ )
709
+ .unwrap();
710
+
711
+ let result = local_query(&directory, vec![segment], "Lilian audience", 2, vec![]);
712
+ assert_eq!(result.evidence.len(), 2);
713
+ assert!(result.truncated);
714
+ assert_eq!(result.coverage.transcript_indexes_ready, 1);
715
+ assert_eq!(result.coverage.ocr_indexes_ready, 1);
716
+ fs::remove_dir_all(directory).unwrap();
717
+ }
718
+
719
+ #[test]
720
+ fn renderer_result_has_no_archive_path_field() {
721
+ let directory = test_dir("path-denial");
722
+ let segment = segment(&directory, "secret-segment", false);
723
+ fs::write(
724
+ directory.join("events.jsonl"),
725
+ r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Zoom","windowTitle":"needle","bundleId":null,"source":"accessibility"}
726
+ "#,
727
+ )
728
+ .unwrap();
729
+ let result = local_query(&directory, vec![segment], "needle", 5, vec![]);
730
+ let json = serde_json::to_value(result).unwrap();
731
+ assert!(json.get("path").is_none());
732
+ assert!(json["evidence"][0].get("path").is_none());
733
+ assert_eq!(json["evidence"][0]["segmentId"], "secret-segment");
734
+ fs::remove_dir_all(directory).unwrap();
735
+ }
736
+
737
+ #[test]
738
+ fn tainted_segments_are_excluded_and_report_no_match() {
739
+ let directory = test_dir("tainted");
740
+ let segment = segment(&directory, "tainted", true);
741
+ fs::write(
742
+ directory.join("events.jsonl"),
743
+ r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Secrets","windowTitle":"needle","bundleId":null,"source":"accessibility"}
744
+ "#,
745
+ )
746
+ .unwrap();
747
+ let result = local_query(&directory, vec![segment], "needle", 5, vec![]);
748
+ assert!(result.evidence.is_empty());
749
+ assert_eq!(result.coverage.segments_considered, 0);
750
+ fs::remove_dir_all(directory).unwrap();
751
+ }
752
+
753
+ #[test]
754
+ fn replay_slice_clamps_without_pre_range_leakage() {
755
+ assert_eq!(replay_slice(60_000, 20_000).unwrap(), (15_000, 30_000));
756
+ assert_eq!(replay_slice(60_000, 2_000).unwrap(), (0, 12_000));
757
+ assert_eq!(replay_slice(60_000, 58_000).unwrap(), (53_000, 60_000));
758
+ assert!(replay_slice(60_000, 60_000).is_err());
759
+ assert!(replay_slice(0, 0).is_err());
760
+ }
761
+
762
+ #[test]
763
+ fn replay_rejects_invalid_segment_ids_before_path_lookup() {
764
+ for id in ["../segment", "/tmp/archive.mp4", "a b", ""] {
765
+ assert!(
766
+ id.is_empty()
767
+ || id.len() > 255
768
+ || !id.chars().all(|character| character.is_ascii_alphanumeric()
769
+ || matches!(character, '-' | '_'))
770
+ );
771
+ }
772
+ }
773
+
774
+ #[test]
775
+ fn replay_rejects_tainted_and_gapped_evidence() {
776
+ let directory = test_dir("replay-rejection");
777
+ let tainted = segment(&directory, "tainted", true);
778
+ assert!(validate_replay_candidate(&tainted, false)
779
+ .unwrap_err()
780
+ .contains("privacy-tainted"));
781
+ let clean = segment(&directory, "clean", false);
782
+ assert!(validate_replay_candidate(&clean, true)
783
+ .unwrap_err()
784
+ .contains("capture gap"));
785
+ fs::remove_dir_all(directory).unwrap();
786
+ }
787
+
788
+ #[test]
789
+ fn query_terms_and_excerpts_are_bounded() {
790
+ let oversized = "x".repeat(MAX_EXCERPT_BYTES + 100);
791
+ let bounded = bounded_text(&oversized, MAX_EXCERPT_BYTES);
792
+ assert!(bounded.len() <= MAX_EXCERPT_BYTES + '…'.len_utf8());
793
+ assert_eq!(
794
+ normalized_terms("What did I do in the meeting?"),
795
+ vec!["meeting"]
796
+ );
797
+ let terms = normalized_terms("orchid submarine 731");
798
+ let ocr = "unrelated Plaud content\norchid subnlfv 731\nmore unrelated content";
799
+ assert!(match_score(ocr, &terms) > 0);
800
+ assert_eq!(matching_excerpt(ocr, &terms, MAX_EXCERPT_BYTES), ocr);
801
+ assert_eq!(match_score("unrelated item 731", &terms), 0);
802
+ }
803
+ }