@agent-native/core 0.114.6 → 0.114.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +15 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/cli/mcp-config-writers.ts +6 -4
  5. package/corpus/core/src/cli/mcp.ts +124 -0
  6. package/corpus/core/src/cli/skills-content/help.ts +4 -3
  7. package/corpus/core/src/cli/skills-content/index.ts +1 -0
  8. package/corpus/core/src/cli/skills-content/rewind-skill.ts +52 -0
  9. package/corpus/core/src/cli/skills.ts +115 -13
  10. package/corpus/core/src/mcp/screen-memory-stdio.ts +1360 -104
  11. package/corpus/core/src/mcp-client/index.ts +9 -0
  12. package/corpus/core/src/mcp-client/screen-memory-local.ts +847 -13
  13. package/corpus/core/src/server/agent-chat-plugin.ts +16 -9
  14. package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +19 -7
  15. package/corpus/templates/analytics/app/hooks/use-chart-tooltip-portal.ts +60 -0
  16. package/corpus/templates/analytics/app/routes/chart.tsx +1 -1
  17. package/corpus/templates/analytics/changelog/2026-07-20-dashboard-requests-no-longer-stall-without-progress.md +6 -0
  18. package/corpus/templates/analytics/server/plugins/agent-chat.ts +6 -0
  19. package/corpus/templates/clips/actions/apply-rewind-extension.ts +195 -0
  20. package/corpus/templates/clips/actions/delete-agent-recording-if-unpromoted.ts +64 -0
  21. package/corpus/templates/clips/actions/get-rewind-extension-request.ts +22 -0
  22. package/corpus/templates/clips/actions/get-screen-memory-status.ts +18 -2
  23. package/corpus/templates/clips/actions/list-rewind-extension-requests.ts +52 -0
  24. package/corpus/templates/clips/actions/make-recording-private-for-rewind.ts +55 -0
  25. package/corpus/templates/clips/actions/query-screen-memory-context.ts +4 -4
  26. package/corpus/templates/clips/actions/request-rewind-extension.ts +81 -0
  27. package/corpus/templates/clips/actions/update-rewind-extension-request.ts +47 -0
  28. package/corpus/templates/clips/app/components/editor/editor-layout.tsx +25 -0
  29. package/corpus/templates/clips/app/components/editor/editor-toolbar.tsx +36 -0
  30. package/corpus/templates/clips/app/components/editor/rewind-extension-dialog.tsx +338 -0
  31. package/corpus/templates/clips/app/components/editor/stitch-manager.tsx +1 -16
  32. package/corpus/templates/clips/app/components/editor/timeline.tsx +26 -0
  33. package/corpus/templates/clips/app/i18n/en-US.ts +18 -0
  34. package/corpus/templates/clips/app/lib/ffmpeg-export.ts +19 -12
  35. package/corpus/templates/clips/app/lib/rewind-visibility.ts +31 -0
  36. package/corpus/templates/clips/app/lib/timestamp-mapping.ts +7 -0
  37. package/corpus/templates/clips/app/lib/upload-file-client.ts +17 -0
  38. package/corpus/templates/clips/changelog/2026-07-14-rewind-keeps-recent-screen-context-local-and-turns-it-into-clips.md +6 -0
  39. package/corpus/templates/clips/changelog/2026-07-17-active-clips-keep-rewind-stable-and-settings-clear.md +12 -0
  40. package/corpus/templates/clips/changelog/2026-07-17-clicking-the-clips-icon-during-a-recording-now-opens-the-app.md +6 -0
  41. package/corpus/templates/clips/changelog/2026-07-17-default-mic-now-uses-the-mac-default-and-transcription-shares-one-capture.md +6 -0
  42. package/corpus/templates/clips/changelog/2026-07-17-full-screen-clips-finalize-even-when-screencapturekit-stop-hangs.md +6 -0
  43. package/corpus/templates/clips/changelog/2026-07-17-normal-full-screen-clips-no-longer-stop-immediately-after-th.md +6 -0
  44. package/corpus/templates/clips/changelog/2026-07-17-rewind-can-hand-one-bounded-private-clip-to-your-agent.md +12 -0
  45. package/corpus/templates/clips/changelog/2026-07-17-rewind-home-can-copy-a-local-agent-prompt.md +6 -0
  46. package/corpus/templates/clips/changelog/2026-07-17-rewind-now-has-a-calm-home-status-focused-first-run-consent-.md +6 -0
  47. package/corpus/templates/clips/changelog/2026-07-17-rewind-settings-are-smaller-clearer-and-agent-first.md +6 -0
  48. package/corpus/templates/clips/changelog/2026-07-17-the-full-clips-interface-stays-available-while-recording.md +6 -0
  49. package/corpus/templates/clips/changelog/2026-07-18-rewind-organizes-recent-work-into-local-searchable-chapters.md +8 -0
  50. package/corpus/templates/clips/changelog/2026-07-18-searchable-rewind-chapters.md +6 -0
  51. package/corpus/templates/clips/changelog/2026-07-19-rewind-backed-recordings-now-start-immediately-after-the-cou.md +6 -0
  52. package/corpus/templates/clips/changelog/2026-07-20-rewind-can-set-up-your-local-agent-once-so-future-requests-w.md +6 -0
  53. package/corpus/templates/clips/desktop/package.json +1 -0
  54. package/corpus/templates/clips/desktop/scripts/sign-macos-local.ts +12 -1
  55. package/corpus/templates/clips/desktop/src/app.tsx +2498 -239
  56. package/corpus/templates/clips/desktop/src/components/ReadinessPanel.tsx +21 -1
  57. package/corpus/templates/clips/desktop/src/components/Switch.tsx +3 -0
  58. package/corpus/templates/clips/desktop/src/hooks/useMeetingTranscription.ts +156 -24
  59. package/corpus/templates/clips/desktop/src/lib/native-recording-warm.ts +6 -7
  60. package/corpus/templates/clips/desktop/src/lib/recorder.ts +1763 -1160
  61. package/corpus/templates/clips/desktop/src/lib/rewind-agent-prompt.ts +9 -0
  62. package/corpus/templates/clips/desktop/src/lib/rewind-recording-start.ts +20 -0
  63. package/corpus/templates/clips/desktop/src/main.tsx +3 -0
  64. package/corpus/templates/clips/desktop/src/overlays/countdown.tsx +14 -14
  65. package/corpus/templates/clips/desktop/src/overlays/onboarding.tsx +4 -4
  66. package/corpus/templates/clips/desktop/src/overlays/preparing.tsx +13 -0
  67. package/corpus/templates/clips/desktop/src/overlays/toolbar.tsx +27 -2
  68. package/corpus/templates/clips/desktop/src/shared/config.ts +16 -0
  69. package/corpus/templates/clips/desktop/src/styles.css +956 -134
  70. package/corpus/templates/clips/desktop/src-tauri/Cargo.toml +5 -0
  71. package/corpus/templates/clips/desktop/src-tauri/build.rs +69 -1
  72. package/corpus/templates/clips/desktop/src-tauri/src/accessibility.rs +386 -12
  73. package/corpus/templates/clips/desktop/src-tauri/src/capture_audio_bus.rs +382 -0
  74. package/corpus/templates/clips/desktop/src-tauri/src/capture_graph.rs +777 -0
  75. package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +249 -22
  76. package/corpus/templates/clips/desktop/src-tauri/src/config.rs +148 -1
  77. package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +58 -4
  78. package/corpus/templates/clips/desktop/src-tauri/src/logfile.rs +13 -1
  79. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +1505 -51
  80. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +124 -0
  81. package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +1006 -95
  82. package/corpus/templates/clips/desktop/src-tauri/src/rewind_capture_suspension.rs +165 -0
  83. package/corpus/templates/clips/desktop/src-tauri/src/rewind_chapters.rs +2434 -0
  84. package/corpus/templates/clips/desktop/src-tauri/src/rewind_clip.rs +1650 -0
  85. package/corpus/templates/clips/desktop/src-tauri/src/rewind_egress.rs +643 -0
  86. package/corpus/templates/clips/desktop/src-tauri/src/rewind_local_ask.rs +803 -0
  87. package/corpus/templates/clips/desktop/src-tauri/src/rewind_meeting_history.rs +595 -0
  88. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory.rs +2900 -91
  89. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory_ocr.rs +360 -0
  90. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory_transcript.rs +408 -0
  91. package/corpus/templates/clips/desktop/src-tauri/src/shortcuts.rs +114 -33
  92. package/corpus/templates/clips/desktop/src-tauri/src/state.rs +2 -2
  93. package/corpus/templates/clips/desktop/src-tauri/src/system_audio.rs +13 -3
  94. package/corpus/templates/clips/desktop/src-tauri/src/tray.rs +4 -1
  95. package/corpus/templates/clips/desktop/src-tauri/src/util.rs +7 -13
  96. package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +198 -11
  97. package/corpus/templates/clips/desktop/src-tauri/tauri.alpha.conf.json +17 -0
  98. package/corpus/toolkit/CHANGELOG.md +6 -0
  99. package/corpus/toolkit/package.json +1 -1
  100. package/dist/cli/mcp-config-writers.d.ts +2 -2
  101. package/dist/cli/mcp-config-writers.d.ts.map +1 -1
  102. package/dist/cli/mcp-config-writers.js +6 -6
  103. package/dist/cli/mcp-config-writers.js.map +1 -1
  104. package/dist/cli/mcp.d.ts +14 -0
  105. package/dist/cli/mcp.d.ts.map +1 -1
  106. package/dist/cli/mcp.js +79 -0
  107. package/dist/cli/mcp.js.map +1 -1
  108. package/dist/cli/skills-content/help.d.ts +1 -1
  109. package/dist/cli/skills-content/help.d.ts.map +1 -1
  110. package/dist/cli/skills-content/help.js +4 -3
  111. package/dist/cli/skills-content/help.js.map +1 -1
  112. package/dist/cli/skills-content/index.d.ts +1 -0
  113. package/dist/cli/skills-content/index.d.ts.map +1 -1
  114. package/dist/cli/skills-content/index.js +1 -0
  115. package/dist/cli/skills-content/index.js.map +1 -1
  116. package/dist/cli/skills-content/rewind-skill.d.ts +2 -0
  117. package/dist/cli/skills-content/rewind-skill.d.ts.map +1 -0
  118. package/dist/cli/skills-content/rewind-skill.js +53 -0
  119. package/dist/cli/skills-content/rewind-skill.js.map +1 -0
  120. package/dist/cli/skills.d.ts +6 -0
  121. package/dist/cli/skills.d.ts.map +1 -1
  122. package/dist/cli/skills.js +90 -14
  123. package/dist/cli/skills.js.map +1 -1
  124. package/dist/collab/struct-routes.d.ts +1 -1
  125. package/dist/mcp/screen-memory-stdio.d.ts +365 -0
  126. package/dist/mcp/screen-memory-stdio.d.ts.map +1 -1
  127. package/dist/mcp/screen-memory-stdio.js +1014 -91
  128. package/dist/mcp/screen-memory-stdio.js.map +1 -1
  129. package/dist/mcp-client/index.d.ts +1 -1
  130. package/dist/mcp-client/index.d.ts.map +1 -1
  131. package/dist/mcp-client/index.js +1 -1
  132. package/dist/mcp-client/index.js.map +1 -1
  133. package/dist/mcp-client/screen-memory-local.d.ts +90 -0
  134. package/dist/mcp-client/screen-memory-local.d.ts.map +1 -1
  135. package/dist/mcp-client/screen-memory-local.js +590 -12
  136. package/dist/mcp-client/screen-memory-local.js.map +1 -1
  137. package/dist/observability/routes.d.ts +1 -1
  138. package/dist/progress/routes.d.ts +1 -1
  139. package/dist/provider-api/actions/provider-api.d.ts +4 -4
  140. package/dist/resources/handlers.d.ts +1 -1
  141. package/dist/server/agent-chat-plugin.d.ts +5 -0
  142. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  143. package/dist/server/agent-chat-plugin.js +10 -9
  144. package/dist/server/agent-chat-plugin.js.map +1 -1
  145. package/package.json +2 -2
  146. package/src/cli/mcp-config-writers.ts +6 -4
  147. package/src/cli/mcp.ts +124 -0
  148. package/src/cli/skills-content/help.ts +4 -3
  149. package/src/cli/skills-content/index.ts +1 -0
  150. package/src/cli/skills-content/rewind-skill.ts +52 -0
  151. package/src/cli/skills.ts +115 -13
  152. package/src/mcp/screen-memory-stdio.ts +1360 -104
  153. package/src/mcp-client/index.ts +9 -0
  154. package/src/mcp-client/screen-memory-local.ts +847 -13
  155. package/src/server/agent-chat-plugin.ts +16 -9
@@ -1,10 +1,21 @@
1
1
  const DEFAULT_CONFIG = {
2
2
  enabled: false,
3
3
  paused: false,
4
- retentionHours: 24,
4
+ retentionHours: 8,
5
5
  maxBytes: 20 * 1024 * 1024 * 1024,
6
6
  segmentSeconds: 5 * 60,
7
7
  sampleIntervalSeconds: 10,
8
+ captureMode: "visuals",
9
+ reviewBeforeSending: true,
10
+ agentClipRetention: "forever",
11
+ excludedBundleIds: [
12
+ "com.1password.1password",
13
+ "com.agilebits.onepassword7",
14
+ "com.bitwarden.desktop",
15
+ "com.dashlane.dashlane",
16
+ "com.lastpass.lastpass",
17
+ ],
18
+ excludePrivateWindows: false,
8
19
  };
9
20
  const JSONL_NAMES = [
10
21
  "context.jsonl",
@@ -12,6 +23,9 @@ const JSONL_NAMES = [
12
23
  "snapshots.jsonl",
13
24
  "screen-memory.jsonl",
14
25
  ];
26
+ const MAX_SOURCE_ROWS = 10_000;
27
+ const MAX_EXCERPT_CHARS = 1_200;
28
+ const MAX_TRANSCRIPT_JOIN_GAP_MS = 2_000;
15
29
  function asRecord(value) {
16
30
  return value && typeof value === "object" && !Array.isArray(value)
17
31
  ? value
@@ -24,15 +38,39 @@ function positiveNumber(value, fallback) {
24
38
  function booleanValue(value, fallback) {
25
39
  return typeof value === "boolean" ? value : fallback;
26
40
  }
41
+ function enumValue(value, allowed, fallback) {
42
+ return typeof value === "string" && allowed.includes(value)
43
+ ? value
44
+ : fallback;
45
+ }
46
+ function bundleIds(value, fallback) {
47
+ if (!Array.isArray(value))
48
+ return [...fallback];
49
+ return [
50
+ ...new Set(value
51
+ .filter((item) => typeof item === "string")
52
+ .map((item) => item.trim().toLowerCase())
53
+ .filter(Boolean)),
54
+ ].sort();
55
+ }
27
56
  function normalizeConfig(value) {
28
57
  const raw = asRecord(value) ?? {};
29
58
  return {
30
59
  enabled: booleanValue(raw.enabled, DEFAULT_CONFIG.enabled),
31
60
  paused: booleanValue(raw.paused, DEFAULT_CONFIG.paused),
32
- retentionHours: positiveNumber(raw.retentionHours, DEFAULT_CONFIG.retentionHours),
61
+ retentionHours: Math.min(positiveNumber(raw.retentionHours, DEFAULT_CONFIG.retentionHours), 24),
33
62
  maxBytes: positiveNumber(raw.maxBytes, DEFAULT_CONFIG.maxBytes),
34
63
  segmentSeconds: positiveNumber(raw.segmentSeconds, DEFAULT_CONFIG.segmentSeconds),
35
64
  sampleIntervalSeconds: positiveNumber(raw.sampleIntervalSeconds, DEFAULT_CONFIG.sampleIntervalSeconds),
65
+ captureMode: enumValue(raw.captureMode, ["visuals", "visuals-audio"], DEFAULT_CONFIG.captureMode),
66
+ reviewBeforeSending: typeof raw.reviewBeforeSending === "boolean"
67
+ ? raw.reviewBeforeSending
68
+ : DEFAULT_CONFIG.reviewBeforeSending,
69
+ agentClipRetention: enumValue(raw.agentClipRetention, ["forever", "24-hours", "7-days", "30-days"], DEFAULT_CONFIG.agentClipRetention),
70
+ excludedBundleIds: bundleIds(raw.excludedBundleIds, DEFAULT_CONFIG.excludedBundleIds),
71
+ excludePrivateWindows: typeof raw.excludePrivateWindows === "boolean"
72
+ ? raw.excludePrivateWindows
73
+ : DEFAULT_CONFIG.excludePrivateWindows,
36
74
  };
37
75
  }
38
76
  async function nodeModules() {
@@ -185,6 +223,20 @@ async function contextFilesFor(dataDirs) {
185
223
  // ignore missing candidate files
186
224
  }
187
225
  }
226
+ try {
227
+ const names = await fs.readdir(dir);
228
+ for (const name of names.filter((entry) => entry.endsWith(".ocr.jsonl") || entry.endsWith(".transcript.jsonl"))) {
229
+ const candidate = path.join(dir, name);
230
+ const stat = await fs.stat(candidate);
231
+ if (stat.isFile()) {
232
+ files.push(candidate);
233
+ storageBytes += stat.size;
234
+ }
235
+ }
236
+ }
237
+ catch {
238
+ // ignore unavailable local directories
239
+ }
188
240
  }
189
241
  return { files, storageBytes };
190
242
  }
@@ -196,6 +248,22 @@ function firstString(raw, keys) {
196
248
  }
197
249
  return null;
198
250
  }
251
+ function firstFiniteNumber(raw, keys) {
252
+ for (const key of keys) {
253
+ const value = raw[key];
254
+ if (typeof value === "number" && Number.isFinite(value))
255
+ return value;
256
+ }
257
+ return null;
258
+ }
259
+ function firstBoolean(raw, keys) {
260
+ for (const key of keys) {
261
+ const value = raw[key];
262
+ if (typeof value === "boolean")
263
+ return value;
264
+ }
265
+ return null;
266
+ }
199
267
  function normalizeContextItem(value, sourceFile) {
200
268
  const raw = asRecord(value);
201
269
  if (!raw)
@@ -231,9 +299,228 @@ function itemTime(item) {
231
299
  const parsed = Date.parse(item.capturedAt);
232
300
  return Number.isFinite(parsed) ? parsed : 0;
233
301
  }
234
- async function readItems(files) {
302
+ function isPrivacyExcludedRow(row, config) {
303
+ const bundleId = row.item.bundleId?.trim().toLowerCase();
304
+ if (bundleId &&
305
+ config.excludedBundleIds.some((excluded) => excluded.trim().toLowerCase() === bundleId)) {
306
+ return true;
307
+ }
308
+ if (!config.excludePrivateWindows)
309
+ return false;
310
+ if (firstBoolean(row.raw, [
311
+ "isPrivate",
312
+ "is_private",
313
+ "privateWindow",
314
+ "private_window",
315
+ "incognito",
316
+ ]) === true) {
317
+ return true;
318
+ }
319
+ const title = row.item.windowTitle?.toLowerCase() ?? "";
320
+ return ["incognito", "inprivate", "private browsing", "private window"].some((marker) => title.includes(marker));
321
+ }
322
+ function rowTimeRange(rows, fallback) {
323
+ const times = rows
324
+ .map(({ item }) => itemTime(item))
325
+ .filter((time) => time > 0);
326
+ if (times.length === 0)
327
+ return fallback;
328
+ return {
329
+ startedAt: new Date(Math.min(...times)).toISOString(),
330
+ endedAt: new Date(Math.max(...times)).toISOString(),
331
+ };
332
+ }
333
+ function stableId(prefix, value) {
334
+ let hash = 0x811c9dc5;
335
+ for (let index = 0; index < value.length; index += 1) {
336
+ hash ^= value.charCodeAt(index);
337
+ hash = Math.imul(hash, 0x01000193);
338
+ }
339
+ return `${prefix}-${(hash >>> 0).toString(36)}`;
340
+ }
341
+ function boundedExcerpt(value) {
342
+ const normalized = value.replace(/\s+/g, " ").trim();
343
+ if (normalized.length <= MAX_EXCERPT_CHARS) {
344
+ return { excerpt: normalized, excerptTruncated: false };
345
+ }
346
+ return {
347
+ excerpt: `${normalized.slice(0, MAX_EXCERPT_CHARS - 1)}…`,
348
+ excerptTruncated: true,
349
+ };
350
+ }
351
+ function redactCredentialText(value) {
352
+ return value
353
+ .replace(/\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9]{12,}|AKIA[A-Z0-9]{16})\b/g, "[REDACTED CREDENTIAL]")
354
+ .replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED]")
355
+ .replace(/\b(api[_-]?key|access[_-]?token|password|secret)\s*[:=]\s*([^\s,;]{4,})/gi, "$1=[REDACTED]");
356
+ }
357
+ function sourceTexts(raw, fallback) {
358
+ const localTranscriptText = firstString(raw, ["text"]) &&
359
+ firstString(raw, ["segmentId", "segment_id"]) &&
360
+ (typeof raw.startMs === "number" || typeof raw.start_ms === "number")
361
+ ? firstString(raw, ["text"])
362
+ : null;
363
+ const values = [
364
+ { sourceType: "transcript", keys: ["transcript", "transcriptText"] },
365
+ { sourceType: "ocr", keys: ["ocrText", "visibleText", "ocr"] },
366
+ ];
367
+ const evidence = values.flatMap(({ sourceType, keys }) => {
368
+ const text = firstString(raw, keys);
369
+ return text ? [{ sourceType, text }] : [];
370
+ });
371
+ if (localTranscriptText) {
372
+ evidence.unshift({
373
+ sourceType: "transcript",
374
+ text: localTranscriptText,
375
+ });
376
+ }
377
+ return evidence.length > 0
378
+ ? evidence
379
+ : fallback
380
+ ? [{ sourceType: "app-context", text: fallback }]
381
+ : [];
382
+ }
383
+ function segmentReferences(raw, capturedAt, segments) {
384
+ const directId = firstString(raw, ["segmentId", "segment_id"]);
385
+ const matched = directId
386
+ ? segments.filter((segment) => segment.id === directId)
387
+ : capturedAt
388
+ ? segments.filter((segment) => {
389
+ const moment = Date.parse(capturedAt);
390
+ const started = segment.startedAt
391
+ ? Date.parse(segment.startedAt)
392
+ : NaN;
393
+ const ended = segment.endedAt ? Date.parse(segment.endedAt) : NaN;
394
+ return moment >= started && moment <= ended;
395
+ })
396
+ : [];
397
+ return matched.slice(0, 3).map((segment) => ({
398
+ id: segment.id,
399
+ startedAt: segment.startedAt,
400
+ endedAt: segment.endedAt,
401
+ }));
402
+ }
403
+ function normalizeEvidence(row, segments) {
404
+ const refs = segmentReferences(row.raw, row.item.capturedAt, segments);
405
+ const base = [
406
+ row.item.sourceFile,
407
+ row.item.capturedAt ?? "unknown-time",
408
+ row.item.appName ?? "",
409
+ row.item.windowTitle ?? "",
410
+ ].join("|");
411
+ const momentId = firstString(row.raw, ["momentId", "moment_id", "eventId", "captureId"]) ??
412
+ stableId("moment", base);
413
+ const appContext = row.item.text ||
414
+ [row.item.appName, row.item.windowTitle, row.item.title, row.item.url]
415
+ .filter((value) => Boolean(value))
416
+ .join(" — ");
417
+ return sourceTexts(row.raw, appContext).map(({ sourceType, text }) => {
418
+ const { excerpt, excerptTruncated } = boundedExcerpt(text);
419
+ const id = firstString(row.raw, ["evidenceId", "evidence_id"]) ??
420
+ stableId("evidence", `${momentId}|${sourceType}|${text}`);
421
+ return {
422
+ id,
423
+ momentId,
424
+ capturedAt: row.item.capturedAt,
425
+ sourceType,
426
+ excerpt,
427
+ excerptTruncated,
428
+ appName: row.item.appName,
429
+ windowTitle: row.item.windowTitle,
430
+ bundleId: row.item.bundleId,
431
+ url: row.item.url,
432
+ title: row.item.title,
433
+ segmentRefs: refs,
434
+ jumpTarget: {
435
+ kind: "screen-memory-moment",
436
+ momentId,
437
+ capturedAt: row.item.capturedAt,
438
+ segmentId: refs[0]?.id ?? null,
439
+ },
440
+ };
441
+ });
442
+ }
443
+ function transcriptSpan(row) {
444
+ const segmentId = firstString(row.raw, ["segmentId", "segment_id"]);
445
+ const source = firstString(row.raw, ["source", "kind"]);
446
+ const startMs = firstFiniteNumber(row.raw, ["startMs", "start_ms"]);
447
+ const endMs = firstFiniteNumber(row.raw, ["endMs", "end_ms"]);
448
+ if (!segmentId ||
449
+ !source ||
450
+ startMs === null ||
451
+ endMs === null ||
452
+ endMs <= startMs ||
453
+ !firstString(row.raw, ["text"])) {
454
+ return null;
455
+ }
456
+ return { row, segmentId, source, startMs, endMs };
457
+ }
458
+ /**
459
+ * Whisper emits phrase-sized rows, which can split an ordinary sentence in
460
+ * the middle. Join only rows that are demonstrably continuous in the same
461
+ * finalized segment and audio source. Stored sidecars remain untouched, and a
462
+ * real pause, source switch, segment boundary, or excerpt bound starts a new
463
+ * evidence item.
464
+ */
465
+ function coalesceTranscriptRows(rows) {
466
+ const passthrough = [];
467
+ const groups = new Map();
468
+ for (const row of rows) {
469
+ const span = transcriptSpan(row);
470
+ if (!span) {
471
+ passthrough.push(row);
472
+ continue;
473
+ }
474
+ const key = [row.item.sourceFile, span.segmentId, span.source].join("\0");
475
+ const group = groups.get(key) ?? [];
476
+ group.push(span);
477
+ groups.set(key, group);
478
+ }
479
+ const joined = [...groups.values()].flatMap((group) => {
480
+ const ordered = group.sort((a, b) => a.startMs - b.startMs);
481
+ const output = [];
482
+ let current = null;
483
+ for (const next of ordered) {
484
+ const currentText = current
485
+ ? (firstString(current.row.raw, ["text"]) ?? "")
486
+ : "";
487
+ const nextText = firstString(next.row.raw, ["text"]) ?? "";
488
+ const gapMs = current ? next.startMs - current.endMs : Infinity;
489
+ const combinedText = [currentText, nextText].filter(Boolean).join(" ");
490
+ const canJoin = current !== null &&
491
+ gapMs >= 0 &&
492
+ gapMs <= MAX_TRANSCRIPT_JOIN_GAP_MS &&
493
+ combinedText.length <= MAX_EXCERPT_CHARS;
494
+ if (!current || !canJoin) {
495
+ if (current)
496
+ output.push(current.row);
497
+ current = {
498
+ ...next,
499
+ row: {
500
+ item: { ...next.row.item },
501
+ raw: { ...next.row.raw },
502
+ },
503
+ };
504
+ continue;
505
+ }
506
+ current.endMs = next.endMs;
507
+ current.row.item.text = combinedText;
508
+ current.row.raw.text = combinedText;
509
+ if ("end_ms" in current.row.raw)
510
+ current.row.raw.end_ms = next.endMs;
511
+ else
512
+ current.row.raw.endMs = next.endMs;
513
+ }
514
+ if (current)
515
+ output.push(current.row);
516
+ return output;
517
+ });
518
+ return [...passthrough, ...joined].sort((a, b) => itemTime(b.item) - itemTime(a.item));
519
+ }
520
+ async function readRows(files) {
235
521
  const { fs } = await nodeModules();
236
- const items = [];
522
+ const rows = [];
523
+ let sourceRowsReadTruncated = false;
237
524
  for (const file of files) {
238
525
  let text = "";
239
526
  try {
@@ -242,19 +529,96 @@ async function readItems(files) {
242
529
  catch {
243
530
  continue;
244
531
  }
245
- const lines = text.split(/\r?\n/).filter(Boolean).slice(-10_000);
532
+ const allLines = text.split(/\r?\n/).filter(Boolean);
533
+ if (allLines.length > MAX_SOURCE_ROWS)
534
+ sourceRowsReadTruncated = true;
535
+ const lines = allLines.slice(-MAX_SOURCE_ROWS);
246
536
  for (const line of lines) {
247
537
  try {
248
- const item = normalizeContextItem(JSON.parse(line), file);
249
- if (item)
250
- items.push(item);
538
+ const raw = asRecord(JSON.parse(line));
539
+ const item = raw ? normalizeContextItem(raw, file) : null;
540
+ if (item && raw)
541
+ rows.push({ item, raw });
251
542
  }
252
543
  catch {
253
544
  // Keep one malformed row from hiding the rest of the local context.
254
545
  }
255
546
  }
256
547
  }
257
- return items.sort((a, b) => itemTime(b) - itemTime(a));
548
+ return {
549
+ rows: coalesceTranscriptRows(rows),
550
+ sourceRowsReadTruncated,
551
+ };
552
+ }
553
+ async function readItems(files) {
554
+ const { rows } = await readRows(files);
555
+ return rows.map(({ item }) => item);
556
+ }
557
+ async function readSegments(dataDirs) {
558
+ const { fs, path } = await nodeModules();
559
+ const segments = [];
560
+ for (const dir of dataDirs) {
561
+ let names = [];
562
+ try {
563
+ names = await fs.readdir(dir);
564
+ }
565
+ catch {
566
+ continue;
567
+ }
568
+ for (const name of names.filter((candidate) => candidate.endsWith(".json"))) {
569
+ const raw = asRecord(await readJson(path.join(dir, name)));
570
+ const id = raw && firstString(raw, ["id"]);
571
+ if (!raw || !id)
572
+ continue;
573
+ const mediaPath = firstString(raw, ["path"]);
574
+ let retained = true;
575
+ if (mediaPath) {
576
+ try {
577
+ await fs.access(mediaPath);
578
+ }
579
+ catch {
580
+ retained = false;
581
+ }
582
+ }
583
+ segments.push({
584
+ id,
585
+ startedAt: firstString(raw, ["startedAt"]),
586
+ endedAt: firstString(raw, ["endedAt"]),
587
+ clean: retained &&
588
+ raw.exclusionTainted !== true &&
589
+ raw.corrupt !== true &&
590
+ !firstString(raw, ["error"]),
591
+ });
592
+ }
593
+ }
594
+ return segments;
595
+ }
596
+ async function readOcrIndexStates(dataDirs) {
597
+ const { fs, path } = await nodeModules();
598
+ const states = [];
599
+ for (const dir of dataDirs) {
600
+ let names = [];
601
+ try {
602
+ names = await fs.readdir(dir);
603
+ }
604
+ catch {
605
+ continue;
606
+ }
607
+ for (const name of names.filter((entry) => entry.endsWith(".ocr-status.json") ||
608
+ entry.endsWith(".transcript-status.json"))) {
609
+ const raw = asRecord(await readJson(path.join(dir, name)));
610
+ const rawState = String(raw?.state);
611
+ const state = rawState === "transcribing" ? "indexing" : rawState;
612
+ const suffix = name.endsWith(".ocr-status.json")
613
+ ? ".ocr-status.json"
614
+ : ".transcript-status.json";
615
+ const segmentId = name.slice(0, -suffix.length);
616
+ if (["pending", "indexing", "ready", "failed", "skipped"].includes(String(state))) {
617
+ states.push({ segmentId, state: state });
618
+ }
619
+ }
620
+ }
621
+ return states;
258
622
  }
259
623
  export async function queryScreenMemoryContext(args = {}, options = {}) {
260
624
  const info = await readConfigInfo(options);
@@ -265,18 +629,118 @@ export async function queryScreenMemoryContext(args = {}, options = {}) {
265
629
  const sinceMinutes = typeof args.sinceMinutes === "number" && Number.isFinite(args.sinceMinutes)
266
630
  ? Math.max(args.sinceMinutes, 0)
267
631
  : null;
268
- const cutoff = sinceMinutes === null ? null : Date.now() - sinceMinutes * 60 * 1000;
632
+ const now = options.now?.() ?? new Date();
633
+ const cutoff = sinceMinutes === null ? null : now.getTime() - sinceMinutes * 60 * 1000;
269
634
  const needle = query?.toLowerCase() ?? null;
270
- const items = (await readItems(files)).filter((item) => {
635
+ const source = await readRows(files);
636
+ const rangeRows = source.rows.filter((row) => {
637
+ const { item } = row;
271
638
  if (cutoff !== null) {
272
639
  const time = itemTime(item);
273
640
  if (!time || time < cutoff)
274
641
  return false;
275
642
  }
643
+ return true;
644
+ });
645
+ const privacyExcludedRows = rangeRows.filter((row) => isPrivacyExcludedRow(row, info.config));
646
+ const candidateRows = rangeRows.filter((row) => {
647
+ if (isPrivacyExcludedRow(row, info.config))
648
+ return false;
276
649
  if (!needle)
277
650
  return true;
278
- return JSON.stringify(item).toLowerCase().includes(needle);
651
+ const { item } = row;
652
+ return JSON.stringify({ item, raw: row.raw })
653
+ .toLowerCase()
654
+ .includes(needle);
279
655
  });
656
+ const segments = await readSegments(paths.dataDirs);
657
+ const cleanSegments = segments.filter((segment) => segment.clean);
658
+ // Modern stores bind every evidence row to retained segment metadata. Once
659
+ // segment metadata exists, refuse rows that only point at tainted, corrupt,
660
+ // or pruned media. Legacy context-only stores (no segment metadata at all)
661
+ // remain readable for backwards compatibility.
662
+ const rows = segments.length === 0
663
+ ? candidateRows
664
+ : candidateRows.filter((row) => segmentReferences(row.raw, row.item.capturedAt, cleanSegments)
665
+ .length > 0);
666
+ const items = rows.map(({ item }) => item);
667
+ const ocrIndexStates = await readOcrIndexStates(paths.dataDirs);
668
+ const evidence = rows.flatMap((row) => normalizeEvidence(row, cleanSegments));
669
+ const returnedEvidence = evidence.slice(0, limit);
670
+ const evidenceTimes = evidence
671
+ .map((item) => (item.capturedAt ? Date.parse(item.capturedAt) : NaN))
672
+ .filter(Number.isFinite);
673
+ const coveredRange = evidenceTimes.length
674
+ ? {
675
+ startedAt: new Date(Math.min(...evidenceTimes)).toISOString(),
676
+ endedAt: new Date(Math.max(...evidenceTimes)).toISOString(),
677
+ }
678
+ : { startedAt: null, endedAt: null };
679
+ const requestedRange = {
680
+ startedAt: cutoff === null ? null : new Date(cutoff).toISOString(),
681
+ endedAt: now.toISOString(),
682
+ };
683
+ const gaps = [];
684
+ if (privacyExcludedRows.length > 0) {
685
+ gaps.push({
686
+ ...rowTimeRange(privacyExcludedRows, requestedRange),
687
+ reason: "privacy-excluded-or-unretained",
688
+ });
689
+ }
690
+ if (candidateRows.length > rows.length) {
691
+ gaps.push({
692
+ ...requestedRange,
693
+ reason: "privacy-excluded-or-unretained",
694
+ });
695
+ }
696
+ for (const index of ocrIndexStates) {
697
+ if (index.state === "ready")
698
+ continue;
699
+ const segment = segments.find((candidate) => candidate.id === index.segmentId);
700
+ const reason = index.state === "failed"
701
+ ? "index-failed"
702
+ : index.state === "skipped"
703
+ ? "index-skipped"
704
+ : "index-pending";
705
+ gaps.push({
706
+ startedAt: segment?.startedAt ?? null,
707
+ endedAt: segment?.endedAt ?? null,
708
+ reason,
709
+ });
710
+ }
711
+ if (files.length === 0) {
712
+ gaps.push({ ...requestedRange, reason: "no-context-files" });
713
+ }
714
+ else if (evidence.length === 0) {
715
+ gaps.push({ ...requestedRange, reason: "no-evidence-in-requested-range" });
716
+ }
717
+ else {
718
+ if (evidence.some((item) => !item.capturedAt)) {
719
+ gaps.push({
720
+ startedAt: null,
721
+ endedAt: null,
722
+ reason: "timestamps-unavailable",
723
+ });
724
+ }
725
+ if (requestedRange.startedAt &&
726
+ coveredRange.startedAt &&
727
+ coveredRange.startedAt > requestedRange.startedAt) {
728
+ gaps.push({
729
+ startedAt: requestedRange.startedAt,
730
+ endedAt: coveredRange.startedAt,
731
+ reason: "missing-before-first-evidence",
732
+ });
733
+ }
734
+ const staleAfterMs = Math.max(info.config.sampleIntervalSeconds * 3 * 1000, 2 * 60 * 1000);
735
+ if (coveredRange.endedAt &&
736
+ Date.parse(coveredRange.endedAt) < now.getTime() - staleAfterMs) {
737
+ gaps.push({
738
+ startedAt: coveredRange.endedAt,
739
+ endedAt: requestedRange.endedAt,
740
+ reason: "capture-stale",
741
+ });
742
+ }
743
+ }
280
744
  return {
281
745
  feature: "screen-memory",
282
746
  localOnly: true,
@@ -286,12 +750,126 @@ export async function queryScreenMemoryContext(args = {}, options = {}) {
286
750
  sinceMinutes,
287
751
  count: items.length,
288
752
  items: items.slice(0, limit),
753
+ evidence: returnedEvidence,
754
+ coverage: { requestedRange, coveredRange, gaps },
755
+ truncation: {
756
+ itemLimit: limit,
757
+ returnedItems: returnedEvidence.length,
758
+ omittedItems: Math.max(evidence.length - returnedEvidence.length, 0),
759
+ maxExcerptChars: MAX_EXCERPT_CHARS,
760
+ excerptsTruncated: returnedEvidence.filter((item) => item.excerptTruncated).length,
761
+ sourceRowsReadLimit: MAX_SOURCE_ROWS,
762
+ sourceRowsReadTruncated: source.sourceRowsReadTruncated,
763
+ },
289
764
  contextFiles: files,
290
765
  note: files.length === 0
291
766
  ? "No local Screen Memory context files were found. Enable Screen Memory in Clips desktop and keep the local MCP capability connected."
292
767
  : "Local Screen Memory context only. Do not treat this as shared, hosted, or exhaustive.",
293
768
  };
294
769
  }
770
+ /**
771
+ * Agent-facing retrieval boundary. Asking an agent to search Rewind is the
772
+ * authorization. This removes filesystem paths, redacts obvious
773
+ * credential-shaped text, and records a content-free activity receipt before
774
+ * it is returned to an action caller.
775
+ */
776
+ export async function queryScreenMemoryForAgent(args = {}, options = {}) {
777
+ const result = await queryScreenMemoryContext(args, options);
778
+ const items = result.items.map((item) => ({
779
+ ...item,
780
+ text: redactCredentialText(item.text),
781
+ sourceFile: "local-screen-memory",
782
+ }));
783
+ const evidence = result.evidence.map((item) => ({
784
+ ...item,
785
+ excerpt: redactCredentialText(item.excerpt),
786
+ }));
787
+ const packet = {
788
+ question: redactCredentialText(args.query?.trim().slice(0, 4_000) || "Recent Screen Memory context"),
789
+ evidence: evidence.slice(0, 20).map((item) => ({
790
+ id: item.id,
791
+ momentId: item.momentId,
792
+ sourceType: item.sourceType,
793
+ capturedAt: item.capturedAt,
794
+ excerpt: item.excerpt.slice(0, MAX_EXCERPT_CHARS),
795
+ })),
796
+ };
797
+ const paths = await resolvePaths(options);
798
+ const { fs, path } = await nodeModules();
799
+ const storeDir = paths.dataDirs[0];
800
+ await fs.mkdir(storeDir, { recursive: true, mode: 0o700 });
801
+ await fs.chmod(storeDir, 0o700);
802
+ const logPath = path.join(storeDir, "egress.jsonl");
803
+ if (await exists(logPath)) {
804
+ const sanitized = (await fs.readFile(logPath, "utf8"))
805
+ .split("\n")
806
+ .filter(Boolean)
807
+ .flatMap((line) => {
808
+ try {
809
+ const event = JSON.parse(line);
810
+ const packet = event.packet;
811
+ if (!event.receipt && Array.isArray(packet?.evidence)) {
812
+ event.receipt = {
813
+ evidence: packet.evidence.map((item) => ({
814
+ id: item.id,
815
+ momentId: item.momentId,
816
+ sourceType: item.sourceType,
817
+ capturedAt: item.capturedAt ?? null,
818
+ })),
819
+ };
820
+ }
821
+ delete event.packet;
822
+ delete event.reason;
823
+ event.packetBytes =
824
+ typeof event.packetBytes === "number" ? event.packetBytes : 0;
825
+ return [JSON.stringify(event)];
826
+ }
827
+ catch {
828
+ return [];
829
+ }
830
+ });
831
+ const temporaryPath = `${logPath}.sanitize-${process.pid}`;
832
+ await fs.writeFile(temporaryPath, sanitized.length > 0 ? `${sanitized.join("\n")}\n` : "", { encoding: "utf8", mode: 0o600 });
833
+ await fs.rename(temporaryPath, logPath);
834
+ }
835
+ const requestId = `egress-action-${Date.now()}-${process.pid}`;
836
+ const prepared = {
837
+ requestId,
838
+ occurredAt: new Date().toISOString(),
839
+ state: "prepared",
840
+ operation: "agent-query",
841
+ receipt: {
842
+ evidence: packet.evidence.map(({ id, momentId, sourceType, capturedAt }) => ({
843
+ id,
844
+ momentId,
845
+ sourceType,
846
+ capturedAt,
847
+ })),
848
+ },
849
+ evidenceCount: packet.evidence.length,
850
+ packetBytes: 0,
851
+ error: null,
852
+ };
853
+ const completed = {
854
+ ...prepared,
855
+ occurredAt: new Date().toISOString(),
856
+ state: "completed",
857
+ receipt: null,
858
+ };
859
+ await fs.appendFile(logPath, `${JSON.stringify(prepared)}\n${JSON.stringify(completed)}\n`, { encoding: "utf8", mode: 0o600 });
860
+ await fs.chmod(logPath, 0o600);
861
+ return {
862
+ ...result,
863
+ items,
864
+ evidence,
865
+ contextFiles: [],
866
+ egress: {
867
+ requestId,
868
+ packet,
869
+ note: "A content-free local activity receipt was recorded before this bounded text packet was returned.",
870
+ },
871
+ };
872
+ }
295
873
  export async function readScreenMemoryStatus(options = {}) {
296
874
  const info = await readConfigInfo(options);
297
875
  const paths = await resolvePaths(options);