@agent-native/core 0.84.21 → 0.84.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/core/CHANGELOG.md +16 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/agent/model-config.ts +1 -1
  5. package/corpus/core/src/cli/skills.ts +17 -3
  6. package/corpus/core/src/client/AssistantChat.tsx +27 -4
  7. package/corpus/core/src/client/guided-questions.tsx +11 -6
  8. package/corpus/core/src/client/tool-display.ts +8 -0
  9. package/corpus/core/src/file-upload/builder.ts +7 -2
  10. package/corpus/core/src/notifications/channels.ts +235 -38
  11. package/corpus/core/src/notifications/index.ts +1 -0
  12. package/corpus/core/src/notifications/registry.ts +13 -1
  13. package/corpus/core/src/server/analytics.ts +32 -6
  14. package/corpus/core/src/server/ssr-handler.ts +31 -12
  15. package/corpus/core/src/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  16. package/corpus/core/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
  17. package/corpus/templates/analytics/.agents/skills/frontend-design/SKILL.md +2 -0
  18. package/corpus/templates/analytics/AGENTS.md +21 -0
  19. package/corpus/templates/analytics/README.md +35 -0
  20. package/corpus/templates/analytics/actions/delete-analytics-alert-rule.ts +23 -0
  21. package/corpus/templates/analytics/actions/list-analytics-alert-rules.ts +20 -0
  22. package/corpus/templates/analytics/actions/run-analytics-alerts.ts +23 -0
  23. package/corpus/templates/analytics/actions/save-analytics-alert-rule.ts +69 -0
  24. package/corpus/templates/analytics/changelog/2026-07-01-analytics-can-alert-when-first-party-events-spike.md +6 -0
  25. package/corpus/templates/analytics/changelog/2026-07-01-session-replays-play-back-in-production-instead-of-showing.md +6 -0
  26. package/corpus/templates/analytics/netlify.toml +3 -0
  27. package/corpus/templates/analytics/scripts/emit-netlify-dashboard-report-cron.ts +126 -0
  28. package/corpus/templates/analytics/server/db/schema.ts +55 -0
  29. package/corpus/templates/analytics/server/handlers/session-replay.ts +5 -2
  30. package/corpus/templates/analytics/server/jobs/analytics-alerts.ts +74 -0
  31. package/corpus/templates/analytics/server/lib/analytics-alerts.ts +803 -0
  32. package/corpus/templates/analytics/server/lib/session-replay.ts +10 -3
  33. package/corpus/templates/analytics/server/plugins/analytics-alert-jobs.ts +54 -0
  34. package/corpus/templates/analytics/server/plugins/db.ts +100 -0
  35. package/corpus/templates/analytics/server/routes/api/analytics-alerts/run.post.ts +60 -0
  36. package/corpus/templates/assets/.agents/skills/frontend-design/SKILL.md +2 -0
  37. package/corpus/templates/assets/README.md +30 -8
  38. package/corpus/templates/brain/.agents/skills/frontend-design/SKILL.md +2 -0
  39. package/corpus/templates/brain/README.md +20 -552
  40. package/corpus/templates/calendar/.agents/skills/frontend-design/SKILL.md +2 -0
  41. package/corpus/templates/calendar/README.md +22 -28
  42. package/corpus/templates/chat/.agents/skills/frontend-design/SKILL.md +2 -0
  43. package/corpus/templates/chat/README.md +32 -0
  44. package/corpus/templates/clips/.agents/skills/frontend-design/SKILL.md +2 -0
  45. package/corpus/templates/clips/.agents/skills/recording/SKILL.md +30 -1
  46. package/corpus/templates/clips/README.md +37 -0
  47. package/corpus/templates/clips/actions/finalize-recording.ts +84 -4
  48. package/corpus/templates/clips/actions/lib/ensure-seekable-video.ts +256 -0
  49. package/corpus/templates/clips/actions/reprocess-recording.ts +160 -0
  50. package/corpus/templates/clips/app/components/library/library-grid.tsx +18 -7
  51. package/corpus/templates/clips/app/components/library/recording-card.tsx +42 -5
  52. package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +41 -1
  53. package/corpus/templates/clips/changelog/2026-07-01-move-to-folder-now-opens-folder-choices-directly-instead-of.md +6 -0
  54. package/corpus/templates/clips/changelog/2026-07-01-shared-clips-now-start-playing-and-scrub-instantly-instead-o.md +6 -0
  55. package/corpus/templates/clips/chrome-extension/public/manifest.json +1 -1
  56. package/corpus/templates/clips/server/lib/video-remux.ts +222 -0
  57. package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts +37 -0
  58. package/corpus/templates/content/.agents/skills/frontend-design/SKILL.md +2 -0
  59. package/corpus/templates/content/AGENTS.md +7 -3
  60. package/corpus/templates/content/README.md +22 -34
  61. package/corpus/templates/content/actions/list-trashed-content-databases.ts +4 -0
  62. package/corpus/templates/content/app/components/editor/SlashCommandMenu.tsx +58 -6
  63. package/corpus/templates/content/app/components/sidebar/DocumentSidebar.tsx +257 -24
  64. package/corpus/templates/content/app/components/sidebar/DocumentTreeItem.tsx +41 -15
  65. package/corpus/templates/content/app/i18n/zh-TW.ts +25 -7
  66. package/corpus/templates/content/app/i18n-data.ts +278 -40
  67. package/corpus/templates/content/changelog/2026-06-30-database-once-again-creates-an-inline-database-in-the-curren.md +6 -0
  68. package/corpus/templates/content/shared/api.ts +1 -0
  69. package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +75 -19
  70. package/corpus/templates/design/.agents/skills/frontend-design/SKILL.md +2 -0
  71. package/corpus/templates/design/AGENTS.md +7 -6
  72. package/corpus/templates/design/README.md +33 -0
  73. package/corpus/templates/design/actions/delete-file.ts +86 -10
  74. package/corpus/templates/design/actions/get-design-snapshot.ts +13 -0
  75. package/corpus/templates/design/actions/present-design-variants.ts +11 -13
  76. package/corpus/templates/design/app/hooks/use-question-flow.ts +2 -2
  77. package/corpus/templates/design/app/pages/DesignEditor.tsx +13 -3
  78. package/corpus/templates/design/changelog/2026-07-01-design-chat-no-longer-shows-an-empty-state-selection-chip-in-the-composer.md +6 -0
  79. package/corpus/templates/dispatch/.agents/skills/frontend-design/SKILL.md +2 -0
  80. package/corpus/templates/dispatch/README.md +34 -0
  81. package/corpus/templates/forms/.agents/skills/frontend-design/SKILL.md +2 -0
  82. package/corpus/templates/forms/README.md +34 -0
  83. package/corpus/templates/macros/.agents/skills/frontend-design/SKILL.md +2 -0
  84. package/corpus/templates/macros/README.md +28 -0
  85. package/corpus/templates/mail/.agents/skills/frontend-design/SKILL.md +2 -0
  86. package/corpus/templates/mail/README.md +35 -0
  87. package/corpus/templates/plan/.agents/skills/frontend-design/SKILL.md +2 -0
  88. package/corpus/templates/plan/README.md +30 -118
  89. package/corpus/templates/slides/.agents/skills/frontend-design/SKILL.md +2 -0
  90. package/corpus/templates/slides/README.md +34 -0
  91. package/corpus/templates/videos/.agents/skills/frontend-design/SKILL.md +2 -0
  92. package/corpus/templates/videos/README.md +22 -201
  93. package/dist/agent/model-config.js +1 -1
  94. package/dist/agent/model-config.js.map +1 -1
  95. package/dist/cli/skills.d.ts.map +1 -1
  96. package/dist/cli/skills.js +17 -3
  97. package/dist/cli/skills.js.map +1 -1
  98. package/dist/client/AssistantChat.d.ts.map +1 -1
  99. package/dist/client/AssistantChat.js +28 -4
  100. package/dist/client/AssistantChat.js.map +1 -1
  101. package/dist/client/guided-questions.d.ts.map +1 -1
  102. package/dist/client/guided-questions.js +10 -6
  103. package/dist/client/guided-questions.js.map +1 -1
  104. package/dist/client/tool-display.d.ts.map +1 -1
  105. package/dist/client/tool-display.js +8 -0
  106. package/dist/client/tool-display.js.map +1 -1
  107. package/dist/collab/routes.d.ts +1 -1
  108. package/dist/file-upload/builder.d.ts.map +1 -1
  109. package/dist/file-upload/builder.js +6 -2
  110. package/dist/file-upload/builder.js.map +1 -1
  111. package/dist/notifications/channels.d.ts +10 -0
  112. package/dist/notifications/channels.d.ts.map +1 -1
  113. package/dist/notifications/channels.js +172 -20
  114. package/dist/notifications/channels.js.map +1 -1
  115. package/dist/notifications/index.d.ts +1 -1
  116. package/dist/notifications/index.d.ts.map +1 -1
  117. package/dist/notifications/index.js +1 -1
  118. package/dist/notifications/index.js.map +1 -1
  119. package/dist/notifications/registry.d.ts +5 -0
  120. package/dist/notifications/registry.d.ts.map +1 -1
  121. package/dist/notifications/registry.js +4 -1
  122. package/dist/notifications/registry.js.map +1 -1
  123. package/dist/progress/routes.d.ts +1 -1
  124. package/dist/resources/handlers.d.ts +1 -1
  125. package/dist/server/analytics.d.ts +16 -0
  126. package/dist/server/analytics.d.ts.map +1 -1
  127. package/dist/server/analytics.js +30 -7
  128. package/dist/server/analytics.js.map +1 -1
  129. package/dist/server/ssr-handler.d.ts.map +1 -1
  130. package/dist/server/ssr-handler.js +28 -12
  131. package/dist/server/ssr-handler.js.map +1 -1
  132. package/dist/server/transcribe-voice.d.ts +1 -1
  133. package/dist/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  134. package/dist/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
  135. package/package.json +1 -1
  136. package/src/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  137. package/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
@@ -0,0 +1,32 @@
1
+ # Chat
2
+
3
+ The minimal agent-native starter app — a clean, ChatGPT-style shell with chat at
4
+ the center, durable threads, standard app navigation, auth, live sync, and
5
+ actions. Start here when you want a real browser app to build on without
6
+ committing to a domain template.
7
+
8
+ **Live app: [chat.agent-native.com](https://chat.agent-native.com)**
9
+
10
+ Chat is the basic agent-native app starting point. It gives you the app-agent
11
+ loop wired end to end and one example action, so you can add your own UI, data,
12
+ and actions on top.
13
+
14
+ ## Features
15
+
16
+ - ChatGPT-style shell with a threads list and durable chat history.
17
+ - Auth, live sync, and application state wired out of the box.
18
+ - The action surface the agent and UI share, plus one example action to copy.
19
+ - A minimal, brandable base for any domain app.
20
+
21
+ ## Develop locally
22
+
23
+ Scaffold your own copy and run it:
24
+
25
+ ```bash
26
+ npx @agent-native/core@latest create my-app --standalone --template chat
27
+ cd my-app
28
+ pnpm install
29
+ pnpm dev
30
+ ```
31
+
32
+ Full docs: [agent-native.com/docs/template-chat](https://agent-native.com/docs/template-chat).
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -31,9 +31,38 @@ Some recordings are linked to a meeting — when `meeting_id` is non-null on the
31
31
  4. **Record.** Start a `MediaRecorder` with `mimeType: "video/webm;codecs=vp9,opus"` (fallback to vp8, then browser default). Use `timeslice: 2000` so chunks arrive every 2s.
32
32
  5. **Upload each chunk.** `ondataavailable` POSTs the chunk bytes to `/api/uploads/chunk` with headers `X-Recording-Id` and `X-Chunk-Index`. Don't retry inline — buffer failed chunks in `IndexedDB` and let a background worker re-send.
33
33
  6. **Live transcription.** Alongside the MediaRecorder, `useLiveTranscription` runs the Web Speech API to accumulate transcript text in real time. On stop, the client calls `save-browser-transcript` to persist the result immediately — no API key needed. Desktop recordings use local Whisper/macOS speech first when available, and fall back to Web Speech in the webview on non-mac before relying on upload transcription.
34
- 7. **Finalize.** On stop, send the final chunk to `/api/uploads/:id/chunk?isFinal=1`. The route calls `finalize-recording`, which stitches chunks, uploads the finished media when storage is configured, transitions `status` to `ready`, then kicks off `request-transcript` for higher-quality output (see `ai-video-tools`).
34
+ 7. **Finalize.** On stop, send the final chunk to `/api/uploads/:id/chunk?isFinal=1`. The route calls `finalize-recording`, which stitches chunks, makes the media seekable (see below), uploads the finished media when storage is configured, transitions `status` to `ready`, then kicks off `request-transcript` for higher-quality output (see `ai-video-tools`).
35
35
  8. **Navigate.** Once the row is `ready` the UI navigates to `/r/:id`.
36
36
 
37
+ ## Seekable playback (don't ship raw MediaRecorder output)
38
+
39
+ Raw `MediaRecorder` files are not friendly to progressive HTTP playback, which
40
+ shows up as "clip takes minutes to load" and "re-buffers every time I seek"
41
+ even though the file downloads fine:
42
+
43
+ - **MP4** is written with the `moov` metadata atom *after* `mdat`, so a player
44
+ must fetch the whole file before it can start or seek.
45
+ - **WebM** is a live stream with no Cues (seek index) and an unknown Segment
46
+ duration, so Chrome won't honor `currentTime = X` and has to scan/download.
47
+
48
+ `finalize-recording` fixes this before upload: MP4 gets pure-TS faststart
49
+ (`server/lib/faststart.ts`), WebM gets a lossless `ffmpeg -c copy` remux that
50
+ writes a SeekHead + Cues + real duration (`server/lib/video-remux.ts`). Both are
51
+ best-effort — on failure we upload the original, never block finalize. Recordings
52
+ above `CLIPS_INLINE_REMUX_MAX_BYTES` (default 200 MB) skip the inline pass and
53
+ are repaired in the background.
54
+
55
+ The **streaming/resumable** upload path forwards raw bytes straight to the
56
+ provider and cannot rewrite them inline, so `finalize-recording` schedules a
57
+ background `ensureRecordingSeekable` pass for those.
58
+
59
+ To repair clips uploaded before this existed (or via streaming), call the
60
+ `reprocess-recording` action: `--id`, `--ids='[...]'`, or `--all --limit=N`. It
61
+ re-fetches provider media, rewrites it, re-uploads, and repoints the row. It's
62
+ idempotent (already-seekable clips are skipped unless `--force`) and only touches
63
+ provider-hosted clips owned by the caller. This is the right tool when a user
64
+ reports a specific slow/buffering clip.
65
+
37
66
  ## Loom import
38
67
 
39
68
  Use `import-loom-recording` for Loom share or embed URLs. The action validates
@@ -0,0 +1,37 @@
1
+ # Clips
2
+
3
+ An open-source, agent-native alternative to Loom, Granola, and Wispr Flow rolled
4
+ into one app. Screen recording, calendar-synced meeting notes, and push-to-talk
5
+ voice dictation — all transcribed, searchable, and yours to own.
6
+
7
+ **Live app: [clips.agent-native.com](https://clips.agent-native.com)**
8
+
9
+ The agent transcribes, titles, summarizes, and indexes everything you capture,
10
+ then lets you ask "find the clip where we discussed the rollout plan" across every
11
+ transcript. Shared clips are agent-readable: paste a share link into an agent and
12
+ it can read the transcript and see timestamped frames without the raw video.
13
+
14
+ ## Features
15
+
16
+ - Screen recording with webcam overlay, audio capture, and pause/trim.
17
+ - Calendar-synced meeting recordings with AI summary, notes, and action items.
18
+ - Push-to-talk (Fn-hold) dictation with searchable history and vocabulary biasing.
19
+ - Auto-generated titles, summaries, and chapter markers for every recording.
20
+ - Full-text search across recordings, meetings, and dictations in one library.
21
+ - Sharing with per-clip permissions, agent-readable links, and Slack unfurls.
22
+ - Chrome extension for capturing browser logs with a recording.
23
+
24
+ ## Develop locally
25
+
26
+ Scaffold your own copy and run it:
27
+
28
+ ```bash
29
+ npx @agent-native/core@latest create my-clips --standalone --template clips
30
+ cd my-clips
31
+ pnpm install
32
+ pnpm dev
33
+ ```
34
+
35
+ Clips needs a video storage backend (Builder.io or an S3-compatible bucket)
36
+ before recordings can upload — see the docs for setup.
37
+ Full docs: [agent-native.com/docs/template-clips](https://agent-native.com/docs/template-clips).
@@ -40,12 +40,27 @@ import {
40
40
  getResumableSession,
41
41
  } from "../server/lib/resumable-session.js";
42
42
  import { isStreamingUploadDisabled } from "../server/lib/streaming-upload-mode.js";
43
+ import { remuxWebmToSeekable } from "../server/lib/video-remux.js";
43
44
  import {
44
45
  requiresConfiguredVideoStorage,
45
46
  STORAGE_SETUP_REQUIRED_REASON,
46
47
  } from "../server/lib/video-storage.js";
48
+ import {
49
+ ensureRecordingSeekable,
50
+ markRecordingSeekable,
51
+ } from "./lib/ensure-seekable-video.js";
47
52
  import requestTranscript from "./request-transcript.js";
48
53
 
54
+ // Recordings up to this size get their seekable rewrite applied inline during
55
+ // finalize (we already hold the assembled bytes). Larger recordings are handed
56
+ // off to the background/reprocess path so we don't stretch the finalize
57
+ // request or exhaust serverless /tmp. Override with CLIPS_INLINE_REMUX_MAX_BYTES.
58
+ function inlineRemuxMaxBytes(): number {
59
+ const raw = Number(process.env.CLIPS_INLINE_REMUX_MAX_BYTES ?? "");
60
+ if (Number.isFinite(raw) && raw > 0) return Math.floor(raw);
61
+ return 200 * 1024 * 1024;
62
+ }
63
+
49
64
  /**
50
65
  * Decode a base64 string back into a Uint8Array.
51
66
  * We store chunks as base64 in application_state because the SQL JSON
@@ -118,6 +133,10 @@ async function markRecordingReady(params: {
118
133
  finalHasAudio: boolean;
119
134
  finalHasCamera: boolean;
120
135
  existingTitle: string;
136
+ // Whether a seekable rewrite (MP4 faststart / WebM Cues remux) was already
137
+ // applied to the uploaded bytes. When false, a best-effort background repair
138
+ // is triggered so streamed/raw uploads still become seekable.
139
+ seekableApplied: boolean;
121
140
  }) {
122
141
  const {
123
142
  id,
@@ -131,6 +150,7 @@ async function markRecordingReady(params: {
131
150
  finalHasAudio,
132
151
  finalHasCamera,
133
152
  existingTitle,
153
+ seekableApplied,
134
154
  } = params;
135
155
  const db = getDb();
136
156
  const now = new Date().toISOString();
@@ -181,6 +201,30 @@ async function markRecordingReady(params: {
181
201
  });
182
202
  await writeAppState("refresh-signal", { ts: Date.now() });
183
203
 
204
+ if (seekableApplied) {
205
+ // Uploaded bytes are already start-playable and seekable — remember it so
206
+ // later reprocess sweeps skip this clip.
207
+ await markRecordingSeekable(id, videoUrl).catch((err) => {
208
+ console.warn("[finalize] failed to write seekable marker", {
209
+ id,
210
+ error: err instanceof Error ? err.message : String(err),
211
+ });
212
+ });
213
+ } else {
214
+ // Streaming/resumable (or oversized) uploads shipped raw MediaRecorder
215
+ // bytes with no seekable rewrite: an MP4 with a trailing moov or a WebM
216
+ // without a Cues index buffers on load and re-buffers on every seek. Fix
217
+ // it in the background so playback is smooth without blocking finalize.
218
+ void Promise.resolve(
219
+ ensureRecordingSeekable({ recordingId: id, ownerEmail }),
220
+ ).catch((err: unknown) => {
221
+ console.warn("[finalize] background seekable remux failed", {
222
+ id,
223
+ error: err instanceof Error ? err.message : String(err),
224
+ });
225
+ });
226
+ }
227
+
184
228
  // Kick off transcription in the background — fire-and-forget so the chunk
185
229
  // endpoint gets a quick response. The request context (user email via
186
230
  // AsyncLocalStorage) carries through to async continuations.
@@ -375,6 +419,10 @@ export default defineAction({
375
419
  ...readyParams,
376
420
  videoUrl,
377
421
  videoSizeBytes: resumableSession.bytesUploaded,
422
+ // Streaming path forwards raw MediaRecorder bytes straight to the
423
+ // provider — no faststart/Cues rewrite happened. Repair in the
424
+ // background.
425
+ seekableApplied: false,
378
426
  });
379
427
  // Delete only after durable state is written — so a retry before
380
428
  // this point can still find the session and re-enter this path.
@@ -521,9 +569,14 @@ export default defineAction({
521
569
  // recordings.
522
570
  parts.length = 0;
523
571
 
524
- // Apply faststart to MP4 files moves the moov atom before mdat so
525
- // browsers can begin playback immediately via HTTP range requests.
572
+ // Make the assembled recording seekable before upload we already hold
573
+ // the full bytes, so a viewer never has to wait through a non-seekable
574
+ // first play. MP4: relocate moov ahead of mdat (pure TS). WebM: remux to
575
+ // add a Cues index + real duration (ffmpeg -c copy). When neither runs
576
+ // (unknown format, oversized, or ffmpeg unavailable) `seekableApplied`
577
+ // stays false and markRecordingReady schedules a background repair.
526
578
  let uploadData = assembled;
579
+ let seekableApplied = false;
527
580
  if (videoFormat === "mp4") {
528
581
  try {
529
582
  uploadData = applyFaststart(assembled);
@@ -561,6 +614,32 @@ export default defineAction({
561
614
  }
562
615
  throw err;
563
616
  }
617
+ // moov is present and validated — the MP4 is start-playable/seekable.
618
+ seekableApplied = true;
619
+ } else if (videoFormat === "webm") {
620
+ // MediaRecorder WebM has no Cues index and an unknown duration, so
621
+ // Chrome buffers on load and re-buffers on every seek. A lossless
622
+ // `ffmpeg -c copy` remux rewrites it with a SeekHead + Cues + real
623
+ // duration. Bounded by size so finalize stays fast; larger clips get a
624
+ // background pass. Best-effort: on any failure we upload the original.
625
+ if (assembled.byteLength <= inlineRemuxMaxBytes()) {
626
+ try {
627
+ const seekable = await remuxWebmToSeekable(uploadData);
628
+ if (seekable.changed) {
629
+ uploadData = seekable.bytes;
630
+ seekableApplied = true;
631
+ debugLog("[finalize] webm remux applied", {
632
+ id,
633
+ bytes: uploadData.byteLength,
634
+ });
635
+ }
636
+ } catch (err) {
637
+ console.warn("[finalize] webm remux failed, uploading as-is", {
638
+ id,
639
+ err: err instanceof Error ? err.message : String(err),
640
+ });
641
+ }
642
+ }
564
643
  }
565
644
 
566
645
  let upload: Awaited<ReturnType<typeof uploadFile>>;
@@ -721,12 +800,13 @@ export default defineAction({
721
800
  debugLog("[finalize] done", {
722
801
  id,
723
802
  videoUrl: upload.url,
724
- bytes: assembled.byteLength,
803
+ bytes: uploadData.byteLength,
725
804
  });
726
805
  return markRecordingReady({
727
806
  ...readyParams,
728
807
  videoUrl: upload.url,
729
- videoSizeBytes: assembled.byteLength,
808
+ videoSizeBytes: uploadData.byteLength,
809
+ seekableApplied,
730
810
  });
731
811
  } finally {
732
812
  // Unconditional chunk scratch-space cleanup. Runs on success AND on
@@ -0,0 +1,256 @@
1
+ /**
2
+ * Repair an already-stored recording so it plays back and seeks instantly.
3
+ *
4
+ * This is the fetch-based counterpart to the inline seekable pass in
5
+ * `finalize-recording`. It re-fetches a recording's stored provider media,
6
+ * runs the format-appropriate seekable rewrite (MP4 faststart / WebM Cues
7
+ * remux), and, when that changed the bytes, re-uploads the fixed file and
8
+ * repoints the recording at it.
9
+ *
10
+ * Used by:
11
+ * - `reprocess-recording` — owner/agent triggered backfill of clips that were
12
+ * uploaded before the seekable pass existed (or via the streaming path,
13
+ * which never applied it).
14
+ * - `finalize-recording` — a best-effort background pass for the resumable
15
+ * streaming path, whose bytes live at the provider and were never buffered
16
+ * server-side for an inline rewrite.
17
+ *
18
+ * Everything is best-effort and non-destructive: if we can't fetch, can't
19
+ * improve, or can't re-upload, the existing recording is left exactly as-is.
20
+ */
21
+
22
+ import {
23
+ readAppState,
24
+ writeAppState,
25
+ } from "@agent-native/core/application-state";
26
+ import { uploadFile } from "@agent-native/core/file-upload";
27
+ import { MAX_UPLOAD_BYTES } from "@shared/upload-limits.js";
28
+ import { and, eq } from "drizzle-orm";
29
+
30
+ import { getDb, schema } from "../../server/db/index.js";
31
+ import { ownerEmailMatches } from "../../server/lib/recordings.js";
32
+ import {
33
+ makeSeekable,
34
+ type VideoFormat,
35
+ } from "../../server/lib/video-remux.js";
36
+ import { isLoomRecordingSource } from "../../shared/loom.js";
37
+
38
+ const PROVIDER_FETCH_TIMEOUT_MS = 60_000;
39
+
40
+ export type EnsureSeekableStatus =
41
+ | "optimized"
42
+ | "already-optimized"
43
+ | "skipped-not-ready"
44
+ | "skipped-no-media"
45
+ | "skipped-local-media"
46
+ | "skipped-too-large"
47
+ | "skipped-fetch-failed"
48
+ | "skipped-upload-failed"
49
+ | "not-found";
50
+
51
+ export interface EnsureSeekableResult {
52
+ recordingId: string;
53
+ status: EnsureSeekableStatus;
54
+ changed: boolean;
55
+ videoUrl?: string | null;
56
+ detail?: string;
57
+ }
58
+
59
+ /** application_state key marking a recording's media as already made seekable. */
60
+ export function seekableMarkerKey(recordingId: string): string {
61
+ return `recording-seekable-${recordingId}`;
62
+ }
63
+
64
+ /**
65
+ * Record that `videoUrl` for `recordingId` has been made seekable, so later
66
+ * sweeps skip it. Scoped by URL so a re-upload that changes the URL is not
67
+ * mistaken for already-processed.
68
+ */
69
+ export async function markRecordingSeekable(
70
+ recordingId: string,
71
+ videoUrl: string | null | undefined,
72
+ ): Promise<void> {
73
+ await writeAppState(seekableMarkerKey(recordingId), {
74
+ recordingId,
75
+ videoUrl: videoUrl ?? null,
76
+ at: new Date().toISOString(),
77
+ });
78
+ }
79
+
80
+ async function isAlreadyMarked(
81
+ recordingId: string,
82
+ videoUrl: string | null,
83
+ ): Promise<boolean> {
84
+ const marker = await readAppState(seekableMarkerKey(recordingId)).catch(
85
+ () => null,
86
+ );
87
+ return Boolean(
88
+ marker &&
89
+ typeof marker === "object" &&
90
+ (marker as { videoUrl?: unknown }).videoUrl === videoUrl,
91
+ );
92
+ }
93
+
94
+ function isRemoteProviderUrl(videoUrl: string | null | undefined): boolean {
95
+ return typeof videoUrl === "string" && /^https:\/\//i.test(videoUrl.trim());
96
+ }
97
+
98
+ async function fetchProviderBytes(
99
+ videoUrl: string,
100
+ ): Promise<
101
+ { ok: true; bytes: Uint8Array } | { ok: false; reason: EnsureSeekableStatus }
102
+ > {
103
+ const controller = new AbortController();
104
+ const timeout = setTimeout(
105
+ () => controller.abort(),
106
+ PROVIDER_FETCH_TIMEOUT_MS,
107
+ );
108
+ try {
109
+ const res = await fetch(videoUrl, { signal: controller.signal });
110
+ if (!res.ok) return { ok: false, reason: "skipped-fetch-failed" };
111
+
112
+ const declaredLength = Number(res.headers.get("content-length") ?? "0");
113
+ if (Number.isFinite(declaredLength) && declaredLength > MAX_UPLOAD_BYTES) {
114
+ return { ok: false, reason: "skipped-too-large" };
115
+ }
116
+
117
+ const buf = new Uint8Array(await res.arrayBuffer());
118
+ if (buf.byteLength === 0)
119
+ return { ok: false, reason: "skipped-fetch-failed" };
120
+ if (buf.byteLength > MAX_UPLOAD_BYTES) {
121
+ return { ok: false, reason: "skipped-too-large" };
122
+ }
123
+ return { ok: true, bytes: buf };
124
+ } catch {
125
+ return { ok: false, reason: "skipped-fetch-failed" };
126
+ } finally {
127
+ clearTimeout(timeout);
128
+ }
129
+ }
130
+
131
+ /**
132
+ * Ensure a single recording's stored media is seekable. Owner-scoped: pass the
133
+ * resolved owner email so the DB lookup can only touch that owner's rows.
134
+ */
135
+ export async function ensureRecordingSeekable(params: {
136
+ recordingId: string;
137
+ ownerEmail: string;
138
+ force?: boolean;
139
+ }): Promise<EnsureSeekableResult> {
140
+ const { recordingId, ownerEmail, force = false } = params;
141
+ const db = getDb();
142
+
143
+ const [rec] = await db
144
+ .select({
145
+ id: schema.recordings.id,
146
+ status: schema.recordings.status,
147
+ videoUrl: schema.recordings.videoUrl,
148
+ videoFormat: schema.recordings.videoFormat,
149
+ sourceAppName: schema.recordings.sourceAppName,
150
+ })
151
+ .from(schema.recordings)
152
+ .where(
153
+ and(
154
+ eq(schema.recordings.id, recordingId),
155
+ ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
156
+ ),
157
+ );
158
+
159
+ if (!rec) {
160
+ return { recordingId, status: "not-found", changed: false };
161
+ }
162
+ if (rec.status !== "ready") {
163
+ return { recordingId, status: "skipped-not-ready", changed: false };
164
+ }
165
+ if (isLoomRecordingSource(rec)) {
166
+ return { recordingId, status: "skipped-local-media", changed: false };
167
+ }
168
+ if (!rec.videoUrl) {
169
+ return { recordingId, status: "skipped-no-media", changed: false };
170
+ }
171
+ // Only provider-hosted media can be re-fetched and re-uploaded. Local/dev
172
+ // blobs (served from application_state via /api/video) and other relative
173
+ // URLs are left untouched.
174
+ if (!isRemoteProviderUrl(rec.videoUrl)) {
175
+ return {
176
+ recordingId,
177
+ status: "skipped-local-media",
178
+ changed: false,
179
+ videoUrl: rec.videoUrl,
180
+ };
181
+ }
182
+
183
+ if (!force && (await isAlreadyMarked(recordingId, rec.videoUrl))) {
184
+ return {
185
+ recordingId,
186
+ status: "already-optimized",
187
+ changed: false,
188
+ videoUrl: rec.videoUrl,
189
+ };
190
+ }
191
+
192
+ const fetched = await fetchProviderBytes(rec.videoUrl);
193
+ if (!fetched.ok) {
194
+ return { recordingId, status: fetched.reason, changed: false };
195
+ }
196
+
197
+ const videoFormat: VideoFormat = rec.videoFormat === "mp4" ? "mp4" : "webm";
198
+ const seekable = await makeSeekable({
199
+ mediaBytes: fetched.bytes,
200
+ videoFormat,
201
+ });
202
+
203
+ if (!seekable.changed) {
204
+ // Already seekable (or unimprovable) — remember so we don't refetch it.
205
+ await markRecordingSeekable(recordingId, rec.videoUrl);
206
+ return {
207
+ recordingId,
208
+ status: "already-optimized",
209
+ changed: false,
210
+ videoUrl: rec.videoUrl,
211
+ };
212
+ }
213
+
214
+ const mimeType = videoFormat === "mp4" ? "video/mp4" : "video/webm";
215
+ const upload = await uploadFile({
216
+ data: seekable.bytes,
217
+ filename: `${recordingId}.${videoFormat}`,
218
+ mimeType,
219
+ ownerEmail,
220
+ skipCompressionWait: true,
221
+ });
222
+
223
+ if (!upload?.url) {
224
+ // Could not re-upload — do NOT touch the row; the original still plays.
225
+ return {
226
+ recordingId,
227
+ status: "skipped-upload-failed",
228
+ changed: false,
229
+ videoUrl: rec.videoUrl,
230
+ };
231
+ }
232
+
233
+ await db
234
+ .update(schema.recordings)
235
+ .set({
236
+ videoUrl: upload.url,
237
+ videoSizeBytes: seekable.bytes.byteLength,
238
+ updatedAt: new Date().toISOString(),
239
+ })
240
+ .where(
241
+ and(
242
+ eq(schema.recordings.id, recordingId),
243
+ ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
244
+ ),
245
+ );
246
+
247
+ await markRecordingSeekable(recordingId, upload.url);
248
+ await writeAppState("refresh-signal", { ts: Date.now() });
249
+
250
+ return {
251
+ recordingId,
252
+ status: "optimized",
253
+ changed: true,
254
+ videoUrl: upload.url,
255
+ };
256
+ }
@@ -0,0 +1,160 @@
1
+ /**
2
+ * Repair the playback of already-stored recordings.
3
+ *
4
+ * Clips uploaded via the streaming path (or before the seekable rewrite
5
+ * existed) can ship raw MediaRecorder output: an MP4 with a trailing `moov`
6
+ * atom or a WebM with no Cues index. Those load slowly and re-buffer on every
7
+ * seek even though the file downloads fine. This action re-fetches the stored
8
+ * media, rewrites it to be start-playable and seekable (MP4 faststart / WebM
9
+ * Cues remux), re-uploads the fixed file, and repoints the recording at it.
10
+ *
11
+ * It's idempotent and non-destructive: clips that are already seekable are
12
+ * skipped, and any clip we can't improve or re-upload is left untouched.
13
+ *
14
+ * Usage:
15
+ * pnpm action reprocess-recording --id=<recordingId>
16
+ * pnpm action reprocess-recording --ids='["id1","id2"]'
17
+ * pnpm action reprocess-recording --all --limit=20
18
+ * pnpm action reprocess-recording --id=<recordingId> --force
19
+ */
20
+
21
+ import { defineAction } from "@agent-native/core";
22
+ import { and, desc, eq, isNotNull } from "drizzle-orm";
23
+ import { z } from "zod";
24
+
25
+ import { getDb, schema } from "../server/db/index.js";
26
+ import {
27
+ getCurrentOwnerEmail,
28
+ ownerEmailMatches,
29
+ } from "../server/lib/recordings.js";
30
+ import {
31
+ ensureRecordingSeekable,
32
+ type EnsureSeekableResult,
33
+ } from "./lib/ensure-seekable-video.js";
34
+
35
+ const MAX_TARGETS_PER_CALL = 100;
36
+ const DEFAULT_ALL_LIMIT = 20;
37
+
38
+ function parseIds(value: unknown): string[] {
39
+ if (Array.isArray(value)) {
40
+ return value.filter((v): v is string => typeof v === "string" && !!v);
41
+ }
42
+ if (typeof value === "string" && value.trim()) {
43
+ try {
44
+ const parsed = JSON.parse(value);
45
+ if (Array.isArray(parsed)) {
46
+ return parsed.filter((v): v is string => typeof v === "string" && !!v);
47
+ }
48
+ } catch {
49
+ // Fall back to a single comma-separated / plain id string.
50
+ return value
51
+ .split(",")
52
+ .map((s) => s.trim())
53
+ .filter(Boolean);
54
+ }
55
+ }
56
+ return [];
57
+ }
58
+
59
+ export default defineAction({
60
+ description:
61
+ "Repair slow-loading or buffering recordings by rewriting their stored video to be start-playable and seekable (MP4 faststart / WebM Cues remux) and re-uploading it. Use for clips that take a long time to load or re-buffer when scrubbing. Pass `id` for one clip, `ids` for several, or `all: true` to sweep the caller's ready clips (bounded by `limit`). Already-seekable clips are skipped unless `force` is true.",
62
+ schema: z.object({
63
+ id: z.string().optional().describe("A single recording id to repair."),
64
+ ids: z
65
+ .union([z.array(z.string()), z.string()])
66
+ .optional()
67
+ .describe(
68
+ "Multiple recording ids (array, or JSON/comma string for CLI).",
69
+ ),
70
+ all: z.coerce
71
+ .boolean()
72
+ .optional()
73
+ .describe(
74
+ "Repair the caller's own ready clips, most recent first, up to `limit`.",
75
+ ),
76
+ limit: z.coerce
77
+ .number()
78
+ .int()
79
+ .min(1)
80
+ .max(MAX_TARGETS_PER_CALL)
81
+ .optional()
82
+ .describe(
83
+ `Max clips to process when using \`all\` (default ${DEFAULT_ALL_LIMIT}).`,
84
+ ),
85
+ force: z.coerce
86
+ .boolean()
87
+ .optional()
88
+ .describe("Re-run even on clips already marked seekable."),
89
+ }),
90
+ run: async (args) => {
91
+ const db = getDb();
92
+ const ownerEmail = getCurrentOwnerEmail();
93
+ const force = Boolean(args.force);
94
+
95
+ let targetIds: string[] = [];
96
+ if (args.id) targetIds.push(args.id);
97
+ targetIds.push(...parseIds(args.ids));
98
+
99
+ if (args.all) {
100
+ const limit = args.limit ?? DEFAULT_ALL_LIMIT;
101
+ const rows = await db
102
+ .select({ id: schema.recordings.id })
103
+ .from(schema.recordings)
104
+ .where(
105
+ and(
106
+ ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
107
+ eq(schema.recordings.status, "ready"),
108
+ isNotNull(schema.recordings.videoUrl),
109
+ ),
110
+ )
111
+ .orderBy(desc(schema.recordings.createdAt))
112
+ .limit(limit);
113
+ targetIds.push(...rows.map((r) => r.id));
114
+ }
115
+
116
+ // De-dupe while preserving order, and bound the batch so a single call
117
+ // can't run unboundedly under the hosted foreground budget.
118
+ targetIds = Array.from(new Set(targetIds)).slice(0, MAX_TARGETS_PER_CALL);
119
+
120
+ if (targetIds.length === 0) {
121
+ return {
122
+ ok: true,
123
+ processed: 0,
124
+ changed: 0,
125
+ results: [] as EnsureSeekableResult[],
126
+ message:
127
+ "No recordings to reprocess. Pass id / ids, or all: true to sweep your clips.",
128
+ };
129
+ }
130
+
131
+ const results: EnsureSeekableResult[] = [];
132
+ for (const recordingId of targetIds) {
133
+ try {
134
+ results.push(
135
+ await ensureRecordingSeekable({ recordingId, ownerEmail, force }),
136
+ );
137
+ } catch (err) {
138
+ console.warn("[reprocess-recording] failed for", recordingId, err);
139
+ results.push({
140
+ recordingId,
141
+ status: "skipped-fetch-failed",
142
+ changed: false,
143
+ detail: err instanceof Error ? err.message : String(err),
144
+ });
145
+ }
146
+ }
147
+
148
+ const changed = results.filter((r) => r.changed).length;
149
+ console.log(
150
+ `Reprocessed ${results.length} recording(s); ${changed} rewritten for smoother playback.`,
151
+ );
152
+
153
+ return {
154
+ ok: true,
155
+ processed: results.length,
156
+ changed,
157
+ results,
158
+ };
159
+ },
160
+ });