@agent-native/core 0.84.22 → 0.84.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/core/CHANGELOG.md +14 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/agent/model-config.ts +1 -1
  5. package/corpus/core/src/cli/skills.ts +17 -3
  6. package/corpus/core/src/client/AssistantChat.tsx +45 -12
  7. package/corpus/core/src/client/active-run-state.ts +22 -0
  8. package/corpus/core/src/client/chat/message-components.tsx +2 -1
  9. package/corpus/core/src/client/chat/tool-call-display.tsx +18 -2
  10. package/corpus/core/src/client/sse-event-processor.ts +157 -40
  11. package/corpus/core/src/notifications/index.ts +1 -0
  12. package/corpus/core/src/notifications/registry.ts +13 -1
  13. package/corpus/core/src/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  14. package/corpus/core/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
  15. package/corpus/templates/analytics/.agents/skills/frontend-design/SKILL.md +2 -0
  16. package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/SqlChartCard.tsx +15 -13
  17. package/corpus/templates/analytics/scripts/emit-netlify-dashboard-report-cron.ts +18 -1
  18. package/corpus/templates/analytics/server/db/schema.ts +1 -1
  19. package/corpus/templates/analytics/server/jobs/analytics-alerts.ts +4 -1
  20. package/corpus/templates/analytics/server/lib/analytics-alerts.ts +76 -7
  21. package/corpus/templates/assets/.agents/skills/frontend-design/SKILL.md +2 -0
  22. package/corpus/templates/brain/.agents/skills/frontend-design/SKILL.md +2 -0
  23. package/corpus/templates/calendar/.agents/skills/frontend-design/SKILL.md +2 -0
  24. package/corpus/templates/chat/.agents/skills/frontend-design/SKILL.md +2 -0
  25. package/corpus/templates/clips/.agents/skills/frontend-design/SKILL.md +2 -0
  26. package/corpus/templates/clips/.agents/skills/recording/SKILL.md +30 -1
  27. package/corpus/templates/clips/actions/finalize-recording.ts +84 -4
  28. package/corpus/templates/clips/actions/lib/ensure-seekable-video.ts +256 -0
  29. package/corpus/templates/clips/actions/reprocess-recording.ts +160 -0
  30. package/corpus/templates/clips/app/components/player/share-dialog.tsx +2 -2
  31. package/corpus/templates/clips/app/i18n/ar-SA.ts +1 -0
  32. package/corpus/templates/clips/app/i18n/de-DE.ts +1 -0
  33. package/corpus/templates/clips/app/i18n/en-US.ts +1 -0
  34. package/corpus/templates/clips/app/i18n/es-ES.ts +1 -0
  35. package/corpus/templates/clips/app/i18n/fr-FR.ts +1 -0
  36. package/corpus/templates/clips/app/i18n/hi-IN.ts +1 -0
  37. package/corpus/templates/clips/app/i18n/ja-JP.ts +1 -0
  38. package/corpus/templates/clips/app/i18n/ko-KR.ts +1 -0
  39. package/corpus/templates/clips/app/i18n/pt-BR.ts +1 -0
  40. package/corpus/templates/clips/app/i18n/zh-CN.ts +1 -0
  41. package/corpus/templates/clips/app/i18n/zh-TW.ts +1 -0
  42. package/corpus/templates/clips/app/routes/embed.$shareId.tsx +31 -3
  43. package/corpus/templates/clips/changelog/2026-07-01-clip-embeds-now-render-as-a-clean-video-only-player-without-.md +6 -0
  44. package/corpus/templates/clips/changelog/2026-07-01-shared-clips-now-start-playing-and-scrub-instantly-instead-o.md +6 -0
  45. package/corpus/templates/clips/chrome-extension/public/manifest.json +1 -1
  46. package/corpus/templates/clips/chrome-extension/src/github-preview-content.ts +2 -2
  47. package/corpus/templates/clips/chrome-extension/src/github-preview.ts +41 -31
  48. package/corpus/templates/clips/server/lib/video-remux.ts +222 -0
  49. package/corpus/templates/content/.agents/skills/frontend-design/SKILL.md +2 -0
  50. package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +75 -19
  51. package/corpus/templates/design/.agents/skills/frontend-design/SKILL.md +2 -0
  52. package/corpus/templates/design/app/pages/DesignEditor.tsx +11 -1
  53. package/corpus/templates/design/changelog/2026-07-01-design-chat-no-longer-shows-an-empty-state-selection-chip-in-the-composer.md +6 -0
  54. package/corpus/templates/dispatch/.agents/skills/frontend-design/SKILL.md +2 -0
  55. package/corpus/templates/forms/.agents/skills/frontend-design/SKILL.md +2 -0
  56. package/corpus/templates/macros/.agents/skills/frontend-design/SKILL.md +2 -0
  57. package/corpus/templates/mail/.agents/skills/frontend-design/SKILL.md +2 -0
  58. package/corpus/templates/plan/.agents/skills/frontend-design/SKILL.md +2 -0
  59. package/corpus/templates/slides/.agents/skills/frontend-design/SKILL.md +2 -0
  60. package/corpus/templates/videos/.agents/skills/frontend-design/SKILL.md +2 -0
  61. package/dist/agent/model-config.js +1 -1
  62. package/dist/agent/model-config.js.map +1 -1
  63. package/dist/cli/skills.d.ts.map +1 -1
  64. package/dist/cli/skills.js +17 -3
  65. package/dist/cli/skills.js.map +1 -1
  66. package/dist/client/AssistantChat.d.ts +2 -1
  67. package/dist/client/AssistantChat.d.ts.map +1 -1
  68. package/dist/client/AssistantChat.js +37 -15
  69. package/dist/client/AssistantChat.js.map +1 -1
  70. package/dist/client/active-run-state.d.ts +2 -0
  71. package/dist/client/active-run-state.d.ts.map +1 -1
  72. package/dist/client/active-run-state.js +17 -0
  73. package/dist/client/active-run-state.js.map +1 -1
  74. package/dist/client/chat/message-components.d.ts.map +1 -1
  75. package/dist/client/chat/message-components.js +3 -1
  76. package/dist/client/chat/message-components.js.map +1 -1
  77. package/dist/client/chat/tool-call-display.d.ts +3 -1
  78. package/dist/client/chat/tool-call-display.d.ts.map +1 -1
  79. package/dist/client/chat/tool-call-display.js +8 -8
  80. package/dist/client/chat/tool-call-display.js.map +1 -1
  81. package/dist/client/sse-event-processor.d.ts +1 -0
  82. package/dist/client/sse-event-processor.d.ts.map +1 -1
  83. package/dist/client/sse-event-processor.js +136 -35
  84. package/dist/client/sse-event-processor.js.map +1 -1
  85. package/dist/notifications/index.d.ts +1 -1
  86. package/dist/notifications/index.d.ts.map +1 -1
  87. package/dist/notifications/index.js +1 -1
  88. package/dist/notifications/index.js.map +1 -1
  89. package/dist/notifications/registry.d.ts +5 -0
  90. package/dist/notifications/registry.d.ts.map +1 -1
  91. package/dist/notifications/registry.js +4 -1
  92. package/dist/notifications/registry.js.map +1 -1
  93. package/dist/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  94. package/dist/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
  95. package/package.json +1 -1
  96. package/src/templates/default/.agents/skills/frontend-design/SKILL.md +2 -0
  97. package/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +2 -0
@@ -1,8 +1,8 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
 
3
- import { notify } from "@agent-native/core/notifications";
3
+ import { notifyWithDelivery } from "@agent-native/core/notifications";
4
4
  import { recordChange } from "@agent-native/core/server";
5
- import { and, asc, desc, eq, gte, isNull, lte, sql } from "drizzle-orm";
5
+ import { and, asc, desc, eq, gte, isNull, lte, or, sql } from "drizzle-orm";
6
6
 
7
7
  import { getDb, schema } from "../db/index.js";
8
8
 
@@ -21,7 +21,12 @@ export interface AnalyticsAlertFilter {
21
21
 
22
22
  export type AnalyticsAlertThresholdMode = "event_count" | "distinct_count";
23
23
  export type AnalyticsAlertSeverity = "warning" | "critical";
24
- export type AnalyticsAlertStatus = "ok" | "triggered" | "cooldown" | "error";
24
+ export type AnalyticsAlertStatus =
25
+ | "ok"
26
+ | "triggered"
27
+ | "cooldown"
28
+ | "error"
29
+ | "running";
25
30
 
26
31
  export interface AnalyticsAlertRuleInput {
27
32
  id?: string;
@@ -108,6 +113,8 @@ function nowIso(): string {
108
113
  return new Date().toISOString();
109
114
  }
110
115
 
116
+ const ALERT_RULE_RUNNING_STALE_MS = 15 * 60 * 1000;
117
+
111
118
  function safeJsonParse<T>(raw: unknown, fallback: T): T {
112
119
  if (typeof raw !== "string" || !raw.trim()) return fallback;
113
120
  try {
@@ -372,6 +379,7 @@ export async function listEnabledAnalyticsAlertRules(options: {
372
379
  options.orgId ? eq(table.orgId, options.orgId) : isNull(table.orgId),
373
380
  );
374
381
  }
382
+ clauses.push(alertRuleNotRunningWhere(new Date()));
375
383
  const rows = await db
376
384
  .select()
377
385
  .from(table)
@@ -385,6 +393,52 @@ export async function listEnabledAnalyticsAlertRules(options: {
385
393
  return rows.map(rowToRule);
386
394
  }
387
395
 
396
+ function alertRuleNotRunningWhere(now: Date) {
397
+ const table = schema.analyticsAlertRules;
398
+ const staleRunningBefore = new Date(
399
+ now.getTime() - ALERT_RULE_RUNNING_STALE_MS,
400
+ ).toISOString();
401
+ return or(
402
+ isNull(table.lastStatus),
403
+ sql`${table.lastStatus} <> 'running'`,
404
+ isNull(table.lastEvaluatedAt),
405
+ lte(table.lastEvaluatedAt, staleRunningBefore),
406
+ );
407
+ }
408
+
409
+ function alertRulePreviousEvaluationWhere(rule: AnalyticsAlertRule) {
410
+ const table = schema.analyticsAlertRules;
411
+ return rule.lastEvaluatedAt
412
+ ? eq(table.lastEvaluatedAt, rule.lastEvaluatedAt)
413
+ : isNull(table.lastEvaluatedAt);
414
+ }
415
+
416
+ export async function claimAnalyticsAlertRuleEvaluation(
417
+ rule: AnalyticsAlertRule,
418
+ now: Date = new Date(),
419
+ ): Promise<boolean> {
420
+ const db = getDb() as any;
421
+ const table = schema.analyticsAlertRules;
422
+ const claimedAt = now.toISOString();
423
+ const rows = await db
424
+ .update(table)
425
+ .set({
426
+ lastEvaluatedAt: claimedAt,
427
+ lastStatus: "running",
428
+ lastError: null,
429
+ })
430
+ .where(
431
+ and(
432
+ eq(table.id, rule.id),
433
+ eq(table.enabled, true),
434
+ alertRuleNotRunningWhere(now),
435
+ alertRulePreviousEvaluationWhere(rule),
436
+ ),
437
+ )
438
+ .returning({ id: table.id });
439
+ return rows.length > 0;
440
+ }
441
+
388
442
  export async function evaluateAndNotifyAnalyticsAlertRule(
389
443
  rule: AnalyticsAlertRule,
390
444
  now: Date = new Date(),
@@ -416,12 +470,13 @@ export async function evaluateAndNotifyAnalyticsAlertRule(
416
470
  }
417
471
 
418
472
  const body = alertBody(rule, evaluation);
419
- const stored = await notify(
473
+ const channels = ensureInboxNotificationChannel(rule.channels);
474
+ const delivery = await notifyWithDelivery(
420
475
  {
421
476
  severity: rule.severity,
422
477
  title: `Analytics alert: ${rule.name}`,
423
478
  body,
424
- channels: rule.channels,
479
+ channels,
425
480
  metadata: {
426
481
  kind: "analytics_alert",
427
482
  ruleId: rule.id,
@@ -438,13 +493,22 @@ export async function evaluateAndNotifyAnalyticsAlertRule(
438
493
  filters: rule.filters,
439
494
  sampleEvents: evaluation.sampleEvents,
440
495
  emailRecipients: rule.emailRecipients,
496
+ requestedChannels: rule.channels,
441
497
  },
442
498
  },
443
499
  { owner: rule.ownerEmail },
444
500
  );
501
+ if (delivery.deliveredChannels.length === 0) {
502
+ await markRuleStatus(rule.id, {
503
+ lastEvaluatedAt: evaluatedAt,
504
+ lastStatus: "error",
505
+ lastError: "Analytics alert notification was not delivered.",
506
+ });
507
+ return { ruleId: rule.id, status: "error", ...evaluation };
508
+ }
445
509
 
446
510
  await recordIncident(rule, {
447
- notificationId: stored?.id,
511
+ notificationId: delivery.notification?.id,
448
512
  triggeredAt: evaluatedAt,
449
513
  windowStart,
450
514
  windowEnd,
@@ -459,11 +523,16 @@ export async function evaluateAndNotifyAnalyticsAlertRule(
459
523
  return {
460
524
  ruleId: rule.id,
461
525
  status: "triggered",
462
- notificationId: stored?.id,
526
+ notificationId: delivery.notification?.id,
463
527
  ...evaluation,
464
528
  };
465
529
  }
466
530
 
531
+ function ensureInboxNotificationChannel(channels: string[]): string[] {
532
+ const normalized = channels.map((channel) => channel.trim()).filter(Boolean);
533
+ return normalized.includes("inbox") ? normalized : ["inbox", ...normalized];
534
+ }
535
+
467
536
  export async function markAnalyticsAlertRuleError(
468
537
  id: string,
469
538
  error: unknown,
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -53,6 +53,8 @@ Default to Apple/Linear-level restraint: make the primary workflow obvious, then
53
53
  - **Visual assets**: Websites, games, and object-focused pages need real or generated media when images help users understand the subject.
54
54
  - **Responsive fit**: Text must not overflow buttons, cards, tabs, sidebars, or fixed-format tools. Use stable dimensions for boards, grids, toolbars, and counters.
55
55
 
56
+ **Beat convergence, not just defaults.** You sample toward the "on-distribution" center, so naming what to avoid is not enough: every "don't" needs a "do", or you converge on the next safe option (ban Inter and you reach for Roboto; ban purple gradients and you reach for Space Grotesk + a teal accent on every screen). Commit to one named direction, pair any reference with the reason it fits ("Linear: the quiet confidence of its spacing" — a bare "Linear" collapses back to the average), and match implementation effort to the vision: maximalist wants elaborate motion and effects, minimal wants restraint and precise spacing. When building on an existing app, inspect its tokens/type/components first and treat any drift back to a default as a missing token to pin, not something to re-prompt.
57
+
56
58
  ## Agent-Native UI Rules
57
59
 
58
60
  - Agent-native apps use React, Vite, Tailwind CSS, shadcn/ui, and `@tabler/icons-react`.
@@ -31,9 +31,38 @@ Some recordings are linked to a meeting — when `meeting_id` is non-null on the
31
31
  4. **Record.** Start a `MediaRecorder` with `mimeType: "video/webm;codecs=vp9,opus"` (fallback to vp8, then browser default). Use `timeslice: 2000` so chunks arrive every 2s.
32
32
  5. **Upload each chunk.** `ondataavailable` POSTs the chunk bytes to `/api/uploads/chunk` with headers `X-Recording-Id` and `X-Chunk-Index`. Don't retry inline — buffer failed chunks in `IndexedDB` and let a background worker re-send.
33
33
  6. **Live transcription.** Alongside the MediaRecorder, `useLiveTranscription` runs the Web Speech API to accumulate transcript text in real time. On stop, the client calls `save-browser-transcript` to persist the result immediately — no API key needed. Desktop recordings use local Whisper/macOS speech first when available, and fall back to Web Speech in the webview on non-mac before relying on upload transcription.
34
- 7. **Finalize.** On stop, send the final chunk to `/api/uploads/:id/chunk?isFinal=1`. The route calls `finalize-recording`, which stitches chunks, uploads the finished media when storage is configured, transitions `status` to `ready`, then kicks off `request-transcript` for higher-quality output (see `ai-video-tools`).
34
+ 7. **Finalize.** On stop, send the final chunk to `/api/uploads/:id/chunk?isFinal=1`. The route calls `finalize-recording`, which stitches chunks, makes the media seekable (see below), uploads the finished media when storage is configured, transitions `status` to `ready`, then kicks off `request-transcript` for higher-quality output (see `ai-video-tools`).
35
35
  8. **Navigate.** Once the row is `ready` the UI navigates to `/r/:id`.
36
36
 
37
+ ## Seekable playback (don't ship raw MediaRecorder output)
38
+
39
+ Raw `MediaRecorder` files are not friendly to progressive HTTP playback, which
40
+ shows up as "clip takes minutes to load" and "re-buffers every time I seek"
41
+ even though the file downloads fine:
42
+
43
+ - **MP4** is written with the `moov` metadata atom *after* `mdat`, so a player
44
+ must fetch the whole file before it can start or seek.
45
+ - **WebM** is a live stream with no Cues (seek index) and an unknown Segment
46
+ duration, so Chrome won't honor `currentTime = X` and has to scan/download.
47
+
48
+ `finalize-recording` fixes this before upload: MP4 gets pure-TS faststart
49
+ (`server/lib/faststart.ts`), WebM gets a lossless `ffmpeg -c copy` remux that
50
+ writes a SeekHead + Cues + real duration (`server/lib/video-remux.ts`). Both are
51
+ best-effort — on failure we upload the original, never block finalize. Recordings
52
+ above `CLIPS_INLINE_REMUX_MAX_BYTES` (default 200 MB) skip the inline pass and
53
+ are repaired in the background.
54
+
55
+ The **streaming/resumable** upload path forwards raw bytes straight to the
56
+ provider and cannot rewrite them inline, so `finalize-recording` schedules a
57
+ background `ensureRecordingSeekable` pass for those.
58
+
59
+ To repair clips uploaded before this existed (or via streaming), call the
60
+ `reprocess-recording` action: `--id`, `--ids='[...]'`, or `--all --limit=N`. It
61
+ re-fetches provider media, rewrites it, re-uploads, and repoints the row. It's
62
+ idempotent (already-seekable clips are skipped unless `--force`) and only touches
63
+ provider-hosted clips owned by the caller. This is the right tool when a user
64
+ reports a specific slow/buffering clip.
65
+
37
66
  ## Loom import
38
67
 
39
68
  Use `import-loom-recording` for Loom share or embed URLs. The action validates
@@ -40,12 +40,27 @@ import {
40
40
  getResumableSession,
41
41
  } from "../server/lib/resumable-session.js";
42
42
  import { isStreamingUploadDisabled } from "../server/lib/streaming-upload-mode.js";
43
+ import { remuxWebmToSeekable } from "../server/lib/video-remux.js";
43
44
  import {
44
45
  requiresConfiguredVideoStorage,
45
46
  STORAGE_SETUP_REQUIRED_REASON,
46
47
  } from "../server/lib/video-storage.js";
48
+ import {
49
+ ensureRecordingSeekable,
50
+ markRecordingSeekable,
51
+ } from "./lib/ensure-seekable-video.js";
47
52
  import requestTranscript from "./request-transcript.js";
48
53
 
54
+ // Recordings up to this size get their seekable rewrite applied inline during
55
+ // finalize (we already hold the assembled bytes). Larger recordings are handed
56
+ // off to the background/reprocess path so we don't stretch the finalize
57
+ // request or exhaust serverless /tmp. Override with CLIPS_INLINE_REMUX_MAX_BYTES.
58
+ function inlineRemuxMaxBytes(): number {
59
+ const raw = Number(process.env.CLIPS_INLINE_REMUX_MAX_BYTES ?? "");
60
+ if (Number.isFinite(raw) && raw > 0) return Math.floor(raw);
61
+ return 200 * 1024 * 1024;
62
+ }
63
+
49
64
  /**
50
65
  * Decode a base64 string back into a Uint8Array.
51
66
  * We store chunks as base64 in application_state because the SQL JSON
@@ -118,6 +133,10 @@ async function markRecordingReady(params: {
118
133
  finalHasAudio: boolean;
119
134
  finalHasCamera: boolean;
120
135
  existingTitle: string;
136
+ // Whether a seekable rewrite (MP4 faststart / WebM Cues remux) was already
137
+ // applied to the uploaded bytes. When false, a best-effort background repair
138
+ // is triggered so streamed/raw uploads still become seekable.
139
+ seekableApplied: boolean;
121
140
  }) {
122
141
  const {
123
142
  id,
@@ -131,6 +150,7 @@ async function markRecordingReady(params: {
131
150
  finalHasAudio,
132
151
  finalHasCamera,
133
152
  existingTitle,
153
+ seekableApplied,
134
154
  } = params;
135
155
  const db = getDb();
136
156
  const now = new Date().toISOString();
@@ -181,6 +201,30 @@ async function markRecordingReady(params: {
181
201
  });
182
202
  await writeAppState("refresh-signal", { ts: Date.now() });
183
203
 
204
+ if (seekableApplied) {
205
+ // Uploaded bytes are already start-playable and seekable — remember it so
206
+ // later reprocess sweeps skip this clip.
207
+ await markRecordingSeekable(id, videoUrl).catch((err) => {
208
+ console.warn("[finalize] failed to write seekable marker", {
209
+ id,
210
+ error: err instanceof Error ? err.message : String(err),
211
+ });
212
+ });
213
+ } else {
214
+ // Streaming/resumable (or oversized) uploads shipped raw MediaRecorder
215
+ // bytes with no seekable rewrite: an MP4 with a trailing moov or a WebM
216
+ // without a Cues index buffers on load and re-buffers on every seek. Fix
217
+ // it in the background so playback is smooth without blocking finalize.
218
+ void Promise.resolve(
219
+ ensureRecordingSeekable({ recordingId: id, ownerEmail }),
220
+ ).catch((err: unknown) => {
221
+ console.warn("[finalize] background seekable remux failed", {
222
+ id,
223
+ error: err instanceof Error ? err.message : String(err),
224
+ });
225
+ });
226
+ }
227
+
184
228
  // Kick off transcription in the background — fire-and-forget so the chunk
185
229
  // endpoint gets a quick response. The request context (user email via
186
230
  // AsyncLocalStorage) carries through to async continuations.
@@ -375,6 +419,10 @@ export default defineAction({
375
419
  ...readyParams,
376
420
  videoUrl,
377
421
  videoSizeBytes: resumableSession.bytesUploaded,
422
+ // Streaming path forwards raw MediaRecorder bytes straight to the
423
+ // provider — no faststart/Cues rewrite happened. Repair in the
424
+ // background.
425
+ seekableApplied: false,
378
426
  });
379
427
  // Delete only after durable state is written — so a retry before
380
428
  // this point can still find the session and re-enter this path.
@@ -521,9 +569,14 @@ export default defineAction({
521
569
  // recordings.
522
570
  parts.length = 0;
523
571
 
524
- // Apply faststart to MP4 files moves the moov atom before mdat so
525
- // browsers can begin playback immediately via HTTP range requests.
572
+ // Make the assembled recording seekable before upload we already hold
573
+ // the full bytes, so a viewer never has to wait through a non-seekable
574
+ // first play. MP4: relocate moov ahead of mdat (pure TS). WebM: remux to
575
+ // add a Cues index + real duration (ffmpeg -c copy). When neither runs
576
+ // (unknown format, oversized, or ffmpeg unavailable) `seekableApplied`
577
+ // stays false and markRecordingReady schedules a background repair.
526
578
  let uploadData = assembled;
579
+ let seekableApplied = false;
527
580
  if (videoFormat === "mp4") {
528
581
  try {
529
582
  uploadData = applyFaststart(assembled);
@@ -561,6 +614,32 @@ export default defineAction({
561
614
  }
562
615
  throw err;
563
616
  }
617
+ // moov is present and validated — the MP4 is start-playable/seekable.
618
+ seekableApplied = true;
619
+ } else if (videoFormat === "webm") {
620
+ // MediaRecorder WebM has no Cues index and an unknown duration, so
621
+ // Chrome buffers on load and re-buffers on every seek. A lossless
622
+ // `ffmpeg -c copy` remux rewrites it with a SeekHead + Cues + real
623
+ // duration. Bounded by size so finalize stays fast; larger clips get a
624
+ // background pass. Best-effort: on any failure we upload the original.
625
+ if (assembled.byteLength <= inlineRemuxMaxBytes()) {
626
+ try {
627
+ const seekable = await remuxWebmToSeekable(uploadData);
628
+ if (seekable.changed) {
629
+ uploadData = seekable.bytes;
630
+ seekableApplied = true;
631
+ debugLog("[finalize] webm remux applied", {
632
+ id,
633
+ bytes: uploadData.byteLength,
634
+ });
635
+ }
636
+ } catch (err) {
637
+ console.warn("[finalize] webm remux failed, uploading as-is", {
638
+ id,
639
+ err: err instanceof Error ? err.message : String(err),
640
+ });
641
+ }
642
+ }
564
643
  }
565
644
 
566
645
  let upload: Awaited<ReturnType<typeof uploadFile>>;
@@ -721,12 +800,13 @@ export default defineAction({
721
800
  debugLog("[finalize] done", {
722
801
  id,
723
802
  videoUrl: upload.url,
724
- bytes: assembled.byteLength,
803
+ bytes: uploadData.byteLength,
725
804
  });
726
805
  return markRecordingReady({
727
806
  ...readyParams,
728
807
  videoUrl: upload.url,
729
- videoSizeBytes: assembled.byteLength,
808
+ videoSizeBytes: uploadData.byteLength,
809
+ seekableApplied,
730
810
  });
731
811
  } finally {
732
812
  // Unconditional chunk scratch-space cleanup. Runs on success AND on
@@ -0,0 +1,256 @@
1
+ /**
2
+ * Repair an already-stored recording so it plays back and seeks instantly.
3
+ *
4
+ * This is the fetch-based counterpart to the inline seekable pass in
5
+ * `finalize-recording`. It re-fetches a recording's stored provider media,
6
+ * runs the format-appropriate seekable rewrite (MP4 faststart / WebM Cues
7
+ * remux), and, when that changed the bytes, re-uploads the fixed file and
8
+ * repoints the recording at it.
9
+ *
10
+ * Used by:
11
+ * - `reprocess-recording` — owner/agent triggered backfill of clips that were
12
+ * uploaded before the seekable pass existed (or via the streaming path,
13
+ * which never applied it).
14
+ * - `finalize-recording` — a best-effort background pass for the resumable
15
+ * streaming path, whose bytes live at the provider and were never buffered
16
+ * server-side for an inline rewrite.
17
+ *
18
+ * Everything is best-effort and non-destructive: if we can't fetch, can't
19
+ * improve, or can't re-upload, the existing recording is left exactly as-is.
20
+ */
21
+
22
+ import {
23
+ readAppState,
24
+ writeAppState,
25
+ } from "@agent-native/core/application-state";
26
+ import { uploadFile } from "@agent-native/core/file-upload";
27
+ import { MAX_UPLOAD_BYTES } from "@shared/upload-limits.js";
28
+ import { and, eq } from "drizzle-orm";
29
+
30
+ import { getDb, schema } from "../../server/db/index.js";
31
+ import { ownerEmailMatches } from "../../server/lib/recordings.js";
32
+ import {
33
+ makeSeekable,
34
+ type VideoFormat,
35
+ } from "../../server/lib/video-remux.js";
36
+ import { isLoomRecordingSource } from "../../shared/loom.js";
37
+
38
+ const PROVIDER_FETCH_TIMEOUT_MS = 60_000;
39
+
40
+ export type EnsureSeekableStatus =
41
+ | "optimized"
42
+ | "already-optimized"
43
+ | "skipped-not-ready"
44
+ | "skipped-no-media"
45
+ | "skipped-local-media"
46
+ | "skipped-too-large"
47
+ | "skipped-fetch-failed"
48
+ | "skipped-upload-failed"
49
+ | "not-found";
50
+
51
+ export interface EnsureSeekableResult {
52
+ recordingId: string;
53
+ status: EnsureSeekableStatus;
54
+ changed: boolean;
55
+ videoUrl?: string | null;
56
+ detail?: string;
57
+ }
58
+
59
+ /** application_state key marking a recording's media as already made seekable. */
60
+ export function seekableMarkerKey(recordingId: string): string {
61
+ return `recording-seekable-${recordingId}`;
62
+ }
63
+
64
+ /**
65
+ * Record that `videoUrl` for `recordingId` has been made seekable, so later
66
+ * sweeps skip it. Scoped by URL so a re-upload that changes the URL is not
67
+ * mistaken for already-processed.
68
+ */
69
+ export async function markRecordingSeekable(
70
+ recordingId: string,
71
+ videoUrl: string | null | undefined,
72
+ ): Promise<void> {
73
+ await writeAppState(seekableMarkerKey(recordingId), {
74
+ recordingId,
75
+ videoUrl: videoUrl ?? null,
76
+ at: new Date().toISOString(),
77
+ });
78
+ }
79
+
80
+ async function isAlreadyMarked(
81
+ recordingId: string,
82
+ videoUrl: string | null,
83
+ ): Promise<boolean> {
84
+ const marker = await readAppState(seekableMarkerKey(recordingId)).catch(
85
+ () => null,
86
+ );
87
+ return Boolean(
88
+ marker &&
89
+ typeof marker === "object" &&
90
+ (marker as { videoUrl?: unknown }).videoUrl === videoUrl,
91
+ );
92
+ }
93
+
94
+ function isRemoteProviderUrl(videoUrl: string | null | undefined): boolean {
95
+ return typeof videoUrl === "string" && /^https:\/\//i.test(videoUrl.trim());
96
+ }
97
+
98
+ async function fetchProviderBytes(
99
+ videoUrl: string,
100
+ ): Promise<
101
+ { ok: true; bytes: Uint8Array } | { ok: false; reason: EnsureSeekableStatus }
102
+ > {
103
+ const controller = new AbortController();
104
+ const timeout = setTimeout(
105
+ () => controller.abort(),
106
+ PROVIDER_FETCH_TIMEOUT_MS,
107
+ );
108
+ try {
109
+ const res = await fetch(videoUrl, { signal: controller.signal });
110
+ if (!res.ok) return { ok: false, reason: "skipped-fetch-failed" };
111
+
112
+ const declaredLength = Number(res.headers.get("content-length") ?? "0");
113
+ if (Number.isFinite(declaredLength) && declaredLength > MAX_UPLOAD_BYTES) {
114
+ return { ok: false, reason: "skipped-too-large" };
115
+ }
116
+
117
+ const buf = new Uint8Array(await res.arrayBuffer());
118
+ if (buf.byteLength === 0)
119
+ return { ok: false, reason: "skipped-fetch-failed" };
120
+ if (buf.byteLength > MAX_UPLOAD_BYTES) {
121
+ return { ok: false, reason: "skipped-too-large" };
122
+ }
123
+ return { ok: true, bytes: buf };
124
+ } catch {
125
+ return { ok: false, reason: "skipped-fetch-failed" };
126
+ } finally {
127
+ clearTimeout(timeout);
128
+ }
129
+ }
130
+
131
+ /**
132
+ * Ensure a single recording's stored media is seekable. Owner-scoped: pass the
133
+ * resolved owner email so the DB lookup can only touch that owner's rows.
134
+ */
135
+ export async function ensureRecordingSeekable(params: {
136
+ recordingId: string;
137
+ ownerEmail: string;
138
+ force?: boolean;
139
+ }): Promise<EnsureSeekableResult> {
140
+ const { recordingId, ownerEmail, force = false } = params;
141
+ const db = getDb();
142
+
143
+ const [rec] = await db
144
+ .select({
145
+ id: schema.recordings.id,
146
+ status: schema.recordings.status,
147
+ videoUrl: schema.recordings.videoUrl,
148
+ videoFormat: schema.recordings.videoFormat,
149
+ sourceAppName: schema.recordings.sourceAppName,
150
+ })
151
+ .from(schema.recordings)
152
+ .where(
153
+ and(
154
+ eq(schema.recordings.id, recordingId),
155
+ ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
156
+ ),
157
+ );
158
+
159
+ if (!rec) {
160
+ return { recordingId, status: "not-found", changed: false };
161
+ }
162
+ if (rec.status !== "ready") {
163
+ return { recordingId, status: "skipped-not-ready", changed: false };
164
+ }
165
+ if (isLoomRecordingSource(rec)) {
166
+ return { recordingId, status: "skipped-local-media", changed: false };
167
+ }
168
+ if (!rec.videoUrl) {
169
+ return { recordingId, status: "skipped-no-media", changed: false };
170
+ }
171
+ // Only provider-hosted media can be re-fetched and re-uploaded. Local/dev
172
+ // blobs (served from application_state via /api/video) and other relative
173
+ // URLs are left untouched.
174
+ if (!isRemoteProviderUrl(rec.videoUrl)) {
175
+ return {
176
+ recordingId,
177
+ status: "skipped-local-media",
178
+ changed: false,
179
+ videoUrl: rec.videoUrl,
180
+ };
181
+ }
182
+
183
+ if (!force && (await isAlreadyMarked(recordingId, rec.videoUrl))) {
184
+ return {
185
+ recordingId,
186
+ status: "already-optimized",
187
+ changed: false,
188
+ videoUrl: rec.videoUrl,
189
+ };
190
+ }
191
+
192
+ const fetched = await fetchProviderBytes(rec.videoUrl);
193
+ if (!fetched.ok) {
194
+ return { recordingId, status: fetched.reason, changed: false };
195
+ }
196
+
197
+ const videoFormat: VideoFormat = rec.videoFormat === "mp4" ? "mp4" : "webm";
198
+ const seekable = await makeSeekable({
199
+ mediaBytes: fetched.bytes,
200
+ videoFormat,
201
+ });
202
+
203
+ if (!seekable.changed) {
204
+ // Already seekable (or unimprovable) — remember so we don't refetch it.
205
+ await markRecordingSeekable(recordingId, rec.videoUrl);
206
+ return {
207
+ recordingId,
208
+ status: "already-optimized",
209
+ changed: false,
210
+ videoUrl: rec.videoUrl,
211
+ };
212
+ }
213
+
214
+ const mimeType = videoFormat === "mp4" ? "video/mp4" : "video/webm";
215
+ const upload = await uploadFile({
216
+ data: seekable.bytes,
217
+ filename: `${recordingId}.${videoFormat}`,
218
+ mimeType,
219
+ ownerEmail,
220
+ skipCompressionWait: true,
221
+ });
222
+
223
+ if (!upload?.url) {
224
+ // Could not re-upload — do NOT touch the row; the original still plays.
225
+ return {
226
+ recordingId,
227
+ status: "skipped-upload-failed",
228
+ changed: false,
229
+ videoUrl: rec.videoUrl,
230
+ };
231
+ }
232
+
233
+ await db
234
+ .update(schema.recordings)
235
+ .set({
236
+ videoUrl: upload.url,
237
+ videoSizeBytes: seekable.bytes.byteLength,
238
+ updatedAt: new Date().toISOString(),
239
+ })
240
+ .where(
241
+ and(
242
+ eq(schema.recordings.id, recordingId),
243
+ ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail),
244
+ ),
245
+ );
246
+
247
+ await markRecordingSeekable(recordingId, upload.url);
248
+ await writeAppState("refresh-signal", { ts: Date.now() });
249
+
250
+ return {
251
+ recordingId,
252
+ status: "optimized",
253
+ changed: true,
254
+ videoUrl: upload.url,
255
+ };
256
+ }