@meframe/core 0.0.1 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +17 -4
  2. package/dist/Meframe.d.ts.map +1 -1
  3. package/dist/Meframe.js +2 -4
  4. package/dist/Meframe.js.map +1 -1
  5. package/dist/cache/CacheManager.d.ts.map +1 -1
  6. package/dist/cache/CacheManager.js +8 -1
  7. package/dist/cache/CacheManager.js.map +1 -1
  8. package/dist/config/defaults.d.ts.map +1 -1
  9. package/dist/config/defaults.js +2 -9
  10. package/dist/config/defaults.js.map +1 -1
  11. package/dist/config/types.d.ts +3 -4
  12. package/dist/config/types.d.ts.map +1 -1
  13. package/dist/controllers/PlaybackController.d.ts +4 -2
  14. package/dist/controllers/PlaybackController.d.ts.map +1 -1
  15. package/dist/controllers/PlaybackController.js +7 -13
  16. package/dist/controllers/PlaybackController.js.map +1 -1
  17. package/dist/controllers/PreRenderService.d.ts +3 -2
  18. package/dist/controllers/PreRenderService.d.ts.map +1 -1
  19. package/dist/controllers/PreRenderService.js.map +1 -1
  20. package/dist/controllers/PreviewHandle.d.ts +2 -0
  21. package/dist/controllers/PreviewHandle.d.ts.map +1 -1
  22. package/dist/controllers/PreviewHandle.js +6 -0
  23. package/dist/controllers/PreviewHandle.js.map +1 -1
  24. package/dist/controllers/index.d.ts +1 -1
  25. package/dist/controllers/index.d.ts.map +1 -1
  26. package/dist/controllers/types.d.ts +2 -12
  27. package/dist/controllers/types.d.ts.map +1 -1
  28. package/dist/event/events.d.ts +5 -59
  29. package/dist/event/events.d.ts.map +1 -1
  30. package/dist/event/events.js +1 -6
  31. package/dist/event/events.js.map +1 -1
  32. package/dist/model/CompositionModel.js +1 -2
  33. package/dist/model/CompositionModel.js.map +1 -1
  34. package/dist/orchestrator/CompositionPlanner.d.ts.map +1 -1
  35. package/dist/orchestrator/CompositionPlanner.js +1 -0
  36. package/dist/orchestrator/CompositionPlanner.js.map +1 -1
  37. package/dist/orchestrator/Orchestrator.d.ts.map +1 -1
  38. package/dist/orchestrator/Orchestrator.js +3 -13
  39. package/dist/orchestrator/Orchestrator.js.map +1 -1
  40. package/dist/orchestrator/VideoClipSession.d.ts.map +1 -1
  41. package/dist/orchestrator/VideoClipSession.js +4 -5
  42. package/dist/orchestrator/VideoClipSession.js.map +1 -1
  43. package/dist/orchestrator/types.d.ts +1 -1
  44. package/dist/orchestrator/types.d.ts.map +1 -1
  45. package/dist/stages/compose/GlobalAudioSession.d.ts.map +1 -1
  46. package/dist/stages/compose/GlobalAudioSession.js +3 -2
  47. package/dist/stages/compose/GlobalAudioSession.js.map +1 -1
  48. package/dist/stages/compose/VideoComposer.d.ts.map +1 -1
  49. package/dist/stages/compose/types.d.ts +3 -1
  50. package/dist/stages/compose/types.d.ts.map +1 -1
  51. package/dist/stages/decode/AudioChunkDecoder.d.ts.map +1 -1
  52. package/dist/stages/decode/VideoChunkDecoder.d.ts +0 -1
  53. package/dist/stages/decode/VideoChunkDecoder.d.ts.map +1 -1
  54. package/dist/stages/demux/MP4Demuxer.d.ts +2 -1
  55. package/dist/stages/demux/MP4Demuxer.d.ts.map +1 -1
  56. package/dist/stages/load/EventHandlers.d.ts +2 -11
  57. package/dist/stages/load/EventHandlers.d.ts.map +1 -1
  58. package/dist/stages/load/EventHandlers.js +1 -24
  59. package/dist/stages/load/EventHandlers.js.map +1 -1
  60. package/dist/stages/load/ResourceLoader.d.ts.map +1 -1
  61. package/dist/stages/load/ResourceLoader.js +11 -13
  62. package/dist/stages/load/ResourceLoader.js.map +1 -1
  63. package/dist/stages/load/TaskManager.d.ts +1 -1
  64. package/dist/stages/load/TaskManager.d.ts.map +1 -1
  65. package/dist/stages/load/TaskManager.js +3 -2
  66. package/dist/stages/load/TaskManager.js.map +1 -1
  67. package/dist/stages/load/types.d.ts +2 -0
  68. package/dist/stages/load/types.d.ts.map +1 -1
  69. package/dist/utils/time-utils.d.ts +3 -2
  70. package/dist/utils/time-utils.d.ts.map +1 -1
  71. package/dist/utils/time-utils.js +2 -1
  72. package/dist/utils/time-utils.js.map +1 -1
  73. package/dist/vite-plugin.d.ts +19 -0
  74. package/dist/vite-plugin.d.ts.map +1 -0
  75. package/dist/vite-plugin.js +145 -0
  76. package/dist/vite-plugin.js.map +1 -0
  77. package/dist/worker/WorkerPool.d.ts +7 -4
  78. package/dist/worker/WorkerPool.d.ts.map +1 -1
  79. package/dist/worker/WorkerPool.js +29 -18
  80. package/dist/worker/WorkerPool.js.map +1 -1
  81. package/dist/{stages/demux → workers}/MP4Demuxer.js +17 -15
  82. package/dist/workers/MP4Demuxer.js.map +1 -0
  83. package/dist/workers/WorkerChannel.js +486 -0
  84. package/dist/workers/WorkerChannel.js.map +1 -0
  85. package/dist/workers/mp4box.all.js +7049 -0
  86. package/dist/workers/mp4box.all.js.map +1 -0
  87. package/dist/workers/stages/compose/audio-compose.worker.js +1063 -0
  88. package/dist/workers/stages/compose/audio-compose.worker.js.map +1 -0
  89. package/dist/workers/stages/compose/video-compose.worker.js +1209 -0
  90. package/dist/workers/stages/compose/video-compose.worker.js.map +1 -0
  91. package/dist/{stages → workers/stages}/decode/decode.worker.js +401 -20
  92. package/dist/workers/stages/decode/decode.worker.js.map +1 -0
  93. package/dist/{stages → workers/stages}/demux/audio-demux.worker.js +184 -4
  94. package/dist/workers/stages/demux/audio-demux.worker.js.map +1 -0
  95. package/dist/{stages → workers/stages}/demux/video-demux.worker.js +7 -30
  96. package/dist/workers/stages/demux/video-demux.worker.js.map +1 -0
  97. package/dist/{stages → workers/stages}/encode/encode.worker.js +238 -5
  98. package/dist/workers/stages/encode/encode.worker.js.map +1 -0
  99. package/dist/{stages/mux/MP4Muxer.js → workers/stages/mux/mux.worker.js} +244 -5
  100. package/dist/workers/stages/mux/mux.worker.js.map +1 -0
  101. package/package.json +27 -21
  102. package/dist/model/types.js +0 -5
  103. package/dist/model/types.js.map +0 -1
  104. package/dist/plugins/BackpressureMonitor.js +0 -62
  105. package/dist/plugins/BackpressureMonitor.js.map +0 -1
  106. package/dist/stages/compose/AudioDucker.js +0 -161
  107. package/dist/stages/compose/AudioDucker.js.map +0 -1
  108. package/dist/stages/compose/AudioMixer.js +0 -373
  109. package/dist/stages/compose/AudioMixer.js.map +0 -1
  110. package/dist/stages/compose/FilterProcessor.js +0 -226
  111. package/dist/stages/compose/FilterProcessor.js.map +0 -1
  112. package/dist/stages/compose/LayerRenderer.js +0 -215
  113. package/dist/stages/compose/LayerRenderer.js.map +0 -1
  114. package/dist/stages/compose/TransitionProcessor.js +0 -189
  115. package/dist/stages/compose/TransitionProcessor.js.map +0 -1
  116. package/dist/stages/compose/VideoComposer.js +0 -186
  117. package/dist/stages/compose/VideoComposer.js.map +0 -1
  118. package/dist/stages/compose/audio-compose.worker.d.ts +0 -79
  119. package/dist/stages/compose/audio-compose.worker.d.ts.map +0 -1
  120. package/dist/stages/compose/audio-compose.worker.js +0 -541
  121. package/dist/stages/compose/audio-compose.worker.js.map +0 -1
  122. package/dist/stages/compose/video-compose.worker.d.ts +0 -60
  123. package/dist/stages/compose/video-compose.worker.d.ts.map +0 -1
  124. package/dist/stages/compose/video-compose.worker.js +0 -369
  125. package/dist/stages/compose/video-compose.worker.js.map +0 -1
  126. package/dist/stages/decode/AudioChunkDecoder.js +0 -83
  127. package/dist/stages/decode/AudioChunkDecoder.js.map +0 -1
  128. package/dist/stages/decode/BaseDecoder.js +0 -130
  129. package/dist/stages/decode/BaseDecoder.js.map +0 -1
  130. package/dist/stages/decode/VideoChunkDecoder.js +0 -209
  131. package/dist/stages/decode/VideoChunkDecoder.js.map +0 -1
  132. package/dist/stages/decode/decode.worker.d.ts +0 -70
  133. package/dist/stages/decode/decode.worker.d.ts.map +0 -1
  134. package/dist/stages/decode/decode.worker.js.map +0 -1
  135. package/dist/stages/demux/MP3FrameParser.js +0 -186
  136. package/dist/stages/demux/MP3FrameParser.js.map +0 -1
  137. package/dist/stages/demux/MP4Demuxer.js.map +0 -1
  138. package/dist/stages/demux/audio-demux.worker.d.ts +0 -51
  139. package/dist/stages/demux/audio-demux.worker.d.ts.map +0 -1
  140. package/dist/stages/demux/audio-demux.worker.js.map +0 -1
  141. package/dist/stages/demux/video-demux.worker.d.ts +0 -48
  142. package/dist/stages/demux/video-demux.worker.d.ts.map +0 -1
  143. package/dist/stages/demux/video-demux.worker.js.map +0 -1
  144. package/dist/stages/encode/AudioChunkEncoder.js +0 -37
  145. package/dist/stages/encode/AudioChunkEncoder.js.map +0 -1
  146. package/dist/stages/encode/BaseEncoder.js +0 -164
  147. package/dist/stages/encode/BaseEncoder.js.map +0 -1
  148. package/dist/stages/encode/VideoChunkEncoder.js +0 -50
  149. package/dist/stages/encode/VideoChunkEncoder.js.map +0 -1
  150. package/dist/stages/encode/encode.worker.d.ts +0 -3
  151. package/dist/stages/encode/encode.worker.d.ts.map +0 -1
  152. package/dist/stages/encode/encode.worker.js.map +0 -1
  153. package/dist/stages/mux/MP4Muxer.js.map +0 -1
  154. package/dist/stages/mux/mux.worker.d.ts +0 -65
  155. package/dist/stages/mux/mux.worker.d.ts.map +0 -1
  156. package/dist/stages/mux/mux.worker.js +0 -219
  157. package/dist/stages/mux/mux.worker.js.map +0 -1
  158. package/dist/stages/mux/utils.js +0 -34
  159. package/dist/stages/mux/utils.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@meframe/core",
3
- "version": "0.0.1",
3
+ "version": "0.0.3",
4
4
  "description": "Next generation media processing framework based on WebCodecs",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -26,6 +26,10 @@
26
26
  "./plugins": {
27
27
  "types": "./dist/plugins/index.d.ts",
28
28
  "import": "./dist/plugins/index.js"
29
+ },
30
+ "./vite-plugin": {
31
+ "types": "./dist/vite-plugin.d.ts",
32
+ "import": "./dist/vite-plugin.js"
29
33
  }
30
34
  },
31
35
  "files": [
@@ -34,25 +38,6 @@
34
38
  "LICENSE",
35
39
  "CHANGELOG.md"
36
40
  ],
37
- "scripts": {
38
- "dev": "vite",
39
- "build": "tsc && vite build",
40
- "preview": "vite preview",
41
- "prepublishOnly": "npm run lint && npm run type-check && npm run build",
42
- "test": "vitest run",
43
- "test:watch": "vitest",
44
- "test:coverage": "vitest run --coverage",
45
- "test:ui": "vitest --ui",
46
- "test:integration": "vitest run --config vitest.integration.config.ts",
47
- "test:e2e": "playwright test --project=chromium",
48
- "test:e2e:ui": "playwright test --project=chromium --ui",
49
- "test:e2e:debug": "playwright test --project=chromium --debug",
50
- "lint": "eslint src --ignore-pattern '**/__tests__' --ignore-pattern '**/*.test.ts' --ignore-pattern '**/*.spec.ts'",
51
- "lint:fix": "eslint src --fix --ignore-pattern '**/__tests__' --ignore-pattern '**/*.test.ts' --ignore-pattern '**/*.spec.ts'",
52
- "type-check": "tsc --noEmit",
53
- "clean": "rm -rf dist coverage",
54
- "playwright:install": "playwright install chromium"
55
- },
56
41
  "dependencies": {
57
42
  "mp4box": "^0.5.2"
58
43
  },
@@ -69,6 +54,8 @@
69
54
  "happy-dom": "^18.0.1",
70
55
  "jsdom": "^26.1.0",
71
56
  "playwright": "^1.55.0",
57
+ "rollup-plugin-delete": "^3.0.1",
58
+ "terser": "^5.44.0",
72
59
  "ts-node": "^10.9.2",
73
60
  "typescript": "^5.3.3",
74
61
  "vite": "^5.1.4",
@@ -101,5 +88,24 @@
101
88
  "publishConfig": {
102
89
  "access": "public",
103
90
  "registry": "https://registry.npmjs.org/"
91
+ },
92
+ "scripts": {
93
+ "dev": "vite",
94
+ "build": "tsc && vite build && pnpm build:workers",
95
+ "build:workers": "vite build --config vite.worker.config.ts",
96
+ "preview": "vite preview",
97
+ "test": "vitest run",
98
+ "test:watch": "vitest",
99
+ "test:coverage": "vitest run --coverage",
100
+ "test:ui": "vitest --ui",
101
+ "test:integration": "vitest run --config vitest.integration.config.ts",
102
+ "test:e2e": "playwright test --project=chromium",
103
+ "test:e2e:ui": "playwright test --project=chromium --ui",
104
+ "test:e2e:debug": "playwright test --project=chromium --debug",
105
+ "lint": "eslint src --ignore-pattern '**/__tests__' --ignore-pattern '**/*.test.ts' --ignore-pattern '**/*.spec.ts' --ignore-pattern '**/*.worker.ts'",
106
+ "lint:fix": "eslint src --fix --ignore-pattern '**/__tests__' --ignore-pattern '**/*.test.ts' --ignore-pattern '**/*.spec.ts' --ignore-pattern '**/*.worker.ts'",
107
+ "type-check": "tsc --noEmit",
108
+ "clean": "rm -rf dist coverage",
109
+ "playwright:install": "playwright install chromium"
104
110
  }
105
- }
111
+ }
@@ -1,5 +0,0 @@
1
- const MICROSECONDS_PER_SECOND = 1e6;
2
- export {
3
- MICROSECONDS_PER_SECOND
4
- };
5
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sources":["../../src/model/types.ts"],"sourcesContent":["// All time values in microseconds (µs)\nexport type TimeUs = number; // 1 second = 1_000_000 µs\n\n// Helper constants\nexport const MICROSECONDS_PER_SECOND = 1_000_000;\nexport const MICROSECONDS_PER_MILLISECOND = 1_000;\n\n// ────── Root Object ──────\nexport interface CompositionModelData {\n version: '1.0';\n fps: 24 | 25 | 30 | 60;\n durationUs: TimeUs;\n\n tracks: Track[];\n resources: Record<string, Resource>;\n\n renderConfig?: RenderConfig;\n\n ext?: Record<string, unknown>;\n}\n\nexport interface RenderConfig {\n width: number;\n height: number;\n backgroundColor?: string;\n}\n\n// ────── Track ──────\nexport interface Track {\n id: string;\n kind: 'video' | 'audio' | 'caption' | 'fx';\n clips: Clip[];\n\n effects?: Effect[];\n duckingRules?: DuckingRule[];\n}\n\n// ────── Clip ──────\nexport interface Clip {\n id: string;\n resourceId: string;\n startUs: TimeUs;\n durationUs: TimeUs;\n trackId?: string;\n trackKind?: 'video' | 'audio' | 'caption' | 'fx';\n\n trimStartUs?: TimeUs;\n trimEndUs?: TimeUs;\n\n effects?: Effect[];\n attachments?: Attachment[];\n\n transitionIn?: Transition;\n transitionOut?: Transition;\n}\n\n// ────── Resource ──────\nexport interface Resource {\n id: string;\n type: 'video' | 'image' | 'audio' | 'json' | string;\n uri: string;\n metadata?: Record<string, unknown>;\n clipIds?: string[];\n // Runtime state maintained by engine\n state?: 'pending' | 'loading' | 'ready' | 'error';\n}\n\n// ────── Common Structures ──────\nexport interface Effect {\n id: string;\n effectType: 'filter' | 'lut' | 'animation' | string;\n params?: Record<string, unknown>;\n}\n\nexport interface Transition {\n id: string;\n transitionType: 'fade' | 'wipe' | 'slide' | string;\n durationUs: TimeUs;\n curve?: 'linear' | 'ease-in' | 'ease-out' | string;\n params?: Record<string, unknown>;\n}\n\nexport interface Attachment {\n id: string;\n kind: 'caption' | 'sticker' | 'mask' | string;\n startUs: TimeUs;\n durationUs: TimeUs;\n data: Record<string, unknown>;\n}\n\nexport interface DuckingRule {\n targetTrackKind: 'voice' | 'audio' | string;\n ratio: number;\n attackMs: number;\n releaseMs: number;\n}\n\n// ────── Patch System ──────\nexport interface CompositionPatch {\n operations: PatchOperation[];\n metadata?: {\n timestamp: number;\n source?: string;\n version?: string;\n };\n}\n\nexport type PatchOperation =\n | TrackOperation\n | ClipOperation\n | ResourceOperation\n | AttachmentOperation\n | TransitionOperation\n | EffectOperation\n | RenderConfigOperation;\n\n// Track operations\nexport interface TrackOperation {\n type: 'addTrack' | 'updateTrack' | 'removeTrack';\n trackId?: string;\n track?: Partial<Track>;\n}\n\n// Clip operations\nexport interface ClipOperation {\n type: 'addClip' | 'updateClip' | 'removeClip' | 'moveClip';\n trackId: string;\n clipId?: string;\n clip?: Partial<Clip>;\n targetTrackId?: string;\n targetStartUs?: TimeUs;\n}\n\n// Resource operations\nexport interface ResourceOperation {\n type: 'addResource' | 'updateResource' | 'removeResource';\n resourceId: string;\n resource?: Partial<Resource>;\n}\n\n// Attachment operations\nexport interface AttachmentOperation {\n type: 'addAttachment' | 'updateAttachment' | 'removeAttachment';\n trackId: string;\n clipId: string;\n attachmentId?: string;\n attachment?: Partial<Attachment>;\n}\n\n// Transition operations\nexport interface TransitionOperation {\n type: 'addTransition' | 'updateTransition' | 'removeTransition';\n trackId: string;\n clipId: string;\n position: 'in' | 'out';\n transition?: Partial<Transition>;\n}\n\n// Render config operations\nexport interface RenderConfigOperation {\n type: 'updateRenderConfig';\n renderConfig?: Partial<RenderConfig>;\n}\n\n// Effect operations\nexport interface EffectOperation {\n type: 'addEffect' | 'updateEffect' | 'removeEffect';\n targetType: 'track' | 'clip';\n targetId: string;\n effectId?: string;\n effect?: Partial<Effect>;\n}\n\n// ────── Dirty Range ──────\nexport interface DirtyRange {\n trackId: string;\n startUs: TimeUs;\n endUs: TimeUs;\n reason: string;\n}\n\n// ────── Validation ──────\nexport interface ValidationError {\n path: string;\n message: string;\n value: any;\n}\n"],"names":[],"mappings":"AAIO,MAAM,0BAA0B;"}
@@ -1,62 +0,0 @@
1
- class BackpressureMonitor {
2
- metrics = /* @__PURE__ */ new Map();
3
- /**
4
- * Update metrics for a stage
5
- */
6
- updateMetrics(stage, desiredSize, queueSize = 0) {
7
- const isPaused = desiredSize <= 0;
8
- this.metrics.set(stage, {
9
- desiredSize,
10
- queueSize,
11
- isPaused,
12
- lastUpdate: Date.now()
13
- });
14
- }
15
- /**
16
- * Get current metrics snapshot
17
- */
18
- getSnapshot() {
19
- const now = Date.now();
20
- const snapshot = {};
21
- for (const [stage, metrics] of this.metrics) {
22
- snapshot[stage] = {
23
- ...metrics,
24
- age: now - metrics.lastUpdate
25
- };
26
- }
27
- return snapshot;
28
- }
29
- /**
30
- * Check if any stage is experiencing backpressure
31
- */
32
- hasBackpressure() {
33
- for (const metrics of this.metrics.values()) {
34
- if (metrics.isPaused) {
35
- return true;
36
- }
37
- }
38
- return false;
39
- }
40
- /**
41
- * Get stages currently experiencing backpressure
42
- */
43
- getBottlenecks() {
44
- const bottlenecks = [];
45
- for (const [stage, metrics] of this.metrics) {
46
- if (metrics.isPaused) {
47
- bottlenecks.push(stage);
48
- }
49
- }
50
- return bottlenecks;
51
- }
52
- /**
53
- * Clear all metrics
54
- */
55
- clear() {
56
- this.metrics.clear();
57
- }
58
- }
59
- export {
60
- BackpressureMonitor
61
- };
62
- //# sourceMappingURL=BackpressureMonitor.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"BackpressureMonitor.js","sources":["../../src/plugins/BackpressureMonitor.ts"],"sourcesContent":["/**\n * Monitor and report backpressure status across pipeline stages\n * This is a runtime monitoring tool used by plugins\n */\nexport class BackpressureMonitor {\n private metrics = new Map<\n string,\n {\n desiredSize: number;\n queueSize: number;\n isPaused: boolean;\n lastUpdate: number;\n }\n >();\n\n /**\n * Update metrics for a stage\n */\n updateMetrics(stage: string, desiredSize: number, queueSize: number = 0): void {\n const isPaused = desiredSize <= 0;\n this.metrics.set(stage, {\n desiredSize,\n queueSize,\n isPaused,\n lastUpdate: Date.now(),\n });\n }\n\n /**\n * Get current metrics snapshot\n */\n getSnapshot(): Record<\n string,\n {\n desiredSize: number;\n queueSize: number;\n isPaused: boolean;\n age: number;\n }\n > {\n const now = Date.now();\n const snapshot: Record<string, any> = {};\n\n for (const [stage, metrics] of this.metrics) {\n snapshot[stage] = {\n ...metrics,\n age: now - metrics.lastUpdate,\n };\n }\n\n return snapshot;\n }\n\n /**\n * Check if any stage is experiencing backpressure\n */\n hasBackpressure(): boolean {\n for (const metrics of this.metrics.values()) {\n if (metrics.isPaused) {\n return true;\n }\n }\n return false;\n }\n\n /**\n * Get stages currently experiencing backpressure\n */\n getBottlenecks(): string[] {\n const bottlenecks: string[] = [];\n for (const [stage, metrics] of this.metrics) {\n if (metrics.isPaused) {\n bottlenecks.push(stage);\n }\n }\n return bottlenecks;\n }\n\n /**\n * Clear all metrics\n */\n clear(): void {\n this.metrics.clear();\n }\n}\n"],"names":[],"mappings":"AAIO,MAAM,oBAAoB;AAAA,EACvB,8BAAc,IAAA;AAAA;AAAA;AAAA;AAAA,EAatB,cAAc,OAAe,aAAqB,YAAoB,GAAS;AAC7E,UAAM,WAAW,eAAe;AAChC,SAAK,QAAQ,IAAI,OAAO;AAAA,MACtB;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,KAAK,IAAA;AAAA,IAAI,CACtB;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,cAQE;AACA,UAAM,MAAM,KAAK,IAAA;AACjB,UAAM,WAAgC,CAAA;AAEtC,eAAW,CAAC,OAAO,OAAO,KAAK,KAAK,SAAS;AAC3C,eAAS,KAAK,IAAI;AAAA,QAChB,GAAG;AAAA,QACH,KAAK,MAAM,QAAQ;AAAA,MAAA;AAAA,IAEvB;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,kBAA2B;AACzB,eAAW,WAAW,KAAK,QAAQ,OAAA,GAAU;AAC3C,UAAI,QAAQ,UAAU;AACpB,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,iBAA2B;AACzB,UAAM,cAAwB,CAAA;AAC9B,eAAW,CAAC,OAAO,OAAO,KAAK,KAAK,SAAS;AAC3C,UAAI,QAAQ,UAAU;AACpB,oBAAY,KAAK,KAAK;AAAA,MACxB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,SAAK,QAAQ,MAAA;AAAA,EACf;AACF;"}
@@ -1,161 +0,0 @@
1
- class AudioDucker {
2
- config = null;
3
- sampleRate;
4
- constructor(sampleRate) {
5
- this.sampleRate = sampleRate;
6
- }
7
- configure(config) {
8
- this.config = config;
9
- }
10
- /**
11
- * Analyze trigger tracks (voice) and generate ducking envelope
12
- * Returns gain values (0-1) to apply to target tracks (BGM)
13
- */
14
- async generateDuckingEnvelope(tracks, frameCount) {
15
- if (!this.config?.enabled) {
16
- return new Float32Array(frameCount).fill(1);
17
- }
18
- const envelope = new Float32Array(frameCount);
19
- envelope.fill(1);
20
- const triggerTracks = tracks.filter((t) => this.config.triggerTracks.includes(t.trackId));
21
- if (triggerTracks.length === 0) {
22
- return envelope;
23
- }
24
- for (const track of triggerTracks) {
25
- const voiceActivity = await this.detectVoiceActivity(track.audioData);
26
- this.applyDuckingToEnvelope(envelope, voiceActivity);
27
- }
28
- return envelope;
29
- }
30
- /**
31
- * Voice Activity Detection (VAD)
32
- * Simple energy-based detection with smoothing
33
- * More sophisticated implementations could use:
34
- * - Zero-crossing rate (ZCR) for speech/music discrimination
35
- * - Spectral centroid for voice frequency detection
36
- * - Machine learning models for robust VAD
37
- */
38
- async detectVoiceActivity(audioData) {
39
- const frameCount = audioData.numberOfFrames;
40
- const activity = new Float32Array(frameCount);
41
- const monoData = new Float32Array(frameCount);
42
- const channelData = new Float32Array(frameCount);
43
- for (let ch = 0; ch < audioData.numberOfChannels; ch++) {
44
- audioData.copyTo(channelData, {
45
- planeIndex: ch,
46
- format: "f32-planar"
47
- });
48
- for (let i = 0; i < frameCount; i++) {
49
- if (monoData && channelData) {
50
- monoData[i] = (monoData[i] || 0) + (channelData[i] || 0) / audioData.numberOfChannels;
51
- }
52
- }
53
- }
54
- const windowSize = Math.floor(this.sampleRate * 0.02);
55
- const hopSize = Math.floor(windowSize / 2);
56
- for (let i = 0; i < frameCount; i += hopSize) {
57
- const end = Math.min(i + windowSize, frameCount);
58
- let energy = 0;
59
- for (let j = i; j < end; j++) {
60
- if (monoData && monoData[j] !== void 0) {
61
- const sample = monoData[j];
62
- if (sample !== void 0) {
63
- energy += sample * sample;
64
- }
65
- }
66
- }
67
- energy = Math.sqrt(energy / (end - i));
68
- const threshold = 0.01;
69
- const isVoice = energy > threshold;
70
- for (let j = i; j < end; j++) {
71
- activity[j] = isVoice ? 1 : 0;
72
- }
73
- }
74
- return this.smoothActivityDetection(activity);
75
- }
76
- /**
77
- * Smooth voice activity detection to avoid choppy ducking
78
- * Uses a simple moving average filter
79
- */
80
- smoothActivityDetection(activity) {
81
- const smoothed = new Float32Array(activity.length);
82
- const smoothWindow = Math.floor(this.sampleRate * 0.05);
83
- for (let i = 0; i < activity.length; i++) {
84
- let sum = 0;
85
- let count = 0;
86
- for (let j = Math.max(0, i - smoothWindow); j <= Math.min(activity.length - 1, i + smoothWindow); j++) {
87
- if (activity && activity[j] !== void 0) {
88
- const val = activity[j];
89
- if (val !== void 0) {
90
- sum += val;
91
- }
92
- }
93
- count++;
94
- }
95
- smoothed[i] = sum / count;
96
- }
97
- return smoothed;
98
- }
99
- /**
100
- * Apply ducking based on voice activity
101
- * Implements attack/release envelope shaping
102
- */
103
- applyDuckingToEnvelope(envelope, voiceActivity) {
104
- if (!this.config) return;
105
- const duckingLevel = 1 - this.config.duckingLevel;
106
- const attackSamples = Math.floor(this.config.attackTimeMs / 1e3 * this.sampleRate);
107
- const releaseSamples = Math.floor(this.config.releaseTimeMs / 1e3 * this.sampleRate);
108
- const lookAheadSamples = this.config.lookAheadMs ? Math.floor(this.config.lookAheadMs / 1e3 * this.sampleRate) : 0;
109
- let currentGain = 1;
110
- let releaseCounter = 0;
111
- for (let i = 0; i < envelope.length; i++) {
112
- const lookAheadIndex = Math.min(i + lookAheadSamples, voiceActivity.length - 1);
113
- const activity = voiceActivity[lookAheadIndex];
114
- if (activity !== void 0 && activity > 0.5) {
115
- if (currentGain > duckingLevel) {
116
- currentGain = Math.max(duckingLevel, currentGain - (1 - duckingLevel) / attackSamples);
117
- } else {
118
- currentGain = duckingLevel;
119
- }
120
- releaseCounter = 0;
121
- } else if (currentGain < 1) {
122
- releaseCounter++;
123
- if (releaseCounter > releaseSamples * 0.1) {
124
- currentGain = Math.min(1, currentGain + (1 - duckingLevel) / releaseSamples);
125
- }
126
- }
127
- envelope[i] = Math.min(envelope[i] || 1, currentGain);
128
- }
129
- }
130
- /**
131
- * Apply ducking envelope to audio buffer
132
- * This modulates the volume over time according to the envelope
133
- */
134
- applyEnvelopeToVolume(baseVolume, envelope) {
135
- const result = new Float32Array(envelope.length);
136
- for (let i = 0; i < envelope.length; i++) {
137
- result[i] = baseVolume * (envelope[i] || 1);
138
- }
139
- return result;
140
- }
141
- /**
142
- * Calculate dynamic range to avoid over-compression
143
- * Returns the difference between peak and RMS levels in dB
144
- */
145
- calculateDynamicRange(envelope) {
146
- let peak = 0;
147
- let sumSquares = 0;
148
- for (const value of envelope) {
149
- peak = Math.max(peak, value);
150
- sumSquares += value * value;
151
- }
152
- const rms = Math.sqrt(sumSquares / envelope.length);
153
- const peakDb = 20 * Math.log10(peak);
154
- const rmsDb = 20 * Math.log10(rms);
155
- return peakDb - rmsDb;
156
- }
157
- }
158
- export {
159
- AudioDucker
160
- };
161
- //# sourceMappingURL=AudioDucker.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"AudioDucker.js","sources":["../../../src/stages/compose/AudioDucker.ts"],"sourcesContent":["import type { DuckingConfig, MixRequest } from './types';\n\n/**\n * AudioDucker - Automatic volume ducking for background music\n *\n * Ducking: Audio engineering technique where the volume of one audio source\n * is automatically reduced when another audio source is present.\n *\n * Common use case: Reduce background music volume when voice/narration plays\n * to improve speech intelligibility without completely muting the music.\n *\n * Key parameters:\n * - Threshold: Level at which ducking triggers\n * - Ratio: How much to reduce volume (e.g., 3:1 means reduce by 1/3)\n * - Attack: How quickly volume reduces (typically 10-50ms)\n * - Release: How quickly volume returns to normal (typically 100-500ms)\n * - Hold: Time to maintain ducking after trigger ends\n */\nexport class AudioDucker {\n private config: DuckingConfig | null = null;\n private sampleRate: number;\n\n constructor(sampleRate: number) {\n this.sampleRate = sampleRate;\n }\n\n configure(config: DuckingConfig): void {\n this.config = config;\n }\n\n /**\n * Analyze trigger tracks (voice) and generate ducking envelope\n * Returns gain values (0-1) to apply to target tracks (BGM)\n */\n async generateDuckingEnvelope(\n tracks: MixRequest['tracks'],\n frameCount: number\n ): Promise<Float32Array> {\n if (!this.config?.enabled) {\n return new Float32Array(frameCount).fill(1.0);\n }\n\n // Initialize envelope with no ducking (gain = 1.0)\n const envelope = new Float32Array(frameCount);\n envelope.fill(1.0);\n\n // Find trigger tracks (typically voice/narration)\n const triggerTracks = tracks.filter((t) => this.config!.triggerTracks.includes(t.trackId));\n\n if (triggerTracks.length === 0) {\n return envelope;\n }\n\n // Analyze each trigger track for voice activity\n for (const track of triggerTracks) {\n const voiceActivity = await this.detectVoiceActivity(track.audioData);\n this.applyDuckingToEnvelope(envelope, voiceActivity);\n }\n\n return envelope;\n }\n\n /**\n * Voice Activity Detection (VAD)\n * Simple energy-based detection with smoothing\n * More sophisticated implementations could use:\n * - Zero-crossing rate (ZCR) for speech/music discrimination\n * - Spectral centroid for voice frequency detection\n * - Machine learning models for robust VAD\n */\n private async detectVoiceActivity(audioData: AudioData): Promise<Float32Array> {\n const frameCount = audioData.numberOfFrames;\n const activity = new Float32Array(frameCount);\n\n // Convert to mono for analysis\n const monoData = new Float32Array(frameCount);\n const channelData = new Float32Array(frameCount);\n\n for (let ch = 0; ch < audioData.numberOfChannels; ch++) {\n audioData.copyTo(channelData, {\n planeIndex: ch,\n format: 'f32-planar' as const,\n });\n\n for (let i = 0; i < frameCount; i++) {\n if (monoData && channelData) {\n monoData[i] = (monoData[i] || 0) + (channelData[i] || 0) / audioData.numberOfChannels;\n }\n }\n }\n\n // Energy calculation with windowing\n // Window size: 20ms is typical for speech analysis\n const windowSize = Math.floor(this.sampleRate * 0.02);\n const hopSize = Math.floor(windowSize / 2); // 50% overlap\n\n for (let i = 0; i < frameCount; i += hopSize) {\n const end = Math.min(i + windowSize, frameCount);\n\n // Calculate RMS energy in window\n let energy = 0;\n for (let j = i; j < end; j++) {\n if (monoData && monoData[j] !== undefined) {\n const sample = monoData[j];\n if (sample !== undefined) {\n energy += sample * sample;\n }\n }\n }\n energy = Math.sqrt(energy / (end - i));\n\n // Simple threshold-based VAD\n // Typical speech energy threshold: -40dB to -30dB\n const threshold = 0.01; // Approximately -40dB\n const isVoice = energy > threshold;\n\n // Fill activity array for this window\n for (let j = i; j < end; j++) {\n activity[j] = isVoice ? 1.0 : 0.0;\n }\n }\n\n // Smooth activity detection to avoid rapid changes\n return this.smoothActivityDetection(activity);\n }\n\n /**\n * Smooth voice activity detection to avoid choppy ducking\n * Uses a simple moving average filter\n */\n private smoothActivityDetection(activity: Float32Array): Float32Array {\n const smoothed = new Float32Array(activity.length);\n const smoothWindow = Math.floor(this.sampleRate * 0.05); // 50ms smoothing\n\n for (let i = 0; i < activity.length; i++) {\n let sum = 0;\n let count = 0;\n\n for (\n let j = Math.max(0, i - smoothWindow);\n j <= Math.min(activity.length - 1, i + smoothWindow);\n j++\n ) {\n if (activity && activity[j] !== undefined) {\n const val = activity[j];\n if (val !== undefined) {\n sum += val;\n }\n }\n count++;\n }\n\n smoothed[i] = sum / count;\n }\n\n return smoothed;\n }\n\n /**\n * Apply ducking based on voice activity\n * Implements attack/release envelope shaping\n */\n private applyDuckingToEnvelope(envelope: Float32Array, voiceActivity: Float32Array): void {\n if (!this.config) return;\n\n const duckingLevel = 1.0 - this.config.duckingLevel;\n const attackSamples = Math.floor((this.config.attackTimeMs / 1000) * this.sampleRate);\n const releaseSamples = Math.floor((this.config.releaseTimeMs / 1000) * this.sampleRate);\n const lookAheadSamples = this.config.lookAheadMs\n ? Math.floor((this.config.lookAheadMs / 1000) * this.sampleRate)\n : 0;\n\n let currentGain = 1.0;\n let releaseCounter = 0;\n\n for (let i = 0; i < envelope.length; i++) {\n // Look ahead for upcoming voice activity\n const lookAheadIndex = Math.min(i + lookAheadSamples, voiceActivity.length - 1);\n const activity = voiceActivity[lookAheadIndex];\n\n if (activity !== undefined && activity > 0.5) {\n // Voice detected - apply ducking with attack curve\n if (currentGain > duckingLevel) {\n // Attack phase - reduce gain\n currentGain = Math.max(duckingLevel, currentGain - (1.0 - duckingLevel) / attackSamples);\n } else {\n currentGain = duckingLevel;\n }\n releaseCounter = 0;\n } else if (currentGain < 1.0) {\n // No voice - apply release curve\n releaseCounter++;\n if (releaseCounter > releaseSamples * 0.1) {\n // Small hold time\n currentGain = Math.min(1.0, currentGain + (1.0 - duckingLevel) / releaseSamples);\n }\n }\n\n // Apply the calculated gain\n envelope[i] = Math.min(envelope[i] || 1, currentGain);\n }\n }\n\n /**\n * Apply ducking envelope to audio buffer\n * This modulates the volume over time according to the envelope\n */\n applyEnvelopeToVolume(baseVolume: number, envelope: Float32Array): Float32Array {\n const result = new Float32Array(envelope.length);\n for (let i = 0; i < envelope.length; i++) {\n result[i] = baseVolume * (envelope[i] || 1);\n }\n return result;\n }\n\n /**\n * Calculate dynamic range to avoid over-compression\n * Returns the difference between peak and RMS levels in dB\n */\n calculateDynamicRange(envelope: Float32Array): number {\n let peak = 0;\n let sumSquares = 0;\n\n for (const value of envelope) {\n peak = Math.max(peak, value);\n sumSquares += value * value;\n }\n\n const rms = Math.sqrt(sumSquares / envelope.length);\n\n // Convert to dB (20 * log10(ratio))\n const peakDb = 20 * Math.log10(peak);\n const rmsDb = 20 * Math.log10(rms);\n\n return peakDb - rmsDb;\n }\n}\n"],"names":[],"mappings":"AAkBO,MAAM,YAAY;AAAA,EACf,SAA+B;AAAA,EAC/B;AAAA,EAER,YAAY,YAAoB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,UAAU,QAA6B;AACrC,SAAK,SAAS;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,wBACJ,QACA,YACuB;AACvB,QAAI,CAAC,KAAK,QAAQ,SAAS;AACzB,aAAO,IAAI,aAAa,UAAU,EAAE,KAAK,CAAG;AAAA,IAC9C;AAGA,UAAM,WAAW,IAAI,aAAa,UAAU;AAC5C,aAAS,KAAK,CAAG;AAGjB,UAAM,gBAAgB,OAAO,OAAO,CAAC,MAAM,KAAK,OAAQ,cAAc,SAAS,EAAE,OAAO,CAAC;AAEzF,QAAI,cAAc,WAAW,GAAG;AAC9B,aAAO;AAAA,IACT;AAGA,eAAW,SAAS,eAAe;AACjC,YAAM,gBAAgB,MAAM,KAAK,oBAAoB,MAAM,SAAS;AACpE,WAAK,uBAAuB,UAAU,aAAa;AAAA,IACrD;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAc,oBAAoB,WAA6C;AAC7E,UAAM,aAAa,UAAU;AAC7B,UAAM,WAAW,IAAI,aAAa,UAAU;AAG5C,UAAM,WAAW,IAAI,aAAa,UAAU;AAC5C,UAAM,cAAc,IAAI,aAAa,UAAU;AAE/C,aAAS,KAAK,GAAG,KAAK,UAAU,kBAAkB,MAAM;AACtD,gBAAU,OAAO,aAAa;AAAA,QAC5B,YAAY;AAAA,QACZ,QAAQ;AAAA,MAAA,CACT;AAED,eAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,YAAI,YAAY,aAAa;AAC3B,mBAAS,CAAC,KAAK,SAAS,CAAC,KAAK,MAAM,YAAY,CAAC,KAAK,KAAK,UAAU;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAIA,UAAM,aAAa,KAAK,MAAM,KAAK,aAAa,IAAI;AACpD,UAAM,UAAU,KAAK,MAAM,aAAa,CAAC;AAEzC,aAAS,IAAI,GAAG,IAAI,YAAY,KAAK,SAAS;AAC5C,YAAM,MAAM,KAAK,IAAI,IAAI,YAAY,UAAU;AAG/C,UAAI,SAAS;AACb,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,YAAI,YAAY,SAAS,CAAC,MAAM,QAAW;AACzC,gBAAM,SAAS,SAAS,CAAC;AACzB,cAAI,WAAW,QAAW;AACxB,sBAAU,SAAS;AAAA,UACrB;AAAA,QACF;AAAA,MACF;AACA,eAAS,KAAK,KAAK,UAAU,MAAM,EAAE;AAIrC,YAAM,YAAY;AAClB,YAAM,UAAU,SAAS;AAGzB,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,iBAAS,CAAC,IAAI,UAAU,IAAM;AAAA,MAChC;AAAA,IACF;AAGA,WAAO,KAAK,wBAAwB,QAAQ;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,wBAAwB,UAAsC;AACpE,UAAM,WAAW,IAAI,aAAa,SAAS,MAAM;AACjD,UAAM,eAAe,KAAK,MAAM,KAAK,aAAa,IAAI;AAEtD,aAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAI,MAAM;AACV,UAAI,QAAQ;AAEZ,eACM,IAAI,KAAK,IAAI,GAAG,IAAI,YAAY,GACpC,KAAK,KAAK,IAAI,SAAS,SAAS,GAAG,IAAI,YAAY,GACnD,KACA;AACA,YAAI,YAAY,SAAS,CAAC,MAAM,QAAW;AACzC,gBAAM,MAAM,SAAS,CAAC;AACtB,cAAI,QAAQ,QAAW;AACrB,mBAAO;AAAA,UACT;AAAA,QACF;AACA;AAAA,MACF;AAEA,eAAS,CAAC,IAAI,MAAM;AAAA,IACtB;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,uBAAuB,UAAwB,eAAmC;AACxF,QAAI,CAAC,KAAK,OAAQ;AAElB,UAAM,eAAe,IAAM,KAAK,OAAO;AACvC,UAAM,gBAAgB,KAAK,MAAO,KAAK,OAAO,eAAe,MAAQ,KAAK,UAAU;AACpF,UAAM,iBAAiB,KAAK,MAAO,KAAK,OAAO,gBAAgB,MAAQ,KAAK,UAAU;AACtF,UAAM,mBAAmB,KAAK,OAAO,cACjC,KAAK,MAAO,KAAK,OAAO,cAAc,MAAQ,KAAK,UAAU,IAC7D;AAEJ,QAAI,cAAc;AAClB,QAAI,iBAAiB;AAErB,aAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AAExC,YAAM,iBAAiB,KAAK,IAAI,IAAI,kBAAkB,cAAc,SAAS,CAAC;AAC9E,YAAM,WAAW,cAAc,cAAc;AAE7C,UAAI,aAAa,UAAa,WAAW,KAAK;AAE5C,YAAI,cAAc,cAAc;AAE9B,wBAAc,KAAK,IAAI,cAAc,eAAe,IAAM,gBAAgB,aAAa;AAAA,QACzF,OAAO;AACL,wBAAc;AAAA,QAChB;AACA,yBAAiB;AAAA,MACnB,WAAW,cAAc,GAAK;AAE5B;AACA,YAAI,iBAAiB,iBAAiB,KAAK;AAEzC,wBAAc,KAAK,IAAI,GAAK,eAAe,IAAM,gBAAgB,cAAc;AAAA,QACjF;AAAA,MACF;AAGA,eAAS,CAAC,IAAI,KAAK,IAAI,SAAS,CAAC,KAAK,GAAG,WAAW;AAAA,IACtD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,sBAAsB,YAAoB,UAAsC;AAC9E,UAAM,SAAS,IAAI,aAAa,SAAS,MAAM;AAC/C,aAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAO,CAAC,IAAI,cAAc,SAAS,CAAC,KAAK;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,sBAAsB,UAAgC;AACpD,QAAI,OAAO;AACX,QAAI,aAAa;AAEjB,eAAW,SAAS,UAAU;AAC5B,aAAO,KAAK,IAAI,MAAM,KAAK;AAC3B,oBAAc,QAAQ;AAAA,IACxB;AAEA,UAAM,MAAM,KAAK,KAAK,aAAa,SAAS,MAAM;AAGlD,UAAM,SAAS,KAAK,KAAK,MAAM,IAAI;AACnC,UAAM,QAAQ,KAAK,KAAK,MAAM,GAAG;AAEjC,WAAO,SAAS;AAAA,EAClB;AACF;"}