@remotion/media 4.0.355 → 4.0.357

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/audio/audio-for-preview.d.ts +30 -0
  2. package/dist/audio/audio-for-preview.js +213 -0
  3. package/dist/audio/audio-for-rendering.js +63 -12
  4. package/dist/audio/audio.js +8 -50
  5. package/dist/audio/props.d.ts +12 -3
  6. package/dist/audio-extraction/audio-cache.d.ts +1 -1
  7. package/dist/audio-extraction/audio-cache.js +5 -1
  8. package/dist/audio-extraction/audio-iterator.d.ts +7 -3
  9. package/dist/audio-extraction/audio-iterator.js +35 -12
  10. package/dist/audio-extraction/audio-manager.d.ts +10 -38
  11. package/dist/audio-extraction/audio-manager.js +40 -11
  12. package/dist/audio-extraction/extract-audio.d.ts +11 -3
  13. package/dist/audio-extraction/extract-audio.js +37 -17
  14. package/dist/caches.d.ts +11 -45
  15. package/dist/convert-audiodata/apply-tonefrequency.d.ts +2 -0
  16. package/dist/convert-audiodata/apply-tonefrequency.js +43 -0
  17. package/dist/convert-audiodata/combine-audiodata.js +2 -23
  18. package/dist/convert-audiodata/convert-audiodata.d.ts +1 -5
  19. package/dist/convert-audiodata/convert-audiodata.js +16 -24
  20. package/dist/convert-audiodata/wsola.d.ts +13 -0
  21. package/dist/convert-audiodata/wsola.js +197 -0
  22. package/dist/esm/index.mjs +2265 -589
  23. package/dist/extract-frame-and-audio.d.ts +7 -7
  24. package/dist/extract-frame-and-audio.js +69 -26
  25. package/dist/get-sink-weak.d.ts +3 -8
  26. package/dist/get-sink-weak.js +3 -11
  27. package/dist/get-sink.d.ts +13 -0
  28. package/dist/get-sink.js +15 -0
  29. package/dist/get-time-in-seconds.d.ts +10 -0
  30. package/dist/get-time-in-seconds.js +25 -0
  31. package/dist/index.d.ts +13 -3
  32. package/dist/index.js +12 -2
  33. package/dist/is-network-error.d.ts +6 -0
  34. package/dist/is-network-error.js +17 -0
  35. package/dist/render-timestamp-range.d.ts +1 -0
  36. package/dist/render-timestamp-range.js +9 -0
  37. package/dist/video/media-player.d.ts +91 -0
  38. package/dist/video/media-player.js +484 -0
  39. package/dist/video/props.d.ts +37 -18
  40. package/dist/video/resolve-playback-time.d.ts +8 -0
  41. package/dist/video/resolve-playback-time.js +22 -0
  42. package/dist/video/timeout-utils.d.ts +2 -0
  43. package/dist/video/timeout-utils.js +18 -0
  44. package/dist/video/video-for-preview.d.ts +25 -0
  45. package/dist/video/video-for-preview.js +241 -0
  46. package/dist/video/video-for-rendering.d.ts +26 -2
  47. package/dist/video/video-for-rendering.js +95 -19
  48. package/dist/video/video.js +13 -18
  49. package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts +19 -6
  50. package/dist/video-extraction/extract-frame-via-broadcast-channel.js +67 -4
  51. package/dist/video-extraction/extract-frame.d.ts +21 -2
  52. package/dist/video-extraction/extract-frame.js +46 -9
  53. package/dist/video-extraction/get-frames-since-keyframe.d.ts +17 -10
  54. package/dist/video-extraction/get-frames-since-keyframe.js +77 -21
  55. package/dist/video-extraction/keyframe-bank.d.ts +3 -2
  56. package/dist/video-extraction/keyframe-bank.js +32 -12
  57. package/dist/video-extraction/keyframe-manager.d.ts +3 -8
  58. package/dist/video-extraction/keyframe-manager.js +25 -10
  59. package/package.json +4 -4
@@ -1,27 +1,8 @@
1
1
  import type { AudioSampleSink } from 'mediabunny';
2
- import type { LogLevel } from 'remotion';
2
+ import { type LogLevel } from 'remotion';
3
3
  import type { RememberActualMatroskaTimestamps } from '../video-extraction/remember-actual-matroska-timestamps';
4
+ import type { AudioSampleIterator } from './audio-iterator';
4
5
  export declare const makeAudioManager: () => {
5
- makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
6
- timeInSeconds: number;
7
- src: string;
8
- audioSampleSink: AudioSampleSink;
9
- isMatroska: boolean;
10
- actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
11
- }) => {
12
- src: string;
13
- getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
14
- waitForCompletion: () => Promise<boolean>;
15
- canSatisfyRequestedTime: (timestamp: number) => boolean;
16
- logOpenFrames: (logLevel: LogLevel) => void;
17
- getCacheStats: () => {
18
- count: number;
19
- size: number;
20
- };
21
- getLastUsed: () => number;
22
- prepareForDeletion: () => Promise<void>;
23
- startTimestamp: number;
24
- };
25
6
  getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
26
7
  src: string;
27
8
  timeInSeconds: number;
@@ -29,20 +10,7 @@ export declare const makeAudioManager: () => {
29
10
  isMatroska: boolean;
30
11
  actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
31
12
  logLevel: LogLevel;
32
- }) => Promise<{
33
- src: string;
34
- getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
35
- waitForCompletion: () => Promise<boolean>;
36
- canSatisfyRequestedTime: (timestamp: number) => boolean;
37
- logOpenFrames: (logLevel: LogLevel) => void;
38
- getCacheStats: () => {
39
- count: number;
40
- size: number;
41
- };
42
- getLastUsed: () => number;
43
- prepareForDeletion: () => Promise<void>;
44
- startTimestamp: number;
45
- }>;
13
+ }) => Promise<AudioSampleIterator>;
46
14
  getCacheStats: () => {
47
15
  count: number;
48
16
  totalSize: number;
@@ -52,14 +20,18 @@ export declare const makeAudioManager: () => {
52
20
  getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
53
21
  waitForCompletion: () => Promise<boolean>;
54
22
  canSatisfyRequestedTime: (timestamp: number) => boolean;
55
- logOpenFrames: (logLevel: LogLevel) => void;
23
+ logOpenFrames: () => void;
56
24
  getCacheStats: () => {
57
25
  count: number;
58
26
  size: number;
59
27
  };
60
28
  getLastUsed: () => number;
61
- prepareForDeletion: () => Promise<void>;
29
+ prepareForDeletion: () => void;
62
30
  startTimestamp: number;
31
+ clearBeforeThreshold: (threshold: number) => void;
32
+ getOldestTimestamp: () => number;
33
+ getNewestTimestamp: () => number | null;
63
34
  } | null;
64
- logOpenFrames: (logLevel: LogLevel) => void;
35
+ logOpenFrames: () => void;
36
+ deleteDuplicateIterators: (logLevel: LogLevel) => void;
65
37
  };
@@ -1,14 +1,16 @@
1
+ import { Internals } from 'remotion';
1
2
  import { getMaxVideoCacheSize, getTotalCacheStats } from '../caches';
2
3
  import { makeAudioIterator } from './audio-iterator';
3
4
  export const makeAudioManager = () => {
4
5
  const iterators = [];
5
- const makeIterator = ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }) => {
6
+ const makeIterator = ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
6
7
  const iterator = makeAudioIterator({
7
8
  audioSampleSink,
8
9
  isMatroska,
9
10
  startTimestamp: timeInSeconds,
10
11
  src,
11
12
  actualMatroskaTimestamps,
13
+ logLevel,
12
14
  });
13
15
  iterators.push(iterator);
14
16
  return iterator;
@@ -25,17 +27,30 @@ export const makeAudioManager = () => {
25
27
  }
26
28
  return mostInThePastIterator;
27
29
  };
28
- const deleteOldestIterator = async () => {
30
+ const deleteOldestIterator = () => {
29
31
  const iterator = getIteratorMostInThePast();
30
32
  if (iterator) {
31
- await iterator.prepareForDeletion();
33
+ iterator.prepareForDeletion();
32
34
  iterators.splice(iterators.indexOf(iterator), 1);
33
35
  }
34
36
  };
37
+ const deleteDuplicateIterators = (logLevel) => {
38
+ const seenKeys = new Set();
39
+ for (let i = 0; i < iterators.length; i++) {
40
+ const iterator = iterators[i];
41
+ const key = `${iterator.src}-${iterator.getOldestTimestamp()}-${iterator.getNewestTimestamp()}`;
42
+ if (seenKeys.has(key)) {
43
+ iterator.prepareForDeletion();
44
+ iterators.splice(iterators.indexOf(iterator), 1);
45
+ Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Deleted duplicate iterator for ${iterator.src}`);
46
+ }
47
+ seenKeys.add(key);
48
+ }
49
+ };
35
50
  const getIterator = async ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
36
51
  const maxCacheSize = getMaxVideoCacheSize(logLevel);
37
52
  while ((await getTotalCacheStats()).totalSize > maxCacheSize) {
38
- await deleteOldestIterator();
53
+ deleteOldestIterator();
39
54
  }
40
55
  for (const iterator of iterators) {
41
56
  if (iterator.src === src &&
@@ -44,19 +59,22 @@ export const makeAudioManager = () => {
44
59
  return iterator;
45
60
  }
46
61
  }
47
- for (const iterator of iterators) {
48
- // delete iterator with same starting timestamp
62
+ for (let i = 0; i < iterators.length; i++) {
63
+ const iterator = iterators[i];
64
+ // delete iterator with same starting timestamp as requested
49
65
  if (iterator.src === src && iterator.startTimestamp === timeInSeconds) {
50
- await iterator.prepareForDeletion();
66
+ iterator.prepareForDeletion();
51
67
  iterators.splice(iterators.indexOf(iterator), 1);
52
68
  }
53
69
  }
70
+ deleteDuplicateIterators(logLevel);
54
71
  return makeIterator({
55
72
  src,
56
73
  timeInSeconds,
57
74
  audioSampleSink,
58
75
  isMatroska,
59
76
  actualMatroskaTimestamps,
77
+ logLevel,
60
78
  });
61
79
  };
62
80
  const getCacheStats = () => {
@@ -69,16 +87,27 @@ export const makeAudioManager = () => {
69
87
  }
70
88
  return { count: totalCount, totalSize };
71
89
  };
72
- const logOpenFrames = (logLevel) => {
90
+ const logOpenFrames = () => {
73
91
  for (const iterator of iterators) {
74
- iterator.logOpenFrames(logLevel);
92
+ iterator.logOpenFrames();
75
93
  }
76
94
  };
95
+ let queue = Promise.resolve(undefined);
77
96
  return {
78
- makeIterator,
79
- getIterator,
97
+ getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
98
+ queue = queue.then(() => getIterator({
99
+ src,
100
+ timeInSeconds,
101
+ audioSampleSink,
102
+ isMatroska,
103
+ actualMatroskaTimestamps,
104
+ logLevel,
105
+ }));
106
+ return queue;
107
+ },
80
108
  getCacheStats,
81
109
  getIteratorMostInThePast,
82
110
  logOpenFrames,
111
+ deleteDuplicateIterators,
83
112
  };
84
113
  };
@@ -1,13 +1,21 @@
1
1
  import { type LogLevel } from 'remotion';
2
2
  import type { PcmS16AudioData } from '../convert-audiodata/convert-audiodata';
3
- export declare const extractAudio: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, logLevel, loop, playbackRate, }: {
3
+ type ExtractAudioReturnType = Awaited<ReturnType<typeof extractAudioInternal>>;
4
+ type ExtractAudioParams = {
4
5
  src: string;
5
6
  timeInSeconds: number;
6
7
  durationInSeconds: number;
7
8
  logLevel: LogLevel;
8
9
  loop: boolean;
9
10
  playbackRate: number;
10
- }) => Promise<{
11
+ audioStreamIndex: number;
12
+ trimBefore: number | undefined;
13
+ trimAfter: number | undefined;
14
+ fps: number;
15
+ };
16
+ declare const extractAudioInternal: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, }: ExtractAudioParams) => Promise<{
11
17
  data: PcmS16AudioData | null;
12
18
  durationInSeconds: number | null;
13
- }>;
19
+ } | "cannot-decode" | "unknown-container-format">;
20
+ export declare const extractAudio: (params: ExtractAudioParams) => Promise<ExtractAudioReturnType>;
21
+ export {};
@@ -1,20 +1,37 @@
1
1
  import { audioManager } from '../caches';
2
2
  import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
3
3
  import { convertAudioData } from '../convert-audiodata/convert-audiodata';
4
- import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
5
- import { getSinkWeak } from '../get-sink-weak';
6
- export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, logLevel, loop, playbackRate, }) => {
7
- const { audio, actualMatroskaTimestamps, isMatroska, getDuration } = await getSinkWeak(src, logLevel);
8
- let duration = null;
4
+ import { getSink } from '../get-sink';
5
+ import { getTimeInSeconds } from '../get-time-in-seconds';
6
+ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, }) => {
7
+ const { getAudio, actualMatroskaTimestamps, isMatroska, getDuration } = await getSink(src, logLevel);
8
+ let mediaDurationInSeconds = null;
9
9
  if (loop) {
10
- duration = await getDuration();
10
+ mediaDurationInSeconds = await getDuration();
11
11
  }
12
- if (audio === null) {
12
+ const audio = await getAudio(audioStreamIndex);
13
+ if (audio === 'no-audio-track') {
13
14
  return { data: null, durationInSeconds: null };
14
15
  }
15
- const timeInSeconds = loop
16
- ? unloopedTimeInSeconds % duration
17
- : unloopedTimeInSeconds;
16
+ if (audio === 'cannot-decode-audio') {
17
+ return 'cannot-decode';
18
+ }
19
+ if (audio === 'unknown-container-format') {
20
+ return 'unknown-container-format';
21
+ }
22
+ const timeInSeconds = getTimeInSeconds({
23
+ loop,
24
+ mediaDurationInSeconds,
25
+ unloopedTimeInSeconds,
26
+ src,
27
+ trimAfter,
28
+ playbackRate,
29
+ trimBefore,
30
+ fps,
31
+ });
32
+ if (timeInSeconds === null) {
33
+ return { data: null, durationInSeconds: mediaDurationInSeconds };
34
+ }
18
35
  const sampleIterator = await audioManager.getIterator({
19
36
  src,
20
37
  timeInSeconds,
@@ -23,8 +40,9 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
23
40
  actualMatroskaTimestamps,
24
41
  logLevel,
25
42
  });
43
+ const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
26
44
  const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
27
- audioManager.logOpenFrames(logLevel);
45
+ audioManager.logOpenFrames();
28
46
  const audioDataArray = [];
29
47
  for (let i = 0; i < samples.length; i++) {
30
48
  const sample = samples[i];
@@ -44,14 +62,13 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
44
62
  // amount of samples to shave from start and end
45
63
  let trimStartInSeconds = 0;
46
64
  let trimEndInSeconds = 0;
47
- // TODO: Apply tone frequency
48
65
  if (isFirstSample) {
49
66
  trimStartInSeconds = timeInSeconds - sample.timestamp;
50
67
  if (trimStartInSeconds < 0 && trimStartInSeconds > -1e-10) {
51
68
  trimStartInSeconds = 0;
52
69
  }
53
70
  if (trimStartInSeconds < 0) {
54
- throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}`);
71
+ throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}. ${JSON.stringify({ timeInSeconds, ts: sample.timestamp, d: sample.duration, isFirstSample, isLastSample, durationInSeconds, i, st: samples.map((s) => s.timestamp) })}`);
55
72
  }
56
73
  }
57
74
  if (isLastSample) {
@@ -63,10 +80,8 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
63
80
  }
64
81
  const audioData = convertAudioData({
65
82
  audioData: audioDataRaw,
66
- newSampleRate: TARGET_SAMPLE_RATE,
67
83
  trimStartInSeconds,
68
84
  trimEndInSeconds,
69
- targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
70
85
  playbackRate,
71
86
  });
72
87
  audioDataRaw.close();
@@ -76,8 +91,13 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
76
91
  audioDataArray.push(audioData);
77
92
  }
78
93
  if (audioDataArray.length === 0) {
79
- return { data: null, durationInSeconds: duration };
94
+ return { data: null, durationInSeconds: mediaDurationInSeconds };
80
95
  }
81
96
  const combined = combineAudioDataAndClosePrevious(audioDataArray);
82
- return { data: combined, durationInSeconds: duration };
97
+ return { data: combined, durationInSeconds: mediaDurationInSeconds };
98
+ };
99
+ let queue = Promise.resolve(undefined);
100
+ export const extractAudio = (params) => {
101
+ queue = queue.then(() => extractAudioInternal(params));
102
+ return queue;
83
103
  };
package/dist/caches.d.ts CHANGED
@@ -2,44 +2,19 @@ import { type LogLevel } from 'remotion';
2
2
  export declare const SAFE_BACK_WINDOW_IN_SECONDS = 1;
3
3
  export declare const keyframeManager: {
4
4
  requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
5
- timestamp: number;
6
5
  packetSink: import("mediabunny").EncodedPacketSink;
6
+ timestamp: number;
7
7
  videoSampleSink: import("mediabunny").VideoSampleSink;
8
8
  src: string;
9
9
  logLevel: LogLevel;
10
- }) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
11
- addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
12
- src: string;
13
- bank: Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
14
- startTimestampInSeconds: number;
15
- }) => void;
10
+ }) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank | null>;
16
11
  getCacheStats: () => Promise<{
17
12
  count: number;
18
13
  totalSize: number;
19
14
  }>;
20
- clearAll: () => Promise<void>;
15
+ clearAll: (logLevel: LogLevel) => Promise<void>;
21
16
  };
22
17
  export declare const audioManager: {
23
- makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
24
- timeInSeconds: number;
25
- src: string;
26
- audioSampleSink: import("mediabunny").AudioSampleSink;
27
- isMatroska: boolean;
28
- actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
29
- }) => {
30
- src: string;
31
- getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
32
- waitForCompletion: () => Promise<boolean>;
33
- canSatisfyRequestedTime: (timestamp: number) => boolean;
34
- logOpenFrames: (logLevel: LogLevel) => void;
35
- getCacheStats: () => {
36
- count: number;
37
- size: number;
38
- };
39
- getLastUsed: () => number;
40
- prepareForDeletion: () => Promise<void>;
41
- startTimestamp: number;
42
- };
43
18
  getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
44
19
  src: string;
45
20
  timeInSeconds: number;
@@ -47,20 +22,7 @@ export declare const audioManager: {
47
22
  isMatroska: boolean;
48
23
  actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
49
24
  logLevel: LogLevel;
50
- }) => Promise<{
51
- src: string;
52
- getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
53
- waitForCompletion: () => Promise<boolean>;
54
- canSatisfyRequestedTime: (timestamp: number) => boolean;
55
- logOpenFrames: (logLevel: LogLevel) => void;
56
- getCacheStats: () => {
57
- count: number;
58
- size: number;
59
- };
60
- getLastUsed: () => number;
61
- prepareForDeletion: () => Promise<void>;
62
- startTimestamp: number;
63
- }>;
25
+ }) => Promise<import("./audio-extraction/audio-iterator").AudioSampleIterator>;
64
26
  getCacheStats: () => {
65
27
  count: number;
66
28
  totalSize: number;
@@ -70,16 +32,20 @@ export declare const audioManager: {
70
32
  getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
71
33
  waitForCompletion: () => Promise<boolean>;
72
34
  canSatisfyRequestedTime: (timestamp: number) => boolean;
73
- logOpenFrames: (logLevel: LogLevel) => void;
35
+ logOpenFrames: () => void;
74
36
  getCacheStats: () => {
75
37
  count: number;
76
38
  size: number;
77
39
  };
78
40
  getLastUsed: () => number;
79
- prepareForDeletion: () => Promise<void>;
41
+ prepareForDeletion: () => void;
80
42
  startTimestamp: number;
43
+ clearBeforeThreshold: (threshold: number) => void;
44
+ getOldestTimestamp: () => number;
45
+ getNewestTimestamp: () => number | null;
81
46
  } | null;
82
- logOpenFrames: (logLevel: LogLevel) => void;
47
+ logOpenFrames: () => void;
48
+ deleteDuplicateIterators: (logLevel: LogLevel) => void;
83
49
  };
84
50
  export declare const getTotalCacheStats: () => Promise<{
85
51
  count: number;
@@ -0,0 +1,2 @@
1
+ import { type PcmS16AudioData } from './convert-audiodata';
2
+ export declare const applyToneFrequency: (audioData: PcmS16AudioData, toneFrequency: number) => PcmS16AudioData;
@@ -0,0 +1,43 @@
1
+ import { FORMAT } from './convert-audiodata';
2
+ import { resampleAudioData, TARGET_SAMPLE_RATE } from './resample-audiodata';
3
+ export const applyToneFrequency = (audioData, toneFrequency) => {
4
+ // In FFmpeg, we apply toneFrequency as follows:
5
+ // `asetrate=${DEFAULT_SAMPLE_RATE}*${toneFrequency},aresample=${DEFAULT_SAMPLE_RATE},atempo=1/${toneFrequency}`
6
+ // So there are 2 steps:
7
+ // 1. Change the assumed sample rate
8
+ // 2. Resample to 48Khz
9
+ // 3. Apply playback rate
10
+ const step1 = {
11
+ ...audioData,
12
+ sampleRate: audioData.sampleRate * toneFrequency,
13
+ };
14
+ const newNumberOfFrames = Math.round(audioData.numberOfFrames * (TARGET_SAMPLE_RATE / step1.sampleRate));
15
+ const step2Data = new Int16Array(newNumberOfFrames * audioData.numberOfChannels);
16
+ const chunkSize = audioData.numberOfFrames / newNumberOfFrames;
17
+ resampleAudioData({
18
+ srcNumberOfChannels: step1.numberOfChannels,
19
+ sourceChannels: step1.data,
20
+ destination: step2Data,
21
+ targetFrames: newNumberOfFrames,
22
+ chunkSize,
23
+ });
24
+ const step2AudioData = {
25
+ data: step2Data,
26
+ format: FORMAT,
27
+ numberOfChannels: step1.numberOfChannels,
28
+ numberOfFrames: newNumberOfFrames,
29
+ sampleRate: TARGET_SAMPLE_RATE,
30
+ timestamp: audioData.timestamp,
31
+ };
32
+ const step3Data = wsolaInt16Interleaved(step2AudioData.data, step2AudioData.numberOfChannels, toneFrequency);
33
+ // Target per-channel length and interleave
34
+ const targetPerChan = Math.max(1, Math.round(step2AudioData.numberOfFrames * toneFrequency));
35
+ const targetTotal = targetPerChan * step2AudioData.numberOfChannels;
36
+ return {
37
+ data: step3Data,
38
+ numberOfChannels: step2AudioData.numberOfChannels,
39
+ numberOfFrames: targetTotal,
40
+ sampleRate: TARGET_SAMPLE_RATE,
41
+ timestamp: audioData.timestamp,
42
+ };
43
+ };
@@ -1,30 +1,11 @@
1
+ import { TARGET_NUMBER_OF_CHANNELS } from './resample-audiodata';
1
2
  export const combineAudioDataAndClosePrevious = (audioDataArray) => {
2
3
  let numberOfFrames = 0;
3
- let numberOfChannels = null;
4
- let sampleRate = null;
5
4
  const { timestamp } = audioDataArray[0];
6
5
  for (const audioData of audioDataArray) {
7
6
  numberOfFrames += audioData.numberOfFrames;
8
- if (!numberOfChannels) {
9
- numberOfChannels = audioData.numberOfChannels;
10
- }
11
- else if (numberOfChannels !== audioData.numberOfChannels) {
12
- throw new Error('Number of channels do not match');
13
- }
14
- if (!sampleRate) {
15
- sampleRate = audioData.sampleRate;
16
- }
17
- else if (sampleRate !== audioData.sampleRate) {
18
- throw new Error('Sample rates do not match');
19
- }
20
7
  }
21
- if (!numberOfChannels) {
22
- throw new Error('Number of channels is not set');
23
- }
24
- if (!sampleRate) {
25
- throw new Error('Sample rate is not set');
26
- }
27
- const arr = new Int16Array(numberOfFrames * numberOfChannels);
8
+ const arr = new Int16Array(numberOfFrames * TARGET_NUMBER_OF_CHANNELS);
28
9
  let offset = 0;
29
10
  for (const audioData of audioDataArray) {
30
11
  arr.set(audioData.data, offset);
@@ -32,9 +13,7 @@ export const combineAudioDataAndClosePrevious = (audioDataArray) => {
32
13
  }
33
14
  return {
34
15
  data: arr,
35
- numberOfChannels,
36
16
  numberOfFrames,
37
- sampleRate,
38
17
  timestamp,
39
18
  };
40
19
  };
@@ -1,16 +1,12 @@
1
1
  export type ConvertAudioDataOptions = {
2
2
  audioData: AudioData;
3
- newSampleRate: number;
4
3
  trimStartInSeconds: number;
5
4
  trimEndInSeconds: number;
6
- targetNumberOfChannels: number;
7
5
  playbackRate: number;
8
6
  };
9
7
  export type PcmS16AudioData = {
10
8
  data: Int16Array;
11
- sampleRate: number;
12
- numberOfChannels: number;
13
9
  numberOfFrames: number;
14
10
  timestamp: number;
15
11
  };
16
- export declare const convertAudioData: ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, playbackRate, }: ConvertAudioDataOptions) => PcmS16AudioData;
12
+ export declare const convertAudioData: ({ audioData, trimStartInSeconds, trimEndInSeconds, playbackRate, }: ConvertAudioDataOptions) => PcmS16AudioData;
@@ -1,25 +1,22 @@
1
- import { resampleAudioData } from './resample-audiodata';
1
+ import { resampleAudioData, TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from './resample-audiodata';
2
2
  const FORMAT = 's16';
3
- const roundButRoundDownZeroPointFive = (value) => {
4
- if (value % 1 <= 0.5) {
5
- return Math.floor(value);
6
- }
7
- return Math.ceil(value);
8
- };
9
- export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, playbackRate, }) => {
3
+ export const convertAudioData = ({ audioData, trimStartInSeconds, trimEndInSeconds, playbackRate, }) => {
10
4
  const { numberOfChannels: srcNumberOfChannels, sampleRate: currentSampleRate, numberOfFrames, } = audioData;
11
- const ratio = currentSampleRate / newSampleRate;
12
- const frameOffset = roundButRoundDownZeroPointFive(trimStartInSeconds * audioData.sampleRate);
5
+ const ratio = currentSampleRate / TARGET_SAMPLE_RATE;
6
+ // Always rounding down start timestamps and rounding up end durations
7
+ // to ensure there are no gaps when the samples don't align
8
+ // In @remotion/renderer inline audio mixing, we also round down the sample start
9
+ // timestamp and round up the end timestamp
10
+ // This might lead to overlapping, hopefully aligning perfectly!
11
+ // Test case: https://github.com/remotion-dev/remotion/issues/5758
12
+ const frameOffset = Math.floor(trimStartInSeconds * audioData.sampleRate);
13
13
  const unroundedFrameCount = numberOfFrames -
14
14
  (trimEndInSeconds + trimStartInSeconds) * audioData.sampleRate;
15
- const frameCount = Math.round(unroundedFrameCount);
16
- const newNumberOfFrames = Math.round(unroundedFrameCount / ratio / playbackRate);
15
+ const frameCount = Math.ceil(unroundedFrameCount);
16
+ const newNumberOfFrames = Math.ceil(unroundedFrameCount / ratio / playbackRate);
17
17
  if (newNumberOfFrames === 0) {
18
18
  throw new Error('Cannot resample - the given sample rate would result in less than 1 sample');
19
19
  }
20
- if (newSampleRate < 3000 || newSampleRate > 768000) {
21
- throw new Error('newSampleRate must be between 3000 and 768000');
22
- }
23
20
  const srcChannels = new Int16Array(srcNumberOfChannels * frameCount);
24
21
  audioData.copyTo(srcChannels, {
25
22
  planeIndex: 0,
@@ -27,17 +24,15 @@ export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds,
27
24
  frameOffset,
28
25
  frameCount,
29
26
  });
30
- const data = new Int16Array(newNumberOfFrames * targetNumberOfChannels);
27
+ const data = new Int16Array(newNumberOfFrames * TARGET_NUMBER_OF_CHANNELS);
31
28
  const chunkSize = frameCount / newNumberOfFrames;
32
29
  if (newNumberOfFrames === frameCount &&
33
- targetNumberOfChannels === srcNumberOfChannels &&
30
+ TARGET_NUMBER_OF_CHANNELS === srcNumberOfChannels &&
34
31
  playbackRate === 1) {
35
32
  return {
36
33
  data: srcChannels,
37
- numberOfChannels: targetNumberOfChannels,
38
34
  numberOfFrames: newNumberOfFrames,
39
- sampleRate: newSampleRate,
40
- timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
35
+ timestamp: audioData.timestamp + (frameOffset / audioData.sampleRate) * 1000000,
41
36
  };
42
37
  }
43
38
  resampleAudioData({
@@ -49,11 +44,8 @@ export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds,
49
44
  });
50
45
  const newAudioData = {
51
46
  data,
52
- format: FORMAT,
53
- numberOfChannels: targetNumberOfChannels,
54
47
  numberOfFrames: newNumberOfFrames,
55
- sampleRate: newSampleRate,
56
- timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
48
+ timestamp: audioData.timestamp + (frameOffset / audioData.sampleRate) * 1000000,
57
49
  };
58
50
  return newAudioData;
59
51
  };
@@ -0,0 +1,13 @@
1
+ /**
2
+ * WSOLA time-scale modification for interleaved Int16 PCM (multi-channel).
3
+ * - Preserves pitch approximately while changing tempo by factor f.
4
+ * - Works for N interleaved channels.
5
+ * - Mitigates head/tail fade-out via overlap-weight normalization and boundary reinforcement.
6
+ *
7
+ * @param input Interleaved Int16 PCM (e.g., LRLRLR... for stereo)
8
+ * @param channels Number of channels (>=1)
9
+ * @param f Tempo factor: >1 = faster/shorter, <1 = slower/longer
10
+ * @param opts Optional tuning parameters
11
+ * @returns Interleaved Int16Array with length ≈ round(input.length * f)
12
+ */
13
+ export declare function wsolaInt16Interleaved(input: Int16Array, channels: number, f: number): Int16Array;