@omnimedia/omnitool 1.1.0-3 → 1.1.0-31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +15 -10
- package/s/context.ts +0 -7
- package/s/demo/demo.bundle.ts +39 -5
- package/s/demo/demo.css +5 -0
- package/s/demo/routines/filmstrip-test.ts +2 -2
- package/s/demo/routines/transcode-test.ts +8 -4
- package/s/demo/routines/transcriber-test.ts +34 -0
- package/s/demo/routines/transitions-test.ts +43 -0
- package/s/demo/routines/waveform-test.ts +3 -2
- package/s/driver/driver.ts +19 -11
- package/s/driver/fns/host.ts +7 -6
- package/s/driver/fns/schematic.ts +47 -24
- package/s/driver/fns/work.ts +165 -156
- package/s/driver/utils/load-decoder-source.ts +3 -4
- package/s/features/speech/transcribe/default-spec.ts +11 -0
- package/s/features/speech/transcribe/parts/load-pipe.ts +19 -0
- package/s/features/speech/transcribe/parts/prep-audio.ts +23 -0
- package/s/features/speech/transcribe/parts/transcribe.ts +70 -0
- package/s/features/speech/transcribe/transcriber.ts +46 -0
- package/s/features/speech/transcribe/types.ts +82 -0
- package/s/features/speech/transcribe/worker.bundle.ts +40 -0
- package/s/features/transition/parts/fragment.ts +24 -0
- package/s/features/transition/parts/types.ts +94 -0
- package/s/features/transition/parts/uniforms.ts +29 -0
- package/s/features/transition/parts/vertex.ts +31 -0
- package/s/features/transition/transition.ts +60 -0
- package/s/index.html.ts +6 -1
- package/s/timeline/index.ts +1 -0
- package/s/timeline/parts/basics.ts +1 -1
- package/s/timeline/parts/compositor/export.ts +77 -0
- package/s/timeline/parts/compositor/parts/html-tree.ts +37 -0
- package/s/timeline/parts/compositor/parts/schedulers.ts +94 -0
- package/s/timeline/parts/compositor/parts/tree-builder.ts +196 -0
- package/s/timeline/parts/compositor/parts/webcodecs-tree.ts +30 -0
- package/s/timeline/parts/compositor/playback.ts +94 -0
- package/s/timeline/parts/compositor/samplers/html.ts +115 -0
- package/s/timeline/parts/compositor/samplers/webcodecs.ts +61 -0
- package/s/timeline/parts/item.ts +48 -6
- package/s/timeline/parts/media.ts +21 -0
- package/s/timeline/parts/waveform.ts +3 -4
- package/s/timeline/sugar/builders.ts +102 -0
- package/s/timeline/sugar/o.ts +163 -38
- package/s/timeline/sugar/omni-test.ts +5 -3
- package/s/timeline/sugar/omni.ts +26 -11
- package/s/timeline/types.ts +29 -0
- package/s/timeline/utils/audio-stream.ts +15 -0
- package/s/timeline/utils/checksum.ts +2 -1
- package/s/timeline/utils/matrix.ts +33 -0
- package/s/timeline/utils/video-cursor.ts +40 -0
- package/x/context.d.ts +1 -4
- package/x/context.js +1 -5
- package/x/context.js.map +1 -1
- package/x/demo/demo.bundle.js +26 -5
- package/x/demo/demo.bundle.js.map +1 -1
- package/x/demo/demo.bundle.min.js +606 -36
- package/x/demo/demo.bundle.min.js.map +4 -4
- package/x/demo/demo.css +5 -0
- package/x/demo/routines/filmstrip-test.d.ts +1 -1
- package/x/demo/routines/filmstrip-test.js +2 -2
- package/x/demo/routines/filmstrip-test.js.map +1 -1
- package/x/demo/routines/transcode-test.js +8 -4
- package/x/demo/routines/transcode-test.js.map +1 -1
- package/x/demo/routines/transcriber-test.d.ts +4 -0
- package/x/demo/routines/transcriber-test.js +33 -0
- package/x/demo/routines/transcriber-test.js.map +1 -0
- package/x/demo/routines/transitions-test.d.ts +5 -0
- package/x/demo/routines/transitions-test.js +35 -0
- package/x/demo/routines/transitions-test.js.map +1 -0
- package/x/demo/routines/waveform-test.d.ts +2 -1
- package/x/demo/routines/waveform-test.js +2 -2
- package/x/demo/routines/waveform-test.js.map +1 -1
- package/x/driver/driver.d.ts +4 -6
- package/x/driver/driver.js +17 -10
- package/x/driver/driver.js.map +1 -1
- package/x/driver/driver.worker.bundle.min.js +2537 -148
- package/x/driver/driver.worker.bundle.min.js.map +4 -4
- package/x/driver/fns/host.d.ts +9 -2
- package/x/driver/fns/host.js +3 -3
- package/x/driver/fns/host.js.map +1 -1
- package/x/driver/fns/schematic.d.ts +41 -23
- package/x/driver/fns/work.d.ts +11 -4
- package/x/driver/fns/work.js +113 -107
- package/x/driver/fns/work.js.map +1 -1
- package/x/driver/utils/load-decoder-source.d.ts +2 -1
- package/x/driver/utils/load-decoder-source.js +2 -3
- package/x/driver/utils/load-decoder-source.js.map +1 -1
- package/x/features/speech/transcribe/default-spec.d.ts +2 -0
- package/x/features/speech/transcribe/default-spec.js +8 -0
- package/x/features/speech/transcribe/default-spec.js.map +1 -0
- package/x/features/speech/transcribe/parts/load-pipe.d.ts +2 -0
- package/x/features/speech/transcribe/parts/load-pipe.js +13 -0
- package/x/features/speech/transcribe/parts/load-pipe.js.map +1 -0
- package/x/features/speech/transcribe/parts/prep-audio.d.ts +5 -0
- package/x/features/speech/transcribe/parts/prep-audio.js +21 -0
- package/x/features/speech/transcribe/parts/prep-audio.js.map +1 -0
- package/x/features/speech/transcribe/parts/transcribe.d.ts +5 -0
- package/x/features/speech/transcribe/parts/transcribe.js +56 -0
- package/x/features/speech/transcribe/parts/transcribe.js.map +1 -0
- package/x/features/speech/transcribe/transcriber.d.ts +5 -0
- package/x/features/speech/transcribe/transcriber.js +33 -0
- package/x/features/speech/transcribe/transcriber.js.map +1 -0
- package/x/features/speech/transcribe/types.d.ts +66 -0
- package/x/features/speech/transcribe/types.js +2 -0
- package/x/features/speech/transcribe/types.js.map +1 -0
- package/x/features/speech/transcribe/worker.bundle.d.ts +1 -0
- package/x/features/speech/transcribe/worker.bundle.js +33 -0
- package/x/features/speech/transcribe/worker.bundle.js.map +1 -0
- package/x/features/speech/transcribe/worker.bundle.min.js +2916 -0
- package/x/features/speech/transcribe/worker.bundle.min.js.map +7 -0
- package/x/features/transition/parts/fragment.d.ts +1 -0
- package/x/features/transition/parts/fragment.js +25 -0
- package/x/features/transition/parts/fragment.js.map +1 -0
- package/x/features/transition/parts/types.d.ts +23 -0
- package/x/features/transition/parts/types.js +2 -0
- package/x/features/transition/parts/types.js.map +1 -0
- package/x/features/transition/parts/uniforms.d.ts +31 -0
- package/x/features/transition/parts/uniforms.js +27 -0
- package/x/features/transition/parts/uniforms.js.map +1 -0
- package/x/features/transition/parts/vertex.d.ts +1 -0
- package/x/features/transition/parts/vertex.js +32 -0
- package/x/features/transition/parts/vertex.js.map +1 -0
- package/x/features/transition/transition.d.ts +5 -0
- package/x/features/transition/transition.js +50 -0
- package/x/features/transition/transition.js.map +1 -0
- package/x/index.html +13 -3
- package/x/index.html.js +6 -1
- package/x/index.html.js.map +1 -1
- package/x/timeline/index.d.ts +1 -0
- package/x/timeline/index.js +1 -0
- package/x/timeline/index.js.map +1 -1
- package/x/timeline/parts/basics.d.ts +1 -1
- package/x/timeline/parts/compositor/export.d.ts +11 -0
- package/x/timeline/parts/compositor/export.js +64 -0
- package/x/timeline/parts/compositor/export.js.map +1 -0
- package/x/timeline/parts/compositor/parts/html-tree.d.ts +3 -0
- package/x/timeline/parts/compositor/parts/html-tree.js +40 -0
- package/x/timeline/parts/compositor/parts/html-tree.js.map +1 -0
- package/x/timeline/parts/compositor/parts/schedulers.d.ts +15 -0
- package/x/timeline/parts/compositor/parts/schedulers.js +69 -0
- package/x/timeline/parts/compositor/parts/schedulers.js.map +1 -0
- package/x/timeline/parts/compositor/parts/tree-builder.d.ts +37 -0
- package/x/timeline/parts/compositor/parts/tree-builder.js +160 -0
- package/x/timeline/parts/compositor/parts/tree-builder.js.map +1 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.d.ts +3 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.js +28 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.js.map +1 -0
- package/x/timeline/parts/compositor/playback.d.ts +26 -0
- package/x/timeline/parts/compositor/playback.js +79 -0
- package/x/timeline/parts/compositor/playback.js.map +1 -0
- package/x/timeline/parts/compositor/samplers/html.d.ts +3 -0
- package/x/timeline/parts/compositor/samplers/html.js +106 -0
- package/x/timeline/parts/compositor/samplers/html.js.map +1 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.d.ts +3 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.js +52 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.js.map +1 -0
- package/x/timeline/parts/item.d.ts +42 -8
- package/x/timeline/parts/item.js +7 -3
- package/x/timeline/parts/item.js.map +1 -1
- package/x/timeline/parts/media.d.ts +3 -0
- package/x/timeline/parts/media.js +17 -0
- package/x/timeline/parts/media.js.map +1 -1
- package/x/timeline/parts/waveform.d.ts +2 -1
- package/x/timeline/parts/waveform.js +2 -4
- package/x/timeline/parts/waveform.js.map +1 -1
- package/x/timeline/sugar/builders.d.ts +1 -0
- package/x/timeline/sugar/builders.js +104 -0
- package/x/timeline/sugar/builders.js.map +1 -0
- package/x/timeline/sugar/o.d.ts +27 -5
- package/x/timeline/sugar/o.js +137 -38
- package/x/timeline/sugar/o.js.map +1 -1
- package/x/timeline/sugar/omni-test.js +4 -2
- package/x/timeline/sugar/omni-test.js.map +1 -1
- package/x/timeline/sugar/omni.d.ts +8 -2
- package/x/timeline/sugar/omni.js +22 -9
- package/x/timeline/sugar/omni.js.map +1 -1
- package/x/timeline/types.d.ts +24 -0
- package/x/timeline/types.js +2 -0
- package/x/timeline/types.js.map +1 -0
- package/x/timeline/utils/audio-stream.d.ts +6 -0
- package/x/timeline/utils/audio-stream.js +17 -0
- package/x/timeline/utils/audio-stream.js.map +1 -0
- package/x/timeline/utils/checksum.js +2 -1
- package/x/timeline/utils/checksum.js.map +1 -1
- package/x/timeline/utils/matrix.d.ts +8 -0
- package/x/timeline/utils/matrix.js +26 -0
- package/x/timeline/utils/matrix.js.map +1 -0
- package/x/timeline/utils/video-cursor.d.ts +10 -0
- package/x/timeline/utils/video-cursor.js +36 -0
- package/x/timeline/utils/video-cursor.js.map +1 -0
package/x/driver/fns/host.d.ts
CHANGED
|
@@ -2,10 +2,17 @@ import { Machina } from "../parts/machina.js";
|
|
|
2
2
|
export declare const setupDriverHost: (machina: Machina) => import("@e280/comrade").SetupHost<{
|
|
3
3
|
work: {
|
|
4
4
|
hello(): Promise<void>;
|
|
5
|
-
|
|
5
|
+
decodeAudio(input: {
|
|
6
6
|
source: import("./schematic.js").DecoderSource;
|
|
7
|
-
video: WritableStream<VideoFrame>;
|
|
8
7
|
audio: WritableStream<AudioData>;
|
|
8
|
+
start?: number;
|
|
9
|
+
end?: number;
|
|
10
|
+
}): Promise<void>;
|
|
11
|
+
decodeVideo(input: {
|
|
12
|
+
source: import("./schematic.js").DecoderSource;
|
|
13
|
+
video: WritableStream<VideoFrame>;
|
|
14
|
+
start?: number;
|
|
15
|
+
end?: number;
|
|
9
16
|
}): Promise<void>;
|
|
10
17
|
encode(input: import("./schematic.js").EncoderInput & {
|
|
11
18
|
bridge: WritableStream<StreamTargetChunk>;
|
package/x/driver/fns/host.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Comrade } from "@e280/comrade";
|
|
2
|
-
export const setupDriverHost = (machina) => Comrade.host(
|
|
2
|
+
export const setupDriverHost = (machina) => (Comrade.host(_shell => ({
|
|
3
3
|
async world() {
|
|
4
4
|
machina.count++;
|
|
5
|
-
}
|
|
6
|
-
}));
|
|
5
|
+
},
|
|
6
|
+
})));
|
|
7
7
|
//# sourceMappingURL=host.js.map
|
package/x/driver/fns/host.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host.js","sourceRoot":"","sources":["../../../s/driver/fns/host.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AAIrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAkB,CAAC,
|
|
1
|
+
{"version":3,"file":"host.js","sourceRoot":"","sources":["../../../s/driver/fns/host.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AAIrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,CACpD,OAAO,CAAC,IAAI,CAAkB,MAAM,CAAC,EAAE,CAAC,CAAC;IACxC,KAAK,CAAC,KAAK;QACV,OAAO,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;CACD,CAAC,CAAC,CACH,CAAA"}
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
+
import { TextStyleOptions } from "pixi.js";
|
|
1
2
|
import { AsSchematic } from "@e280/comrade";
|
|
2
3
|
import type { AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig } from "mediabunny";
|
|
4
|
+
import { Mat6 } from "../../timeline/utils/matrix.js";
|
|
3
5
|
export type DriverSchematic = AsSchematic<{
|
|
4
6
|
work: {
|
|
5
7
|
hello(): Promise<void>;
|
|
6
|
-
|
|
8
|
+
decodeAudio(input: {
|
|
7
9
|
source: DecoderSource;
|
|
8
|
-
video: WritableStream<VideoFrame>;
|
|
9
10
|
audio: WritableStream<AudioData>;
|
|
11
|
+
start?: number;
|
|
12
|
+
end?: number;
|
|
13
|
+
}): Promise<void>;
|
|
14
|
+
decodeVideo(input: {
|
|
15
|
+
source: DecoderSource;
|
|
16
|
+
video: WritableStream<VideoFrame>;
|
|
17
|
+
start?: number;
|
|
18
|
+
end?: number;
|
|
10
19
|
}): Promise<void>;
|
|
11
20
|
encode(input: EncoderInput & {
|
|
12
21
|
bridge: WritableStream<StreamTargetChunk>;
|
|
@@ -18,18 +27,19 @@ export type DriverSchematic = AsSchematic<{
|
|
|
18
27
|
};
|
|
19
28
|
}>;
|
|
20
29
|
export interface EncoderInput {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
25
|
-
config: {
|
|
26
|
-
video: VideoEncodingConfig;
|
|
27
|
-
audio: AudioEncodingConfig;
|
|
28
|
-
};
|
|
30
|
+
video?: ReadableStream<VideoFrame>;
|
|
31
|
+
audio?: ReadableStream<AudioData>;
|
|
32
|
+
config: RenderConfig;
|
|
29
33
|
}
|
|
30
|
-
export
|
|
34
|
+
export interface RenderConfig {
|
|
35
|
+
video: VideoEncodingConfig;
|
|
36
|
+
audio: AudioEncodingConfig;
|
|
37
|
+
}
|
|
38
|
+
export type DecoderSource = Blob | string | URL;
|
|
31
39
|
export interface DecoderInput {
|
|
32
40
|
source: DecoderSource;
|
|
41
|
+
start?: number;
|
|
42
|
+
end?: number;
|
|
33
43
|
onFrame?: (frame: VideoFrame) => Promise<VideoFrame>;
|
|
34
44
|
}
|
|
35
45
|
export interface MuxOpts {
|
|
@@ -46,21 +56,29 @@ export interface MuxOpts {
|
|
|
46
56
|
};
|
|
47
57
|
}
|
|
48
58
|
export type Composition = Layer | (Layer | Composition)[];
|
|
49
|
-
export type Transform = {
|
|
50
|
-
x?: number;
|
|
51
|
-
y?: number;
|
|
52
|
-
scale?: number;
|
|
53
|
-
opacity?: number;
|
|
54
|
-
anchor?: number;
|
|
55
|
-
};
|
|
56
59
|
export type TextLayer = {
|
|
57
60
|
kind: 'text';
|
|
58
61
|
content: string;
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
62
|
+
style?: TextStyleOptions;
|
|
63
|
+
matrix?: Mat6;
|
|
64
|
+
};
|
|
62
65
|
export type ImageLayer = {
|
|
63
66
|
kind: 'image';
|
|
64
67
|
frame: VideoFrame;
|
|
65
|
-
|
|
66
|
-
|
|
68
|
+
matrix?: Mat6;
|
|
69
|
+
};
|
|
70
|
+
export type TransitionLayer = {
|
|
71
|
+
kind: 'transition';
|
|
72
|
+
name: string;
|
|
73
|
+
progress: number;
|
|
74
|
+
from: VideoFrame;
|
|
75
|
+
to: VideoFrame;
|
|
76
|
+
};
|
|
77
|
+
export type GapLayer = {
|
|
78
|
+
kind: 'gap';
|
|
79
|
+
};
|
|
80
|
+
export type Audio = {
|
|
81
|
+
kind: "audio";
|
|
82
|
+
data: AudioData;
|
|
83
|
+
};
|
|
84
|
+
export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer;
|
package/x/driver/fns/work.d.ts
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { Input } from "mediabunny";
|
|
2
|
-
import { Composition } from "./schematic.js";
|
|
2
|
+
import { Composition, DecoderSource } from "./schematic.js";
|
|
3
3
|
export declare const setupDriverWork: import("@e280/comrade").SetupWork<{
|
|
4
4
|
work: {
|
|
5
5
|
hello(): Promise<void>;
|
|
6
|
-
|
|
7
|
-
source:
|
|
8
|
-
video: WritableStream<VideoFrame>;
|
|
6
|
+
decodeAudio(input: {
|
|
7
|
+
source: DecoderSource;
|
|
9
8
|
audio: WritableStream<AudioData>;
|
|
9
|
+
start?: number;
|
|
10
|
+
end?: number;
|
|
11
|
+
}): Promise<void>;
|
|
12
|
+
decodeVideo(input: {
|
|
13
|
+
source: DecoderSource;
|
|
14
|
+
video: WritableStream<VideoFrame>;
|
|
15
|
+
start?: number;
|
|
16
|
+
end?: number;
|
|
10
17
|
}): Promise<void>;
|
|
11
18
|
encode(input: import("./schematic.js").EncoderInput & {
|
|
12
19
|
bridge: WritableStream<Input>;
|
package/x/driver/fns/work.js
CHANGED
|
@@ -1,76 +1,72 @@
|
|
|
1
1
|
import { Comrade } from "@e280/comrade";
|
|
2
|
-
import { Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource } from "mediabunny";
|
|
3
2
|
import { autoDetectRenderer, Container, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter } from "pixi.js";
|
|
3
|
+
import { Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource } from "mediabunny";
|
|
4
|
+
import { mat6ToMatrix } from "../../timeline/utils/matrix.js";
|
|
5
|
+
import { makeTransition } from "../../features/transition/transition.js";
|
|
4
6
|
DOMAdapter.set(WebWorkerAdapter);
|
|
5
|
-
|
|
7
|
+
const loadSource = async (source) => {
|
|
8
|
+
if (source instanceof Blob) {
|
|
9
|
+
return new BlobSource(source);
|
|
10
|
+
}
|
|
11
|
+
else {
|
|
12
|
+
return new UrlSource(source);
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
export const setupDriverWork = (Comrade.work(shell => ({
|
|
6
16
|
async hello() {
|
|
7
|
-
await host.world();
|
|
17
|
+
await shell.host.world();
|
|
8
18
|
},
|
|
9
|
-
async
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
19
|
+
async decodeAudio({ source, audio, start, end }) {
|
|
20
|
+
const input = new Input({
|
|
21
|
+
source: await loadSource(source),
|
|
22
|
+
formats: ALL_FORMATS
|
|
23
|
+
});
|
|
24
|
+
const audioTrack = await input.getPrimaryAudioTrack();
|
|
25
|
+
const audioDecodable = await audioTrack?.canDecode();
|
|
26
|
+
const audioWriter = audio.getWriter();
|
|
27
|
+
if (audioDecodable && audioTrack) {
|
|
28
|
+
const sink = new AudioSampleSink(audioTrack);
|
|
29
|
+
for await (const sample of sink.samples(start, end)) {
|
|
30
|
+
const frame = sample.toAudioData();
|
|
31
|
+
await audioWriter.write(frame);
|
|
32
|
+
sample.close();
|
|
33
|
+
frame.close();
|
|
17
34
|
}
|
|
18
|
-
|
|
35
|
+
await audioWriter.close();
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
async decodeVideo({ source, video, start, end }) {
|
|
19
39
|
const input = new Input({
|
|
20
|
-
source: await loadSource(),
|
|
40
|
+
source: await loadSource(source),
|
|
21
41
|
formats: ALL_FORMATS
|
|
22
42
|
});
|
|
23
|
-
const
|
|
24
|
-
input.getPrimaryVideoTrack(),
|
|
25
|
-
input.getPrimaryAudioTrack()
|
|
26
|
-
]);
|
|
43
|
+
const videoTrack = await input.getPrimaryVideoTrack();
|
|
27
44
|
const videoDecodable = await videoTrack?.canDecode();
|
|
28
|
-
const audioDecodable = await audioTrack?.canDecode();
|
|
29
45
|
const videoWriter = video.getWriter();
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
await videoWriter.close();
|
|
42
|
-
}
|
|
43
|
-
})(),
|
|
44
|
-
(async () => {
|
|
45
|
-
if (audioDecodable && audioTrack) {
|
|
46
|
-
const sink = new AudioSampleSink(audioTrack);
|
|
47
|
-
for await (const sample of sink.samples()) {
|
|
48
|
-
const frame = sample.toAudioData();
|
|
49
|
-
await audioWriter.write(frame);
|
|
50
|
-
sample.close();
|
|
51
|
-
frame.close();
|
|
52
|
-
}
|
|
53
|
-
await audioWriter.close();
|
|
54
|
-
}
|
|
55
|
-
})()
|
|
56
|
-
]);
|
|
46
|
+
if (videoDecodable && videoTrack) {
|
|
47
|
+
const sink = new VideoSampleSink(videoTrack);
|
|
48
|
+
for await (const sample of sink.samples(start, end)) {
|
|
49
|
+
const frame = sample.toVideoFrame();
|
|
50
|
+
await videoWriter.write(frame);
|
|
51
|
+
sample.close();
|
|
52
|
+
frame.close();
|
|
53
|
+
}
|
|
54
|
+
await videoWriter.close();
|
|
55
|
+
}
|
|
57
56
|
},
|
|
58
|
-
async encode({
|
|
57
|
+
async encode({ video, audio, config, bridge }) {
|
|
59
58
|
const output = new Output({
|
|
60
59
|
format: new Mp4OutputFormat(),
|
|
61
60
|
target: new StreamTarget(bridge, { chunked: true })
|
|
62
61
|
});
|
|
63
|
-
const videoSource = new VideoSampleSource(config.video);
|
|
64
|
-
output.addVideoTrack(videoSource);
|
|
65
62
|
// since AudioSample is not transferable it fails to transfer encoder bitrate config
|
|
66
63
|
// so it needs to be hardcoded not set through constants eg QUALITY_LOW
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
(async () => {
|
|
64
|
+
const promises = [];
|
|
65
|
+
if (video) {
|
|
66
|
+
const videoSource = new VideoSampleSource(config.video);
|
|
67
|
+
output.addVideoTrack(videoSource);
|
|
68
|
+
const videoReader = video.getReader();
|
|
69
|
+
promises.push((async () => {
|
|
74
70
|
while (true) {
|
|
75
71
|
const { done, value } = await videoReader.read();
|
|
76
72
|
if (done)
|
|
@@ -79,8 +75,13 @@ export const setupDriverWork = Comrade.work(({ host }, rig) => ({
|
|
|
79
75
|
await videoSource.add(sample);
|
|
80
76
|
sample.close();
|
|
81
77
|
}
|
|
82
|
-
})()
|
|
83
|
-
|
|
78
|
+
})());
|
|
79
|
+
}
|
|
80
|
+
if (audio) {
|
|
81
|
+
const audioSource = new AudioSampleSource(config.audio);
|
|
82
|
+
output.addAudioTrack(audioSource);
|
|
83
|
+
const audioReader = audio.getReader();
|
|
84
|
+
promises.push((async () => {
|
|
84
85
|
while (true) {
|
|
85
86
|
const { done, value } = await audioReader.read();
|
|
86
87
|
if (done)
|
|
@@ -90,30 +91,30 @@ export const setupDriverWork = Comrade.work(({ host }, rig) => ({
|
|
|
90
91
|
sample.close();
|
|
91
92
|
value.close();
|
|
92
93
|
}
|
|
93
|
-
})()
|
|
94
|
-
|
|
94
|
+
})());
|
|
95
|
+
}
|
|
96
|
+
await output.start();
|
|
97
|
+
await Promise.all(promises);
|
|
95
98
|
await output.finalize();
|
|
96
99
|
},
|
|
97
100
|
async composite(composition) {
|
|
98
101
|
const { stage, renderer } = await renderPIXI(1920, 1080);
|
|
99
102
|
stage.removeChildren();
|
|
100
|
-
const {
|
|
103
|
+
const { dispose } = await renderLayer(composition, stage);
|
|
101
104
|
renderer.render(stage);
|
|
102
105
|
// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
|
|
103
106
|
// if its very slow on eg edge try chrome
|
|
104
107
|
const frame = new VideoFrame(renderer.canvas, {
|
|
105
|
-
timestamp:
|
|
106
|
-
duration:
|
|
108
|
+
timestamp: 0,
|
|
109
|
+
duration: 0,
|
|
107
110
|
});
|
|
108
|
-
baseFrame?.close();
|
|
109
111
|
renderer.clear();
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
rig.transfer = [frame];
|
|
112
|
+
dispose();
|
|
113
|
+
shell.transfer = [frame];
|
|
114
114
|
return frame;
|
|
115
115
|
}
|
|
116
|
-
}));
|
|
116
|
+
})));
|
|
117
|
+
// TODO suspicious global, probably bad
|
|
117
118
|
let pixi = null;
|
|
118
119
|
async function renderPIXI(width, height) {
|
|
119
120
|
if (pixi)
|
|
@@ -129,64 +130,69 @@ async function renderPIXI(width, height) {
|
|
|
129
130
|
pixi = { renderer, stage };
|
|
130
131
|
return pixi;
|
|
131
132
|
}
|
|
132
|
-
|
|
133
|
+
const transitions = new Map();
|
|
134
|
+
async function renderLayer(layer, parent) {
|
|
133
135
|
if (Array.isArray(layer)) {
|
|
134
|
-
|
|
136
|
+
layer.reverse();
|
|
137
|
+
const disposers = [];
|
|
135
138
|
for (const child of layer) {
|
|
136
|
-
const result = await renderLayer(child, parent
|
|
137
|
-
|
|
139
|
+
const result = await renderLayer(child, parent);
|
|
140
|
+
disposers.push(result.dispose);
|
|
138
141
|
}
|
|
139
|
-
return {
|
|
140
|
-
}
|
|
141
|
-
if (!isRenderableLayer(layer)) {
|
|
142
|
-
console.warn('Invalid layer', layer);
|
|
143
|
-
return { disposables };
|
|
142
|
+
return { dispose: () => disposers.forEach(d => d()) };
|
|
144
143
|
}
|
|
145
144
|
switch (layer.kind) {
|
|
146
145
|
case 'text':
|
|
147
|
-
return renderTextLayer(layer, parent
|
|
146
|
+
return renderTextLayer(layer, parent);
|
|
148
147
|
case 'image':
|
|
149
|
-
return renderImageLayer(layer, parent
|
|
148
|
+
return renderImageLayer(layer, parent);
|
|
149
|
+
case 'transition':
|
|
150
|
+
return renderTransitionLayer(layer, parent);
|
|
151
|
+
case 'gap': {
|
|
152
|
+
pixi?.renderer.clear();
|
|
153
|
+
return { dispose: () => { } };
|
|
154
|
+
}
|
|
150
155
|
default:
|
|
151
156
|
console.warn('Unknown layer kind', layer.kind);
|
|
152
|
-
return {
|
|
157
|
+
return { dispose: () => { } };
|
|
153
158
|
}
|
|
154
159
|
}
|
|
155
|
-
function
|
|
156
|
-
return !!layer && typeof layer === 'object' && typeof layer.kind === 'string';
|
|
157
|
-
}
|
|
158
|
-
function renderTextLayer(layer, parent, disposables) {
|
|
160
|
+
function renderTextLayer(layer, parent) {
|
|
159
161
|
const text = new Text({
|
|
160
162
|
text: layer.content,
|
|
161
|
-
style:
|
|
162
|
-
fontFamily: 'sans-serif',
|
|
163
|
-
fontSize: layer.fontSize ?? 48,
|
|
164
|
-
fill: layer.color ?? 'white'
|
|
165
|
-
}
|
|
163
|
+
style: layer.style
|
|
166
164
|
});
|
|
167
|
-
applyTransform(text, layer);
|
|
165
|
+
applyTransform(text, layer.matrix);
|
|
168
166
|
parent.addChild(text);
|
|
169
|
-
|
|
170
|
-
return { disposables };
|
|
167
|
+
return { dispose: () => text.destroy(true) };
|
|
171
168
|
}
|
|
172
|
-
function renderImageLayer(layer, parent
|
|
169
|
+
function renderImageLayer(layer, parent) {
|
|
173
170
|
const texture = Texture.from(layer.frame);
|
|
174
171
|
const sprite = new Sprite(texture);
|
|
175
|
-
applyTransform(sprite, layer);
|
|
172
|
+
applyTransform(sprite, layer.matrix);
|
|
173
|
+
parent.addChild(sprite);
|
|
174
|
+
return { dispose: () => {
|
|
175
|
+
sprite.destroy(true);
|
|
176
|
+
texture.destroy(true);
|
|
177
|
+
layer.frame.close();
|
|
178
|
+
} };
|
|
179
|
+
}
|
|
180
|
+
function renderTransitionLayer({ from, to, progress, name }, parent) {
|
|
181
|
+
const transition = transitions.get(name) ??
|
|
182
|
+
(transitions.set(name, makeTransition({
|
|
183
|
+
name: "circle",
|
|
184
|
+
renderer: pixi.renderer
|
|
185
|
+
})),
|
|
186
|
+
transitions.get(name));
|
|
187
|
+
const texture = transition.render({ from, to, progress, width: from.displayWidth, height: from.displayHeight });
|
|
188
|
+
const sprite = new Sprite(texture);
|
|
176
189
|
parent.addChild(sprite);
|
|
177
|
-
|
|
178
|
-
return { baseFrame: layer.frame, disposables };
|
|
190
|
+
return { dispose: () => sprite.destroy(false) };
|
|
179
191
|
}
|
|
180
|
-
function applyTransform(target,
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
if (t.scale)
|
|
186
|
-
target.scale.set(t.scale);
|
|
187
|
-
if (t.opacity)
|
|
188
|
-
target.alpha = t.opacity;
|
|
189
|
-
if (t.anchor && 'anchor' in target)
|
|
190
|
-
target.anchor.set(t.anchor);
|
|
192
|
+
function applyTransform(target, worldMatrix) {
|
|
193
|
+
if (!worldMatrix)
|
|
194
|
+
return;
|
|
195
|
+
const mx = mat6ToMatrix(worldMatrix);
|
|
196
|
+
target.setFromMatrix(mx);
|
|
191
197
|
}
|
|
192
198
|
//# sourceMappingURL=work.js.map
|
package/x/driver/fns/work.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"work.js","sourceRoot":"","sources":["../../../s/driver/fns/work.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,
|
|
1
|
+
{"version":3,"file":"work.js","sourceRoot":"","sources":["../../../s/driver/fns/work.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,kBAAkB,EAAE,SAAS,EAAY,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAS,MAAM,SAAS,CAAA;AAC5H,OAAO,EAAC,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAE7M,OAAO,EAAO,YAAY,EAAC,MAAM,gCAAgC,CAAA;AACjE,OAAO,EAAC,cAAc,EAAC,MAAM,yCAAyC,CAAA;AAGtE,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;AAEhC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAqB,EAAE,EAAE;IAClD,IAAG,MAAM,YAAY,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;SAAM,CAAC;QACP,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,CAC9B,OAAO,CAAC,IAAI,CAAkB,KAAK,CAAC,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,KAAK;QACV,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACvB,MAAM,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAA;QACrD,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,SAAS,EAAE,CAAA;QACpD,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;QAErC,IAAI,cAAc,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,UAAU,CAAC,CAAA;YAC5C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;gBAClC,MAAM,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC9B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACd,KAAK,CAAC,KAAK,EAAE,CAAA;YACd,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACvB,MAAM,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAA;QACrD,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,SAAS,EAAE,CAAA;QACpD,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;QAErC,IAAI,cAAc,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,UAAU,CAAC,CAAA;YAC5C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;gBACnC,MAAM,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC9B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACd,KAAK,CAAC,KAAK,EAAE,CAAA;YACd,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;YACzB,MAAM,EAAE,IAAI,eAAe,EAAE;YAC7B,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,EAAE,EAAC,OAAO,EAAE,IAAI,EAAC,CAAC;SACjD,CAAC,CAAA;QACF,oFAAoF;QACpF,uEAAuE;QAEvE,MAAM,QAAQ,GAAG,EAAE,CAAA;QAEnB,IAAG,KAAK,EAAE,CAAC;YACV,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACvD,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAA;YACjC,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;YACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;gBACzB,OAAO,IAAI,EAAE,CAAC;oBACb,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;oBAC9C,IAAI,IAAI;wBAAE,MAAK;oBACf,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;oBACrC,MAAM,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACf,CAAC;YACF,CAAC,CAAC,EAAE,CAAC,CAAA;QACN,CAAC;QAED,IAAG,KAAK,EAAE,CAAC;YACV,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACvD,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAA;YACjC,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;YACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;gBACzB,OAAO,IAAI,EAAE,CAAC;oBACb,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;oBAC9C,IAAI,IAAI;wBAAE,MAAK;oBACf,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;oBACrC,MAAM,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,KAAK,CAAC,KAAK,EAAE,CAAA;gBACd,CAAC;YACF,CAAC,CAAC,EAAE,CAAC,CAAA;QACN,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC3B,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAW;QAC1B,MAAM,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACtD,KAAK,CAAC,cAAc,EAAE,CAAA;QAEtB,MAAM,EAAC,OAAO,EAAC,GAAG,MAAM,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QACvD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEtB,gGAAgG;QAChG,yCAAyC;QACzC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE;YAC7C,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;SACX,CAAC,CAAA;QAEF,QAAQ,CAAC,KAAK,EAAE,CAAA;QAChB,OAAO,EAAE,CAAA;QAET,KAAK,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,CAAA;QACxB,OAAO,KAAK,CAAA;IACb,CAAC;CACD,CAAC,CAAC,CACH,CAAA;AAED,uCAAuC;AACvC,IAAI,IAAI,GAGG,IAAI,CAAA;AAEf,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,MAAc;IACtD,IAAI,IAAI;QACP,OAAO,IAAI,CAAA;IAEZ,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC;QACzC,KAAK;QACL,MAAM;QACN,UAAU,EAAE,OAAO,EAAE,iDAAiD;QACtE,UAAU,EAAE,OAAO;QACnB,kBAAkB,EAAE,CAAC;KACrB,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAA;IAC7B,IAAI,GAAG,EAAC,QAAQ,EAAE,KAAK,EAAC,CAAA;IAExB,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,WAAW,GAAmD,IAAI,GAAG,EAAE,CAAA;AAI7E,KAAK,UAAU,WAAW,CACzB,KAA0B,EAC1B,MAAiB;IAEjB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,OAAO,EAAE,CAAA;QACf,MAAM,SAAS,GAAmB,EAAE,CAAA;QACpC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAC/C,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC/B,CAAC;QACD,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAC,CAAA;IACpD,CAAC;IAED,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,MAAM;YACV,OAAO,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACtC,KAAK,OAAO;YACX,OAAO,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACvC,KAAK,YAAY;YAChB,OAAO,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAC5C,KAAK,KAAK,CAAC,CAAC,CAAC;YACZ,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;YACtB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAC,CAAA;QAC3B,CAAC;QACD;YACC,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAG,KAAa,CAAC,IAAI,CAAC,CAAA;YACvD,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAC,CAAA;IAC5B,CAAC;AACF,CAAC;AAED,SAAS,eAAe,CACvB,KAAqC,EACrC,MAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC;QACrB,IAAI,EAAE,KAAK,CAAC,OAAO;QACnB,KAAK,EAAE,KAAK,CAAC,KAAK;KAClB,CAAC,CAAA;IACF,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAC,CAAA;AAC3C,CAAC;AAED,SAAS,gBAAgB,CACxB,KAAsC,EACtC,MAAiB;IAEjB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACzC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IACpC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACvB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACpB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACrB,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACpB,CAAC,EAAC,CAAA;AACH,CAAC;AAED,SAAS,qBAAqB,CAC7B,EAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAuC,EAChE,MAAiB;IAEjB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;QACvC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,cAAc,CAAC;YACrC,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,IAAK,CAAC,QAAQ;SACxB,CAAC,CAAC;YACF,WAAW,CAAC,GAAG,CAAC,IAAI,CAAE,CACvB,CAAA;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAC,CAAC,CAAA;IAC7G,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACvB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAC,CAAA;AAC9C,CAAC;AAED,SAAS,cAAc,CAAC,MAAqB,EAAE,WAAkB;IAC/D,IAAI,CAAC,WAAW;QAAE,OAAM;IACzB,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;IACnC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;AAC1B,CAAC"}
|
|
@@ -1,2 +1,3 @@
|
|
|
1
|
+
import { BlobSource, UrlSource } from "mediabunny";
|
|
1
2
|
import { DecoderSource } from "../fns/schematic.js";
|
|
2
|
-
export declare function loadDecoderSource(source: DecoderSource): Promise<
|
|
3
|
+
export declare function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource>;
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { BlobSource, UrlSource } from "mediabunny";
|
|
2
2
|
// only streamable sources
|
|
3
3
|
export async function loadDecoderSource(source) {
|
|
4
|
-
if (source instanceof
|
|
5
|
-
|
|
6
|
-
return new BlobSource(file);
|
|
4
|
+
if (source instanceof Blob) {
|
|
5
|
+
return new BlobSource(source);
|
|
7
6
|
}
|
|
8
7
|
else {
|
|
9
8
|
return new UrlSource(source);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load-decoder-source.js","sourceRoot":"","sources":["../../../s/driver/utils/load-decoder-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAGhD,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAqB;IAC5D,IAAG,MAAM,YAAY,
|
|
1
|
+
{"version":3,"file":"load-decoder-source.js","sourceRoot":"","sources":["../../../s/driver/utils/load-decoder-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAGhD,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAqB;IAC5D,IAAG,MAAM,YAAY,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;SAAM,CAAC;QACP,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-spec.js","sourceRoot":"","sources":["../../../../s/features/speech/transcribe/default-spec.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAoB,EAAE,CAAC,CAAC;IAC7D,KAAK,EAAE,yCAAyC;IAChD,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,MAAM;IACd,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,CAAC;CACf,CAAC,CAAA"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { pipeline } from "@huggingface/transformers";
|
|
2
|
+
export async function loadPipe(options) {
|
|
3
|
+
const { spec, onLoading } = options;
|
|
4
|
+
const pipe = await pipeline("automatic-speech-recognition", spec.model, {
|
|
5
|
+
device: spec.device,
|
|
6
|
+
dtype: spec.dtype,
|
|
7
|
+
progress_callback: (data) => {
|
|
8
|
+
onLoading({ total: data.total, progress: data.progress });
|
|
9
|
+
},
|
|
10
|
+
});
|
|
11
|
+
return pipe;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=load-pipe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-pipe.js","sourceRoot":"","sources":["../../../../../s/features/speech/transcribe/parts/load-pipe.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAIlD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAA+B;IAC7D,MAAM,EAAC,IAAI,EAAE,SAAS,EAAC,GAAG,OAAO,CAAA;IAEjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,8BAA8B,EAAE,IAAI,CAAC,KAAK,EAAE;QACvE,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,iBAAiB,EAAE,CAAC,IAAS,EAAE,EAAE;YAChC,SAAS,CAAC,EAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAC,CAAC,CAAA;QACxD,CAAC;KACD,CAAC,CAAA;IAEF,OAAO,IAAI,CAAA;AACZ,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export async function prepAudio(driver, source) {
|
|
2
|
+
const arrayBuffer = await source.arrayBuffer();
|
|
3
|
+
const audioCTX = new AudioContext({ sampleRate: 16000 });
|
|
4
|
+
const audioData = await audioCTX.decodeAudioData(arrayBuffer);
|
|
5
|
+
let audio;
|
|
6
|
+
if (audioData.numberOfChannels === 2) {
|
|
7
|
+
const SCALING_FACTOR = Math.sqrt(2);
|
|
8
|
+
const left = audioData.getChannelData(0);
|
|
9
|
+
const right = audioData.getChannelData(1);
|
|
10
|
+
audio = new Float32Array(left.length);
|
|
11
|
+
for (let i = 0; i < audioData.length; ++i) {
|
|
12
|
+
audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
audio = audioData.getChannelData(0);
|
|
17
|
+
}
|
|
18
|
+
const duration = await driver.getAudioDuration(source);
|
|
19
|
+
return { audio, duration };
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=prep-audio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prep-audio.js","sourceRoot":"","sources":["../../../../../s/features/speech/transcribe/parts/prep-audio.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,MAAY;IAC3D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,WAAW,EAAE,CAAA;IAC9C,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,EAAC,UAAU,EAAE,KAAK,EAAC,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;IAC7D,IAAI,KAAmB,CAAA;IACvB,IAAI,SAAS,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,IAAI,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;QACzC,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;SAAM,CAAC;QACP,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;IACpC,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAA;IACtD,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,CAAA;AACzB,CAAC"}
|