@omnimedia/omnitool 1.1.0-1 → 1.1.0-10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +12 -9
- package/s/context.ts +1 -1
- package/s/demo/demo.bundle.ts +6 -2
- package/s/demo/demo.css +5 -0
- package/s/demo/routines/filmstrip-test.ts +2 -2
- package/s/demo/routines/transcode-test.ts +4 -2
- package/s/demo/routines/transcriber-test.ts +34 -0
- package/s/demo/routines/transitions-test.ts +43 -0
- package/s/driver/driver.ts +17 -9
- package/s/driver/fns/host.ts +7 -6
- package/s/driver/fns/schematic.ts +45 -22
- package/s/driver/fns/work.ts +163 -151
- package/s/driver/utils/load-decoder-source.ts +3 -4
- package/s/features/speech/transcribe/default-spec.ts +11 -0
- package/s/features/speech/transcribe/parts/load-pipe.ts +19 -0
- package/s/features/speech/transcribe/parts/prep-audio.ts +23 -0
- package/s/features/speech/transcribe/parts/transcribe.ts +70 -0
- package/s/features/speech/transcribe/transcriber.ts +46 -0
- package/s/features/speech/transcribe/types.ts +82 -0
- package/s/features/speech/transcribe/worker.bundle.ts +40 -0
- package/s/features/transition/parts/fragment.ts +24 -0
- package/s/features/transition/parts/types.ts +94 -0
- package/s/features/transition/parts/uniforms.ts +29 -0
- package/s/features/transition/parts/vertex.ts +31 -0
- package/s/features/transition/transition.ts +60 -0
- package/s/index.html.ts +6 -1
- package/s/timeline/index.ts +1 -0
- package/s/timeline/parts/basics.ts +1 -1
- package/s/timeline/parts/compositor/export.ts +77 -0
- package/s/timeline/parts/compositor/parts/html-tree.ts +37 -0
- package/s/timeline/parts/compositor/parts/schedulers.ts +85 -0
- package/s/timeline/parts/compositor/parts/tree-builder.ts +184 -0
- package/s/timeline/parts/compositor/parts/webcodecs-tree.ts +30 -0
- package/s/timeline/parts/compositor/playback.ts +81 -0
- package/s/timeline/parts/compositor/samplers/html.ts +115 -0
- package/s/timeline/parts/compositor/samplers/webcodecs.ts +60 -0
- package/s/timeline/parts/item.ts +38 -6
- package/s/timeline/parts/media.ts +21 -0
- package/s/timeline/parts/waveform.ts +1 -1
- package/s/timeline/sugar/builders.ts +102 -0
- package/s/timeline/sugar/o.ts +117 -27
- package/s/timeline/sugar/omni-test.ts +2 -2
- package/s/timeline/sugar/omni.ts +21 -11
- package/s/timeline/types.ts +29 -0
- package/s/timeline/utils/audio-stream.ts +15 -0
- package/s/timeline/utils/checksum.ts +2 -1
- package/s/timeline/utils/matrix.ts +33 -0
- package/s/timeline/utils/video-cursor.ts +40 -0
- package/s/tools/common/loader.ts +26 -0
- package/s/tools/common/transformer-pipeline.ts +26 -0
- package/s/tools/speech-recognition/common/model.ts +26 -0
- package/s/tools/speech-recognition/whisper/fns/host.ts +25 -0
- package/s/tools/speech-recognition/whisper/fns/schematic.ts +23 -0
- package/s/tools/speech-recognition/whisper/fns/work.ts +91 -0
- package/s/tools/speech-recognition/whisper/parts/types.ts +38 -0
- package/s/tools/speech-recognition/whisper/parts/worker.bundle.ts +7 -0
- package/s/tools/speech-recognition/whisper/tool.ts +70 -0
- package/x/context.js +1 -1
- package/x/context.js.map +1 -1
- package/x/demo/demo.bundle.js +6 -2
- package/x/demo/demo.bundle.js.map +1 -1
- package/x/demo/demo.bundle.min.js +39 -37
- package/x/demo/demo.bundle.min.js.map +4 -4
- package/x/demo/demo.css +5 -0
- package/x/demo/routines/filmstrip-test.d.ts +1 -1
- package/x/demo/routines/filmstrip-test.js +2 -2
- package/x/demo/routines/filmstrip-test.js.map +1 -1
- package/x/demo/routines/transcode-test.js +4 -2
- package/x/demo/routines/transcode-test.js.map +1 -1
- package/x/demo/routines/transcriber-test.d.ts +4 -0
- package/x/demo/routines/transcriber-test.js +33 -0
- package/x/demo/routines/transcriber-test.js.map +1 -0
- package/x/demo/routines/transitions-test.d.ts +5 -0
- package/x/demo/routines/transitions-test.js +35 -0
- package/x/demo/routines/transitions-test.js.map +1 -0
- package/x/driver/driver.d.ts +3 -5
- package/x/driver/driver.js +16 -9
- package/x/driver/driver.js.map +1 -1
- package/x/driver/driver.worker.bundle.min.js +2537 -148
- package/x/driver/driver.worker.bundle.min.js.map +4 -4
- package/x/driver/fns/host.d.ts +9 -2
- package/x/driver/fns/host.js +3 -3
- package/x/driver/fns/host.js.map +1 -1
- package/x/driver/fns/schematic.d.ts +39 -21
- package/x/driver/fns/work.d.ts +11 -4
- package/x/driver/fns/work.js +111 -102
- package/x/driver/fns/work.js.map +1 -1
- package/x/driver/utils/load-decoder-source.d.ts +2 -1
- package/x/driver/utils/load-decoder-source.js +2 -3
- package/x/driver/utils/load-decoder-source.js.map +1 -1
- package/x/features/speech/transcribe/default-spec.d.ts +2 -0
- package/x/features/speech/transcribe/default-spec.js +8 -0
- package/x/features/speech/transcribe/default-spec.js.map +1 -0
- package/x/features/speech/transcribe/parts/load-pipe.d.ts +2 -0
- package/x/features/speech/transcribe/parts/load-pipe.js +13 -0
- package/x/features/speech/transcribe/parts/load-pipe.js.map +1 -0
- package/x/features/speech/transcribe/parts/prep-audio.d.ts +5 -0
- package/x/features/speech/transcribe/parts/prep-audio.js +21 -0
- package/x/features/speech/transcribe/parts/prep-audio.js.map +1 -0
- package/x/features/speech/transcribe/parts/transcribe.d.ts +5 -0
- package/x/features/speech/transcribe/parts/transcribe.js +56 -0
- package/x/features/speech/transcribe/parts/transcribe.js.map +1 -0
- package/x/features/speech/transcribe/transcriber.d.ts +5 -0
- package/x/features/speech/transcribe/transcriber.js +33 -0
- package/x/features/speech/transcribe/transcriber.js.map +1 -0
- package/x/features/speech/transcribe/types.d.ts +66 -0
- package/x/features/speech/transcribe/types.js +2 -0
- package/x/features/speech/transcribe/types.js.map +1 -0
- package/x/features/speech/transcribe/worker.bundle.d.ts +1 -0
- package/x/features/speech/transcribe/worker.bundle.js +33 -0
- package/x/features/speech/transcribe/worker.bundle.js.map +1 -0
- package/x/features/speech/transcribe/worker.bundle.min.js +2916 -0
- package/x/features/speech/transcribe/worker.bundle.min.js.map +7 -0
- package/x/features/transition/parts/fragment.d.ts +1 -0
- package/x/features/transition/parts/fragment.js +25 -0
- package/x/features/transition/parts/fragment.js.map +1 -0
- package/x/features/transition/parts/types.d.ts +23 -0
- package/x/features/transition/parts/types.js +2 -0
- package/x/features/transition/parts/types.js.map +1 -0
- package/x/features/transition/parts/uniforms.d.ts +31 -0
- package/x/features/transition/parts/uniforms.js +27 -0
- package/x/features/transition/parts/uniforms.js.map +1 -0
- package/x/features/transition/parts/vertex.d.ts +1 -0
- package/x/features/transition/parts/vertex.js +32 -0
- package/x/features/transition/parts/vertex.js.map +1 -0
- package/x/features/transition/transition.d.ts +5 -0
- package/x/features/transition/transition.js +50 -0
- package/x/features/transition/transition.js.map +1 -0
- package/x/index.html +13 -3
- package/x/index.html.js +6 -1
- package/x/index.html.js.map +1 -1
- package/x/timeline/index.d.ts +1 -0
- package/x/timeline/index.js +1 -0
- package/x/timeline/index.js.map +1 -1
- package/x/timeline/parts/basics.d.ts +1 -1
- package/x/timeline/parts/compositor/export.d.ts +9 -0
- package/x/timeline/parts/compositor/export.js +64 -0
- package/x/timeline/parts/compositor/export.js.map +1 -0
- package/x/timeline/parts/compositor/parts/html-tree.d.ts +3 -0
- package/x/timeline/parts/compositor/parts/html-tree.js +40 -0
- package/x/timeline/parts/compositor/parts/html-tree.js.map +1 -0
- package/x/timeline/parts/compositor/parts/schedulers.d.ts +15 -0
- package/x/timeline/parts/compositor/parts/schedulers.js +64 -0
- package/x/timeline/parts/compositor/parts/schedulers.js.map +1 -0
- package/x/timeline/parts/compositor/parts/tree-builder.d.ts +37 -0
- package/x/timeline/parts/compositor/parts/tree-builder.js +147 -0
- package/x/timeline/parts/compositor/parts/tree-builder.js.map +1 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.d.ts +3 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.js +28 -0
- package/x/timeline/parts/compositor/parts/webcodecs-tree.js.map +1 -0
- package/x/timeline/parts/compositor/playback.d.ts +19 -0
- package/x/timeline/parts/compositor/playback.js +71 -0
- package/x/timeline/parts/compositor/playback.js.map +1 -0
- package/x/timeline/parts/compositor/samplers/html.d.ts +3 -0
- package/x/timeline/parts/compositor/samplers/html.js +106 -0
- package/x/timeline/parts/compositor/samplers/html.js.map +1 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.d.ts +2 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.js +55 -0
- package/x/timeline/parts/compositor/samplers/webcodecs.js.map +1 -0
- package/x/timeline/parts/item.d.ts +34 -8
- package/x/timeline/parts/item.js +6 -3
- package/x/timeline/parts/item.js.map +1 -1
- package/x/timeline/parts/media.d.ts +3 -0
- package/x/timeline/parts/media.js +17 -0
- package/x/timeline/parts/media.js.map +1 -1
- package/x/timeline/parts/waveform.js +1 -1
- package/x/timeline/parts/waveform.js.map +1 -1
- package/x/timeline/sugar/builders.d.ts +1 -0
- package/x/timeline/sugar/builders.js +104 -0
- package/x/timeline/sugar/builders.js.map +1 -0
- package/x/timeline/sugar/o.d.ts +23 -5
- package/x/timeline/sugar/o.js +93 -27
- package/x/timeline/sugar/o.js.map +1 -1
- package/x/timeline/sugar/omni-test.js +1 -1
- package/x/timeline/sugar/omni-test.js.map +1 -1
- package/x/timeline/sugar/omni.d.ts +5 -2
- package/x/timeline/sugar/omni.js +17 -9
- package/x/timeline/sugar/omni.js.map +1 -1
- package/x/timeline/types.d.ts +24 -0
- package/x/timeline/types.js +2 -0
- package/x/timeline/types.js.map +1 -0
- package/x/timeline/utils/audio-stream.d.ts +6 -0
- package/x/timeline/utils/audio-stream.js +17 -0
- package/x/timeline/utils/audio-stream.js.map +1 -0
- package/x/timeline/utils/checksum.js +2 -1
- package/x/timeline/utils/checksum.js.map +1 -1
- package/x/timeline/utils/matrix.d.ts +8 -0
- package/x/timeline/utils/matrix.js +26 -0
- package/x/timeline/utils/matrix.js.map +1 -0
- package/x/timeline/utils/video-cursor.d.ts +10 -0
- package/x/timeline/utils/video-cursor.js +36 -0
- package/x/timeline/utils/video-cursor.js.map +1 -0
- package/x/tools/common/loader.d.ts +19 -0
- package/x/tools/common/loader.js +18 -0
- package/x/tools/common/loader.js.map +1 -0
- package/x/tools/common/transformer-pipeline.d.ts +8 -0
- package/x/tools/common/transformer-pipeline.js +24 -0
- package/x/tools/common/transformer-pipeline.js.map +1 -0
- package/x/tools/speech-recognition/common/model.d.ts +14 -0
- package/x/tools/speech-recognition/common/model.js +16 -0
- package/x/tools/speech-recognition/common/model.js.map +1 -0
- package/x/tools/speech-recognition/whisper/fns/host.d.ts +13 -0
- package/x/tools/speech-recognition/whisper/fns/host.js +19 -0
- package/x/tools/speech-recognition/whisper/fns/host.js.map +1 -0
- package/x/tools/speech-recognition/whisper/fns/schematic.d.ts +19 -0
- package/x/tools/speech-recognition/whisper/fns/schematic.js +2 -0
- package/x/tools/speech-recognition/whisper/fns/schematic.js.map +1 -0
- package/x/tools/speech-recognition/whisper/fns/work.d.ts +12 -0
- package/x/tools/speech-recognition/whisper/fns/work.js +74 -0
- package/x/tools/speech-recognition/whisper/fns/work.js.map +1 -0
- package/x/tools/speech-recognition/whisper/parts/types.d.ts +31 -0
- package/x/tools/speech-recognition/whisper/parts/types.js +2 -0
- package/x/tools/speech-recognition/whisper/parts/types.js.map +1 -0
- package/x/tools/speech-recognition/whisper/parts/worker.bundle.d.ts +1 -0
- package/x/tools/speech-recognition/whisper/parts/worker.bundle.js +4 -0
- package/x/tools/speech-recognition/whisper/parts/worker.bundle.js.map +1 -0
- package/x/tools/speech-recognition/whisper/parts/worker.bundle.min.js +8 -0
- package/x/tools/speech-recognition/whisper/parts/worker.bundle.min.js.map +7 -0
- package/x/tools/speech-recognition/whisper/tool.d.ts +12 -0
- package/x/tools/speech-recognition/whisper/tool.js +63 -0
- package/x/tools/speech-recognition/whisper/tool.js.map +1 -0
package/x/driver/fns/host.d.ts
CHANGED
|
@@ -2,10 +2,17 @@ import { Machina } from "../parts/machina.js";
|
|
|
2
2
|
export declare const setupDriverHost: (machina: Machina) => import("@e280/comrade").SetupHost<{
|
|
3
3
|
work: {
|
|
4
4
|
hello(): Promise<void>;
|
|
5
|
-
|
|
5
|
+
decodeAudio(input: {
|
|
6
6
|
source: import("./schematic.js").DecoderSource;
|
|
7
|
-
video: WritableStream<VideoFrame>;
|
|
8
7
|
audio: WritableStream<AudioData>;
|
|
8
|
+
start?: number;
|
|
9
|
+
end?: number;
|
|
10
|
+
}): Promise<void>;
|
|
11
|
+
decodeVideo(input: {
|
|
12
|
+
source: import("./schematic.js").DecoderSource;
|
|
13
|
+
video: WritableStream<VideoFrame>;
|
|
14
|
+
start?: number;
|
|
15
|
+
end?: number;
|
|
9
16
|
}): Promise<void>;
|
|
10
17
|
encode(input: import("./schematic.js").EncoderInput & {
|
|
11
18
|
bridge: WritableStream<StreamTargetChunk>;
|
package/x/driver/fns/host.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Comrade } from "@e280/comrade";
|
|
2
|
-
export const setupDriverHost = (machina) => Comrade.host(
|
|
2
|
+
export const setupDriverHost = (machina) => (Comrade.host(_shell => ({
|
|
3
3
|
async world() {
|
|
4
4
|
machina.count++;
|
|
5
|
-
}
|
|
6
|
-
}));
|
|
5
|
+
},
|
|
6
|
+
})));
|
|
7
7
|
//# sourceMappingURL=host.js.map
|
package/x/driver/fns/host.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host.js","sourceRoot":"","sources":["../../../s/driver/fns/host.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AAIrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAkB,CAAC,
|
|
1
|
+
{"version":3,"file":"host.js","sourceRoot":"","sources":["../../../s/driver/fns/host.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AAIrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,CACpD,OAAO,CAAC,IAAI,CAAkB,MAAM,CAAC,EAAE,CAAC,CAAC;IACxC,KAAK,CAAC,KAAK;QACV,OAAO,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;CACD,CAAC,CAAC,CACH,CAAA"}
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { AsSchematic } from "@e280/comrade";
|
|
2
2
|
import type { AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig } from "mediabunny";
|
|
3
|
+
import { Mat6 } from "../../timeline/utils/matrix.js";
|
|
3
4
|
export type DriverSchematic = AsSchematic<{
|
|
4
5
|
work: {
|
|
5
6
|
hello(): Promise<void>;
|
|
6
|
-
|
|
7
|
+
decodeAudio(input: {
|
|
7
8
|
source: DecoderSource;
|
|
8
|
-
video: WritableStream<VideoFrame>;
|
|
9
9
|
audio: WritableStream<AudioData>;
|
|
10
|
+
start?: number;
|
|
11
|
+
end?: number;
|
|
12
|
+
}): Promise<void>;
|
|
13
|
+
decodeVideo(input: {
|
|
14
|
+
source: DecoderSource;
|
|
15
|
+
video: WritableStream<VideoFrame>;
|
|
16
|
+
start?: number;
|
|
17
|
+
end?: number;
|
|
10
18
|
}): Promise<void>;
|
|
11
19
|
encode(input: EncoderInput & {
|
|
12
20
|
bridge: WritableStream<StreamTargetChunk>;
|
|
@@ -18,18 +26,19 @@ export type DriverSchematic = AsSchematic<{
|
|
|
18
26
|
};
|
|
19
27
|
}>;
|
|
20
28
|
export interface EncoderInput {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
25
|
-
config: {
|
|
26
|
-
video: VideoEncodingConfig;
|
|
27
|
-
audio: AudioEncodingConfig;
|
|
28
|
-
};
|
|
29
|
+
video?: ReadableStream<VideoFrame>;
|
|
30
|
+
audio?: ReadableStream<AudioData>;
|
|
31
|
+
config: RenderConfig;
|
|
29
32
|
}
|
|
30
|
-
export
|
|
33
|
+
export interface RenderConfig {
|
|
34
|
+
video: VideoEncodingConfig;
|
|
35
|
+
audio: AudioEncodingConfig;
|
|
36
|
+
}
|
|
37
|
+
export type DecoderSource = Blob | string | URL;
|
|
31
38
|
export interface DecoderInput {
|
|
32
39
|
source: DecoderSource;
|
|
40
|
+
start?: number;
|
|
41
|
+
end?: number;
|
|
33
42
|
onFrame?: (frame: VideoFrame) => Promise<VideoFrame>;
|
|
34
43
|
}
|
|
35
44
|
export interface MuxOpts {
|
|
@@ -46,21 +55,30 @@ export interface MuxOpts {
|
|
|
46
55
|
};
|
|
47
56
|
}
|
|
48
57
|
export type Composition = Layer | (Layer | Composition)[];
|
|
49
|
-
export type Transform = {
|
|
50
|
-
x?: number;
|
|
51
|
-
y?: number;
|
|
52
|
-
scale?: number;
|
|
53
|
-
opacity?: number;
|
|
54
|
-
anchor?: number;
|
|
55
|
-
};
|
|
56
58
|
export type TextLayer = {
|
|
57
59
|
kind: 'text';
|
|
58
60
|
content: string;
|
|
59
61
|
fontSize?: number;
|
|
60
62
|
color?: string;
|
|
61
|
-
|
|
63
|
+
matrix?: Mat6;
|
|
64
|
+
};
|
|
62
65
|
export type ImageLayer = {
|
|
63
66
|
kind: 'image';
|
|
64
67
|
frame: VideoFrame;
|
|
65
|
-
|
|
66
|
-
|
|
68
|
+
matrix?: Mat6;
|
|
69
|
+
};
|
|
70
|
+
export type TransitionLayer = {
|
|
71
|
+
kind: 'transition';
|
|
72
|
+
name: string;
|
|
73
|
+
progress: number;
|
|
74
|
+
from: VideoFrame;
|
|
75
|
+
to: VideoFrame;
|
|
76
|
+
};
|
|
77
|
+
export type GapLayer = {
|
|
78
|
+
kind: 'gap';
|
|
79
|
+
};
|
|
80
|
+
export type Audio = {
|
|
81
|
+
kind: "audio";
|
|
82
|
+
data: AudioData;
|
|
83
|
+
};
|
|
84
|
+
export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer;
|
package/x/driver/fns/work.d.ts
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { Input } from "mediabunny";
|
|
2
|
-
import { Composition } from "./schematic.js";
|
|
2
|
+
import { Composition, DecoderSource } from "./schematic.js";
|
|
3
3
|
export declare const setupDriverWork: import("@e280/comrade").SetupWork<{
|
|
4
4
|
work: {
|
|
5
5
|
hello(): Promise<void>;
|
|
6
|
-
|
|
7
|
-
source:
|
|
8
|
-
video: WritableStream<VideoFrame>;
|
|
6
|
+
decodeAudio(input: {
|
|
7
|
+
source: DecoderSource;
|
|
9
8
|
audio: WritableStream<AudioData>;
|
|
9
|
+
start?: number;
|
|
10
|
+
end?: number;
|
|
11
|
+
}): Promise<void>;
|
|
12
|
+
decodeVideo(input: {
|
|
13
|
+
source: DecoderSource;
|
|
14
|
+
video: WritableStream<VideoFrame>;
|
|
15
|
+
start?: number;
|
|
16
|
+
end?: number;
|
|
10
17
|
}): Promise<void>;
|
|
11
18
|
encode(input: import("./schematic.js").EncoderInput & {
|
|
12
19
|
bridge: WritableStream<Input>;
|
package/x/driver/fns/work.js
CHANGED
|
@@ -1,76 +1,72 @@
|
|
|
1
1
|
import { Comrade } from "@e280/comrade";
|
|
2
|
-
import { Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource } from "mediabunny";
|
|
3
2
|
import { autoDetectRenderer, Container, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter } from "pixi.js";
|
|
3
|
+
import { Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource } from "mediabunny";
|
|
4
|
+
import { mat6ToMatrix } from "../../timeline/utils/matrix.js";
|
|
5
|
+
import { makeTransition } from "../../features/transition/transition.js";
|
|
4
6
|
DOMAdapter.set(WebWorkerAdapter);
|
|
5
|
-
|
|
7
|
+
const loadSource = async (source) => {
|
|
8
|
+
if (source instanceof Blob) {
|
|
9
|
+
return new BlobSource(source);
|
|
10
|
+
}
|
|
11
|
+
else {
|
|
12
|
+
return new UrlSource(source);
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
export const setupDriverWork = (Comrade.work(shell => ({
|
|
6
16
|
async hello() {
|
|
7
|
-
await host.world();
|
|
17
|
+
await shell.host.world();
|
|
8
18
|
},
|
|
9
|
-
async
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
19
|
+
async decodeAudio({ source, audio, start, end }) {
|
|
20
|
+
const input = new Input({
|
|
21
|
+
source: await loadSource(source),
|
|
22
|
+
formats: ALL_FORMATS
|
|
23
|
+
});
|
|
24
|
+
const audioTrack = await input.getPrimaryAudioTrack();
|
|
25
|
+
const audioDecodable = await audioTrack?.canDecode();
|
|
26
|
+
const audioWriter = audio.getWriter();
|
|
27
|
+
if (audioDecodable && audioTrack) {
|
|
28
|
+
const sink = new AudioSampleSink(audioTrack);
|
|
29
|
+
for await (const sample of sink.samples(start, end)) {
|
|
30
|
+
const frame = sample.toAudioData();
|
|
31
|
+
await audioWriter.write(frame);
|
|
32
|
+
sample.close();
|
|
33
|
+
frame.close();
|
|
17
34
|
}
|
|
18
|
-
|
|
35
|
+
await audioWriter.close();
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
async decodeVideo({ source, video, start, end }) {
|
|
19
39
|
const input = new Input({
|
|
20
|
-
source: await loadSource(),
|
|
40
|
+
source: await loadSource(source),
|
|
21
41
|
formats: ALL_FORMATS
|
|
22
42
|
});
|
|
23
|
-
const
|
|
24
|
-
input.getPrimaryVideoTrack(),
|
|
25
|
-
input.getPrimaryAudioTrack()
|
|
26
|
-
]);
|
|
43
|
+
const videoTrack = await input.getPrimaryVideoTrack();
|
|
27
44
|
const videoDecodable = await videoTrack?.canDecode();
|
|
28
|
-
const audioDecodable = await audioTrack?.canDecode();
|
|
29
45
|
const videoWriter = video.getWriter();
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
await videoWriter.close();
|
|
42
|
-
}
|
|
43
|
-
})(),
|
|
44
|
-
(async () => {
|
|
45
|
-
if (audioDecodable && audioTrack) {
|
|
46
|
-
const sink = new AudioSampleSink(audioTrack);
|
|
47
|
-
for await (const sample of sink.samples()) {
|
|
48
|
-
const frame = sample.toAudioData();
|
|
49
|
-
await audioWriter.write(frame);
|
|
50
|
-
sample.close();
|
|
51
|
-
frame.close();
|
|
52
|
-
}
|
|
53
|
-
await audioWriter.close();
|
|
54
|
-
}
|
|
55
|
-
})()
|
|
56
|
-
]);
|
|
46
|
+
if (videoDecodable && videoTrack) {
|
|
47
|
+
const sink = new VideoSampleSink(videoTrack);
|
|
48
|
+
for await (const sample of sink.samples(start, end)) {
|
|
49
|
+
const frame = sample.toVideoFrame();
|
|
50
|
+
await videoWriter.write(frame);
|
|
51
|
+
sample.close();
|
|
52
|
+
frame.close();
|
|
53
|
+
}
|
|
54
|
+
await videoWriter.close();
|
|
55
|
+
}
|
|
57
56
|
},
|
|
58
|
-
async encode({
|
|
57
|
+
async encode({ video, audio, config, bridge }) {
|
|
59
58
|
const output = new Output({
|
|
60
59
|
format: new Mp4OutputFormat(),
|
|
61
60
|
target: new StreamTarget(bridge, { chunked: true })
|
|
62
61
|
});
|
|
63
|
-
const videoSource = new VideoSampleSource(config.video);
|
|
64
|
-
output.addVideoTrack(videoSource);
|
|
65
62
|
// since AudioSample is not transferable it fails to transfer encoder bitrate config
|
|
66
63
|
// so it needs to be hardcoded not set through constants eg QUALITY_LOW
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
(async () => {
|
|
64
|
+
const promises = [];
|
|
65
|
+
if (video) {
|
|
66
|
+
const videoSource = new VideoSampleSource(config.video);
|
|
67
|
+
output.addVideoTrack(videoSource);
|
|
68
|
+
const videoReader = video.getReader();
|
|
69
|
+
promises.push((async () => {
|
|
74
70
|
while (true) {
|
|
75
71
|
const { done, value } = await videoReader.read();
|
|
76
72
|
if (done)
|
|
@@ -79,8 +75,13 @@ export const setupDriverWork = Comrade.work(({ host }, rig) => ({
|
|
|
79
75
|
await videoSource.add(sample);
|
|
80
76
|
sample.close();
|
|
81
77
|
}
|
|
82
|
-
})()
|
|
83
|
-
|
|
78
|
+
})());
|
|
79
|
+
}
|
|
80
|
+
if (audio) {
|
|
81
|
+
const audioSource = new AudioSampleSource(config.audio);
|
|
82
|
+
output.addAudioTrack(audioSource);
|
|
83
|
+
const audioReader = audio.getReader();
|
|
84
|
+
promises.push((async () => {
|
|
84
85
|
while (true) {
|
|
85
86
|
const { done, value } = await audioReader.read();
|
|
86
87
|
if (done)
|
|
@@ -90,30 +91,30 @@ export const setupDriverWork = Comrade.work(({ host }, rig) => ({
|
|
|
90
91
|
sample.close();
|
|
91
92
|
value.close();
|
|
92
93
|
}
|
|
93
|
-
})()
|
|
94
|
-
|
|
94
|
+
})());
|
|
95
|
+
}
|
|
96
|
+
await output.start();
|
|
97
|
+
await Promise.all(promises);
|
|
95
98
|
await output.finalize();
|
|
96
99
|
},
|
|
97
100
|
async composite(composition) {
|
|
98
101
|
const { stage, renderer } = await renderPIXI(1920, 1080);
|
|
99
102
|
stage.removeChildren();
|
|
100
|
-
const {
|
|
103
|
+
const { dispose } = await renderLayer(composition, stage);
|
|
101
104
|
renderer.render(stage);
|
|
102
105
|
// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
|
|
103
106
|
// if its very slow on eg edge try chrome
|
|
104
107
|
const frame = new VideoFrame(renderer.canvas, {
|
|
105
|
-
timestamp:
|
|
106
|
-
duration:
|
|
108
|
+
timestamp: 0,
|
|
109
|
+
duration: 0,
|
|
107
110
|
});
|
|
108
|
-
baseFrame?.close();
|
|
109
111
|
renderer.clear();
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
rig.transfer = [frame];
|
|
112
|
+
dispose();
|
|
113
|
+
shell.transfer = [frame];
|
|
114
114
|
return frame;
|
|
115
115
|
}
|
|
116
|
-
}));
|
|
116
|
+
})));
|
|
117
|
+
// TODO suspicious global, probably bad
|
|
117
118
|
let pixi = null;
|
|
118
119
|
async function renderPIXI(width, height) {
|
|
119
120
|
if (pixi)
|
|
@@ -129,33 +130,33 @@ async function renderPIXI(width, height) {
|
|
|
129
130
|
pixi = { renderer, stage };
|
|
130
131
|
return pixi;
|
|
131
132
|
}
|
|
132
|
-
|
|
133
|
+
const transitions = new Map();
|
|
134
|
+
async function renderLayer(layer, parent) {
|
|
133
135
|
if (Array.isArray(layer)) {
|
|
134
|
-
|
|
136
|
+
const disposers = [];
|
|
135
137
|
for (const child of layer) {
|
|
136
|
-
const result = await renderLayer(child, parent
|
|
137
|
-
|
|
138
|
+
const result = await renderLayer(child, parent);
|
|
139
|
+
disposers.push(result.dispose);
|
|
138
140
|
}
|
|
139
|
-
return {
|
|
140
|
-
}
|
|
141
|
-
if (!isRenderableLayer(layer)) {
|
|
142
|
-
console.warn('Invalid layer', layer);
|
|
143
|
-
return { disposables };
|
|
141
|
+
return { dispose: () => disposers.forEach(d => d()) };
|
|
144
142
|
}
|
|
145
143
|
switch (layer.kind) {
|
|
146
144
|
case 'text':
|
|
147
|
-
return renderTextLayer(layer, parent
|
|
145
|
+
return renderTextLayer(layer, parent);
|
|
148
146
|
case 'image':
|
|
149
|
-
return renderImageLayer(layer, parent
|
|
147
|
+
return renderImageLayer(layer, parent);
|
|
148
|
+
case 'transition':
|
|
149
|
+
return renderTransitionLayer(layer, parent);
|
|
150
|
+
case 'gap': {
|
|
151
|
+
pixi?.renderer.clear();
|
|
152
|
+
return { dispose: () => { } };
|
|
153
|
+
}
|
|
150
154
|
default:
|
|
151
155
|
console.warn('Unknown layer kind', layer.kind);
|
|
152
|
-
return {
|
|
156
|
+
return { dispose: () => { } };
|
|
153
157
|
}
|
|
154
158
|
}
|
|
155
|
-
function
|
|
156
|
-
return !!layer && typeof layer === 'object' && typeof layer.kind === 'string';
|
|
157
|
-
}
|
|
158
|
-
function renderTextLayer(layer, parent, disposables) {
|
|
159
|
+
function renderTextLayer(layer, parent) {
|
|
159
160
|
const text = new Text({
|
|
160
161
|
text: layer.content,
|
|
161
162
|
style: {
|
|
@@ -164,29 +165,37 @@ function renderTextLayer(layer, parent, disposables) {
|
|
|
164
165
|
fill: layer.color ?? 'white'
|
|
165
166
|
}
|
|
166
167
|
});
|
|
167
|
-
applyTransform(text, layer);
|
|
168
|
+
applyTransform(text, layer.matrix);
|
|
168
169
|
parent.addChild(text);
|
|
169
|
-
|
|
170
|
-
return { disposables };
|
|
170
|
+
return { dispose: () => text.destroy(true) };
|
|
171
171
|
}
|
|
172
|
-
function renderImageLayer(layer, parent
|
|
172
|
+
function renderImageLayer(layer, parent) {
|
|
173
173
|
const texture = Texture.from(layer.frame);
|
|
174
174
|
const sprite = new Sprite(texture);
|
|
175
|
-
applyTransform(sprite, layer);
|
|
175
|
+
applyTransform(sprite, layer.matrix);
|
|
176
|
+
parent.addChild(sprite);
|
|
177
|
+
return { dispose: () => {
|
|
178
|
+
sprite.destroy(true);
|
|
179
|
+
texture.destroy(true);
|
|
180
|
+
layer.frame.close();
|
|
181
|
+
} };
|
|
182
|
+
}
|
|
183
|
+
function renderTransitionLayer({ from, to, progress, name }, parent) {
|
|
184
|
+
const transition = transitions.get(name) ??
|
|
185
|
+
(transitions.set(name, makeTransition({
|
|
186
|
+
name: "circle",
|
|
187
|
+
renderer: pixi.renderer
|
|
188
|
+
})),
|
|
189
|
+
transitions.get(name));
|
|
190
|
+
const texture = transition.render({ from, to, progress, width: from.displayWidth, height: from.displayHeight });
|
|
191
|
+
const sprite = new Sprite(texture);
|
|
176
192
|
parent.addChild(sprite);
|
|
177
|
-
|
|
178
|
-
return { baseFrame: layer.frame, disposables };
|
|
193
|
+
return { dispose: () => sprite.destroy(false) };
|
|
179
194
|
}
|
|
180
|
-
function applyTransform(target,
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
if (t.scale)
|
|
186
|
-
target.scale.set(t.scale);
|
|
187
|
-
if (t.opacity)
|
|
188
|
-
target.alpha = t.opacity;
|
|
189
|
-
if (t.anchor && 'anchor' in target)
|
|
190
|
-
target.anchor.set(t.anchor);
|
|
195
|
+
function applyTransform(target, worldMatrix) {
|
|
196
|
+
if (!worldMatrix)
|
|
197
|
+
return;
|
|
198
|
+
const mx = mat6ToMatrix(worldMatrix);
|
|
199
|
+
target.setFromMatrix(mx);
|
|
191
200
|
}
|
|
192
201
|
//# sourceMappingURL=work.js.map
|
package/x/driver/fns/work.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"work.js","sourceRoot":"","sources":["../../../s/driver/fns/work.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,
|
|
1
|
+
{"version":3,"file":"work.js","sourceRoot":"","sources":["../../../s/driver/fns/work.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,kBAAkB,EAAE,SAAS,EAAY,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAS,MAAM,SAAS,CAAA;AAC5H,OAAO,EAAC,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAE7M,OAAO,EAAO,YAAY,EAAC,MAAM,gCAAgC,CAAA;AACjE,OAAO,EAAC,cAAc,EAAC,MAAM,yCAAyC,CAAA;AAGtE,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;AAEhC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAqB,EAAE,EAAE;IAClD,IAAG,MAAM,YAAY,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;SAAM,CAAC;QACP,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,CAC9B,OAAO,CAAC,IAAI,CAAkB,KAAK,CAAC,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,KAAK;QACV,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACvB,MAAM,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAA;QACrD,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,SAAS,EAAE,CAAA;QACpD,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;QAErC,IAAI,cAAc,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,UAAU,CAAC,CAAA;YAC5C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;gBAClC,MAAM,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC9B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACd,KAAK,CAAC,KAAK,EAAE,CAAA;YACd,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACvB,MAAM,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAA;QACrD,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,SAAS,EAAE,CAAA;QACpD,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;QAErC,IAAI,cAAc,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,UAAU,CAAC,CAAA;YAC5C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;gBACnC,MAAM,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC9B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACd,KAAK,CAAC,KAAK,EAAE,CAAA;YACd,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;YACzB,MAAM,EAAE,IAAI,eAAe,EAAE;YAC7B,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,EAAE,EAAC,OAAO,EAAE,IAAI,EAAC,CAAC;SACjD,CAAC,CAAA;QACF,oFAAoF;QACpF,uEAAuE;QAEvE,MAAM,QAAQ,GAAG,EAAE,CAAA;QAEnB,IAAG,KAAK,EAAE,CAAC;YACV,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACvD,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAA;YACjC,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;YACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;gBACzB,OAAO,IAAI,EAAE,CAAC;oBACb,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;oBAC9C,IAAI,IAAI;wBAAE,MAAK;oBACf,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;oBACrC,MAAM,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,CAAC,KAAK,EAAE,CAAA;gBACf,CAAC;YACF,CAAC,CAAC,EAAE,CAAC,CAAA;QACN,CAAC;QAED,IAAG,KAAK,EAAE,CAAC;YACV,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACvD,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAA;YACjC,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;YACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;gBACzB,OAAO,IAAI,EAAE,CAAC;oBACb,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAA;oBAC9C,IAAI,IAAI;wBAAE,MAAK;oBACf,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;oBACrC,MAAM,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,KAAK,CAAC,KAAK,EAAE,CAAA;gBACd,CAAC;YACF,CAAC,CAAC,EAAE,CAAC,CAAA;QACN,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC3B,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAW;QAC1B,MAAM,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACtD,KAAK,CAAC,cAAc,EAAE,CAAA;QAEtB,MAAM,EAAC,OAAO,EAAC,GAAG,MAAM,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QACvD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEtB,gGAAgG;QAChG,yCAAyC;QACzC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE;YAC7C,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;SACX,CAAC,CAAA;QAEF,QAAQ,CAAC,KAAK,EAAE,CAAA;QAChB,OAAO,EAAE,CAAA;QAET,KAAK,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,CAAA;QACxB,OAAO,KAAK,CAAA;IACb,CAAC;CACD,CAAC,CAAC,CACH,CAAA;AAED,uCAAuC;AACvC,IAAI,IAAI,GAGG,IAAI,CAAA;AAEf,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,MAAc;IACtD,IAAI,IAAI;QACP,OAAO,IAAI,CAAA;IAEZ,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC;QACzC,KAAK;QACL,MAAM;QACN,UAAU,EAAE,OAAO,EAAE,iDAAiD;QACtE,UAAU,EAAE,OAAO;QACnB,kBAAkB,EAAE,CAAC;KACrB,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAA;IAC7B,IAAI,GAAG,EAAC,QAAQ,EAAE,KAAK,EAAC,CAAA;IAExB,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,WAAW,GAAmD,IAAI,GAAG,EAAE,CAAA;AAI7E,KAAK,UAAU,WAAW,CACzB,KAA0B,EAC1B,MAAiB;IAEjB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAmB,EAAE,CAAA;QACpC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAC/C,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC/B,CAAC;QACD,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAC,CAAA;IACpD,CAAC;IAED,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,MAAM;YACV,OAAO,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACtC,KAAK,OAAO;YACX,OAAO,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACvC,KAAK,YAAY;YAChB,OAAO,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAC5C,KAAK,KAAK,CAAC,CAAC,CAAC;YACZ,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;YACtB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAC,CAAA;QAC3B,CAAC;QACD;YACC,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAG,KAAa,CAAC,IAAI,CAAC,CAAA;YACvD,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAC,CAAA;IAC5B,CAAC;AACF,CAAC;AAED,SAAS,eAAe,CACvB,KAAqC,EACrC,MAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC;QACrB,IAAI,EAAE,KAAK,CAAC,OAAO;QACnB,KAAK,EAAE;YACN,UAAU,EAAE,YAAY;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE;YAC9B,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,OAAO;SAC5B;KACD,CAAC,CAAA;IACF,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAC,CAAA;AAC3C,CAAC;AAED,SAAS,gBAAgB,CACxB,KAAsC,EACtC,MAAiB;IAEjB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACzC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IACpC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACvB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACpB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACrB,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACpB,CAAC,EAAC,CAAA;AACH,CAAC;AAED,SAAS,qBAAqB,CAC7B,EAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAuC,EAChE,MAAiB;IAEjB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;QACvC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,cAAc,CAAC;YACrC,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,IAAK,CAAC,QAAQ;SACxB,CAAC,CAAC;YACF,WAAW,CAAC,GAAG,CAAC,IAAI,CAAE,CACvB,CAAA;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAC,CAAC,CAAA;IAC7G,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACvB,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAC,CAAA;AAC9C,CAAC;AAED,SAAS,cAAc,CAAC,MAAqB,EAAE,WAAkB;IAC/D,IAAI,CAAC,WAAW;QAAE,OAAM;IACzB,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;IACnC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;AAC1B,CAAC"}
|
|
@@ -1,2 +1,3 @@
|
|
|
1
|
+
import { BlobSource, UrlSource } from "mediabunny";
|
|
1
2
|
import { DecoderSource } from "../fns/schematic.js";
|
|
2
|
-
export declare function loadDecoderSource(source: DecoderSource): Promise<
|
|
3
|
+
export declare function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource>;
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { BlobSource, UrlSource } from "mediabunny";
|
|
2
2
|
// only streamable sources
|
|
3
3
|
export async function loadDecoderSource(source) {
|
|
4
|
-
if (source instanceof
|
|
5
|
-
|
|
6
|
-
return new BlobSource(file);
|
|
4
|
+
if (source instanceof Blob) {
|
|
5
|
+
return new BlobSource(source);
|
|
7
6
|
}
|
|
8
7
|
else {
|
|
9
8
|
return new UrlSource(source);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load-decoder-source.js","sourceRoot":"","sources":["../../../s/driver/utils/load-decoder-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAGhD,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAqB;IAC5D,IAAG,MAAM,YAAY,
|
|
1
|
+
{"version":3,"file":"load-decoder-source.js","sourceRoot":"","sources":["../../../s/driver/utils/load-decoder-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAGhD,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAqB;IAC5D,IAAG,MAAM,YAAY,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;SAAM,CAAC;QACP,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-spec.js","sourceRoot":"","sources":["../../../../s/features/speech/transcribe/default-spec.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAoB,EAAE,CAAC,CAAC;IAC7D,KAAK,EAAE,yCAAyC;IAChD,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,MAAM;IACd,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,CAAC;CACf,CAAC,CAAA"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { pipeline } from "@huggingface/transformers";
|
|
2
|
+
export async function loadPipe(options) {
|
|
3
|
+
const { spec, onLoading } = options;
|
|
4
|
+
const pipe = await pipeline("automatic-speech-recognition", spec.model, {
|
|
5
|
+
device: spec.device,
|
|
6
|
+
dtype: spec.dtype,
|
|
7
|
+
progress_callback: (data) => {
|
|
8
|
+
onLoading({ total: data.total, progress: data.progress });
|
|
9
|
+
},
|
|
10
|
+
});
|
|
11
|
+
return pipe;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=load-pipe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-pipe.js","sourceRoot":"","sources":["../../../../../s/features/speech/transcribe/parts/load-pipe.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAIlD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAA+B;IAC7D,MAAM,EAAC,IAAI,EAAE,SAAS,EAAC,GAAG,OAAO,CAAA;IAEjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,8BAA8B,EAAE,IAAI,CAAC,KAAK,EAAE;QACvE,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,iBAAiB,EAAE,CAAC,IAAS,EAAE,EAAE;YAChC,SAAS,CAAC,EAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAC,CAAC,CAAA;QACxD,CAAC;KACD,CAAC,CAAA;IAEF,OAAO,IAAI,CAAA;AACZ,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export async function prepAudio(driver, source) {
|
|
2
|
+
const arrayBuffer = await source.arrayBuffer();
|
|
3
|
+
const audioCTX = new AudioContext({ sampleRate: 16000 });
|
|
4
|
+
const audioData = await audioCTX.decodeAudioData(arrayBuffer);
|
|
5
|
+
let audio;
|
|
6
|
+
if (audioData.numberOfChannels === 2) {
|
|
7
|
+
const SCALING_FACTOR = Math.sqrt(2);
|
|
8
|
+
const left = audioData.getChannelData(0);
|
|
9
|
+
const right = audioData.getChannelData(1);
|
|
10
|
+
audio = new Float32Array(left.length);
|
|
11
|
+
for (let i = 0; i < audioData.length; ++i) {
|
|
12
|
+
audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
audio = audioData.getChannelData(0);
|
|
17
|
+
}
|
|
18
|
+
const duration = await driver.getAudioDuration(source);
|
|
19
|
+
return { audio, duration };
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=prep-audio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prep-audio.js","sourceRoot":"","sources":["../../../../../s/features/speech/transcribe/parts/prep-audio.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,MAAY;IAC3D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,WAAW,EAAE,CAAA;IAC9C,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,EAAC,UAAU,EAAE,KAAK,EAAC,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;IAC7D,IAAI,KAAmB,CAAA;IACvB,IAAI,SAAS,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,IAAI,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;QACzC,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;SAAM,CAAC;QACP,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;IACpC,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAA;IACtD,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,CAAA;AACzB,CAAC"}
|