@editframe/vite-plugin 0.5.0-beta.9 → 0.6.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packages/vite-plugin/forbidRelativePaths.cjs +8 -0
- package/dist/packages/vite-plugin/forbidRelativePaths.d.ts +3 -0
- package/dist/packages/vite-plugin/index.cjs +65 -0
- package/dist/packages/vite-plugin/index.d.ts +9 -0
- package/dist/packages/vite-plugin/{src/vite-plugin-editframe.mjs → index.js} +5 -9
- package/dist/packages/vite-plugin/sendTaskResult.cjs +54 -0
- package/dist/packages/vite-plugin/sendTaskResult.d.ts +5 -0
- package/dist/packages/vite-plugin/{src/sendTaskResult.mjs → sendTaskResult.js} +4 -4
- package/package.json +14 -3
- package/dist/lib/assets/Probe.mjs +0 -176
- package/dist/lib/assets/idempotentTask.mjs +0 -57
- package/dist/lib/assets/md5.mjs +0 -28
- package/dist/lib/assets/mp4FileWritable.mjs +0 -21
- package/dist/lib/assets/tasks/cacheImage.mjs +0 -22
- package/dist/lib/assets/tasks/findOrCreateCaptions.mjs +0 -26
- package/dist/lib/assets/tasks/generateTrack.mjs +0 -52
- package/dist/lib/assets/tasks/generateTrackFragmentIndex.mjs +0 -71
- package/dist/lib/av/MP4File.mjs +0 -160
- package/dist/lib/util/execPromise.mjs +0 -6
- /package/dist/packages/vite-plugin/{src/forbidRelativePaths.mjs → forbidRelativePaths.js} +0 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
3
|
+
const forbidRelativePaths = (req) => {
|
|
4
|
+
if (req.url?.includes("..")) {
|
|
5
|
+
throw new Error("Relative paths are forbidden");
|
|
6
|
+
}
|
|
7
|
+
};
|
|
8
|
+
exports.forbidRelativePaths = forbidRelativePaths;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
3
|
+
const path = require("node:path");
|
|
4
|
+
const debug = require("debug");
|
|
5
|
+
const assets = require("@editframe/assets");
|
|
6
|
+
const forbidRelativePaths = require("./forbidRelativePaths.cjs");
|
|
7
|
+
const sendTaskResult = require("./sendTaskResult.cjs");
|
|
8
|
+
const vitePluginEditframe = (options) => {
|
|
9
|
+
return {
|
|
10
|
+
name: "vite-plugin-editframe",
|
|
11
|
+
configureServer(server) {
|
|
12
|
+
server.middlewares.use(async (req, res, next) => {
|
|
13
|
+
const log = debug("@ef:vite-plugin");
|
|
14
|
+
if (req.url?.startsWith("/@ef")) {
|
|
15
|
+
forbidRelativePaths.forbidRelativePaths(req);
|
|
16
|
+
} else {
|
|
17
|
+
return next();
|
|
18
|
+
}
|
|
19
|
+
log(`Handling ${req.url}`);
|
|
20
|
+
const requestPath = req.url.replace(/^\/@ef-[^\/]+\//, "");
|
|
21
|
+
const assetPath = requestPath.replace(/\?.*$/, "");
|
|
22
|
+
const absolutePath = path.join(options.root, assetPath).replace("dist/", "src/");
|
|
23
|
+
options.cacheRoot = options.cacheRoot.replace("dist/", "src/");
|
|
24
|
+
const efPrefix = req.url.split("/")[1];
|
|
25
|
+
switch (efPrefix) {
|
|
26
|
+
case "@ef-asset": {
|
|
27
|
+
if (req.method !== "HEAD") {
|
|
28
|
+
res.writeHead(405, { Allow: "HEAD" });
|
|
29
|
+
res.end();
|
|
30
|
+
}
|
|
31
|
+
assets.md5FilePath(absolutePath).then((md5) => {
|
|
32
|
+
res.writeHead(200, {
|
|
33
|
+
etag: md5
|
|
34
|
+
});
|
|
35
|
+
res.end();
|
|
36
|
+
}).catch(next);
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
case "@ef-track-fragment-index": {
|
|
40
|
+
log(`Serving track fragment index for ${absolutePath}`);
|
|
41
|
+
assets.generateTrackFragmentIndex(options.cacheRoot, absolutePath).then((taskResult) => sendTaskResult.sendTaskResult(req, res, taskResult)).catch(next);
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
case "@ef-track": {
|
|
45
|
+
log(`Serving track for ${absolutePath}`);
|
|
46
|
+
assets.generateTrack(options.cacheRoot, absolutePath, req.url).then((taskResult) => sendTaskResult.sendTaskResult(req, res, taskResult)).catch(next);
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
case "@ef-captions":
|
|
50
|
+
log(`Serving captions for ${absolutePath}`);
|
|
51
|
+
assets.findOrCreateCaptions(options.cacheRoot, absolutePath).then((taskResult) => sendTaskResult.sendTaskResult(req, res, taskResult)).catch(next);
|
|
52
|
+
break;
|
|
53
|
+
case "@ef-image":
|
|
54
|
+
log(`Serving image file ${absolutePath}`);
|
|
55
|
+
assets.cacheImage(options.cacheRoot, absolutePath).then((taskResult) => sendTaskResult.sendTaskResult(req, res, taskResult)).catch(next);
|
|
56
|
+
break;
|
|
57
|
+
default:
|
|
58
|
+
log(`Unknown asset type ${efPrefix}`);
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
};
|
|
65
|
+
exports.vitePluginEditframe = vitePluginEditframe;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
interface VitePluginEditframeOptions {
|
|
2
|
+
root: string;
|
|
3
|
+
cacheRoot: string;
|
|
4
|
+
}
|
|
5
|
+
export declare const vitePluginEditframe: (options: VitePluginEditframeOptions) => {
|
|
6
|
+
name: string;
|
|
7
|
+
configureServer(this: void, server: import('vite').ViteDevServer): void;
|
|
8
|
+
};
|
|
9
|
+
export {};
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import path from "path";
|
|
1
|
+
import path from "node:path";
|
|
2
2
|
import debug from "debug";
|
|
3
|
-
import { findOrCreateCaptions } from "
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { cacheImage } from "../../../lib/assets/tasks/cacheImage.mjs";
|
|
7
|
-
import { md5FilePath } from "../../../lib/assets/md5.mjs";
|
|
8
|
-
import { forbidRelativePaths } from "./forbidRelativePaths.mjs";
|
|
9
|
-
import { sendTaskResult } from "./sendTaskResult.mjs";
|
|
3
|
+
import { cacheImage, findOrCreateCaptions, generateTrack, generateTrackFragmentIndex, md5FilePath } from "@editframe/assets";
|
|
4
|
+
import { forbidRelativePaths } from "./forbidRelativePaths.js";
|
|
5
|
+
import { sendTaskResult } from "./sendTaskResult.js";
|
|
10
6
|
const vitePluginEditframe = (options) => {
|
|
11
7
|
return {
|
|
12
8
|
name: "vite-plugin-editframe",
|
|
@@ -19,7 +15,7 @@ const vitePluginEditframe = (options) => {
|
|
|
19
15
|
return next();
|
|
20
16
|
}
|
|
21
17
|
log(`Handling ${req.url}`);
|
|
22
|
-
const requestPath = req.url.replace(
|
|
18
|
+
const requestPath = req.url.replace(/^\/@ef-[^\/]+\//, "");
|
|
23
19
|
const assetPath = requestPath.replace(/\?.*$/, "");
|
|
24
20
|
const absolutePath = path.join(options.root, assetPath).replace("dist/", "src/");
|
|
25
21
|
options.cacheRoot = options.cacheRoot.replace("dist/", "src/");
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
3
|
+
const node_fs = require("node:fs");
|
|
4
|
+
const mime = require("mime");
|
|
5
|
+
const debug = require("debug");
|
|
6
|
+
const sendTaskResult = (req, res, taskResult) => {
|
|
7
|
+
const { cachePath, md5Sum } = taskResult;
|
|
8
|
+
const filePath = cachePath;
|
|
9
|
+
const headers = {
|
|
10
|
+
etag: md5Sum
|
|
11
|
+
};
|
|
12
|
+
const log = debug("@ef:sendfile");
|
|
13
|
+
try {
|
|
14
|
+
log(`Sending file ${filePath}`);
|
|
15
|
+
const stats = node_fs.statSync(filePath);
|
|
16
|
+
if (req.headers.range) {
|
|
17
|
+
const [x, y] = req.headers.range.replace("bytes=", "").split("-");
|
|
18
|
+
let end = Number.parseInt(y ?? "0", 10) || stats.size - 1;
|
|
19
|
+
const start = Number.parseInt(x ?? "0", 10) || 0;
|
|
20
|
+
if (end >= stats.size) {
|
|
21
|
+
end = stats.size - 1;
|
|
22
|
+
}
|
|
23
|
+
if (start >= stats.size) {
|
|
24
|
+
log("Range start is greater than file size");
|
|
25
|
+
res.setHeader("Content-Range", `bytes */${stats.size}`);
|
|
26
|
+
res.statusCode = 416;
|
|
27
|
+
return res.end();
|
|
28
|
+
}
|
|
29
|
+
res.writeHead(206, {
|
|
30
|
+
...headers,
|
|
31
|
+
"Content-Type": mime.getType(filePath) || "text/plain",
|
|
32
|
+
"Content-Range": `bytes ${start}-${end}/${stats.size}`,
|
|
33
|
+
"Content-Length": end - start + 1,
|
|
34
|
+
"Accept-Ranges": "bytes"
|
|
35
|
+
});
|
|
36
|
+
log(`Sending ${filePath} range ${start}-${end}/${stats.size}`);
|
|
37
|
+
const readStream = node_fs.createReadStream(filePath, { start, end });
|
|
38
|
+
readStream.pipe(res);
|
|
39
|
+
} else {
|
|
40
|
+
res.writeHead(200, {
|
|
41
|
+
...headers,
|
|
42
|
+
"Content-Type": mime.getType(filePath) || "text/plain",
|
|
43
|
+
"Contetn-Length": stats.size
|
|
44
|
+
});
|
|
45
|
+
log(`Sending ${filePath}`);
|
|
46
|
+
const readStream = node_fs.createReadStream(filePath);
|
|
47
|
+
readStream.pipe(res);
|
|
48
|
+
}
|
|
49
|
+
} catch (error) {
|
|
50
|
+
log("Error sending file", error);
|
|
51
|
+
console.error(error);
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
exports.sendTaskResult = sendTaskResult;
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { ServerResponse } from 'node:http';
|
|
2
|
+
import { IncomingMessage } from 'connect';
|
|
3
|
+
import { TaskResult } from '../../assets';
|
|
4
|
+
|
|
5
|
+
export declare const sendTaskResult: (req: IncomingMessage, res: ServerResponse<IncomingMessage>, taskResult: TaskResult) => ServerResponse<IncomingMessage> | undefined;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { statSync, createReadStream } from "fs";
|
|
1
|
+
import { statSync, createReadStream } from "node:fs";
|
|
2
2
|
import mime from "mime";
|
|
3
3
|
import debug from "debug";
|
|
4
4
|
const sendTaskResult = (req, res, taskResult) => {
|
|
@@ -12,9 +12,9 @@ const sendTaskResult = (req, res, taskResult) => {
|
|
|
12
12
|
log(`Sending file ${filePath}`);
|
|
13
13
|
const stats = statSync(filePath);
|
|
14
14
|
if (req.headers.range) {
|
|
15
|
-
|
|
16
|
-
let end = parseInt(y ?? "0", 10) || stats.size - 1;
|
|
17
|
-
|
|
15
|
+
const [x, y] = req.headers.range.replace("bytes=", "").split("-");
|
|
16
|
+
let end = Number.parseInt(y ?? "0", 10) || stats.size - 1;
|
|
17
|
+
const start = Number.parseInt(x ?? "0", 10) || 0;
|
|
18
18
|
if (end >= stats.size) {
|
|
19
19
|
end = stats.size - 1;
|
|
20
20
|
}
|
package/package.json
CHANGED
|
@@ -1,23 +1,34 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@editframe/vite-plugin",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0-beta.10",
|
|
4
4
|
"description": "Editframe vite plugin",
|
|
5
5
|
"exports": {
|
|
6
|
-
".":
|
|
6
|
+
".": {
|
|
7
|
+
"import": "./dist/packages/vite-plugin/index.js",
|
|
8
|
+
"require": "./dist/packages/vite-plugin/index.cjs"
|
|
9
|
+
}
|
|
7
10
|
},
|
|
11
|
+
"types": "./dist/packages/vite-plugin/index.d.ts",
|
|
8
12
|
"scripts": {
|
|
9
|
-
"
|
|
13
|
+
"typecheck": "tsc --noEmit --emitDeclarationOnly false",
|
|
14
|
+
"build": "vite build",
|
|
15
|
+
"build:watch": "vite build --watch"
|
|
10
16
|
},
|
|
17
|
+
"type": "module",
|
|
11
18
|
"author": "",
|
|
12
19
|
"license": "UNLICENSED",
|
|
13
20
|
"dependencies": {
|
|
21
|
+
"@editframe/assets": "0.6.0-beta.10",
|
|
14
22
|
"debug": "^4.3.4",
|
|
15
23
|
"mime": "^4.0.3",
|
|
16
24
|
"node-html-parser": "^6.1.13",
|
|
17
25
|
"vite": "^5.2.11"
|
|
18
26
|
},
|
|
19
27
|
"devDependencies": {
|
|
28
|
+
"@types/dom-webcodecs": "^0.1.11",
|
|
29
|
+
"@types/node": "^20.14.9",
|
|
20
30
|
"connect": "^3.7.0",
|
|
31
|
+
"vite-plugin-dts": "^3.9.1",
|
|
21
32
|
"vite-tsconfig-paths": "^4.3.2"
|
|
22
33
|
}
|
|
23
34
|
}
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
import { spawn } from "child_process";
|
|
2
|
-
import { execPromise } from "../util/execPromise.mjs";
|
|
3
|
-
import * as z from "zod";
|
|
4
|
-
import { createReadStream } from "fs";
|
|
5
|
-
const AudioStreamSchema = z.object({
|
|
6
|
-
index: z.number(),
|
|
7
|
-
codec_name: z.string(),
|
|
8
|
-
codec_long_name: z.string(),
|
|
9
|
-
codec_type: z.literal("audio"),
|
|
10
|
-
codec_tag_string: z.string(),
|
|
11
|
-
codec_tag: z.string(),
|
|
12
|
-
sample_fmt: z.string(),
|
|
13
|
-
sample_rate: z.string(),
|
|
14
|
-
channels: z.number(),
|
|
15
|
-
channel_layout: z.string(),
|
|
16
|
-
bits_per_sample: z.number(),
|
|
17
|
-
initial_padding: z.number().optional(),
|
|
18
|
-
r_frame_rate: z.string(),
|
|
19
|
-
avg_frame_rate: z.string(),
|
|
20
|
-
time_base: z.string(),
|
|
21
|
-
start_pts: z.number(),
|
|
22
|
-
start_time: z.coerce.number(),
|
|
23
|
-
duration_ts: z.number(),
|
|
24
|
-
duration: z.coerce.number(),
|
|
25
|
-
bit_rate: z.string(),
|
|
26
|
-
disposition: z.record(z.unknown())
|
|
27
|
-
});
|
|
28
|
-
const VideoStreamSchema = z.object({
|
|
29
|
-
index: z.number(),
|
|
30
|
-
codec_name: z.string(),
|
|
31
|
-
codec_long_name: z.string(),
|
|
32
|
-
codec_type: z.literal("video"),
|
|
33
|
-
codec_tag_string: z.string(),
|
|
34
|
-
codec_tag: z.string(),
|
|
35
|
-
width: z.number(),
|
|
36
|
-
height: z.number(),
|
|
37
|
-
coded_width: z.number(),
|
|
38
|
-
coded_height: z.number(),
|
|
39
|
-
r_frame_rate: z.string(),
|
|
40
|
-
avg_frame_rate: z.string(),
|
|
41
|
-
time_base: z.string(),
|
|
42
|
-
start_pts: z.number().optional(),
|
|
43
|
-
start_time: z.coerce.number().optional(),
|
|
44
|
-
duration_ts: z.number().optional(),
|
|
45
|
-
duration: z.coerce.number().optional(),
|
|
46
|
-
bit_rate: z.string().optional(),
|
|
47
|
-
disposition: z.record(z.unknown())
|
|
48
|
-
});
|
|
49
|
-
const ProbeFormatSchema = z.object({
|
|
50
|
-
filename: z.string(),
|
|
51
|
-
nb_streams: z.number(),
|
|
52
|
-
nb_programs: z.number(),
|
|
53
|
-
format_name: z.string(),
|
|
54
|
-
format_long_name: z.string(),
|
|
55
|
-
start_time: z.string().optional(),
|
|
56
|
-
duration: z.string().optional(),
|
|
57
|
-
size: z.string(),
|
|
58
|
-
bit_rate: z.string().optional(),
|
|
59
|
-
probe_score: z.number()
|
|
60
|
-
});
|
|
61
|
-
const StreamSchema = z.discriminatedUnion("codec_type", [
|
|
62
|
-
AudioStreamSchema,
|
|
63
|
-
VideoStreamSchema
|
|
64
|
-
]);
|
|
65
|
-
const ProbeSchema = z.object({
|
|
66
|
-
streams: z.array(StreamSchema),
|
|
67
|
-
format: ProbeFormatSchema
|
|
68
|
-
});
|
|
69
|
-
class Probe {
|
|
70
|
-
constructor(absolutePath, rawData) {
|
|
71
|
-
this.absolutePath = absolutePath;
|
|
72
|
-
this.data = ProbeSchema.parse(rawData);
|
|
73
|
-
}
|
|
74
|
-
static async probePath(absolutePath) {
|
|
75
|
-
const probeResult = await execPromise(
|
|
76
|
-
`ffprobe -v error -show_format -show_streams -of json ${absolutePath}`
|
|
77
|
-
);
|
|
78
|
-
const json = JSON.parse(probeResult.stdout);
|
|
79
|
-
return new Probe(absolutePath, json);
|
|
80
|
-
}
|
|
81
|
-
get audioStreams() {
|
|
82
|
-
return this.data.streams.filter(
|
|
83
|
-
(stream) => stream.codec_type === "audio"
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
get videoStreams() {
|
|
87
|
-
return this.data.streams.filter(
|
|
88
|
-
(stream) => stream.codec_type === "video"
|
|
89
|
-
);
|
|
90
|
-
}
|
|
91
|
-
get streams() {
|
|
92
|
-
return this.data.streams;
|
|
93
|
-
}
|
|
94
|
-
get format() {
|
|
95
|
-
return this.data.format;
|
|
96
|
-
}
|
|
97
|
-
get mustReencodeAudio() {
|
|
98
|
-
return this.audioStreams.some((stream) => stream.codec_name !== "aac");
|
|
99
|
-
}
|
|
100
|
-
get mustReencodeVideo() {
|
|
101
|
-
return false;
|
|
102
|
-
}
|
|
103
|
-
get mustRemux() {
|
|
104
|
-
return this.format.format_name !== "mp4";
|
|
105
|
-
}
|
|
106
|
-
get hasAudio() {
|
|
107
|
-
return this.audioStreams.length > 0;
|
|
108
|
-
}
|
|
109
|
-
get hasVideo() {
|
|
110
|
-
return this.videoStreams.length > 0;
|
|
111
|
-
}
|
|
112
|
-
get isAudioOnly() {
|
|
113
|
-
return this.audioStreams.length > 0 && this.videoStreams.length === 0;
|
|
114
|
-
}
|
|
115
|
-
get isVideoOnly() {
|
|
116
|
-
return this.audioStreams.length === 0 && this.videoStreams.length > 0;
|
|
117
|
-
}
|
|
118
|
-
get mustProcess() {
|
|
119
|
-
return this.mustReencodeAudio || this.mustReencodeVideo || this.mustRemux;
|
|
120
|
-
}
|
|
121
|
-
get ffmpegAudioOptions() {
|
|
122
|
-
if (!this.hasAudio) {
|
|
123
|
-
return [];
|
|
124
|
-
}
|
|
125
|
-
if (this.mustReencodeAudio) {
|
|
126
|
-
return [
|
|
127
|
-
"-c:a",
|
|
128
|
-
"aac",
|
|
129
|
-
"-b:a",
|
|
130
|
-
"192k",
|
|
131
|
-
"-ar",
|
|
132
|
-
"48000"
|
|
133
|
-
];
|
|
134
|
-
}
|
|
135
|
-
return ["-c:a", "copy"];
|
|
136
|
-
}
|
|
137
|
-
get ffmpegVideoOptions() {
|
|
138
|
-
if (!this.hasVideo) {
|
|
139
|
-
return [];
|
|
140
|
-
}
|
|
141
|
-
if (this.mustReencodeVideo) {
|
|
142
|
-
return [
|
|
143
|
-
"-c:v",
|
|
144
|
-
"h264",
|
|
145
|
-
"-pix_fmt",
|
|
146
|
-
"yuv420p"
|
|
147
|
-
];
|
|
148
|
-
}
|
|
149
|
-
return ["-c:v", "copy"];
|
|
150
|
-
}
|
|
151
|
-
createConformingReadstream() {
|
|
152
|
-
if (!this.mustProcess) {
|
|
153
|
-
return createReadStream(this.absolutePath);
|
|
154
|
-
}
|
|
155
|
-
return spawn("ffmpeg", [
|
|
156
|
-
"-i",
|
|
157
|
-
this.absolutePath,
|
|
158
|
-
...this.ffmpegAudioOptions,
|
|
159
|
-
...this.ffmpegVideoOptions,
|
|
160
|
-
"-f",
|
|
161
|
-
"mp4",
|
|
162
|
-
"-frag_duration",
|
|
163
|
-
"8000000",
|
|
164
|
-
"-movflags",
|
|
165
|
-
"frag_keyframe+empty_moov",
|
|
166
|
-
"pipe:1"
|
|
167
|
-
], {
|
|
168
|
-
stdio: ["ignore", "pipe", "inherit"]
|
|
169
|
-
}).stdout;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
export {
|
|
173
|
-
AudioStreamSchema,
|
|
174
|
-
Probe,
|
|
175
|
-
VideoStreamSchema
|
|
176
|
-
};
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import { existsSync, createWriteStream } from "fs";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import { md5FilePath } from "./md5.mjs";
|
|
4
|
-
import debug from "debug";
|
|
5
|
-
import { mkdir, writeFile } from "fs/promises";
|
|
6
|
-
import { Readable } from "stream";
|
|
7
|
-
const idempotentTask = ({
|
|
8
|
-
label,
|
|
9
|
-
filename,
|
|
10
|
-
runner
|
|
11
|
-
}) => {
|
|
12
|
-
const tasks = {};
|
|
13
|
-
return async (rootDir, absolutePath, ...args) => {
|
|
14
|
-
const log = debug(`@ef:${label}`);
|
|
15
|
-
const md5 = await md5FilePath(absolutePath);
|
|
16
|
-
const cacheDir = path.join(rootDir, ".cache", md5);
|
|
17
|
-
log(`Cache dir: ${cacheDir}`);
|
|
18
|
-
await mkdir(cacheDir, { recursive: true });
|
|
19
|
-
const cachePath = path.join(cacheDir, filename(absolutePath, ...args));
|
|
20
|
-
const key = cachePath;
|
|
21
|
-
if (existsSync(cachePath)) {
|
|
22
|
-
log(`Returning cached @ef:${label} task for ${key}`);
|
|
23
|
-
return { cachePath, md5Sum: md5 };
|
|
24
|
-
}
|
|
25
|
-
const maybeTask = tasks[key];
|
|
26
|
-
if (maybeTask) {
|
|
27
|
-
log(`Returning existing @ef:${label} task for ${key}`);
|
|
28
|
-
await maybeTask;
|
|
29
|
-
return { cachePath, md5Sum: md5 };
|
|
30
|
-
}
|
|
31
|
-
log(`Creating new @ef:${label} task for ${key}`);
|
|
32
|
-
const task = runner(absolutePath, ...args);
|
|
33
|
-
tasks[key] = task;
|
|
34
|
-
log(`Awaiting task for ${key}`);
|
|
35
|
-
const result = await task;
|
|
36
|
-
if (result instanceof Readable) {
|
|
37
|
-
log(`Piping task for ${key} to cache`);
|
|
38
|
-
const writeStream = createWriteStream(cachePath);
|
|
39
|
-
result.pipe(writeStream);
|
|
40
|
-
await new Promise((resolve, reject) => {
|
|
41
|
-
result.on("error", reject);
|
|
42
|
-
writeStream.on("error", reject);
|
|
43
|
-
writeStream.on("finish", resolve);
|
|
44
|
-
});
|
|
45
|
-
return { cachePath, md5Sum: md5 };
|
|
46
|
-
}
|
|
47
|
-
log(`Writing to ${cachePath}`);
|
|
48
|
-
await writeFile(cachePath, result);
|
|
49
|
-
return {
|
|
50
|
-
md5Sum: md5,
|
|
51
|
-
cachePath
|
|
52
|
-
};
|
|
53
|
-
};
|
|
54
|
-
};
|
|
55
|
-
export {
|
|
56
|
-
idempotentTask
|
|
57
|
-
};
|
package/dist/lib/assets/md5.mjs
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { createReadStream } from "fs";
|
|
2
|
-
import crypto from "crypto";
|
|
3
|
-
async function md5FilePath(filePath) {
|
|
4
|
-
const readStream = createReadStream(filePath);
|
|
5
|
-
return md5ReadStream(readStream);
|
|
6
|
-
}
|
|
7
|
-
function md5ReadStream(readStream) {
|
|
8
|
-
return new Promise((resolve, reject) => {
|
|
9
|
-
const hash = crypto.createHash("md5");
|
|
10
|
-
readStream.on("data", (data) => {
|
|
11
|
-
hash.update(data);
|
|
12
|
-
});
|
|
13
|
-
readStream.on("error", reject);
|
|
14
|
-
readStream.on("end", () => {
|
|
15
|
-
resolve(addDashesToUUID(hash.digest("hex")));
|
|
16
|
-
});
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
function addDashesToUUID(uuidWithoutDashes) {
|
|
20
|
-
if (uuidWithoutDashes.length !== 32) {
|
|
21
|
-
throw new Error("Invalid UUID without dashes. Expected 32 characters.");
|
|
22
|
-
}
|
|
23
|
-
return uuidWithoutDashes.slice(0, 8) + "-" + uuidWithoutDashes.slice(8, 12) + "-" + uuidWithoutDashes.slice(12, 16) + "-" + uuidWithoutDashes.slice(16, 20) + "-" + uuidWithoutDashes.slice(20, 32);
|
|
24
|
-
}
|
|
25
|
-
export {
|
|
26
|
-
md5FilePath,
|
|
27
|
-
md5ReadStream
|
|
28
|
-
};
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { Writable } from "stream";
|
|
2
|
-
const mp4FileWritable = (mp4File) => {
|
|
3
|
-
let arrayBufferStart = 0;
|
|
4
|
-
return new Writable({
|
|
5
|
-
write: (chunk, _encoding, callback) => {
|
|
6
|
-
const mp4BoxBuffer = chunk.buffer;
|
|
7
|
-
mp4BoxBuffer.fileStart = arrayBufferStart;
|
|
8
|
-
arrayBufferStart += chunk.length;
|
|
9
|
-
mp4File.appendBuffer(mp4BoxBuffer, false);
|
|
10
|
-
callback();
|
|
11
|
-
},
|
|
12
|
-
final: (callback) => {
|
|
13
|
-
mp4File.flush();
|
|
14
|
-
mp4File.processSamples(true);
|
|
15
|
-
callback();
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
};
|
|
19
|
-
export {
|
|
20
|
-
mp4FileWritable
|
|
21
|
-
};
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { idempotentTask } from "../idempotentTask.mjs";
|
|
2
|
-
import { createReadStream } from "fs";
|
|
3
|
-
import path from "path";
|
|
4
|
-
const cacheImageTask = idempotentTask({
|
|
5
|
-
label: "image",
|
|
6
|
-
filename: (absolutePath) => path.basename(absolutePath),
|
|
7
|
-
runner: async (absolutePath) => {
|
|
8
|
-
return createReadStream(absolutePath);
|
|
9
|
-
}
|
|
10
|
-
});
|
|
11
|
-
const cacheImage = async (cacheRoot, absolutePath) => {
|
|
12
|
-
try {
|
|
13
|
-
return await cacheImageTask(cacheRoot, absolutePath);
|
|
14
|
-
} catch (error) {
|
|
15
|
-
console.error(error);
|
|
16
|
-
console.trace("Error generating track fragment index", error);
|
|
17
|
-
throw error;
|
|
18
|
-
}
|
|
19
|
-
};
|
|
20
|
-
export {
|
|
21
|
-
cacheImage
|
|
22
|
-
};
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { execPromise } from "../../util/execPromise.mjs";
|
|
2
|
-
import { idempotentTask } from "../idempotentTask.mjs";
|
|
3
|
-
import debug from "debug";
|
|
4
|
-
import { basename } from "path";
|
|
5
|
-
const log = debug("@ef:generateCaptions");
|
|
6
|
-
const generateCaptionData = idempotentTask({
|
|
7
|
-
label: "captions",
|
|
8
|
-
filename: (absolutePath) => `${basename(absolutePath)}.captions.json`,
|
|
9
|
-
runner: async (absolutePath) => {
|
|
10
|
-
const command = `whisper_timestamped --language en --efficient --output_format vtt ${absolutePath}`;
|
|
11
|
-
log(`Running command: ${command}`);
|
|
12
|
-
const { stdout } = await execPromise(command);
|
|
13
|
-
return stdout;
|
|
14
|
-
}
|
|
15
|
-
});
|
|
16
|
-
const findOrCreateCaptions = async (cacheRoot, absolutePath) => {
|
|
17
|
-
try {
|
|
18
|
-
return await generateCaptionData(cacheRoot, absolutePath);
|
|
19
|
-
} catch (error) {
|
|
20
|
-
console.trace("Error finding or creating captions", error);
|
|
21
|
-
throw error;
|
|
22
|
-
}
|
|
23
|
-
};
|
|
24
|
-
export {
|
|
25
|
-
findOrCreateCaptions
|
|
26
|
-
};
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { idempotentTask } from "../idempotentTask.mjs";
|
|
2
|
-
import { MP4File } from "../../av/MP4File.mjs";
|
|
3
|
-
import debug from "debug";
|
|
4
|
-
import { mp4FileWritable } from "../mp4FileWritable.mjs";
|
|
5
|
-
import { PassThrough } from "stream";
|
|
6
|
-
import { basename } from "path";
|
|
7
|
-
import { Probe } from "../Probe.mjs";
|
|
8
|
-
const generateTrackTask = idempotentTask({
|
|
9
|
-
label: "track",
|
|
10
|
-
filename: (absolutePath, trackId) => `${basename(absolutePath)}.track-${trackId}.mp4`,
|
|
11
|
-
runner: async (absolutePath, trackId) => {
|
|
12
|
-
const log = debug("@ef:generateTrackFragment");
|
|
13
|
-
const probe = await Probe.probePath(absolutePath);
|
|
14
|
-
const readStream = probe.createConformingReadstream();
|
|
15
|
-
const mp4File = new MP4File();
|
|
16
|
-
log(`Generating track fragment index for ${absolutePath}`);
|
|
17
|
-
readStream.pipe(mp4FileWritable(mp4File));
|
|
18
|
-
await new Promise((resolve, reject) => {
|
|
19
|
-
readStream.on("end", resolve);
|
|
20
|
-
readStream.on("error", reject);
|
|
21
|
-
});
|
|
22
|
-
const trackStream = new PassThrough();
|
|
23
|
-
for await (const fragment of mp4File.fragmentIterator()) {
|
|
24
|
-
if (fragment.track !== trackId) {
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
trackStream.write(Buffer.from(fragment.data), "binary");
|
|
28
|
-
}
|
|
29
|
-
trackStream.end();
|
|
30
|
-
return trackStream;
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
const generateTrack = async (cacheRoot, absolutePath, url) => {
|
|
34
|
-
try {
|
|
35
|
-
const trackId = new URL(
|
|
36
|
-
`http://localhost${url}` ?? "bad-url"
|
|
37
|
-
).searchParams.get("trackId");
|
|
38
|
-
if (trackId === null) {
|
|
39
|
-
throw new Error(
|
|
40
|
-
"No trackId provided. IT must be specified in the query string: ?trackId=0"
|
|
41
|
-
);
|
|
42
|
-
}
|
|
43
|
-
return await generateTrackTask(cacheRoot, absolutePath, Number(trackId));
|
|
44
|
-
} catch (error) {
|
|
45
|
-
console.error(error);
|
|
46
|
-
console.trace("Error generating track fragment index", error);
|
|
47
|
-
throw error;
|
|
48
|
-
}
|
|
49
|
-
};
|
|
50
|
-
export {
|
|
51
|
-
generateTrack
|
|
52
|
-
};
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
import { idempotentTask } from "../idempotentTask.mjs";
|
|
2
|
-
import { MP4File } from "../../av/MP4File.mjs";
|
|
3
|
-
import debug from "debug";
|
|
4
|
-
import { mp4FileWritable } from "../mp4FileWritable.mjs";
|
|
5
|
-
import { basename } from "path";
|
|
6
|
-
import { Probe } from "../Probe.mjs";
|
|
7
|
-
const generateTrackFragmentIndexFromPath = async (absolutePath) => {
|
|
8
|
-
const log = debug("@ef:generateTrackFragment");
|
|
9
|
-
const probe = await Probe.probePath(absolutePath);
|
|
10
|
-
const readStream = probe.createConformingReadstream();
|
|
11
|
-
const mp4File = new MP4File();
|
|
12
|
-
log(`Generating track fragment index for ${absolutePath}`);
|
|
13
|
-
readStream.pipe(mp4FileWritable(mp4File));
|
|
14
|
-
await new Promise((resolve, reject) => {
|
|
15
|
-
readStream.on("end", resolve);
|
|
16
|
-
readStream.on("error", reject);
|
|
17
|
-
});
|
|
18
|
-
const trackFragmentIndexes = {};
|
|
19
|
-
const trackByteOffsets = {};
|
|
20
|
-
for await (const fragment of mp4File.fragmentIterator()) {
|
|
21
|
-
const track = mp4File.getInfo().tracks.find((track2) => track2.id === fragment.track);
|
|
22
|
-
if (!track) {
|
|
23
|
-
throw new Error("Track not found");
|
|
24
|
-
}
|
|
25
|
-
if (fragment.segment === "init") {
|
|
26
|
-
trackByteOffsets[fragment.track] = fragment.data.byteLength;
|
|
27
|
-
trackFragmentIndexes[fragment.track] = {
|
|
28
|
-
track: fragment.track,
|
|
29
|
-
type: track?.type ?? "video",
|
|
30
|
-
timescale: track.timescale,
|
|
31
|
-
duration: 0,
|
|
32
|
-
initSegment: {
|
|
33
|
-
offset: 0,
|
|
34
|
-
size: fragment.data.byteLength
|
|
35
|
-
},
|
|
36
|
-
segments: []
|
|
37
|
-
};
|
|
38
|
-
} else {
|
|
39
|
-
const fragmentIndex = trackFragmentIndexes[fragment.track];
|
|
40
|
-
if (!fragmentIndex) {
|
|
41
|
-
throw new Error("Fragment index not found");
|
|
42
|
-
}
|
|
43
|
-
fragmentIndex.duration += fragment.duration;
|
|
44
|
-
fragmentIndex.segments.push({
|
|
45
|
-
cts: fragment.cts,
|
|
46
|
-
dts: fragment.dts,
|
|
47
|
-
duration: fragment.duration,
|
|
48
|
-
offset: trackByteOffsets[fragment.track],
|
|
49
|
-
size: fragment.data.byteLength
|
|
50
|
-
});
|
|
51
|
-
trackByteOffsets[fragment.track] += fragment.data.byteLength;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
return JSON.stringify(trackFragmentIndexes, null, 2);
|
|
55
|
-
};
|
|
56
|
-
const generateTrackFragmentIndexTask = idempotentTask({
|
|
57
|
-
label: "trackFragmentIndex",
|
|
58
|
-
filename: (absolutePath) => `${basename(absolutePath)}.tracks.json`,
|
|
59
|
-
runner: generateTrackFragmentIndexFromPath
|
|
60
|
-
});
|
|
61
|
-
const generateTrackFragmentIndex = async (cacheRoot, absolutePath) => {
|
|
62
|
-
try {
|
|
63
|
-
return await generateTrackFragmentIndexTask(cacheRoot, absolutePath);
|
|
64
|
-
} catch (error) {
|
|
65
|
-
console.trace("Error generating track fragment index", error);
|
|
66
|
-
throw error;
|
|
67
|
-
}
|
|
68
|
-
};
|
|
69
|
-
export {
|
|
70
|
-
generateTrackFragmentIndex
|
|
71
|
-
};
|
package/dist/lib/av/MP4File.mjs
DELETED
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
import * as MP4Box from "mp4box";
|
|
2
|
-
class MP4File extends MP4Box.ISOFile {
|
|
3
|
-
constructor() {
|
|
4
|
-
super(...arguments);
|
|
5
|
-
this.readyPromise = new Promise((resolve, reject) => {
|
|
6
|
-
this.onReady = () => resolve();
|
|
7
|
-
this.onError = reject;
|
|
8
|
-
});
|
|
9
|
-
this.waitingForSamples = [];
|
|
10
|
-
this._hasSeenLastSamples = false;
|
|
11
|
-
this._arrayBufferFileStart = 0;
|
|
12
|
-
}
|
|
13
|
-
setSegmentOptions(id, user, options) {
|
|
14
|
-
const trak = this.getTrackById(id);
|
|
15
|
-
if (trak) {
|
|
16
|
-
trak.nextSample = 0;
|
|
17
|
-
this.fragmentedTracks.push({
|
|
18
|
-
id,
|
|
19
|
-
user,
|
|
20
|
-
trak,
|
|
21
|
-
segmentStream: null,
|
|
22
|
-
nb_samples: "nbSamples" in options && options.nbSamples || 1e3,
|
|
23
|
-
rapAlignement: ("rapAlignement" in options && options.rapAlignement) ?? true
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Fragments all tracks in a file into separate array buffers.
|
|
29
|
-
*/
|
|
30
|
-
async fragmentAllTracks() {
|
|
31
|
-
let trackBuffers = {};
|
|
32
|
-
for await (const segment of this.fragmentIterator()) {
|
|
33
|
-
(trackBuffers[segment.track] ??= []).push(segment.data);
|
|
34
|
-
}
|
|
35
|
-
return trackBuffers;
|
|
36
|
-
}
|
|
37
|
-
async *fragmentIterator() {
|
|
38
|
-
await this.readyPromise;
|
|
39
|
-
const trackInfo = {};
|
|
40
|
-
for (const videoTrack of this.getInfo().videoTracks) {
|
|
41
|
-
trackInfo[videoTrack.id] = { index: 0, complete: false };
|
|
42
|
-
this.setSegmentOptions(videoTrack.id, null, {
|
|
43
|
-
rapAlignement: true
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
for (const audioTrack of this.getInfo().audioTracks) {
|
|
47
|
-
trackInfo[audioTrack.id] = { index: 0, complete: false };
|
|
48
|
-
const sampleRate = audioTrack.audio.sample_rate;
|
|
49
|
-
const probablePacketSize = 1024;
|
|
50
|
-
const probableFourSecondsOfSamples = Math.ceil(
|
|
51
|
-
sampleRate / probablePacketSize * 4
|
|
52
|
-
);
|
|
53
|
-
this.setSegmentOptions(audioTrack.id, null, {
|
|
54
|
-
nbSamples: probableFourSecondsOfSamples
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
const initSegments = this.initializeSegmentation();
|
|
58
|
-
for (const initSegment of initSegments) {
|
|
59
|
-
yield {
|
|
60
|
-
track: initSegment.id,
|
|
61
|
-
segment: "init",
|
|
62
|
-
data: initSegment.buffer,
|
|
63
|
-
complete: false
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
const fragmentStartSamples = {};
|
|
67
|
-
let finishedReading = false;
|
|
68
|
-
const allTracksFinished = () => {
|
|
69
|
-
for (const fragmentedTrack of this.fragmentedTracks) {
|
|
70
|
-
if (!trackInfo[fragmentedTrack.id]?.complete) {
|
|
71
|
-
return false;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
return true;
|
|
75
|
-
};
|
|
76
|
-
while (!(finishedReading && allTracksFinished())) {
|
|
77
|
-
for (const fragTrak of this.fragmentedTracks) {
|
|
78
|
-
const trak = fragTrak.trak;
|
|
79
|
-
if (trak.nextSample === void 0) {
|
|
80
|
-
throw new Error("trak.nextSample is undefined");
|
|
81
|
-
}
|
|
82
|
-
if (trak.samples === void 0) {
|
|
83
|
-
throw new Error("trak.samples is undefined");
|
|
84
|
-
}
|
|
85
|
-
eachSample: while (trak.nextSample < trak.samples.length) {
|
|
86
|
-
let result = void 0;
|
|
87
|
-
if (trak?.samples[trak.nextSample]) {
|
|
88
|
-
fragmentStartSamples[fragTrak.id] ||= trak.samples[trak.nextSample];
|
|
89
|
-
}
|
|
90
|
-
try {
|
|
91
|
-
result = this.createFragment(
|
|
92
|
-
fragTrak.id,
|
|
93
|
-
trak.nextSample,
|
|
94
|
-
fragTrak.segmentStream
|
|
95
|
-
);
|
|
96
|
-
} catch (error) {
|
|
97
|
-
console.log("Failed to createFragment", error);
|
|
98
|
-
}
|
|
99
|
-
if (result) {
|
|
100
|
-
fragTrak.segmentStream = result;
|
|
101
|
-
trak.nextSample++;
|
|
102
|
-
} else {
|
|
103
|
-
finishedReading = await this.waitForMoreSamples();
|
|
104
|
-
break eachSample;
|
|
105
|
-
}
|
|
106
|
-
const nextSample = trak.samples[trak.nextSample];
|
|
107
|
-
const emitSegment = (
|
|
108
|
-
// if rapAlignement is true, we emit a fragment when we have a rap sample coming up next
|
|
109
|
-
fragTrak.rapAlignement === true && nextSample?.is_sync || // if rapAlignement is false, we emit a fragment when we have the required number of samples
|
|
110
|
-
!fragTrak.rapAlignement && trak.nextSample % fragTrak.nb_samples === 0 || // // if this is the last sample, we emit the fragment
|
|
111
|
-
// finished ||
|
|
112
|
-
// if we have more samples than the number of samples requested, we emit the fragment
|
|
113
|
-
trak.nextSample >= trak.samples.length
|
|
114
|
-
);
|
|
115
|
-
if (emitSegment) {
|
|
116
|
-
if (trak.nextSample >= trak.samples.length) {
|
|
117
|
-
trackInfo[fragTrak.id].complete = true;
|
|
118
|
-
}
|
|
119
|
-
const startSample = fragmentStartSamples[fragTrak.id];
|
|
120
|
-
const endSample = trak.samples[trak.nextSample - 1];
|
|
121
|
-
if (!startSample || !endSample) {
|
|
122
|
-
throw new Error("startSample or endSample is undefined");
|
|
123
|
-
}
|
|
124
|
-
yield {
|
|
125
|
-
track: fragTrak.id,
|
|
126
|
-
segment: trackInfo[fragTrak.id].index,
|
|
127
|
-
data: fragTrak.segmentStream.buffer,
|
|
128
|
-
complete: trackInfo[fragTrak.id].complete,
|
|
129
|
-
cts: startSample.cts,
|
|
130
|
-
dts: startSample.dts,
|
|
131
|
-
duration: endSample.dts - startSample.dts + endSample.duration
|
|
132
|
-
};
|
|
133
|
-
trackInfo[fragTrak.id].index += 1;
|
|
134
|
-
fragTrak.segmentStream = null;
|
|
135
|
-
delete fragmentStartSamples[fragTrak.id];
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
finishedReading = await this.waitForMoreSamples();
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
waitForMoreSamples() {
|
|
143
|
-
if (this._hasSeenLastSamples) {
|
|
144
|
-
return Promise.resolve(true);
|
|
145
|
-
}
|
|
146
|
-
return new Promise((resolve) => {
|
|
147
|
-
this.waitingForSamples.push(resolve);
|
|
148
|
-
});
|
|
149
|
-
}
|
|
150
|
-
processSamples(last) {
|
|
151
|
-
this._hasSeenLastSamples = last;
|
|
152
|
-
for (const observer of this.waitingForSamples) {
|
|
153
|
-
observer(last);
|
|
154
|
-
}
|
|
155
|
-
this.waitingForSamples = [];
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
export {
|
|
159
|
-
MP4File
|
|
160
|
-
};
|
|
File without changes
|