@storyteller-platform/align 0.1.56 → 0.1.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/align/align.cjs +38 -19
- package/dist/align/align.d.cts +12 -1
- package/dist/align/align.d.ts +12 -1
- package/dist/align/align.js +38 -19
- package/dist/align/ctc/Aligner.cjs +800 -0
- package/dist/align/ctc/Aligner.d.cts +56 -0
- package/dist/align/ctc/Aligner.d.ts +56 -0
- package/dist/align/ctc/Aligner.js +789 -0
- package/dist/align/ctc/align.cjs +189 -0
- package/dist/align/ctc/align.d.cts +14 -0
- package/dist/align/ctc/align.d.ts +14 -0
- package/dist/align/ctc/align.js +132 -0
- package/dist/{process/mime.cjs → align/ctc/emissions.cjs} +7 -20
- package/dist/align/ctc/emissions.d.cts +6 -0
- package/dist/align/ctc/emissions.d.ts +6 -0
- package/dist/align/ctc/emissions.js +7 -0
- package/dist/align/ctc/forcedAlign.cjs +273 -0
- package/dist/align/ctc/forcedAlign.d.cts +21 -0
- package/dist/align/ctc/forcedAlign.d.ts +21 -0
- package/dist/align/ctc/forcedAlign.js +248 -0
- package/dist/align/ctc/greedyDecode.cjs +61 -0
- package/dist/align/ctc/greedyDecode.d.cts +15 -0
- package/dist/align/ctc/greedyDecode.d.ts +15 -0
- package/dist/align/ctc/greedyDecode.js +37 -0
- package/dist/align/ctc/mediaOverlay.cjs +248 -0
- package/dist/align/ctc/mediaOverlay.d.cts +16 -0
- package/dist/align/ctc/mediaOverlay.d.ts +16 -0
- package/dist/align/ctc/mediaOverlay.js +224 -0
- package/dist/align/ctc/report.cjs +66 -0
- package/dist/align/ctc/report.d.cts +79 -0
- package/dist/align/ctc/report.d.ts +79 -0
- package/dist/align/ctc/report.js +42 -0
- package/dist/align/ctc/search.cjs +437 -0
- package/dist/align/ctc/search.d.cts +46 -0
- package/dist/align/ctc/search.d.ts +46 -0
- package/dist/align/ctc/search.js +412 -0
- package/dist/align/getSentenceRanges.cjs +9 -3
- package/dist/align/getSentenceRanges.d.cts +1 -0
- package/dist/align/getSentenceRanges.d.ts +1 -0
- package/dist/align/getSentenceRanges.js +9 -3
- package/dist/align/parse.cjs +17 -4
- package/dist/align/parse.d.cts +8 -4
- package/dist/align/parse.d.ts +8 -4
- package/dist/align/parse.js +19 -4
- package/dist/align/search.cjs +76 -23
- package/dist/align/search.d.cts +8 -4
- package/dist/align/search.d.ts +8 -4
- package/dist/align/search.js +76 -24
- package/dist/align/slugify.cjs +3 -2
- package/dist/align/slugify.d.cts +1 -1
- package/dist/align/slugify.d.ts +1 -1
- package/dist/align/slugify.js +3 -2
- package/dist/cli/bin.cjs +204 -114
- package/dist/cli/bin.js +194 -103
- package/dist/common/ffmpeg.cjs +10 -5
- package/dist/common/ffmpeg.js +10 -5
- package/dist/common/logging.cjs +33 -1
- package/dist/common/logging.js +33 -1
- package/dist/emit/emit.cjs +105 -0
- package/dist/emit/emit.d.cts +15 -0
- package/dist/emit/emit.d.ts +15 -0
- package/dist/emit/emit.js +86 -0
- package/dist/emit/fs.cjs +227 -0
- package/dist/emit/fs.d.cts +22 -0
- package/dist/emit/fs.d.ts +22 -0
- package/dist/emit/fs.js +162 -0
- package/dist/emit/parse.cjs +68 -0
- package/dist/emit/parse.d.cts +26 -0
- package/dist/emit/parse.d.ts +26 -0
- package/dist/emit/parse.js +58 -0
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/markup/model.cjs +0 -16
- package/dist/markup/model.d.cts +1 -1
- package/dist/markup/model.d.ts +1 -1
- package/dist/markup/model.js +0 -12
- package/dist/markup/parseDom.cjs +13 -14
- package/dist/markup/parseDom.d.cts +2 -2
- package/dist/markup/parseDom.d.ts +2 -2
- package/dist/markup/parseDom.js +13 -18
- package/dist/markup/resolvedPos.d.cts +1 -1
- package/dist/markup/resolvedPos.d.ts +1 -1
- package/dist/markup/segmentation.cjs +3 -1
- package/dist/markup/segmentation.js +3 -1
- package/dist/markup/serializeDom.cjs +1 -1
- package/dist/markup/serializeDom.d.cts +1 -1
- package/dist/markup/serializeDom.d.ts +1 -1
- package/dist/markup/serializeDom.js +1 -1
- package/dist/markup/transform.cjs +5 -2
- package/dist/markup/transform.d.cts +1 -1
- package/dist/markup/transform.d.ts +1 -1
- package/dist/markup/transform.js +5 -4
- package/dist/{model-CZ2mMHJO.d.cts → model-CeOMDDgt.d.cts} +3 -11
- package/dist/{model-CZ2mMHJO.d.ts → model-CeOMDDgt.d.ts} +3 -11
- package/dist/process/processAudiobook.cjs +12 -23
- package/dist/process/processAudiobook.js +14 -30
- package/dist/process/ranges.cjs +2 -2
- package/dist/process/ranges.js +2 -2
- package/dist/readium/guidedNavigation.cjs +8 -8
- package/dist/readium/guidedNavigation.js +8 -8
- package/dist/readium/manifest.cjs +5 -4
- package/dist/readium/manifest.js +5 -4
- package/dist/snapshot/ctc/snapshot.cjs +261 -0
- package/dist/snapshot/ctc/snapshot.d.cts +6 -0
- package/dist/snapshot/ctc/snapshot.d.ts +6 -0
- package/dist/snapshot/ctc/snapshot.js +200 -0
- package/dist/snapshot/parse.cjs +29 -16
- package/dist/snapshot/parse.d.cts +14 -10
- package/dist/snapshot/parse.d.ts +14 -10
- package/dist/snapshot/parse.js +32 -17
- package/dist/transcribe/parse.d.cts +6 -6
- package/dist/transcribe/parse.d.ts +6 -6
- package/dist/transcribe/transcribe.cjs +4 -2
- package/dist/transcribe/transcribe.js +4 -2
- package/package.json +6 -4
- package/dist/process/mime.d.cts +0 -3
- package/dist/process/mime.d.ts +0 -3
- package/dist/process/mime.js +0 -24
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var emit_exports = {};
|
|
20
|
+
__export(emit_exports, {
|
|
21
|
+
emit: () => emit
|
|
22
|
+
});
|
|
23
|
+
module.exports = __toCommonJS(emit_exports);
|
|
24
|
+
var import_promises = require("node:fs/promises");
|
|
25
|
+
var import_node_path = require("node:path");
|
|
26
|
+
var import_itertools = require("itertools");
|
|
27
|
+
var import_ghost_story = require("@storyteller-platform/ghost-story");
|
|
28
|
+
var import_media_types = require("@storyteller-platform/media-types");
|
|
29
|
+
var import_fs = require("./fs.cjs");
|
|
30
|
+
async function emit(input, output, options) {
|
|
31
|
+
const controller = new AbortController();
|
|
32
|
+
const signal = AbortSignal.any([
|
|
33
|
+
...options.signal ? [options.signal] : [],
|
|
34
|
+
controller.signal
|
|
35
|
+
]);
|
|
36
|
+
await (0, import_promises.mkdir)(output, { recursive: true });
|
|
37
|
+
const allFiles = await (0, import_promises.readdir)(input, { recursive: true });
|
|
38
|
+
const filenames = allFiles.toSorted().filter((f) => import_media_types.MediaType.fromPath(f)?.kind === "audio");
|
|
39
|
+
if (!filenames.length) {
|
|
40
|
+
throw new Error(
|
|
41
|
+
`Failed to transcribe audio: found no audio files in ${input}`
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
const engine = "wav2vec2";
|
|
45
|
+
const model = options.model ?? "onnx-community/mms-300m-1130-forced-aligner-ONNX";
|
|
46
|
+
const emissions = [];
|
|
47
|
+
function aborted() {
|
|
48
|
+
return signal.aborted;
|
|
49
|
+
}
|
|
50
|
+
const timing = (0, import_ghost_story.createAggregator)();
|
|
51
|
+
timing.setMetadata("engine", engine);
|
|
52
|
+
timing.setMetadata("model", model);
|
|
53
|
+
for (const [i, filename] of (0, import_itertools.enumerate)(filenames)) {
|
|
54
|
+
let onFileProgress2 = function(progress) {
|
|
55
|
+
const updatedProgress = (i + progress) / filenames.length;
|
|
56
|
+
options.logger?.info(`Progress: ${Math.floor(updatedProgress * 100)}%`);
|
|
57
|
+
options.onProgress?.(updatedProgress);
|
|
58
|
+
};
|
|
59
|
+
var onFileProgress = onFileProgress2;
|
|
60
|
+
if (aborted()) throw new Error("Aborted");
|
|
61
|
+
const filepath = (0, import_node_path.join)(input, filename);
|
|
62
|
+
const emissionFilepath = (0, import_node_path.join)(
|
|
63
|
+
output,
|
|
64
|
+
`${(0, import_node_path.basename)(filename, (0, import_node_path.extname)(filename))}.json`
|
|
65
|
+
);
|
|
66
|
+
try {
|
|
67
|
+
await (0, import_promises.readFile)(emissionFilepath, {
|
|
68
|
+
encoding: "utf-8",
|
|
69
|
+
signal
|
|
70
|
+
});
|
|
71
|
+
options.logger?.info(`Found existing transcription for ${filepath}`);
|
|
72
|
+
emissions.push(emissionFilepath);
|
|
73
|
+
options.onProgress?.((emissions.length + 1) / filenames.length);
|
|
74
|
+
continue;
|
|
75
|
+
} catch {
|
|
76
|
+
}
|
|
77
|
+
if (aborted()) throw new Error("Aborted");
|
|
78
|
+
const result = await (0, import_ghost_story.emit)(filepath, {
|
|
79
|
+
...options,
|
|
80
|
+
signal,
|
|
81
|
+
engine,
|
|
82
|
+
options: {
|
|
83
|
+
model,
|
|
84
|
+
...options.device && { device: options.device },
|
|
85
|
+
...options.dtype && { dtype: options.dtype },
|
|
86
|
+
onProgress: onFileProgress2
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
options.logger?.info(
|
|
90
|
+
(0, import_ghost_story.formatSingleReport)(
|
|
91
|
+
result.timing,
|
|
92
|
+
`Transcription Timing Report for ${filepath}`
|
|
93
|
+
)
|
|
94
|
+
);
|
|
95
|
+
timing.add(result.timing);
|
|
96
|
+
await (0, import_fs.writeEmissionsFile)(emissionFilepath, result, { signal });
|
|
97
|
+
emissions.push(emissionFilepath);
|
|
98
|
+
options.onProgress?.((i + 1) / filenames.length);
|
|
99
|
+
}
|
|
100
|
+
return timing;
|
|
101
|
+
}
|
|
102
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
103
|
+
0 && (module.exports = {
|
|
104
|
+
emit
|
|
105
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Logger } from 'pino';
|
|
2
|
+
import { TimingAggregator } from '@storyteller-platform/ghost-story';
|
|
3
|
+
import { Wav2Vec2Model, Wav2Vec2Device, Wav2Vec2DataType } from '@storyteller-platform/ghost-story/constants';
|
|
4
|
+
|
|
5
|
+
interface EmitOptions {
|
|
6
|
+
onProgress?: ((progress: number) => void) | null | undefined;
|
|
7
|
+
signal?: AbortSignal | null | undefined;
|
|
8
|
+
model?: Wav2Vec2Model | null | undefined;
|
|
9
|
+
device?: Wav2Vec2Device | null | undefined;
|
|
10
|
+
dtype?: Wav2Vec2DataType | null | undefined;
|
|
11
|
+
logger?: Logger | null | undefined;
|
|
12
|
+
}
|
|
13
|
+
declare function emit(input: string, output: string, options: EmitOptions): Promise<TimingAggregator>;
|
|
14
|
+
|
|
15
|
+
export { type EmitOptions, emit };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Logger } from 'pino';
|
|
2
|
+
import { TimingAggregator } from '@storyteller-platform/ghost-story';
|
|
3
|
+
import { Wav2Vec2Model, Wav2Vec2Device, Wav2Vec2DataType } from '@storyteller-platform/ghost-story/constants';
|
|
4
|
+
|
|
5
|
+
interface EmitOptions {
|
|
6
|
+
onProgress?: ((progress: number) => void) | null | undefined;
|
|
7
|
+
signal?: AbortSignal | null | undefined;
|
|
8
|
+
model?: Wav2Vec2Model | null | undefined;
|
|
9
|
+
device?: Wav2Vec2Device | null | undefined;
|
|
10
|
+
dtype?: Wav2Vec2DataType | null | undefined;
|
|
11
|
+
logger?: Logger | null | undefined;
|
|
12
|
+
}
|
|
13
|
+
declare function emit(input: string, output: string, options: EmitOptions): Promise<TimingAggregator>;
|
|
14
|
+
|
|
15
|
+
export { type EmitOptions, emit };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import "../chunk-BIEQXUOY.js";
|
|
2
|
+
import { mkdir, readFile, readdir } from "node:fs/promises";
|
|
3
|
+
import { basename, extname, join } from "node:path";
|
|
4
|
+
import { enumerate } from "itertools";
|
|
5
|
+
import {
|
|
6
|
+
createAggregator,
|
|
7
|
+
emit as emitFile,
|
|
8
|
+
formatSingleReport
|
|
9
|
+
} from "@storyteller-platform/ghost-story";
|
|
10
|
+
import { MediaType } from "@storyteller-platform/media-types";
|
|
11
|
+
import { writeEmissionsFile } from "./fs.js";
|
|
12
|
+
async function emit(input, output, options) {
|
|
13
|
+
const controller = new AbortController();
|
|
14
|
+
const signal = AbortSignal.any([
|
|
15
|
+
...options.signal ? [options.signal] : [],
|
|
16
|
+
controller.signal
|
|
17
|
+
]);
|
|
18
|
+
await mkdir(output, { recursive: true });
|
|
19
|
+
const allFiles = await readdir(input, { recursive: true });
|
|
20
|
+
const filenames = allFiles.toSorted().filter((f) => MediaType.fromPath(f)?.kind === "audio");
|
|
21
|
+
if (!filenames.length) {
|
|
22
|
+
throw new Error(
|
|
23
|
+
`Failed to transcribe audio: found no audio files in ${input}`
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
const engine = "wav2vec2";
|
|
27
|
+
const model = options.model ?? "onnx-community/mms-300m-1130-forced-aligner-ONNX";
|
|
28
|
+
const emissions = [];
|
|
29
|
+
function aborted() {
|
|
30
|
+
return signal.aborted;
|
|
31
|
+
}
|
|
32
|
+
const timing = createAggregator();
|
|
33
|
+
timing.setMetadata("engine", engine);
|
|
34
|
+
timing.setMetadata("model", model);
|
|
35
|
+
for (const [i, filename] of enumerate(filenames)) {
|
|
36
|
+
let onFileProgress2 = function(progress) {
|
|
37
|
+
const updatedProgress = (i + progress) / filenames.length;
|
|
38
|
+
options.logger?.info(`Progress: ${Math.floor(updatedProgress * 100)}%`);
|
|
39
|
+
options.onProgress?.(updatedProgress);
|
|
40
|
+
};
|
|
41
|
+
var onFileProgress = onFileProgress2;
|
|
42
|
+
if (aborted()) throw new Error("Aborted");
|
|
43
|
+
const filepath = join(input, filename);
|
|
44
|
+
const emissionFilepath = join(
|
|
45
|
+
output,
|
|
46
|
+
`${basename(filename, extname(filename))}.json`
|
|
47
|
+
);
|
|
48
|
+
try {
|
|
49
|
+
await readFile(emissionFilepath, {
|
|
50
|
+
encoding: "utf-8",
|
|
51
|
+
signal
|
|
52
|
+
});
|
|
53
|
+
options.logger?.info(`Found existing transcription for ${filepath}`);
|
|
54
|
+
emissions.push(emissionFilepath);
|
|
55
|
+
options.onProgress?.((emissions.length + 1) / filenames.length);
|
|
56
|
+
continue;
|
|
57
|
+
} catch {
|
|
58
|
+
}
|
|
59
|
+
if (aborted()) throw new Error("Aborted");
|
|
60
|
+
const result = await emitFile(filepath, {
|
|
61
|
+
...options,
|
|
62
|
+
signal,
|
|
63
|
+
engine,
|
|
64
|
+
options: {
|
|
65
|
+
model,
|
|
66
|
+
...options.device && { device: options.device },
|
|
67
|
+
...options.dtype && { dtype: options.dtype },
|
|
68
|
+
onProgress: onFileProgress2
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
options.logger?.info(
|
|
72
|
+
formatSingleReport(
|
|
73
|
+
result.timing,
|
|
74
|
+
`Transcription Timing Report for ${filepath}`
|
|
75
|
+
)
|
|
76
|
+
);
|
|
77
|
+
timing.add(result.timing);
|
|
78
|
+
await writeEmissionsFile(emissionFilepath, result, { signal });
|
|
79
|
+
emissions.push(emissionFilepath);
|
|
80
|
+
options.onProgress?.((i + 1) / filenames.length);
|
|
81
|
+
}
|
|
82
|
+
return timing;
|
|
83
|
+
}
|
|
84
|
+
export {
|
|
85
|
+
emit
|
|
86
|
+
};
|
package/dist/emit/fs.cjs
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __knownSymbol = (name, symbol) => (symbol = Symbol[name]) ? symbol : Symbol.for("Symbol." + name);
|
|
7
|
+
var __typeError = (msg) => {
|
|
8
|
+
throw TypeError(msg);
|
|
9
|
+
};
|
|
10
|
+
var __export = (target, all) => {
|
|
11
|
+
for (var name in all)
|
|
12
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
13
|
+
};
|
|
14
|
+
var __copyProps = (to, from, except, desc) => {
|
|
15
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
16
|
+
for (let key of __getOwnPropNames(from))
|
|
17
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
18
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
19
|
+
}
|
|
20
|
+
return to;
|
|
21
|
+
};
|
|
22
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
23
|
+
var __using = (stack, value, async) => {
|
|
24
|
+
if (value != null) {
|
|
25
|
+
if (typeof value !== "object" && typeof value !== "function") __typeError("Object expected");
|
|
26
|
+
var dispose, inner;
|
|
27
|
+
if (async) dispose = value[__knownSymbol("asyncDispose")];
|
|
28
|
+
if (dispose === void 0) {
|
|
29
|
+
dispose = value[__knownSymbol("dispose")];
|
|
30
|
+
if (async) inner = dispose;
|
|
31
|
+
}
|
|
32
|
+
if (typeof dispose !== "function") __typeError("Object not disposable");
|
|
33
|
+
if (inner) dispose = function() {
|
|
34
|
+
try {
|
|
35
|
+
inner.call(this);
|
|
36
|
+
} catch (e) {
|
|
37
|
+
return Promise.reject(e);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
stack.push([async, dispose, value]);
|
|
41
|
+
} else if (async) {
|
|
42
|
+
stack.push([async]);
|
|
43
|
+
}
|
|
44
|
+
return value;
|
|
45
|
+
};
|
|
46
|
+
var __callDispose = (stack, error, hasError) => {
|
|
47
|
+
var E = typeof SuppressedError === "function" ? SuppressedError : function(e, s, m, _) {
|
|
48
|
+
return _ = Error(m), _.name = "SuppressedError", _.error = e, _.suppressed = s, _;
|
|
49
|
+
};
|
|
50
|
+
var fail = (e) => error = hasError ? new E(e, error, "An error was suppressed during disposal") : (hasError = true, e);
|
|
51
|
+
var next = (it) => {
|
|
52
|
+
while (it = stack.pop()) {
|
|
53
|
+
try {
|
|
54
|
+
var result = it[1] && it[1].call(it[2]);
|
|
55
|
+
if (it[0]) return Promise.resolve(result).then(next, (e) => (fail(e), next()));
|
|
56
|
+
} catch (e) {
|
|
57
|
+
fail(e);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (hasError) throw error;
|
|
61
|
+
};
|
|
62
|
+
return next();
|
|
63
|
+
};
|
|
64
|
+
var fs_exports = {};
|
|
65
|
+
__export(fs_exports, {
|
|
66
|
+
EmissionsReader: () => EmissionsReader,
|
|
67
|
+
readEmissionsFile: () => readEmissionsFile,
|
|
68
|
+
readEmissionsFileHead: () => readEmissionsFileHead,
|
|
69
|
+
writeEmissionsFile: () => writeEmissionsFile
|
|
70
|
+
});
|
|
71
|
+
module.exports = __toCommonJS(fs_exports);
|
|
72
|
+
var import_promises = require("node:fs/promises");
|
|
73
|
+
var import_node_path = require("node:path");
|
|
74
|
+
var import_ghost_story = require("@storyteller-platform/ghost-story");
|
|
75
|
+
async function writeEmissionsFile(path, data, options = {}) {
|
|
76
|
+
const encoded = (0, import_ghost_story.encodeEmissions)(data);
|
|
77
|
+
await (0, import_promises.writeFile)(path, encoded, options);
|
|
78
|
+
}
|
|
79
|
+
async function readEmissionsFile(path, framesFrom, framesTo) {
|
|
80
|
+
var _stack = [];
|
|
81
|
+
try {
|
|
82
|
+
const file = __using(_stack, await (0, import_promises.open)(path, "r"), true);
|
|
83
|
+
async function getBytes(bytesFrom, bytesTo) {
|
|
84
|
+
const length = bytesTo - bytesFrom;
|
|
85
|
+
const buffer = new Uint8Array(length);
|
|
86
|
+
let read = 0;
|
|
87
|
+
while (read < length) {
|
|
88
|
+
const { bytesRead } = await file.read(
|
|
89
|
+
buffer,
|
|
90
|
+
read,
|
|
91
|
+
length - read,
|
|
92
|
+
bytesFrom + read
|
|
93
|
+
);
|
|
94
|
+
if (bytesRead === 0) break;
|
|
95
|
+
read += bytesRead;
|
|
96
|
+
}
|
|
97
|
+
return buffer;
|
|
98
|
+
}
|
|
99
|
+
return await (0, import_ghost_story.decodeEmissions)(getBytes, framesFrom, framesTo);
|
|
100
|
+
} catch (_) {
|
|
101
|
+
var _error = _, _hasError = true;
|
|
102
|
+
} finally {
|
|
103
|
+
var _promise = __callDispose(_stack, _error, _hasError);
|
|
104
|
+
_promise && await _promise;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async function readEmissionsFileHead(path) {
|
|
108
|
+
return await readEmissionsFile(path, 0, 0);
|
|
109
|
+
}
|
|
110
|
+
async function packJsonEmissionsFile(path) {
|
|
111
|
+
const directory = (0, import_node_path.dirname)(path);
|
|
112
|
+
const filebase = (0, import_node_path.basename)(path, (0, import_node_path.extname)(path));
|
|
113
|
+
const emissionsName = (0, import_node_path.join)(directory, `${filebase}.emissions`);
|
|
114
|
+
const jsonData = await (0, import_promises.readFile)(path, { encoding: "utf-8" });
|
|
115
|
+
const parsed = JSON.parse(jsonData);
|
|
116
|
+
await writeEmissionsFile(emissionsName, {
|
|
117
|
+
...parsed,
|
|
118
|
+
vocab: new Map(Object.entries(parsed.vocab)),
|
|
119
|
+
numFrames: parsed.emissions.length
|
|
120
|
+
});
|
|
121
|
+
await (0, import_promises.rm)(path);
|
|
122
|
+
return emissionsName;
|
|
123
|
+
}
|
|
124
|
+
class EmissionsReader {
|
|
125
|
+
constructor(numFrames = 0, frameMap, vocab, blankId, timings) {
|
|
126
|
+
this.numFrames = numFrames;
|
|
127
|
+
this.frameMap = frameMap;
|
|
128
|
+
this.vocab = vocab;
|
|
129
|
+
this.blankId = blankId;
|
|
130
|
+
this.timings = timings;
|
|
131
|
+
}
|
|
132
|
+
static async getFilepaths(fileOrDirectory) {
|
|
133
|
+
if ((0, import_node_path.extname)(fileOrDirectory) === ".emissions" || (0, import_node_path.extname)(fileOrDirectory) === ".json") {
|
|
134
|
+
return [fileOrDirectory];
|
|
135
|
+
}
|
|
136
|
+
const allFiles = await (0, import_promises.readdir)(fileOrDirectory, { recursive: true });
|
|
137
|
+
return allFiles.filter((f) => (0, import_node_path.extname)(f) === ".emissions" || (0, import_node_path.extname)(f) === ".json").toSorted().map((f) => (0, import_node_path.join)(fileOrDirectory, f));
|
|
138
|
+
}
|
|
139
|
+
static async from(fileOrDirectory) {
|
|
140
|
+
let vocab = null;
|
|
141
|
+
let blankId = null;
|
|
142
|
+
const timings = [];
|
|
143
|
+
const frameMap = [];
|
|
144
|
+
let numFrames = 0;
|
|
145
|
+
const emissionsFiles = await EmissionsReader.getFilepaths(fileOrDirectory);
|
|
146
|
+
let f = 0;
|
|
147
|
+
for (let path of emissionsFiles) {
|
|
148
|
+
if ((0, import_node_path.extname)(path) === ".json") {
|
|
149
|
+
path = await packJsonEmissionsFile(path);
|
|
150
|
+
}
|
|
151
|
+
const {
|
|
152
|
+
numFrames: numFrames2,
|
|
153
|
+
vocab: v,
|
|
154
|
+
blankIndex: b,
|
|
155
|
+
emissions: _,
|
|
156
|
+
...emissionTimings
|
|
157
|
+
} = await readEmissionsFileHead(path);
|
|
158
|
+
vocab ??= v;
|
|
159
|
+
blankId ??= b;
|
|
160
|
+
timings.push(emissionTimings);
|
|
161
|
+
frameMap.push([path, f, f + numFrames2]);
|
|
162
|
+
f += numFrames2;
|
|
163
|
+
}
|
|
164
|
+
numFrames = f;
|
|
165
|
+
if (vocab === null || blankId === null) {
|
|
166
|
+
throw new Error("Failed to initialize EmissionsReader");
|
|
167
|
+
}
|
|
168
|
+
const reader = new EmissionsReader(
|
|
169
|
+
numFrames,
|
|
170
|
+
frameMap,
|
|
171
|
+
vocab,
|
|
172
|
+
blankId,
|
|
173
|
+
timings
|
|
174
|
+
);
|
|
175
|
+
return reader;
|
|
176
|
+
}
|
|
177
|
+
async *frames() {
|
|
178
|
+
let fileIndex = 0;
|
|
179
|
+
let chunkFrom = 0;
|
|
180
|
+
let frameIndex = 0;
|
|
181
|
+
while (frameIndex < this.numFrames) {
|
|
182
|
+
const [path, framesStart, framesEnd] = this.frameMap[fileIndex];
|
|
183
|
+
const chunkTo = Math.min(chunkFrom + 1e3, framesEnd - framesStart);
|
|
184
|
+
const { emissions: chunk } = await readEmissionsFile(
|
|
185
|
+
path,
|
|
186
|
+
chunkFrom,
|
|
187
|
+
chunkTo
|
|
188
|
+
);
|
|
189
|
+
for (const e of chunk) {
|
|
190
|
+
yield e;
|
|
191
|
+
}
|
|
192
|
+
frameIndex += chunkTo - chunkFrom;
|
|
193
|
+
if (chunkTo < chunkFrom + 1e3) {
|
|
194
|
+
fileIndex++;
|
|
195
|
+
chunkFrom = 0;
|
|
196
|
+
} else {
|
|
197
|
+
chunkFrom += 1e3;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
async readFrames(framesFrom, framesTo) {
|
|
202
|
+
const startFile = this.frameMap.findIndex(([, , end]) => end > framesFrom);
|
|
203
|
+
if (startFile === -1) return [];
|
|
204
|
+
let endFile = this.frameMap.findLastIndex(([, start]) => start < framesTo);
|
|
205
|
+
if (endFile === -1) endFile = this.frameMap.length - 1;
|
|
206
|
+
const frames = [];
|
|
207
|
+
for (let i = startFile; i <= endFile; i++) {
|
|
208
|
+
const [path, from, to] = this.frameMap[i];
|
|
209
|
+
const { emissions } = await readEmissionsFile(
|
|
210
|
+
path,
|
|
211
|
+
Math.max(framesFrom, from) - from,
|
|
212
|
+
Math.min(to, framesTo) - from
|
|
213
|
+
);
|
|
214
|
+
for (const e of emissions) {
|
|
215
|
+
frames.push(e);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return frames;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
222
|
+
0 && (module.exports = {
|
|
223
|
+
EmissionsReader,
|
|
224
|
+
readEmissionsFile,
|
|
225
|
+
readEmissionsFileHead,
|
|
226
|
+
writeEmissionsFile
|
|
227
|
+
});
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { EmissionData } from '@storyteller-platform/ghost-story';
|
|
2
|
+
import { EmissionTimings } from '../align/ctc/emissions.cjs';
|
|
3
|
+
|
|
4
|
+
declare function writeEmissionsFile(path: string, data: EmissionData, options?: {
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
}): Promise<void>;
|
|
7
|
+
declare function readEmissionsFile(path: string, framesFrom?: number, framesTo?: number): Promise<EmissionData>;
|
|
8
|
+
declare function readEmissionsFileHead(path: string): Promise<EmissionData>;
|
|
9
|
+
declare class EmissionsReader {
|
|
10
|
+
numFrames: number;
|
|
11
|
+
frameMap: [string, number, number][];
|
|
12
|
+
vocab: Map<string, number>;
|
|
13
|
+
blankId: number;
|
|
14
|
+
timings: EmissionTimings[];
|
|
15
|
+
private constructor();
|
|
16
|
+
private static getFilepaths;
|
|
17
|
+
static from(fileOrDirectory: string): Promise<EmissionsReader>;
|
|
18
|
+
frames(): AsyncGenerator<number[], void, unknown>;
|
|
19
|
+
readFrames(framesFrom: number, framesTo: number): Promise<number[][]>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export { EmissionsReader, readEmissionsFile, readEmissionsFileHead, writeEmissionsFile };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { EmissionData } from '@storyteller-platform/ghost-story';
|
|
2
|
+
import { EmissionTimings } from '../align/ctc/emissions.js';
|
|
3
|
+
|
|
4
|
+
declare function writeEmissionsFile(path: string, data: EmissionData, options?: {
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
}): Promise<void>;
|
|
7
|
+
declare function readEmissionsFile(path: string, framesFrom?: number, framesTo?: number): Promise<EmissionData>;
|
|
8
|
+
declare function readEmissionsFileHead(path: string): Promise<EmissionData>;
|
|
9
|
+
declare class EmissionsReader {
|
|
10
|
+
numFrames: number;
|
|
11
|
+
frameMap: [string, number, number][];
|
|
12
|
+
vocab: Map<string, number>;
|
|
13
|
+
blankId: number;
|
|
14
|
+
timings: EmissionTimings[];
|
|
15
|
+
private constructor();
|
|
16
|
+
private static getFilepaths;
|
|
17
|
+
static from(fileOrDirectory: string): Promise<EmissionsReader>;
|
|
18
|
+
frames(): AsyncGenerator<number[], void, unknown>;
|
|
19
|
+
readFrames(framesFrom: number, framesTo: number): Promise<number[][]>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export { EmissionsReader, readEmissionsFile, readEmissionsFileHead, writeEmissionsFile };
|
package/dist/emit/fs.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import {
|
|
2
|
+
__callDispose,
|
|
3
|
+
__using
|
|
4
|
+
} from "../chunk-BIEQXUOY.js";
|
|
5
|
+
import { open, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
6
|
+
import { basename, dirname, extname, join } from "node:path";
|
|
7
|
+
import {
|
|
8
|
+
decodeEmissions,
|
|
9
|
+
encodeEmissions
|
|
10
|
+
} from "@storyteller-platform/ghost-story";
|
|
11
|
+
async function writeEmissionsFile(path, data, options = {}) {
|
|
12
|
+
const encoded = encodeEmissions(data);
|
|
13
|
+
await writeFile(path, encoded, options);
|
|
14
|
+
}
|
|
15
|
+
async function readEmissionsFile(path, framesFrom, framesTo) {
|
|
16
|
+
var _stack = [];
|
|
17
|
+
try {
|
|
18
|
+
const file = __using(_stack, await open(path, "r"), true);
|
|
19
|
+
async function getBytes(bytesFrom, bytesTo) {
|
|
20
|
+
const length = bytesTo - bytesFrom;
|
|
21
|
+
const buffer = new Uint8Array(length);
|
|
22
|
+
let read = 0;
|
|
23
|
+
while (read < length) {
|
|
24
|
+
const { bytesRead } = await file.read(
|
|
25
|
+
buffer,
|
|
26
|
+
read,
|
|
27
|
+
length - read,
|
|
28
|
+
bytesFrom + read
|
|
29
|
+
);
|
|
30
|
+
if (bytesRead === 0) break;
|
|
31
|
+
read += bytesRead;
|
|
32
|
+
}
|
|
33
|
+
return buffer;
|
|
34
|
+
}
|
|
35
|
+
return await decodeEmissions(getBytes, framesFrom, framesTo);
|
|
36
|
+
} catch (_) {
|
|
37
|
+
var _error = _, _hasError = true;
|
|
38
|
+
} finally {
|
|
39
|
+
var _promise = __callDispose(_stack, _error, _hasError);
|
|
40
|
+
_promise && await _promise;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async function readEmissionsFileHead(path) {
|
|
44
|
+
return await readEmissionsFile(path, 0, 0);
|
|
45
|
+
}
|
|
46
|
+
async function packJsonEmissionsFile(path) {
|
|
47
|
+
const directory = dirname(path);
|
|
48
|
+
const filebase = basename(path, extname(path));
|
|
49
|
+
const emissionsName = join(directory, `${filebase}.emissions`);
|
|
50
|
+
const jsonData = await readFile(path, { encoding: "utf-8" });
|
|
51
|
+
const parsed = JSON.parse(jsonData);
|
|
52
|
+
await writeEmissionsFile(emissionsName, {
|
|
53
|
+
...parsed,
|
|
54
|
+
vocab: new Map(Object.entries(parsed.vocab)),
|
|
55
|
+
numFrames: parsed.emissions.length
|
|
56
|
+
});
|
|
57
|
+
await rm(path);
|
|
58
|
+
return emissionsName;
|
|
59
|
+
}
|
|
60
|
+
class EmissionsReader {
|
|
61
|
+
constructor(numFrames = 0, frameMap, vocab, blankId, timings) {
|
|
62
|
+
this.numFrames = numFrames;
|
|
63
|
+
this.frameMap = frameMap;
|
|
64
|
+
this.vocab = vocab;
|
|
65
|
+
this.blankId = blankId;
|
|
66
|
+
this.timings = timings;
|
|
67
|
+
}
|
|
68
|
+
static async getFilepaths(fileOrDirectory) {
|
|
69
|
+
if (extname(fileOrDirectory) === ".emissions" || extname(fileOrDirectory) === ".json") {
|
|
70
|
+
return [fileOrDirectory];
|
|
71
|
+
}
|
|
72
|
+
const allFiles = await readdir(fileOrDirectory, { recursive: true });
|
|
73
|
+
return allFiles.filter((f) => extname(f) === ".emissions" || extname(f) === ".json").toSorted().map((f) => join(fileOrDirectory, f));
|
|
74
|
+
}
|
|
75
|
+
static async from(fileOrDirectory) {
|
|
76
|
+
let vocab = null;
|
|
77
|
+
let blankId = null;
|
|
78
|
+
const timings = [];
|
|
79
|
+
const frameMap = [];
|
|
80
|
+
let numFrames = 0;
|
|
81
|
+
const emissionsFiles = await EmissionsReader.getFilepaths(fileOrDirectory);
|
|
82
|
+
let f = 0;
|
|
83
|
+
for (let path of emissionsFiles) {
|
|
84
|
+
if (extname(path) === ".json") {
|
|
85
|
+
path = await packJsonEmissionsFile(path);
|
|
86
|
+
}
|
|
87
|
+
const {
|
|
88
|
+
numFrames: numFrames2,
|
|
89
|
+
vocab: v,
|
|
90
|
+
blankIndex: b,
|
|
91
|
+
emissions: _,
|
|
92
|
+
...emissionTimings
|
|
93
|
+
} = await readEmissionsFileHead(path);
|
|
94
|
+
vocab ??= v;
|
|
95
|
+
blankId ??= b;
|
|
96
|
+
timings.push(emissionTimings);
|
|
97
|
+
frameMap.push([path, f, f + numFrames2]);
|
|
98
|
+
f += numFrames2;
|
|
99
|
+
}
|
|
100
|
+
numFrames = f;
|
|
101
|
+
if (vocab === null || blankId === null) {
|
|
102
|
+
throw new Error("Failed to initialize EmissionsReader");
|
|
103
|
+
}
|
|
104
|
+
const reader = new EmissionsReader(
|
|
105
|
+
numFrames,
|
|
106
|
+
frameMap,
|
|
107
|
+
vocab,
|
|
108
|
+
blankId,
|
|
109
|
+
timings
|
|
110
|
+
);
|
|
111
|
+
return reader;
|
|
112
|
+
}
|
|
113
|
+
async *frames() {
|
|
114
|
+
let fileIndex = 0;
|
|
115
|
+
let chunkFrom = 0;
|
|
116
|
+
let frameIndex = 0;
|
|
117
|
+
while (frameIndex < this.numFrames) {
|
|
118
|
+
const [path, framesStart, framesEnd] = this.frameMap[fileIndex];
|
|
119
|
+
const chunkTo = Math.min(chunkFrom + 1e3, framesEnd - framesStart);
|
|
120
|
+
const { emissions: chunk } = await readEmissionsFile(
|
|
121
|
+
path,
|
|
122
|
+
chunkFrom,
|
|
123
|
+
chunkTo
|
|
124
|
+
);
|
|
125
|
+
for (const e of chunk) {
|
|
126
|
+
yield e;
|
|
127
|
+
}
|
|
128
|
+
frameIndex += chunkTo - chunkFrom;
|
|
129
|
+
if (chunkTo < chunkFrom + 1e3) {
|
|
130
|
+
fileIndex++;
|
|
131
|
+
chunkFrom = 0;
|
|
132
|
+
} else {
|
|
133
|
+
chunkFrom += 1e3;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async readFrames(framesFrom, framesTo) {
|
|
138
|
+
const startFile = this.frameMap.findIndex(([, , end]) => end > framesFrom);
|
|
139
|
+
if (startFile === -1) return [];
|
|
140
|
+
let endFile = this.frameMap.findLastIndex(([, start]) => start < framesTo);
|
|
141
|
+
if (endFile === -1) endFile = this.frameMap.length - 1;
|
|
142
|
+
const frames = [];
|
|
143
|
+
for (let i = startFile; i <= endFile; i++) {
|
|
144
|
+
const [path, from, to] = this.frameMap[i];
|
|
145
|
+
const { emissions } = await readEmissionsFile(
|
|
146
|
+
path,
|
|
147
|
+
Math.max(framesFrom, from) - from,
|
|
148
|
+
Math.min(to, framesTo) - from
|
|
149
|
+
);
|
|
150
|
+
for (const e of emissions) {
|
|
151
|
+
frames.push(e);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return frames;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
export {
|
|
158
|
+
EmissionsReader,
|
|
159
|
+
readEmissionsFile,
|
|
160
|
+
readEmissionsFileHead,
|
|
161
|
+
writeEmissionsFile
|
|
162
|
+
};
|