@storyteller-platform/align 0.1.57 → 0.1.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/align/align.cjs +38 -19
- package/dist/align/align.d.cts +12 -1
- package/dist/align/align.d.ts +12 -1
- package/dist/align/align.js +38 -19
- package/dist/align/ctc/Aligner.cjs +800 -0
- package/dist/align/ctc/Aligner.d.cts +56 -0
- package/dist/align/ctc/Aligner.d.ts +56 -0
- package/dist/align/ctc/Aligner.js +789 -0
- package/dist/align/ctc/align.cjs +189 -0
- package/dist/align/ctc/align.d.cts +14 -0
- package/dist/align/ctc/align.d.ts +14 -0
- package/dist/align/ctc/align.js +132 -0
- package/dist/{process/mime.cjs → align/ctc/emissions.cjs} +7 -20
- package/dist/align/ctc/emissions.d.cts +6 -0
- package/dist/align/ctc/emissions.d.ts +6 -0
- package/dist/align/ctc/emissions.js +7 -0
- package/dist/align/ctc/forcedAlign.cjs +273 -0
- package/dist/align/ctc/forcedAlign.d.cts +21 -0
- package/dist/align/ctc/forcedAlign.d.ts +21 -0
- package/dist/align/ctc/forcedAlign.js +248 -0
- package/dist/align/ctc/greedyDecode.cjs +61 -0
- package/dist/align/ctc/greedyDecode.d.cts +15 -0
- package/dist/align/ctc/greedyDecode.d.ts +15 -0
- package/dist/align/ctc/greedyDecode.js +37 -0
- package/dist/align/ctc/mediaOverlay.cjs +248 -0
- package/dist/align/ctc/mediaOverlay.d.cts +16 -0
- package/dist/align/ctc/mediaOverlay.d.ts +16 -0
- package/dist/align/ctc/mediaOverlay.js +224 -0
- package/dist/align/ctc/report.cjs +66 -0
- package/dist/align/ctc/report.d.cts +79 -0
- package/dist/align/ctc/report.d.ts +79 -0
- package/dist/align/ctc/report.js +42 -0
- package/dist/align/ctc/search.cjs +437 -0
- package/dist/align/ctc/search.d.cts +46 -0
- package/dist/align/ctc/search.d.ts +46 -0
- package/dist/align/ctc/search.js +412 -0
- package/dist/align/getSentenceRanges.cjs +9 -3
- package/dist/align/getSentenceRanges.d.cts +1 -0
- package/dist/align/getSentenceRanges.d.ts +1 -0
- package/dist/align/getSentenceRanges.js +9 -3
- package/dist/align/parse.cjs +17 -4
- package/dist/align/parse.d.cts +8 -4
- package/dist/align/parse.d.ts +8 -4
- package/dist/align/parse.js +19 -4
- package/dist/align/search.cjs +76 -23
- package/dist/align/search.d.cts +8 -4
- package/dist/align/search.d.ts +8 -4
- package/dist/align/search.js +76 -24
- package/dist/align/slugify.cjs +3 -2
- package/dist/align/slugify.d.cts +1 -1
- package/dist/align/slugify.d.ts +1 -1
- package/dist/align/slugify.js +3 -2
- package/dist/cli/bin.cjs +204 -114
- package/dist/cli/bin.js +194 -103
- package/dist/common/ffmpeg.cjs +10 -5
- package/dist/common/ffmpeg.js +10 -5
- package/dist/common/logging.cjs +33 -1
- package/dist/common/logging.js +33 -1
- package/dist/emit/emit.cjs +105 -0
- package/dist/emit/emit.d.cts +15 -0
- package/dist/emit/emit.d.ts +15 -0
- package/dist/emit/emit.js +86 -0
- package/dist/emit/fs.cjs +227 -0
- package/dist/emit/fs.d.cts +22 -0
- package/dist/emit/fs.d.ts +22 -0
- package/dist/emit/fs.js +162 -0
- package/dist/emit/parse.cjs +68 -0
- package/dist/emit/parse.d.cts +26 -0
- package/dist/emit/parse.d.ts +26 -0
- package/dist/emit/parse.js +58 -0
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/markup/model.cjs +0 -16
- package/dist/markup/model.d.cts +1 -1
- package/dist/markup/model.d.ts +1 -1
- package/dist/markup/model.js +0 -12
- package/dist/markup/parseDom.cjs +13 -14
- package/dist/markup/parseDom.d.cts +2 -2
- package/dist/markup/parseDom.d.ts +2 -2
- package/dist/markup/parseDom.js +13 -18
- package/dist/markup/resolvedPos.d.cts +1 -1
- package/dist/markup/resolvedPos.d.ts +1 -1
- package/dist/markup/segmentation.cjs +3 -1
- package/dist/markup/segmentation.js +3 -1
- package/dist/markup/serializeDom.cjs +1 -1
- package/dist/markup/serializeDom.d.cts +1 -1
- package/dist/markup/serializeDom.d.ts +1 -1
- package/dist/markup/serializeDom.js +1 -1
- package/dist/markup/transform.cjs +5 -2
- package/dist/markup/transform.d.cts +1 -1
- package/dist/markup/transform.d.ts +1 -1
- package/dist/markup/transform.js +5 -4
- package/dist/{model-CZ2mMHJO.d.cts → model-CeOMDDgt.d.cts} +3 -11
- package/dist/{model-CZ2mMHJO.d.ts → model-CeOMDDgt.d.ts} +3 -11
- package/dist/process/processAudiobook.cjs +12 -23
- package/dist/process/processAudiobook.js +14 -30
- package/dist/process/ranges.cjs +2 -2
- package/dist/process/ranges.js +2 -2
- package/dist/readium/guidedNavigation.cjs +8 -8
- package/dist/readium/guidedNavigation.js +8 -8
- package/dist/readium/manifest.cjs +5 -4
- package/dist/readium/manifest.js +5 -4
- package/dist/snapshot/ctc/snapshot.cjs +261 -0
- package/dist/snapshot/ctc/snapshot.d.cts +6 -0
- package/dist/snapshot/ctc/snapshot.d.ts +6 -0
- package/dist/snapshot/ctc/snapshot.js +200 -0
- package/dist/snapshot/parse.cjs +29 -16
- package/dist/snapshot/parse.d.cts +14 -10
- package/dist/snapshot/parse.d.ts +14 -10
- package/dist/snapshot/parse.js +32 -17
- package/dist/transcribe/parse.d.cts +6 -6
- package/dist/transcribe/parse.d.ts +6 -6
- package/dist/transcribe/transcribe.cjs +4 -2
- package/dist/transcribe/transcribe.js +4 -2
- package/package.json +6 -4
- package/dist/process/mime.d.cts +0 -3
- package/dist/process/mime.d.ts +0 -3
- package/dist/process/mime.js +0 -24
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Match } from './search.cjs';
|
|
2
|
+
import '../../emit/fs.cjs';
|
|
3
|
+
import '@storyteller-platform/ghost-story';
|
|
4
|
+
import './emissions.cjs';
|
|
5
|
+
|
|
6
|
+
interface TokenAlignment {
|
|
7
|
+
label: number;
|
|
8
|
+
startFrame: number;
|
|
9
|
+
endFrame: number;
|
|
10
|
+
score: number;
|
|
11
|
+
}
|
|
12
|
+
declare function ctcViterbiAlign(emissions: number[][], labels: number[], blankId: number, starId: number, starScore: number): {
|
|
13
|
+
label: number;
|
|
14
|
+
startFrame: number;
|
|
15
|
+
endFrame: number;
|
|
16
|
+
score: number;
|
|
17
|
+
}[];
|
|
18
|
+
declare function detectDroppedLabels(alignments: TokenAlignment[], startOffset: number, endOffset: number, minLength?: number): boolean;
|
|
19
|
+
declare function ctcForcedAlign(baseline: string, emissions: number[][], anchors: Match[], vocab: Map<string, number>, blankId: number, dropLabels?: (labelOffset: number, alignments: TokenAlignment[]) => [number, number][]): TokenAlignment[];
|
|
20
|
+
|
|
21
|
+
export { type TokenAlignment, ctcForcedAlign, ctcViterbiAlign, detectDroppedLabels };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Match } from './search.js';
|
|
2
|
+
import '../../emit/fs.js';
|
|
3
|
+
import '@storyteller-platform/ghost-story';
|
|
4
|
+
import './emissions.js';
|
|
5
|
+
|
|
6
|
+
interface TokenAlignment {
|
|
7
|
+
label: number;
|
|
8
|
+
startFrame: number;
|
|
9
|
+
endFrame: number;
|
|
10
|
+
score: number;
|
|
11
|
+
}
|
|
12
|
+
declare function ctcViterbiAlign(emissions: number[][], labels: number[], blankId: number, starId: number, starScore: number): {
|
|
13
|
+
label: number;
|
|
14
|
+
startFrame: number;
|
|
15
|
+
endFrame: number;
|
|
16
|
+
score: number;
|
|
17
|
+
}[];
|
|
18
|
+
declare function detectDroppedLabels(alignments: TokenAlignment[], startOffset: number, endOffset: number, minLength?: number): boolean;
|
|
19
|
+
declare function ctcForcedAlign(baseline: string, emissions: number[][], anchors: Match[], vocab: Map<string, number>, blankId: number, dropLabels?: (labelOffset: number, alignments: TokenAlignment[]) => [number, number][]): TokenAlignment[];
|
|
20
|
+
|
|
21
|
+
export { type TokenAlignment, ctcForcedAlign, ctcViterbiAlign, detectDroppedLabels };
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import "../../chunk-BIEQXUOY.js";
|
|
2
|
+
import { enumerate, range } from "itertools";
|
|
3
|
+
function buildStateGraph(labels, starId, blankId) {
|
|
4
|
+
const tokens = [blankId].concat(labels.flatMap((l) => [l, blankId]));
|
|
5
|
+
const skipLegal = tokens.map(
|
|
6
|
+
(t, i) => i >= 2 && (t !== tokens[i - 2] || t === starId)
|
|
7
|
+
);
|
|
8
|
+
return { tokens, skipLegal };
|
|
9
|
+
}
|
|
10
|
+
function ctcViterbiAlign(emissions, labels, blankId, starId, starScore) {
|
|
11
|
+
if (!labels.length) return [];
|
|
12
|
+
function emitScore(row, token) {
|
|
13
|
+
return token === starId ? starScore : row[token];
|
|
14
|
+
}
|
|
15
|
+
const { tokens, skipLegal } = buildStateGraph(labels, starId, blankId);
|
|
16
|
+
const numStates = tokens.length;
|
|
17
|
+
const numFrames = emissions.length;
|
|
18
|
+
if (!numFrames) return [];
|
|
19
|
+
let prevScores = new Float32Array(numStates).fill(-Infinity);
|
|
20
|
+
const backpointers = new Uint8Array(numFrames * numStates);
|
|
21
|
+
prevScores[0] = emissions[0][blankId];
|
|
22
|
+
prevScores[1] = emitScore(emissions[0], tokens[1]);
|
|
23
|
+
if (labels[0] === starId) {
|
|
24
|
+
prevScores[2] = emissions[0][blankId];
|
|
25
|
+
prevScores[3] = emitScore(emissions[0], tokens[3]);
|
|
26
|
+
}
|
|
27
|
+
for (let t = 1; t < numFrames; t++) {
|
|
28
|
+
const row = emissions[t];
|
|
29
|
+
const currScores = new Float32Array(numStates).fill(-Infinity);
|
|
30
|
+
for (let s = 0; s < numStates; s++) {
|
|
31
|
+
const token = tokens[s];
|
|
32
|
+
let best = prevScores[s];
|
|
33
|
+
let bestOp = 0;
|
|
34
|
+
if (s > 0) {
|
|
35
|
+
const move = prevScores[s - 1];
|
|
36
|
+
if (move > best) {
|
|
37
|
+
best = move;
|
|
38
|
+
bestOp = 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (skipLegal[s]) {
|
|
42
|
+
const move = prevScores[s - 2];
|
|
43
|
+
if (move > best) {
|
|
44
|
+
best = move;
|
|
45
|
+
bestOp = 2;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
currScores[s] = emitScore(row, token) + best;
|
|
49
|
+
backpointers[t * numStates + s] = bestOp;
|
|
50
|
+
}
|
|
51
|
+
prevScores = currScores;
|
|
52
|
+
}
|
|
53
|
+
const terminalCandidates = [
|
|
54
|
+
numStates - 1,
|
|
55
|
+
numStates - 2,
|
|
56
|
+
// Same as above, only allow skipping the last label
|
|
57
|
+
// if it's a star
|
|
58
|
+
...labels.at(-1) === starId ? [numStates - 3, numStates - 4] : []
|
|
59
|
+
].filter((s) => s >= 0);
|
|
60
|
+
let terminal = terminalCandidates[0];
|
|
61
|
+
for (const s of terminalCandidates) {
|
|
62
|
+
if (prevScores[s] > prevScores[terminal]) terminal = s;
|
|
63
|
+
}
|
|
64
|
+
if (prevScores[terminal] === -Infinity) {
|
|
65
|
+
return labels.map((label) => ({
|
|
66
|
+
label,
|
|
67
|
+
startFrame: -1,
|
|
68
|
+
endFrame: -1,
|
|
69
|
+
score: -Infinity
|
|
70
|
+
}));
|
|
71
|
+
}
|
|
72
|
+
const path = new Int32Array(numFrames);
|
|
73
|
+
path[numFrames - 1] = terminal;
|
|
74
|
+
for (let t = numFrames - 1; t > 0; t--) {
|
|
75
|
+
const s = path[t];
|
|
76
|
+
const op = backpointers[t * numStates + s];
|
|
77
|
+
path[t - 1] = s - op;
|
|
78
|
+
}
|
|
79
|
+
const firstFrames = new Int32Array(numStates).fill(-1);
|
|
80
|
+
const lastFrames = new Int32Array(numStates).fill(-1);
|
|
81
|
+
const scoreSums = new Float64Array(numStates);
|
|
82
|
+
const frameCounts = new Int32Array(numStates);
|
|
83
|
+
for (let t = 0; t < numFrames; t++) {
|
|
84
|
+
const s = path[t];
|
|
85
|
+
if (firstFrames[s] === -1) firstFrames[s] = t;
|
|
86
|
+
lastFrames[s] = t;
|
|
87
|
+
const row = emissions[t];
|
|
88
|
+
const token = tokens[s];
|
|
89
|
+
scoreSums[s] += emitScore(row, token);
|
|
90
|
+
frameCounts[s]++;
|
|
91
|
+
}
|
|
92
|
+
const alignments = [];
|
|
93
|
+
for (let l = 0; l < labels.length; l++) {
|
|
94
|
+
const state = 2 * l + 1;
|
|
95
|
+
const startFrame = firstFrames[state];
|
|
96
|
+
const endFrame = lastFrames[state];
|
|
97
|
+
const scoreSum = scoreSums[state];
|
|
98
|
+
const frameCount = frameCounts[state];
|
|
99
|
+
const meanScore = frameCount > 0 ? scoreSum / frameCount : -Infinity;
|
|
100
|
+
alignments.push({
|
|
101
|
+
label: labels[l],
|
|
102
|
+
startFrame,
|
|
103
|
+
endFrame,
|
|
104
|
+
score: meanScore
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return alignments;
|
|
108
|
+
}
|
|
109
|
+
function detectDroppedLabels(alignments, startOffset, endOffset, minLength = 8) {
|
|
110
|
+
if (endOffset < startOffset || endOffset - startOffset <= minLength) {
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
const endFrame = alignments[endOffset]?.endFrame;
|
|
114
|
+
if (endFrame === void 0) {
|
|
115
|
+
throw new Error(`endOffset ${endOffset} out of bounds`);
|
|
116
|
+
}
|
|
117
|
+
const startFrame = alignments[startOffset]?.startFrame;
|
|
118
|
+
if (startFrame === void 0) {
|
|
119
|
+
throw new Error(`startOffset ${startOffset} out of bounds`);
|
|
120
|
+
}
|
|
121
|
+
const framesPerChar = (endFrame - startFrame) / (endOffset - startOffset);
|
|
122
|
+
return framesPerChar < 2;
|
|
123
|
+
}
|
|
124
|
+
function removeLabelRanges(input, removals) {
|
|
125
|
+
const inputIndices = [];
|
|
126
|
+
let nextRemovalIndex = 0;
|
|
127
|
+
const result = [];
|
|
128
|
+
let i = 0;
|
|
129
|
+
while (i < input.length) {
|
|
130
|
+
const l = input[i];
|
|
131
|
+
const nextRemoval = removals[nextRemovalIndex];
|
|
132
|
+
if (!nextRemoval) {
|
|
133
|
+
inputIndices.push(i);
|
|
134
|
+
result.push(l);
|
|
135
|
+
i++;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (i >= nextRemoval[0] && i <= nextRemoval[1]) {
|
|
139
|
+
i++;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (i > nextRemoval[1]) {
|
|
143
|
+
nextRemovalIndex++;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
inputIndices.push(i);
|
|
147
|
+
result.push(l);
|
|
148
|
+
i++;
|
|
149
|
+
}
|
|
150
|
+
return { result, inputIndices };
|
|
151
|
+
}
|
|
152
|
+
const STAR_SCORE = -3;
|
|
153
|
+
function ctcForcedAlign(baseline, emissions, anchors, vocab, blankId, dropLabels = () => []) {
|
|
154
|
+
const star = vocab.size;
|
|
155
|
+
const segments = [];
|
|
156
|
+
for (let i = 0; i < anchors.length + 1; i++) {
|
|
157
|
+
const prevAnchor = anchors[i - 1];
|
|
158
|
+
const nextAnchor = anchors[i];
|
|
159
|
+
segments.push({
|
|
160
|
+
labels: baseline.slice(prevAnchor?.offset, nextAnchor?.offset).split("").map((c) => vocab.get(c) ?? star),
|
|
161
|
+
labelsStart: prevAnchor?.offset ?? 0,
|
|
162
|
+
emissions: emissions.slice(prevAnchor?.position, nextAnchor?.position),
|
|
163
|
+
emissionsStart: prevAnchor?.position ?? 0
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
segments[0]?.labels.unshift(star);
|
|
167
|
+
segments.at(-1)?.labels.push(star);
|
|
168
|
+
const alignmentsWithRemovals = [];
|
|
169
|
+
const inputIndices = [];
|
|
170
|
+
for (const segment of segments) {
|
|
171
|
+
let stripBoundingStars2 = function(arr) {
|
|
172
|
+
let result = arr;
|
|
173
|
+
if (segment === segments[0]) {
|
|
174
|
+
result = result.slice(1);
|
|
175
|
+
}
|
|
176
|
+
if (segment === segments.at(-1)) {
|
|
177
|
+
result = result.slice(0, -1);
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
};
|
|
181
|
+
var stripBoundingStars = stripBoundingStars2;
|
|
182
|
+
let indices = Array.from(range(stripBoundingStars2(segment.labels).length));
|
|
183
|
+
let unmapped = ctcViterbiAlign(
|
|
184
|
+
segment.emissions,
|
|
185
|
+
segment.labels,
|
|
186
|
+
blankId,
|
|
187
|
+
star,
|
|
188
|
+
STAR_SCORE
|
|
189
|
+
);
|
|
190
|
+
const dropped = dropLabels(
|
|
191
|
+
segment.labelsStart,
|
|
192
|
+
stripBoundingStars2(unmapped)
|
|
193
|
+
).map(([s, e]) => [
|
|
194
|
+
s - segment.labelsStart,
|
|
195
|
+
e - segment.labelsStart
|
|
196
|
+
]);
|
|
197
|
+
if (dropped.length) {
|
|
198
|
+
const { result: adjustedLabels, inputIndices: adjustedIndices } = removeLabelRanges(stripBoundingStars2(segment.labels), dropped);
|
|
199
|
+
indices = adjustedIndices;
|
|
200
|
+
if (segment === segments[0]) {
|
|
201
|
+
adjustedLabels.unshift(star);
|
|
202
|
+
}
|
|
203
|
+
if (segment === segments.at(-1)) {
|
|
204
|
+
adjustedLabels.push(star);
|
|
205
|
+
}
|
|
206
|
+
unmapped = ctcViterbiAlign(
|
|
207
|
+
segment.emissions,
|
|
208
|
+
adjustedLabels,
|
|
209
|
+
blankId,
|
|
210
|
+
star,
|
|
211
|
+
STAR_SCORE
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
const segmentAlignments = unmapped.map((a) => ({
|
|
215
|
+
...a,
|
|
216
|
+
startFrame: a.startFrame + segment.emissionsStart,
|
|
217
|
+
endFrame: a.endFrame + segment.emissionsStart
|
|
218
|
+
}));
|
|
219
|
+
inputIndices.push(...indices.map((i) => i + segment.labelsStart));
|
|
220
|
+
alignmentsWithRemovals.push(...segmentAlignments);
|
|
221
|
+
}
|
|
222
|
+
const alignments = [];
|
|
223
|
+
for (const [k, a] of enumerate(alignmentsWithRemovals.slice(1, -1))) {
|
|
224
|
+
const i = inputIndices[k];
|
|
225
|
+
const lastUndroppedIndex = alignments.length - 1;
|
|
226
|
+
const lastUndropped = alignments[lastUndroppedIndex] ?? {
|
|
227
|
+
startFrame: 0,
|
|
228
|
+
endFrame: 0,
|
|
229
|
+
score: -Infinity,
|
|
230
|
+
label: star
|
|
231
|
+
};
|
|
232
|
+
for (const j of range(lastUndroppedIndex, i - 1)) {
|
|
233
|
+
alignments.push({
|
|
234
|
+
startFrame: lastUndropped.endFrame,
|
|
235
|
+
endFrame: lastUndropped.endFrame,
|
|
236
|
+
score: -Infinity,
|
|
237
|
+
label: vocab.get(baseline[j])
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
alignments[i] = a;
|
|
241
|
+
}
|
|
242
|
+
return alignments;
|
|
243
|
+
}
|
|
244
|
+
export {
|
|
245
|
+
ctcForcedAlign,
|
|
246
|
+
ctcViterbiAlign,
|
|
247
|
+
detectDroppedLabels
|
|
248
|
+
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var greedyDecode_exports = {};
|
|
20
|
+
__export(greedyDecode_exports, {
|
|
21
|
+
argmax: () => argmax,
|
|
22
|
+
ctcGreedyDecode: () => ctcGreedyDecode
|
|
23
|
+
});
|
|
24
|
+
module.exports = __toCommonJS(greedyDecode_exports);
|
|
25
|
+
function argmax(vector) {
|
|
26
|
+
let best = 0;
|
|
27
|
+
for (let i = 1; i < vector.length; i++) {
|
|
28
|
+
if (vector[i] > vector[best]) best = i;
|
|
29
|
+
}
|
|
30
|
+
return best;
|
|
31
|
+
}
|
|
32
|
+
async function ctcGreedyDecode(emissionsReader) {
|
|
33
|
+
const charByToken = /* @__PURE__ */ new Map();
|
|
34
|
+
for (const [char, id] of emissionsReader.vocab.entries()) {
|
|
35
|
+
if (id !== emissionsReader.blankId && char.length === 1 && char !== "|") {
|
|
36
|
+
charByToken.set(id, char);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const chars = [];
|
|
40
|
+
const frames = [];
|
|
41
|
+
let previous = -1;
|
|
42
|
+
let t = 0;
|
|
43
|
+
for await (const row of emissionsReader.frames()) {
|
|
44
|
+
const best = argmax(row);
|
|
45
|
+
if (best !== previous) {
|
|
46
|
+
const char = charByToken.get(best);
|
|
47
|
+
if (char !== void 0) {
|
|
48
|
+
chars.push(char);
|
|
49
|
+
frames.push(t);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
previous = best;
|
|
53
|
+
t++;
|
|
54
|
+
}
|
|
55
|
+
return { text: chars.join(""), frames };
|
|
56
|
+
}
|
|
57
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
58
|
+
0 && (module.exports = {
|
|
59
|
+
argmax,
|
|
60
|
+
ctcGreedyDecode
|
|
61
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { EmissionsReader } from '../../emit/fs.cjs';
|
|
2
|
+
import '@storyteller-platform/ghost-story';
|
|
3
|
+
import './emissions.cjs';
|
|
4
|
+
|
|
5
|
+
declare function argmax(vector: number[]): number;
|
|
6
|
+
/**
|
|
7
|
+
* Returns the decoded text along with, for each character, the emission
|
|
8
|
+
* frame at which it was first observed.
|
|
9
|
+
*/
|
|
10
|
+
declare function ctcGreedyDecode(emissionsReader: EmissionsReader): Promise<{
|
|
11
|
+
text: string;
|
|
12
|
+
frames: number[];
|
|
13
|
+
}>;
|
|
14
|
+
|
|
15
|
+
export { argmax, ctcGreedyDecode };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { EmissionsReader } from '../../emit/fs.js';
|
|
2
|
+
import '@storyteller-platform/ghost-story';
|
|
3
|
+
import './emissions.js';
|
|
4
|
+
|
|
5
|
+
declare function argmax(vector: number[]): number;
|
|
6
|
+
/**
|
|
7
|
+
* Returns the decoded text along with, for each character, the emission
|
|
8
|
+
* frame at which it was first observed.
|
|
9
|
+
*/
|
|
10
|
+
declare function ctcGreedyDecode(emissionsReader: EmissionsReader): Promise<{
|
|
11
|
+
text: string;
|
|
12
|
+
frames: number[];
|
|
13
|
+
}>;
|
|
14
|
+
|
|
15
|
+
export { argmax, ctcGreedyDecode };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import "../../chunk-BIEQXUOY.js";
|
|
2
|
+
function argmax(vector) {
|
|
3
|
+
let best = 0;
|
|
4
|
+
for (let i = 1; i < vector.length; i++) {
|
|
5
|
+
if (vector[i] > vector[best]) best = i;
|
|
6
|
+
}
|
|
7
|
+
return best;
|
|
8
|
+
}
|
|
9
|
+
async function ctcGreedyDecode(emissionsReader) {
|
|
10
|
+
const charByToken = /* @__PURE__ */ new Map();
|
|
11
|
+
for (const [char, id] of emissionsReader.vocab.entries()) {
|
|
12
|
+
if (id !== emissionsReader.blankId && char.length === 1 && char !== "|") {
|
|
13
|
+
charByToken.set(id, char);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
const chars = [];
|
|
17
|
+
const frames = [];
|
|
18
|
+
let previous = -1;
|
|
19
|
+
let t = 0;
|
|
20
|
+
for await (const row of emissionsReader.frames()) {
|
|
21
|
+
const best = argmax(row);
|
|
22
|
+
if (best !== previous) {
|
|
23
|
+
const char = charByToken.get(best);
|
|
24
|
+
if (char !== void 0) {
|
|
25
|
+
chars.push(char);
|
|
26
|
+
frames.push(t);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
previous = best;
|
|
30
|
+
t++;
|
|
31
|
+
}
|
|
32
|
+
return { text: chars.join(""), frames };
|
|
33
|
+
}
|
|
34
|
+
export {
|
|
35
|
+
argmax,
|
|
36
|
+
ctcGreedyDecode
|
|
37
|
+
};
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var mediaOverlay_exports = {};
|
|
20
|
+
__export(mediaOverlay_exports, {
|
|
21
|
+
buildFragmentMaps: () => buildFragmentMaps,
|
|
22
|
+
createMediaOverlay: () => createMediaOverlay
|
|
23
|
+
});
|
|
24
|
+
module.exports = __toCommonJS(mediaOverlay_exports);
|
|
25
|
+
var import_posix = require("node:path/posix");
|
|
26
|
+
var import_itertools = require("itertools");
|
|
27
|
+
var import_epub = require("@storyteller-platform/epub");
|
|
28
|
+
var import_textFragments = require("../textFragments.cjs");
|
|
29
|
+
function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToBlockFragment, sentenceIdToFragment, wordIdToFragment) {
|
|
30
|
+
const subSequences = sentenceIdToBlockFragment ? createTextRangeLargeSequences(
|
|
31
|
+
chapter,
|
|
32
|
+
granularity,
|
|
33
|
+
sentenceRanges,
|
|
34
|
+
wordRanges,
|
|
35
|
+
sentenceIdToBlockFragment,
|
|
36
|
+
sentenceIdToFragment,
|
|
37
|
+
wordIdToFragment
|
|
38
|
+
) : createTextRangeSmallSequences(
|
|
39
|
+
chapter,
|
|
40
|
+
granularity,
|
|
41
|
+
sentenceRanges,
|
|
42
|
+
wordRanges,
|
|
43
|
+
sentenceIdToFragment,
|
|
44
|
+
wordIdToFragment
|
|
45
|
+
);
|
|
46
|
+
return [
|
|
47
|
+
import_epub.Epub.createXmlElement(
|
|
48
|
+
"smil",
|
|
49
|
+
{
|
|
50
|
+
xmlns: "http://www.w3.org/ns/SMIL",
|
|
51
|
+
"xmlns:epub": "http://www.idpf.org/2007/ops",
|
|
52
|
+
version: "3.0"
|
|
53
|
+
},
|
|
54
|
+
[
|
|
55
|
+
import_epub.Epub.createXmlElement("body", {}, [
|
|
56
|
+
import_epub.Epub.createXmlElement(
|
|
57
|
+
"seq",
|
|
58
|
+
{
|
|
59
|
+
id: `${chapter.id}_overlay`,
|
|
60
|
+
"epub:textref": `../${chapter.href}`,
|
|
61
|
+
"epub:type": "chapter"
|
|
62
|
+
},
|
|
63
|
+
subSequences
|
|
64
|
+
)
|
|
65
|
+
])
|
|
66
|
+
]
|
|
67
|
+
)
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
function createTextRangeLargeSequences(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToBlockFragment, sentenceIdToFragment, wordIdToFragment) {
|
|
71
|
+
const blockStarts = sentenceIdToBlockFragment.entries().toArray().toSorted(([a], [b]) => a - b);
|
|
72
|
+
return blockStarts.map(([sentenceId, fragment], index) => {
|
|
73
|
+
const blockEnd = index === blockStarts.length - 1 ? sentenceRanges.length - 1 : (
|
|
74
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
75
|
+
blockStarts[index + 1][0]
|
|
76
|
+
);
|
|
77
|
+
const sentences = sentenceRanges.slice(sentenceId, blockEnd);
|
|
78
|
+
return import_epub.Epub.createXmlElement(
|
|
79
|
+
"seq",
|
|
80
|
+
{
|
|
81
|
+
id: `${chapter.id}-b${index}`,
|
|
82
|
+
"epub:type": "text-range-large",
|
|
83
|
+
"epub:textref": `../${chapter.href}#${fragment}`
|
|
84
|
+
},
|
|
85
|
+
createTextRangeSmallSequences(
|
|
86
|
+
chapter,
|
|
87
|
+
granularity,
|
|
88
|
+
sentences,
|
|
89
|
+
wordRanges,
|
|
90
|
+
sentenceIdToFragment,
|
|
91
|
+
wordIdToFragment
|
|
92
|
+
)
|
|
93
|
+
);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
function createTextRangeSmallSequences(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToFragment, wordIdToFragment) {
|
|
97
|
+
return sentenceRanges.map((sentenceRange) => {
|
|
98
|
+
if (granularity === "sentence" || !wordRanges.has(sentenceRange.id)) {
|
|
99
|
+
return import_epub.Epub.createXmlElement(
|
|
100
|
+
"par",
|
|
101
|
+
{
|
|
102
|
+
id: `${chapter.id}-s${sentenceRange.id}`
|
|
103
|
+
},
|
|
104
|
+
[
|
|
105
|
+
import_epub.Epub.createXmlElement("text", {
|
|
106
|
+
src: `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
|
|
107
|
+
}),
|
|
108
|
+
import_epub.Epub.createXmlElement("audio", {
|
|
109
|
+
src: `../Audio/${(0, import_posix.basename)(sentenceRange.audiofile)}`,
|
|
110
|
+
clipBegin: `${sentenceRange.start.toFixed(3)}s`,
|
|
111
|
+
clipEnd: `${sentenceRange.end.toFixed(3)}s`
|
|
112
|
+
})
|
|
113
|
+
]
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
const words = wordRanges.get(sentenceRange.id);
|
|
117
|
+
const wordToFragment = wordIdToFragment.get(sentenceRange.id);
|
|
118
|
+
return import_epub.Epub.createXmlElement(
|
|
119
|
+
"seq",
|
|
120
|
+
{
|
|
121
|
+
id: `${chapter.id}-s${sentenceRange.id}`,
|
|
122
|
+
"epub:type": "text-range-small",
|
|
123
|
+
"epub:textref": `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
|
|
124
|
+
},
|
|
125
|
+
words.map(
|
|
126
|
+
(word) => import_epub.Epub.createXmlElement(
|
|
127
|
+
"par",
|
|
128
|
+
{
|
|
129
|
+
id: `${chapter.id}-s${sentenceRange.id}-w${word.id}`
|
|
130
|
+
},
|
|
131
|
+
[
|
|
132
|
+
import_epub.Epub.createXmlElement("text", {
|
|
133
|
+
src: `../${chapter.href}#${wordToFragment.get(word.id)}`
|
|
134
|
+
}),
|
|
135
|
+
import_epub.Epub.createXmlElement("audio", {
|
|
136
|
+
src: `../Audio/${(0, import_posix.basename)(word.audiofile)}`,
|
|
137
|
+
clipBegin: `${word.start.toFixed(3)}s`,
|
|
138
|
+
clipEnd: `${word.end.toFixed(3)}s`
|
|
139
|
+
})
|
|
140
|
+
]
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
function buildFragmentMaps(sentenceRanges, wordRanges, sentences, locale, textRef, granularity) {
|
|
147
|
+
let sentenceIdToBlockFragment = null;
|
|
148
|
+
const sentenceIdToFragment = new Map(
|
|
149
|
+
sentenceRanges.map((range) => [
|
|
150
|
+
range.id,
|
|
151
|
+
`${range.chapterId}-s${range.id}`
|
|
152
|
+
])
|
|
153
|
+
);
|
|
154
|
+
const wordIdToFragment = new Map(
|
|
155
|
+
wordRanges.map((ranges) => [
|
|
156
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
157
|
+
ranges[0].sentenceId,
|
|
158
|
+
new Map(
|
|
159
|
+
ranges.map((range) => [
|
|
160
|
+
range.id,
|
|
161
|
+
`${range.chapterId}-s${range.sentenceId}-w${range.id}`
|
|
162
|
+
])
|
|
163
|
+
)
|
|
164
|
+
])
|
|
165
|
+
);
|
|
166
|
+
const wordRangeMap = new Map(wordRanges.map((w) => [w[0].sentenceId, w]));
|
|
167
|
+
if (textRef === "text-fragment") {
|
|
168
|
+
sentenceIdToBlockFragment = /* @__PURE__ */ new Map();
|
|
169
|
+
const blocks = [[]];
|
|
170
|
+
for (const [i, sentence] of (0, import_itertools.enumerate)(sentences)) {
|
|
171
|
+
const text = sentence.text;
|
|
172
|
+
blocks.at(-1)?.push(text);
|
|
173
|
+
if (text.includes("\n") && i < sentences.length - 1) {
|
|
174
|
+
blocks.push([]);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const blockFactory = new import_textFragments.TextFragmentFactory(
|
|
178
|
+
blocks.map((block) => block.join("")),
|
|
179
|
+
locale
|
|
180
|
+
);
|
|
181
|
+
let sentenceRangeIndex = 0;
|
|
182
|
+
for (const [i, block] of (0, import_itertools.enumerate)(blocks)) {
|
|
183
|
+
sentenceIdToBlockFragment.set(
|
|
184
|
+
sentenceRangeIndex,
|
|
185
|
+
blockFactory.findMinimalFragment(i)
|
|
186
|
+
);
|
|
187
|
+
const sentenceFactory = new import_textFragments.TextFragmentFactory(
|
|
188
|
+
block.map((s) => s.replace("\n", "")),
|
|
189
|
+
locale
|
|
190
|
+
);
|
|
191
|
+
const blockRanges = sentenceRanges.slice(
|
|
192
|
+
sentenceRangeIndex,
|
|
193
|
+
sentenceRangeIndex + block.length
|
|
194
|
+
);
|
|
195
|
+
for (const [j, range] of (0, import_itertools.enumerate)(blockRanges)) {
|
|
196
|
+
sentenceIdToFragment.set(
|
|
197
|
+
range.id,
|
|
198
|
+
sentenceFactory.findMinimalFragment(j)
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
if (granularity === "word") {
|
|
202
|
+
const allWords = [];
|
|
203
|
+
for (const range of blockRanges) {
|
|
204
|
+
const sentence = sentences[range.id];
|
|
205
|
+
const words = [];
|
|
206
|
+
for (const w of sentence.words.entries) {
|
|
207
|
+
if (w.isPunctuation) {
|
|
208
|
+
const lastWord = words.at(-1);
|
|
209
|
+
if (lastWord === void 0) {
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
words[words.length - 1] = lastWord + w.text.replace("\n", "");
|
|
213
|
+
} else {
|
|
214
|
+
words.push(w.text);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
allWords.push(...words);
|
|
218
|
+
}
|
|
219
|
+
const wordFactory = new import_textFragments.TextFragmentFactory(allWords);
|
|
220
|
+
let wordRangeIndex = 0;
|
|
221
|
+
for (const range of blockRanges) {
|
|
222
|
+
const ranges = wordRangeMap.get(range.id);
|
|
223
|
+
const toFragment = wordIdToFragment.get(range.id);
|
|
224
|
+
if (!ranges || !toFragment) continue;
|
|
225
|
+
for (const [k, wordRange] of (0, import_itertools.enumerate)(ranges)) {
|
|
226
|
+
toFragment.set(
|
|
227
|
+
wordRange.id,
|
|
228
|
+
wordFactory.findMinimalFragment(k + wordRangeIndex)
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
wordRangeIndex += ranges.length;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
sentenceRangeIndex += block.length;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
sentenceIdToBlockFragment,
|
|
239
|
+
sentenceIdToFragment,
|
|
240
|
+
wordIdToFragment,
|
|
241
|
+
wordRangeMap
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
245
|
+
0 && (module.exports = {
|
|
246
|
+
buildFragmentMaps,
|
|
247
|
+
createMediaOverlay
|
|
248
|
+
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import * as _storyteller_platform_epub from '@storyteller-platform/epub';
|
|
2
|
+
import { ManifestItem } from '@storyteller-platform/epub';
|
|
3
|
+
import { SegmentationResult } from '@echogarden/text-segmentation';
|
|
4
|
+
import { SentenceRange, WordRange } from '../getSentenceRanges.cjs';
|
|
5
|
+
import '@storyteller-platform/ghost-story';
|
|
6
|
+
import '@storyteller-platform/mapping';
|
|
7
|
+
|
|
8
|
+
declare function createMediaOverlay(chapter: ManifestItem, granularity: "sentence" | "word", sentenceRanges: SentenceRange[], wordRanges: Map<number, WordRange[]>, sentenceIdToBlockFragment: Map<number, string> | null, sentenceIdToFragment: Map<number, string>, wordIdToFragment: Map<number, Map<number, string>>): _storyteller_platform_epub.XmlElement<"smil">[];
|
|
9
|
+
declare function buildFragmentMaps(sentenceRanges: SentenceRange[], wordRanges: WordRange[][], sentences: SegmentationResult["sentences"], locale: Intl.Locale, textRef: "id-fragment" | "text-fragment", granularity: "sentence" | "word"): {
|
|
10
|
+
sentenceIdToBlockFragment: Map<number, string> | null;
|
|
11
|
+
sentenceIdToFragment: Map<number, string>;
|
|
12
|
+
wordIdToFragment: Map<number, Map<number, string>>;
|
|
13
|
+
wordRangeMap: Map<number, WordRange[]>;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export { buildFragmentMaps, createMediaOverlay };
|