@makefully/dutifully 1.19.0 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +75 -3
- package/package.json +4 -2
- package/src/classes/LipSync.js +110 -24
- package/src/classes/Packfully.js +38 -8
- package/src/classes/Transcription.js +106 -52
- package/src/classes/Translate.js +47 -0
- package/src/helpers/jobExports.js +131 -0
- package/src/index.js +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 1.21.0
|
|
6
|
+
|
|
7
|
+
- Add `jobExports` mirror for multi-format diff checks in LipSync and Transcription.
|
|
8
|
+
- Packfully difference checks honor `exports: null` → all atlas formats.
|
|
9
|
+
- Document `exports` string | array | null semantics for lipsync and transcription jobs.
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
- Add `translate` client for Amazon Translate (Cheerfully script JSON or language-matrix CSV/TSV).
|
|
13
|
+
- Support `ignore` for phrases left untranslated.
|
|
14
|
+
|
|
5
15
|
## 1.19.0
|
|
6
16
|
|
|
7
17
|
- Authenticate Showfully yap chores with `Authorization: Bearer` and `POST /yap/:service` (token no longer in the URL path).
|
package/README.md
CHANGED
|
@@ -288,10 +288,31 @@ This audio-to-lip-flap service requires audio as input and will download a mouth
|
|
|
288
288
|
}
|
|
289
289
|
```
|
|
290
290
|
|
|
291
|
-
Export formats include those supported by Rhubarb as well as
|
|
291
|
+
Export formats include those supported by Rhubarb as well as `mp3`, which embeds lipsync timings into the MP3's ID3 metadata.
|
|
292
|
+
|
|
293
|
+
Set `exportFormat`, `exports`, or omit/`null` for export selection:
|
|
294
|
+
|
|
295
|
+
* **string** — one format (for example `"json"` or `"mp3"`)
|
|
296
|
+
* **array** — multiple formats in one run (for example `"exports": ["json", "mp3"]`)
|
|
297
|
+
* **`null` / omit** — all supported formats (difference checks require every format to be current)
|
|
298
|
+
|
|
299
|
+
MP3 export does **not** include `mouthCues.json` unless `json` is also requested.
|
|
292
300
|
|
|
293
301
|
For exporting to JSON, you can set `compress` to `true` for the format to use JSON arrays similar to Rhubarb's shorter `tsv` format.
|
|
294
302
|
|
|
303
|
+
```javascript
|
|
304
|
+
{
|
|
305
|
+
"rhubarb": [{
|
|
306
|
+
"exports": ["json", "mp3"],
|
|
307
|
+
"options": {
|
|
308
|
+
"compress": true
|
|
309
|
+
},
|
|
310
|
+
"src": "./example/vo/en/",
|
|
311
|
+
"output": "./example/lipsync/en/"
|
|
312
|
+
}]
|
|
313
|
+
}
|
|
314
|
+
```
|
|
315
|
+
|
|
295
316
|
### Rhubarb
|
|
296
317
|
|
|
297
318
|
This audio-to-lip-flap service requires audio as input and will download a mouth cues JSON file delineating mouth shapes for particular timestamps using Rhubarb Lip-Sync. You can set Rhubarb settings in `cheerfully.json` like so:
|
|
@@ -309,7 +330,9 @@ This audio-to-lip-flap service requires audio as input and will download a mouth
|
|
|
309
330
|
}
|
|
310
331
|
```
|
|
311
332
|
|
|
312
|
-
Export formats include those supported by Rhubarb as well as
|
|
333
|
+
Export formats include those supported by Rhubarb as well as `mp3`, which embeds lipsync timings into the MP3's ID3 metadata.
|
|
334
|
+
|
|
335
|
+
Use `exportFormat`, `exports`, or omit/`null` with the same string | array | all semantics as Rhubarb (see Allosaurus above).
|
|
313
336
|
|
|
314
337
|
For exporting to JSON, you can set `compress` to `true` for the format to use JSON arrays similar to Rhubarb's shorter `tsv` format.
|
|
315
338
|
|
|
@@ -341,6 +364,24 @@ Transcription supports the following formats:
|
|
|
341
364
|
* SRT
|
|
342
365
|
* VTT
|
|
343
366
|
|
|
367
|
+
Use `format`, `exports`, or omit/`null`:
|
|
368
|
+
|
|
369
|
+
* **string** — one format (for example `"vtt"`)
|
|
370
|
+
* **array** — multiple formats (for example `"exports": ["json", "vtt", "srt"]`)
|
|
371
|
+
* **`null` / omit on `exports`** — all supported formats in one Cheerfully run
|
|
372
|
+
|
|
373
|
+
```javascript
|
|
374
|
+
{
|
|
375
|
+
"transcription": [{
|
|
376
|
+
"exports": ["json", "vtt", "srt"],
|
|
377
|
+
"language": "en-US",
|
|
378
|
+
"script": "./example/script.json",
|
|
379
|
+
"src": "./example/vo/en/",
|
|
380
|
+
"output": "./example/captions/en/"
|
|
381
|
+
}]
|
|
382
|
+
}
|
|
383
|
+
```
|
|
384
|
+
|
|
344
385
|
Specify `limit` to set the maximum number of characters for a single caption line. The default is `40`.
|
|
345
386
|
|
|
346
387
|
If your scripts include any phrases that should remain unbroken by the captioning process, add them to the `nowrap` list. For example: `"nowrap": ["Makefully Studios"]`.
|
|
@@ -355,9 +396,40 @@ Script lines may also include an optional `events` array for one-shot timed mark
|
|
|
355
396
|
|
|
356
397
|
Cheerfully resolves these against caption timings and exports them in parallel using the same caption `format` (`events.json`, `{id}.events.vtt` / `.srt` / `.sami` / `.smi` / `.lrc`, or a separate MP3 SYLT frame with content type `EVENTS` labeled `"events"`).
|
|
357
398
|
|
|
399
|
+
### Translate
|
|
400
|
+
|
|
401
|
+
This text-to-text service translates Cheerfully scripts or language-matrix CSV/TSV via Amazon Translate. Set jobs in `cheerfully.json`:
|
|
402
|
+
|
|
403
|
+
```javascript
|
|
404
|
+
{
|
|
405
|
+
"translate": [{
|
|
406
|
+
"script": "./example/script.json",
|
|
407
|
+
"language": "en-US",
|
|
408
|
+
"languages": ["es-ES", "fr-FR"],
|
|
409
|
+
"output": "./example/scripts/",
|
|
410
|
+
"ignore": ["Makefully Studios"]
|
|
411
|
+
}, {
|
|
412
|
+
"script": "./example/strings.csv",
|
|
413
|
+
"language": "en-US",
|
|
414
|
+
"languages": ["es-ES", "fr-FR"],
|
|
415
|
+
"output": "./example/scripts/"
|
|
416
|
+
}]
|
|
417
|
+
}
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
* `script` — path to a Cheerfully `.json` script or a language-matrix `.csv` / `.tsv` (first column = ids, first row = language tags).
|
|
421
|
+
* `language` — source locale (CSV: source column). Matrix jobs require this; JSON may use `auto`.
|
|
422
|
+
* `languages` — target locales to produce or fill.
|
|
423
|
+
* `format` / `exports` — optional. Defaults to the source extension (`json`, `csv`, or `tsv`).
|
|
424
|
+
* `ignore` — phrases left untranslated.
|
|
425
|
+
|
|
426
|
+
JSON sources write `{lang}/{basename}.{ext}` under `output`. CSV/TSV sources write an updated matrix file and leave existing non-empty cells unchanged.
|
|
427
|
+
|
|
428
|
+
Supported formats: `json`, `i18n`, `csv`, `tsv`, `po`, `xliff`, `properties`, `yml` / `yaml`, `strings`, `xml`.
|
|
429
|
+
|
|
358
430
|
## Example
|
|
359
431
|
|
|
360
|
-
Find an example script in the `./examples/` folder. You can run either `npm run test-elevenlabs` or `npm run test-polly` to create VO tracks in the examples folder that you can then use to try out `npm run test-rhubarb` and `npm run test-transcription`. Sample sprites for atlas packing are under `./example/sprites/`; run `npm run test-packfully` to pack them into `./example/atlases/`.
|
|
432
|
+
Find an example script in the `./examples/` folder. You can run either `npm run test-elevenlabs` or `npm run test-polly` to create VO tracks in the examples folder that you can then use to try out `npm run test-rhubarb` and `npm run test-transcription`. Sample sprites for atlas packing are under `./example/sprites/`; run `npm run test-packfully` to pack them into `./example/atlases/`. Run `npm run test-translate` to translate `./example/script.json` and `./example/strings.csv` into `./example/scripts/`.
|
|
361
433
|
|
|
362
434
|
These examples are specified using a JSON definition in `./cheerfully.json` which you can copy or modify to set different generation parameters as supported by Cheerfully.
|
|
363
435
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@makefully/dutifully",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.21.0",
|
|
4
4
|
"description": "Client tools for Cheerfully service.",
|
|
5
5
|
"main": "./src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"test-sharp": "node ./cheer.js -service sharp",
|
|
29
29
|
"test-stackfully": "node ./cheer.js -service stackfully",
|
|
30
30
|
"test-transcription": "node ./cheer.js -service transcription",
|
|
31
|
+
"test-translate": "node ./cheer.js -service translate",
|
|
31
32
|
"say": "node ./say.js"
|
|
32
33
|
},
|
|
33
34
|
"repository": {
|
|
@@ -50,7 +51,8 @@
|
|
|
50
51
|
"packfully",
|
|
51
52
|
"atlas",
|
|
52
53
|
"spritesheet",
|
|
53
|
-
"classfully"
|
|
54
|
+
"classfully",
|
|
55
|
+
"translate"
|
|
54
56
|
],
|
|
55
57
|
"author": "Derek Detweiler <derek@makefullystudios.com>",
|
|
56
58
|
"contributors": [
|
package/src/classes/LipSync.js
CHANGED
|
@@ -2,6 +2,7 @@ const
|
|
|
2
2
|
Cheer = require('./Cheer'),
|
|
3
3
|
fs = require('fs').promises,
|
|
4
4
|
getJSON = require('../helpers/getJSON'),
|
|
5
|
+
{resolveJobExports} = require('../helpers/jobExports'),
|
|
5
6
|
{ensureDir, ensureParentDir, readdirOrEmpty} = require('../helpers/dirs'),
|
|
6
7
|
id3 = require('node-id3').Promise,
|
|
7
8
|
filter = (fileType, file) => (file.indexOf('.') !== 0) && (file.slice(-(fileType.length + 1)) === `.${fileType}`),
|
|
@@ -14,7 +15,7 @@ const
|
|
|
14
15
|
const
|
|
15
16
|
file = files[i];
|
|
16
17
|
|
|
17
|
-
if (fileType === 'mp3') {
|
|
18
|
+
if (fileType === 'mp3') {
|
|
18
19
|
const
|
|
19
20
|
{synchronisedLyrics} = await id3.read(`${path}${file}`);
|
|
20
21
|
|
|
@@ -54,9 +55,13 @@ const
|
|
|
54
55
|
|
|
55
56
|
async checkDifference () {
|
|
56
57
|
const
|
|
57
|
-
{config} = this,
|
|
58
|
-
{files = {}, options = {}, output, src} = config,
|
|
59
|
-
|
|
58
|
+
{config, service} = this,
|
|
59
|
+
{files = {}, options = {}, output, src, exports: configExports} = config,
|
|
60
|
+
exportFormats = resolveJobExports({
|
|
61
|
+
exports: configExports,
|
|
62
|
+
options
|
|
63
|
+
}, service, {defaultFormats: ['json']}),
|
|
64
|
+
usesJsonAggregate = exportFormats.includes('json'),
|
|
60
65
|
file = 'mouthCues.json';
|
|
61
66
|
|
|
62
67
|
if (output) {
|
|
@@ -64,47 +69,128 @@ const
|
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
const
|
|
67
|
-
raw =
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
+
raw = usesJsonAggregate ? await getJSON(`${output}${file}`) ?? {} : null,
|
|
73
|
+
formatLists = {},
|
|
74
|
+
loadFormatList = async (formatId) => {
|
|
75
|
+
if (formatId === 'json') {
|
|
76
|
+
return raw ? Object.keys(raw) : [];
|
|
77
|
+
}
|
|
72
78
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
formatLists[formatId] = formatLists[formatId] ?? await combine(output, formatId);
|
|
80
|
+
return formatLists[formatId];
|
|
81
|
+
},
|
|
82
|
+
markDone = (id) => {
|
|
83
|
+
if (raw) {
|
|
84
|
+
delete raw[id];
|
|
78
85
|
}
|
|
86
|
+
|
|
87
|
+
exportFormats.forEach((formatId) => {
|
|
88
|
+
if (formatId === 'json') {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const
|
|
93
|
+
list = formatLists[formatId];
|
|
94
|
+
|
|
95
|
+
if (!list) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const
|
|
100
|
+
index = list.indexOf(id);
|
|
101
|
+
|
|
102
|
+
if (index >= 0) {
|
|
103
|
+
list.splice(index, 1);
|
|
104
|
+
}
|
|
105
|
+
});
|
|
79
106
|
},
|
|
80
|
-
|
|
81
|
-
for (let i = 0; i <
|
|
107
|
+
isCurrent = async (id) => {
|
|
108
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
82
109
|
const
|
|
83
|
-
|
|
110
|
+
formatId = exportFormats[i],
|
|
111
|
+
list = await loadFormatList(formatId);
|
|
112
|
+
|
|
113
|
+
if (list.indexOf(id) < 0) {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
markDone(id);
|
|
119
|
+
return true;
|
|
120
|
+
},
|
|
121
|
+
mergeLipSync = async (newLipSync = false) => {
|
|
122
|
+
const
|
|
123
|
+
staleIds = new Set();
|
|
124
|
+
|
|
125
|
+
if (raw) {
|
|
126
|
+
Object.keys(raw).forEach((key) => staleIds.add(key));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
exportFormats.forEach((formatId) => {
|
|
130
|
+
if (formatId === 'json') {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
(formatLists[formatId] || []).forEach((key) => staleIds.add(key));
|
|
135
|
+
});
|
|
84
136
|
|
|
137
|
+
for (const key of staleIds) {
|
|
85
138
|
if (raw) {
|
|
86
139
|
delete raw[key];
|
|
87
|
-
} else {
|
|
88
|
-
await fs.rm(`${output}${key}.${exportFormat}`);
|
|
89
140
|
}
|
|
141
|
+
|
|
142
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
143
|
+
const
|
|
144
|
+
formatId = exportFormats[i];
|
|
145
|
+
|
|
146
|
+
if (formatId !== 'json') {
|
|
147
|
+
await fs.rm(`${output}${key}.${formatId}`, {force: true});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
90
151
|
console.log(`Removed "${key}"`);
|
|
91
152
|
}
|
|
153
|
+
|
|
92
154
|
if (raw) {
|
|
93
155
|
await ensureParentDir(`${output}${file}`);
|
|
94
156
|
await fs.writeFile(`${output}${file}`, JSON.stringify(sortKeys({
|
|
95
157
|
...raw,
|
|
96
|
-
...newLipSync ? await getJSON(`${output}${file}`) ?? {} : {}
|
|
158
|
+
...newLipSync ? await getJSON(`${output}${file}`) ?? {} : {}
|
|
97
159
|
}), null, 4));
|
|
98
160
|
}
|
|
99
161
|
|
|
100
|
-
return
|
|
162
|
+
return staleIds.size;
|
|
101
163
|
},
|
|
102
|
-
list =
|
|
164
|
+
list = [];
|
|
165
|
+
|
|
166
|
+
for (const fileName of (await readdirOrEmpty(src)).filter(filter.bind(null, 'mp3'))) {
|
|
167
|
+
const
|
|
168
|
+
id = fileName.substring(0, fileName.length - 4);
|
|
169
|
+
|
|
170
|
+
if (!(await isCurrent(id))) {
|
|
171
|
+
list.push(fileName);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (exportFormats.length === 1) {
|
|
176
|
+
config.options = {
|
|
177
|
+
...options,
|
|
178
|
+
exportFormat: exportFormats[0]
|
|
179
|
+
};
|
|
180
|
+
delete config.exports;
|
|
181
|
+
} else {
|
|
182
|
+
config.exports = exportFormats;
|
|
183
|
+
config.options = {
|
|
184
|
+
...options,
|
|
185
|
+
exportFormat: undefined,
|
|
186
|
+
f: undefined
|
|
187
|
+
};
|
|
188
|
+
}
|
|
103
189
|
|
|
104
190
|
this.differenceOnly = true;
|
|
191
|
+
this.exportFormats = exportFormats;
|
|
105
192
|
|
|
106
193
|
if (list.length === 0) {
|
|
107
|
-
// We'll still run the merge in case any have been removed.
|
|
108
194
|
if (await mergeLipSync()) {
|
|
109
195
|
throw Error('Old lip-sync removed, but no new ones required generation.');
|
|
110
196
|
} else {
|
|
@@ -138,4 +224,4 @@ const
|
|
|
138
224
|
}
|
|
139
225
|
};
|
|
140
226
|
|
|
141
|
-
module.exports = LipSync;
|
|
227
|
+
module.exports = LipSync;
|
package/src/classes/Packfully.js
CHANGED
|
@@ -10,18 +10,48 @@ const
|
|
|
10
10
|
listImageFiles
|
|
11
11
|
} = require('../helpers/imageHash'),
|
|
12
12
|
{ensureDir, readdirOrEmpty} = require('../helpers/dirs'),
|
|
13
|
+
{resolveJobExports} = require('../helpers/jobExports'),
|
|
13
14
|
HASH_KEY = HASH_KEYS.packfully,
|
|
14
|
-
|
|
15
|
+
packfullyAlias = (format) => {
|
|
15
16
|
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
id = String(format || 'texturepacker').toLowerCase();
|
|
18
|
+
|
|
19
|
+
switch (id) {
|
|
20
|
+
case 'texturepacker':
|
|
21
|
+
case 'json-hash':
|
|
22
|
+
case 'jsonhash':
|
|
23
|
+
case 'pixi':
|
|
24
|
+
case 'phaser-hash':
|
|
25
|
+
case 'phaserhash':
|
|
26
|
+
return 'texturepacker';
|
|
27
|
+
case 'json-array':
|
|
28
|
+
case 'jsonarray':
|
|
29
|
+
case 'phaser-array':
|
|
30
|
+
case 'phaserarray':
|
|
31
|
+
return 'json-array';
|
|
32
|
+
case 'createjs':
|
|
33
|
+
case 'easeljs':
|
|
34
|
+
return 'createjs';
|
|
35
|
+
case 'spine':
|
|
36
|
+
case 'atlas':
|
|
37
|
+
return 'spine';
|
|
38
|
+
default:
|
|
39
|
+
return id;
|
|
20
40
|
}
|
|
41
|
+
},
|
|
42
|
+
listFormats = (job = {}) => {
|
|
21
43
|
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
44
|
+
raw = job.exports ?? job.format ?? job.options?.format,
|
|
45
|
+
normalizedJob = raw === undefined
|
|
46
|
+
? job
|
|
47
|
+
: {
|
|
48
|
+
...job,
|
|
49
|
+
exports: (Array.isArray(raw) ? raw : [raw]).map(packfullyAlias),
|
|
50
|
+
format: undefined
|
|
51
|
+
},
|
|
52
|
+
list = resolveJobExports(normalizedJob, 'packfully', {defaultFormats: ['texturepacker']});
|
|
53
|
+
|
|
54
|
+
return [...new Set(list.map(packfullyAlias))];
|
|
25
55
|
},
|
|
26
56
|
Packfully = class Packfully extends Cheer {
|
|
27
57
|
async checkDifference () {
|
|
@@ -5,6 +5,7 @@ const
|
|
|
5
5
|
getCaptions = require('../helpers/getCaptions'),
|
|
6
6
|
getFileData = require('../helpers/getFileData'),
|
|
7
7
|
getJSON = require('../helpers/getJSON'),
|
|
8
|
+
{resolveJobExports} = require('../helpers/jobExports'),
|
|
8
9
|
{ensureDir, readdirOrEmpty} = require('../helpers/dirs'),
|
|
9
10
|
SEPARATE_CHARACTER = '|',
|
|
10
11
|
JOIN_CHARACTER = '^',
|
|
@@ -22,7 +23,6 @@ const
|
|
|
22
23
|
this.updateList = null;
|
|
23
24
|
await this.replaceConfigPathWithJSON('script', 'files');
|
|
24
25
|
|
|
25
|
-
// create standardized transcripts and an accompanying hash for each.
|
|
26
26
|
const
|
|
27
27
|
{files, voice} = this.config,
|
|
28
28
|
standardizedFiles = {},
|
|
@@ -44,45 +44,68 @@ const
|
|
|
44
44
|
async checkDifference () {
|
|
45
45
|
const
|
|
46
46
|
{config} = this,
|
|
47
|
-
{files, format = 'json', output, src} = config,
|
|
48
|
-
|
|
47
|
+
{files, format = 'json', exports, output, src} = config,
|
|
48
|
+
exportFormats = resolveJobExports({format, exports}, 'transcription', {
|
|
49
|
+
defaultFormats: [format || 'json']
|
|
50
|
+
}),
|
|
51
|
+
formatState = {};
|
|
49
52
|
|
|
50
53
|
if (output) {
|
|
51
54
|
await ensureDir(output);
|
|
52
55
|
}
|
|
53
56
|
|
|
57
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
58
|
+
const
|
|
59
|
+
fileType = exportFormats[i];
|
|
60
|
+
|
|
61
|
+
formatState[fileType] = await getCaptions(output, fileType);
|
|
62
|
+
}
|
|
63
|
+
|
|
54
64
|
const
|
|
55
|
-
|
|
56
|
-
eventsJsonExisting =
|
|
65
|
+
jsonState = formatState.json,
|
|
66
|
+
eventsJsonExisting = exportFormats.includes('json') ? await getJSON(`${output}events.json`) : null,
|
|
57
67
|
eventsFile = eventsJsonExisting ? await getFileData(`${output}events.json`, 'json') : null,
|
|
58
|
-
|
|
68
|
+
staleByFormat = exportFormats.reduce((obj, fileType) => {
|
|
69
|
+
obj[fileType] = {...(formatState[fileType]?.captions ?? {})};
|
|
70
|
+
return obj;
|
|
71
|
+
}, {}),
|
|
72
|
+
checkFormat = (fileType, id, caption) => {
|
|
59
73
|
const
|
|
60
|
-
|
|
61
|
-
|
|
74
|
+
state = formatState[fileType],
|
|
75
|
+
file = state?.files?.[id],
|
|
76
|
+
original = state?.captions?.[id],
|
|
62
77
|
hash = file?.getHash();
|
|
63
|
-
|
|
64
|
-
delete
|
|
65
|
-
|
|
78
|
+
|
|
79
|
+
delete staleByFormat[fileType][id];
|
|
80
|
+
|
|
66
81
|
if (hash) {
|
|
67
82
|
const
|
|
68
83
|
same = hash === this.transcripts[id].getHash();
|
|
69
84
|
|
|
70
85
|
if (!same) {
|
|
71
|
-
console.log(`Updating "${id}".`);
|
|
86
|
+
console.log(`Updating "${id}" (${fileType}).`);
|
|
72
87
|
}
|
|
73
88
|
|
|
74
89
|
return same;
|
|
75
|
-
}
|
|
76
|
-
const
|
|
77
|
-
cap = (Array.isArray(caption) ? caption : [caption]).map((cap) => cap?.caption ?? cap ?? null).filter((cap) => cap !== null).join(' ').replaceAll(JOIN_CHARACTER, ' ').replaceAll(SEPARATE_CHARACTER, ' ');
|
|
78
|
-
|
|
79
|
-
if (cap && cap !== original) {
|
|
80
|
-
console.log(`Updating "${id}".`);
|
|
81
|
-
}
|
|
90
|
+
}
|
|
82
91
|
|
|
83
|
-
|
|
84
|
-
|
|
92
|
+
const
|
|
93
|
+
cap = (Array.isArray(caption) ? caption : [caption]).map((cap) => cap?.caption ?? cap ?? null).filter((cap) => cap !== null).join(' ').replaceAll(JOIN_CHARACTER, ' ').replaceAll(SEPARATE_CHARACTER, ' ');
|
|
94
|
+
|
|
95
|
+
if (cap && cap !== original) {
|
|
96
|
+
console.log(`Updating "${id}" (${fileType}).`);
|
|
85
97
|
}
|
|
98
|
+
|
|
99
|
+
return cap === '' || cap === original;
|
|
100
|
+
},
|
|
101
|
+
check = (id, caption) => {
|
|
102
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
103
|
+
if (!checkFormat(exportFormats[i], id, caption)) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return true;
|
|
86
109
|
},
|
|
87
110
|
saveEventsFile = async () => {
|
|
88
111
|
if (!eventsFile) {
|
|
@@ -95,30 +118,40 @@ const
|
|
|
95
118
|
await eventsFile.save();
|
|
96
119
|
}
|
|
97
120
|
},
|
|
98
|
-
mergeCaptions = async (newCaptions = false) => {
|
|
121
|
+
mergeCaptions = async (newCaptions = false) => {
|
|
99
122
|
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
123
|
+
staleIds = new Set();
|
|
124
|
+
|
|
125
|
+
exportFormats.forEach((fileType) => {
|
|
126
|
+
Object.keys(staleByFormat[fileType] || {}).forEach((id) => staleIds.add(id));
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
for (const caption of staleIds) {
|
|
103
130
|
const
|
|
104
|
-
|
|
105
|
-
|
|
131
|
+
captionsFile = jsonState?.file;
|
|
132
|
+
|
|
106
133
|
if (captionsFile) {
|
|
107
134
|
captionsFile.removeKey(caption);
|
|
108
135
|
eventsFile?.removeKey(caption);
|
|
109
136
|
} else {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
137
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
138
|
+
const
|
|
139
|
+
fileType = exportFormats[i];
|
|
140
|
+
|
|
141
|
+
if (fileType !== 'json') {
|
|
142
|
+
await fs.rm(`${output}${caption}.${fileType}`, {force: true});
|
|
143
|
+
await fs.rm(`${output}${caption}.events.${fileType}`, {force: true});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
113
146
|
}
|
|
147
|
+
|
|
114
148
|
console.log(`Removed "${caption}"`);
|
|
115
149
|
}
|
|
116
|
-
|
|
150
|
+
|
|
151
|
+
if (jsonState?.file) {
|
|
117
152
|
if (newCaptions) {
|
|
118
|
-
|
|
153
|
+
jsonState.file.mergeData(await getJSON(`${output}captions.json`));
|
|
119
154
|
|
|
120
|
-
// Drop events for every regenerated caption, then merge whatever the server returned.
|
|
121
|
-
// Captions that no longer export events stay removed (no orphan keys).
|
|
122
155
|
const
|
|
123
156
|
updated = this.updateList ?? [],
|
|
124
157
|
newEvents = await getJSON(`${output}events.json`) ?? {};
|
|
@@ -136,14 +169,14 @@ const
|
|
|
136
169
|
} else if (Object.keys(newEvents).length === 0) {
|
|
137
170
|
await fs.rm(`${output}events.json`, {force: true});
|
|
138
171
|
}
|
|
139
|
-
|
|
140
|
-
} else if (olds.length) {
|
|
172
|
+
} else if (staleIds.size) {
|
|
141
173
|
await saveEventsFile();
|
|
142
174
|
}
|
|
143
|
-
|
|
175
|
+
|
|
176
|
+
await jsonState.file.save();
|
|
144
177
|
}
|
|
145
|
-
|
|
146
|
-
return
|
|
178
|
+
|
|
179
|
+
return staleIds.size;
|
|
147
180
|
},
|
|
148
181
|
list = (await readdirOrEmpty(src)).filter(filter.bind(null, 'mp3')).filter((file) => {
|
|
149
182
|
const
|
|
@@ -153,10 +186,18 @@ const
|
|
|
153
186
|
return !check(id, caption);
|
|
154
187
|
});
|
|
155
188
|
|
|
189
|
+
if (exportFormats.length === 1) {
|
|
190
|
+
config.format = exportFormats[0];
|
|
191
|
+
delete config.exports;
|
|
192
|
+
} else {
|
|
193
|
+
config.exports = exportFormats;
|
|
194
|
+
delete config.format;
|
|
195
|
+
}
|
|
196
|
+
|
|
156
197
|
this.differenceOnly = true;
|
|
198
|
+
this.exportFormats = exportFormats;
|
|
157
199
|
|
|
158
200
|
if (list.length === 0) {
|
|
159
|
-
// We'll still run the merge in case any have been removed.
|
|
160
201
|
if (await mergeCaptions()) {
|
|
161
202
|
throw Error('Old captions removed, but no new captions required generation.');
|
|
162
203
|
} else {
|
|
@@ -168,13 +209,17 @@ const
|
|
|
168
209
|
config.files = mapReduction(files, list);
|
|
169
210
|
this.mergeCaptions = mergeCaptions;
|
|
170
211
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
212
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
213
|
+
const
|
|
214
|
+
fileType = exportFormats[i];
|
|
215
|
+
|
|
216
|
+
if (fileType !== 'json' && fileType !== 'mp3') {
|
|
217
|
+
for (let j = 0; j < list.length; j++) {
|
|
218
|
+
const
|
|
219
|
+
id = list[j].substring(0, list[j].length - 4);
|
|
176
220
|
|
|
177
|
-
|
|
221
|
+
await fs.rm(`${output}${id}.events.${fileType}`, {force: true});
|
|
222
|
+
}
|
|
178
223
|
}
|
|
179
224
|
}
|
|
180
225
|
}
|
|
@@ -195,23 +240,32 @@ const
|
|
|
195
240
|
|
|
196
241
|
async afterExport (...args) {
|
|
197
242
|
const
|
|
198
|
-
{config, transcripts, updateList} = this,
|
|
199
|
-
{
|
|
243
|
+
{config, transcripts, updateList, exportFormats = [config.format || 'json']} = this,
|
|
244
|
+
{output} = config;
|
|
200
245
|
|
|
201
246
|
if (this.mergeCaptions) {
|
|
202
247
|
await this.mergeCaptions(true);
|
|
203
248
|
}
|
|
204
|
-
|
|
205
|
-
|
|
249
|
+
|
|
250
|
+
for (let i = 0; i < exportFormats.length; i++) {
|
|
251
|
+
const
|
|
252
|
+
formatId = exportFormats[i];
|
|
253
|
+
|
|
254
|
+
if (formatId === 'json') {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
for (let j = 0; j < updateList.length; j++) {
|
|
206
259
|
const
|
|
207
|
-
filename = updateList[
|
|
260
|
+
filename = updateList[j],
|
|
208
261
|
id = filename.substring(0, filename.length - 4),
|
|
209
|
-
file = await getFileData(`${output}${id}.${
|
|
262
|
+
file = await getFileData(`${output}${id}.${formatId}`, formatId);
|
|
210
263
|
|
|
211
264
|
file.addHash(transcripts[id].getHash());
|
|
212
265
|
await file.save();
|
|
213
266
|
}
|
|
214
267
|
}
|
|
268
|
+
|
|
215
269
|
super.afterExport(...args)
|
|
216
270
|
}
|
|
217
271
|
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
const
|
|
2
|
+
path = require('path'),
|
|
3
|
+
Cheer = require('./Cheer'),
|
|
4
|
+
MATRIX_EXTS = new Set(['csv', 'tsv']),
|
|
5
|
+
Translate = class Translate extends Cheer {
|
|
6
|
+
async prepare (data) {
|
|
7
|
+
const
|
|
8
|
+
{config} = this,
|
|
9
|
+
scriptPath = config.script;
|
|
10
|
+
|
|
11
|
+
this.matrixMode = false;
|
|
12
|
+
this.scriptPath = null;
|
|
13
|
+
|
|
14
|
+
if (scriptPath) {
|
|
15
|
+
const
|
|
16
|
+
ext = path.extname(scriptPath).slice(1).toLowerCase(),
|
|
17
|
+
basename = path.basename(scriptPath, path.extname(scriptPath));
|
|
18
|
+
|
|
19
|
+
this.scriptPath = scriptPath;
|
|
20
|
+
config.basename = config.basename ?? basename;
|
|
21
|
+
|
|
22
|
+
if (!config.format && !config.exports) {
|
|
23
|
+
config.format = MATRIX_EXTS.has(ext) ? ext : 'json';
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if (MATRIX_EXTS.has(ext)) {
|
|
27
|
+
this.matrixMode = true;
|
|
28
|
+
config.matrixFile = path.basename(scriptPath);
|
|
29
|
+
delete config.script;
|
|
30
|
+
} else {
|
|
31
|
+
await this.replaceConfigPathWithJSON('script', 'files');
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
return super.prepare(data);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
beforeSend (archive) {
|
|
39
|
+
if (this.matrixMode && this.scriptPath) {
|
|
40
|
+
archive.file(this.scriptPath, {name: this.config.matrixFile});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return super.beforeSend(archive);
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
module.exports = Translate;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side mirror of Cheerfully server/helpers/jobExports.js
|
|
3
|
+
* Keep catalog subsets in sync with the canonical Cheerfully module.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const LYRIC_FORMATS = ['json', 'lrc', 'mp3', 'sami', 'smi', 'srt', 'vtt', 'tsv', 'xml', 'dat'];
|
|
7
|
+
const TRANSCRIPTION_FORMATS = ['json', 'lrc', 'mp3', 'sami', 'smi', 'srt', 'vtt'];
|
|
8
|
+
const PACKFULLY_FORMATS = ['texturepacker', 'json-array', 'createjs', 'spine', 'css', 'starling', 'unity', 'godot'];
|
|
9
|
+
const TRANSLATE_FORMATS = ['json', 'i18n', 'po', 'xliff', 'properties', 'yml', 'yaml', 'strings', 'xml', 'csv', 'tsv'];
|
|
10
|
+
|
|
11
|
+
const EXPORT_CATALOG = {
|
|
12
|
+
transcription: {
|
|
13
|
+
keys: ['format', 'exportFormat', 'exports', 'f'],
|
|
14
|
+
formats: TRANSCRIPTION_FORMATS,
|
|
15
|
+
constraints: {mp3: {sourceExt: 'mp3'}}
|
|
16
|
+
},
|
|
17
|
+
rhubarb: {
|
|
18
|
+
keys: ['exportFormat', 'exports', 'f'],
|
|
19
|
+
formats: LYRIC_FORMATS,
|
|
20
|
+
constraints: {mp3: {sourceExt: 'mp3'}}
|
|
21
|
+
},
|
|
22
|
+
allosaurus: {
|
|
23
|
+
keys: ['exportFormat', 'exports', 'f'],
|
|
24
|
+
formats: LYRIC_FORMATS,
|
|
25
|
+
constraints: {mp3: {sourceExt: 'mp3'}}
|
|
26
|
+
},
|
|
27
|
+
translate: {
|
|
28
|
+
keys: ['format', 'exports'],
|
|
29
|
+
formats: TRANSLATE_FORMATS
|
|
30
|
+
},
|
|
31
|
+
packfully: {
|
|
32
|
+
keys: ['format', 'exports', 'exportFormat'],
|
|
33
|
+
formats: PACKFULLY_FORMATS
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const normalizeFormatId = (format) => {
|
|
38
|
+
const
|
|
39
|
+
id = String(format).toLowerCase().replace(/^\./, '');
|
|
40
|
+
|
|
41
|
+
if (id === 'jpeg') {
|
|
42
|
+
return 'jpg';
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return id;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const extractRawValue = (job = {}, keys = []) => {
|
|
49
|
+
for (let i = 0; i < keys.length; i++) {
|
|
50
|
+
const
|
|
51
|
+
key = keys[i];
|
|
52
|
+
|
|
53
|
+
if (job[key] !== undefined) {
|
|
54
|
+
return job[key];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (job.options?.[key] !== undefined) {
|
|
58
|
+
return job.options[key];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return undefined;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const applyConstraints = (formats, constraints = {}, {sourceExt} = {}) => formats.filter((format) => {
|
|
66
|
+
const
|
|
67
|
+
rule = constraints[format];
|
|
68
|
+
|
|
69
|
+
if (!rule) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (rule.sourceExt && sourceExt && rule.sourceExt !== sourceExt) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return true;
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
const resolveJobExports = (job = {}, serviceId, {sourceExt, defaultFormats} = {}) => {
|
|
81
|
+
const
|
|
82
|
+
catalog = EXPORT_CATALOG[serviceId];
|
|
83
|
+
|
|
84
|
+
if (!catalog) {
|
|
85
|
+
throw new Error(`Unknown export service "${serviceId}"`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const
|
|
89
|
+
raw = extractRawValue(job, catalog.keys);
|
|
90
|
+
|
|
91
|
+
if (raw === undefined) {
|
|
92
|
+
const
|
|
93
|
+
fallback = defaultFormats?.length
|
|
94
|
+
? defaultFormats.map(normalizeFormatId)
|
|
95
|
+
: [catalog.formats[0]];
|
|
96
|
+
|
|
97
|
+
return applyConstraints([...new Set(fallback)], catalog.constraints, {sourceExt});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (raw === null || raw === '') {
|
|
101
|
+
return applyConstraints([...catalog.formats], catalog.constraints, {sourceExt});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const
|
|
105
|
+
list = (Array.isArray(raw) ? raw : [raw])
|
|
106
|
+
.map(normalizeFormatId)
|
|
107
|
+
.filter(Boolean),
|
|
108
|
+
seen = new Set(),
|
|
109
|
+
out = [];
|
|
110
|
+
|
|
111
|
+
list.forEach((format) => {
|
|
112
|
+
if (!catalog.formats.includes(format)) {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (!seen.has(format)) {
|
|
117
|
+
seen.add(format);
|
|
118
|
+
out.push(format);
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
const
|
|
123
|
+
constrained = applyConstraints(out.length ? out : [catalog.formats[0]], catalog.constraints, {sourceExt});
|
|
124
|
+
|
|
125
|
+
return constrained.length ? constrained : applyConstraints([catalog.formats[0]], catalog.constraints, {sourceExt});
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
module.exports = {
|
|
129
|
+
EXPORT_CATALOG,
|
|
130
|
+
resolveJobExports
|
|
131
|
+
};
|
package/src/index.js
CHANGED
|
@@ -13,7 +13,8 @@ const
|
|
|
13
13
|
rhubarb: LipSync,
|
|
14
14
|
sharp: require('./classes/Sharp'),
|
|
15
15
|
stackfully: require('./classes/Stackfully'),
|
|
16
|
-
transcription: require('./classes/Transcription')
|
|
16
|
+
transcription: require('./classes/Transcription'),
|
|
17
|
+
translate: require('./classes/Translate')
|
|
17
18
|
},
|
|
18
19
|
getJSON = require('./helpers/getJSON'),
|
|
19
20
|
send = async function (contents) {
|