@openmoca/moca-convert 0.0.0-stage → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,109 @@
1
+ // Turns YouTube's json3 caption track into clean WebVTT.
2
+ //
3
+ // YouTube's own VTT for automatic captions repeats each line as it scrolls
4
+ // and carries per-word timing tags. json3 has one event per phrase instead,
5
+ // with durations that overlap the next phrase and newline-only "append"
6
+ // events in between. Each phrase with text becomes one cue, ending where the
7
+ // next one starts, so the cues never overlap and the text appears once.
8
+
9
+ /**
10
+ * @param {{ events?: object[] }} json3
11
+ * @returns {{ start: number, end: number, text: string }[]} times in milliseconds
12
+ */
13
+ export function json3Cues(json3) {
14
+ const phrases = [];
15
+ for (const event of json3.events ?? []) {
16
+ if (!Array.isArray(event.segs)) continue;
17
+ const text = event.segs.map((s) => s.utf8 ?? '').join('').replace(/\s+/g, ' ').trim();
18
+ if (!text) continue;
19
+ const start = event.tStartMs ?? 0;
20
+ phrases.push({ start, end: start + (event.dDurationMs ?? 0), text });
21
+ }
22
+ phrases.sort((a, b) => a.start - b.start);
23
+ const cues = [];
24
+ for (let i = 0; i < phrases.length; i++) {
25
+ const next = phrases[i + 1];
26
+ const end = next ? Math.min(phrases[i].end, next.start) : phrases[i].end;
27
+ // A phrase that starts at the same instant as the next has no time of
28
+ // its own; fold its text into the next rather than emit an empty cue.
29
+ if (next && end <= phrases[i].start) {
30
+ next.text = `${phrases[i].text} ${next.text}`;
31
+ continue;
32
+ }
33
+ cues.push({ start: phrases[i].start, end: Math.max(end, phrases[i].start + 1), text: phrases[i].text });
34
+ }
35
+ return cues;
36
+ }
37
+
38
+ /**
39
+ * @param {{ start: number, end: number, text: string }[]} cues
40
+ * @param {string} [note] a NOTE block written after the header
41
+ * @returns {string}
42
+ */
43
+ export function cuesToVtt(cues, note) {
44
+ const blocks = ['WEBVTT'];
45
+ if (note) blocks.push(`NOTE ${note.replace(/-->/g, '->').replace(/\n+/g, ' ')}`);
46
+ cues.forEach((cue, i) => {
47
+ blocks.push(`${i + 1}\n${vttTime(cue.start)} --> ${vttTime(cue.end)}\n${escapeCueText(cue.text)}`);
48
+ });
49
+ return `${blocks.join('\n\n')}\n`;
50
+ }
51
+
52
+ /** @param {object} json3 @param {string} [note] */
53
+ export function json3ToVtt(json3, note) {
54
+ return cuesToVtt(json3Cues(json3), note);
55
+ }
56
+
57
+ /** Milliseconds as WebVTT hh:mm:ss.ttt. */
58
+ export function vttTime(ms) {
59
+ const total = Math.max(0, Math.round(ms));
60
+ const h = Math.floor(total / 3_600_000);
61
+ const m = Math.floor((total % 3_600_000) / 60_000);
62
+ const s = Math.floor((total % 60_000) / 1000);
63
+ const t = total % 1000;
64
+ return `${pad(h)}:${pad(m)}:${pad(s)}.${String(t).padStart(3, '0')}`;
65
+ }
66
+
67
+ function pad(n) {
68
+ return String(n).padStart(2, '0');
69
+ }
70
+
71
+ // Cue payload may not contain "-->", and "&" and "<" start character
72
+ // references and tags, so they are written as references.
73
+ function escapeCueText(text) {
74
+ return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
75
+ }
76
+
77
+ /**
78
+ * Parses the cues back out of a WebVTT file written by cuesToVtt (or any
79
+ * simple WebVTT): used by the adapter to build the transcript.
80
+ * @param {string} text
81
+ * @returns {{ start: number, end: number, text: string }[]}
82
+ */
83
+ export function parseVtt(text) {
84
+ const cues = [];
85
+ for (const block of text.replace(/\r\n?/g, '\n').split(/\n{2,}/)) {
86
+ const lines = block.split('\n');
87
+ const at = lines.findIndex((l) => l.includes('-->'));
88
+ if (at === -1) continue;
89
+ const [from, to] = lines[at].split('-->').map((p) => parseTime(p.trim().split(/\s+/)[0]));
90
+ if (from === undefined || to === undefined) continue;
91
+ const payload = lines
92
+ .slice(at + 1)
93
+ .join(' ')
94
+ .replace(/<[^>]*>/g, '')
95
+ .replace(/&lt;/g, '<')
96
+ .replace(/&gt;/g, '>')
97
+ .replace(/&nbsp;/g, ' ')
98
+ .replace(/&amp;/g, '&')
99
+ .trim();
100
+ cues.push({ start: from, end: to, text: payload });
101
+ }
102
+ return cues;
103
+ }
104
+
105
+ function parseTime(value) {
106
+ const m = /^(?:(\d+):)?(\d{2}):(\d{2})\.(\d{3})$/.exec(value ?? '');
107
+ if (!m) return undefined;
108
+ return ((Number(m[1] ?? 0) * 60 + Number(m[2])) * 60 + Number(m[3])) * 1000 + Number(m[4]);
109
+ }
@@ -0,0 +1,58 @@
1
+ // Runs yt-dlp (https://github.com/yt-dlp/yt-dlp), which does the talking to
2
+ // YouTube. It is spawned directly, never through a shell, so the same code
3
+ // runs on macOS, Linux and Windows.
4
+ import { execFile } from 'node:child_process';
5
+ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
6
+ import { tmpdir } from 'node:os';
7
+ import { join } from 'node:path';
8
+ import { UsageError } from '../target.js';
9
+
10
+ const MAX_BUFFER = 512 * 1024 * 1024;
11
+
12
+ /**
13
+ * @param {{ bin?: string }} [options]
14
+ * @returns {{ json: (args: string[]) => Promise<object>, captions: (info: object, track: { language: string, manual: boolean }) => Promise<object> }}
15
+ */
16
+ export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp' } = {}) {
17
+ const run = (args) =>
18
+ new Promise((resolve, reject) => {
19
+ execFile(bin, args, { maxBuffer: MAX_BUFFER, windowsHide: true }, (err, stdout, stderr) => {
20
+ if (err?.code === 'ENOENT') {
21
+ reject(new UsageError(`Cannot run "${bin}". Install yt-dlp (https://github.com/yt-dlp/yt-dlp#installation) or pass --yt-dlp <path>.`));
22
+ } else if (err) {
23
+ reject(new Error(`yt-dlp ${args.join(' ')} failed: ${(stderr || err.message).trim()}`));
24
+ } else {
25
+ resolve(stdout);
26
+ }
27
+ });
28
+ });
29
+
30
+ return {
31
+ async json(args) {
32
+ return JSON.parse(await run(['--dump-single-json', '--no-warnings', ...args]));
33
+ },
34
+
35
+ // Reuses the metadata already fetched, so YouTube is only asked for the
36
+ // caption track itself.
37
+ async captions(info, { language, manual }) {
38
+ const dir = mkdtempSync(join(tmpdir(), 'moca-youtube-'));
39
+ try {
40
+ const infoPath = join(dir, 'info.json');
41
+ writeFileSync(infoPath, JSON.stringify(info));
42
+ await run([
43
+ '--load-info-json', infoPath,
44
+ '--skip-download', '--no-warnings',
45
+ manual ? '--write-subs' : '--write-auto-subs',
46
+ '--sub-langs', language,
47
+ '--sub-format', 'json3',
48
+ '-o', join(dir, 'captions.%(ext)s'),
49
+ ]);
50
+ const file = readdirSync(dir).find((f) => f.endsWith('.json3'));
51
+ if (!file) throw new Error(`yt-dlp wrote no ${language} caption track for ${info.id}.`);
52
+ return JSON.parse(readFileSync(join(dir, file), 'utf8'));
53
+ } finally {
54
+ rmSync(dir, { recursive: true, force: true });
55
+ }
56
+ },
57
+ };
58
+ }
package/package.json CHANGED
@@ -1,6 +1,44 @@
1
1
  {
2
2
  "name": "@openmoca/moca-convert",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.6.0",
4
+ "description": "Create MOCA packages (OKF content bundles) from Markdown folders, Obsidian vaults, OpenAPI documents, PDFs and YouTube videos.",
5
+ "license": "Apache-2.0",
6
+ "type": "module",
7
+ "engines": {
8
+ "node": ">=22"
9
+ },
10
+ "bin": {
11
+ "moca-convert": "./bin/moca-convert.js",
12
+ "moca-youtube": "./bin/moca-youtube.js"
13
+ },
14
+ "scripts": {
15
+ "test": "node --test test/*.test.js"
16
+ },
17
+ "dependencies": {
18
+ "@openmoca/moca-core": "^0.6.0",
19
+ "commander": "^12.1.0",
20
+ "js-yaml": "^5.2.2",
21
+ "pdfjs-dist": "^6.3.289"
22
+ },
23
+ "publishConfig": {
24
+ "access": "public"
25
+ },
26
+ "repository": {
27
+ "type": "git",
28
+ "url": "git+https://github.com/openmoca/moca-convert.git"
29
+ },
30
+ "homepage": "https://github.com/openmoca/moca-convert#readme",
31
+ "bugs": {
32
+ "url": "https://github.com/openmoca/moca-convert/issues"
33
+ },
34
+ "files": [
35
+ "bin/",
36
+ "lib/",
37
+ "README.md"
38
+ ],
39
+ "exports": {
40
+ ".": "./lib/index.js",
41
+ "./lib/*": "./lib/*",
42
+ "./package.json": "./package.json"
43
+ }
44
+ }