@openmoca/moca-convert 0.0.0-stage → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,150 @@
1
+ // Collects what the youtube adapter converts: a capture folder holding each
2
+ // video's metadata and captions, in the order YouTube lists them.
3
+ //
4
+ // youtube-capture.json what was captured, and the video order
5
+ // videos/<id>/info.json the video's metadata, stable fields only
6
+ // videos/<id>/captions.vtt its captions as clean WebVTT, when it has any
7
+ //
8
+ // Everything that changes from one fetch to the next (view counts, signed
9
+ // media URLs, thumbnails) is left out, so a capture is worth reviewing and
10
+ // converting it twice gives the same package.
11
+ import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { UsageError } from '../target.js';
14
+ import { json3ToVtt } from './json3-to-vtt.js';
15
+
16
+ export const CAPTURE_FILE = 'youtube-capture.json';
17
+ /** A video or playlist id: letters, digits, `-` and `_`. It names a folder and goes into links. */
18
+ export const VIDEO_ID = /^[A-Za-z0-9_-]{1,64}$/;
19
+
20
+ /** The hosts a capture may read from: YouTube's own. */
21
+ const YOUTUBE_HOSTS = new Set(['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be']);
22
+
23
+ /**
24
+ * Checks that `url` is an http(s) URL on a YouTube host, and returns it
25
+ * rebuilt. Throws UsageError otherwise.
26
+ * @param {string} url
27
+ */
28
+ export function youtubeUrl(url) {
29
+ let parsed;
30
+ try {
31
+ parsed = new URL(url);
32
+ } catch {
33
+ throw new UsageError(`"${url}" is not a URL.`);
34
+ }
35
+ if ((parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || parsed.username || parsed.password || !YOUTUBE_HOSTS.has(parsed.hostname.toLowerCase())) {
36
+ throw new UsageError(`"${url}" is not a YouTube URL (${[...YOUTUBE_HOSTS].join(', ')}).`);
37
+ }
38
+ return parsed.href;
39
+ }
40
+
41
+ /**
42
+ * @param {object} params
43
+ * @param {string} params.url a YouTube video or playlist URL
44
+ * @param {string} params.captureDir
45
+ * @param {string} [params.language] caption language, BCP 47 (default en)
46
+ * @param {boolean} [params.force]
47
+ * @param {ReturnType<import('./ytdlp.js').ytDlp>} params.ytdlp
48
+ * @param {(line: string) => void} [params.log]
49
+ * @returns {Promise<{ kind: string, videos: number, warnings: string[] }>}
50
+ */
51
+ export async function capture({ url, captureDir, language = 'en', force = false, ytdlp, log = () => {} }) {
52
+ if (existsSync(captureDir) && readdirSync(captureDir).length > 0 && !force) {
53
+ throw new UsageError(`Capture directory "${captureDir}" is not empty; pass --force to overwrite.`);
54
+ }
55
+ const checked = youtubeUrl(url);
56
+ const warnings = [];
57
+ log(`reading ${checked}`);
58
+ // `--` ends yt-dlp's options, so the URL is only ever read as a URL.
59
+ const top = await ytdlp.json(['--flat-playlist', '--', checked]);
60
+ const isPlaylist = top._type === 'playlist';
61
+ if (typeof top.id !== 'string' || !VIDEO_ID.test(top.id)) throw new Error(`yt-dlp gave an id that is not a plain id: ${JSON.stringify(top.id)}`);
62
+ const entries = [];
63
+ for (const e of isPlaylist ? (top.entries ?? []) : [top]) {
64
+ if (!e?.id) continue;
65
+ // A video id names a folder, so it must be a plain id.
66
+ if (typeof e.id !== 'string' || !VIDEO_ID.test(e.id)) warnings.push(`Skipped an entry whose id is not a plain id: ${JSON.stringify(e.id)}`);
67
+ else entries.push(e);
68
+ }
69
+
70
+ const videos = [];
71
+ for (const [i, entry] of entries.entries()) {
72
+ const label = `${i + 1}/${entries.length} ${entry.title ?? entry.id}`;
73
+ let info;
74
+ try {
75
+ info = isPlaylist ? await ytdlp.json(['--skip-download', '--', watchUrl(entry.id)]) : entry;
76
+ if (info?.id !== entry.id) throw new Error(`yt-dlp returned another video (${JSON.stringify(info?.id)})`);
77
+ } catch (err) {
78
+ warnings.push(`Skipped ${entry.id}: ${err.message}`);
79
+ log(`skip ${label}`);
80
+ continue;
81
+ }
82
+ const track = chooseTrack(info, language);
83
+ let vtt;
84
+ if (track) {
85
+ log(`fetch ${label} (${track.manual ? 'captions' : 'automatic captions'}, ${track.language})`);
86
+ const json3 = await ytdlp.captions(info, track);
87
+ vtt = json3ToVtt(json3, `${track.manual ? 'Captions' : 'Automatic captions'} for ${watchUrl(info.id)}, from YouTube.`);
88
+ } else {
89
+ warnings.push(`${info.id} has no ${language} captions.`);
90
+ log(`fetch ${label} (no ${language} captions)`);
91
+ }
92
+ const dir = join(captureDir, 'videos', info.id);
93
+ mkdirSync(dir, { recursive: true });
94
+ writeJson(join(dir, 'info.json'), trimInfo(info, track));
95
+ if (vtt) writeFileSync(join(dir, 'captions.vtt'), vtt);
96
+ videos.push(info.id);
97
+ }
98
+
99
+ const first = entries[0] ?? {};
100
+ writeJson(join(captureDir, CAPTURE_FILE), {
101
+ kind: isPlaylist ? 'playlist' : 'video',
102
+ id: top.id,
103
+ url: isPlaylist ? `https://www.youtube.com/playlist?list=${top.id}` : watchUrl(top.id),
104
+ title: top.title,
105
+ description: top.description ?? '',
106
+ channel: top.channel ?? top.uploader ?? first.channel ?? null,
107
+ channel_url: top.channel_url ?? top.uploader_url ?? null,
108
+ videos,
109
+ });
110
+ return { kind: isPlaylist ? 'playlist' : 'video', videos: videos.length, warnings };
111
+ }
112
+
113
+ /**
114
+ * Picks the caption track: captions the creator supplied in `language`, or
115
+ * failing that YouTube's automatic captions in the spoken language
116
+ * (`<language>-orig`), then its automatic ones in `language`.
117
+ * @returns {{ language: string, manual: boolean } | null}
118
+ */
119
+ export function chooseTrack(info, language) {
120
+ const manual = Object.keys(info.subtitles ?? {}).filter((k) => k !== 'live_chat');
121
+ const auto = Object.keys(info.automatic_captions ?? {});
122
+ const own = manual.find((k) => k === language) ?? manual.find((k) => k.startsWith(`${language}-`));
123
+ if (own) return { language: own, manual: true };
124
+ const asr = [`${language}-orig`, language].find((k) => auto.includes(k));
125
+ return asr ? { language: asr, manual: false } : null;
126
+ }
127
+
128
+ export function watchUrl(id) {
129
+ return `https://www.youtube.com/watch?v=${id}`;
130
+ }
131
+
132
+ function trimInfo(info, track) {
133
+ return {
134
+ id: info.id,
135
+ title: info.title,
136
+ description: info.description ?? '',
137
+ channel: info.channel ?? info.uploader ?? null,
138
+ channel_url: info.channel_url ?? info.uploader_url ?? null,
139
+ upload_date: info.upload_date ?? null,
140
+ duration: info.duration ?? null,
141
+ webpage_url: watchUrl(info.id),
142
+ license: info.license ?? null,
143
+ chapters: (info.chapters ?? []).map((c) => ({ start: c.start_time, end: c.end_time, title: c.title })),
144
+ captions: track ? { language: track.language.replace(/-orig$/, ''), automatic: !track.manual } : null,
145
+ };
146
+ }
147
+
148
+ function writeJson(path, value) {
149
+ writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
150
+ }
@@ -0,0 +1,109 @@
1
+ // Turns YouTube's json3 caption track into clean WebVTT.
2
+ //
3
+ // YouTube's own VTT for automatic captions repeats each line as it scrolls
4
+ // and carries per-word timing tags. json3 has one event per phrase instead,
5
+ // with durations that overlap the next phrase and newline-only "append"
6
+ // events in between. Each phrase with text becomes one cue, ending where the
7
+ // next one starts, so the cues never overlap and the text appears once.
8
+
9
+ /**
10
+ * @param {{ events?: object[] }} json3
11
+ * @returns {{ start: number, end: number, text: string }[]} times in milliseconds
12
+ */
13
+ export function json3Cues(json3) {
14
+ const phrases = [];
15
+ for (const event of json3.events ?? []) {
16
+ if (!Array.isArray(event.segs)) continue;
17
+ const text = event.segs.map((s) => s.utf8 ?? '').join('').replace(/\s+/g, ' ').trim();
18
+ if (!text) continue;
19
+ const start = event.tStartMs ?? 0;
20
+ phrases.push({ start, end: start + (event.dDurationMs ?? 0), text });
21
+ }
22
+ phrases.sort((a, b) => a.start - b.start);
23
+ const cues = [];
24
+ for (let i = 0; i < phrases.length; i++) {
25
+ const next = phrases[i + 1];
26
+ const end = next ? Math.min(phrases[i].end, next.start) : phrases[i].end;
27
+ // A phrase that starts at the same instant as the next has no time of
28
+ // its own; fold its text into the next rather than emit an empty cue.
29
+ if (next && end <= phrases[i].start) {
30
+ next.text = `${phrases[i].text} ${next.text}`;
31
+ continue;
32
+ }
33
+ cues.push({ start: phrases[i].start, end: Math.max(end, phrases[i].start + 1), text: phrases[i].text });
34
+ }
35
+ return cues;
36
+ }
37
+
38
+ /**
39
+ * @param {{ start: number, end: number, text: string }[]} cues
40
+ * @param {string} [note] a NOTE block written after the header
41
+ * @returns {string}
42
+ */
43
+ export function cuesToVtt(cues, note) {
44
+ const blocks = ['WEBVTT'];
45
+ if (note) blocks.push(`NOTE ${note.replace(/-->/g, '->').replace(/\n+/g, ' ')}`);
46
+ cues.forEach((cue, i) => {
47
+ blocks.push(`${i + 1}\n${vttTime(cue.start)} --> ${vttTime(cue.end)}\n${escapeCueText(cue.text)}`);
48
+ });
49
+ return `${blocks.join('\n\n')}\n`;
50
+ }
51
+
52
+ /** @param {object} json3 @param {string} [note] */
53
+ export function json3ToVtt(json3, note) {
54
+ return cuesToVtt(json3Cues(json3), note);
55
+ }
56
+
57
+ /** Milliseconds as WebVTT hh:mm:ss.ttt. */
58
+ export function vttTime(ms) {
59
+ const total = Math.max(0, Math.round(ms));
60
+ const h = Math.floor(total / 3_600_000);
61
+ const m = Math.floor((total % 3_600_000) / 60_000);
62
+ const s = Math.floor((total % 60_000) / 1000);
63
+ const t = total % 1000;
64
+ return `${pad(h)}:${pad(m)}:${pad(s)}.${String(t).padStart(3, '0')}`;
65
+ }
66
+
67
+ function pad(n) {
68
+ return String(n).padStart(2, '0');
69
+ }
70
+
71
+ // Cue payload may not contain "-->", and "&" and "<" start character
72
+ // references and tags, so they are written as references.
73
+ function escapeCueText(text) {
74
+ return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
75
+ }
76
+
77
+ /**
78
+ * Parses the cues back out of a WebVTT file written by cuesToVtt (or any
79
+ * simple WebVTT): used by the adapter to build the transcript.
80
+ * @param {string} text
81
+ * @returns {{ start: number, end: number, text: string }[]}
82
+ */
83
+ export function parseVtt(text) {
84
+ const cues = [];
85
+ for (const block of text.replace(/\r\n?/g, '\n').split(/\n{2,}/)) {
86
+ const lines = block.split('\n');
87
+ const at = lines.findIndex((l) => l.includes('-->'));
88
+ if (at === -1) continue;
89
+ const [from, to] = lines[at].split('-->').map((p) => parseTime(p.trim().split(/\s+/)[0]));
90
+ if (from === undefined || to === undefined) continue;
91
+ const payload = lines
92
+ .slice(at + 1)
93
+ .join(' ')
94
+ .replace(/<[^>]*>/g, '')
95
+ .replace(/&lt;/g, '<')
96
+ .replace(/&gt;/g, '>')
97
+ .replace(/&nbsp;/g, ' ')
98
+ .replace(/&amp;/g, '&')
99
+ .trim();
100
+ cues.push({ start: from, end: to, text: payload });
101
+ }
102
+ return cues;
103
+ }
104
+
105
+ function parseTime(value) {
106
+ const m = /^(?:(\d+):)?(\d{2}):(\d{2})\.(\d{3})$/.exec(value ?? '');
107
+ if (!m) return undefined;
108
+ return ((Number(m[1] ?? 0) * 60 + Number(m[2])) * 60 + Number(m[3])) * 1000 + Number(m[4]);
109
+ }
@@ -0,0 +1,64 @@
1
+ // Runs yt-dlp (https://github.com/yt-dlp/yt-dlp), which does the talking to
2
+ // YouTube. It is spawned directly, never through a shell, so the same code
3
+ // runs on macOS, Linux and Windows.
4
+ import { execFile } from 'node:child_process';
5
+ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
6
+ import { tmpdir } from 'node:os';
7
+ import { join } from 'node:path';
8
+ import { UsageError } from '../target.js';
9
+
10
+ const MAX_BUFFER = 512 * 1024 * 1024;
11
+ /** The longest one yt-dlp run may take. */
12
+ export const YTDLP_TIMEOUT_MS = 5 * 60_000;
13
+
14
+ /**
15
+ * yt-dlp runs with --ignore-config, so a user's or system configuration file
16
+ * cannot add options, and each run is stopped after `timeoutMs`.
17
+ * @param {{ bin?: string, timeoutMs?: number }} [options]
18
+ * @returns {{ json: (args: string[]) => Promise<object>, captions: (info: object, track: { language: string, manual: boolean }) => Promise<object> }}
19
+ */
20
+ export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp', timeoutMs = YTDLP_TIMEOUT_MS } = {}) {
21
+ const run = (args) =>
22
+ new Promise((resolve, reject) => {
23
+ execFile(bin, ['--ignore-config', ...args], { maxBuffer: MAX_BUFFER, windowsHide: true, timeout: timeoutMs, killSignal: 'SIGKILL' }, (err, stdout, stderr) => {
24
+ if (err?.code === 'ENOENT') {
25
+ reject(new UsageError(`Cannot run "${bin}". Install yt-dlp (https://github.com/yt-dlp/yt-dlp#installation) or pass --yt-dlp <path>.`));
26
+ } else if (err?.killed) {
27
+ reject(new Error(`yt-dlp took longer than ${Math.round(timeoutMs / 1000)} s and was stopped.`));
28
+ } else if (err) {
29
+ reject(new Error(`yt-dlp ${args.join(' ')} failed: ${(stderr || err.message).trim()}`));
30
+ } else {
31
+ resolve(stdout);
32
+ }
33
+ });
34
+ });
35
+
36
+ return {
37
+ async json(args) {
38
+ return JSON.parse(await run(['--dump-single-json', '--no-warnings', ...args]));
39
+ },
40
+
41
+ // Reuses the metadata already fetched, so YouTube is only asked for the
42
+ // caption track itself.
43
+ async captions(info, { language, manual }) {
44
+ const dir = mkdtempSync(join(tmpdir(), 'moca-youtube-'));
45
+ try {
46
+ const infoPath = join(dir, 'info.json');
47
+ writeFileSync(infoPath, JSON.stringify(info));
48
+ await run([
49
+ '--load-info-json', infoPath,
50
+ '--skip-download', '--no-warnings',
51
+ manual ? '--write-subs' : '--write-auto-subs',
52
+ '--sub-langs', language,
53
+ '--sub-format', 'json3',
54
+ '-o', join(dir, 'captions.%(ext)s'),
55
+ ]);
56
+ const file = readdirSync(dir).find((f) => f.endsWith('.json3'));
57
+ if (!file) throw new Error(`yt-dlp wrote no ${language} caption track for ${info.id}.`);
58
+ return JSON.parse(readFileSync(join(dir, file), 'utf8'));
59
+ } finally {
60
+ rmSync(dir, { recursive: true, force: true });
61
+ }
62
+ },
63
+ };
64
+ }
package/package.json CHANGED
@@ -1,6 +1,44 @@
1
1
  {
2
2
  "name": "@openmoca/moca-convert",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.7.0",
4
+ "description": "Create MOCA packages (OKF content bundles) from Markdown folders, Obsidian vaults, OpenAPI documents, PDFs and YouTube videos.",
5
+ "license": "Apache-2.0",
6
+ "type": "module",
7
+ "engines": {
8
+ "node": ">=22"
9
+ },
10
+ "bin": {
11
+ "moca-convert": "./bin/moca-convert.js",
12
+ "moca-youtube": "./bin/moca-youtube.js"
13
+ },
14
+ "scripts": {
15
+ "test": "node --test test/*.test.js"
16
+ },
17
+ "dependencies": {
18
+ "@openmoca/moca-core": "^0.7.0",
19
+ "commander": "^12.1.0",
20
+ "js-yaml": "^5.2.2",
21
+ "pdfjs-dist": "^6.3.289"
22
+ },
23
+ "publishConfig": {
24
+ "access": "public"
25
+ },
26
+ "repository": {
27
+ "type": "git",
28
+ "url": "git+https://github.com/openmoca/moca-convert.git"
29
+ },
30
+ "homepage": "https://github.com/openmoca/moca-convert#readme",
31
+ "bugs": {
32
+ "url": "https://github.com/openmoca/moca-convert/issues"
33
+ },
34
+ "files": [
35
+ "bin/",
36
+ "lib/",
37
+ "README.md"
38
+ ],
39
+ "exports": {
40
+ ".": "./lib/index.js",
41
+ "./lib/*": "./lib/*",
42
+ "./package.json": "./package.json"
43
+ }
44
+ }