@openmoca/moca-convert 0.0.0-stage → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,309 @@
1
+ // `pdf` adapter: one or more PDFs become content nodes, one per section. A
2
+ // section starts at a heading of the first level that occurs more than once
3
+ // in the document (chapters in a book, sections in a chapter); deeper headings
4
+ // stay inside the node. Every node cites the pages it came from as RFC 3778
5
+ // `page=` fragments, and structure.ttl keeps the documents and their sections
6
+ // in order.
7
+ //
8
+ // Input is a PDF file, a folder of PDFs, or a folder with a `pdfs.json`
9
+ // that lists the documents in order, with an optional title, source URL and
10
+ // SHA-256 for each (see README).
11
+ import { createHash } from 'node:crypto';
12
+ import { closeSync, constants, existsSync, fstatSync, openSync, readFileSync, realpathSync, statSync } from 'node:fs';
13
+ import { basename, join, sep } from 'node:path';
14
+ import { joinFrontmatter } from '@openmoca/moca-core';
15
+ import { buildManifest } from '../manifest.js';
16
+ import { slugify } from '../slug.js';
17
+ import { UsageError } from '../target.js';
18
+ import { walkFiles } from '../walk.js';
19
+ import { escapeInline, escapeLineStart, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
20
+ import { extractPdf, PdfLimitError, PDF_LIMITS } from '../pdf/extract.js';
21
+ import { layoutPdf } from '../pdf/layout.js';
22
+
23
+ export const name = 'pdf';
24
+ export const DEFAULT_TYPE = 'Document';
25
+ export const DESCRIPTOR = 'pdfs.json';
26
+
27
+ const RFC3778 = 'http://tools.ietf.org/rfc/rfc3778';
28
+ /** The largest PDF read, in bytes, unless --max-pdf-mb raises it. */
29
+ export const MAX_PDF_BYTES = 256 * 1024 * 1024;
30
+ const SOURCE_MODES = ['copy', 'link'];
31
+
32
+ /** @param {string} inputPath */
33
+ export function detect(inputPath) {
34
+ if (!existsSync(inputPath)) return false;
35
+ const st = statSync(inputPath);
36
+ if (st.isFile()) return /\.pdf$/i.test(inputPath);
37
+ if (!st.isDirectory()) return false;
38
+ if (existsSync(join(inputPath, DESCRIPTOR))) return true;
39
+ const files = walkFiles(inputPath);
40
+ return files.some((f) => /\.pdf$/i.test(f)) && !files.some((f) => f.endsWith('.md'));
41
+ }
42
+
43
+ /**
44
+ * @param {{ inputPath: string, options: object }} ctx
45
+ * @returns {Promise<import('./index.js').PackageDraft>}
46
+ */
47
+ export async function convert({ inputPath, options }) {
48
+ const sourcesMode = options.sources ?? 'copy';
49
+ if (!SOURCE_MODES.includes(sourcesMode)) throw new UsageError(`--sources must be one of: ${SOURCE_MODES.join(', ')}.`);
50
+ const listing = listDocuments(inputPath);
51
+ if (sourcesMode === 'link') {
52
+ const missing = listing.documents.filter((d) => !d.url);
53
+ if (missing.length > 0) throw new UsageError(`--sources link needs a url for every document in ${DESCRIPTOR}; ${missing[0].file} has none.`);
54
+ }
55
+
56
+ const limits = {
57
+ maxBytes: options.maxPdfBytes ?? MAX_PDF_BYTES,
58
+ maxPages: options.maxPdfPages ?? PDF_LIMITS.maxPages,
59
+ timeoutMs: options.pdfTimeoutMs ?? PDF_LIMITS.timeoutMs,
60
+ };
61
+ const warnings = [];
62
+ const docs = [];
63
+ const docSlugs = new Set(['index']);
64
+ for (const entry of listing.documents) {
65
+ const bytes = readPdf(entry, inputPath, limits.maxBytes);
66
+ let pdf;
67
+ try {
68
+ pdf = await extractPdf(new Uint8Array(bytes), limits);
69
+ } catch (err) {
70
+ if (err instanceof PdfLimitError) throw new UsageError(`${entry.file}: ${err.message}. Raise the limit with --max-pdf-pages or --pdf-timeout.`);
71
+ throw err;
72
+ }
73
+ const { blocks, labels } = layoutPdf(pdf);
74
+ const title = entry.title ?? pdf.title ?? firstHeading(blocks) ?? basename(entry.file, '.pdf');
75
+ if (!blocks.some((b) => b.kind !== 'heading')) {
76
+ warnings.push({ code: 'PDF_NO_TEXT', message: `${entry.file} has no text layer (a scan?); it was left out. Run OCR on it first.`, file: entry.file });
77
+ continue;
78
+ }
79
+ const slug = uniqueSlug(slugify(title), docSlugs);
80
+ docs.push({ entry, bytes, title, slug, labels, sections: splitSections(blocks, title) });
81
+ }
82
+ if (docs.length === 0) throw new UsageError(`${inputPath}: none of the PDFs has any text.`);
83
+
84
+ const multi = docs.length > 1;
85
+ const manifest = buildManifest({
86
+ ...options,
87
+ title: options.title ?? listing.title ?? (multi ? undefined : docs[0].title),
88
+ description: options.description ?? listing.description,
89
+ });
90
+ const iri = (fragment) => `${manifest.id}#${fragment}`;
91
+ const type = options.type ?? DEFAULT_TYPE;
92
+ const contentNodes = [];
93
+ const files = [];
94
+
95
+ for (const doc of docs) {
96
+ const dir = multi ? `content/${doc.slug}` : 'content';
97
+ const up = multi ? '../../' : '../';
98
+ const pdfPath = `sources/${doc.slug}.pdf`;
99
+ if (sourcesMode === 'copy') files.push({ path: pdfPath, body: doc.bytes });
100
+ const pdfRef = sourcesMode === 'copy' ? `${up}${pdfPath}` : doc.entry.url;
101
+ const sectionSlugs = new Set(['index']);
102
+ for (const section of doc.sections) {
103
+ section.slug = uniqueSlug(slugify(section.title), sectionSlugs);
104
+ const pageList = section.pages;
105
+ const evidence = pageList.map((n) => ({
106
+ source: 'pdf',
107
+ selector: { type: 'FragmentSelector', conformsTo: RFC3778, value: `page=${n}` },
108
+ ...(doc.labels.get(n) ? { note: `p. ${doc.labels.get(n)}` } : {}),
109
+ }));
110
+ const firstText = section.blocks.find((b) => b.kind === 'paragraph');
111
+ const data = {
112
+ type,
113
+ title: section.title,
114
+ ...(firstText ? { description: summary(firstText.text) } : {}),
115
+ ...(doc.entry.url ? { resource: `${doc.entry.url}#page=${pageList[0]}` } : {}),
116
+ sources: [{ id: 'pdf', resource: pdfRef, title: `${doc.title} (PDF)` }],
117
+ moca: {
118
+ concepts: [{ iri: iri(`${doc.slug}/${section.slug}`), role: 'primary' }],
119
+ evidence,
120
+ },
121
+ };
122
+ contentNodes.push({ path: `${dir}/${section.slug}.md`, body: joinFrontmatter(data, sectionBody(section, doc, pdfRef)) });
123
+ }
124
+ if (multi) contentNodes.push({ path: `${dir}/index.md`, body: documentIndex(doc, pdfRef) });
125
+ }
126
+ contentNodes.push({ path: 'content/index.md', body: multi ? collectionIndex(manifest, docs) : documentIndex(docs[0], sourcesMode === 'copy' ? `../sources/${docs[0].slug}.pdf` : docs[0].entry.url) });
127
+ files.push({ path: 'structure.ttl', body: structure(manifest, docs, { iri, lang: options.language }) });
128
+ return { manifest, contentNodes, files, warnings };
129
+ }
130
+
131
+ /**
132
+ * The documents to convert, in order.
133
+ * @returns {{ title?: string, description?: string, documents: Array<{ file: string, abs: string, url?: string, sha256?: string, title?: string }> }}
134
+ */
135
+ export function listDocuments(inputPath) {
136
+ if (!existsSync(inputPath)) throw new UsageError(`Input does not exist: ${inputPath}`);
137
+ if (statSync(inputPath).isFile()) {
138
+ if (!/\.pdf$/i.test(inputPath)) throw new UsageError(`${inputPath} is not a PDF.`);
139
+ return { documents: [{ file: basename(inputPath), abs: inputPath }] };
140
+ }
141
+ const descriptor = join(inputPath, DESCRIPTOR);
142
+ if (!existsSync(descriptor)) {
143
+ const found = walkFiles(inputPath).filter((f) => /\.pdf$/i.test(f)).sort();
144
+ if (found.length === 0) throw new UsageError(`${inputPath} holds no PDFs.`);
145
+ return { documents: found.map((file) => ({ file, abs: join(inputPath, file) })) };
146
+ }
147
+ let spec;
148
+ try {
149
+ spec = JSON.parse(readFileSync(descriptor, 'utf8'));
150
+ } catch (err) {
151
+ throw new UsageError(`${descriptor} is not valid JSON: ${err.message}`);
152
+ }
153
+ if (!Array.isArray(spec?.documents) || spec.documents.length === 0) throw new UsageError(`${descriptor} needs a non-empty "documents" list.`);
154
+ const documents = spec.documents.map((d, i) => {
155
+ if (typeof d?.file !== 'string' || d.file.split(/[\\/]/).includes('..')) throw new UsageError(`${descriptor}: documents[${i}] needs a "file" inside the folder.`);
156
+ let url;
157
+ if (d.url !== undefined) {
158
+ url = httpUrl(d.url);
159
+ if (!url) throw new UsageError(`${descriptor}: documents[${i}].url must be an http(s) URL, with no user name or password.`);
160
+ if (new URL(url).hash) throw new UsageError(`${descriptor}: documents[${i}].url must not have a #fragment; moca-convert adds #page= itself.`);
161
+ }
162
+ if (d.sha256 !== undefined && !/^[0-9a-f]{64}$/.test(d.sha256)) throw new UsageError(`${descriptor}: documents[${i}].sha256 must be 64 lowercase hex digits.`);
163
+ return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['sha256', 'title']), ...(url ? { url } : {}) };
164
+ });
165
+ return { ...pick(spec, ['title', 'description']), documents };
166
+ }
167
+
168
+ function readPdf(entry, inputPath, maxBytes) {
169
+ // A listed file is a regular file inside the input folder: not a symbolic link, and not reached through one. It is
170
+ // opened once, without following a link, and checked and read through that one handle.
171
+ let fd;
172
+ try {
173
+ fd = openSync(entry.abs, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
174
+ } catch (err) {
175
+ if (err.code === 'ENOENT') throw new UsageError(`${entry.file} is missing${entry.url ? `; download it from ${entry.url}` : ''}.`);
176
+ if (err.code === 'ELOOP' || err.code === 'EISDIR') throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
177
+ throw err;
178
+ }
179
+ let bytes;
180
+ try {
181
+ const st = fstatSync(fd);
182
+ if (!st.isFile()) throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
183
+ if (statSync(inputPath).isDirectory()) {
184
+ const root = realpathSync(inputPath);
185
+ if (!realpathSync(entry.abs).startsWith(root + sep)) throw new UsageError(`${entry.file} is outside ${inputPath}.`);
186
+ }
187
+ if (st.size > maxBytes) {
188
+ throw new UsageError(`${entry.file} is ${Math.ceil(st.size / 1024 / 1024)} MB, over the ${Math.round(maxBytes / 1024 / 1024)} MB limit. Raise it with --max-pdf-mb.`);
189
+ }
190
+ bytes = readFileSync(fd);
191
+ } finally {
192
+ closeSync(fd);
193
+ }
194
+ if (entry.sha256) {
195
+ const actual = createHash('sha256').update(bytes).digest('hex');
196
+ if (actual !== entry.sha256) throw new UsageError(`${entry.file} has SHA-256 ${actual}, but ${DESCRIPTOR} expects ${entry.sha256}.`);
197
+ }
198
+ return bytes;
199
+ }
200
+
201
+ /**
202
+ * Splits blocks into sections at the first heading level that occurs more
203
+ * than once. Text before the first such heading is an opening section, named
204
+ * by the headings above it or by the document title.
205
+ * @param {import('../pdf/layout.js').Block[]} blocks
206
+ * @returns {Array<{ title: string, level: number, blocks: import('../pdf/layout.js').Block[], pages: number[] }>}
207
+ */
208
+ export function splitSections(blocks, docTitle) {
209
+ const counts = new Map();
210
+ for (const b of blocks) if (b.kind === 'heading') counts.set(b.level, (counts.get(b.level) ?? 0) + 1);
211
+ const splitLevel = [...counts.keys()].sort((a, b) => a - b).find((l) => counts.get(l) > 1);
212
+
213
+ const sections = [];
214
+ let current = { title: null, level: splitLevel ?? 1, blocks: [], pages: new Set() };
215
+ const push = () => {
216
+ if (current.blocks.some((b) => b.kind !== 'heading')) sections.push(current);
217
+ };
218
+ for (const block of blocks) {
219
+ if (splitLevel && block.kind === 'heading' && block.level <= splitLevel) {
220
+ if (block.level === splitLevel || current.blocks.some((b) => b.kind !== 'heading')) {
221
+ push();
222
+ current = { title: block.text, level: splitLevel, blocks: [], pages: new Set([block.page]) };
223
+ continue;
224
+ }
225
+ // A heading above the split level before any text (a chapter title)
226
+ // names the opening section.
227
+ current.title = current.title ? `${current.title}: ${block.text}` : block.text;
228
+ current.pages.add(block.page);
229
+ continue;
230
+ }
231
+ current.blocks.push(block);
232
+ for (let p = block.page; p <= (block.lastPage ?? block.page); p++) current.pages.add(p);
233
+ }
234
+ push();
235
+ return sections.map((s) => ({ ...s, title: s.title ?? docTitle, pages: [...s.pages].sort((a, b) => a - b) }));
236
+ }
237
+
238
+ function sectionBody(section, doc, pdfRef) {
239
+ const range = pageRange(section.pages, doc.labels);
240
+ const out = [`# ${escapeInline(section.title)}`, `From ${escapeInline(doc.title)}, ${range} ([PDF](<${pdfRef}#page=${section.pages[0]}>)).`];
241
+ for (const block of section.blocks) {
242
+ if (block.kind === 'heading') out.push(`${'#'.repeat(Math.min(6, Math.max(2, block.level - section.level + 1)))} ${escapeInline(block.text)}`);
243
+ else if (block.kind === 'item') out.push(`${block.number ? `${block.number}.` : '-'} ${escapeInline(block.text)}`);
244
+ else if (block.kind === 'caption') out.push(`*${escapeInline(block.text)}*`);
245
+ else out.push(escapeLineStart(escapeInline(block.text)));
246
+ }
247
+ return `\n${joinBlocks(out, section.blocks)}\n`;
248
+ }
249
+
250
+ // List items that follow each other form one list; everything else is
251
+ // separated by a blank line.
252
+ function joinBlocks(lines, blocks) {
253
+ const kinds = [null, null, ...blocks.map((b) => (b.kind === 'item' ? (b.number ? 'ol' : 'ul') : b.kind))];
254
+ const listed = (k) => k === 'ol' || k === 'ul';
255
+ return lines.reduce((acc, line, i) => (i === 0 ? line : `${acc}${listed(kinds[i]) && kinds[i] === kinds[i - 1] ? '\n' : '\n\n'}${line}`), '');
256
+ }
257
+
258
+ // Printed page numbers when both ends have one, otherwise the PDF's own page
259
+ // numbers, said as such so they are not mistaken for printed ones.
260
+ function pageRange(pages, labels) {
261
+ const first = pages[0];
262
+ const last = pages.at(-1);
263
+ const printed = labels.has(first) && labels.has(last);
264
+ const show = (n) => (printed ? escapeInline(labels.get(n)) : String(n));
265
+ const noun = `${printed ? '' : 'PDF '}page${first === last ? '' : 's'}`;
266
+ return first === last ? `${noun} ${show(first)}` : `${noun} ${show(first)} to ${show(last)}`;
267
+ }
268
+
269
+ function documentIndex(doc, pdfRef) {
270
+ const lines = [`# ${escapeInline(doc.title)}`, '', `Sections of [the PDF](<${pdfRef}>) in order:`, ''];
271
+ doc.sections.forEach((s, i) => lines.push(`${i + 1}. [${escapeInline(s.title)}](${s.slug}.md)`));
272
+ return `${lines.join('\n')}\n`;
273
+ }
274
+
275
+ function collectionIndex(manifest, docs) {
276
+ const lines = [`# ${escapeInline(manifest.title)}`, ''];
277
+ if (manifest.description) lines.push(escapeInline(manifest.description), '');
278
+ lines.push('Documents in order:', '');
279
+ docs.forEach((d, i) => lines.push(`${i + 1}. [${escapeInline(d.title)}](${d.slug}/index.md)`));
280
+ return `${lines.join('\n')}\n`;
281
+ }
282
+
283
+ function structure(manifest, docs, { iri, lang }) {
284
+ const label = (text) => `${turtleString(text)}${lang ? `@${lang}` : ''}`;
285
+ const ref = (fragment) => `<${iri(fragment)}>`;
286
+ const blocks = ['@prefix skos: <http://www.w3.org/2004/02/skos/core#> .\n@prefix dct: <http://purl.org/dc/terms/> .'];
287
+ const ordered = (fragment, text, members) =>
288
+ `${ref(fragment)} a skos:OrderedCollection ;\n skos:prefLabel ${label(text)} ;\n skos:memberList (\n ${members.join('\n ')}\n ) .`;
289
+ if (docs.length > 1) {
290
+ const members = docs.map((d) => ref(d.slug));
291
+ blocks.push(`${ref('collection')} a skos:Concept ;\n skos:prefLabel ${label(manifest.title)} ;\n dct:hasPart ${members.join(',\n ')} .`);
292
+ blocks.push(ordered('collection-order', `${manifest.title} (document order)`, members));
293
+ }
294
+ for (const doc of docs) {
295
+ const members = doc.sections.map((s) => ref(`${doc.slug}/${s.slug}`));
296
+ blocks.push(`${ref(doc.slug)} a skos:Concept ;\n skos:prefLabel ${label(doc.title)} ;\n dct:hasPart ${members.join(',\n ')} .`);
297
+ blocks.push(ordered(`${doc.slug}/order`, `${doc.title} (section order)`, members));
298
+ for (const s of doc.sections) blocks.push(`${ref(`${doc.slug}/${s.slug}`)} a skos:Concept ;\n skos:prefLabel ${label(s.title)} .`);
299
+ }
300
+ return `${blocks.join('\n\n')}\n`;
301
+ }
302
+
303
+ function firstHeading(blocks) {
304
+ return blocks.find((b) => b.kind === 'heading')?.text;
305
+ }
306
+
307
+ function pick(obj, keys) {
308
+ return Object.fromEntries(keys.filter((k) => typeof obj?.[k] === 'string' && obj[k].trim()).map((k) => [k, obj[k].trim()]));
309
+ }
@@ -0,0 +1,259 @@
1
+ // `youtube` adapter: a capture folder written by `moca-youtube` (see
2
+ // lib/youtube/capture.js) becomes one content node per video. The captions
3
+ // travel under media/ and the description under sources/, so every citation
4
+ // can be checked without a network; a playlist also gets a structure.ttl that
5
+ // keeps the videos in playlist order.
6
+ import { existsSync, readFileSync, statSync } from 'node:fs';
7
+ import { join } from 'node:path';
8
+ import { joinFrontmatter } from '@openmoca/moca-core';
9
+ import { buildManifest } from '../manifest.js';
10
+ import { slugify } from '../slug.js';
11
+ import { UsageError } from '../target.js';
12
+ import { compactDate, escapeInline, escapeText, firstLine, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
13
+ import { CAPTURE_FILE, VIDEO_ID, watchUrl } from '../youtube/capture.js';
14
+ import { parseVtt } from '../youtube/json3-to-vtt.js';
15
+
16
+ export const name = 'youtube';
17
+ export const DEFAULT_TYPE = 'Video';
18
+
19
+ const MEDIA_FRAGS = 'http://www.w3.org/TR/media-frags/';
20
+ const PARAGRAPH_MS = 60_000;
21
+ // A caption language as a BCP 47 tag; it is part of a file name.
22
+ const LANGUAGE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/;
23
+
24
+ /** @param {string} inputPath */
25
+ export function detect(inputPath) {
26
+ return existsSync(inputPath) && statSync(inputPath).isDirectory() && existsSync(join(inputPath, CAPTURE_FILE));
27
+ }
28
+
29
+ /**
30
+ * @param {{ inputPath: string, options: object }} ctx
31
+ * @returns {import('./index.js').PackageDraft}
32
+ */
33
+ export function convert({ inputPath, options }) {
34
+ if (!detect(inputPath)) throw new UsageError(`${inputPath} is not a YouTube capture folder (no ${CAPTURE_FILE}); create one with moca-youtube.`);
35
+ const capture = readCapture(inputPath);
36
+ if (capture.videos.length === 0) throw new UsageError(`${inputPath} holds no videos.`);
37
+ const playlist = capture.kind === 'playlist';
38
+ const manifest = buildManifest({
39
+ ...options,
40
+ title: options.title ?? capture.title,
41
+ description: options.description ?? summary(capture.description),
42
+ });
43
+ const type = options.type ?? DEFAULT_TYPE;
44
+ const lang = options.language;
45
+ const iri = (fragment) => `${manifest.id}#${fragment}`;
46
+
47
+ const warnings = [];
48
+ const contentNodes = [];
49
+ const files = [];
50
+ const taken = new Set(playlist ? ['index'] : []);
51
+ const videos = capture.videos.map((id) => {
52
+ const dir = join(inputPath, 'videos', id);
53
+ const info = readInfo(id, join(dir, 'info.json'));
54
+ const vttPath = join(dir, 'captions.vtt');
55
+ const vtt = existsSync(vttPath) ? readFileSync(vttPath, 'utf8').replace(/\r\n?/g, '\n') : undefined;
56
+ return { info, vtt, slug: uniqueSlug(slugify(info.title), taken) };
57
+ });
58
+
59
+ for (const { info, vtt, slug } of videos) {
60
+ const captionLang = info.captions?.language ?? lang ?? 'und';
61
+ const captionsPath = `media/${slug}.${captionLang}.vtt`;
62
+ const descriptionPath = `sources/${slug}.description.txt`;
63
+ const description = info.description.replace(/\r\n?/g, '\n').trim();
64
+
65
+ const sources = [{ id: 'video', resource: info.webpage_url, title: `${info.title} (YouTube${info.channel ? `, ${info.channel}` : ''})` }];
66
+ const evidence = [];
67
+ if (description) {
68
+ files.push({ path: descriptionPath, body: `${description}\n` });
69
+ sources.push({ id: 'description', resource: `../${descriptionPath}`, title: 'Video description' });
70
+ evidence.push({ source: 'description', selector: { type: 'TextQuoteSelector', exact: firstLine(description) } });
71
+ }
72
+ const cues = vtt ? parseVtt(vtt) : [];
73
+ if (vtt) {
74
+ files.push({ path: captionsPath, body: vtt });
75
+ sources.push({ id: 'captions', resource: `../${captionsPath}`, title: `${info.captions?.automatic ? 'Automatic captions' : 'Captions'} (${captionLang})` });
76
+ for (const chapter of sections(info, cues)) {
77
+ evidence.push({
78
+ source: 'captions',
79
+ selector: { type: 'FragmentSelector', conformsTo: MEDIA_FRAGS, value: `t=${chapter.start},${chapter.end}` },
80
+ ...(chapter.title ? { note: chapter.title } : {}),
81
+ });
82
+ }
83
+ } else {
84
+ warnings.push({ code: 'YT_NO_CAPTIONS', message: `${info.title} has no captions; its node has no transcript.`, file: `videos/${info.id}` });
85
+ }
86
+
87
+ // Package spec §5.7: the video plays through its YouTube page, so it is described with the page's type. The
88
+ // duration bounds the chapter time ranges, which can end after the last caption.
89
+ const media = typeof info.duration === 'number' && info.duration >= 0
90
+ ? [{ source: 'video', type: 'text/html', duration: info.duration, ...(vtt ? { transcript: 'captions' } : {}) }]
91
+ : [];
92
+
93
+ const data = {
94
+ type,
95
+ title: info.title,
96
+ ...(description ? { description: summary(description) } : {}),
97
+ resource: info.webpage_url,
98
+ sources,
99
+ moca: {
100
+ ...(playlist ? { concepts: [{ iri: iri(`video-${info.id}`), role: 'primary' }] } : {}),
101
+ ...(evidence.length > 0 ? { evidence } : {}),
102
+ ...(media.length > 0 ? { media } : {}),
103
+ },
104
+ };
105
+ if (Object.keys(data.moca).length === 0) delete data.moca;
106
+ contentNodes.push({ path: `content/${slug}.md`, body: joinFrontmatter(data, videoBody(info, description, cues)) });
107
+ }
108
+
109
+ if (playlist) {
110
+ contentNodes.push({ path: 'content/index.md', body: playlistIndex(capture, videos) });
111
+ files.push({ path: 'structure.ttl', body: playlistStructure(capture, videos, { iri, lang, title: manifest.title }) });
112
+ }
113
+ return { manifest, contentNodes, files, warnings };
114
+ }
115
+
116
+ function videoBody(info, description, cues) {
117
+ const facts = [`[Watch on YouTube](<${info.webpage_url}>)`];
118
+ if (info.channel) facts.push(info.channel_url ? `[${escapeInline(info.channel)}](<${info.channel_url}>)` : escapeInline(info.channel));
119
+ if (info.upload_date) facts.push(`published ${info.upload_date}`);
120
+ if (info.duration) facts.push(clock(info.duration));
121
+ const out = [`# ${escapeInline(info.title)}`, facts.join(' · ')];
122
+ if (description) out.push('## Description', escapeText(description));
123
+ if (cues.length > 0) {
124
+ out.push('## Transcript');
125
+ for (const section of sections(info, cues)) {
126
+ if (section.title) out.push(`### ${clock(section.start)} ${escapeInline(section.title)}`);
127
+ for (const para of paragraphs(section.cues)) {
128
+ out.push(`[${clock(para.start / 1000)}](<https://youtu.be/${info.id}?t=${Math.floor(para.start / 1000)}>) ${escapeInline(para.text)}`);
129
+ }
130
+ }
131
+ }
132
+ return `\n${out.join('\n\n')}\n`;
133
+ }
134
+
135
+ /**
136
+ * The video's chapters with the cues each holds (by cue start), or the whole
137
+ * video as one untitled section. Times are whole seconds, except an end that
138
+ * would otherwise fall after the end of the video.
139
+ */
140
+ function sections(info, cues) {
141
+ // The end of the video: its duration, or the end of the last caption when later. A time range never ends after it,
142
+ // so it is kept exact rather than rounded up (Reader contract §7).
143
+ const end = Math.max(info.duration ?? 0, (cues.at(-1)?.end ?? 0) / 1000);
144
+ const chapters = info.chapters?.length > 0
145
+ ? info.chapters.map((c, i) => ({ start: Math.floor(c.start), end: Math.min(Math.ceil(c.end ?? end), end), title: chapterTitle(c.title, i) }))
146
+ : [{ start: 0, end, title: null }];
147
+ return chapters
148
+ .map((c, i) => ({
149
+ ...c,
150
+ cues: cues.filter((cue) => cue.start >= c.start * 1000 && (i === chapters.length - 1 || cue.start < chapters[i + 1].start * 1000)),
151
+ }))
152
+ .filter((c) => c.cues.length > 0 && c.end > c.start);
153
+ }
154
+
155
+ // YouTube names a chapter it had to invent, before the first timestamp in
156
+ // the description, "<Untitled Chapter 1>".
157
+ function chapterTitle(title, index) {
158
+ if (title && !/^<Untitled Chapter \d+>$/.test(title)) return title;
159
+ return index === 0 ? 'Opening' : `Chapter ${index + 1}`;
160
+ }
161
+
162
+ function paragraphs(cues) {
163
+ const out = [];
164
+ for (const cue of cues) {
165
+ const last = out.at(-1);
166
+ if (last && cue.start - last.start < PARAGRAPH_MS) last.text += ` ${cue.text}`;
167
+ else out.push({ start: cue.start, text: cue.text });
168
+ }
169
+ return out;
170
+ }
171
+
172
+ function playlistIndex(capture, videos) {
173
+ const lines = [`# ${escapeInline(capture.title)}`, ''];
174
+ const description = (capture.description ?? '').trim();
175
+ if (description) lines.push(escapeText(description), '');
176
+ lines.push(capture.url ? `Videos in [playlist](<${capture.url}>) order:` : 'Videos in playlist order:', '');
177
+ videos.forEach(({ info, slug }, i) => lines.push(`${i + 1}. [${escapeInline(info.title)}](${slug}.md)`));
178
+ return `${lines.join('\n')}\n`;
179
+ }
180
+
181
+ function playlistStructure(capture, videos, { iri, lang, title }) {
182
+ const label = (text) => `${turtleString(text)}${lang ? `@${lang}` : ''}`;
183
+ const ref = (fragment) => `<${iri(fragment)}>`;
184
+ const members = videos.map(({ info }) => ref(`video-${info.id}`));
185
+ const blocks = [
186
+ '@prefix skos: <http://www.w3.org/2004/02/skos/core#> .\n@prefix dct: <http://purl.org/dc/terms/> .',
187
+ `${ref('playlist')} a skos:Concept ;\n skos:prefLabel ${label(capture.title ?? title)} ;\n dct:hasPart ${members.join(',\n ')} .`,
188
+ `${ref('playlist-order')} a skos:OrderedCollection ;\n skos:prefLabel ${label(`${capture.title ?? title} (playlist order)`)} ;\n skos:memberList (\n ${members.join('\n ')}\n ) .`,
189
+ ...videos.map(({ info }) => `${ref(`video-${info.id}`)} a skos:Concept ;\n skos:prefLabel ${label(info.title)} .`),
190
+ ];
191
+ return `${blocks.join('\n\n')}\n`;
192
+ }
193
+
194
+ /** Seconds as m:ss or h:mm:ss, as YouTube shows them. */
195
+ function clock(seconds) {
196
+ const s = Math.floor(seconds);
197
+ const h = Math.floor(s / 3600);
198
+ const m = Math.floor((s % 3600) / 60);
199
+ const ss = String(s % 60).padStart(2, '0');
200
+ return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${ss}` : `${m}:${ss}`;
201
+ }
202
+
203
+ function readJson(path) {
204
+ try {
205
+ return JSON.parse(readFileSync(path, 'utf8'));
206
+ } catch (err) {
207
+ throw new UsageError(`${path} is not valid JSON: ${err.message}`);
208
+ }
209
+ }
210
+
211
+ const text = (value) => (typeof value === 'string' ? value : undefined);
212
+ const seconds = (value) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined);
213
+
214
+ /**
215
+ * The capture file, checked as it is read: video ids are plain ids (they name
216
+ * folders and go into links), and the playlist URL is parsed and rebuilt.
217
+ */
218
+ function readCapture(inputPath) {
219
+ const raw = readJson(join(inputPath, CAPTURE_FILE));
220
+ if (!Array.isArray(raw?.videos)) throw new UsageError(`${CAPTURE_FILE} needs a "videos" list.`);
221
+ for (const id of raw.videos) {
222
+ if (typeof id !== 'string' || !VIDEO_ID.test(id)) throw new UsageError(`${CAPTURE_FILE}: ${JSON.stringify(id)} is not a video id.`);
223
+ }
224
+ return {
225
+ kind: raw.kind === 'playlist' ? 'playlist' : 'video',
226
+ title: text(raw.title),
227
+ description: text(raw.description) ?? '',
228
+ url: httpUrl(raw.url),
229
+ videos: raw.videos,
230
+ };
231
+ }
232
+
233
+ /**
234
+ * A video's info.json, checked as it is read. It must be the video the
235
+ * capture lists. URLs are parsed and rebuilt (the watch page falls back to the
236
+ * one the id gives), the date must be a real YYYYMMDD date, times are numbers,
237
+ * and the caption language is a language tag. Anything else is left out.
238
+ */
239
+ function readInfo(id, path) {
240
+ const raw = readJson(path);
241
+ if (raw?.id !== id) throw new UsageError(`${path} is not the info for video ${id}.`);
242
+ const captions = raw.captions && typeof raw.captions === 'object'
243
+ ? { language: LANGUAGE.test(text(raw.captions.language) ?? '') ? raw.captions.language : 'und', automatic: raw.captions.automatic === true }
244
+ : null;
245
+ return {
246
+ id,
247
+ title: text(raw.title)?.trim() || id,
248
+ description: text(raw.description) ?? '',
249
+ channel: text(raw.channel)?.trim() || null,
250
+ channel_url: httpUrl(raw.channel_url) ?? null,
251
+ upload_date: compactDate(raw.upload_date),
252
+ duration: seconds(raw.duration) ?? null,
253
+ webpage_url: httpUrl(raw.webpage_url) ?? watchUrl(id),
254
+ chapters: (Array.isArray(raw.chapters) ? raw.chapters : [])
255
+ .filter((c) => seconds(c?.start) !== undefined)
256
+ .map((c) => ({ start: c.start, ...(seconds(c.end) !== undefined ? { end: c.end } : {}), title: text(c.title) })),
257
+ captions,
258
+ };
259
+ }