@openmoca/moca-convert 0.0.0-stage → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +133 -2
- package/bin/moca-convert.js +170 -0
- package/bin/moca-youtube.js +93 -0
- package/lib/adapters/directory.js +134 -0
- package/lib/adapters/index.js +60 -0
- package/lib/adapters/markdown.js +65 -0
- package/lib/adapters/obsidian.js +199 -0
- package/lib/adapters/openapi/parse.js +114 -0
- package/lib/adapters/openapi/render.js +141 -0
- package/lib/adapters/openapi/suitability.js +41 -0
- package/lib/adapters/openapi.js +77 -0
- package/lib/adapters/pdf.js +270 -0
- package/lib/adapters/youtube.js +194 -0
- package/lib/detect.js +128 -0
- package/lib/frontmatter.js +42 -0
- package/lib/glob.js +33 -0
- package/lib/index.js +4 -0
- package/lib/manifest.js +23 -0
- package/lib/pdf/extract.js +176 -0
- package/lib/pdf/layout.js +270 -0
- package/lib/slug.js +19 -0
- package/lib/target.js +35 -0
- package/lib/text.js +59 -0
- package/lib/walk.js +34 -0
- package/lib/write.js +98 -0
- package/lib/youtube/capture.js +117 -0
- package/lib/youtube/json3-to-vtt.js +109 -0
- package/lib/youtube/ytdlp.js +58 -0
- package/package.json +42 -4
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// Turns the positioned lines from lib/pdf/extract.js into a flat list of
|
|
2
|
+
// headings and paragraphs. Pure functions: no pdf.js here, so the rules can be
|
|
3
|
+
// tested with hand-made pages.
|
|
4
|
+
//
|
|
5
|
+
// - Running headers and footers (lines in the page margins that repeat across
|
|
6
|
+
// pages, numbers aside) and bare page numbers are dropped.
|
|
7
|
+
// - Headings come from the outline when the PDF has one, otherwise from type
|
|
8
|
+
// noticeably larger than the body text.
|
|
9
|
+
// - Body lines are joined into paragraphs; a paragraph may run on across a
|
|
10
|
+
// column or a page break. Words hyphenated at a line end are rejoined.
|
|
11
|
+
|
|
12
|
+
const MARGIN = 0.08;
|
|
13
|
+
const HEADING_RATIO = 1.12;
|
|
14
|
+
const SMALL_RATIO = 0.8;
|
|
15
|
+
const MAX_HEADING_CHARS = 150;
|
|
16
|
+
// Bullet glyphs, including the private-use ones Symbol and Wingdings map to.
|
|
17
|
+
const BULLET = /^[•▪■●◦‣∙·–⦁○□➢➤►]\s*/u;
|
|
18
|
+
// "1. Power", "12.Loss of control", "3) Configuration"; not "91.417 requires…".
|
|
19
|
+
const NUMBERED = /^(\d{1,2})[.)](?:\s+(?=\S)|(?=\p{L}))/u;
|
|
20
|
+
// "Figure 4-6. The…", "Table 2: …", "Figure 4-1 A-F. …"; not "Figure 1-5 shows…".
|
|
21
|
+
const CAPTION = /^(figure|table|fig\.)\s+\d+(?:[-–.]\d+)*(?:\s*[A-Z](?:[-–][A-Z])?)?[.:]/i;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @typedef {{ kind: 'heading', level: number, text: string, page: number }} HeadingBlock
|
|
25
|
+
* @typedef {{ kind: 'paragraph' | 'item' | 'caption', text: string, page: number, lastPage?: number, number?: number }} TextBlock
|
|
26
|
+
* `lastPage` is set when the text runs on to a later page; an item with
|
|
27
|
+
* `number` is from a numbered list
|
|
28
|
+
* @typedef {HeadingBlock | TextBlock} Block
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {import('./extract.js').PdfText} pdf
|
|
33
|
+
* @returns {{ blocks: Block[], bodySize: number, labels: Map<number, string> }}
|
|
34
|
+
* labels: printed page label by physical page number, where one is known
|
|
35
|
+
*/
|
|
36
|
+
export function layoutPdf(pdf) {
|
|
37
|
+
const pages = dropRunningLines(pdf.pages);
|
|
38
|
+
const bodySize = bodyFontSize(pages);
|
|
39
|
+
const lines = pages.flatMap((page) => page.lines.map((line) => ({ ...line, page: page.number })));
|
|
40
|
+
const levels = usefulOutline(pdf.outline, pages.length) ? outlineHeadings(lines, pdf.outline) : fontHeadings(lines, bodySize);
|
|
41
|
+
const labels = new Map(pages.filter((p) => p.label).map((p) => [p.number, p.label]));
|
|
42
|
+
return { blocks: toBlocks(lines, levels, bodySize), bodySize, labels };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Removes lines in the top and bottom margins that repeat on at least a third
|
|
47
|
+
* of the pages (digits ignored, so "4-12" and "4-13" match), and margin lines
|
|
48
|
+
* that are only a page number. A page without a label in the PDF takes the
|
|
49
|
+
* printed page number it drops ("4-12").
|
|
50
|
+
* @param {import('./extract.js').PdfPage[]} pages
|
|
51
|
+
*/
|
|
52
|
+
export function dropRunningLines(pages) {
|
|
53
|
+
const inMargin = (page, line) => line.y > page.height * (1 - MARGIN) || line.y < page.height * MARGIN;
|
|
54
|
+
const key = (text) => text.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
|
|
55
|
+
const seen = new Map();
|
|
56
|
+
for (const page of pages) {
|
|
57
|
+
const keys = new Set(page.lines.filter((l) => inMargin(page, l)).map((l) => key(l.text)));
|
|
58
|
+
for (const k of keys) seen.set(k, (seen.get(k) ?? 0) + 1);
|
|
59
|
+
}
|
|
60
|
+
const repeats = Math.max(2, Math.ceil(pages.length / 3));
|
|
61
|
+
return pages.map((page) => {
|
|
62
|
+
let label = page.label;
|
|
63
|
+
const lines = page.lines.filter((l) => {
|
|
64
|
+
if (!inMargin(page, l)) return true;
|
|
65
|
+
if (isPageNumber(l.text, page.label)) {
|
|
66
|
+
if (!label && /\d/.test(l.text)) label = l.text.trim().replace(/^page\s+/i, '');
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
return !(pages.length > 2 && seen.get(key(l.text)) >= repeats);
|
|
70
|
+
});
|
|
71
|
+
return { ...page, ...(label ? { label } : {}), lines };
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function isPageNumber(text, label) {
|
|
76
|
+
const t = text.trim();
|
|
77
|
+
return t === label || /^(page\s+)?[\dA-Z]{0,3}[-–.]?\d{1,4}$/i.test(t) || /^[ivxlc]+$/i.test(t);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The size of most of the text, weighted by characters. */
|
|
81
|
+
export function bodyFontSize(pages) {
|
|
82
|
+
const weight = new Map();
|
|
83
|
+
for (const page of pages) {
|
|
84
|
+
for (const line of page.lines) weight.set(line.size, (weight.get(line.size) ?? 0) + line.text.length);
|
|
85
|
+
}
|
|
86
|
+
let best = 0;
|
|
87
|
+
let most = -1;
|
|
88
|
+
for (const [size, n] of weight) {
|
|
89
|
+
if (n > most) {
|
|
90
|
+
best = size;
|
|
91
|
+
most = n;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return best;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Heading levels from type size: every size clearly larger than the body is
|
|
99
|
+
* a level, largest first.
|
|
100
|
+
* @returns {Map<number, number>} line index -> level
|
|
101
|
+
*/
|
|
102
|
+
export function fontHeadings(lines, bodySize) {
|
|
103
|
+
const headingText = (line) => line.text.length <= MAX_HEADING_CHARS && /\p{L}/u.test(line.text) && !CAPTION.test(line.text);
|
|
104
|
+
// Large type may be a single letter (a glossary's "A", "B", …).
|
|
105
|
+
const large = (line) => line.size >= bodySize * HEADING_RATIO && headingText(line) && !/[,;]$/.test(line.text);
|
|
106
|
+
// A whole line in bold at body size, not ending like a sentence, is a
|
|
107
|
+
// run-in heading one level below the smallest large heading. It needs two
|
|
108
|
+
// letters in a row, so bold figure callouts ("A", "1a") are not headings.
|
|
109
|
+
const bold = (line) => line.bold && !large(line) && line.size >= bodySize * 0.95 && headingText(line)
|
|
110
|
+
&& /\p{L}{2}/u.test(line.text) && !/[.,;:]$/.test(line.text);
|
|
111
|
+
const sizes = [...new Set(lines.filter(large).map((l) => l.size))].sort((a, b) => b - a).slice(0, 5);
|
|
112
|
+
const out = new Map();
|
|
113
|
+
lines.forEach((line, i) => {
|
|
114
|
+
if (large(line)) out.set(i, sizes.indexOf(line.size) + 1);
|
|
115
|
+
else if (bold(line)) out.set(i, sizes.length + 1);
|
|
116
|
+
});
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Whether the outline describes the whole document. Some tools export only a
|
|
122
|
+
* "Structure Bookmarks" placeholder and the title, or bookmarks for a few
|
|
123
|
+
* sections near the end; the type sizes are a better guide then.
|
|
124
|
+
*/
|
|
125
|
+
export function usefulOutline(outline, pageCount) {
|
|
126
|
+
const placed = (outline ?? []).filter((e) => e.page);
|
|
127
|
+
if (placed.length < 3 || new Set(placed.map((e) => e.page)).size < 2) return false;
|
|
128
|
+
return Math.min(...placed.map((e) => e.page)) <= Math.max(2, Math.ceil(pageCount * 0.1));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Heading levels from the outline: each entry marks the first line on its
|
|
133
|
+
* page, or a later page, whose text starts the entry's title. An entry whose
|
|
134
|
+
* title is not found marks the first line of its page.
|
|
135
|
+
* @param {Array<{ page: number, text: string }>} lines
|
|
136
|
+
* @param {import('./extract.js').OutlineEntry[]} outline
|
|
137
|
+
* @returns {Map<number, number>} line index -> level
|
|
138
|
+
*/
|
|
139
|
+
export function outlineHeadings(lines, outline) {
|
|
140
|
+
const out = new Map();
|
|
141
|
+
const norm = (s) => s.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
|
|
142
|
+
let from = 0;
|
|
143
|
+
for (const entry of outline) {
|
|
144
|
+
if (!entry.page) continue;
|
|
145
|
+
const title = norm(entry.title);
|
|
146
|
+
const start = Math.max(from, lines.findIndex((l) => l.page >= entry.page));
|
|
147
|
+
if (start < 0 || start >= lines.length) continue;
|
|
148
|
+
let hit = -1;
|
|
149
|
+
for (let i = start; i < lines.length && lines[i].page <= entry.page + 1; i++) {
|
|
150
|
+
const text = norm(lines[i].text);
|
|
151
|
+
if (text && (title.startsWith(text) || text.startsWith(title)) && text.length >= Math.min(4, title.length)) {
|
|
152
|
+
hit = i;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const at = hit >= 0 ? hit : start;
|
|
157
|
+
if (!out.has(at)) out.set(at, Math.min(entry.level, 6));
|
|
158
|
+
from = at + 1;
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* @param {Array<import('./extract.js').PdfLine & { page: number }>} lines
|
|
165
|
+
* @param {Map<number, number>} levels
|
|
166
|
+
* @param {number} bodySize
|
|
167
|
+
* @returns {Block[]}
|
|
168
|
+
*/
|
|
169
|
+
export function toBlocks(lines, levels, bodySize) {
|
|
170
|
+
const pitch = linePitch(lines, bodySize) / (bodySize || 1);
|
|
171
|
+
const blocks = [];
|
|
172
|
+
let para = null;
|
|
173
|
+
let prev = null;
|
|
174
|
+
const flush = () => {
|
|
175
|
+
if (para && para.text.trim()) {
|
|
176
|
+
blocks.push({
|
|
177
|
+
kind: para.kind, text: para.text.trim(), page: para.page,
|
|
178
|
+
...(para.lastPage > para.page ? { lastPage: para.lastPage } : {}),
|
|
179
|
+
...(para.number ? { number: para.number } : {}),
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
para = null;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
lines.forEach((line, i) => {
|
|
186
|
+
const level = levels.get(i);
|
|
187
|
+
if (level) {
|
|
188
|
+
flush();
|
|
189
|
+
const last = blocks.at(-1);
|
|
190
|
+
// A heading set on two lines is one heading.
|
|
191
|
+
if (last?.kind === 'heading' && last.level === level && prev && levels.get(i - 1) === level
|
|
192
|
+
&& last.page === line.page && prev.y - line.y < line.size * 2 && prev.y > line.y) {
|
|
193
|
+
last.text = joinLine(last.text, line.text);
|
|
194
|
+
} else {
|
|
195
|
+
blocks.push({ kind: 'heading', level, text: line.text, page: line.page });
|
|
196
|
+
}
|
|
197
|
+
prev = line;
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (line.size < bodySize * SMALL_RATIO && !CAPTION.test(line.text)) {
|
|
201
|
+
// Small type is figure labelling or a footnote marker; left out.
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
const numbered = NUMBERED.exec(line.text);
|
|
205
|
+
const kind = CAPTION.test(line.text) ? 'caption' : BULLET.test(line.text) || numbered ? 'item' : 'paragraph';
|
|
206
|
+
const text = kind !== 'item' ? line.text : numbered ? line.text.slice(numbered[0].length) : line.text.replace(BULLET, '');
|
|
207
|
+
if (!para || kind !== 'paragraph' || breaksParagraph(prev, line, para, pitch)) {
|
|
208
|
+
flush();
|
|
209
|
+
para = { kind, text, page: line.page, ...(numbered && kind === 'item' ? { number: Number(numbered[1]) } : {}) };
|
|
210
|
+
} else {
|
|
211
|
+
para.text = joinLine(para.text, text);
|
|
212
|
+
para.lastPage = line.page;
|
|
213
|
+
}
|
|
214
|
+
prev = line;
|
|
215
|
+
});
|
|
216
|
+
flush();
|
|
217
|
+
return blocks;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The usual distance between baselines of body text: the most common gap
|
|
222
|
+
* between consecutive body-size lines on a page.
|
|
223
|
+
*/
|
|
224
|
+
export function linePitch(lines, bodySize) {
|
|
225
|
+
const counts = new Map();
|
|
226
|
+
for (let i = 1; i < lines.length; i++) {
|
|
227
|
+
const [a, b] = [lines[i - 1], lines[i]];
|
|
228
|
+
if (a.page !== b.page || Math.abs(a.size - bodySize) > 0.5 || Math.abs(b.size - bodySize) > 0.5) continue;
|
|
229
|
+
const gap = Math.round((a.y - b.y) * 2) / 2;
|
|
230
|
+
if (gap > 0 && gap < bodySize * 3) counts.set(gap, (counts.get(gap) ?? 0) + 1);
|
|
231
|
+
}
|
|
232
|
+
let best = bodySize * 1.2;
|
|
233
|
+
let most = 0;
|
|
234
|
+
for (const [gap, n] of counts) {
|
|
235
|
+
if (n > most) {
|
|
236
|
+
best = gap;
|
|
237
|
+
most = n;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return best;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Whether `line` starts a new paragraph after `prev`.
|
|
245
|
+
* @param {number} pitch line pitch as a multiple of the font size
|
|
246
|
+
*/
|
|
247
|
+
function breaksParagraph(prev, line, para, pitch) {
|
|
248
|
+
if (!prev) return true;
|
|
249
|
+
const ended = /[.!?:"”)]$/.test(para.text);
|
|
250
|
+
if (Math.abs(line.size - prev.size) > prev.size * 0.15) return true;
|
|
251
|
+
if (line.page !== prev.page || line.y > prev.y + prev.size) {
|
|
252
|
+
// A new page or a new column: carry on unless the text had finished.
|
|
253
|
+
return ended;
|
|
254
|
+
}
|
|
255
|
+
// More space between the lines than the usual line pitch, scaled to the
|
|
256
|
+
// size of this text.
|
|
257
|
+
return prev.y - line.y > pitch * prev.size * 1.25;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Joins two lines, rejoining a word hyphenated at the break ("encap-" +
|
|
262
|
+
* "sulating"). A hyphen after anything but a lowercase letter ("U.S.-" +
|
|
263
|
+
* "registered") and a slash ("accelerate/" + "decelerate") are kept, with no
|
|
264
|
+
* space added.
|
|
265
|
+
*/
|
|
266
|
+
export function joinLine(a, b) {
|
|
267
|
+
if (/\p{Ll}-$/u.test(a) && /^\p{Ll}/u.test(b)) return a.slice(0, -1) + b;
|
|
268
|
+
if (/[-/]$/.test(a) && /^\p{L}/u.test(b)) return a + b;
|
|
269
|
+
return `${a} ${b}`;
|
|
270
|
+
}
|
package/lib/slug.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic, filesystem-safe, ASCII-only slug from a title string.
|
|
3
|
+
* NFKD-normalizing first turns an accented character into a base letter
|
|
4
|
+
* plus a separate combining-mark codepoint (e.g. "é" -> "e" + U+0301), so
|
|
5
|
+
* simply dropping every non-ASCII codepoint afterward strips diacritics
|
|
6
|
+
* while keeping the underlying letters.
|
|
7
|
+
* @param {string} text
|
|
8
|
+
* @returns {string}
|
|
9
|
+
*/
|
|
10
|
+
export function slugify(text) {
|
|
11
|
+
return (
|
|
12
|
+
text
|
|
13
|
+
.normalize('NFKD')
|
|
14
|
+
.replace(/[^\x00-\x7F]/g, '')
|
|
15
|
+
.toLowerCase()
|
|
16
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
17
|
+
.replace(/^-+|-+$/g, '') || 'untitled'
|
|
18
|
+
);
|
|
19
|
+
}
|
package/lib/target.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { existsSync, statSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Resolves and validates a conversion input path exists on disk, and reports
|
|
5
|
+
* whether it's a file or a directory. Glob inputs (containing *, ?, or [)
|
|
6
|
+
* are not resolvable via stat and are passed through unchanged for the
|
|
7
|
+
* markdown adapter to expand itself.
|
|
8
|
+
*
|
|
9
|
+
* @param {string} inputPath
|
|
10
|
+
* @returns {{ inputPath: string, isGlob: boolean, isDirectory: boolean, isFile: boolean }}
|
|
11
|
+
*/
|
|
12
|
+
export function resolveInputTarget(inputPath) {
|
|
13
|
+
if (isGlobPattern(inputPath)) {
|
|
14
|
+
return { inputPath, isGlob: true, isDirectory: false, isFile: false };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if (!existsSync(inputPath)) {
|
|
18
|
+
throw new UsageError(`Input does not exist: ${inputPath}`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const stats = statSync(inputPath);
|
|
22
|
+
return {
|
|
23
|
+
inputPath,
|
|
24
|
+
isGlob: false,
|
|
25
|
+
isDirectory: stats.isDirectory(),
|
|
26
|
+
isFile: stats.isFile(),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** @param {string} inputPath */
|
|
31
|
+
export function isGlobPattern(inputPath) {
|
|
32
|
+
return /[*?[\]]/.test(inputPath);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class UsageError extends Error {}
|
package/lib/text.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Helpers shared by adapters that write Markdown from text that was not
|
|
2
|
+
// written as Markdown (YouTube descriptions, text extracted from PDFs).
|
|
3
|
+
|
|
4
|
+
// Source text is shown as written, not interpreted as Markdown: characters
|
|
5
|
+
// that would start emphasis, links or HTML are escaped. Bare URLs become
|
|
6
|
+
// autolinks.
|
|
7
|
+
export function escapeInline(text) {
|
|
8
|
+
return String(text)
|
|
9
|
+
.split(/(https?:\/\/[^\s<>]+)/)
|
|
10
|
+
.map((part, i) => (i % 2 === 1 ? `<${part}>` : part.replace(/([\\`*_[\]<>|])/g, '\\$1')))
|
|
11
|
+
.join('');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// Multi-line text: blank lines separate paragraphs and single line breaks are
|
|
15
|
+
// kept as hard breaks. A line that would start a heading, list, quote or
|
|
16
|
+
// code block is escaped.
|
|
17
|
+
export function escapeText(text) {
|
|
18
|
+
return text
|
|
19
|
+
.split(/\n\s*\n/)
|
|
20
|
+
.map((para) =>
|
|
21
|
+
para
|
|
22
|
+
.split('\n')
|
|
23
|
+
.map((line) => escapeLineStart(escapeInline(line.trim())))
|
|
24
|
+
.filter(Boolean)
|
|
25
|
+
.join('\\\n'))
|
|
26
|
+
.filter(Boolean)
|
|
27
|
+
.join('\n\n');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Escapes a line start that Markdown would read as a heading, list or quote. */
|
|
31
|
+
export function escapeLineStart(line) {
|
|
32
|
+
return line.replace(/^([#>+-])/, '\\$1').replace(/^(\d+)([.)])/, '$1\\$2');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** First non-empty line, trimmed. */
|
|
36
|
+
export function firstLine(text) {
|
|
37
|
+
return text.split('\n').map((l) => l.trim()).find(Boolean) ?? '';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A one-line summary for `description`: the first line, cut at a sentence when long. */
|
|
41
|
+
export function summary(text) {
|
|
42
|
+
const line = firstLine(text ?? '');
|
|
43
|
+
if (line.length <= 300) return line || undefined;
|
|
44
|
+
const cut = line.slice(0, 300);
|
|
45
|
+
const stop = cut.lastIndexOf('. ');
|
|
46
|
+
return stop > 80 ? cut.slice(0, stop + 1) : `${cut.slice(0, cut.lastIndexOf(' '))}…`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** `base`, or `base-2`, `base-3`… when `taken` already has it; records the result. */
|
|
50
|
+
export function uniqueSlug(base, taken) {
|
|
51
|
+
let slug = base;
|
|
52
|
+
for (let n = 2; taken.has(slug); n++) slug = `${base}-${n}`;
|
|
53
|
+
taken.add(slug);
|
|
54
|
+
return slug;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function turtleString(text) {
|
|
58
|
+
return `"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`;
|
|
59
|
+
}
|
package/lib/walk.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Recursively lists files under `dir`, returning paths relative to `rootDir`
|
|
6
|
+
* (using forward slashes regardless of platform), sorted lexicographically
|
|
7
|
+
* for deterministic iteration order.
|
|
8
|
+
*
|
|
9
|
+
* @param {string} dir
|
|
10
|
+
* @param {string} rootDir
|
|
11
|
+
* @returns {string[]}
|
|
12
|
+
*/
|
|
13
|
+
export function walkFiles(dir, rootDir = dir) {
|
|
14
|
+
if (!existsSyncSafe(dir)) return [];
|
|
15
|
+
const entries = readdirSync(dir, { withFileTypes: true });
|
|
16
|
+
const results = [];
|
|
17
|
+
for (const entry of entries) {
|
|
18
|
+
const full = join(dir, entry.name);
|
|
19
|
+
if (entry.isDirectory()) {
|
|
20
|
+
results.push(...walkFiles(full, rootDir));
|
|
21
|
+
} else if (entry.isFile()) {
|
|
22
|
+
results.push(relative(rootDir, full).split('\\').join('/'));
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return results.sort();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function existsSyncSafe(dir) {
|
|
29
|
+
try {
|
|
30
|
+
return statSync(dir).isDirectory();
|
|
31
|
+
} catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
}
|
package/lib/write.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Writes a PackageDraft (see lib/adapters/index.js) to disk and gates
|
|
2
|
+
// success on the reference Reader reporting zero error-severity diagnostics:
|
|
3
|
+
// read first, refuse to leave invalid output behind.
|
|
4
|
+
import { existsSync, mkdirSync, mkdtempSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import { dirname, join } from 'node:path';
|
|
7
|
+
import { readPackage } from '@openmoca/moca-core';
|
|
8
|
+
import { UsageError } from './target.js';
|
|
9
|
+
|
|
10
|
+
export { UsageError };
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {object} params
|
|
14
|
+
* @param {import('./adapters/index.js').PackageDraft} params.draft
|
|
15
|
+
* @param {string} params.outDir
|
|
16
|
+
* @param {boolean} [params.force]
|
|
17
|
+
* @param {boolean} [params.strict]
|
|
18
|
+
* @param {(entry: string) => void} [params.onLog]
|
|
19
|
+
* @returns {Promise<{ findings: object[], digest: string }>}
|
|
20
|
+
*/
|
|
21
|
+
export async function writeDraft({ draft, outDir, force = false, strict = false, onLog = () => {} }) {
|
|
22
|
+
checkFiles(draft);
|
|
23
|
+
const outDirExists = existsSync(outDir);
|
|
24
|
+
const outDirNonEmpty = outDirExists && readdirSync(outDir).length > 0;
|
|
25
|
+
|
|
26
|
+
if (outDirNonEmpty && !force) {
|
|
27
|
+
throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to overwrite.`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const useScratch = outDirNonEmpty;
|
|
31
|
+
const stageDir = useScratch ? mkdtempSync(join(tmpdir(), 'moca-convert-')) : outDir;
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
stageDraft(draft, stageDir);
|
|
35
|
+
|
|
36
|
+
onLog(`reading ${stageDir}`);
|
|
37
|
+
const result = await readPackage(stageDir, { strict });
|
|
38
|
+
const findings = result.diagnostics;
|
|
39
|
+
const errors = findings.filter((f) => f.severity === 'error');
|
|
40
|
+
|
|
41
|
+
if (errors.length > 0) {
|
|
42
|
+
throw new ConversionFailedError(
|
|
43
|
+
'The converted package failed its own validation; no output was written.',
|
|
44
|
+
findings
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (useScratch) {
|
|
49
|
+
rmSync(outDir, { recursive: true, force: true });
|
|
50
|
+
renameSync(stageDir, outDir);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return { findings, digest: result.digest };
|
|
54
|
+
} catch (err) {
|
|
55
|
+
// Never leave partial/invalid output behind: on any failure, remove the
|
|
56
|
+
// staged directory. In scratch mode that's a temp dir, so the
|
|
57
|
+
// pre-existing outDir is untouched; in direct mode stageDir === outDir,
|
|
58
|
+
// so a freshly-created output directory is fully cleaned up.
|
|
59
|
+
if (existsSync(stageDir)) {
|
|
60
|
+
rmSync(stageDir, { recursive: true, force: true });
|
|
61
|
+
}
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function stageDraft(draft, stageDir) {
|
|
67
|
+
mkdirSync(stageDir, { recursive: true });
|
|
68
|
+
writeFileSync(join(stageDir, 'moca.json'), `${JSON.stringify(draft.manifest, null, 2)}\n`);
|
|
69
|
+
|
|
70
|
+
for (const node of [...draft.contentNodes, ...(draft.files ?? [])]) {
|
|
71
|
+
const absPath = join(stageDir, node.path);
|
|
72
|
+
mkdirSync(dirname(absPath), { recursive: true });
|
|
73
|
+
writeFileSync(absPath, node.body);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Adapters may add the evidence layer and the structure layer beside
|
|
78
|
+
// content/, and nothing else.
|
|
79
|
+
const OTHER_FILE = /^(?:(?:sources|media)\/(?:[^/]+\/)*[^/]+|structure\.ttl)$/;
|
|
80
|
+
|
|
81
|
+
function checkFiles(draft) {
|
|
82
|
+
for (const file of draft.files ?? []) {
|
|
83
|
+
if (!OTHER_FILE.test(file.path) || file.path.split('/').some((seg) => seg === '..' || seg === '.')) {
|
|
84
|
+
throw new UsageError(`An adapter may only add files under sources/ or media/, or structure.ttl; refusing "${file.path}".`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export class ConversionFailedError extends Error {
|
|
90
|
+
/**
|
|
91
|
+
* @param {string} message
|
|
92
|
+
* @param {object[]} findings
|
|
93
|
+
*/
|
|
94
|
+
constructor(message, findings) {
|
|
95
|
+
super(message);
|
|
96
|
+
this.findings = findings;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// Collects what the youtube adapter converts: a capture folder holding each
|
|
2
|
+
// video's metadata and captions, in the order YouTube lists them.
|
|
3
|
+
//
|
|
4
|
+
// youtube-capture.json what was captured, and the video order
|
|
5
|
+
// videos/<id>/info.json the video's metadata, stable fields only
|
|
6
|
+
// videos/<id>/captions.vtt its captions as clean WebVTT, when it has any
|
|
7
|
+
//
|
|
8
|
+
// Everything that changes from one fetch to the next (view counts, signed
|
|
9
|
+
// media URLs, thumbnails) is left out, so a capture is worth reviewing and
|
|
10
|
+
// converting it twice gives the same package.
|
|
11
|
+
import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { UsageError } from '../target.js';
|
|
14
|
+
import { json3ToVtt } from './json3-to-vtt.js';
|
|
15
|
+
|
|
16
|
+
export const CAPTURE_FILE = 'youtube-capture.json';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @param {object} params
|
|
20
|
+
* @param {string} params.url
|
|
21
|
+
* @param {string} params.captureDir
|
|
22
|
+
* @param {string} [params.language] caption language, BCP 47 (default en)
|
|
23
|
+
* @param {boolean} [params.force]
|
|
24
|
+
* @param {ReturnType<import('./ytdlp.js').ytDlp>} params.ytdlp
|
|
25
|
+
* @param {(line: string) => void} [params.log]
|
|
26
|
+
* @returns {Promise<{ kind: string, videos: number, warnings: string[] }>}
|
|
27
|
+
*/
|
|
28
|
+
export async function capture({ url, captureDir, language = 'en', force = false, ytdlp, log = () => {} }) {
|
|
29
|
+
if (existsSync(captureDir) && readdirSync(captureDir).length > 0 && !force) {
|
|
30
|
+
throw new UsageError(`Capture directory "${captureDir}" is not empty; pass --force to overwrite.`);
|
|
31
|
+
}
|
|
32
|
+
const warnings = [];
|
|
33
|
+
log(`reading ${url}`);
|
|
34
|
+
const top = await ytdlp.json(['--flat-playlist', url]);
|
|
35
|
+
const isPlaylist = top._type === 'playlist';
|
|
36
|
+
const entries = isPlaylist ? (top.entries ?? []).filter((e) => e?.id) : [top];
|
|
37
|
+
|
|
38
|
+
const videos = [];
|
|
39
|
+
for (const [i, entry] of entries.entries()) {
|
|
40
|
+
const label = `${i + 1}/${entries.length} ${entry.title ?? entry.id}`;
|
|
41
|
+
let info;
|
|
42
|
+
try {
|
|
43
|
+
info = isPlaylist ? await ytdlp.json(['--skip-download', watchUrl(entry.id)]) : entry;
|
|
44
|
+
} catch (err) {
|
|
45
|
+
warnings.push(`Skipped ${entry.id}: ${err.message}`);
|
|
46
|
+
log(`skip ${label}`);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const track = chooseTrack(info, language);
|
|
50
|
+
let vtt;
|
|
51
|
+
if (track) {
|
|
52
|
+
log(`fetch ${label} (${track.manual ? 'captions' : 'automatic captions'}, ${track.language})`);
|
|
53
|
+
const json3 = await ytdlp.captions(info, track);
|
|
54
|
+
vtt = json3ToVtt(json3, `${track.manual ? 'Captions' : 'Automatic captions'} for ${watchUrl(info.id)}, from YouTube.`);
|
|
55
|
+
} else {
|
|
56
|
+
warnings.push(`${info.id} has no ${language} captions.`);
|
|
57
|
+
log(`fetch ${label} (no ${language} captions)`);
|
|
58
|
+
}
|
|
59
|
+
const dir = join(captureDir, 'videos', info.id);
|
|
60
|
+
mkdirSync(dir, { recursive: true });
|
|
61
|
+
writeJson(join(dir, 'info.json'), trimInfo(info, track));
|
|
62
|
+
if (vtt) writeFileSync(join(dir, 'captions.vtt'), vtt);
|
|
63
|
+
videos.push(info.id);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const first = entries[0] ?? {};
|
|
67
|
+
writeJson(join(captureDir, CAPTURE_FILE), {
|
|
68
|
+
kind: isPlaylist ? 'playlist' : 'video',
|
|
69
|
+
id: top.id,
|
|
70
|
+
url: isPlaylist ? `https://www.youtube.com/playlist?list=${top.id}` : watchUrl(top.id),
|
|
71
|
+
title: top.title,
|
|
72
|
+
description: top.description ?? '',
|
|
73
|
+
channel: top.channel ?? top.uploader ?? first.channel ?? null,
|
|
74
|
+
channel_url: top.channel_url ?? top.uploader_url ?? null,
|
|
75
|
+
videos,
|
|
76
|
+
});
|
|
77
|
+
return { kind: isPlaylist ? 'playlist' : 'video', videos: videos.length, warnings };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Picks the caption track: captions the creator supplied in `language`, or
|
|
82
|
+
* failing that YouTube's automatic captions in the spoken language
|
|
83
|
+
* (`<language>-orig`), then its automatic ones in `language`.
|
|
84
|
+
* @returns {{ language: string, manual: boolean } | null}
|
|
85
|
+
*/
|
|
86
|
+
export function chooseTrack(info, language) {
|
|
87
|
+
const manual = Object.keys(info.subtitles ?? {}).filter((k) => k !== 'live_chat');
|
|
88
|
+
const auto = Object.keys(info.automatic_captions ?? {});
|
|
89
|
+
const own = manual.find((k) => k === language) ?? manual.find((k) => k.startsWith(`${language}-`));
|
|
90
|
+
if (own) return { language: own, manual: true };
|
|
91
|
+
const asr = [`${language}-orig`, language].find((k) => auto.includes(k));
|
|
92
|
+
return asr ? { language: asr, manual: false } : null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function watchUrl(id) {
|
|
96
|
+
return `https://www.youtube.com/watch?v=${id}`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function trimInfo(info, track) {
|
|
100
|
+
return {
|
|
101
|
+
id: info.id,
|
|
102
|
+
title: info.title,
|
|
103
|
+
description: info.description ?? '',
|
|
104
|
+
channel: info.channel ?? info.uploader ?? null,
|
|
105
|
+
channel_url: info.channel_url ?? info.uploader_url ?? null,
|
|
106
|
+
upload_date: info.upload_date ?? null,
|
|
107
|
+
duration: info.duration ?? null,
|
|
108
|
+
webpage_url: watchUrl(info.id),
|
|
109
|
+
license: info.license ?? null,
|
|
110
|
+
chapters: (info.chapters ?? []).map((c) => ({ start: c.start_time, end: c.end_time, title: c.title })),
|
|
111
|
+
captions: track ? { language: track.language.replace(/-orig$/, ''), automatic: !track.manual } : null,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function writeJson(path, value) {
|
|
116
|
+
writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
117
|
+
}
|