@openmoca/moca-convert 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +191 -2
- package/bin/moca-convert.js +198 -0
- package/bin/moca-youtube.js +93 -0
- package/lib/adapters/directory.js +134 -0
- package/lib/adapters/index.js +60 -0
- package/lib/adapters/markdown.js +65 -0
- package/lib/adapters/obsidian.js +206 -0
- package/lib/adapters/openapi/parse.js +143 -0
- package/lib/adapters/openapi/render.js +149 -0
- package/lib/adapters/openapi/suitability.js +41 -0
- package/lib/adapters/openapi.js +79 -0
- package/lib/adapters/pdf.js +309 -0
- package/lib/adapters/youtube.js +259 -0
- package/lib/detect.js +128 -0
- package/lib/frontmatter.js +42 -0
- package/lib/glob.js +33 -0
- package/lib/index.js +4 -0
- package/lib/manifest.js +23 -0
- package/lib/pdf/extract.js +206 -0
- package/lib/pdf/layout.js +270 -0
- package/lib/slug.js +19 -0
- package/lib/target.js +35 -0
- package/lib/text.js +114 -0
- package/lib/walk.js +37 -0
- package/lib/write.js +123 -0
- package/lib/youtube/capture.js +150 -0
- package/lib/youtube/json3-to-vtt.js +109 -0
- package/lib/youtube/ytdlp.js +64 -0
- package/package.json +42 -4
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// Turns the positioned lines from lib/pdf/extract.js into a flat list of
|
|
2
|
+
// headings and paragraphs. Pure functions: no pdf.js here, so the rules can be
|
|
3
|
+
// tested with hand-made pages.
|
|
4
|
+
//
|
|
5
|
+
// - Running headers and footers (lines in the page margins that repeat across
|
|
6
|
+
// pages, numbers aside) and bare page numbers are dropped.
|
|
7
|
+
// - Headings come from the outline when the PDF has one, otherwise from type
|
|
8
|
+
// noticeably larger than the body text.
|
|
9
|
+
// - Body lines are joined into paragraphs; a paragraph may run on across a
|
|
10
|
+
// column or a page break. Words hyphenated at a line end are rejoined.
|
|
11
|
+
|
|
12
|
+
const MARGIN = 0.08;
|
|
13
|
+
const HEADING_RATIO = 1.12;
|
|
14
|
+
const SMALL_RATIO = 0.8;
|
|
15
|
+
const MAX_HEADING_CHARS = 150;
|
|
16
|
+
// Bullet glyphs, including the private-use ones Symbol and Wingdings map to.
|
|
17
|
+
const BULLET = /^[•▪■●◦‣∙·–⦁○□➢➤►]\s*/u;
|
|
18
|
+
// "1. Power", "12.Loss of control", "3) Configuration"; not "91.417 requires…".
|
|
19
|
+
const NUMBERED = /^(\d{1,2})[.)](?:\s+(?=\S)|(?=\p{L}))/u;
|
|
20
|
+
// "Figure 4-6. The…", "Table 2: …", "Figure 4-1 A-F. …"; not "Figure 1-5 shows…".
|
|
21
|
+
const CAPTION = /^(figure|table|fig\.)\s+\d+(?:[-–.]\d+)*(?:\s*[A-Z](?:[-–][A-Z])?)?[.:]/i;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @typedef {{ kind: 'heading', level: number, text: string, page: number }} HeadingBlock
|
|
25
|
+
* @typedef {{ kind: 'paragraph' | 'item' | 'caption', text: string, page: number, lastPage?: number, number?: number }} TextBlock
|
|
26
|
+
* `lastPage` is set when the text runs on to a later page; an item with
|
|
27
|
+
* `number` is from a numbered list
|
|
28
|
+
* @typedef {HeadingBlock | TextBlock} Block
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {import('./extract.js').PdfText} pdf
|
|
33
|
+
* @returns {{ blocks: Block[], bodySize: number, labels: Map<number, string> }}
|
|
34
|
+
* labels: printed page label by physical page number, where one is known
|
|
35
|
+
*/
|
|
36
|
+
export function layoutPdf(pdf) {
|
|
37
|
+
const pages = dropRunningLines(pdf.pages);
|
|
38
|
+
const bodySize = bodyFontSize(pages);
|
|
39
|
+
const lines = pages.flatMap((page) => page.lines.map((line) => ({ ...line, page: page.number })));
|
|
40
|
+
const levels = usefulOutline(pdf.outline, pages.length) ? outlineHeadings(lines, pdf.outline) : fontHeadings(lines, bodySize);
|
|
41
|
+
const labels = new Map(pages.filter((p) => p.label).map((p) => [p.number, p.label]));
|
|
42
|
+
return { blocks: toBlocks(lines, levels, bodySize), bodySize, labels };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Removes lines in the top and bottom margins that repeat on at least a third
|
|
47
|
+
* of the pages (digits ignored, so "4-12" and "4-13" match), and margin lines
|
|
48
|
+
* that are only a page number. A page without a label in the PDF takes the
|
|
49
|
+
* printed page number it drops ("4-12").
|
|
50
|
+
* @param {import('./extract.js').PdfPage[]} pages
|
|
51
|
+
*/
|
|
52
|
+
export function dropRunningLines(pages) {
|
|
53
|
+
const inMargin = (page, line) => line.y > page.height * (1 - MARGIN) || line.y < page.height * MARGIN;
|
|
54
|
+
const key = (text) => text.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
|
|
55
|
+
const seen = new Map();
|
|
56
|
+
for (const page of pages) {
|
|
57
|
+
const keys = new Set(page.lines.filter((l) => inMargin(page, l)).map((l) => key(l.text)));
|
|
58
|
+
for (const k of keys) seen.set(k, (seen.get(k) ?? 0) + 1);
|
|
59
|
+
}
|
|
60
|
+
const repeats = Math.max(2, Math.ceil(pages.length / 3));
|
|
61
|
+
return pages.map((page) => {
|
|
62
|
+
let label = page.label;
|
|
63
|
+
const lines = page.lines.filter((l) => {
|
|
64
|
+
if (!inMargin(page, l)) return true;
|
|
65
|
+
if (isPageNumber(l.text, page.label)) {
|
|
66
|
+
if (!label && /\d/.test(l.text)) label = l.text.trim().replace(/^page\s+/i, '');
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
return !(pages.length > 2 && seen.get(key(l.text)) >= repeats);
|
|
70
|
+
});
|
|
71
|
+
return { ...page, ...(label ? { label } : {}), lines };
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function isPageNumber(text, label) {
|
|
76
|
+
const t = text.trim();
|
|
77
|
+
return t === label || /^(page\s+)?[\dA-Z]{0,3}[-–.]?\d{1,4}$/i.test(t) || /^[ivxlc]+$/i.test(t);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The size of most of the text, weighted by characters. */
|
|
81
|
+
export function bodyFontSize(pages) {
|
|
82
|
+
const weight = new Map();
|
|
83
|
+
for (const page of pages) {
|
|
84
|
+
for (const line of page.lines) weight.set(line.size, (weight.get(line.size) ?? 0) + line.text.length);
|
|
85
|
+
}
|
|
86
|
+
let best = 0;
|
|
87
|
+
let most = -1;
|
|
88
|
+
for (const [size, n] of weight) {
|
|
89
|
+
if (n > most) {
|
|
90
|
+
best = size;
|
|
91
|
+
most = n;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return best;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Heading levels from type size: every size clearly larger than the body is
|
|
99
|
+
* a level, largest first.
|
|
100
|
+
* @returns {Map<number, number>} line index -> level
|
|
101
|
+
*/
|
|
102
|
+
export function fontHeadings(lines, bodySize) {
|
|
103
|
+
const headingText = (line) => line.text.length <= MAX_HEADING_CHARS && /\p{L}/u.test(line.text) && !CAPTION.test(line.text);
|
|
104
|
+
// Large type may be a single letter (a glossary's "A", "B", …).
|
|
105
|
+
const large = (line) => line.size >= bodySize * HEADING_RATIO && headingText(line) && !/[,;]$/.test(line.text);
|
|
106
|
+
// A whole line in bold at body size, not ending like a sentence, is a
|
|
107
|
+
// run-in heading one level below the smallest large heading. It needs two
|
|
108
|
+
// letters in a row, so bold figure callouts ("A", "1a") are not headings.
|
|
109
|
+
const bold = (line) => line.bold && !large(line) && line.size >= bodySize * 0.95 && headingText(line)
|
|
110
|
+
&& /\p{L}{2}/u.test(line.text) && !/[.,;:]$/.test(line.text);
|
|
111
|
+
const sizes = [...new Set(lines.filter(large).map((l) => l.size))].sort((a, b) => b - a).slice(0, 5);
|
|
112
|
+
const out = new Map();
|
|
113
|
+
lines.forEach((line, i) => {
|
|
114
|
+
if (large(line)) out.set(i, sizes.indexOf(line.size) + 1);
|
|
115
|
+
else if (bold(line)) out.set(i, sizes.length + 1);
|
|
116
|
+
});
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Whether the outline describes the whole document. Some tools export only a
|
|
122
|
+
* "Structure Bookmarks" placeholder and the title, or bookmarks for a few
|
|
123
|
+
* sections near the end; the type sizes are a better guide then.
|
|
124
|
+
*/
|
|
125
|
+
export function usefulOutline(outline, pageCount) {
|
|
126
|
+
const placed = (outline ?? []).filter((e) => e.page);
|
|
127
|
+
if (placed.length < 3 || new Set(placed.map((e) => e.page)).size < 2) return false;
|
|
128
|
+
return Math.min(...placed.map((e) => e.page)) <= Math.max(2, Math.ceil(pageCount * 0.1));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Heading levels from the outline: each entry marks the first line on its
|
|
133
|
+
* page, or a later page, whose text starts the entry's title. An entry whose
|
|
134
|
+
* title is not found marks the first line of its page.
|
|
135
|
+
* @param {Array<{ page: number, text: string }>} lines
|
|
136
|
+
* @param {import('./extract.js').OutlineEntry[]} outline
|
|
137
|
+
* @returns {Map<number, number>} line index -> level
|
|
138
|
+
*/
|
|
139
|
+
export function outlineHeadings(lines, outline) {
|
|
140
|
+
const out = new Map();
|
|
141
|
+
const norm = (s) => s.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
|
|
142
|
+
let from = 0;
|
|
143
|
+
for (const entry of outline) {
|
|
144
|
+
if (!entry.page) continue;
|
|
145
|
+
const title = norm(entry.title);
|
|
146
|
+
const start = Math.max(from, lines.findIndex((l) => l.page >= entry.page));
|
|
147
|
+
if (start < 0 || start >= lines.length) continue;
|
|
148
|
+
let hit = -1;
|
|
149
|
+
for (let i = start; i < lines.length && lines[i].page <= entry.page + 1; i++) {
|
|
150
|
+
const text = norm(lines[i].text);
|
|
151
|
+
if (text && (title.startsWith(text) || text.startsWith(title)) && text.length >= Math.min(4, title.length)) {
|
|
152
|
+
hit = i;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const at = hit >= 0 ? hit : start;
|
|
157
|
+
if (!out.has(at)) out.set(at, Math.min(entry.level, 6));
|
|
158
|
+
from = at + 1;
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* @param {Array<import('./extract.js').PdfLine & { page: number }>} lines
|
|
165
|
+
* @param {Map<number, number>} levels
|
|
166
|
+
* @param {number} bodySize
|
|
167
|
+
* @returns {Block[]}
|
|
168
|
+
*/
|
|
169
|
+
export function toBlocks(lines, levels, bodySize) {
|
|
170
|
+
const pitch = linePitch(lines, bodySize) / (bodySize || 1);
|
|
171
|
+
const blocks = [];
|
|
172
|
+
let para = null;
|
|
173
|
+
let prev = null;
|
|
174
|
+
const flush = () => {
|
|
175
|
+
if (para && para.text.trim()) {
|
|
176
|
+
blocks.push({
|
|
177
|
+
kind: para.kind, text: para.text.trim(), page: para.page,
|
|
178
|
+
...(para.lastPage > para.page ? { lastPage: para.lastPage } : {}),
|
|
179
|
+
...(para.number ? { number: para.number } : {}),
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
para = null;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
lines.forEach((line, i) => {
|
|
186
|
+
const level = levels.get(i);
|
|
187
|
+
if (level) {
|
|
188
|
+
flush();
|
|
189
|
+
const last = blocks.at(-1);
|
|
190
|
+
// A heading set on two lines is one heading.
|
|
191
|
+
if (last?.kind === 'heading' && last.level === level && prev && levels.get(i - 1) === level
|
|
192
|
+
&& last.page === line.page && prev.y - line.y < line.size * 2 && prev.y > line.y) {
|
|
193
|
+
last.text = joinLine(last.text, line.text);
|
|
194
|
+
} else {
|
|
195
|
+
blocks.push({ kind: 'heading', level, text: line.text, page: line.page });
|
|
196
|
+
}
|
|
197
|
+
prev = line;
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (line.size < bodySize * SMALL_RATIO && !CAPTION.test(line.text)) {
|
|
201
|
+
// Small type is figure labelling or a footnote marker; left out.
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
const numbered = NUMBERED.exec(line.text);
|
|
205
|
+
const kind = CAPTION.test(line.text) ? 'caption' : BULLET.test(line.text) || numbered ? 'item' : 'paragraph';
|
|
206
|
+
const text = kind !== 'item' ? line.text : numbered ? line.text.slice(numbered[0].length) : line.text.replace(BULLET, '');
|
|
207
|
+
if (!para || kind !== 'paragraph' || breaksParagraph(prev, line, para, pitch)) {
|
|
208
|
+
flush();
|
|
209
|
+
para = { kind, text, page: line.page, ...(numbered && kind === 'item' ? { number: Number(numbered[1]) } : {}) };
|
|
210
|
+
} else {
|
|
211
|
+
para.text = joinLine(para.text, text);
|
|
212
|
+
para.lastPage = line.page;
|
|
213
|
+
}
|
|
214
|
+
prev = line;
|
|
215
|
+
});
|
|
216
|
+
flush();
|
|
217
|
+
return blocks;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The usual distance between baselines of body text: the most common gap
|
|
222
|
+
* between consecutive body-size lines on a page.
|
|
223
|
+
*/
|
|
224
|
+
export function linePitch(lines, bodySize) {
|
|
225
|
+
const counts = new Map();
|
|
226
|
+
for (let i = 1; i < lines.length; i++) {
|
|
227
|
+
const [a, b] = [lines[i - 1], lines[i]];
|
|
228
|
+
if (a.page !== b.page || Math.abs(a.size - bodySize) > 0.5 || Math.abs(b.size - bodySize) > 0.5) continue;
|
|
229
|
+
const gap = Math.round((a.y - b.y) * 2) / 2;
|
|
230
|
+
if (gap > 0 && gap < bodySize * 3) counts.set(gap, (counts.get(gap) ?? 0) + 1);
|
|
231
|
+
}
|
|
232
|
+
let best = bodySize * 1.2;
|
|
233
|
+
let most = 0;
|
|
234
|
+
for (const [gap, n] of counts) {
|
|
235
|
+
if (n > most) {
|
|
236
|
+
best = gap;
|
|
237
|
+
most = n;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return best;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Whether `line` starts a new paragraph after `prev`.
|
|
245
|
+
* @param {number} pitch line pitch as a multiple of the font size
|
|
246
|
+
*/
|
|
247
|
+
function breaksParagraph(prev, line, para, pitch) {
|
|
248
|
+
if (!prev) return true;
|
|
249
|
+
const ended = /[.!?:"”)]$/.test(para.text);
|
|
250
|
+
if (Math.abs(line.size - prev.size) > prev.size * 0.15) return true;
|
|
251
|
+
if (line.page !== prev.page || line.y > prev.y + prev.size) {
|
|
252
|
+
// A new page or a new column: carry on unless the text had finished.
|
|
253
|
+
return ended;
|
|
254
|
+
}
|
|
255
|
+
// More space between the lines than the usual line pitch, scaled to the
|
|
256
|
+
// size of this text.
|
|
257
|
+
return prev.y - line.y > pitch * prev.size * 1.25;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Joins two lines, rejoining a word hyphenated at the break ("encap-" +
|
|
262
|
+
* "sulating"). A hyphen after anything but a lowercase letter ("U.S.-" +
|
|
263
|
+
* "registered") and a slash ("accelerate/" + "decelerate") are kept, with no
|
|
264
|
+
* space added.
|
|
265
|
+
*/
|
|
266
|
+
export function joinLine(a, b) {
|
|
267
|
+
if (/\p{Ll}-$/u.test(a) && /^\p{Ll}/u.test(b)) return a.slice(0, -1) + b;
|
|
268
|
+
if (/[-/]$/.test(a) && /^\p{L}/u.test(b)) return a + b;
|
|
269
|
+
return `${a} ${b}`;
|
|
270
|
+
}
|
package/lib/slug.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic, filesystem-safe, ASCII-only slug from a title string.
|
|
3
|
+
* NFKD-normalizing first turns an accented character into a base letter
|
|
4
|
+
* plus a separate combining-mark codepoint (e.g. "é" -> "e" + U+0301), so
|
|
5
|
+
* simply dropping every non-ASCII codepoint afterward strips diacritics
|
|
6
|
+
* while keeping the underlying letters.
|
|
7
|
+
* @param {string} text
|
|
8
|
+
* @returns {string}
|
|
9
|
+
*/
|
|
10
|
+
export function slugify(text) {
|
|
11
|
+
return (
|
|
12
|
+
text
|
|
13
|
+
.normalize('NFKD')
|
|
14
|
+
.replace(/[^\x00-\x7F]/g, '')
|
|
15
|
+
.toLowerCase()
|
|
16
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
17
|
+
.replace(/^-+|-+$/g, '') || 'untitled'
|
|
18
|
+
);
|
|
19
|
+
}
|
package/lib/target.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { existsSync, statSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Resolves and validates a conversion input path exists on disk, and reports
|
|
5
|
+
* whether it's a file or a directory. Glob inputs (containing *, ?, or [)
|
|
6
|
+
* are not resolvable via stat and are passed through unchanged for the
|
|
7
|
+
* markdown adapter to expand itself.
|
|
8
|
+
*
|
|
9
|
+
* @param {string} inputPath
|
|
10
|
+
* @returns {{ inputPath: string, isGlob: boolean, isDirectory: boolean, isFile: boolean }}
|
|
11
|
+
*/
|
|
12
|
+
export function resolveInputTarget(inputPath) {
|
|
13
|
+
if (isGlobPattern(inputPath)) {
|
|
14
|
+
return { inputPath, isGlob: true, isDirectory: false, isFile: false };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if (!existsSync(inputPath)) {
|
|
18
|
+
throw new UsageError(`Input does not exist: ${inputPath}`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const stats = statSync(inputPath);
|
|
22
|
+
return {
|
|
23
|
+
inputPath,
|
|
24
|
+
isGlob: false,
|
|
25
|
+
isDirectory: stats.isDirectory(),
|
|
26
|
+
isFile: stats.isFile(),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** @param {string} inputPath */
|
|
31
|
+
export function isGlobPattern(inputPath) {
|
|
32
|
+
return /[*?[\]]/.test(inputPath);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class UsageError extends Error {}
|
package/lib/text.js
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Helpers shared by adapters that write Markdown from text that was not
|
|
2
|
+
// written as Markdown (YouTube descriptions, text extracted from PDFs).
|
|
3
|
+
|
|
4
|
+
// Source text is shown as written, not interpreted as Markdown: characters
|
|
5
|
+
// that would start emphasis, links or HTML are escaped. Bare URLs become
|
|
6
|
+
// autolinks.
|
|
7
|
+
export function escapeInline(text) {
|
|
8
|
+
return String(text)
|
|
9
|
+
.split(/(https?:\/\/[^\s<>]+)/)
|
|
10
|
+
.map((part, i) => (i % 2 === 1 ? `<${part}>` : part.replace(/([\\`*_[\]<>|])/g, '\\$1')))
|
|
11
|
+
.join('');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// Multi-line text: blank lines separate paragraphs and single line breaks are
|
|
15
|
+
// kept as hard breaks. A line that would start a heading, list, quote or
|
|
16
|
+
// code block is escaped.
|
|
17
|
+
export function escapeText(text) {
|
|
18
|
+
return text
|
|
19
|
+
.split(/\n\s*\n/)
|
|
20
|
+
.map((para) =>
|
|
21
|
+
para
|
|
22
|
+
.split('\n')
|
|
23
|
+
.map((line) => escapeLineStart(escapeInline(line.trim())))
|
|
24
|
+
.filter(Boolean)
|
|
25
|
+
.join('\\\n'))
|
|
26
|
+
.filter(Boolean)
|
|
27
|
+
.join('\n\n');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Escapes a line start that Markdown would read as a heading, list or quote. */
|
|
31
|
+
export function escapeLineStart(line) {
|
|
32
|
+
return line.replace(/^([#>+-])/, '\\$1').replace(/^(\d+)([.)])/, '$1\\$2');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** First non-empty line, trimmed. */
|
|
36
|
+
export function firstLine(text) {
|
|
37
|
+
return text.split('\n').map((l) => l.trim()).find(Boolean) ?? '';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A one-line summary for `description`: the first line, cut at a sentence when long. */
|
|
41
|
+
export function summary(text) {
|
|
42
|
+
const line = firstLine(text ?? '');
|
|
43
|
+
if (line.length <= 300) return line || undefined;
|
|
44
|
+
const cut = line.slice(0, 300);
|
|
45
|
+
const stop = cut.lastIndexOf('. ');
|
|
46
|
+
return stop > 80 ? cut.slice(0, stop + 1) : `${cut.slice(0, cut.lastIndexOf(' '))}…`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** `base`, or `base-2`, `base-3`… when `taken` already has it; records the result. */
|
|
50
|
+
export function uniqueSlug(base, taken) {
|
|
51
|
+
let slug = base;
|
|
52
|
+
for (let n = 2; taken.has(slug); n++) slug = `${base}-${n}`;
|
|
53
|
+
taken.add(slug);
|
|
54
|
+
return slug;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function turtleString(text) {
|
|
58
|
+
return `"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* An http or https URL from input, parsed and rebuilt, or undefined. The
|
|
63
|
+
* rebuilt form percent-encodes spaces, quotes and angle brackets, so it is
|
|
64
|
+
* safe inside a Markdown link destination (`<...>`) and in YAML. URLs with
|
|
65
|
+
* other schemes, or with a user name or password, are refused.
|
|
66
|
+
* @param {unknown} value
|
|
67
|
+
* @returns {string|undefined}
|
|
68
|
+
*/
|
|
69
|
+
export function httpUrl(value) {
|
|
70
|
+
if (typeof value !== 'string' || value.length > 2048 || /[\u0000-\u001f\u007f\s]/.test(value)) return undefined;
|
|
71
|
+
let url;
|
|
72
|
+
try {
|
|
73
|
+
url = new URL(value);
|
|
74
|
+
} catch {
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
if ((url.protocol !== 'http:' && url.protocol !== 'https:') || url.username || url.password || !url.hostname) return undefined;
|
|
78
|
+
return url.href;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* A path as an IRI fragment or path: each `/`-separated segment
|
|
83
|
+
* percent-encoded, so spaces, quotes and angle brackets cannot end a Turtle
|
|
84
|
+
* IRI (`<...>`) early.
|
|
85
|
+
* @param {string} path
|
|
86
|
+
*/
|
|
87
|
+
export function iriPath(path) {
|
|
88
|
+
return path.split('/').map((segment) => encodeURIComponent(segment)).join('/');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Short text from input shown on one line, such as a PDF page label: control
|
|
93
|
+
* and format characters removed, white space collapsed, at most `max`
|
|
94
|
+
* characters. Undefined when nothing is left.
|
|
95
|
+
* @param {unknown} value
|
|
96
|
+
* @param {number} [max]
|
|
97
|
+
*/
|
|
98
|
+
export function oneLine(value, max = 32) {
|
|
99
|
+
if (typeof value !== 'string') return undefined;
|
|
100
|
+
const text = value.replace(/[\p{Cc}\p{Cf}]/gu, ' ').replace(/\s+/g, ' ').trim().slice(0, max).trim();
|
|
101
|
+
return text || undefined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* A YYYYMMDD date that exists, as YYYY-MM-DD, or undefined.
|
|
106
|
+
* @param {unknown} value
|
|
107
|
+
*/
|
|
108
|
+
export function compactDate(value) {
|
|
109
|
+
const m = typeof value === 'string' ? /^(\d{4})(\d{2})(\d{2})$/.exec(value) : null;
|
|
110
|
+
if (!m) return undefined;
|
|
111
|
+
const iso = `${m[1]}-${m[2]}-${m[3]}`;
|
|
112
|
+
const d = new Date(`${iso}T00:00:00Z`);
|
|
113
|
+
return !Number.isNaN(d.getTime()) && d.toISOString().startsWith(iso) ? iso : undefined;
|
|
114
|
+
}
|
package/lib/walk.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Recursively lists files under `dir`, returning paths relative to `rootDir`
|
|
6
|
+
* (using forward slashes regardless of platform), sorted lexicographically
|
|
7
|
+
* for deterministic iteration order. Hidden entries (a name starting with
|
|
8
|
+
* ".": .git/, .obsidian/, .trash/ and the like) are skipped, as a Reader
|
|
9
|
+
* ignores them, and so are symbolic links.
|
|
10
|
+
*
|
|
11
|
+
* @param {string} dir
|
|
12
|
+
* @param {string} rootDir
|
|
13
|
+
* @returns {string[]}
|
|
14
|
+
*/
|
|
15
|
+
export function walkFiles(dir, rootDir = dir) {
|
|
16
|
+
if (!existsSyncSafe(dir)) return [];
|
|
17
|
+
const entries = readdirSync(dir, { withFileTypes: true });
|
|
18
|
+
const results = [];
|
|
19
|
+
for (const entry of entries) {
|
|
20
|
+
if (entry.name.startsWith('.')) continue;
|
|
21
|
+
const full = join(dir, entry.name);
|
|
22
|
+
if (entry.isDirectory()) {
|
|
23
|
+
results.push(...walkFiles(full, rootDir));
|
|
24
|
+
} else if (entry.isFile()) {
|
|
25
|
+
results.push(relative(rootDir, full).split('\\').join('/'));
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return results.sort();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function existsSyncSafe(dir) {
|
|
32
|
+
try {
|
|
33
|
+
return statSync(dir).isDirectory();
|
|
34
|
+
} catch {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
package/lib/write.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Writes a PackageDraft (see lib/adapters/index.js) to disk and gates
|
|
2
|
+
// success on the reference Reader reporting zero error-severity diagnostics:
|
|
3
|
+
// read first, refuse to leave invalid output behind.
|
|
4
|
+
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { randomBytes } from 'node:crypto';
|
|
6
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
7
|
+
import { readPackage } from '@openmoca/moca-core';
|
|
8
|
+
import { UsageError } from './target.js';
|
|
9
|
+
|
|
10
|
+
export { UsageError };
|
|
11
|
+
|
|
12
|
+
/** A hidden name beside `outDir`, on the same file system, so a rename never crosses devices. */
|
|
13
|
+
const besideName = (outDir, tag) => join(dirname(resolve(outDir)), `.${basename(resolve(outDir))}.${randomBytes(6).toString('hex')}.${tag}`);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Writes the draft to a folder beside `outDir`, checks it with the reference
|
|
17
|
+
* Reader, and only then moves it into place. Anything already at `outDir` is
|
|
18
|
+
* moved aside first and deleted once the new package is in; if the move fails,
|
|
19
|
+
* it is put back. On any failure the staged folder is removed and `outDir` is
|
|
20
|
+
* as it was.
|
|
21
|
+
*
|
|
22
|
+
* `force` replaces a non-empty `outDir` only when it is a previous package (it
|
|
23
|
+
* has a moca.json); anything else is refused.
|
|
24
|
+
*
|
|
25
|
+
* @param {object} params
|
|
26
|
+
* @param {import('./adapters/index.js').PackageDraft} params.draft
|
|
27
|
+
* @param {string} params.outDir
|
|
28
|
+
* @param {boolean} [params.force]
|
|
29
|
+
* @param {boolean} [params.strict]
|
|
30
|
+
* @param {(entry: string) => void} [params.onLog]
|
|
31
|
+
* @returns {Promise<{ findings: object[], digest: string }>}
|
|
32
|
+
*/
|
|
33
|
+
export async function writeDraft({ draft, outDir, force = false, strict = false, onLog = () => {} }) {
|
|
34
|
+
checkFiles(draft);
|
|
35
|
+
const outDirExists = existsSync(outDir);
|
|
36
|
+
if (outDirExists && !statSync(outDir).isDirectory()) {
|
|
37
|
+
throw new UsageError(`Output "${outDir}" exists and is not a directory.`);
|
|
38
|
+
}
|
|
39
|
+
const outDirNonEmpty = outDirExists && readdirSync(outDir).length > 0;
|
|
40
|
+
if (outDirNonEmpty && !force) {
|
|
41
|
+
throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to replace it.`);
|
|
42
|
+
}
|
|
43
|
+
if (outDirNonEmpty && !existsSync(join(outDir, 'moca.json'))) {
|
|
44
|
+
throw new UsageError(`Output directory "${outDir}" is not a previous package (it has no moca.json), so --force will not replace it.`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
mkdirSync(dirname(resolve(outDir)), { recursive: true });
|
|
48
|
+
const stageDir = besideName(outDir, 'partial');
|
|
49
|
+
try {
|
|
50
|
+
stageDraft(draft, stageDir);
|
|
51
|
+
|
|
52
|
+
onLog(`reading ${stageDir}`);
|
|
53
|
+
const result = await readPackage(stageDir, { strict });
|
|
54
|
+
const findings = result.diagnostics;
|
|
55
|
+
const errors = findings.filter((f) => f.severity === 'error');
|
|
56
|
+
|
|
57
|
+
if (errors.length > 0) {
|
|
58
|
+
throw new ConversionFailedError(
|
|
59
|
+
'The converted package failed its own validation; no output was written.',
|
|
60
|
+
findings
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
replaceWith(stageDir, outDir, onLog);
|
|
65
|
+
return { findings, digest: result.digest };
|
|
66
|
+
} finally {
|
|
67
|
+
// Never leave partial or invalid output behind. After a successful move
|
|
68
|
+
// the staged folder no longer exists.
|
|
69
|
+
rmSync(stageDir, { recursive: true, force: true });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Moves the staged folder to `outDir`: the old one aside first, removed last. */
|
|
74
|
+
function replaceWith(stageDir, outDir, onLog) {
|
|
75
|
+
if (!existsSync(outDir)) {
|
|
76
|
+
renameSync(stageDir, outDir);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const aside = besideName(outDir, 'old');
|
|
80
|
+
onLog(`moving the previous ${outDir} aside`);
|
|
81
|
+
renameSync(outDir, aside);
|
|
82
|
+
try {
|
|
83
|
+
renameSync(stageDir, outDir);
|
|
84
|
+
} catch (err) {
|
|
85
|
+
renameSync(aside, outDir);
|
|
86
|
+
throw err;
|
|
87
|
+
}
|
|
88
|
+
rmSync(aside, { recursive: true, force: true });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function stageDraft(draft, stageDir) {
|
|
92
|
+
mkdirSync(stageDir);
|
|
93
|
+
writeFileSync(join(stageDir, 'moca.json'), `${JSON.stringify(draft.manifest, null, 2)}\n`);
|
|
94
|
+
|
|
95
|
+
for (const node of [...draft.contentNodes, ...(draft.files ?? [])]) {
|
|
96
|
+
const absPath = join(stageDir, node.path);
|
|
97
|
+
mkdirSync(dirname(absPath), { recursive: true });
|
|
98
|
+
writeFileSync(absPath, node.body);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Adapters may add the evidence layer and the structure layer beside
|
|
103
|
+
// content/, and nothing else.
|
|
104
|
+
const OTHER_FILE = /^(?:(?:sources|media)\/(?:[^/]+\/)*[^/]+|structure\.ttl)$/;
|
|
105
|
+
|
|
106
|
+
function checkFiles(draft) {
|
|
107
|
+
for (const file of draft.files ?? []) {
|
|
108
|
+
if (!OTHER_FILE.test(file.path) || file.path.split('/').some((seg) => seg === '..' || seg === '.')) {
|
|
109
|
+
throw new UsageError(`An adapter may only add files under sources/ or media/, or structure.ttl; refusing "${file.path}".`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export class ConversionFailedError extends Error {
|
|
115
|
+
/**
|
|
116
|
+
* @param {string} message
|
|
117
|
+
* @param {object[]} findings
|
|
118
|
+
*/
|
|
119
|
+
constructor(message, findings) {
|
|
120
|
+
super(message);
|
|
121
|
+
this.findings = findings;
|
|
122
|
+
}
|
|
123
|
+
}
|