@openmoca/moca-convert 0.0.0-stage → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +133 -2
- package/bin/moca-convert.js +170 -0
- package/bin/moca-youtube.js +93 -0
- package/lib/adapters/directory.js +134 -0
- package/lib/adapters/index.js +60 -0
- package/lib/adapters/markdown.js +65 -0
- package/lib/adapters/obsidian.js +199 -0
- package/lib/adapters/openapi/parse.js +114 -0
- package/lib/adapters/openapi/render.js +141 -0
- package/lib/adapters/openapi/suitability.js +41 -0
- package/lib/adapters/openapi.js +77 -0
- package/lib/adapters/pdf.js +270 -0
- package/lib/adapters/youtube.js +194 -0
- package/lib/detect.js +128 -0
- package/lib/frontmatter.js +42 -0
- package/lib/glob.js +33 -0
- package/lib/index.js +4 -0
- package/lib/manifest.js +23 -0
- package/lib/pdf/extract.js +176 -0
- package/lib/pdf/layout.js +270 -0
- package/lib/slug.js +19 -0
- package/lib/target.js +35 -0
- package/lib/text.js +59 -0
- package/lib/walk.js +34 -0
- package/lib/write.js +98 -0
- package/lib/youtube/capture.js +117 -0
- package/lib/youtube/json3-to-vtt.js +109 -0
- package/lib/youtube/ytdlp.js +58 -0
- package/package.json +42 -4
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// Renders parsed OpenAPI operations into content nodes: either one node per
|
|
2
|
+
// operation, or one node per tag with each operation as a subsection.
|
|
3
|
+
import { joinFrontmatter } from '@openmoca/moca-core';
|
|
4
|
+
import { slugify } from '../../slug.js';
|
|
5
|
+
import { resolveSchema } from './parse.js';
|
|
6
|
+
|
|
7
|
+
const OPERATION_ID_PATTERN = /^[A-Za-z0-9_.:-]+$/;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {string} path
|
|
11
|
+
* @param {string} method
|
|
12
|
+
* @param {object} operation
|
|
13
|
+
* @param {object} doc
|
|
14
|
+
* @returns {import('../index.js').ContentNode}
|
|
15
|
+
*/
|
|
16
|
+
export function renderOperationNode(path, method, operation, doc) {
|
|
17
|
+
const title = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
|
|
18
|
+
const operationId = safeOperationId(operation.operationId);
|
|
19
|
+
const frontmatter = {
|
|
20
|
+
type: 'API Operation',
|
|
21
|
+
title,
|
|
22
|
+
resource: `${method.toUpperCase()} ${path}`,
|
|
23
|
+
...(operationId ? { tags: [operationId] } : {}),
|
|
24
|
+
};
|
|
25
|
+
const body = `# ${title}\n\n${renderSections(operation, doc, 2)}`;
|
|
26
|
+
return {
|
|
27
|
+
path: `content/${method}-${slugifyOperationPath(path)}.md`,
|
|
28
|
+
body: joinFrontmatter(frontmatter, body),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} tag
|
|
34
|
+
* @param {Array<{ path: string, method: string, operation: object }>} operations
|
|
35
|
+
* @param {object} doc
|
|
36
|
+
* @returns {import('../index.js').ContentNode}
|
|
37
|
+
*/
|
|
38
|
+
export function renderTagNode(tag, operations, doc) {
|
|
39
|
+
const sections = operations
|
|
40
|
+
.map(({ path, method, operation }) => {
|
|
41
|
+
const opTitle = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
|
|
42
|
+
return `## ${method.toUpperCase()} ${path} — ${opTitle}\n\n${renderSections(operation, doc, 3)}`;
|
|
43
|
+
})
|
|
44
|
+
.join('\n');
|
|
45
|
+
const body = `# ${tag}\n\n${sections}`;
|
|
46
|
+
return {
|
|
47
|
+
path: `content/${slugify(tag)}.md`,
|
|
48
|
+
body: joinFrontmatter({ type: 'API Tag', title: tag }, body),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Renders the Description/Request/Responses subsections for one operation,
|
|
54
|
+
* starting at the given heading level (2 for a standalone operation node's
|
|
55
|
+
* "##" sections, 3 when nested one level deeper under a tag node's "## METHOD
|
|
56
|
+
* path" heading).
|
|
57
|
+
*/
|
|
58
|
+
function renderSections(operation, doc, level) {
|
|
59
|
+
const h = '#'.repeat(level);
|
|
60
|
+
const hSub = '#'.repeat(level + 1);
|
|
61
|
+
const lines = [];
|
|
62
|
+
|
|
63
|
+
lines.push(`${h} Description`, '', describe(operation), '');
|
|
64
|
+
|
|
65
|
+
const parameters = operation.parameters ?? [];
|
|
66
|
+
const requestBody = operation.requestBody;
|
|
67
|
+
if (parameters.length > 0 || requestBody) {
|
|
68
|
+
lines.push(`${h} Request`, '');
|
|
69
|
+
if (parameters.length > 0) {
|
|
70
|
+
lines.push(...parameterTable(parameters), '');
|
|
71
|
+
}
|
|
72
|
+
const requestSchema = requestBody && firstJsonSchema(requestBody.content);
|
|
73
|
+
if (requestSchema) {
|
|
74
|
+
lines.push('```json', JSON.stringify(resolveSchema(requestSchema, doc), null, 2), '```', '');
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const responses = operation.responses ?? {};
|
|
79
|
+
const statusCodes = Object.keys(responses).sort();
|
|
80
|
+
if (statusCodes.length > 0) {
|
|
81
|
+
lines.push(`${h} Responses`, '');
|
|
82
|
+
for (const status of statusCodes) {
|
|
83
|
+
const response = responses[status] ?? {};
|
|
84
|
+
lines.push(`${hSub} ${status}`, '');
|
|
85
|
+
if (typeof response.description === 'string' && response.description.trim()) {
|
|
86
|
+
lines.push(response.description.trim(), '');
|
|
87
|
+
}
|
|
88
|
+
const responseSchema = firstJsonSchema(response.content);
|
|
89
|
+
if (responseSchema) {
|
|
90
|
+
lines.push('```json', JSON.stringify(resolveSchema(responseSchema, doc), null, 2), '```', '');
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return lines
|
|
96
|
+
.join('\n')
|
|
97
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
98
|
+
.trimEnd() + '\n';
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function describe(operation) {
|
|
102
|
+
return firstNonEmpty(operation.description, operation.summary, '_No description provided._').trim();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function parameterTable(parameters) {
|
|
106
|
+
const rows = ['| Parameter | In | Required | Type |', '|---|---|---|---|'];
|
|
107
|
+
for (const param of parameters) {
|
|
108
|
+
rows.push(`| ${param.name ?? ''} | ${param.in ?? ''} | ${param.required ? 'yes' : 'no'} | ${schemaType(param.schema)} |`);
|
|
109
|
+
}
|
|
110
|
+
return rows;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function schemaType(schema) {
|
|
114
|
+
if (!schema || typeof schema !== 'object') return '';
|
|
115
|
+
if (typeof schema.type === 'string') return schema.type;
|
|
116
|
+
if (typeof schema.$ref === 'string') return schema.$ref.split('/').pop();
|
|
117
|
+
return '';
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function firstJsonSchema(content) {
|
|
121
|
+
if (!content || typeof content !== 'object') return null;
|
|
122
|
+
const mediaType = content['application/json'] ?? Object.values(content)[0];
|
|
123
|
+
return mediaType?.schema ?? null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function safeOperationId(operationId) {
|
|
127
|
+
return typeof operationId === 'string' && OPERATION_ID_PATTERN.test(operationId) ? operationId : undefined;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function slugifyOperationPath(path) {
|
|
131
|
+
const segments = path
|
|
132
|
+
.replace(/[{}]/g, '')
|
|
133
|
+
.split('/')
|
|
134
|
+
.filter(Boolean)
|
|
135
|
+
.map((segment) => slugify(segment));
|
|
136
|
+
return segments.length > 0 ? segments.join('-') : 'root';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function firstNonEmpty(...values) {
|
|
140
|
+
return values.find((v) => typeof v === 'string' && v.trim().length > 0) ?? values[values.length - 1];
|
|
141
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// "Suitable" per the moca-convert design means an OpenAPI document whose
|
|
2
|
+
// operations carry real human-readable summary/description fields, not a
|
|
3
|
+
// bare machine-generated schema dump -- converting the latter would emit a
|
|
4
|
+
// low-value package that merely echoes the raw schema back as prose.
|
|
5
|
+
import { collectOperations } from './parse.js';
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_MIN_DESCRIPTION_RATIO = 0.5;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {object} doc
|
|
11
|
+
* @param {number} [minRatio]
|
|
12
|
+
* @returns {{ suitable: true } | { suitable: false, reason: string }}
|
|
13
|
+
*/
|
|
14
|
+
export function checkSuitability(doc, minRatio = DEFAULT_MIN_DESCRIPTION_RATIO) {
|
|
15
|
+
const operations = collectOperations(doc);
|
|
16
|
+
const total = operations.length;
|
|
17
|
+
|
|
18
|
+
if (total === 0) {
|
|
19
|
+
return { suitable: false, reason: 'No operations found under "paths".' };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const described = operations.filter(({ operation }) => hasDescription(operation)).length;
|
|
23
|
+
const ratio = described / total;
|
|
24
|
+
|
|
25
|
+
if (ratio < minRatio) {
|
|
26
|
+
return {
|
|
27
|
+
suitable: false,
|
|
28
|
+
reason: `Only ${described}/${total} operation(s) (${Math.round(ratio * 100)}%) have a summary or description; need at least ${Math.round(minRatio * 100)}%.`,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { suitable: true };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function hasDescription(operation) {
|
|
36
|
+
return isNonEmptyString(operation.summary) || isNonEmptyString(operation.description);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isNonEmptyString(value) {
|
|
40
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
41
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// `openapi` adapter: a "suitable" OpenAPI 3.x document (JSON or YAML) --
|
|
2
|
+
// one whose operations carry real summary/description text -- becomes one
|
|
3
|
+
// content node per operation (default) or per tag (--chunker tag).
|
|
4
|
+
import { existsSync, statSync } from 'node:fs';
|
|
5
|
+
import { UsageError } from '../target.js';
|
|
6
|
+
import { parseOpenApiDocument, collectOperations } from './openapi/parse.js';
|
|
7
|
+
import { checkSuitability, DEFAULT_MIN_DESCRIPTION_RATIO } from './openapi/suitability.js';
|
|
8
|
+
import { renderOperationNode, renderTagNode } from './openapi/render.js';
|
|
9
|
+
import { buildManifest } from '../manifest.js';
|
|
10
|
+
|
|
11
|
+
export const name = 'openapi';
|
|
12
|
+
|
|
13
|
+
const OPENAPI_FILE_EXT = /\.(json|ya?ml)$/i;
|
|
14
|
+
const CHUNKER_MODES = ['operation', 'tag'];
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @param {string} inputPath
|
|
18
|
+
* @returns {boolean} true for a single JSON/YAML file that parses as an
|
|
19
|
+
* OpenAPI 3.x document.
|
|
20
|
+
*/
|
|
21
|
+
export function detect(inputPath) {
|
|
22
|
+
if (!existsSync(inputPath) || !statSync(inputPath).isFile()) return false;
|
|
23
|
+
if (!OPENAPI_FILE_EXT.test(inputPath)) return false;
|
|
24
|
+
try {
|
|
25
|
+
parseOpenApiDocument(inputPath);
|
|
26
|
+
return true;
|
|
27
|
+
} catch {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {object} ctx
|
|
34
|
+
* @param {string} ctx.inputPath
|
|
35
|
+
* @param {{ id?: string, title?: string, version?: string, minDescriptionRatio?: number, chunker?: string }} ctx.options
|
|
36
|
+
* @returns {import('./index.js').PackageDraft}
|
|
37
|
+
*/
|
|
38
|
+
export function convert({ inputPath, options }) {
|
|
39
|
+
const chunker = options.chunker ?? 'operation';
|
|
40
|
+
if (!CHUNKER_MODES.includes(chunker)) {
|
|
41
|
+
throw new UsageError(`Unknown --chunker "${chunker}"; expected one of: ${CHUNKER_MODES.join(', ')}.`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const doc = parseOpenApiDocument(inputPath);
|
|
45
|
+
|
|
46
|
+
const minRatio = options.minDescriptionRatio ?? DEFAULT_MIN_DESCRIPTION_RATIO;
|
|
47
|
+
const suitability = checkSuitability(doc, minRatio);
|
|
48
|
+
if (!suitability.suitable) {
|
|
49
|
+
throw new UsageError(`"${inputPath}" is not suitable for conversion: ${suitability.reason}`);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const title = options.title ?? (typeof doc.info?.title === 'string' ? doc.info.title : undefined);
|
|
53
|
+
if (!title) {
|
|
54
|
+
throw new UsageError(`"${inputPath}" has no "info.title" and no --title was given.`);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const description = typeof doc.info?.description === 'string' ? doc.info.description.trim() : '';
|
|
58
|
+
const manifest = buildManifest({ ...options, title, description });
|
|
59
|
+
const operations = collectOperations(doc);
|
|
60
|
+
const contentNodes = chunker === 'tag' ? buildTagNodes(operations, doc) : buildOperationNodes(operations, doc);
|
|
61
|
+
|
|
62
|
+
return { manifest, contentNodes, warnings: [] };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function buildOperationNodes(operations, doc) {
|
|
66
|
+
return operations.map(({ path, method, operation }) => renderOperationNode(path, method, operation, doc));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function buildTagNodes(operations, doc) {
|
|
70
|
+
const groups = new Map();
|
|
71
|
+
for (const entry of operations) {
|
|
72
|
+
const tag = entry.operation.tags?.[0] ?? 'Untagged';
|
|
73
|
+
if (!groups.has(tag)) groups.set(tag, []);
|
|
74
|
+
groups.get(tag).push(entry);
|
|
75
|
+
}
|
|
76
|
+
return [...groups.keys()].sort().map((tag) => renderTagNode(tag, groups.get(tag), doc));
|
|
77
|
+
}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// `pdf` adapter: one or more PDFs become content nodes, one per section. A
|
|
2
|
+
// section starts at a heading of the first level that occurs more than once
|
|
3
|
+
// in the document (chapters in a book, sections in a chapter); deeper headings
|
|
4
|
+
// stay inside the node. Every node cites the pages it came from as RFC 3778
|
|
5
|
+
// `page=` fragments, and structure.ttl keeps the documents and their sections
|
|
6
|
+
// in order.
|
|
7
|
+
//
|
|
8
|
+
// Input is a PDF file, a folder of PDFs, or a folder with a `pdfs.json`
|
|
9
|
+
// that lists the documents in order, with an optional title, source URL and
|
|
10
|
+
// SHA-256 for each (see README).
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
13
|
+
import { basename, join } from 'node:path';
|
|
14
|
+
import { joinFrontmatter } from '@openmoca/moca-core';
|
|
15
|
+
import { buildManifest } from '../manifest.js';
|
|
16
|
+
import { slugify } from '../slug.js';
|
|
17
|
+
import { UsageError } from '../target.js';
|
|
18
|
+
import { walkFiles } from '../walk.js';
|
|
19
|
+
import { escapeInline, escapeLineStart, summary, turtleString, uniqueSlug } from '../text.js';
|
|
20
|
+
import { extractPdf } from '../pdf/extract.js';
|
|
21
|
+
import { layoutPdf } from '../pdf/layout.js';
|
|
22
|
+
|
|
23
|
+
export const name = 'pdf';
|
|
24
|
+
export const DEFAULT_TYPE = 'Document';
|
|
25
|
+
export const DESCRIPTOR = 'pdfs.json';
|
|
26
|
+
|
|
27
|
+
const RFC3778 = 'http://tools.ietf.org/rfc/rfc3778';
|
|
28
|
+
const SOURCE_MODES = ['copy', 'link'];
|
|
29
|
+
|
|
30
|
+
/** @param {string} inputPath */
|
|
31
|
+
export function detect(inputPath) {
|
|
32
|
+
if (!existsSync(inputPath)) return false;
|
|
33
|
+
const st = statSync(inputPath);
|
|
34
|
+
if (st.isFile()) return /\.pdf$/i.test(inputPath);
|
|
35
|
+
if (!st.isDirectory()) return false;
|
|
36
|
+
if (existsSync(join(inputPath, DESCRIPTOR))) return true;
|
|
37
|
+
const files = walkFiles(inputPath);
|
|
38
|
+
return files.some((f) => /\.pdf$/i.test(f)) && !files.some((f) => f.endsWith('.md'));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {{ inputPath: string, options: object }} ctx
|
|
43
|
+
* @returns {Promise<import('./index.js').PackageDraft>}
|
|
44
|
+
*/
|
|
45
|
+
export async function convert({ inputPath, options }) {
|
|
46
|
+
const sourcesMode = options.sources ?? 'copy';
|
|
47
|
+
if (!SOURCE_MODES.includes(sourcesMode)) throw new UsageError(`--sources must be one of: ${SOURCE_MODES.join(', ')}.`);
|
|
48
|
+
const listing = listDocuments(inputPath);
|
|
49
|
+
if (sourcesMode === 'link') {
|
|
50
|
+
const missing = listing.documents.filter((d) => !d.url);
|
|
51
|
+
if (missing.length > 0) throw new UsageError(`--sources link needs a url for every document in ${DESCRIPTOR}; ${missing[0].file} has none.`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const warnings = [];
|
|
55
|
+
const docs = [];
|
|
56
|
+
const docSlugs = new Set(['index']);
|
|
57
|
+
for (const entry of listing.documents) {
|
|
58
|
+
const bytes = readPdf(entry);
|
|
59
|
+
const pdf = await extractPdf(new Uint8Array(bytes));
|
|
60
|
+
const { blocks, labels } = layoutPdf(pdf);
|
|
61
|
+
const title = entry.title ?? pdf.title ?? firstHeading(blocks) ?? basename(entry.file, '.pdf');
|
|
62
|
+
if (!blocks.some((b) => b.kind !== 'heading')) {
|
|
63
|
+
warnings.push({ code: 'PDF_NO_TEXT', message: `${entry.file} has no text layer (a scan?); it was left out. Run OCR on it first.`, file: entry.file });
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const slug = uniqueSlug(slugify(title), docSlugs);
|
|
67
|
+
docs.push({ entry, bytes, title, slug, labels, sections: splitSections(blocks, title) });
|
|
68
|
+
}
|
|
69
|
+
if (docs.length === 0) throw new UsageError(`${inputPath}: none of the PDFs has any text.`);
|
|
70
|
+
|
|
71
|
+
const multi = docs.length > 1;
|
|
72
|
+
const manifest = buildManifest({
|
|
73
|
+
...options,
|
|
74
|
+
title: options.title ?? listing.title ?? (multi ? undefined : docs[0].title),
|
|
75
|
+
description: options.description ?? listing.description,
|
|
76
|
+
});
|
|
77
|
+
const iri = (fragment) => `${manifest.id}#${fragment}`;
|
|
78
|
+
const type = options.type ?? DEFAULT_TYPE;
|
|
79
|
+
const contentNodes = [];
|
|
80
|
+
const files = [];
|
|
81
|
+
|
|
82
|
+
for (const doc of docs) {
|
|
83
|
+
const dir = multi ? `content/${doc.slug}` : 'content';
|
|
84
|
+
const up = multi ? '../../' : '../';
|
|
85
|
+
const pdfPath = `sources/${doc.slug}.pdf`;
|
|
86
|
+
if (sourcesMode === 'copy') files.push({ path: pdfPath, body: doc.bytes });
|
|
87
|
+
const pdfRef = sourcesMode === 'copy' ? `${up}${pdfPath}` : doc.entry.url;
|
|
88
|
+
const sectionSlugs = new Set(['index']);
|
|
89
|
+
for (const section of doc.sections) {
|
|
90
|
+
section.slug = uniqueSlug(slugify(section.title), sectionSlugs);
|
|
91
|
+
const pageList = section.pages;
|
|
92
|
+
const evidence = pageList.map((n) => ({
|
|
93
|
+
source: 'pdf',
|
|
94
|
+
selector: { type: 'FragmentSelector', conformsTo: RFC3778, value: `page=${n}` },
|
|
95
|
+
...(doc.labels.get(n) ? { note: `p. ${doc.labels.get(n)}` } : {}),
|
|
96
|
+
}));
|
|
97
|
+
const firstText = section.blocks.find((b) => b.kind === 'paragraph');
|
|
98
|
+
const data = {
|
|
99
|
+
type,
|
|
100
|
+
title: section.title,
|
|
101
|
+
...(firstText ? { description: summary(firstText.text) } : {}),
|
|
102
|
+
...(doc.entry.url ? { resource: `${doc.entry.url}#page=${pageList[0]}` } : {}),
|
|
103
|
+
sources: [{ id: 'pdf', resource: pdfRef, title: `${doc.title} (PDF)` }],
|
|
104
|
+
moca: {
|
|
105
|
+
concepts: [{ iri: iri(`${doc.slug}/${section.slug}`), role: 'primary' }],
|
|
106
|
+
evidence,
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
contentNodes.push({ path: `${dir}/${section.slug}.md`, body: joinFrontmatter(data, sectionBody(section, doc, pdfRef)) });
|
|
110
|
+
}
|
|
111
|
+
if (multi) contentNodes.push({ path: `${dir}/index.md`, body: documentIndex(doc, pdfRef) });
|
|
112
|
+
}
|
|
113
|
+
contentNodes.push({ path: 'content/index.md', body: multi ? collectionIndex(manifest, docs) : documentIndex(docs[0], sourcesMode === 'copy' ? `../sources/${docs[0].slug}.pdf` : docs[0].entry.url) });
|
|
114
|
+
files.push({ path: 'structure.ttl', body: structure(manifest, docs, { iri, lang: options.language }) });
|
|
115
|
+
return { manifest, contentNodes, files, warnings };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The documents to convert, in order.
|
|
120
|
+
* @returns {{ title?: string, description?: string, documents: Array<{ file: string, abs: string, url?: string, sha256?: string, title?: string }> }}
|
|
121
|
+
*/
|
|
122
|
+
export function listDocuments(inputPath) {
|
|
123
|
+
if (!existsSync(inputPath)) throw new UsageError(`Input does not exist: ${inputPath}`);
|
|
124
|
+
if (statSync(inputPath).isFile()) {
|
|
125
|
+
if (!/\.pdf$/i.test(inputPath)) throw new UsageError(`${inputPath} is not a PDF.`);
|
|
126
|
+
return { documents: [{ file: basename(inputPath), abs: inputPath }] };
|
|
127
|
+
}
|
|
128
|
+
const descriptor = join(inputPath, DESCRIPTOR);
|
|
129
|
+
if (!existsSync(descriptor)) {
|
|
130
|
+
const found = walkFiles(inputPath).filter((f) => /\.pdf$/i.test(f)).sort();
|
|
131
|
+
if (found.length === 0) throw new UsageError(`${inputPath} holds no PDFs.`);
|
|
132
|
+
return { documents: found.map((file) => ({ file, abs: join(inputPath, file) })) };
|
|
133
|
+
}
|
|
134
|
+
let spec;
|
|
135
|
+
try {
|
|
136
|
+
spec = JSON.parse(readFileSync(descriptor, 'utf8'));
|
|
137
|
+
} catch (err) {
|
|
138
|
+
throw new UsageError(`${descriptor} is not valid JSON: ${err.message}`);
|
|
139
|
+
}
|
|
140
|
+
if (!Array.isArray(spec?.documents) || spec.documents.length === 0) throw new UsageError(`${descriptor} needs a non-empty "documents" list.`);
|
|
141
|
+
const documents = spec.documents.map((d, i) => {
|
|
142
|
+
if (typeof d?.file !== 'string' || d.file.split(/[\\/]/).includes('..')) throw new UsageError(`${descriptor}: documents[${i}] needs a "file" inside the folder.`);
|
|
143
|
+
if (d.url !== undefined && !/^https?:\/\/\S+$/.test(d.url)) throw new UsageError(`${descriptor}: documents[${i}].url must be an http(s) URL.`);
|
|
144
|
+
if (d.sha256 !== undefined && !/^[0-9a-f]{64}$/.test(d.sha256)) throw new UsageError(`${descriptor}: documents[${i}].sha256 must be 64 lowercase hex digits.`);
|
|
145
|
+
return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['url', 'sha256', 'title']) };
|
|
146
|
+
});
|
|
147
|
+
return { ...pick(spec, ['title', 'description']), documents };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function readPdf(entry) {
|
|
151
|
+
if (!existsSync(entry.abs)) {
|
|
152
|
+
throw new UsageError(`${entry.file} is missing${entry.url ? `; download it from ${entry.url}` : ''}.`);
|
|
153
|
+
}
|
|
154
|
+
const bytes = readFileSync(entry.abs);
|
|
155
|
+
if (entry.sha256) {
|
|
156
|
+
const actual = createHash('sha256').update(bytes).digest('hex');
|
|
157
|
+
if (actual !== entry.sha256) throw new UsageError(`${entry.file} has SHA-256 ${actual}, but ${DESCRIPTOR} expects ${entry.sha256}.`);
|
|
158
|
+
}
|
|
159
|
+
return bytes;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Splits blocks into sections at the first heading level that occurs more
|
|
164
|
+
* than once. Text before the first such heading is an opening section, named
|
|
165
|
+
* by the headings above it or by the document title.
|
|
166
|
+
* @param {import('../pdf/layout.js').Block[]} blocks
|
|
167
|
+
* @returns {Array<{ title: string, level: number, blocks: import('../pdf/layout.js').Block[], pages: number[] }>}
|
|
168
|
+
*/
|
|
169
|
+
export function splitSections(blocks, docTitle) {
|
|
170
|
+
const counts = new Map();
|
|
171
|
+
for (const b of blocks) if (b.kind === 'heading') counts.set(b.level, (counts.get(b.level) ?? 0) + 1);
|
|
172
|
+
const splitLevel = [...counts.keys()].sort((a, b) => a - b).find((l) => counts.get(l) > 1);
|
|
173
|
+
|
|
174
|
+
const sections = [];
|
|
175
|
+
let current = { title: null, level: splitLevel ?? 1, blocks: [], pages: new Set() };
|
|
176
|
+
const push = () => {
|
|
177
|
+
if (current.blocks.some((b) => b.kind !== 'heading')) sections.push(current);
|
|
178
|
+
};
|
|
179
|
+
for (const block of blocks) {
|
|
180
|
+
if (splitLevel && block.kind === 'heading' && block.level <= splitLevel) {
|
|
181
|
+
if (block.level === splitLevel || current.blocks.some((b) => b.kind !== 'heading')) {
|
|
182
|
+
push();
|
|
183
|
+
current = { title: block.text, level: splitLevel, blocks: [], pages: new Set([block.page]) };
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
// A heading above the split level before any text (a chapter title)
|
|
187
|
+
// names the opening section.
|
|
188
|
+
current.title = current.title ? `${current.title}: ${block.text}` : block.text;
|
|
189
|
+
current.pages.add(block.page);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
current.blocks.push(block);
|
|
193
|
+
for (let p = block.page; p <= (block.lastPage ?? block.page); p++) current.pages.add(p);
|
|
194
|
+
}
|
|
195
|
+
push();
|
|
196
|
+
return sections.map((s) => ({ ...s, title: s.title ?? docTitle, pages: [...s.pages].sort((a, b) => a - b) }));
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function sectionBody(section, doc, pdfRef) {
|
|
200
|
+
const range = pageRange(section.pages, doc.labels);
|
|
201
|
+
const out = [`# ${escapeInline(section.title)}`, `From ${escapeInline(doc.title)}, ${range} ([PDF](<${pdfRef}#page=${section.pages[0]}>)).`];
|
|
202
|
+
for (const block of section.blocks) {
|
|
203
|
+
if (block.kind === 'heading') out.push(`${'#'.repeat(Math.min(6, Math.max(2, block.level - section.level + 1)))} ${escapeInline(block.text)}`);
|
|
204
|
+
else if (block.kind === 'item') out.push(`${block.number ? `${block.number}.` : '-'} ${escapeInline(block.text)}`);
|
|
205
|
+
else if (block.kind === 'caption') out.push(`*${escapeInline(block.text)}*`);
|
|
206
|
+
else out.push(escapeLineStart(escapeInline(block.text)));
|
|
207
|
+
}
|
|
208
|
+
return `\n${joinBlocks(out, section.blocks)}\n`;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// List items that follow each other form one list; everything else is
|
|
212
|
+
// separated by a blank line.
|
|
213
|
+
function joinBlocks(lines, blocks) {
|
|
214
|
+
const kinds = [null, null, ...blocks.map((b) => (b.kind === 'item' ? (b.number ? 'ol' : 'ul') : b.kind))];
|
|
215
|
+
const listed = (k) => k === 'ol' || k === 'ul';
|
|
216
|
+
return lines.reduce((acc, line, i) => (i === 0 ? line : `${acc}${listed(kinds[i]) && kinds[i] === kinds[i - 1] ? '\n' : '\n\n'}${line}`), '');
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Printed page numbers when both ends have one, otherwise the PDF's own page
|
|
220
|
+
// numbers, said as such so they are not mistaken for printed ones.
|
|
221
|
+
function pageRange(pages, labels) {
|
|
222
|
+
const first = pages[0];
|
|
223
|
+
const last = pages.at(-1);
|
|
224
|
+
const printed = labels.has(first) && labels.has(last);
|
|
225
|
+
const show = (n) => (printed ? labels.get(n) : String(n));
|
|
226
|
+
const noun = `${printed ? '' : 'PDF '}page${first === last ? '' : 's'}`;
|
|
227
|
+
return first === last ? `${noun} ${show(first)}` : `${noun} ${show(first)} to ${show(last)}`;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function documentIndex(doc, pdfRef) {
|
|
231
|
+
const lines = [`# ${escapeInline(doc.title)}`, '', `Sections of [the PDF](<${pdfRef}>) in order:`, ''];
|
|
232
|
+
doc.sections.forEach((s, i) => lines.push(`${i + 1}. [${escapeInline(s.title)}](${s.slug}.md)`));
|
|
233
|
+
return `${lines.join('\n')}\n`;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function collectionIndex(manifest, docs) {
|
|
237
|
+
const lines = [`# ${escapeInline(manifest.title)}`, ''];
|
|
238
|
+
if (manifest.description) lines.push(escapeInline(manifest.description), '');
|
|
239
|
+
lines.push('Documents in order:', '');
|
|
240
|
+
docs.forEach((d, i) => lines.push(`${i + 1}. [${escapeInline(d.title)}](${d.slug}/index.md)`));
|
|
241
|
+
return `${lines.join('\n')}\n`;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function structure(manifest, docs, { iri, lang }) {
|
|
245
|
+
const label = (text) => `${turtleString(text)}${lang ? `@${lang}` : ''}`;
|
|
246
|
+
const ref = (fragment) => `<${iri(fragment)}>`;
|
|
247
|
+
const blocks = ['@prefix skos: <http://www.w3.org/2004/02/skos/core#> .\n@prefix dct: <http://purl.org/dc/terms/> .'];
|
|
248
|
+
const ordered = (fragment, text, members) =>
|
|
249
|
+
`${ref(fragment)} a skos:OrderedCollection ;\n skos:prefLabel ${label(text)} ;\n skos:memberList (\n ${members.join('\n ')}\n ) .`;
|
|
250
|
+
if (docs.length > 1) {
|
|
251
|
+
const members = docs.map((d) => ref(d.slug));
|
|
252
|
+
blocks.push(`${ref('collection')} a skos:Concept ;\n skos:prefLabel ${label(manifest.title)} ;\n dct:hasPart ${members.join(',\n ')} .`);
|
|
253
|
+
blocks.push(ordered('collection-order', `${manifest.title} (document order)`, members));
|
|
254
|
+
}
|
|
255
|
+
for (const doc of docs) {
|
|
256
|
+
const members = doc.sections.map((s) => ref(`${doc.slug}/${s.slug}`));
|
|
257
|
+
blocks.push(`${ref(doc.slug)} a skos:Concept ;\n skos:prefLabel ${label(doc.title)} ;\n dct:hasPart ${members.join(',\n ')} .`);
|
|
258
|
+
blocks.push(ordered(`${doc.slug}/order`, `${doc.title} (section order)`, members));
|
|
259
|
+
for (const s of doc.sections) blocks.push(`${ref(`${doc.slug}/${s.slug}`)} a skos:Concept ;\n skos:prefLabel ${label(s.title)} .`);
|
|
260
|
+
}
|
|
261
|
+
return `${blocks.join('\n\n')}\n`;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function firstHeading(blocks) {
|
|
265
|
+
return blocks.find((b) => b.kind === 'heading')?.text;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function pick(obj, keys) {
|
|
269
|
+
return Object.fromEntries(keys.filter((k) => typeof obj?.[k] === 'string' && obj[k].trim()).map((k) => [k, obj[k].trim()]));
|
|
270
|
+
}
|