@openmoca/moca-convert 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -13
- package/bin/moca-convert.js +33 -5
- package/lib/adapters/directory.js +4 -4
- package/lib/adapters/obsidian.js +18 -11
- package/lib/adapters/openapi/parse.js +39 -10
- package/lib/adapters/openapi/render.js +18 -10
- package/lib/adapters/openapi.js +4 -2
- package/lib/adapters/pdf.js +52 -13
- package/lib/adapters/youtube.js +79 -14
- package/lib/manifest.js +1 -1
- package/lib/pdf/extract.js +33 -3
- package/lib/text.js +55 -0
- package/lib/walk.js +4 -1
- package/lib/write.js +46 -21
- package/lib/youtube/capture.js +38 -5
- package/lib/youtube/ytdlp.js +9 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -8,8 +8,8 @@ concept document with a `type`, and the package id must be an absolute URI.
|
|
|
8
8
|
moca-convert <input> -o <dir> --id <uri> [--title text] [--version semver] [--type type]
|
|
9
9
|
[--language bcp47] [--license spdx] [--from directory|markdown|obsidian|openapi|pdf|youtube]
|
|
10
10
|
[--exclude glob...] [--chunker operation|tag] [--min-description-ratio 0-1]
|
|
11
|
-
[--sources copy|link]
|
|
12
|
-
[--force] [--strict] [--format text|json]
|
|
11
|
+
[--sources copy|link] [--max-pdf-mb n] [--max-pdf-pages n] [--pdf-timeout seconds]
|
|
12
|
+
[--force] [--strict] [--format text|json] [--log-file file]
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
| Adapter | Input | Default `type` |
|
|
@@ -25,8 +25,23 @@ moca-convert <input> -o <dir> --id <uri> [--title text] [--version semver] [--ty
|
|
|
25
25
|
`type` (and `title`, if missing) added at the top of its frontmatter.
|
|
26
26
|
- The adapter is detected from the input unless `--from` is given; ambiguous
|
|
27
27
|
input is refused, not guessed.
|
|
28
|
-
-
|
|
29
|
-
|
|
28
|
+
- Hidden files and folders (`.git/`, `.obsidian/`, Obsidian's `.trash/` of
|
|
29
|
+
deleted notes) and symbolic links in an input folder are not read.
|
|
30
|
+
- Obsidian wikilinks are rewritten in the note's body only, never in its
|
|
31
|
+
front matter, and each link target is percent-encoded.
|
|
32
|
+
- OpenAPI: each operation's parameters include its path's and those it
|
|
33
|
+
references (`$ref`). Schema references are expanded up to 1,000 objects a
|
|
34
|
+
schema; beyond that they are shown as `{"$ref": ...}`. Two operations whose
|
|
35
|
+
paths give the same file name get distinct files (`get-a-b.md`,
|
|
36
|
+
`get-a-b-2.md`).
|
|
37
|
+
- Output is written to a folder beside `-o`, read back with the reference
|
|
38
|
+
Reader, and only then moved into place. If it would be invalid, nothing is
|
|
39
|
+
written.
|
|
40
|
+
- `--force` replaces an output folder only when it holds a previous package
|
|
41
|
+
(it has a `moca.json`); it refuses any other folder. The previous package is
|
|
42
|
+
moved aside, the new one moved in, and only then is the old one deleted.
|
|
43
|
+
- `--log-file <file>` records the adapter, timings and any failure, on every
|
|
44
|
+
exit.
|
|
30
45
|
|
|
31
46
|
## PDF
|
|
32
47
|
|
|
@@ -51,6 +66,12 @@ PDFs with a text layer become one node per section:
|
|
|
51
66
|
its `url` instead, for PDFs too large to carry.
|
|
52
67
|
- A PDF without a text layer (a scan) is reported and left out; run OCR on
|
|
53
68
|
it first.
|
|
69
|
+
- A PDF over 256 MB or 2,000 pages, or that takes over 10 minutes to read, is
|
|
70
|
+
refused; `--max-pdf-mb`, `--max-pdf-pages` and `--pdf-timeout` raise the
|
|
71
|
+
limits.
|
|
72
|
+
- Text from the PDF itself (headings, paragraphs, page labels) is written as
|
|
73
|
+
text: characters Markdown would read as links or HTML are escaped. Page
|
|
74
|
+
labels are kept short and on one line.
|
|
54
75
|
|
|
55
76
|
A folder may list its documents, in order, in `pdfs.json`:
|
|
56
77
|
|
|
@@ -64,8 +85,10 @@ A folder may list its documents, in order, in `pdfs.json`:
|
|
|
64
85
|
}
|
|
65
86
|
```
|
|
66
87
|
|
|
67
|
-
Only `file` is required
|
|
68
|
-
|
|
88
|
+
Only `file` is required: a regular file inside the folder, not a symbolic
|
|
89
|
+
link. `title` falls back to the PDF's own title. `url` is the published copy
|
|
90
|
+
that `--sources link` cites: an `http` or `https` URL with no user name,
|
|
91
|
+
password or `#fragment`, parsed and written back in its normal form. A file whose SHA-256 is
|
|
69
92
|
not `sha256` is refused, so a conversion never silently uses a changed
|
|
70
93
|
document. Without `pdfs.json`, every PDF in the folder is converted in path
|
|
71
94
|
order. moca-spec's [`scripts/fetch-pdfs.mjs`](https://github.com/openmoca/moca-spec/blob/main/scripts/fetch-pdfs.mjs) downloads the
|
|
@@ -89,7 +112,9 @@ moca-youtube <url> -o <dir> --id <uri> [--title text] [--version semver] [--type
|
|
|
89
112
|
captions grouped by chapter. The captions are written under `media/` as
|
|
90
113
|
WebVTT and the description under `sources/`, and the node cites each
|
|
91
114
|
chapter as a time range (`t=start,end`) and quotes the description, so the
|
|
92
|
-
package is `self-contained-evidence`.
|
|
115
|
+
package is `self-contained-evidence`. The node describes the video in
|
|
116
|
+
`moca.media`: its page's type (`text/html`), its length and its captions, so
|
|
117
|
+
the last chapter can run past the last caption.
|
|
93
118
|
- A playlist also gets `content/index.md` and a `structure.ttl` that keeps the
|
|
94
119
|
videos in playlist order.
|
|
95
120
|
- Captions the creator supplied in `--language` (default `en`) are preferred,
|
|
@@ -100,6 +125,15 @@ moca-youtube <url> -o <dir> --id <uri> [--title text] [--version semver] [--type
|
|
|
100
125
|
capture goes to a temporary directory.
|
|
101
126
|
- yt-dlp is run directly, without a shell, so this works on macOS, Linux and
|
|
102
127
|
Windows. Point at it with `--yt-dlp` or `$YT_DLP` when it is not on `PATH`.
|
|
128
|
+
It runs with `--ignore-config`, so your yt-dlp configuration files are not
|
|
129
|
+
used, and each run is stopped after 5 minutes.
|
|
130
|
+
- The URL must be on `youtube.com` (with `www.`, `m.` or `music.`) or
|
|
131
|
+
`youtu.be`. Video and playlist ids must be plain ids (letters, digits, `-`
|
|
132
|
+
and `_`); an entry with any other id is skipped and reported.
|
|
133
|
+
- When a capture is converted, it is checked as it is read: video ids, URLs
|
|
134
|
+
(parsed and rebuilt; `http` and `https` only), the upload date and the
|
|
135
|
+
caption language. A value that fails is left out, or the capture is
|
|
136
|
+
refused.
|
|
103
137
|
|
|
104
138
|
See [`examples/youtube`](https://github.com/openmoca/moca-spec/blob/main/examples/youtube).
|
|
105
139
|
|
|
@@ -117,13 +151,37 @@ npm test
|
|
|
117
151
|
## Release
|
|
118
152
|
|
|
119
153
|
1. Publish `@openmoca/moca-core` first if it changed.
|
|
120
|
-
2.
|
|
121
|
-
|
|
154
|
+
2. Name its released range in `package.json` in place of the `file:` path,
|
|
155
|
+
and run `npm install`.
|
|
156
|
+
3. Add a dated section to [CHANGELOG.md](CHANGELOG.md) and update the version.
|
|
157
|
+
4. Point the `MOCA_*_REF` pins in `.github/workflows/test.yml` at the release tags
|
|
122
158
|
([moca-spec docs/releasing.md](https://github.com/openmoca/moca-spec/blob/main/docs/releasing.md#moving-the-pins)).
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
section.
|
|
126
|
-
|
|
159
|
+
5. `npm test` passes.
|
|
160
|
+
6. Tag `vX.Y.Z` on main and publish a GitHub release whose notes are the
|
|
161
|
+
changelog section. The release workflow publishes to npm through trusted
|
|
162
|
+
publishing, with provenance; it stops if the tag is not on main, the
|
|
163
|
+
version does not match, or `@openmoca/moca-core` is not on npm.
|
|
164
|
+
|
|
165
|
+
## Standards
|
|
166
|
+
|
|
167
|
+
What moca-convert reads and writes, and how strongly. **Follow** means it
|
|
168
|
+
implements the standard and the tests check it where they can.
|
|
169
|
+
|
|
170
|
+
| Standard | Where | How strongly |
|
|
171
|
+
| --- | --- | --- |
|
|
172
|
+
| MOCA ([moca-spec](https://github.com/openmoca/moca-spec)) 0.7 | every package it writes, checked with the reference Reader | follow |
|
|
173
|
+
| [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format/blob/ad30107c31c06aec8a7d5636e0d1058118604e6f/SPEC.md) v0.2, pinned at commit `ad30107` | `content/`: each node an OKF concept document with a `type` | follow (pinned) |
|
|
174
|
+
| [CommonMark](https://spec.commonmark.org/) | node bodies; text from input is escaped, link destinations percent-encoded | follow |
|
|
175
|
+
| [YAML 1.2](https://yaml.org/spec/1.2.2/) | front matter, and OpenAPI documents in YAML | follow |
|
|
176
|
+
| [SKOS](https://www.w3.org/TR/skos-reference/) and [Dublin Core terms](https://www.dublincore.org/specifications/dublin-core/dcmi-terms/) in [RDF 1.1 Turtle](https://www.w3.org/TR/turtle/) | `structure.ttl`; IRIs percent-encoded ([RFC 3987](https://www.rfc-editor.org/rfc/rfc3987)) | follow (the named terms) |
|
|
177
|
+
| [RFC 3778](https://www.rfc-editor.org/rfc/rfc3778) `page=` fragments | PDF evidence | follow |
|
|
178
|
+
| [Media Fragments](https://www.w3.org/TR/media-frags/) `t=` | YouTube evidence | follow |
|
|
179
|
+
| [WebVTT](https://www.w3.org/TR/webvtt1/) | YouTube captions under `media/` | follow |
|
|
180
|
+
| [OpenAPI 3.x](https://spec.openapis.org/oas/v3.1.0) | the `openapi` adapter's input; local `$ref` only | follow (the parts it reads) |
|
|
181
|
+
| [WHATWG URL](https://url.spec.whatwg.org/) | URLs from input, parsed and rebuilt; `http` and `https` only | follow |
|
|
182
|
+
| [BCP 47](https://www.rfc-editor.org/info/bcp47) | `--language`, and caption languages | follow |
|
|
183
|
+
|
|
184
|
+
Built with the help of AI assistants; every change is reviewed and accepted by the maintainer.
|
|
127
185
|
|
|
128
186
|
## Licence
|
|
129
187
|
|
package/bin/moca-convert.js
CHANGED
|
@@ -12,8 +12,18 @@ const ADAPTER_ONLY_FLAGS = {
|
|
|
12
12
|
minDescriptionRatio: 'openapi',
|
|
13
13
|
chunker: 'openapi',
|
|
14
14
|
sources: 'pdf',
|
|
15
|
+
maxPdfMb: 'pdf',
|
|
16
|
+
maxPdfPages: 'pdf',
|
|
17
|
+
pdfTimeout: 'pdf',
|
|
15
18
|
};
|
|
16
19
|
|
|
20
|
+
function parsePositive(value) {
|
|
21
|
+
if (!/^[0-9]+$/.test(value) || Number(value) < 1 || !Number.isSafeInteger(Number(value))) {
|
|
22
|
+
throw new InvalidArgumentError('must be a positive whole number.');
|
|
23
|
+
}
|
|
24
|
+
return Number(value);
|
|
25
|
+
}
|
|
26
|
+
|
|
17
27
|
function parseRatio(value) {
|
|
18
28
|
const num = Number(value);
|
|
19
29
|
if (Number.isNaN(num) || num < 0 || num > 1) {
|
|
@@ -48,8 +58,11 @@ program
|
|
|
48
58
|
.option('--min-description-ratio <ratio>', 'openapi only: minimum fraction of operations needing a summary/description, 0-1 (default 0.5)', parseRatio)
|
|
49
59
|
.option('--chunker <mode>', 'openapi only: operation|tag content-node grouping (default operation)')
|
|
50
60
|
.option('--sources <mode>', 'pdf only: copy the PDFs into sources/ (copy, the default) or cite each by its pdfs.json url (link)')
|
|
61
|
+
.option('--max-pdf-mb <n>', 'pdf only: the largest PDF to read, in MB (default 256)', parsePositive)
|
|
62
|
+
.option('--max-pdf-pages <n>', 'pdf only: the most pages a PDF may have (default 2000)', parsePositive)
|
|
63
|
+
.option('--pdf-timeout <seconds>', 'pdf only: the longest one PDF may take to read (default 600)', parsePositive)
|
|
51
64
|
.option('--no-structure', 'directory only: do not write structure.ttl from the folders, or bind nodes to it')
|
|
52
|
-
.option('--force', '
|
|
65
|
+
.option('--force', 'replace the output directory when it holds a previous package', false)
|
|
53
66
|
.option('--strict', 'escalate warning-level output-validation findings to errors', false)
|
|
54
67
|
.option('--format <fmt>', 'summary output format: text|json', 'text')
|
|
55
68
|
.option('--report <file>', 'write the formatted summary to a file (in addition to console output)')
|
|
@@ -59,6 +72,8 @@ program
|
|
|
59
72
|
.option('--no-color', 'disable colored output')
|
|
60
73
|
.action(async (input, options) => {
|
|
61
74
|
await run(input, options);
|
|
75
|
+
// The log file is written by now; exit even if a library left a handle open.
|
|
76
|
+
process.exit(process.exitCode ?? 0);
|
|
62
77
|
});
|
|
63
78
|
|
|
64
79
|
// Commander's own usage errors (missing required options, unknown flags,
|
|
@@ -104,6 +119,8 @@ async function run(input, options) {
|
|
|
104
119
|
const adapterName = resolveAdapterName(target, options.from);
|
|
105
120
|
assertAdapterSpecificFlags(adapterName, options);
|
|
106
121
|
const adapter = getAdapter(adapterName);
|
|
122
|
+
logger.log(`adapter ${adapterName}`);
|
|
123
|
+
const started = performance.now();
|
|
107
124
|
|
|
108
125
|
const draft = await adapter.convert({
|
|
109
126
|
inputPath: input,
|
|
@@ -115,6 +132,9 @@ async function run(input, options) {
|
|
|
115
132
|
minDescriptionRatio: options.minDescriptionRatio,
|
|
116
133
|
chunker: options.chunker,
|
|
117
134
|
sources: options.sources,
|
|
135
|
+
...(options.maxPdfMb ? { maxPdfBytes: options.maxPdfMb * 1024 * 1024 } : {}),
|
|
136
|
+
...(options.maxPdfPages ? { maxPdfPages: options.maxPdfPages } : {}),
|
|
137
|
+
...(options.pdfTimeout ? { pdfTimeoutMs: options.pdfTimeout * 1000 } : {}),
|
|
118
138
|
structure: options.structure,
|
|
119
139
|
type: options.type,
|
|
120
140
|
language: options.language,
|
|
@@ -122,6 +142,7 @@ async function run(input, options) {
|
|
|
122
142
|
},
|
|
123
143
|
});
|
|
124
144
|
|
|
145
|
+
logger.log(`converted ${draft.contentNodes.length} content node(s) in ${Math.round(performance.now() - started)} ms`);
|
|
125
146
|
const { findings, digest } = await writeDraft({
|
|
126
147
|
draft,
|
|
127
148
|
outDir: options.output,
|
|
@@ -130,6 +151,7 @@ async function run(input, options) {
|
|
|
130
151
|
onLog: logger.log,
|
|
131
152
|
});
|
|
132
153
|
|
|
154
|
+
logger.log(`wrote ${options.output} in ${Math.round(performance.now() - started)} ms in all`);
|
|
133
155
|
if (!options.quiet) {
|
|
134
156
|
console.log(`Wrote ${options.output} (${draft.contentNodes.length} content node(s), ${draft.warnings.length} warning(s))\n${digest}`);
|
|
135
157
|
}
|
|
@@ -139,18 +161,24 @@ async function run(input, options) {
|
|
|
139
161
|
if (findings.length > 0) {
|
|
140
162
|
writeReport(options, formatFindings(findings, options.format));
|
|
141
163
|
}
|
|
142
|
-
process.
|
|
164
|
+
process.exitCode = 0;
|
|
143
165
|
} catch (err) {
|
|
166
|
+
// Set the exit code and return, so the finally block below writes the log
|
|
167
|
+
// file on every path.
|
|
144
168
|
if (err instanceof UsageError) {
|
|
145
169
|
console.error(`moca-convert: ${err.message}`);
|
|
146
|
-
process.
|
|
170
|
+
process.exitCode = 2;
|
|
171
|
+
return;
|
|
147
172
|
}
|
|
148
173
|
if (err instanceof ConversionFailedError) {
|
|
149
174
|
console.error(`moca-convert: ${err.message}`);
|
|
150
175
|
writeReport(options, formatFindings(err.findings, options.format));
|
|
151
|
-
process.
|
|
176
|
+
process.exitCode = 1;
|
|
177
|
+
return;
|
|
152
178
|
}
|
|
153
|
-
|
|
179
|
+
logger.log(`failed: ${err?.stack ?? err}`);
|
|
180
|
+
console.error(`moca-convert: failed: ${err?.message ?? err}`);
|
|
181
|
+
process.exitCode = 1;
|
|
154
182
|
} finally {
|
|
155
183
|
logger.flush();
|
|
156
184
|
}
|
|
@@ -11,7 +11,7 @@ import { walkFiles } from '../walk.js';
|
|
|
11
11
|
import { readContentFile, ensureOkf, deriveTitle } from '../frontmatter.js';
|
|
12
12
|
import { buildManifest } from '../manifest.js';
|
|
13
13
|
import { UsageError } from '../target.js';
|
|
14
|
-
import { turtleString } from '../text.js';
|
|
14
|
+
import { iriPath, turtleString } from '../text.js';
|
|
15
15
|
|
|
16
16
|
export const name = 'directory';
|
|
17
17
|
export const DEFAULT_TYPE = 'Document';
|
|
@@ -65,9 +65,9 @@ export function convert({ inputPath, options }) {
|
|
|
65
65
|
return result;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
/** A file's concept fragment: its path without .md. */
|
|
68
|
+
/** A file's concept fragment: its path without .md, each segment percent-encoded. */
|
|
69
69
|
function fileFragment(relPath) {
|
|
70
|
-
return relPath.replace(/\.md$/, '');
|
|
70
|
+
return iriPath(relPath.replace(/\.md$/, ''));
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
function isFolderPage(relPath) {
|
|
@@ -115,7 +115,7 @@ function structure(manifest, files, titles, { iri, lang }) {
|
|
|
115
115
|
children.get(parent).add(child);
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
|
-
const fragment = (path) => (path === '' ? 'collection' : path.endsWith('.md') ? fileFragment(path) : path);
|
|
118
|
+
const fragment = (path) => (path === '' ? 'collection' : path.endsWith('.md') ? fileFragment(path) : iriPath(path));
|
|
119
119
|
const order = (path) => [...children.get(path)].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
120
120
|
const blocks = ['@prefix skos: <http://www.w3.org/2004/02/skos/core#> .\n@prefix dct: <http://purl.org/dc/terms/> .'];
|
|
121
121
|
for (const container of [...children.keys()].sort()) {
|
package/lib/adapters/obsidian.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// rewriting [[wikilink]] syntax to relative Markdown links.
|
|
4
4
|
import { existsSync, statSync } from 'node:fs';
|
|
5
5
|
import { basename, join, posix } from 'node:path';
|
|
6
|
+
import { splitFrontmatter } from '@openmoca/moca-core';
|
|
6
7
|
import { walkFiles } from '../walk.js';
|
|
7
8
|
import { readContentFile, ensureOkf, deriveTitle } from '../frontmatter.js';
|
|
8
9
|
import { buildManifest } from '../manifest.js';
|
|
@@ -36,9 +37,8 @@ export function convert({ inputPath, options }) {
|
|
|
36
37
|
const manifest = buildManifest({ ...options, title: options.title ?? basename(inputPath) });
|
|
37
38
|
const type = options.type ?? DEFAULT_TYPE;
|
|
38
39
|
const excludePatterns = options.exclude ?? [];
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
.filter((f) => f !== '.obsidian' && !f.startsWith('.obsidian/'));
|
|
40
|
+
// walkFiles skips hidden folders: .obsidian/, and .trash/ where Obsidian keeps deleted notes.
|
|
41
|
+
const allNotePaths = walkFiles(inputPath).filter((f) => f.endsWith('.md'));
|
|
42
42
|
|
|
43
43
|
if (allNotePaths.length === 0) {
|
|
44
44
|
throw new UsageError(`No notes found under ${inputPath}.`);
|
|
@@ -61,7 +61,11 @@ export function convert({ inputPath, options }) {
|
|
|
61
61
|
|
|
62
62
|
function buildContentNode(note, noteIndex, warnings, type) {
|
|
63
63
|
const title = deriveTitle(note.data, note.content, basename(note.relPath, '.md'));
|
|
64
|
-
|
|
64
|
+
// Wikilinks are rewritten in the body only: the front matter is kept as it is.
|
|
65
|
+
const text = ensureOkf(note.raw, note.data, { type, title });
|
|
66
|
+
const split = splitFrontmatter(text);
|
|
67
|
+
const head = split.present && !split.error ? Buffer.from(text, 'utf8').subarray(0, split.bodyOffset).toString('utf8') : '';
|
|
68
|
+
const body = head + rewriteWikilinks(text.slice(head.length), note, noteIndex, warnings);
|
|
65
69
|
return { path: `content/${note.relPath}`, body };
|
|
66
70
|
}
|
|
67
71
|
|
|
@@ -126,15 +130,18 @@ function rewriteMatch({ match, bang, rawTarget, heading, alias, sourceNote, note
|
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
/**
|
|
129
|
-
* A
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
* @param {string} destination
|
|
133
|
+
* A link destination with each path segment percent-encoded: vault file names
|
|
134
|
+
* often hold spaces ("Setup Guide.md") and may hold any other character, and
|
|
135
|
+
* an encoded destination is always one link. `.` and `..` segments are kept.
|
|
136
|
+
* @param {string} destination a relative path, with an optional #anchor
|
|
134
137
|
*/
|
|
135
138
|
function formatLinkDestination(destination) {
|
|
136
|
-
|
|
137
|
-
|
|
139
|
+
const hash = destination.indexOf('#');
|
|
140
|
+
const path = hash === -1 ? destination : destination.slice(0, hash);
|
|
141
|
+
const anchor = hash === -1 ? '' : destination.slice(hash);
|
|
142
|
+
// Parentheses too: an unbalanced one would end a bare link destination.
|
|
143
|
+
const encode = (seg) => encodeURIComponent(seg).replace(/[()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
144
|
+
return path.split('/').map((seg) => (seg === '.' || seg === '..' ? seg : encode(seg))).join('/') + anchor;
|
|
138
145
|
}
|
|
139
146
|
|
|
140
147
|
/**
|
|
@@ -40,7 +40,10 @@ export function parseOpenApiDocument(inputPath) {
|
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
42
|
* @param {object} doc
|
|
43
|
-
* @returns {Array<{ path: string, method: string, operation: object }>}
|
|
43
|
+
* @returns {Array<{ path: string, method: string, operation: object, parameters: object[] }>}
|
|
44
|
+
* `parameters` are the operation's own and its path's, with local $refs
|
|
45
|
+
* resolved; the operation's own replaces a path parameter with the same
|
|
46
|
+
* name and location.
|
|
44
47
|
* sorted by path (lexicographic), then by a fixed HTTP method order --
|
|
45
48
|
* not object-insertion order, which isn't guaranteed stable across
|
|
46
49
|
* different JSON/YAML parsers.
|
|
@@ -53,7 +56,8 @@ export function collectOperations(doc) {
|
|
|
53
56
|
const pathItem = paths[path] ?? {};
|
|
54
57
|
for (const method of HTTP_METHODS) {
|
|
55
58
|
if (pathItem[method]) {
|
|
56
|
-
|
|
59
|
+
const operation = pathItem[method];
|
|
60
|
+
operations.push({ path, method, operation, parameters: mergeParameters(doc, pathItem.parameters, operation.parameters) });
|
|
57
61
|
}
|
|
58
62
|
}
|
|
59
63
|
}
|
|
@@ -66,6 +70,17 @@ export function collectOperations(doc) {
|
|
|
66
70
|
return operations;
|
|
67
71
|
}
|
|
68
72
|
|
|
73
|
+
/** OpenAPI: a path's parameters apply to each of its operations, unless the operation lists one with the same name and `in`. */
|
|
74
|
+
function mergeParameters(doc, pathLevel, operationLevel) {
|
|
75
|
+
const resolve = (list) => (Array.isArray(list) ? list : [])
|
|
76
|
+
.map((p) => (typeof p?.$ref === 'string' ? resolveLocalRef(doc, p.$ref) ?? { name: p.$ref.split('/').pop(), in: '' } : p))
|
|
77
|
+
.filter((p) => p && typeof p === 'object');
|
|
78
|
+
const own = resolve(operationLevel);
|
|
79
|
+
const key = (p) => `${p.name}\u0000${p.in}`;
|
|
80
|
+
const ownKeys = new Set(own.map(key));
|
|
81
|
+
return [...resolve(pathLevel).filter((p) => !ownKeys.has(key(p))), ...own];
|
|
82
|
+
}
|
|
83
|
+
|
|
69
84
|
/**
|
|
70
85
|
* Resolves a local (same-document) JSON Pointer $ref, e.g.
|
|
71
86
|
* "#/components/schemas/Widget". Returns undefined if the ref doesn't
|
|
@@ -86,29 +101,43 @@ function decodeRefSegment(segment) {
|
|
|
86
101
|
return segment.replace(/~1/g, '/').replace(/~0/g, '~');
|
|
87
102
|
}
|
|
88
103
|
|
|
104
|
+
/** The most objects and arrays one rendered schema may hold, however its references fan out. */
|
|
105
|
+
export const MAX_SCHEMA_NODES = 1_000;
|
|
106
|
+
|
|
89
107
|
/**
|
|
90
108
|
* Recursively expands local $ref pointers within a schema/value so rendered
|
|
91
109
|
* code blocks show the actual shape rather than a bare pointer. An external
|
|
92
110
|
* ref (doesn't start with "#/"), an unresolvable ref, or a ref cycle is left
|
|
93
111
|
* as a literal `{ "$ref": "..." }` marker instead of failing.
|
|
112
|
+
*
|
|
113
|
+
* The expansion is bounded: once it has made `maxNodes` objects and arrays,
|
|
114
|
+
* every further reference is left as its `{ "$ref": "..." }` marker and every
|
|
115
|
+
* further object as "…", so references that fan out at each level cannot
|
|
116
|
+
* take exponential time or memory.
|
|
94
117
|
* @param {*} value
|
|
95
118
|
* @param {object} doc
|
|
96
|
-
* @param {
|
|
119
|
+
* @param {{ maxNodes?: number }} [limits]
|
|
120
|
+
* @returns {*} the expanded value; `truncated` is not part of it
|
|
97
121
|
*/
|
|
98
|
-
export function resolveSchema(value, doc,
|
|
99
|
-
|
|
100
|
-
|
|
122
|
+
export function resolveSchema(value, doc, { maxNodes = MAX_SCHEMA_NODES } = {}) {
|
|
123
|
+
const ctx = { doc, nodes: 0, maxNodes };
|
|
124
|
+
return expand(value, ctx, new Set());
|
|
125
|
+
}
|
|
101
126
|
|
|
127
|
+
function expand(value, ctx, seen) {
|
|
128
|
+
if (value === null || typeof value !== 'object') return value;
|
|
102
129
|
if (typeof value.$ref === 'string') {
|
|
103
|
-
if (!value.$ref.startsWith('#/') || seen.has(value.$ref)) return { $ref: value.$ref };
|
|
104
|
-
const target = resolveLocalRef(doc, value.$ref);
|
|
130
|
+
if (!value.$ref.startsWith('#/') || seen.has(value.$ref) || ctx.nodes >= ctx.maxNodes) return { $ref: value.$ref };
|
|
131
|
+
const target = resolveLocalRef(ctx.doc, value.$ref);
|
|
105
132
|
if (target === undefined) return { $ref: value.$ref };
|
|
106
|
-
return
|
|
133
|
+
return expand(target, ctx, new Set(seen).add(value.$ref));
|
|
107
134
|
}
|
|
135
|
+
if (++ctx.nodes > ctx.maxNodes) return '…';
|
|
136
|
+
if (Array.isArray(value)) return value.map((item) => expand(item, ctx, seen));
|
|
108
137
|
|
|
109
138
|
const result = {};
|
|
110
139
|
for (const [key, nested] of Object.entries(value)) {
|
|
111
|
-
result[key] =
|
|
140
|
+
result[key] = expand(nested, ctx, seen);
|
|
112
141
|
}
|
|
113
142
|
return result;
|
|
114
143
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// operation, or one node per tag with each operation as a subsection.
|
|
3
3
|
import { joinFrontmatter } from '@openmoca/moca-core';
|
|
4
4
|
import { slugify } from '../../slug.js';
|
|
5
|
+
import { uniqueSlug } from '../../text.js';
|
|
5
6
|
import { resolveSchema } from './parse.js';
|
|
6
7
|
|
|
7
8
|
const OPERATION_ID_PATTERN = /^[A-Za-z0-9_.:-]+$/;
|
|
@@ -11,9 +12,12 @@ const OPERATION_ID_PATTERN = /^[A-Za-z0-9_.:-]+$/;
|
|
|
11
12
|
* @param {string} method
|
|
12
13
|
* @param {object} operation
|
|
13
14
|
* @param {object} doc
|
|
15
|
+
* @param {{ parameters?: object[], taken?: Set<string> }} [more] `parameters`: the
|
|
16
|
+
* operation's and its path's (collectOperations); `taken`: file names already used,
|
|
17
|
+
* so two operations whose paths slug alike get distinct files
|
|
14
18
|
* @returns {import('../index.js').ContentNode}
|
|
15
19
|
*/
|
|
16
|
-
export function renderOperationNode(path, method, operation, doc) {
|
|
20
|
+
export function renderOperationNode(path, method, operation, doc, { parameters = operation.parameters ?? [], taken = new Set() } = {}) {
|
|
17
21
|
const title = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
|
|
18
22
|
const operationId = safeOperationId(operation.operationId);
|
|
19
23
|
const frontmatter = {
|
|
@@ -22,9 +26,9 @@ export function renderOperationNode(path, method, operation, doc) {
|
|
|
22
26
|
resource: `${method.toUpperCase()} ${path}`,
|
|
23
27
|
...(operationId ? { tags: [operationId] } : {}),
|
|
24
28
|
};
|
|
25
|
-
const body = `# ${title}\n\n${renderSections(operation, doc, 2)}`;
|
|
29
|
+
const body = `# ${title}\n\n${renderSections(operation, doc, 2, parameters)}`;
|
|
26
30
|
return {
|
|
27
|
-
path: `content/${method}-${slugifyOperationPath(path)}.md`,
|
|
31
|
+
path: `content/${uniqueSlug(`${method}-${slugifyOperationPath(path)}`, taken)}.md`,
|
|
28
32
|
body: joinFrontmatter(frontmatter, body),
|
|
29
33
|
};
|
|
30
34
|
}
|
|
@@ -35,16 +39,16 @@ export function renderOperationNode(path, method, operation, doc) {
|
|
|
35
39
|
* @param {object} doc
|
|
36
40
|
* @returns {import('../index.js').ContentNode}
|
|
37
41
|
*/
|
|
38
|
-
export function renderTagNode(tag, operations, doc) {
|
|
42
|
+
export function renderTagNode(tag, operations, doc, taken = new Set()) {
|
|
39
43
|
const sections = operations
|
|
40
|
-
.map(({ path, method, operation }) => {
|
|
44
|
+
.map(({ path, method, operation, parameters }) => {
|
|
41
45
|
const opTitle = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
|
|
42
|
-
return `## ${method.toUpperCase()} ${path} — ${opTitle}\n\n${renderSections(operation, doc, 3)}`;
|
|
46
|
+
return `## ${method.toUpperCase()} ${path} — ${opTitle}\n\n${renderSections(operation, doc, 3, parameters ?? operation.parameters ?? [])}`;
|
|
43
47
|
})
|
|
44
48
|
.join('\n');
|
|
45
49
|
const body = `# ${tag}\n\n${sections}`;
|
|
46
50
|
return {
|
|
47
|
-
path: `content/${slugify(tag)}.md`,
|
|
51
|
+
path: `content/${uniqueSlug(slugify(tag), taken)}.md`,
|
|
48
52
|
body: joinFrontmatter({ type: 'API Tag', title: tag }, body),
|
|
49
53
|
};
|
|
50
54
|
}
|
|
@@ -55,14 +59,13 @@ export function renderTagNode(tag, operations, doc) {
|
|
|
55
59
|
* "##" sections, 3 when nested one level deeper under a tag node's "## METHOD
|
|
56
60
|
* path" heading).
|
|
57
61
|
*/
|
|
58
|
-
function renderSections(operation, doc, level) {
|
|
62
|
+
function renderSections(operation, doc, level, parameters) {
|
|
59
63
|
const h = '#'.repeat(level);
|
|
60
64
|
const hSub = '#'.repeat(level + 1);
|
|
61
65
|
const lines = [];
|
|
62
66
|
|
|
63
67
|
lines.push(`${h} Description`, '', describe(operation), '');
|
|
64
68
|
|
|
65
|
-
const parameters = operation.parameters ?? [];
|
|
66
69
|
const requestBody = operation.requestBody;
|
|
67
70
|
if (parameters.length > 0 || requestBody) {
|
|
68
71
|
lines.push(`${h} Request`, '');
|
|
@@ -105,11 +108,16 @@ function describe(operation) {
|
|
|
105
108
|
function parameterTable(parameters) {
|
|
106
109
|
const rows = ['| Parameter | In | Required | Type |', '|---|---|---|---|'];
|
|
107
110
|
for (const param of parameters) {
|
|
108
|
-
rows.push(`| ${param.name
|
|
111
|
+
rows.push(`| ${cell(param.name)} | ${cell(param.in)} | ${param.required ? 'yes' : 'no'} | ${cell(schemaType(param.schema))} |`);
|
|
109
112
|
}
|
|
110
113
|
return rows;
|
|
111
114
|
}
|
|
112
115
|
|
|
116
|
+
/** A table cell from the document: one line, with `|` escaped so it cannot add a column. */
|
|
117
|
+
function cell(value) {
|
|
118
|
+
return typeof value === 'string' ? value.replace(/[\r\n]+/g, ' ').replace(/\\/g, '\\\\').replace(/\|/g, '\\|') : '';
|
|
119
|
+
}
|
|
120
|
+
|
|
113
121
|
function schemaType(schema) {
|
|
114
122
|
if (!schema || typeof schema !== 'object') return '';
|
|
115
123
|
if (typeof schema.type === 'string') return schema.type;
|
package/lib/adapters/openapi.js
CHANGED
|
@@ -63,7 +63,8 @@ export function convert({ inputPath, options }) {
|
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
function buildOperationNodes(operations, doc) {
|
|
66
|
-
|
|
66
|
+
const taken = new Set();
|
|
67
|
+
return operations.map(({ path, method, operation, parameters }) => renderOperationNode(path, method, operation, doc, { parameters, taken }));
|
|
67
68
|
}
|
|
68
69
|
|
|
69
70
|
function buildTagNodes(operations, doc) {
|
|
@@ -73,5 +74,6 @@ function buildTagNodes(operations, doc) {
|
|
|
73
74
|
if (!groups.has(tag)) groups.set(tag, []);
|
|
74
75
|
groups.get(tag).push(entry);
|
|
75
76
|
}
|
|
76
|
-
|
|
77
|
+
const taken = new Set();
|
|
78
|
+
return [...groups.keys()].sort().map((tag) => renderTagNode(tag, groups.get(tag), doc, taken));
|
|
77
79
|
}
|
package/lib/adapters/pdf.js
CHANGED
|
@@ -9,15 +9,15 @@
|
|
|
9
9
|
// that lists the documents in order, with an optional title, source URL and
|
|
10
10
|
// SHA-256 for each (see README).
|
|
11
11
|
import { createHash } from 'node:crypto';
|
|
12
|
-
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
13
|
-
import { basename, join } from 'node:path';
|
|
12
|
+
import { closeSync, constants, existsSync, fstatSync, openSync, readFileSync, realpathSync, statSync } from 'node:fs';
|
|
13
|
+
import { basename, join, sep } from 'node:path';
|
|
14
14
|
import { joinFrontmatter } from '@openmoca/moca-core';
|
|
15
15
|
import { buildManifest } from '../manifest.js';
|
|
16
16
|
import { slugify } from '../slug.js';
|
|
17
17
|
import { UsageError } from '../target.js';
|
|
18
18
|
import { walkFiles } from '../walk.js';
|
|
19
|
-
import { escapeInline, escapeLineStart, summary, turtleString, uniqueSlug } from '../text.js';
|
|
20
|
-
import { extractPdf } from '../pdf/extract.js';
|
|
19
|
+
import { escapeInline, escapeLineStart, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
|
|
20
|
+
import { extractPdf, PdfLimitError, PDF_LIMITS } from '../pdf/extract.js';
|
|
21
21
|
import { layoutPdf } from '../pdf/layout.js';
|
|
22
22
|
|
|
23
23
|
export const name = 'pdf';
|
|
@@ -25,6 +25,8 @@ export const DEFAULT_TYPE = 'Document';
|
|
|
25
25
|
export const DESCRIPTOR = 'pdfs.json';
|
|
26
26
|
|
|
27
27
|
const RFC3778 = 'http://tools.ietf.org/rfc/rfc3778';
|
|
28
|
+
/** The largest PDF read, in bytes, unless --max-pdf-mb raises it. */
|
|
29
|
+
export const MAX_PDF_BYTES = 256 * 1024 * 1024;
|
|
28
30
|
const SOURCE_MODES = ['copy', 'link'];
|
|
29
31
|
|
|
30
32
|
/** @param {string} inputPath */
|
|
@@ -51,12 +53,23 @@ export async function convert({ inputPath, options }) {
|
|
|
51
53
|
if (missing.length > 0) throw new UsageError(`--sources link needs a url for every document in ${DESCRIPTOR}; ${missing[0].file} has none.`);
|
|
52
54
|
}
|
|
53
55
|
|
|
56
|
+
const limits = {
|
|
57
|
+
maxBytes: options.maxPdfBytes ?? MAX_PDF_BYTES,
|
|
58
|
+
maxPages: options.maxPdfPages ?? PDF_LIMITS.maxPages,
|
|
59
|
+
timeoutMs: options.pdfTimeoutMs ?? PDF_LIMITS.timeoutMs,
|
|
60
|
+
};
|
|
54
61
|
const warnings = [];
|
|
55
62
|
const docs = [];
|
|
56
63
|
const docSlugs = new Set(['index']);
|
|
57
64
|
for (const entry of listing.documents) {
|
|
58
|
-
const bytes = readPdf(entry);
|
|
59
|
-
|
|
65
|
+
const bytes = readPdf(entry, inputPath, limits.maxBytes);
|
|
66
|
+
let pdf;
|
|
67
|
+
try {
|
|
68
|
+
pdf = await extractPdf(new Uint8Array(bytes), limits);
|
|
69
|
+
} catch (err) {
|
|
70
|
+
if (err instanceof PdfLimitError) throw new UsageError(`${entry.file}: ${err.message}. Raise the limit with --max-pdf-pages or --pdf-timeout.`);
|
|
71
|
+
throw err;
|
|
72
|
+
}
|
|
60
73
|
const { blocks, labels } = layoutPdf(pdf);
|
|
61
74
|
const title = entry.title ?? pdf.title ?? firstHeading(blocks) ?? basename(entry.file, '.pdf');
|
|
62
75
|
if (!blocks.some((b) => b.kind !== 'heading')) {
|
|
@@ -140,18 +153,44 @@ export function listDocuments(inputPath) {
|
|
|
140
153
|
if (!Array.isArray(spec?.documents) || spec.documents.length === 0) throw new UsageError(`${descriptor} needs a non-empty "documents" list.`);
|
|
141
154
|
const documents = spec.documents.map((d, i) => {
|
|
142
155
|
if (typeof d?.file !== 'string' || d.file.split(/[\\/]/).includes('..')) throw new UsageError(`${descriptor}: documents[${i}] needs a "file" inside the folder.`);
|
|
143
|
-
|
|
156
|
+
let url;
|
|
157
|
+
if (d.url !== undefined) {
|
|
158
|
+
url = httpUrl(d.url);
|
|
159
|
+
if (!url) throw new UsageError(`${descriptor}: documents[${i}].url must be an http(s) URL, with no user name or password.`);
|
|
160
|
+
if (new URL(url).hash) throw new UsageError(`${descriptor}: documents[${i}].url must not have a #fragment; moca-convert adds #page= itself.`);
|
|
161
|
+
}
|
|
144
162
|
if (d.sha256 !== undefined && !/^[0-9a-f]{64}$/.test(d.sha256)) throw new UsageError(`${descriptor}: documents[${i}].sha256 must be 64 lowercase hex digits.`);
|
|
145
|
-
return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['
|
|
163
|
+
return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['sha256', 'title']), ...(url ? { url } : {}) };
|
|
146
164
|
});
|
|
147
165
|
return { ...pick(spec, ['title', 'description']), documents };
|
|
148
166
|
}
|
|
149
167
|
|
|
150
|
-
function readPdf(entry) {
|
|
151
|
-
|
|
152
|
-
|
|
168
|
+
function readPdf(entry, inputPath, maxBytes) {
|
|
169
|
+
// A listed file is a regular file inside the input folder: not a symbolic link, and not reached through one. It is
|
|
170
|
+
// opened once, without following a link, and checked and read through that one handle.
|
|
171
|
+
let fd;
|
|
172
|
+
try {
|
|
173
|
+
fd = openSync(entry.abs, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
174
|
+
} catch (err) {
|
|
175
|
+
if (err.code === 'ENOENT') throw new UsageError(`${entry.file} is missing${entry.url ? `; download it from ${entry.url}` : ''}.`);
|
|
176
|
+
if (err.code === 'ELOOP' || err.code === 'EISDIR') throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
|
|
177
|
+
throw err;
|
|
178
|
+
}
|
|
179
|
+
let bytes;
|
|
180
|
+
try {
|
|
181
|
+
const st = fstatSync(fd);
|
|
182
|
+
if (!st.isFile()) throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
|
|
183
|
+
if (statSync(inputPath).isDirectory()) {
|
|
184
|
+
const root = realpathSync(inputPath);
|
|
185
|
+
if (!realpathSync(entry.abs).startsWith(root + sep)) throw new UsageError(`${entry.file} is outside ${inputPath}.`);
|
|
186
|
+
}
|
|
187
|
+
if (st.size > maxBytes) {
|
|
188
|
+
throw new UsageError(`${entry.file} is ${Math.ceil(st.size / 1024 / 1024)} MB, over the ${Math.round(maxBytes / 1024 / 1024)} MB limit. Raise it with --max-pdf-mb.`);
|
|
189
|
+
}
|
|
190
|
+
bytes = readFileSync(fd);
|
|
191
|
+
} finally {
|
|
192
|
+
closeSync(fd);
|
|
153
193
|
}
|
|
154
|
-
const bytes = readFileSync(entry.abs);
|
|
155
194
|
if (entry.sha256) {
|
|
156
195
|
const actual = createHash('sha256').update(bytes).digest('hex');
|
|
157
196
|
if (actual !== entry.sha256) throw new UsageError(`${entry.file} has SHA-256 ${actual}, but ${DESCRIPTOR} expects ${entry.sha256}.`);
|
|
@@ -222,7 +261,7 @@ function pageRange(pages, labels) {
|
|
|
222
261
|
const first = pages[0];
|
|
223
262
|
const last = pages.at(-1);
|
|
224
263
|
const printed = labels.has(first) && labels.has(last);
|
|
225
|
-
const show = (n) => (printed ? labels.get(n) : String(n));
|
|
264
|
+
const show = (n) => (printed ? escapeInline(labels.get(n)) : String(n));
|
|
226
265
|
const noun = `${printed ? '' : 'PDF '}page${first === last ? '' : 's'}`;
|
|
227
266
|
return first === last ? `${noun} ${show(first)}` : `${noun} ${show(first)} to ${show(last)}`;
|
|
228
267
|
}
|
package/lib/adapters/youtube.js
CHANGED
|
@@ -9,8 +9,8 @@ import { joinFrontmatter } from '@openmoca/moca-core';
|
|
|
9
9
|
import { buildManifest } from '../manifest.js';
|
|
10
10
|
import { slugify } from '../slug.js';
|
|
11
11
|
import { UsageError } from '../target.js';
|
|
12
|
-
import { escapeInline, escapeText, firstLine, summary, turtleString, uniqueSlug } from '../text.js';
|
|
13
|
-
import { CAPTURE_FILE } from '../youtube/capture.js';
|
|
12
|
+
import { compactDate, escapeInline, escapeText, firstLine, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
|
|
13
|
+
import { CAPTURE_FILE, VIDEO_ID, watchUrl } from '../youtube/capture.js';
|
|
14
14
|
import { parseVtt } from '../youtube/json3-to-vtt.js';
|
|
15
15
|
|
|
16
16
|
export const name = 'youtube';
|
|
@@ -18,6 +18,8 @@ export const DEFAULT_TYPE = 'Video';
|
|
|
18
18
|
|
|
19
19
|
const MEDIA_FRAGS = 'http://www.w3.org/TR/media-frags/';
|
|
20
20
|
const PARAGRAPH_MS = 60_000;
|
|
21
|
+
// A caption language as a BCP 47 tag; it is part of a file name.
|
|
22
|
+
const LANGUAGE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/;
|
|
21
23
|
|
|
22
24
|
/** @param {string} inputPath */
|
|
23
25
|
export function detect(inputPath) {
|
|
@@ -30,7 +32,7 @@ export function detect(inputPath) {
|
|
|
30
32
|
*/
|
|
31
33
|
export function convert({ inputPath, options }) {
|
|
32
34
|
if (!detect(inputPath)) throw new UsageError(`${inputPath} is not a YouTube capture folder (no ${CAPTURE_FILE}); create one with moca-youtube.`);
|
|
33
|
-
const capture =
|
|
35
|
+
const capture = readCapture(inputPath);
|
|
34
36
|
if (capture.videos.length === 0) throw new UsageError(`${inputPath} holds no videos.`);
|
|
35
37
|
const playlist = capture.kind === 'playlist';
|
|
36
38
|
const manifest = buildManifest({
|
|
@@ -48,7 +50,7 @@ export function convert({ inputPath, options }) {
|
|
|
48
50
|
const taken = new Set(playlist ? ['index'] : []);
|
|
49
51
|
const videos = capture.videos.map((id) => {
|
|
50
52
|
const dir = join(inputPath, 'videos', id);
|
|
51
|
-
const info =
|
|
53
|
+
const info = readInfo(id, join(dir, 'info.json'));
|
|
52
54
|
const vttPath = join(dir, 'captions.vtt');
|
|
53
55
|
const vtt = existsSync(vttPath) ? readFileSync(vttPath, 'utf8').replace(/\r\n?/g, '\n') : undefined;
|
|
54
56
|
return { info, vtt, slug: uniqueSlug(slugify(info.title), taken) };
|
|
@@ -82,6 +84,12 @@ export function convert({ inputPath, options }) {
|
|
|
82
84
|
warnings.push({ code: 'YT_NO_CAPTIONS', message: `${info.title} has no captions; its node has no transcript.`, file: `videos/${info.id}` });
|
|
83
85
|
}
|
|
84
86
|
|
|
87
|
+
// Package spec §5.7: the video plays through its YouTube page, so it is described with the page's type. The
|
|
88
|
+
// duration bounds the chapter time ranges, which can end after the last caption.
|
|
89
|
+
const media = typeof info.duration === 'number' && info.duration >= 0
|
|
90
|
+
? [{ source: 'video', type: 'text/html', duration: info.duration, ...(vtt ? { transcript: 'captions' } : {}) }]
|
|
91
|
+
: [];
|
|
92
|
+
|
|
85
93
|
const data = {
|
|
86
94
|
type,
|
|
87
95
|
title: info.title,
|
|
@@ -91,6 +99,7 @@ export function convert({ inputPath, options }) {
|
|
|
91
99
|
moca: {
|
|
92
100
|
...(playlist ? { concepts: [{ iri: iri(`video-${info.id}`), role: 'primary' }] } : {}),
|
|
93
101
|
...(evidence.length > 0 ? { evidence } : {}),
|
|
102
|
+
...(media.length > 0 ? { media } : {}),
|
|
94
103
|
},
|
|
95
104
|
};
|
|
96
105
|
if (Object.keys(data.moca).length === 0) delete data.moca;
|
|
@@ -105,9 +114,9 @@ export function convert({ inputPath, options }) {
|
|
|
105
114
|
}
|
|
106
115
|
|
|
107
116
|
function videoBody(info, description, cues) {
|
|
108
|
-
const facts = [`[Watch on YouTube](
|
|
109
|
-
if (info.channel) facts.push(info.channel_url ? `[${escapeInline(info.channel)}](
|
|
110
|
-
if (info.upload_date) facts.push(`published ${info.upload_date
|
|
117
|
+
const facts = [`[Watch on YouTube](<${info.webpage_url}>)`];
|
|
118
|
+
if (info.channel) facts.push(info.channel_url ? `[${escapeInline(info.channel)}](<${info.channel_url}>)` : escapeInline(info.channel));
|
|
119
|
+
if (info.upload_date) facts.push(`published ${info.upload_date}`);
|
|
111
120
|
if (info.duration) facts.push(clock(info.duration));
|
|
112
121
|
const out = [`# ${escapeInline(info.title)}`, facts.join(' · ')];
|
|
113
122
|
if (description) out.push('## Description', escapeText(description));
|
|
@@ -116,7 +125,7 @@ function videoBody(info, description, cues) {
|
|
|
116
125
|
for (const section of sections(info, cues)) {
|
|
117
126
|
if (section.title) out.push(`### ${clock(section.start)} ${escapeInline(section.title)}`);
|
|
118
127
|
for (const para of paragraphs(section.cues)) {
|
|
119
|
-
out.push(`[${clock(para.start / 1000)}](https://youtu.be/${info.id}?t=${Math.floor(para.start / 1000)}) ${escapeInline(para.text)}`);
|
|
128
|
+
out.push(`[${clock(para.start / 1000)}](<https://youtu.be/${info.id}?t=${Math.floor(para.start / 1000)}>) ${escapeInline(para.text)}`);
|
|
120
129
|
}
|
|
121
130
|
}
|
|
122
131
|
}
|
|
@@ -125,13 +134,15 @@ function videoBody(info, description, cues) {
|
|
|
125
134
|
|
|
126
135
|
/**
|
|
127
136
|
* The video's chapters with the cues each holds (by cue start), or the whole
|
|
128
|
-
* video as one untitled section. Times are whole seconds
|
|
137
|
+
* video as one untitled section. Times are whole seconds, except an end that
|
|
138
|
+
* would otherwise fall after the end of the video.
|
|
129
139
|
*/
|
|
130
140
|
function sections(info, cues) {
|
|
131
|
-
|
|
132
|
-
|
|
141
|
+
// The end of the video: its duration, or the end of the last caption when later. A time range never ends after it,
|
|
142
|
+
// so it is kept exact rather than rounded up (Reader contract §7).
|
|
143
|
+
const end = Math.max(info.duration ?? 0, (cues.at(-1)?.end ?? 0) / 1000);
|
|
133
144
|
const chapters = info.chapters?.length > 0
|
|
134
|
-
? info.chapters.map((c, i) => ({ start: Math.floor(c.start), end: Math.ceil(c.end ?? end), title: chapterTitle(c.title, i) }))
|
|
145
|
+
? info.chapters.map((c, i) => ({ start: Math.floor(c.start), end: Math.min(Math.ceil(c.end ?? end), end), title: chapterTitle(c.title, i) }))
|
|
135
146
|
: [{ start: 0, end, title: null }];
|
|
136
147
|
return chapters
|
|
137
148
|
.map((c, i) => ({
|
|
@@ -162,7 +173,7 @@ function playlistIndex(capture, videos) {
|
|
|
162
173
|
const lines = [`# ${escapeInline(capture.title)}`, ''];
|
|
163
174
|
const description = (capture.description ?? '').trim();
|
|
164
175
|
if (description) lines.push(escapeText(description), '');
|
|
165
|
-
lines.push(`Videos in [playlist](
|
|
176
|
+
lines.push(capture.url ? `Videos in [playlist](<${capture.url}>) order:` : 'Videos in playlist order:', '');
|
|
166
177
|
videos.forEach(({ info, slug }, i) => lines.push(`${i + 1}. [${escapeInline(info.title)}](${slug}.md)`));
|
|
167
178
|
return `${lines.join('\n')}\n`;
|
|
168
179
|
}
|
|
@@ -190,5 +201,59 @@ function clock(seconds) {
|
|
|
190
201
|
}
|
|
191
202
|
|
|
192
203
|
function readJson(path) {
|
|
193
|
-
|
|
204
|
+
try {
|
|
205
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
206
|
+
} catch (err) {
|
|
207
|
+
throw new UsageError(`${path} is not valid JSON: ${err.message}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const text = (value) => (typeof value === 'string' ? value : undefined);
|
|
212
|
+
const seconds = (value) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined);
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The capture file, checked as it is read: video ids are plain ids (they name
|
|
216
|
+
* folders and go into links), and the playlist URL is parsed and rebuilt.
|
|
217
|
+
*/
|
|
218
|
+
function readCapture(inputPath) {
|
|
219
|
+
const raw = readJson(join(inputPath, CAPTURE_FILE));
|
|
220
|
+
if (!Array.isArray(raw?.videos)) throw new UsageError(`${CAPTURE_FILE} needs a "videos" list.`);
|
|
221
|
+
for (const id of raw.videos) {
|
|
222
|
+
if (typeof id !== 'string' || !VIDEO_ID.test(id)) throw new UsageError(`${CAPTURE_FILE}: ${JSON.stringify(id)} is not a video id.`);
|
|
223
|
+
}
|
|
224
|
+
return {
|
|
225
|
+
kind: raw.kind === 'playlist' ? 'playlist' : 'video',
|
|
226
|
+
title: text(raw.title),
|
|
227
|
+
description: text(raw.description) ?? '',
|
|
228
|
+
url: httpUrl(raw.url),
|
|
229
|
+
videos: raw.videos,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* A video's info.json, checked as it is read. It must be the video the
|
|
235
|
+
* capture lists. URLs are parsed and rebuilt (the watch page falls back to the
|
|
236
|
+
* one the id gives), the date must be a real YYYYMMDD date, times are numbers,
|
|
237
|
+
* and the caption language is a language tag. Anything else is left out.
|
|
238
|
+
*/
|
|
239
|
+
function readInfo(id, path) {
|
|
240
|
+
const raw = readJson(path);
|
|
241
|
+
if (raw?.id !== id) throw new UsageError(`${path} is not the info for video ${id}.`);
|
|
242
|
+
const captions = raw.captions && typeof raw.captions === 'object'
|
|
243
|
+
? { language: LANGUAGE.test(text(raw.captions.language) ?? '') ? raw.captions.language : 'und', automatic: raw.captions.automatic === true }
|
|
244
|
+
: null;
|
|
245
|
+
return {
|
|
246
|
+
id,
|
|
247
|
+
title: text(raw.title)?.trim() || id,
|
|
248
|
+
description: text(raw.description) ?? '',
|
|
249
|
+
channel: text(raw.channel)?.trim() || null,
|
|
250
|
+
channel_url: httpUrl(raw.channel_url) ?? null,
|
|
251
|
+
upload_date: compactDate(raw.upload_date),
|
|
252
|
+
duration: seconds(raw.duration) ?? null,
|
|
253
|
+
webpage_url: httpUrl(raw.webpage_url) ?? watchUrl(id),
|
|
254
|
+
chapters: (Array.isArray(raw.chapters) ? raw.chapters : [])
|
|
255
|
+
.filter((c) => seconds(c?.start) !== undefined)
|
|
256
|
+
.map((c) => ({ start: c.start, ...(seconds(c.end) !== undefined ? { end: c.end } : {}), title: text(c.title) })),
|
|
257
|
+
captions,
|
|
258
|
+
};
|
|
194
259
|
}
|
package/lib/manifest.js
CHANGED
|
@@ -12,7 +12,7 @@ export function buildManifest(o) {
|
|
|
12
12
|
if (!o.title) throw new UsageError('--title is required.');
|
|
13
13
|
return {
|
|
14
14
|
$schema: 'https://w3id.org/moca/schemas/v1/manifest.schema.json',
|
|
15
|
-
mocaVersion: '0.
|
|
15
|
+
mocaVersion: '0.7',
|
|
16
16
|
id: o.id,
|
|
17
17
|
version: o.version ?? '1.0.0',
|
|
18
18
|
title: o.title,
|
package/lib/pdf/extract.js
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
// are made here; lib/pdf/layout.js turns the lines into headings and
|
|
4
4
|
// paragraphs.
|
|
5
5
|
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
6
|
+
import { oneLine } from '../text.js';
|
|
7
|
+
|
|
8
|
+
/** Limits on one PDF: pages, and time to read it. */
|
|
9
|
+
export const PDF_LIMITS = Object.freeze({ maxPages: 2_000, timeoutMs: 10 * 60_000 });
|
|
10
|
+
|
|
11
|
+
export class PdfLimitError extends Error {}
|
|
6
12
|
|
|
7
13
|
/**
|
|
8
14
|
* @typedef {object} PdfLine
|
|
@@ -32,9 +38,28 @@ import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
|
32
38
|
|
|
33
39
|
/**
|
|
34
40
|
* @param {Uint8Array} data
|
|
35
|
-
* @
|
|
41
|
+
* @param {{ maxPages?: number, timeoutMs?: number }} [limits] PDF_LIMITS by default
|
|
42
|
+
* @returns {Promise<PdfText>} rejects with PdfLimitError over a limit
|
|
36
43
|
*/
|
|
37
|
-
export async function extractPdf(data) {
|
|
44
|
+
export async function extractPdf(data, limits = {}) {
|
|
45
|
+
const { maxPages, timeoutMs } = { ...PDF_LIMITS, ...limits };
|
|
46
|
+
let timer;
|
|
47
|
+
let task;
|
|
48
|
+
const timeout = new Promise((_, reject) => {
|
|
49
|
+
timer = setTimeout(() => {
|
|
50
|
+
reject(new PdfLimitError(`reading it took longer than ${Math.round(timeoutMs / 1000)} s`));
|
|
51
|
+
task?.destroy();
|
|
52
|
+
}, timeoutMs);
|
|
53
|
+
});
|
|
54
|
+
try {
|
|
55
|
+
const deadline = Date.now() + timeoutMs;
|
|
56
|
+
return await Promise.race([read(data, { maxPages, deadline, timeoutMs }, (t) => { task = t; }), timeout]);
|
|
57
|
+
} finally {
|
|
58
|
+
clearTimeout(timer);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function read(data, { maxPages, deadline, timeoutMs }, onTask) {
|
|
38
63
|
const task = getDocument({
|
|
39
64
|
data,
|
|
40
65
|
isEvalSupported: false,
|
|
@@ -42,16 +67,21 @@ export async function extractPdf(data) {
|
|
|
42
67
|
useSystemFonts: false,
|
|
43
68
|
verbosity: 0,
|
|
44
69
|
});
|
|
70
|
+
onTask(task);
|
|
45
71
|
const doc = await task.promise;
|
|
46
72
|
try {
|
|
73
|
+
if (doc.numPages > maxPages) throw new PdfLimitError(`it has ${doc.numPages} pages, over the ${maxPages}-page limit`);
|
|
47
74
|
const labels = await doc.getPageLabels().catch(() => null);
|
|
48
75
|
const pages = [];
|
|
49
76
|
for (let n = 1; n <= doc.numPages; n++) {
|
|
77
|
+
if (Date.now() >= deadline) throw new PdfLimitError(`reading it took longer than ${Math.round(timeoutMs / 1000)} s`);
|
|
50
78
|
const page = await doc.getPage(n);
|
|
51
79
|
const { items } = await page.getTextContent();
|
|
52
80
|
const [, , , height] = page.view;
|
|
53
81
|
const bold = await boldFonts(page, items);
|
|
54
|
-
|
|
82
|
+
// A page label is the PDF's own text: kept short, on one line, with no control characters.
|
|
83
|
+
const label = oneLine(labels?.[n - 1]);
|
|
84
|
+
pages.push({ number: n, ...(label ? { label } : {}), height, lines: toLines(items, bold) });
|
|
55
85
|
page.cleanup();
|
|
56
86
|
}
|
|
57
87
|
const meta = await doc.getMetadata().catch(() => null);
|
package/lib/text.js
CHANGED
|
@@ -57,3 +57,58 @@ export function uniqueSlug(base, taken) {
|
|
|
57
57
|
export function turtleString(text) {
|
|
58
58
|
return `"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`;
|
|
59
59
|
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* An http or https URL from input, parsed and rebuilt, or undefined. The
|
|
63
|
+
* rebuilt form percent-encodes spaces, quotes and angle brackets, so it is
|
|
64
|
+
* safe inside a Markdown link destination (`<...>`) and in YAML. URLs with
|
|
65
|
+
* other schemes, or with a user name or password, are refused.
|
|
66
|
+
* @param {unknown} value
|
|
67
|
+
* @returns {string|undefined}
|
|
68
|
+
*/
|
|
69
|
+
export function httpUrl(value) {
|
|
70
|
+
if (typeof value !== 'string' || value.length > 2048 || /[\u0000-\u001f\u007f\s]/.test(value)) return undefined;
|
|
71
|
+
let url;
|
|
72
|
+
try {
|
|
73
|
+
url = new URL(value);
|
|
74
|
+
} catch {
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
if ((url.protocol !== 'http:' && url.protocol !== 'https:') || url.username || url.password || !url.hostname) return undefined;
|
|
78
|
+
return url.href;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* A path as an IRI fragment or path: each `/`-separated segment
|
|
83
|
+
* percent-encoded, so spaces, quotes and angle brackets cannot end a Turtle
|
|
84
|
+
* IRI (`<...>`) early.
|
|
85
|
+
* @param {string} path
|
|
86
|
+
*/
|
|
87
|
+
export function iriPath(path) {
|
|
88
|
+
return path.split('/').map((segment) => encodeURIComponent(segment)).join('/');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Short text from input shown on one line, such as a PDF page label: control
|
|
93
|
+
* and format characters removed, white space collapsed, at most `max`
|
|
94
|
+
* characters. Undefined when nothing is left.
|
|
95
|
+
* @param {unknown} value
|
|
96
|
+
* @param {number} [max]
|
|
97
|
+
*/
|
|
98
|
+
export function oneLine(value, max = 32) {
|
|
99
|
+
if (typeof value !== 'string') return undefined;
|
|
100
|
+
const text = value.replace(/[\p{Cc}\p{Cf}]/gu, ' ').replace(/\s+/g, ' ').trim().slice(0, max).trim();
|
|
101
|
+
return text || undefined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* A YYYYMMDD date that exists, as YYYY-MM-DD, or undefined.
|
|
106
|
+
* @param {unknown} value
|
|
107
|
+
*/
|
|
108
|
+
export function compactDate(value) {
|
|
109
|
+
const m = typeof value === 'string' ? /^(\d{4})(\d{2})(\d{2})$/.exec(value) : null;
|
|
110
|
+
if (!m) return undefined;
|
|
111
|
+
const iso = `${m[1]}-${m[2]}-${m[3]}`;
|
|
112
|
+
const d = new Date(`${iso}T00:00:00Z`);
|
|
113
|
+
return !Number.isNaN(d.getTime()) && d.toISOString().startsWith(iso) ? iso : undefined;
|
|
114
|
+
}
|
package/lib/walk.js
CHANGED
|
@@ -4,7 +4,9 @@ import { join, relative } from 'node:path';
|
|
|
4
4
|
/**
|
|
5
5
|
* Recursively lists files under `dir`, returning paths relative to `rootDir`
|
|
6
6
|
* (using forward slashes regardless of platform), sorted lexicographically
|
|
7
|
-
* for deterministic iteration order.
|
|
7
|
+
* for deterministic iteration order. Hidden entries (a name starting with
|
|
8
|
+
* ".": .git/, .obsidian/, .trash/ and the like) are skipped, as a Reader
|
|
9
|
+
* ignores them, and so are symbolic links.
|
|
8
10
|
*
|
|
9
11
|
* @param {string} dir
|
|
10
12
|
* @param {string} rootDir
|
|
@@ -15,6 +17,7 @@ export function walkFiles(dir, rootDir = dir) {
|
|
|
15
17
|
const entries = readdirSync(dir, { withFileTypes: true });
|
|
16
18
|
const results = [];
|
|
17
19
|
for (const entry of entries) {
|
|
20
|
+
if (entry.name.startsWith('.')) continue;
|
|
18
21
|
const full = join(dir, entry.name);
|
|
19
22
|
if (entry.isDirectory()) {
|
|
20
23
|
results.push(...walkFiles(full, rootDir));
|
package/lib/write.js
CHANGED
|
@@ -1,15 +1,27 @@
|
|
|
1
1
|
// Writes a PackageDraft (see lib/adapters/index.js) to disk and gates
|
|
2
2
|
// success on the reference Reader reporting zero error-severity diagnostics:
|
|
3
3
|
// read first, refuse to leave invalid output behind.
|
|
4
|
-
import { existsSync, mkdirSync,
|
|
5
|
-
import {
|
|
6
|
-
import { dirname, join } from 'node:path';
|
|
4
|
+
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { randomBytes } from 'node:crypto';
|
|
6
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
7
7
|
import { readPackage } from '@openmoca/moca-core';
|
|
8
8
|
import { UsageError } from './target.js';
|
|
9
9
|
|
|
10
10
|
export { UsageError };
|
|
11
11
|
|
|
12
|
+
/** A hidden name beside `outDir`, on the same file system, so a rename never crosses devices. */
|
|
13
|
+
const besideName = (outDir, tag) => join(dirname(resolve(outDir)), `.${basename(resolve(outDir))}.${randomBytes(6).toString('hex')}.${tag}`);
|
|
14
|
+
|
|
12
15
|
/**
|
|
16
|
+
* Writes the draft to a folder beside `outDir`, checks it with the reference
|
|
17
|
+
* Reader, and only then moves it into place. Anything already at `outDir` is
|
|
18
|
+
* moved aside first and deleted once the new package is in; if the move fails,
|
|
19
|
+
* it is put back. On any failure the staged folder is removed and `outDir` is
|
|
20
|
+
* as it was.
|
|
21
|
+
*
|
|
22
|
+
* `force` replaces a non-empty `outDir` only when it is a previous package (it
|
|
23
|
+
* has a moca.json); anything else is refused.
|
|
24
|
+
*
|
|
13
25
|
* @param {object} params
|
|
14
26
|
* @param {import('./adapters/index.js').PackageDraft} params.draft
|
|
15
27
|
* @param {string} params.outDir
|
|
@@ -21,15 +33,19 @@ export { UsageError };
|
|
|
21
33
|
export async function writeDraft({ draft, outDir, force = false, strict = false, onLog = () => {} }) {
|
|
22
34
|
checkFiles(draft);
|
|
23
35
|
const outDirExists = existsSync(outDir);
|
|
36
|
+
if (outDirExists && !statSync(outDir).isDirectory()) {
|
|
37
|
+
throw new UsageError(`Output "${outDir}" exists and is not a directory.`);
|
|
38
|
+
}
|
|
24
39
|
const outDirNonEmpty = outDirExists && readdirSync(outDir).length > 0;
|
|
25
|
-
|
|
26
40
|
if (outDirNonEmpty && !force) {
|
|
27
|
-
throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to
|
|
41
|
+
throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to replace it.`);
|
|
42
|
+
}
|
|
43
|
+
if (outDirNonEmpty && !existsSync(join(outDir, 'moca.json'))) {
|
|
44
|
+
throw new UsageError(`Output directory "${outDir}" is not a previous package (it has no moca.json), so --force will not replace it.`);
|
|
28
45
|
}
|
|
29
46
|
|
|
30
|
-
|
|
31
|
-
const stageDir =
|
|
32
|
-
|
|
47
|
+
mkdirSync(dirname(resolve(outDir)), { recursive: true });
|
|
48
|
+
const stageDir = besideName(outDir, 'partial');
|
|
33
49
|
try {
|
|
34
50
|
stageDraft(draft, stageDir);
|
|
35
51
|
|
|
@@ -45,26 +61,35 @@ export async function writeDraft({ draft, outDir, force = false, strict = false,
|
|
|
45
61
|
);
|
|
46
62
|
}
|
|
47
63
|
|
|
48
|
-
|
|
49
|
-
rmSync(outDir, { recursive: true, force: true });
|
|
50
|
-
renameSync(stageDir, outDir);
|
|
51
|
-
}
|
|
52
|
-
|
|
64
|
+
replaceWith(stageDir, outDir, onLog);
|
|
53
65
|
return { findings, digest: result.digest };
|
|
66
|
+
} finally {
|
|
67
|
+
// Never leave partial or invalid output behind. After a successful move
|
|
68
|
+
// the staged folder no longer exists.
|
|
69
|
+
rmSync(stageDir, { recursive: true, force: true });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Moves the staged folder to `outDir`: the old one aside first, removed last. */
|
|
74
|
+
function replaceWith(stageDir, outDir, onLog) {
|
|
75
|
+
if (!existsSync(outDir)) {
|
|
76
|
+
renameSync(stageDir, outDir);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const aside = besideName(outDir, 'old');
|
|
80
|
+
onLog(`moving the previous ${outDir} aside`);
|
|
81
|
+
renameSync(outDir, aside);
|
|
82
|
+
try {
|
|
83
|
+
renameSync(stageDir, outDir);
|
|
54
84
|
} catch (err) {
|
|
55
|
-
|
|
56
|
-
// staged directory. In scratch mode that's a temp dir, so the
|
|
57
|
-
// pre-existing outDir is untouched; in direct mode stageDir === outDir,
|
|
58
|
-
// so a freshly-created output directory is fully cleaned up.
|
|
59
|
-
if (existsSync(stageDir)) {
|
|
60
|
-
rmSync(stageDir, { recursive: true, force: true });
|
|
61
|
-
}
|
|
85
|
+
renameSync(aside, outDir);
|
|
62
86
|
throw err;
|
|
63
87
|
}
|
|
88
|
+
rmSync(aside, { recursive: true, force: true });
|
|
64
89
|
}
|
|
65
90
|
|
|
66
91
|
function stageDraft(draft, stageDir) {
|
|
67
|
-
mkdirSync(stageDir
|
|
92
|
+
mkdirSync(stageDir);
|
|
68
93
|
writeFileSync(join(stageDir, 'moca.json'), `${JSON.stringify(draft.manifest, null, 2)}\n`);
|
|
69
94
|
|
|
70
95
|
for (const node of [...draft.contentNodes, ...(draft.files ?? [])]) {
|
package/lib/youtube/capture.js
CHANGED
|
@@ -14,10 +14,33 @@ import { UsageError } from '../target.js';
|
|
|
14
14
|
import { json3ToVtt } from './json3-to-vtt.js';
|
|
15
15
|
|
|
16
16
|
export const CAPTURE_FILE = 'youtube-capture.json';
|
|
17
|
+
/** A video or playlist id: letters, digits, `-` and `_`. It names a folder and goes into links. */
|
|
18
|
+
export const VIDEO_ID = /^[A-Za-z0-9_-]{1,64}$/;
|
|
19
|
+
|
|
20
|
+
/** The hosts a capture may read from: YouTube's own. */
|
|
21
|
+
const YOUTUBE_HOSTS = new Set(['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be']);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Checks that `url` is an http(s) URL on a YouTube host, and returns it
|
|
25
|
+
* rebuilt. Throws UsageError otherwise.
|
|
26
|
+
* @param {string} url
|
|
27
|
+
*/
|
|
28
|
+
export function youtubeUrl(url) {
|
|
29
|
+
let parsed;
|
|
30
|
+
try {
|
|
31
|
+
parsed = new URL(url);
|
|
32
|
+
} catch {
|
|
33
|
+
throw new UsageError(`"${url}" is not a URL.`);
|
|
34
|
+
}
|
|
35
|
+
if ((parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || parsed.username || parsed.password || !YOUTUBE_HOSTS.has(parsed.hostname.toLowerCase())) {
|
|
36
|
+
throw new UsageError(`"${url}" is not a YouTube URL (${[...YOUTUBE_HOSTS].join(', ')}).`);
|
|
37
|
+
}
|
|
38
|
+
return parsed.href;
|
|
39
|
+
}
|
|
17
40
|
|
|
18
41
|
/**
|
|
19
42
|
* @param {object} params
|
|
20
|
-
* @param {string} params.url
|
|
43
|
+
* @param {string} params.url a YouTube video or playlist URL
|
|
21
44
|
* @param {string} params.captureDir
|
|
22
45
|
* @param {string} [params.language] caption language, BCP 47 (default en)
|
|
23
46
|
* @param {boolean} [params.force]
|
|
@@ -29,18 +52,28 @@ export async function capture({ url, captureDir, language = 'en', force = false,
|
|
|
29
52
|
if (existsSync(captureDir) && readdirSync(captureDir).length > 0 && !force) {
|
|
30
53
|
throw new UsageError(`Capture directory "${captureDir}" is not empty; pass --force to overwrite.`);
|
|
31
54
|
}
|
|
55
|
+
const checked = youtubeUrl(url);
|
|
32
56
|
const warnings = [];
|
|
33
|
-
log(`reading ${
|
|
34
|
-
|
|
57
|
+
log(`reading ${checked}`);
|
|
58
|
+
// `--` ends yt-dlp's options, so the URL is only ever read as a URL.
|
|
59
|
+
const top = await ytdlp.json(['--flat-playlist', '--', checked]);
|
|
35
60
|
const isPlaylist = top._type === 'playlist';
|
|
36
|
-
|
|
61
|
+
if (typeof top.id !== 'string' || !VIDEO_ID.test(top.id)) throw new Error(`yt-dlp gave an id that is not a plain id: ${JSON.stringify(top.id)}`);
|
|
62
|
+
const entries = [];
|
|
63
|
+
for (const e of isPlaylist ? (top.entries ?? []) : [top]) {
|
|
64
|
+
if (!e?.id) continue;
|
|
65
|
+
// A video id names a folder, so it must be a plain id.
|
|
66
|
+
if (typeof e.id !== 'string' || !VIDEO_ID.test(e.id)) warnings.push(`Skipped an entry whose id is not a plain id: ${JSON.stringify(e.id)}`);
|
|
67
|
+
else entries.push(e);
|
|
68
|
+
}
|
|
37
69
|
|
|
38
70
|
const videos = [];
|
|
39
71
|
for (const [i, entry] of entries.entries()) {
|
|
40
72
|
const label = `${i + 1}/${entries.length} ${entry.title ?? entry.id}`;
|
|
41
73
|
let info;
|
|
42
74
|
try {
|
|
43
|
-
info = isPlaylist ? await ytdlp.json(['--skip-download', watchUrl(entry.id)]) : entry;
|
|
75
|
+
info = isPlaylist ? await ytdlp.json(['--skip-download', '--', watchUrl(entry.id)]) : entry;
|
|
76
|
+
if (info?.id !== entry.id) throw new Error(`yt-dlp returned another video (${JSON.stringify(info?.id)})`);
|
|
44
77
|
} catch (err) {
|
|
45
78
|
warnings.push(`Skipped ${entry.id}: ${err.message}`);
|
|
46
79
|
log(`skip ${label}`);
|
package/lib/youtube/ytdlp.js
CHANGED
|
@@ -8,17 +8,23 @@ import { join } from 'node:path';
|
|
|
8
8
|
import { UsageError } from '../target.js';
|
|
9
9
|
|
|
10
10
|
const MAX_BUFFER = 512 * 1024 * 1024;
|
|
11
|
+
/** The longest one yt-dlp run may take. */
|
|
12
|
+
export const YTDLP_TIMEOUT_MS = 5 * 60_000;
|
|
11
13
|
|
|
12
14
|
/**
|
|
13
|
-
*
|
|
15
|
+
* yt-dlp runs with --ignore-config, so a user's or system configuration file
|
|
16
|
+
* cannot add options, and each run is stopped after `timeoutMs`.
|
|
17
|
+
* @param {{ bin?: string, timeoutMs?: number }} [options]
|
|
14
18
|
* @returns {{ json: (args: string[]) => Promise<object>, captions: (info: object, track: { language: string, manual: boolean }) => Promise<object> }}
|
|
15
19
|
*/
|
|
16
|
-
export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp' } = {}) {
|
|
20
|
+
export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp', timeoutMs = YTDLP_TIMEOUT_MS } = {}) {
|
|
17
21
|
const run = (args) =>
|
|
18
22
|
new Promise((resolve, reject) => {
|
|
19
|
-
execFile(bin, args, { maxBuffer: MAX_BUFFER, windowsHide: true }, (err, stdout, stderr) => {
|
|
23
|
+
execFile(bin, ['--ignore-config', ...args], { maxBuffer: MAX_BUFFER, windowsHide: true, timeout: timeoutMs, killSignal: 'SIGKILL' }, (err, stdout, stderr) => {
|
|
20
24
|
if (err?.code === 'ENOENT') {
|
|
21
25
|
reject(new UsageError(`Cannot run "${bin}". Install yt-dlp (https://github.com/yt-dlp/yt-dlp#installation) or pass --yt-dlp <path>.`));
|
|
26
|
+
} else if (err?.killed) {
|
|
27
|
+
reject(new Error(`yt-dlp took longer than ${Math.round(timeoutMs / 1000)} s and was stopped.`));
|
|
22
28
|
} else if (err) {
|
|
23
29
|
reject(new Error(`yt-dlp ${args.join(' ')} failed: ${(stderr || err.message).trim()}`));
|
|
24
30
|
} else {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openmoca/moca-convert",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Create MOCA packages (OKF content bundles) from Markdown folders, Obsidian vaults, OpenAPI documents, PDFs and YouTube videos.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"test": "node --test test/*.test.js"
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@openmoca/moca-core": "^0.
|
|
18
|
+
"@openmoca/moca-core": "^0.7.0",
|
|
19
19
|
"commander": "^12.1.0",
|
|
20
20
|
"js-yaml": "^5.2.2",
|
|
21
21
|
"pdfjs-dist": "^6.3.289"
|