@openmoca/moca-convert 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -8,8 +8,8 @@ concept document with a `type`, and the package id must be an absolute URI.
8
8
  moca-convert <input> -o <dir> --id <uri> [--title text] [--version semver] [--type type]
9
9
  [--language bcp47] [--license spdx] [--from directory|markdown|obsidian|openapi|pdf|youtube]
10
10
  [--exclude glob...] [--chunker operation|tag] [--min-description-ratio 0-1]
11
- [--sources copy|link]
12
- [--force] [--strict] [--format text|json]
11
+ [--sources copy|link] [--max-pdf-mb n] [--max-pdf-pages n] [--pdf-timeout seconds]
12
+ [--force] [--strict] [--format text|json] [--log-file file]
13
13
  ```
14
14
 
15
15
  | Adapter | Input | Default `type` |
@@ -25,8 +25,23 @@ moca-convert <input> -o <dir> --id <uri> [--title text] [--version semver] [--ty
25
25
  `type` (and `title`, if missing) added at the top of its frontmatter.
26
26
  - The adapter is detected from the input unless `--from` is given; ambiguous
27
27
  input is refused, not guessed.
28
- - Output is read back with the reference Reader before anything is left on
29
- disk. If it would be invalid, nothing is written.
28
+ - Hidden files and folders (`.git/`, `.obsidian/`, Obsidian's `.trash/` of
29
+ deleted notes) and symbolic links in an input folder are not read.
30
+ - Obsidian wikilinks are rewritten in the note's body only, never in its
31
+ front matter, and each link target is percent-encoded.
32
+ - OpenAPI: each operation's parameters include its path's and those it
33
+ references (`$ref`). Schema references are expanded up to 1,000 objects a
34
+ schema; beyond that they are shown as `{"$ref": ...}`. Two operations whose
35
+ paths give the same file name get distinct files (`get-a-b.md`,
36
+ `get-a-b-2.md`).
37
+ - Output is written to a folder beside `-o`, read back with the reference
38
+ Reader, and only then moved into place. If it would be invalid, nothing is
39
+ written.
40
+ - `--force` replaces an output folder only when it holds a previous package
41
+ (it has a `moca.json`); it refuses any other folder. The previous package is
42
+ moved aside, the new one moved in, and only then is the old one deleted.
43
+ - `--log-file <file>` records the adapter, timings and any failure, on every
44
+ exit.
30
45
 
31
46
  ## PDF
32
47
 
@@ -51,6 +66,12 @@ PDFs with a text layer become one node per section:
51
66
  its `url` instead, for PDFs too large to carry.
52
67
  - A PDF without a text layer (a scan) is reported and left out; run OCR on
53
68
  it first.
69
+ - A PDF over 256 MB or 2,000 pages, or that takes over 10 minutes to read, is
70
+ refused; `--max-pdf-mb`, `--max-pdf-pages` and `--pdf-timeout` raise the
71
+ limits.
72
+ - Text from the PDF itself (headings, paragraphs, page labels) is written as
73
+ text: characters Markdown would read as links or HTML are escaped. Page
74
+ labels are kept short and on one line.
54
75
 
55
76
  A folder may list its documents, in order, in `pdfs.json`:
56
77
 
@@ -64,8 +85,10 @@ A folder may list its documents, in order, in `pdfs.json`:
64
85
  }
65
86
  ```
66
87
 
67
- Only `file` is required. `title` falls back to the PDF's own title, `url` is
68
- the published copy that `--sources link` cites, and a file whose SHA-256 is
88
+ Only `file` is required: a regular file inside the folder, not a symbolic
89
+ link. `title` falls back to the PDF's own title. `url` is the published copy
90
+ that `--sources link` cites: an `http` or `https` URL with no user name,
91
+ password or `#fragment`, parsed and written back in its normal form. A file whose SHA-256 is
69
92
  not `sha256` is refused, so a conversion never silently uses a changed
70
93
  document. Without `pdfs.json`, every PDF in the folder is converted in path
71
94
  order. moca-spec's [`scripts/fetch-pdfs.mjs`](https://github.com/openmoca/moca-spec/blob/main/scripts/fetch-pdfs.mjs) downloads the
@@ -89,7 +112,9 @@ moca-youtube <url> -o <dir> --id <uri> [--title text] [--version semver] [--type
89
112
  captions grouped by chapter. The captions are written under `media/` as
90
113
  WebVTT and the description under `sources/`, and the node cites each
91
114
  chapter as a time range (`t=start,end`) and quotes the description, so the
92
- package is `self-contained-evidence`.
115
+ package is `self-contained-evidence`. The node describes the video in
116
+ `moca.media`: its page's type (`text/html`), its length and its captions, so
117
+ the last chapter can run past the last caption.
93
118
  - A playlist also gets `content/index.md` and a `structure.ttl` that keeps the
94
119
  videos in playlist order.
95
120
  - Captions the creator supplied in `--language` (default `en`) are preferred,
@@ -100,6 +125,15 @@ moca-youtube <url> -o <dir> --id <uri> [--title text] [--version semver] [--type
100
125
  capture goes to a temporary directory.
101
126
  - yt-dlp is run directly, without a shell, so this works on macOS, Linux and
102
127
  Windows. Point at it with `--yt-dlp` or `$YT_DLP` when it is not on `PATH`.
128
+ It runs with `--ignore-config`, so your yt-dlp configuration files are not
129
+ used, and each run is stopped after 5 minutes.
130
+ - The URL must be on `youtube.com` (with `www.`, `m.` or `music.`) or
131
+ `youtu.be`. Video and playlist ids must be plain ids (letters, digits, `-`
132
+ and `_`); an entry with any other id is skipped and reported.
133
+ - When a capture is converted, it is checked as it is read: video ids, URLs
134
+ (parsed and rebuilt; `http` and `https` only), the upload date and the
135
+ caption language. A value that fails is left out, or the capture is
136
+ refused.
103
137
 
104
138
  See [`examples/youtube`](https://github.com/openmoca/moca-spec/blob/main/examples/youtube).
105
139
 
@@ -117,13 +151,37 @@ npm test
117
151
  ## Release
118
152
 
119
153
  1. Publish `@openmoca/moca-core` first if it changed.
120
- 2. Add a dated section to [CHANGELOG.md](CHANGELOG.md) and update the version.
121
- 3. Point the `MOCA_*_REF` pins in `.github/workflows/test.yml` at the release tags
154
+ 2. Name its released range in `package.json` in place of the `file:` path,
155
+ and run `npm install`.
156
+ 3. Add a dated section to [CHANGELOG.md](CHANGELOG.md) and update the version.
157
+ 4. Point the `MOCA_*_REF` pins in `.github/workflows/test.yml` at the release tags
122
158
  ([moca-spec docs/releasing.md](https://github.com/openmoca/moca-spec/blob/main/docs/releasing.md#moving-the-pins)).
123
- 4. `npm test` passes.
124
- 5. Tag `vX.Y.Z` and publish a GitHub release whose notes are the changelog
125
- section.
126
- 6. `npm publish --access public` from the tagged commit.
159
+ 5. `npm test` passes.
160
+ 6. Tag `vX.Y.Z` on main and publish a GitHub release whose notes are the
161
+ changelog section. The release workflow publishes to npm through trusted
162
+ publishing, with provenance; it stops if the tag is not on main, the
163
+ version does not match, or `@openmoca/moca-core` is not on npm.
164
+
165
+ ## Standards
166
+
167
+ What moca-convert reads and writes, and how strongly. **Follow** means it
168
+ implements the standard and the tests check it where they can.
169
+
170
+ | Standard | Where | How strongly |
171
+ | --- | --- | --- |
172
+ | MOCA ([moca-spec](https://github.com/openmoca/moca-spec)) 0.7 | every package it writes, checked with the reference Reader | follow |
173
+ | [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format/blob/ad30107c31c06aec8a7d5636e0d1058118604e6f/SPEC.md) v0.2, pinned at commit `ad30107` | `content/`: each node an OKF concept document with a `type` | follow (pinned) |
174
+ | [CommonMark](https://spec.commonmark.org/) | node bodies; text from input is escaped, link destinations percent-encoded | follow |
175
+ | [YAML 1.2](https://yaml.org/spec/1.2.2/) | front matter, and OpenAPI documents in YAML | follow |
176
+ | [SKOS](https://www.w3.org/TR/skos-reference/) and [Dublin Core terms](https://www.dublincore.org/specifications/dublin-core/dcmi-terms/) in [RDF 1.1 Turtle](https://www.w3.org/TR/turtle/) | `structure.ttl`; IRIs percent-encoded ([RFC 3987](https://www.rfc-editor.org/rfc/rfc3987)) | follow (the named terms) |
177
+ | [RFC 3778](https://www.rfc-editor.org/rfc/rfc3778) `page=` fragments | PDF evidence | follow |
178
+ | [Media Fragments](https://www.w3.org/TR/media-frags/) `t=` | YouTube evidence | follow |
179
+ | [WebVTT](https://www.w3.org/TR/webvtt1/) | YouTube captions under `media/` | follow |
180
+ | [OpenAPI 3.x](https://spec.openapis.org/oas/v3.1.0) | the `openapi` adapter's input; local `$ref` only | follow (the parts it reads) |
181
+ | [WHATWG URL](https://url.spec.whatwg.org/) | URLs from input, parsed and rebuilt; `http` and `https` only | follow |
182
+ | [BCP 47](https://www.rfc-editor.org/info/bcp47) | `--language`, and caption languages | follow |
183
+
184
+ Built with the help of AI assistants; every change is reviewed and accepted by the maintainer.
127
185
 
128
186
  ## Licence
129
187
 
@@ -12,8 +12,18 @@ const ADAPTER_ONLY_FLAGS = {
12
12
  minDescriptionRatio: 'openapi',
13
13
  chunker: 'openapi',
14
14
  sources: 'pdf',
15
+ maxPdfMb: 'pdf',
16
+ maxPdfPages: 'pdf',
17
+ pdfTimeout: 'pdf',
15
18
  };
16
19
 
20
+ function parsePositive(value) {
21
+ if (!/^[0-9]+$/.test(value) || Number(value) < 1 || !Number.isSafeInteger(Number(value))) {
22
+ throw new InvalidArgumentError('must be a positive whole number.');
23
+ }
24
+ return Number(value);
25
+ }
26
+
17
27
  function parseRatio(value) {
18
28
  const num = Number(value);
19
29
  if (Number.isNaN(num) || num < 0 || num > 1) {
@@ -48,8 +58,11 @@ program
48
58
  .option('--min-description-ratio <ratio>', 'openapi only: minimum fraction of operations needing a summary/description, 0-1 (default 0.5)', parseRatio)
49
59
  .option('--chunker <mode>', 'openapi only: operation|tag content-node grouping (default operation)')
50
60
  .option('--sources <mode>', 'pdf only: copy the PDFs into sources/ (copy, the default) or cite each by its pdfs.json url (link)')
61
+ .option('--max-pdf-mb <n>', 'pdf only: the largest PDF to read, in MB (default 256)', parsePositive)
62
+ .option('--max-pdf-pages <n>', 'pdf only: the most pages a PDF may have (default 2000)', parsePositive)
63
+ .option('--pdf-timeout <seconds>', 'pdf only: the longest one PDF may take to read (default 600)', parsePositive)
51
64
  .option('--no-structure', 'directory only: do not write structure.ttl from the folders, or bind nodes to it')
52
- .option('--force', 'allow writing into a non-empty output directory', false)
65
+ .option('--force', 'replace the output directory when it holds a previous package', false)
53
66
  .option('--strict', 'escalate warning-level output-validation findings to errors', false)
54
67
  .option('--format <fmt>', 'summary output format: text|json', 'text')
55
68
  .option('--report <file>', 'write the formatted summary to a file (in addition to console output)')
@@ -59,6 +72,8 @@ program
59
72
  .option('--no-color', 'disable colored output')
60
73
  .action(async (input, options) => {
61
74
  await run(input, options);
75
+ // The log file is written by now; exit even if a library left a handle open.
76
+ process.exit(process.exitCode ?? 0);
62
77
  });
63
78
 
64
79
  // Commander's own usage errors (missing required options, unknown flags,
@@ -104,6 +119,8 @@ async function run(input, options) {
104
119
  const adapterName = resolveAdapterName(target, options.from);
105
120
  assertAdapterSpecificFlags(adapterName, options);
106
121
  const adapter = getAdapter(adapterName);
122
+ logger.log(`adapter ${adapterName}`);
123
+ const started = performance.now();
107
124
 
108
125
  const draft = await adapter.convert({
109
126
  inputPath: input,
@@ -115,6 +132,9 @@ async function run(input, options) {
115
132
  minDescriptionRatio: options.minDescriptionRatio,
116
133
  chunker: options.chunker,
117
134
  sources: options.sources,
135
+ ...(options.maxPdfMb ? { maxPdfBytes: options.maxPdfMb * 1024 * 1024 } : {}),
136
+ ...(options.maxPdfPages ? { maxPdfPages: options.maxPdfPages } : {}),
137
+ ...(options.pdfTimeout ? { pdfTimeoutMs: options.pdfTimeout * 1000 } : {}),
118
138
  structure: options.structure,
119
139
  type: options.type,
120
140
  language: options.language,
@@ -122,6 +142,7 @@ async function run(input, options) {
122
142
  },
123
143
  });
124
144
 
145
+ logger.log(`converted ${draft.contentNodes.length} content node(s) in ${Math.round(performance.now() - started)} ms`);
125
146
  const { findings, digest } = await writeDraft({
126
147
  draft,
127
148
  outDir: options.output,
@@ -130,6 +151,7 @@ async function run(input, options) {
130
151
  onLog: logger.log,
131
152
  });
132
153
 
154
+ logger.log(`wrote ${options.output} in ${Math.round(performance.now() - started)} ms in all`);
133
155
  if (!options.quiet) {
134
156
  console.log(`Wrote ${options.output} (${draft.contentNodes.length} content node(s), ${draft.warnings.length} warning(s))\n${digest}`);
135
157
  }
@@ -139,18 +161,24 @@ async function run(input, options) {
139
161
  if (findings.length > 0) {
140
162
  writeReport(options, formatFindings(findings, options.format));
141
163
  }
142
- process.exit(0);
164
+ process.exitCode = 0;
143
165
  } catch (err) {
166
+ // Set the exit code and return, so the finally block below writes the log
167
+ // file on every path.
144
168
  if (err instanceof UsageError) {
145
169
  console.error(`moca-convert: ${err.message}`);
146
- process.exit(2);
170
+ process.exitCode = 2;
171
+ return;
147
172
  }
148
173
  if (err instanceof ConversionFailedError) {
149
174
  console.error(`moca-convert: ${err.message}`);
150
175
  writeReport(options, formatFindings(err.findings, options.format));
151
- process.exit(1);
176
+ process.exitCode = 1;
177
+ return;
152
178
  }
153
- throw err;
179
+ logger.log(`failed: ${err?.stack ?? err}`);
180
+ console.error(`moca-convert: failed: ${err?.message ?? err}`);
181
+ process.exitCode = 1;
154
182
  } finally {
155
183
  logger.flush();
156
184
  }
@@ -11,7 +11,7 @@ import { walkFiles } from '../walk.js';
11
11
  import { readContentFile, ensureOkf, deriveTitle } from '../frontmatter.js';
12
12
  import { buildManifest } from '../manifest.js';
13
13
  import { UsageError } from '../target.js';
14
- import { turtleString } from '../text.js';
14
+ import { iriPath, turtleString } from '../text.js';
15
15
 
16
16
  export const name = 'directory';
17
17
  export const DEFAULT_TYPE = 'Document';
@@ -65,9 +65,9 @@ export function convert({ inputPath, options }) {
65
65
  return result;
66
66
  }
67
67
 
68
- /** A file's concept fragment: its path without .md. */
68
+ /** A file's concept fragment: its path without .md, each segment percent-encoded. */
69
69
  function fileFragment(relPath) {
70
- return relPath.replace(/\.md$/, '');
70
+ return iriPath(relPath.replace(/\.md$/, ''));
71
71
  }
72
72
 
73
73
  function isFolderPage(relPath) {
@@ -115,7 +115,7 @@ function structure(manifest, files, titles, { iri, lang }) {
115
115
  children.get(parent).add(child);
116
116
  }
117
117
  }
118
- const fragment = (path) => (path === '' ? 'collection' : path.endsWith('.md') ? fileFragment(path) : path);
118
+ const fragment = (path) => (path === '' ? 'collection' : path.endsWith('.md') ? fileFragment(path) : iriPath(path));
119
119
  const order = (path) => [...children.get(path)].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
120
120
  const blocks = ['@prefix skos: <http://www.w3.org/2004/02/skos/core#> .\n@prefix dct: <http://purl.org/dc/terms/> .'];
121
121
  for (const container of [...children.keys()].sort()) {
@@ -3,6 +3,7 @@
3
3
  // rewriting [[wikilink]] syntax to relative Markdown links.
4
4
  import { existsSync, statSync } from 'node:fs';
5
5
  import { basename, join, posix } from 'node:path';
6
+ import { splitFrontmatter } from '@openmoca/moca-core';
6
7
  import { walkFiles } from '../walk.js';
7
8
  import { readContentFile, ensureOkf, deriveTitle } from '../frontmatter.js';
8
9
  import { buildManifest } from '../manifest.js';
@@ -36,9 +37,8 @@ export function convert({ inputPath, options }) {
36
37
  const manifest = buildManifest({ ...options, title: options.title ?? basename(inputPath) });
37
38
  const type = options.type ?? DEFAULT_TYPE;
38
39
  const excludePatterns = options.exclude ?? [];
39
- const allNotePaths = walkFiles(inputPath)
40
- .filter((f) => f.endsWith('.md'))
41
- .filter((f) => f !== '.obsidian' && !f.startsWith('.obsidian/'));
40
+ // walkFiles skips hidden folders: .obsidian/, and .trash/ where Obsidian keeps deleted notes.
41
+ const allNotePaths = walkFiles(inputPath).filter((f) => f.endsWith('.md'));
42
42
 
43
43
  if (allNotePaths.length === 0) {
44
44
  throw new UsageError(`No notes found under ${inputPath}.`);
@@ -61,7 +61,11 @@ export function convert({ inputPath, options }) {
61
61
 
62
62
  function buildContentNode(note, noteIndex, warnings, type) {
63
63
  const title = deriveTitle(note.data, note.content, basename(note.relPath, '.md'));
64
- const body = rewriteWikilinks(ensureOkf(note.raw, note.data, { type, title }), note, noteIndex, warnings);
64
+ // Wikilinks are rewritten in the body only: the front matter is kept as it is.
65
+ const text = ensureOkf(note.raw, note.data, { type, title });
66
+ const split = splitFrontmatter(text);
67
+ const head = split.present && !split.error ? Buffer.from(text, 'utf8').subarray(0, split.bodyOffset).toString('utf8') : '';
68
+ const body = head + rewriteWikilinks(text.slice(head.length), note, noteIndex, warnings);
65
69
  return { path: `content/${note.relPath}`, body };
66
70
  }
67
71
 
@@ -126,15 +130,18 @@ function rewriteMatch({ match, bang, rawTarget, heading, alias, sourceNote, note
126
130
  }
127
131
 
128
132
  /**
129
- * A CommonMark link destination containing an unescaped space is not
130
- * parsed as a link at all -- and vault filenames commonly contain spaces
131
- * (e.g. "Setup Guide.md") -- so wrap the destination in angle brackets
132
- * whenever it needs them, per the CommonMark link-destination grammar.
133
- * @param {string} destination
133
+ * A link destination with each path segment percent-encoded: vault file names
134
+ * often hold spaces ("Setup Guide.md") and may hold any other character, and
135
+ * an encoded destination is always one link. `.` and `..` segments are kept.
136
+ * @param {string} destination a relative path, with an optional #anchor
134
137
  */
135
138
  function formatLinkDestination(destination) {
136
- if (!/[\s<>]/.test(destination)) return destination;
137
- return `<${destination.replace(/[<>]/g, '\\$&')}>`;
139
+ const hash = destination.indexOf('#');
140
+ const path = hash === -1 ? destination : destination.slice(0, hash);
141
+ const anchor = hash === -1 ? '' : destination.slice(hash);
142
+ // Parentheses too: an unbalanced one would end a bare link destination.
143
+ const encode = (seg) => encodeURIComponent(seg).replace(/[()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
144
+ return path.split('/').map((seg) => (seg === '.' || seg === '..' ? seg : encode(seg))).join('/') + anchor;
138
145
  }
139
146
 
140
147
  /**
@@ -40,7 +40,10 @@ export function parseOpenApiDocument(inputPath) {
40
40
 
41
41
  /**
42
42
  * @param {object} doc
43
- * @returns {Array<{ path: string, method: string, operation: object }>}
43
+ * @returns {Array<{ path: string, method: string, operation: object, parameters: object[] }>}
44
+ * `parameters` are the operation's own and its path's, with local $refs
45
+ * resolved; the operation's own replaces a path parameter with the same
46
+ * name and location.
44
47
  * sorted by path (lexicographic), then by a fixed HTTP method order --
45
48
  * not object-insertion order, which isn't guaranteed stable across
46
49
  * different JSON/YAML parsers.
@@ -53,7 +56,8 @@ export function collectOperations(doc) {
53
56
  const pathItem = paths[path] ?? {};
54
57
  for (const method of HTTP_METHODS) {
55
58
  if (pathItem[method]) {
56
- operations.push({ path, method, operation: pathItem[method] });
59
+ const operation = pathItem[method];
60
+ operations.push({ path, method, operation, parameters: mergeParameters(doc, pathItem.parameters, operation.parameters) });
57
61
  }
58
62
  }
59
63
  }
@@ -66,6 +70,17 @@ export function collectOperations(doc) {
66
70
  return operations;
67
71
  }
68
72
 
73
+ /** OpenAPI: a path's parameters apply to each of its operations, unless the operation lists one with the same name and `in`. */
74
+ function mergeParameters(doc, pathLevel, operationLevel) {
75
+ const resolve = (list) => (Array.isArray(list) ? list : [])
76
+ .map((p) => (typeof p?.$ref === 'string' ? resolveLocalRef(doc, p.$ref) ?? { name: p.$ref.split('/').pop(), in: '' } : p))
77
+ .filter((p) => p && typeof p === 'object');
78
+ const own = resolve(operationLevel);
79
+ const key = (p) => `${p.name}\u0000${p.in}`;
80
+ const ownKeys = new Set(own.map(key));
81
+ return [...resolve(pathLevel).filter((p) => !ownKeys.has(key(p))), ...own];
82
+ }
83
+
69
84
  /**
70
85
  * Resolves a local (same-document) JSON Pointer $ref, e.g.
71
86
  * "#/components/schemas/Widget". Returns undefined if the ref doesn't
@@ -86,29 +101,43 @@ function decodeRefSegment(segment) {
86
101
  return segment.replace(/~1/g, '/').replace(/~0/g, '~');
87
102
  }
88
103
 
104
+ /** The most objects and arrays one rendered schema may hold, however its references fan out. */
105
+ export const MAX_SCHEMA_NODES = 1_000;
106
+
89
107
  /**
90
108
  * Recursively expands local $ref pointers within a schema/value so rendered
91
109
  * code blocks show the actual shape rather than a bare pointer. An external
92
110
  * ref (doesn't start with "#/"), an unresolvable ref, or a ref cycle is left
93
111
  * as a literal `{ "$ref": "..." }` marker instead of failing.
112
+ *
113
+ * The expansion is bounded: once it has made `maxNodes` objects and arrays,
114
+ * every further reference is left as its `{ "$ref": "..." }` marker and every
115
+ * further object as "…", so references that fan out at each level cannot
116
+ * take exponential time or memory.
94
117
  * @param {*} value
95
118
  * @param {object} doc
96
- * @param {Set<string>} [seen]
119
+ * @param {{ maxNodes?: number }} [limits]
120
+ * @returns {*} the expanded value; `truncated` is not part of it
97
121
  */
98
- export function resolveSchema(value, doc, seen = new Set()) {
99
- if (value === null || typeof value !== 'object') return value;
100
- if (Array.isArray(value)) return value.map((item) => resolveSchema(item, doc, seen));
122
+ export function resolveSchema(value, doc, { maxNodes = MAX_SCHEMA_NODES } = {}) {
123
+ const ctx = { doc, nodes: 0, maxNodes };
124
+ return expand(value, ctx, new Set());
125
+ }
101
126
 
127
+ function expand(value, ctx, seen) {
128
+ if (value === null || typeof value !== 'object') return value;
102
129
  if (typeof value.$ref === 'string') {
103
- if (!value.$ref.startsWith('#/') || seen.has(value.$ref)) return { $ref: value.$ref };
104
- const target = resolveLocalRef(doc, value.$ref);
130
+ if (!value.$ref.startsWith('#/') || seen.has(value.$ref) || ctx.nodes >= ctx.maxNodes) return { $ref: value.$ref };
131
+ const target = resolveLocalRef(ctx.doc, value.$ref);
105
132
  if (target === undefined) return { $ref: value.$ref };
106
- return resolveSchema(target, doc, new Set(seen).add(value.$ref));
133
+ return expand(target, ctx, new Set(seen).add(value.$ref));
107
134
  }
135
+ if (++ctx.nodes > ctx.maxNodes) return '…';
136
+ if (Array.isArray(value)) return value.map((item) => expand(item, ctx, seen));
108
137
 
109
138
  const result = {};
110
139
  for (const [key, nested] of Object.entries(value)) {
111
- result[key] = resolveSchema(nested, doc, seen);
140
+ result[key] = expand(nested, ctx, seen);
112
141
  }
113
142
  return result;
114
143
  }
@@ -2,6 +2,7 @@
2
2
  // operation, or one node per tag with each operation as a subsection.
3
3
  import { joinFrontmatter } from '@openmoca/moca-core';
4
4
  import { slugify } from '../../slug.js';
5
+ import { uniqueSlug } from '../../text.js';
5
6
  import { resolveSchema } from './parse.js';
6
7
 
7
8
  const OPERATION_ID_PATTERN = /^[A-Za-z0-9_.:-]+$/;
@@ -11,9 +12,12 @@ const OPERATION_ID_PATTERN = /^[A-Za-z0-9_.:-]+$/;
11
12
  * @param {string} method
12
13
  * @param {object} operation
13
14
  * @param {object} doc
15
+ * @param {{ parameters?: object[], taken?: Set<string> }} [more] `parameters`: the
16
+ * operation's and its path's (collectOperations); `taken`: file names already used,
17
+ * so two operations whose paths slug alike get distinct files
14
18
  * @returns {import('../index.js').ContentNode}
15
19
  */
16
- export function renderOperationNode(path, method, operation, doc) {
20
+ export function renderOperationNode(path, method, operation, doc, { parameters = operation.parameters ?? [], taken = new Set() } = {}) {
17
21
  const title = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
18
22
  const operationId = safeOperationId(operation.operationId);
19
23
  const frontmatter = {
@@ -22,9 +26,9 @@ export function renderOperationNode(path, method, operation, doc) {
22
26
  resource: `${method.toUpperCase()} ${path}`,
23
27
  ...(operationId ? { tags: [operationId] } : {}),
24
28
  };
25
- const body = `# ${title}\n\n${renderSections(operation, doc, 2)}`;
29
+ const body = `# ${title}\n\n${renderSections(operation, doc, 2, parameters)}`;
26
30
  return {
27
- path: `content/${method}-${slugifyOperationPath(path)}.md`,
31
+ path: `content/${uniqueSlug(`${method}-${slugifyOperationPath(path)}`, taken)}.md`,
28
32
  body: joinFrontmatter(frontmatter, body),
29
33
  };
30
34
  }
@@ -35,16 +39,16 @@ export function renderOperationNode(path, method, operation, doc) {
35
39
  * @param {object} doc
36
40
  * @returns {import('../index.js').ContentNode}
37
41
  */
38
- export function renderTagNode(tag, operations, doc) {
42
+ export function renderTagNode(tag, operations, doc, taken = new Set()) {
39
43
  const sections = operations
40
- .map(({ path, method, operation }) => {
44
+ .map(({ path, method, operation, parameters }) => {
41
45
  const opTitle = firstNonEmpty(operation.summary, `${method.toUpperCase()} ${path}`);
42
- return `## ${method.toUpperCase()} ${path} — ${opTitle}\n\n${renderSections(operation, doc, 3)}`;
46
+ return `## ${method.toUpperCase()} ${path} — ${opTitle}\n\n${renderSections(operation, doc, 3, parameters ?? operation.parameters ?? [])}`;
43
47
  })
44
48
  .join('\n');
45
49
  const body = `# ${tag}\n\n${sections}`;
46
50
  return {
47
- path: `content/${slugify(tag)}.md`,
51
+ path: `content/${uniqueSlug(slugify(tag), taken)}.md`,
48
52
  body: joinFrontmatter({ type: 'API Tag', title: tag }, body),
49
53
  };
50
54
  }
@@ -55,14 +59,13 @@ export function renderTagNode(tag, operations, doc) {
55
59
  * "##" sections, 3 when nested one level deeper under a tag node's "## METHOD
56
60
  * path" heading).
57
61
  */
58
- function renderSections(operation, doc, level) {
62
+ function renderSections(operation, doc, level, parameters) {
59
63
  const h = '#'.repeat(level);
60
64
  const hSub = '#'.repeat(level + 1);
61
65
  const lines = [];
62
66
 
63
67
  lines.push(`${h} Description`, '', describe(operation), '');
64
68
 
65
- const parameters = operation.parameters ?? [];
66
69
  const requestBody = operation.requestBody;
67
70
  if (parameters.length > 0 || requestBody) {
68
71
  lines.push(`${h} Request`, '');
@@ -105,11 +108,16 @@ function describe(operation) {
105
108
  function parameterTable(parameters) {
106
109
  const rows = ['| Parameter | In | Required | Type |', '|---|---|---|---|'];
107
110
  for (const param of parameters) {
108
- rows.push(`| ${param.name ?? ''} | ${param.in ?? ''} | ${param.required ? 'yes' : 'no'} | ${schemaType(param.schema)} |`);
111
+ rows.push(`| ${cell(param.name)} | ${cell(param.in)} | ${param.required ? 'yes' : 'no'} | ${cell(schemaType(param.schema))} |`);
109
112
  }
110
113
  return rows;
111
114
  }
112
115
 
116
+ /** A table cell from the document: one line, with `|` escaped so it cannot add a column. */
117
+ function cell(value) {
118
+ return typeof value === 'string' ? value.replace(/[\r\n]+/g, ' ').replace(/\\/g, '\\\\').replace(/\|/g, '\\|') : '';
119
+ }
120
+
113
121
  function schemaType(schema) {
114
122
  if (!schema || typeof schema !== 'object') return '';
115
123
  if (typeof schema.type === 'string') return schema.type;
@@ -63,7 +63,8 @@ export function convert({ inputPath, options }) {
63
63
  }
64
64
 
65
65
  function buildOperationNodes(operations, doc) {
66
- return operations.map(({ path, method, operation }) => renderOperationNode(path, method, operation, doc));
66
+ const taken = new Set();
67
+ return operations.map(({ path, method, operation, parameters }) => renderOperationNode(path, method, operation, doc, { parameters, taken }));
67
68
  }
68
69
 
69
70
  function buildTagNodes(operations, doc) {
@@ -73,5 +74,6 @@ function buildTagNodes(operations, doc) {
73
74
  if (!groups.has(tag)) groups.set(tag, []);
74
75
  groups.get(tag).push(entry);
75
76
  }
76
- return [...groups.keys()].sort().map((tag) => renderTagNode(tag, groups.get(tag), doc));
77
+ const taken = new Set();
78
+ return [...groups.keys()].sort().map((tag) => renderTagNode(tag, groups.get(tag), doc, taken));
77
79
  }
@@ -9,15 +9,15 @@
9
9
  // that lists the documents in order, with an optional title, source URL and
10
10
  // SHA-256 for each (see README).
11
11
  import { createHash } from 'node:crypto';
12
- import { existsSync, readFileSync, statSync } from 'node:fs';
13
- import { basename, join } from 'node:path';
12
+ import { closeSync, constants, existsSync, fstatSync, openSync, readFileSync, realpathSync, statSync } from 'node:fs';
13
+ import { basename, join, sep } from 'node:path';
14
14
  import { joinFrontmatter } from '@openmoca/moca-core';
15
15
  import { buildManifest } from '../manifest.js';
16
16
  import { slugify } from '../slug.js';
17
17
  import { UsageError } from '../target.js';
18
18
  import { walkFiles } from '../walk.js';
19
- import { escapeInline, escapeLineStart, summary, turtleString, uniqueSlug } from '../text.js';
20
- import { extractPdf } from '../pdf/extract.js';
19
+ import { escapeInline, escapeLineStart, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
20
+ import { extractPdf, PdfLimitError, PDF_LIMITS } from '../pdf/extract.js';
21
21
  import { layoutPdf } from '../pdf/layout.js';
22
22
 
23
23
  export const name = 'pdf';
@@ -25,6 +25,8 @@ export const DEFAULT_TYPE = 'Document';
25
25
  export const DESCRIPTOR = 'pdfs.json';
26
26
 
27
27
  const RFC3778 = 'http://tools.ietf.org/rfc/rfc3778';
28
+ /** The largest PDF read, in bytes, unless --max-pdf-mb raises it. */
29
+ export const MAX_PDF_BYTES = 256 * 1024 * 1024;
28
30
  const SOURCE_MODES = ['copy', 'link'];
29
31
 
30
32
  /** @param {string} inputPath */
@@ -51,12 +53,23 @@ export async function convert({ inputPath, options }) {
51
53
  if (missing.length > 0) throw new UsageError(`--sources link needs a url for every document in ${DESCRIPTOR}; ${missing[0].file} has none.`);
52
54
  }
53
55
 
56
+ const limits = {
57
+ maxBytes: options.maxPdfBytes ?? MAX_PDF_BYTES,
58
+ maxPages: options.maxPdfPages ?? PDF_LIMITS.maxPages,
59
+ timeoutMs: options.pdfTimeoutMs ?? PDF_LIMITS.timeoutMs,
60
+ };
54
61
  const warnings = [];
55
62
  const docs = [];
56
63
  const docSlugs = new Set(['index']);
57
64
  for (const entry of listing.documents) {
58
- const bytes = readPdf(entry);
59
- const pdf = await extractPdf(new Uint8Array(bytes));
65
+ const bytes = readPdf(entry, inputPath, limits.maxBytes);
66
+ let pdf;
67
+ try {
68
+ pdf = await extractPdf(new Uint8Array(bytes), limits);
69
+ } catch (err) {
70
+ if (err instanceof PdfLimitError) throw new UsageError(`${entry.file}: ${err.message}. Raise the limit with --max-pdf-pages or --pdf-timeout.`);
71
+ throw err;
72
+ }
60
73
  const { blocks, labels } = layoutPdf(pdf);
61
74
  const title = entry.title ?? pdf.title ?? firstHeading(blocks) ?? basename(entry.file, '.pdf');
62
75
  if (!blocks.some((b) => b.kind !== 'heading')) {
@@ -140,18 +153,44 @@ export function listDocuments(inputPath) {
140
153
  if (!Array.isArray(spec?.documents) || spec.documents.length === 0) throw new UsageError(`${descriptor} needs a non-empty "documents" list.`);
141
154
  const documents = spec.documents.map((d, i) => {
142
155
  if (typeof d?.file !== 'string' || d.file.split(/[\\/]/).includes('..')) throw new UsageError(`${descriptor}: documents[${i}] needs a "file" inside the folder.`);
143
- if (d.url !== undefined && !/^https?:\/\/\S+$/.test(d.url)) throw new UsageError(`${descriptor}: documents[${i}].url must be an http(s) URL.`);
156
+ let url;
157
+ if (d.url !== undefined) {
158
+ url = httpUrl(d.url);
159
+ if (!url) throw new UsageError(`${descriptor}: documents[${i}].url must be an http(s) URL, with no user name or password.`);
160
+ if (new URL(url).hash) throw new UsageError(`${descriptor}: documents[${i}].url must not have a #fragment; moca-convert adds #page= itself.`);
161
+ }
144
162
  if (d.sha256 !== undefined && !/^[0-9a-f]{64}$/.test(d.sha256)) throw new UsageError(`${descriptor}: documents[${i}].sha256 must be 64 lowercase hex digits.`);
145
- return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['url', 'sha256', 'title']) };
163
+ return { file: d.file, abs: join(inputPath, d.file), ...pick(d, ['sha256', 'title']), ...(url ? { url } : {}) };
146
164
  });
147
165
  return { ...pick(spec, ['title', 'description']), documents };
148
166
  }
149
167
 
150
- function readPdf(entry) {
151
- if (!existsSync(entry.abs)) {
152
- throw new UsageError(`${entry.file} is missing${entry.url ? `; download it from ${entry.url}` : ''}.`);
168
+ function readPdf(entry, inputPath, maxBytes) {
169
+ // A listed file is a regular file inside the input folder: not a symbolic link, and not reached through one. It is
170
+ // opened once, without following a link, and checked and read through that one handle.
171
+ let fd;
172
+ try {
173
+ fd = openSync(entry.abs, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
174
+ } catch (err) {
175
+ if (err.code === 'ENOENT') throw new UsageError(`${entry.file} is missing${entry.url ? `; download it from ${entry.url}` : ''}.`);
176
+ if (err.code === 'ELOOP' || err.code === 'EISDIR') throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
177
+ throw err;
178
+ }
179
+ let bytes;
180
+ try {
181
+ const st = fstatSync(fd);
182
+ if (!st.isFile()) throw new UsageError(`${entry.file} is not a regular file (a symbolic link or a folder); list the file itself.`);
183
+ if (statSync(inputPath).isDirectory()) {
184
+ const root = realpathSync(inputPath);
185
+ if (!realpathSync(entry.abs).startsWith(root + sep)) throw new UsageError(`${entry.file} is outside ${inputPath}.`);
186
+ }
187
+ if (st.size > maxBytes) {
188
+ throw new UsageError(`${entry.file} is ${Math.ceil(st.size / 1024 / 1024)} MB, over the ${Math.round(maxBytes / 1024 / 1024)} MB limit. Raise it with --max-pdf-mb.`);
189
+ }
190
+ bytes = readFileSync(fd);
191
+ } finally {
192
+ closeSync(fd);
153
193
  }
154
- const bytes = readFileSync(entry.abs);
155
194
  if (entry.sha256) {
156
195
  const actual = createHash('sha256').update(bytes).digest('hex');
157
196
  if (actual !== entry.sha256) throw new UsageError(`${entry.file} has SHA-256 ${actual}, but ${DESCRIPTOR} expects ${entry.sha256}.`);
@@ -222,7 +261,7 @@ function pageRange(pages, labels) {
222
261
  const first = pages[0];
223
262
  const last = pages.at(-1);
224
263
  const printed = labels.has(first) && labels.has(last);
225
- const show = (n) => (printed ? labels.get(n) : String(n));
264
+ const show = (n) => (printed ? escapeInline(labels.get(n)) : String(n));
226
265
  const noun = `${printed ? '' : 'PDF '}page${first === last ? '' : 's'}`;
227
266
  return first === last ? `${noun} ${show(first)}` : `${noun} ${show(first)} to ${show(last)}`;
228
267
  }
@@ -9,8 +9,8 @@ import { joinFrontmatter } from '@openmoca/moca-core';
9
9
  import { buildManifest } from '../manifest.js';
10
10
  import { slugify } from '../slug.js';
11
11
  import { UsageError } from '../target.js';
12
- import { escapeInline, escapeText, firstLine, summary, turtleString, uniqueSlug } from '../text.js';
13
- import { CAPTURE_FILE } from '../youtube/capture.js';
12
+ import { compactDate, escapeInline, escapeText, firstLine, httpUrl, summary, turtleString, uniqueSlug } from '../text.js';
13
+ import { CAPTURE_FILE, VIDEO_ID, watchUrl } from '../youtube/capture.js';
14
14
  import { parseVtt } from '../youtube/json3-to-vtt.js';
15
15
 
16
16
  export const name = 'youtube';
@@ -18,6 +18,8 @@ export const DEFAULT_TYPE = 'Video';
18
18
 
19
19
  const MEDIA_FRAGS = 'http://www.w3.org/TR/media-frags/';
20
20
  const PARAGRAPH_MS = 60_000;
21
+ // A caption language as a BCP 47 tag; it is part of a file name.
22
+ const LANGUAGE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/;
21
23
 
22
24
  /** @param {string} inputPath */
23
25
  export function detect(inputPath) {
@@ -30,7 +32,7 @@ export function detect(inputPath) {
30
32
  */
31
33
  export function convert({ inputPath, options }) {
32
34
  if (!detect(inputPath)) throw new UsageError(`${inputPath} is not a YouTube capture folder (no ${CAPTURE_FILE}); create one with moca-youtube.`);
33
- const capture = readJson(join(inputPath, CAPTURE_FILE));
35
+ const capture = readCapture(inputPath);
34
36
  if (capture.videos.length === 0) throw new UsageError(`${inputPath} holds no videos.`);
35
37
  const playlist = capture.kind === 'playlist';
36
38
  const manifest = buildManifest({
@@ -48,7 +50,7 @@ export function convert({ inputPath, options }) {
48
50
  const taken = new Set(playlist ? ['index'] : []);
49
51
  const videos = capture.videos.map((id) => {
50
52
  const dir = join(inputPath, 'videos', id);
51
- const info = readJson(join(dir, 'info.json'));
53
+ const info = readInfo(id, join(dir, 'info.json'));
52
54
  const vttPath = join(dir, 'captions.vtt');
53
55
  const vtt = existsSync(vttPath) ? readFileSync(vttPath, 'utf8').replace(/\r\n?/g, '\n') : undefined;
54
56
  return { info, vtt, slug: uniqueSlug(slugify(info.title), taken) };
@@ -82,6 +84,12 @@ export function convert({ inputPath, options }) {
82
84
  warnings.push({ code: 'YT_NO_CAPTIONS', message: `${info.title} has no captions; its node has no transcript.`, file: `videos/${info.id}` });
83
85
  }
84
86
 
87
+ // Package spec §5.7: the video plays through its YouTube page, so it is described with the page's type. The
88
+ // duration bounds the chapter time ranges, which can end after the last caption.
89
+ const media = typeof info.duration === 'number' && info.duration >= 0
90
+ ? [{ source: 'video', type: 'text/html', duration: info.duration, ...(vtt ? { transcript: 'captions' } : {}) }]
91
+ : [];
92
+
85
93
  const data = {
86
94
  type,
87
95
  title: info.title,
@@ -91,6 +99,7 @@ export function convert({ inputPath, options }) {
91
99
  moca: {
92
100
  ...(playlist ? { concepts: [{ iri: iri(`video-${info.id}`), role: 'primary' }] } : {}),
93
101
  ...(evidence.length > 0 ? { evidence } : {}),
102
+ ...(media.length > 0 ? { media } : {}),
94
103
  },
95
104
  };
96
105
  if (Object.keys(data.moca).length === 0) delete data.moca;
@@ -105,9 +114,9 @@ export function convert({ inputPath, options }) {
105
114
  }
106
115
 
107
116
  function videoBody(info, description, cues) {
108
- const facts = [`[Watch on YouTube](${info.webpage_url})`];
109
- if (info.channel) facts.push(info.channel_url ? `[${escapeInline(info.channel)}](${info.channel_url})` : escapeInline(info.channel));
110
- if (info.upload_date) facts.push(`published ${info.upload_date.replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3')}`);
117
+ const facts = [`[Watch on YouTube](<${info.webpage_url}>)`];
118
+ if (info.channel) facts.push(info.channel_url ? `[${escapeInline(info.channel)}](<${info.channel_url}>)` : escapeInline(info.channel));
119
+ if (info.upload_date) facts.push(`published ${info.upload_date}`);
111
120
  if (info.duration) facts.push(clock(info.duration));
112
121
  const out = [`# ${escapeInline(info.title)}`, facts.join(' · ')];
113
122
  if (description) out.push('## Description', escapeText(description));
@@ -116,7 +125,7 @@ function videoBody(info, description, cues) {
116
125
  for (const section of sections(info, cues)) {
117
126
  if (section.title) out.push(`### ${clock(section.start)} ${escapeInline(section.title)}`);
118
127
  for (const para of paragraphs(section.cues)) {
119
- out.push(`[${clock(para.start / 1000)}](https://youtu.be/${info.id}?t=${Math.floor(para.start / 1000)}) ${escapeInline(para.text)}`);
128
+ out.push(`[${clock(para.start / 1000)}](<https://youtu.be/${info.id}?t=${Math.floor(para.start / 1000)}>) ${escapeInline(para.text)}`);
120
129
  }
121
130
  }
122
131
  }
@@ -125,13 +134,15 @@ function videoBody(info, description, cues) {
125
134
 
126
135
  /**
127
136
  * The video's chapters with the cues each holds (by cue start), or the whole
128
- * video as one untitled section. Times are whole seconds.
137
+ * video as one untitled section. Times are whole seconds, except an end that
138
+ * would otherwise fall after the end of the video.
129
139
  */
130
140
  function sections(info, cues) {
131
- const lastEnd = Math.ceil((cues.at(-1)?.end ?? 0) / 1000);
132
- const end = Math.max(info.duration ?? 0, lastEnd);
141
+ // The end of the video: its duration, or the end of the last caption when later. A time range never ends after it,
142
+ // so it is kept exact rather than rounded up (Reader contract §7).
143
+ const end = Math.max(info.duration ?? 0, (cues.at(-1)?.end ?? 0) / 1000);
133
144
  const chapters = info.chapters?.length > 0
134
- ? info.chapters.map((c, i) => ({ start: Math.floor(c.start), end: Math.ceil(c.end ?? end), title: chapterTitle(c.title, i) }))
145
+ ? info.chapters.map((c, i) => ({ start: Math.floor(c.start), end: Math.min(Math.ceil(c.end ?? end), end), title: chapterTitle(c.title, i) }))
135
146
  : [{ start: 0, end, title: null }];
136
147
  return chapters
137
148
  .map((c, i) => ({
@@ -162,7 +173,7 @@ function playlistIndex(capture, videos) {
162
173
  const lines = [`# ${escapeInline(capture.title)}`, ''];
163
174
  const description = (capture.description ?? '').trim();
164
175
  if (description) lines.push(escapeText(description), '');
165
- lines.push(`Videos in [playlist](${capture.url}) order:`, '');
176
+ lines.push(capture.url ? `Videos in [playlist](<${capture.url}>) order:` : 'Videos in playlist order:', '');
166
177
  videos.forEach(({ info, slug }, i) => lines.push(`${i + 1}. [${escapeInline(info.title)}](${slug}.md)`));
167
178
  return `${lines.join('\n')}\n`;
168
179
  }
@@ -190,5 +201,59 @@ function clock(seconds) {
190
201
  }
191
202
 
192
203
  function readJson(path) {
193
- return JSON.parse(readFileSync(path, 'utf8'));
204
+ try {
205
+ return JSON.parse(readFileSync(path, 'utf8'));
206
+ } catch (err) {
207
+ throw new UsageError(`${path} is not valid JSON: ${err.message}`);
208
+ }
209
+ }
210
+
211
+ const text = (value) => (typeof value === 'string' ? value : undefined);
212
+ const seconds = (value) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined);
213
+
214
+ /**
215
+ * The capture file, checked as it is read: video ids are plain ids (they name
216
+ * folders and go into links), and the playlist URL is parsed and rebuilt.
217
+ */
218
+ function readCapture(inputPath) {
219
+ const raw = readJson(join(inputPath, CAPTURE_FILE));
220
+ if (!Array.isArray(raw?.videos)) throw new UsageError(`${CAPTURE_FILE} needs a "videos" list.`);
221
+ for (const id of raw.videos) {
222
+ if (typeof id !== 'string' || !VIDEO_ID.test(id)) throw new UsageError(`${CAPTURE_FILE}: ${JSON.stringify(id)} is not a video id.`);
223
+ }
224
+ return {
225
+ kind: raw.kind === 'playlist' ? 'playlist' : 'video',
226
+ title: text(raw.title),
227
+ description: text(raw.description) ?? '',
228
+ url: httpUrl(raw.url),
229
+ videos: raw.videos,
230
+ };
231
+ }
232
+
233
+ /**
234
+ * A video's info.json, checked as it is read. It must be the video the
235
+ * capture lists. URLs are parsed and rebuilt (the watch page falls back to the
236
+ * one the id gives), the date must be a real YYYYMMDD date, times are numbers,
237
+ * and the caption language is a language tag. Anything else is left out.
238
+ */
239
+ function readInfo(id, path) {
240
+ const raw = readJson(path);
241
+ if (raw?.id !== id) throw new UsageError(`${path} is not the info for video ${id}.`);
242
+ const captions = raw.captions && typeof raw.captions === 'object'
243
+ ? { language: LANGUAGE.test(text(raw.captions.language) ?? '') ? raw.captions.language : 'und', automatic: raw.captions.automatic === true }
244
+ : null;
245
+ return {
246
+ id,
247
+ title: text(raw.title)?.trim() || id,
248
+ description: text(raw.description) ?? '',
249
+ channel: text(raw.channel)?.trim() || null,
250
+ channel_url: httpUrl(raw.channel_url) ?? null,
251
+ upload_date: compactDate(raw.upload_date),
252
+ duration: seconds(raw.duration) ?? null,
253
+ webpage_url: httpUrl(raw.webpage_url) ?? watchUrl(id),
254
+ chapters: (Array.isArray(raw.chapters) ? raw.chapters : [])
255
+ .filter((c) => seconds(c?.start) !== undefined)
256
+ .map((c) => ({ start: c.start, ...(seconds(c.end) !== undefined ? { end: c.end } : {}), title: text(c.title) })),
257
+ captions,
258
+ };
194
259
  }
package/lib/manifest.js CHANGED
@@ -12,7 +12,7 @@ export function buildManifest(o) {
12
12
  if (!o.title) throw new UsageError('--title is required.');
13
13
  return {
14
14
  $schema: 'https://w3id.org/moca/schemas/v1/manifest.schema.json',
15
- mocaVersion: '0.6',
15
+ mocaVersion: '0.7',
16
16
  id: o.id,
17
17
  version: o.version ?? '1.0.0',
18
18
  title: o.title,
@@ -3,6 +3,12 @@
3
3
  // are made here; lib/pdf/layout.js turns the lines into headings and
4
4
  // paragraphs.
5
5
  import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
6
+ import { oneLine } from '../text.js';
7
+
8
+ /** Limits on one PDF: pages, and time to read it. */
9
+ export const PDF_LIMITS = Object.freeze({ maxPages: 2_000, timeoutMs: 10 * 60_000 });
10
+
11
+ export class PdfLimitError extends Error {}
6
12
 
7
13
  /**
8
14
  * @typedef {object} PdfLine
@@ -32,9 +38,28 @@ import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
32
38
 
33
39
  /**
34
40
  * @param {Uint8Array} data
35
- * @returns {Promise<PdfText>}
41
+ * @param {{ maxPages?: number, timeoutMs?: number }} [limits] PDF_LIMITS by default
42
+ * @returns {Promise<PdfText>} rejects with PdfLimitError over a limit
36
43
  */
37
- export async function extractPdf(data) {
44
+ export async function extractPdf(data, limits = {}) {
45
+ const { maxPages, timeoutMs } = { ...PDF_LIMITS, ...limits };
46
+ let timer;
47
+ let task;
48
+ const timeout = new Promise((_, reject) => {
49
+ timer = setTimeout(() => {
50
+ reject(new PdfLimitError(`reading it took longer than ${Math.round(timeoutMs / 1000)} s`));
51
+ task?.destroy();
52
+ }, timeoutMs);
53
+ });
54
+ try {
55
+ const deadline = Date.now() + timeoutMs;
56
+ return await Promise.race([read(data, { maxPages, deadline, timeoutMs }, (t) => { task = t; }), timeout]);
57
+ } finally {
58
+ clearTimeout(timer);
59
+ }
60
+ }
61
+
62
+ async function read(data, { maxPages, deadline, timeoutMs }, onTask) {
38
63
  const task = getDocument({
39
64
  data,
40
65
  isEvalSupported: false,
@@ -42,16 +67,21 @@ export async function extractPdf(data) {
42
67
  useSystemFonts: false,
43
68
  verbosity: 0,
44
69
  });
70
+ onTask(task);
45
71
  const doc = await task.promise;
46
72
  try {
73
+ if (doc.numPages > maxPages) throw new PdfLimitError(`it has ${doc.numPages} pages, over the ${maxPages}-page limit`);
47
74
  const labels = await doc.getPageLabels().catch(() => null);
48
75
  const pages = [];
49
76
  for (let n = 1; n <= doc.numPages; n++) {
77
+ if (Date.now() >= deadline) throw new PdfLimitError(`reading it took longer than ${Math.round(timeoutMs / 1000)} s`);
50
78
  const page = await doc.getPage(n);
51
79
  const { items } = await page.getTextContent();
52
80
  const [, , , height] = page.view;
53
81
  const bold = await boldFonts(page, items);
54
- pages.push({ number: n, ...(labels?.[n - 1] ? { label: labels[n - 1] } : {}), height, lines: toLines(items, bold) });
82
+ // A page label is the PDF's own text: kept short, on one line, with no control characters.
83
+ const label = oneLine(labels?.[n - 1]);
84
+ pages.push({ number: n, ...(label ? { label } : {}), height, lines: toLines(items, bold) });
55
85
  page.cleanup();
56
86
  }
57
87
  const meta = await doc.getMetadata().catch(() => null);
package/lib/text.js CHANGED
@@ -57,3 +57,58 @@ export function uniqueSlug(base, taken) {
57
57
  export function turtleString(text) {
58
58
  return `"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`;
59
59
  }
60
+
61
+ /**
62
+ * An http or https URL from input, parsed and rebuilt, or undefined. The
63
+ * rebuilt form percent-encodes spaces, quotes and angle brackets, so it is
64
+ * safe inside a Markdown link destination (`<...>`) and in YAML. URLs with
65
+ * other schemes, or with a user name or password, are refused.
66
+ * @param {unknown} value
67
+ * @returns {string|undefined}
68
+ */
69
+ export function httpUrl(value) {
70
+ if (typeof value !== 'string' || value.length > 2048 || /[\u0000-\u001f\u007f\s]/.test(value)) return undefined;
71
+ let url;
72
+ try {
73
+ url = new URL(value);
74
+ } catch {
75
+ return undefined;
76
+ }
77
+ if ((url.protocol !== 'http:' && url.protocol !== 'https:') || url.username || url.password || !url.hostname) return undefined;
78
+ return url.href;
79
+ }
80
+
81
+ /**
82
+ * A path as an IRI fragment or path: each `/`-separated segment
83
+ * percent-encoded, so spaces, quotes and angle brackets cannot end a Turtle
84
+ * IRI (`<...>`) early.
85
+ * @param {string} path
86
+ */
87
+ export function iriPath(path) {
88
+ return path.split('/').map((segment) => encodeURIComponent(segment)).join('/');
89
+ }
90
+
91
+ /**
92
+ * Short text from input shown on one line, such as a PDF page label: control
93
+ * and format characters removed, white space collapsed, at most `max`
94
+ * characters. Undefined when nothing is left.
95
+ * @param {unknown} value
96
+ * @param {number} [max]
97
+ */
98
+ export function oneLine(value, max = 32) {
99
+ if (typeof value !== 'string') return undefined;
100
+ const text = value.replace(/[\p{Cc}\p{Cf}]/gu, ' ').replace(/\s+/g, ' ').trim().slice(0, max).trim();
101
+ return text || undefined;
102
+ }
103
+
104
+ /**
105
+ * A YYYYMMDD date that exists, as YYYY-MM-DD, or undefined.
106
+ * @param {unknown} value
107
+ */
108
+ export function compactDate(value) {
109
+ const m = typeof value === 'string' ? /^(\d{4})(\d{2})(\d{2})$/.exec(value) : null;
110
+ if (!m) return undefined;
111
+ const iso = `${m[1]}-${m[2]}-${m[3]}`;
112
+ const d = new Date(`${iso}T00:00:00Z`);
113
+ return !Number.isNaN(d.getTime()) && d.toISOString().startsWith(iso) ? iso : undefined;
114
+ }
package/lib/walk.js CHANGED
@@ -4,7 +4,9 @@ import { join, relative } from 'node:path';
4
4
  /**
5
5
  * Recursively lists files under `dir`, returning paths relative to `rootDir`
6
6
  * (using forward slashes regardless of platform), sorted lexicographically
7
- * for deterministic iteration order.
7
+ * for deterministic iteration order. Hidden entries (a name starting with
8
+ * ".": .git/, .obsidian/, .trash/ and the like) are skipped, as a Reader
9
+ * ignores them, and so are symbolic links.
8
10
  *
9
11
  * @param {string} dir
10
12
  * @param {string} rootDir
@@ -15,6 +17,7 @@ export function walkFiles(dir, rootDir = dir) {
15
17
  const entries = readdirSync(dir, { withFileTypes: true });
16
18
  const results = [];
17
19
  for (const entry of entries) {
20
+ if (entry.name.startsWith('.')) continue;
18
21
  const full = join(dir, entry.name);
19
22
  if (entry.isDirectory()) {
20
23
  results.push(...walkFiles(full, rootDir));
package/lib/write.js CHANGED
@@ -1,15 +1,27 @@
1
1
  // Writes a PackageDraft (see lib/adapters/index.js) to disk and gates
2
2
  // success on the reference Reader reporting zero error-severity diagnostics:
3
3
  // read first, refuse to leave invalid output behind.
4
- import { existsSync, mkdirSync, mkdtempSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
5
- import { tmpdir } from 'node:os';
6
- import { dirname, join } from 'node:path';
4
+ import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
5
+ import { randomBytes } from 'node:crypto';
6
+ import { basename, dirname, join, resolve } from 'node:path';
7
7
  import { readPackage } from '@openmoca/moca-core';
8
8
  import { UsageError } from './target.js';
9
9
 
10
10
  export { UsageError };
11
11
 
12
+ /** A hidden name beside `outDir`, on the same file system, so a rename never crosses devices. */
13
+ const besideName = (outDir, tag) => join(dirname(resolve(outDir)), `.${basename(resolve(outDir))}.${randomBytes(6).toString('hex')}.${tag}`);
14
+
12
15
  /**
16
+ * Writes the draft to a folder beside `outDir`, checks it with the reference
17
+ * Reader, and only then moves it into place. Anything already at `outDir` is
18
+ * moved aside first and deleted once the new package is in; if the move fails,
19
+ * it is put back. On any failure the staged folder is removed and `outDir` is
20
+ * as it was.
21
+ *
22
+ * `force` replaces a non-empty `outDir` only when it is a previous package (it
23
+ * has a moca.json); anything else is refused.
24
+ *
13
25
  * @param {object} params
14
26
  * @param {import('./adapters/index.js').PackageDraft} params.draft
15
27
  * @param {string} params.outDir
@@ -21,15 +33,19 @@ export { UsageError };
21
33
  export async function writeDraft({ draft, outDir, force = false, strict = false, onLog = () => {} }) {
22
34
  checkFiles(draft);
23
35
  const outDirExists = existsSync(outDir);
36
+ if (outDirExists && !statSync(outDir).isDirectory()) {
37
+ throw new UsageError(`Output "${outDir}" exists and is not a directory.`);
38
+ }
24
39
  const outDirNonEmpty = outDirExists && readdirSync(outDir).length > 0;
25
-
26
40
  if (outDirNonEmpty && !force) {
27
- throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to overwrite.`);
41
+ throw new UsageError(`Output directory "${outDir}" already exists and is not empty; pass --force to replace it.`);
42
+ }
43
+ if (outDirNonEmpty && !existsSync(join(outDir, 'moca.json'))) {
44
+ throw new UsageError(`Output directory "${outDir}" is not a previous package (it has no moca.json), so --force will not replace it.`);
28
45
  }
29
46
 
30
- const useScratch = outDirNonEmpty;
31
- const stageDir = useScratch ? mkdtempSync(join(tmpdir(), 'moca-convert-')) : outDir;
32
-
47
+ mkdirSync(dirname(resolve(outDir)), { recursive: true });
48
+ const stageDir = besideName(outDir, 'partial');
33
49
  try {
34
50
  stageDraft(draft, stageDir);
35
51
 
@@ -45,26 +61,35 @@ export async function writeDraft({ draft, outDir, force = false, strict = false,
45
61
  );
46
62
  }
47
63
 
48
- if (useScratch) {
49
- rmSync(outDir, { recursive: true, force: true });
50
- renameSync(stageDir, outDir);
51
- }
52
-
64
+ replaceWith(stageDir, outDir, onLog);
53
65
  return { findings, digest: result.digest };
66
+ } finally {
67
+ // Never leave partial or invalid output behind. After a successful move
68
+ // the staged folder no longer exists.
69
+ rmSync(stageDir, { recursive: true, force: true });
70
+ }
71
+ }
72
+
73
+ /** Moves the staged folder to `outDir`: the old one aside first, removed last. */
74
+ function replaceWith(stageDir, outDir, onLog) {
75
+ if (!existsSync(outDir)) {
76
+ renameSync(stageDir, outDir);
77
+ return;
78
+ }
79
+ const aside = besideName(outDir, 'old');
80
+ onLog(`moving the previous ${outDir} aside`);
81
+ renameSync(outDir, aside);
82
+ try {
83
+ renameSync(stageDir, outDir);
54
84
  } catch (err) {
55
- // Never leave partial/invalid output behind: on any failure, remove the
56
- // staged directory. In scratch mode that's a temp dir, so the
57
- // pre-existing outDir is untouched; in direct mode stageDir === outDir,
58
- // so a freshly-created output directory is fully cleaned up.
59
- if (existsSync(stageDir)) {
60
- rmSync(stageDir, { recursive: true, force: true });
61
- }
85
+ renameSync(aside, outDir);
62
86
  throw err;
63
87
  }
88
+ rmSync(aside, { recursive: true, force: true });
64
89
  }
65
90
 
66
91
  function stageDraft(draft, stageDir) {
67
- mkdirSync(stageDir, { recursive: true });
92
+ mkdirSync(stageDir);
68
93
  writeFileSync(join(stageDir, 'moca.json'), `${JSON.stringify(draft.manifest, null, 2)}\n`);
69
94
 
70
95
  for (const node of [...draft.contentNodes, ...(draft.files ?? [])]) {
@@ -14,10 +14,33 @@ import { UsageError } from '../target.js';
14
14
  import { json3ToVtt } from './json3-to-vtt.js';
15
15
 
16
16
  export const CAPTURE_FILE = 'youtube-capture.json';
17
+ /** A video or playlist id: letters, digits, `-` and `_`. It names a folder and goes into links. */
18
+ export const VIDEO_ID = /^[A-Za-z0-9_-]{1,64}$/;
19
+
20
+ /** The hosts a capture may read from: YouTube's own. */
21
+ const YOUTUBE_HOSTS = new Set(['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be']);
22
+
23
+ /**
24
+ * Checks that `url` is an http(s) URL on a YouTube host, and returns it
25
+ * rebuilt. Throws UsageError otherwise.
26
+ * @param {string} url
27
+ */
28
+ export function youtubeUrl(url) {
29
+ let parsed;
30
+ try {
31
+ parsed = new URL(url);
32
+ } catch {
33
+ throw new UsageError(`"${url}" is not a URL.`);
34
+ }
35
+ if ((parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || parsed.username || parsed.password || !YOUTUBE_HOSTS.has(parsed.hostname.toLowerCase())) {
36
+ throw new UsageError(`"${url}" is not a YouTube URL (${[...YOUTUBE_HOSTS].join(', ')}).`);
37
+ }
38
+ return parsed.href;
39
+ }
17
40
 
18
41
  /**
19
42
  * @param {object} params
20
- * @param {string} params.url
43
+ * @param {string} params.url a YouTube video or playlist URL
21
44
  * @param {string} params.captureDir
22
45
  * @param {string} [params.language] caption language, BCP 47 (default en)
23
46
  * @param {boolean} [params.force]
@@ -29,18 +52,28 @@ export async function capture({ url, captureDir, language = 'en', force = false,
29
52
  if (existsSync(captureDir) && readdirSync(captureDir).length > 0 && !force) {
30
53
  throw new UsageError(`Capture directory "${captureDir}" is not empty; pass --force to overwrite.`);
31
54
  }
55
+ const checked = youtubeUrl(url);
32
56
  const warnings = [];
33
- log(`reading ${url}`);
34
- const top = await ytdlp.json(['--flat-playlist', url]);
57
+ log(`reading ${checked}`);
58
+ // `--` ends yt-dlp's options, so the URL is only ever read as a URL.
59
+ const top = await ytdlp.json(['--flat-playlist', '--', checked]);
35
60
  const isPlaylist = top._type === 'playlist';
36
- const entries = isPlaylist ? (top.entries ?? []).filter((e) => e?.id) : [top];
61
+ if (typeof top.id !== 'string' || !VIDEO_ID.test(top.id)) throw new Error(`yt-dlp gave an id that is not a plain id: ${JSON.stringify(top.id)}`);
62
+ const entries = [];
63
+ for (const e of isPlaylist ? (top.entries ?? []) : [top]) {
64
+ if (!e?.id) continue;
65
+ // A video id names a folder, so it must be a plain id.
66
+ if (typeof e.id !== 'string' || !VIDEO_ID.test(e.id)) warnings.push(`Skipped an entry whose id is not a plain id: ${JSON.stringify(e.id)}`);
67
+ else entries.push(e);
68
+ }
37
69
 
38
70
  const videos = [];
39
71
  for (const [i, entry] of entries.entries()) {
40
72
  const label = `${i + 1}/${entries.length} ${entry.title ?? entry.id}`;
41
73
  let info;
42
74
  try {
43
- info = isPlaylist ? await ytdlp.json(['--skip-download', watchUrl(entry.id)]) : entry;
75
+ info = isPlaylist ? await ytdlp.json(['--skip-download', '--', watchUrl(entry.id)]) : entry;
76
+ if (info?.id !== entry.id) throw new Error(`yt-dlp returned another video (${JSON.stringify(info?.id)})`);
44
77
  } catch (err) {
45
78
  warnings.push(`Skipped ${entry.id}: ${err.message}`);
46
79
  log(`skip ${label}`);
@@ -8,17 +8,23 @@ import { join } from 'node:path';
8
8
  import { UsageError } from '../target.js';
9
9
 
10
10
  const MAX_BUFFER = 512 * 1024 * 1024;
11
+ /** The longest one yt-dlp run may take. */
12
+ export const YTDLP_TIMEOUT_MS = 5 * 60_000;
11
13
 
12
14
  /**
13
- * @param {{ bin?: string }} [options]
15
+ * yt-dlp runs with --ignore-config, so a user's or system configuration file
16
+ * cannot add options, and each run is stopped after `timeoutMs`.
17
+ * @param {{ bin?: string, timeoutMs?: number }} [options]
14
18
  * @returns {{ json: (args: string[]) => Promise<object>, captions: (info: object, track: { language: string, manual: boolean }) => Promise<object> }}
15
19
  */
16
- export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp' } = {}) {
20
+ export function ytDlp({ bin = process.env.YT_DLP || 'yt-dlp', timeoutMs = YTDLP_TIMEOUT_MS } = {}) {
17
21
  const run = (args) =>
18
22
  new Promise((resolve, reject) => {
19
- execFile(bin, args, { maxBuffer: MAX_BUFFER, windowsHide: true }, (err, stdout, stderr) => {
23
+ execFile(bin, ['--ignore-config', ...args], { maxBuffer: MAX_BUFFER, windowsHide: true, timeout: timeoutMs, killSignal: 'SIGKILL' }, (err, stdout, stderr) => {
20
24
  if (err?.code === 'ENOENT') {
21
25
  reject(new UsageError(`Cannot run "${bin}". Install yt-dlp (https://github.com/yt-dlp/yt-dlp#installation) or pass --yt-dlp <path>.`));
26
+ } else if (err?.killed) {
27
+ reject(new Error(`yt-dlp took longer than ${Math.round(timeoutMs / 1000)} s and was stopped.`));
22
28
  } else if (err) {
23
29
  reject(new Error(`yt-dlp ${args.join(' ')} failed: ${(stderr || err.message).trim()}`));
24
30
  } else {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openmoca/moca-convert",
3
- "version": "0.6.0",
3
+ "version": "0.7.0",
4
4
  "description": "Create MOCA packages (OKF content bundles) from Markdown folders, Obsidian vaults, OpenAPI documents, PDFs and YouTube videos.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -15,7 +15,7 @@
15
15
  "test": "node --test test/*.test.js"
16
16
  },
17
17
  "dependencies": {
18
- "@openmoca/moca-core": "^0.6.0",
18
+ "@openmoca/moca-core": "^0.7.0",
19
19
  "commander": "^12.1.0",
20
20
  "js-yaml": "^5.2.2",
21
21
  "pdfjs-dist": "^6.3.289"