@markup-carve/pandoc-carve 0.1.3 → 0.1.5

package/README.md

@@ -46,6 +46,12 @@ pandoc-carve doc.crv -t latex -s -o doc.tex
 # Just the Pandoc JSON AST (no pandoc needed)
 pandoc-carve doc.crv -t json
 
+# Expand contained includes; named files default to their own directory
+pandoc-carve book/main.crv -t json --include-root "$PWD/book"
+
+# Keep directives literal for a named file
+pandoc-carve book/main.crv -t json --no-includes
+
 # Read from stdin, pass extra args through to pandoc after --
 cat doc.crv | pandoc-carve - -t latex -- --toc
 
@@ -62,25 +68,51 @@ carve doc.crv --to-json | pandoc-carve - -f carve-json -t latex
 ```
 
 Anything Carve cannot map faithfully is reported on stderr as a
-`pandoc-carve: degraded ...` warning - nothing degrades silently. For migration
-automation, `--diagnostics report.json` writes the same findings as structured
-JSON without mixing them into document output. `--fail-on-loss` exits with code
-3 for `lossy` or `unsupported` findings, while harmless `normalized` findings
-do not fail CI. Use `--diagnostics -` for JSON on stderr.
+`pandoc-carve: degraded ...` warning - nothing degrades silently. Unresolved or
+refused includes are reported as dropped, while safe include renames and heading
+clamps are normalized. For migration
+automation, `--diagnostics report.json` writes the same findings in a versioned
+`{ schemaVersion, sourceFormat, diagnostics }` JSON envelope without mixing
+them into document output. `--fail-on-loss` exits with code 3 for degraded or
+dropped findings, while preserved and normalized findings do not fail CI.
+Imports through a Pandoc reader other than `-f json` also exit
+3 because fidelity before the Pandoc JSON boundary cannot be verified; stderr
+names that worst-case gate explicitly. Use `--diagnostics -` for JSON on stderr.
 
 ## API
 
 ```js
 import { carveToPandoc, carveToPandocJson } from '@markup-carve/pandoc-carve';
 
-const { doc, warnings, diagnostics } = carveToPandoc('Hello /world/!');
+const { doc, warnings, diagnostics, report } = carveToPandoc('Hello /world/!');
 // doc = { 'pandoc-api-version': [1, 23, 1], meta: {...}, blocks: [...] }
 // warnings = ['degraded: ...'] for lossy constructs
-// diagnostics = [{ code, direction, severity, message, details?, path?, sourceLocation? }]
+// diagnostics = [{ code, direction, class, severity, fidelity, confidence, message, ... }]
+// report = { schemaVersion: 2, sourceFormat: 'carve', diagnostics }
 
 const json = carveToPandocJson('Hello /world/!'); // stringified doc
 ```
 
+String conversion leaves `{{ path }}` directives literal. File-backed hosts can
+opt into contained expansion and receive warnings and dependency identities:
+
+```js
+import { readFileSync } from 'node:fs';
+import { carveToPandocWithIncludes } from '@markup-carve/pandoc-carve/node';
+
+const sourcePath = '/srv/book/main.crv';
+const result = carveToPandocWithIncludes(readFileSync(sourcePath, 'utf8'), {
+  includeRoot: '/srv/book',
+  sourcePath,
+});
+// includeWarnings and suppressedIncludeWarnings describe expansion diagnostics;
+// dependencies contains resolved and unresolved include identities.
+```
+
+Both API paths must be absolute, and `sourcePath` must stay inside
+`includeRoot`. The CLI accepts ordinary relative input paths but requires an
+absolute `--include-root`. Stdin stays literal unless that option is supplied.
+
 What the converter reads is the **serialized AST of the Carve spec's PART 12**
 (the shape `resources/ast-schema.json` pins), not any implementation's runtime
 tree - so a document another engine already parsed converts the same way,
@@ -103,7 +135,7 @@ serializer), so output formatting carries fmt's guarantees:
 ```js
 import { pandocToCarve, pandocToCarveAst } from '@markup-carve/pandoc-carve';
 
-const { carve, warnings, diagnostics } = pandocToCarve(pandocJsonString);
+const { carve, warnings, diagnostics, report } = pandocToCarve(pandocJsonString);
 
 // Preserve structured fields that Carve 0.1 source cannot spell, including
 // Pandoc's optional short figure/table caption.
@@ -160,6 +192,36 @@ The complete node-by-node contract lives in the test goldens. Worked
 input/output pairs in both directions - including how interactive constructs
 degrade for print formats - are in [`examples/`](examples/README.md).
 
+## Dangerous URL schemes are blanked
+
+A link or image destination whose scheme Carve's spec denies (PART 9 §25 -
+`javascript:`, `vbscript:`, `data:`, `file:`, and the OS protocol-handler class
+such as `ms-msdt:`, `search-ms:`, `shell:`, `vscode:` and `jar:`) leaves this
+bridge as an **empty target**, and an `unsafe-url-scheme` diagnostic is emitted
+naming the scheme and the destination that was refused.
+
+Carve's own HTML, Markdown and ANSI writers all blank it, and the clause binds
+every target that emits a resolvable URL. Pandoc's targets are not HTML, but
+`pandoc -f json -t html` is one command away - passing the scheme through here
+would not be a narrower policy, it would be the same sink one step removed.
+
+The diagnostic class is `lossy`, so `--fail-on-loss` stops on its dropped
+fidelity and a caller reading
+`diagnostics` can act on it. The scheme list and the scheme probe are mirrored
+from the engine rather than invented, and a test drives the engine's own writer
+over both to keep the mirror honest - the probe strips control characters and
+Unicode whitespace before it reads a scheme, so `<U+202F>javascript:` and
+`java<DEL>script:` are caught along with the plain spelling.
+
+Nothing else is touched. An `https:`, `mailto:`, `tel:`, `ftp:`, relative or
+fragment destination reaches pandoc exactly as written, and emits no
+diagnostic.
+
+> [!NOTE]
+> This covers link and image **destinations**. An `{onclick=…}`, `{style=…}` or
+> `{srcdoc=…}` attribute, which Carve's HTML writer also refuses, still reaches
+> pandoc - see [#163](https://github.com/markup-carve/pandoc-carve/issues/163).
+
 ## Why a bridge, not a pandoc reader?
 
 A native `Text.Pandoc.Readers.Carve` upstream would be a fourth full Carve
@@ -327,16 +389,4 @@ a processor with neither extension enabled would render.
 
 ## Development
 
-The test suite converts the whole shared spec corpus, which arrives as a
-submodule:
-
-```sh
-git clone --recurse-submodules https://github.com/markup-carve/pandoc-carve
-# or, in an existing clone:
-git submodule update --init
-npm ci && npm test
-```
-
-Without the submodule the corpus tests FAIL rather than skip - a skipped corpus
-reads like a converted one, which is how a stack overflow reachable from a
-two-word document stayed live on the published package.
+Contributor setup, testing, and maintenance notes are in the [development guide](docs/development.md).

package/dist/ast-json.js

@@ -47,6 +47,8 @@ const CHILD_FIELDS = [
     'shortCaption',
     'title',
     'target',
+    'old',
+    'new',
 ];
 function isObject(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
@@ -107,6 +109,19 @@ function normalize(node) {
             // reads the wire has no reason to know the other spelling existed.
             set('type', 'critic_comment');
             break;
+        case 'substitution': {
+            if (!Array.isArray(current['old']) && typeof current['oldText'] === 'string') {
+                set('old', [{ type: 'text', value: current['oldText'] }]);
+            }
+            if (!Array.isArray(current['new']) && typeof current['newText'] === 'string') {
+                set('new', [{ type: 'text', value: current['newText'] }]);
+            }
+            if (current['oldText'] !== undefined || current['newText'] !== undefined) {
+                const { oldText: _oldText, newText: _newText, ...rest } = out ?? { ...node };
+                out = rest;
+            }
+            break;
+        }
         case 'footnote': {
             // The pre-split inline node (carve#405), whose name COLLIDES with
             // the wire `footnote` - which is the DEFINITION block, carrying

package/dist/attribute-sanitize.d.ts

@@ -0,0 +1,6 @@
+/** Attribute names Carve never emits, regardless of value. */
+export declare const DANGEROUS_ATTR_NAMES: readonly string[];
+/** Mirror of Carve's always-on PART 9 §25 attribute-name policy. */
+export declare function isDangerousAttrName(name: string): boolean;
+/** Mirror of Carve's always-on PART 9 §25 attribute-value policy. */
+export declare function renderedAttrValue(name: string, value: string): string;

package/dist/attribute-sanitize.js

@@ -0,0 +1,55 @@
+import { DANGEROUS_URL_SCHEMES, SCHEME_PROBE_STRIP_RE } from './url-scheme.js';
+/** Attribute names Carve never emits, regardless of value. */
+export const DANGEROUS_ATTR_NAMES = ['srcdoc', 'formaction'];
+const DANGEROUS_NAMES = new Set(DANGEROUS_ATTR_NAMES);
+const DANGEROUS_VALUE_SCHEMES = new Set(DANGEROUS_URL_SCHEMES);
+const ASCII_WHITESPACE = '\\t\\n\\f\\r ';
+const URL_LIST_SEPARATORS = new Map([
+    ['srcset', new RegExp(`[,${ASCII_WHITESPACE}]+`)],
+    ['imagesrcset', new RegExp(`[,${ASCII_WHITESPACE}]+`)],
+    ['ping', new RegExp(`[${ASCII_WHITESPACE}]+`)],
+    ['attributionsrc', new RegExp(`[${ASCII_WHITESPACE}]+`)],
+]);
+/** Mirror of Carve's always-on PART 9 §25 attribute-name policy. */
+export function isDangerousAttrName(name) {
+    const normalized = name.toLowerCase();
+    return normalized.startsWith('on') || DANGEROUS_NAMES.has(normalized);
+}
+function hasDeniedValueScheme(value) {
+    const colon = value.indexOf(':');
+    if (colon === -1)
+        return false;
+    const scheme = value.slice(0, colon).replace(SCHEME_PROBE_STRIP_RE, '').toLowerCase();
+    return DANGEROUS_VALUE_SCHEMES.has(scheme);
+}
+function decodeCssEscapes(value) {
+    return value.replace(/\\([0-9a-f]{1,6}\s?|[\s\S])/gi, (_match, escape) => {
+        if (!/^[0-9a-f]/i.test(escape))
+            return escape;
+        const codePoint = Number.parseInt(escape.trim(), 16);
+        return Number.isFinite(codePoint) && codePoint <= 0x10ffff
+            ? String.fromCodePoint(codePoint)
+            : '';
+    });
+}
+function hasDangerousCss(value) {
+    const compact = decodeCssEscapes(value.replace(/\/\*[\s\S]*?\*\//g, ''))
+        .toLowerCase()
+        .replace(/\s+/g, '');
+    return compact.includes('expression(')
+        || compact.includes('url(')
+        || compact.includes('@import')
+        || compact.includes('behavior:')
+        || compact.includes('-moz-binding');
+}
+/** Mirror of Carve's always-on PART 9 §25 attribute-value policy. */
+export function renderedAttrValue(name, value) {
+    if (hasDeniedValueScheme(value))
+        return '';
+    const separator = URL_LIST_SEPARATORS.get(name.toLowerCase());
+    if (separator && value.split(separator).some((part) => part !== '' && hasDeniedValueScheme(part)))
+        return '';
+    if (name.toLowerCase() === 'style' && hasDangerousCss(value))
+        return '';
+    return value;
+}

package/dist/cli.js

@@ -8,9 +8,11 @@
  *
  * Everything after `--` is passed through to pandoc verbatim.
  */
-import { readFileSync, writeFileSync } from 'node:fs';
+import { readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
 import { carveAstToPandoc, carveToPandoc } from './index.js';
-import { hasLoss } from './diagnostics.js';
+import { carveToPandocWithIncludes } from './includes.js';
+import { hasLoss, migrationReport } from './diagnostics.js';
 function usage(exitCode) {
     const text = `Usage: pandoc-carve <input | -> [options] [-- pandoc-args...]
 
@@ -36,6 +38,8 @@ Common options:
                      human warnings are suppressed)
   --fail-on-loss     exit 3 when lossy or unsupported diagnostics are present
   --symbols FILE     JSON map resolving :name: symbols to text (export)
+  --include-root DIR containment root for includes (defaults to input folder)
+  --no-includes      leave include directives literal
   --pandoc PATH      pandoc executable (default: $PANDOC or "pandoc")
   -h, --help         show this help
 `;
@@ -50,6 +54,7 @@ function parseArgs(argv) {
         roundtrip: false,
         listTable: true,
         citations: true,
+        includes: true,
         failOnLoss: false,
         pandocPath: process.env.PANDOC ?? 'pandoc',
         passthrough: [],
@@ -88,6 +93,12 @@ function parseArgs(argv) {
         else if (a === '--symbols') {
             args.symbolsFile = argv[++i] ?? usage(1);
         }
+        else if (a === '--include-root') {
+            args.includeRoot = argv[++i] ?? usage(1);
+        }
+        else if (a === '--no-includes') {
+            args.includes = false;
+        }
         else if (a === '--diagnostics') {
             args.diagnosticsFile = argv[++i] ?? usage(1);
         }
@@ -112,6 +123,9 @@ function parseArgs(argv) {
 }
 async function main() {
     const args = parseArgs(process.argv.slice(2));
+    if (args.from && args.from !== 'carve' && (args.includeRoot || !args.includes)) {
+        throw new Error('--include-root and --no-includes only apply to Carve source input');
+    }
     // `carve` and `carve-json` are the two EXPORT inputs - Carve source, and the
     // serialized AST any engine writes with --to-json. Every other `-f` names a
     // pandoc reader, which is the import direction.
@@ -128,17 +142,38 @@ async function main() {
         citations: args.citations,
         symbols,
     };
-    const { doc, warnings, diagnostics } = args.from === 'carve-json'
-        ? carveAstToPandoc(source, options)
-        : carveToPandoc(source, options);
-    report(args, warnings, diagnostics);
+    if (args.includeRoot && !path.isAbsolute(args.includeRoot)) {
+        throw new Error('--include-root must be an absolute path');
+    }
+    let includeResult;
+    let converted;
+    if (args.from === 'carve-json') {
+        converted = carveAstToPandoc(source, options);
+    }
+    else if (args.includes && (args.input !== '-' || args.includeRoot)) {
+        includeResult = carveToPandocWithIncludes(source, {
+            includeRoot: path.resolve(args.includeRoot ?? path.dirname(args.input)),
+            ...(args.input === '-' ? {} : { sourcePath: path.resolve(args.input) }),
+        }, options);
+        converted = includeResult;
+    }
+    else {
+        converted = carveToPandoc(source, options);
+    }
+    const includeWarnings = includeResult?.includeWarnings ?? [];
+    const suppressedIncludeWarnings = includeResult?.suppressedIncludeWarnings ?? 0;
+    const { doc, warnings, diagnostics } = converted;
+    const root = args.includeRoot ?? (args.input === '-' ? undefined : path.dirname(path.resolve(args.input)));
+    const includeDiagnostics = diagnosticsForIncludes(includeWarnings, suppressedIncludeWarnings, root);
+    const allDiagnostics = [...diagnostics, ...includeDiagnostics];
+    report(args, warnings, allDiagnostics, 'carve', includeWarnings, suppressedIncludeWarnings, root);
     const json = JSON.stringify(doc);
     if (args.to === 'json') {
         if (args.output)
             writeFileSync(args.output, json + '\n');
         else
             process.stdout.write(json + '\n');
-        if (args.failOnLoss && hasLoss(diagnostics))
+        if (args.failOnLoss && hasLoss(allDiagnostics))
             process.exitCode = 3;
         return;
     }
@@ -172,7 +207,7 @@ async function main() {
         }
         throw result.error;
     }
-    process.exit(result.status || (args.failOnLoss && hasLoss(diagnostics) ? 3 : 0));
+    process.exit(result.status || (args.failOnLoss && hasLoss(allDiagnostics) ? 3 : 0));
 }
 /** Reverse direction: pandoc-readable input -> Carve source. */
 async function importToCarve(args) {
@@ -201,17 +236,29 @@ async function importToCarve(args) {
     }
     const { pandocToCarve } = await import('./index.js');
     const { carve, warnings, diagnostics } = pandocToCarve(json);
-    report(args, warnings, diagnostics);
+    const reportDiagnostics = args.from === 'json' ? diagnostics : [
+        ...diagnostics,
+        {
+            code: 'fidelity-unverified',
+            direction: 'pandoc-to-carve',
+            class: 'unsupported',
+            severity: 'error',
+            fidelity: 'dropped',
+            confidence: 'fallback',
+            message: 'Fidelity before the Pandoc JSON boundary was not reported; dropped is a conservative worst-case release-gate classification',
+        },
+    ];
+    report(args, warnings, reportDiagnostics, 'pandoc-json');
     if (args.output)
         writeFileSync(args.output, carve);
     else
         process.stdout.write(carve);
-    if (args.failOnLoss && hasLoss(diagnostics))
+    if (args.failOnLoss && hasLoss(reportDiagnostics))
         process.exitCode = 3;
 }
-function report(args, warnings, diagnostics) {
+function report(args, warnings, diagnostics, sourceFormat, includeWarnings = [], suppressedIncludeWarnings = 0, includeRoot) {
     if (args.diagnosticsFile !== undefined) {
-        const json = JSON.stringify(diagnostics, null, 2) + '\n';
+        const json = JSON.stringify(migrationReport(diagnostics, sourceFormat), null, 2) + '\n';
         if (args.diagnosticsFile === '-')
             process.stderr.write(json);
         else
@@ -220,6 +267,72 @@ function report(args, warnings, diagnostics) {
     }
     for (const warning of warnings)
         process.stderr.write(`pandoc-carve: degraded: ${warning}\n`);
+    for (const warning of includeWarnings) {
+        const file = displayIncludeFile(warning.file, includeRoot, args.input);
+        process.stderr.write(`pandoc-carve: ${file}:${warning.line}:${warning.column} ${warning.rule} - ${warning.message}\n`);
+    }
+    if (suppressedIncludeWarnings) {
+        process.stderr.write(`pandoc-carve: ${suppressedIncludeWarnings} additional include warning(s) suppressed\n`);
+    }
+    for (const diagnostic of diagnostics) {
+        if (diagnostic.code === 'fidelity-unverified') {
+            process.stderr.write(`pandoc-carve: dropped: ${diagnostic.message}\n`);
+        }
+    }
+}
+function diagnosticsForIncludes(warnings, suppressed, root) {
+    const diagnostics = warnings.map((warning) => {
+        const normalized = NORMALIZED_INCLUDE_RULES.has(warning.rule);
+        const file = warning.file ? displayIncludeFile(warning.file, root, '') : undefined;
+        return {
+            code: warning.rule,
+            direction: 'carve-to-pandoc',
+            class: normalized ? 'normalized' : 'lossy',
+            severity: normalized ? 'info' : 'error',
+            fidelity: normalized ? 'normalized' : 'dropped',
+            confidence: 'exact',
+            message: warning.message,
+            details: {
+                ...(file ? { file } : {}),
+                line: warning.line,
+                column: warning.column,
+            },
+        };
+    });
+    if (suppressed)
+        diagnostics.push({
+            code: 'include-warnings-suppressed',
+            direction: 'carve-to-pandoc',
+            class: 'degraded',
+            severity: 'warning',
+            fidelity: 'degraded',
+            confidence: 'exact',
+            message: `${suppressed} additional include warning(s) suppressed`,
+            details: { count: suppressed },
+        });
+    return diagnostics;
+}
+const NORMALIZED_INCLUDE_RULES = new Set([
+    'include-footnote-rename',
+    'include-heading-clamp',
+    'include-heading-id-rename',
+]);
+function displayIncludeFile(file, root, fallback) {
+    if (!file || !root)
+        return fallback;
+    const canonicalRoot = realpathSync(root);
+    let canonicalFile = file;
+    try {
+        canonicalFile = realpathSync(file);
+    }
+    catch {
+        // A resolver may report an unresolved identity. Keep it contained below.
+    }
+    const relative = path.relative(canonicalRoot, canonicalFile);
+    if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+        return '[outside-include-root]';
+    }
+    return `[include-root]/${relative || '.'}`;
 }
 main().catch((err) => {
     process.stderr.write(`pandoc-carve: ${err instanceof Error ? err.message : String(err)}\n`);

package/dist/convert.d.ts

@@ -11,11 +11,12 @@
  */
 import type { CarveAstDocument } from './ast-json.js';
 import * as P from './pandoc.js';
-import { type ConversionDiagnostic } from './diagnostics.js';
+import { type ConversionDiagnostic, type MigrationReport } from './diagnostics.js';
 export interface ConvertResult {
     doc: P.PandocDoc;
     warnings: string[];
     diagnostics: ConversionDiagnostic[];
+    report: MigrationReport;
 }
 export interface ConvertOptions {
     /**