@xml-auditfile/auditfile 0.0.0-stage → 0.0.2-alpha.3

package/src/internal/streams.js

@@ -0,0 +1,232 @@
+import { OPEN, PartInfo, SourceUsageError } from '../reading/source.js';
+import { PartEncoding, PartEncodingError } from './encoding.js';
+import { Sha256 } from './sha256.js';
+
+const PREFIX_SIZE = 1024;
+
+/** A part could not be opened or read; reported as XAF-X002, never thrown to the caller. */
+export class SourceError extends Error {
+    constructor(part, cause) {
+        super(cause?.message ?? String(cause), { cause });
+        this.name = 'SourceError';
+        this.part = part;
+    }
+}
+
+/** Runs a call into the caller's stream or factory; any failure becomes a SourceError for that part. */
+async function guarded(part, call) {
+    try {
+        return await call();
+    } catch (error) {
+        throw error instanceof SourceError || error instanceof SourceUsageError ? error : new SourceError(part, error);
+    }
+}
+
+/** @param {unknown} chunk */
+function toBytes(chunk) {
+    if (chunk instanceof Uint8Array) {
+        return chunk;
+    }
+
+    if (chunk instanceof ArrayBuffer) {
+        return new Uint8Array(chunk);
+    }
+
+    if (ArrayBuffer.isView(chunk)) {
+        return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+    }
+
+    throw new TypeError('A stream must produce bytes (Uint8Array); text is refused because its encoding is unknown.');
+}
+
+async function* single(bytes) {
+    yield bytes;
+}
+
+async function* fromReader(stream, keepOpen) {
+    const reader = stream.getReader();
+    let done = false;
+    try {
+        for (;;) {
+            const result = await reader.read();
+            if (result.done) {
+                done = true;
+                return;
+            }
+
+            yield result.value;
+        }
+    } finally {
+        if (!done && !keepOpen) {
+            await reader.cancel().catch(() => {});
+        }
+
+        reader.releaseLock();
+    }
+}
+
+/**
+ * An async iterator over the chunks of any supported stream.
+ * With keepOpen the caller's stream is not destroyed when reading stops early.
+ * @returns {AsyncIterator<unknown>}
+ */
+export function iterate(stream, keepOpen) {
+    if (stream instanceof Uint8Array || stream instanceof ArrayBuffer || ArrayBuffer.isView(stream)) {
+        return single(toBytes(stream));
+    }
+
+    if (typeof Blob !== 'undefined' && stream instanceof Blob) {
+        return iterate(stream.stream(), false);
+    }
+
+    if (stream !== null && typeof stream === 'object') {
+        // Node Readable: do not destroy a caller-owned stream.
+        if (typeof stream.iterator === 'function' && typeof stream.pipe === 'function') {
+            return stream.iterator({ destroyOnReturn: !keepOpen });
+        }
+
+        // Web ReadableStream.
+        if (typeof stream.getReader === 'function') {
+            return fromReader(stream, keepOpen);
+        }
+
+        if (typeof stream[Symbol.asyncIterator] === 'function') {
+            return stream[Symbol.asyncIterator]();
+        }
+
+        if (typeof stream[Symbol.iterator] === 'function') {
+            return (async function* () {
+                yield* stream;
+            })();
+        }
+    }
+
+    throw new TypeError('Unsupported stream: pass a Readable, ReadableStream, Blob, Uint8Array or an iterable of Uint8Array.');
+}
+
+/**
+ * Reads the parts of a source one after another as one document.
+ * For continuation parts a leading byte order mark and XML declaration are skipped; a part in another encoding
+ * than the first is refused (PartEncodingError), because it would be decoded with the encoding of the first part.
+ * Collects size, SHA-256 and line count per part when collectStatistics is set.
+ */
+export class ConcatenatedStream {
+    #parts;
+    /** @type {PartInfo[]} Statistics of the parts read completely so far. */
+    completed = [];
+    /** Number (1-based) of the part the most recent chunk came from. */
+    part = 0;
+    /** Bytes handed out so far, over all parts. */
+    position = 0;
+    /** @type {PartEncoding | null} */
+    #encoding = null;
+
+    /**
+     * @param {readonly import('../reading/source.js').SourcePart[]} parts
+     * @param {boolean} collectStatistics
+     */
+    constructor(parts, collectStatistics) {
+        this.#parts = parts;
+        this.collectStatistics = collectStatistics;
+    }
+
+    /** @returns {AsyncGenerator<Uint8Array>} */
+    async *chunks() {
+        for (const part of this.#parts) {
+            this.part = part.number;
+            const { stream, keepOpen } = await guarded(part.name, () => part[OPEN]());
+            const iterator = iterate(stream, keepOpen);
+            const next = () => guarded(part.name, () => iterator.next());
+            const statistics = this.collectStatistics ? { hash: Sha256.create(), size: 0, lines: 1, lastWasCarriageReturn: false } : null;
+            const track = (bytes) => {
+                if (statistics === null) {
+                    return;
+                }
+
+                statistics.size += bytes.length;
+                statistics.hash.update(bytes);
+                for (const b of bytes) {
+                    if (b === 0x0a) {
+                        if (!statistics.lastWasCarriageReturn) {
+                            statistics.lines++;
+                        }
+
+                        statistics.lastWasCarriageReturn = false;
+                    } else if (b === 0x0d) {
+                        statistics.lines++;
+                        statistics.lastWasCarriageReturn = true;
+                    } else {
+                        statistics.lastWasCarriageReturn = false;
+                    }
+                }
+            };
+
+            try {
+                const prefix = [];
+                let prefixLength = 0;
+                let done = false;
+                while (prefixLength < PREFIX_SIZE) {
+                    const result = await next();
+                    if (result.done) {
+                        done = true;
+                        break;
+                    }
+
+                    const bytes = toBytes(result.value);
+                    prefix.push(bytes);
+                    prefixLength += bytes.length;
+                }
+
+                const first = concat(prefix, prefixLength);
+                track(first);
+                const encoding = PartEncoding.detect(first);
+                if (part.number === 1) {
+                    this.#encoding = encoding;
+                } else if (!encoding.sameAs(this.#encoding)) {
+                    throw new PartEncodingError(part.number, encoding, this.#encoding);
+                }
+
+                const rest = part.number === 1 ? first : first.subarray(encoding.skipDeclaration(first, Math.min(first.length, PREFIX_SIZE)));
+                if (rest.length > 0) {
+                    this.position += rest.length;
+                    yield rest;
+                }
+
+                while (!done) {
+                    const result = await next();
+                    if (result.done) {
+                        break;
+                    }
+
+                    const bytes = toBytes(result.value);
+                    track(bytes);
+                    if (bytes.length > 0) {
+                        this.position += bytes.length;
+                        yield bytes;
+                    }
+                }
+
+                if (statistics !== null) {
+                    this.completed.push(new PartInfo(part.number, part.name, statistics.size, statistics.hash.digestHex(), statistics.lines));
+                }
+            } finally {
+                await iterator.return?.();
+            }
+        }
+    }
+}
+
+function concat(chunks, length) {
+    if (chunks.length === 1) {
+        return chunks[0];
+    }
+
+    const bytes = new Uint8Array(length);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.length;
+    }
+
+    return bytes;
+}

package/src/internal/xml-reader.js

@@ -0,0 +1,247 @@
+import { SaxesParser } from 'saxes';
+import { PartEncoding, XmlEncodingError } from './encoding.js';
+
+/** Event kinds. */
+export const START = 1;
+export const END = 2;
+export const TEXT = 3;
+
+const SLICE_SIZE = 64 * 1024;
+
+/** The document is not well-formed XML, or its bytes cannot be decoded. */
+export class XmlSyntaxError extends Error {
+    constructor(message, line, column, part) {
+        super(message);
+        this.name = 'XmlSyntaxError';
+        this.line = line;
+        this.column = column;
+        this.part = part;
+    }
+}
+
+export { XmlEncodingError };
+
+function createDecoder(encoding) {
+    try {
+        return new TextDecoder(encoding, { fatal: true });
+    } catch {
+        throw new XmlEncodingError(encoding);
+    }
+}
+
+/**
+ * Pull reader over a byte stream: start, text and end events with line, column, depth and source part.
+ * DTDs are refused, entities other than the five predefined ones are errors, and namespaces are resolved.
+ */
+export class XmlReader {
+    #stream;
+    #chunks;
+    #parser;
+    #decoder = null;
+    #queue = [];
+    #head = 0;
+    #depth = 0;
+    #ended = false;
+    #pendingText = null;
+    #tagLine = 1;
+    #tagColumn = 1;
+    #mapFrom = null;
+    #mapTo = null;
+    #fed = 0;
+    #characters = 0;
+    #maxCharacters = 0;
+
+    /** @param {import('./streams.js').ConcatenatedStream} stream */
+    /**
+     * @param {import('./streams.js').ConcatenatedStream} stream
+     * @param {{ maxCharacters?: number }} [limits] maxCharacters: characters in the whole document; 0 is no limit.
+     */
+    constructor(stream, { maxCharacters = 0 } = {}) {
+        this.#maxCharacters = maxCharacters;
+        this.#stream = stream;
+        this.#chunks = stream.chunks();
+        const parser = new SaxesParser({ xmlns: true, position: true, forceXMLVersion: true, defaultXMLVersion: '1.0' });
+        this.#parser = parser;
+
+        parser.on('error', (error) => {
+            throw new XmlSyntaxError(error.message.replace(/^\d+:\d+: /, ''), parser.line, parser.column + 1, this.#stream.part);
+        });
+        parser.on('doctype', () => {
+            throw new XmlSyntaxError('For security reasons DTD is prohibited in this XML document.', parser.line, parser.column + 1, this.#stream.part);
+        });
+        parser.on('opentagstart', (tag) => {
+            this.#flushText();
+            // The parser stands after the name; report the position of the name itself (1-based), as .NET does.
+            this.#tagLine = parser.line;
+            this.#tagColumn = Math.max(1, parser.column - tag.name.length + 1);
+        });
+        parser.on('opentag', (tag) => {
+            const attributes = [];
+            for (const attribute of Object.values(tag.attributes)) {
+                attributes.push({ ns: attribute.uri, local: attribute.local, prefix: attribute.prefix, value: attribute.value });
+            }
+
+            this.#queue.push({
+                kind: START, ns: tag.uri, local: tag.local, prefix: tag.prefix, attributes,
+                line: this.#tagLine, column: this.#tagColumn, depth: this.#depth, part: this.#stream.part,
+            });
+            if (this.#depth === 0) {
+                this.rootOffset = this.#fed;
+            }
+
+            this.#depth++;
+        });
+        parser.on('closetag', (tag) => {
+            this.#flushText();
+            this.#depth--;
+            this.#queue.push({ kind: END, ns: tag.uri, local: tag.local, line: parser.line, column: parser.column, depth: this.#depth, part: this.#stream.part });
+        });
+        const text = (value) => {
+            if (this.#pendingText === null) {
+                this.#pendingText = { kind: TEXT, text: value, line: parser.line, column: parser.column, part: this.#stream.part };
+            } else {
+                this.#pendingText.text += value;
+            }
+        };
+        parser.on('text', text);
+        parser.on('cdata', text);
+    }
+
+    /** Bytes before the slice in which the root element starts; precise to one slice (64 KB). */
+    rootOffset = null;
+
+    /** Bytes handed to the parser so far. */
+    get position() {
+        return this.#stream.position;
+    }
+
+    /** The source part being parsed. */
+    get part() {
+        return this.#stream.part;
+    }
+
+    /** Presents one namespace as another, e.g. an alias as the official namespace. */
+    mapNamespace(from, to) {
+        this.#mapFrom = from;
+        this.#mapTo = to;
+    }
+
+    /**
+     * The next event when one is already parsed; undefined when the caller must await {@link XmlReader#next}.
+     * Saves a promise per event, which is most of the cost of a large file.
+     */
+    poll() {
+        if (this.#head >= this.#queue.length) {
+            return undefined;
+        }
+
+        const event = this.#queue[this.#head];
+        this.#queue[this.#head++] = undefined;
+        return this.#map(event);
+    }
+
+    /**
+     * The next event, or null at the end of the document.
+     * @throws {XmlSyntaxError | XmlEncodingError}
+     */
+    async next() {
+        while (this.#head >= this.#queue.length) {
+            this.#queue.length = 0;
+            this.#head = 0;
+            if (this.#ended) {
+                return null;
+            }
+
+            await this.#feed();
+        }
+
+        const event = this.#queue[this.#head];
+        this.#queue[this.#head++] = undefined;
+        return this.#map(event);
+    }
+
+    /** Puts an event back so the next call returns it again. */
+    pushBack(event) {
+        if (this.#head > 0) {
+            this.#queue[--this.#head] = event;
+        } else {
+            this.#queue.unshift(event);
+        }
+    }
+
+    /** Stops reading and closes the streams it opened. */
+    async close() {
+        this.#ended = true;
+        await this.#chunks.return();
+    }
+
+    async #feed() {
+        let result;
+        try {
+            result = await this.#chunks.next();
+        } catch (error) {
+            this.#ended = true;
+            throw error;
+        }
+
+        try {
+            if (result.done) {
+                this.#ended = true;
+                if (this.#decoder !== null) {
+                    this.#write(this.#decoder.decode());
+                }
+
+                this.#parser.close();
+                this.#flushText();
+                return;
+            }
+
+            this.#decoder ??= createDecoder(PartEncoding.detect(result.value).name);
+            // Slices keep the byte position of events meaningful when a caller hands over the whole file at once.
+            for (let offset = 0; offset < result.value.length; offset += SLICE_SIZE) {
+                const slice = result.value.subarray(offset, offset + SLICE_SIZE);
+                this.#write(this.#decoder.decode(slice, { stream: true }));
+                this.#fed += slice.length;
+            }
+        } catch (error) {
+            this.#ended = true;
+            if (error instanceof TypeError && /decode|encoded data/i.test(error.message)) {
+                throw new XmlSyntaxError(`Invalid byte sequence for the encoding: ${error.message}`, this.#parser.line, this.#parser.column + 1, this.#stream.part);
+            }
+
+            throw error;
+        }
+    }
+
+    #write(text) {
+        if (text.length > 0) {
+            this.#characters += text.length;
+            if (this.#maxCharacters > 0 && this.#characters > this.#maxCharacters) {
+                throw new XmlSyntaxError(`The document has more than ${this.#maxCharacters} characters.`, this.#parser.line, this.#parser.column + 1, this.#stream.part);
+            }
+
+            this.#parser.write(text);
+            // Text is handed out per slice, so a huge text node is never joined into one string here.
+            this.#flushText();
+        }
+    }
+
+    #flushText() {
+        if (this.#pendingText !== null) {
+            this.#queue.push(this.#pendingText);
+            this.#pendingText = null;
+        }
+    }
+
+    #map(event) {
+        if (this.#mapFrom === null || event.kind === TEXT) {
+            return event;
+        }
+
+        const ns = event.ns === this.#mapFrom ? this.#mapTo : event.ns;
+        const attributes = event.kind === START && event.attributes.some((a) => a.ns === this.#mapFrom)
+            ? event.attributes.map((a) => (a.ns === this.#mapFrom ? { ...a, ns: this.#mapTo } : a))
+            : event.attributes;
+        return ns === event.ns && attributes === event.attributes ? event : { ...event, ns, attributes };
+    }
+}

package/src/memory/memory-sink.js

@@ -0,0 +1,68 @@
+import { TextLanguage } from '../definitions/localized-text.js';
+import { SinkCapabilities } from '../sinks/sink.js';
+
+/** Keeps all rows in memory, per table. Useful for tests, small files and the browser. */
+export class MemorySink {
+    /** @type {Map<string, import('../reading/row.js').AuditfileRow[]>} */
+    #tables = new Map();
+
+    capabilities = SinkCapabilities.None;
+    /** @type {import('../schema/schema.js').AuditfileSchema | null} The import schema, after begin. */
+    schema = null;
+    /** @type {import('../diagnostics/report.js').AuditfileReport | null} The report, after completion or abort. */
+    report = null;
+    /** True when the import was aborted; no rows are kept then. */
+    isAborted = false;
+
+    /**
+     * Rows of a table; empty when the table has no rows.
+     * @param {string} table
+     * @returns {readonly import('../reading/row.js').AuditfileRow[]}
+     */
+    rows(table) {
+        return this.#tables.get(table) ?? [];
+    }
+
+    /**
+     * Runs a view, e.g. `sink.run(sink.schema.version.findView('trial_balance'))`, over the rows in memory.
+     * @param {import('../definitions/view-definition.js').ViewDefinition} view
+     * @param {Record<string, unknown> | Map<string, unknown> | null} [parameters]
+     * @param {'nl' | 'en'} [language]
+     * @throws {Error} Nothing has been read yet.
+     */
+    run(view, parameters = null, language = TextLanguage.Dutch) {
+        if (this.schema === null) {
+            throw new Error('Read an audit file into this sink first.');
+        }
+
+        return view.run(this.schema, (table) => this.rows(table), parameters, language);
+    }
+
+    begin(context) {
+        this.#tables.clear();
+        this.schema = context.schema;
+        this.isAborted = false;
+    }
+
+    write(table, rows) {
+        let list = this.#tables.get(table.name);
+        if (list === undefined) {
+            list = [];
+            this.#tables.set(table.name, list);
+        }
+
+        for (const row of rows) {
+            list.push(row);
+        }
+    }
+
+    complete(report) {
+        this.report = report;
+    }
+
+    abort(report) {
+        this.#tables.clear();
+        this.report = report;
+        this.isAborted = true;
+    }
+}

package/src/reading/mapping-plan.js

@@ -0,0 +1,91 @@
+/**
+ * The paths of an import as a tree, built once per import: every node knows its table, column or group.
+ * Reading an element is then one lookup of its local name under the current node, instead of building the
+ * full path as a string and looking that up. Only known paths have a node, so a file cannot make the tree grow.
+ */
+export class MappingPlan {
+    /** @type {Map<string, PathNode>} */
+    #nodes = new Map();
+    /** Above the root element: its only child is the root element of the version. */
+    root = new PathNode('');
+
+    /** @param {import('../schema/schema.js').AuditfileSchema} schema */
+    constructor(schema) {
+        const plans = [];
+        for (const table of schema.tables) {
+            const plan = new TablePlan(table);
+            plans.push(plan);
+            this.#node(table.path).table = plan;
+
+            for (const column of table.columns) {
+                if (column.path !== '.') {
+                    this.#node(`${table.path}/${column.path}`).column = { table: plan, column };
+                }
+            }
+        }
+
+        for (const node of this.#nodes.values()) {
+            node.isGroup = node.table === null && node.column === null;
+        }
+
+        for (const plan of plans) {
+            plan.required = plan.table.columns
+                .filter((c) => c.minOccurs >= 1 && c.path !== '.')
+                .map((c) => {
+                    const slash = c.path.lastIndexOf('/');
+                    return { column: c, group: slash > 0 ? this.#nodes.get(`${plan.table.path}/${c.path.slice(0, slash)}`) : null };
+                });
+        }
+    }
+
+    #node(path) {
+        let node = this.#nodes.get(path);
+        if (node !== undefined) {
+            return node;
+        }
+
+        const slash = path.lastIndexOf('/');
+        const parent = slash < 0 ? this.root : this.#node(path.slice(0, slash));
+        const name = slash < 0 ? path : path.slice(slash + 1);
+        node = new PathNode(path);
+        (name.startsWith('@') ? parent.attributes : parent.children).set(name.replace(/^@/, ''), node);
+        this.#nodes.set(path, node);
+        return node;
+    }
+}
+
+/** A known element or attribute path. */
+export class PathNode {
+    /** @type {TablePlan | null} */
+    table = null;
+    /** @type {{ table: TablePlan, column: import('../schema/schema.js').SchemaColumn } | null} */
+    column = null;
+    /** A flattened element or pure container: no table and no column. */
+    isGroup = false;
+    /** @type {Map<string, PathNode>} Child elements by local name. */
+    children = new Map();
+    /** @type {Map<string, PathNode>} Attributes by local name. */
+    attributes = new Map();
+
+    /** @param {string} path The full path, e.g. "auditfile/company/transactions"; attributes as ".../@name". */
+    constructor(path) {
+        this.path = path;
+    }
+
+    /** The path of an unknown child, for a diagnostic. */
+    childPath(localName) {
+        return this.path === '' ? localName : `${this.path}/${localName}`;
+    }
+}
+
+export class TablePlan {
+    /** @type {{ column: import('../schema/schema.js').SchemaColumn, group: PathNode | null }[]} Columns with minOccurs ≥ 1, with the group they are in. */
+    required = [];
+
+    /** @param {import('../schema/schema.js').SchemaTable} table */
+    constructor(table) {
+        this.table = table;
+        /** A repeating simple element: one "value" column read from the element text. */
+        this.isValueTable = table.columns.length === 1 && table.columns[0].path === '.';
+    }
+}