@xml-auditfile/auditfile 0.0.0-stage → 0.0.2-alpha.1

package/src/reading/row-loader.js

@@ -0,0 +1,362 @@
+import { Location } from '../diagnostics/collector.js';
+import { ColumnType } from '../definitions/manifest.js';
+import { DialectAction } from '../definitions/names.js';
+import { SourceError } from '../internal/streams.js';
+import { END, START, TEXT, XmlEncodingError, XmlSyntaxError } from '../internal/xml-reader.js';
+import { MappingPlan } from './mapping-plan.js';
+import { AuditfileRow } from './row.js';
+import { CANCELLATION_INTERVAL, ReadMode } from './read-options.js';
+import { tryConvert } from './value-converter.js';
+
+const XMLNS = 'http://www.w3.org/2000/xmlns/';
+const XSI = 'http://www.w3.org/2001/XMLSchema-instance';
+
+/** An open table element. */
+class Frame {
+    #ordinals = null;
+
+    constructor(plan, id, parentId, ordinal, part, node) {
+        this.plan = plan;
+        this.id = id;
+        this.parentId = parentId;
+        this.ordinal = ordinal;
+        this.part = part;
+        /** @type {import('./mapping-plan.js').PathNode} */
+        this.node = node;
+        this.values = new Array(plan.table.columns.length).fill(null);
+        /** Columns that occurred, also when their value was invalid. */
+        this.present = new Array(plan.table.columns.length).fill(false);
+        /** @type {Set<import('./mapping-plan.js').PathNode>} Groups that occurred within this row. */
+        this.groups = new Set();
+    }
+
+    nextOrdinal(child) {
+        this.#ordinals ??= new Map();
+        const next = (this.#ordinals.get(child) ?? 0) + 1;
+        this.#ordinals.set(child, next);
+        return next;
+    }
+}
+
+/**
+ * The load phase: walks the XML once and turns elements into rows using the mapping plan.
+ * A row is emitted when its element closes, so all its columns are known; ids are assigned when it opens,
+ * so they follow document order. Only the open elements are kept in memory.
+ */
+export class RowLoader {
+    #reader;
+    #plan;
+    #resolution;
+    #options;
+    #diagnostics;
+    #observers;
+    #validate;
+    #namespace;
+    #softwarePath;
+    /** @type {import('./mapping-plan.js').PathNode[]} The open elements. */
+    #nodes = [];
+    #frames = [];
+    #ready = [];
+    #reported = new Set();
+    #corrections = new Map();
+    #nextId = 0;
+    #skipDepth = null;
+    #capture = null;
+    #event = null;
+    #checkRequired;
+
+    /** True when reading stopped on a fatal error. */
+    stopped = false;
+
+    /**
+     * @param {import('../internal/xml-reader.js').XmlReader} reader
+     * @param {((event: object) => void) | null} validate Feeds the XSD validator, when validating during load.
+     */
+    constructor(reader, schema, resolution, options, diagnostics, observers, validate) {
+        this.#reader = reader;
+        this.#plan = new MappingPlan(schema);
+        this.#resolution = resolution;
+        this.#options = options;
+        this.#diagnostics = diagnostics;
+        this.#observers = observers;
+        this.#validate = validate;
+        this.#namespace = resolution.version.namespace;
+        this.#softwarePath = `${resolution.version.rootElement}/header/softwareDesc`;
+        this.#useDialect(resolution.dialect);
+        // Without the XSD nobody else notices a missing required field; with it, XAF-D001 already does.
+        this.#checkRequired = options.mode === ReadMode.Unverified;
+    }
+
+    /** @returns {AsyncGenerator<AuditfileRow>} Rows in document order of their closing tags. */
+    async *rows() {
+        const signal = this.#options.signal;
+        let events = 0;
+        for (;;) {
+            if (events++ % CANCELLATION_INTERVAL === 0) {
+                signal?.throwIfAborted();
+            }
+
+            let event;
+            try {
+                event = this.#reader.poll() ?? await this.#reader.next();
+            } catch (error) {
+                if (error instanceof XmlSyntaxError) {
+                    this.#diagnostics.add('XAF-X001', new Location(error.part, error.line, error.column, this.#currentPath), { detail: error.message });
+                } else if (error instanceof XmlEncodingError) {
+                    this.#diagnostics.add('XAF-X004', new Location(error.part, null, null, this.#currentPath), { encoding: error.encoding });
+                } else if (error instanceof SourceError) {
+                    this.#diagnostics.add('XAF-X002', new Location(null, null, null, this.#currentPath), { part: error.part, detail: error.message });
+                } else {
+                    throw error;
+                }
+
+                this.stopped = true;
+                return;
+            }
+
+            if (event === null) {
+                return;
+            }
+
+            this.#event = event;
+            this.#validate?.(event);
+            switch (event.kind) {
+                case START:
+                    this.#start(event);
+                    break;
+                case END:
+                    this.#end(event);
+                    break;
+                case TEXT:
+                    if (this.#capture !== null) {
+                        // Beyond the maximum the value is only counted, never kept.
+                        this.#capture.length += event.text.length;
+                        if (this.#capture.length <= this.#options.maxValueLength) {
+                            this.#capture.text += event.text;
+                        }
+                    }
+
+                    break;
+            }
+
+            if (this.#ready.length > 0) {
+                for (const row of this.#ready) {
+                    yield row;
+                }
+
+                this.#ready.length = 0;
+            }
+
+            if (this.stopped) {
+                return;
+            }
+        }
+    }
+
+    get #currentPath() {
+        return this.#nodes.at(-1)?.path ?? null;
+    }
+
+    /** The open element, or the node above the root element before it opens. */
+    get #current() {
+        return this.#nodes.at(-1) ?? this.#plan.root;
+    }
+
+    get #top() {
+        return this.#frames.at(-1) ?? null;
+    }
+
+    #start(event) {
+        if (this.#skipDepth !== null) {
+            return;
+        }
+
+        if (this.#capture !== null) {
+            this.#diagnostics.add('XAF-X001', this.#here(this.#capture.path),
+                { detail: `Element '${event.local}' is not allowed inside '${this.#capture.path}', which holds a value.` });
+            this.stopped = true;
+            return;
+        }
+
+        if (event.depth >= this.#options.maxDepth) {
+            this.#diagnostics.add('XAF-X003', this.#here(this.#currentPath), { max: this.#options.maxDepth });
+            this.stopped = true;
+            return;
+        }
+
+        const node = event.ns === this.#namespace ? this.#current.children.get(event.local) : undefined;
+        if (node === undefined) {
+            this.#skip(this.#current.childPath(event.local), event);
+            return;
+        }
+
+        const owner = this.#top;
+        if (node.table !== null) {
+            this.#startTable(node, node.table, event);
+        } else if (node.column !== null && owner !== null && owner.plan === node.column.table) {
+            this.#capture = { frame: owner, column: node.column.column, path: node.path, text: '', length: 0, emit: false };
+        } else if (node.isGroup) {
+            if (owner !== null) {
+                owner.groups.add(node);
+                this.#readAttributes(node, owner, event);
+            }
+
+            this.#nodes.push(node);
+        } else {
+            this.#skip(node.path, event);
+        }
+    }
+
+    #startTable(node, table, event) {
+        const parent = this.#top;
+        const frame = new Frame(table, ++this.#nextId, parent?.id ?? null, parent?.nextOrdinal(table) ?? 1, event.part, node);
+        this.#readAttributes(node, frame, event);
+        if (table.isValueTable) {
+            this.#capture = { frame, column: table.table.columns[0], path: node.path, text: '', length: 0, emit: true };
+            return;
+        }
+
+        this.#frames.push(frame);
+        this.#nodes.push(node);
+    }
+
+    #end(event) {
+        if (this.#skipDepth !== null) {
+            if (event.depth === this.#skipDepth) {
+                this.#skipDepth = null;
+            }
+
+            return;
+        }
+
+        if (this.#capture !== null) {
+            const { frame, column, path, text, length, emit } = this.#capture;
+            this.#capture = null;
+            if (length > this.#options.maxValueLength) {
+                frame.present[column.index] = true;
+                this.#diagnostics.add('XAF-S008', this.#here(path),
+                    { table: frame.plan.table.name, column: column.name, length, max: this.#options.maxValueLength });
+            } else {
+                this.#setValue(frame, column, text, path);
+            }
+            if (emit) {
+                this.#emit(frame);
+            }
+
+            return;
+        }
+
+        const node = this.#nodes.pop();
+        if (node !== undefined && this.#top?.node === node) {
+            this.#emit(this.#frames.pop());
+        }
+    }
+
+    #skip(path, event) {
+        const key = `S003:${path}`;
+        if (!this.#reported.has(key)) {
+            this.#reported.add(key);
+            this.#diagnostics.add('XAF-S003', this.#here(path), { path });
+        }
+
+        this.#skipDepth = event.depth;
+    }
+
+    #readAttributes(element, frame, event) {
+        for (const attribute of event.attributes) {
+            if (attribute.ns === XMLNS || attribute.ns === XSI || attribute.local === 'xmlns' || attribute.prefix === 'xmlns') {
+                continue;
+            }
+
+            const target = element.attributes.get(attribute.local);
+            if (target?.column != null && target.column.table === frame.plan) {
+                this.#setValue(frame, target.column.column, attribute.value, target.path);
+            }
+        }
+    }
+
+    #setValue(frame, column, raw, path) {
+        const corrections = this.#corrections.get(path);
+        if (corrections !== undefined) {
+            raw = this.#correct(raw, corrections, column, path);
+        }
+
+        const { ok, value } = tryConvert(raw, column.type);
+        if (ok) {
+            frame.values[column.index] = value;
+        } else {
+            this.#diagnostics.add('XAF-S004', this.#here(path), { value: raw, column: `${frame.plan.table.name}.${column.name}`, type: column.type });
+        }
+
+        frame.present[column.index] = true;
+        if (path === this.#softwarePath && typeof value === 'string' && this.#resolution.useSoftware(value, this.#options.applyDialects)) {
+            this.#useDialect(this.#resolution.dialect);
+        }
+    }
+
+    #correct(raw, corrections, column, path) {
+        for (const correction of corrections) {
+            let corrected = raw;
+            if (correction.action === DialectAction.DecimalComma && raw.includes(',') && !raw.includes('.')) {
+                corrected = raw.replaceAll(',', '.');
+            } else if (correction.action === DialectAction.DateFromDateTime && column.type === ColumnType.Date && raw.trim().length > 10) {
+                corrected = raw.trim().slice(0, 10);
+            }
+
+            const key = `${correction.code}:${path}`;
+            if (corrected !== raw && !this.#reported.has(key)) {
+                this.#reported.add(key);
+                this.#diagnostics.add(correction.code, this.#here(path), { path });
+            }
+
+            raw = corrected;
+        }
+
+        return raw;
+    }
+
+    #useDialect(dialect) {
+        this.#corrections = new Map();
+        for (const correction of dialect?.corrections ?? []) {
+            if (correction.action === DialectAction.ReplaceNamespace) {
+                continue;
+            }
+
+            for (const path of correction.paths) {
+                const list = this.#corrections.get(path) ?? [];
+                list.push(correction);
+                this.#corrections.set(path, list);
+            }
+        }
+    }
+
+    /**
+     * XAF-S006 for each required column the row lacks. minOccurs is relative to the column's XML parent:
+     * a column inside an optional group is only required when that group occurs.
+     */
+    #checkRequiredColumns(frame) {
+        for (const { column, group } of frame.plan.required) {
+            if (!frame.present[column.index] && (group === null || frame.groups.has(group))) {
+                this.#diagnostics.add('XAF-S006', this.#here(frame.node.path), { column: column.name, table: frame.plan.table.name });
+            }
+        }
+    }
+
+    #emit(frame) {
+        if (this.#checkRequired) {
+            this.#checkRequiredColumns(frame);
+        }
+
+        const row = new AuditfileRow(frame.plan.table, frame.id, frame.parentId, frame.ordinal, frame.part, frame.values);
+        for (const observer of this.#observers) {
+            observer.onRow(row);
+        }
+
+        this.#ready.push(row);
+    }
+
+    #here(path) {
+        const event = this.#event;
+        return new Location(event?.part ?? null, event?.line ?? null, event?.column ?? null, path);
+    }
+}

package/src/reading/row.js

@@ -0,0 +1,123 @@
+import { ColumnType } from '../definitions/manifest.js';
+import { Decimal } from '../internal/decimal.js';
+
+/** The accessor for each column type, for messages. */
+const ACCESSORS = Object.freeze({
+    [ColumnType.String]: 'getString',
+    [ColumnType.Decimal]: 'getDecimal',
+    [ColumnType.Integer]: 'getInteger',
+    [ColumnType.Date]: 'getDate',
+    [ColumnType.DateTime]: 'getDateTime',
+    [ColumnType.Time]: 'getTime',
+    [ColumnType.Boolean]: 'getBoolean',
+});
+
+/**
+ * One row of a table. {@link AuditfileRow#values} follows the column order of {@link AuditfileRow#table};
+ * the JavaScript type of a value follows the column type (string, Decimal, number, Date, string "HH:MM:SS", boolean) or is null.
+ *
+ * Prefer the typed accessors ({@link AuditfileRow#getDecimal}, {@link AuditfileRow#getString}, ...) over {@link AuditfileRow#get}:
+ * they check the column type and say what is wrong. Every accessor takes a column name or a `SchemaColumn`; in a loop over
+ * many rows, look the column up once with `table.findColumn()` and pass it.
+ */
+export class AuditfileRow {
+    /** @internal */
+    constructor(table, id, parentId, ordinal, part, values) {
+        /** @type {import('../schema/schema.js').SchemaTable} The table this row belongs to. */
+        this.table = table;
+        /** Unique within the import; assigned in document order. */
+        this.id = id;
+        /** Id of the parent row, or null for the root. */
+        this.parentId = parentId;
+        /** Position under the parent row (1-based), per table. */
+        this.ordinal = ordinal;
+        /** Source part the row came from (1-based). */
+        this.part = part;
+        /** @type {readonly unknown[]} Column values in column order. */
+        this.values = Object.freeze(values);
+        Object.freeze(this);
+    }
+
+    /**
+     * Value of a column.
+     * @param {string | import('../schema/schema.js').SchemaColumn} column
+     * @throws {RangeError} The column does not exist in this table.
+     */
+    get(column) {
+        return this.values[this.#column(column).index];
+    }
+
+    /**
+     * True when this table has the column (a column may exist in one version and not in another).
+     * @param {string} column
+     */
+    has(column) {
+        return this.table.findColumn(column) !== null;
+    }
+
+    /** @returns {string | null} A string column. @throws {RangeError | TypeError} */
+    getString(column) {
+        return this.#typed(column, ColumnType.String);
+    }
+
+    /**
+     * A decimal column, or an integer column as a Decimal (a column can be integer in one version and decimal in another).
+     * @returns {Decimal | null}
+     * @throws {RangeError | TypeError}
+     */
+    getDecimal(column) {
+        const value = this.#typed(column, ColumnType.Decimal, ColumnType.Integer);
+        return typeof value === 'number' ? Decimal.from(value) : value;
+    }
+
+    /** @returns {number | null} An integer column. @throws {RangeError | TypeError} */
+    getInteger(column) {
+        return this.#typed(column, ColumnType.Integer);
+    }
+
+    /** @returns {Date | null} A date column (midnight UTC). @throws {RangeError | TypeError} */
+    getDate(column) {
+        return this.#typed(column, ColumnType.Date);
+    }
+
+    /** @returns {Date | null} A date-time column, or a date column at midnight UTC. @throws {RangeError | TypeError} */
+    getDateTime(column) {
+        return this.#typed(column, ColumnType.DateTime, ColumnType.Date);
+    }
+
+    /** @returns {string | null} A time column, "HH:MM:SS". @throws {RangeError | TypeError} */
+    getTime(column) {
+        return this.#typed(column, ColumnType.Time);
+    }
+
+    /** @returns {boolean | null} A boolean column. @throws {RangeError | TypeError} */
+    getBoolean(column) {
+        return this.#typed(column, ColumnType.Boolean);
+    }
+
+    #column(column) {
+        if (typeof column === 'string') {
+            const found = this.table.findColumn(column);
+            if (found === null) {
+                throw new RangeError(`Column '${column}' does not exist in ${this.table.name}.`);
+            }
+
+            return found;
+        }
+
+        if (this.table.columns[column?.index] !== column) {
+            throw new RangeError(`Column '${column?.name}' does not belong to ${this.table.name}.`);
+        }
+
+        return column;
+    }
+
+    #typed(column, type, alternative = null) {
+        const found = this.#column(column);
+        if (found.type !== type && found.type !== alternative) {
+            throw new TypeError(`${this.table.name}.${found.name} is a ${found.type} column, not ${type}; use ${ACCESSORS[found.type]}.`);
+        }
+
+        return this.values[found.index];
+    }
+}

package/src/reading/source.js

@@ -0,0 +1,146 @@
+/** Opens a part; internal so callers cannot bypass the single-use check. */
+export const OPEN = Symbol('open');
+
+/** A source used in a way it does not allow, such as reading a single-use stream twice: the caller's mistake. */
+export class SourceUsageError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SourceUsageError';
+    }
+}
+
+/**
+ * Where an audit file comes from: one stream, or several streams for the parts of a split audit file.
+ * The reader never opens files itself; the caller supplies the streams, so the same code runs on a server and in the browser.
+ * A factory can be read twice (scan, then load); a plain stream only once.
+ *
+ * A stream is a Node Readable, a web ReadableStream, a Blob, a Uint8Array, an ArrayBuffer,
+ * or any (async) iterable of Uint8Array chunks.
+ */
+export class AuditfileSource {
+    /** @param {SourcePart[]} parts */
+    constructor(parts) {
+        if (parts.length === 0) {
+            throw new RangeError('A source needs at least one part.');
+        }
+
+        /** @type {readonly SourcePart[]} The parts in reading order. */
+        this.parts = Object.freeze(parts);
+        Object.freeze(this);
+    }
+
+    /** True when every part can be opened again, so the reader can scan before loading. */
+    get canReadTwice() {
+        return this.parts.every((p) => p.reopenable);
+    }
+
+    /**
+     * A stream that can be read only once. The reader does not close or destroy it.
+     * @param {unknown} stream
+     * @param {string} [name]
+     */
+    static fromStream(stream, name = 'stream') {
+        return AuditfileSource.fromStreams([name, stream]);
+    }
+
+    /**
+     * A split audit file as streams that can be read only once: the first part followed by its continuation parts, in order.
+     * The reader does not close or destroy the streams.
+     * @param {...[string, unknown]} parts
+     */
+    static fromStreams(...parts) {
+        return new AuditfileSource(parts.map((part, i) => {
+            const [name, stream] = pair(part, 'stream');
+            if (stream === null || stream === undefined) {
+                throw new TypeError(`Part ${i + 1} has no stream.`);
+            }
+
+            let used = false;
+            return new SourcePart(i + 1, name, () => {
+                if (used) {
+                    throw new SourceUsageError('This stream source can only be read once.');
+                }
+
+                used = true;
+                return { stream, keepOpen: true };
+            }, false);
+        }));
+    }
+
+    /**
+     * A stream factory that returns a fresh stream (or a promise of one) on every call, so the source can be read twice.
+     * The reader closes the streams it gets.
+     * @example
+     * AuditfileSource.fromFactory(() => fs.createReadStream('admin.xaf'), 'admin.xaf');
+     * AuditfileSource.fromFactory(() => bytes);
+     * @param {() => unknown} open
+     * @param {string} [name]
+     */
+    static fromFactory(open, name = 'stream') {
+        return AuditfileSource.fromFactories([name, open]);
+    }
+
+    /**
+     * A split audit file as stream factories: the first part followed by its continuation parts, in order.
+     * Every factory returns a fresh stream on every call, so the source can be read twice. The reader closes the streams it gets.
+     * @param {...[string, () => unknown]} parts
+     */
+    static fromFactories(...parts) {
+        return new AuditfileSource(parts.map((part, i) => {
+            const [name, open] = pair(part, 'factory');
+            if (typeof open !== 'function') {
+                throw new TypeError(`Part ${i + 1} needs a factory function.`);
+            }
+
+            return new SourcePart(i + 1, name, async () => ({ stream: await open(), keepOpen: false }), true);
+        }));
+    }
+}
+
+function pair(part, what) {
+    if (!Array.isArray(part) || part.length !== 2 || typeof part[0] !== 'string') {
+        throw new TypeError(`Every part must be a [name, ${what}] pair.`);
+    }
+
+    return part;
+}
+
+/** One part of a source. */
+export class SourcePart {
+    #open;
+
+    /** @internal */
+    constructor(number, name, open, reopenable) {
+        /** Part number (1-based). */
+        this.number = number;
+        /** File or stream name. */
+        this.name = name;
+        /** True when the part can be opened more than once. */
+        this.reopenable = reopenable;
+        this.#open = open;
+        Object.freeze(this);
+    }
+
+    /** @internal @returns {Promise<{ stream: unknown, keepOpen: boolean }>} */
+    async [OPEN]() {
+        return this.#open();
+    }
+}
+
+/** Size, checksum and line count of a source part, known after it has been read once. */
+export class PartInfo {
+    /** @internal */
+    constructor(number, name, size, sha256, lines) {
+        /** Part number (1-based). */
+        this.number = number;
+        /** File or stream name. */
+        this.name = name;
+        /** Size in bytes. */
+        this.size = size;
+        /** SHA-256 of the bytes as read, lowercase hex. */
+        this.sha256 = sha256;
+        /** Number of lines. */
+        this.lines = lines;
+        Object.freeze(this);
+    }
+}