@xml-auditfile/auditfile 0.0.0-stage → 0.0.2-alpha.3

package/src/reading/value-converter.js

@@ -0,0 +1,136 @@
+import { ColumnType } from '../definitions/manifest.js';
+import { Decimal } from '../internal/decimal.js';
+
+/** Converts XML text to typed values, always in XSD notation and independent of locale. */
+
+const INTEGER = /^[+-]?\d+$/;
+const DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
+const TIME_ZONE = /^(?:|Z|[+-]\d{2}:\d{2})$/;
+// Only the XSD notation: a date-time like 03/01/2017 10:00:00 or with a space instead of T is refused, never guessed.
+const DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,7}))?(Z|[+-]\d{2}:\d{2})?$/;
+// hh:mm:ss with an optional fraction (dropped) and time zone (ignored), as for a date.
+const TIME = /^(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?$/;
+
+// The range of .NET long, so every implementation accepts the same integers; JavaScript numbers must also be safe.
+const MIN_INTEGER = -(2n ** 63n);
+const MAX_INTEGER = 2n ** 63n - 1n;
+
+/** A converted value, or failure. */
+const FAILED = Object.freeze({ ok: false, value: null });
+
+function ok(value) {
+    return { ok: true, value };
+}
+
+/** A UTC date, or null when the parts do not form a real date. */
+function utcDate(year, month, day, hour = 0, minute = 0, second = 0, millisecond = 0) {
+    const date = new Date(Date.UTC(year, month - 1, day, hour, minute, second, millisecond));
+    date.setUTCFullYear(year);
+    return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day
+        && date.getUTCHours() === hour && date.getUTCMinutes() === minute && date.getUTCSeconds() === second
+        ? date
+        : null;
+}
+
+/**
+ * Empty text becomes null. Returns `{ ok: false }` when the text is not valid for the type.
+ * @param {string} raw
+ * @param {string} type One of {@link ColumnType}.
+ * @returns {{ ok: boolean, value: unknown }}
+ */
+export function tryConvert(raw, type) {
+    if (type === ColumnType.String) {
+        return ok(raw);
+    }
+
+    const text = raw.trim();
+    if (text.length === 0) {
+        return ok(null);
+    }
+
+    switch (type) {
+        case ColumnType.Decimal: {
+            const value = Decimal.parse(text);
+            return value === null ? FAILED : ok(value);
+        }
+
+        case ColumnType.Integer: {
+            if (!INTEGER.test(text) || text.length > 21) {
+                return FAILED;
+            }
+
+            const value = BigInt(text);
+            return value < MIN_INTEGER || value > MAX_INTEGER || value < BigInt(Number.MIN_SAFE_INTEGER) || value > BigInt(Number.MAX_SAFE_INTEGER)
+                ? FAILED
+                : ok(Number(value));
+        }
+
+        case ColumnType.Date: {
+            // The date as written; an optional time zone (Z, +hh:mm, -hh:mm) is ignored, never converted.
+            const match = DATE.exec(text.slice(0, 10));
+            if (match === null || !TIME_ZONE.test(text.slice(10))) {
+                return FAILED;
+            }
+
+            const date = utcDate(Number(match[1]), Number(match[2]), Number(match[3]));
+            return date === null ? FAILED : ok(date);
+        }
+
+        case ColumnType.DateTime: {
+            const match = DATE_TIME.exec(text);
+            if (match === null) {
+                return FAILED;
+            }
+
+            const [, year, month, day, hour, minute, second, fraction = '', zone] = match;
+            const local = utcDate(Number(year), Number(month), Number(day), Number(hour), Number(minute), Number(second),
+                Math.floor(Number(`0.${fraction || '0'}`) * 1000));
+            if (local === null) {
+                return FAILED;
+            }
+
+            if (zone === undefined || zone === 'Z') {
+                return ok(local);
+            }
+
+            const offset = (Number(zone.slice(1, 3)) * 60 + Number(zone.slice(4, 6))) * (zone[0] === '-' ? -1 : 1);
+            return ok(new Date(local.getTime() - offset * 60000));
+        }
+
+        case ColumnType.Time: {
+            const match = TIME.exec(text);
+            if (match === null || Number(match[1]) > 23 || Number(match[2]) > 59 || Number(match[3]) > 59) {
+                return FAILED;
+            }
+
+            return ok(`${match[1]}:${match[2]}:${match[3]}`);
+        }
+
+        case ColumnType.Boolean:
+            if (text === 'true' || text === '1') {
+                return ok(true);
+            }
+
+            return text === 'false' || text === '0' ? ok(false) : FAILED;
+
+        default:
+            return ok(raw);
+    }
+}
+
+/**
+ * Text form of a value for keys and messages: dates as yyyy-mm-dd, decimals as written.
+ * @param {unknown} value
+ */
+export function toText(value) {
+    if (value === null || value === undefined) {
+        return '';
+    }
+
+    if (value instanceof Date) {
+        const iso = value.toISOString();
+        return iso.endsWith('T00:00:00.000Z') ? iso.slice(0, 10) : iso;
+    }
+
+    return String(value);
+}

package/src/schema/schema.js

@@ -0,0 +1,256 @@
+import { ManifestKey, ManifestReference, parseColumnType } from '../definitions/manifest.js';
+import { bool, int, objects, optionalInt, optionalString, parseObject, string, strings } from '../internal/json.js';
+import { AuditfileVersion } from '../versioning/version.js';
+
+/** How the version of an import was determined. */
+export const VersionSource = Object.freeze({
+    /** The file uses the official namespace. */
+    Official: 'official',
+    /** The file uses a known alias namespace. */
+    Alias: 'alias',
+    /** The caller forced the version. */
+    Forced: 'forced',
+    /** The resolver derived the version from the namespace. */
+    Resolved: 'resolved',
+});
+
+const SOURCES = new Set(Object.values(VersionSource));
+
+/** Format version written by toJson; fromJson refuses any other. */
+const FORMAT_VERSION = 1;
+
+/**
+ * The schema of one import: tables, columns, types, keys, references, row counts and origin.
+ * This is the contract everything after the reader builds on, regardless of where the data is stored.
+ * Converts to and from JSON without loss (definitions are re-attached from the version when known).
+ */
+export class AuditfileSchema {
+    #byName;
+
+    /** @internal */
+    constructor(versionKey, fileNamespace, source, dialect, hasRowCounts, tables) {
+        /** Version key, e.g. "xaf/3.2". */
+        this.versionKey = versionKey;
+        /** @type {AuditfileVersion | null} The version, or null when this reader does not know it (schema loaded from JSON). */
+        this.version = AuditfileVersion.find(versionKey);
+        /** Namespace as found in the file. */
+        this.namespace = fileNamespace;
+        /** How the version was determined, one of {@link VersionSource}. */
+        this.source = source;
+        /** Name of the applied dialect, or null. */
+        this.dialect = dialect;
+        /** True when the row count is known for every table. */
+        this.hasRowCounts = hasRowCounts;
+        /** @type {readonly SchemaTable[]} All tables in document order. */
+        this.tables = Object.freeze([...tables]);
+        this.#byName = new Map(this.tables.map((t) => [t.name, t]));
+        Object.freeze(this);
+    }
+
+    /**
+     * Finds a table by name, or returns null.
+     * @param {string} name
+     * @returns {SchemaTable | null}
+     */
+    findTable(name) {
+        return this.#byName.get(name) ?? null;
+    }
+
+    /**
+     * Returns a table by name.
+     * @param {string} name
+     * @throws {RangeError} The table does not exist.
+     */
+    table(name) {
+        const table = this.findTable(name);
+        if (table === null) {
+            throw new RangeError(`Table '${name}' does not exist.`);
+        }
+
+        return table;
+    }
+
+    /** Serializes the schema to compact JSON, e.g. for storage in a database. */
+    toJson() {
+        return JSON.stringify({
+            formatVersion: FORMAT_VERSION,
+            version: this.versionKey,
+            namespace: this.namespace,
+            source: this.source,
+            dialect: this.dialect,
+            hasRowCounts: this.hasRowCounts,
+            tables: this.tables.map((table) => ({
+                name: table.name,
+                xmlName: table.xmlName,
+                path: table.path,
+                parent: table.parent,
+                minOccurs: table.minOccurs,
+                maxOccurs: table.maxOccurs,
+                required: table.isRequired,
+                expected: table.isExpected,
+                rowCount: table.rowCount,
+                keys: table.keys.map((key) => ({ name: key.name, columns: key.columns })),
+                references: table.references.map((reference) => ({
+                    name: reference.name, columns: reference.columns, table: reference.table, key: reference.key,
+                })),
+                columns: table.columns.map((column) => ({
+                    name: column.name,
+                    xmlName: column.xmlName,
+                    path: column.path,
+                    kind: column.kind,
+                    type: column.type,
+                    xsdType: column.xsdType,
+                    minOccurs: column.minOccurs,
+                })),
+            })),
+        });
+    }
+
+    /**
+     * Reads a schema written by {@link AuditfileSchema#toJson}.
+     * @param {string} json
+     * @throws {SyntaxError} The JSON is not a valid schema, or uses a format version, column type or version source this reader does not know.
+     */
+    static fromJson(json) {
+        if (typeof json !== 'string') {
+            throw new TypeError('json must be a string.');
+        }
+
+        const root = parseObject(json, 'Schema JSON');
+        if (root.formatVersion !== FORMAT_VERSION) {
+            throw new SyntaxError(`Schema JSON format version ${root.formatVersion ?? '(missing)'} is not supported; this reader reads format version ${FORMAT_VERSION}.`);
+        }
+
+        const versionKey = string(root, 'version');
+        const definitions = AuditfileVersion.find(versionKey)?.definitions ?? null;
+        const lookup = (key) => definitions?.get(key) ?? null;
+
+        const tables = objects(root, 'tables').map((t, index) => {
+            const name = string(t, 'name');
+            const columns = objects(t, 'columns').map((c, i) => new SchemaColumn(
+                string(c, 'name'), string(c, 'xmlName'), string(c, 'path'), string(c, 'kind'),
+                parseColumnType(string(c, 'type')), string(c, 'xsdType'), int(c, 'minOccurs'), i,
+                lookup(`${name}.${string(c, 'name')}`)));
+
+            return new SchemaTable(
+                name, string(t, 'xmlName'), string(t, 'path'), optionalString(t, 'parent'),
+                int(t, 'minOccurs'), optionalInt(t, 'maxOccurs'), bool(t, 'required'), bool(t, 'expected'),
+                optionalInt(t, 'rowCount'), index, columns,
+                objects(t, 'keys').map((k) => new ManifestKey(string(k, 'name'), strings(k, 'columns'))),
+                objects(t, 'references').map((r) => new ManifestReference(string(r, 'name'), strings(r, 'columns'), string(r, 'table'), string(r, 'key'))),
+                lookup(name));
+        });
+
+        const source = string(root, 'source');
+        if (!SOURCES.has(source)) {
+            throw new SyntaxError(`Unknown version source '${source}' in schema JSON.`);
+        }
+
+        return new AuditfileSchema(versionKey, string(root, 'namespace'), source,
+            optionalString(root, 'dialect'), bool(root, 'hasRowCounts'), tables);
+    }
+
+    /**
+     * @internal
+     * @param {AuditfileVersion} version
+     * @param {Map<string, number> | null} rowCounts
+     */
+    static create(version, fileNamespace, source, dialect, rowCounts) {
+        const manifest = version.manifest;
+        const definitions = version.definitions;
+        const required = new Set(manifest.requiredTables);
+        const expected = new Set(manifest.expectedTables);
+
+        const tables = manifest.tables.map((t) => new SchemaTable(
+            t.name, t.xmlName, t.path, t.parent, t.minOccurs, t.maxOccurs, required.has(t.name), expected.has(t.name),
+            rowCounts === null ? null : rowCounts.get(t.name) ?? 0,
+            t.index,
+            t.columns.map((c) => new SchemaColumn(c.name, c.xmlName, c.path, c.kind, c.type, c.xsdType, c.minOccurs, c.index,
+                definitions.get(`${t.name}.${c.name}`) ?? null)),
+            t.keys, t.references, definitions.get(t.name) ?? null));
+
+        return new AuditfileSchema(version.key, fileNamespace, source, dialect, rowCounts !== null, tables);
+    }
+}
+
+/** A table of an import. */
+export class SchemaTable {
+    #byName;
+
+    /** @internal */
+    constructor(name, xmlName, path, parent, minOccurs, maxOccurs, isRequired, isExpected, rowCount, index, columns, keys, references, definition) {
+        /** Table name, e.g. "tr_line". */
+        this.name = name;
+        /** Element name, e.g. "trLine". */
+        this.xmlName = xmlName;
+        /** Full XML path. */
+        this.path = path;
+        /** Parent table name, or null for the root. */
+        this.parent = parent;
+        /** Minimum occurrences relative to the XML parent element. */
+        this.minOccurs = minOccurs;
+        /** Maximum occurrences relative to the XML parent element; null means unbounded. */
+        this.maxOccurs = maxOccurs;
+        /** Required by the standard. */
+        this.isRequired = isRequired;
+        /** Expected in practice. */
+        this.isExpected = isExpected;
+        /** Number of rows, when known from the scan phase; otherwise null. */
+        this.rowCount = rowCount;
+        /** Position in the schema. */
+        this.index = index;
+        /** @type {readonly SchemaColumn[]} Columns; {@link AuditfileRow#values} uses the same order. */
+        this.columns = Object.freeze([...columns]);
+        /** @type {readonly ManifestKey[]} Natural keys. */
+        this.keys = Object.freeze([...keys]);
+        /** @type {readonly ManifestReference[]} References to keys of other tables. */
+        this.references = Object.freeze([...references]);
+        /** @type {import('../definitions/element-definition.js').ElementDefinition | null} Documentation, when available. */
+        this.definition = definition;
+        this.#byName = new Map(this.columns.map((c) => [c.name, c]));
+        Object.freeze(this);
+    }
+
+    /**
+     * Finds a column by name, or returns null.
+     * @param {string} name
+     * @returns {SchemaColumn | null}
+     */
+    findColumn(name) {
+        return this.#byName.get(name) ?? null;
+    }
+
+    toString() {
+        return this.name;
+    }
+}
+
+/** A column of an import table. */
+export class SchemaColumn {
+    /** @internal */
+    constructor(name, xmlName, path, kind, type, xsdType, minOccurs, index, definition) {
+        /** Column name. */
+        this.name = name;
+        /** Element or attribute name. */
+        this.xmlName = xmlName;
+        /** Path relative to the table element. */
+        this.path = path;
+        /** "element" or "attribute". */
+        this.kind = kind;
+        /** Logical type, one of {@link ColumnType}; determines the JavaScript type of the value. */
+        this.type = type;
+        /** XSD type name. */
+        this.xsdType = xsdType;
+        /** Minimum occurrences relative to the XML parent element. */
+        this.minOccurs = minOccurs;
+        /** Position in {@link AuditfileRow#values}. */
+        this.index = index;
+        /** @type {import('../definitions/element-definition.js').ElementDefinition | null} Documentation, when available. */
+        this.definition = definition;
+        Object.freeze(this);
+    }
+
+    toString() {
+        return this.name;
+    }
+}

package/src/sinks/sink.js

@@ -0,0 +1,53 @@
+/** What a sink needs from the reader (bit flags). */
+export const SinkCapabilities = Object.freeze({
+    /** Nothing special. */
+    None: 0,
+    /** The sink wants row counts before the first row (e.g. to pre-allocate); the reader scans first. */
+    NeedsRowCounts: 1,
+});
+
+/**
+ * Receives an import: first the schema, then rows per table in batches, then completion or abort.
+ * All validation, merging and reporting happens before the sink, so a custom sink gets the same result as a built-in one.
+ * Every method may return a promise.
+ *
+ * @typedef {object} Sink
+ * @property {number} [capabilities] What the sink needs, as {@link SinkCapabilities} flags.
+ * @property {(context: ImportContext) => void | Promise<void>} begin Start of an import. The schema contains row counts when the reader scanned first.
+ * @property {(table: import('../schema/schema.js').SchemaTable, rows: readonly import('../reading/row.js').AuditfileRow[]) => void | Promise<void>} write
+ *   A batch of rows of one table. Child rows can arrive before their parent row; ids are already assigned.
+ * @property {(report: import('../diagnostics/report.js').AuditfileReport) => void | Promise<void>} complete The import is complete; commit.
+ * @property {(report: import('../diagnostics/report.js').AuditfileReport) => void | Promise<void>} abort The import failed; discard everything received since begin.
+ */
+
+/** Information at the start of an import. */
+export class ImportContext {
+    /** @internal */
+    constructor(schema, parts) {
+        /** @type {import('../schema/schema.js').AuditfileSchema} The import schema. */
+        this.schema = schema;
+        /** @type {readonly import('../reading/source.js').SourcePart[]} The source parts. */
+        this.parts = parts;
+        Object.freeze(this);
+    }
+}
+
+/** Discards everything; used for validation. */
+export const NULL_SINK = Object.freeze({
+    capabilities: SinkCapabilities.None,
+    begin() {},
+    write() {},
+    complete() {},
+    abort() {},
+});
+
+/** @param {unknown} sink */
+export function checkSink(sink) {
+    for (const method of ['begin', 'write', 'complete', 'abort']) {
+        if (typeof sink?.[method] !== 'function') {
+            throw new TypeError(`sink must have a ${method} method.`);
+        }
+    }
+
+    return sink;
+}

package/src/validation/reference-checker.js

@@ -0,0 +1,146 @@
+import { Location } from '../diagnostics/collector.js';
+import { SEPARATOR } from '../internal/composite-key.js';
+import { toText } from '../reading/value-converter.js';
+
+const MAX_PENDING_PER_REFERENCE = 100_000;
+
+/**
+ * Checks keys (unique) and references (existing) from the schema.
+ * Only key values and unresolved reference values are kept in memory; references are resolved at the end,
+ * so a reference to a key that appears later (e.g. in a later part) is still found.
+ */
+export class ReferenceChecker {
+    #diagnostics;
+    /** @type {Map<string, KeyState>} */
+    #keysByName = new Map();
+    /** @type {Map<object, KeyState[]>} */
+    #keysByTable = new Map();
+    /** @type {Map<object, ReferenceState[]>} */
+    #referencesByTable = new Map();
+
+    constructor(schema, diagnostics) {
+        this.#diagnostics = diagnostics;
+
+        for (const table of schema.tables) {
+            for (const key of table.keys) {
+                const state = new KeyState(table, indexes(table, key.columns));
+                this.#keysByName.set(key.name, state);
+                add(this.#keysByTable, table, state);
+            }
+        }
+
+        for (const table of schema.tables) {
+            for (const reference of table.references) {
+                const key = this.#keysByName.get(reference.key);
+                if (key !== undefined) {
+                    add(this.#referencesByTable, table, new ReferenceState(table, reference.columns.join(', '), indexes(table, reference.columns), key));
+                }
+            }
+        }
+    }
+
+    onRow(row) {
+        for (const key of this.#keysByTable.get(row.table) ?? []) {
+            const value = composite(row, key.columns);
+            if (value !== null) {
+                if (key.values.has(value)) {
+                    this.#diagnostics.add('XAF-R004', new Location(row.part, null, null, row.table.path), { value: display(value), table: row.table.name });
+                } else {
+                    key.values.add(value);
+                }
+            }
+        }
+
+        for (const reference of this.#referencesByTable.get(row.table) ?? []) {
+            const value = composite(row, reference.columns);
+            if (value === null || reference.key.values.has(value) || reference.pending.has(value)) {
+                continue;
+            }
+
+            if (reference.pending.size < MAX_PENDING_PER_REFERENCE) {
+                reference.pending.add(value);
+            } else if (!reference.overflowed) {
+                // From here on new values are not checked; say so once, so a clean result is not mistaken for a complete one.
+                reference.overflowed = true;
+                this.#diagnostics.add('XAF-R017', new Location(row.part, null, null, row.table.path), {
+                    table: row.table.name, column: reference.columnNames, target: reference.key.table.name, max: MAX_PENDING_PER_REFERENCE,
+                });
+            }
+        }
+    }
+
+    onEnd() {
+        for (const references of this.#referencesByTable.values()) {
+            for (const reference of references) {
+                const missing = [...reference.pending].filter((v) => !reference.key.values.has(v)).sort(ordinal);
+                for (const value of missing) {
+                    this.#diagnostics.add('XAF-R003', new Location(null, null, null, reference.table.path), {
+                        table: reference.table.name, column: reference.columnNames, value: display(value), target: reference.key.table.name,
+                    });
+                }
+            }
+        }
+    }
+}
+
+/** Ordinal string order (UTF-16 code units), as everywhere in this reader. */
+function ordinal(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+
+function composite(row, columns) {
+    const parts = [];
+    for (const column of columns) {
+        const value = row.values[column];
+        if (value === null || value === undefined) {
+            return null;
+        }
+
+        parts.push(toText(value));
+    }
+
+    return parts.join(SEPARATOR);
+}
+
+function display(value) {
+    return value.replaceAll(SEPARATOR, '/');
+}
+
+function indexes(table, columns) {
+    return columns.map((name) => {
+        const column = table.findColumn(name);
+        if (column === null) {
+            throw new Error(`Key column ${name} not in ${table.name}.`);
+        }
+
+        return column.index;
+    });
+}
+
+function add(map, table, item) {
+    const list = map.get(table) ?? [];
+    list.push(item);
+    map.set(table, list);
+}
+
+class KeyState {
+    values = new Set();
+
+    constructor(table, columns) {
+        this.table = table;
+        this.columns = columns;
+    }
+}
+
+class ReferenceState {
+    pending = new Set();
+    /** True once the pending limit was reached and XAF-R017 was reported. */
+    overflowed = false;
+
+    constructor(table, columnNames, columns, key) {
+        this.table = table;
+        this.columnNames = columnNames;
+        this.columns = columns;
+        this.key = key;
+    }
+}