node-opcua-address-space 2.181.1 → 2.182.0

package/api/loader/nodeset_builtin_formats.ts

@@ -0,0 +1,143 @@
+/**
+ * @module node-opcua-address-space
+ *
+ * The two serialisations the loader has always read, expressed as {@link NodesetFormat}s.
+ *
+ * Registering them here rather than branching on a boolean inside the loader is the whole point
+ * of the registry: the built-ins get no privilege a third-party format does not also get, so the
+ * contract cannot quietly grow a dependency on something only the built-ins can reach. If these
+ * two can be written against it, so can a fourth.
+ *
+ * Importing this module registers them. The loader does that once, at its own import.
+ */
+
+import { getMinOPCUADate } from "node-opcua-date-time";
+import type { NodeSetLoaderOptions } from "../interfaces/nodeset_loader_options.js";
+import {
+    type NodesetDocument,
+    type NodesetFormat,
+    type NodesetHead,
+    type NodesetModel,
+    type NodesetModelInfo,
+    registerNodesetFormat
+} from "./nodeset_format.js";
+import { imageLinesToRecords, readNodesetImageInfo } from "./nodeset_image.js";
+import type { NodesetRecord } from "./nodeset_record.js";
+import { headerComplete, parseDependencies, sliceHeader } from "./nodeset_xml_header.js";
+import { xmlNodesetRecords } from "./nodeset_xml_producer.js";
+
+/** the name the NodeSet2 XML format is registered under */
+export const NODESET2_XML_FORMAT = "nodeset2-xml";
+/** the name the NDJSON image format is registered under */
+export const NDJSON_IMAGE_FORMAT = "ndjson";
+
+/**
+ * NodeSet2 XML: the fallback.
+ *
+ * It claims every document no other format recognised, which is exactly what the loader did
+ * before there was a registry, and it is the right default for a reason that outlives this
+ * refactor: XML has no compact magic number worth sniffing. A document may open with a byte-order
+ * mark, an XML declaration, a comment, a doctype or whitespace before `<UANodeSet` ever appears,
+ * and a sniffer strict enough to be sure would reject documents the loader reads today. So it
+ * does not guess. It takes what is left, and reports a real parse error if it was wrong, which is
+ * a better diagnostic than "unrecognised format" ever was.
+ */
+const nodeset2XmlFormat: NodesetFormat = {
+    name: NODESET2_XML_FORMAT,
+    extensions: [".xml"],
+    priority: -100,
+    sniff(_head: NodesetHead): boolean {
+        return true;
+    },
+    async readModels(document: NodesetDocument): Promise<NodesetModelInfo> {
+        const head = await document.readHead(headerComplete);
+        return await parseDependencies(sliceHeader(head, document.name));
+    },
+    records(document: NodesetDocument, _options?: NodeSetLoaderOptions): AsyncIterable<NodesetRecord> {
+        return xmlNodesetRecords(textChunks(document));
+    }
+};
+
+/** the chunks of a document as text: the XML producer takes text, and a source may deliver bytes */
+async function* textChunks(document: NodesetDocument): AsyncGenerator<string> {
+    const decoder = new TextDecoder("utf-8");
+    for await (const chunk of document.chunks()) {
+        yield typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
+    }
+}
+
+/**
+ * The NDJSON image: node-opcua's own cache of a document it has already parsed.
+ *
+ * Its first line is a header record, so it is told from every other line-delimited nodeset format
+ * by the two fields that open it: `{"kind":"header","schema":<n>`. That is a stronger test than
+ * "begins with a brace", which is all the loader used to do, and it is what lets a foreign
+ * `.jsonl` sit in the same directory without being mistaken for one of ours.
+ */
+const ndjsonImageFormat: NodesetFormat = {
+    name: NDJSON_IMAGE_FORMAT,
+    extensions: [".ndjson", ".ndjson.gz"],
+    priority: 100,
+    sniff(head: NodesetHead): boolean {
+        return isNdjsonHeaderLine(head.firstLine);
+    },
+    async readModels(document: NodesetDocument): Promise<NodesetModelInfo> {
+        // an image is small, a few hundred KB, and its header is line 1: reading it whole costs
+        // nothing that the load itself was not about to spend
+        const { header } = await readNodesetImageInfo(await document.rawBytes());
+        const models: NodesetModel[] = header.models.map((m) => ({
+            modelUri: m.modelUri,
+            version: m.version,
+            publicationDate: m.publicationDate ? new Date(m.publicationDate) : getMinOPCUADate(),
+            requiredModel: m.requiredModels.map((r) => ({
+                ...r,
+                publicationDate: new Date(r.publicationDate ?? Number.NaN)
+            }))
+        }));
+        if (models.length === 0 && header.namespaceUris.length >= 1) {
+            models.push({
+                modelUri: header.namespaceUris[0],
+                version: "1",
+                publicationDate: getMinOPCUADate(),
+                requiredModel: []
+            });
+        }
+        return { models, namespaceUris: header.namespaceUris };
+    },
+    async *records(document: NodesetDocument, _options?: NodeSetLoaderOptions): AsyncGenerator<NodesetRecord> {
+        yield* imageLinesToRecords(await document.lines());
+    }
+};
+
+/**
+ * whether this line is the header record of an NDJSON image.
+ *
+ * Deliberately a string test before it is a parse: the discriminating fields are written first
+ * and the line can be hundreds of kilobytes, so there is no reason to build an object to answer
+ * a question the first thirty characters already settle. The parse is the confirmation, not the
+ * test, and it is only reached for a line that already looks like ours.
+ */
+export function isNdjsonHeaderLine(line: string): boolean {
+    const start = line.slice(0, 200);
+    if (!start.startsWith("{") || !start.includes('"kind"') || !start.includes('"header"')) {
+        return false;
+    }
+    try {
+        const parsed = JSON.parse(line) as { kind?: unknown; schema?: unknown };
+        return parsed?.kind === "header" && typeof parsed.schema === "number";
+    } catch {
+        return false;
+    }
+}
+
+let registered = false;
+
+/** register the built-in formats; called by the loader, and safe to call more than once */
+export function registerBuiltinNodesetFormats(): void {
+    if (registered) return;
+    registered = true;
+    registerNodesetFormat(nodeset2XmlFormat);
+    registerNodesetFormat(ndjsonImageFormat);
+}
+
+registerBuiltinNodesetFormats();

package/api/loader/nodeset_format.ts

@@ -0,0 +1,153 @@
+/**
+ * @module node-opcua-address-space
+ *
+ * Which serialisation a nodeset document is written in, and who can read it.
+ *
+ * The loader is built as `producer -> NodesetRecord -> consumer`: whatever a document is written
+ * in, it becomes a stream of records, and everything downstream of that -- the address space, the
+ * NodeSet2 XML writer, the image writer -- neither knows nor asks. What was missing is the other
+ * half of that seam. The *choice* of producer was a hard-coded pair, XML or image, decided on one
+ * byte, so a third serialisation had no way in short of editing the loader.
+ *
+ * A {@link NodesetFormat} is that way in. It answers three questions and nothing else: are these
+ * bytes mine, what does this document depend on, and what are its records. Formats are found by
+ * sniffing the document, never by its file name: an extension is a claim about a file, and the
+ * loader reads sources that are not files at all.
+ *
+ * Two formats are built in and registered by the loader itself, so the default behaviour is
+ * exactly what it always was: NodeSet2 XML, and the NDJSON image. Anything else is registered by
+ * the code that implements it, the way the Node.js entry point installs its zlib codecs.
+ */
+
+import type { RequiredModel } from "node-opcua-address-space-base";
+import type { NodeSetLoaderOptions } from "../interfaces/nodeset_loader_options.js";
+import type { NodesetRecord } from "./nodeset_record.js";
+import type { NodesetChunk } from "./nodeset_source.js";
+
+/** a model a document defines, together with the models it says it needs */
+export interface NodesetModel extends RequiredModel {
+    requiredModel: RequiredModel[];
+}
+
+/**
+ * what a document says about itself before its body is read: enough, and only enough, to sort a
+ * set of documents into the order they have to be loaded in.
+ */
+export interface NodesetModelInfo {
+    namespaceUris: string[];
+    models: NodesetModel[];
+}
+
+/**
+ * the beginning of a document, for {@link NodesetFormat.sniff}.
+ *
+ * `text` is inflated when the document arrived gzipped, so a sniffer never has to care whether it
+ * is looking at a compressed file: compression is a property of how a document was stored, never
+ * of what it says. It is decoded as UTF-8, which is lossless for the text formats and, for a
+ * binary container, faithful across the ASCII and NUL runs that container formats put their magic
+ * in (a tar header, for instance, is ASCII and NUL for its whole first block).
+ */
+export interface NodesetHead {
+    /** the first bytes of the document, decoded as UTF-8, inflated if the document was gzipped */
+    readonly text: string;
+    /** `text` up to the first line feed: what the line-delimited formats decide on */
+    readonly firstLine: string;
+    /** whether the document was gzip on the wire */
+    readonly gzip: boolean;
+    /** the source's name, for diagnostics only; never a basis for the decision */
+    readonly name: string;
+}
+
+/**
+ * a document handed to a format, in whichever shape that format finds convenient. Every accessor
+ * inflates first when the document was gzipped, and the results are computed once and shared, so
+ * a format that asks for both the lines and the bytes pays for one inflate.
+ */
+export interface NodesetDocument {
+    readonly name: string;
+    /** the whole document, inflated */
+    bytes(): Promise<Uint8Array>;
+    /** the whole document as text, inflated and decoded */
+    text(): Promise<string>;
+    /** the document's lines, for the line-delimited formats */
+    lines(): Promise<string[]>;
+    /** the bytes exactly as they arrived, still compressed if they were */
+    rawBytes(): Promise<Uint8Array>;
+    /** the document as it arrives, for a format that parses incrementally rather than whole */
+    chunks(): AsyncIterable<NodesetChunk>;
+    /**
+     * read from the start until `complete(text)` holds, or the document ends. What was read stays
+     * in the document and is delivered again, first, by {@link NodesetDocument.chunks}, so a
+     * format that reads a header this way has not spent the source.
+     *
+     * This is how a format avoids reading a body it does not need: the XML header pre-pass stops
+     * after a few hundred bytes of a four megabyte file.
+     */
+    readHead(complete: (text: string) => boolean): Promise<string>;
+    /** the SHA-256 of the bytes as they arrived, when the loader was asked to compute one */
+    digest(): string | undefined;
+}
+
+/** a serialisation the loader can read */
+export interface NodesetFormat {
+    /** how this format is named in diagnostics and in the tools; unique among registered formats */
+    readonly name: string;
+    /**
+     * the file extensions this format is usually written with, lowercase and dotted.
+     * Documentation and tooling only: the loader never decides on a name.
+     */
+    readonly extensions?: readonly string[];
+    /**
+     * which format wins when more than one claims a document. Higher is stronger; the default is
+     * 0. NodeSet2 XML registers at -100 because it is the fallback: it claims anything no other
+     * format recognised, which is the behaviour the loader has always had.
+     */
+    readonly priority?: number;
+    /** whether this format claims the document that begins like this */
+    sniff(head: NodesetHead): boolean;
+    /** what the document depends on, read without parsing its body */
+    readModels(document: NodesetDocument): Promise<NodesetModelInfo>;
+    /** the document's records, in document order */
+    records(document: NodesetDocument, options?: NodeSetLoaderOptions): AsyncIterable<NodesetRecord>;
+}
+
+const formats = new Map<string, NodesetFormat>();
+
+/**
+ * make a format available to the loader. Returns a function that removes it again, which is what
+ * a test uses to put the registry back as it found it; production code registers once, at import.
+ *
+ * Registering a name twice replaces the earlier format, so a caller can override a built-in.
+ */
+export function registerNodesetFormat(format: NodesetFormat): () => void {
+    formats.set(format.name, format);
+    return () => {
+        if (formats.get(format.name) === format) {
+            formats.delete(format.name);
+        }
+    };
+}
+
+/** every registered format, strongest claim first */
+export function nodesetFormats(): readonly NodesetFormat[] {
+    return [...formats.values()].sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
+}
+
+/** the registered format of that name, if any */
+export function nodesetFormatByName(name: string): NodesetFormat | undefined {
+    return formats.get(name);
+}
+
+/**
+ * which format claims a document that begins like this, or `undefined` when none does. Formats
+ * are asked strongest first, so a specific format is always given the chance to claim a document
+ * before the XML fallback sees it.
+ */
+export function findNodesetFormat(head: NodesetHead): NodesetFormat | undefined {
+    for (const format of nodesetFormats()) {
+        if (format.sniff(head)) {
+            return format;
+        }
+    }
+    return undefined;
+}

package/api/loader/nodeset_image.ts

@@ -31,9 +31,24 @@ import type { NodesetChunk, NodesetChunkStream } from "./nodeset_source.js";
 
 export { NodesetImageError, type NodesetImageHeader, type NodesetImageTrailer } from "./nodeset_image_codec.js";
 
-/** the two bytes every gzip stream starts with: how an image is told from XML */
+const GZIP_MAGIC_0 = 0x1f;
+const GZIP_MAGIC_1 = 0x8b;
+/** `{`: the first byte of a document whose header record was not compressed */
+const OPEN_BRACE = 0x7b;
+
+/** whether these bytes are gzip, and so need inflating before they are text */
+export function isGzip(firstBytes: Uint8Array): boolean {
+    return firstBytes.length >= 2 && firstBytes[0] === GZIP_MAGIC_0 && firstBytes[1] === GZIP_MAGIC_1;
+}
+
+/**
+ * a NodeSet-NDJSON document, compressed or not, told from NodeSet2 XML by its first byte: a gzip
+ * stream starts `1f 8b`, an XML document `<`, and an uncompressed document `{`. Compression is a
+ * property of how a document was stored, never of what it says, so a reader that took only the
+ * compressed form would be refusing the same document written plainly.
+ */
 export function isNodesetImage(firstBytes: Uint8Array): boolean {
-    return firstBytes.length >= 2 && firstBytes[0] === 0x1f && firstBytes[1] === 0x8b;
+    return isGzip(firstBytes) || (firstBytes.length >= 1 && firstBytes[0] === OPEN_BRACE);
 }
 
 async function toReadableStream(source: Uint8Array | NodesetChunkStream): Promise<ReadableStream<Uint8Array>> {
@@ -59,11 +74,28 @@ async function toReadableStream(source: Uint8Array | NodesetChunkStream): Promis
     });
 }
 
-async function gzip(text: string): Promise<Uint8Array> {
+/** how the text of an image is compressed; see {@link setImageDeflater} */
+export type ImageDeflater = (text: string) => Promise<Uint8Array>;
+
+const deflateWithCompressionStream: ImageDeflater = async (text) => {
     const compressed = new Blob([text])
         .stream()
         .pipeThrough(new CompressionStream("gzip") as unknown as ReadableWritablePair<Uint8Array, Uint8Array>);
     return new Uint8Array(await new Response(compressed).arrayBuffer());
+};
+
+let gzip: ImageDeflater = deflateWithCompressionStream;
+
+/**
+ * how an image is compressed: CompressionStream everywhere by default, which offers no way to ask
+ * for a compression level and so writes at the default one; the Node.js entry point installs zlib
+ * at level 9, four percent smaller on the published nodesets for no change to the reader -- every
+ * gzip stream inflates the same whatever level wrote it. Returns the deflater installed before.
+ */
+export function setImageDeflater(deflater: ImageDeflater): ImageDeflater {
+    const previous = gzip;
+    gzip = deflater;
+    return previous;
 }
 
 /**
@@ -115,7 +147,8 @@ export function inflatedImageLines(image: Uint8Array): Promise<string[]> {
         lines = (async () => {
             let text: string;
             try {
-                text = await inflateImage(image);
+                // an uncompressed document is already its own text; only gzip needs inflating
+                text = isGzip(image) ? await inflateImage(image) : new TextDecoder("utf-8").decode(image);
             } catch (err) {
                 throw new NodesetImageError(`the image cannot be inflated: ${(err as Error).message}`);
             }