layout-data 0.2.0 → 0.3.0

package/src/layout.js

@@ -0,0 +1,303 @@
+import { readFileSync } from "node:fs";
+import { parseDocument } from "yaml";
+import { CODECS } from "./codecs.js";
+import { register, registered_of, set_compiled } from "./registry.js";
+import { build_structure, validate_structure, TYPE_BITS } from "./structure.js";
+
+/** @import { Layout_Source, Registered_Layouts, Layout } from "./typedefs.js" */
+/** @import { Data_Struct } from "./internal_types.js" */
+
+/**
+ * The canonical text of a value: object keys sorted, `comment` left out (it
+ * does not change how a layout works) and the type names in lower case.
+ */
+function canonical(value, key) {
+	if (Array.isArray(value)) return `[${value.map((v) => canonical(v)).join(",")}]`;
+	if (value !== null && typeof value === "object") {
+		const fields = Object.keys(value).filter((k) => k !== "comment" && value[k] !== undefined).sort();
+		return `{${fields.map((k) => `${JSON.stringify(k)}:${canonical(value[k], k)}`).join(",")}}`;
+	}
+	if (key === "type" && typeof value === "string") return JSON.stringify(value.toLowerCase());
+	return JSON.stringify(value) ?? "null";
+}
+
+/**
+ * Compiles an expanded structure (the result of build_structure) into a Layout
+ * (doc/design.md §3) and registers it under its name, the one source of the
+ * layouts an Ldata is created from. Registering a structure whose canonical
+ * text equals the registered one of that name gives the registered Layout, so
+ * the same layout described twice is compiled once; another structure under a
+ * used name throws. Throws on an unknown type or a duplicated tag name; the
+ * other layout rules are checked by validate_structure. Internal.
+ *
+ * @param {Data_Struct} data_struct
+ * @returns {Layout}
+ * @throws {Error} When the name is used by another layout.
+ */
+export function register_structure(data_struct) {
+	const { name } = data_struct;
+	if (typeof name !== "string" || name === "") throw new Error("a layout needs a name");
+	const key = key_of(data_struct);
+	const known = registered_of(name);
+	if (known) {
+		if (known.key !== key) throw new Error(`another layout is already named ${name}`);
+		return known.layout;
+	}
+	const layout = compile_structure(data_struct);
+	register(name, { key, layout });
+	return layout;
+}
+
+function key_of(data_struct) {
+	return canonical({ name: data_struct.name, length: data_struct.length, items: data_struct.items });
+}
+
+function compile_structure(data_struct) {
+	const size = data_struct.length >> 3;
+	const entries = new Map();
+	const tags_at_byte = Array.from({ length: size }, () => []);
+	for (const item of data_struct.items) {
+		const { name, offset, init_value } = item;
+		const type = item.type.toLowerCase();
+		const codec = CODECS[type];
+		const fail = (message) => {
+			throw new Error(`${data_struct.name}.${name}: ${message}`);
+		};
+		if (!codec) fail(`unknown tag type ${item.type}`);
+		if (entries.has(name)) fail("duplicated tag name");
+		// defensive checks of what an Ldata relies on; validate_structure reports them all
+		if (type !== "bool" && offset % 8 !== 0) fail(`bit offset ${offset % 8} on a ${item.type}`);
+		if (codec.size > 1 && (offset >> 3) % 2 !== 0) fail(`${item.type} at the odd byte ${offset >> 3}`);
+		if (!["number", "boolean"].includes(typeof init_value)) fail(`init_value ${init_value} is not a number or boolean`);
+		const byte = offset >> 3;
+		const entry = {
+			name, type, init_value, byte,
+			index: entries.size,
+			size: codec.size,
+			mask: type === "bool" ? 1 << (offset % 8) : 0xff,
+			codec,
+		};
+		entries.set(name, entry);
+		// entries keep the item order: a combined word comes before its bits
+		for (let i = byte; i < byte + codec.size && i < size; i++) tags_at_byte[i].push(entry);
+	}
+
+	// the public layout holds the name, the sizes and frozen copies of the items;
+	// the entries and the lookup table stay out of reach (doc/spec-data-api.md §6.6)
+	const layout = Object.freeze({
+		name: data_struct.name,
+		length: data_struct.length,
+		size,
+		items: Object.freeze(data_struct.items.map(public_tag)),
+	});
+	set_compiled(layout, { entries, tags_at_byte });
+	return layout;
+}
+
+/**
+ * The frozen description of a tag of the public layout: the documented fields
+ * only, the length filled in from the type when the item has none.
+ */
+function public_tag(item) {
+	const tag = {
+		name: item.name,
+		type: item.type,
+		offset: item.offset,
+		length: item.length ?? TYPE_BITS[item.type.toLowerCase()],
+		init_value: item.init_value,
+	};
+	if (item.comment !== undefined) tag.comment = item.comment;
+	tag.coupling = Object.freeze([...(item.coupling ?? [])]);
+	return Object.freeze(tag);
+}
+
+/**
+ * Resolves the fragment references of a list of items: `coupling` and
+ * `includes` given as a fragment name are replaced by a copy of that
+ * fragment, recursively. Omitted lengths are filled in from the type.
+ */
+function resolve_items(list, fragments, stack, where, errors) {
+	if (!Array.isArray(list)) {
+		errors.push(`${where}: expected a list of items`);
+		return [];
+	}
+	const resolve_ref = (ref, field) => {
+		if (Array.isArray(ref)) return resolve_items(ref, fragments, stack, where, errors);
+		if (typeof ref !== "string") {
+			errors.push(`${where}: ${field} must be a fragment name or a list of items`);
+			return [];
+		}
+		if (stack.includes(ref)) {
+			errors.push(`${where}: circular fragment reference ${[...stack, ref].join(" -> ")}`);
+			return [];
+		}
+		if (!Object.hasOwn(fragments, ref)) {
+			errors.push(`${where}: unknown fragment ${ref}`);
+			return [];
+		}
+		return resolve_items(fragments[ref], fragments, [...stack, ref], `fragment ${ref}`, errors);
+	};
+	return list.map((_item) => {
+		const item = { ..._item };
+		if (item.coupling !== undefined) item.coupling = resolve_ref(item.coupling, "coupling");
+		if (item.includes !== undefined) item.includes = resolve_ref(item.includes, "includes");
+		const bits = TYPE_BITS[item.type?.toLowerCase()];
+		if (item.length === undefined && bits !== undefined) item.length = bits;
+		return item;
+	});
+}
+
+/**
+ * Reads the sources of the layouts (doc/design.md §3), merges their layouts
+ * and fragments into one description and registers it with
+ * register_layout_data. Each source is the path of a YAML layout file or a
+ * parsed description; a list of sources is merged, so a layout may name a
+ * fragment of any source whatever the order. The root key `layouts` of a
+ * source maps layout names to item lists; every other root key is a fragment
+ * referenced by name.
+ *
+ * @param {Layout_Source | Layout_Source[]} sources
+ * @returns {Registered_Layouts}
+ * @throws {Error} With `errors`, every violation found, when the sources are not valid.
+ */
+export function register_sources(sources) {
+	const errors = [];
+	const list = Array.isArray(sources) ? sources : [sources];
+	if (list.length === 0) errors.push("no source is given");
+	const origins = [];
+	const layout_defs = {};
+	const fragment_defs = {};
+	// the origin of each layout and fragment, to name both in a duplicate
+	const owners = { layout: new Map(), fragment: new Map() };
+	const add = (origin, root) => {
+		origins.push(origin);
+		if (root === null || typeof root !== "object" || Array.isArray(root)) {
+			errors.push(`${origin}: the root must be a mapping`);
+			return;
+		}
+		const { layouts, ...fragments } = root;
+		if (layouts !== undefined && (layouts === null || typeof layouts !== "object" || Array.isArray(layouts))) {
+			errors.push(`${origin}: the root key \`layouts\` must map layout names to item lists`);
+			return;
+		}
+		const merge = (kind, defs, into) => {
+			for (const [name, value] of Object.entries(defs)) {
+				if (owners[kind].has(name)) {
+					errors.push(`${kind} ${name} is defined in ${owners[kind].get(name)} and ${origin}`);
+					continue;
+				}
+				owners[kind].set(name, origin);
+				into[name] = value;
+			}
+		};
+		merge("layout", layouts ?? {}, layout_defs);
+		merge("fragment", fragments, fragment_defs);
+	};
+	list.forEach((source, index) => {
+		const label = Array.isArray(sources) ? `layout description #${index + 1}` : "layout description";
+		if (Array.isArray(source)) {
+			errors.push(`${label}: a list of sources must not contain a list`);
+		} else if (typeof source === "string") {
+			let text;
+			try {
+				text = readFileSync(source, "utf8");
+			} catch (error) {
+				errors.push(error.message);
+				return;
+			}
+			const doc = parseDocument(text, { version: "1.2" });
+			if (doc.errors.length > 0) errors.push(...doc.errors.map((e) => `${source}: ${e.message}`));
+			else add(source, doc.toJS());
+		} else {
+			add(label, source);
+		}
+	});
+	if (errors.length === 0 && Object.keys(layout_defs).length === 0) errors.push("no layout is defined in the sources");
+	if (errors.length > 0) throw layout_error("layout sources", errors);
+	return register_layout_data({ layouts: layout_defs, ...fragment_defs }, origins.join(", "));
+}
+
+/** Set once a call of register_layout has registered the layouts. */
+let registered_once = false;
+
+/**
+ * Registers the layouts of the sources, once per process: a layout is
+ * described, registered under its name, and then used by that name
+ * (doc/spec-data-api.md §6.4). Nothing is registered when a source is not
+ * valid, and such a call does not use up the single call.
+ *
+ * @param {Layout_Source | Layout_Source[]} sources - The path of a YAML layout file, a parsed description, or a list of them.
+ * @returns {Registered_Layouts}
+ * @throws {Error} With `errors`, every violation found, when the sources are not valid.
+ * @throws {Error} When the layouts were registered by an earlier call.
+ */
+export function register_layout(sources) {
+	if (registered_once) throw new Error("register_layout can be called only once");
+	const result = register_sources(sources);
+	registered_once = true;
+	return result;
+}
+
+function layout_error(subject, errors) {
+	const error = new Error(`invalid ${subject}:\n${errors.join("\n")}`);
+	error.errors = errors;
+	return error;
+}
+
+/**
+ * The conflicts of structures with the layouts registered under their names.
+ *
+ * @param {Record<string, Data_Struct>} structs - By layout name.
+ * @returns {string[]}
+ */
+function find_conflicts(structs) {
+	const conflicts = [];
+	for (const [name, struct] of Object.entries(structs)) {
+		const known = registered_of(name);
+		if (known && known.key !== key_of(struct)) conflicts.push(`layout ${name}: another layout is already named ${name}`);
+	}
+	return conflicts;
+}
+
+/**
+ * Registers the layouts of one description that is parsed and complete: the
+ * root key `layouts` and the fragments the layouts name. Internal.
+ *
+ * @param {Record<string, any>} root - The parsed description: `layouts` and the fragments.
+ * @param {string} [origin] - Names the source in errors.
+ * @returns {Registered_Layouts}
+ * @throws {Error} With `errors`, every violation found, when the description is not valid.
+ */
+export function register_layout_data(root, origin = "the given object") {
+	const fail = (errors) => {
+		throw layout_error(`layout description from ${origin}`, errors);
+	};
+	if (root === null || typeof root !== "object" || Array.isArray(root)) fail(["the root must be a mapping"]);
+	const { layouts: layout_defs, ...fragment_defs } = root;
+	if (layout_defs === null || typeof layout_defs !== "object" || Array.isArray(layout_defs)) {
+		fail(["the root key `layouts` must map layout names to item lists"]);
+	}
+
+	const errors = [];
+	const fragments = {};
+	for (const [name, list] of Object.entries(fragment_defs)) {
+		fragments[name] = resolve_items(list, fragment_defs, [name], `fragment ${name}`, errors);
+	}
+	const structs = {};
+	for (const [name, list] of Object.entries(layout_defs)) {
+		const items = resolve_items(list, fragment_defs, [], `layout ${name}`, errors);
+		structs[name] = build_structure({ name }, items);
+	}
+	if (errors.length === 0) {
+		for (const struct of Object.values(structs)) errors.push(...validate_structure(struct));
+	}
+	if (errors.length === 0) {
+		// all or none: a conflict is found before any of the layouts is registered
+		errors.push(...find_conflicts(structs));
+	}
+	if (errors.length > 0) fail(errors);
+
+	const layouts = {};
+	for (const [name, struct] of Object.entries(structs)) layouts[name] = register_structure(struct);
+	return { layouts, fragments };
+}

package/src/observer.js

@@ -0,0 +1,147 @@
+import { Buffer } from "node:buffer";
+
+/** @import { Compiled_Layout, Layout_Entry } from "./internal_types.js" */
+
+// a change made by a change handler is detected in a further round
+const MAX_DIFF_ROUNDS = 16;
+
+const same_value = (a, b) => a === b || (Number.isNaN(a) && Number.isNaN(b));
+
+/**
+ * Detects the changes of an Ldata buffer by comparing it with a snapshot
+ * (doc/design.md §5.1) and reports each changed tag once.
+ *
+ * Every write path of the Ldata registers the byte range it wrote with
+ * touch(), then calls diff() (in a transaction: at its commit). Only the
+ * registered range is compared, the other bytes equal the snapshot by
+ * construction. A diff() called while the changes are being reported, by a
+ * handler writing a tag, is only recorded; the outer diff() runs another
+ * round once the current changes are reported.
+ */
+export class Observer {
+	/** @type {Buffer} */
+	#buffer;
+	/** @type {string} */
+	#name;
+	/** @type {Compiled_Layout} */
+	#compiled;
+	/** @type {Buffer} */
+	#snapshot;
+	/** @type {(entry: Layout_Entry, old_value: any, new_value: any, errors: unknown[]) => void} */
+	#report;
+	#diffing = false;
+	#pending = false;
+	/** The byte range [#start, #end) written since the last collection. */
+	#start = Number.POSITIVE_INFINITY;
+	#end = 0;
+	/** @type {Uint32Array} */
+	#seen;
+	#mark = 0;
+
+	/**
+	 * @param {Buffer} buffer - The buffer of the Ldata.
+	 * @param {string} name - The layout name, for errors.
+	 * @param {Compiled_Layout} compiled - The compiled data of the layout of the Ldata.
+	 * @param {(entry: Layout_Entry, old_value: any, new_value: any, errors: unknown[]) => void} report - Called for each change; what a handler throws goes to `errors`.
+	 */
+	constructor(buffer, name, compiled, report) {
+		this.#buffer = buffer;
+		this.#name = name;
+		this.#compiled = compiled;
+		this.#report = report;
+		this.#snapshot = Buffer.from(buffer);
+		this.#seen = new Uint32Array(compiled.entries.size);
+	}
+
+	/**
+	 * Registers the byte range [start, end) as written.
+	 *
+	 * @param {number} start
+	 * @param {number} end - Exclusive.
+	 */
+	touch(start, end) {
+		if (start >= end) return;
+		if (start < this.#start) this.#start = start;
+		if (end > this.#end) this.#end = end;
+	}
+
+	/**
+	 * Forgets the registered range: the buffer was restored to the snapshot.
+	 */
+	discard() {
+		this.#start = Number.POSITIVE_INFINITY;
+		this.#end = 0;
+	}
+
+	/**
+	 * Reports every tag changed since the last diff. Nothing is thrown: what
+	 * the handlers throw, and the exception of changes that do not settle, are
+	 * pushed to `errors` after every change has been reported (doc/spec-data-api.md §4.4).
+	 *
+	 * @param {unknown[]} errors
+	 */
+	diff(errors) {
+		if (this.#diffing) {
+			this.#pending = true;
+			return;
+		}
+		this.#diffing = true;
+		try {
+			let rounds = 0;
+			do {
+				this.#pending = false;
+				if (++rounds > MAX_DIFF_ROUNDS) {
+					errors.push(new Error(`${this.#name}: changes do not settle after ${MAX_DIFF_ROUNDS} rounds`));
+					break;
+				}
+				for (const [entry, old_value, new_value] of this.#collect()) {
+					this.#report(entry, old_value, new_value, errors);
+				}
+			} while (this.#pending);
+		} finally {
+			this.#diffing = false;
+		}
+	}
+
+	/**
+	 * Collects the changed tags, in layout order, and updates the snapshot.
+	 * Old values are decoded from the snapshot, so no value is cached.
+	 */
+	#collect() {
+		const buffer = this.#buffer;
+		const snapshot = this.#snapshot;
+		const tags_at_byte = this.#compiled.tags_at_byte;
+		// entries already checked in this collection, by entry index
+		const seen = this.#seen;
+		if (this.#mark === 0xffffffff) {
+			// the marks wrap around: clear them
+			seen.fill(0);
+			this.#mark = 0;
+		}
+		const mark = ++this.#mark;
+		const changes = [];
+		let in_order = true;
+		let last_index = -1;
+		// a write during the reporting registers its range for the next round
+		const start = this.#start;
+		const end = Math.min(this.#end, buffer.length);
+		this.discard();
+		for (let i = start; i < end; i++) {
+			if (buffer[i] === snapshot[i]) continue;
+			for (const entry of tags_at_byte[i]) {
+				if (seen[entry.index] === mark) continue;
+				seen[entry.index] = mark;
+				const { codec, byte, mask } = entry;
+				const old_value = codec.get(snapshot, byte, mask);
+				const new_value = codec.get(buffer, byte, mask);
+				if (same_value(old_value, new_value)) continue;
+				changes.push([entry, old_value, new_value]);
+				if (entry.index < last_index) in_order = false;
+				last_index = entry.index;
+			}
+		}
+		if (start < end) buffer.copy(snapshot, start, start, end);
+		if (!in_order) changes.sort((a, b) => a[0].index - b[0].index);
+		return changes;
+	}
+}

package/src/registry.js

@@ -0,0 +1,83 @@
+// The registry of the compiled layouts, by layout name (doc/design.md §3).
+// A name identifies one layout: it is the only source of the layouts an Ldata
+// is created from. Written by register_structure (layout.js), read by Ldata.
+
+/** @import { Layout } from "./typedefs.js" */
+/** @import { Compiled_Layout } from "./internal_types.js" */
+
+/**
+ * @typedef {object} Registered
+ * @property {string} key - The canonical text of the structure, to tell the same layout from another one.
+ * @property {Layout} layout
+ */
+
+/** @type {Map<string, Registered>} */
+const registered = new Map();
+
+/**
+ * The compiled data of the layouts (doc/design.md §3), kept here so that the
+ * public Layout holds none of it.
+ *
+ * @type {WeakMap<Layout, Compiled_Layout>}
+ */
+const compiled = new WeakMap();
+
+/**
+ * @param {Layout} layout
+ * @param {Compiled_Layout} data
+ */
+export function set_compiled(layout, data) {
+	compiled.set(layout, data);
+}
+
+/**
+ * The compiled data of a layout made by this package.
+ *
+ * @param {Layout} layout
+ * @returns {Compiled_Layout}
+ * @throws {Error} For an object that is not a compiled layout.
+ */
+export function compiled_of(layout) {
+	const data = compiled.get(layout);
+	if (!data) throw new Error(`${layout?.name} is not a registered layout`);
+	return data;
+}
+
+/**
+ * @param {string} name
+ * @returns {Registered | undefined}
+ */
+export function registered_of(name) {
+	return registered.get(name);
+}
+
+/**
+ * @param {string} name
+ * @param {Registered} entry
+ */
+export function register(name, entry) {
+	registered.set(name, entry);
+}
+
+/**
+ * The registered layout of a name, undefined when there is none.
+ *
+ * @param {string} name
+ * @returns {Layout | undefined}
+ */
+export function find_layout(name) {
+	return registered.get(name)?.layout;
+}
+
+/**
+ * The registered layout of a name (doc/spec-data-api.md §6.5).
+ *
+ * @param {string} name
+ * @returns {Layout}
+ * @throws {Error} When no layout of that name is registered.
+ */
+export function get_layout(name) {
+	const layout = find_layout(name);
+	if (!layout) throw new Error(`unknown layout ${name}`);
+	return layout;
+}

package/src/reserved.js

@@ -0,0 +1,35 @@
+// The reserved names (doc/spec-data-api.md §2.10, doc/design.md §4). Internal:
+// index.js does not export this file.
+
+/**
+ * Names a tag must not have: the members of Ldata, the method and field names
+ * of EventEmitter in Node.js 20.11 and the names of Object.prototype. A static
+ * list, so that it does not depend on the Node.js version that runs the program.
+ */
+const RESERVED_NAMES = new Set([
+	// members of Ldata
+	"name", "layout", "size", "get", "transaction",
+	"read_bytes", "write_bytes", "read_bit", "write_bit", "load_tag", "store_tag",
+	// EventEmitter
+	"addListener", "emit", "eventNames", "getMaxListeners", "listenerCount", "listeners",
+	"off", "on", "once", "prependListener", "prependOnceListener", "rawListeners",
+	"removeAllListeners", "removeListener", "setMaxListeners",
+	"_events", "_eventsCount", "_maxListeners",
+	// Object
+	"constructor", "hasOwnProperty", "isPrototypeOf", "propertyIsEnumerable",
+	"toLocaleString", "toString", "valueOf", "__proto__",
+	"__defineGetter__", "__defineSetter__", "__lookupGetter__", "__lookupSetter__",
+]);
+
+/** Event names of the library; a user does not emit them or use them for its own events. */
+export const RESERVED_EVENTS = Object.freeze(["change", "write", "access", "read"]);
+
+/**
+ * Whether a tag name would shadow a member of a data block.
+ *
+ * @param {string} name
+ * @returns {boolean}
+ */
+export function is_reserved_name(name) {
+	return RESERVED_NAMES.has(name);
+}