al-sem 0.0.1

package/src/parser/native/parse-index-worker.ts

@@ -0,0 +1,94 @@
+// src/parser/native/parse-index-worker.ts
+// Worker entry: parse + index a single AL source unit. The native parser and the JS
+// indexers are CPU-bound; spreading them across a worker pool yields near-linear
+// scaling on multi-core hosts when the workload is many independent files (e.g.
+// Microsoft Base Application's 7,634 .al sources).
+//
+// Each worker holds its own native parser instance (each `Parser` wrapper instance
+// owns an independent TSParser handle via `al_shim_parser_new`). Trees never cross
+// the worker boundary — they're created, walked, and freed inside the worker. Only
+// plain-JS results (objects, tables, routines, diagnostics) are postMessaged back.
+
+import { sha256Hex } from "../../hash.ts";
+import { indexObjects } from "../../index/object-indexer.ts";
+import { indexRoutines } from "../../index/routine-indexer.ts";
+import type { ObjectDecl, Table } from "../../model/entities.ts";
+import { NativeParserUnavailableError, parseALSource } from "../parser-init.ts";
+
+declare const self: Worker;
+
+interface WorkerRequest {
+	jobId: number;
+	relativePath: string;
+	content: string;
+	appGuid: string;
+	sourceUnitId: string;
+	modelInstanceId: string;
+}
+
+interface WorkerResultOk {
+	jobId: number;
+	relativePath: string;
+	ok: true;
+	objects: ObjectDecl[];
+	tables: Table[];
+	routines: ReturnType<typeof indexRoutines>;
+}
+
+interface WorkerResultError {
+	jobId: number;
+	relativePath: string;
+	ok: false;
+	errorKind: "parser-unavailable" | "parse-error";
+	errorMessage: string;
+}
+
+export type WorkerResult = WorkerResultOk | WorkerResultError;
+
+self.onmessage = async (event: MessageEvent<WorkerRequest>) => {
+	const { jobId, relativePath, content, appGuid, sourceUnitId, modelInstanceId } = event.data;
+	try {
+		const tree = await parseALSource(content);
+		try {
+			const sourceHash = sha256Hex(content);
+			const objResults = indexObjects({ tree, appGuid, sourceUnitId, modelInstanceId, sourceHash });
+			const objects: ObjectDecl[] = [];
+			const tables: Table[] = [];
+			let routines: ReturnType<typeof indexRoutines> = [];
+			for (const objResult of objResults) {
+				if (!objResult.object || !objResult.objectNode) continue;
+				objects.push(objResult.object);
+				if (objResult.table) tables.push(objResult.table);
+				routines = routines.concat(
+					indexRoutines({
+						objectNode: objResult.objectNode,
+						object: objResult.object,
+						sourceUnitId,
+						modelInstanceId,
+					}),
+				);
+			}
+			const result: WorkerResultOk = {
+				jobId,
+				relativePath,
+				ok: true,
+				objects,
+				tables,
+				routines,
+			};
+			self.postMessage(result);
+		} finally {
+			tree.delete();
+		}
+	} catch (err) {
+		const isParserDown = err instanceof NativeParserUnavailableError;
+		const result: WorkerResultError = {
+			jobId,
+			relativePath,
+			ok: false,
+			errorKind: isParserDown ? "parser-unavailable" : "parse-error",
+			errorMessage: (err as Error).message,
+		};
+		self.postMessage(result);
+	}
+};

package/src/parser/native/wrapper.ts

@@ -0,0 +1,353 @@
+// src/parser/native/wrapper.ts
+// JS façade around the al_shim FFI surface. API mirrors the subset of
+// web-tree-sitter's Node/Tree/TreeCursor that al-sem's indexers use today.
+// Memory model:
+//   - Parser handle: process-lifetime (cached in parser-init.ts).
+//   - Tree handle: owned by the Tree wrapper; freed by Tree.delete().
+//     Tree retains the source UTF-8 buffer for byte-accurate Node.text.
+//   - Node: lightweight (Uint8Array of NODE_SIZE bytes + back-ref to Tree).
+//     The wrapper interns numeric `id`s per-tree (collision-free).
+//   - TreeCursor: heap-allocated by shim; freed by TreeCursor.delete().
+
+import { type Pointer, ptr } from "bun:ffi";
+import { NativeParserUnavailableError, loadShim } from "./ffi.ts";
+
+export { NativeParserUnavailableError };
+
+const decoder = new TextDecoder("utf-8");
+const encoder = new TextEncoder();
+
+/**
+ * Per-process cache of UTF-8 encodings for tree-sitter field names. The indexer hits
+ * `childForFieldName` with the same handful of field names ("name", "function", "object",
+ * "member", etc.) millions of times across a workspace; without this cache each call
+ * pays an `encoder.encode()` UTF-8 round-trip + a Uint8Array allocation.
+ */
+const FIELD_NAME_BYTES = new Map<string, Uint8Array>();
+function fieldNameBytes(name: string): Uint8Array {
+	let bytes = FIELD_NAME_BYTES.get(name);
+	if (bytes === undefined) {
+		bytes = encoder.encode(name);
+		FIELD_NAME_BYTES.set(name, bytes);
+	}
+	return bytes;
+}
+
+// Module-level shim handle + NODE_SIZE constant. Lazy-init on first wrapper use.
+let SHIM: ReturnType<typeof loadShim> | null = null;
+let NODE_SIZE = 0;
+function shim(): ReturnType<typeof loadShim> {
+	if (!SHIM) {
+		SHIM = loadShim();
+		NODE_SIZE = SHIM.symbols.al_shim_node_size();
+		if (NODE_SIZE === 0 || NODE_SIZE > 256) {
+			throw new NativeParserUnavailableError(
+				"abi-mismatch",
+				`al-sem: native parser reported invalid TSNode size ${NODE_SIZE}. Library may be wrong version.`,
+			);
+		}
+	}
+	return SHIM;
+}
+
+export class Parser {
+	private handle: Pointer;
+	private disposed = false;
+	constructor() {
+		const s = shim();
+		const h = s.symbols.al_shim_parser_new();
+		if (!h) throw new Error("al-sem: al_shim_parser_new returned null");
+		this.handle = h;
+		const lang = s.symbols.al_shim_language();
+		if (!lang) throw new Error("al-sem: al_shim_language returned null");
+		const ok = s.symbols.al_shim_parser_set_language(h, lang);
+		if (ok === 0) throw new Error("al-sem: al_shim_parser_set_language failed");
+	}
+
+	parse(source: string): Tree {
+		if (this.disposed) throw new Error("Parser is disposed");
+		const s = shim();
+		const bytes = encoder.encode(source);
+		const treePtr = s.symbols.al_shim_parse_utf8(this.handle, ptr(bytes), bytes.byteLength);
+		if (!treePtr) throw new Error("al-sem: parse returned null (tree-sitter halt)");
+		return new Tree(treePtr, bytes);
+	}
+
+	delete(): void {
+		if (this.disposed) return;
+		shim().symbols.al_shim_parser_delete(this.handle);
+		this.disposed = true;
+	}
+}
+
+export class Tree {
+	/** @internal */ readonly handle: Pointer;
+	/** @internal */ readonly sourceBytes: Uint8Array;
+	/** @internal */ readonly nodeIds = new Map<string, number>();
+	/** @internal */ nextId = 1;
+	/** @internal */ disposed = false;
+	private _root: Node | undefined;
+
+	constructor(handle: Pointer, sourceBytes: Uint8Array) {
+		this.handle = handle;
+		this.sourceBytes = sourceBytes;
+	}
+
+	get rootNode(): Node {
+		if (this.disposed) throw new Error("Tree is disposed");
+		if (!this._root) {
+			const buf = new Uint8Array(NODE_SIZE);
+			shim().symbols.al_shim_tree_root_node(this.handle, ptr(buf));
+			this._root = new Node(this, buf);
+		}
+		return this._root;
+	}
+
+	walk(): TreeCursor {
+		return new TreeCursor(this, this.rootNode);
+	}
+
+	delete(): void {
+		if (this.disposed) return;
+		shim().symbols.al_shim_tree_delete(this.handle);
+		this.disposed = true;
+	}
+}
+
+export class Node {
+	/** @internal */ readonly tree: Tree;
+	/** @internal */ readonly buf: Uint8Array;
+	private _children: Node[] | undefined;
+	private _namedChildren: Node[] | undefined;
+	private _id: number | undefined;
+	// FFI-getter caches. The underlying TSNode buffer is immutable for this wrapper's
+	// lifetime, so every getter result is safe to memoize. The hot path (indexRoutines)
+	// re-reads .type millions of times per cold run; caching here removed >1M FFI calls
+	// on Continia Delivery Network alone.
+	private _type: string | undefined;
+	private _startIndex: number | undefined;
+	private _endIndex: number | undefined;
+	private _startPosition: { row: number; column: number } | undefined;
+	private _endPosition: { row: number; column: number } | undefined;
+	private _childCount: number | undefined;
+	private _namedChildCount: number | undefined;
+	private _hasError: boolean | undefined;
+	private _isNamed: boolean | undefined;
+
+	constructor(tree: Tree, buf: Uint8Array) {
+		this.tree = tree;
+		this.buf = buf;
+	}
+
+	private check(): ReturnType<typeof loadShim> {
+		if (this.tree.disposed) throw new Error("Node references a disposed Tree");
+		return shim();
+	}
+
+	get type(): string {
+		if (this._type !== undefined) return this._type;
+		this._type = this.check().symbols.al_shim_node_type(ptr(this.buf))?.toString() ?? "";
+		return this._type;
+	}
+	get startIndex(): number {
+		if (this._startIndex !== undefined) return this._startIndex;
+		this._startIndex = this.check().symbols.al_shim_node_start_byte(ptr(this.buf));
+		return this._startIndex;
+	}
+	get endIndex(): number {
+		if (this._endIndex !== undefined) return this._endIndex;
+		this._endIndex = this.check().symbols.al_shim_node_end_byte(ptr(this.buf));
+		return this._endIndex;
+	}
+	get startPosition(): { row: number; column: number } {
+		if (this._startPosition !== undefined) return this._startPosition;
+		const s = this.check();
+		this._startPosition = {
+			row: s.symbols.al_shim_node_start_row(ptr(this.buf)),
+			column: s.symbols.al_shim_node_start_column(ptr(this.buf)),
+		};
+		return this._startPosition;
+	}
+	get endPosition(): { row: number; column: number } {
+		if (this._endPosition !== undefined) return this._endPosition;
+		const s = this.check();
+		this._endPosition = {
+			row: s.symbols.al_shim_node_end_row(ptr(this.buf)),
+			column: s.symbols.al_shim_node_end_column(ptr(this.buf)),
+		};
+		return this._endPosition;
+	}
+	get text(): string {
+		const start = this.startIndex;
+		const end = this.endIndex;
+		return decoder.decode(this.tree.sourceBytes.subarray(start, end));
+	}
+	/**
+	 * Raw UTF-8 byte slice of the node's source range — no string allocation.
+	 * Use for hashing or byte-level comparison; for human consumption use `.text`.
+	 */
+	get textBytes(): Uint8Array {
+		return this.tree.sourceBytes.subarray(this.startIndex, this.endIndex);
+	}
+	get childCount(): number {
+		if (this._childCount !== undefined) return this._childCount;
+		this._childCount = this.check().symbols.al_shim_node_child_count(ptr(this.buf));
+		return this._childCount;
+	}
+	get namedChildCount(): number {
+		if (this._namedChildCount !== undefined) return this._namedChildCount;
+		this._namedChildCount = this.check().symbols.al_shim_node_named_child_count(ptr(this.buf));
+		return this._namedChildCount;
+	}
+
+	child(i: number): Node | null {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_child(ptr(this.buf), i, ptr(out));
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+	namedChild(i: number): Node | null {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_named_child(ptr(this.buf), i, ptr(out));
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+	childForFieldName(name: string): Node | null {
+		const s = this.check();
+		const nameBytes = fieldNameBytes(name);
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_child_by_field_name(
+			ptr(this.buf),
+			ptr(nameBytes),
+			nameBytes.byteLength,
+			ptr(out),
+		);
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+
+	get children(): Node[] {
+		if (this._children) return this._children;
+		const s = this.check();
+		const n = this.childCount;
+		const out: Node[] = new Array(n);
+		// Tree-sitter contract: child(i) for i < childCount is never null. Skip the
+		// is_null FFI call we'd otherwise pay per child — collapses 2 FFI calls to 1
+		// per child for the millions of children walked during indexing.
+		for (let i = 0; i < n; i++) {
+			const buf = new Uint8Array(NODE_SIZE);
+			s.symbols.al_shim_node_child(ptr(this.buf), i, ptr(buf));
+			out[i] = new Node(this.tree, buf);
+		}
+		this._children = out;
+		return out;
+	}
+	get namedChildren(): Node[] {
+		if (this._namedChildren) return this._namedChildren;
+		const s = this.check();
+		const n = this.namedChildCount;
+		const out: Node[] = new Array(n);
+		// Same dead-is_null elimination as `children` above.
+		for (let i = 0; i < n; i++) {
+			const buf = new Uint8Array(NODE_SIZE);
+			s.symbols.al_shim_node_named_child(ptr(this.buf), i, ptr(buf));
+			out[i] = new Node(this.tree, buf);
+		}
+		this._namedChildren = out;
+		return out;
+	}
+
+	get parent(): Node | null {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_parent(ptr(this.buf), ptr(out));
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+	get previousSibling(): Node | null {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_previous_sibling(ptr(this.buf), ptr(out));
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+	get nextSibling(): Node | null {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_node_next_sibling(ptr(this.buf), ptr(out));
+		return s.symbols.al_shim_node_is_null(ptr(out)) !== 0 ? null : new Node(this.tree, out);
+	}
+	get isNamed(): boolean {
+		if (this._isNamed !== undefined) return this._isNamed;
+		this._isNamed = this.check().symbols.al_shim_node_is_named(ptr(this.buf)) !== 0;
+		return this._isNamed;
+	}
+	get hasError(): boolean {
+		if (this._hasError !== undefined) return this._hasError;
+		this._hasError = this.check().symbols.al_shim_node_has_error(ptr(this.buf)) !== 0;
+		return this._hasError;
+	}
+
+	/** Per-tree numeric interning. Same TSNode bytes in the same tree → same id. */
+	get id(): number {
+		if (this._id !== undefined) return this._id;
+		// Key by binary-string of the buffer (collision-free; fast for ~32-byte buffers).
+		let key = "";
+		for (let i = 0; i < this.buf.byteLength; i++) key += String.fromCharCode(this.buf[i] ?? 0);
+		const existing = this.tree.nodeIds.get(key);
+		if (existing !== undefined) {
+			this._id = existing;
+			return existing;
+		}
+		const id = this.tree.nextId++;
+		this.tree.nodeIds.set(key, id);
+		this._id = id;
+		return id;
+	}
+
+	walk(): TreeCursor {
+		return new TreeCursor(this.tree, this);
+	}
+}
+
+export class TreeCursor {
+	/** @internal */ readonly tree: Tree;
+	private handle: Pointer;
+	private disposed = false;
+
+	constructor(tree: Tree, startNode: Node) {
+		this.tree = tree;
+		const h = shim().symbols.al_shim_cursor_new(ptr(startNode.buf));
+		if (!h) throw new Error("al-sem: al_shim_cursor_new returned null (cursor allocation failed)");
+		this.handle = h;
+	}
+
+	private check(): ReturnType<typeof loadShim> {
+		if (this.disposed) throw new Error("TreeCursor is disposed");
+		if (this.tree.disposed) throw new Error("TreeCursor references a disposed Tree");
+		return shim();
+	}
+
+	gotoFirstChild(): boolean {
+		return this.check().symbols.al_shim_cursor_goto_first_child(this.handle) !== 0;
+	}
+	gotoNextSibling(): boolean {
+		return this.check().symbols.al_shim_cursor_goto_next_sibling(this.handle) !== 0;
+	}
+	gotoParent(): boolean {
+		return this.check().symbols.al_shim_cursor_goto_parent(this.handle) !== 0;
+	}
+	get currentNode(): Node {
+		const s = this.check();
+		const out = new Uint8Array(NODE_SIZE);
+		s.symbols.al_shim_cursor_current_node(this.handle, ptr(out));
+		return new Node(this.tree, out);
+	}
+	get currentFieldName(): string | null {
+		const s = this.check();
+		const cs = s.symbols.al_shim_cursor_current_field_name(this.handle);
+		return cs ? cs.toString() : null;
+	}
+	delete(): void {
+		if (this.disposed) return;
+		shim().symbols.al_shim_cursor_delete(this.handle);
+		this.disposed = true;
+	}
+}

package/src/parser/parser-init.ts

@@ -0,0 +1,43 @@
+// src/parser/parser-init.ts
+// Native parser bootstrap. Lazy-loads the shim, caches the parser handle for
+// the process lifetime, and surfaces a missing/incompatible native library as
+// a single typed NativeParserUnavailableError — consumers (`indexer.ts`,
+// `dependency-pipeline.ts`) catch it and emit one diagnostic instead of
+// thousands.
+
+import { NativeParserUnavailableError, Parser, type Tree } from "./native/wrapper.ts";
+
+export { NativeParserUnavailableError } from "./native/wrapper.ts";
+export type { Tree } from "./native/wrapper.ts";
+
+let cachedParser: Parser | null = null;
+let loadError: NativeParserUnavailableError | null = null;
+
+function ensureParser(): Parser {
+	if (cachedParser) return cachedParser;
+	if (loadError) throw loadError;
+	try {
+		cachedParser = new Parser();
+		return cachedParser;
+	} catch (err) {
+		if (err instanceof NativeParserUnavailableError) {
+			loadError = err;
+			throw err;
+		}
+		// Unexpected: wrap so callers can still pattern-match.
+		loadError = new NativeParserUnavailableError(
+			"missing",
+			`unexpected parser init failure: ${(err as Error).message}`,
+		);
+		throw loadError;
+	}
+}
+
+/**
+ * Parse AL source code into a syntax tree. Native; sync internally; async-shaped
+ * for source-compat with existing callers. Throws `NativeParserUnavailableError`
+ * if the native parser cannot be loaded.
+ */
+export async function parseALSource(source: string): Promise<Tree> {
+	return ensureParser().parse(source);
+}

package/src/perf/profiler.ts

@@ -0,0 +1,66 @@
+// Structured phase profiler — one sink for all the pipeline's phase timers.
+//
+// Two consumers:
+//   1. `AL_SEM_PROFILE=1` — writes each phase to stderr (the existing ad-hoc behaviour).
+//   2. The benchmark harness — calls `collectPhases(fn)` to capture every phase emitted while
+//      `fn` runs as a structured `PhaseSample[]`, with no stderr noise.
+//
+// Zero-cost when neither is active: `makeLap`/`recordPhase` short-circuit on a single boolean
+// check before reading the clock, so leaving the timers in hot loops costs nothing in normal runs.
+
+export interface PhaseSample {
+	/** Hierarchical label, e.g. "analyze:runDetectors", "ingest:Base Application:pool-parse". */
+	label: string;
+	ms: number;
+}
+
+// The active collection sink. Non-null only inside a `collectPhases` call.
+let sink: PhaseSample[] | null = null;
+
+/** True when phases should be recorded (a collector is active OR stderr profiling is on). */
+export function profilingActive(): boolean {
+	return sink !== null || process.env.AL_SEM_PROFILE === "1";
+}
+
+/** Record one phase sample to the active collector and/or stderr. */
+export function recordPhase(label: string, ms: number): void {
+	if (sink !== null) sink.push({ label, ms });
+	if (process.env.AL_SEM_PROFILE === "1") {
+		process.stderr.write(`  [phase] ${label} ${ms.toFixed(0)}ms\n`);
+	}
+}
+
+/**
+ * Make a lap timer. Each call records the elapsed time since the previous lap (or since creation)
+ * under `prefix + label`. No-op (beyond resetting the clock) when profiling is inactive.
+ */
+export function makeLap(prefix = ""): (label: string) => void {
+	let t = performance.now();
+	return (label: string): void => {
+		if (!profilingActive()) {
+			t = performance.now();
+			return;
+		}
+		const now = performance.now();
+		recordPhase(prefix + label, now - t);
+		t = now;
+	};
+}
+
+/**
+ * Run `fn` while collecting every phase it emits into a fresh sink. Returns the result plus the
+ * collected samples. Nestable: restores the outer sink afterward. Used by the benchmark harness.
+ */
+export async function collectPhases<T>(
+	fn: () => Promise<T> | T,
+): Promise<{ result: T; phases: PhaseSample[] }> {
+	const prev = sink;
+	const mine: PhaseSample[] = [];
+	sink = mine;
+	try {
+		const result = await fn();
+		return { result, phases: mine };
+	} finally {
+		sink = prev;
+	}
+}

package/src/policy/policy-default.yaml

@@ -0,0 +1,83 @@
+version: 1
+description: "al-sem bundled default policy — high-precision rules for common Business Central correctness/security concerns."
+defaults:
+  onUnknown: fail-open
+  requireCoverage: partial
+rules:
+  - id: no-commit-in-event-subscribers
+    title: "Event subscribers must not perform Commit"
+    severity: high
+    when:
+      root.kinds: event-subscriber
+      capability.op: commit
+    onUnknown: fail-closed
+    facts: any
+
+  - id: no-commit-in-triggers
+    title: "Database triggers must not perform Commit"
+    severity: high
+    when:
+      root.kinds: trigger-table
+      capability.op: commit
+    onUnknown: fail-closed
+    facts: any
+
+  - id: api-no-interactive-ui
+    title: "API/web-service roots must not use interactive UI"
+    severity: high
+    when:
+      root.kinds: [api-page, web-service-exposed]
+      capability.op: [ui-confirm, ui-message]
+    onUnknown: fail-closed
+    facts: any
+
+  - id: api-no-user-dynamic-dispatch
+    title: "API/web-service roots must not perform user-driven dynamic dispatch"
+    severity: high
+    when:
+      root.kinds: [api-page, web-service-exposed]
+      capability.op: execute
+      capability.confidence: [userDynamic, configDynamic]
+    onUnknown: fail-closed
+    facts: any
+
+  - id: no-http-from-table-triggers
+    title: "Database triggers should not perform HTTP calls"
+    severity: medium
+    when:
+      root.kinds: trigger-table
+      capability.resourceKind: http
+    facts: any
+
+  - id: install-codeunit-no-business-data-writes
+    title: "Install codeunits should not write outside setup/migration tables"
+    severity: medium
+    when:
+      root.kinds: install-codeunit
+      capability.op: [insert, modify, delete]
+    except:
+      any:
+        - capability.resource.table.name: "* Setup"
+        - capability.resource.table.name: "* Setup *"
+        - capability.resource.table.name: "* Migration *"
+        - capability.resource.table.name: "* Buffer"
+    facts: any
+
+  - id: exposed-root-no-isolated-storage-write
+    title: "API/web-service roots must not write to IsolatedStorage"
+    severity: high
+    when:
+      root.kinds: [api-page, web-service-exposed]
+      capability.op: [store-write, store-delete]
+    onUnknown: fail-closed
+    facts: any
+
+  - id: exposed-root-no-direct-ledger-write
+    title: "API/web-service roots must not directly write ledger tables"
+    severity: high
+    when:
+      root.kinds: [api-page, web-service-exposed]
+      capability.op: [insert, modify, delete]
+      capability.resource.table.name: "* Ledger Entry"
+    onUnknown: fail-closed
+    facts: any