@amxts/core 0.1.0

package/scripts/prebuilt.ts

@@ -0,0 +1,268 @@
+// Modules that come compiled: a module package from npm carries its owner
+// plugin's .aot for each system in prebuilt/, with prebuilt/manifest.json
+// saying what it was compiled from. A build takes it in place of compiling
+// the module - seconds of asc and wamrc on every machine - while the project
+// would compile the same thing; when it would not, the build says why and
+// compiles it.
+//
+//   bun scripts/prebuilt.ts      in a project: compile every module it lists
+//                                into the module's prebuilt/, for every
+//                                system (scripts/publish.ts runs it on the
+//                                packages it stages)
+//
+// What a module's .aot is made of, and so what the manifest holds:
+//
+// - the core: the compile's scripts, the patched AssemblyScript and its
+//   Binaryen, the API in as/, the natives table wamrc reads and the WAMR
+//   patch - which also decides whether the server's module loads the .aot.
+//   A package from the registry is its version: what npm has under a
+//   version never changes. A folder on this machine (a checkout) is its
+//   content, so the version and a hash are both kept;
+// - every module package the compile reads - the module, and the ones it
+//   imports (menu-core reads config-core for its proxy) - the same way;
+// - the options amxts.config.ts gives them: setup is compiled with them;
+// - how the includes declare a forward the module raises with something
+//   other than text: a Team, a number or a Float crosses as the declaration
+//   says. The server's includes change nothing else of a module;
+// - the project's plugins folder, which lies over the core's as/ in the
+//   tree a compile reads: a file there at a place the module reads would be
+//   compiled in its stead;
+// - the system: one .aot each, compiled from one wasm.
+//
+// TypeScript is compared without its comments: `amxts prepare` writes the
+// tooltips of the core and the modules in the project's language, which is
+// no change to the code.
+import type { NativesBeside, PluginNative } from './plugin-natives';
+import type { ModulePackage, Options, Sources } from './project';
+import type { System } from './system';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { join, relative, resolve, sep } from 'node:path';
+import ts from 'typescript';
+import { compileToMachineCode, compileToWasm } from './compile';
+import { codeFiles } from './compile-cache';
+import { includeForward, nativesBeside } from './plugin-natives';
+import { CORE_DIR, CORE_PLUGINS, loadProject, optionsOf, sourcesFor } from './project';
+import { SYSTEM_NAME, SYSTEMS, wamrcPath } from './system';
+import { c, log, since } from './ui';
+
+/** Where a module package keeps its compiled owner: prebuilt/<system>/<name>.aot, and the manifest. */
+export const PREBUILT_DIR = 'prebuilt';
+const MANIFEST = 'manifest.json';
+/** Goes up when the manifest changes shape: a build reads only its own. */
+const FORMAT = 1;
+const CORE_NAME = '@amxts/core';
+
+/** A package as a prebuilt module was compiled with it. */
+interface Built {
+	version: string;
+	/** Its content: the code files, TypeScript without comments. */
+	hash: string;
+}
+
+export interface PrebuiltManifest {
+	format: number;
+	module: string;
+	version: string;
+	/** The core and every module package the compile read, by name. */
+	from: Record<string, Built>;
+	/** The options each of those modules was compiled with. */
+	options: Record<string, Options>;
+	/** The forwards it raises with something other than text, as the includes declared them (null: none did). */
+	forwards: Record<string, string | null>;
+	/** The places under `~/` the compile read: a project's plugins folder must not hold them. */
+	places: string[];
+	/** Its natives, for the include a Pawn plugin compiles against. */
+	natives: PluginNative[];
+	beside: NativesBeside;
+	systems: Partial<Record<System, { file: string; size: number; sha256: string }>>;
+}
+
+/** What a build does with a module's prebuilt .aot: takes it, or compiles the module and says why. */
+export type PrebuiltUse = { aot: Uint8Array; natives: PluginNative[]; beside: NativesBeside } | { why: string };
+
+const posix = (path: string) => path.replace(/\\/g, '/');
+const sha256 = (data: Uint8Array) => createHash('sha256').update(data).digest('hex');
+
+/** Whether a package's folder is one a package manager installed, not a folder on this machine. */
+export function fromRegistry(dir: string): boolean {
+	return realpathSync(dir).split(sep).includes('node_modules');
+}
+
+const printer = ts.createPrinter({ removeComments: true });
+
+/** A file as a compile sees it: TypeScript without its comments, anything else as it is. */
+function codeOf(file: string): string | Buffer {
+	const bytes = readFileSync(file);
+	if (!file.endsWith('.ts')) return bytes;
+	return printer.printFile(ts.createSourceFile(file, bytes.toString('utf8'), ts.ScriptTarget.Latest, false, ts.ScriptKind.TS));
+}
+
+/** The files under `dir`, relative, but for node_modules, prebuilt/ and dot-folders. */
+function filesIn(dir: string, at = ''): string[] {
+	return readdirSync(join(dir, at), { withFileTypes: true }).flatMap((entry) => {
+		const path = at ? `${at}/${entry.name}` : entry.name;
+		if (!entry.isDirectory()) return [path];
+		return entry.name === 'node_modules' || entry.name === PREBUILT_DIR || entry.name.startsWith('.') ? [] : filesIn(dir, path);
+	});
+}
+
+/** One hash of `files` (relative to `base`) by their names and code, with `extra` before them. */
+function hashFiles(base: string, files: string[], extra = ''): string {
+	const hash = createHash('sha1').update(`${extra}\0`);
+	for (const file of [...files].sort()) hash.update(`${file}\0`).update(codeOf(join(base, file))).update('\0');
+	return hash.digest('hex');
+}
+
+/**
+ * A module package's content: the TypeScript and the includes it publishes
+ * (its package.json's `files`, when it lists them), and what its
+ * package.json says to the build.
+ */
+function packageHash(pkg: ModulePackage): string {
+	const json = JSON.parse(readFileSync(join(pkg.dir, 'package.json'), 'utf8'));
+	const listed: string[] | null = Array.isArray(json.files) ? json.files.map((each: string) => each.replace(/^\.\/|\/$/g, '')) : null;
+	const published = (file: string) => !listed || listed.some(each => file === each || file.startsWith(`${each}/`));
+	const files = filesIn(pkg.dir).filter(file => /\.(?:ts|inc)$/.test(file) && published(file));
+	return hashFiles(pkg.dir, files, JSON.stringify({ name: json.name, version: json.version, description: json.description, amxts: json.amxts }));
+}
+
+let coreHashed: string | null = null;
+
+/** The core's content: the compile's code, the API, the natives table and the WAMR patch. */
+function coreHash(): string {
+	if (coreHashed) return coreHashed;
+	const code = codeFiles([join(CORE_DIR, 'scripts/compile.ts')]).filter(existsSync).map(file => posix(relative(CORE_DIR, file)));
+	const api = filesIn(CORE_PLUGINS).filter(file => file.endsWith('.ts')).map(file => `as/${file}`);
+	const patches = readdirSync(join(CORE_DIR, 'runtime/patches')).map(file => `runtime/patches/${file}`);
+	coreHashed = hashFiles(CORE_DIR, [...code, ...api, ...patches, 'runtime/natives.txt']);
+	return coreHashed;
+}
+
+/** The core or a module package the project has, by name, as a prebuilt module is checked against it. */
+function installed(name: string, sources: Sources): { version: string; dir: string; hash: () => string } | null {
+	if (name === CORE_NAME) {
+		return { version: String(JSON.parse(readFileSync(join(CORE_DIR, 'package.json'), 'utf8')).version), dir: CORE_DIR, hash: coreHash };
+	}
+	const pkg = sources.project.packages.find(each => each.name === name);
+	return pkg ? { version: pkg.version, dir: pkg.dir, hash: () => packageHash(pkg) } : null;
+}
+
+/** How a forward's declaration is compared: what decides how its arguments cross. */
+function declarationOf(name: string): string | null {
+	const found = includeForward(name);
+	return found && JSON.stringify(found.params.map(param => [param.type, Boolean(param.isArray)]));
+}
+
+/**
+ * The prebuilt .aot of a module from npm for `system`, when it is what the
+ * project would compile; else why not. Null for a module that has none - a
+ * folder on this machine, or a package without prebuilt/ - which is compiled
+ * as any plugin is.
+ */
+export function prebuiltOf(pkg: ModulePackage, system: System, sources: Sources): PrebuiltUse | null {
+	const file = join(pkg.dir, PREBUILT_DIR, MANIFEST);
+	if (!fromRegistry(pkg.dir) || !existsSync(file)) return null;
+	const named = `${pkg.short} ${pkg.version}`;
+	let manifest: PrebuiltManifest;
+	try {
+		manifest = JSON.parse(readFileSync(file, 'utf8'));
+	} catch {
+		return { why: `${named}: its ${PREBUILT_DIR}/${MANIFEST} cannot be read` };
+	}
+	if (manifest.format !== FORMAT) return { why: `${named} comes compiled for another build of amxts` };
+	const aot = manifest.systems[system];
+	if (!aot) return { why: `${named} comes compiled for ${Object.keys(manifest.systems).map(each => SYSTEM_NAME[each as System]).join(' and ')}, not ${SYSTEM_NAME[system]}` };
+
+	for (const [name, built] of Object.entries(manifest.from)) {
+		const have = installed(name, sources);
+		if (!have) return { why: `${named} was built with ${name}, which the project does not have` };
+		if (have.version !== built.version) return { why: `${named} was built for ${name} ${built.version}, the project has ${have.version}` };
+		if (!fromRegistry(have.dir) && have.hash() !== built.hash) {
+			return { why: `${named} was built from another ${name} ${built.version} than the one in ${posix(relative(sources.project.dir, have.dir)) || '.'}` };
+		}
+	}
+	for (const [name, options] of Object.entries(manifest.options)) {
+		const module = sources.project.packages.find(each => each.name === name)!;
+		if (JSON.stringify(optionsOf(sources.project, module.definition)) !== JSON.stringify(options)) {
+			return { why: `${named} was built with ${name === pkg.name ? 'its' : `${module.short}'s`} default options, and amxts.config.ts sets ${module.definition.configKey}` };
+		}
+	}
+	for (const [name, declared] of Object.entries(manifest.forwards)) {
+		if (declarationOf(name) !== declared) return { why: `the includes declare the forward ${name} otherwise than when ${named} was built` };
+	}
+	const own = resolve(sources.project.pluginsDir) !== resolve(CORE_PLUGINS);
+	const shadow = own ? manifest.places.find(place => existsSync(join(sources.project.pluginsDir, place))) : undefined;
+	if (shadow) return { why: `${posix(relative(sources.project.dir, join(sources.project.pluginsDir, shadow)))} takes the place of a file ${named} was built from` };
+
+	const bytes = new Uint8Array(readFileSync(join(pkg.dir, aot.file)));
+	if (sha256(bytes) !== aot.sha256) return { why: `${named}: ${aot.file} is not the file its manifest lists` };
+	return { aot: bytes, natives: manifest.natives, beside: manifest.beside };
+}
+
+/** The `new Forward<...>("name")` calls of a package whose arguments are not all text: their names. */
+function typedForwards(pkg: ModulePackage): string[] {
+	const names: string[] = [];
+	for (const file of filesIn(pkg.dir).filter(each => each.endsWith('.ts'))) {
+		const text = readFileSync(join(pkg.dir, file), 'utf8');
+		for (const at of text.matchAll(/\bnew\s+Forward\b/g)) {
+			const call = text.slice(at.index).match(/^new\s+Forward(?:\s*<([^<>]*)>)?\s*\(\s*["']([^"']+)["']/);
+			if (!call) throw new Error(`${pkg.name}: ${file} - a Forward whose name and types the prebuilt build cannot read: new Forward<...>("name") with plain types`);
+			if ((call[1] ?? '').split(',').some(type => type.trim() !== 'string')) names.push(call[2]);
+		}
+	}
+	return names;
+}
+
+/** Compiles one module into its package's prebuilt/: one wasm, an .aot for each system, the manifest. */
+async function prebuild(pkg: ModulePackage, sources: Sources): Promise<string> {
+	const started = performance.now();
+	const dir = join(pkg.dir, PREBUILT_DIR);
+	rmSync(dir, { recursive: true, force: true });
+	mkdirSync(dir, { recursive: true });
+	const owner = sources.ownerSource(pkg);
+	const plugin = { source: owner, output: '', root: CORE_PLUGINS, wamrc: wamrcPath(), signatures: join(CORE_DIR, 'runtime/natives.txt'), quick: false };
+	const natives: PluginNative[] = [];
+	const wasm = join(dir, `${pkg.short}.wasm`);
+	const problem = await compileToWasm(plugin, wasm, false, natives);
+	if (problem) throw new Error(`${pkg.name} does not compile:\n${problem}`);
+
+	const systems: PrebuiltManifest['systems'] = {};
+	for (const system of SYSTEMS) {
+		const output = join(dir, system, `${pkg.short}.aot`);
+		const failed = compileToMachineCode({ ...plugin, output, system }, wasm, natives);
+		if (failed) throw new Error(`${pkg.name} for ${SYSTEM_NAME[system]}:\n${failed}`);
+		// The include is written from the natives where the build puts the .aot.
+		for (const name of readdirSync(join(dir, system)).filter(name => name.endsWith('.inc'))) rmSync(join(dir, system, name));
+		const bytes = readFileSync(output);
+		systems[system] = { file: `${PREBUILT_DIR}/${system}/${pkg.short}.aot`, size: bytes.length, sha256: sha256(bytes) };
+	}
+	rmSync(wasm, { force: true });
+
+	// What the compile read: its places in the tree, and the packages they are in.
+	const reached = sources.reach(join(CORE_PLUGINS, 'facade.ts'), sources.reach(join(CORE_PLUGINS, sources.entry(owner))));
+	const places = [...reached].map(place => posix(relative(CORE_PLUGINS, place))).sort();
+	const used = sources.project.modules.filter(each => each === pkg || [...reached].some(place => sources.packageOf(sources.real(place) ?? place) === each));
+	const core = installed(CORE_NAME, sources)!;
+	const manifest: PrebuiltManifest = {
+		format: FORMAT,
+		module: pkg.name,
+		version: pkg.version,
+		from: Object.fromEntries([[CORE_NAME, { version: core.version, hash: core.hash() }], ...used.map(each => [each.name, { version: each.version, hash: packageHash(each) }])]),
+		options: Object.fromEntries(used.map(each => [each.name, optionsOf(sources.project, each.definition)])),
+		forwards: Object.fromEntries(used.flatMap(typedForwards).map(name => [name, declarationOf(name)])),
+		places,
+		natives,
+		beside: nativesBeside(natives),
+		systems,
+	};
+	writeFileSync(join(dir, MANIFEST), `${JSON.stringify(manifest, null, '\t')}\n`);
+	const sizes = SYSTEMS.map(system => `${SYSTEM_NAME[system]} ${(systems[system]!.size / 1024).toFixed(0)} KB`).join(', ');
+	return `prebuilt ${c.bold(pkg.short)} ${c.dim(`${sizes} (${since(started)})`)}`;
+}
+
+if (import.meta.main) {
+	const project = loadProject(process.cwd());
+	if (project.problems.length) throw new Error(project.problems.join('\n'));
+	for (const pkg of project.modules) log.success(await prebuild(pkg, sourcesFor(CORE_PLUGINS)));
+}

package/scripts/prepare.ts

@@ -0,0 +1,134 @@
+// What an editor needs to read a project: .amxts/tsconfig.json, which the
+// project's tsconfig.json extends.
+//
+//   npx amxts prepare        (also the first step of `amxts build`, with --quiet:
+//                            the line it prints only under --debug)
+//
+// The project's own tsconfig.json extends it: { "extends": "./.amxts/tsconfig.json" }.
+// It says what the build says - `~/` is the project's plugins over the
+// core's as/, "@amxts/core" is the facade, a module package is its module
+// file - and it takes in amxts.config.ts and every module the config lists, so
+// that the options each module declares (ModuleOptions) type the config.
+// Beside it, imports.d.ts makes what plugins use without an import - the
+// facade's API, the modules' namespaces - globals for the editor
+// (scripts/auto-imports.ts).
+import { existsSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, isAbsolute, join, relative } from 'node:path';
+import { applyProject, docsLang } from './apply-docs';
+import { importsDeclaration } from './auto-imports';
+import { CONFIG_FILE, CORE_DIR, CORE_PLUGINS, loadProject } from './project';
+import { c, debug, log } from './ui';
+
+function fail(message: string): void {
+	log.error(message);
+}
+
+const project = loadProject(process.cwd());
+if (project.problems.length) {
+	for (const problem of project.problems) fail(problem);
+	const missing = project.problems.map(problem => problem.match(/module "([^"]+)" is not installed/)?.[1]).filter(Boolean);
+	if (missing.length) log.hint(`Install ${missing.length > 1 ? 'them' : 'it'}: amxts module add ${missing.join(' ')}`);
+	process.exit(1);
+}
+
+// The hood's tooltips - the core's and the modules' - in the language .env
+// picks (AMXTS_DOCS_LANG), as the generated API has them. A core it cannot
+// write to keeps English: the editor still has words, only not these.
+let docs = '';
+try {
+	docs = await applyProject(project.dir, docsLang());
+} catch (error) {
+	fail(`docs: ${(error as Error).message}`);
+}
+
+// A module's own folder, without a project config, has its own tsconfig.json
+// (extending the core's as/tsconfig.json): nothing to write there.
+if (!project.config && project.packages.some(pkg => pkg.dir === project.dir)) process.exit(0);
+
+const out = join(project.dir, '.amxts');
+/** A path as the generated config writes it: from .amxts/, with forward slashes. */
+function from(path: string) {
+	const rel = relative(out, path).replace(/\\/g, '/');
+	// Another drive has no relative path: the absolute one, then.
+	if (isAbsolute(rel)) return rel;
+	return rel.startsWith('.') ? rel : `./${rel}`;
+}
+
+// AssemblyScript's own config - its types for the standard library - is the
+// core's: the project does not install AssemblyScript itself.
+const assembly = createRequire(join(CORE_DIR, 'package.json')).resolve('assemblyscript/std/assembly.json');
+
+/**
+ * Whether `name` is found from the project as a package - in node_modules, as
+ * Node looks, and the same folder the build takes (a link counts as its
+ * target). Such a package is resolved by its package.json ("exports",
+ * "types"), not mapped to a file: tools that read tsconfig (knip) take a
+ * mapped name for a folder of the project, and its dependency for unused.
+ */
+function installed(name: string, dir: string) {
+	for (let at = project.dir; ; at = dirname(at)) {
+		const path = join(at, 'node_modules', name);
+		if (existsSync(path)) return realpathSync(path) === realpathSync(dir);
+		if (dirname(at) === at) return false;
+	}
+}
+
+// `~/` is not a package. The core and the modules are, once installed; one
+// that is not (a module in modules/, a core run from elsewhere) is mapped to
+// its file, as the build finds it.
+const paths: Record<string, string[]> = {
+	'~/*': [...new Set([`${from(project.pluginsDir)}/*`, `${from(CORE_PLUGINS)}/*`])],
+};
+if (!installed('@amxts/core', CORE_DIR)) {
+	paths['@amxts/core'] = [from(join(CORE_PLUGINS, 'facade.ts'))];
+	paths['@amxts/core/*'] = [`${from(CORE_PLUGINS)}/*`];
+}
+for (const pkg of project.modules) {
+	if (!installed(pkg.name, pkg.dir)) paths[pkg.name] = [from(pkg.module)];
+}
+
+const tsconfig = {
+	extends: from(assembly),
+	compilerOptions: {
+		// The core's: the lowest target TypeScript 7 keeps. `for (const x of
+		// list)` checks against the iterator the core's patched typings give
+		// arrays and strings; the compiler lowers the loop itself.
+		target: 'es2015',
+		// A package by its package.json "exports", as Bun and TypeScript 6 and
+		// later read it; TypeScript 5.9 takes "bundler" only with an ES module.
+		// asc does not read this: scripts/project.ts answers the names for it.
+		module: 'esnext',
+		moduleResolution: 'bundler',
+		// Every file is a module, one without an import line too: what a
+		// plugin declares is its own, never a global another plugin's name
+		// collides with.
+		moduleDetection: 'force',
+		paths,
+	},
+	include: [
+		`${from(project.pluginsDir)}/**/*.ts`,
+		from(join(project.dir, CONFIG_FILE)),
+		`${from(CORE_PLUGINS)}/*.d.ts`,
+		'./modules.d.ts',
+		'./imports.d.ts',
+	],
+};
+
+mkdirSync(out, { recursive: true });
+writeFileSync(join(out, 'tsconfig.json'), `${JSON.stringify(tsconfig, null, '\t')}\n`);
+writeFileSync(join(out, 'modules.d.ts'), [
+	'// GENERATED by amxts prepare: the modules amxts.config.ts lists, for the',
+	'// options each one adds to ModuleOptions.',
+	...project.modules.map(pkg => `import "${pkg.name}";`),
+	'',
+	'export {};',
+	'',
+].join('\n'));
+
+writeFileSync(join(out, 'imports.d.ts'), importsDeclaration(project.autoImports));
+
+// A step of dev, build and typecheck, which say what the project is
+// themselves: they pass --quiet, and --debug shows it all the same.
+const modules = project.modules.map(pkg => pkg.short).join(', ') || 'no modules';
+if (!process.argv.includes('--quiet') || debug()) log.step(c.dim(`prepared ${(relative(process.cwd(), out) || '.').replace(/\\/g, '/')}/tsconfig.json (${modules})${docs ? ` · ${docs}` : ''}`));