gofront 1.6.1 → 1.8.0

package/src/compiler.js

@@ -19,26 +19,40 @@
 
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname, join, relative, resolve, sep } from "node:path";
-import {
-	buildSourceMap,
-	buildWasmSourceMap,
-	CodeGen,
-} from "./backend/js/index.js";
+import { CodeGen } from "./backend/js/index.js";
 import { compileWasmModule, optimizeWasm } from "./backend/wasm/index.js";
-import { log } from "./colors.js";
+import { log } from "./cli/colors.js";
 import { parseDts } from "./dts-parser.js";
-import { Lexer } from "./lexer.js";
+import { Lexer } from "./lexer/lexer.js";
+import { TemplLexer } from "./lexer/templ-lexer.js";
+import {
+	attachSourceMap,
+	buildWasmMap,
+	mergeCompilationChunks,
+	WASM_MAP_URL,
+} from "./link/sourcemap.js";
+import {
+	checkJsonCodecs,
+	finishStdlibJs,
+	importStdlibPackages,
+} from "./link/stdlib.js";
+import {
+	linkIntoBundle,
+	registerPackageUnit,
+	registerWasmUnit,
+	wasmMarker,
+} from "./link/wasm.js";
 import { lower } from "./lower/index.js";
 import { Parser } from "./parser/index.js";
+import { TemplParser } from "./parser/templ.js";
 import {
 	isBuiltinPackage,
 	isLocalPath,
 	resolveAll,
 	resolveGwDir,
+	resolveStdlibSource,
 } from "./resolver.js";
-import { TemplLexer } from "./templ-lexer.js";
-import { TemplParser } from "./templ-parser.js";
-import { TypeCheckError, TypeChecker } from "./typechecker/index.js";
+import { EXCLUDED_STDLIB_PACKAGES, TypeChecker } from "./typechecker/index.js";
 import { WASM_SUPPORTED_STDLIB } from "./typechecker/types.js";
 
 // ── Parse cache ──────────────────────────────────────────────
@@ -89,98 +103,6 @@ export function gwFilesIn(dir, { includeTests = false } = {}) {
 		.map((f) => join(dir, f));
 }
 
-function normalizeChunk(c) {
-	if (typeof c === "string") {
-		return { js: c, sourceFiles: [], sourcesContent: [], mappings: [] };
-	}
-	return c;
-}
-
-function appendChunkMappings(dest, mappings, localToUnifiedIdx, lineOffset) {
-	if (!mappings) return;
-	for (const m of mappings) {
-		dest.push({
-			genLine: m.genLine + lineOffset,
-			srcLine: m.srcLine,
-			srcFileIdx: localToUnifiedIdx[m.srcFileIdx ?? 0] ?? 0,
-		});
-	}
-}
-
-function mergeCompilationChunks(chunks) {
-	const activeChunks = chunks
-		.filter(Boolean)
-		.map(normalizeChunk)
-		.filter((c) => c.js.length > 0);
-
-	if (activeChunks.length === 0) {
-		return { js: "", sourceFiles: [], sourcesContent: [], mappings: [] };
-	}
-
-	const unifiedSources = [];
-	const unifiedSourcesContent = [];
-	const fileIdxMap = new Map();
-
-	function getOrAddFile(filePath, content) {
-		const key = resolve(filePath);
-		let idx = fileIdxMap.get(key);
-		if (idx === undefined) {
-			idx = unifiedSources.length;
-			fileIdxMap.set(key, idx);
-			unifiedSources.push(key);
-			unifiedSourcesContent.push(content);
-		}
-		return idx;
-	}
-
-	const combinedJsParts = [];
-	const mergedMappings = [];
-	let currentLineOffset = 0;
-
-	for (const chunk of activeChunks) {
-		combinedJsParts.push(chunk.js);
-		const localToUnifiedIdx = (chunk.sourceFiles ?? []).map((file, i) =>
-			getOrAddFile(file, chunk.sourcesContent?.[i] ?? ""),
-		);
-		appendChunkMappings(
-			mergedMappings,
-			chunk.mappings,
-			localToUnifiedIdx,
-			currentLineOffset,
-		);
-		currentLineOffset += chunk.js.split("\n").length;
-	}
-
-	return {
-		js: combinedJsParts.join("\n"),
-		sourceFiles: unifiedSources,
-		sourcesContent: unifiedSourcesContent,
-		mappings: mergedMappings,
-	};
-}
-
-function attachSourceMap(merged, outputDir) {
-	const relativeSources = merged.sourceFiles.map((f) => relative(outputDir, f));
-	const mapJson = buildSourceMap(
-		relativeSources,
-		merged.mappings,
-		merged.sourcesContent,
-	);
-	const b64 = Buffer.from(mapJson).toString("base64");
-	merged.js += `\n//# sourceMappingURL=data:application/json;base64,${b64}`;
-}
-
-const WASM_MAP_URL = "app.wasm.map";
-
-// Function-level `app.wasm.map` (sources relative to the output dir).
-function buildWasmMap(info, outputDir) {
-	if (!info || info.mappings.length === 0) return null;
-	return buildWasmSourceMap(
-		info.sources.map((f) => relative(outputDir, f)),
-		info.mappings,
-	);
-}
-
 // Shared tail of compileFiles / compileSingleFile: link `app.wasm`, run
 // wasm-opt when requested and produce the wasm source map.
 function finishWasm(options, merged, pkgTarget, outputDir) {
@@ -213,171 +135,16 @@ function finishWasm(options, merged, pkgTarget, outputDir) {
 	return { wasm, wat, wasmOptInfo, wasmSourceMap };
 }
 
-// ── WASM linking ──────────────────────────────────────────────
-//
-// Every `wasm`/`both` package reached during a build registers a unit in the
-// shared `options.wasmUnits` array.  The root compile links all units into a
-// single `app.wasm` and splices the generated JS facade into the bundle where
-// the first wasm package's JS would have been.
-
-const WASM_MARKER_RE = /^\/\*__GOFRONT_WASM_UNIT:[^*]*\*\/$/m;
-
-function wasmMarker(pkgName) {
-	return `/*__GOFRONT_WASM_UNIT:${pkgName}*/`;
-}
-
-function registerWasmUnit(options, unit) {
-	const units = options.wasmUnits;
-	if (!units) return;
-	if (units.some((u) => u.key === unit.key)) return;
-	units.push(unit);
-}
-
-// Registers a `wasm`/`both` package as a link unit and returns the target the
-// package's own code is emitted for.  `options.wasmTests` runs a `both`
-// package's tests through the wasm backend (the package is then treated as
-// `wasm` for linking so its test functions are exposed by the facade).
-function registerPackageUnit(options, unit) {
-	const { pkgTarget, programs } = unit;
-	if (pkgTarget !== "wasm" && pkgTarget !== "both") return pkgTarget;
-	const emitTarget =
-		options.wasmTests && options.isTest && pkgTarget === "both"
-			? "wasm"
-			: pkgTarget;
-	registerWasmUnit(options, {
-		...unit,
-		target: emitTarget,
-		programs:
-			emitTarget === pkgTarget
-				? programs
-				: programs.map((p) => ({ ...p, target: emitTarget })),
-	});
-	return emitTarget;
-}
-
-function linkWasmUnits(
-	units,
-	{ emitWat = false, rootTarget = "js", sourceMappingURL = null } = {},
-) {
-	if (!units.some((u) => u.target === "wasm")) return null;
-	checkLinkCollisions(units);
-
-	const types = new Map();
-	const bundledPackages = new Set();
-	const programs = [];
-	for (const u of units) {
-		for (const [k, v] of u.types) types.set(k, v);
-		for (const b of u.bundledPackages) bundledPackages.add(b);
-		programs.push(...u.programs);
-	}
-	const checker = {
-		types,
-		errors: [],
-		err(msg, node) {
-			const e = new TypeCheckError(
-				msg,
-				node,
-				this._currentFile,
-				this._currentSource,
-			);
-			this.errors.push(e);
-			return e;
-		},
-	};
-	const lowerRes = lower(programs, checker);
-	return compileWasmModule(programs, checker, lowerRes, {
-		boundary: true,
-		bundledPackages,
-		emitWat,
-		callMain: rootTarget === "wasm",
-		sourceMappingURL,
-	});
-}
-
-// The linked module is one flat namespace (like the JS bundle), so a top-level
-// name declared by two different packages would silently resolve to whichever
-// was emitted last.  Reject it up front.
-function topLevelDeclNames(decl) {
-	switch (decl.kind) {
-		case "FuncDecl":
-			return [decl.name];
-		case "MethodDecl":
-			return [`${decl.recvType?.name ?? "?"}.${decl.name}`];
-		case "TypeDecl":
-			return [decl.name];
-		case "VarDecl":
-		case "ConstDecl":
-			return (decl.decls ?? []).flatMap((spec) => spec.names ?? []);
-		default:
-			return [];
-	}
-}
-
-function checkLinkCollisions(units) {
-	const owner = new Map(); // name -> pkgName
-	const errors = [];
-	for (const u of units) {
-		for (const p of u.programs) {
-			for (const d of p.decls ?? []) {
-				for (const name of topLevelDeclNames(d)) {
-					if (name === "_") continue;
-					const prev = owner.get(name);
-					if (prev === undefined) owner.set(name, u.pkgName);
-					else if (prev !== u.pkgName)
-						errors.push(
-							`cannot link wasm packages: '${name}' is declared in both '${prev}' and '${u.pkgName}' (linked wasm packages share one namespace)`,
-						);
-				}
-			}
-		}
-	}
-	if (errors.length > 0) throw new Error([...new Set(errors)].join("\n"));
-}
-
-// Replaces the first wasm marker with the facade (dropping the others) and
-// shifts source-map lines that follow it.
-function spliceFacade(merged, facade) {
-	const lines = merged.js.split("\n");
-	const first = lines.findIndex((l) => WASM_MARKER_RE.test(l));
-	if (first < 0) {
-		merged.js = `${facade}\n${merged.js}`;
-		const shift = facade.split("\n").length;
-		for (const m of merged.mappings) m.genLine += shift;
-		return;
-	}
-	const facadeLines = facade.split("\n");
-	const delta = facadeLines.length - 1;
-	for (const m of merged.mappings) {
-		if (m.genLine > first) m.genLine += delta;
-	}
-	const out = [...lines.slice(0, first), ...facadeLines];
-	for (let i = first + 1; i < lines.length; i++) {
-		if (!WASM_MARKER_RE.test(lines[i])) out.push(lines[i]);
-	}
-	merged.js = out.join("\n");
-}
-
-// Root compiles only: link every registered unit into app.wasm and splice the
-// facade into the merged bundle.  Returns `{ wasm, wat, sourceMapInfo }`
-// (nulls when there is nothing to link).
-function linkIntoBundle(options, merged, pkgTarget, { sourceMappingURL } = {}) {
-	const none = { wasm: null, wat: null, sourceMapInfo: null };
-	if (options.isDependency) return none;
-	const linked = linkWasmUnits(options.wasmUnits, {
-		emitWat: options.emitWat,
-		rootTarget: pkgTarget,
-		sourceMappingURL,
-	});
-	if (!linked) return none;
-	spliceFacade(merged, linked.facade);
-	return {
-		wasm: linked.wasm,
-		wat: linked.wat ?? null,
-		sourceMapInfo: linked.sourceMapInfo ?? null,
-	};
+// Prints non-fatal diagnostics (e.g. JS Number precision of a constant).
+function logWarnings(warnings) {
+	for (const w of warnings) log.warn(w);
 }
 
 // ── Import resolution ─────────────────────────────────────────
+
+// What link/stdlib.js needs from this module.  Injected rather than imported
+// so that link/stdlib.js does not depend on compiler.js.
+const STDLIB_DEPS = { gwFilesIn, resolveImports, TypeChecker };
 //
 // Shared by compileFiles (multi-file) and the single-file path in index.js.
 // Mutates checker, jsImports, bundledPackages, and preambles in place.
@@ -428,7 +195,12 @@ function resolveImports(
 							"'gofront/shared' requires a wasm-only package (//gofront:target wasm): a both package has no linear memory on the JS side",
 							impNode,
 						);
-					} else if (!isLocalPath(path)) {
+					} else if (
+						!isLocalPath(path) &&
+						!EXCLUDED_STDLIB_PACKAGES.has(path) &&
+						(isBuiltinPackage(path) ||
+							!resolveStdlibSource(path, options.stdlibRoot))
+					) {
 						if (!isBuiltinPackage(path)) {
 							checker.recordBlocker("js: import", path);
 							checker.err(
@@ -469,13 +241,27 @@ function resolveImports(
 		...imp,
 		imports: imp.imports.filter(({ alias }) => alias !== "_"),
 	}));
-	const resolved = resolveAll(allImportsNoSideEffect, fromFile, parseDts);
+	const resolved = resolveAll(
+		allImportsNoSideEffect,
+		fromFile,
+		parseDts,
+		options.stdlibRoot,
+	);
 	for (const [path, info] of resolved) {
 		if (!info) continue;
 		checker.addDefinitions(info.types, info.values);
 		jsImports.set(path, [...info.values.keys()]);
 	}
 
+	importStdlibPackages(
+		programs,
+		checker,
+		preambles,
+		bundledDirs,
+		options,
+		STDLIB_DEPS,
+	);
+
 	// local GoFront packages (./subdir)
 	const seenLocalPaths = new Set();
 	for (const p of programs) {
@@ -541,7 +327,10 @@ function resolveImports(
 
 				if (alias === "_") continue;
 				if (alias === ".") {
-					checker.addDefinitions(dep.exportedTypes, dep.exportedSymbols);
+					checker.addDefinitions(dep.exportedTypes, dep.exportedSymbols, {
+						external: false,
+						consts: dep.exportedConsts,
+					});
 					continue;
 				}
 				const nameUsed = alias ?? dep.pkgName;
@@ -551,6 +340,7 @@ function resolveImports(
 					dep.exportedSymbols,
 					dep.exportedTypes,
 					dep.target,
+					dep.exportedConsts,
 				);
 				checker.trackImport(nameUsed, { _line }, p._filename, p._source);
 			}
@@ -590,7 +380,13 @@ export function compileSingleFile(inputPath, options = {}) {
 		dumpTokens = false,
 		dumpAst = false,
 	} = options;
-	options = { ...options, wasmUnits: options.wasmUnits ?? [] };
+	options = {
+		...options,
+		wasmUnits: options.wasmUnits ?? [],
+		stdlibCache: options.stdlibCache ?? new Map(),
+		jsRootPrograms: options.jsRootPrograms ?? [],
+		bundledDirs: options.bundledDirs ?? new Set(),
+	};
 
 	let source;
 	try {
@@ -661,6 +457,7 @@ export function compileSingleFile(inputPath, options = {}) {
 	});
 	const mainJs =
 		pkgTarget === "wasm" ? wasmMarker(checker.pkgName) : cg.generate(ast);
+	logWarnings(cg.warnings);
 
 	const mainChunk = {
 		js: mainJs,
@@ -670,6 +467,7 @@ export function compileSingleFile(inputPath, options = {}) {
 	};
 
 	const merged = mergeCompilationChunks([...preambles, mainChunk]);
+	finishStdlibJs(merged, options, [ast], pkgTarget);
 	const { wasm, wat, wasmOptInfo, wasmSourceMap } = finishWasm(
 		{ ...options, sourceMap },
 		merged,
@@ -716,7 +514,10 @@ export function compileFiles(files, options = {}) {
 	options = {
 		...options,
 		wasmUnits: options.wasmUnits ?? [],
+		stdlibCache: options.stdlibCache ?? new Map(),
+		jsRootPrograms: options.jsRootPrograms ?? [],
 		rootDir: options.rootDir ?? fromDir,
+		bundledDirs: options.bundledDirs ?? new Set(),
 	};
 	const outputDir =
 		options.outputDir ??
@@ -792,6 +593,13 @@ export function compileFiles(files, options = {}) {
 
 	// ── 3. Type-check ─────────────────────────────────────────────
 	const errors = checker.checkAll(programs);
+	const codecs = checkJsonCodecs(
+		programs,
+		checker,
+		preambles,
+		options,
+		STDLIB_DEPS,
+	);
 	checker.reportUnusedImports();
 	if (errors.length > 0) {
 		const msgs = errors.map((e) => e.message).join("\n");
@@ -804,7 +612,7 @@ export function compileFiles(files, options = {}) {
 		key: resolve(fromDir),
 		pkgName,
 		pkgTarget,
-		programs,
+		programs: [...programs, ...codecs],
 		types: checker.types,
 		bundledPackages,
 	});
@@ -820,6 +628,7 @@ export function compileFiles(files, options = {}) {
 			: codegen.generateAll(programs, {
 					isTest: options.isTest ?? false,
 				});
+	logWarnings(codegen.warnings);
 
 	const mainChunk = {
 		js: mainJs,
@@ -829,6 +638,7 @@ export function compileFiles(files, options = {}) {
 	};
 
 	const merged = mergeCompilationChunks([...preambles, mainChunk]);
+	finishStdlibJs(merged, options, programs, emitTarget);
 	const { wasm, wat, wasmOptInfo, wasmSourceMap } = finishWasm(
 		options,
 		merged,
@@ -857,8 +667,36 @@ export function compileFiles(files, options = {}) {
 		programs,
 		exportedSymbols: checker.getExportedSymbols(),
 		exportedTypes: checker.getExportedTypes(),
+		exportedConsts: checker.getExportedConsts(),
 		sourceFiles: merged.sourceFiles,
 		sourcesContent: merged.sourcesContent,
 		mappings: merged.mappings,
 	};
 }
+
+export function compileWasm(source, options = {}) {
+	const filename = options.filename ?? "main.go";
+	const tokens = new Lexer(source, filename).tokenize();
+	const ast = new Parser(tokens, filename, source).parse();
+
+	const checker = new TypeChecker();
+	checker.target = "wasm";
+	checker.pkgName = ast.pkg?.name ?? "main";
+
+	const errors = checker.check(ast);
+	if (errors.length > 0) {
+		return { wasm: null, errors };
+	}
+
+	const lowerRes = lower([ast], checker);
+	const res = compileWasmModule([ast], checker, lowerRes, {
+		...options,
+		exportAll: options.exportAll ?? true,
+	});
+
+	if (checker.errors?.length > 0) {
+		return { wasm: null, errors: checker.errors };
+	}
+
+	return { ...res, errors: [] };
+}

package/src/dts-parser.js

@@ -203,9 +203,48 @@ export class DtsParser {
 
 	// ── Type parsing ─────────────────────────────────────────────
 
+	_parseTupleElement() {
+		this.skip();
+		if (this.startsWith("...")) this.pos += 3;
+
+		const savedPos = this.pos;
+		const ident = this.readIdent();
+		if (ident) {
+			this.skip();
+			if (this.ch() === "?") {
+				this.pos++;
+				this.skip();
+			}
+			if (this.ch() === ":") {
+				this.pos++;
+				return this.parseType(false);
+			}
+		}
+		this.pos = savedPos;
+		return this.parseType(false);
+	}
+
+	_parseTupleType(inReturn) {
+		this.consume("[");
+		const elements = [];
+		while (!this.eof() && this.peek() !== "]") {
+			this.skip();
+			if (this.ch() === "]") break;
+			elements.push(this._parseTupleElement());
+			this.skip();
+			if (this.ch() !== ",") break;
+			this.pos++;
+		}
+		this.consume("]");
+		if (inReturn) {
+			return { kind: "tuple", types: elements };
+		}
+		return { kind: "slice", elem: ANY };
+	}
+
 	// Parse a TypeScript type expression and return a GoFront type.
 	// Always returns something; falls back to ANY for complex types.
-	parseType() {
+	parseType(inReturn = false) {
 		this.skip();
 
 		// Skip modifiers
@@ -226,13 +265,16 @@ export class DtsParser {
 		this.skip();
 		if (this.ch() === "|" || this.ch() === "&") this.pos++;
 
-		let base = this._parseBaseType();
+		let base = this._parseBaseType(inReturn);
 
 		this.skip();
 		// Array suffix: T[] or T[][]
 		while (this.startsWith("[]")) {
 			this.pos += 2;
-			base = { kind: "slice", elem: base };
+			base = {
+				kind: "slice",
+				elem: base.kind === "tuple" ? { kind: "slice", elem: ANY } : base,
+			};
 			this.skip();
 		}
 
@@ -285,14 +327,11 @@ export class DtsParser {
 		return { kind: "basic", name: "any", alias: name };
 	}
 
-	_parseBaseType() {
+	_parseBaseType(inReturn = false) {
 		this.skip();
 		const c = this.ch();
 		if (c === "[") {
-			this.pos++;
-			this.skipTypeExpr("]");
-			this.consume("]");
-			return ANY;
+			return this._parseTupleType(inReturn);
 		}
 		if (c === "{") {
 			this.skipBlock();
@@ -399,15 +438,24 @@ export class DtsParser {
 		let ret = VOID;
 		if (this.ch() === ":") {
 			this.pos++;
-			ret = this.parseType();
+			ret = this.parseType(true);
 		}
 		this.consume(";");
+		const isVariadic = params.some((p) => p.variadic);
 		return {
 			name,
 			val: {
 				kind: "func",
-				params: params.map((p) => p.type),
-				returns: [ret],
+				params: params.map((p) =>
+					p.variadic && p.type?.kind === "slice" ? p.type.elem : p.type,
+				),
+				returns:
+					ret.kind === "tuple"
+						? ret.types.length > 0
+							? ret.types
+							: [VOID]
+						: [ret],
+				...(isVariadic ? { variadic: true } : {}),
 			},
 		};
 	}
@@ -545,13 +593,22 @@ export class DtsParser {
 			let ret = VOID;
 			if (this.ch() === ":") {
 				this.pos++;
-				ret = this.parseType();
+				ret = this.parseType(true);
 			}
 			this.consume(";");
+			const isVariadic = params.some((p) => p.variadic);
 			out[name] = {
 				kind: "func",
-				params: params.map((p) => p.type),
-				returns: [ret],
+				params: params.map((p) =>
+					p.variadic && p.type?.kind === "slice" ? p.type.elem : p.type,
+				),
+				returns:
+					ret.kind === "tuple"
+						? ret.types.length > 0
+							? ret.types
+							: [VOID]
+						: [ret],
+				...(isVariadic ? { variadic: true } : {}),
 			};
 		} else if (this.ch() === ":") {
 			this.pos++;