gofront 1.8.0 → 1.9.0

package/src/compiler.js

@@ -32,17 +32,19 @@ import {
 	WASM_MAP_URL,
 } from "./link/sourcemap.js";
 import {
-	checkJsonCodecs,
+	checkGeneratedGo,
 	finishStdlibJs,
 	importStdlibPackages,
+	mangleCollidingTypes,
+	renameLocalTypes,
+	withOriginalTypeNames,
 } from "./link/stdlib.js";
 import {
 	linkIntoBundle,
 	registerPackageUnit,
-	registerWasmUnit,
 	wasmMarker,
 } from "./link/wasm.js";
-import { lower } from "./lower/index.js";
+import { lower, takesElementAddress } from "./lower/index.js";
 import { Parser } from "./parser/index.js";
 import { TemplParser } from "./parser/templ.js";
 import {
@@ -60,10 +62,21 @@ import { WASM_SUPPORTED_STDLIB } from "./typechecker/types.js";
 // Map<filePath, { mtime: number, ast: object }>
 const _parseCache = new Map();
 
+// A cached AST keeps everything the build attached to it (checked types,
+// lowered stdlib programs), so a build of a Go-source stdlib importer pins
+// megabytes.  The cache is therefore bounded; the least recently used file goes
+// first.
+const PARSE_CACHE_LIMIT = 16;
+
 export function clearParseCache() {
 	_parseCache.clear();
 }
 
+// Drops one file, e.g. a throwaway source that was deleted after its build.
+export function evictParseCache(filePath) {
+	_parseCache.delete(resolve(filePath));
+}
+
 export function parseCacheSize() {
 	return _parseCache.size;
 }
@@ -71,9 +84,14 @@ export function parseCacheSize() {
 // ── Helpers ──────────────────────────────────────────────────
 
 function parseGoFrontFile(filePath) {
+	const key = resolve(filePath);
 	const mtime = statSync(filePath).mtimeMs;
-	const cached = _parseCache.get(filePath);
-	if (cached && cached.mtime === mtime) return cached.ast;
+	const cached = _parseCache.get(key);
+	if (cached && cached.mtime === mtime) {
+		_parseCache.delete(key);
+		_parseCache.set(key, cached);
+		return cached.ast;
+	}
 
 	const source = readFileSync(filePath, "utf8");
 	const filename = basename(filePath);
@@ -88,7 +106,9 @@ function parseGoFrontFile(filePath) {
 	ast._path = resolve(filePath);
 	// compileFiles overwrites `target` with the resolved one; keep the directive.
 	ast._declaredTarget = ast.target;
-	_parseCache.set(filePath, { mtime, ast });
+	_parseCache.set(key, { mtime, ast });
+	if (_parseCache.size > PARSE_CACHE_LIMIT)
+		_parseCache.delete(_parseCache.keys().next().value);
 	return ast;
 }
 
@@ -107,7 +127,7 @@ export function gwFilesIn(dir, { includeTests = false } = {}) {
 // wasm-opt when requested and produce the wasm source map.
 function finishWasm(options, merged, pkgTarget, outputDir) {
 	const wantMap = Boolean(options.sourceMap && !options.isDependency);
-	let { wasm, wat, sourceMapInfo } = linkIntoBundle(
+	let { wasm, wat, sourceMapInfo, stringTable } = linkIntoBundle(
 		options,
 		merged,
 		pkgTarget,
@@ -132,7 +152,7 @@ function finishWasm(options, merged, pkgTarget, outputDir) {
 		wasmOptInfo = opt;
 		if (opt.sourceMap) wasmSourceMap = opt.sourceMap;
 	}
-	return { wasm, wat, wasmOptInfo, wasmSourceMap };
+	return { wasm, wat, wasmOptInfo, wasmSourceMap, stringTable };
 }
 
 // Prints non-fatal diagnostics (e.g. JS Number precision of a constant).
@@ -184,12 +204,6 @@ function resolveImports(
 							"js: imports are not allowed in wasm packages",
 							impNode,
 						);
-					} else if (path === "gom") {
-						checker.recordBlocker("gom usage", "gom");
-						checker.err(
-							"package 'gom' is not available in wasm packages",
-							impNode,
-						);
 					} else if (path === "gofront/shared" && pkgTarget === "both") {
 						checker.err(
 							"'gofront/shared' requires a wasm-only package (//gofront:target wasm): a both package has no linear memory on the JS side",
@@ -287,6 +301,9 @@ function resolveImports(
 					isDependency: true,
 				});
 				if (!alreadyBundled) preambles.push(dep);
+				// A dependency that hands out `&xs[i]` makes whole-element
+				// stores in this package write in place too.
+				if (dep.elemAddrTaken) checker.depElemAddrTaken = true;
 
 				// Target import rules for local packages
 				const impNode = {
@@ -374,120 +391,29 @@ export function resolvePackageTarget(declared, pkgDir, options = {}) {
 }
 
 export function compileSingleFile(inputPath, options = {}) {
-	const {
-		sourceMap = false,
-		outputFile = null,
-		dumpTokens = false,
-		dumpAst = false,
-	} = options;
-	options = {
-		...options,
-		wasmUnits: options.wasmUnits ?? [],
-		stdlibCache: options.stdlibCache ?? new Map(),
-		jsRootPrograms: options.jsRootPrograms ?? [],
-		bundledDirs: options.bundledDirs ?? new Set(),
-	};
-
-	let source;
-	try {
-		source = readFileSync(inputPath, "utf8");
-	} catch (e) {
-		throw new Error(`cannot read '${inputPath}': ${e.message}`);
-	}
-
-	const tokens = new Lexer(source, basename(inputPath)).tokenize();
-	if (dumpTokens) return { tokens };
-
-	const ast = new Parser(tokens, basename(inputPath), source).parse();
-	ast._source = source;
-	ast._path = resolve(inputPath);
-	if (dumpAst) return { ast };
-
-	const outputDir =
-		options.outputDir ??
-		(outputFile ? dirname(resolve(outputFile)) : resolve("."));
-
-	const checker = new TypeChecker();
-	const jsImports = new Map();
-	const bundledPackages = new Set();
-	const preambles = [];
-
-	const pkgTarget = resolvePackageTarget(
-		ast.target ?? options.target ?? "js",
-		dirname(resolve(inputPath)),
-		options,
-	);
-	ast.target = pkgTarget;
-	checker.target = pkgTarget;
-	checker.pkgName = ast.pkg?.name ?? "main";
-
-	resolveImports(
-		[ast],
-		inputPath,
-		checker,
-		jsImports,
-		bundledPackages,
-		preambles,
-		options.bundledDirs ?? new Set(),
-		{ ...options, outputDir },
-	);
-
-	const errors = checker.check(ast);
-	checker.reportUnusedImports();
-	if (errors.length > 0) {
-		throw new Error(errors.map((e) => e.message).join("\n"));
-	}
-	for (const w of checker.warnings) log.warn(w);
+	const { outputFile = null, dumpTokens = false, dumpAst = false } = options;
 
-	if (pkgTarget === "wasm" || pkgTarget === "both") {
-		registerWasmUnit(options, {
-			key: resolve(inputPath),
-			pkgName: checker.pkgName,
-			target: pkgTarget,
-			programs: [ast],
-			types: checker.types,
-			bundledPackages,
-		});
+	if (dumpTokens || dumpAst) {
+		let source;
+		try {
+			source = readFileSync(inputPath, "utf8");
+		} catch (e) {
+			throw new Error(`cannot read '${inputPath}': ${e.message}`);
+		}
+		const tokens = new Lexer(source, basename(inputPath)).tokenize();
+		if (dumpTokens) return { tokens };
+		const ast = new Parser(tokens, basename(inputPath), source).parse();
+		ast._source = source;
+		ast._path = resolve(inputPath);
+		return { ast };
 	}
 
-	const isStrict = Boolean(options.strict || pkgTarget === "both");
-	const cg = new CodeGen(checker, jsImports, bundledPackages, {
-		target: pkgTarget,
-		strict: isStrict,
+	return compileFiles([resolve(inputPath)], {
+		...options,
+		outputDir:
+			options.outputDir ??
+			(outputFile ? dirname(resolve(outputFile)) : resolve(".")),
 	});
-	const mainJs =
-		pkgTarget === "wasm" ? wasmMarker(checker.pkgName) : cg.generate(ast);
-	logWarnings(cg.warnings);
-
-	const mainChunk = {
-		js: mainJs,
-		sourceFiles: [resolve(inputPath)],
-		sourcesContent: [source],
-		mappings: pkgTarget === "wasm" ? [] : cg.getMappings(),
-	};
-
-	const merged = mergeCompilationChunks([...preambles, mainChunk]);
-	finishStdlibJs(merged, options, [ast], pkgTarget);
-	const { wasm, wat, wasmOptInfo, wasmSourceMap } = finishWasm(
-		{ ...options, sourceMap },
-		merged,
-		pkgTarget,
-		outputDir,
-	);
-
-	if (sourceMap && !options.isDependency) {
-		attachSourceMap(merged, outputDir);
-	}
-
-	return {
-		js: merged.js,
-		css: cg.getCss(),
-		target: pkgTarget,
-		wasm,
-		wat,
-		wasmOptInfo,
-		wasmSourceMap,
-	};
 }
 
 export function compileDir(dir, options = {}) {
@@ -518,6 +444,8 @@ export function compileFiles(files, options = {}) {
 		jsRootPrograms: options.jsRootPrograms ?? [],
 		rootDir: options.rootDir ?? fromDir,
 		bundledDirs: options.bundledDirs ?? new Set(),
+		typeOwners: options.typeOwners ?? new Map(),
+		jsonBuild: options.jsonBuild ?? { js: false, wasm: false },
 	};
 	const outputDir =
 		options.outputDir ??
@@ -545,6 +473,15 @@ export function compileFiles(files, options = {}) {
 		);
 	const pkgName = pkgNames[0];
 
+	// Local packages share one flat JS module: a type name another package
+	// already declared is renamed here (`Pt` -> `Pt$$b`).
+	const renamedTypes = mangleLocalTypes(
+		programs,
+		resolve(fromDir),
+		pkgName,
+		options.typeOwners,
+	);
+
 	// Validate target directives across files in this package
 	const declaredTargets = new Map();
 	for (const p of programs) {
@@ -571,6 +508,11 @@ export function compileFiles(files, options = {}) {
 	);
 	for (const p of programs) p.target = pkgTarget;
 
+	// The test runner's driver is a source that exists only in memory: it is
+	// parsed and checked with the package but never written next to it.
+	const driver = testDriverProgram(options, programs, pkgTarget, fromDir);
+	if (driver) programs.push(driver);
+
 	// ── 2. Resolve imports ────────────────────────────────────────
 	const checker = new TypeChecker();
 	checker.target = pkgTarget;
@@ -593,7 +535,7 @@ export function compileFiles(files, options = {}) {
 
 	// ── 3. Type-check ─────────────────────────────────────────────
 	const errors = checker.checkAll(programs);
-	const codecs = checkJsonCodecs(
+	const codecs = checkGeneratedGo(
 		programs,
 		checker,
 		preambles,
@@ -621,11 +563,12 @@ export function compileFiles(files, options = {}) {
 	const codegen = new CodeGen(checker, jsImports, bundledPackages, {
 		target: pkgTarget,
 		strict: isStrict,
+		natives: options.natives,
 	});
 	const mainJs =
 		emitTarget === "wasm"
 			? wasmMarker(pkgName)
-			: codegen.generateAll(programs, {
+			: codegen.generateAll([...programs, ...codecs], {
 					isTest: options.isTest ?? false,
 				});
 	logWarnings(codegen.warnings);
@@ -638,8 +581,8 @@ export function compileFiles(files, options = {}) {
 	};
 
 	const merged = mergeCompilationChunks([...preambles, mainChunk]);
-	finishStdlibJs(merged, options, programs, emitTarget);
-	const { wasm, wat, wasmOptInfo, wasmSourceMap } = finishWasm(
+	finishStdlibJs(merged, options, [...programs, ...codecs], emitTarget);
+	const { wasm, wat, wasmOptInfo, wasmSourceMap, stringTable } = finishWasm(
 		options,
 		merged,
 		pkgTarget,
@@ -658,15 +601,22 @@ export function compileFiles(files, options = {}) {
 	return {
 		pkgName,
 		target: pkgTarget,
+		elemAddrTaken:
+			checker.depElemAddrTaken === true ||
+			programs.some((p) => takesElementAddress(p.decls)),
 		js: merged.js,
 		css: allCss,
 		wasm,
 		wat,
 		wasmOptInfo,
 		wasmSourceMap,
-		programs,
+		stringTable,
+		programs: driver ? programs.filter((p) => p !== driver) : programs,
 		exportedSymbols: checker.getExportedSymbols(),
-		exportedTypes: checker.getExportedTypes(),
+		exportedTypes: withOriginalTypeNames(
+			checker.getExportedTypes(),
+			renamedTypes,
+		),
 		exportedConsts: checker.getExportedConsts(),
 		sourceFiles: merged.sourceFiles,
 		sourcesContent: merged.sourcesContent,
@@ -674,6 +624,34 @@ export function compileFiles(files, options = {}) {
 	};
 }
 
+// Parses the in-memory driver source `options.testDriver(programs, target)`
+// returns (null: no driver) as a `_test.go` file of the package.
+function testDriverProgram(options, programs, pkgTarget, fromDir) {
+	if (!options.testDriver || options.isDependency) return null;
+	const source = options.testDriver(programs, pkgTarget);
+	if (!source) return null;
+	const filename = "gofront_testmain_test.go";
+	const ast = new Parser(
+		new Lexer(source, filename).tokenize(),
+		filename,
+		source,
+	).parse();
+	ast._source = source;
+	ast._path = join(fromDir, filename);
+	ast._declaredTarget = ast.target;
+	ast.target = pkgTarget;
+	return ast;
+}
+
+// The renamed ASTs must not be reused from the parse cache by a later build.
+function mangleLocalTypes(programs, pkgKey, pkgName, owners) {
+	const renamed = mangleCollidingTypes(programs, pkgKey, pkgName, owners);
+	const locals = renameLocalTypes(programs);
+	if (renamed.size > 0 || locals)
+		for (const p of programs) _parseCache.delete(p._path);
+	return renamed;
+}
+
 export function compileWasm(source, options = {}) {
 	const filename = options.filename ?? "main.go";
 	const tokens = new Lexer(source, filename).tokenize();

package/src/lexer/lexer.js

@@ -119,6 +119,31 @@ const SIMPLE_ESCAPES = {
 	"\\": 92,
 };
 
+// Decodes the raw bytes of consecutive \xNN / octal escapes in a string
+// literal: valid UTF-8 sequences become their code point, every other byte
+// 0x80-0xff becomes the lone surrogate U+DC00+byte so the runtime can turn it
+// back into that byte when the string is converted to []byte.
+function decodeRawBytes(bytes) {
+	const dec = new TextDecoder("utf-8", { fatal: true });
+	let out = "";
+	for (let i = 0; i < bytes.length; ) {
+		const b = bytes[i];
+		const n = b < 0x80 ? 1 : b >= 0xf0 ? 4 : b >= 0xe0 ? 3 : b >= 0xc0 ? 2 : 0;
+		if (n > 0 && i + n <= bytes.length) {
+			try {
+				out += dec.decode(Uint8Array.from(bytes.slice(i, i + n)));
+				i += n;
+				continue;
+			} catch {
+				// invalid sequence: fall through to the lone-byte case
+			}
+		}
+		out += String.fromCharCode(b < 0x80 ? b : 0xdc00 + b);
+		i++;
+	}
+	return out;
+}
+
 export class Lexer {
 	constructor(source, filename = null) {
 		this.src = source;
@@ -130,6 +155,7 @@ export class Lexer {
 		this.target = null;
 		this.targetDirective = null;
 		this.intrinsics = new Map(); // line -> name of a //gofront:intrinsic
+		this.natives = new Map(); // line -> name of a //gofront:native
 		this.embedDirectives = []; // {line, col} of each //go:embed line comment
 	}
 
@@ -185,6 +211,9 @@ export class Lexer {
 					this.embedDirectives.push({ line, col });
 				const intrinsic = comment.match(/^\/\/gofront:intrinsic\s+(\S+)/);
 				if (intrinsic) this.intrinsics.set(line, intrinsic[1]);
+				const native = comment.match(/^\/\/gofront:native\s+(\S+)/);
+				if (native) this.natives.set(line, native[1]);
+				if (!intrinsic && !native) this.forwardDirectives(line, start);
 				const match = comment.match(/^\/\/gofront:target\s+(\S+)/);
 				if (match) {
 					this.target = match[1];
@@ -249,7 +278,7 @@ export class Lexer {
 				if (parts === null) parts = [];
 				if (this.pos > start) parts.push(this.src.slice(start, this.pos));
 				this.advance(); // skip \
-				parts.push(String.fromCodePoint(this.readEscape('"')));
+				parts.push(this.readStringEscape());
 				start = this.pos;
 			} else {
 				this.advance();
@@ -258,10 +287,41 @@ export class Lexer {
 		throw new LexError("Unterminated string", l, c, this.filename, this.src);
 	}
 
+	// The escape after a consumed backslash in a string literal.  \xNN and octal
+	// escapes are raw bytes; a run of them is decoded together so "\xc3\xa9" is
+	// "é" and "\xff" is one invalid byte (a lone surrogate, see decodeRawBytes).
+	readStringEscape() {
+		if (!this.isByteEscapeAt(this.pos))
+			return String.fromCodePoint(this.readEscape('"'));
+		const bytes = [this.readEscape('"')];
+		while (this.peek() === "\\" && this.isByteEscapeAt(this.pos + 1)) {
+			this.advance();
+			bytes.push(this.readEscape('"'));
+		}
+		return decodeRawBytes(bytes);
+	}
+
+	// A doc comment between a directive and its func carries the directive down
+	// so the parser still finds it on the line above the func.
+	forwardDirectives(line, start) {
+		const before = this.src.slice(
+			this.src.lastIndexOf("\n", start - 1) + 1,
+			start,
+		);
+		if (!/^[ \t]*$/.test(before)) return;
+		for (const map of [this.intrinsics, this.natives])
+			if (map.has(line - 1)) map.set(line, map.get(line - 1));
+	}
+
+	isByteEscapeAt(i) {
+		const n = this.src[i];
+		return n === "x" || (n >= "0" && n <= "7");
+	}
+
 	// Reads the escape after a consumed backslash and returns its code point.
 	// `quote` is the enclosing delimiter: `\'` is valid only in runes and
-	// `\"` only in strings. `\xNN` and octal escapes give U+0000-U+00FF (see
-	// plan §1.1: strings are JS strings, so \x80-\xff are code points, not raw bytes).
+	// `\"` only in strings. `\xNN` and octal escapes give 0-255 (a code point in
+	// rune literals; string literals decode runs of them as UTF-8 bytes).
 	readEscape(quote) {
 		const l = this.line,
 			c = this.col;
@@ -328,6 +388,7 @@ export class Lexer {
 		} else if (this.peek() !== "'") {
 			code = this.src.codePointAt(this.pos);
 			this.advance();
+			if (code > 0xffff) this.pos++; // astral: skip the low surrogate
 		} else {
 			throw new LexError("Empty rune literal", l, c, this.filename, this.src);
 		}
@@ -451,12 +512,10 @@ export class Lexer {
 				break;
 			}
 		}
-		if (
-			this.pos + 1 < this.src.length &&
-			this.src.charCodeAt(this.pos) === 46
-		) {
+		if (this.src.charCodeAt(this.pos) === 46) {
+			// `1.` is a float literal; `1..` and `1.x` keep the dot separate.
 			const next = this.src.charCodeAt(this.pos + 1);
-			if (next >= 48 && next <= 57) {
+			if (this._dotContinuesNumber(next)) {
 				isFloat = true;
 				n += this.advance();
 				while (this.pos < this.src.length) {
@@ -469,6 +528,7 @@ export class Lexer {
 						break;
 					}
 				}
+				if (n.endsWith(".")) n += "0";
 			}
 		}
 		if (this.pos < this.src.length) {
@@ -481,6 +541,32 @@ export class Lexer {
 		return { n, isFloat };
 	}
 
+	// Whether the "." at the cursor belongs to the decimal literal before it.
+	// `next` is the code after the dot: a digit, an exponent (`1.e3`), an
+	// imaginary suffix (`1.i`) or anything that cannot start a selector or a
+	// range-style `..` makes `1.` a float; `1.x` and `1..` keep the dot apart.
+	_dotContinuesNumber(next) {
+		if (Number.isNaN(next)) return true;
+		if (next >= 48 && next <= 57) return true;
+		if (next === 46) return false;
+		const after = this.src.charCodeAt(this.pos + 2);
+		const identAfter =
+			(after >= 97 && after <= 122) ||
+			(after >= 65 && after <= 90) ||
+			after === 95 ||
+			(after >= 48 && after <= 57);
+		if (next === 101 || next === 69) {
+			return (after >= 48 && after <= 57) || after === 43 || after === 45;
+		}
+		if (next === 105) return !identAfter;
+		return !(
+			(next >= 97 && next <= 122) ||
+			(next >= 65 && next <= 90) ||
+			next === 95 ||
+			next > 127
+		);
+	}
+
 	_startsUnicodeIdent() {
 		return (
 			this.peek().charCodeAt(0) > 127 &&
@@ -584,6 +670,7 @@ export class Lexer {
 		this.tokens.target = this.target;
 		this.tokens.targetDirective = this.targetDirective;
 		this.tokens.intrinsics = this.intrinsics;
+		this.tokens.natives = this.natives;
 		this.tokens.embedDirectives = this.embedDirectives;
 		return this.tokens;
 	}
@@ -617,8 +704,10 @@ export class Lexer {
 
 	_readAmpOp(l, c) {
 		if (this.match("&")) this.push(T.AND, "&&", l, c);
-		else if (this.match("^")) this.push(T.AND_NOT, "&^", l, c);
-		else if (this.match("=")) this.push(T.AMP_ASSIGN, "&=", l, c);
+		else if (this.match("^")) {
+			if (this.match("=")) this.push(T.AND_NOT_ASSIGN, "&^=", l, c);
+			else this.push(T.AND_NOT, "&^", l, c);
+		} else if (this.match("=")) this.push(T.AMP_ASSIGN, "&=", l, c);
 		else this.push(T.AMP, "&", l, c);
 	}
 
@@ -660,7 +749,12 @@ export class Lexer {
 			case ".":
 				if (/[0-9]/.test(this.peek() ?? "")) {
 					const { n } = this._readLeadingDotFloat();
-					this.push(T.FLOAT, n, l, c);
+					if (this.src[this.pos] === "i") {
+						this.advance();
+						this.push(T.IMAG, n, l, c);
+					} else {
+						this.push(T.FLOAT, n, l, c);
+					}
 				} else if (this.peek() === "." && this.peek(1) === ".") {
 					this.advance();
 					this.advance();