@bramas/simulant-lang 0.0.0-stage → 0.1.1

package/dist/elab/lower.js

@@ -0,0 +1,372 @@
+import { T } from "../types.js";
+/** Expressions that can be substituted for a parameter without memory. */
+function isConstant(e) {
+    switch (e.t) {
+        case 'num':
+        case 'str':
+        case 'bool':
+        case 'ctor':
+        case 'unit': return true;
+        case 'name': return /^[A-Z]\w*\.\w+$/.test(e.name); // Move.cw
+        case 'list': return e.items.every(isConstant);
+        case 'unop': return e.op === '-' && e.e.t === 'num';
+        default: return false;
+    }
+}
+export function lower(name, procs, main, sensors, oblivious, host) {
+    const locals = new Map();
+    let fresh = 0;
+    const sensorConsts = new Map(sensors.map(s => [s, { e: 'get', on: { e: 'var', name: 'view' }, field: s }]));
+    // 1. Inlining
+    let waitLocal;
+    const inline = (stmts, env, stack) => stmts.flatMap((s) => {
+        const at = { span: s.span, env };
+        switch (s.s) {
+            case 'move': return [{ ...at, k: 'move', dir: s.dir, actions: s.actions }];
+            case 'stay': return [{ ...at, k: 'stay', actions: s.actions }];
+            case 'halt': return [{ ...at, k: 'halt', actions: s.actions }];
+            case 'wait': {
+                if (!waitLocal) {
+                    waitLocal = `wait_${fresh++}`;
+                    locals.set(waitLocal, { type: T.nat, span: s.span });
+                }
+                // `#wait` is not a surface name: the counter cannot clash with the algorithm's names.
+                const w = { ...env, locals: new Map(env.locals).set('#wait', waitLocal) };
+                const span = s.span;
+                const left = { t: 'name', name: '#wait', span };
+                return [
+                    { span, env, k: 'set', local: waitLocal, value: s.count },
+                    { span, env: w, k: 'while', cond: { t: 'binop', op: '>', l: left, r: { t: 'num', value: 0, span }, span }, body: [
+                            { span, env: w, k: 'set', local: waitLocal, value: { t: 'binop', op: '-', l: left, r: { t: 'num', value: 1, span }, span } },
+                            { span, env: w, k: 'stay' },
+                        ] },
+                ];
+            }
+            case 'repeat': return [{ ...at, k: 'repeat', body: inline(s.body, env, stack) }];
+            case 'while': return [{ ...at, k: 'while', cond: s.cond, body: inline(s.body, env, stack) }];
+            case 'if': return [{ ...at, k: 'if', cond: s.cond, then: inline(s.then, env, stack), else: inline(s.else ?? [], env, stack) }];
+            case 'let': {
+                // Locals are scoped to their procedure (one declaration per name).
+                if (env.locals.has(s.name))
+                    host.fail(`${s.name} is already declared in this procedure`, s.span);
+                const local = `${s.name}_${fresh++}`;
+                locals.set(local, { type: s.type && host.type(s.type), span: s.span });
+                env.locals.set(s.name, local);
+                return [{ ...at, k: 'set', local, value: s.value }];
+            }
+            case 'assign': {
+                if (s.name === 'pub')
+                    return [{ ...at, k: 'set', local: 'pub', value: s.value }];
+                const local = env.locals.get(s.name) ?? host.fail(`${s.name} is not a local variable (declare it with let)`, s.span);
+                return [{ ...at, k: 'set', local, value: s.value }];
+            }
+            case 'call': {
+                const p = procs.get(s.name) ?? host.fail(`unknown procedure ${s.name}`, s.span);
+                if (stack.includes(s.name))
+                    host.fail(`procedures cannot be recursive: ${[...stack, s.name].join(' → ')}`, s.span);
+                if (s.args.length !== p.params.length)
+                    host.fail(`${s.name} takes ${p.params.length} argument(s)`, s.span);
+                const callee = { consts: new Map(env.consts), locals: new Map(), calls: [...env.calls, s.span.start.line] };
+                const setup = [];
+                p.params.forEach((param, i) => {
+                    const arg = s.args[i];
+                    if (isConstant(arg)) {
+                        callee.consts.set(param.name, host.tr(arg, new Map(), sensorConsts));
+                    }
+                    else {
+                        const local = `${param.name}_${fresh++}`;
+                        locals.set(local, { type: host.type(param.type), span: param.span });
+                        callee.locals.set(param.name, local);
+                        setup.push({ span: arg.span, env, k: 'set', local, value: arg });
+                    }
+                });
+                return [...setup, ...inline(p.body, callee, [...stack, s.name])];
+            }
+        }
+    });
+    const program = inline(main, { consts: new Map(), locals: new Map(), calls: [] }, []);
+    const resumes = new Map();
+    const halts = new Map();
+    const order = [];
+    // Program-counter values are numbered by kind in program order, not by line, so that an
+    // edit that only moves lines (a comment) keeps the experiment, its states and Lean names.
+    const counts = new Map();
+    const pcName = (prefix, _line) => {
+        const k = (counts.get(prefix) ?? 0) + 1;
+        counts.set(prefix, k);
+        return `${prefix}${k}`;
+    };
+    const typedLocals = () => [...locals].filter(([, l]) => l.type).map(([n, l]) => ({ name: n, type: l.type }));
+    const ended = (cont) => cont.every(f => f.rest.length === 0 && !f.loop);
+    let tmp = 0;
+    /** The type of the program counter; its constructors are known at the end. */
+    const PC = { k: 'enum', name: `${name}PC`, ctors: [] };
+    /** The decision ending this activation, resuming at `pc` (with continuation `cont`), with the activation's actions. */
+    const decision = (move, pc, st, span, actions = {}, cont = [], line = span?.start.line ?? null) => {
+        const mem = {};
+        const restart = pc === 'start';
+        for (const l of typedLocals()) {
+            // At a restart nothing is kept: locals go back to their initial values, so equal states stay equal.
+            mem[l.name] = restart
+                ? { e: 'lit', type: l.type, value: defaultValue(l.type, host, span) }
+                : { e: 'var', name: st.values.get(l.name) ?? host.fail('internal: lost local', span) };
+        }
+        if (!restart) {
+            for (const [l, info] of locals) {
+                // An untyped local only matters if what follows reads it before writing it.
+                if (!info.type && st.values.has(l) && live(cont, l) === 'read') {
+                    host.fail(`${l.replace(/_\d+$/, '')} is kept across a move: give its type (let x : T := ...)`, info.span);
+                }
+            }
+        }
+        return {
+            e: 'record',
+            fields: { move, pub: { e: 'var', name: st.pub }, mem: { e: 'record', fields: { pc: { e: 'lit', type: PC, value: pc }, ...mem } }, ...actions },
+            // The statement that decides (display only, see `Expr.at`).
+            ...(line !== null ? { at: line } : {}),
+        };
+    };
+    const actionsOf = (s, st) => {
+        const out = {};
+        for (const item of s.actions ?? []) {
+            if (item.t !== 'field' || !item.value)
+                continue;
+            if (['move', 'pub', 'mem'].includes(item.name))
+                host.fail(`${item.name} is not an action`, item.span);
+            out[item.name] = tr(item.value, s.env, st);
+        }
+        return out;
+    };
+    const scopeOf = (env, st) => {
+        const scope = new Map([['pub', st.pub]]);
+        for (const [surface, local] of env.locals) {
+            const v = st.values.get(local);
+            if (v)
+                scope.set(surface, v);
+        }
+        return scope;
+    };
+    const tr = (e, env, st) => host.tr(e, scopeOf(env, st), new Map([...sensorConsts, ...env.consts]));
+    const run = (cont, st, seen) => {
+        if (!cont.length) {
+            // End of `main`: nothing more this activation; start again at the next one.
+            return decision({ e: 'lit', type: T.move, value: 'stay' }, 'start', st, program[0]?.span ?? main[0]?.span, {}, [], null);
+        }
+        const [frame, ...outer] = cont;
+        if (!frame.rest.length)
+            return run(frame.loop ? [{ rest: [frame.loop] }, ...outer] : outer, st, seen);
+        const [s, ...rest] = frame.rest;
+        const next = [{ rest, loop: frame.loop }, ...outer];
+        switch (s.k) {
+            case 'move':
+            case 'stay': {
+                const move = s.k === 'move' ? tr(s.dir, s.env, st) : { e: 'lit', type: T.move, value: 'stay' };
+                let pc = 'start';
+                if (!ended(next)) {
+                    let r = resumes.get(s);
+                    if (!r) {
+                        r = { pc: pcName(s.k === 'move' ? 'move' : 'stay', s.span.start.line), cont: next, line: s.span.start.line, calls: s.env.calls };
+                        resumes.set(s, r);
+                        order.push(r);
+                    }
+                    pc = r.pc;
+                    if (oblivious)
+                        host.fail(`${name}: in an oblivious model, the algorithm cannot continue after this ${s.k} (it would need memory)`, s.span);
+                }
+                return decision(move, pc, st, s.span, actionsOf(s, st), next);
+            }
+            case 'halt': {
+                if (oblivious)
+                    host.fail(`${name}: in an oblivious model, an algorithm cannot halt (it would need memory)`, s.span);
+                let pc = halts.get(s);
+                if (!pc) {
+                    pc = pcName('halt', s.span.start.line);
+                    halts.set(s, pc);
+                }
+                return decision({ e: 'lit', type: T.move, value: 'stay' }, pc, st, s.span, actionsOf(s, st));
+            }
+            case 'set': {
+                const value = tr(s.value, s.env, st);
+                const v = `${s.local === 'pub' ? 'pub' : s.local.replace(/_\d+$/, '')}_v${tmp++}`;
+                const st2 = s.local === 'pub' ? { ...st, pub: v } : { ...st, values: new Map(st.values).set(s.local, v) };
+                return { e: 'let', name: v, value, body: run(next, st2, seen) };
+            }
+            case 'if':
+                return {
+                    e: 'if', cond: tr(s.cond, s.env, st),
+                    then: run([{ rest: s.then }, ...next], st, seen),
+                    else: run([{ rest: s.else }, ...next], st, seen),
+                };
+            case 'repeat':
+            case 'while': {
+                if (seen.has(s))
+                    host.fail(`this loop can run again without move, stay or halt`, s.span);
+                const seen2 = new Set(seen).add(s);
+                const body = run([{ rest: s.body, loop: s }, ...next], st, seen2);
+                return s.k === 'repeat' ? body : { e: 'if', cond: tr(s.cond, s.env, st), then: body, else: run(next, st, seen) };
+            }
+        }
+    };
+    const readsIn = (e, env, local) => {
+        if (!e || typeof e !== 'object')
+            return false;
+        if (Array.isArray(e))
+            return e.some(x => readsIn(x, env, local));
+        const o = e;
+        if (o.t === 'name' && o.name && env.locals.get(o.name) === local)
+            return true;
+        return Object.entries(o).some(([k, v]) => k !== 'span' && readsIn(v, env, local));
+    };
+    const seq = (stmts, local, loops) => {
+        for (const s of stmts) {
+            const u = stmt(s, local, loops);
+            if (u !== 'none')
+                return u;
+        }
+        return 'none';
+    };
+    const stmt = (s, local, loops) => {
+        switch (s.k) {
+            case 'move': return readsIn(s.dir, s.env, local) || readsIn(s.actions, s.env, local) ? 'read' : 'none';
+            case 'stay':
+            case 'halt': return readsIn(s.actions, s.env, local) ? 'read' : 'none';
+            case 'set':
+                if (readsIn(s.value, s.env, local))
+                    return 'read';
+                return s.local === local ? 'written' : 'none';
+            case 'if': {
+                if (readsIn(s.cond, s.env, local))
+                    return 'read';
+                const a = seq(s.then, local, loops), b = seq(s.else, local, loops);
+                return a === 'read' || b === 'read' ? 'read' : a === 'written' && b === 'written' ? 'written' : 'none';
+            }
+            case 'repeat':
+            case 'while': {
+                if (loops.has(s))
+                    return 'none';
+                const inner = new Set(loops).add(s);
+                if (s.k === 'while' && readsIn(s.cond, s.env, local))
+                    return 'read';
+                const u = seq(s.body, local, inner);
+                return u === 'read' ? 'read' : s.k === 'repeat' ? u : 'none';
+            }
+        }
+    };
+    const live = (cont, local) => {
+        const loops = new Set();
+        for (const f of cont) {
+            const u = seq(f.rest, local, loops);
+            if (u !== 'none')
+                return u;
+            if (f.loop) {
+                const again = stmt(f.loop, local, loops);
+                if (again !== 'none')
+                    return again;
+                loops.add(f.loop);
+            }
+        }
+        return 'none';
+    };
+    // 3. The step function: one case per resume point, generated until no new one appears.
+    const atResume = () => ({ pub: 'pub', values: new Map(typedLocals().map(l => [l.name, `${l.name}_m`])) });
+    /** Reads the saved locals from memory. */
+    const withLocals = (body) => typedLocals().reduceRight((acc, l) => ({ e: 'let', name: `${l.name}_m`, value: { e: 'get', on: { e: 'var', name: 'mem' }, field: l.name }, body: acc }), body);
+    const cases = { start: withLocals(run([{ rest: program }], atResume(), new Set())) };
+    for (let i = 0; i < order.length; i++)
+        cases[order[i].pc] = withLocals(run(order[i].cont, atResume(), new Set()));
+    for (const [s, pc] of halts)
+        cases[pc] = withLocals(decision({ e: 'lit', type: T.move, value: 'stay' }, pc, atResume(), s.span));
+    const pcs = Object.keys(cases);
+    const lines = { start: (program[0] ?? main[0])?.span.start.line ?? 1 };
+    for (const r of order)
+        lines[r.pc] = r.line;
+    for (const [s, pc] of halts)
+        lines[pc] = s.span.start.line;
+    const calls = {};
+    const addCalls = (pc, c) => { if (c?.length)
+        calls[pc] = c; };
+    addCalls('start', program[0]?.env.calls);
+    for (const r of order)
+        addCalls(r.pc, r.calls);
+    for (const [s, pc] of halts)
+        addCalls(pc, s.env.calls);
+    const locs = typedLocals();
+    if (pcs.length === 1 && !locs.length) {
+        // No memory: the program restarts at every activation.
+        const step = stripMem(cases.start);
+        return { mem: T.unit, initMem: { e: 'lit', type: T.unit, value: null }, step, lines, calls };
+    }
+    if (oblivious && locs.length)
+        host.fail(`${name}: in an oblivious model, locals cannot be kept across activations`, locals.get(locs[0].name).span);
+    PC.ctors.push(...pcs);
+    const fields = [{ name: 'pc', type: PC }, ...locs];
+    return {
+        mem: T.record(fields),
+        initMem: { e: 'record', fields: { pc: { e: 'lit', type: PC, value: 'start' }, ...Object.fromEntries(locs.map(l => [l.name, { e: 'lit', type: l.type, value: defaultValue(l.type, host, locals.get(l.name).span) }])) } },
+        step: { e: 'match', on: { e: 'get', on: { e: 'var', name: 'mem' }, field: 'pc' }, cases },
+        line: { e: 'match', on: { e: 'get', on: { e: 'var', name: 'mem' }, field: 'pc' }, cases: Object.fromEntries(pcs.map(pc => [pc, { e: 'lit', type: T.nat, value: lines[pc] }])) },
+        lines,
+        calls,
+    };
+}
+/**
+ * Decisions that omit an action keep its default, but the IR (and Lean) need every decision of
+ * a step to have the same fields: once the model is known, each action that some decision sets
+ * is added, with its default, to the decisions that omit it. Steps that set no action are
+ * unchanged.
+ */
+export function fillActionDefaults(step, actions) {
+    const isDecision = (x) => x.e === 'record' && !!x.fields.move && !!x.fields.pub && !!x.fields.mem;
+    const used = new Set();
+    mapExprs(step, x => {
+        if (isDecision(x))
+            for (const a of actions)
+                if (x.fields[a.name])
+                    used.add(a.name);
+        return x;
+    });
+    if (!used.size)
+        return step;
+    return mapExprs(step, x => {
+        if (!isDecision(x))
+            return x;
+        const fields = { ...x.fields };
+        for (const a of actions)
+            if (used.has(a.name) && !fields[a.name])
+                fields[a.name] = { e: 'lit', type: a.type, value: a.default };
+        return { ...x, fields };
+    });
+}
+/** Without memory, decisions do not mention it. */
+function stripMem(e) {
+    return mapExprs(e, x => x.e === 'record' && x.fields.mem && x.fields.move
+        ? { ...x, fields: { ...x.fields, mem: { e: 'var', name: 'mem' } } } : x);
+}
+function mapExprs(e, f) {
+    const walk = (x) => {
+        if (!x || typeof x !== 'object')
+            return x;
+        if (Array.isArray(x))
+            return x.map(walk);
+        const o = Object.fromEntries(Object.entries(x).map(([k, v]) => [k, walk(v)]));
+        return typeof o.e === 'string' ? f(o) : o;
+    };
+    return walk(e);
+}
+function defaultValue(t, host, span) {
+    switch (t.k) {
+        case 'nat':
+        case 'int': return 0;
+        case 'bool':
+        case 'frame': return false;
+        case 'string': return '';
+        case 'unit':
+        case 'option': return null;
+        case 'list':
+        case 'multiset': return [];
+        case 'move': return 'stay';
+        case 'enum': return t.ctors[0];
+        case 'record': return Object.fromEntries(t.fields.map(f => [f.name, defaultValue(f.type, host, span)]));
+        default: return host.fail(`a local of type ${t.k} cannot be kept in memory`, span);
+    }
+}

package/dist/elab/node.js

@@ -0,0 +1,29 @@
+/** Node.js helpers: reading `.sim` programs from disk. Not for the browser. */
+import { readFileSync } from "node:fs";
+import { dirname, join, resolve, sep } from "node:path";
+import { createRequire } from "node:module";
+import { Elaborator } from "./elaborate.js";
+import { Program } from "./program.js";
+/** Resolve data through its package, both in the workspace and in an npm installation. */
+export const stdlibDir = join(dirname(createRequire(import.meta.url).resolve('@bramas/simulant-stdlib/package.json')), 'sim') + sep;
+/** A host reading files from disk; `std/...` imports resolve in `stdlib`. */
+export function nodeHost(stdlib = stdlibDir) {
+    return {
+        read: p => {
+            try {
+                return readFileSync(p, 'utf8');
+            }
+            catch {
+                return undefined;
+            }
+        },
+        resolve: (from, spec) => spec.startsWith('std/') ? join(stdlib, spec.slice(4)) : resolve(dirname(from), spec),
+    };
+}
+export function loadProgram(entry, stdlib = stdlibDir) {
+    return new Program(nodeHost(stdlib), entry);
+}
+/** An elaborator over a module of the standard library (`models.sim` imports all components and sensors). */
+export function stdlibElaborator(module = 'models.sim') {
+    return new Elaborator(loadProgram(join(stdlibDir, module)));
+}

package/dist/elab/program.js

@@ -0,0 +1,45 @@
+import { LexError } from "../parse/lexer.js";
+import { parse, ParseError } from "../parse/parser.js";
+export class Program {
+    host;
+    entry;
+    modules = new Map();
+    diagnostics = [];
+    /** Every file read, including those that failed: watched for changes. */
+    files = new Set();
+    constructor(host, entry) {
+        this.host = host;
+        this.entry = entry;
+        this.load(entry);
+    }
+    load(path, importedAt) {
+        if (this.modules.has(path) || this.files.has(path))
+            return;
+        this.files.add(path);
+        const source = this.host.read(path);
+        if (source === undefined) {
+            this.diagnostics.push({ message: `cannot read module ${path}`, span: importedAt });
+            return;
+        }
+        let file;
+        try {
+            file = parse(source, path);
+        }
+        catch (e) {
+            if (e instanceof ParseError || e instanceof LexError) {
+                this.diagnostics.push({ message: e.message, span: e.span });
+                return;
+            }
+            throw e;
+        }
+        this.modules.set(path, file);
+        for (const d of file.decls) {
+            if (d.d === 'import')
+                this.load(this.host.resolve(path, d.path), d.span);
+        }
+    }
+    /** All declarations of the program, entry module last. */
+    decls() {
+        return [...this.modules.values()].reverse().flatMap(f => f.decls);
+    }
+}

package/dist/ir.js

@@ -0,0 +1,9 @@
+/**
+ * The Simulant intermediate representation (plan, Sections 3 and 6.1).
+ *
+ * Everything here is plain JSON data: the builder (and later the parser)
+ * produce it, the engine interprets it, and the Lean generator translates it.
+ * Topologies and rounds are not written in the IR: they belong to the
+ * semantic core, and the IR only names them.
+ */
+export const IR_VERSION = 1;

package/dist/layout.js

@@ -0,0 +1,56 @@
+import { resolvePub, T } from "./types.js";
+export const MODEL = 'model';
+export function layout(exp) {
+    const errors = [];
+    const unique = (what, items) => {
+        const seen = new Map();
+        for (const item of items) {
+            const other = seen.get(item.name);
+            if (other !== undefined)
+                errors.push(`${what} '${item.name}' is declared by both ${other} and ${item.owner}`);
+            seen.set(item.name, item.owner);
+        }
+        return items;
+    };
+    const owned = (owner, items) => (items ?? []).map(x => ({ ...x, owner }));
+    const { model, algorithm } = exp;
+    const world = unique('world field', model.components.flatMap(c => owned(c.name, c.world)));
+    const agentFields = unique('agent field', model.components.flatMap(c => owned(c.name, c.agent)));
+    const flags = unique('flag', model.components.flatMap(c => owned(c.name, c.flags)));
+    const sensors = unique('sensor', [...owned(MODEL, model.sensors), ...model.components.flatMap(c => owned(c.name, c.sensors))])
+        .map(s => ({ ...s, type: resolvePub(s.type, algorithm.pub) }));
+    const actions = unique('action', model.components.flatMap(c => owned(c.name, c.actions)));
+    const reserved = new Set(['pos', 'pub', 'mem']);
+    for (const f of [...agentFields, ...flags]) {
+        if (reserved.has(f.name))
+            errors.push(`agent field '${f.name}' of ${f.owner} is reserved`);
+    }
+    for (const f of agentFields) {
+        if (flags.some(g => g.name === f.name))
+            errors.push(`'${f.name}' is both an agent field and a flag`);
+    }
+    for (const a of actions) {
+        if (['move', 'pub', 'mem'].includes(a.name))
+            errors.push(`action '${a.name}' of ${a.owner} is reserved`);
+    }
+    const view = [];
+    for (const name of algorithm.uses) {
+        const sensor = sensors.find(s => s.name === name);
+        if (sensor)
+            view.push({ name, type: sensor.type });
+        else
+            errors.push(`algorithm ${algorithm.name} uses sensor '${name}', which the model does not provide`);
+    }
+    return {
+        world, agentFields, flags, sensors, actions, errors,
+        worldType: T.record(world.map(({ name, type }) => ({ name, type }))),
+        pub: algorithm.pub,
+        mem: algorithm.mem,
+        view: T.record(view),
+        decision: T.record([
+            { name: 'move', type: T.move }, { name: 'pub', type: algorithm.pub }, { name: 'mem', type: algorithm.mem },
+            ...actions.map(({ name, type }) => ({ name, type })),
+        ]),
+        defs: model.defs ?? [],
+    };
+}

package/dist/parse/ast.js

@@ -0,0 +1 @@
+export {};

package/dist/parse/lexer.js

@@ -0,0 +1,151 @@
+/**
+ * Lexer of `.sim` sources. Tokens keep their position and whether they start
+ * a line: the parser uses columns for layout, as Lean does.
+ */
+export const KEYWORDS = new Set([
+    'import', 'def', 'structure', 'inductive', 'sensor', 'task', 'goal', 'algorithm', 'experiment', 'where', 'let',
+    'if', 'then', 'else', 'match', 'with', 'fun', 'choose', 'such', 'that', 'labels', 'param', 'flag', 'uses',
+    'true', 'false', 'raise', 'nothing', 'action',
+]);
+/** Operators, longest first. Unicode and ASCII spellings are both accepted. */
+const OPERATORS = [
+    ':=', '=>', '->', '<-', '++', '==', '!=', '<=', '>=', '&&', '||',
+    '→', '←', '≠', '≤', '≥', '∧', '∨', '¬', '·', '×',
+    '=', '<', '>', '+', '-', '*', '/', '%', '!', '|', ',', ':', ';', '(', ')', '[', ']', '{', '}', '@',
+];
+const NORMALIZE = {
+    '->': '→', '<-': '←', '!=': '≠', '<=': '≤', '>=': '≥', '&&': '∧', '||': '∨', '==': '=',
+};
+export class LexError extends Error {
+    span;
+    constructor(message, span) {
+        super(message);
+        this.span = span;
+    }
+}
+const identStart = /[A-Za-z_À-ɏͰ-Ͽ]/;
+const identPart = /[A-Za-z0-9_'À-ɏͰ-Ͽ₀-₉]/;
+export function lex(source, file = '<input>') {
+    const tokens = [];
+    let i = 0, line = 1, col = 1;
+    let lineStart = true, spaced = true;
+    const pos = () => ({ line, col, offset: i });
+    const advance = (n = 1) => {
+        for (let j = 0; j < n; j++) {
+            if (source[i] === '\n') {
+                line++;
+                col = 1;
+            }
+            else
+                col++;
+            i++;
+        }
+    };
+    const fail = (msg, start) => { throw new LexError(msg, { file, start, end: pos() }); };
+    while (i < source.length) {
+        const c = source[i];
+        if (c === '\n') {
+            advance();
+            lineStart = true;
+            spaced = true;
+            continue;
+        }
+        if (c === ' ' || c === '\t' || c === '\r') {
+            advance();
+            spaced = true;
+            continue;
+        }
+        if (source.startsWith('--', i)) {
+            while (i < source.length && source[i] !== '\n')
+                advance();
+            spaced = true;
+            continue;
+        }
+        if (source.startsWith('/-', i)) {
+            const start = pos();
+            let depth = 0;
+            do {
+                if (source.startsWith('/-', i)) {
+                    depth++;
+                    advance(2);
+                }
+                else if (source.startsWith('-/', i)) {
+                    depth--;
+                    advance(2);
+                }
+                else if (i >= source.length)
+                    fail('unterminated comment', start);
+                else
+                    advance();
+            } while (depth > 0);
+            spaced = true;
+            continue;
+        }
+        const start = pos();
+        const push = (kind, text) => {
+            tokens.push({ kind, text, start, end: pos(), lineStart, spaced });
+            lineStart = false;
+            spaced = false;
+        };
+        const prev = tokens[tokens.length - 1];
+        const afterTerm = !!prev && !spaced && (prev.kind === 'ident' || prev.kind === 'field' || prev.kind === 'dotIdent'
+            || prev.kind === 'number' || prev.kind === 'string' || [')', ']', '}'].includes(prev.text) || prev.text === '·');
+        if (c === '.' && identStart.test(source[i + 1] ?? '')) {
+            // `.x` right after a term is a projection; otherwise a constructor.
+            advance();
+            const s = i;
+            while (i < source.length && identPart.test(source[i]))
+                advance();
+            push(afterTerm ? 'field' : 'dotIdent', source.slice(s, i));
+            continue;
+        }
+        if (identStart.test(c)) {
+            const s = i;
+            while (i < source.length && identPart.test(source[i]))
+                advance();
+            // A dotted name, as in Lean: `Hazards.BlackHole` or `C.world.site`; name resolution
+            // decides which parts are namespaces and which are projections.
+            while (source[i] === '.' && identStart.test(source[i + 1] ?? '')) {
+                advance();
+                while (i < source.length && identPart.test(source[i]))
+                    advance();
+            }
+            const text = source.slice(s, i);
+            push(KEYWORDS.has(text) ? 'keyword' : 'ident', text);
+            continue;
+        }
+        if (/[0-9]/.test(c)) {
+            const s = i;
+            while (i < source.length && /[0-9]/.test(source[i]))
+                advance();
+            push('number', source.slice(s, i));
+            continue;
+        }
+        if (c === '"') {
+            advance();
+            let text = '';
+            while (source[i] !== '"') {
+                if (i >= source.length || source[i] === '\n')
+                    fail('unterminated string', start);
+                if (source[i] === '\\') {
+                    advance();
+                    text += source[i] === 'n' ? '\n' : source[i];
+                    advance();
+                    continue;
+                }
+                text += source[i];
+                advance();
+            }
+            advance();
+            push('string', text);
+            continue;
+        }
+        const op = OPERATORS.find(o => source.startsWith(o, i));
+        if (!op)
+            fail(`unexpected character '${c}'`, start);
+        advance(op.length);
+        push('op', NORMALIZE[op] ?? op);
+    }
+    tokens.push({ kind: 'eof', text: '', start: pos(), end: pos(), lineStart: true, spaced: true });
+    return tokens;
+}