onejs-sl 0.1.4 → 0.1.5

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.1.5
+
+What an editor needs, and errors that fix themselves (`Specs/SL_NEXT.md` 5 and 9). Every program that compiled before compiles to exactly what it did, unless it names something after a keyword or a type.
+
+- `classify` returns every token with its class, comments included, and never throws
+- `SL_KEYWORDS`, `SL_TYPES` and `TYPE_WIDTH` are exported from `onejs-sl/tables`
+- `BUILTIN_DOCS`, `INPUT_DOCS` and `PRELUDE_DOCS` describe every builtin, input and prelude function in a line
+- `SLParseError` carries `text`, its message alone, and `offset`
+- An error carries a one click `fix` where the fix is certain: a GLSL rename or a "did you mean"
+- A GLSL name used as a value, such as `gl_FragCoord`, gets the HLSL name
+- `break`, `continue` and `switch` say why they are not there
+- A keyword or a type cannot name a value, a function or a parameter
+- Something missing at the end of a line or the file is marked on the last token before it
+- Errors from inside the EDSL say `float3`, not `vec3`
+- `BodyTarget.sample` documents what the body expects of a sample
+
 ## 0.1.4
 
 Four things that were errors now compile, the most common mistakes in cold runs of authors writing from the docs alone (`Specs/SL_NEXT.md` 2). Every program that compiled before compiles to exactly what it did.

package/README.md

@@ -35,9 +35,9 @@ only what it calls.
 
 | Entry | Contents |
 |---|---|
-| `onejs-sl` | `parse` and `analyze`, the TypeScript form `sl`, the IR types, `SL_IR_VERSION`, `toJSON`/`fromJSON`, `SLParseError` (file, line, column, length) |
+| `onejs-sl` | `parse` and `analyze`, `classify` (every token with its class, comments kept, never throws), the TypeScript form `sl`, the IR types, `SL_IR_VERSION`, `toJSON`/`fromJSON`, `SLParseError` (file, line, column, offset, length, the bare `text`, and a `fix` where one is certain) |
 | `onejs-sl/core` | the same without the parser: what a game needs at run time |
-| `onejs-sl/tables` | `BUILTINS`, `SL_HLSL`, `INPUTS`, `SL_SDF_SHAPES`, `SL_SDF_PARAMS`: what completion and highlighting read. `BUILTIN_PARAMS`, `SL_SDF_PARAM_NAMES` and `LIB_SIGNATURES` name every parameter, so an editor can show `lerp(x, y, s)` |
+| `onejs-sl/tables` | `BUILTINS`, `SL_HLSL`, `INPUTS`, `SL_SDF_SHAPES`, `SL_SDF_PARAMS`, `SL_KEYWORDS`, `SL_TYPES`, `TYPE_WIDTH`, `PRELUDE_NAMES`: what completion and highlighting read. `BUILTIN_PARAMS`, `SL_SDF_PARAM_NAMES` and `LIB_SIGNATURES` name every parameter, so an editor can show `lerp(x, y, s)`, and `BUILTIN_DOCS`, `INPUT_DOCS` and `PRELUDE_DOCS` give each a line for its tooltip |
 | `onejs-sl/limits` | `vmFit(program)`: whether the VM runs it, and why not, without encoding |
 | `onejs-sl/vm` | `encode`, `SL_WIRE_VERSION` |
 | `onejs-sl/emit/hlsl-body` | `emitBody`: a program as a function body for a host's own frame; `emitLibrary`: the library functions it calls |
@@ -56,7 +56,11 @@ intrinsic at its exact type, since Metal overloads where HLSL converts). The `Bo
 for each input, the float4 holding a uniform slot, a texture sample, whether
 `toLinear` is real (`colour: "linear"`) or the identity (`"gamma"`), and
 optionally a local to assign the result to. It returns the uniform and texture
-slots the body uses and the library functions it calls. OneJS's Unity shader is
+slots the body uses and the library functions it calls. The sample's contract
+is on `BodyTarget.sample`: straight alpha in and out, rgb in the space `colour`
+names (an sRGB texture decoded before filtering when it is `linear`), and
+filtering and wrapping left to the host, which OneJS's hosts take from the
+bound texture's own settings. OneJS's Unity shader is
 one frame over it (`hlsl.ts`); Magerie's compute kernel is another, and its
 target is in `src/body.test.ts` so an opcode cannot change without the text
 Magerie compiles changing in front of a test.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "onejs-sl",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "The OneJS shader language: a small typed language for per pixel programs, compiled to a portable IR and emitted as HLSL, WGSL, GLSL ES or a VM buffer",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body.ts

@@ -31,12 +31,30 @@ export interface BodyTarget {
     inputs: Record<InputName, string>
     /** The float4 holding a uniform slot. The body swizzles it down to the uniform's width. */
     uniform: (slot: number, name: string) => string
-    /** A float4 sample of the texture in `slot` at the float2 expression `uv`, straight alpha. */
+    /**
+     * A float4 sample of the texture in `slot` at the float2 expression `uv`.
+     *
+     * The contract the body is written against, which is what OneJS's own hosts
+     * do with the GPU's sampler:
+     *
+     * - **Straight alpha**, not premultiplied, in and out: the body never
+     *   multiplies or divides by alpha, and its result is straight too.
+     * - **In the space `colour` names.** With `linear`, rgb is linear light, so
+     *   an sRGB texture is decoded before it is filtered, as an sRGB sampler
+     *   does; with `gamma`, rgb is as stored.
+     * - **Filtered and wrapped by the host.** The body does neither. OneJS's
+     *   hosts, the generated shader, the VM and the web backends alike, use the
+     *   bound texture's own filter and wrap modes, so a host that filters by
+     *   hand draws the same picture by following the texture's settings.
+     *
+     * What becomes of the result is the host's: OneJS writes it to the target
+     * as it is, and a host that composites premultiplied multiplies on write.
+     */
     sample: (slot: number, uv: string) => string
     /**
      * `linear`: `toLinear` calls `sl_toLinear`, for a host whose target holds
      * linear light. `gamma`: `toLinear` is the identity, decided here, so hex
-     * colours and ramps stay as written.
+     * colours and ramps stay as written. `sample` reads in the same space.
      */
     colour: "gamma" | "linear"
     /** Assign the result to this local instead of returning it. */

package/src/index.ts

@@ -6,7 +6,7 @@
  * HLSL, WGSL or GLSL ES) is a separate entry, so a host bundles only what it
  * calls:
  *
- *   onejs-sl             this: parse, the TypeScript form, the IR, its JSON
+ *   onejs-sl             this: parse, classify, the TypeScript form, the IR, its JSON
  *   onejs-sl/core        the same without the parser, for a game at run time
  *   onejs-sl/tables      builtins, opcodes, inputs, shapes: what an editor reads
  *   onejs-sl/limits      whether a program fits the VM, without encoding it
@@ -19,6 +19,8 @@
  */
 export * from "./core"
 export {
-    analyze, parse, parseUnit, preludeFunctions, tokenize, PRELUDE_SOURCE, SLParseError,
+    analyze, classify, parse, parseUnit, preludeFunctions, tokenize, PRELUDE_SOURCE, SLParseError,
+} from "./lang"
+export type {
+    Checked, Expr, FuncDecl, ParseOptions, Pos, SLClassifiedToken, SLFix, SLTokenClass, Stmt, Token, Unit,
 } from "./lang"
-export type { Checked, Expr, FuncDecl, ParseOptions, Pos, Stmt, Token, Unit } from "./lang"

package/src/lang/README.md

@@ -168,11 +168,36 @@ lowering it changes recorded behaviour rather than parser behaviour.
 
 ## Errors
 
-Every one carries file, line and column, and an error that starts inside the
-EDSL keeps the EDSL's wording. `"z" is component 3 of a vec2, which has 2` is
-already the right sentence; what it lacked was a place. `errors.test.ts` checks
-the positions as strictly as the words, because a marker under the wrong
-character sends the reader to look at something that is fine.
+Every one carries file, line, column, offset and length, and `text`, the
+message without its tag and its `file:line:col: `, for an editor that puts the
+words in a marker of its own. An error that starts inside the EDSL keeps the
+EDSL's wording, with its type names put in the file's: `"z" is component 3 of
+a float2, which has 2` was already the right sentence; what it lacked was a
+place. `errors.test.ts` checks the positions as strictly as the words, because
+a marker under the wrong character sends the reader to look at something that
+is fine. Something missing at the end of a line or the file marks the last
+token before it, where there is a character to underline.
+
+Where the fix is certain, the error carries it as `fix`: a title and the text
+for the marked range, one click in an editor. A GLSL spelling that means the
+same in HLSL (`mix`, `fract`, `vec3`, `gl_FragCoord`, `iTime`, two argument
+`atan`) and a "did you mean" both have one. `mod` does not, since GLSL's floors
+and `%` truncates, and neither does `int`, since float would change what
+dividing it does. That is Decision 1 kept as `Specs/SL_NEXT.md` 5 A has it:
+one spelling, and every hint a fix.
+
+## For an editor
+
+`classify(source)` is every token with its class (keyword, type, builtin,
+input, prelude, number, hex, comment, punct, member, ident, or invalid), built
+on the parser's own scanner and word lists, so the highlighting is the parser's.
+It never throws, since it runs on every keystroke. It classifies by spelling,
+not scope: a local named `circle` still reads as a builtin where it is used.
+
+`SL_KEYWORDS` and `SL_TYPES` (`words.ts`) are the lists the parser reads, and a
+name may be neither. `onejs-sl/tables` carries them with a one line description
+of every builtin, input and prelude function; a prelude function's is the
+comment above it in `prelude-source.ts`.
 
 ## See also
 

package/src/lang/check.ts

@@ -34,10 +34,22 @@ import { SL_GLSL_HINT } from "../ops"
 import { BUILTINS, NOT_YET } from "./builtins"
 import { SL_SDF_SHAPES } from "../shapes"
 import type { Expr, FuncDecl, Stmt, Unit } from "./ast"
-import { SLParseError, type Pos } from "./lexer"
+import { SLParseError, type Pos, type SLFix } from "./lexer"
 
 const INPUT_NAMES = new Set(Object.keys(INPUTS))
 
+/**
+ * The GLSL spellings whose HLSL name means the same wherever it is written, so
+ * the hint can be a one click fix (`Specs/SL_NEXT.md` 5, Decision 1 A). The
+ * rest of `SL_GLSL_HINT` gets the hint alone: `mod` floors where `%` truncates,
+ * `ivec2` truncates where `float2` does not, `atan` is `atan2` only with two
+ * arguments (handled at the call), and `textureLod` names a builtin that is not
+ * implemented yet.
+ */
+const GLSL_RENAMES = new Set(["mix", "fract", "texture", "vec2", "vec3", "vec4", "gl_FragCoord", "iTime", "iResolution"])
+
+const rename = (from: string, to: string): SLFix => ({ title: `Replace ${from} with ${to}`, replacement: to })
+
 export interface Checked {
     unit: Unit
     /** Every callable function by name, the file's shadowing the prelude's. */
@@ -65,9 +77,9 @@ export function check(unit: Unit, prelude: FuncDecl[], options: CheckOptions = {
     // terminating control flow when the callee's `never` comes from an explicit
     // type, so without this every `if (x.k !== "ident") fail(...)` below would
     // fail to narrow.
-    const fail: (message: string, pos: Pos, length?: number) => never =
-        (message, pos, length = 1) => {
-            throw new SLParseError(message, file, pos, length)
+    const fail: (message: string, pos: Pos, length?: number, fix?: SLFix) => never =
+        (message, pos, length = 1, fix) => {
+            throw new SLParseError(message, file, pos, length, fix)
         }
 
     const funcs = new Map<string, FuncDecl>()
@@ -316,9 +328,7 @@ export function check(unit: Unit, prelude: FuncDecl[], options: CheckOptions = {
                             target.pos, n.length,
                         )
                     }
-                    if (!scope.has(n)) {
-                        fail(unknown(n, scope), target.pos, n.length)
-                    }
+                    if (!scope.has(n)) unknown(n, scope, target.pos)
                     checkExpr(fn, s.value, scope)
                     break
                 }
@@ -382,7 +392,8 @@ export function check(unit: Unit, prelude: FuncDecl[], options: CheckOptions = {
                 if (funcs.has(e.name) || BUILTINS[e.name] !== undefined) {
                     fail(`${e.name} is a function; call it, as in \`${e.name}(...)\``, e.pos, e.name.length)
                 }
-                fail(unknown(e.name, scope), e.pos, e.name.length)
+                glsl(e.name, e.pos)
+                unknown(e.name, scope, e.pos)
                 break
             }
             case "member":
@@ -493,10 +504,16 @@ export function check(unit: Unit, prelude: FuncDecl[], options: CheckOptions = {
                 callee.pos, n.length,
             )
         }
-        if (SL_GLSL_HINT[n] !== undefined) {
-            fail(`${n} is GLSL; this is HLSL, so write ${SL_GLSL_HINT[n]}`, callee.pos, n.length)
-        }
-        fail(unknown(n, scope), callee.pos, n.length)
+        glsl(n, callee.pos, e.args.length)
+        unknown(n, scope, callee.pos)
+    }
+
+    /** Refuses a GLSL spelling by its HLSL name, with the fix where the rename is certain. */
+    function glsl(n: string, pos: Pos, args?: number): void {
+        const hint = SL_GLSL_HINT[n]
+        if (hint === undefined) return
+        const certain = GLSL_RENAMES.has(n) || (n === "atan" && args === 2)
+        fail(`${n} is GLSL; this is HLSL, so write ${hint}`, pos, n.length, certain ? rename(n, hint) : undefined)
     }
 
     function arity(e: Extract<Expr, { k: "call" }>, n: string, min: number, max: number): void {
@@ -505,13 +522,14 @@ export function check(unit: Unit, prelude: FuncDecl[], options: CheckOptions = {
         fail(`${n} takes ${want} argument${max === 1 ? "" : "s"}, got ${e.args.length}`, e.pos)
     }
 
-    /** "unknown x" plus the nearest thing that is spelled almost like it. */
-    function unknown(n: string, scope: Set<string>): string {
+    /** Refuses a name nothing declares, offering the nearest one spelled almost like it. */
+    function unknown(n: string, scope: Set<string>, pos: Pos): never {
         const near = nearest(n, [
             ...scope, ...INPUT_NAMES, ...uniforms.keys(), ...textures.keys(), ...consts.keys(),
             ...funcs.keys(), ...Object.keys(BUILTINS),
         ])
-        return near === null ? `"${n}" is not declared` : `"${n}" is not declared; did you mean ${near}?`
+        if (near === null) fail(`"${n}" is not declared`, pos, n.length)
+        fail(`"${n}" is not declared; did you mean ${near}?`, pos, n.length, rename(n, near))
     }
 
     return { unit, funcs, uniforms, textures, consts }

package/src/lang/classify.ts

@@ -0,0 +1,70 @@
+/**
+ * Every token of a file with what it is, for an editor's highlighting.
+ *
+ * `Specs/SL_NEXT.md` 9. Built on the parser's own scanner and the parser's own
+ * word lists, so what highlights as a keyword is exactly what parses as one,
+ * and a builtin is exactly what `BUILTINS` holds. It never throws: it runs on
+ * every keystroke over half typed text, and a character it cannot read comes
+ * back as `invalid` rather than ending the pass.
+ *
+ * Classification is by spelling and position alone, not by scope. A local
+ * that takes a builtin's name (`float circle = ...`) still reads `builtin`
+ * where it is used; telling the two apart needs `analyze`, and an editor that
+ * wants that should ask it.
+ */
+
+import { INPUTS } from "../ir"
+import { BUILTINS } from "./builtins"
+import { lex } from "./lexer"
+import { PRELUDE_NAMES } from "./prelude-source"
+import { KEYWORD_SET, TYPE_SET } from "./words"
+
+export type SLTokenClass =
+    | "keyword" | "type" | "builtin" | "input" | "prelude"
+    | "number" | "hex" | "comment" | "punct"
+    /** A component after a dot, as in `c.rgb`. */
+    | "member"
+    /** Any other name: a uniform, a local, a function the file declares. */
+    | "ident"
+    /** A character or `#word` the language cannot read, and would refuse. */
+    | "invalid"
+
+export interface SLClassifiedToken {
+    kind: SLTokenClass
+    text: string
+    /** 1 based. */
+    line: number
+    /** 1 based. */
+    col: number
+    offset: number
+    length: number
+}
+
+const INPUT_NAMES: ReadonlySet<string> = new Set(Object.keys(INPUTS))
+
+/** The tokens of `source`, comments included, in order, each with its class. */
+export function classify(source: string): SLClassifiedToken[] {
+    const tokens = lex(source, "", true)
+    const out: SLClassifiedToken[] = []
+    // The last token that was not a comment, which decides what a name after a dot is.
+    let before = ""
+    let beforeThat = ""
+    for (const t of tokens) {
+        if (t.kind === "eof") break
+        let kind: SLTokenClass
+        if (t.kind === "ident") {
+            if (before === ".") kind = beforeThat === "sdf" ? "builtin" : "member"
+            else if (KEYWORD_SET.has(t.text)) kind = "keyword"
+            else if (TYPE_SET.has(t.text)) kind = "type"
+            else if (INPUT_NAMES.has(t.text)) kind = "input"
+            else if (BUILTINS[t.text] !== undefined) kind = "builtin"
+            else if (PRELUDE_NAMES.has(t.text)) kind = "prelude"
+            else kind = "ident"
+        } else {
+            kind = t.kind
+        }
+        out.push({ kind, text: t.text, line: t.line, col: t.col, offset: t.offset, length: t.length })
+        if (t.kind !== "comment") { beforeThat = before; before = t.text }
+    }
+    return out
+}

package/src/lang/index.ts

@@ -30,8 +30,11 @@ export type {
 } from "./ast"
 export { TYPE_WIDTH } from "./ast"
 export { SLParseError } from "./lexer"
-export type { Pos, Token } from "./lexer"
+export type { Pos, SLFix, Token } from "./lexer"
 export { tokenize } from "./lexer"
+export { classify } from "./classify"
+export type { SLClassifiedToken, SLTokenClass } from "./classify"
+export { SL_KEYWORDS, SL_TYPES } from "./words"
 export { parseUnit } from "./parser"
 export { PRELUDE_SOURCE, preludeFunctions } from "./prelude"
 export { BUILTINS, NOT_YET } from "./builtins"

package/src/lang/lexer.ts

@@ -32,6 +32,19 @@ export interface Token extends Pos {
     value?: number
 }
 
+/**
+ * A replacement for the characters an error marks, offered as one click.
+ *
+ * Only where the fix is certain: `mix` to `lerp` is, but `mod` to `%` is not
+ * (GLSL's `mod` floors and `%` truncates), so `mod` gets the hint and no fix.
+ */
+export interface SLFix {
+    /** What the editor's action says, as in "Replace mix with lerp". */
+    title: string
+    /** Text for the marked range, `offset` to `offset + length`. */
+    replacement: string
+}
+
 /**
  * An error with a place in a file.
  *
@@ -43,15 +56,23 @@ export class SLParseError extends SLError {
     readonly file: string
     readonly line: number
     readonly column: number
+    /** Into the source, where the marked range starts. */
+    readonly offset: number
     readonly length: number
+    /** The message alone, without `message`'s tag and its `file:line:col: `. */
+    readonly text: string
+    readonly fix?: SLFix
 
-    constructor(message: string, file: string, pos: Pos, length = 1) {
+    constructor(message: string, file: string, pos: Pos, length = 1, fix?: SLFix) {
         super(`${file}:${pos.line}:${pos.col}: ${message}`)
         this.name = "SLParseError"
         this.file = file
         this.line = pos.line
         this.column = pos.col
+        this.offset = pos.offset
         this.length = length
+        this.text = message
+        if (fix !== undefined) this.fix = fix
     }
 }
 
@@ -79,14 +100,37 @@ const isIdentStart = (c: string) => c === "_" || (c >= "a" && c <= "z") || (c >=
 const isIdentPart = (c: string) => isIdentStart(c) || isDigit(c)
 
 export function tokenize(source: string, file: string): Token[] {
-    const out: Token[] = []
+    return lex(source, file, false)
+}
+
+/** A token as `classify` sees it: comments kept, and whatever cannot be read marked rather than thrown. */
+export interface LexedToken extends Omit<Token, "kind"> {
+    kind: TokenKind | "comment" | "invalid"
+    /** Characters it covers, which for `1.0f` is one more than its text. */
+    length: number
+}
+
+/**
+ * The scanner behind `tokenize` and `classify`.
+ *
+ * With `keep` off it is `tokenize`: comments dropped, the first unreadable
+ * character thrown. With it on, nothing throws and nothing is dropped, because
+ * a highlighter runs on every keystroke over text that is half typed.
+ */
+export function lex(source: string, file: string, keep: true): LexedToken[]
+export function lex(source: string, file: string, keep: false): Token[]
+export function lex(source: string, file: string, keep: boolean): Token[] | LexedToken[] {
+    const out: LexedToken[] = []
     let i = 0
     let line = 1
     let lineStart = 0
     const here = (): Pos => ({ line, col: i - lineStart + 1, offset: i })
-    const fail = (message: string, at: Pos = here(), length = 1): never => {
-        throw new SLParseError(message, file, at, length)
+    /** Throws, or with `keep` records the span as invalid and lets the caller skip it. */
+    const fail = (message: string, at: Pos = here(), length = 1): void => {
+        if (!keep) throw new SLParseError(message, file, at, length)
+        out.push({ kind: "invalid", text: source.slice(at.offset, at.offset + length), length, ...at })
     }
+    const push = (t: Omit<LexedToken, "length">, end: number) => { out.push({ ...t, length: end - t.offset }) }
 
     while (i < source.length) {
         const c = source[i]!
@@ -95,18 +139,25 @@ export function tokenize(source: string, file: string): Token[] {
         if (c === " " || c === "\t" || c === "\r") { i++; continue }
 
         if (c === "/" && source[i + 1] === "/") {
+            const start = here()
             while (i < source.length && source[i] !== "\n") i++
+            if (keep) push({ kind: "comment", text: source.slice(start.offset, i), ...start }, i)
             continue
         }
         if (c === "/" && source[i + 1] === "*") {
             const open = here()
             i += 2
             for (;;) {
-                if (i >= source.length) fail("this block comment is never closed", open, 2)
+                if (i >= source.length) {
+                    // Highlighted to the end as the comment it will become once closed.
+                    if (!keep) fail("this block comment is never closed", open, 2)
+                    break
+                }
                 if (source[i] === "*" && source[i + 1] === "/") { i += 2; break }
                 if (source[i] === "\n") { line++; lineStart = i + 1 }
                 i++
             }
+            if (keep) push({ kind: "comment", text: source.slice(open.offset, i), ...open }, i)
             continue
         }
 
@@ -122,11 +173,11 @@ export function tokenize(source: string, file: string): Token[] {
                     `function, which inlines.`,
                     start, body.length + 1,
                 )
-            }
-            if (body.length === 0 || ![...body].every(isHex)) {
+            } else if (body.length === 0 || ![...body].every(isHex)) {
                 fail(`"#${body}" is not a colour; use #rgb, #rrggbb or #rrggbbaa`, start, body.length + 1)
+            } else {
+                push({ kind: "hex", text: "#" + body, ...start }, j)
             }
-            out.push({ kind: "hex", text: "#" + body, ...start })
             i = j
             continue
         }
@@ -150,7 +201,7 @@ export function tokenize(source: string, file: string): Token[] {
             }
             const value = Number(text)
             if (!Number.isFinite(value)) fail(`"${text}" is not a number`, start, text.length)
-            out.push({ kind: "number", text, value, ...start })
+            else push({ kind: "number", text, value, ...start }, j)
             i = j
             continue
         }
@@ -159,18 +210,24 @@ export function tokenize(source: string, file: string): Token[] {
             const start = here()
             let j = i
             while (j < source.length && isIdentPart(source[j]!)) j++
-            out.push({ kind: "ident", text: source.slice(i, j), ...start })
+            push({ kind: "ident", text: source.slice(i, j), ...start }, j)
             i = j
             continue
         }
 
         const start = here()
         const p = PUNCT.find((op) => source.startsWith(op, i))
-        if (p === undefined) fail(`"${c}" means nothing here`, start)
-        out.push({ kind: "punct", text: p!, ...start })
-        i += p!.length
+        if (p === undefined) {
+            fail(`"${c}" means nothing here`, start)
+            i++
+            continue
+        }
+        push({ kind: "punct", text: p, ...start }, i + p.length)
+        i += p.length
     }
 
-    out.push({ kind: "eof", text: "", line, col: i - lineStart + 1, offset: i })
-    return out
+    out.push({ kind: "eof", text: "", line, col: i - lineStart + 1, offset: i, length: 0 })
+    if (keep) return out
+    // `tokenize`'s tokens are the ones it always returned, with no length.
+    return out.map(({ length: _, ...t }) => t as Token)
 }

package/src/lang/lower.ts

@@ -106,7 +106,7 @@ export function lower(checked: Checked): Program {
     const located = (e: unknown, pos: Pos): unknown => {
         if (e instanceof SLParseError) return e
         const raw = e instanceof Error ? e.message : String(e)
-        return new SLParseError(raw.replace(/^\[onejs sl] /, ""), file, pos)
+        return new SLParseError(inFileWords(raw.replace(/^\[onejs sl] /, "")), file, pos)
     }
 
     const fail: (message: string, pos: Pos, length?: number) => never =
@@ -676,3 +676,17 @@ function widthType(w: SLType): TypeName {
     return (["float", "float2", "float3", "float4"] as const)[w - 1]!
 }
 
+/**
+ * An EDSL message in the words of a `.sl` file.
+ *
+ * The EDSL names its types after its constructors, `sl.vec3`, so "cannot
+ * combine a vec3 with a vec2" is right for a TypeScript author and wrong for
+ * one who wrote float3 (`Specs/SL_NEXT.md` 5). One rewrite here, rather than a
+ * second set of messages, keeps both surfaces saying the same thing. A quoted
+ * name is the author's own and left as written.
+ */
+function inFileWords(message: string): string {
+    return message.split(/("[^"]*")/).map((part, i) =>
+        i % 2 === 1 ? part : part.replace(/\bsl\.vec([234])\(/g, "float$1(").replace(/\bvec([234])\b/g, "float$1"),
+    ).join("")
+}

package/src/lang/parser.ts

@@ -12,13 +12,11 @@
  */
 
 import {
-    TYPE_WIDTH,
     type AssignOp, type BinaryOp, type Expr, type FuncDecl, type Param, type Stmt,
     type TextureDecl, type TypeName, type UniformDecl, type Unit,
 } from "./ast"
-import { SLParseError, tokenize, type Pos, type Token } from "./lexer"
-
-const TYPE_NAMES = new Set(Object.keys(TYPE_WIDTH))
+import { SLParseError, tokenize, type Pos, type SLFix, type Token } from "./lexer"
+import { KEYWORD_SET, TYPE_SET } from "./words"
 
 /**
  * Type spellings that exist in HLSL or GLSL and not here, each with the reason.
@@ -45,6 +43,11 @@ const NOT_A_TYPE: Record<string, string> = {
     struct: "there are no structs",
 }
 
+/** The spellings above whose replacement is certain, so the error can offer it as a fix. */
+const TYPE_FIX: Record<string, TypeName> = {
+    vec2: "float2", vec3: "float3", vec4: "float4", half: "float", double: "float", fixed: "float",
+}
+
 /** Binding power per binary operator, HLSL's. Higher binds tighter. */
 const BINDING: Record<string, number> = {
     "||": 1,
@@ -73,14 +76,21 @@ class Parser {
     private at(text: string): boolean { const t = this.peek(); return t.kind !== "eof" && t.text === text }
     private eat(text: string): boolean { if (this.at(text)) { this.i++; return true } return false }
 
-    private fail(message: string, at: Token = this.peek()): never {
-        throw new SLParseError(message, this.file, at as Pos, Math.max(1, at.text.length))
+    private fail(message: string, at: Token = this.peek(), fix?: SLFix): never {
+        // The end of the file is no character, so an error there marks the last
+        // token instead of one past it, where an editor has nothing to underline.
+        if (at.kind === "eof" && this.tokens.length > 1) at = this.tokens[this.tokens.length - 2]!
+        throw new SLParseError(message, this.file, at as Pos, Math.max(1, at.text.length), fix)
     }
 
     private expect(text: string, what: string): Token {
         if (!this.at(text)) {
             const got = this.peek()
-            this.fail(`expected "${text}" ${what}, got ${describe(got)}`, got)
+            // Something missing at the end of a line is missing after the last
+            // token on it, which is where the author will look, not at the next line.
+            const prev = this.tokens[this.i - 1]
+            const at = prev !== undefined && got.line > prev.line ? prev : got
+            this.fail(`expected "${text}" ${what}, got ${describe(got)}`, at)
         }
         return this.next()
     }
@@ -91,11 +101,21 @@ class Parser {
         return this.next()
     }
 
+    /** A name being declared, which may not be a word the parser reads as syntax. */
+    private expectName(what: string): Token {
+        const t = this.expectIdent(what)
+        if (KEYWORD_SET.has(t.text)) this.fail(`"${t.text}" is a keyword, so it cannot be ${what}`, t)
+        if (TYPE_SET.has(t.text)) this.fail(`"${t.text}" is a type, so it cannot be ${what}`, t)
+        return t
+    }
+
     private expectType(what: string): TypeName {
         const t = this.expectIdent(what)
-        if (TYPE_NAMES.has(t.text)) return t.text as TypeName
+        if (TYPE_SET.has(t.text)) return t.text as TypeName
         const why = NOT_A_TYPE[t.text]
-        if (why !== undefined) this.fail(`"${t.text}" is not a type here: ${why}`, t)
+        const to = TYPE_FIX[t.text]
+        const fix = to === undefined ? undefined : { title: `Replace ${t.text} with ${to}`, replacement: to }
+        if (why !== undefined) this.fail(`"${t.text}" is not a type here: ${why}`, t, fix)
         this.fail(`"${t.text}" is not a type; the types are float, float2, float3 and float4`, t)
     }
 
@@ -146,7 +166,7 @@ class Parser {
     private parseUniform(): UniformDecl {
         const kw = this.next()
         const type = this.expectType("a type after uniform")
-        const name = this.expectIdent("a uniform name")
+        const name = this.expectName("a uniform name")
         let init: Expr | null = null
         if (this.eat("=")) init = this.parseExpr()
         this.expect(";", "after a uniform declaration")
@@ -155,7 +175,7 @@ class Parser {
 
     private parseTexture(): TextureDecl {
         const kw = this.next()
-        const name = this.expectIdent("a texture name")
+        const name = this.expectName("a texture name")
         if (this.at("=")) {
             this.fail(
                 "a texture has no default; the host binds it by name through the textures prop",
@@ -169,7 +189,7 @@ class Parser {
     private parseConst(): Extract<Stmt, { k: "const" }> {
         const kw = this.next()
         const type = this.expectType("a type after const")
-        const name = this.expectIdent("a name")
+        const name = this.expectName("a const's name")
         this.expect("=", "after a const's name; a const has to have a value")
         const init = this.parseExpr()
         this.expect(";", "after a const")
@@ -179,7 +199,7 @@ class Parser {
     private parseFunction(): FuncDecl {
         const start = this.peek()
         const ret = this.expectType("a return type")
-        const name = this.expectIdent("a function name")
+        const name = this.expectName("a function name")
         if (!this.at("(")) {
             this.fail(
                 `expected "(" after ${name.text}. Only uniforms, textures, consts and functions live ` +
@@ -193,7 +213,7 @@ class Parser {
             for (;;) {
                 const p = this.peek()
                 const ptype = this.expectType("a parameter type")
-                const pname = this.expectIdent("a parameter name")
+                const pname = this.expectName("a parameter name")
                 params.push({ type: ptype, name: pname.text, pos: p })
                 if (!this.eat(",")) break
             }
@@ -238,6 +258,16 @@ class Parser {
         if (t.text === "discard") {
             this.fail("there is no discard; return a colour with alpha 0 instead", t)
         }
+        if (t.text === "break" || t.text === "continue") {
+            this.fail(
+                `there is no ${t.text}: a for loop unrolls, so every iteration runs. Put the rest of ` +
+                `the body under an if instead`,
+                t,
+            )
+        }
+        if (t.text === "switch") {
+            this.fail("there is no switch; write it as an if and else if", t)
+        }
         if (t.text === "return") {
             this.next()
             const value = this.parseExpr()
@@ -253,7 +283,7 @@ class Parser {
         // statement may be is the left of an assignment.
         if (t.kind === "ident" && this.peek(1).kind === "ident") {
             const type = this.expectType("a type")
-            const name = this.expectIdent("a name")
+            const name = this.expectName("a local's name")
             if (this.at("(")) {
                 this.fail(
                     "a function is declared at the top level of the file, not inside another function",
@@ -318,7 +348,7 @@ class Parser {
             )
         }
         this.next()
-        const counter = this.expectIdent("a counter name")
+        const counter = this.expectName("a counter name")
         this.expect("=", "after the counter")
         const from = this.parseExpr()
         this.expect(";", "after the counter's start")

package/src/lang/prelude-source.ts

@@ -0,0 +1,53 @@
+/**
+ * The prelude's source, and what an editor reads from it without parsing it.
+ *
+ * Kept out of `prelude.ts`, which parses it, so that `onejs-sl/tables` can offer
+ * the prelude's names and descriptions with no parser in the bundle. Each
+ * function's description is the comment above it, so there is one place to
+ * write it.
+ */
+
+export const PRELUDE_SOURCE = `
+// Rotate a point about the origin. Translate first if you want another centre.
+float2 rotate(float2 p, float angle) {
+    float c = cos(angle);
+    float s = sin(angle);
+    return float2(p.x * c - p.y * s, p.x * s + p.y * c);
+}
+
+// Cartesian to polar: x is the angle in radians, y is the radius.
+float2 polar(float2 p) {
+    return float2(atan2(p.y, p.x), length(p));
+}
+
+// Signed distance to a circle at the origin. Negative inside.
+float circle(float2 p, float r) {
+    return length(p) - r;
+}
+
+// Signed distance to an axis aligned box of the given half extents.
+float box(float2 p, float2 size) {
+    float2 d = abs(p) - size;
+    return length(max(d, 0)) + min(max(d.x, d.y), 0);
+}
+
+// Repeat a unit cell n times across the input, centred on zero in each cell.
+float2 tile(float2 p, float n) {
+    return frac(p * n) - 0.5;
+}
+
+// Inigo Quilez's cosine palette. Four float3s in, a colour out, no texture.
+float3 palette(float t, float3 a, float3 b, float3 c, float3 d) {
+    return a + b * cos(6.28318530718 * (c * t + d));
+}
+`
+
+/** Each prelude function's name, in the order the source declares them. */
+const DECLARED = /* @__PURE__ */ [...PRELUDE_SOURCE.matchAll(/^\/\/ (.+)\n\w+ (\w+)\(/gm)]
+
+export const PRELUDE_NAMES: ReadonlySet<string> = /* @__PURE__ */ new Set(DECLARED.map((m) => m[2]!))
+
+/** Each prelude function's one line description: the comment above it. */
+export const PRELUDE_DOCS: Readonly<Record<string, string>> = /* @__PURE__ */ Object.fromEntries(
+    DECLARED.map((m) => [m[2]!, m[1]!]),
+)

package/src/lang/prelude.ts

@@ -21,41 +21,9 @@
 import type { FuncDecl } from "./ast"
 import { check } from "./check"
 import { parseUnit } from "./parser"
+import { PRELUDE_SOURCE } from "./prelude-source"
 
-export const PRELUDE_SOURCE = `
-// Rotate a point about the origin. Translate first if you want another centre.
-float2 rotate(float2 p, float angle) {
-    float c = cos(angle);
-    float s = sin(angle);
-    return float2(p.x * c - p.y * s, p.x * s + p.y * c);
-}
-
-// Cartesian to polar: x is the angle in radians, y is the radius.
-float2 polar(float2 p) {
-    return float2(atan2(p.y, p.x), length(p));
-}
-
-// Signed distance to a circle at the origin. Negative inside.
-float circle(float2 p, float r) {
-    return length(p) - r;
-}
-
-// Signed distance to an axis aligned box of the given half extents.
-float box(float2 p, float2 size) {
-    float2 d = abs(p) - size;
-    return length(max(d, 0)) + min(max(d.x, d.y), 0);
-}
-
-// Repeat a unit cell n times across the input, centred on zero in each cell.
-float2 tile(float2 p, float n) {
-    return frac(p * n) - 0.5;
-}
-
-// Inigo Quilez's cosine palette. Four float3s in, a colour out, no texture.
-float3 palette(float t, float3 a, float3 b, float3 c, float3 d) {
-    return a + b * cos(6.28318530718 * (c * t + d));
-}
-`
+export { PRELUDE_SOURCE }
 
 let cached: FuncDecl[] | null = null
 

package/src/lang/words.ts

@@ -0,0 +1,23 @@
+/**
+ * The words the parser reads as syntax, and the value types.
+ *
+ * One list, read by the parser (a name may not be one of these) and by
+ * `classify` (these highlight as keywords and types), so an editor that
+ * highlights with it agrees with the parser by construction rather than by a
+ * list it keeps in step by hand.
+ */
+
+import { TYPE_WIDTH, type TypeName } from "./ast"
+
+/**
+ * Declaration and statement words. `int` is here rather than among the types
+ * because it is only ever a for loop's counter, `for (int i = 0; i < 4; i++)`;
+ * a value cannot be declared as one.
+ */
+export const SL_KEYWORDS = ["uniform", "texture2D", "const", "if", "else", "for", "return", "int"] as const
+
+/** The types a value, a uniform or a function can have. */
+export const SL_TYPES = Object.keys(TYPE_WIDTH) as readonly TypeName[]
+
+export const KEYWORD_SET: ReadonlySet<string> = new Set(SL_KEYWORDS)
+export const TYPE_SET: ReadonlySet<string> = new Set(SL_TYPES)

package/src/params.ts

@@ -20,6 +20,7 @@ import { BUILTINS } from "./lang/builtins"
 import { LIB_FUNCTIONS, SDF_CALLS } from "./lib/table"
 import { LIB_PARAM_NAMES } from "./lib/names"
 import { SL_SDF_PARAMS, SL_SDF_SHAPES, type SlSdfKind } from "./shapes"
+import type { InputName } from "./ir"
 
 /** One function of the helper library, as its source declares it. */
 export interface LibSignature {
@@ -111,3 +112,63 @@ function shapeNames(): Record<SlSdfKind, readonly string[]> {
     }
     return out
 }
+
+/**
+ * One line on what each builtin does, for a completion's tooltip and the docs
+ * page: the other half of `BUILTIN_PARAMS`, and named in its words, so a line
+ * can say `s` and mean the third argument.
+ */
+export const BUILTIN_DOCS: Readonly<Record<string, string>> = {
+    sin: "The sine of x, in radians.",
+    cos: "The cosine of x, in radians.",
+    tan: "The tangent of x, in radians.",
+    asin: "The angle whose sine is x, with x clamped to -1 to 1.",
+    acos: "The angle whose cosine is x, with x clamped to -1 to 1.",
+    exp: "e raised to the power x.",
+    log: "The natural logarithm of x.",
+    sqrt: "The square root of x.",
+    sign: "-1, 0 or 1, by the sign of x.",
+    ceil: "x rounded up to a whole number.",
+    round: "x rounded to the nearest whole number.",
+    toLinear: "A colour as written, in sRGB, converted to the space the target holds; alpha is left alone.",
+    abs: "x without its sign.",
+    floor: "x rounded down to a whole number.",
+    frac: "The part of x after the decimal point, 0 up to 1.",
+    saturate: "x clamped to 0 to 1.",
+    rcp: "1 / x.",
+    normalize: "x scaled to length 1.",
+    pow: "x raised to the power y.",
+    min: "The smaller of x and y, per component.",
+    max: "The larger of x and y, per component.",
+    clamp: "x held between min and max.",
+    atan2: "The angle of the point (x, y) from the positive x axis, in radians.",
+    length: "The length of the vector x.",
+    distance: "The distance between the points x and y.",
+    dot: "The dot product of x and y.",
+    cross: "The cross product of two float3s.",
+    reflect: "The direction i reflected off a surface with normal n.",
+    lerp: "Mixes x into y by s: x at 0, y at 1.",
+    step: "0 where x is below y, else 1.",
+    smoothstep: "0 below min, 1 above max, and a smooth curve between.",
+    remap: "v moved from the range fromMin to fromMax into the range toMin to toMax.",
+    hsv2rgb: "A colour from hue, saturation and value, each 0 to 1.",
+    luminance: "How bright a colour looks, as one number.",
+    noise: "Smooth value noise at p, about 0 to 1. Offset p for a different field.",
+    simplex: "Simplex noise at p, about 0 to 1, with no grid pattern.",
+    voronoi: "The distance from p to the nearest point of a jittered grid: cells.",
+    fbm: "Value noise layered in octaves, 1 to 4 (3 when left out): clouds and terrain.",
+    turbulence: "Simplex creases layered in octaves, 1 to 4: fire, smoke, marble.",
+    ridged: "Turbulence made bright at its creases: ridges, lightning, cracks.",
+    tex2D: "The colour of texture s at the uv t, with straight alpha.",
+    sdf: "The signed distance from p to a shape, negative inside: `sdf.circle(p, r)`.",
+    ramp: "t mapped through evenly spaced colour stops, from the first at 0 to the last at 1.",
+}
+
+/** One line on each input, for the same tooltips. */
+export const INPUT_DOCS: Readonly<Record<InputName, string>> = {
+    uv: "Where this pixel is in the element, 0 to 1 on each axis, with y up.",
+    fragCoord: "Where this pixel is in the element, in pixels, with y up.",
+    resolution: "The element's size in pixels.",
+    time: "Seconds since the effect started.",
+    aspect: "The element's width divided by its height.",
+}

package/src/tables.ts

@@ -2,8 +2,10 @@
  * `onejs-sl/tables`: what completion and highlighting read.
  *
  * The same tables the checker and the emitters use, so an editor offers
- * exactly the builtins that compile, with the arities they take and the names
- * of their parameters (`params.ts`), and the helper library's signatures.
+ * exactly the builtins that compile, with the arities they take, the names of
+ * their parameters and a line on each (`params.ts`), the words the parser
+ * reads as syntax, and the helper library's signatures. `classify`, in the
+ * main entry, is the highlighter built on the same lists.
  */
 export { BUILTINS, NOT_YET } from "./lang/builtins"
 export type { Builtin } from "./lang/builtins"
@@ -13,6 +15,10 @@ export {
 export type { SLSurface } from "./ops"
 export { INPUTS } from "./ir"
 export { SL_SDF_SHAPES, SL_SDF_PARAMS } from "./shapes"
-export { BUILTIN_PARAMS, LIB_SIGNATURES, SL_SDF_PARAM_NAMES } from "./params"
+export { BUILTIN_DOCS, BUILTIN_PARAMS, INPUT_DOCS, LIB_SIGNATURES, SL_SDF_PARAM_NAMES } from "./params"
+export { PRELUDE_DOCS, PRELUDE_NAMES } from "./lang/prelude-source"
+export { SL_KEYWORDS, SL_TYPES } from "./lang/words"
+export { TYPE_WIDTH } from "./lang/ast"
+export type { TypeName } from "./lang/ast"
 export type { LibSignature } from "./params"
 export type { SlSdfKind } from "./shapes"