@8bitscript/compiler 0.1.0

package/src/lexer/index.mjs

@@ -0,0 +1,383 @@
+// The lexer: raw text in, tokens out.
+//
+// First layer of the pipeline described in docs/compiler.md. It is deliberately
+// the only layer that exists today, because it is the one that pays off before
+// a parser does: it finds unterminated strings, stray characters, and unbalanced
+// brackets, which is already enough to put real errors under the cursor.
+//
+// Every token carries its offset and length. Diagnostics are built from those,
+// so a position is never recomputed by guesswork later.
+import { Codes, diagnostic } from '../diagnostics/index.mjs';
+import { INTEGER_TYPE_NAMES } from '../types/index.mjs';
+
+export const TokenKind = {
+  Comment: 'comment',
+  String: 'string',
+  // A backtick string with `${...}` fields — `TICK ${ticks} OPTION ${option}`.
+  // One token for the whole thing; `parts` records where its literal text
+  // and its field sources sit, and the parser re-lexes each field.
+  Template: 'template',
+  Number: 'number',
+  Identifier: 'identifier',
+  Keyword: 'keyword',
+  Type: 'type',
+  Decorator: 'decorator',
+  // `#frames` — a function the compiler evaluates, never the target. The
+  // `#` is the one spelling that says "8bitscript resolves this before any
+  // target toolchain runs"; a plain `name(...)` always runs on the machine.
+  CompileTime: 'compileTime',
+  AsmBlock: 'asm',
+  Punctuation: 'punctuation',
+  Operator: 'operator',
+};
+
+export const KEYWORDS = new Set([
+  'let', 'const', 'function', 'return', 'export', 'import', 'from', 'as',
+  'if', 'else', 'while', 'for', 'do', 'break', 'continue',
+  'switch', 'case', 'default', 'true', 'false', 'asm6502', 'namespace',
+]);
+
+// Every primitive integer spelling comes from the shared registry — the
+// canonical names (`utinyint`, `int`, ...) and the low-level aliases (`u8`,
+// `i32`, ...) — so this set can't drift out of sync with the checker, the
+// backends, or hover/completion.
+export const TYPE_NAMES = new Set([
+  ...INTEGER_TYPE_NAMES,
+  'bool', 'void', 'string', 'ptr', 'array', 'volatile',
+]);
+
+const BRACKET_PAIRS = { ')': '(', ']': '[', '}': '{' };
+const OPEN_BRACKETS = new Set(['(', '[', '{']);
+
+// Operators, longest first, matched by maximal munch against this list only.
+// Greedily globbing operator characters is how `x=-1` ends up lexed as the
+// non-operator `=-`; matching real operators cannot produce a token that no
+// rule of the language recognises.
+const OPERATORS = [
+  '<<=', '>>=',
+  '==', '!=', '<=', '>=', '&&', '||', '<<', '>>', '++', '--',
+  '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=',
+  '+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '<', '>', '=',
+  '?', ':', ';', ',', '.',
+];
+
+const DIGITS_FOR_RADIX = { 2: /[01_]/, 10: /[0-9_]/, 16: /[0-9a-fA-F_]/ };
+
+// `$` is not an identifier character: it introduces hex literals ($900F). A
+// language aimed at this hardware gives the assembly spelling priority.
+const isIdentStart = (c) => /[A-Za-z_]/.test(c);
+const isIdentPart = (c) => /[A-Za-z0-9_]/.test(c);
+const isDigit = (c) => c >= '0' && c <= '9';
+
+/**
+ * Tokenize a source file.
+ *
+ * Always returns both tokens and diagnostics: lexing never throws, because an
+ * editor asks for tokens on every keystroke and half-typed source is the normal
+ * case, not an exceptional one.
+ *
+ * @param {string} text
+ * @param {string} file
+ * @returns {{ tokens: object[], diagnostics: object[] }}
+ */
+export function tokenize(text, file = '<unknown>') {
+  const tokens = [];
+  const diagnostics = [];
+  const brackets = [];
+  let i = 0;
+
+  const push = (kind, start, end, extra = {}) =>
+    tokens.push({ kind, start, length: end - start, text: text.slice(start, end), ...extra });
+
+  while (i < text.length) {
+    const c = text[i];
+
+    if (c === ' ' || c === '\t' || c === '\r' || c === '\n') {
+      i += 1;
+      continue;
+    }
+
+    // Comments, both spellings.
+    if (c === '/' && text[i + 1] === '/') {
+      const start = i;
+      while (i < text.length && text[i] !== '\n') i += 1;
+      push(TokenKind.Comment, start, i);
+      continue;
+    }
+    if (c === '/' && text[i + 1] === '*') {
+      const start = i;
+      i += 2;
+      while (i < text.length && !(text[i] === '*' && text[i + 1] === '/')) i += 1;
+      if (i >= text.length) {
+        diagnostics.push(
+          diagnostic(
+            Codes.UNTERMINATED_BLOCK_COMMENT,
+            'unterminated block comment',
+            file, start, text.length - start,
+          ),
+        );
+        push(TokenKind.Comment, start, text.length);
+        break;
+      }
+      i += 2;
+      push(TokenKind.Comment, start, i);
+      continue;
+    }
+
+    // Strings. A newline ends the search: an unterminated string should report
+    // on its own line rather than swallowing the rest of the file.
+    if (c === '"' || c === "'") {
+      const start = i;
+      const quote = c;
+      i += 1;
+      let closed = false;
+      while (i < text.length) {
+        if (text[i] === '\\') { i += 2; continue; }
+        if (text[i] === quote) { i += 1; closed = true; break; }
+        if (text[i] === '\n') break;
+        i += 1;
+      }
+      if (!closed) {
+        diagnostics.push(
+          diagnostic(
+            Codes.UNTERMINATED_STRING,
+            'unterminated string literal',
+            file, start, i - start,
+          ),
+        );
+      }
+      push(TokenKind.String, start, i, closed ? {} : { unterminated: true });
+      continue;
+    }
+
+    // Template strings: `TICK ${ticks} OPTION ${option}`. Lexed as one token
+    // so the parser sees a single literal, with `parts` marking each run of
+    // text and each `${...}` field's source span (the field's own tokens are
+    // produced by the parser re-lexing that span, offsets intact). Braces
+    // nest inside a field so a future `{`-bearing expression still ends at
+    // the right `}`. Single-line, like the other strings.
+    if (c === '`') {
+      const start = i;
+      i += 1;
+      const parts = [];
+      let textStart = i;
+      let closed = false;
+      const flushText = (end) => {
+        if (end > textStart) parts.push({ kind: 'text', start: textStart, end });
+      };
+      while (i < text.length) {
+        if (text[i] === '\\') { i += 2; continue; }
+        if (text[i] === '`') { flushText(i); i += 1; closed = true; break; }
+        if (text[i] === '\n') break;
+        if (text[i] === '$' && text[i + 1] === '{') {
+          flushText(i);
+          const fieldStart = i;
+          i += 2;
+          const sourceStart = i;
+          let depth = 1;
+          while (i < text.length && text[i] !== '\n') {
+            if (text[i] === '{') depth += 1;
+            else if (text[i] === '}') { depth -= 1; if (depth === 0) break; }
+            i += 1;
+          }
+          if (depth !== 0) {
+            diagnostics.push(diagnostic(
+              Codes.UNTERMINATED_STRING, "unterminated '${' field in template string", file, fieldStart, i - fieldStart,
+            ));
+            // Recorded anyway: a field being typed has no `}` yet, and
+            // hover and completion inside it are exactly what a person
+            // wants at that moment (half-typed source is the normal input,
+            // see this file's header).
+            parts.push({ kind: 'field', start: fieldStart, end: i, sourceStart, sourceEnd: i });
+            closed = true; // one diagnostic, not this plus "unterminated template"
+            break;
+          }
+          parts.push({ kind: 'field', start: fieldStart, end: i + 1, sourceStart, sourceEnd: i });
+          i += 1;
+          textStart = i;
+          continue;
+        }
+        i += 1;
+      }
+      if (!closed) {
+        diagnostics.push(diagnostic(
+          Codes.UNTERMINATED_STRING, 'unterminated template string', file, start, i - start,
+        ));
+      }
+      push(TokenKind.Template, start, i, { parts });
+      continue;
+    }
+
+    // Numbers, in the C spellings and the assembly spellings.
+    //
+    // `%` is also the modulo operator, so `%101` is a binary literal only where
+    // a value is expected: after an identifier, a literal, or a closing bracket
+    // the `%` in `x%2` has to be modulo. `$` has no such conflict.
+    const prev = tokens[tokens.length - 1];
+    const prevIsOperand = prev && (
+      prev.kind === TokenKind.Identifier ||
+      prev.kind === TokenKind.Number ||
+      prev.kind === TokenKind.String ||
+      prev.kind === TokenKind.Template ||
+      prev.kind === TokenKind.Type ||
+      [')', ']'].includes(prev.text)
+    );
+    const startsBinaryLiteral = c === '%' && /[01]/.test(text[i + 1] ?? '') && !prevIsOperand;
+    const startsHexLiteral = c === '$' && /[0-9a-fA-F]/.test(text[i + 1] ?? '');
+
+    if (isDigit(c) || startsHexLiteral || startsBinaryLiteral) {
+      const start = i;
+      let radix = 10;
+      if (c === '$') { radix = 16; i += 1; }
+      else if (c === '%') { radix = 2; i += 1; }
+      else if (c === '0' && /[xX]/.test(text[i + 1] ?? '')) { radix = 16; i += 2; }
+      else if (c === '0' && /[bB]/.test(text[i + 1] ?? '')) { radix = 2; i += 2; }
+      const digitsStart = i;
+      const digitPattern = DIGITS_FOR_RADIX[radix];
+      while (i < text.length && digitPattern.test(text[i])) i += 1;
+      const digits = text.slice(digitsStart, i).replace(/_/g, '');
+      if (digits === '') {
+        // `0x` with nothing after it. Reported here so the value can never be
+        // a silent NaN travelling through the checker.
+        diagnostics.push(
+          diagnostic(
+            Codes.INVALID_NUMBER,
+            `invalid number literal '${text.slice(start, i)}'`,
+            file, start, i - start,
+          ),
+        );
+        push(TokenKind.Number, start, i, { value: 0, radix });
+        continue;
+      }
+      // A decimal fraction — `0.5` — radix 10 only, and only when a digit
+      // actually follows the `.`: a bare `1.` stays `1` then the `.`
+      // operator (unchanged), and `array<u8, 16>`-style code elsewhere in
+      // the grammar never wants a Number token to swallow a trailing `.`.
+      // Recorded as an exact numerator/denominator pair, never as a
+      // floating-point value used for arithmetic — the only thing that ever
+      // reads `isDecimal`/`numerator`/`denominator` is the `#frames(...)`
+      // compile-time fold (packages/compiler/src/fold), which works in
+      // exact integers throughout; `value` here is cosmetic only (kept for
+      // uniformity with plain-integer Number tokens).
+      if (radix === 10 && text[i] === '.' && isDigit(text[i + 1] ?? '')) {
+        i += 1; // the '.'
+        const fracStart = i;
+        while (i < text.length && digitPattern.test(text[i])) i += 1;
+        const fracDigits = text.slice(fracStart, i).replace(/_/g, '');
+        push(TokenKind.Number, start, i, {
+          value: Number.parseFloat(text.slice(start, i)),
+          radix,
+          isDecimal: true,
+          numerator: Number.parseInt(digits + fracDigits, 10),
+          denominator: 10 ** fracDigits.length,
+        });
+        continue;
+      }
+
+      push(TokenKind.Number, start, i, { value: Number.parseInt(digits, radix), radix });
+      continue;
+    }
+
+    // `#frames(...)` — a compile-time function (see TokenKind.CompileTime).
+    if (c === '#' && isIdentStart(text[i + 1] ?? '')) {
+      const start = i;
+      i += 1;
+      while (i < text.length && isIdentPart(text[i])) i += 1;
+      push(TokenKind.CompileTime, start, i);
+      continue;
+    }
+
+    // `@address(0x900F)` and friends.
+    if (c === '@' && isIdentStart(text[i + 1] ?? '')) {
+      const start = i;
+      i += 1;
+      while (i < text.length && isIdentPart(text[i])) i += 1;
+      push(TokenKind.Decorator, start, i);
+      continue;
+    }
+
+    if (isIdentStart(c)) {
+      const start = i;
+      while (i < text.length && isIdentPart(text[i])) i += 1;
+      const word = text.slice(start, i);
+      const kind = KEYWORDS.has(word)
+        ? TokenKind.Keyword
+        : TYPE_NAMES.has(word)
+          ? TokenKind.Type
+          : TokenKind.Identifier;
+      push(kind, start, i);
+
+      // The body of an `asm6502 { ... }` block is 6502 assembly, not
+      // 8BitScript. Lexing it as 8BitScript is simply wrong — `lda #$06` would
+      // report `#` as an unexpected character — so the whole block is taken as
+      // one opaque token and handed to the backend untouched.
+      if (word === 'asm6502') {
+        let j = i;
+        while (j < text.length && /\s/.test(text[j])) j += 1;
+        if (text[j] === '{') {
+          const bodyStart = j;
+          let depth = 0;
+          while (j < text.length) {
+            if (text[j] === '{') depth += 1;
+            else if (text[j] === '}') {
+              depth -= 1;
+              if (depth === 0) { j += 1; break; }
+            }
+            j += 1;
+          }
+          if (depth !== 0) {
+            diagnostics.push(
+              diagnostic(
+                Codes.UNTERMINATED_ASM_BLOCK,
+                'unterminated asm6502 block',
+                file, bodyStart, text.length - bodyStart,
+              ),
+            );
+          }
+          push(TokenKind.AsmBlock, bodyStart, j);
+          i = j;
+        }
+      }
+      continue;
+    }
+
+    if (OPEN_BRACKETS.has(c)) {
+      brackets.push({ char: c, offset: i });
+      push(TokenKind.Punctuation, i, i + 1);
+      i += 1;
+      continue;
+    }
+    if (c in BRACKET_PAIRS) {
+      const top = brackets.pop();
+      if (!top || top.char !== BRACKET_PAIRS[c]) {
+        diagnostics.push(
+          diagnostic(Codes.UNMATCHED_BRACKET, `unmatched '${c}'`, file, i, 1),
+        );
+        if (top) brackets.push(top);
+      }
+      push(TokenKind.Punctuation, i, i + 1);
+      i += 1;
+      continue;
+    }
+
+    const operator = OPERATORS.find((op) => text.startsWith(op, i));
+    if (operator) {
+      push(TokenKind.Operator, i, i + operator.length);
+      i += operator.length;
+      continue;
+    }
+
+    diagnostics.push(
+      diagnostic(Codes.UNEXPECTED_CHARACTER, `unexpected character '${c}'`, file, i, 1),
+    );
+    i += 1;
+  }
+
+  for (const open of brackets) {
+    diagnostics.push(
+      diagnostic(Codes.UNCLOSED_BRACKET, `unclosed '${open.char}'`, file, open.offset, 1),
+    );
+  }
+
+  return { tokens, diagnostics };
+}

package/src/linker/hazards.mjs

@@ -0,0 +1,121 @@
+// Hardware hazards: writes a target documents as able to damage the machine.
+//
+// The root AGENTS.md warns against turning one game's trivia into a compiler
+// diagnostic. This table is the deliberate exception, and its bar is high: a
+// write goes here only when the target's own documentation says it can
+// destroy hardware — not crash the program, not garble the screen, destroy
+// hardware. There is one entry today. The PET's "killer poke" (POKE 59458,62
+// — `$E842`, the 6522 VIA's data-direction register B) makes port bit PB5
+// an output. PB5 is the vertical-retrace *input*; on the CRTC models (the
+// 12-inch 4032, 8032 and later) it is wired to the CRTC's vertical sync,
+// which also drives the monitor, and driving that line drags its level down
+// until the monitor's vertical deflection misbehaves and, in time, the
+// flyback fails. See packages/pet/AGENTS.md ("Hazards").
+//
+// The rule is narrow on purpose: the register has legitimate uses (its
+// other bits steer the cassette motor and IEEE-488 lines), so a write whose
+// value 8bitscript can see at compile time — a literal or a const — with bit
+// 5 clear is allowed, and only a write it cannot prove safe is refused: a
+// constant with bit 5 set, or a runtime value. Reads are always fine.
+//
+// The check runs in the linker, not the checker, because it needs two
+// things only the linker has: the machine being built for, and every const
+// already inlined (so `memory.write(VIA_DDRB, 62)` with `VIA_DDRB` imported
+// from another module is the same write as the literal). `8bs check` and the
+// editor analyse files without a machine, so this class of diagnostic is a
+// build-time one; docs/compiler.md and docs/language-server.md say so.
+
+import { Codes, diagnostic } from '../diagnostics/index.mjs';
+
+/**
+ * Per machine: the addresses whose writes are checked. `forbiddenBits` is
+ * the mask a compile-time value must have clear to be allowed; `name` and
+ * `why` are the words of the message.
+ */
+export const HARDWARE_HAZARDS = {
+  pet: [
+    {
+      address: 0xE842,
+      forbiddenBits: 0x20,
+      name: 'the VIA\'s data-direction register B ($E842, POKE 59458)',
+      why: 'a value with bit 5 set makes PB5 an output — the PET "killer poke", which drives the monitor\'s '
+        + 'vertical sync on the CRTC models (4032, 8032 and later) and can destroy a 12-inch PET display. '
+        + 'Only a compile-time value with bit 5 clear may be written here; see packages/pet/AGENTS.md',
+    },
+  ],
+};
+
+const hex = (n) => `$${n.toString(16).toUpperCase().padStart(4, '0')}`;
+
+/**
+ * Walk the linked program's functions for writes to a hazardous address on
+ * `machine`, reporting each as `8BS3003` in the file it was written in.
+ *
+ * Three ways to write an address are covered: `memory.write(address, v)`
+ * with a compile-time address; assignment to a scalar global declared
+ * `@address(...)` there; and an element store into an `@address` array that
+ * covers it (a compile-time index that lands on it, or a runtime index,
+ * which cannot be proved to miss). Everything else — a runtime address to
+ * `memory.write`, say — is the program's own responsibility, as on the
+ * machine.
+ *
+ * @param {object} ir            the linked IR (globals carry output names and addresses)
+ * @param {string|undefined} machine
+ * @param {Map<object, string>} fileOf  each function to the file it came from
+ * @param {object[]} diagnostics
+ */
+export function checkHardwareHazards(ir, machine, fileOf, diagnostics) {
+  const hazards = machine ? HARDWARE_HAZARDS[machine] : undefined;
+  if (!hazards) return;
+
+  // The globals that map a hazardous address, by output name: scalars at
+  // it exactly, arrays that span it (with the element offset that is it).
+  const scalars = new Map();
+  const arrays = new Map();
+  for (const g of ir.globals) {
+    if (g.address === null || g.address === undefined) continue;
+    for (const hazard of hazards) {
+      if (!g.array && g.address === hazard.address) scalars.set(g.name, hazard);
+      if (g.array && g.address <= hazard.address && hazard.address < g.address + g.array) {
+        arrays.set(g.name, { hazard, offset: hazard.address - g.address });
+      }
+    }
+  }
+
+  const safe = (value, hazard) => value?.kind === 'const' && (value.value & hazard.forbiddenBits) === 0;
+  const report = (node, file, hazard, how) => diagnostics.push(diagnostic(
+    Codes.HARDWARE_HAZARD,
+    `${how} ${hazard.name} on the ${machine}: ${hazard.why}`,
+    file, node.start ?? 0, node.length ?? 0,
+  ));
+  const describe = (value) => (value?.kind === 'const' ? `writing ${value.value} to` : 'writing a runtime value to');
+
+  const visit = (s, file) => {
+    if (s.kind === 'memoryWrite' && s.address?.kind === 'const') {
+      const hazard = hazards.find((h) => h.address === s.address.value);
+      if (hazard && !safe(s.value, hazard)) report(s, file, hazard, describe(s.value));
+    } else if (s.kind === 'assign' && scalars.has(s.target)) {
+      const hazard = scalars.get(s.target);
+      if (!safe(s.value, hazard)) report(s, file, hazard, describe(s.value));
+    } else if (s.kind === 'storeIndex' && arrays.has(s.array?.name)) {
+      const { hazard, offset } = arrays.get(s.array.name);
+      const index = s.index;
+      if (index?.kind === 'const' && index.value !== offset) return;
+      if (!safe(s.value, hazard)) {
+        const how = index?.kind === 'const'
+          ? describe(s.value)
+          : `an element store with a runtime index cannot be proved to miss ${hex(hazard.address)}, so this counts as writing to`;
+        report(s, file, hazard, how);
+      }
+    }
+  };
+  const walk = (body, file) => {
+    for (const s of body) {
+      visit(s, file);
+      if (s.kind === 'if') { walk(s.then, file); if (s.else) walk(s.else, file); }
+      else if (s.kind === 'while' || s.kind === 'block') walk(s.body, file);
+      else if (s.kind === 'for') { if (s.init) visit(s.init, file); if (s.update) visit(s.update, file); walk(s.body, file); }
+    }
+  };
+  for (const fn of ir.functions) walk(fn.body, fileOf.get(fn) ?? '');
+}