openalgo-script 0.1.0-alpha.0

package/src/core/lex/numbers.ts

@@ -0,0 +1,77 @@
+import { DOT, isDigit, isHexDigit, UNDERSCORE } from './characters.js';
+
+/**
+ * A number literal, in every form of language.md 3.5: decimal with or without a
+ * fractional part, a fractional part with no leading digit, an exponent, digit
+ * group underscores and the hexadecimal form. There is no octal form, so `010`
+ * is ten, which falls out of reading the digits rather than being a special
+ * case.
+ */
+export interface NumberLiteral {
+  /** One past the last character of the literal. */
+  readonly end: number;
+  readonly value: number;
+}
+
+/**
+ * An underscore separates digit groups, so it only belongs to the literal when
+ * a digit follows it. `1_` and `1__0` therefore end the literal at the first
+ * underscore, and the caller reports what follows as a name that starts with a
+ * digit, which is what they are.
+ */
+function runOfDigits(text: string, from: number, digit: (code: number) => boolean): number {
+  let i = from;
+  for (;;) {
+    const code = text.charCodeAt(i);
+    if (digit(code)) {
+      i++;
+      continue;
+    }
+    if (code === UNDERSCORE && digit(text.charCodeAt(i + 1))) {
+      i += 2;
+      continue;
+    }
+    return i;
+  }
+}
+
+/**
+ * Reads the literal starting at `start`, which the caller has established
+ * begins a number: a digit, or a dot with a digit after it.
+ *
+ * The value is computed here and carried on the token so that no later stage
+ * re-reads the text and has to agree a second time about underscores and about
+ * the hexadecimal form.
+ */
+export function scanNumber(text: string, start: number): NumberLiteral {
+  const second = text.charCodeAt(start + 1);
+  const isHex =
+    text.charCodeAt(start) === 0x30 &&
+    (second === 0x78 || second === 0x58) &&
+    isHexDigit(text.charCodeAt(start + 2));
+
+  if (isHex) {
+    const end = runOfDigits(text, start + 2, isHexDigit);
+    return { end, value: Number(text.slice(start, end).replace(/_/g, '')) };
+  }
+
+  let i = runOfDigits(text, start, isDigit);
+
+  // A dot only belongs to the literal when a digit follows it, so `x.y` reads as
+  // an element access and `1.` reads as a number and a dot.
+  if (text.charCodeAt(i) === DOT && isDigit(text.charCodeAt(i + 1))) {
+    i = runOfDigits(text, i + 1, isDigit);
+  }
+
+  const exponent = text.charCodeAt(i);
+  if (exponent === 0x65 || exponent === 0x45) {
+    const sign = text.charCodeAt(i + 1);
+    const digits = sign === 0x2b || sign === 0x2d ? i + 2 : i + 1;
+    // Without a digit the `e` is not an exponent, and the caller reports the
+    // letter sitting against a number rather than this reading a literal that
+    // is not there.
+    if (isDigit(text.charCodeAt(digits))) i = runOfDigits(text, digits, isDigit);
+  }
+
+  return { end: i, value: Number(text.slice(start, i).replace(/_/g, '')) };
+}

package/src/core/lex/statements.ts

@@ -0,0 +1,94 @@
+import type { Token, TokenKind } from '../tokens/index.js';
+
+/**
+ * Where a statement ends, and whether it opens a block. Both questions are
+ * asked of the tokens of a line, which is the whole of what language.md 3.10
+ * and 3.11 need to know about a statement before it is parsed.
+ */
+
+/**
+ * The tokens that promise a right-hand side, from 3.11: a binary operator, a
+ * comma, a `?`, a `:` or an `=`. A line ending in one of them continues onto
+ * the next line.
+ *
+ * The compound assignments are here on the same terms as `=`: each one promises
+ * a value just as plainly. `.` is deliberately absent, because 3.11 does not
+ * list it and a line that ends in a dot is a mistake rather than a wrap.
+ */
+const CONTINUING: ReadonlySet<TokenKind> = new Set<TokenKind>([
+  '+',
+  '-',
+  '*',
+  '/',
+  '%',
+  '==',
+  '!=',
+  '<',
+  '<=',
+  '>',
+  '>=',
+  '=',
+  '+=',
+  '-=',
+  '*=',
+  '/=',
+  '%=',
+  ',',
+  '?',
+  ':',
+  'and',
+  'or',
+]);
+
+/** The words that introduce a block, per 3.10 and the grammar of section 19. */
+const HEADERS: ReadonlySet<TokenKind> = new Set<TokenKind>([
+  'if',
+  'else',
+  'for',
+  'while',
+  'switch',
+  'case',
+  'default',
+]);
+
+/**
+ * Whether the last two tokens are the `=>` of a function declaration.
+ *
+ * Section 3.12 does not list `=>` among the punctuation, so it arrives as an
+ * `=` and a `>` that touch. Both readings need saying apart here: a line whose
+ * last token is `>` continues onto the next line, and a line that ends in `=>`
+ * is a function header whose body is the indented block below it, so it must
+ * not.
+ */
+export function endsWithArrow(tokens: readonly Token[]): boolean {
+  const last = tokens[tokens.length - 1];
+  const before = tokens[tokens.length - 2];
+  if (last === undefined || before === undefined) return false;
+  return (
+    last.kind === '>' &&
+    before.kind === '=' &&
+    before.span.offset + before.span.length === last.span.offset
+  );
+}
+
+/** Whether the statement so far promises more, so the next line belongs to it. */
+export function continuesLine(tokens: readonly Token[]): boolean {
+  const last = tokens[tokens.length - 1];
+  if (last === undefined) return false;
+  if (endsWithArrow(tokens)) return false;
+  return CONTINUING.has(last.kind);
+}
+
+/**
+ * Whether a statement that begins with this token and ends this way opens a
+ * block.
+ *
+ * `fn` is the one header with a form that fits on one line: with its body after
+ * the `=>` it opens nothing, and with nothing after the `=>` the block below it
+ * is the body (3.10, 11.1). Every other header opens a block always, because
+ * there is no statement separator to end a one-line body with.
+ */
+export function opensBlock(firstKind: TokenKind, arrowAtEnd: boolean): boolean {
+  if (firstKind === 'fn') return arrowAtEnd;
+  return HEADERS.has(firstKind);
+}

package/src/core/lex/strings.ts

@@ -0,0 +1,119 @@
+import { BACKSLASH, isHexDigit, LINE_FEED } from './characters.js';
+
+/** A backslash sequence the language does not define, for OS1005. */
+export interface BadEscape {
+  readonly offset: number;
+  readonly length: number;
+  /** The backslash and what followed it, which is the {sequence} of OS1005. */
+  readonly text: string;
+}
+
+export interface StringLiteral {
+  /** One past the closing quote, or the end of the line when there was none. */
+  readonly end: number;
+  /** The text the literal denotes, with its escapes already resolved. */
+  readonly value: string;
+  readonly unterminated: boolean;
+  readonly badEscapes: readonly BadEscape[];
+}
+
+/** The escapes of language.md 3.6, apart from \uXXXX which carries its own digits. */
+const SIMPLE = new Map<number, string>([
+  [0x5c, '\\'],
+  [0x22, '"'],
+  [0x27, "'"],
+  [0x6e, '\n'],
+  [0x74, '\t'],
+  [0x72, '\r'],
+  [0x30, '\u0000'],
+]);
+
+const UNICODE_ESCAPE = 0x75;
+
+/** How many hexadecimal digits follow, up to the four a \uXXXX escape needs. */
+function hexDigitsAfter(text: string, from: number): number {
+  let count = 0;
+  while (count < 4 && isHexDigit(text.charCodeAt(from + count))) count++;
+  return count;
+}
+
+/**
+ * Reads the literal opening at `start`, which is the quote character, in either
+ * delimiter of language.md 3.6.
+ *
+ * A literal may not span a line, so the scan stops at the newline and says so
+ * rather than swallowing the rest of the file. The caller reports OS1004 at the
+ * opening quote, which is where the mistake is, and keeps the text that was
+ * read so that the statement around it still parses and the reader is not shown
+ * a second error for the same missing character.
+ */
+export function scanString(text: string, start: number): StringLiteral {
+  const quote = text.charCodeAt(start);
+  const badEscapes: BadEscape[] = [];
+  let value = '';
+  let plainFrom = start + 1;
+  let i = start + 1;
+
+  const takePlain = (upTo: number): void => {
+    if (upTo > plainFrom) value += text.slice(plainFrom, upTo);
+  };
+
+  while (i < text.length) {
+    const code = text.charCodeAt(i);
+
+    if (code === LINE_FEED) break;
+
+    if (code === quote) {
+      takePlain(i);
+      return { end: i + 1, value, unterminated: false, badEscapes };
+    }
+
+    if (code !== BACKSLASH) {
+      i++;
+      continue;
+    }
+
+    const next = text.charCodeAt(i + 1);
+    // A backslash at the end of the line escapes nothing, because the line is
+    // where the literal has to end. That is one missing quote, not two errors.
+    if (Number.isNaN(next) || next === LINE_FEED) break;
+
+    takePlain(i);
+
+    const simple = SIMPLE.get(next);
+    if (simple !== undefined) {
+      value += simple;
+      i += 2;
+      plainFrom = i;
+      continue;
+    }
+
+    if (next === UNICODE_ESCAPE) {
+      const digits = hexDigitsAfter(text, i + 2);
+      if (digits === 4) {
+        value += String.fromCharCode(Number.parseInt(text.slice(i + 2, i + 6), 16));
+        i += 6;
+        plainFrom = i;
+        continue;
+      }
+      const length = 2 + digits;
+      badEscapes.push({ offset: i, length, text: text.slice(i, i + length) });
+      value += text.slice(i + 1, i + length);
+      i += length;
+      plainFrom = i;
+      continue;
+    }
+
+    // The written character, whole: an astral character is two code units, and
+    // reporting half of one puts a broken surrogate in the message.
+    const written = String.fromCodePoint(text.codePointAt(i + 1) ?? next);
+    const length = 1 + written.length;
+    badEscapes.push({ offset: i, length, text: text.slice(i, i + length) });
+    value += written;
+    i += length;
+    plainFrom = i;
+  }
+
+  takePlain(i);
+  return { end: i, value, unterminated: true, badEscapes };
+}

package/src/core/lex/unexpected.ts

@@ -0,0 +1,125 @@
+/**
+ * OS1001: naming the character that was found, and the plain spelling to write
+ * instead.
+ *
+ * The replacement sentences are the substitution table of errors.md section 7,
+ * which is the one part of an OS1001 message that errors.json does not carry:
+ * the catalogue holds the message and the fix, and section 7 holds the third
+ * column that fills {suggestion}. The table is transcribed here rather than
+ * paraphrased, and the moment it moves into errors.json this file is generated
+ * away instead of being edited.
+ *
+ * A character the table does not name takes the generic sentence, which is what
+ * section 7 says to do with one.
+ */
+
+export interface Unexpected {
+  /** The {char} slot: what was found, quoted, and named when it cannot be seen. */
+  readonly char: string;
+  /** The {suggestion} slot: one sentence naming the plain spelling to use. */
+  readonly suggestion: string;
+}
+
+const PLAIN_SPACE = 'Use a plain space instead.';
+const STRAIGHT_QUOTE = 'Use a straight quote instead.';
+const ASCII_NAME = 'Use the ASCII spelling of the name instead.';
+const INDENTATION = 'Use indentation instead, which is how a block is written.';
+const PUT_IN_A_STRING = 'Delete it, or put the text in a string.';
+const GENERIC = 'Delete it, or move it inside a string literal.';
+
+/** The rows of section 7 that name an exact spelling. */
+const SPELLINGS = new Map<string, string>([
+  ['!', 'Write not instead.'],
+  ['&&', 'Write and instead.'],
+  ['||', 'Write or instead.'],
+  ['^', 'Write pow(a, b) instead.'],
+  ['**', 'Write pow(a, b) instead.'],
+  ['++', 'Write a += 1 instead.'],
+  ['{', INDENTATION],
+  ['}', INDENTATION],
+  ['#', PUT_IN_A_STRING],
+  ['$', PUT_IN_A_STRING],
+  ['@', PUT_IN_A_STRING],
+]);
+
+/**
+ * Names for the characters a reader cannot see on their screen.
+ *
+ * A message that quotes an invisible character quotes nothing, and the whole
+ * point of OS1001 is that an invisible character produces a baffling error
+ * three tokens later. The name and the code point are what makes the message
+ * act on: a reader can search their file for the one and read the other aloud.
+ */
+const NAMES = new Map<number, string>([
+  [0x0009, 'a tab'],
+  [0x000d, 'a carriage return'],
+  [0x00a0, 'a no-break space'],
+  [0x2002, 'an en space'],
+  [0x2003, 'an em space'],
+  [0x2007, 'a figure space'],
+  [0x2009, 'a thin space'],
+  [0x200a, 'a hair space'],
+  [0x202f, 'a narrow no-break space'],
+  [0x3000, 'an ideographic space'],
+  [0x200b, 'a zero width space'],
+  [0x200c, 'a zero width non-joiner'],
+  [0x200d, 'a zero width joiner'],
+  [0xfeff, 'a byte order mark'],
+  [0x2018, 'a left single quotation mark'],
+  [0x2019, 'a right single quotation mark'],
+  [0x201c, 'a left double quotation mark'],
+  [0x201d, 'a right double quotation mark'],
+  [0x2013, 'an en dash'],
+  [0x2014, 'an em dash'],
+  [0x2026, 'a horizontal ellipsis'],
+]);
+
+const TYPOGRAPHIC_QUOTES = '\u2018\u2019\u201c\u201d\u201a\u201b\u201e\u201f';
+
+const SPACE_SEPARATOR = /\p{Zs}/u;
+const LETTER = /\p{L}/u;
+const ASCII_GRAPHIC = /^[\x21-\x7e]+$/;
+
+function codePointLabel(text: string): string {
+  const point = text.codePointAt(0) ?? 0;
+  return `U+${point.toString(16).toUpperCase().padStart(4, '0')}`;
+}
+
+/**
+ * The {char} slot. An ASCII graphic speaks for itself; everything else is
+ * quoted and then identified, because the quotes may well hold nothing a reader
+ * can see.
+ */
+function describe(written: string): string {
+  if (ASCII_GRAPHIC.test(written)) return `"${written}"`;
+  const name = NAMES.get(written.codePointAt(0) ?? -1);
+  const label = codePointLabel(written);
+  return name === undefined ? `"${written}" (${label})` : `"${written}" (${name}, ${label})`;
+}
+
+function suggestionFor(written: string): string {
+  const spelling = SPELLINGS.get(written);
+  if (spelling !== undefined) return spelling;
+  // A tab is a space that happens to be a control character, and the table's
+  // answer to every other invisible space is the same plain space.
+  if (written === '\t' || SPACE_SEPARATOR.test(written)) return PLAIN_SPACE;
+  if (TYPOGRAPHIC_QUOTES.includes(written)) return STRAIGHT_QUOTE;
+  if (LETTER.test(written)) return ASCII_NAME;
+  return GENERIC;
+}
+
+/** What OS1001 says about one rejected character, or one rejected operator. */
+export function describeUnexpected(written: string): Unexpected {
+  return { char: describe(written), suggestion: suggestionFor(written) };
+}
+
+/**
+ * Whether a code point is a letter the language does not allow in a name.
+ *
+ * It is asked once per word, at the character that ended the word, so that
+ * `l\u00e4ngd` is one name with one error on the one character 3.3 refuses,
+ * rather than two names with a stray character between them.
+ */
+export function isNonAsciiLetter(point: number): boolean {
+  return point > 0x7f && LETTER.test(String.fromCodePoint(point));
+}

package/src/core/parse/annotations.ts

@@ -0,0 +1,60 @@
+import type { TypeAnnotation } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import { spanning } from '../span/index.js';
+import type { TokenKind } from '../tokens/index.js';
+import { RESERVED_WORDS } from '../tokens/index.js';
+import type { Cursor } from './cursor.js';
+
+/**
+ * A type as a script writes it, on a parameter or a `var`.
+ *
+ * The grammar of section 19 admits only the real type names. The prose wins
+ * here, because OS2016 and OS2019 are check stage: this reads any word into a
+ * `namedType` and lets the checker be the one to say that `whole` is not a
+ * type, which is the difference between a caret with no sentence and a sentence
+ * naming the types that do exist.
+ */
+
+const RESERVED: ReadonlySet<TokenKind> = new Set<TokenKind>(RESERVED_WORDS);
+
+/**
+ * What may stand as a type name.
+ *
+ * A literal is taken as well as a word, so `len: 5` reaches OS2016 with `5` in
+ * its message instead of failing here with a caret and nothing to act on.
+ */
+function isTypeWord(kind: TokenKind): boolean {
+  return (
+    kind === 'identifier' ||
+    kind === 'numberLiteral' ||
+    kind === 'stringLiteral' ||
+    RESERVED.has(kind)
+  );
+}
+
+export function parseTypeAnnotation(cursor: Cursor): TypeAnnotation {
+  const token = cursor.token;
+
+  if (token.kind === 'series') {
+    cursor.advance();
+    const element = parseTypeAnnotation(cursor);
+    return makeNode('seriesType', spanning(token.span, element.span), { element });
+  }
+
+  if (token.kind === 'array' && cursor.peek().kind === '<') {
+    cursor.advance();
+    cursor.advance();
+    const element = parseTypeAnnotation(cursor);
+    const close = cursor.take('>');
+    if (close === undefined) cursor.unexpected(cursor.token);
+    return makeNode('arrayType', spanning(token.span, close?.span ?? element.span), { element });
+  }
+
+  if (isTypeWord(token.kind)) {
+    cursor.advance();
+    return makeNode('namedType', token.span, { name: token.text });
+  }
+
+  cursor.unexpected(token);
+  return makeNode('namedType', cursor.holeSpan(), { name: '' });
+}

package/src/core/parse/blocks.ts

@@ -0,0 +1,52 @@
+import type { Block, FunctionDeclaration, Statement } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import type { Span } from '../span/index.js';
+import { spanning } from '../span/index.js';
+import type { Cursor } from './cursor.js';
+import { parseStatement } from './statements.js';
+
+/**
+ * The indented lines under a header, language.md 3.10.
+ *
+ * The lexer has already decided where a block begins and ends, so this reads an
+ * `indent`, statements, and a `dedent`, and the one rule it owns is what to do
+ * when the `indent` is not there: OS1010, reported against the header rather
+ * than against the line that failed to be a body, because the header is what
+ * the reader has to change.
+ *
+ * The statement loop insists on progress. A rule that reports without consuming
+ * anything would otherwise spin on the token it could not read, and a compiler
+ * that hangs on a malformed file is worse than one that gives up on it.
+ */
+export function parseBlock(cursor: Cursor, header: string, at: Span): Block {
+  if (!cursor.at('indent')) {
+    // A header that was already reported on has nothing to add by saying its
+    // body is missing too: the body is missing because of what it was told.
+    if (!cursor.reportedHere) cursor.report('OS1010', at, { header });
+    const hole = cursor.holeSpan();
+    return makeNode('block', hole, { statements: [] });
+  }
+
+  if (!cursor.openBlock(cursor.token.span)) {
+    const span = cursor.token.span;
+    cursor.skipBlockBody();
+    return makeNode('block', span, { statements: [] });
+  }
+
+  const indent = cursor.advance();
+  const statements: (Statement | FunctionDeclaration)[] = [];
+
+  while (!cursor.at('dedent') && !cursor.at('endOfFile')) {
+    const before = cursor.position;
+    const statement = parseStatement(cursor);
+    if (statement !== undefined) statements.push(statement);
+    if (cursor.position === before) cursor.skipStatement();
+  }
+
+  cursor.closeBlock();
+  cursor.take('dedent');
+
+  const last = statements[statements.length - 1];
+  const span = last === undefined ? indent.span : spanning(indent.span, last.span);
+  return makeNode('block', span, { statements });
+}

package/src/core/parse/branches.ts

@@ -0,0 +1,77 @@
+import type { ElseBranch, IfBranch, IfStatement } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import type { Span } from '../span/index.js';
+import { spanning } from '../span/index.js';
+import type { Token } from '../tokens/index.js';
+import type { Cursor } from './cursor.js';
+import { parseBlock } from './blocks.js';
+import { parseCondition } from './expressions.js';
+import { finishStatement } from './statements.js';
+
+/**
+ * `if`, `else if` and `else`, language.md 10.2.
+ *
+ * The branches are kept beside each other rather than nested, because `else if`
+ * is two words on one line and does not increase indentation, so a reader sees
+ * one statement with several arms and every pass over the tree should see the
+ * same thing.
+ *
+ * An `else` pairs with the `if` at its own indentation. The lexer has already
+ * closed the blocks between them, so the `else` that arrives here is the one
+ * that belongs to this `if`; what is left to check is that the two line up, and
+ * OS1016 when they do not.
+ */
+export function parseIfStatement(cursor: Cursor): IfStatement {
+  const keyword = cursor.token;
+  cursor.noteIf(keyword.span.column);
+
+  const branches: IfBranch[] = [parseBranch(cursor, 'if', keyword.span)];
+  let elseBranch: ElseBranch | undefined;
+  let end = branches[0]?.span ?? keyword.span;
+
+  while (cursor.at('else')) {
+    const word = cursor.token;
+    pairWithIf(cursor, word, keyword);
+    cursor.advance();
+
+    if (cursor.at('if')) {
+      const branch = parseBranch(cursor, 'else if', word.span);
+      branches.push(branch);
+      end = branch.span;
+      continue;
+    }
+
+    elseBranch = parseElse(cursor, word);
+    end = elseBranch.span;
+    break;
+  }
+
+  return makeNode('ifStatement', spanning(keyword.span, end), { branches, elseBranch });
+}
+
+/** One `if` or `else if` header and the block under it. */
+function parseBranch(cursor: Cursor, header: string, start: Span): IfBranch {
+  cursor.beginStatement();
+  const keyword = cursor.advance();
+  const condition = parseCondition(cursor);
+  finishStatement(cursor);
+  const body = parseBlock(cursor, header, spanning(start, keyword.span));
+  return makeNode('ifBranch', spanning(start, body.span), { condition, body });
+}
+
+function parseElse(cursor: Cursor, word: Token): ElseBranch {
+  cursor.beginStatement();
+  // Nothing may follow `else` on its line: there is no statement separator to
+  // end a one line body with, so a body is written indented underneath.
+  finishStatement(cursor);
+  const body = parseBlock(cursor, 'else', word.span);
+  return makeNode('elseBranch', spanning(word.span, body.span), { body });
+}
+
+function pairWithIf(cursor: Cursor, word: Token, keyword: Token): void {
+  if (word.span.column === keyword.span.column) return;
+  cursor.report('OS1016', word.span, {
+    found: word.span.column - 1,
+    expected: keyword.span.column - 1,
+  });
+}