openalgo-script 0.1.0-alpha.0

package/src/core/render/terminal.ts

@@ -0,0 +1,62 @@
+import type { Diagnostic } from '../diagnostics/index.js';
+import type { SourceFile } from '../source/index.js';
+
+/**
+ * The renderer for a terminal, and the shape is the one errors.md section 2
+ * fixes as the contract:
+ *
+ *     14 |     len = 9
+ *        |     ^^^
+ *     OS2002: len is already declared at line 1, so a second one cannot be declared here.
+ *     Fix: rename this one, or drop the inner declaration and let the assignment update the len at line 1.
+ *
+ * The line, then the caret, then what happened, then what to do. The fix is on
+ * its own line because it is the line the reader acts on, and a reader who has
+ * understood the message from the caret alone should be able to skip to it.
+ */
+
+/** A tab is one column wide here, so the caret below the line still lines up. */
+const TAB_WIDTH_FOR_DISPLAY = ' ';
+
+function countCharacters(text: string): number {
+  // Code points, not code units: a terminal draws one character per code point,
+  // and padding by code units would push the caret right by one for every
+  // astral character earlier on the line. See the span module on columns.
+  return [...text].length;
+}
+
+export function renderDiagnostic(file: SourceFile, diagnostic: Diagnostic): string {
+  const { line, column, length } = diagnostic.span;
+  const shown = file.lineText(line).replace(/\t/g, TAB_WIDTH_FOR_DISPLAY);
+
+  const lineNumber = String(line);
+  const gutter = ' '.repeat(lineNumber.length);
+
+  const beforeCaret = shown.slice(0, Math.max(column - 1, 0));
+  // A span that runs past the end of its line, which an unterminated bracket
+  // produces, is drawn to the end of the line rather than off it.
+  const underCaret = shown.slice(beforeCaret.length, beforeCaret.length + length);
+  const caretWidth = Math.max(countCharacters(underCaret), 1);
+
+  return [
+    `${lineNumber} | ${shown}`,
+    `${gutter} | ${' '.repeat(countCharacters(beforeCaret))}${'^'.repeat(caretWidth)}`,
+    `${diagnostic.code}: ${diagnostic.message}`,
+    `Fix: ${diagnostic.fix}`,
+  ].join('\n');
+}
+
+/**
+ * Every diagnostic, in the order a reader walks the file, separated by a blank
+ * line. The file's name is printed once above them rather than on each one,
+ * because a compile reports on one file and repeating its name on twenty
+ * diagnostics buries the twenty.
+ */
+export function renderDiagnostics(
+  file: SourceFile,
+  diagnostics: readonly Diagnostic[],
+): string {
+  if (diagnostics.length === 0) return '';
+  const body = diagnostics.map((diagnostic) => renderDiagnostic(file, diagnostic));
+  return [`${file.name}`, ...body].join('\n\n');
+}

package/src/core/source/index.ts

@@ -0,0 +1,6 @@
+/**
+ * The source text and the index over it. The one place that normalises a file,
+ * so every offset in the compiler means the same thing.
+ */
+export type { SourceFile, SourcePosition } from './source.js';
+export { normaliseSource, sourceFile } from './source.js';

package/src/core/source/source.ts

@@ -0,0 +1,102 @@
+import { makeSpan } from '../span/index.js';
+import type { Span } from '../span/index.js';
+
+/** A place in the source, one-based on both axes. See the span module on columns. */
+export interface SourcePosition {
+  readonly line: number;
+  readonly column: number;
+}
+
+/**
+ * A file the compiler is working on: its normalised text, and the index that
+ * turns an offset into a line and a column.
+ *
+ * Every stage needs the same two answers, and computing them by scanning from
+ * the top of the file each time is how a compiler becomes quadratic on the one
+ * input that has many diagnostics. The line starts are built once.
+ */
+export interface SourceFile {
+  /** What the file is called, for the first line of a rendered diagnostic. */
+  readonly name: string;
+  /** The text after normalisation. Offsets and spans index this, not the bytes on disk. */
+  readonly text: string;
+  readonly lineCount: number;
+  /** The line's text, without its newline. An out of range line is empty. */
+  lineText(line: number): string;
+  /** The offset the line starts at. */
+  lineStart(line: number): number;
+  positionAt(offset: number): SourcePosition;
+  offsetAt(position: SourcePosition): number;
+  /** The span covering length code units from offset, with its line and column worked out. */
+  spanAt(offset: number, length: number): Span;
+}
+
+const BYTE_ORDER_MARK = '';
+
+/**
+ * Drops a leading byte order mark and turns CRLF into LF, which language.md 3.1
+ * requires to happen before anything else reads the file, so that a script
+ * written on one operating system compiles identically on another.
+ *
+ * A lone carriage return is deliberately left alone. It is not a line ending
+ * the language accepts, so it belongs to the lexer as OS1001 on the character
+ * rather than being silently repaired here.
+ */
+export function normaliseSource(raw: string): string {
+  const withoutMark = raw.startsWith(BYTE_ORDER_MARK) ? raw.slice(BYTE_ORDER_MARK.length) : raw;
+  return withoutMark.includes('\r\n') ? withoutMark.replace(/\r\n/g, '\n') : withoutMark;
+}
+
+function lineStartsOf(text: string): readonly number[] {
+  const starts = [0];
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) === 10) starts.push(i + 1);
+  }
+  return starts;
+}
+
+export function sourceFile(name: string, raw: string): SourceFile {
+  const text = normaliseSource(raw);
+  const starts = lineStartsOf(text);
+
+  const clampLine = (line: number): number => Math.min(Math.max(Math.trunc(line), 1), starts.length);
+
+  const lineStart = (line: number): number => starts[clampLine(line) - 1] ?? 0;
+
+  const lineEnd = (line: number): number => {
+    const next = starts[clampLine(line)];
+    if (next === undefined) return text.length;
+    // The next line starts one past the newline, which this line does not own.
+    return next - 1;
+  };
+
+  const positionAt = (offset: number): SourcePosition => {
+    const at = Math.min(Math.max(Math.trunc(offset), 0), text.length);
+    let low = 0;
+    let high = starts.length - 1;
+    while (low < high) {
+      const middle = (low + high + 1) >> 1;
+      if ((starts[middle] ?? 0) <= at) low = middle;
+      else high = middle - 1;
+    }
+    return { line: low + 1, column: at - (starts[low] ?? 0) + 1 };
+  };
+
+  return {
+    name,
+    text,
+    lineCount: starts.length,
+    lineText: (line) => text.slice(lineStart(line), lineEnd(line)),
+    lineStart,
+    positionAt,
+    offsetAt: (position) => {
+      const start = lineStart(position.line);
+      const end = lineEnd(position.line);
+      return Math.min(start + Math.max(Math.trunc(position.column), 1) - 1, end);
+    },
+    spanAt: (offset, length) => {
+      const { line, column } = positionAt(offset);
+      return makeSpan(offset, Math.max(length, 0), line, column);
+    },
+  };
+}

package/src/core/span/index.ts

@@ -0,0 +1,2 @@
+export type { Positioned, Span } from './span.js';
+export { containsOffset, endOffset, makeSpan, spanning } from './span.js';

package/src/core/span/span.ts

@@ -0,0 +1,84 @@
+/**
+ * Where something is in the source text.
+ *
+ * Every token and every node carries one of these, and every diagnostic is
+ * drawn under one, so the definition of a column here decides where a caret
+ * lands for every reader of the language. It is stated rather than implied.
+ *
+ * ## A column counts UTF-16 code units
+ *
+ * An editor and a terminal disagree about what a column is, and there is no
+ * answer that is right for both, so the question is which one gets the stored
+ * number and which one converts.
+ *
+ * The stored number is UTF-16 code units, one-based, because that is what the
+ * things that consume a column ask for: a browser text component indexes the
+ * text it holds in UTF-16, and the language server protocol an editor speaks
+ * counts positions in UTF-16 by default. Storing anything else means converting
+ * at every token on the hot path of a compile that runs while somebody waits,
+ * and converting is exactly where an off-by-one caret comes from.
+ *
+ * A terminal counts characters, not code units, so the renderer does not use
+ * the column at all. It takes the source line and the span's offsets and counts
+ * code points itself when it pads the caret, which is why a caret sitting after
+ * a string literal holding an astral character still lands under the right
+ * character in a terminal.
+ *
+ * The practical reach of the difference is small and worth knowing: outside a
+ * string literal the language accepts ASCII only (language.md 3.1), so a column
+ * can only diverge from a character count on a line that holds a string literal
+ * with non-ASCII text in it.
+ *
+ * ## Offsets index the normalised text
+ *
+ * A byte order mark is dropped and CRLF becomes LF before anything else reads
+ * the file (language.md 3.1), so an offset counts into the text after those two
+ * changes. A host that has to point back into the file on disk adds the mark
+ * and the carriage returns back itself; see the source module, which is the one
+ * place that does the normalising.
+ */
+export interface Span {
+  /** Zero-based UTF-16 code unit index into the normalised source text. */
+  readonly offset: number;
+  /** Length in UTF-16 code units. Zero where a diagnostic points between two characters. */
+  readonly length: number;
+  /** One-based, because that is the first line an editor shows. */
+  readonly line: number;
+  /** One-based UTF-16 code unit count from the start of the line. */
+  readonly column: number;
+}
+
+/** Anything the compiler can point a caret at. */
+export interface Positioned {
+  readonly span: Span;
+}
+
+export function makeSpan(offset: number, length: number, line: number, column: number): Span {
+  return { offset, length, line, column };
+}
+
+/** One past the last code unit the span covers. */
+export function endOffset(span: Span): number {
+  return span.offset + span.length;
+}
+
+/**
+ * The span from the start of the first to the end of the last.
+ *
+ * This is how a node gets a span: a call expression runs from its callee to its
+ * closing bracket, and the caret under it covers the whole call rather than the
+ * one token the parser happened to be holding.
+ */
+export function spanning(first: Span, last: Span): Span {
+  return {
+    offset: first.offset,
+    length: Math.max(endOffset(last) - first.offset, 0),
+    line: first.line,
+    column: first.column,
+  };
+}
+
+/** Whether an offset falls inside the span. A zero length span contains nothing. */
+export function containsOffset(span: Span, offset: number): boolean {
+  return offset >= span.offset && offset < endOffset(span);
+}

package/src/core/tokens/index.ts

@@ -0,0 +1,15 @@
+/**
+ * The token vocabulary of language.md section 3. Types and the two tables the
+ * types are derived from; the lexer that produces them is the next stage.
+ */
+export type {
+  KeywordKind,
+  LayoutKind,
+  LiteralKind,
+  NameKind,
+  PunctuationKind,
+  TokenKind,
+} from './kind.js';
+export { PUNCTUATORS, RESERVED_WORDS } from './kind.js';
+
+export type { NumberToken, SimpleToken, StringToken, Token } from './token.js';

package/src/core/tokens/kind.ts

@@ -0,0 +1,123 @@
+/**
+ * Every kind of token language.md section 3 describes. Types and the two tables
+ * they are derived from, and no scanning: the lexer is the next stage.
+ *
+ * A keyword's kind is the word, and a punctuation token's kind is the symbol,
+ * so a parser reads as `token.kind === 'if'` and `token.kind === '('` rather
+ * than through a second vocabulary that has to be learned and kept in step.
+ */
+
+/**
+ * The reserved words of language.md 3.4, which cannot be used as names.
+ *
+ * `import`, `map`, `matrix`, `type` and `as` are reserved and unused in version
+ * 1. They are here so that adding them later cannot break a script that used
+ * one as a variable name.
+ *
+ * Two words the grammar treats as keywords are deliberately absent, because 3.4
+ * does not reserve them: `version` and `limits` are recognised by position in
+ * the parser and are ordinary identifiers to the lexer.
+ */
+export const RESERVED_WORDS = [
+  'and',
+  'array',
+  'as',
+  'bool',
+  'break',
+  'case',
+  'color',
+  'continue',
+  'default',
+  'else',
+  'false',
+  'fn',
+  'for',
+  'if',
+  'import',
+  'in',
+  'is',
+  'live',
+  'map',
+  'matrix',
+  'none',
+  'not',
+  'number',
+  'or',
+  'return',
+  'series',
+  'step',
+  'string',
+  'strategy',
+  'study',
+  'switch',
+  'to',
+  'true',
+  'type',
+  'var',
+  'while',
+] as const;
+
+export type KeywordKind = (typeof RESERVED_WORDS)[number];
+
+/**
+ * The operator and punctuation tokens of language.md 3.12, longest first so a
+ * lexer taking them in order takes the longest match.
+ *
+ * `!` is not in the list and is not an operator on its own: it exists only as
+ * the first half of `!=`, and a lone one is OS1001 with the fix to write `not`.
+ * There are no bitwise, increment or exponent operators for the same reason
+ * they are missing from the specification, so there is nothing here to add.
+ */
+export const PUNCTUATORS = [
+  '==',
+  '!=',
+  '<=',
+  '>=',
+  '+=',
+  '-=',
+  '*=',
+  '/=',
+  '%=',
+  '+',
+  '-',
+  '*',
+  '/',
+  '%',
+  '<',
+  '>',
+  '=',
+  '(',
+  ')',
+  '[',
+  ']',
+  ',',
+  '.',
+  '?',
+  ':',
+] as const;
+
+export type PunctuationKind = (typeof PUNCTUATORS)[number];
+
+/**
+ * A name, and the literals that are not words.
+ *
+ * A named colour such as `aqua` is an identifier rather than a literal: 3.8
+ * makes the nineteen names ordinary globals so the standard library can add
+ * more without a grammar change. `hexColor` is only the `#rrggbb` and
+ * `#rrggbbaa` forms. `true`, `false` and `none` are keywords, being words.
+ */
+export type LiteralKind = 'numberLiteral' | 'stringLiteral' | 'hexColor';
+
+export type NameKind = 'identifier';
+
+/**
+ * The layout tokens of 3.10.
+ *
+ * A comment and a line carrying no token produce nothing at all, which is the
+ * whole of the rule that lets a commented-out statement sit at column zero
+ * inside a block without closing it. So there is no comment kind and no blank
+ * line kind here, and nothing downstream has to skip them.
+ */
+export type LayoutKind = 'newline' | 'indent' | 'dedent' | 'endOfFile';
+
+export type TokenKind = KeywordKind | PunctuationKind | LiteralKind | NameKind | LayoutKind;

package/src/core/tokens/token.ts

@@ -0,0 +1,39 @@
+import type { Span } from '../span/index.js';
+import type { TokenKind } from './kind.js';
+
+interface TokenFields {
+  readonly kind: TokenKind;
+  readonly span: Span;
+  /**
+   * The source text the token was read from, exactly as written.
+   *
+   * An indent token's text is the leading spaces of the line it opens, which is
+   * what OS1003 counts when it says how deeply a line was indented. A dedent, a
+   * newline and the end of the file carry no text.
+   */
+  readonly text: string;
+}
+
+/**
+ * A number literal and the value it denotes.
+ *
+ * The value is carried rather than re-read later because the literal forms of
+ * 3.5 include digit group underscores and a hexadecimal form, and a second
+ * reader of the text would be a second place that has to agree about them.
+ */
+export interface NumberToken extends TokenFields {
+  readonly kind: 'numberLiteral';
+  readonly value: number;
+}
+
+/** A string literal with its escape sequences already resolved. */
+export interface StringToken extends TokenFields {
+  readonly kind: 'stringLiteral';
+  readonly value: string;
+}
+
+export interface SimpleToken extends TokenFields {
+  readonly kind: Exclude<TokenKind, 'numberLiteral' | 'stringLiteral'>;
+}
+
+export type Token = NumberToken | StringToken | SimpleToken;