openalgo-script 0.1.0-alpha.0

package/src/core/parse/names.ts

@@ -0,0 +1,135 @@
+import type { Name } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import type { TokenKind } from '../tokens/index.js';
+import { RESERVED_WORDS } from '../tokens/index.js';
+import type { Cursor } from './cursor.js';
+import { describeToken } from './cursor.js';
+
+/**
+ * Reading the places a name is written, which is where OS1019 lives.
+ *
+ * The lexer cannot make this call. A reserved word before an `=` is a legal
+ * label inside a call and is OS1019 inside a parameter list (language.md 3.4),
+ * and only the parser knows which of the two lists it is reading. So a reserved
+ * word arrives carrying its keyword kind, and the readers below are the
+ * answers: `takeName` refuses one, `takeLabel` and `takeMember` accept one,
+ * because a label and a member are matched against a published list rather than
+ * looked up in a scope.
+ */
+
+const RESERVED: ReadonlySet<TokenKind> = new Set<TokenKind>(RESERVED_WORDS);
+
+/** Whether a token may stand where a word is expected. */
+export function isWordKind(kind: TokenKind): boolean {
+  return kind === 'identifier' || RESERVED.has(kind);
+}
+
+/**
+ * The reserved words a reader can only have meant as a name.
+ *
+ * A reserved word written where a value belongs is one of two things, and which
+ * one depends on the word. Six of them are read by a rule standing next to the
+ * expression: `and` and `or` join two operands and `not` takes one, and `in`,
+ * `to` and `step` are read by a `for` header the moment the expression before
+ * them ends. A word that opens a statement is the same answer again, because an
+ * unclosed bracket carries the statement onto the lines below it and the `if`
+ * down there belongs to its own line rather than to this expression. Taking any
+ * of those as a name would swallow a token the rule around it is waiting for,
+ * and the second diagnostic would then be about the rule rather than about the
+ * mistake.
+ *
+ * What is left is the four value type names, the two words that build a type,
+ * and the six the language reserves and version 1 does not implement. No rule
+ * of the grammar reads one of them anywhere near an expression, so a reader who
+ * wrote one where a value belongs wrote a name, and the answer is OS1019: the
+ * same answer `takeName` gives for the same word written as a target.
+ */
+const MEANT_AS_A_NAME: ReadonlySet<TokenKind> = new Set<TokenKind>([
+  'array',
+  'as',
+  'bool',
+  'color',
+  'import',
+  'is',
+  'map',
+  'matrix',
+  'number',
+  'series',
+  'string',
+  'type',
+]);
+
+/** Whether a word standing where a value belongs is a name the language has taken. */
+export function isMeantAsAName(kind: TokenKind): boolean {
+  return MEANT_AS_A_NAME.has(kind);
+}
+
+/**
+ * A name that is not reserved, derived from one that is.
+ *
+ * OS1019 promises a near name that keeps the meaning, and the derivation has to
+ * work for all thirty six words rather than for the five anyone remembers, so
+ * it is mechanical: the word is what the reader meant, and the suffix is what
+ * makes it theirs to use.
+ */
+function unreservedNear(word: string): string {
+  return `${word}Value`;
+}
+
+/** No word where one was required, which leaves the statement without its subject. */
+function missingWord(cursor: Cursor): Name {
+  const hole = cursor.holeSpan();
+  cursor.report('OS1022', hole, {
+    token: describeToken(cursor.previousMeaningful() ?? cursor.token),
+  });
+  return makeNode('name', hole, { text: '' });
+}
+
+/**
+ * The name at the cursor: an identifier, or a reserved word that is OS1019.
+ *
+ * The word is taken either way. A script that named something `type` still
+ * meant to declare it, and every later diagnostic about that name is more use
+ * to the reader than a hole where the declaration should have been.
+ */
+export function takeName(cursor: Cursor): Name {
+  const token = cursor.token;
+  if (!isWordKind(token.kind)) return missingWord(cursor);
+
+  cursor.advance();
+  if (token.kind !== 'identifier') {
+    cursor.report('OS1019', token.span, {
+      word: token.text,
+      suggestion: unreservedNear(token.text),
+    });
+  }
+  return makeNode('name', token.span, { text: token.text });
+}
+
+/**
+ * The label of a named argument, where a reserved word is legal.
+ *
+ * A label is matched against the callee's parameter list and is never looked up
+ * in a scope, so `plot(v, "V", color = aqua)` is correct and is not OS1019.
+ */
+export function takeLabel(cursor: Cursor): Name {
+  const token = cursor.advance();
+  return makeNode('name', token.span, { text: token.text });
+}
+
+/**
+ * The word after a dot.
+ *
+ * A member is matched against the namespace or the object type it was read
+ * from, which is the same kind of published list a label is matched against, so
+ * a reserved word is not OS1019 here either. Whether the member exists at all
+ * is OS2009, and the checker owns it.
+ */
+export function takeMember(cursor: Cursor): Name {
+  return isWordKind(cursor.kind) ? takeLabel(cursor) : missingWord(cursor);
+}
+
+/** Whether the cursor is on a named argument's label: a word, then a bare `=`. */
+export function atLabel(cursor: Cursor): boolean {
+  return isWordKind(cursor.kind) && cursor.peek().kind === '=';
+}

package/src/core/parse/script.ts

@@ -0,0 +1,64 @@
+import type { Script, TopLevelItem } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import type { DiagnosticSink } from '../diagnostics/index.js';
+import { lex } from '../lex/index.js';
+import type { SourceFile } from '../source/index.js';
+import type { Token } from '../tokens/index.js';
+import { Cursor } from './cursor.js';
+import { parseFunctionDeclaration } from './functions.js';
+import { parseStatement } from './statements.js';
+
+/**
+ * A file, which is a list of statements and function declarations.
+ *
+ * Section 19 writes a file as a version line, a declaration and a limits line
+ * followed by the rest. The prose wins: the declaration being missing is
+ * OS2007, two of them is OS2008, and a version line that is not first is
+ * OS1021, all of which name a line and suggest a move. So every line here is an
+ * item in source order, a file with no declaration still has a tree, and the
+ * checker is left holding the questions that are its to answer.
+ *
+ * Nothing in this file throws and nothing stops early. A statement that could
+ * not be read costs its own line and, where it opened one, its block; the next
+ * line is read as though nothing had happened, which is what lets a file with
+ * three mistakes in it produce three diagnostics from one compile.
+ */
+export function parse(file: SourceFile, diagnostics: DiagnosticSink): Script {
+  return parseTokens(file, lex(file, diagnostics), diagnostics);
+}
+
+/** The same, for a host that has already lexed the file and kept the tokens. */
+export function parseTokens(
+  file: SourceFile,
+  tokens: readonly Token[],
+  diagnostics: DiagnosticSink,
+): Script {
+  const cursor = new Cursor(file, tokens, diagnostics);
+  const items: TopLevelItem[] = [];
+
+  while (!cursor.at('endOfFile')) {
+    const before = cursor.position;
+
+    if (cursor.at('newline') || cursor.at('dedent') || cursor.at('indent')) {
+      // Layout with no statement to belong to, which a recovery can leave
+      // behind. Taking it here keeps the loop below reading statements only.
+      if (cursor.at('indent')) cursor.skipBlockBody();
+      else cursor.advance();
+      continue;
+    }
+
+    if (cursor.at('fn')) {
+      cursor.beginStatement();
+      const declaration = parseFunctionDeclaration(cursor);
+      cursor.noteStatement(declaration.span.line);
+      items.push(declaration);
+    } else {
+      const statement = parseStatement(cursor);
+      if (statement !== undefined) items.push(statement);
+    }
+
+    if (cursor.position === before) cursor.skipStatement();
+  }
+
+  return makeNode('script', file.spanAt(0, file.text.length), { items });
+}

package/src/core/parse/statements.ts

@@ -0,0 +1,329 @@
+import type {
+  AssignmentOperator,
+  Expression,
+  FunctionDeclaration,
+  Statement,
+  TypeAnnotation,
+} from '../ast/index.js';
+import { ASSIGNMENT_OPERATORS, makeNode, withoutGrouping } from '../ast/index.js';
+import { spanning } from '../span/index.js';
+import type { NumberToken, TokenKind } from '../tokens/index.js';
+import { parseTypeAnnotation } from './annotations.js';
+import { parseIfStatement } from './branches.js';
+import type { Cursor } from './cursor.js';
+import { missingExpression, parseArgumentList, parseExpression } from './expressions.js';
+import { parseFunctionDeclaration } from './functions.js';
+import { parseForStatement, parseWhileStatement } from './loops.js';
+import { isWordKind, takeName } from './names.js';
+import { parseSwitchStatement } from './switches.js';
+
+/**
+ * One statement, and the choice of which one it is.
+ *
+ * The choice is made from the first token and at most one more, which is what
+ * keeps the language readable: a line that begins with `if` is an `if` and
+ * nothing else, and the only lookahead anywhere is the token after a word,
+ * which says whether the word is being assigned to.
+ *
+ * Three of the forms here describe the file rather than compute anything. They
+ * are statements like the rest and may be written anywhere a statement may be,
+ * because the errors for putting one in the wrong place name a line and suggest
+ * a move (OS1021, OS2007, OS2008, OS3014), and a tree that refused to hold them
+ * could not carry a line number for any of those to report.
+ */
+
+const ASSIGNMENT: ReadonlySet<TokenKind> = new Set<TokenKind>(ASSIGNMENT_OPERATORS);
+
+export function parseStatement(
+  cursor: Cursor,
+): Statement | FunctionDeclaration | undefined {
+  cursor.beginStatement();
+  const statement = dispatch(cursor);
+  if (statement !== undefined) cursor.noteStatement(statement.span.line);
+  return statement;
+}
+
+/**
+ * The end of a statement: a newline, and OS1018 for anything before it.
+ *
+ * A line holds at most one statement, so whatever is left over is either a
+ * missing operator or a second statement that lost its newline. It is only
+ * reported when the statement has nothing else wrong with it, because leftovers
+ * are what every other mistake on the line produces and the reader should be
+ * told about the mistake rather than about its wreckage.
+ */
+export function finishStatement(cursor: Cursor): void {
+  if (cursor.take('newline') !== undefined) return;
+  if (cursor.atStatementEnd()) return;
+  if (!cursor.reportedHere) cursor.unexpected(cursor.token);
+  cursor.skipToEndOfLine();
+}
+
+function dispatch(cursor: Cursor): Statement | FunctionDeclaration | undefined {
+  switch (cursor.kind) {
+    case 'if':
+      return parseIfStatement(cursor);
+    case 'for':
+      return parseForStatement(cursor);
+    case 'while':
+      return parseWhileStatement(cursor);
+    case 'switch':
+      return parseSwitchStatement(cursor);
+    case 'break':
+    case 'continue':
+      return parseJump(cursor);
+    case 'return':
+      return parseReturn(cursor);
+    case 'var':
+      return parseVarDeclaration(cursor, false);
+    case 'live':
+      return cursor.peek().kind === 'var'
+        ? parseVarDeclaration(cursor, true)
+        : parseSimpleStatement(cursor);
+    case 'study':
+    case 'strategy':
+      return parseScriptDeclaration(cursor);
+    case 'else':
+      return strayElse(cursor);
+    case 'case':
+    case 'default':
+      return strayArm(cursor);
+    case 'fn':
+      return nestedFunction(cursor);
+    default:
+      return parseSimpleStatement(cursor);
+  }
+}
+
+function parseSimpleStatement(cursor: Cursor): Statement | undefined {
+  const token = cursor.token;
+
+  // A block whose header was not understood, so nothing is left that could own
+  // it. Skipping it whole reports the header once instead of every line of it.
+  if (token.kind === 'indent') {
+    cursor.skipBlockBody();
+    return undefined;
+  }
+
+  if (token.kind === 'identifier') {
+    const next = cursor.peek();
+    // `version` and `limits` are not reserved words: a script written before
+    // either existed keeps compiling, so both are recognised by position.
+    if (token.text === 'version' && next.kind === 'numberLiteral') {
+      return parseVersionLine(cursor, next);
+    }
+    if (token.text === 'limits' && next.kind === '(') return parseLimitsLine(cursor);
+  }
+
+  if (isWordKind(token.kind) && ASSIGNMENT.has(cursor.peek().kind)) {
+    return parseAssignment(cursor);
+  }
+
+  if (cursor.atStatementEnd()) {
+    cursor.take('newline');
+    return undefined;
+  }
+
+  const expression = parseExpression(cursor);
+  if (ASSIGNMENT.has(cursor.kind)) {
+    const refused = assignmentToSomethingOtherThanAName(cursor, expression);
+    if (refused !== undefined) return refused;
+  }
+
+  finishStatement(cursor);
+  return makeNode('expressionStatement', expression.span, { expression });
+}
+
+/**
+ * An assignment whose target is an index or a member: OS1024 and OS1025.
+ *
+ * The grammar of section 19 has one assignment target and it is an identifier,
+ * so what is wrong on this line is the target and not the operator after it.
+ * That is why neither of these is OS1018, whose message would say the statement
+ * had ended before an operator the reader wrote in the middle of it. Two codes
+ * rather than one because the two have different fixes: an array element is
+ * written with a call, and a member is not written at all.
+ *
+ * Anything else the grammar cannot assign to, a literal or a call, is left to
+ * the caller and reported as OS1018, which is the nearest true thing the
+ * catalogue has for it.
+ *
+ * The value is read rather than skipped, so a mistake in it is reported on the
+ * same compile, and the statement reaches the tree as the target that was
+ * written. There is no node for an assignment whose target is not a name, and
+ * building one over a name taken out of the target, `prices` out of
+ * `prices[0]`, would put a different program in the tree from the one on the
+ * page.
+ */
+function assignmentToSomethingOtherThanAName(
+  cursor: Cursor,
+  target: Expression,
+): Statement | undefined {
+  const written = withoutGrouping(target);
+
+  if (written.kind === 'index') {
+    cursor.report('OS1024', target.span, { name: cursor.textOf(written.target.span) });
+  } else if (written.kind === 'member') {
+    cursor.report('OS1025', target.span, {
+      name: cursor.textOf(written.object.span),
+      member: written.member.text,
+    });
+  } else {
+    return undefined;
+  }
+
+  // The operator, and then the value, which is read so that a mistake inside it
+  // is reported on this compile rather than on the one after the target is
+  // fixed.
+  cursor.advance();
+  const value = parseExpression(cursor);
+  finishStatement(cursor);
+  return makeNode('expressionStatement', spanning(target.span, value.span), {
+    expression: target,
+  });
+}
+
+function parseAssignment(cursor: Cursor): Statement {
+  const target = takeName(cursor);
+  const operator = cursor.advance().kind as AssignmentOperator;
+  const value = parseExpression(cursor);
+  finishStatement(cursor);
+  return makeNode('assignment', spanning(target.span, value.span), { target, operator, value });
+}
+
+function parseVarDeclaration(cursor: Cursor, live: boolean): Statement {
+  const start = cursor.advance();
+  if (live) cursor.advance();
+
+  const name = takeName(cursor);
+  const annotation: TypeAnnotation | undefined =
+    cursor.take(':') !== undefined ? parseTypeAnnotation(cursor) : undefined;
+
+  let initialiser: Expression;
+  if (cursor.take('=') !== undefined) {
+    initialiser = parseExpression(cursor);
+  } else {
+    // The declaration and the first assignment are one statement, so a
+    // persistent value always has something in it and no bar can read it
+    // before it exists.
+    cursor.report('OS1011', spanning(start.span, name.span), { name: name.text });
+    initialiser = missingExpression(cursor);
+  }
+
+  finishStatement(cursor);
+  return makeNode('varDeclaration', spanning(start.span, initialiser.span), {
+    live,
+    name,
+    annotation,
+    initialiser,
+  });
+}
+
+function parseScriptDeclaration(cursor: Cursor): Statement {
+  const keyword = cursor.advance();
+  const open = cursor.take('(');
+  const list = open === undefined ? undefined : parseArgumentList(cursor, open);
+  finishStatement(cursor);
+
+  return makeNode('scriptDeclaration', spanning(keyword.span, list?.end ?? keyword.span), {
+    form: keyword.kind === 'study' ? 'study' : 'strategy',
+    args: list?.items ?? [],
+  });
+}
+
+function parseVersionLine(cursor: Cursor, number: NumberToken): Statement {
+  const word = cursor.advance();
+  cursor.advance();
+  const span = spanning(word.span, number.span);
+
+  const before = cursor.firstStatementLine;
+  if (before !== undefined) cursor.report('OS1021', span, { line: before });
+
+  finishStatement(cursor);
+  return makeNode('versionLine', span, {
+    version: makeNode('numberLiteral', number.span, { value: number.value }),
+  });
+}
+
+function parseLimitsLine(cursor: Cursor): Statement {
+  const word = cursor.advance();
+  const list = parseArgumentList(cursor, cursor.advance());
+  finishStatement(cursor);
+  return makeNode('limitsLine', spanning(word.span, list.end), { args: list.items });
+}
+
+function parseJump(cursor: Cursor): Statement {
+  const keyword = cursor.advance();
+  if (!cursor.insideLoop) cursor.report('OS1009', keyword.span, { word: keyword.text });
+  finishStatement(cursor);
+  return keyword.kind === 'break'
+    ? makeNode('breakStatement', keyword.span, {})
+    : makeNode('continueStatement', keyword.span, {});
+}
+
+function parseReturn(cursor: Cursor): Statement {
+  const keyword = cursor.advance();
+  // A bare `return` exits with `none`, so the value is genuinely absent here
+  // rather than a hole something went missing from.
+  const value = cursor.atStatementEnd() ? undefined : parseExpression(cursor);
+  finishStatement(cursor);
+  return makeNode(
+    'returnStatement',
+    value === undefined ? keyword.span : spanning(keyword.span, value.span),
+    { value },
+  );
+}
+
+/**
+ * An `else` that no `if` took.
+ *
+ * `parseIfStatement` takes the `else` that belongs to it, so one that reaches
+ * here has no `if` at its own indentation: either the statement above it is
+ * something else, or the lexer has already put the two in different blocks.
+ */
+function strayElse(cursor: Cursor): undefined {
+  const token = cursor.token;
+  cursor.report('OS1016', token.span, {
+    found: token.span.column - 1,
+    expected: cursor.nearestIfColumn - 1,
+  });
+  cursor.skipStatement();
+  return undefined;
+}
+
+function strayArm(cursor: Cursor): undefined {
+  const token = cursor.token;
+  cursor.report('OS1017', token.span, { word: token.text });
+  cursor.skipStatement();
+  return undefined;
+}
+
+/**
+ * A `fn` inside a block, which language.md 11.1 forbids: OS1023.
+ *
+ * It is read and it is kept. Reading it keeps the lines under it from being
+ * taken for statements of the block around it. Keeping it is what lets anything
+ * after the parser see the function at all: a declaration dropped here is gone
+ * from the tree, so a call to it could only ever be reported as a name nobody
+ * declared, and a nested function nobody calls would go unreported for good. It
+ * stays where the reader wrote it rather than being lifted to the top level,
+ * which would compile a file the language refuses.
+ *
+ * The caret covers `fn` and the name, which is the part of the declaration the
+ * reader has to move, and the same span OS1011 puts under `var name`.
+ *
+ * A header with no name at all has already been told that a name was expected,
+ * and the message here has nothing to name, so it waits: the node is kept
+ * either way, and the declaration is reported as nested once it has a name for
+ * the sentence to use and for the fix to tell the reader to call.
+ */
+function nestedFunction(cursor: Cursor): FunctionDeclaration {
+  const keyword = cursor.token;
+  const declaration = parseFunctionDeclaration(cursor);
+  if (declaration.name.text !== '') {
+    cursor.report('OS1023', spanning(keyword.span, declaration.name.span), {
+      name: declaration.name.text,
+    });
+  }
+  return declaration;
+}

package/src/core/parse/switches.ts

@@ -0,0 +1,105 @@
+import type { Expression, SwitchCase, SwitchDefault, SwitchStatement } from '../ast/index.js';
+import { makeNode } from '../ast/index.js';
+import type { Span } from '../span/index.js';
+import { spanning } from '../span/index.js';
+import { parseBlock } from './blocks.js';
+import type { Cursor } from './cursor.js';
+import { parseCondition } from './expressions.js';
+import { finishStatement } from './statements.js';
+
+/**
+ * `switch`, in both forms of language.md 10.6.
+ *
+ * With a subject each case value is compared against it, and without one each
+ * case value is a condition. The difference is the absent subject and needs no
+ * flag beside it.
+ *
+ * The arms are the whole of a switch body. A `default` that is not last would
+ * leave the arms after it unreachable, so the arm that follows one is OS1017,
+ * and the arm is kept in the tree all the same: a reader who moves the default
+ * should not then be told about the arms one at a time.
+ */
+export function parseSwitchStatement(cursor: Cursor): SwitchStatement {
+  const keyword = cursor.advance();
+  const subject = cursor.atStatementEnd() ? undefined : parseCondition(cursor);
+  finishStatement(cursor);
+
+  const cases: SwitchCase[] = [];
+  let defaultCase: SwitchDefault | undefined;
+  let end: Span = keyword.span;
+
+  if (!cursor.at('indent')) {
+    if (!cursor.reportedHere) cursor.report('OS1010', keyword.span, { header: 'switch' });
+    return makeNode('switchStatement', spanning(keyword.span, end), {
+      subject,
+      cases,
+      defaultCase,
+    });
+  }
+
+  if (!cursor.openBlock(cursor.token.span)) {
+    cursor.skipBlockBody();
+    return makeNode('switchStatement', spanning(keyword.span, end), {
+      subject,
+      cases,
+      defaultCase,
+    });
+  }
+  cursor.advance();
+
+  while (!cursor.at('dedent') && !cursor.at('endOfFile')) {
+    const before = cursor.position;
+    const word = cursor.token;
+
+    if (cursor.at('case')) {
+      if (defaultCase !== undefined) cursor.report('OS1017', word.span, { word: word.text });
+      const arm = parseCase(cursor);
+      cases.push(arm);
+      end = arm.span;
+    } else if (cursor.at('default')) {
+      const arm = parseDefault(cursor);
+      if (defaultCase === undefined) defaultCase = arm;
+      else cursor.report('OS1017', word.span, { word: word.text });
+      end = arm.span;
+    } else {
+      // A line in a switch body that is not an arm. The catalogue has no code
+      // for it, and OS1017 carries the rule it breaks: the arms are the whole
+      // of the body, so this line belongs inside one of them or above the
+      // switch.
+      cursor.report('OS1017', word.span, { word: 'case' });
+      cursor.skipStatement();
+    }
+
+    if (cursor.position === before) cursor.skipStatement();
+  }
+
+  cursor.closeBlock();
+  cursor.take('dedent');
+
+  return makeNode('switchStatement', spanning(keyword.span, end), { subject, cases, defaultCase });
+}
+
+/** One arm. Several values separated by commas are one arm, not several. */
+function parseCase(cursor: Cursor): SwitchCase {
+  cursor.beginStatement();
+  const keyword = cursor.advance();
+
+  const values: Expression[] = [parseCondition(cursor)];
+  while (cursor.take(',') !== undefined) {
+    const before = cursor.position;
+    values.push(parseCondition(cursor));
+    if (cursor.position === before) break;
+  }
+
+  finishStatement(cursor);
+  const body = parseBlock(cursor, 'case', keyword.span);
+  return makeNode('switchCase', spanning(keyword.span, body.span), { values, body });
+}
+
+function parseDefault(cursor: Cursor): SwitchDefault {
+  cursor.beginStatement();
+  const keyword = cursor.advance();
+  finishStatement(cursor);
+  const body = parseBlock(cursor, 'default', keyword.span);
+  return makeNode('switchDefault', spanning(keyword.span, body.span), { body });
+}

package/src/core/render/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Turning a diagnostic into something a reader sees.
+ *
+ * Only the terminal shape lives here. An editor puts the same diagnostic in a
+ * gutter and a chart puts it over a pane, and both take the Diagnostic and the
+ * span rather than this text, which is why rendering is a module of its own
+ * rather than a method on the diagnostic.
+ */
+export { renderDiagnostic, renderDiagnostics } from './terminal.js';