openalgo-script 0.2.0 → 0.5.0

package/src/editor/manifest.ts

@@ -0,0 +1,119 @@
+/**
+ * The standard library manifest, as the three functions that read it need it.
+ *
+ * `complete`, `hover` and `signature` all want the same things about a name:
+ * which signature the writer means, how to spell that signature, what each
+ * parameter defaults to, and the line the specification gives it. Every one of
+ * those is a question the compiler already answers, and the whole value of this
+ * file is that it asks rather than knows.
+ *
+ *     which entries a name has    the manifest index, `libraryEntries`
+ *     what a parameter defaults   the manifest, or the emitter for the eight
+ *                                 declaration calls
+ *     what the call is for        `proseFor`, read out of `stdlib.md`'s own
+ *                                 tables at build time, with the warmup its row
+ *                                 states beside it
+ *     what a type is called       `typeText`, the checker's own spelling
+ *
+ * A list of function names typed into this repository would be right on the day
+ * it was typed and wrong at the next release, and a missing completion looks
+ * exactly like a completion that has not loaded yet, so nobody would report it.
+ *
+ * ## The default is the compiler's, including for the eight that hide it
+ *
+ * Most calls carry their default in the manifest, spelled as `stdlib.md` prints
+ * it. A `plot`, a `level`, a `table` and the five like them do not: their
+ * optional arguments become fields of a declaration rather than arguments an
+ * engine is passed, so their values live in the emitter
+ * (`compiled-program.md` 2.3). A tooltip built on the manifest alone would show
+ * a writer nothing beside `width?` while the compiler writes 1.5.
+ *
+ * So both are read, through `declarationDefaultText`, which is the same answer
+ * `scripts/check-defaults.mjs` holds to what the specification prints. A default
+ * shown here cannot differ from the one the compiler applies without failing the
+ * build, which is the only way a tooltip's number is worth anything.
+ */
+import {
+  DECLARATION_CALLS,
+  declarationDefaultText,
+  libraryEntries,
+  typeText,
+} from '../core/index.js';
+import type { LibraryEntry } from '../core/index.js';
+
+/** One parameter of a call, with everything a writer is shown about it. */
+export interface Parameter {
+  readonly name: string;
+  /** The type as the checker spells it: `series number`, `array<line>`. */
+  readonly type: string;
+  /** Whether the call is refused without it, which is OS3012. */
+  readonly required: boolean;
+  /**
+   * What an omitted argument resolves to, as the specification prints it.
+   *
+   * Nothing where the specification states no value: `spec/default-exceptions.json`
+   * records those and why each one has none, and a number invented to fill the
+   * gap would be the defect that file exists to keep out.
+   */
+  readonly defaultText: string | undefined;
+  /** The closed set of strings it accepts, where the library fixes one: OS3008. */
+  readonly values: readonly string[];
+}
+
+/** Every parameter of one signature, with the defaults from both homes. */
+export function parametersOf(entry: LibraryEntry): readonly Parameter[] {
+  const declared = DECLARATION_CALLS.has(entry.name);
+  return entry.parameters.map((one) => ({
+    name: one.name,
+    type: typeText(one.type),
+    required: !one.optional,
+    defaultText: declared
+      ? declarationDefaultText(entry.name, one.name)
+      : one.defaultText,
+    values: entry.values[one.name] ?? [],
+  }));
+}
+
+/**
+ * One signature, written out the way the specification's tables write one.
+ *
+ * `ema(src: series number, len: number) -> series number`. The parameter names,
+ * the types and the defaults are the manifest's; nothing about the spelling is
+ * decided here except the punctuation between them.
+ */
+export function signatureTextOf(entry: LibraryEntry): string {
+  if (!entry.callable) return `${entry.name}: ${typeText(entry.returns)}`;
+  const written = parametersOf(entry).map((one) => {
+    const head = `${one.name}${one.required ? '' : '?'}: ${one.type}`;
+    return one.defaultText === undefined ? head : `${head} = ${one.defaultText}`;
+  });
+  return `${entry.name}(${written.join(', ')}) -> ${typeText(entry.returns)}`;
+}
+
+/**
+ * The signature a half written call means, chosen by the rule that can be
+ * applied to a half written call.
+ *
+ * `stdlib.md` 2.2 resolves an overload by arity first and by argument type
+ * second, and the checker does exactly that. The second rule needs a type for
+ * every argument, which needs a tree, which a call that is still being typed has
+ * not got: `clear(` is two signatures until the argument arrives. So only the
+ * first rule is applied here, and where it leaves more than one candidate the
+ * first is taken, which is the order `stdlib.md` states them in.
+ *
+ * That is a narrowing of the checker's answer rather than a second opinion about
+ * it: the tooltip may show the array signature of `clear` while the writer meant
+ * the grid one, and the moment the argument is written the checker settles it
+ * and `diagnose` is what says so. It is recorded in `spec/editor-narrowings.json`.
+ */
+export function entryAt(name: string, given: number): LibraryEntry | undefined {
+  const entries = libraryEntries(name).filter((one) => one.callable);
+  const first = entries[0];
+  if (first === undefined || entries.length === 1) return first;
+
+  const byArity = entries.filter((one) => {
+    const required = one.parameters.filter((parameter) => !parameter.optional).length;
+    return given >= required && given <= one.parameters.length;
+  });
+  return byArity[0] ?? first;
+}

package/src/editor/reading.ts

@@ -0,0 +1,69 @@
+/**
+ * The first step of every function in this tier: text becomes a file, and the
+ * file becomes tokens.
+ *
+ * It is one place rather than four because all four begin identically and
+ * because one of them beginning differently is a bug nobody would look for.
+ *
+ * The bag is created and, for `highlight`, thrown away. That is not waste: the
+ * lexer reports rather than throws, so handing it somewhere to report is how it
+ * is told to carry on, and a highlighter has nothing to say about a character
+ * the language does not have. `diagnose` is what says it.
+ */
+import { DiagnosticBag, check, lex, parseTokens, sourceFile } from '../core/index.js';
+import type { CheckedScript, Script, SourceFile, Token } from '../core/index.js';
+
+/**
+ * The name the file is read under, and why none of these functions takes one.
+ *
+ * A `SourceFile` carries a name, and the renderer in the core puts it on the
+ * first line of a rendered diagnostic. Nothing this tier returns carries it: a
+ * diagnostic holds a span, and a highlighted piece holds a span, and neither
+ * holds a file. So a name parameter here would be a parameter a caller can only
+ * get wrong, with nothing anywhere able to notice that they had. A host that
+ * renders a diagnostic with its own file name on it builds the `SourceFile`
+ * itself, from the core, which is where the renderer lives.
+ */
+const UNNAMED = 'script.oscript';
+
+export interface Read {
+  readonly file: SourceFile;
+  readonly bag: DiagnosticBag;
+  readonly tokens: readonly Token[];
+}
+
+export interface ReadTree extends Read {
+  readonly script: Script;
+}
+
+export interface ReadChecked extends ReadTree {
+  readonly checked: CheckedScript;
+}
+
+/** Text to a file and its tokens. */
+export function read(source: string): Read {
+  const file = sourceFile(UNNAMED, source);
+  const bag = new DiagnosticBag();
+  return { file, bag, tokens: lex(file, bag) };
+}
+
+/** The same, with the tree, for the functions that need one. */
+export function readTree(source: string): ReadTree {
+  const held = read(source);
+  return { ...held, script: parseTokens(held.file, held.tokens, held.bag) };
+}
+
+/**
+ * The same again, checked, for the two that ask what a name means.
+ *
+ * `complete` and `hover` both need the checker's answers: which names the file
+ * has declared, where each was declared, and what type each one holds. Nothing
+ * here is a lighter version of the checker. It is the checker, run on a file
+ * that is halfway through being typed, which it is built to survive: a name
+ * that does not resolve becomes `unknown` and the lines around it are still
+ * checked, so the names above the cursor are still names.
+ */
+export function readChecked(source: string): ReadChecked {
+  const held = readTree(source);
+  return { ...held, checked: check(held.file, held.script, held.bag) };
+}

package/src/editor/scan.ts

@@ -0,0 +1,162 @@
+/**
+ * The covering scan: every character of a file, exactly once, with what it is.
+ *
+ * ## The property that decides the shape
+ *
+ * A host draws highlighting behind or over a text component, and it draws it by
+ * walking a list. Drop one character anywhere on a line and everything after it
+ * on that line is drawn one column to the left, so the caret stops sitting where
+ * the text is. That is not a cosmetic bug: it is the editor feeling broken, and
+ * it is reported as "the cursor is in the wrong place" rather than as a
+ * highlighting fault, which is why it survives.
+ *
+ * So the guarantee here is not "here are the tokens". It is **the spans tile the
+ * source**: they are in order, they do not overlap, none is empty, and their
+ * texts concatenate to the file exactly. A host that renders the spans and
+ * nothing else cannot lose a character, because there is nowhere for one to go.
+ * `tests/editor/highlight.test.ts` holds that over every script in the
+ * repository and over a corpus of malformed ones.
+ *
+ * ## Where the comments come from
+ *
+ * The lexer emits no token for a comment, because the parser has no use for one
+ * (`language.md` 3.10, and it is the rule that lets a commented-out statement
+ * sit at column zero inside a block without closing it). A highlighter does have
+ * a use for one, so they have to be recovered, and this is the one place in the
+ * project that does it. Leaving it to each host is what produced the two
+ * problems this file exists to remove: a platform recovering comments from the
+ * gaps between token spans, and doing it with its own scan for `//`.
+ *
+ * A scan of its own is the wrong answer, because a `//` inside a string literal
+ * is ordinary text (3.2) and finding that out means lexing strings a second
+ * time. This does not scan the file. It walks the tokens the real lexer
+ * produced, and only looks at what is left between them, which by construction
+ * holds no string literal: a `//` in a gap is always a comment, with no second
+ * opinion about the language anywhere in it.
+ *
+ * What else can be in a gap is small and worth naming, because a reader of this
+ * file will want to know whether it can be surprised: blanks, a line continuation
+ * backslash, a region somebody meant as a block comment, and the characters the
+ * lexer refused and reported. All of those are `unknown`, which means "the lexer
+ * took no token from this", and `diagnose` is what says whether any of it is a
+ * mistake.
+ */
+import { endOffset } from '../core/index.js';
+import type { SourceFile, Span, Token } from '../core/index.js';
+import { kindOf } from './kinds.js';
+import type { HighlightKind } from './kinds.js';
+
+/** One piece of the file, with what it is and the text it covers. */
+export interface Highlight {
+  readonly kind: HighlightKind;
+  readonly span: Span;
+  /**
+   * The source the span covers, carried rather than left to the caller to slice.
+   *
+   * A host that concatenates these gets the file back, so a renderer never
+   * indexes the buffer and never has to agree with this module about what an
+   * offset means. It is also what makes the covering property a one-line test.
+   */
+  readonly text: string;
+}
+
+const SLASH = 47;
+const LINE_FEED = 10;
+
+/** Whether a comment opens at `at`: two slashes, which run to the end of the line. */
+function opensComment(text: string, at: number): boolean {
+  return text.charCodeAt(at) === SLASH && text.charCodeAt(at + 1) === SLASH;
+}
+
+function isBlank(code: number): boolean {
+  // The three the lexer passes over between tokens. A tab is OS1002 in leading
+  // whitespace and is still whitespace to look at, so it is painted as such and
+  // the compiler is left to be the one that objects to it. A lone carriage
+  // return is not one of these: `language.md` 3.1 does not accept it, the lexer
+  // reports it, and painting it as a space would hide the one thing about it
+  // worth knowing.
+  return code === 32 || code === 9 || code === LINE_FEED;
+}
+
+/** The end of the line `at` sits on, or the end of the file. */
+function endOfLine(text: string, at: number): number {
+  const found = text.indexOf('\n', at);
+  return found === -1 ? text.length : found;
+}
+
+/**
+ * A run of one kind inside a gap, from `from` up to but not including its end.
+ *
+ * The three cases are tried in the order they can be told apart: a blank is a
+ * blank, two slashes open a comment to the end of the line, and anything else
+ * runs until one of those two begins.
+ */
+function runIn(text: string, from: number, to: number): { end: number; kind: HighlightKind } {
+  if (isBlank(text.charCodeAt(from))) {
+    let i = from;
+    while (i < to && isBlank(text.charCodeAt(i))) i += 1;
+    return { end: i, kind: 'whitespace' };
+  }
+
+  if (opensComment(text, from)) {
+    return { end: Math.min(endOfLine(text, from), to), kind: 'comment' };
+  }
+
+  let i = from;
+  while (i < to && !isBlank(text.charCodeAt(i)) && !opensComment(text, i)) i += 1;
+  return { end: i, kind: 'unknown' };
+}
+
+/**
+ * Every piece of a file, in order, covering it exactly.
+ *
+ * The token list is walked rather than sorted, and a token that would overlap
+ * the piece before it is skipped rather than believed. Both of those are
+ * defences rather than expectations: the property this returns is what a host
+ * draws with, and it has to hold for a stream from a file somebody is halfway
+ * through typing as firmly as for one from a script that compiles.
+ */
+export function scan(file: SourceFile, tokens: readonly Token[]): readonly Highlight[] {
+  const text = file.text;
+  const out: Highlight[] = [];
+  let at = 0;
+
+  const fill = (to: number): void => {
+    let from = at;
+    while (from < to) {
+      const run = runIn(text, from, to);
+      out.push({
+        kind: run.kind,
+        span: file.spanAt(from, run.end - from),
+        text: text.slice(from, run.end),
+      });
+      from = run.end;
+    }
+  };
+
+  const read: Token[] = [];
+  for (const token of tokens) {
+    if (token.span.length === 0) {
+      // A dedent and the end of the file are positions rather than text, and a
+      // newline at the end of a file is one too. Nothing to paint, but the
+      // token still counts as read for the dotted name question next door.
+      read.push(token);
+      continue;
+    }
+    if (token.span.offset < at) continue;
+    if (token.span.offset > at) {
+      fill(token.span.offset);
+      at = token.span.offset;
+    }
+    out.push({
+      kind: kindOf(token, read),
+      span: token.span,
+      text: text.slice(token.span.offset, endOffset(token.span)),
+    });
+    at = endOffset(token.span);
+    read.push(token);
+  }
+
+  fill(text.length);
+  return out;
+}

package/src/editor/scope.ts

@@ -0,0 +1,122 @@
+/**
+ * Which of the file's own names may be written at a position.
+ *
+ * A completion list that offered every name in the file would offer a loop
+ * variable three blocks away and a function parameter from another function,
+ * and a writer accepting one of those gets OS2001 from the compiler a moment
+ * later. So the two rules the language states are applied, and both are asked of
+ * the compiler's own answers rather than worked out again here.
+ *
+ *     declared above     OS2001 is "not defined at this point in the file", so a
+ *                        name is offered from the end of the statement that
+ *                        declares it. Not from the end of the name: `x` is not
+ *                        yet a name inside `x = ema(x, 9)`, and `f` is not one
+ *                        inside `f`'s own body, because recursion is OS2005
+ *     and still inside   a name belongs to the block that declared it
+ *                        (`language.md` 12), so it is offered inside that block
+ *                        and nowhere else
+ *
+ * The bindings are the checker's, not a second walk of the tree: `declaredAt` is
+ * the span the checker points OS2002 and OS8010 at, and the kind, the type and
+ * the persistence are what it worked out. What is derived here is one thing, the
+ * region a binding belongs to, and it is derived from the tree the parser
+ * produced by asking which nodes enclose the declaration.
+ */
+import { containsOffset, endOffset, isStatement, pathAtOffset } from '../core/index.js';
+import type { AstNode, Binding, CheckedScript, NodeKind } from '../core/index.js';
+
+/**
+ * The node kinds that hold a scope of their own.
+ *
+ * A block is the obvious one. A function declaration is here because its
+ * parameters are declared in its header rather than in its body, and a loop
+ * because its variable is declared in the header too. Anything else a name is
+ * declared inside belongs to whichever of these encloses it, and a file level
+ * name belongs to the file.
+ */
+const SCOPES: ReadonlySet<NodeKind> = new Set<NodeKind>([
+  'block',
+  'functionDeclaration',
+  'forRangeStatement',
+  'forInStatement',
+]);
+
+/**
+ * The region a declaration belongs to, and the offset it starts being a name at.
+ *
+ * Both come from one walk of the path to the declaration. The region is the
+ * innermost enclosing scope, or the file. The offset is the end of the statement
+ * that declares it, except for a parameter and a loop variable, whose statement
+ * is the function or the loop they are read inside: those two are names from the
+ * end of the header that declares them.
+ */
+interface Region {
+  readonly node: AstNode | undefined;
+  readonly from: number;
+}
+
+const HEADERS: ReadonlySet<Binding['kind']> = new Set<Binding['kind']>(['parameter', 'loop']);
+
+function regionOf(script: CheckedScript, binding: Binding): Region {
+  const path = pathAtOffset(script.script, binding.declaredAt.offset);
+  let node: AstNode | undefined;
+  let statement: AstNode | undefined;
+
+  for (let i = path.length - 1; i >= 0; i -= 1) {
+    const held = path[i];
+    if (held === undefined) continue;
+    if (statement === undefined && isStatement(held)) statement = held;
+    if (SCOPES.has(held.kind)) {
+      node = held;
+      break;
+    }
+  }
+
+  const from = HEADERS.has(binding.kind)
+    ? endOffset(binding.declaredAt)
+    : endOffset(statement?.span ?? binding.declaredAt);
+  return { node, from };
+}
+
+/**
+ * Every name the file itself declares that may be written at an offset.
+ *
+ * A library name is not here: the manifest answers for those, and a binding of
+ * kind `library` is the checker's record of the global scope rather than
+ * something the file declared.
+ */
+export function inScopeAt(script: CheckedScript, offset: number): readonly Binding[] {
+  return script.bindings.filter((binding) => {
+    if (binding.kind === 'library') return false;
+    const region = regionOf(script, binding);
+    if (region.from > offset) return false;
+    // No enclosing scope node means the file, and the file holds every offset,
+    // including the one past its last character where somebody is typing.
+    return region.node === undefined || containsOffset(region.node.span, offset);
+  });
+}
+
+/**
+ * The binding a name means at an offset, or nothing where the file declares none.
+ *
+ * The declaration itself counts, which the scope rule above deliberately does
+ * not: a name is not in scope inside its own right hand side, and pointing at it
+ * on the line that declares it is still pointing at it. Without this, a hover
+ * over the `x` in `x = ema(close, 9)` would say nothing at all.
+ */
+export function bindingNamed(
+  script: CheckedScript,
+  name: string,
+  offset: number,
+): Binding | undefined {
+  const declaring = script.bindings.find(
+    (binding) => binding.name === name && containsOffset(binding.declaredAt, offset),
+  );
+  if (declaring !== undefined) return declaring;
+
+  const held = inScopeAt(script, offset).filter((binding) => binding.name === name);
+  // The innermost declaration wins, which is the one declared last among those
+  // still in scope: a block level name shadows a file level one of the same
+  // spelling for as long as its block lasts.
+  return held[held.length - 1];
+}

package/src/editor/signature.ts

@@ -0,0 +1,150 @@
+/**
+ * `signature`: the call being written, and which parameter the cursor is in.
+ *
+ * The call is found from the brackets rather than from the tree, because a call
+ * being written has no finished tree node: `ema(close, ` is the state every call
+ * is in while somebody types one, and that is exactly when this is asked.
+ * `site.ts` does that part.
+ *
+ * ## The default shown is the default the compiler applies
+ *
+ * This is the trap worth naming, because getting it wrong produces a tooltip
+ * that is confidently wrong rather than absent. `stdlib.md` states a default for
+ * every optional argument, and `scripts/check-defaults.mjs` already holds the
+ * compiler to those: a value the surface carries that the specification does not
+ * state fails the build, and so does one the specification states that the
+ * compiler does not apply.
+ *
+ * So this reads the same two places that check reads, through `parametersOf`:
+ * the library manifest for an ordinary call, and the emitter for the eight
+ * declaration calls whose optional arguments become fields of a declaration
+ * rather than arguments. A number shown here therefore cannot differ from the
+ * number the compiler fills in; if it ever does, the defaults check fails before
+ * anybody sees the tooltip.
+ *
+ * A writer's own function is answered too, from the checker's record of it. Its
+ * parameter types are the ones the checker worked out, and a default it declares
+ * is quoted from the source text rather than printed from the tree, because the
+ * text is what the writer wrote.
+ */
+import { endOffset, entryFor, fillTemplate, proseFor, typeText } from '../core/index.js';
+import type { CheckedScript, Span } from '../core/index.js';
+import { entryAt, parametersOf, signatureTextOf } from './manifest.js';
+import type { Parameter } from './manifest.js';
+import { readChecked } from './reading.js';
+import { siteAt } from './site.js';
+import type { CallSite } from './site.js';
+
+export interface SignatureHelp {
+  /** The callee as written: `ema`, `draw.box`, or a name the file declared. */
+  readonly name: string;
+  /** Whether the library holds it or the file declares it. */
+  readonly of: 'library' | 'declared';
+  /** The signature written out, the way the specification's tables write one. */
+  readonly signature: string;
+  readonly parameters: readonly Parameter[];
+  /**
+   * Which parameter the cursor is in, or -1 where it is in none.
+   *
+   * A label decides it where one is written, because `plot(close, color = ` is
+   * in `color` whatever position it sits in; otherwise the count of commas
+   * decides. It is -1 past the last parameter, which is a call with more
+   * arguments than the signature takes and is OS3001 the moment it is compiled.
+   */
+  readonly active: number;
+  /** The line the specification gives the call, where it gives one. */
+  readonly summary: string | undefined;
+  /** Named in the library and not implemented in this version: OS2020. */
+  readonly planned: boolean;
+  readonly refusal: string | undefined;
+  /** The callee's own name, for a host anchoring a tooltip to the call. */
+  readonly span: Span;
+}
+
+/** Which parameter a position in the argument list means. */
+function activeIn(parameters: readonly Parameter[], call: CallSite): number {
+  if (call.label !== undefined) {
+    return parameters.findIndex((one) => one.name === call.label);
+  }
+  return call.argument < parameters.length ? call.argument : -1;
+}
+
+/**
+ * A function the file declares, as the checker recorded it.
+ *
+ * `language.md` 11.1 lets a parameter carry a default, and the text of that
+ * default is quoted from the source rather than printed back from the tree: the
+ * tree holds an expression and printing one would be a second formatter, while
+ * the source holds what the writer typed.
+ */
+function declaredSignature(
+  checked: CheckedScript,
+  text: string,
+  call: CallSite,
+): SignatureHelp | undefined {
+  const held = checked.functions.find((one) => one.declaration.name.text === call.name);
+  if (held === undefined) return undefined;
+
+  const parameters: Parameter[] = held.declaration.parameters.map((one, index) => {
+    const value = one.defaultValue;
+    const binding = held.parameters[index];
+    return {
+      name: one.name.text,
+      type: binding === undefined ? 'unknown' : typeText(binding.type),
+      required: value === undefined,
+      defaultText:
+        value === undefined ? undefined : text.slice(value.span.offset, endOffset(value.span)),
+      values: [],
+    };
+  });
+
+  const written = parameters.map((one) => {
+    const head = `${one.name}${one.required ? '' : '?'}: ${one.type}`;
+    return one.defaultText === undefined ? head : `${head} = ${one.defaultText}`;
+  });
+
+  return {
+    name: call.name,
+    of: 'declared',
+    signature: `fn ${call.name}(${written.join(', ')})`,
+    parameters,
+    active: activeIn(parameters, call),
+    summary: undefined,
+    planned: false,
+    refusal: undefined,
+    span: call.nameSpan,
+  };
+}
+
+/**
+ * The call the cursor is inside, or nothing where it is inside none.
+ *
+ * Nothing is the answer for a cursor outside every bracket, and for one inside a
+ * bracket whose callee the language does not have: a name that resolves to
+ * neither the library nor the file is OS2001, and `diagnose` is what says so.
+ * A tooltip that invented a signature for it would be a second opinion about
+ * what the file declares.
+ */
+export function signature(source: string, offset: number): SignatureHelp | undefined {
+  const { file, tokens, checked } = readChecked(source);
+  const call = siteAt(file, tokens, offset).call;
+  if (call === undefined) return undefined;
+
+  const entry = entryAt(call.name, call.argument + 1);
+  if (entry === undefined) return declaredSignature(checked, file.text, call);
+
+  const parameters = parametersOf(entry);
+  return {
+    name: call.name,
+    of: 'library',
+    signature: signatureTextOf(entry),
+    parameters,
+    active: activeIn(parameters, call),
+    summary: proseFor(call.name)?.summary,
+    planned: entry.planned,
+    refusal: entry.planned
+      ? fillTemplate(entryFor('OS2020').message, { name: call.name })
+      : undefined,
+    span: call.nameSpan,
+  };
+}