openalgo-script 0.1.0-alpha.1 → 0.4.0

package/src/editor/format.ts

@@ -0,0 +1,85 @@
+/**
+ * `format`: source text in, the same program in its canonical layout out.
+ *
+ * ## The rule that makes a format button safe to press
+ *
+ * **Formatting a source must not change what it means.** A trader presses this
+ * on a script that is holding a position. If a reprint can change a number, the
+ * button is a hazard whatever else it does for the diff.
+ *
+ * That is not asserted here, it is proved twice over:
+ *
+ * - **Per call, before this function returns.** The output is lexed again and
+ *   compared with the tokens that went in, token for token. If anything moved,
+ *   the source is returned untouched. The parser reads nothing but the token
+ *   stream, so an identical stream is an identical tree, and an identical tree
+ *   is an identical program. A formatter that is wrong about a rule therefore
+ *   fails by doing nothing, which is the only acceptable way for it to fail.
+ * - **Over a corpus, in the suite.** `tests/editor/format.test.ts` formats every
+ *   example in the repository and every script the phase gates run, compiles
+ *   both texts, and asserts the compiled programs are identical. That is the
+ *   test that says the formatter works; the check above is what says it is safe
+ *   when it does not.
+ *
+ * ## A source that does not parse
+ *
+ * It is returned unchanged. Not reformatted as best it can be, and not partly
+ * reformatted: unchanged, exactly, including its line endings.
+ *
+ * The reason is what a diagnostic from either stage means. The lexer reports a
+ * character it could not read and emits no token for it, so a reprint from the
+ * token stream would delete it, and a reader would watch a character disappear
+ * from their file. The parser reports a line it could not read and recovers, so
+ * the tree describes a program nobody wrote and the three questions the spacing
+ * rules ask of it have no trustworthy answers. Neither is a file to lay out. A
+ * host formats on demand rather than on every keystroke, and a file mid-edit is
+ * simply left alone until it reads.
+ */
+import type { Token } from '../core/index.js';
+import { layOut } from './layout.js';
+import { readTree } from './reading.js';
+import { scan } from './scan.js';
+import { shapesOf } from './spacing.js';
+
+/**
+ * Whether two token streams are the same stream.
+ *
+ * Kind and text for everything, except the indent token, whose text is the
+ * leading whitespace of the line it opens and is exactly what a formatter is
+ * allowed to change. It is the only token that carries layout as text: a
+ * newline, a dedent and the end of the file have none.
+ *
+ * Spans are not compared, and must not be: every one of them moves when a line
+ * is re-indented, which is the whole point of the exercise.
+ */
+function sameStream(before: readonly Token[], after: readonly Token[]): boolean {
+  if (before.length !== after.length) return false;
+  for (const [at, token] of before.entries()) {
+    const other = after[at];
+    if (other === undefined || other.kind !== token.kind) return false;
+    if (token.kind !== 'indent' && other.text !== token.text) return false;
+  }
+  return true;
+}
+
+/**
+ * The canonical layout of a source, or the source itself where it cannot be
+ * given one safely.
+ *
+ * The output always ends in a single line ending and uses LF, because a file is
+ * normalised before anything reads it (`language.md` 3.1). A host holding CRLF
+ * text gets LF back, which is the same file to the compiler and to every other
+ * part of this project.
+ */
+export function format(source: string): string {
+  const held = readTree(source);
+  if (!held.bag.isEmpty) return source;
+
+  const shapes = shapesOf(held.script, held.tokens);
+  const out = layOut(held.file, held.tokens, scan(held.file, held.tokens), shapes);
+
+  const again = readTree(out);
+  if (!again.bag.isEmpty) return source;
+  if (!sameStream(held.tokens, again.tokens)) return source;
+  return out;
+}

package/src/editor/highlight.ts

@@ -0,0 +1,110 @@
+/**
+ * `highlight`: source text in, painted spans out.
+ *
+ * It is the real lexer and nothing else. A word added to the language is
+ * coloured on the day it lexes, because the only thing between the lexer and a
+ * colour here is a table lookup in `kinds.ts` that asks the language's own
+ * tables which sort of thing a token is.
+ *
+ * ## Two shapes, and why both
+ *
+ * A host paints a document one of two ways and there is no third: it draws over
+ * the whole text, or it renders a line at a time. So both shapes are here, and
+ * the second is derived from the first rather than scanned again.
+ *
+ * `highlight` is the flat one: every piece of the file in order, covering it
+ * exactly once. It is the primitive because the covering property is a property
+ * of a flat list, and because a host holding one can produce anything else.
+ *
+ * `highlightLines` is the per-line one, and it exists because the split is where
+ * the mistake is made. A host slicing the flat list at line boundaries has to
+ * decide what to do with a piece of whitespace that runs from the end of one
+ * line to the start of the next, and getting it wrong costs a line its
+ * indentation or gives it somebody else's. Done once here, each line's pieces
+ * concatenate to exactly that line's text, newline excluded, which is what a
+ * line renderer wants and what the test asserts.
+ *
+ * ## Offsets are into the normalised text
+ *
+ * A byte order mark is dropped and CRLF becomes LF before anything else reads a
+ * file (`language.md` 3.1), so a span's offset counts into the text after those
+ * two changes, exactly as a diagnostic's does. A host that keeps its buffer
+ * exactly as the file was written normalises it once on the way in, or draws
+ * from each piece's own `text` and never indexes its buffer at all.
+ */
+import type { SourceFile, Span } from '../core/index.js';
+import { read } from './reading.js';
+import { scan } from './scan.js';
+import type { Highlight } from './scan.js';
+
+/** One line's pieces, with the line they are on so a caller counts nothing. */
+export interface HighlightedLine {
+  /** One-based, the same line a diagnostic's span carries. */
+  readonly line: number;
+  readonly pieces: readonly Highlight[];
+}
+
+/**
+ * The pieces of a file, in order, covering every character exactly once.
+ *
+ * Nothing here throws and nothing here reports. A file being typed into is
+ * malformed most of the time, and a highlighter that stopped at the first
+ * character the language does not have would leave the rest of a trader's
+ * script grey while they finished the word. `diagnose` is where the compiler
+ * says what is wrong with it.
+ */
+export function highlight(source: string): readonly Highlight[] {
+  const held = read(source);
+  return scan(held.file, held.tokens);
+}
+
+/**
+ * The same pieces, grouped by line, with every line of the file present.
+ *
+ * A piece that runs across a line ending is cut at it, and the line ending
+ * itself belongs to no line: a renderer draws the text of a line and puts the
+ * break there itself. A blank line is an entry with no pieces rather than a
+ * missing entry, so a caller can walk lines one to `lineCount` and never index
+ * past the end of a shorter list.
+ */
+export function highlightLines(source: string): readonly HighlightedLine[] {
+  const { file, tokens } = read(source);
+  const pieces = scan(file, tokens);
+  const lines: Highlight[][] = [];
+  for (let line = 0; line < file.lineCount; line += 1) lines.push([]);
+
+  for (const piece of pieces) {
+    for (const part of split(file, piece)) {
+      lines[part.span.line - 1]?.push(part);
+    }
+  }
+
+  return lines.map((held, index) => ({ line: index + 1, pieces: held }));
+}
+
+/**
+ * One piece as the pieces of the lines it covers, the line endings dropped.
+ *
+ * Only whitespace can reach here holding a line ending, since no token and no
+ * comment crosses one, but the split is written for any piece rather than for
+ * that one case: a piece shape that grows a new kind should not need this
+ * function to be revisited to stay correct.
+ */
+function split(file: SourceFile, piece: Highlight): readonly Highlight[] {
+  if (!piece.text.includes('\n')) return [piece];
+
+  const out: Highlight[] = [];
+  let from = 0;
+  for (let i = 0; i <= piece.text.length; i += 1) {
+    const ending = i === piece.text.length || piece.text.charCodeAt(i) === 10;
+    if (!ending) continue;
+    if (i > from) out.push(partOf(file, piece, from, i));
+    from = i + 1;
+  }
+  return out;
+}
+
+function partOf(file: SourceFile, piece: Highlight, from: number, to: number): Highlight {
+  const span: Span = file.spanAt(piece.span.offset + from, to - from);
+  return { kind: piece.kind, span, text: piece.text.slice(from, to) };
+}

package/src/editor/hover.ts

@@ -0,0 +1,199 @@
+/**
+ * `hover`: what the thing under the pointer is.
+ *
+ * Three answers, and the difference between them is what the compiler says the
+ * word is rather than anything guessed from its spelling:
+ *
+ *     a library name    its signatures from the manifest, the line the
+ *                       specification gives it, the first bar its own row
+ *                       promises a value on, and the channels of a named colour
+ *     a declared name   the type the checker worked out, how it is held between
+ *                       bars, and where in the file it was declared
+ *     a reserved word   that it is one, and nothing else. See below
+ *
+ * ## What a hover wants and cannot have, said plainly
+ *
+ * **There is no per-word explanation of a keyword in a machine readable form.**
+ * `language.md` 3.4 reserves the words in one block and explains each one in the
+ * prose of the section that uses it, which is a page rather than a field. So a
+ * hover over `var` says it is a reserved word and stops. The alternative was to
+ * type thirty-five sentences into this file, and that is the copy that drifts:
+ * it would be right today, the specification would improve next month, and the
+ * only copy a writer ever reads would be the stale one. A host that wants more
+ * links the word to the specification, which is one page and stays current.
+ *
+ * **A name with several signatures has one description.** The specification
+ * gives every signature its own row, and the map read here is keyed by name, so
+ * `close` is described by its bar series row rather than by the order function's.
+ * Every signature is still listed: it is the sentence beside them that is the
+ * first row's.
+ *
+ * **A parameter has no description of its own anywhere.** `stdlib.md` describes
+ * a call in one line and names its parameters in the signature; only section
+ * 13's shared input arguments have a table with a meaning per argument. So
+ * `signature` shows a parameter's name, type, default and accepted values, which
+ * are facts, and no sentence, which would have to be invented.
+ *
+ * Each of those is recorded in `spec/editor-narrowings.json` beside what the
+ * adapter narrows, so a host reads them in one place rather than discovering
+ * them.
+ */
+import {
+  endOffset,
+  entryFor,
+  fillTemplate,
+  isNamespace,
+  libraryEntries,
+  namedColour,
+  proseFor,
+  typeText,
+} from '../core/index.js';
+import type { Colour, SourceFile, Span, Token } from '../core/index.js';
+import { isReserved } from './kinds.js';
+import { signatureTextOf } from './manifest.js';
+import { readChecked } from './reading.js';
+import { bindingNamed } from './scope.js';
+import { tokenOn } from './site.js';
+
+export type HoverKind = 'library' | 'declared' | 'keyword';
+
+export interface Hover {
+  readonly kind: HoverKind;
+  /** The word the pointer is over, which is what a host underlines. */
+  readonly span: Span;
+  /** The name as written, dotted where the word is half of a namespaced name. */
+  readonly name: string;
+  /**
+   * Every signature the name has, in the order the manifest holds them.
+   *
+   * More than one where the language gives the name more than one, which
+   * `stdlib.md` 2.2 allows: a hover has no arguments to choose between them
+   * with, so it shows both rather than picking.
+   */
+  readonly signatures: readonly string[];
+  /** The line the specification gives it, where it gives one. */
+  readonly summary: string | undefined;
+  /** The first bar it can produce a value for, as its own row states it. */
+  readonly warmup: string | undefined;
+  /** The type the checker gave a declared name. */
+  readonly type: string | undefined;
+  /** Where a declared name was declared, which is what a host jumps to. */
+  readonly declaredAt: Span | undefined;
+  /** The channels of a named colour, for a swatch. */
+  readonly colour: Colour | undefined;
+  /** Named in the library and not implemented in this version: OS2020. */
+  readonly planned: boolean;
+  /** The catalogue's OS2020 message for this name, on a planned name and no other. */
+  readonly refusal: string | undefined;
+}
+
+/**
+ * The name a word is part of, which is the dotted one when it is half of one.
+ *
+ * Pointing at `box` in `draw.box` is pointing at `draw.box`: the member on its
+ * own is an ordinary name the library does not have, and a hover that said so
+ * would be wrong about the word under the pointer.
+ */
+function nameAt(
+  file: SourceFile,
+  tokens: readonly Token[],
+  token: Token,
+): { name: string; span: Span } {
+  const index = tokens.indexOf(token);
+  const dot = tokens[index - 1];
+  const namespace = tokens[index - 2];
+  if (dot?.kind === '.' && namespace?.kind === 'identifier' && isNamespace(namespace.text)) {
+    return {
+      name: `${namespace.text}.${token.text}`,
+      span: file.spanAt(namespace.span.offset, endOffset(token.span) - namespace.span.offset),
+    };
+  }
+  // A namespace with a member after it is pointed at as the whole name too, so
+  // that hovering either half of `draw.box` says the same thing.
+  const next = tokens[index + 1];
+  const member = tokens[index + 2];
+  if (
+    isNamespace(token.text) &&
+    next?.kind === '.' &&
+    member?.kind === 'identifier' &&
+    libraryEntries(`${token.text}.${member.text}`).length > 0
+  ) {
+    return {
+      name: `${token.text}.${member.text}`,
+      span: file.spanAt(token.span.offset, endOffset(member.span) - token.span.offset),
+    };
+  }
+  return { name: file.text.slice(token.span.offset, endOffset(token.span)), span: token.span };
+}
+
+const NOTHING = {
+  summary: undefined,
+  warmup: undefined,
+  type: undefined,
+  declaredAt: undefined,
+  colour: undefined,
+  planned: false,
+  refusal: undefined,
+} as const;
+
+/**
+ * What the word at an offset is, or nothing where the offset is not on a word.
+ *
+ * A pointer over a space, a bracket, a number or a string gets nothing: those
+ * are not names, and a tooltip that appeared over a comma would be in the way
+ * rather than useful. A host that wants a literal's value has it in the text.
+ */
+export function hover(source: string, offset: number): Hover | undefined {
+  const { file, tokens, checked } = readChecked(source);
+  const token = tokenOn(tokens, offset);
+  if (token === undefined) return undefined;
+
+  if (isReserved(token.kind)) {
+    // A reserved word, and every one of them is also a word a script may not
+    // declare. What it means is the specification's, and there is no per-word
+    // field to read it from, which the file header states and the narrowings
+    // record.
+    return { kind: 'keyword', span: token.span, name: token.text, signatures: [], ...NOTHING };
+  }
+  if (token.kind !== 'identifier') return undefined;
+
+  const { name, span } = nameAt(file, tokens, token);
+
+  const binding = bindingNamed(checked, name, offset);
+  if (binding !== undefined) {
+    const persistence = binding.persistence === 'none' ? '' : `${binding.persistence} `;
+    return {
+      kind: 'declared',
+      span,
+      name,
+      signatures: [`${persistence}${name}: ${typeText(binding.type)}`],
+      summary: undefined,
+      warmup: undefined,
+      type: typeText(binding.type),
+      declaredAt: binding.declaredAt,
+      colour: undefined,
+      planned: false,
+      refusal: undefined,
+    };
+  }
+
+  const entries = libraryEntries(name);
+  const first = entries[0];
+  if (first === undefined) return undefined;
+
+  const prose = proseFor(name);
+  const planned = entries.every((one) => one.planned);
+  return {
+    kind: 'library',
+    span,
+    name,
+    signatures: entries.map(signatureTextOf),
+    summary: prose?.summary,
+    warmup: prose?.warmup,
+    type: typeText(first.returns),
+    declaredAt: undefined,
+    colour: namedColour(name),
+    planned,
+    refusal: planned ? fillTemplate(entryFor('OS2020').message, { name }) : undefined,
+  };
+}

package/src/editor/index.ts

@@ -0,0 +1,65 @@
+/**
+ * OpenScript, the language intelligence, headless.
+ *
+ * Six pure functions, text in and data out: highlight, complete, diagnose,
+ * hover, signature, format. Three of them are here. No DOM anywhere, no package,
+ * no browser global, and nothing on screen: a host supplies the text component,
+ * the panel, the apply button, saving and revisions, and keeps its own design
+ * system. `scripts/check-layering.mjs` fails the build if this tier so much as
+ * mentions a document object, because an editor half that reaches for one is the
+ * thing that stops a platform embedding it.
+ *
+ * **None of it is hand written.** That is the whole reason it is worth shipping
+ * rather than leaving to each host, and it is a property of how each function is
+ * built rather than an intention:
+ *
+ *     highlight    the real lexer, and the language's own tables of words and
+ *                  marks decide what each token is painted as
+ *     diagnose     the whole compiler, and every message and fix comes from the
+ *                  error catalogue the documentation is generated from
+ *     format       the real lexer for the tokens and the real parser for the
+ *                  three questions spacing cannot answer without a tree
+ *
+ * A hand written version of any of them drifts from the language and nobody
+ * notices for a release: a word is added and stays grey, a diagnostic's wording
+ * improves in one place. The editor is the compiler wearing a different hat, not
+ * a second implementation to keep in step.
+ *
+ *     complete     the standard library manifest, the checker's bindings for
+ *                  what the file itself declares, and the error catalogue for
+ *                  what a planned call would be refused with
+ *     hover        the manifest again, and the specification's own tables, read
+ *                  at build time rather than retyped here
+ *     signature    the manifest and the emitter, which between them are where
+ *                  every default the compiler applies lives
+ *
+ * All six exist, so `openalgo-script/editor` is an entry point in the package's
+ * export map. It was not one while three of them were missing, because an entry
+ * point that resolves to half a tier fails at a consumer's run time rather than
+ * honestly at install.
+ *
+ * What a host loses by taking the drop-in adapter beside this rather than
+ * calling these six itself, and the three places a hover has no sentence to show
+ * because none exists in a machine readable form, are recorded together in
+ * `spec/editor-narrowings.json`.
+ */
+export type { Diagnostic, Span } from '../core/index.js';
+
+export type { HighlightKind } from './kinds.js';
+export type { Highlight } from './scan.js';
+export type { HighlightedLine } from './highlight.js';
+export { highlight, highlightLines } from './highlight.js';
+
+export { diagnose } from './diagnose.js';
+
+export { format } from './format.js';
+
+export type { Completion, CompletionKind } from './complete.js';
+export { complete } from './complete.js';
+
+export type { Hover, HoverKind } from './hover.js';
+export { hover } from './hover.js';
+
+export type { Parameter } from './manifest.js';
+export type { SignatureHelp } from './signature.js';
+export { signature } from './signature.js';

package/src/editor/kinds.ts

@@ -0,0 +1,157 @@
+/**
+ * What a host paints, and where each kind comes from.
+ *
+ * A highlighter is the one part of a language tool that is always written twice:
+ * once in the compiler, where the vocabulary is decided, and once in a theme
+ * file, where somebody types the words out again from memory. The second copy is
+ * wrong from the day a word is added to the language, and nobody notices, because
+ * an unpainted keyword still compiles.
+ *
+ * So nothing below is a list of words. Every kind is decided by asking the
+ * compiler a question it already answers:
+ *
+ *     keyword      the token's kind is in the language's reserved word table
+ *     builtin      the standard library manifest has an entry for the name
+ *     operator     the mark is in one of the language's operator tables
+ *     punctuation  a mark the grammar has that no operator table claims
+ *
+ * A word added to `RESERVED_WORDS`, a function added to the manifest and a mark
+ * added to `PUNCTUATORS` are each painted the day they lex, without this file
+ * being edited.
+ *
+ * ## Why `reserved` is a question and not a comparison
+ *
+ * A token's kind is its own text for a keyword and for a punctuation mark, which
+ * is what lets a parser read `token.kind === 'if'`. A consuming platform read
+ * that as "a reserved word is a token whose kind equals its text", which is true
+ * and is folklore: it is nowhere in the token vocabulary, it silently takes in
+ * every punctuation mark as well, and it would stop being true the first time a
+ * kind and a spelling came apart. `kindOf` answers the question instead, from
+ * the same table the lexer classifies with.
+ */
+import { PUNCTUATORS, RESERVED_WORDS } from '../core/index.js';
+import {
+  ARITHMETIC_OPERATORS,
+  ASSIGNMENT_OPERATORS,
+  COMPARISON_OPERATORS,
+  EQUALITY_OPERATORS,
+  isLibraryName,
+  isNamespace,
+} from '../core/index.js';
+import type { Token, TokenKind } from '../core/index.js';
+
+/**
+ * What a span of source is, for the purpose of giving it a colour.
+ *
+ * Eleven kinds, and the set is closed: a host writes eleven theme rules once and
+ * never has to add a twelfth, because a new word in the language arrives as one
+ * of these rather than as a kind nobody has a colour for.
+ *
+ * `whitespace` and `unknown` exist because every character of the file is
+ * covered exactly once (see `scan.ts`), so the spaces between tokens have to be
+ * something. `unknown` is source the lexer took no token from and that is not a
+ * space or a comment: a character the language does not have, a region somebody
+ * meant as a block comment, or a line continuation backslash. It does not mean
+ * "wrong": `diagnose` says what is wrong, and a highlighter that had a second
+ * opinion about it would be a second opinion about the language.
+ */
+export type HighlightKind =
+  | 'keyword'
+  | 'builtin'
+  | 'name'
+  | 'number'
+  | 'string'
+  | 'color'
+  | 'comment'
+  | 'operator'
+  | 'punctuation'
+  | 'whitespace'
+  | 'unknown';
+
+const RESERVED: ReadonlySet<string> = new Set<string>(RESERVED_WORDS);
+
+/**
+ * The marks that compute something, as against the marks that group and
+ * separate.
+ *
+ * Built from the language's own operator tables rather than by dividing the
+ * punctuation list by hand, so that an operator added to the language is painted
+ * as one. The tables of words, `and` and `or` and `not`, are reserved words and
+ * are painted as keywords, which is what they are to the lexer.
+ *
+ * What is left over is punctuation: the brackets, the comma, the dot, and the
+ * two marks of the ternary. They group and separate rather than compute, and a
+ * theme that wants them dimmer than the arithmetic can have that.
+ */
+const OPERATORS: ReadonlySet<string> = new Set<string>([
+  ...ARITHMETIC_OPERATORS,
+  ...COMPARISON_OPERATORS,
+  ...EQUALITY_OPERATORS,
+  ...ASSIGNMENT_OPERATORS,
+]);
+
+const PUNCTUATION: ReadonlySet<string> = new Set<string>(PUNCTUATORS);
+
+/** Whether a token's kind is a reserved word of language.md 3.4. */
+export function isReserved(kind: TokenKind): boolean {
+  return RESERVED.has(kind);
+}
+
+/**
+ * Whether a name is one the standard library already has.
+ *
+ * The manifest is the same index the checker resolves against and the same one
+ * the example check reads its globals from, so a function added to the library
+ * is painted as a built-in on the day it is added. A namespace is included
+ * because `draw` on its own is not a value a script can hold: it is half of a
+ * dotted name, and `dotted` below finishes the question for the other half.
+ */
+function isBuiltin(name: string): boolean {
+  return isLibraryName(name) || isNamespace(name);
+}
+
+/**
+ * The member half of a namespaced name, which is only a built-in in company.
+ *
+ * `box` is an ordinary name, and `draw.box` is a library entry. The manifest is
+ * asked about the dotted spelling, so nothing here decides what a namespace
+ * holds: `language.md` 15.2 and the manifest do.
+ */
+function dotted(before: readonly Token[], name: string): boolean {
+  const dot = before[before.length - 1];
+  const namespace = before[before.length - 2];
+  if (dot?.kind !== '.' || namespace?.kind !== 'identifier') return false;
+  return isNamespace(namespace.text) && isLibraryName(`${namespace.text}.${name}`);
+}
+
+/**
+ * The kind a token is painted as, given the tokens before it.
+ *
+ * The tokens before it are needed for exactly one question, the dotted name
+ * above. Everything else is decided by the token alone.
+ */
+export function kindOf(token: Token, before: readonly Token[]): HighlightKind {
+  switch (token.kind) {
+    case 'numberLiteral':
+      return 'number';
+    case 'stringLiteral':
+      return 'string';
+    case 'hexColor':
+      // The language's own spelling of the type is `color`, so that is the kind
+      // a host matches on, whatever the prose around it calls the thing.
+      return 'color';
+    case 'identifier':
+      return isBuiltin(token.text) || dotted(before, token.text) ? 'builtin' : 'name';
+    case 'newline':
+    case 'indent':
+    case 'dedent':
+    case 'endOfFile':
+      // Layout is the file's shape rather than something to read, and an indent
+      // token's span is the leading spaces of its line, which are spaces.
+      return 'whitespace';
+    default:
+      if (isReserved(token.kind)) return 'keyword';
+      if (OPERATORS.has(token.kind)) return 'operator';
+      return PUNCTUATION.has(token.kind) ? 'punctuation' : 'unknown';
+  }
+}