openalgo-script 0.2.0 → 0.5.0

package/src/editor/hover.ts

@@ -0,0 +1,199 @@
+/**
+ * `hover`: what the thing under the pointer is.
+ *
+ * Three answers, and the difference between them is what the compiler says the
+ * word is rather than anything guessed from its spelling:
+ *
+ *     a library name    its signatures from the manifest, the line the
+ *                       specification gives it, the first bar its own row
+ *                       promises a value on, and the channels of a named colour
+ *     a declared name   the type the checker worked out, how it is held between
+ *                       bars, and where in the file it was declared
+ *     a reserved word   that it is one, and nothing else. See below
+ *
+ * ## What a hover wants and cannot have, said plainly
+ *
+ * **There is no per-word explanation of a keyword in a machine readable form.**
+ * `language.md` 3.4 reserves the words in one block and explains each one in the
+ * prose of the section that uses it, which is a page rather than a field. So a
+ * hover over `var` says it is a reserved word and stops. The alternative was to
+ * type thirty-five sentences into this file, and that is the copy that drifts:
+ * it would be right today, the specification would improve next month, and the
+ * only copy a writer ever reads would be the stale one. A host that wants more
+ * links the word to the specification, which is one page and stays current.
+ *
+ * **A name with several signatures has one description.** The specification
+ * gives every signature its own row, and the map read here is keyed by name, so
+ * `close` is described by its bar series row rather than by the order function's.
+ * Every signature is still listed: it is the sentence beside them that is the
+ * first row's.
+ *
+ * **A parameter has no description of its own anywhere.** `stdlib.md` describes
+ * a call in one line and names its parameters in the signature; only section
+ * 13's shared input arguments have a table with a meaning per argument. So
+ * `signature` shows a parameter's name, type, default and accepted values, which
+ * are facts, and no sentence, which would have to be invented.
+ *
+ * Each of those is recorded in `spec/editor-narrowings.json` beside what the
+ * adapter narrows, so a host reads them in one place rather than discovering
+ * them.
+ */
+import {
+  endOffset,
+  entryFor,
+  fillTemplate,
+  isNamespace,
+  libraryEntries,
+  namedColour,
+  proseFor,
+  typeText,
+} from '../core/index.js';
+import type { Colour, SourceFile, Span, Token } from '../core/index.js';
+import { isReserved } from './kinds.js';
+import { signatureTextOf } from './manifest.js';
+import { readChecked } from './reading.js';
+import { bindingNamed } from './scope.js';
+import { tokenOn } from './site.js';
+
+export type HoverKind = 'library' | 'declared' | 'keyword';
+
+export interface Hover {
+  readonly kind: HoverKind;
+  /** The word the pointer is over, which is what a host underlines. */
+  readonly span: Span;
+  /** The name as written, dotted where the word is half of a namespaced name. */
+  readonly name: string;
+  /**
+   * Every signature the name has, in the order the manifest holds them.
+   *
+   * More than one where the language gives the name more than one, which
+   * `stdlib.md` 2.2 allows: a hover has no arguments to choose between them
+   * with, so it shows both rather than picking.
+   */
+  readonly signatures: readonly string[];
+  /** The line the specification gives it, where it gives one. */
+  readonly summary: string | undefined;
+  /** The first bar it can produce a value for, as its own row states it. */
+  readonly warmup: string | undefined;
+  /** The type the checker gave a declared name. */
+  readonly type: string | undefined;
+  /** Where a declared name was declared, which is what a host jumps to. */
+  readonly declaredAt: Span | undefined;
+  /** The channels of a named colour, for a swatch. */
+  readonly colour: Colour | undefined;
+  /** Named in the library and not implemented in this version: OS2020. */
+  readonly planned: boolean;
+  /** The catalogue's OS2020 message for this name, on a planned name and no other. */
+  readonly refusal: string | undefined;
+}
+
+/**
+ * The name a word is part of, which is the dotted one when it is half of one.
+ *
+ * Pointing at `box` in `draw.box` is pointing at `draw.box`: the member on its
+ * own is an ordinary name the library does not have, and a hover that said so
+ * would be wrong about the word under the pointer.
+ */
+function nameAt(
+  file: SourceFile,
+  tokens: readonly Token[],
+  token: Token,
+): { name: string; span: Span } {
+  const index = tokens.indexOf(token);
+  const dot = tokens[index - 1];
+  const namespace = tokens[index - 2];
+  if (dot?.kind === '.' && namespace?.kind === 'identifier' && isNamespace(namespace.text)) {
+    return {
+      name: `${namespace.text}.${token.text}`,
+      span: file.spanAt(namespace.span.offset, endOffset(token.span) - namespace.span.offset),
+    };
+  }
+  // A namespace with a member after it is pointed at as the whole name too, so
+  // that hovering either half of `draw.box` says the same thing.
+  const next = tokens[index + 1];
+  const member = tokens[index + 2];
+  if (
+    isNamespace(token.text) &&
+    next?.kind === '.' &&
+    member?.kind === 'identifier' &&
+    libraryEntries(`${token.text}.${member.text}`).length > 0
+  ) {
+    return {
+      name: `${token.text}.${member.text}`,
+      span: file.spanAt(token.span.offset, endOffset(member.span) - token.span.offset),
+    };
+  }
+  return { name: file.text.slice(token.span.offset, endOffset(token.span)), span: token.span };
+}
+
+const NOTHING = {
+  summary: undefined,
+  warmup: undefined,
+  type: undefined,
+  declaredAt: undefined,
+  colour: undefined,
+  planned: false,
+  refusal: undefined,
+} as const;
+
+/**
+ * What the word at an offset is, or nothing where the offset is not on a word.
+ *
+ * A pointer over a space, a bracket, a number or a string gets nothing: those
+ * are not names, and a tooltip that appeared over a comma would be in the way
+ * rather than useful. A host that wants a literal's value has it in the text.
+ */
+export function hover(source: string, offset: number): Hover | undefined {
+  const { file, tokens, checked } = readChecked(source);
+  const token = tokenOn(tokens, offset);
+  if (token === undefined) return undefined;
+
+  if (isReserved(token.kind)) {
+    // A reserved word, and every one of them is also a word a script may not
+    // declare. What it means is the specification's, and there is no per-word
+    // field to read it from, which the file header states and the narrowings
+    // record.
+    return { kind: 'keyword', span: token.span, name: token.text, signatures: [], ...NOTHING };
+  }
+  if (token.kind !== 'identifier') return undefined;
+
+  const { name, span } = nameAt(file, tokens, token);
+
+  const binding = bindingNamed(checked, name, offset);
+  if (binding !== undefined) {
+    const persistence = binding.persistence === 'none' ? '' : `${binding.persistence} `;
+    return {
+      kind: 'declared',
+      span,
+      name,
+      signatures: [`${persistence}${name}: ${typeText(binding.type)}`],
+      summary: undefined,
+      warmup: undefined,
+      type: typeText(binding.type),
+      declaredAt: binding.declaredAt,
+      colour: undefined,
+      planned: false,
+      refusal: undefined,
+    };
+  }
+
+  const entries = libraryEntries(name);
+  const first = entries[0];
+  if (first === undefined) return undefined;
+
+  const prose = proseFor(name);
+  const planned = entries.every((one) => one.planned);
+  return {
+    kind: 'library',
+    span,
+    name,
+    signatures: entries.map(signatureTextOf),
+    summary: prose?.summary,
+    warmup: prose?.warmup,
+    type: typeText(first.returns),
+    declaredAt: undefined,
+    colour: namedColour(name),
+    planned,
+    refusal: planned ? fillTemplate(entryFor('OS2020').message, { name }) : undefined,
+  };
+}

package/src/editor/index.ts

@@ -0,0 +1,65 @@
+/**
+ * OpenScript, the language intelligence, headless.
+ *
+ * Six pure functions, text in and data out: highlight, complete, diagnose,
+ * hover, signature, format. Three of them are here. No DOM anywhere, no package,
+ * no browser global, and nothing on screen: a host supplies the text component,
+ * the panel, the apply button, saving and revisions, and keeps its own design
+ * system. `scripts/check-layering.mjs` fails the build if this tier so much as
+ * mentions a document object, because an editor half that reaches for one is the
+ * thing that stops a platform embedding it.
+ *
+ * **None of it is hand written.** That is the whole reason it is worth shipping
+ * rather than leaving to each host, and it is a property of how each function is
+ * built rather than an intention:
+ *
+ *     highlight    the real lexer, and the language's own tables of words and
+ *                  marks decide what each token is painted as
+ *     diagnose     the whole compiler, and every message and fix comes from the
+ *                  error catalogue the documentation is generated from
+ *     format       the real lexer for the tokens and the real parser for the
+ *                  three questions spacing cannot answer without a tree
+ *
+ * A hand written version of any of them drifts from the language and nobody
+ * notices for a release: a word is added and stays grey, a diagnostic's wording
+ * improves in one place. The editor is the compiler wearing a different hat, not
+ * a second implementation to keep in step.
+ *
+ *     complete     the standard library manifest, the checker's bindings for
+ *                  what the file itself declares, and the error catalogue for
+ *                  what a planned call would be refused with
+ *     hover        the manifest again, and the specification's own tables, read
+ *                  at build time rather than retyped here
+ *     signature    the manifest and the emitter, which between them are where
+ *                  every default the compiler applies lives
+ *
+ * All six exist, so `openalgo-script/editor` is an entry point in the package's
+ * export map. It was not one while three of them were missing, because an entry
+ * point that resolves to half a tier fails at a consumer's run time rather than
+ * honestly at install.
+ *
+ * What a host loses by taking the drop-in adapter beside this rather than
+ * calling these six itself, and the three places a hover has no sentence to show
+ * because none exists in a machine readable form, are recorded together in
+ * `spec/editor-narrowings.json`.
+ */
+export type { Diagnostic, Span } from '../core/index.js';
+
+export type { HighlightKind } from './kinds.js';
+export type { Highlight } from './scan.js';
+export type { HighlightedLine } from './highlight.js';
+export { highlight, highlightLines } from './highlight.js';
+
+export { diagnose } from './diagnose.js';
+
+export { format } from './format.js';
+
+export type { Completion, CompletionKind } from './complete.js';
+export { complete } from './complete.js';
+
+export type { Hover, HoverKind } from './hover.js';
+export { hover } from './hover.js';
+
+export type { Parameter } from './manifest.js';
+export type { SignatureHelp } from './signature.js';
+export { signature } from './signature.js';

package/src/editor/kinds.ts

@@ -0,0 +1,157 @@
+/**
+ * What a host paints, and where each kind comes from.
+ *
+ * A highlighter is the one part of a language tool that is always written twice:
+ * once in the compiler, where the vocabulary is decided, and once in a theme
+ * file, where somebody types the words out again from memory. The second copy is
+ * wrong from the day a word is added to the language, and nobody notices, because
+ * an unpainted keyword still compiles.
+ *
+ * So nothing below is a list of words. Every kind is decided by asking the
+ * compiler a question it already answers:
+ *
+ *     keyword      the token's kind is in the language's reserved word table
+ *     builtin      the standard library manifest has an entry for the name
+ *     operator     the mark is in one of the language's operator tables
+ *     punctuation  a mark the grammar has that no operator table claims
+ *
+ * A word added to `RESERVED_WORDS`, a function added to the manifest and a mark
+ * added to `PUNCTUATORS` are each painted the day they lex, without this file
+ * being edited.
+ *
+ * ## Why `reserved` is a question and not a comparison
+ *
+ * A token's kind is its own text for a keyword and for a punctuation mark, which
+ * is what lets a parser read `token.kind === 'if'`. A consuming platform read
+ * that as "a reserved word is a token whose kind equals its text", which is true
+ * and is folklore: it is nowhere in the token vocabulary, it silently takes in
+ * every punctuation mark as well, and it would stop being true the first time a
+ * kind and a spelling came apart. `kindOf` answers the question instead, from
+ * the same table the lexer classifies with.
+ */
+import { PUNCTUATORS, RESERVED_WORDS } from '../core/index.js';
+import {
+  ARITHMETIC_OPERATORS,
+  ASSIGNMENT_OPERATORS,
+  COMPARISON_OPERATORS,
+  EQUALITY_OPERATORS,
+  isLibraryName,
+  isNamespace,
+} from '../core/index.js';
+import type { Token, TokenKind } from '../core/index.js';
+
+/**
+ * What a span of source is, for the purpose of giving it a colour.
+ *
+ * Eleven kinds, and the set is closed: a host writes eleven theme rules once and
+ * never has to add a twelfth, because a new word in the language arrives as one
+ * of these rather than as a kind nobody has a colour for.
+ *
+ * `whitespace` and `unknown` exist because every character of the file is
+ * covered exactly once (see `scan.ts`), so the spaces between tokens have to be
+ * something. `unknown` is source the lexer took no token from and that is not a
+ * space or a comment: a character the language does not have, a region somebody
+ * meant as a block comment, or a line continuation backslash. It does not mean
+ * "wrong": `diagnose` says what is wrong, and a highlighter that had a second
+ * opinion about it would be a second opinion about the language.
+ */
+export type HighlightKind =
+  | 'keyword'
+  | 'builtin'
+  | 'name'
+  | 'number'
+  | 'string'
+  | 'color'
+  | 'comment'
+  | 'operator'
+  | 'punctuation'
+  | 'whitespace'
+  | 'unknown';
+
+const RESERVED: ReadonlySet<string> = new Set<string>(RESERVED_WORDS);
+
+/**
+ * The marks that compute something, as against the marks that group and
+ * separate.
+ *
+ * Built from the language's own operator tables rather than by dividing the
+ * punctuation list by hand, so that an operator added to the language is painted
+ * as one. The tables of words, `and` and `or` and `not`, are reserved words and
+ * are painted as keywords, which is what they are to the lexer.
+ *
+ * What is left over is punctuation: the brackets, the comma, the dot, and the
+ * two marks of the ternary. They group and separate rather than compute, and a
+ * theme that wants them dimmer than the arithmetic can have that.
+ */
+const OPERATORS: ReadonlySet<string> = new Set<string>([
+  ...ARITHMETIC_OPERATORS,
+  ...COMPARISON_OPERATORS,
+  ...EQUALITY_OPERATORS,
+  ...ASSIGNMENT_OPERATORS,
+]);
+
+const PUNCTUATION: ReadonlySet<string> = new Set<string>(PUNCTUATORS);
+
+/** Whether a token's kind is a reserved word of language.md 3.4. */
+export function isReserved(kind: TokenKind): boolean {
+  return RESERVED.has(kind);
+}
+
+/**
+ * Whether a name is one the standard library already has.
+ *
+ * The manifest is the same index the checker resolves against and the same one
+ * the example check reads its globals from, so a function added to the library
+ * is painted as a built-in on the day it is added. A namespace is included
+ * because `draw` on its own is not a value a script can hold: it is half of a
+ * dotted name, and `dotted` below finishes the question for the other half.
+ */
+function isBuiltin(name: string): boolean {
+  return isLibraryName(name) || isNamespace(name);
+}
+
+/**
+ * The member half of a namespaced name, which is only a built-in in company.
+ *
+ * `box` is an ordinary name, and `draw.box` is a library entry. The manifest is
+ * asked about the dotted spelling, so nothing here decides what a namespace
+ * holds: `language.md` 15.2 and the manifest do.
+ */
+function dotted(before: readonly Token[], name: string): boolean {
+  const dot = before[before.length - 1];
+  const namespace = before[before.length - 2];
+  if (dot?.kind !== '.' || namespace?.kind !== 'identifier') return false;
+  return isNamespace(namespace.text) && isLibraryName(`${namespace.text}.${name}`);
+}
+
+/**
+ * The kind a token is painted as, given the tokens before it.
+ *
+ * The tokens before it are needed for exactly one question, the dotted name
+ * above. Everything else is decided by the token alone.
+ */
+export function kindOf(token: Token, before: readonly Token[]): HighlightKind {
+  switch (token.kind) {
+    case 'numberLiteral':
+      return 'number';
+    case 'stringLiteral':
+      return 'string';
+    case 'hexColor':
+      // The language's own spelling of the type is `color`, so that is the kind
+      // a host matches on, whatever the prose around it calls the thing.
+      return 'color';
+    case 'identifier':
+      return isBuiltin(token.text) || dotted(before, token.text) ? 'builtin' : 'name';
+    case 'newline':
+    case 'indent':
+    case 'dedent':
+    case 'endOfFile':
+      // Layout is the file's shape rather than something to read, and an indent
+      // token's span is the leading spaces of its line, which are spaces.
+      return 'whitespace';
+    default:
+      if (isReserved(token.kind)) return 'keyword';
+      if (OPERATORS.has(token.kind)) return 'operator';
+      return PUNCTUATION.has(token.kind) ? 'punctuation' : 'unknown';
+  }
+}

package/src/editor/layout.ts

@@ -0,0 +1,189 @@
+/**
+ * The canonical layout of a file, line by line.
+ *
+ * `language.md` 3.13 states the rules; this applies them. The two halves are
+ * separate because the specification is what another implementation writes a
+ * formatter from, and a rule that lived only here would be a rule only this
+ * formatter has.
+ *
+ * ## What is preserved, and why that is the safe direction
+ *
+ * The writer's line breaks are kept. A formatter that joined a call spanning
+ * four lines into one would be correct and unusable: it would take the layout
+ * somebody chose to make a long argument list readable and hand back a line
+ * nobody can read. So this decides indentation and spacing, which have one right
+ * answer, and leaves the choice of where to break a long statement to the person
+ * writing it, which does not.
+ *
+ * It is also the safe direction. A break that was there before is a break the
+ * lexer already accepted, so preserving them cannot turn a legal file into an
+ * illegal one, while inventing or removing one can: a statement continues onto
+ * the next line only under the three conditions of 3.11, and a formatter that
+ * moved a break would have to know them, which is the language's rule
+ * implemented twice.
+ *
+ * ## Comments
+ *
+ * A comment on a line of its own takes the indentation of the next line that
+ * carries a token, so a commented-out statement sits with the block it belongs
+ * to. This is safe for a reason worth stating: such a line produces no token at
+ * all, so its indentation says nothing about block structure and cannot open or
+ * close one (3.10). A comment after code stays where it is, two spaces clear of
+ * the code.
+ */
+import type { SourceFile, Token } from '../core/index.js';
+import type { Highlight } from './scan.js';
+import { spaceBetween } from './spacing.js';
+import type { Shapes } from './spacing.js';
+
+/** One level of block, `language.md` 3.13. */
+const INDENT = '    ';
+
+/**
+ * What a continuation line is indented past the line that began its statement.
+ *
+ * Eight rather than four so that a continuation can never be read as the body of
+ * the header above it. `if crossUp(fast,` with its second half indented four
+ * would sit exactly where the block's first statement sits, which is legal and
+ * unreadable.
+ */
+const CONTINUATION = '        ';
+
+/** Two, so a trailing comment is visibly not part of the code it follows. */
+const BEFORE_COMMENT = '  ';
+
+const BACKSLASH = '\\';
+
+interface CodeLine {
+  readonly tokens: Token[];
+  readonly indent: string;
+}
+
+/**
+ * The code lines of a file, each with the indentation the canonical layout gives
+ * it.
+ *
+ * Depth comes from the lexer's own indent and dedent tokens rather than from
+ * counting spaces, so the blocks this prints are the blocks the compiler read. A
+ * line that begins a statement is indented by its depth; a line that continues
+ * one is indented past the line that began it.
+ */
+function codeLines(tokens: readonly Token[]): ReadonlyMap<number, CodeLine> {
+  const lines = new Map<number, CodeLine>();
+  let depth = 0;
+  let starting = true;
+
+  for (const token of tokens) {
+    switch (token.kind) {
+      case 'indent':
+        depth += 1;
+        continue;
+      case 'dedent':
+        depth -= 1;
+        continue;
+      case 'newline':
+        starting = true;
+        continue;
+      case 'endOfFile':
+        continue;
+      default:
+        break;
+    }
+
+    let line = lines.get(token.span.line);
+    if (line === undefined) {
+      const body = INDENT.repeat(Math.max(depth, 0));
+      line = { tokens: [], indent: starting ? body : body + CONTINUATION };
+      lines.set(token.span.line, line);
+    }
+    line.tokens.push(token);
+    starting = false;
+  }
+
+  return lines;
+}
+
+/** One code line as text: its indentation, then its tokens with their spacing. */
+function printCode(file: SourceFile, line: CodeLine, shapes: Shapes): string {
+  let text = line.indent;
+  for (const [at, token] of line.tokens.entries()) {
+    const before = line.tokens[at - 1];
+    if (before !== undefined && spaceBetween(before, token, shapes)) text += ' ';
+    text += file.text.slice(token.span.offset, token.span.offset + token.span.length);
+  }
+  return text;
+}
+
+/** The indentation of the next line that carries a token, for a comment to take. */
+function indentBelow(lines: ReadonlyMap<number, CodeLine>, from: number, last: number): string {
+  for (let line = from + 1; line <= last; line += 1) {
+    const found = lines.get(line);
+    if (found !== undefined) return found.indent;
+  }
+  return '';
+}
+
+/**
+ * Blank lines, as the canonical layout has them: never two in a row, never
+ * before the first line, and one line ending at the end of the file.
+ *
+ * A blank line between two paragraphs of a script is the writer's, and it is
+ * kept. A run of four is a file that has been edited for a year, and one of them
+ * says everything the four said.
+ */
+function joinLines(printed: readonly (string | undefined)[]): string {
+  const kept: string[] = [];
+  for (const line of printed) {
+    if (line !== undefined) {
+      kept.push(line);
+      continue;
+    }
+    if (kept.length === 0 || kept[kept.length - 1] === '') continue;
+    kept.push('');
+  }
+  while (kept.length > 0 && kept[kept.length - 1] === '') kept.pop();
+  return kept.length === 0 ? '' : `${kept.join('\n')}\n`;
+}
+
+/**
+ * The whole file, laid out.
+ *
+ * The pieces are the covering scan, which is where the comments come from and
+ * where a line continuation backslash is found. A backslash is kept wherever it
+ * was written rather than worked out again: whether a statement needs one is the
+ * lexer's rule, and the file in front of this one has already been through the
+ * lexer with it in place.
+ */
+export function layOut(
+  file: SourceFile,
+  tokens: readonly Token[],
+  pieces: readonly Highlight[],
+  shapes: Shapes,
+): string {
+  const lines = codeLines(tokens);
+  const comments = new Map<number, string>();
+  const continued = new Set<number>();
+
+  for (const piece of pieces) {
+    if (piece.kind === 'comment') comments.set(piece.span.line, piece.text.trimEnd());
+    else if (piece.kind === 'unknown' && piece.text === BACKSLASH) continued.add(piece.span.line);
+  }
+
+  const printed: (string | undefined)[] = [];
+  for (let line = 1; line <= file.lineCount; line += 1) {
+    const code = lines.get(line);
+    const comment = comments.get(line);
+
+    if (code === undefined) {
+      printed.push(comment === undefined ? undefined : indentBelow(lines, line, file.lineCount) + comment);
+      continue;
+    }
+
+    let text = printCode(file, code, shapes);
+    if (continued.has(line)) text += ` ${BACKSLASH}`;
+    if (comment !== undefined) text += BEFORE_COMMENT + comment;
+    printed.push(text);
+  }
+
+  return joinLines(printed);
+}