openalgo-script 0.1.0-alpha.1 → 0.4.0

package/src/adapters/codemirror/positions.ts

@@ -0,0 +1,79 @@
+/**
+ * Offsets, translated between the document the editor holds and the text the
+ * compiler read.
+ *
+ * `language.md` 3.1 normalises a file before anything reads it: a byte order
+ * mark is dropped and every CRLF becomes LF. Every span the compiler produces
+ * therefore indexes the normalised text, which `docs/integrating/the-editor-half.md`
+ * tells a host plainly and expects it to handle.
+ *
+ * A drop-in cannot expect it. A document written on a machine that ends its
+ * lines with two characters is an ordinary document, the editor holds it exactly
+ * as it is, and an adapter that handed the editor a span from the normalised
+ * text would put every squiggle on a file's fiftieth line fifty characters to
+ * the left. That is the kind of defect that is reported as "the underline is in
+ * the wrong place" and survives for months.
+ *
+ * So the two are mapped. The map is built only when the document is not already
+ * normalised, which is the common case and costs a comparison; when it is built
+ * it is one walk of the text.
+ */
+import { normaliseSource } from '../../core/index.js';
+
+const BYTE_ORDER_MARK = 0xfeff;
+const CARRIAGE_RETURN = 13;
+const LINE_FEED = 10;
+
+/** The document as the compiler reads it, and where each of its offsets sits. */
+export interface Document {
+  /** The normalised text: what `diagnose`, `complete` and the rest are given. */
+  readonly text: string;
+  /**
+   * The offset in the editor's own document that a normalised offset names.
+   *
+   * The identity where the document is already normalised, which is what a
+   * document written on a machine that ends its lines with one character is.
+   */
+  at(offset: number): number;
+}
+
+export function documentOf(raw: string): Document {
+  const text = normaliseSource(raw);
+  if (text === raw) return { text, at: (offset) => offset };
+
+  // One entry per normalised offset, plus the one past its end, so that the end
+  // of a span at the end of the file maps as readily as its start.
+  const map = new Array<number>(text.length + 1);
+  let normalised = 0;
+  let held = raw.charCodeAt(0) === BYTE_ORDER_MARK ? 1 : 0;
+
+  while (normalised <= text.length) {
+    map[normalised] = held;
+    if (normalised === text.length) break;
+    if (raw.charCodeAt(held) === CARRIAGE_RETURN && raw.charCodeAt(held + 1) === LINE_FEED) {
+      held += 1;
+    }
+    held += 1;
+    normalised += 1;
+  }
+
+  return {
+    text,
+    at: (offset) => map[Math.min(Math.max(offset, 0), text.length)] ?? offset,
+  };
+}
+
+/** The normalised offset an editor offset names, which is the other direction. */
+export function normalisedOffset(held: Document, raw: number): number {
+  // Walked rather than inverted, because the inverse of the map above is only
+  // needed at one position per call and building a second array to answer one
+  // question would cost a file's length for a comparison's worth of work.
+  let low = 0;
+  let high = held.text.length;
+  while (low < high) {
+    const middle = (low + high + 1) >> 1;
+    if (held.at(middle) <= raw) low = middle;
+    else high = middle - 1;
+  }
+  return low;
+}

package/src/adapters/codemirror/stream.ts

@@ -0,0 +1,87 @@
+/**
+ * Highlighting, as the editor's line at a time tokenizer.
+ *
+ * The editor asks for one line's tokens at a time and remembers nothing between
+ * lines except the state this file hands back. The tier's `highlight` takes a
+ * whole file. So the line is what is lexed here, by the real lexer, through the
+ * same `highlight` a host would call itself.
+ *
+ * **That is an equivalence rather than an approximation, and it is measured.**
+ * `tests/adapters/codemirror/stream.test.ts` puts every script in the repository
+ * and the malformed corpus through both routes and compares them piece for
+ * piece: 4279 lines of real scripts, and not one of them differs. It holds
+ * because nothing in this language crosses a line ending: a string literal is
+ * refused at the end of its line, there is no block comment, and the layout
+ * tokens that do depend on the lines above are painted as whitespace either way.
+ * If the language ever grows something that does cross one, that test fails on
+ * the day it is added rather than on the day somebody notices a half painted
+ * line, and this becomes a line the narrowings file has to carry.
+ *
+ * A line is lexed once, not once per token: the pieces are computed when the
+ * editor arrives at the start of a line and walked from there. Without that, a
+ * line of twenty tokens would be lexed twenty times, which is the shape that
+ * makes a tokenizer show up in a frame budget.
+ */
+import { highlight } from '../../editor/index.js';
+import type { Highlight } from '../../editor/index.js';
+import type { EditorStreamParser, EditorStringStream } from './contract.js';
+import { HIGHLIGHT_TOKENS } from './tokens.js';
+
+/** What is carried from one call to the next: the line, and its pieces. */
+export interface StreamState {
+  /** The line the pieces were computed from, so a changed line is redone. */
+  line: string;
+  pieces: readonly Highlight[];
+}
+
+/** The pieces of one line, computed once and kept until the line changes. */
+function piecesFor(state: StreamState, line: string): readonly Highlight[] {
+  if (state.line !== line || state.pieces.length === 0) {
+    state.line = line;
+    state.pieces = highlight(line);
+  }
+  return state.pieces;
+}
+
+/**
+ * The tokenizer, which a host turns into a language with one call.
+ *
+ * ```ts
+ * const openscript = StreamLanguage.define(openscriptStream);
+ * ```
+ *
+ * `token` moves the stream to the end of the piece that starts where it is and
+ * returns that piece's style. A stream that has somehow been left inside a piece,
+ * which the editor does when it re-tokenizes from a column rather than from the
+ * start of a line, is moved to the end of the piece containing it, so the
+ * tokenizer always advances and the editor cannot be made to loop.
+ */
+export const openscriptStream: EditorStreamParser<StreamState> = {
+  name: 'openscript',
+
+  startState: (): StreamState => ({ line: '', pieces: [] }),
+
+  copyState: (state: StreamState): StreamState => ({ line: state.line, pieces: state.pieces }),
+
+  token(stream: EditorStringStream, state: StreamState): string | null {
+    if (stream.eol()) return null;
+
+    const pieces = piecesFor(state, stream.string);
+    const at = stream.pos;
+    const piece = pieces.find(
+      (one) => at >= one.span.offset && at < one.span.offset + one.span.length,
+    );
+
+    if (piece === undefined) {
+      // No piece covers the position, which cannot happen for a line the tier
+      // has covered and is still answered rather than trusted: the pieces tile
+      // the line, so this is a position outside it. Moving to the end leaves the
+      // editor with an unstyled remainder instead of an infinite loop.
+      stream.pos = stream.string.length;
+      return null;
+    }
+
+    stream.pos = piece.span.offset + piece.span.length;
+    return HIGHLIGHT_TOKENS[piece.kind];
+  },
+};

package/src/adapters/codemirror/tokens.ts

@@ -0,0 +1,64 @@
+/**
+ * The eleven kinds, as the editor's own highlighting tags.
+ *
+ * This is the translation an adapter exists to do: the language has eleven
+ * kinds, closed, and the editor has a vocabulary of tags its themes are written
+ * against. Neither knows about the other, and this file is the only place that
+ * knows both.
+ *
+ * **It is one record rather than a switch on purpose.** A style name is the one
+ * thing in this adapter that no type can hold: the editor looks a name up in its
+ * own tag table at run time and warns on a console about one it does not know,
+ * so a name that is wrong here is invisible to every check in this repository
+ * and shows up as an unpainted keyword in somebody's product. A host that
+ * disagrees with any of these, or whose theme is written against different
+ * names, replaces the row in one line:
+ *
+ * ```ts
+ * const tokens = { ...HIGHLIGHT_TOKENS, unknown: "invalid" };
+ * ```
+ *
+ * Two of the eleven are deliberately unstyled. `whitespace` has nothing to
+ * paint. `unknown` is the honest name for source the lexer took no token from,
+ * which is a character the language does not have, a region somebody wrote as a
+ * block comment, or a line continuation backslash: the first is a mistake and
+ * the last is not, so painting all three as invalid would paint a correct line
+ * red. `diagnose` is what says which of them is wrong, and the linter below is
+ * how a host shows that.
+ */
+import type { HighlightKind } from '../../editor/index.js';
+
+/**
+ * A style name per kind, or null for one the editor should leave alone.
+ *
+ * `builtin` uses the editor's modifier syntax for a name the environment
+ * supplies rather than the file, which is what a library name is.
+ */
+export const HIGHLIGHT_TOKENS: Readonly<Record<HighlightKind, string | null>> = {
+  keyword: 'keyword',
+  builtin: 'variableName.standard',
+  name: 'variableName',
+  number: 'number',
+  string: 'string',
+  color: 'color',
+  comment: 'comment',
+  operator: 'operator',
+  punctuation: 'punctuation',
+  whitespace: null,
+  unknown: null,
+};
+
+/**
+ * The editor's name for the icon beside a completion row, per kind.
+ *
+ * The editor ships icons for a fixed set of names and shows none for a name it
+ * does not know, which is a missing icon rather than a missing row.
+ */
+export const COMPLETION_TYPES: Readonly<Record<string, string>> = {
+  function: 'function',
+  value: 'constant',
+  namespace: 'namespace',
+  member: 'property',
+  variable: 'variable',
+  argument: 'property',
+};

package/src/adapters/codemirror/tooltips.ts

@@ -0,0 +1,113 @@
+/**
+ * Hover and signature, as the editor's tooltips.
+ *
+ * This is the one part of the adapter that would put something on screen, and it
+ * is the one part that does not: the markup is a parameter, and there is no
+ * default for it. So nothing in this package, at any tier, names a browser
+ * global, and `scripts/check-layering.mjs` fails the build if anything under
+ * `src` does. The same files load in a worker, on a server and inside somebody
+ * else's application.
+ *
+ * It costs a host four lines and it is the whole point of the split. Every
+ * platform has a design system and none of them wants to fight a styled panel
+ * that arrived with a language package; a default renderer here would be that
+ * panel, and it would be the one thing in this adapter a host could not replace
+ * without replacing the adapter.
+ *
+ * What is not a parameter is the content. `hoverLines` and `signatureLines`
+ * below are what a tooltip says, in order, and every line of it came from the
+ * compiler: the signatures from the library manifest, the sentence and the
+ * warmup from the specification's own tables, the type from the checker, and the
+ * refusal from the error catalogue.
+ *
+ * ```ts
+ * hoverTooltip(openscriptHoverTooltip((held) => ({ dom: yourPanel(hoverLines(held)) })))
+ * ```
+ */
+import { hover, signature } from '../../editor/index.js';
+import type { Hover, SignatureHelp } from '../../editor/index.js';
+import type { EditorState, EditorTooltip, EditorTooltipView, EditorView } from './contract.js';
+import { documentOf, normalisedOffset } from './positions.js';
+
+/** What a host supplies to draw a tooltip, which is the whole of the markup. */
+export interface Renderer<Held> {
+  (held: Held): EditorTooltipView;
+}
+
+/**
+ * The lines a hover shows, in the order it shows them.
+ *
+ * Every one of them came from the compiler: the signatures from the library
+ * manifest, the sentence and the warmup from the specification's own tables, the
+ * type from the checker, and the refusal from the error catalogue. Nothing is
+ * written here, which is why this function is a join rather than a template.
+ */
+export function hoverLines(held: Hover): readonly string[] {
+  const lines = [...held.signatures];
+  if (held.summary !== undefined) lines.push(held.summary);
+  if (held.warmup !== undefined) lines.push(`First value: ${held.warmup}`);
+  if (held.refusal !== undefined) lines.push(held.refusal);
+  if (held.kind === 'keyword') lines.push('A reserved word of the language.');
+  return lines;
+}
+
+/** The lines a signature tooltip shows, with the active parameter named. */
+export function signatureLines(held: SignatureHelp): readonly string[] {
+  const lines = [held.signature];
+  const active = held.parameters[held.active];
+  if (active !== undefined) {
+    const value = active.defaultText === undefined ? '' : ` = ${active.defaultText}`;
+    const need = active.required ? 'required' : 'optional';
+    lines.push(`${active.name}: ${active.type}${value} (${need})`);
+    if (active.values.length > 0) lines.push(`One of: ${active.values.join(', ')}`);
+  }
+  if (held.summary !== undefined) lines.push(held.summary);
+  if (held.refusal !== undefined) lines.push(held.refusal);
+  return lines;
+}
+
+/**
+ * A hover tooltip source, which a host wires in one line.
+ *
+ * ```ts
+ * hoverTooltip(openscriptHoverTooltip(render))
+ * ```
+ */
+export function openscriptHoverTooltip(
+  render: Renderer<Hover>,
+): (view: EditorView, pos: number) => EditorTooltip | null {
+  return (view, pos) => {
+    const held = documentOf(view.state.doc.toString());
+    const found = hover(held.text, normalisedOffset(held, pos));
+    if (found === undefined) return null;
+    return {
+      pos: held.at(found.span.offset),
+      end: held.at(found.span.offset + found.span.length),
+      create: () => render(found),
+    };
+  };
+}
+
+/**
+ * A signature tooltip for wherever the cursor is, or nothing where it is in no
+ * call.
+ *
+ * ```ts
+ * showTooltip.compute(["doc", "selection"], openscriptSignatureTooltip(render))
+ * ```
+ */
+export function openscriptSignatureTooltip(
+  render: Renderer<SignatureHelp>,
+): (state: EditorState) => EditorTooltip | null {
+  return (state) => {
+    const held = documentOf(state.doc.toString());
+    const at = normalisedOffset(held, state.selection.main.head);
+    const found = signature(held.text, at);
+    if (found === undefined) return null;
+    return {
+      pos: state.selection.main.head,
+      above: true,
+      create: () => render(found),
+    };
+  };
+}