openalgo-script 0.2.0 → 0.4.0

package/src/core/index.ts

@@ -170,6 +170,7 @@ export type {
   InputKind,
   LibraryEntry,
   LibraryParameter,
+  LibraryProse,
   ObjectKind,
   RequestMode,
   Storage,
@@ -192,6 +193,7 @@ export {
   atLeastBar,
   check,
   delayed,
+  describedNames,
   earlier,
   elementOf,
   isLibraryName,
@@ -205,6 +207,7 @@ export {
   membersOf,
   sameType,
   seriesOf,
+  proseFor,
   typeText,
 } from './check/index.js';
 export { REQUEST_NAMES } from './check/index.js';
@@ -223,9 +226,21 @@ export { COMPILED_FORMAT_VERSION, VERSION } from './version/index.js';
  */
 export { emit } from './emit/index.js';
 export type { EmitOptions, EmitResult } from './emit/index.js';
-export type { CompiledProgram } from './emit/index.js';
+export type { Colour, CompiledProgram } from './emit/index.js';
 export { canonicalise, programHash, sourceHash } from './emit/index.js';
 
+/**
+ * Two answers the emitter holds that a tool built on the language needs.
+ *
+ * A declaration call's defaults and the channels a colour name denotes are both
+ * facts the compiler applies and neither is in the library manifest, so a tier
+ * above this one that read the manifest alone would show a writer nothing where
+ * the compiler has a value. They are here rather than reached for behind the
+ * emitter's door, and `scripts/check-defaults.mjs` holds the first of them to
+ * what `stdlib.md` prints.
+ */
+export { DECLARATION_CALLS, declarationDefaultText, namedColour } from './emit/index.js';
+
 /**
  * The engine: a program in, one bar at a time, values out.
  *
@@ -267,3 +282,72 @@ export { isAbsent } from './engine/index.js';
 export type { Value } from './engine/index.js';
 export { LANGUAGE_VERSIONS, capabilitiesFor, verify } from './engine/index.js';
 export type { VerifyOptions, VerifyResult } from './engine/index.js';
+
+/**
+ * The money, and the run it is folded from (`stdlib.md` 17.1, 17.7).
+ *
+ * Two modules rather than one, and the line between them is the design. The
+ * accounting shapes are arithmetic over portable data and know nothing about an
+ * engine, so a stored record can be reported again with no engine present and
+ * the engine can one day call the same arithmetic with no second implementation
+ * to disagree with. The backtest shapes are what one run was carried out under
+ * and what it produced, and a record of one is a conformance case a second
+ * engine can be handed whole.
+ *
+ * They are here rather than behind an entry point of their own because they are
+ * the same layer: a host that compiles and runs a program is the host that
+ * reports what it did.
+ */
+export type {
+  BarMark,
+  ChargeBase,
+  ChargeBreakdown,
+  ChargeLine,
+  ChargeSchedule,
+  ChargeSide,
+  Contract,
+  EquityPoint,
+  Money,
+  MonthlyReturn,
+  RecordedFill,
+  Report,
+  Summary,
+  Trade,
+  TradeMarker,
+} from './accounting/index.js';
+export { chargeFor, scheduleFromDeclaration, scheduleProblem } from './accounting/index.js';
+export { markersOf, monthlyOver, reportOf } from './accounting/index.js';
+export { tradesOf } from './accounting/index.js';
+export { equityOver, summaryOf } from './accounting/index.js';
+export {
+  DEFAULT_FILL,
+  EXACT,
+  WHOLE_RANGE,
+  RECORD_VERSION,
+  backtest,
+  barsHash,
+  checkSettings,
+  compareRuns,
+  recordFromJson,
+  recordToJson,
+  recordOf,
+  replay,
+  rerun,
+  runBytes,
+  settingsFor,
+  windowFor,
+} from './backtest/index.js';
+export type { BacktestResult, ReplayResult, ReportWindow } from './backtest/index.js';
+export type {
+  BacktestSettings,
+  BarsInRecord,
+  DateRange,
+  FillPolicy,
+  RecordedBar,
+  RecordedDiagnostic,
+  RecordedFrame,
+  RecordedOrder,
+  RunComparison,
+  RunRecord,
+  Tolerance,
+} from './backtest/index.js';

package/src/core/version/version.generated.ts

@@ -4,7 +4,7 @@
 // spec/compiled-program.md, so neither can drift from the thing it names.
 
 /** This build's package version. The release refuses to publish unless the tag, the manifest and this agree. */
-export const VERSION = "0.2.0";
+export const VERSION = "0.4.0";
 
 /**
  * The compiled program format this compiler emits.

package/src/editor/complete.ts

@@ -0,0 +1,266 @@
+/**
+ * `complete`: what may be written at a position.
+ *
+ * Four kinds of answer, and every one of them is read out of something the
+ * compiler already holds:
+ *
+ *     the members of a namespace   `membersOf`, after `draw.`
+ *     the named arguments of the   the manifest's parameters for the call the
+ *     call being written           brackets say the cursor is inside
+ *     the file's own names         the checker's bindings, filtered by the two
+ *                                  scope rules of `scope.ts`
+ *     the library's names          `libraryNames`, the same index the checker
+ *                                  resolves against and the example check reads
+ *                                  its globals from
+ *
+ * **A hand written list of function names is the exact failure this project
+ * keeps removing.** It would be right on the day it was written and wrong at the
+ * next release, and nobody would report it, because a missing completion is
+ * indistinguishable from a completion that has not loaded yet. Nothing below
+ * names a single function of the language.
+ *
+ * ## A planned call is offered, and is marked as one
+ *
+ * `stdlib.md` lists calls that are named and not implemented, so that the gap in
+ * the surface is visible rather than looking like an oversight. Writing one is
+ * OS2020.
+ *
+ * Three answers were possible: leave them out, offer them as though they worked,
+ * or offer them marked. Leaving them out is the worst of the three, because the
+ * writer then types the name from the documentation, gets no completion, and
+ * learns nothing about why; offering them unmarked walks somebody into OS2020
+ * with a plot that will not compile.
+ *
+ * So they are offered, they sort after everything a script may write today, and
+ * each one carries `refusal`, which is **the catalogue's own OS2020 message
+ * filled with that name**. A host greys the row, shows the sentence, or drops
+ * the row with one field, and the sentence it shows is the same sentence the
+ * compiler would produce, because it came from the same catalogue.
+ */
+import {
+  NAMESPACES,
+  entryFor,
+  fillTemplate,
+  isNamespace,
+  libraryEntries,
+  libraryNames,
+  membersOf,
+  proseFor,
+  typeText,
+} from '../core/index.js';
+import type { Binding, LibraryEntry, Span } from '../core/index.js';
+import { entryAt, parametersOf, signatureTextOf } from './manifest.js';
+import { readChecked } from './reading.js';
+import { inScopeAt } from './scope.js';
+import { siteAt } from './site.js';
+
+/**
+ * What a completion is, which decides the icon a host puts beside it.
+ *
+ * `value` is a library name read without brackets, `close` and `aqua` among
+ * them; `function` is one that is called. The two are the manifest's own
+ * `callable`, not a guess from the spelling.
+ */
+export type CompletionKind =
+  | 'function'
+  | 'value'
+  | 'namespace'
+  | 'member'
+  | 'variable'
+  | 'argument';
+
+export interface Completion {
+  /** What the row reads as: the name, or the parameter for a named argument. */
+  readonly label: string;
+  /**
+   * What a host puts into the text, which is not always the label.
+   *
+   * A named argument is written `len = `, so accepting the row leaves the caret
+   * where the value goes rather than beside a label with no equals sign.
+   */
+  readonly insert: string;
+  readonly kind: CompletionKind;
+  /** The signature or the type, spelled by the manifest and the checker. */
+  readonly detail: string;
+  /** The line the specification gives it, where it gives one. */
+  readonly summary: string | undefined;
+  /** Named in the library and not implemented in this version: OS2020. */
+  readonly planned: boolean;
+  /** The catalogue's OS2020 message for this name, on a planned row and no other. */
+  readonly refusal: string | undefined;
+  /** The text the row replaces, which is the word being typed or nothing. */
+  readonly replace: Span;
+}
+
+/** The order a host shows them in, decided here so two hosts agree. */
+const ORDER: Readonly<Record<CompletionKind, number>> = {
+  argument: 0,
+  variable: 1,
+  function: 2,
+  member: 2,
+  value: 3,
+  namespace: 4,
+};
+
+/**
+ * The order, and the two things that decide it.
+ *
+ * Named arguments keep the order the signature states them in, because that is
+ * the order they are written in and an alphabetical list of them would put
+ * `len` before `src` in a call whose first argument is the source. Everything
+ * else is sorted: a planned call after every call a script may write today,
+ * then the kinds above, then the name.
+ *
+ * The name comparison has no locale in it. `compiled-program.md` 8.4 forbids
+ * one, and it is the same rule here for the same reason: two machines set
+ * differently would order a list differently, and the machine set differently is
+ * the one nobody is looking at.
+ */
+function ordered(completions: readonly Completion[]): readonly Completion[] {
+  const written = completions.filter((one) => one.kind === 'argument');
+  const rest = completions.filter((one) => one.kind !== 'argument');
+  rest.sort((a, b) => {
+    if (a.planned !== b.planned) return a.planned ? 1 : -1;
+    const kind = (ORDER[a.kind] ?? 0) - (ORDER[b.kind] ?? 0);
+    if (kind !== 0) return kind;
+    if (a.label === b.label) return 0;
+    return a.label < b.label ? -1 : 1;
+  });
+  return [...written, ...rest];
+}
+
+/** The OS2020 sentence, filled with the name, for a call that is not here yet. */
+function refusalFor(name: string): string {
+  return fillTemplate(entryFor('OS2020').message, { name });
+}
+
+function fromEntry(
+  entry: LibraryEntry,
+  label: string,
+  kind: CompletionKind,
+  replace: Span,
+): Completion {
+  return {
+    label,
+    insert: label,
+    kind,
+    detail: signatureTextOf(entry),
+    summary: proseFor(entry.name)?.summary,
+    planned: entry.planned,
+    refusal: entry.planned ? refusalFor(entry.name) : undefined,
+    replace,
+  };
+}
+
+/**
+ * One row per library name, taking the first signature for the detail.
+ *
+ * A name with several signatures is one row and not several: `clear` is one
+ * thing to write and two things to write it about, and a list that showed it
+ * twice would be a list about the manifest rather than about the language.
+ * `signature` is what tells a writer which one they are in once they have
+ * opened the bracket.
+ */
+function libraryRow(name: string, replace: Span): Completion | undefined {
+  const entries = libraryEntries(name);
+  const entry = entries[0];
+  if (entry === undefined) return undefined;
+  return fromEntry(entry, name, entry.callable ? 'function' : 'value', replace);
+}
+
+/** A name the file declared, with the type and the storage the checker gave it. */
+function declaredRow(binding: Binding, replace: Span): Completion {
+  const kind: CompletionKind = binding.kind === 'function' ? 'function' : 'variable';
+  const persistence = binding.persistence === 'none' ? '' : `${binding.persistence} `;
+  return {
+    label: binding.name,
+    insert: binding.name,
+    kind,
+    detail: `${persistence}${binding.name}: ${typeText(binding.type)}`,
+    summary: undefined,
+    planned: false,
+    refusal: undefined,
+    replace,
+  };
+}
+
+/**
+ * Everything that may be written at an offset, most useful first.
+ *
+ * Rows are filtered by the word already typed, which is compared exactly:
+ * `language.md` makes names case sensitive, so `EM` is not a start of `ema` and
+ * offering it would be offering a name that does not compile.
+ */
+export function complete(source: string, offset: number): readonly Completion[] {
+  const { file, tokens, checked } = readChecked(source);
+  const site = siteAt(file, tokens, offset);
+  const at = site.replace.offset;
+
+  const rows: Completion[] = [];
+
+  // After a dot, the only thing that may be written is a member of the
+  // namespace, and only where the name before the dot is one. A dotted read of
+  // anything else is not a member read at all in version 1 (`language.md` 15.2),
+  // so there is nothing to offer rather than the whole library offered wrongly.
+  if (site.afterDot) {
+    const namespace = site.namespace;
+    if (namespace !== undefined && isNamespace(namespace)) {
+      for (const member of membersOf(namespace)) {
+        const entry = libraryEntries(`${namespace}.${member}`)[0];
+        if (entry !== undefined) rows.push(fromEntry(entry, member, 'member', site.replace));
+      }
+    }
+    return ordered(rows.filter((row) => row.label.startsWith(site.word)));
+  }
+
+  // The named arguments of the call being written, where one is being written
+  // and the label of this argument has not been decided yet.
+  const call = site.call;
+  if (call !== undefined && call.label === undefined) {
+    const entry = entryAt(call.name, call.argument + 1);
+    if (entry !== undefined) {
+      for (const parameter of parametersOf(entry)) {
+        if (call.labels.includes(parameter.name)) continue;
+        rows.push({
+          label: parameter.name,
+          insert: `${parameter.name} = `,
+          kind: 'argument',
+          detail:
+            `${parameter.name}: ${parameter.type}` +
+            (parameter.defaultText === undefined ? '' : ` = ${parameter.defaultText}`),
+          summary: undefined,
+          planned: entry.planned,
+          refusal: entry.planned ? refusalFor(entry.name) : undefined,
+          replace: site.replace,
+        });
+      }
+    }
+  }
+
+  for (const binding of inScopeAt(checked, at)) {
+    rows.push(declaredRow(binding, site.replace));
+  }
+
+  for (const name of libraryNames()) {
+    // A dotted name is reached through its namespace and not written whole from
+    // nothing, so the namespace is the row and `membersOf` is the rest.
+    if (name.includes('.')) continue;
+    const row = libraryRow(name, site.replace);
+    if (row !== undefined) rows.push(row);
+  }
+
+  for (const namespace of NAMESPACES) {
+    rows.push({
+      label: namespace,
+      insert: `${namespace}.`,
+      kind: 'namespace',
+      detail: `${namespace}.`,
+      summary: undefined,
+      planned: false,
+      refusal: undefined,
+      replace: site.replace,
+    });
+  }
+
+  return ordered(rows.filter((row) => row.label.startsWith(site.word)));
+}

package/src/editor/diagnose.ts

@@ -0,0 +1,71 @@
+/**
+ * `diagnose`: source text in, everything the compiler has to say about it out.
+ *
+ * This is the one a trader sees most. It runs on a debounce while somebody
+ * types, so two things about it are decisions rather than details: what it
+ * costs, and what it does with a file that does not parse.
+ *
+ * ## It is the whole compile, on purpose
+ *
+ * Lexer, parser, checker, emitter. Not a subset, and not a faster partial
+ * compiler written for the editor, because a second implementation of the
+ * language is the thing this project refuses to have: the editor is the compiler
+ * wearing a different hat.
+ *
+ * It matters concretely as well as in principle. There are codes a script can
+ * carry that the emitter raises and nothing before it does, so an editor that
+ * stopped after the checker would show a clean file and then refuse it the
+ * moment somebody pressed apply. What a host sees here is exactly what a compile
+ * produces, so nothing new can appear at apply, and a test holds that by taking
+ * a file whose only fault the emitter is the one to find.
+ *
+ * ## What it costs
+ *
+ * A full compile of a ninety line study is a third of a millisecond, and the
+ * same file with a bracket left open mid way through, which is what a file looks
+ * like the moment somebody types one, is about a millisecond. Both are in the
+ * benchmark suite with a budget, as `diagnose-heavy` and `diagnose-typing`, so a
+ * change that makes the editor cost ten times what it costs today fails the
+ * build rather than being felt by a trader.
+ *
+ * That is why there is no partial compiler here. At this price, one is not worth
+ * the second implementation it would cost; if a script ever arrives that makes
+ * it worth one, the number to beat is in the benchmark table rather than in
+ * somebody's impression.
+ *
+ * ## A file that does not parse
+ *
+ * That is the normal state of a file being typed into, so it is the case this
+ * has to be good at rather than the case it survives. Every stage of this
+ * compiler recovers and reports instead of throwing: a lexical error costs its
+ * own character, a statement that will not parse costs its own line, a name that
+ * does not resolve becomes `unknown` and the lines around it are still checked.
+ * So a half typed file produces the diagnostics its finished lines have earned,
+ * plus the one about the line being typed, and nothing here needs a try block to
+ * make that true. `tests/editor/diagnose.test.ts` puts a corpus of malformed
+ * files through it, including text that is not the language at all.
+ */
+import { check, emit } from '../core/index.js';
+import type { Diagnostic } from '../core/index.js';
+import { readTree } from './reading.js';
+
+/**
+ * Every diagnostic a compile of this source produces, in the order a reader
+ * walks the file.
+ *
+ * Each one carries its code, its span, its message and its fix, because that is
+ * what a `Diagnostic` is: the stage that raised it supplied the code and the
+ * span, and everything a person reads came from the error catalogue. There is no
+ * editor-shaped diagnostic type here for the same reason there is no editor
+ * compiler. A host that renders one in a panel and a terminal that renders the
+ * same one under a caret are looking at the same record.
+ *
+ * The order is the bag's own: by offset, then by the span's length, then by code
+ * point of the code. It depends on nothing outside the file, so two machines
+ * list them identically.
+ */
+export function diagnose(source: string): readonly Diagnostic[] {
+  const { file, bag, script } = readTree(source);
+  emit(file, check(file, script, bag), bag, {});
+  return bag.ordered();
+}

package/src/editor/format.ts

@@ -0,0 +1,85 @@
+/**
+ * `format`: source text in, the same program in its canonical layout out.
+ *
+ * ## The rule that makes a format button safe to press
+ *
+ * **Formatting a source must not change what it means.** A trader presses this
+ * on a script that is holding a position. If a reprint can change a number, the
+ * button is a hazard whatever else it does for the diff.
+ *
+ * That is not asserted here, it is proved twice over:
+ *
+ * - **Per call, before this function returns.** The output is lexed again and
+ *   compared with the tokens that went in, token for token. If anything moved,
+ *   the source is returned untouched. The parser reads nothing but the token
+ *   stream, so an identical stream is an identical tree, and an identical tree
+ *   is an identical program. A formatter that is wrong about a rule therefore
+ *   fails by doing nothing, which is the only acceptable way for it to fail.
+ * - **Over a corpus, in the suite.** `tests/editor/format.test.ts` formats every
+ *   example in the repository and every script the phase gates run, compiles
+ *   both texts, and asserts the compiled programs are identical. That is the
+ *   test that says the formatter works; the check above is what says it is safe
+ *   when it does not.
+ *
+ * ## A source that does not parse
+ *
+ * It is returned unchanged. Not reformatted as best it can be, and not partly
+ * reformatted: unchanged, exactly, including its line endings.
+ *
+ * The reason is what a diagnostic from either stage means. The lexer reports a
+ * character it could not read and emits no token for it, so a reprint from the
+ * token stream would delete it, and a reader would watch a character disappear
+ * from their file. The parser reports a line it could not read and recovers, so
+ * the tree describes a program nobody wrote and the three questions the spacing
+ * rules ask of it have no trustworthy answers. Neither is a file to lay out. A
+ * host formats on demand rather than on every keystroke, and a file mid-edit is
+ * simply left alone until it reads.
+ */
+import type { Token } from '../core/index.js';
+import { layOut } from './layout.js';
+import { readTree } from './reading.js';
+import { scan } from './scan.js';
+import { shapesOf } from './spacing.js';
+
+/**
+ * Whether two token streams are the same stream.
+ *
+ * Kind and text for everything, except the indent token, whose text is the
+ * leading whitespace of the line it opens and is exactly what a formatter is
+ * allowed to change. It is the only token that carries layout as text: a
+ * newline, a dedent and the end of the file have none.
+ *
+ * Spans are not compared, and must not be: every one of them moves when a line
+ * is re-indented, which is the whole point of the exercise.
+ */
+function sameStream(before: readonly Token[], after: readonly Token[]): boolean {
+  if (before.length !== after.length) return false;
+  for (const [at, token] of before.entries()) {
+    const other = after[at];
+    if (other === undefined || other.kind !== token.kind) return false;
+    if (token.kind !== 'indent' && other.text !== token.text) return false;
+  }
+  return true;
+}
+
+/**
+ * The canonical layout of a source, or the source itself where it cannot be
+ * given one safely.
+ *
+ * The output always ends in a single line ending and uses LF, because a file is
+ * normalised before anything reads it (`language.md` 3.1). A host holding CRLF
+ * text gets LF back, which is the same file to the compiler and to every other
+ * part of this project.
+ */
+export function format(source: string): string {
+  const held = readTree(source);
+  if (!held.bag.isEmpty) return source;
+
+  const shapes = shapesOf(held.script, held.tokens);
+  const out = layOut(held.file, held.tokens, scan(held.file, held.tokens), shapes);
+
+  const again = readTree(out);
+  if (!again.bag.isEmpty) return source;
+  if (!sameStream(held.tokens, again.tokens)) return source;
+  return out;
+}

package/src/editor/highlight.ts

@@ -0,0 +1,110 @@
+/**
+ * `highlight`: source text in, painted spans out.
+ *
+ * It is the real lexer and nothing else. A word added to the language is
+ * coloured on the day it lexes, because the only thing between the lexer and a
+ * colour here is a table lookup in `kinds.ts` that asks the language's own
+ * tables which sort of thing a token is.
+ *
+ * ## Two shapes, and why both
+ *
+ * A host paints a document one of two ways and there is no third: it draws over
+ * the whole text, or it renders a line at a time. So both shapes are here, and
+ * the second is derived from the first rather than scanned again.
+ *
+ * `highlight` is the flat one: every piece of the file in order, covering it
+ * exactly once. It is the primitive because the covering property is a property
+ * of a flat list, and because a host holding one can produce anything else.
+ *
+ * `highlightLines` is the per-line one, and it exists because the split is where
+ * the mistake is made. A host slicing the flat list at line boundaries has to
+ * decide what to do with a piece of whitespace that runs from the end of one
+ * line to the start of the next, and getting it wrong costs a line its
+ * indentation or gives it somebody else's. Done once here, each line's pieces
+ * concatenate to exactly that line's text, newline excluded, which is what a
+ * line renderer wants and what the test asserts.
+ *
+ * ## Offsets are into the normalised text
+ *
+ * A byte order mark is dropped and CRLF becomes LF before anything else reads a
+ * file (`language.md` 3.1), so a span's offset counts into the text after those
+ * two changes, exactly as a diagnostic's does. A host that keeps its buffer
+ * exactly as the file was written normalises it once on the way in, or draws
+ * from each piece's own `text` and never indexes its buffer at all.
+ */
+import type { SourceFile, Span } from '../core/index.js';
+import { read } from './reading.js';
+import { scan } from './scan.js';
+import type { Highlight } from './scan.js';
+
+/** One line's pieces, with the line they are on so a caller counts nothing. */
+export interface HighlightedLine {
+  /** One-based, the same line a diagnostic's span carries. */
+  readonly line: number;
+  readonly pieces: readonly Highlight[];
+}
+
+/**
+ * The pieces of a file, in order, covering every character exactly once.
+ *
+ * Nothing here throws and nothing here reports. A file being typed into is
+ * malformed most of the time, and a highlighter that stopped at the first
+ * character the language does not have would leave the rest of a trader's
+ * script grey while they finished the word. `diagnose` is where the compiler
+ * says what is wrong with it.
+ */
+export function highlight(source: string): readonly Highlight[] {
+  const held = read(source);
+  return scan(held.file, held.tokens);
+}
+
+/**
+ * The same pieces, grouped by line, with every line of the file present.
+ *
+ * A piece that runs across a line ending is cut at it, and the line ending
+ * itself belongs to no line: a renderer draws the text of a line and puts the
+ * break there itself. A blank line is an entry with no pieces rather than a
+ * missing entry, so a caller can walk lines one to `lineCount` and never index
+ * past the end of a shorter list.
+ */
+export function highlightLines(source: string): readonly HighlightedLine[] {
+  const { file, tokens } = read(source);
+  const pieces = scan(file, tokens);
+  const lines: Highlight[][] = [];
+  for (let line = 0; line < file.lineCount; line += 1) lines.push([]);
+
+  for (const piece of pieces) {
+    for (const part of split(file, piece)) {
+      lines[part.span.line - 1]?.push(part);
+    }
+  }
+
+  return lines.map((held, index) => ({ line: index + 1, pieces: held }));
+}
+
+/**
+ * One piece as the pieces of the lines it covers, the line endings dropped.
+ *
+ * Only whitespace can reach here holding a line ending, since no token and no
+ * comment crosses one, but the split is written for any piece rather than for
+ * that one case: a piece shape that grows a new kind should not need this
+ * function to be revisited to stay correct.
+ */
+function split(file: SourceFile, piece: Highlight): readonly Highlight[] {
+  if (!piece.text.includes('\n')) return [piece];
+
+  const out: Highlight[] = [];
+  let from = 0;
+  for (let i = 0; i <= piece.text.length; i += 1) {
+    const ending = i === piece.text.length || piece.text.charCodeAt(i) === 10;
+    if (!ending) continue;
+    if (i > from) out.push(partOf(file, piece, from, i));
+    from = i + 1;
+  }
+  return out;
+}
+
+function partOf(file: SourceFile, piece: Highlight, from: number, to: number): Highlight {
+  const span: Span = file.spanAt(piece.span.offset + from, to - from);
+  return { kind: piece.kind, span, text: piece.text.slice(from, to) };
+}