@adecore/editor-core 0.0.1 → 0.17.0-beta.1

package/src/text-span.ts

@@ -0,0 +1,230 @@
+export interface TextSpan {
+    /* UTF-16 offsets into the text before the change. */
+    readonly start: number;
+    readonly end: number;
+    /* What stands between them after it. */
+    readonly text: string;
+}
+
+function isHighSurrogate(code: number): boolean {
+    return code >= 0xd800 && code <= 0xdbff;
+}
+
+function isLowSurrogate(code: number): boolean {
+    return code >= 0xdc00 && code <= 0xdfff;
+}
+
+/*
+ * The one stretch that differs between two texts, found from both ends. Replacing only that keeps a
+ * cursor, a selection and the scroll where they were anywhere outside it, which setting the whole
+ * text would not. Null when the two are the same.
+ */
+export function changedSpan(before: string, after: string): TextSpan | null {
+    if (before === after) {
+        return null;
+    }
+    const limit = Math.min(before.length, after.length);
+    let start = 0;
+    while (start < limit && before.charCodeAt(start) === after.charCodeAt(start)) {
+        start += 1;
+    }
+    // Never between the halves of a surrogate pair, which would leave a lone half on either side.
+    if (start > 0 && isHighSurrogate(before.charCodeAt(start - 1))) {
+        start -= 1;
+    }
+    let tail = 0;
+    while (tail < limit - start && before.charCodeAt(before.length - 1 - tail) === after.charCodeAt(after.length - 1 - tail)) {
+        tail += 1;
+    }
+    if (tail > 0 && isLowSurrogate(before.charCodeAt(before.length - tail))) {
+        tail -= 1;
+    }
+    return { start, end: before.length - tail, text: after.slice(start, after.length - tail) };
+}
+
+/* A line diff stops at this much work (snake steps and diagonals visited) and falls back to the one span. */
+const DIFF_WORK_LIMIT = 4_000_000;
+/* More edit lines than this is a rewrite, where one span is as good as any. */
+const DIFF_EDIT_LIMIT = 1_000;
+/* Past this much text between the first and last change the intern table costs more than the diff is worth. */
+const DIFF_TEXT_LIMIT = 16 * 1024 * 1024;
+
+interface LineHunk {
+    readonly beforeFrom: number;
+    readonly beforeTo: number;
+    readonly afterFrom: number;
+    readonly afterTo: number;
+}
+
+function splitLines(text: string): string[] {
+    const lines: string[] = [];
+    let from = 0;
+    for (let at = text.indexOf('\n'); at >= 0; at = text.indexOf('\n', from)) {
+        lines.push(text.slice(from, at + 1));
+        from = at + 1;
+    }
+    if (from < text.length) {
+        lines.push(text.slice(from));
+    }
+    return lines;
+}
+
+/*
+ * The shortest edit script between two line sequences (Myers), as the runs of lines that differ.
+ * Null once the work or the number of edited lines passes its limit.
+ */
+function diffLineHunks(before: Int32Array, after: Int32Array): LineHunk[] | null {
+    const rows = before.length;
+    const columns = after.length;
+    const maxEdits = Math.min(rows + columns, DIFF_EDIT_LIMIT);
+    const base = maxEdits + 1;
+    const frontier = new Int32Array(2 * base + 2);
+    const trace: Int32Array[] = [];
+    let work = 0;
+    let found = -1;
+    for (let edits = 0; edits <= maxEdits && found < 0; edits++) {
+        trace.push(frontier.slice());
+        for (let diagonal = -edits; diagonal <= edits; diagonal += 2) {
+            let x: number;
+            if (diagonal === -edits || (diagonal !== edits && frontier[base + diagonal - 1]! < frontier[base + diagonal + 1]!)) {
+                x = frontier[base + diagonal + 1]!;
+            } else {
+                x = frontier[base + diagonal - 1]! + 1;
+            }
+            let y = x - diagonal;
+            while (x < rows && y < columns && before[x] === after[y]) {
+                x++;
+                y++;
+                work++;
+            }
+            work++;
+            frontier[base + diagonal] = x;
+            if (x >= rows && y >= columns) {
+                found = edits;
+                break;
+            }
+        }
+        if (work > DIFF_WORK_LIMIT) {
+            return null;
+        }
+    }
+    if (found < 0) {
+        return null;
+    }
+    const operations: { kind: 'delete' | 'insert'; x: number; y: number }[] = [];
+    let x = rows;
+    let y = columns;
+    for (let edits = found; edits > 0; edits--) {
+        const previous = trace[edits]!;
+        const diagonal = x - y;
+        const fromAbove = diagonal === -edits || (diagonal !== edits && previous[base + diagonal - 1]! < previous[base + diagonal + 1]!);
+        const previousDiagonal = fromAbove ? diagonal + 1 : diagonal - 1;
+        const previousX = previous[base + previousDiagonal]!;
+        const previousY = previousX - previousDiagonal;
+        while (x > previousX && y > previousY) {
+            x--;
+            y--;
+        }
+        if (fromAbove) {
+            y--;
+            operations.push({ kind: 'insert', x, y });
+        } else {
+            x--;
+            operations.push({ kind: 'delete', x, y });
+        }
+    }
+    operations.reverse();
+    const hunks: LineHunk[] = [];
+    let open: { beforeFrom: number; afterFrom: number; beforeTo: number; afterTo: number } | null = null;
+    for (const operation of operations) {
+        if (open !== null && open.beforeTo === operation.x && open.afterTo === operation.y) {
+            if (operation.kind === 'delete') {
+                open.beforeTo++;
+            } else {
+                open.afterTo++;
+            }
+            continue;
+        }
+        if (open !== null) {
+            hunks.push(open);
+        }
+        open = {
+            beforeFrom: operation.x,
+            afterFrom: operation.y,
+            beforeTo: operation.x + (operation.kind === 'delete' ? 1 : 0),
+            afterTo: operation.y + (operation.kind === 'insert' ? 1 : 0)
+        };
+    }
+    if (open !== null) {
+        hunks.push(open);
+    }
+    return hunks;
+}
+
+/*
+ * The stretches that differ between two texts, found by a line diff with a bounded cost, in the
+ * coordinates of `before`. Applied as one batch they keep every cursor, fold, marker and the scroll
+ * wherever the text around them stayed, also between two far apart changes, where the one span of
+ * `changedSpan` would cover everything in between. A rewrite that passes the bounds is that one span.
+ */
+export function changedSpans(before: string, after: string): TextSpan[] {
+    const outer = changedSpan(before, after);
+    if (outer === null) {
+        return [];
+    }
+    const afterEnd = after.length - (before.length - outer.end);
+    if (outer.end - outer.start + (afterEnd - outer.start) > DIFF_TEXT_LIMIT) {
+        return [outer];
+    }
+    // Whole lines on both sides of the change, so the diff compares lines and not their halves.
+    const lineStart = outer.start === 0 ? 0 : before.lastIndexOf('\n', outer.start - 1) + 1;
+    const lineEnd = (at: string, from: number): number => {
+        const newline = at.indexOf('\n', from);
+        return newline < 0 ? at.length : newline + 1;
+    };
+    const beforeEnd = lineEnd(before, outer.end > outer.start ? outer.end - 1 : outer.end);
+    const afterLineEnd = beforeEnd - outer.end + afterEnd;
+    const beforeLines = splitLines(before.slice(lineStart, beforeEnd));
+    const afterLines = splitLines(after.slice(lineStart, afterLineEnd));
+    if (beforeLines.length < 2 && afterLines.length < 2) {
+        return [outer];
+    }
+    const ids = new Map<string, number>();
+    const intern = (lines: string[]): Int32Array => {
+        const result = new Int32Array(lines.length);
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i]!;
+            let id = ids.get(line);
+            if (id === undefined) {
+                id = ids.size;
+                ids.set(line, id);
+            }
+            result[i] = id;
+        }
+        return result;
+    };
+    const hunks = diffLineHunks(intern(beforeLines), intern(afterLines));
+    if (hunks === null || hunks.length === 0) {
+        return [outer];
+    }
+    const beforeOffsets = lineOffsets(beforeLines, lineStart);
+    const afterOffsets = lineOffsets(afterLines, lineStart);
+    const spans: TextSpan[] = [];
+    for (const hunk of hunks) {
+        const beforeFrom = beforeOffsets[hunk.beforeFrom]!;
+        const afterFrom = afterOffsets[hunk.afterFrom]!;
+        const inner = changedSpan(before.slice(beforeFrom, beforeOffsets[hunk.beforeTo]), after.slice(afterFrom, afterOffsets[hunk.afterTo]));
+        if (inner !== null) {
+            spans.push({ start: beforeFrom + inner.start, end: beforeFrom + inner.end, text: inner.text });
+        }
+    }
+    return spans;
+}
+
+function lineOffsets(lines: readonly string[], base: number): number[] {
+    const offsets = [base];
+    for (const line of lines) {
+        offsets.push(offsets[offsets.length - 1]! + line.length);
+    }
+    return offsets;
+}

package/src/types.ts

@@ -0,0 +1,141 @@
+/* Offsets are UTF-16 code units from the start of the document. */
+export interface Selection {
+    anchor: number;
+    head: number;
+}
+
+/* A replacement of `[from, to)`. A batch of edits is simultaneous, in the coordinates of the document before it. */
+export interface TextEdit {
+    from: number;
+    to: number;
+    text: string;
+}
+
+/* A replaced range in the coordinates of the document before the change, with the length that took its place. */
+export interface DocumentChange {
+    from: number;
+    to: number;
+    insertedLength: number;
+}
+
+/* Zero-based line and UTF-16 column. */
+export interface Position {
+    line: number;
+    column: number;
+}
+
+/* A replacement as a language server's `didChange` wants it: a range of the document as it stands after the edits listed before this one. */
+export interface ContentEdit {
+    start: Position;
+    end: Position;
+    text: string;
+}
+
+export interface Disposable {
+    dispose(): void;
+}
+
+export type ChangeSource = 'input' | 'command' | 'external';
+
+export interface EditorSnapshot {
+    /* Flattened on first read, so a listener that only needs the selection never pays for it. */
+    text: string;
+    selections: readonly Selection[];
+    revision: number;
+    canUndo: boolean;
+    canRedo: boolean;
+    /* One entry per transaction, present when the text changed. */
+    changes?: readonly (readonly DocumentChange[])[];
+    /* The same change in order, as a sequence a language server can follow; present when the text changed. */
+    contentEdits?: readonly ContentEdit[];
+    source?: ChangeSource;
+}
+
+export interface EditOptions {
+    /* Selections after the edit. Without them the current ones are mapped through the edits. */
+    selections?: readonly Selection[];
+    source?: ChangeSource;
+    /* Edits with the same group join one undo step until a selection change or an undo closes it. */
+    historyGroup?: string;
+}
+
+export type EditorCommand =
+    | 'undo'
+    | 'redo'
+    | 'selectAll'
+    | 'smartHome'
+    | 'smartEnd'
+    | 'selectSmartHome'
+    | 'selectSmartEnd'
+    | 'wordLeft'
+    | 'wordRight'
+    | 'selectWordLeft'
+    | 'selectWordRight'
+    | 'deleteWordLeft'
+    | 'deleteWordRight'
+    | 'camelLeft'
+    | 'camelRight'
+    | 'selectCamelLeft'
+    | 'selectCamelRight'
+    | 'deleteCamelLeft'
+    | 'deleteCamelRight'
+    | 'expandSelection'
+    | 'shrinkSelection'
+    | 'smartBackspace'
+    | 'deleteForward'
+    | 'duplicateLine'
+    | 'deleteLine'
+    | 'moveLineUp'
+    | 'moveLineDown'
+    | 'toggleLineComment'
+    | 'toggleBlockComment'
+    | 'insertTab'
+    | 'indent'
+    | 'outdent'
+    | 'insertNewline'
+    | 'startNewLine'
+    | 'startNewLineBefore'
+    | 'splitLine'
+    | 'joinLines'
+    | 'toggleCase'
+    | 'autoIndentLines'
+    | 'addCaretAbove'
+    | 'addCaretBelow'
+    | 'selectNextOccurrence'
+    | 'unselectOccurrence'
+    | 'selectAllOccurrences'
+    | 'addCaretPerSelectedLine';
+
+export interface CommandOptions {
+    /* Clamped to 1 through 16. */
+    tabSize?: number;
+    insertSpaces?: boolean;
+    /* Replaces the line comment marker of the language. */
+    commentToken?: string;
+    /* Whether the plain word commands stop at camel humps. Off unless true; the explicit camel commands always do. */
+    camelCase?: boolean;
+    /* Turns pairing off for brackets and quotes at once; the two below turn off one of them. */
+    autoClosingPairs?: boolean;
+    autoClosingBrackets?: boolean;
+    autoClosingQuotes?: boolean;
+    /* A bracket or quote typed over a selection wraps it. On unless false. */
+    surroundSelection?: boolean;
+    /* A language id such as `typescript` or `php`; it decides what counts as a comment or a string. */
+    language?: string;
+    smartSemicolon?: boolean;
+    /* Tab steps over a closer the editor inserted. On unless false. */
+    tabOutOfClosers?: boolean;
+    /* Enter computes the indentation of the new line, continues comments and closes braces. On unless false. */
+    smartEnter?: boolean;
+    /* A pasted block moves to the indentation of the line it lands on. On unless false. */
+    indentOnPaste?: boolean;
+    /* Where the row of the screen an offset is on starts and ends, so Home and End go by wrapped rows. Without it a line is one row. */
+    visualLine?: (offset: number) => { start: number; end: number };
+    /*
+     * The lines that stand on one row of the screen: a collapsed fold is a row, from its first line to its last.
+     * Line commands act on the whole row, a word move treats it as one stop and an added caret skips its inside.
+     */
+    lineSpan?: (line: number) => { first: number; last: number };
+    /* Told which lines a move went to (`from` before, `to` after), so a host can take what it keeps by line along. */
+    onLinesMoved?: (moves: readonly { from: number; to: number }[]) => void;
+}

package/src/typing-context.ts

@@ -0,0 +1,329 @@
+import {
+    hasBlockComments,
+    hasDashComments,
+    hasHashComments,
+    hashNeedsWordStart,
+    hasMarkupComments,
+    hasSlashComments,
+    isPhp,
+    isScript,
+    isWordCharacter,
+    openers,
+    startsExpression,
+    startsExpressionAfter,
+    startsWord
+} from './lexical.ts';
+import type { DocumentLine } from './rope.ts';
+
+export interface TypingBracket {
+    at: number;
+    close: string;
+    previous?: TypingBracket;
+    /* An `${` whose closer resumes the template text. */
+    template?: boolean;
+    /* The parenthesis of `if`, `for` and the like. */
+    control?: boolean;
+    /* A `{` that opens a block, as opposed to an object literal. */
+    block?: boolean;
+}
+
+/* `html` is the markup of a PHP file around its tags, and `heredoc` the body of a heredoc or nowdoc, which `quote` holds the label of. */
+type Mode = 'code' | 'line-comment' | 'block-comment' | 'html-comment' | 'quote' | 'template' | 'regex' | 'html' | 'heredoc';
+
+/* What the lexer knows at one offset. Bracket stacks are immutable, so states share them. */
+export interface TypingContext {
+    mode: Mode;
+    quote?: string;
+    escaped: boolean;
+    inClass: boolean;
+    bracket?: TypingBracket;
+    /* Whether an expression may start here, which makes a `/` a regex. */
+    expression: boolean;
+    /* The last word, or `control-close` after the parenthesis of a control statement. */
+    word: string;
+    /* Where the comment the lexer is in began, in the document. */
+    commentStart?: number;
+}
+
+interface Rules {
+    script: boolean;
+    php: boolean;
+    hash: boolean;
+    hashAtWordStart: boolean;
+    dash: boolean;
+    slash: boolean;
+    block: boolean;
+    markup: boolean;
+}
+
+const PHP_HEREDOC = /^<<<[ \t]*(["']?)([A-Za-z_]\w*)\1[ \t]*$/;
+const PHP_OPEN = /<\?(?:php\b|=)/;
+const controlWords = /^(for|for-await|if|while|switch|catch|with)$/;
+const blockWords = /^(else|try|finally|do)$/;
+
+function initial(): TypingContext {
+    return { mode: 'code', escaped: false, inClass: false, expression: true, word: '' };
+}
+
+function rulesOf(language: string): Rules {
+    return {
+        script: isScript(language),
+        php: isPhp(language),
+        hash: hasHashComments(language),
+        hashAtWordStart: hashNeedsWordStart(language),
+        dash: hasDashComments(language),
+        slash: hasSlashComments(language),
+        block: hasBlockComments(language),
+        markup: hasMarkupComments(language)
+    };
+}
+
+function scanComment(state: TypingContext, text: string, at: number, end: number): number {
+    const close = state.mode === 'block-comment' ? '*/' : '-->';
+    if (at + close.length <= end && text.startsWith(close, at)) {
+        state.mode = 'code';
+        state.commentStart = undefined;
+        return at + close.length - 1;
+    }
+    return at;
+}
+
+function scanLiteral(state: TypingContext, text: string, at: number, end: number, offset: number): number {
+    const char = text[at];
+    if (state.escaped) {
+        state.escaped = false;
+    } else if (char === '\\') {
+        state.escaped = true;
+    } else if (state.mode === 'regex') {
+        if (char === '[') {
+            state.inClass = true;
+        } else if (char === ']') {
+            state.inClass = false;
+        } else if (char === '/' && !state.inClass) {
+            state.mode = 'code';
+            state.expression = false;
+        }
+    } else if (state.mode === 'template' && char === '$' && text[at + 1] === '{' && at + 1 < end) {
+        state.bracket = { at: offset + at + 1, close: '}', previous: state.bracket, template: true };
+        state.mode = 'code';
+        state.expression = true;
+        state.word = '';
+        return at + 1;
+    } else if (char === state.quote) {
+        state.mode = 'code';
+        state.expression = false;
+        state.quote = undefined;
+    }
+    return at;
+}
+
+function scanCode(state: TypingContext, rules: Rules, text: string, at: number, end: number, offset: number): number {
+    const char = text[at];
+    const next = text[at + 1];
+
+    if (/\s/u.test(char)) {
+        return at;
+    }
+    if (rules.markup && text.startsWith('<!--', at)) {
+        state.mode = 'html-comment';
+        state.commentStart = offset + at;
+        return at + 3;
+    }
+    if (rules.hash && char === '#' && !(rules.php && next === '[') && (!rules.hashAtWordStart || startsWord(text, at))) {
+        state.mode = 'line-comment';
+        state.commentStart = offset + at;
+        return at;
+    }
+    if (rules.dash && char === '-' && next === '-') {
+        state.mode = 'line-comment';
+        state.commentStart = offset + at;
+        return at + 1;
+    }
+    if (rules.slash && char === '/' && next === '/') {
+        state.mode = 'line-comment';
+        state.commentStart = offset + at;
+        return at + 1;
+    }
+    if (rules.block && char === '/' && next === '*') {
+        state.mode = 'block-comment';
+        state.commentStart = offset + at;
+        return at + 1;
+    }
+    if (rules.php && char === '<' && text.startsWith('<<<', at)) {
+        const opener = PHP_HEREDOC.exec(text.slice(at));
+        if (opener !== null) {
+            state.mode = 'heredoc';
+            state.quote = opener[2];
+            return text.length;
+        }
+    }
+    if (rules.php && char === '?' && next === '>') {
+        state.mode = 'html';
+        return at + 1;
+    }
+    if (char === '"' || char === "'" || char === '`') {
+        state.mode = rules.script && char === '`' ? 'template' : 'quote';
+        state.quote = char;
+        state.escaped = false;
+        state.word = '';
+        return at;
+    }
+    if (rules.script && char === '/' && state.expression) {
+        state.mode = 'regex';
+        state.inClass = false;
+        state.escaped = false;
+        return at;
+    }
+    if ((char === '+' || char === '-') && next === char) {
+        state.word = '';
+        return at + 1;
+    }
+    if (openers[char]) {
+        state.bracket = {
+            at: offset + at,
+            close: openers[char],
+            previous: state.bracket,
+            control: char === '(' && controlWords.test(state.word),
+            block: char === '{' && (!state.expression || state.word === 'control-close' || blockWords.test(state.word))
+        };
+        state.expression = true;
+        state.word = '';
+    } else if (/[)\]}]/.test(char)) {
+        closeBracket(state, char);
+    } else if (isWordCharacter(char)) {
+        const from = at;
+        while (at + 1 < end && isWordCharacter(text[at + 1])) {
+            at++;
+        }
+        const word = text.slice(from, at + 1);
+        state.word = word === 'await' && state.word === 'for' ? 'for-await' : word;
+        state.expression = startsExpression(word);
+    } else {
+        state.expression = startsExpressionAfter(char);
+        state.word = '';
+    }
+    return at;
+}
+
+function closeBracket(state: TypingContext, char: string): void {
+    const bracket = state.bracket;
+    if (bracket?.close !== char) {
+        state.bracket = undefined;
+        state.expression = false;
+        return;
+    }
+    state.bracket = bracket.previous;
+    if (bracket.template) {
+        state.mode = 'template';
+        state.quote = '`';
+    }
+    state.expression = Boolean(bracket.control || bracket.block);
+    state.word = bracket.control ? 'control-close' : '';
+}
+
+/* Lexes `text` up to `end`, from the state the line started in. `offset` is where the line starts in the document. */
+function scan(text: string, end: number, start: TypingContext, rules: Rules, offset: number): TypingContext {
+    const state = { ...start };
+    for (let at = 0; at < end; at++) {
+        if (state.mode === 'line-comment') {
+            // A PHP line comment ends at its line break and at a closing tag.
+            const tag = rules.php ? text.indexOf('?>', at) : -1;
+            if (tag < 0 || tag >= end) {
+                break;
+            }
+            state.mode = 'html';
+            state.commentStart = undefined;
+            at = tag + 1;
+        } else if (state.mode === 'html') {
+            const open = PHP_OPEN.exec(text.slice(at));
+            if (open === null || at + open.index >= end) {
+                break;
+            }
+            state.mode = 'code';
+            state.expression = true;
+            state.word = '';
+            at += open.index + open[0].length - 1;
+        } else if (state.mode === 'heredoc') {
+            const closing = new RegExp(`^[\\t ]*${state.quote}(?![\\w])`).exec(text);
+            if (closing === null) {
+                break;
+            }
+            state.mode = 'code';
+            state.quote = undefined;
+            state.expression = false;
+            at = closing[0].length - 1;
+        } else if (state.mode === 'block-comment' || state.mode === 'html-comment') {
+            at = scanComment(state, text, at, end);
+        } else if (state.mode === 'code') {
+            at = scanCode(state, rules, text, at, end, offset);
+        } else {
+            at = scanLiteral(state, text, at, end, offset);
+        }
+    }
+    return state;
+}
+
+function sameState(left: TypingContext, right: TypingContext): boolean {
+    return (
+        left.mode === right.mode &&
+        left.quote === right.quote &&
+        left.escaped === right.escaped &&
+        left.inClass === right.inClass &&
+        left.bracket === right.bracket &&
+        left.expression === right.expression &&
+        left.word === right.word &&
+        left.commentStart === right.commentStart
+    );
+}
+
+/* The state of the lexer at the start of each line, computed on demand. An edit drops the lines from its own on. */
+export class TypingContexts {
+    private readonly line: (index: number) => DocumentLine;
+    private states: TypingContext[] = [initial()];
+    private language = '';
+
+    constructor(line: (index: number) => DocumentLine) {
+        this.line = line;
+    }
+
+    invalidate(line = 0): void {
+        this.states.length = Math.min(this.states.length, Math.max(1, line + 1));
+        if (line === 0) {
+            this.language = '';
+        }
+    }
+
+    /* A PHP file that opens with markup, and not with a tag, starts outside the PHP. */
+    private initialState(language: string): TypingContext {
+        const state = initial();
+        if (language === 'php' && /^\s*<(?![?<])/.test(this.line(0).text)) {
+            state.mode = 'html';
+        }
+        return state;
+    }
+
+    at(line: number, column: number, language: string): TypingContext {
+        const id = language.toLowerCase();
+        if (this.language !== id) {
+            this.language = id;
+            this.states = [this.initialState(id)];
+        }
+        const rules = rulesOf(id);
+        while (this.states.length <= line) {
+            const previous = this.states.at(-1)!;
+            const source = this.line(this.states.length - 1);
+            const next = scan(source.text, source.text.length, previous, rules, source.start);
+            // Only PHP and Rust strings run past a line break without a trailing backslash.
+            if (next.mode === 'line-comment' || next.mode === 'regex' || (next.mode === 'quote' && !/^(php|rust|rs)$/i.test(id) && !next.escaped)) {
+                next.mode = 'code';
+                next.quote = undefined;
+                next.commentStart = undefined;
+                next.expression = true;
+            }
+            next.escaped = false;
+            this.states.push(sameState(next, previous) ? previous : next);
+        }
+        const source = this.line(line);
+        return scan(source.text, column, this.states[line], rules, source.start);
+    }
+}