@trebco/treb 36.1.4 → 37.0.1

package/build/treb-parser/src/parser.js

@@ -0,0 +1,2418 @@
+/*
+ * This file is part of TREB.
+ *
+ * TREB is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * TREB is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with TREB. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Copyright 2022-2026 trebco, llc.
+ * info@treb.app
+ *
+ */
+import { ArgumentSeparatorType, DecimalMarkType, DefaultParserConfig } from './parser-types';
+/**
+ * regex determines if a sheet name requires quotes. centralizing
+ * this to simplify maintenance and reduce overlap/errors
+ */
+export const QuotedSheetNameRegex = /[\s-+=<>!()]/;
+/**
+ * similarly, illegal sheet name. we don't actually handle this in
+ * the parser, but it seems like a reasonable place to keep this
+ * definition.
+ */
+export const IllegalSheetNameRegex = /['*\\]/;
+const DOUBLE_QUOTE = 0x22; // '"'.charCodeAt(0);
+const SINGLE_QUOTE = 0x27; // `'`.charCodeAt(0);
+const NON_BREAKING_SPACE = 0xa0;
+const SPACE = 0x20;
+const TAB = 0x09;
+const CR = 0x0a;
+const LF = 0x0d;
+const ZERO = 0x30;
+const NINE = 0x39;
+const PERIOD = 0x2e;
+const PLUS = 0x2b;
+const MINUS = 0x2d;
+const OPEN_PAREN = 0x28;
+const CLOSE_PAREN = 0x29;
+const COMMA = 0x2c;
+const PERCENT = 0x25;
+const UNDERSCORE = 0x5f;
+const DOLLAR_SIGN = 0x24;
+const OPEN_BRACE = 0x7b;
+const CLOSE_BRACE = 0x7d;
+const OPEN_SQUARE_BRACKET = 0x5b;
+const CLOSE_SQUARE_BRACKET = 0x5d;
+const QUESTION_MARK = 0x3f;
+const EXCLAMATION_MARK = 0x21;
+// const COLON = 0x3a; // became an operator
+const SEMICOLON = 0x3b;
+const HASH = 0x23; // #
+const AT = 0x40; // @
+const UC_A = 0x41;
+const LC_A = 0x61;
+const UC_E = 0x45;
+const LC_E = 0x65;
+const UC_Z = 0x5a;
+const LC_Z = 0x7a;
+const LC_I = 0x69;
+// const LC_J = 0x6a;
+// there are a couple of characters we don't want in this 
+// range; we should split into separate ranges. also we 
+// probably have characters we don't need (atm)
+const ACCENTED_RANGE_START = 192;
+const ACCENTED_RANGE_END = 382; // bumping up for polish // 312;
+/**
+ * precedence map
+ */
+const binary_operators_precendence = {
+    '==': 6,
+    '!=': 6, // FIXME: we should not support these (legacy)
+    '<>': 6,
+    '=': 6, // these are the appropriate equality operators for SL
+    '<': 7,
+    '>': 7,
+    '<=': 7,
+    '>=': 7,
+    '+': 9,
+    '-': 9,
+    '&': 9,
+    '*': 10,
+    '/': 10,
+    '^': 11, // highest math op
+    ':': 13, // range operator
+};
+/* *
+ * binary ops are sorted by length so we can compare long ops first
+   switching to a composite w/ unary operators
+ * /
+const binary_operators = Object.keys(binary_operators_precendence).sort(
+  (a, b) => b.length - a.length,
+);
+
+/**
+ * unary operators. atm we have no precedence issues, unary operators
+ * always have absolute precedence. (for numbers, these are properly part
+ * of the number, but consider `=-SUM(1,2)` -- this is an operator).
+ *
+ * implicit intersection operator should now have precedence over +/-.
+ */
+const unary_operators = { '@': 50, '-': 100, '+': 100 };
+/**
+ * to avoid the double - and +, we're just adding our one extra unary
+ * operator. doing this dynamically would be silly, although this does
+ * make this code more fragile.
+ */
+const composite_operators = [...Object.keys(binary_operators_precendence), '@'].sort((a, b) => b.length - a.length);
+/**
+ * parser for spreadsheet language.
+ *
+ * FIXME: this is stateless, think about exporting a singleton.
+ *
+ * (there is internal state, but it's only used during a Parse() call,
+ * which runs synchronously). one benefit of using a singleton would be
+ * consistency in decimal mark, we'd only have to set once.
+ *
+ * FIXME: the internal state is starting to grate. there's no reason for
+ * it and it just confuses things because parsing is stateless (except for
+ * configuration). internal state just keeps results from the last parse
+ * operation. we should refactor so parsing is clean and returns all
+ * results directly, caller can store if necessary.
+ *
+ * FIXME: split rendering into a separate class? would be a little cleaner.
+ *
+ * FIXME: we don't currently handle full-width punctuation. it would be
+ * simple to parse, a little more complicated to keep track of if we wanted
+ * to be able to rewrite. TODO/FIXME.
+ *
+ */
+export class Parser {
+    /**
+     * accessor replacing old field. the actual value is moved to flags,
+     * and should be set via the SetLocaleSettings method.
+     */
+    get argument_separator() {
+        return this.flags.argument_separator;
+    }
+    /**
+     * accessor replacing old field. the actual value is moved to flags,
+     * and should be set via the SetLocaleSettings method.
+     */
+    get decimal_mark() {
+        return this.flags.decimal_mark;
+    }
+    /**
+     * unifying flags
+     */
+    flags = {
+        ...DefaultParserConfig,
+    };
+    /**
+     * FIXME: why is this a class member? at a minimum it could be static
+     * FIXME: why are we doing this with a regex?
+     */
+    r1c1_regex = /[rR]((?:\[[-+]{0,1}\d+\]|\d*))[cC]((?:\[[-+]{0,1}\d+\]|\d*))$/;
+    /**
+     * internal argument separator, as a number. this is set internally on
+     * parse call, following the argument_separator value.
+     */
+    argument_separator_char = COMMA;
+    /**
+     * internal decimal mark, as a number.
+     */
+    decimal_mark_char = PERIOD;
+    /**
+     * imaginary number value. this is "i", except for those EE weirdos who
+     * use "j". although I guess those guys put it in front, so it won't really
+     * work anyway... let's stick with "i" for now.
+     */
+    imaginary_char = LC_I;
+    /**
+     * imaginary number as text for matching
+     */
+    imaginary_number = 'i';
+    /**
+     * internal counter for incrementing IDs
+     */
+    id_counter = 0;
+    expression = '';
+    data = [];
+    index = 0;
+    length = 0;
+    /** success flag */
+    valid = true;
+    /** rolling error state */
+    error_position;
+    /** rolling error state */
+    error;
+    dependencies = {
+        addresses: {},
+        ranges: {},
+    };
+    // referenced addresses -- used to merge ranges/addresses, although I'm
+    // not sure that's actually all that useful
+    address_refcount = {};
+    /**
+     * full list of referenced addresses and ranges. we're adding this
+     * to support highlighting, for which we need multiple instances
+     * of a single address. the original dep list was used for graph dependencies,
+     * so we compressed the list.
+     *
+     * FIXME: use a single list, i.e. something like
+     *
+     *   address -> [instance, instance]
+     *
+     * because that's a big API change it's going to have to wait. for now,
+     * use a second list.
+     *
+     * UPDATE: adding (otherwise unused) tokens, which could be named ranges.
+     * in the future we may pass in a list of names at parse time, and resolve
+     * them; for now we are just listing names.
+     */
+    full_reference_list = [];
+    /**
+     * cache for storing/restoring parser state, if we toggle it
+     */
+    parser_state_cache = [];
+    /**
+     * step towards protecting these values and setting them in one
+     * operation.
+     *
+     * UPDATE: switch order. argument separator is optional and implied.
+     */
+    SetLocaleSettings(decimal_mark, argument_separator) {
+        if (typeof argument_separator === 'undefined') {
+            argument_separator = (decimal_mark === DecimalMarkType.Comma) ?
+                ArgumentSeparatorType.Semicolon :
+                ArgumentSeparatorType.Comma;
+        }
+        // I suppose semicolon and period is allowable, although no one 
+        // uses it. this test only works because we know the internal type
+        // representation, but that's fragile and not a good idea. FIXME
+        if (argument_separator === decimal_mark) {
+            throw new Error('invalid locale setting');
+        }
+        this.flags.argument_separator = argument_separator;
+        this.flags.decimal_mark = decimal_mark;
+    }
+    /**
+     * save local configuration to a buffer, so it can be restored. we're doing
+     * this because in a lot of places we're caching parser flagss, changing
+     * them, and then restoring them. that's become repetitive, fragile to
+     * changes or new flags, and annoying.
+     *
+     * config is managed in a list with push/pop semantics. we store it as
+     * JSON so there's no possibility we'll accidentally mutate.
+     *
+     * FIXME: while we're at it why not migrate the separators -> flags, so
+     * there's a single location for this kind of state? (...TODO)
+     *
+     */
+    Save() {
+        this.parser_state_cache.push(JSON.stringify(this.flags));
+    }
+    /**
+     * restore persisted config
+     * @see Save
+     */
+    Restore() {
+        const json = this.parser_state_cache.shift();
+        if (json) {
+            try {
+                this.flags = JSON.parse(json);
+            }
+            catch (err) {
+                console.error(err);
+            }
+        }
+        else {
+            console.warn("No parser state to restore");
+        }
+    }
+    /**
+     * recursive tree walk that allows substitution. this should be
+     * a drop-in replacement for the original Walk function but I'm
+     * keeping it separate temporarily just in case it breaks something.
+     *
+     * @param func - in this version function can return `true` (continue
+     * walking subtree), `false` (don't walk subtree), or an ExpressionUnit.
+     * in the last case, we'll replace the original unit with the substitution.
+     * obviously in that case we don't recurse.
+     */
+    Walk2(unit, func) {
+        const result = func(unit);
+        if (typeof result === 'object') {
+            return result;
+        }
+        switch (unit.type) {
+            case 'address':
+            case 'missing':
+            case 'literal':
+            case 'complex':
+            case 'identifier':
+            case 'operator':
+            case 'structured-reference':
+                break;
+            case 'dimensioned':
+                if (result) {
+                    unit.expression = this.Walk2(unit.expression, func); // could be an issue
+                    unit.unit = this.Walk2(unit.unit, func); // could be an issue
+                }
+                break;
+            case 'range':
+                if (func(unit)) {
+                    unit.start = this.Walk2(unit.start, func); // could be an issue
+                    unit.end = this.Walk2(unit.end, func); // could be an issue
+                }
+                break;
+            case 'binary':
+                if (func(unit)) {
+                    unit.left = this.Walk2(unit.left, func);
+                    unit.right = this.Walk2(unit.right, func);
+                }
+                break;
+            case 'unary':
+                if (func(unit)) {
+                    unit.operand = this.Walk2(unit.operand, func);
+                }
+                break;
+            case 'group':
+                if (func(unit)) {
+                    unit.elements = unit.elements.map(source => this.Walk2(source, func));
+                }
+                break;
+            case 'implicit-call':
+                if (func(unit)) {
+                    unit.call = this.Walk2(unit.call, func);
+                    unit.args = unit.args.map(source => this.Walk2(source, func));
+                }
+                break;
+            case 'call':
+                if (func(unit)) {
+                    unit.args = unit.args.map(source => this.Walk2(source, func));
+                }
+                break;
+        }
+        return unit;
+    }
+    /**
+     * recursive tree walk.
+     *
+     * @param func function called on each node. for nodes that have children
+     * (operations, calls, groups) return false to skip the subtree, or true to
+     * traverse.
+     */
+    Walk(unit, func) {
+        switch (unit.type) {
+            case 'address':
+            case 'missing':
+            case 'literal':
+            case 'complex':
+            case 'identifier':
+            case 'operator':
+            case 'structured-reference':
+                func(unit);
+                return;
+            case 'dimensioned':
+                if (func(unit)) {
+                    this.Walk(unit.expression, func);
+                    this.Walk(unit.unit, func);
+                }
+                return;
+            case 'range':
+                if (func(unit)) {
+                    this.Walk(unit.start, func);
+                    this.Walk(unit.end, func);
+                }
+                return;
+            case 'binary':
+                if (func(unit)) {
+                    this.Walk(unit.left, func);
+                    this.Walk(unit.right, func);
+                }
+                return;
+            case 'unary':
+                if (func(unit)) {
+                    this.Walk(unit.operand, func);
+                }
+                return;
+            case 'group':
+                if (func(unit)) {
+                    // unit.elements.forEach((element) => this.Walk(element, func));
+                    for (const element of unit.elements) {
+                        this.Walk(element, func);
+                    }
+                }
+                return;
+            case 'implicit-call':
+                if (func(unit)) {
+                    this.Walk(unit.call, func);
+                    for (const arg of unit.args) {
+                        this.Walk(arg, func);
+                    }
+                }
+                return;
+            case 'call':
+                if (func(unit)) {
+                    for (const arg of unit.args) {
+                        this.Walk(arg, func);
+                    }
+                    // unit.args.forEach((arg) => this.Walk(arg, func));
+                }
+        }
+    }
+    /** utility: transpose array */
+    Transpose(arr) {
+        const m = arr.length;
+        const transposed = [];
+        let n = 0;
+        for (let i = 0; i < m; i++) {
+            if (Array.isArray(arr[i])) {
+                n = Math.max(n, arr[i].length);
+            }
+        }
+        for (let i = 0; i < n; i++) {
+            transposed[i] = [];
+            for (let j = 0; j < m; j++) {
+                transposed[i][j] = arr[j] ? arr[j][i] : undefined;
+            }
+        }
+        return transposed;
+    }
+    /**
+     * renders the passed expression as a string.
+     * @param unit base expression
+     * @param offset offset for addresses, used to offset relative addresses
+     * (and ranges). this is for copy-and-paste or move operations.
+     * @param missing string to represent missing values (can be '', for functions)
+     *
+     * FIXME: we're accumulating too many arguments. need to switch to an
+     * options object. do that after the structured reference stuff merges.
+     *
+     */
+    Render(unit, options = {}) {
+        // defaults
+        const offset = options.offset || { rows: 0, columns: 0 };
+        const missing = options.missing ?? '(missing)';
+        // the rest are optional
+        /*
+          offset: { rows: number; columns: number } = { rows: 0, columns: 0 },
+          missing = '(missing)',
+          convert_decimal?: DecimalMarkType,
+          convert_argument_separator?: ArgumentSeparatorType,
+          convert_imaginary_number?: 'i'|'j',
+          long_structured_references?: boolean,
+          table_name?: string,
+      
+        ): string {
+          */
+        const { convert_decimal, convert_argument_separator, 
+        // convert_imaginary_number, 
+        long_structured_references, table_name, } = options;
+        // use default separator, unless we're explicitly converting.
+        let separator = this.flags.argument_separator + ' ';
+        if (convert_argument_separator === ArgumentSeparatorType.Comma) {
+            separator = ', ';
+        }
+        else if (convert_argument_separator === ArgumentSeparatorType.Semicolon) {
+            separator = '; ';
+        }
+        /*
+        let imaginary_character = this.imaginary_number;
+        if (convert_imaginary_number) {
+          imaginary_character = convert_imaginary_number;
+        }
+        */
+        // this is only used if we're converting.
+        const decimal = convert_decimal === DecimalMarkType.Comma ? ',' : '.';
+        const decimal_rex = this.flags.decimal_mark === DecimalMarkType.Comma ? /,/ : /\./;
+        // we need this for complex numbers, but I don't want to change the 
+        // original at the moment, just in case. we can run through that later.
+        const decimal_rex_g = this.flags.decimal_mark === DecimalMarkType.Comma ? /,/g : /\./g;
+        switch (unit.type) {
+            case 'address':
+                if (options.pass_through_addresses) {
+                    return unit.label;
+                }
+                return options.r1c1 ? this.R1C1Label(unit, options) : this.AddressLabel(unit, offset);
+            case 'range':
+                if (options.pass_through_addresses) {
+                    return unit.label;
+                }
+                return options.r1c1 ?
+                    this.R1C1Label(unit.start, options) + ':' +
+                        this.R1C1Label(unit.end, options) :
+                    this.AddressLabel(unit.start, offset) + ':' + this.AddressLabel(unit.end, offset);
+            case 'missing':
+                return missing;
+            case 'array':
+                // we have to transpose because we're column-major but the 
+                // format is row-major
+                return '{' +
+                    this.Transpose(unit.values).map((row) => row.map((value) => {
+                        if (typeof value === 'string') {
+                            return '"' + value + '"';
+                        }
+                        return value;
+                    }).join(', ')).join('; ') + '}';
+            case 'binary':
+                // in some cases we might see range constructs as binary units
+                // because one side (or maybe both sides) of the range is a 
+                // function. in that case we don't want a space in front of the 
+                // operator.
+                // UPDATE: for aesthetic reasons, also remove spaces around a
+                // power operator (caret, "^")
+                // FIXME: parameterize?
+                {
+                    const separator = ((unit.operator === ':' || unit.operator === '^') ? '' : ' ');
+                    return (this.Render(unit.left, options) +
+                        separator +
+                        unit.operator +
+                        separator +
+                        this.Render(unit.right, options));
+                }
+            case 'unary':
+                return (unit.operator +
+                    this.Render(unit.operand, options));
+            case 'complex':
+                // formatting complex value (note for searching)
+                // this uses small regular "i"
+                // as with literals, we want to preserve the original text,
+                // which might have slight precision differences from what
+                // we would render.
+                if (unit.text) {
+                    if (convert_decimal) {
+                        // we don't support grouping numbers for complex, so there's
+                        // no need to handle grouping
+                        const text = unit.text;
+                        return text.replace(decimal_rex_g, decimal);
+                    }
+                    else {
+                        return unit.text;
+                    }
+                }
+                else {
+                    // if we don't have the original text for whatever reason, format 
+                    // and convert if necessary.
+                    let imaginary_text = Math.abs(unit.imaginary).toString();
+                    if (convert_decimal === DecimalMarkType.Comma || this.flags.decimal_mark === DecimalMarkType.Comma) {
+                        imaginary_text = imaginary_text.replace(/\./, ',');
+                    }
+                    if (unit.real) {
+                        let real_text = unit.real.toString();
+                        if (convert_decimal === DecimalMarkType.Comma || this.flags.decimal_mark === DecimalMarkType.Comma) {
+                            real_text = real_text.replace(/\./, ',');
+                        }
+                        const i = Math.abs(unit.imaginary);
+                        return `${real_text}${unit.imaginary < 0 ? ' - ' : ' + '}${i === 1 ? '' : imaginary_text}i`;
+                    }
+                    else if (unit.imaginary === -1) {
+                        return `-i`;
+                    }
+                    else if (unit.imaginary === 1) {
+                        return `i`;
+                    }
+                    else {
+                        return `${unit.imaginary < 0 ? '-' : ''}${imaginary_text}i`;
+                    }
+                }
+                break;
+            case 'literal':
+                if (typeof unit.value === 'string') {
+                    // escape any quotation marks in string
+                    return '"' + unit.value.replace(/"/g, '""') + '"';
+                }
+                else if (typeof unit.value === 'boolean') {
+                    // use render option (replacement) value; then flags value; then a default
+                    if (unit.value) {
+                        return options.boolean_true || this.flags.boolean_true || 'true'; // default
+                    }
+                    else {
+                        return options.boolean_false || this.flags.boolean_false || 'false'; // default
+                    }
+                }
+                else if (convert_decimal && typeof unit.value === 'number') {
+                    if (unit.text) {
+                        // here we want to translate the literal typed-in value.
+                        // users can type in a decimal point and possibly grouping.
+                        // if we are converting from dot to comma, we need to make
+                        // sure to remove any existing commas. for the time being
+                        // we will just remove them.
+                        // what about the alternate case? in that case, we're not allowing
+                        // users to type in groupings (I think), so we can skip that part.
+                        // ACTUALLY, we don't allow grouping at all. we normalize it
+                        // if you type in a number. why? consider functions, grouping
+                        // looks like parameter separation. so no.
+                        let text = unit.text;
+                        if (convert_decimal === DecimalMarkType.Comma &&
+                            this.flags.decimal_mark === DecimalMarkType.Period) {
+                            text = text.replace(/,/g, ''); // remove grouping
+                        }
+                        return text.replace(decimal_rex, decimal);
+                    }
+                    else {
+                        // this always works because this function is guaranteed
+                        // to return value in dot-decimal format without separators.
+                        return unit.value.toString().replace(/\./, decimal);
+                    }
+                }
+                else if (unit.text)
+                    return unit.text;
+                return unit.value.toString();
+            case 'identifier':
+                return unit.name;
+            case 'operator':
+                return '[' + unit.operator + ']'; // this should be invalid output
+            case 'group':
+                if (unit.explicit) {
+                    return ('(' +
+                        unit.elements
+                            .map((x) => this.Render(x, options)).join(separator) +
+                        ')');
+                }
+                else {
+                    return unit.elements
+                        .map((x) => this.Render(x, options)).join(separator);
+                }
+            case 'implicit-call':
+                return this.Render(unit.call, options) +
+                    '(' + unit.args.map(element => this.Render(element, options)).join(separator) + ')';
+            case 'call':
+                return (unit.name +
+                    '(' +
+                    unit.args
+                        .map((x) => this.Render(x, options)).join(separator) +
+                    ')');
+            case 'dimensioned':
+                return this.Render(unit.expression) + ' ' + this.Render(unit.unit);
+            case 'structured-reference':
+                // not sure of the rules around one or two braces for the 
+                // column name... certainly spaces means you need at least one
+                {
+                    let column = unit.column;
+                    if (/[^A-Za-z]/.test(column)) {
+                        column = '[' + column + ']';
+                    }
+                    let table = unit.table;
+                    // console.info("RENDER SR", unit, table_name, long_structured_references);
+                    if (!table && long_structured_references && table_name) {
+                        table = table_name;
+                    }
+                    switch (unit.scope) {
+                        case 'all':
+                            return `${table}[[#all],${column}]`;
+                        case 'row':
+                            if (long_structured_references) {
+                                return `${table}[[#this row],${column}]`;
+                            }
+                            else {
+                                return `${table}[@${column}]`;
+                            }
+                        case 'column':
+                            return `${table}[${column}]`;
+                    }
+                    // this is here in case we add a new scope in the future,
+                    // so we remember to handle this case
+                    throw new Error('unhandled scope in structured reference');
+                }
+        }
+        return '??';
+    }
+    /**
+     * parses expression and returns the root of the parse tree, plus a
+     * list of dependencies (addresses and ranges) found in the expression.
+     *
+     * NOTE that in the new address parsing structure, we will overlap ranges
+     * and addresses (range corners). this is OK because ranges are mapped
+     * to individual address dependencies. it's just sloppy (FIXME: refcount?)
+     */
+    Parse(expression) {
+        // normalize
+        expression = expression.trim();
+        // remove leading =
+        if (expression[0] === '=') {
+            expression = expression.substr(1).trim();
+        }
+        this.expression = expression;
+        this.data = [];
+        this.length = expression.length;
+        this.index = 0;
+        this.valid = true;
+        this.error_position = undefined;
+        this.error = undefined;
+        this.dependencies.addresses = {};
+        this.dependencies.ranges = {};
+        this.address_refcount = {};
+        this.full_reference_list = [];
+        // reset ID
+        this.id_counter = 0;
+        // set separator
+        switch (this.flags.argument_separator) {
+            case ArgumentSeparatorType.Semicolon:
+                this.argument_separator_char = SEMICOLON;
+                break;
+            default:
+                this.argument_separator_char = COMMA;
+                break;
+        }
+        // and decimal mark
+        switch (this.flags.decimal_mark) {
+            case DecimalMarkType.Comma:
+                this.decimal_mark_char = COMMA;
+                break;
+            default:
+                this.decimal_mark_char = PERIOD;
+                break;
+        }
+        // NOTE on this function: charCodeAt returns UTF-16. codePointAt returns
+        // unicode. length returns UTF-16 length. any characters that are not
+        // representable as a single character in UTF-16 will be 'the first unit
+        // of a surrogate pair...' and so on.
+        //
+        // we want UTF-16, not unicode. for the parser itself, we are only really
+        // looking for ASCII, so it's not material. for anything else, if we
+        // construct strings from the original data we want to map the UTF-16,
+        // otherwise we will construct the string incorrectly. this applies to
+        // strings, function names, and anything else.
+        //
+        // which is all a long way of saying, don't be tempted to replace this
+        // with codePointAt.
+        for (let i = 0; i < this.length; i++) {
+            this.data[i] = expression.charCodeAt(i);
+        }
+        const expr = this.ParseGeneric();
+        // last pass: convert any remaining imaginary values to complex values.
+        // FIXME: could do this elsewhere? not sure we should be adding yet
+        // another loop...
+        // (moving)
+        // remove extraneous addresses
+        // NOTE: we still may have duplicates that have different absolute/relative
+        // modifiers, e.g. C3 and $C$3 (and $C3 and C$3). not sure what we should
+        // do about that, since some consumers may consider these different -- we
+        // need to establish a contract about this
+        const addresses = {};
+        for (const key of Object.keys(this.dependencies.addresses)) {
+            if (this.address_refcount[key]) {
+                addresses[key] = this.dependencies.addresses[key];
+            }
+        }
+        this.dependencies.addresses = addresses;
+        return {
+            expression: expr || undefined,
+            valid: this.valid,
+            error: this.error,
+            error_position: this.error_position,
+            dependencies: this.dependencies,
+            separator: this.flags.argument_separator,
+            decimal_mark: this.flags.decimal_mark,
+            full_reference_list: this.full_reference_list.slice(0),
+        };
+    }
+    /** generates column label ("A") from column index (0-based) */
+    ColumnLabel(column) {
+        if (column === Infinity) {
+            return '';
+        }
+        let s = String.fromCharCode(65 + (column % 26));
+        while (column > 25) {
+            column = Math.floor(column / 26) - 1;
+            s = String.fromCharCode(65 + (column % 26)) + s;
+        }
+        return s;
+    }
+    /**
+     * generates absolute or relative R1C1 address
+     *
+     * FIXME: not supporting relative (offset) addresses atm? I'd like to
+     * change this but I don't want to break anything...
+     */
+    R1C1Label(address, options) {
+        const force_relative = !!options.r1c1_force_relative;
+        const base = options.r1c1_base;
+        let label = '';
+        if (address.sheet) { // && (!base?.sheet || base?.sheet !== address.sheet)) {
+            label = (QuotedSheetNameRegex.test(address.sheet) ?
+                '\'' + address.sheet + '\'' : address.sheet) + '!';
+        }
+        let row = '';
+        let column = '';
+        if (force_relative && options.r1c1_proper_semantics && base) {
+            if (address.absolute_row) {
+                row = (address.row + 1).toString();
+            }
+            else {
+                const delta_row = address.row - base.row;
+                if (delta_row) {
+                    row = `[${delta_row}]`;
+                }
+            }
+            if (address.absolute_column) {
+                column = (address.column + 1).toString();
+            }
+            else {
+                const delta_column = address.column - base.column;
+                if (delta_column) {
+                    column = `[${delta_column}]`;
+                }
+            }
+        }
+        else if (force_relative && base) {
+            const delta_row = address.row - base.row;
+            const delta_column = address.column - base.column;
+            if (delta_row) {
+                row = `[${delta_row}]`;
+            }
+            if (delta_column) {
+                column = `[${delta_column}]`;
+            }
+        }
+        else {
+            row = address.offset_row ? `[${address.row}]` : (address.row + 1).toString();
+            column = address.offset_column ? `[${address.column}]` : (address.column + 1).toString();
+        }
+        /*
+        const row = (address.absolute_row || !base) ? (address.row + 1).toString() : `[${address.row - base.row}]`;
+        const column = (address.absolute_column || !base) ? (address.column + 1).toString() : `[${address.column - base.column}]`;
+        */
+        label += `R${row}C${column}`;
+        return label;
+    }
+    /**
+     * generates address label ("C3") from address (0-based).
+     *
+     * @param offset - offset by some number of rows or columns
+     * @param r1c1 - if set, return data in R1C1 format.
+     */
+    AddressLabel(address, offset) {
+        let column = address.column;
+        if (!address.absolute_column && address.column !== Infinity)
+            column += offset.columns;
+        let row = address.row;
+        if (!address.absolute_row && address.row !== Infinity)
+            row += offset.rows;
+        if (row < 0 || column < 0 || (row === Infinity && column === Infinity))
+            return '#REF';
+        let label = '';
+        if (address.sheet) {
+            label = (QuotedSheetNameRegex.test(address.sheet) ?
+                '\'' + address.sheet + '\'' : address.sheet) + '!';
+        }
+        if (row === Infinity) {
+            return label +
+                (address.absolute_column ? '$' : '') +
+                this.ColumnLabel(column);
+        }
+        if (column === Infinity) {
+            return label +
+                (address.absolute_row ? '$' : '') +
+                (row + 1);
+        }
+        return (label +
+            (address.absolute_column ? '$' : '') +
+            this.ColumnLabel(column) +
+            (address.absolute_row ? '$' : '') +
+            (row + 1) +
+            (address.spill ? '#' : ''));
+    }
+    /**
+     * base parse routine; may recurse inside parens (either as grouped
+     * operations or in function arguments).
+     *
+     * @param exit exit on specific characters
+     */
+    ParseGeneric(exit = [0], explicit_group = false) {
+        let stream = [];
+        for (; this.index < this.length;) {
+            const unit = this.ParseNext(stream.length === 0);
+            if (typeof unit === 'number') {
+                if (exit.some((test) => unit === test)) {
+                    break;
+                }
+                else if (unit === OPEN_PAREN) {
+                    // note that function calls are handled elsewhere,
+                    // so we only have to worry about grouping. parse
+                    // up to the closing paren...
+                    // actually now we have implicit calls, so we need
+                    // to manage that here. 
+                    this.index++; // open paren
+                    const group = this.ParseGeneric([CLOSE_PAREN], true);
+                    this.index++; // close paren
+                    // and wrap up in a group element to prevent reordering.
+                    // flag indicates that this is a user grouping, not ours
+                    // skip nulls
+                    // ...don't skip nulls? don't know what the rationale was
+                    // but for implicit calls we will need to support empty arguments 
+                    // if (group) {
+                    stream.push({
+                        type: 'group',
+                        id: this.id_counter++,
+                        elements: group ? [group] : [],
+                        explicit: true,
+                    });
+                    //}
+                }
+                else {
+                    // this can probably move to PNext? except for the test
+                    // on looking for a binary operator? (...)
+                    const operator = this.ConsumeOperator();
+                    if (operator) {
+                        stream.push(operator);
+                    }
+                    else if (explicit_group && unit === this.argument_separator_char) {
+                        // adding a new unit type here to explicitly show we're in
+                        // a group; prevents later passes from treating arguments as 
+                        // fractions or something else. we just need to remove these
+                        // later
+                        stream.push({
+                            type: 'group-separator',
+                            position: this.index,
+                            id: this.id_counter++,
+                        });
+                        this.index++;
+                    }
+                    else {
+                        this.error = `unexpected character [1]: ${String.fromCharCode(unit)}, 0x${unit.toString(16)}`;
+                        this.valid = false;
+                        this.index++;
+                    }
+                }
+            }
+            else {
+                stream.push(unit);
+            }
+        }
+        // why do we build ranges after doing reordering? since ranges
+        // have the highest precedence (after complex numbers), why not
+        // just run through them now? also we could merge the complex
+        // composition (or not, since that's optional)
+        // ...
+        // OK, doing that now (testing). a side benefit is that this solves
+        // one of the problems we had with complex numbers, mismatching naked
+        // column identifiers like I:J. if we do ranges first we will not run
+        // into that problem.
+        if (stream.length) {
+            stream = this.BinaryToRange2(stream);
+            // FIXME: fractions should perhaps move, not sure about the proper 
+            // ordering...
+            if (this.flags.fractions) {
+                // the specific pattern we are looking for for a fraction is
+                //
+                // literal (integer)
+                // literal (integer)
+                // operator (/)
+                // literal (integer)
+                // 
+                // NOTE: excel actually translates these functions after you
+                // enter them to remove the fractions. not sure why, but it's 
+                // possible that exporting them to something else (lotus?) wouldn't
+                // work. we can export them to excel, however, so maybe we can just
+                // leave as-is.
+                const rebuilt = [];
+                const IsInteger = (test) => {
+                    return (test.type === 'literal')
+                        && ((typeof test.value) === 'number')
+                        && (test.value % 1 === 0); // bad typescript
+                };
+                let i = 0;
+                for (; i < stream.length - 3; i++) {
+                    if (IsInteger(stream[i])
+                        && IsInteger(stream[i + 1])
+                        && (stream[i + 2].type === 'operator' && stream[i + 2].operator === '/')
+                        && IsInteger(stream[i + 3])) {
+                        const a = stream[i];
+                        const b = stream[i + 1];
+                        const c = stream[i + 3];
+                        const f = ((a.value < 0) ? -1 : 1) * (b.value / c.value);
+                        i += 3;
+                        rebuilt.push({
+                            id: stream[i].id,
+                            type: 'literal',
+                            text: this.expression.substring(a.position, c.position + 1),
+                            value: a.value + f,
+                            position: a.position,
+                        });
+                    }
+                    else {
+                        rebuilt.push(stream[i]);
+                    }
+                }
+                for (; i < stream.length; i++) {
+                    rebuilt.push(stream[i]);
+                }
+                stream = rebuilt;
+            }
+            // so we're moving complex handling to post-reordering, to support
+            // precedence properly. there's still one thing we have to do here,
+            // though: handle those cases of naked imaginary values "i". these
+            // will be text identifiers, because they don't look like anything 
+            // else. the previous routine will have pulled out column ranges like
+            // I:I so we don't have to worry about that anymore.
+            stream = stream.map(test => {
+                if (test.type === 'identifier' && test.name === this.imaginary_number) {
+                    return {
+                        type: 'complex',
+                        real: 0,
+                        imaginary: 1,
+                        position: test.position,
+                        text: test.name,
+                        id: this.id_counter++,
+                    };
+                }
+                return test;
+            });
+            if (this.flags.dimensioned_quantities) {
+                // support dimensioned quantities. we need to think a little about what 
+                // should and should not be supported here -- definitely a literal 
+                // followed by an identifier; definitely not two identifiers in a row; 
+                // (really?) definitely not expressions followed by identifiers...
+                //
+                // what about
+                // group: (3+2)mm [yes]
+                // call: sin(3)mm [yes]
+                // name?: Xmm [...]
+                //
+                // what about space?
+                // 10 fluid ounces
+                // 10 fl oz
+                // 
+                const rebuilt = [];
+                let unit;
+                for (let i = 0; i < stream.length; i++) {
+                    //for (const entry of stream) {
+                    const entry = stream[i];
+                    if (!unit) {
+                        unit = entry;
+                    }
+                    else if (entry.type === 'identifier' && (unit.type === 'literal' || unit.type === 'group' || unit.type === 'call')) {
+                        // check for multi-word unit (unit has spaces)
+                        const identifier = entry;
+                        while (stream[i + 1]?.type === 'identifier') {
+                            identifier.name += (' ' + stream[++i].name);
+                        }
+                        rebuilt.push({
+                            type: 'dimensioned',
+                            expression: unit,
+                            unit: entry,
+                            id: this.id_counter++,
+                        });
+                        unit = undefined; // consume
+                    }
+                    else {
+                        rebuilt.push(unit);
+                        unit = entry;
+                    }
+                }
+                // trailer
+                if (unit) {
+                    rebuilt.push(unit);
+                }
+                stream = rebuilt;
+            }
+        }
+        // console.info("STREAM\n", stream, "\n\n");
+        if (stream.length === 0)
+            return null;
+        if (stream.length === 1)
+            return stream[0];
+        // fix ordering of binary operations based on precedence; also
+        // convert and validate ranges
+        return this.BinaryToComplex(this.ArrangeUnits(stream));
+    }
+    /**
+     * helper function, @see BinaryToRange
+     * @param unit
+     * @returns
+     */
+    UnitToAddress(unit) {
+        // console.info("U2", unit);
+        // for literals, only numbers are valid
+        if (unit.type === 'literal') {
+            if (typeof unit.value === 'number' && unit.value > 0 && !/\./.test(unit.text || '')) {
+                return {
+                    type: 'address',
+                    position: unit.position,
+                    label: unit.value.toString(),
+                    row: unit.value - 1,
+                    id: this.id_counter++,
+                    column: Infinity,
+                };
+            }
+        }
+        else {
+            // UPDATE: sheet names... we may actually need a subparser for this?
+            // or can we do it with a regex? (...)
+            let sheet;
+            let name = unit.name;
+            const tokens = name.split('!');
+            if (tokens.length > 1) {
+                sheet = tokens.slice(0, tokens.length - 1).join('!');
+                name = name.substr(sheet.length + 1);
+                if (sheet[0] === '\'') {
+                    if (sheet.length > 1 && sheet[sheet.length - 1] === '\'') {
+                        sheet = sheet.substr(1, sheet.length - 2);
+                    }
+                    else {
+                        // console.info('mismatched single quote');
+                        return undefined;
+                    }
+                }
+            }
+            const absolute = name[0] === '$';
+            name = (absolute ? name.substr(1) : name).toUpperCase();
+            const as_number = Number(name);
+            // if it looks like a number, consider it a number and then be strict
+            if (!isNaN(as_number)) {
+                if (as_number > 0 && as_number !== Infinity && !/\./.test(name)) {
+                    return {
+                        type: 'address',
+                        position: unit.position,
+                        absolute_row: absolute,
+                        label: unit.name,
+                        row: as_number - 1,
+                        id: this.id_counter++,
+                        column: Infinity,
+                        sheet,
+                    };
+                }
+            }
+            else if (/[A-Z]{1,3}/.test(name)) {
+                let column = -1; // clever
+                for (let i = 0; i < name.length; i++) {
+                    const char = name[i].charCodeAt(0);
+                    column = 26 * (1 + column) + (char - UC_A);
+                }
+                return {
+                    type: 'address',
+                    position: unit.position,
+                    absolute_column: absolute,
+                    label: unit.name,
+                    column,
+                    id: this.id_counter++,
+                    row: Infinity,
+                    sheet,
+                };
+            }
+        }
+        return undefined;
+    }
+    /**
+     * rewrite of binary to range. this version operates on the initial stream,
+     * which should be OK because range has the highest precedence so we would
+     * never reorder a range.
+     *
+     * ACTUALLY this will break in the case of
+     *
+     * -15:16
+     *
+     * (I think that's the only case). we can fix that though. this should
+     * not impact the case of `2-15:16`, because in that case the - will look
+     * like an operator and not part of the number. the same goes for a leading
+     * `+` which will get dropped implicitly but has no effect (we might want
+     * to preserve it for consistency though).
+     *
+     * NOTE: that error existed in the old version, too, and this way is perhaps
+     * better for fixing it. we should merge this into main.
+     *
+     *
+     * old version comments:
+     * ---
+     *
+     * converts binary operations with a colon operator to ranges. this also
+     * validates that there are no colon operations with non-address operands
+     * (which is why it's called after precendence reordering; colon has the
+     * highest preference). recursive only over binary ops AND unary ops.
+     *
+     * NOTE: there are other legal arguments to a colon operator. specifically:
+     *
+     * (1) two numbers, in either order
+     *
+     * 15:16
+     * 16:16
+     * 16:15
+     *
+     * (2) with one or both optionally having a $
+     *
+     * 15:$16
+     * $16:$16
+     *
+     * (3) two column identifiers, in either order
+     *
+     * A:F
+     * B:A
+     *
+     * (4) and the same with $
+     *
+     * $A:F
+     * $A:$F
+     *
+     * because none of these are legal in any other context, we leave the
+     * default treatment of them UNLESS they are arguments to the colon
+     * operator, in which case we will grab them. that does mean we parse
+     * them twice, but (...)
+     *
+     * FIXME: will need some updated to rendering these, we don't have any
+     * handler for rendering infinity
+     */
+    BinaryToRange2(stream) {
+        const result = [];
+        for (let i = 0; i < stream.length; i++) {
+            const a = stream[i];
+            const b = stream[i + 1];
+            const c = stream[i + 2];
+            let range;
+            let label = '';
+            let negative; // this is a fix for the error case `-14:15`, see below
+            if (a && b && c && b.type === 'operator' && b.operator === ':') {
+                if (a.type === 'address' && c.type === 'address') {
+                    // construct a label using the full text. there's a possibility,
+                    // I suppose, that there are spaces (this should probably not be
+                    // legal). this is a canonical label, though (generated)
+                    // it might be better to let this slip, or treat it as an error
+                    // and force a correction... not sure (TODO/FIXME)
+                    const start_index = a.position + a.label.length;
+                    const end_index = c.position;
+                    range = {
+                        type: 'range',
+                        id: this.id_counter++,
+                        position: a.position,
+                        start: a,
+                        end: c,
+                        label: a.label +
+                            this.expression.substring(start_index, end_index) +
+                            c.label,
+                    };
+                    label = range.start.label + ':' + range.end.label;
+                    this.address_refcount[range.start.label]--;
+                    this.address_refcount[range.end.label]--;
+                    // remove entries from the list for start, stop
+                    const positions = [a.position, c.position];
+                    this.full_reference_list = this.full_reference_list.filter((test) => {
+                        return (test.position !== positions[0] && test.position !== positions[1]);
+                    });
+                }
+                else if ((a.type === 'literal' || a.type === 'identifier')
+                    && (c.type === 'literal' || c.type === 'identifier')) {
+                    // see if we can plausibly interpret both of these as rows or columns
+                    // this is a fix for the case of `-14:15`, which is kind of a rare
+                    // case but could happen. in that case we need to invert the first number,
+                    // so it parses as an address properly, and also insert a "-" which
+                    // should be treated as a unary operator.
+                    // if this happens, the first part must look like a negative number,
+                    // e.g. -10, so there are no leading spaces or intervening spaces
+                    // between the - and the value. therefore...
+                    let left = this.UnitToAddress(a);
+                    if (!left && a.type === 'literal' && typeof a.value === 'number' && a.value < 0) {
+                        const test = {
+                            ...a,
+                            text: (a.text || '').replace(/^-/, ''), // <- ...sign always in position 0
+                            position: a.position + 1, // <- ...advance 1
+                            value: -a.value, // <- ...invert value
+                        };
+                        left = this.UnitToAddress(test);
+                        if (left) {
+                            // if that worked, we need to insert an operator into the
+                            // stream to reflect the - sign. we use the original position.
+                            negative = {
+                                type: 'operator',
+                                operator: '-',
+                                position: a.position,
+                                id: this.id_counter++,
+                            };
+                        }
+                    }
+                    const right = this.UnitToAddress(c);
+                    // and they need to match
+                    if (left && right
+                        && ((left.column === Infinity && right.column === Infinity)
+                            || (left.row === Infinity && right.row === Infinity))) {
+                        label = left.label + ':' + right.label;
+                        // we don't support out-of-order ranges, so we should correct.
+                        // they just won't work otherwise. (TODO/FIXME)
+                        range = {
+                            type: 'range',
+                            id: this.id_counter++,
+                            position: left.position,
+                            start: left,
+                            end: right,
+                            label,
+                        };
+                    }
+                }
+            }
+            if (range) {
+                if (negative) {
+                    result.push(negative);
+                }
+                result.push(range);
+                this.dependencies.ranges[label] = range;
+                this.full_reference_list.push(range);
+                // skip
+                i += 2;
+            }
+            else {
+                result.push(a);
+            }
+        }
+        return result;
+    }
+    /**
+     * we've now come full circle. we started with handling ranges as
+     * binary operators; then we added complex composition as a first-pass
+     * function; then we moved ranges to a first-pass function; and now we're
+     * moving complex composition to a lower-level restructuring of binary
+     * operations.
+     *
+     * that allows better precedence handling for (potentially) ambiguous
+     * constructions like =B3 * 2 + 3i. we do have parens, so.
+     *
+     * @param unit
+     * @returns
+     */
+    BinaryToComplex(unit) {
+        if (unit.type === 'binary') {
+            if ((unit.operator === '+' || unit.operator === '-')
+                && unit.left.type === 'literal'
+                && typeof unit.left.value === 'number'
+                && unit.right.type === 'complex' // 'imaginary') {
+                && !unit.right.composited) {
+                // ok, compose
+                // console.info("WANT TO COMPOSE", unit);
+                let text = '';
+                text = this.expression.substring(unit.left.position, unit.right.position + (unit.right.text?.length || 0));
+                let imaginary_value = unit.right.imaginary;
+                if (unit.operator === '-') {
+                    imaginary_value = -imaginary_value;
+                }
+                return {
+                    type: 'complex',
+                    position: unit.left.position,
+                    text: text,
+                    id: this.id_counter++,
+                    imaginary: imaginary_value,
+                    real: unit.left.value,
+                    composited: true,
+                };
+            }
+            else {
+                unit.left = this.BinaryToComplex(unit.left);
+                unit.right = this.BinaryToComplex(unit.right);
+            }
+        }
+        else if (unit.type === 'unary' &&
+            (unit.operator === '-' || unit.operator === '+') &&
+            unit.operand.type === 'complex' &&
+            unit.operand.text === this.imaginary_number) {
+            // sigh... patch fix for very special case of "-i"
+            // actually: why do I care about this? we could let whomever is using
+            // the result deal with this particular case... although it's more
+            // properly our responsibility if we are parsing complex numbers.
+            // we only have to worry about mischaracterizing the range label, 
+            // e.g. "-i:j", but we should have already handled that in a prior pass.
+            return {
+                ...unit.operand,
+                position: unit.position,
+                text: this.expression.substring(unit.position, unit.operand.position + (unit.operand.text || '').length),
+                imaginary: unit.operand.imaginary * (unit.operator === '-' ? -1 : 1),
+            };
+        }
+        return unit;
+    }
+    /**
+     * reorders operations for precendence
+     *
+     * this method was written with the assumption that groups were
+     * always an error. that's no longer true, with implicit calls.
+     * we should still error if it's not an _explicit_ group, i.e. there's
+     * just a bunch of naked tokens.
+     *
+     */
+    ArrangeUnits(stream) {
+        // probably should not happen
+        if (stream.length === 0)
+            return { type: 'missing', id: this.id_counter++ };
+        // this is probably already covered
+        if (stream.length === 1)
+            return stream[0];
+        const stack = [];
+        // work left-to-right (implied precendence), unless there
+        // is actual precendence. spreadsheet language only supports
+        // binary operators, so we always expect unit - operator - unit
+        //
+        // UPDATE: that's incorrect. SL supports unary + and - operators.
+        // which makes this more complicated.
+        //
+        // we explicitly support unfinished expressions for the first pass
+        // to build dependencies, but if they're invalid the resulting
+        // parse tree isn't expected to be correct. in that case we
+        // generally will pass back a bag of parts, with a flag set.
+        for (let index = 0; index < stream.length; index++) {
+            let element = stream[index];
+            if (element.type === 'group-separator') {
+                continue; // drop 
+            }
+            // given that we need to support unary operators, the logic needs
+            // to be a little different. operators are OK at any position, provided
+            // we can construct either a unary or binary operation.
+            if (element.type === 'operator') {
+                if (stack.length === 0 || stack[stack.length - 1].type === 'operator') {
+                    // valid if unary operator and we can construct a unary operation.
+                    // in this case we do it with recursion.
+                    if (unary_operators[element.operator]) {
+                        const right = this.BinaryToComplex(this.ArrangeUnits(stream.slice(index + 1)));
+                        // this ensures we return the highest-level group, even if we recurse
+                        if (!this.valid) {
+                            return {
+                                type: 'group',
+                                id: this.id_counter++,
+                                elements: stream,
+                                explicit: false,
+                            };
+                        }
+                        // if it succeeded, then we need to apply the unary operator to
+                        // the result, or if it's a binary operation, to the left-hand side
+                        // (because we have precedence) -- unless it's a range [this is now
+                        // handled above]
+                        if (right.type === 'binary') {
+                            right.left = {
+                                type: 'unary',
+                                id: this.id_counter++,
+                                operator: element.operator,
+                                operand: right.left,
+                                position: element.position,
+                            };
+                            element = right;
+                        }
+                        else {
+                            // create a unary operation which will replace the element
+                            element = {
+                                type: 'unary',
+                                id: this.id_counter++,
+                                operator: element.operator,
+                                operand: right,
+                                position: element.position,
+                            };
+                        }
+                        // end loop after this pass, because the recurse consumes everything else
+                        index = stream.length;
+                    }
+                    else {
+                        this.error = `unexpected character [2]: ${element.operator}`;
+                        this.error_position = element.position;
+                        this.valid = false;
+                        return {
+                            type: 'group',
+                            id: this.id_counter++,
+                            elements: stream,
+                            explicit: false,
+                        };
+                    }
+                }
+                else {
+                    stack.push(element);
+                    continue;
+                }
+            }
+            //
+            // why is this 2? are we thinking about combining complex numbers?
+            // or ranges? (those would be binary). or was this for dimensioned
+            // quantities? [actually that makes sense] [A: no, it wasn't that]
+            //
+            // actually what's the case where this is triggered and it's _not_
+            // an error? can we find that?
+            // 
+            if (stack.length < 2) {
+                // we know that `element` is not an operator, because we 
+                // would have consumed it
+                if (stack.length === 1) {
+                    const a = stack[0].type;
+                    // support for lambdas
+                    if (element.type === 'group' && element.explicit) {
+                        if (a === 'address' || a === 'call' || a === 'identifier' || a === 'implicit-call') {
+                            // our parser seems to create implicit groups from these
+                            // values in parens. we should fix that, but we can unpack it.
+                            let args = element.elements;
+                            if (args.length === 1 && args[0].type === 'group' && !args[0].explicit) {
+                                args = args[0].elements;
+                            }
+                            // create an implicit call. replace on the stack.
+                            stack[0] = {
+                                type: 'implicit-call',
+                                call: stack[0],
+                                args,
+                                id: this.id_counter++,
+                                position: stack[0].position,
+                            };
+                            continue;
+                        }
+                    }
+                    /*
+                    else if (a !== 'operator') {
+          
+                      // console.warn("unexpected element", stack[0], element);
+          
+                      this.error = `unexpected element [3]: ${element.type}`;
+                      this.error_position = (element.type === 'missing' || element.type === 'group' || element.type === 'dimensioned') ? -1 : element.position;
+                      this.valid = false;
+                      return {
+                        type: 'group',
+                        id: this.id_counter++,
+                        elements: stream,
+                        explicit: false,
+                      };
+          
+                    }
+                    */
+                }
+                stack.push(element);
+            }
+            else if (stack[stack.length - 1].type === 'operator') {
+                const left = stack[stack.length - 2];
+                const operator_unit = stack[stack.length - 1];
+                const operator = operator_unit.operator;
+                // assume we can construct it as follows: [A op B]
+                const operation = {
+                    type: 'binary',
+                    id: this.id_counter++,
+                    left,
+                    operator,
+                    position: operator_unit.position,
+                    right: element,
+                };
+                // we have to reorder if left (A) is a binary operation, and the
+                // precedence of the new operator is higher. note that we will
+                // deal with range operations later, for now just worry about
+                // operator precedence
+                if (left.type === 'binary' &&
+                    binary_operators_precendence[operator] >
+                        binary_operators_precendence[left.operator]) {
+                    // so we have [[A op1 B] op2 C], and we need to re-order this into [A op1 [B op2 C]].
+                    operation.left = left.left; // <- A
+                    operation.operator = left.operator; // <- op1
+                    operation.position = left.position;
+                    operation.right = {
+                        type: 'binary',
+                        id: this.id_counter++,
+                        left: left.right, // <- B
+                        right: element, // <- C
+                        operator, // <- op2
+                        position: operator_unit.position,
+                    };
+                }
+                stack.splice(-2, 2, operation);
+            }
+            else {
+                /*
+                this.error = `multiple expressions`;
+                this.error_position = (element as {position?: number}).position;
+                this.valid = false;
+                return {
+                  type: 'group',
+                  id: this.id_counter++,
+                  elements: stream,
+                  explicit: false,
+                };
+                */
+                stack.push(element);
+            }
+        }
+        if (stack.length > 1) {
+            return {
+                type: 'group',
+                id: this.id_counter++,
+                elements: stack,
+                explicit: false,
+            };
+        }
+        return stack[0];
+    }
+    /**
+     * parses literals and tokens from the stream, ignoring whitespace,
+     * and stopping on unexpected tokens (generally operators or parens).
+     *
+     * @param naked treat -/+ as signs (part of numbers) rather than operators.
+     */
+    ParseNext(naked = true) {
+        this.ConsumeWhiteSpace();
+        const char = this.data[this.index];
+        if (char === DOUBLE_QUOTE) {
+            return {
+                type: 'literal',
+                id: this.id_counter++,
+                position: this.index,
+                value: this.ConsumeString(),
+            };
+        }
+        else if ((char >= ZERO && char <= NINE) || char === this.decimal_mark_char) {
+            return this.ConsumeNumber();
+        }
+        else if (char === OPEN_BRACE) {
+            return this.ConsumeArray();
+        }
+        else if (naked && (char === MINUS || char === PLUS)) {
+            // there's a case where you type '=-func()', which should support
+            // '=+func()' as well, both of which are naked operators and not numbers.
+            // the only way to figure this out is to check for a second number char.
+            // this is turning into lookahead, which we did not want to do...
+            const check = this.data[this.index + 1];
+            if ((check >= ZERO && check <= NINE) ||
+                check === this.decimal_mark_char) {
+                return this.ConsumeNumber();
+            }
+        }
+        else if ((char >= UC_A && char <= UC_Z) ||
+            (char >= LC_A && char <= LC_Z) ||
+            char === UNDERSCORE ||
+            char === HASH || // new: only allowed in position 1, always an error
+            char === SINGLE_QUOTE ||
+            char === DOLLAR_SIGN ||
+            // we used to not allow square brackets to start tokens, because
+            // we only supported them for relative R1C1 references -- hence you'd
+            // need the R first. but we now allow them for "structured references".
+            char === OPEN_SQUARE_BRACKET ||
+            (char >= ACCENTED_RANGE_START && char <= ACCENTED_RANGE_END) // adding accented characters, needs some testing
+        ) {
+            return this.ConsumeToken(char);
+        }
+        // else throw(new Error('Unexpected character: ' + char));
+        return char;
+    }
+    ConsumeArray() {
+        const expression = {
+            type: 'array',
+            id: this.id_counter++,
+            values: [],
+            position: this.index,
+        };
+        this.index++;
+        let row = 0;
+        let column = 0;
+        while (this.index < this.length) {
+            const item = this.ParseNext();
+            const start_position = this.index;
+            if (typeof item === 'number') {
+                this.index++;
+                switch (item) {
+                    case SEMICOLON:
+                        //column = 0;
+                        //row++;
+                        column++;
+                        row = 0;
+                        break;
+                    case COMMA:
+                        //column++;
+                        row++;
+                        break;
+                    case CLOSE_BRACE:
+                        return expression;
+                    default:
+                        if (this.valid) {
+                            this.error = `invalid character in array literal`;
+                            this.error_position = start_position;
+                            this.valid = false;
+                        }
+                        break;
+                }
+            }
+            else {
+                switch (item.type) {
+                    case 'literal':
+                        if (!expression.values[row]) {
+                            expression.values[row] = [];
+                        }
+                        expression.values[row][column] = item.value;
+                        break;
+                    default:
+                        if (this.valid) {
+                            this.error = `invalid value in array literal`;
+                            this.error_position = start_position;
+                            this.valid = false;
+                        }
+                        break;
+                }
+            }
+        }
+        return expression;
+    }
+    ConsumeOperator() {
+        for (const operator of composite_operators) {
+            if (this.expression.substr(this.index, operator.length) === operator) {
+                const position = this.index;
+                this.index += operator.length;
+                return {
+                    type: 'operator',
+                    id: this.id_counter++,
+                    operator,
+                    position,
+                };
+            }
+        }
+        return null;
+    }
+    /** consume function arguments, which can be of any type */
+    ConsumeArguments() {
+        this.index++; // open paren
+        let argument_index = 0;
+        const args = [];
+        for (; this.index < this.length;) {
+            const unit = this.ParseGeneric([
+                this.argument_separator_char,
+                CLOSE_PAREN,
+            ]);
+            if (null !== unit)
+                args.push(unit);
+            // why did parsing stop?
+            const char = this.data[this.index];
+            if (char === this.argument_separator_char) {
+                this.index++;
+                argument_index++;
+                for (let i = args.length; i < argument_index; i++) {
+                    args.push({ type: 'missing', id: this.id_counter++ });
+                }
+            }
+            else if (char === CLOSE_PAREN) {
+                this.index++;
+                return args;
+            }
+            // else console.info('UNEXPECTED (CA)', char);
+        }
+        return args;
+    }
+    /**
+     * consume token. also checks for function call, because parens
+     * have a different meaning (grouping/precedence) when they appear
+     * not immediately after a token.
+     *
+     * regarding periods: as long as there's no intervening whitespace
+     * or operator, period should be a valid token character. tokens
+     * cannot start with a period.
+     *
+     * NOTE: that's true irrespective of decimal mark type.
+     *
+     * you can have tokens (addresses) with single quotes; these are used
+     * to escape sheet names with spaces (which is a bad idea, but hey). this
+     * should only be legal if the token starts with a single quote, and only
+     * for one (closing) quote.
+     *
+     * R1C1 relative notation uses square brackets, like =R2C[-1] or =R[-1]C[-2].
+     * that's pretty easy to see. there's also regular R1C1, like =R1C1.
+     *
+     * "structured references" use square brackets. they can start with
+     * square brackets -- in that case the table source is implicit (has to
+     * be in the table). otherwise they look like =TableName[@ColumnName]. that
+     * @ is optional and (I think) means don't spill.
+     *
+     */
+    ConsumeToken(initial_char) {
+        const token = [initial_char];
+        const position = this.index;
+        let single_quote = (initial_char === SINGLE_QUOTE);
+        let square_bracket = 0; // now balancing // false; // this one can't be initial
+        // this is a set-once flag for square brackets; it can 
+        // short-circuit the check for structured references. 
+        let braces = false;
+        // also watch first char
+        if (initial_char === OPEN_SQUARE_BRACKET) {
+            square_bracket = 1;
+            braces = true;
+        }
+        for (++this.index; this.index < this.length; this.index++) {
+            const char = this.data[this.index];
+            if ((char >= UC_A && char <= UC_Z) ||
+                (char >= LC_A && char <= LC_Z) ||
+                (char >= ACCENTED_RANGE_START && char <= ACCENTED_RANGE_END) ||
+                char === UNDERSCORE ||
+                char === DOLLAR_SIGN ||
+                char === PERIOD ||
+                char === EXCLAMATION_MARK ||
+                single_quote || // ((char === SINGLE_QUOTE || char === SPACE) && single_quote) ||
+                (char >= ZERO && char <= NINE) // tokens can't start with a number, but this loop starts at index 1
+                // we now allow square brackets for structured references; 
+                // minus is still only allowed in R1C1 references, so keep
+                // that restriction
+                || char === OPEN_SQUARE_BRACKET
+                || (square_bracket > 0 && char === CLOSE_SQUARE_BRACKET)
+                || (char === MINUS && this.flags.r1c1 && (square_bracket === 1))
+                // the @ sign can appear after the first square bracket... 
+                // but only immediately? 
+                || (square_bracket > 0 && char === AT && this.data[this.index - 1] === OPEN_SQUARE_BRACKET)
+                // comma can appear in the first level. this is maybe an older
+                // syntax? it looks like `Table2[[#this row],[region]]
+                || (square_bracket === 1 && (char === COMMA || char === SPACE))
+                // structured references allow basically any character, if 
+                // it's in the SECOND bracket. not sure what's up with that.
+                || (square_bracket > 1)
+                // I think that's all the rules for structured references.
+                // testing question marks, which are legal in defined names
+                // (but I think not in table names or column names)
+                || (char === QUESTION_MARK && square_bracket === 0)
+            // moving
+            // || (char === HASH) // FIXME: this should only be allowed at the end...
+            /*
+    
+            || (this.flags.r1c1 && (
+              char === OPEN_SQUARE_BRACKET ||
+              char === CLOSE_SQUARE_BRACKET ||
+              (char === MINUS && square_bracket)
+            ))
+            */
+            ) {
+                token.push(char);
+                if (char === OPEN_SQUARE_BRACKET) {
+                    // square_bracket = true;
+                    square_bracket++;
+                    braces = true;
+                }
+                if (char === CLOSE_SQUARE_BRACKET) {
+                    // square_bracket = false;
+                    square_bracket--;
+                }
+                if (char === SINGLE_QUOTE) {
+                    single_quote = false; // one only
+                }
+            }
+            else
+                break;
+        }
+        // hash at end only
+        if (this.data[this.index] === HASH) {
+            token.push(this.data[this.index++]);
+        }
+        const str = token.map((num) => String.fromCharCode(num)).join('');
+        // special handling: unbalanced single quote (probably sheet name),
+        // this is an error
+        if (single_quote) { // unbalanced
+            this.error = `unbalanced single quote`;
+            this.error_position = position;
+            this.valid = false;
+            return {
+                type: 'identifier',
+                id: this.id_counter++,
+                name: str,
+                position,
+            };
+        }
+        // check unbalanced square bracket as well, could be a runaway structured 
+        // reference
+        if (square_bracket) {
+            this.error = `unbalanced square bracket`;
+            this.error_position = position;
+            this.valid = false;
+            return {
+                type: 'identifier',
+                id: this.id_counter++,
+                name: str,
+                position,
+            };
+        }
+        /* remove special handling
+    
+        // special handling
+    
+        if (str.toLowerCase() === 'true') {
+          return {
+            type: 'literal',
+            id: this.id_counter++,
+            value: true,
+            position,
+          };
+        }
+        if (str.toLowerCase() === 'false') {
+          return {
+            type: 'literal',
+            id: this.id_counter++,
+            value: false,
+            position,
+          };
+        }
+    
+        */
+        // function takes precendence over address? I guess so
+        this.ConsumeWhiteSpace();
+        // UPDATE: UNLESS the token is an address, because that's not
+        // a legal function name. so change that precedence rule, address
+        // comes first.
+        // erm -- that's not 100% correct. LOG10 is a valid cell address
+        // and a valid function name. there might be others as well.
+        if (this.flags.spreadsheet_semantics) {
+            const address = this.ConsumeAddress(str, position);
+            if (address)
+                return address;
+        }
+        // [FIXME: what about braces? (...)]
+        const next_char = this.data[this.index];
+        if (next_char === OPEN_PAREN) {
+            const args = this.ConsumeArguments();
+            return {
+                type: 'call',
+                id: this.id_counter++,
+                name: str,
+                args,
+                position,
+                end: this.index, // testing
+            };
+        }
+        if (this.flags.spreadsheet_semantics) {
+            // check for address. in the case of a range, we'll see an address, the
+            // range operator, and a second address. that will be turned into a range
+            // later.
+            // moved up
+            // const address = this.ConsumeAddress(str, position);
+            // if (address) return address;
+            // check for structured reference, if we had square brackets
+            if (braces) {
+                const structured = this.ConsumeStructuredReference(str, position);
+                if (structured) {
+                    return structured;
+                }
+            }
+        }
+        // move true/false handling here
+        // should we accept english even if it's not the active language? (...)
+        const lc = str.toLowerCase();
+        if (lc === 'true' || (this.flags.boolean_true && lc === this.flags.boolean_true.toLowerCase())) {
+            return {
+                type: 'literal',
+                id: this.id_counter++,
+                value: true,
+                position,
+            };
+        }
+        if (lc === 'false' || (this.flags.boolean_false && lc === this.flags.boolean_false.toLowerCase())) {
+            return {
+                type: 'literal',
+                id: this.id_counter++,
+                value: false,
+                position,
+            };
+        }
+        const identifier = {
+            type: 'identifier',
+            id: this.id_counter++,
+            name: str,
+            position,
+        };
+        this.full_reference_list.push(identifier);
+        return identifier;
+    }
+    /**
+     * like ConsumeAddress, look for a structured reference.
+     */
+    ConsumeStructuredReference(token, position) {
+        // structured references look something like
+        //
+        // [@Column1]
+        // [@[Column with spaces]]
+        // [[#This Row],[Column2]]
+        //
+        // @ means the same as [#This Row]. there are probably other things
+        // that use the # syntax, but I haven't seen them yet. 
+        //
+        // some observations: case is not matched for the "this row" text.
+        // I think that's true of column names as well, but that's not relevant
+        // at this stage. whitespace around that comma is ignored. I _think_
+        // whitespace around column names is also ignored, but spaces within
+        // a column name are OK, at least within the second set of brackets. 
+        // const index = position;
+        const token_length = token.length;
+        const label = token;
+        let table = '';
+        let i = 0;
+        for (; i < token_length; i++) {
+            if (token[i] === '[') {
+                token = token.substring(i);
+                break;
+            }
+            table += token[i];
+        }
+        // after the table, must start and end with brackets
+        if (token[0] !== '[' || token[token.length - 1] !== ']') {
+            return undefined;
+        }
+        token = token.substring(1, token.length - 1);
+        const parts = token.split(',').map(part => part.trim());
+        let scope = 'column';
+        // let this_row = false;
+        let column = '';
+        if (parts.length > 2) {
+            return undefined; // ??
+        }
+        else if (parts.length === 2) {
+            if (/\[#this row\]/i.test(parts[0])) {
+                scope = 'row';
+            }
+            else if (/\[#all\]/i.test(parts[0])) {
+                scope = 'all';
+            }
+            column = parts[1];
+        }
+        else {
+            column = parts[0];
+            if (column[0] === '@') {
+                scope = 'row';
+                column = column.substring(1, column.length);
+            }
+        }
+        if (column[0] === '[' && column[column.length - 1] === ']') {
+            column = column.substring(1, column.length - 1);
+        }
+        const reference = {
+            type: 'structured-reference',
+            id: this.id_counter++,
+            label,
+            position,
+            scope,
+            column,
+            table,
+        };
+        // console.info(reference);
+        this.full_reference_list.push(reference);
+        return reference;
+    }
+    /**
+     * consumes address. this is outside of the normal parse flow;
+     * we already have a token, here we're checking if it's an address.
+     *
+     * this used to check for ranges as well, but we now treat ranges as
+     * an operation on two addresses; that supports whitespace between the
+     * tokens.
+     *
+     * FIXME: that means we can now inline the column/row routines, since
+     * they are not called more than once
+     */
+    ConsumeAddress(token, position) {
+        const index = position;
+        const token_length = token.length;
+        // FIXME: should mark this (!) when it hits, rather than search
+        // UPDATE: ! is legal in sheet names, although it needs to be quoted.
+        let sheet;
+        const tokens = token.split('!');
+        if (tokens.length > 1) {
+            sheet = tokens.slice(0, tokens.length - 1).join('!');
+            position += sheet.length + 1;
+        }
+        // handle first
+        if (this.flags.r1c1) {
+            const match = tokens[tokens.length - 1].match(this.r1c1_regex);
+            if (match) {
+                const r1c1 = {
+                    type: 'address',
+                    id: this.id_counter++,
+                    label: token, // TODO
+                    row: 0,
+                    column: 0,
+                    // absolute_row: false, // TODO: is this supported?
+                    // absolute_column: false, // TODO: is this supported?
+                    position: index,
+                    sheet,
+                    r1c1: true,
+                };
+                if (match[1][0] === '[') { // relative
+                    r1c1.offset_row = true;
+                    r1c1.row = Number(match[1].substring(1, match[1].length - 1));
+                }
+                else if (match[1]) { // absolute
+                    r1c1.row = Number(match[1]) - 1; // R1C1 is 1-based
+                    if (this.flags.r1c1_proper_semantics) {
+                        r1c1.absolute_row = true;
+                    }
+                }
+                else {
+                    r1c1.offset_row = true;
+                    r1c1.row = 0;
+                }
+                if (match[2][0] === '[') { // relative
+                    r1c1.offset_column = true;
+                    r1c1.column = Number(match[2].substring(1, match[2].length - 1));
+                }
+                else if (match[2]) { // absolute
+                    r1c1.column = Number(match[2]) - 1; // R1C1 is 1-based
+                    if (this.flags.r1c1_proper_semantics) {
+                        r1c1.absolute_column = true;
+                    }
+                }
+                else {
+                    r1c1.offset_column = true;
+                    r1c1.column = 0;
+                }
+                return r1c1;
+            }
+        }
+        // FIXME: can inline
+        const c = this.ConsumeAddressColumn(position);
+        if (!c)
+            return null;
+        position = c.position;
+        // things that look like an address but have row 0 are legal
+        // as names. so this should be a token if r === 0.
+        const r = this.ConsumeAddressRow(position);
+        if (!r)
+            return null;
+        position = r.position;
+        // special hack for LOG10. ugh. can't find any other functions with
+        // this problem, in english at least. btw what's the translation for
+        // log10?
+        if (c.column === 8508 && r.row === 9) {
+            return null;
+        }
+        const label = sheet ?
+            sheet + token.substr(sheet.length, position - index).toUpperCase() :
+            token.substr(0, position - index).toUpperCase();
+        if (sheet && sheet[0] === '\'') {
+            sheet = sheet.substr(1, sheet.length - 2);
+        }
+        const addr = {
+            type: 'address',
+            id: this.id_counter++,
+            label, // : token.substr(0, position - index).toUpperCase(),
+            row: r.row,
+            column: c.column,
+            absolute_row: r.absolute,
+            absolute_column: c.absolute,
+            position: index,
+            sheet,
+            spill: r.spill,
+        };
+        // if that's not the complete token, then it's invalid
+        if (token_length !== position - index)
+            return null;
+        // store ref, increment count
+        this.dependencies.addresses[addr.label] = addr;
+        this.address_refcount[addr.label] =
+            (this.address_refcount[addr.label] || 0) + 1;
+        // add to new address list. use the actual object (not a clone or copy);
+        // we update the list later, and we may want to remove it (if it turns
+        // out it's part of a range)
+        this.full_reference_list.push(addr);
+        return addr;
+    }
+    /**
+     * consumes a row, possibly absolute ($). returns the numeric row
+     * (0-based) and metadata.
+     *
+     * note that something like "X0" is a legal token, because 0 is not
+     * a valid row. but at the same time it can't have a $ in it. although
+     * maybe "X$0" is a token but not a valid name? dunno
+     */
+    ConsumeAddressRow(position) {
+        const absolute = this.data[position] === DOLLAR_SIGN;
+        if (absolute)
+            position++;
+        const start = position;
+        let value = 0;
+        for (;; position++) {
+            const char = this.data[position];
+            if (char >= ZERO && char <= NINE) {
+                value *= 10;
+                value += char - ZERO;
+            }
+            else
+                break;
+        }
+        if (start === position) {
+            return false;
+        }
+        // handle token X0. should ~maybe~ handle this only if !absolute
+        // temp leaving this separate from the above test just so it's clear
+        // what we are doing
+        if (value === 0) {
+            return false;
+        }
+        let spill = false;
+        if (this.data[position] === HASH) {
+            position++;
+            spill = true;
+        }
+        return { absolute, row: value - 1, position, spill };
+    }
+    /**
+     * consumes a column, possibly absolute ($). returns the numeric
+     * column (0-based) and metadata
+     */
+    ConsumeAddressColumn(position) {
+        let column = -1; // clever
+        let length = 0; // max 3 chars for column
+        const absolute = this.data[position] === DOLLAR_SIGN;
+        if (absolute)
+            position++;
+        for (;; position++, length++) {
+            if (length >= 4)
+                return false; // max 3 chars for column
+            const char = this.data[position];
+            if (char >= UC_A && char <= UC_Z) {
+                column = 26 * (1 + column) + (char - UC_A);
+            }
+            else if (char >= LC_A && char <= LC_Z) {
+                column = 26 * (1 + column) + (char - LC_A);
+            }
+            else
+                break;
+        }
+        if (column < 0)
+            return false;
+        return { absolute, column, position };
+    }
+    /**
+     * consumes number. supported formats (WIP):
+     *
+     * -3
+     * +3
+     * 100.9
+     * 10.0%
+     * 1e-2.2
+     *
+     * ~1,333,123.22~
+     *
+     * UPDATE: commas (separators) are not acceptable in numbers passed
+     * in formulae, can't distinguish between them and function argument
+     * separators.
+     *
+     * regarding the above, a couple of rules:
+     *
+     * 1. +/- is only legal in position 0 or immediately after e/E
+     * 2. only one decimal point is allowed.
+     * 3. any number of separators, in any position, are legal, but
+     *    only before the decimal point.
+     * 4. only one % is allowed, and only in the last position
+     *
+     * NOTE: this is probably going to break on unfinished strings that
+     * end in - or +... if they're not treated as operators...
+     *
+     * FIXME: find test cases for that so we can fix it
+     *
+     * UPDATE: exporting original text string for preservation/insertion.
+     * this function now returns a tuple of [value, text].
+     *
+     * UPDATE: we now (at least in a branch) consume complex numbers. the last
+     * element of the return array is a boolean which is set if the value is an
+     * imaginary number. when parsing, we will only see the imaginary part;
+     * we'll use a separate step to put complex numbers together.
+     *
+     *
+     */
+    ConsumeNumber() {
+        const starting_position = this.index;
+        // for exponential notation
+        let exponent = 0;
+        let negative_exponent = false;
+        // general
+        let negative = false;
+        let integer = 0;
+        let decimal = 0;
+        let fraction = 0;
+        let state = 'integer';
+        let position = 0;
+        let imaginary = false;
+        const start_index = this.index;
+        for (; this.index < this.length; this.index++, position++) {
+            const char = this.data[this.index];
+            if (char === this.decimal_mark_char) {
+                if (state === 'integer')
+                    state = 'fraction';
+                else
+                    break; // end of token; not consuming
+            }
+            else if (char === PERCENT) {
+                // FIXME: disallow combination of exponential and percent notation
+                integer /= 100; // this is a dumb way to do this
+                fraction /= 100;
+                this.index++; // we are consuming
+                break; // end of token
+            }
+            else if (char === PLUS || char === MINUS) {
+                // NOTE: handling of positive/negative exponent in exponential
+                // notation is handled separately, see below
+                if (position === 0) {
+                    if (char === MINUS)
+                        negative = true;
+                }
+                else
+                    break; // end of token -- not consuming
+            }
+            else if (char === UC_E || char === LC_E) {
+                if (state === 'integer' || state === 'fraction') {
+                    state = 'exponent';
+                    if (this.index < this.length - 1) {
+                        if (this.data[this.index + 1] === PLUS)
+                            this.index++;
+                        else if (this.data[this.index + 1] === MINUS) {
+                            this.index++;
+                            negative_exponent = true;
+                        }
+                    }
+                }
+                else
+                    break; // not sure what this is, then
+            }
+            else if (char === this.imaginary_char) {
+                // FIXME: this should only be set if it's exactly '8i' and not '8in',
+                // since we want to use that for dimensioned quantities. what's legit
+                // after the i and what is not? let's exclude anything in the "word" 
+                // range...
+                // peek
+                const peek = this.data[this.index + 1];
+                if ((peek >= UC_A && peek <= UC_Z) ||
+                    (peek >= LC_A && peek <= LC_Z) ||
+                    (peek >= ACCENTED_RANGE_START && peek <= ACCENTED_RANGE_END) ||
+                    peek === UNDERSCORE) {
+                    break; // start of an identifier
+                }
+                // actually we could use our dimension logic instead of this... turn
+                // this off when using dimensioned quantities and move it in there?
+                if (state === 'integer' || state === 'fraction') {
+                    this.index++; // consume
+                    imaginary = true;
+                    break; // end of token
+                }
+            }
+            else if (char >= ZERO && char <= NINE) {
+                switch (state) {
+                    case 'integer':
+                        integer = integer * 10 + (char - ZERO);
+                        break;
+                    case 'fraction':
+                        fraction = fraction * 10 + (char - ZERO);
+                        decimal++;
+                        break;
+                    case 'exponent':
+                        exponent = exponent * 10 + (char - ZERO);
+                        break;
+                }
+            }
+            else
+                break;
+        }
+        // NOTE: multiplying returns fp noise, but dividing does not? need
+        // to check more browsers... maybe we should store the value in some
+        // other form? (that's a larger TODO)
+        // let value = integer + fraction * Math.pow(10, -decimal);
+        let value = integer + fraction / (Math.pow(10, decimal)); // <- this is cleaner? 
+        if (state === 'exponent') {
+            value = value * Math.pow(10, (negative_exponent ? -1 : 1) * exponent);
+        }
+        // const text = this.expression.substring(start_index, this.index) || '';
+        // return [negative ? -value : value, text, imaginary];
+        if (imaginary) {
+            return {
+                type: 'complex',
+                id: this.id_counter++,
+                position: starting_position,
+                imaginary: negative ? -value : value,
+                real: 0,
+                text: this.expression.substring(start_index, this.index) || '',
+            };
+        }
+        else {
+            return {
+                type: 'literal',
+                id: this.id_counter++,
+                position: starting_position,
+                value: negative ? -value : value,
+                text: this.expression.substring(start_index, this.index) || '',
+            };
+        }
+        /*
+        return {
+          type: imaginary ? 'imaginary' : 'literal',
+          id: this.id_counter++,
+          position: starting_position,
+          value: negative ? -value : value,
+          text: this.expression.substring(start_index, this.index) || '',
+        };
+        */
+    }
+    /**
+     * in spreadsheet language ONLY double-quoted strings are legal. there
+     * are no escape characters, and a backslash is a legal character. to
+     * embed a quotation mark, use "" (double-double quote); that's an escaped
+     * double-quote.
+     */
+    ConsumeString() {
+        this.index++; // open quote
+        const str = [];
+        for (; this.index < this.length; this.index++) {
+            const char = this.data[this.index];
+            if (char === DOUBLE_QUOTE) {
+                // always do this: either it's part of the string (and
+                // we want to skip the next one), or it's the end of the
+                // string and we want to close the literal.
+                this.index++;
+                // check for an escaped double-quote; otherwise close the string
+                // note (1) we already incremented, so check the current value,
+                // and (2) it will increment again on the loop pass so it will
+                // drop the extra one. I note these because this was confusing to
+                // write.
+                if (this.index >= this.length ||
+                    this.data[this.index] !== DOUBLE_QUOTE) {
+                    break;
+                }
+            }
+            str.push(char);
+        }
+        return str.map((char) => String.fromCharCode(char)).join('');
+    }
+    /** run through any intervening whitespace */
+    ConsumeWhiteSpace() {
+        for (; this.index < this.length;) {
+            const char = this.data[this.index];
+            if (char === SPACE ||
+                char === TAB ||
+                char === CR ||
+                char === LF ||
+                char === NON_BREAKING_SPACE) {
+                this.index++;
+            }
+            else
+                return;
+        }
+    }
+}
+//# sourceMappingURL=parser.js.map