@borgar/fx 2.1.1 → 3.0.0

package/lib/{addMeta-test.js → addMeta.spec.js}

@@ -1,10 +1,22 @@
 import { test, Test } from 'tape';
-import { FX_PREFIX, OPERATOR, NUMBER, RANGE, RANGE_BEAM } from './constants.js';
+import { FX_PREFIX, OPERATOR, NUMBER, RANGE, RANGE_BEAM, FUNCTION, WHITESPACE } from './constants.js';
 import { addMeta } from './addMeta.js';
 import { tokenize } from './lexer.js';
 
-Test.prototype.isMetaTokens = function isTokens (expr, result, opts) {
-  this.deepEqual(addMeta(tokenize(expr), opts), result, expr);
+Test.prototype.isMetaTokens = function isTokens (expr, expect, opts) {
+  const actual = addMeta(tokenize(expr), opts);
+  if (actual.length === expect.length) {
+    actual.forEach((d, i) => {
+      const keys = Object.keys(d).concat(Object.keys(expect[i]));
+      keys.forEach(key => {
+        if (actual[i][key] === expect[i][key]) {
+          delete actual[i][key];
+          delete expect[i][key];
+        }
+      });
+    });
+  }
+  this.deepEqual(actual, expect, expr);
 };
 
 test('add extra meta to operators', t => {
@@ -34,9 +46,9 @@ test('add extra meta to operators', t => {
   // don't be fooled by nested curlys
   t.isMetaTokens('={{}}', [
     { index: 0, depth: 0, type: FX_PREFIX, value: '=' },
-    { index: 1, depth: 0, type: OPERATOR, value: '{', groupId: 'fxg1' },
-    { index: 2, depth: 0, type: OPERATOR, value: '{', error: true },
-    { index: 3, depth: 0, type: OPERATOR, value: '}', groupId: 'fxg1' },
+    { index: 1, depth: 1, type: OPERATOR, value: '{', groupId: 'fxg1' },
+    { index: 2, depth: 1, type: OPERATOR, value: '{', error: true },
+    { index: 3, depth: 1, type: OPERATOR, value: '}', groupId: 'fxg1' },
     { index: 4, depth: 0, type: OPERATOR, value: '}', error: true }
   ]);
 
@@ -73,5 +85,26 @@ test('add extra meta to operators', t => {
     { index: 9, depth: 0, type: RANGE_BEAM, value: '[foo]Sheet1!1:1', groupId: 'fxg2' }
   ], { sheetName: 'Sheet1', workbookName: 'foo' });
 
+  t.isMetaTokens('=SUM((1, 2), {3, 4})', [
+    { index: 0, depth: 0, type: FX_PREFIX, value: '=' },
+    { index: 1, depth: 0, type: FUNCTION, value: 'SUM' },
+    { index: 2, depth: 1, type: OPERATOR, value: '(', groupId: 'fxg3' },
+    { index: 3, depth: 2, type: OPERATOR, value: '(', groupId: 'fxg1' },
+    { index: 4, depth: 2, type: NUMBER, value: '1' },
+    { index: 5, depth: 2, type: OPERATOR, value: ',' },
+    { index: 6, depth: 2, type: WHITESPACE, value: ' ' },
+    { index: 7, depth: 2, type: NUMBER, value: '2' },
+    { index: 8, depth: 2, type: OPERATOR, value: ')', groupId: 'fxg1' },
+    { index: 9, depth: 1, type: OPERATOR, value: ',' },
+    { index: 10, depth: 1, type: WHITESPACE, value: ' ' },
+    { index: 11, depth: 2, type: OPERATOR, value: '{', groupId: 'fxg2' },
+    { index: 12, depth: 2, type: NUMBER, value: '3' },
+    { index: 13, depth: 2, type: OPERATOR, value: ',' },
+    { index: 14, depth: 2, type: WHITESPACE, value: ' ' },
+    { index: 15, depth: 2, type: NUMBER, value: '4' },
+    { index: 16, depth: 2, type: OPERATOR, value: '}', groupId: 'fxg2' },
+    { index: 17, depth: 1, type: OPERATOR, value: ')', groupId: 'fxg3' }
+  ], { sheetName: 'Sheet1', workbookName: 'foo' });
+
   t.end();
 });

package/lib/constants.js

@@ -1,101 +1,19 @@
-import { quickVerifyRangeA1, quickVerifyRangeRC } from './quickVerify.js';
-
 export const OPERATOR = 'operator';
 export const BOOLEAN = 'bool';
 export const ERROR = 'error';
 export const NUMBER = 'number';
-export const FUNCTION = 'function';
+export const FUNCTION = 'func';
 export const NEWLINE = 'newline';
 export const WHITESPACE = 'whitespace';
 export const STRING = 'string';
-export const PATH_QUOTE = 'path-quote';
-export const PATH_BRACE = 'path-brace';
-export const PATH_PREFIX = 'path-prefix';
+export const CONTEXT_QUOTE = 'context_quote';
+export const CONTEXT = 'context';
 export const RANGE = 'range';
-export const RANGE_BEAM = 'range-beam';
-export const RANGE_NAMED = 'range-named';
-export const FX_PREFIX = 'fx-prefix';
+export const RANGE_BEAM = 'range_beam';
+export const RANGE_TERNARY = 'range_ternary';
+export const RANGE_NAMED = 'range_named';
+export const FX_PREFIX = 'fx_prefix';
 export const UNKNOWN = 'unknown';
 
 export const MAX_COLS = 2 ** 14 - 1; // 16383
 export const MAX_ROWS = 2 ** 20 - 1; // 1048575
-
-const re_ERROR = /^#(NAME\?|FIELD!|CALC!|VALUE!|REF!|DIV\/0!|NULL!|NUM!|N\/A|GETTING_DATA\b|SPILL!|UNKNOWN!|FIELD\b|CALC\b|SYNTAX\?|ERROR!)/i;
-const re_OPERATOR = /^(<=|>=|<>|[-+/*^%&<>=]|[{},;]|[()]|@|:|!|#)/;
-const re_BOOLEAN = /^(TRUE|FALSE)\b/i;
-const re_FUNCTION = /^[A-Z_]+[A-Z\d_.]+(?=\s*\()/i;
-const re_NEWLINE = /^\n+/;
-const re_WHITESPACE = /^[ \f\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/;
-const re_STRING = /^"(?:""|[^"])*("|$)/;
-const re_PATH_QUOTE = /^'(?:''|[^'])*('|$)/;
-const re_PATH_BRACE = /^\[(?:[^\]])+(\]|$)/;
-const re_PATH_PREFIX = /^([^ \t\n$!"`'#%&(){}<>,;:^@|~=*+-]+)(?=!)/; // Sheets: [^:\\/?*[\]]{0,31} (but WB names?)
-const re_A1COL = /^\$?[A-Z]{1,3}:\$?[A-Z]{1,3}/i;
-const re_A1ROW = /^\$?[1-9][0-9]{0,6}:\$?[1-9][0-9]{0,6}/i;
-const re_A1RANGE = /^\$?[A-Z]{1,3}\$?[1-9][0-9]{0,6}/i;
-const rPart = '(?:R(?:\\[[+-]?\\d+\\]|[1-9][0-9]{0,6})?)';
-const cPart = '(?:C(?:\\[[+-]?\\d+\\]|[1-9][0-9]{0,4})?)';
-const re_RCCOL = new RegExp(`^${cPart}(:${cPart})?(?=\\W|$)`, 'i');
-const re_RCROW = new RegExp(`^${rPart}(:${rPart})?(?=\\W|$)`, 'i');
-const re_RCRANGE = new RegExp(`^(?:(?=[RC])${rPart}${cPart})`, 'i');
-const re_NUMBER = /^(?:\d+(\.\d+)?(?:[eE][+-]?\d+)?|\d+)/;
-const re_NAMED = /^[A-Z\d\\_.?]+/i; // FIXME there are stricter rules for this!
-// const re_NAMED = /^(?![RC]$)[A-ZÀ-ȳ_\\][\\?\wÀ-ȳ.]{0,255}$/i;
-
-export const tokenHandlersA1 = [
-  [ ERROR,       re_ERROR ],
-  [ OPERATOR,    re_OPERATOR ],
-  [ BOOLEAN,     re_BOOLEAN ],
-  [ FUNCTION,    re_FUNCTION ],
-  [ NEWLINE,     re_NEWLINE ],
-  [ WHITESPACE,  re_WHITESPACE ],
-  [ STRING,      re_STRING ],
-  [ PATH_QUOTE,  re_PATH_QUOTE ],
-  [ PATH_BRACE,  re_PATH_BRACE ],
-  [ PATH_PREFIX, re_PATH_PREFIX ],
-  [ RANGE,       re_A1RANGE, quickVerifyRangeA1 ],
-  [ RANGE_BEAM,  re_A1COL, quickVerifyRangeA1 ],
-  [ RANGE_BEAM,  re_A1ROW, quickVerifyRangeA1 ],
-  [ NUMBER,      re_NUMBER ],
-  [ RANGE_NAMED, re_NAMED ]
-];
-
-export const tokenHandlersRC = [
-  [ ERROR,       re_ERROR ],
-  [ OPERATOR,    re_OPERATOR ],
-  [ BOOLEAN,     re_BOOLEAN ],
-  [ FUNCTION,    re_FUNCTION ],
-  [ NEWLINE,     re_NEWLINE ],
-  [ WHITESPACE,  re_WHITESPACE ],
-  [ STRING,      re_STRING ],
-  [ PATH_QUOTE,  re_PATH_QUOTE ],
-  [ PATH_BRACE,  re_PATH_BRACE ],
-  [ PATH_PREFIX, re_PATH_PREFIX ],
-  [ RANGE,       re_RCRANGE, quickVerifyRangeRC ],
-  [ RANGE_BEAM,  re_RCROW, quickVerifyRangeRC ],
-  [ RANGE_BEAM,  re_RCCOL, quickVerifyRangeRC ],
-  [ NUMBER,      re_NUMBER ],
-  [ RANGE_NAMED, re_NAMED ]
-];
-
-export const tokenHandlersRefsA1 = [
-  [ OPERATOR,    /^[!:]/ ],
-  [ PATH_QUOTE,  re_PATH_QUOTE ],
-  [ PATH_BRACE,  re_PATH_BRACE ],
-  [ PATH_PREFIX, re_PATH_PREFIX ],
-  [ RANGE,       re_A1RANGE, quickVerifyRangeA1 ],
-  [ RANGE_BEAM,  re_A1COL, quickVerifyRangeA1 ],
-  [ RANGE_BEAM,  re_A1ROW, quickVerifyRangeA1 ],
-  [ RANGE_NAMED, re_NAMED ]
-];
-
-export const tokenHandlersRefsRC = [
-  [ OPERATOR,    /^!/ ],
-  [ PATH_QUOTE,  re_PATH_QUOTE ],
-  [ PATH_BRACE,  re_PATH_BRACE ],
-  [ PATH_PREFIX, re_PATH_PREFIX ],
-  [ RANGE,       re_RCRANGE, quickVerifyRangeRC ],
-  [ RANGE_BEAM,  re_RCROW, quickVerifyRangeRC ],
-  [ RANGE_BEAM,  re_RCCOL, quickVerifyRangeRC ],
-  [ RANGE_NAMED, re_NAMED ]
-];

package/lib/fixRanges.js

@@ -0,0 +1,41 @@
+import { isRange } from './isType.js';
+import { parseA1Ref, stringifyA1Ref, addRangeBounds } from './a1.js';
+import { tokenize } from './lexer.js';
+
+// There is no R1C1 counerpart to this. This is because without an anchor cell
+// it is impossible to determine if a relative+absolute range (R[1]C[1]:R5C5)
+// needs to be flipped or not. The solution is to convert to A1 first:
+// translateToRC(fixRanges(translateToA1(...)))
+
+export function fixRanges (tokens, options = { addBounds: false }) {
+  if (typeof tokens === 'string') {
+    return fixRanges(tokenize(tokens, options), options)
+      .map(d => d.value)
+      .join('');
+  }
+  if (!Array.isArray(tokens)) {
+    throw new Error('fixRanges expects an array of tokens');
+  }
+  const { addBounds, r1c1 } = options;
+  if (r1c1) {
+    throw new Error('fixRanges does not have an R1C1 mode');
+  }
+  return tokens.map(token => {
+    if (isRange(token)) {
+      const ref = parseA1Ref(token.value, options);
+      const range = ref.range;
+      // fill missing dimensions?
+      if (addBounds) {
+        addRangeBounds(range);
+      }
+      const ret = { ...token };
+      ret.value = stringifyA1Ref(ref);
+      if (ret.range) {
+        ret.range = range;
+      }
+      return ret;
+    }
+    return token;
+  });
+}
+

package/lib/fixRanges.spec.js

@@ -0,0 +1,111 @@
+import { test, Test } from 'tape';
+import { tokenize } from './lexer.js';
+import { addMeta } from './addMeta.js';
+import { fixRanges } from './fixRanges.js';
+import { RANGE } from './constants.js';
+
+Test.prototype.isFixed = function (expr, expected, options = {}) {
+  const result = fixRanges(expr, options);
+  this.is(result, expected, expr + ' → ' + expected);
+};
+
+test('fixRanges basics', t => {
+  const fx = '=SUM([wb]Sheet1!B2:A1)';
+  t.throws(() => fixRanges(123), 'throws on non arrays (number)');
+  t.throws(() => fixRanges(null), 'throws on non arrays (null)');
+  const tokens = addMeta(tokenize(fx, { mergeRanges: true }));
+  tokens[3].foo = 'bar';
+  const fixedTokens = fixRanges(tokens, { debug: 0 });
+  t.ok(tokens !== fixedTokens, 'emits a new array instance');
+  t.ok(tokens[3] !== fixedTokens[3], 'does not mutate existing range tokens');
+  t.deepEqual(tokens[3], {
+    type: RANGE,
+    value: '[wb]Sheet1!B2:A1',
+    index: 3,
+    depth: 1,
+    groupId: 'fxg1',
+    foo: 'bar'
+  }, 'keeps meta (pre-fix range token)');
+  t.deepEqual(fixedTokens[3], {
+    type: RANGE,
+    value: '[wb]Sheet1!A1:B2',
+    index: 3,
+    depth: 1,
+    groupId: 'fxg1',
+    foo: 'bar'
+  }, 'keeps meta (post-fix range token)');
+  // fixes all range meta
+  t.end();
+});
+
+test('fixRanges A1', t => {
+  const opt = { allowTernary: true };
+  // doesn't mess with things that it doesn't have to
+  t.isFixed('=A1', '=A1', opt);
+  t.isFixed('=ZZ123', '=ZZ123', opt);
+  t.isFixed('=A1:B2', '=A1:B2', opt);
+  t.isFixed('=B3:OFFSET(A1,10,10)', '=B3:OFFSET(A1,10,10)', opt);
+  t.isFixed('=A:B', '=A:B', opt);
+  t.isFixed('=C:C', '=C:C', opt);
+  t.isFixed('=3:6', '=3:6', opt);
+  t.isFixed('=3:3', '=3:3', opt);
+  // redundancy
+  t.isFixed('=A1:$A$1', '=A1:$A$1', opt);
+  t.isFixed('=A1:A1', '=A1', opt);
+  // lowercase to uppercase
+  t.isFixed('=a1', '=A1', opt);
+  t.isFixed('=zz123', '=ZZ123', opt);
+  t.isFixed('=a1:b2', '=A1:B2', opt);
+  // flipped rects
+  t.isFixed('=B2:A1', '=A1:B2', opt);
+  t.isFixed('=$B$2:$A$1', '=$A$1:$B$2', opt);
+  // flipped beams
+  t.isFixed('=C:A', '=A:C', opt);
+  t.isFixed('=$D:B', '=B:$D', opt);
+  t.isFixed('=10:1', '=1:10', opt);
+  t.isFixed('=$5:3', '=3:$5', opt);
+  // flipped partials - bottom
+  t.isFixed('=A:A1', '=A1:A', opt);
+  t.isFixed('=A:A$1', '=A$1:A', opt);
+  // flipped partials - right
+  t.isFixed('=1:A1', '=A1:1', opt);
+  // $1:$A1 is rather counter intuitive case:
+  //   This range is parsed as { left=null, top=$1, right=$A, bottom=1 } but,
+  //   because left is null, right and left are flipped around, making this
+  //   end up as { left=$A, top=$1, right=null, bottom=1 } which serializes
+  //   as $A$1:1
+  t.isFixed('=$1:$A1', '=$A$1:1', opt);
+  t.end();
+});
+
+test('fixRanges A1 addBounds', t => {
+  const opt = { allowTernary: true, addBounds: true };
+  t.isFixed('=B3:OFFSET(A1,10,10)', '=B3:OFFSET(A1,10,10)', opt);
+  t.isFixed('=A:A', '=A:A', opt);
+  t.isFixed('=A:A1', '=A:A', opt);
+  t.isFixed('=A:A$1', '=A:A', opt);
+  t.isFixed('=A:$A$1', '=A:$A', opt);
+  // partials - bottom
+  t.isFixed('=A1:A', '=A:A', opt);
+  t.isFixed('=A1:Z', '=A:Z', opt);
+  t.isFixed('=A:A1', '=A:A', opt);
+  t.isFixed('=$A1:A', '=$A:A', opt);
+  t.isFixed('=A$1:A', '=A:A', opt);
+  t.isFixed('=A1:$A', '=A:$A', opt);
+  t.isFixed('=A2:A', '=A2:A1048576', opt);
+  t.isFixed('=B2:B', '=B2:B1048576', opt);
+  t.isFixed('=A:A2', '=A2:A1048576', opt);
+  t.isFixed('=B:B2', '=B2:B1048576', opt);
+  // flipped partials - bottom
+  t.isFixed('=A1:1', '=1:1', opt);
+  t.isFixed('=A1:4', '=1:4', opt);
+  t.isFixed('=1:A1', '=1:1', opt);
+  t.isFixed('=$A1:1', '=1:1', opt);
+  t.isFixed('=A$1:1', '=$1:1', opt);
+  t.isFixed('=A1:$1', '=1:$1', opt);
+  t.isFixed('=B1:1', '=B1:XFD1', opt);
+  t.isFixed('=1:B1', '=B1:XFD1', opt);
+  t.isFixed('=B2:20', '=B2:XFD20', opt);
+  t.isFixed('=2:B20', '=B2:XFD20', opt);
+  t.end();
+});

package/lib/index.js

@@ -4,6 +4,9 @@ export { translateToRC, translateToA1 } from './translate.js';
 export { default as a1 } from './a1.js';
 export { default as rc } from './rc.js';
 export { MAX_COLS, MAX_ROWS } from './constants.js';
+export { isReference, isRange } from './isType.js';
+export { mergeRefTokens as mergeRanges } from './mergeRefTokens.js';
+export { fixRanges } from './fixRanges.js';
 
 import {
   OPERATOR,
@@ -14,11 +17,11 @@ import {
   NEWLINE,
   WHITESPACE,
   STRING,
-  PATH_QUOTE,
-  PATH_BRACE,
-  PATH_PREFIX,
+  CONTEXT,
+  CONTEXT_QUOTE,
   RANGE,
   RANGE_BEAM,
+  RANGE_TERNARY,
   RANGE_NAMED,
   FX_PREFIX,
   UNKNOWN
@@ -33,11 +36,11 @@ export const tokenTypes = {
   NEWLINE,
   WHITESPACE,
   STRING,
-  PATH_QUOTE,
-  PATH_BRACE,
-  PATH_PREFIX,
+  CONTEXT,
+  CONTEXT_QUOTE,
   RANGE,
   RANGE_BEAM,
+  RANGE_TERNARY,
   RANGE_NAMED,
   FX_PREFIX,
   UNKNOWN

package/lib/isType.js

@@ -0,0 +1,18 @@
+import { RANGE, RANGE_BEAM, RANGE_NAMED, RANGE_TERNARY } from './constants.js';
+
+export function isRange (token) {
+  return !!token && (
+    token.type === RANGE ||
+    token.type === RANGE_BEAM ||
+    token.type === RANGE_TERNARY
+  );
+}
+
+export function isReference (token) {
+  return !!token && (
+    token.type === RANGE ||
+    token.type === RANGE_BEAM ||
+    token.type === RANGE_TERNARY ||
+    token.type === RANGE_NAMED
+  );
+}

package/lib/lexer.js

@@ -3,38 +3,74 @@ import {
   NEWLINE,
   NUMBER,
   OPERATOR,
-  PATH_BRACE,
-  PATH_PREFIX,
-  PATH_QUOTE,
-  RANGE,
-  RANGE_BEAM,
   RANGE_NAMED,
   STRING,
   UNKNOWN,
   WHITESPACE,
-  tokenHandlersA1,
-  tokenHandlersRC
+  FUNCTION
 } from './constants.js';
+import { lexers } from './lexerParts.js';
+import { mergeRefTokens } from './mergeRefTokens.js';
 
 const isType = (t, type) => t && t.type === type;
-const isRangeOp = t => t && t.value === ':';
-const isBangOp = t => t && t.value === '!';
 
 const defaultOptions = {
   emitRanges: false,
   mergeRanges: true,
-  negativeNumbers: false,
+  allowTernary: false,
+  negativeNumbers: true,
   r1c1: false
 };
 
+const isTextToken = token => {
+  return (
+    token.type === RANGE_NAMED ||
+    token.type === FUNCTION
+  );
+};
+
+const causesBinaryMinus = token => {
+  return !isType(token, OPERATOR) || (
+    token.value === '%' ||
+    token.value === '}' ||
+    token.value === ')' ||
+    token.value === '#'
+  );
+};
+
 export function getTokens (fx, tokenHandlers, options = {}) {
-  const { emitRanges, mergeRanges, negativeNumbers } = Object.assign({}, defaultOptions, options);
+  const opts = Object.assign({}, defaultOptions, options);
+  const { emitRanges, mergeRanges, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
-  const lookBehind = n => tokens[tokens.length - n];
-  const lookBehindIgnoreWS = n => {
-    const noWs = tokens.filter(t => !isType(t, WHITESPACE) && !isType(t, NEWLINE));
-    return noWs[noWs.length - n];
+
+  let tail0 = null; // last non-whitespace token
+  let tail1 = null; // penultimate non-whitespace token
+  let lastToken = null; // last token
+  const pushToken = token => {
+    const isCurrUnknown = token.type === UNKNOWN;
+    const isLastUnknown = lastToken && lastToken.type === UNKNOWN;
+    if (lastToken && (
+      (isCurrUnknown && isLastUnknown) ||
+      (isCurrUnknown && isTextToken(lastToken)) ||
+      (isLastUnknown && isTextToken(token))
+    )) {
+      // UNKNOWN tokens "contaminate" sibling text tokens
+      lastToken.value += token.value;
+      lastToken.type = UNKNOWN;
+      if (emitRanges) {
+        lastToken.range[1] = token.range[1];
+      }
+    }
+    else {
+      // push token as normally
+      tokens.push(token);
+      lastToken = token;
+      if (token.type !== WHITESPACE && token.type !== NEWLINE) {
+        tail1 = tail0;
+        tail0 = token;
+      }
+    }
   };
 
   if (/^=/.test(fx)) {
@@ -44,7 +80,7 @@ export function getTokens (fx, tokenHandlers, options = {}) {
       ...(emitRanges ? { range: [ 0, 1 ] } : {})
     };
     pos++;
-    tokens.push(token);
+    pushToken(token);
   }
 
   while (pos < fx.length) {
@@ -53,12 +89,11 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     let tokenType = '';
     let tokenValue = '';
     for (let i = 0; i < tokenHandlers.length; i++) {
-      const [ type, reTest, fnTest ] = tokenHandlers[i];
-      const m = reTest.exec(s);
-      if (m && (!fnTest || fnTest(m[0]))) {
-        tokenType = type;
-        tokenValue = m[0];
-        pos += m[0].length;
+      const t = tokenHandlers[i](s, opts);
+      if (t) {
+        tokenType = t.type;
+        tokenValue = t.value;
+        pos += tokenValue.length;
         break;
       }
     }
@@ -69,72 +104,66 @@ export function getTokens (fx, tokenHandlers, options = {}) {
       pos++;
     }
 
-    let token = {
+    const token = {
       type: tokenType,
       value: tokenValue,
       ...(emitRanges ? { range: [ startPos, pos ] } : {})
     };
 
-    if (tokenType === STRING && (tokenValue === '"' || !tokenValue.endsWith('"'))) {
-      token.unterminated = true;
-    }
-    else if (tokenType === PATH_BRACE && !tokenValue.endsWith(']')) {
-      token.unterminated = true;
-    }
-    else if (tokenType === PATH_QUOTE && (tokenValue === "'" || !tokenValue.endsWith("'"))) {
-      token.unterminated = true;
+    // check for termination
+    if (tokenType === STRING) {
+      const l = tokenValue.length;
+      if (tokenValue === '""') {
+        // common case that IS terminated
+      }
+      else if (tokenValue === '"' || tokenValue[l - 1] !== '"') {
+        token.unterminated = true;
+      }
+      else if (tokenValue !== '""' && tokenValue[l - 2] === '"') {
+        let p = l - 1;
+        while (tokenValue[p] === '"') { p--; }
+        const atStart = (p + 1);
+        const oddNum = ((l - p + 1) % 2 === 0);
+        if (!atStart ^ oddNum) {
+          token.unterminated = true;
+        }
+      }
     }
 
     if (negativeNumbers && tokenType === NUMBER) {
-      const last1 = lookBehind(1);
+      const last1 = lastToken;
+      // do we have a number preceded by a minus?
       if (last1 && isType(last1, OPERATOR) && last1.value === '-') {
-        // we have a number preceded by a minus
-        const last2 = lookBehindIgnoreWS(2);
-        // missing last2 means we are at the start of the stream
-        if (!last2 || isType(last2, FX_PREFIX) || (isType(last2, OPERATOR) && ![ '%', '}', ')', '#' ].includes(last2.value))) {
-          tokens.pop();
+        // missing tail1 means we are at the start of the stream
+        if (
+          !tail1 ||
+          isType(tail1, FX_PREFIX) ||
+          !causesBinaryMinus(tail1, OPERATOR)
+        ) {
+          const minus = tokens.pop();
           token.value = '-' + tokenValue;
-        }
-      }
-    }
-
-    if (mergeRanges) {
-      if (tokenType === RANGE || tokenType === RANGE_NAMED || tokenType === RANGE_BEAM) {
-        const merge = [];
-        // join A1:A1 or RC:RC-
-        if (isRangeOp(lookBehind(1)) && isType(lookBehind(2), RANGE) && tokenType === RANGE) {
-          // should not be done this if current or last ranges are A:A or 1:1, or R or C)
-          merge.unshift(...tokens.splice(-2, 2));
-        }
-        // join prefixes
-        if (isBangOp(lookBehind(1))) {
-          if (isType(lookBehind(2), PATH_QUOTE) || isType(lookBehind(2), PATH_BRACE)) {
-            merge.unshift(...tokens.splice(-2, 2));
-          }
-          else if (isType(lookBehind(2), PATH_PREFIX)) {
-            const n = isType(lookBehind(3), PATH_BRACE) ? 3 : 2;
-            merge.unshift(...tokens.splice(-n, n));
+          if (emitRanges) {
+            // ensure range offsets are up to date
+            token.range[0] = minus.range[0];
           }
-        }
-        // want to merge?
-        if (merge.length) {
-          token = {
-            type: tokenType,
-            value: merge.map(d => d.value).join('') + tokenValue,
-            ...(emitRanges ? { range: [ merge[0].range[0], pos ] } : {})
-          };
+          // next step tries to counter the screwing around with the tailing
+          // it should be correct again once we pushToken()
+          tail0 = tail1;
+          lastToken = tokens[tokens.length - 1];
         }
       }
     }
-    tokens.push(token);
+
+    pushToken(token);
+  }
+
+  if (mergeRanges) {
+    return mergeRefTokens(tokens);
   }
 
   return tokens;
 }
 
-// Formulas can either have RC or A1 style refs, not both: because C1 and R1 are both!
-// Refmode: A1 | RC | Anostic emit ranges with RANGE
 export function tokenize (fx, options = {}) {
-  const tokenHandlers = options.r1c1 ? tokenHandlersRC : tokenHandlersA1;
-  return getTokens(fx, tokenHandlers, options);
+  return getTokens(fx, lexers, options);
 }