@borgar/fx 2.1.1 → 3.0.0

package/lib/{lexer-test.js → lexer.spec.js}

@@ -1,9 +1,13 @@
 import { test, Test } from 'tape';
-import { FX_PREFIX, OPERATOR, BOOLEAN, ERROR, NUMBER, FUNCTION, WHITESPACE, STRING, PATH_PREFIX, PATH_QUOTE, PATH_BRACE, RANGE, RANGE_BEAM, RANGE_NAMED } from './constants.js';
+import {
+  FX_PREFIX, UNKNOWN,
+  OPERATOR, BOOLEAN, ERROR, NUMBER, FUNCTION, WHITESPACE, STRING,
+  RANGE, RANGE_BEAM, RANGE_NAMED, RANGE_TERNARY, CONTEXT, CONTEXT_QUOTE, NEWLINE
+} from './constants.js';
 import { tokenize } from './lexer.js';
 
 Test.prototype.isTokens = function isTokens (expr, result, opts) {
-  this.deepEqual(tokenize(expr, (opts || {})), result, expr);
+  this.deepEqual(tokenize(expr, { negativeNumbers: false, ...opts }), result, expr);
 };
 Test.prototype.isTokensNeg = function isTokensNeg (expr, result, opts) {
   this.deepEqual(tokenize(expr, { ...opts, negativeNumbers: true }), result, expr);
@@ -106,7 +110,7 @@ test('tokenize operators', t => {
   ]);
   t.isTokens('=Sheet1!A1', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sheet1' },
+    { type: CONTEXT, value: 'Sheet1' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -229,6 +233,12 @@ test('tokenize functions', t => {
     { type: NUMBER, value: '1' },
     { type: OPERATOR, value: ')' }
   ]);
+  t.isTokens('=N()', [
+    { type: FX_PREFIX, value: '=' },
+    { type: FUNCTION, value: 'N' },
+    { type: OPERATOR, value: '(' },
+    { type: OPERATOR, value: ')' }
+  ]);
   t.isTokens('=@SUM(1)', [
     { type: FX_PREFIX, value: '=' },
     { type: OPERATOR, value: '@' },
@@ -307,9 +317,9 @@ test('tokenize functions', t => {
     { type: NUMBER, value: '1' },
     { type: OPERATOR, value: ')' }
   ]);
-  t.isTokens('=.FOO(1)', [
+  t.isTokens('=\\FOO(1)', [
     { type: FX_PREFIX, value: '=' },
-    { type: RANGE_NAMED, value: '.FOO' },
+    { type: RANGE_NAMED, value: '\\FOO' },
     { type: OPERATOR, value: '(' },
     { type: NUMBER, value: '1' },
     { type: OPERATOR, value: ')' }
@@ -425,7 +435,7 @@ test('tokenize negative numbers', t => {
     { type: NUMBER, value: '-1.5E-10' }
   ]);
   t.isTokensNeg('-1', [
-    { type: 'number', value: '-1' }
+    { type: NUMBER, value: '-1' }
   ]);
 
   //
@@ -436,144 +446,144 @@ test('tokenize negative numbers', t => {
     { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('1--1', [
-    { type: 'number', value: '1' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '-1' }
+    { type: NUMBER, value: '1' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '-1' }
   ]);
   t.isTokensNeg('1 - -1', [
-    { type: 'number', value: '1' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'number', value: '-1' }
+    { type: NUMBER, value: '1' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NUMBER, value: '-1' }
   ]);
   t.isTokensNeg('1 - - 1', [
-    { type: 'number', value: '1' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'number', value: '1' }
+    { type: NUMBER, value: '1' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('1 \n  - \n  -1', [
-    { type: 'number', value: '1' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'newline', value: '\n' },
-    { type: 'whitespace', value: '  ' },
-    { type: 'operator', value: '-' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'newline', value: '\n' },
-    { type: 'whitespace', value: '  ' },
-    { type: 'number', value: '-1' }
+    { type: NUMBER, value: '1' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NEWLINE, value: '\n' },
+    { type: WHITESPACE, value: '  ' },
+    { type: OPERATOR, value: '-' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NEWLINE, value: '\n' },
+    { type: WHITESPACE, value: '  ' },
+    { type: NUMBER, value: '-1' }
   ]);
   t.isTokensNeg('-(-1)', [
-    { type: 'operator', value: '-' },
-    { type: 'operator', value: '(' },
-    { type: 'number', value: '-1' },
-    { type: 'operator', value: ')' }
+    { type: OPERATOR, value: '-' },
+    { type: OPERATOR, value: '(' },
+    { type: NUMBER, value: '-1' },
+    { type: OPERATOR, value: ')' }
   ]);
   t.isTokensNeg('-( -1 )', [
-    { type: 'operator', value: '-' },
-    { type: 'operator', value: '(' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'number', value: '-1' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: ')' }
+    { type: OPERATOR, value: '-' },
+    { type: OPERATOR, value: '(' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NUMBER, value: '-1' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: ')' }
   ]);
 
   t.isTokensNeg('=true-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'bool', value: 'true' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: BOOLEAN, value: 'true' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=true -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'bool', value: 'true' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: BOOLEAN, value: 'true' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=true - 1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'bool', value: 'true' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: BOOLEAN, value: 'true' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=#VALUE!-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'error', value: '#VALUE!' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: ERROR, value: '#VALUE!' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=#VALUE! -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'error', value: '#VALUE!' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: ERROR, value: '#VALUE!' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=SUM(-1) -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'function', value: 'SUM' },
-    { type: 'operator', value: '(' },
-    { type: 'number', value: '-1' },
-    { type: 'operator', value: ')' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: FUNCTION, value: 'SUM' },
+    { type: OPERATOR, value: '(' },
+    { type: NUMBER, value: '-1' },
+    { type: OPERATOR, value: ')' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=SUM( -1)-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'function', value: 'SUM' },
-    { type: 'operator', value: '(' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'number', value: '-1' },
-    { type: 'operator', value: ')' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: FUNCTION, value: 'SUM' },
+    { type: OPERATOR, value: '(' },
+    { type: WHITESPACE, value: ' ' },
+    { type: NUMBER, value: '-1' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=A1-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'range', value: 'A1' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE, value: 'A1' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=A1 -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'range', value: 'A1' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE, value: 'A1' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=foo-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'range-named', value: 'foo' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: 'foo' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('=foo -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'range-named', value: 'foo' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: 'foo' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('="true"-1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'string', value: '"true"' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"true"' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
   t.isTokensNeg('="true" -1', [
-    { type: 'fx-prefix', value: '=' },
-    { type: 'string', value: '"true"' },
-    { type: 'whitespace', value: ' ' },
-    { type: 'operator', value: '-' },
-    { type: 'number', value: '1' }
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"true"' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '-' },
+    { type: NUMBER, value: '1' }
   ]);
 
   t.isTokensNeg('=SUM(1)-1', [
@@ -653,7 +663,7 @@ test('tokenize simple equations', t => {
   ]);
   t.isTokens('    =     (     1.1+2  )   -       3  ', [
     { type: WHITESPACE, value: '    ' },
-    { type: OPERATOR, value: '=' }, // FX_PREFIX
+    { type: OPERATOR, value: '=' }, // FX_PREFIX?
     { type: WHITESPACE, value: '     ' },
     { type: OPERATOR, value: '(' },
     { type: WHITESPACE, value: '     ' },
@@ -866,8 +876,7 @@ test('tokenize R1C1 style references', t => {
 
   t.isTokens('=[filename]Sheetname!R[-2]C:R[-1]C', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_BRACE, value: '[filename]' },
-    { type: PATH_PREFIX, value: 'Sheetname' },
+    { type: CONTEXT, value: '[filename]Sheetname' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'R[-2]C' },
     { type: OPERATOR, value: ':' },
@@ -992,7 +1001,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens('=Sheetname!A1', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sheetname' },
+    { type: CONTEXT, value: 'Sheetname' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -1004,7 +1013,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens('=Sheet1!A1:B2', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sheet1' },
+    { type: CONTEXT, value: 'Sheet1' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' },
     { type: OPERATOR, value: ':' },
@@ -1018,7 +1027,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='Sheet name'!A1:B2", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'Sheet name'" },
+    { type: CONTEXT_QUOTE, value: "'Sheet name'" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' },
     { type: OPERATOR, value: ':' },
@@ -1032,7 +1041,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='Sheets'' name'!A1:B2", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'Sheets'' name'" },
+    { type: CONTEXT_QUOTE, value: "'Sheets'' name'" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' },
     { type: OPERATOR, value: ':' },
@@ -1046,8 +1055,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens('=[filename]Sheetname!A1', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_BRACE, value: '[filename]' },
-    { type: PATH_PREFIX, value: 'Sheetname' },
+    { type: CONTEXT, value: '[filename]Sheetname' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -1059,7 +1067,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='[filename]Sheets'' name'!A1:B2", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'[filename]Sheets'' name'" },
+    { type: CONTEXT_QUOTE, value: "'[filename]Sheets'' name'" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' },
     { type: OPERATOR, value: ':' },
@@ -1073,7 +1081,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='Run forest, run!'!A1", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'Run forest, run!'" },
+    { type: CONTEXT_QUOTE, value: "'Run forest, run!'" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -1085,7 +1093,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='foo'''!A1", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'foo'''" },
+    { type: CONTEXT_QUOTE, value: "'foo'''" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -1097,7 +1105,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens("='foo'''''!A1", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'foo'''''" },
+    { type: CONTEXT_QUOTE, value: "'foo'''''" },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'A1' }
   ], { mergeRanges: false });
@@ -1107,16 +1115,15 @@ test('tokenize A1 style references', t => {
     { type: RANGE, value: '[15]Sheet32!X4' }
   ]);
 
+  // illegal syntax
   t.isTokens('=[15]!named', [
     { type: FX_PREFIX, value: '=' },
-    { type: RANGE_NAMED, value: '[15]!named' }
-  ]);
-  t.isTokens('=[15]!named', [
-    { type: FX_PREFIX, value: '=' },
-    { type: PATH_BRACE, value: '[15]' },
+    { type: UNKNOWN, value: '[' },
+    { type: NUMBER, value: '15' },
+    { type: UNKNOWN, value: ']' },
     { type: OPERATOR, value: '!' },
     { type: RANGE_NAMED, value: 'named' }
-  ], { mergeRanges: false });
+  ]);
 
   t.isTokens('=filename!named', [
     { type: FX_PREFIX, value: '=' },
@@ -1124,15 +1131,14 @@ test('tokenize A1 style references', t => {
   ]);
   t.isTokens('=filename!named', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'filename' },
+    { type: CONTEXT, value: 'filename' },
     { type: OPERATOR, value: '!' },
     { type: RANGE_NAMED, value: 'named' }
   ], { mergeRanges: false });
 
   t.isTokens('=[15]Sheet32!X4', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_BRACE, value: '[15]' },
-    { type: PATH_PREFIX, value: 'Sheet32' },
+    { type: CONTEXT, value: '[15]Sheet32' },
     { type: OPERATOR, value: '!' },
     { type: RANGE, value: 'X4' }
   ], { mergeRanges: false });
@@ -1158,7 +1164,7 @@ test('tokenize A1 style references', t => {
   //
   t.isTokens("='D:\\Reports\\Sales.xlsx'!namedrange", [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'D:\\Reports\\Sales.xlsx'" },
+    { type: CONTEXT_QUOTE, value: "'D:\\Reports\\Sales.xlsx'" },
     { type: OPERATOR, value: '!' },
     { type: RANGE_NAMED, value: 'namedrange' }
   ], { mergeRanges: false });
@@ -1169,7 +1175,7 @@ test('tokenize A1 style references', t => {
 
   t.isTokens('=Sales.xlsx!namedrange', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sales.xlsx' },
+    { type: CONTEXT, value: 'Sales.xlsx' },
     { type: OPERATOR, value: '!' },
     { type: RANGE_NAMED, value: 'namedrange' }
   ], { mergeRanges: false });
@@ -1184,7 +1190,7 @@ test('tokenize A1 style references', t => {
   ]);
   t.isTokens('=Sheet1!A:A', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sheet1' },
+    { type: CONTEXT, value: 'Sheet1' },
     { type: OPERATOR, value: '!' },
     { type: RANGE_BEAM, value: 'A:A' }
   ], { mergeRanges: false });
@@ -1197,7 +1203,7 @@ test('tokenize A1 style references', t => {
   ]);
   t.isTokens('=Sheet1!A:A:B:B', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_PREFIX, value: 'Sheet1' },
+    { type: CONTEXT, value: 'Sheet1' },
     { type: OPERATOR, value: '!' },
     { type: RANGE_BEAM, value: 'A:A' },
     { type: OPERATOR, value: ':' },
@@ -1269,7 +1275,7 @@ test('tokenize errors', t => {
   t.isTokens('=#NONSENSE!', [
     { type: FX_PREFIX, value: '=' },
     { type: OPERATOR, value: '#' },
-    { type: PATH_PREFIX, value: 'NONSENSE' },
+    { type: CONTEXT, value: 'NONSENSE' },
     { type: OPERATOR, value: '!' }
   ]);
 
@@ -1338,26 +1344,323 @@ test('tokenize strings', t => {
   // we should be able to highlight things that are still being typed
   // so open-ended STRING, PATH_BRACE, and PATH_QUOTE tokens at the
   // end of output are tagged.
+  t.isTokens('="incomple', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"incomple', unterminated: true }
+  ]);
+
   t.isTokens('="', [
     { type: FX_PREFIX, value: '=' },
     { type: STRING, value: '"', unterminated: true }
   ]);
-  t.isTokens('="incomple', [
+  t.isTokens('=""', [
     { type: FX_PREFIX, value: '=' },
-    { type: STRING, value: '"incomple', unterminated: true }
+    { type: STRING, value: '""' }
+  ]);
+  t.isTokens('="""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"""', unterminated: true }
+  ]);
+  t.isTokens('=""""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '""""' }
+  ]);
+  t.isTokens('="""""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"""""', unterminated: true }
+  ]);
+  t.isTokens('=""""""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '""""""' }
+  ]);
+  t.isTokens('="aa""ss', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"aa""ss', unterminated: true }
+  ]);
+  t.isTokens('="aa""ss"', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"aa""ss"' }
+  ]);
+  t.isTokens('="aa""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"aa""', unterminated: true }
+  ]);
+  t.isTokens('="aa"""', [
+    { type: FX_PREFIX, value: '=' },
+    { type: STRING, value: '"aa"""' }
+  ]);
+
+  t.end();
+});
+
+test('unknown and unary minus mess with offsets', t => {
+  t.isTokens('=-1', [
+    { type: FX_PREFIX, value: '=', range: [ 0, 1 ] },
+    { type: OPERATOR, value: '-', range: [ 1, 2 ] },
+    { type: NUMBER, value: '1', range: [ 2, 3 ] }
+  ], { emitRanges: true });
+  t.isTokens('=-1', [
+    { type: FX_PREFIX, value: '=', range: [ 0, 1 ] },
+    { type: NUMBER, value: '-1', range: [ 1, 3 ] }
+  ], { emitRanges: true, negativeNumbers: true });
+
+  t.isTokens('=$C', [
+    { type: FX_PREFIX, value: '=', range: [ 0, 1 ] },
+    { type: UNKNOWN, value: '$C', range: [ 1, 3 ] }
+  ], { emitRanges: true });
+  t.isTokens('=$C.foo', [
+    { type: FX_PREFIX, value: '=', range: [ 0, 1 ] },
+    { type: UNKNOWN, value: '$C.foo', range: [ 1, 7 ] }
+  ], { emitRanges: true });
+
+  t.end();
+});
+
+test('unknowns, named ranges and functions', t => {
+  t.isTokens('=foo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: 'foo' }
+  ]);
+  t.isTokens('=_foo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: '_foo' }
+  ]);
+  t.isTokens('=\\foo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: '\\foo' }
+  ]);
+  t.isTokens('=æði', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: 'æði' }
+  ]);
+  t.isTokens('=らーめん', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: 'らーめん' }
+  ]);
+  t.isTokens('=@foo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: OPERATOR, value: '@' },
+    { type: RANGE_NAMED, value: 'foo' }
+  ]);
+  t.isTokens('=9æði', [
+    { type: FX_PREFIX, value: '=' },
+    { type: NUMBER, value: '9' },
+    { type: RANGE_NAMED, value: 'æði' }
+  ]);
+  t.isTokens('=¢mah¢', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_NAMED, value: '¢mah¢' }
+  ]);
+  t.isTokens('=~mah~', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: '~mah~' }
   ]);
-  t.isTokens('=[Foo', [
+  t.isTokens('=$foo', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_BRACE, value: '[Foo', unterminated: true }
+    { type: UNKNOWN, value: '$foo' }
   ]);
-  t.isTokens("='", [
+  t.isTokens('=$zzzz12', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'", unterminated: true }
+    { type: UNKNOWN, value: '$zzzz12' }
   ]);
-  t.isTokens("='Sheet name", [
+  t.isTokens('=~zzzz12()', [
     { type: FX_PREFIX, value: '=' },
-    { type: PATH_QUOTE, value: "'Sheet name", unterminated: true }
+    { type: UNKNOWN, value: '~zzzz12' },
+    { type: OPERATOR, value: '(' },
+    { type: OPERATOR, value: ')' }
+  ]);
+  t.isTokens('=zzzz~12()', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: 'zzzz~' },
+    { type: NUMBER, value: '12' },
+    { type: OPERATOR, value: '(' },
+    { type: OPERATOR, value: ')' }
   ]);
+  t.end();
+});
+
+test('tokenize partial ranges', t => {
+  const opts = { allowTernary: true };
+  t.isTokens('1:D$1', [
+    { type: RANGE_TERNARY, value: '1:D$1' }
+  ], opts);
+
+  t.isTokens('1:D$1', [
+    { type: NUMBER, value: '1' },
+    { type: OPERATOR, value: ':' },
+    { type: RANGE, value: 'D$1' }
+  ]);
+
+  t.isTokens('B2:B', [
+    { type: RANGE_TERNARY, value: 'B2:B' }
+  ], opts);
+  t.isTokens('B2:B', [
+    { type: RANGE, value: 'B2' },
+    { type: OPERATOR, value: ':' },
+    { type: RANGE_NAMED, value: 'B' }
+  ]);
+
+  // form 1
+  t.isTokens('1:A1', [
+    { type: RANGE_TERNARY, value: '1:A1' }
+  ], opts);
+  t.isTokens('$1:A1', [
+    { type: RANGE_TERNARY, value: '$1:A1' }
+  ], opts);
+  t.isTokens('1:$A1', [
+    { type: RANGE_TERNARY, value: '1:$A1' }
+  ], opts);
+  t.isTokens('1:A$1', [
+    { type: RANGE_TERNARY, value: '1:A$1' }
+  ], opts);
+  t.isTokens('1:$A$1', [
+    { type: RANGE_TERNARY, value: '1:$A$1' }
+  ], opts);
+  t.isTokens('$1:A$1', [
+    { type: RANGE_TERNARY, value: '$1:A$1' }
+  ], opts);
+  t.isTokens('$1:$A1', [
+    { type: RANGE_TERNARY, value: '$1:$A1' }
+  ], opts);
+  t.isTokens('$1:$A$1', [
+    { type: RANGE_TERNARY, value: '$1:$A$1' }
+  ], opts);
+
+  // form 2
+  t.isTokens('A1:1', [
+    { type: RANGE_TERNARY, value: 'A1:1' }
+  ], opts);
+  t.isTokens('A1:$1', [
+    { type: RANGE_TERNARY, value: 'A1:$1' }
+  ], opts);
+  t.isTokens('$A1:1', [
+    { type: RANGE_TERNARY, value: '$A1:1' }
+  ], opts);
+  t.isTokens('A$1:1', [
+    { type: RANGE_TERNARY, value: 'A$1:1' }
+  ], opts);
+  t.isTokens('$A$1:1', [
+    { type: RANGE_TERNARY, value: '$A$1:1' }
+  ], opts);
+  t.isTokens('A$1:$1', [
+    { type: RANGE_TERNARY, value: 'A$1:$1' }
+  ], opts);
+  t.isTokens('$A1:$1', [
+    { type: RANGE_TERNARY, value: '$A1:$1' }
+  ], opts);
+  t.isTokens('$A$1:$1', [
+    { type: RANGE_TERNARY, value: '$A$1:$1' }
+  ], opts);
+
+  // form 3
+  t.isTokens('A:A1', [
+    { type: RANGE_TERNARY, value: 'A:A1' }
+  ], opts);
+  t.isTokens('$A:A1', [
+    { type: RANGE_TERNARY, value: '$A:A1' }
+  ], opts);
+  t.isTokens('A:$A1', [
+    { type: RANGE_TERNARY, value: 'A:$A1' }
+  ], opts);
+  t.isTokens('A:A$1', [
+    { type: RANGE_TERNARY, value: 'A:A$1' }
+  ], opts);
+  t.isTokens('A:$A$1', [
+    { type: RANGE_TERNARY, value: 'A:$A$1' }
+  ], opts);
+  t.isTokens('$A:A$1', [
+    { type: RANGE_TERNARY, value: '$A:A$1' }
+  ], opts);
+  t.isTokens('$A:$A1', [
+    { type: RANGE_TERNARY, value: '$A:$A1' }
+  ], opts);
+  t.isTokens('$A:$A$1', [
+    { type: RANGE_TERNARY, value: '$A:$A$1' }
+  ], opts);
+
+  // form 4
+  t.isTokens('A1:A', [
+    { type: RANGE_TERNARY, value: 'A1:A' }
+  ], opts);
+  t.isTokens('A1:$A', [
+    { type: RANGE_TERNARY, value: 'A1:$A' }
+  ], opts);
+  t.isTokens('$A1:A', [
+    { type: RANGE_TERNARY, value: '$A1:A' }
+  ], opts);
+  t.isTokens('A$1:A', [
+    { type: RANGE_TERNARY, value: 'A$1:A' }
+  ], opts);
+  t.isTokens('$A$1:A', [
+    { type: RANGE_TERNARY, value: '$A$1:A' }
+  ], opts);
+  t.isTokens('A$1:$A', [
+    { type: RANGE_TERNARY, value: 'A$1:$A' }
+  ], opts);
+  t.isTokens('$A1:$A', [
+    { type: RANGE_TERNARY, value: '$A1:$A' }
+  ], opts);
+  t.isTokens('$A$1:$A', [
+    { type: RANGE_TERNARY, value: '$A$1:$A' }
+  ], opts);
+
+  t.isTokens('=A10:A+B1:2', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE_TERNARY, value: 'A10:A' },
+    { type: OPERATOR, value: '+' },
+    { type: RANGE_TERNARY, value: 'B1:2' }
+  ], opts);
+  t.isTokens('=SUM(A:A$10,3:B$2)', [
+    { type: FX_PREFIX, value: '=' },
+    { type: FUNCTION, value: 'SUM' },
+    { type: OPERATOR, value: '(' },
+    { type: RANGE_TERNARY, value: 'A:A$10' },
+    { type: OPERATOR, value: ',' },
+    { type: RANGE_TERNARY, value: '3:B$2' },
+    { type: OPERATOR, value: ')' }
+  ], opts);
+  t.isTokens('$A$10:$12', [
+    { type: RANGE_TERNARY, value: '$A$10:$12' }
+  ], opts);
+  t.isTokens('1:D$1', [
+    { type: RANGE_TERNARY, value: '1:D$1' }
+  ], opts);
+  t.isTokens('=A1:IF()', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE, value: 'A1' },
+    { type: OPERATOR, value: ':' },
+    { type: FUNCTION, value: 'IF' },
+    { type: OPERATOR, value: '(' },
+    { type: OPERATOR, value: ')' }
+  ], opts);
+  t.isTokens('=A1:F.DIST()', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE, value: 'A1' },
+    { type: OPERATOR, value: ':' },
+    { type: FUNCTION, value: 'F.DIST' },
+    { type: OPERATOR, value: '(' },
+    { type: OPERATOR, value: ')' }
+  ], opts);
+  t.isTokens('=1:A1.', [
+    { type: FX_PREFIX, value: '=' },
+    { type: NUMBER, value: '1' },
+    { type: OPERATOR, value: ':' },
+    { type: RANGE, value: 'A1' },
+    { type: UNKNOWN, value: '.' }
+  ], opts);
+  t.isTokens('=A1:X$', [
+    { type: FX_PREFIX, value: '=' },
+    { type: RANGE, value: 'A1' },
+    { type: OPERATOR, value: ':' },
+    { type: UNKNOWN, value: 'X$' }
+  ], opts);
+
+  t.isTokens('=[foo]Bar!A:A1', [
+    { type: FX_PREFIX, value: '=' },
+    { type: CONTEXT, value: '[foo]Bar' },
+    { type: OPERATOR, value: '!' },
+    { type: RANGE_TERNARY, value: 'A:A1' }
+  ], { mergeRanges: false, allowTernary: true });
 
   t.end();
 });