@borgar/fx 3.1.0 → 4.0.0-rc.1

package/lib/lexer-srefs.spec.js

@@ -0,0 +1,311 @@
+import { test, Test } from 'tape';
+import {
+  FX_PREFIX, UNKNOWN,
+  OPERATOR, WHITESPACE,
+  REF_NAMED, CONTEXT_QUOTE, REF_STRUCT
+} from './constants.js';
+import { tokenize } from './lexer.js';
+
+Test.prototype.isTokens = function isTokens (expr, result, opts) {
+  this.deepEqual(tokenize(expr, { negativeNumbers: false, ...opts }), result, expr);
+};
+Test.prototype.isTokensNeg = function isTokensNeg (expr, result, opts) {
+  this.deepEqual(tokenize(expr, { ...opts, negativeNumbers: true }), result, expr);
+};
+
+test('tokenize structured references (merges on)', t => {
+  // keyword specifiers
+  t.isTokens('[@]', [
+    { type: REF_STRUCT, value: '[@]' }
+  ]);
+  t.isTokens('[[@]]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[@]' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[@foo]', [
+    { type: REF_STRUCT, value: '[@foo]' }
+  ]);
+  t.isTokens('[Column Name]', [
+    { type: REF_STRUCT, value: '[Column Name]' }
+  ]);
+  t.isTokens('[@foo:bar]', [
+    { type: REF_STRUCT, value: '[@foo:bar]' }
+  ]);
+  t.isTokens('[@[foo]:bar]', [
+    { type: REF_STRUCT, value: '[@[foo]:bar]' }
+  ]);
+  t.isTokens('[@[foo]:[bar]]', [
+    { type: REF_STRUCT, value: '[@[foo]:[bar]]' }
+  ]);
+  t.isTokens('[@foo:[bar]]', [
+    { type: REF_STRUCT, value: '[@foo:[bar]]' }
+  ]);
+  t.isTokens('[@[foo]]', [
+    { type: REF_STRUCT, value: '[@[foo]]' }
+  ]);
+  t.isTokens('[[@foo]]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[@foo]' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[[\'@foo]]', [
+    { type: REF_STRUCT, value: '[[\'@foo]]' }
+  ]);
+  t.isTokens('[#All]', [
+    { type: REF_STRUCT, value: '[#All]' }
+  ]);
+  t.isTokens('[#All]', [
+    { type: REF_STRUCT, value: '[#All]' }
+  ]);
+  t.isTokens('[#Data]', [
+    { type: REF_STRUCT, value: '[#Data]' }
+  ]);
+  t.isTokens('[#Headers]', [
+    { type: REF_STRUCT, value: '[#Headers]' }
+  ]);
+  t.isTokens('[#Totals]', [
+    { type: REF_STRUCT, value: '[#Totals]' }
+  ]);
+  t.isTokens('[#This Row]', [
+    { type: REF_STRUCT, value: '[#This Row]' }
+  ]);
+  t.isTokens('[#Totals]', [
+    { type: REF_STRUCT, value: '[#Totals]' }
+  ]);
+  t.isTokens('[#totals]', [
+    { type: REF_STRUCT, value: '[#totals]' }
+  ]);
+  t.isTokens('[#tOtAlS]', [
+    { type: REF_STRUCT, value: '[#tOtAlS]' }
+  ]);
+  t.isTokens('[#This  Row]', [
+    { type: UNKNOWN, value: '[' },
+    { type: OPERATOR, value: '#' },
+    { type: REF_NAMED, value: 'This' },
+    { type: WHITESPACE, value: '  ' },
+    { type: UNKNOWN, value: 'Row]' }
+  ]);
+  t.isTokens('[ #tOtAlS]', [
+    { type: UNKNOWN, value: '[' },
+    { type: WHITESPACE, value: ' ' },
+    { type: OPERATOR, value: '#' },
+    { type: UNKNOWN, value: 'tOtAlS]' }
+  ]);
+  t.isTokens('[#tOtAlS ]', [
+    { type: UNKNOWN, value: '[' },
+    { type: OPERATOR, value: '#' },
+    { type: REF_NAMED, value: 'tOtAlS' },
+    { type: WHITESPACE, value: ' ' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[# tOtAlS ]', [
+    { type: UNKNOWN, value: '[' },
+    { type: OPERATOR, value: '#' },
+    { type: WHITESPACE, value: ' ' },
+    { type: REF_NAMED, value: 'tOtAlS' },
+    { type: WHITESPACE, value: ' ' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[[#all],@[foo]]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[#all]' },
+    { type: OPERATOR, value: ',' },
+    { type: OPERATOR, value: '@' },
+    { type: REF_STRUCT, value: '[foo]' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[[#all],]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[#all]' },
+    { type: OPERATOR, value: ',' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[[#data][#headers]]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[#data]' },
+    { type: REF_STRUCT, value: '[#headers]' },
+    { type: UNKNOWN, value: ']' }
+  ]);
+  t.isTokens('[[#data]foo]', [
+    { type: UNKNOWN, value: '[' },
+    { type: REF_STRUCT, value: '[#data]' },
+    { type: UNKNOWN, value: 'foo]' }
+  ]);
+  t.isTokens('[[#all],[foo]]', [
+    { type: REF_STRUCT, value: '[[#all],[foo]]' }
+  ]);
+  t.isTokens('[[#all],foo]', [
+    { type: REF_STRUCT, value: '[[#all],foo]' }
+  ]);
+  t.isTokens('[[#all],foo:bar]', [
+    { type: REF_STRUCT, value: '[[#all],foo:bar]' }
+  ]);
+  t.isTokens('[[#all],[foo]:[bar]]', [
+    { type: REF_STRUCT, value: '[[#all],[foo]:[bar]]' }
+  ]);
+  // this may not be what users expect, but "foo:bar baz" is a legit column name
+  t.isTokens('[foo:bar baz]', [
+    { type: REF_STRUCT, value: '[foo:bar baz]' }
+  ]);
+  t.isTokens('[foo:[bar baz]]', [
+    { type: REF_STRUCT, value: '[foo:[bar baz]]' }
+  ]);
+  t.isTokens('[foo:]', [
+    { type: REF_STRUCT, value: '[foo:]' }
+  ]);
+  t.isTokens('[[foo]:[bar baz]]', [
+    { type: REF_STRUCT, value: '[[foo]:[bar baz]]' }
+  ]);
+  t.isTokens('[[#headers],[#data],[#headers],[#data],[#headers],[#data],[Baz]]', [
+    { type: REF_STRUCT, value: '[[#headers],[#data],[#headers],[#data],[#headers],[#data],[Baz]]' }
+  ]);
+  t.isTokens('[[#all],[#all],[#all],[#all],[ColumnName]]', [
+    { type: REF_STRUCT, value: '[[#all],[#all],[#all],[#all],[ColumnName]]' }
+  ]);
+  t.isTokens('[[#Totals],col name:Foo]', [
+    { type: REF_STRUCT, value: '[[#Totals],col name:Foo]' }
+  ]);
+  t.end();
+});
+
+test('tokenize structured references (merges off)', t => {
+  t.isTokens('Table1[[#This Row],[Column]]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[[#This Row],[Column]]' }
+  ], { mergeRefs: false });
+  t.isTokens('DeptSales[[#Headers],[#Data],[% Commission]]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[[#Headers],[#Data],[% Commission]]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[[#This Row],[Column Name]]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[[#This Row],[Column Name]]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[@[Column]]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[@[Column]]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[@Column]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[@Column]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[ [#Data], [Surf]:[Rod] ]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[ [#Data], [Surf]:[Rod] ]' }
+  ], { mergeRefs: false });
+  // Excel does pick this up but normalizes to DeptSales[@[Commission Amount]]
+  t.isTokens('DeptSales[@Commission Amount]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[@Commission Amount]' }
+  ], { mergeRefs: false });
+  t.isTokens('DeptSales[[#Totals],[Sales Amount]:[Commission Amount]]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[[#Totals],[Sales Amount]:[Commission Amount]]' }
+  ], { mergeRefs: false });
+  t.isTokens('DeptSales[[#Headers],[Region]:[Commission Amount]]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[[#Headers],[Region]:[Commission Amount]]' }
+  ], { mergeRefs: false });
+  t.isTokens('DeptSales[\'#OfItems]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[\'#OfItems]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[[#Data],[#Totals],Bar:Baz]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[[#Data],[#Totals],Bar:Baz]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[[Foo]:[Bar]]:Table1[Baz]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[[Foo]:[Bar]]' },
+    { type: OPERATOR, value: ':' },
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[Baz]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[Bar]:Table1[Baz]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[Bar]' },
+    { type: OPERATOR, value: ':' },
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[Baz]' }
+  ], { mergeRefs: false });
+  t.isTokens('Table1[[#Headers],[My\'#thing]]', [
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[[#Headers],[My\'#thing]]' }
+  ], { mergeRefs: false });
+  t.isTokens('DeptSales[Sales Amount]*DeptSales[% Commission]', [
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[Sales Amount]' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_NAMED, value: 'DeptSales' },
+    { type: REF_STRUCT, value: '[% Commission]' }
+  ], { mergeRefs: false });
+  t.isTokens('=\'Sales - May2020.xlsx\'!Table1[ [#Data], [#Totals], [Surf]:[Rod] ]', [
+    { type: FX_PREFIX, value: '=' },
+    { type: CONTEXT_QUOTE, value: '\'Sales - May2020.xlsx\'' },
+    { type: OPERATOR, value: '!' },
+    { type: REF_NAMED, value: 'Table1' },
+    { type: REF_STRUCT, value: '[ [#Data], [#Totals], [Surf]:[Rod] ]' }
+  ], { mergeRefs: false });
+  t.end();
+});
+
+test('tokenize structured references (merges on)', t => {
+  t.isTokens('Table1[[#This Row],[Column]]', [
+    { type: REF_STRUCT, value: 'Table1[[#This Row],[Column]]' }
+  ]);
+  t.isTokens('DeptSales[[#Headers],[#Data],[% Commission]]', [
+    { type: REF_STRUCT, value: 'DeptSales[[#Headers],[#Data],[% Commission]]' }
+  ]);
+  t.isTokens('Table1[[#This Row],[Column Name]]', [
+    { type: REF_STRUCT, value: 'Table1[[#This Row],[Column Name]]' }
+  ]);
+  t.isTokens('Table1[@[Column]]', [
+    { type: REF_STRUCT, value: 'Table1[@[Column]]' }
+  ]);
+  t.isTokens('Table1[@Column]', [
+    { type: REF_STRUCT, value: 'Table1[@Column]' }
+  ]);
+  t.isTokens('Table1[ [#Data], [Surf]:[Rod] ]', [
+    { type: REF_STRUCT, value: 'Table1[ [#Data], [Surf]:[Rod] ]' }
+  ]);
+  // Excel does pick this up but normalizes to DeptSales[@[Commission Amount]]
+  t.isTokens('DeptSales[@Commission Amount]', [
+    { type: REF_STRUCT, value: 'DeptSales[@Commission Amount]' }
+  ]);
+  t.isTokens('DeptSales[[#Totals],[Sales Amount]:[Commission Amount]]', [
+    { type: REF_STRUCT, value: 'DeptSales[[#Totals],[Sales Amount]:[Commission Amount]]' }
+  ]);
+  t.isTokens('DeptSales[[#Headers],[Region]:[Commission Amount]]', [
+    { type: REF_STRUCT, value: 'DeptSales[[#Headers],[Region]:[Commission Amount]]' }
+  ]);
+  t.isTokens('DeptSales[\'#OfItems]', [
+    { type: REF_STRUCT, value: 'DeptSales[\'#OfItems]' }
+  ]);
+  t.isTokens('Table1[[#Data],[#Totals],Bar:Baz]', [
+    { type: REF_STRUCT, value: 'Table1[[#Data],[#Totals],Bar:Baz]' }
+  ]);
+  t.isTokens('Table1[[Foo]:[Bar]]:Table1[Baz]', [
+    { type: REF_STRUCT, value: 'Table1[[Foo]:[Bar]]' },
+    { type: OPERATOR, value: ':' },
+    { type: REF_STRUCT, value: 'Table1[Baz]' }
+  ]);
+  t.isTokens('Table1[Bar]:Table1[Baz]', [
+    { type: REF_STRUCT, value: 'Table1[Bar]' },
+    { type: OPERATOR, value: ':' },
+    { type: REF_STRUCT, value: 'Table1[Baz]' }
+  ]);
+  t.isTokens('Table1[[#Headers],[My\'#thing]]', [
+    { type: REF_STRUCT, value: 'Table1[[#Headers],[My\'#thing]]' }
+  ]);
+  t.isTokens('DeptSales[Sales Amount]*DeptSales[% Commission]', [
+    { type: REF_STRUCT, value: 'DeptSales[Sales Amount]' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_STRUCT, value: 'DeptSales[% Commission]' }
+  ]);
+  t.isTokens('=\'Sales - May2020.xlsx\'!Table1[ [#Data], [#Totals], [Surf]:[Rod] ]', [
+    { type: FX_PREFIX, value: '=' },
+    { type: REF_STRUCT, value: '\'Sales - May2020.xlsx\'!Table1[ [#Data], [#Totals], [Surf]:[Rod] ]' }
+  ]);
+  t.end();
+});

package/lib/lexer.js

@@ -3,7 +3,7 @@ import {
   NEWLINE,
   NUMBER,
   OPERATOR,
-  RANGE_NAMED,
+  REF_NAMED,
   STRING,
   UNKNOWN,
   WHITESPACE,
@@ -15,8 +15,8 @@ import { mergeRefTokens } from './mergeRefTokens.js';
 const isType = (t, type) => t && t.type === type;
 
 const defaultOptions = {
-  emitRanges: false,
-  mergeRanges: true,
+  withLocation: false,
+  mergeRefs: true,
   allowTernary: false,
   negativeNumbers: true,
   r1c1: false
@@ -24,7 +24,7 @@ const defaultOptions = {
 
 const isTextToken = token => {
   return (
-    token.type === RANGE_NAMED ||
+    token.type === REF_NAMED ||
     token.type === FUNCTION
   );
 };
@@ -40,7 +40,7 @@ const causesBinaryMinus = token => {
 
 export function getTokens (fx, tokenHandlers, options = {}) {
   const opts = Object.assign({}, defaultOptions, options);
-  const { emitRanges, mergeRanges, negativeNumbers } = opts;
+  const { withLocation, mergeRefs, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
 
@@ -58,8 +58,8 @@ export function getTokens (fx, tokenHandlers, options = {}) {
       // UNKNOWN tokens "contaminate" sibling text tokens
       lastToken.value += token.value;
       lastToken.type = UNKNOWN;
-      if (emitRanges) {
-        lastToken.range[1] = token.range[1];
+      if (withLocation) {
+        lastToken.loc[1] = token.loc[1];
       }
     }
     else {
@@ -77,7 +77,7 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     const token = {
       type: FX_PREFIX,
       value: '=',
-      ...(emitRanges ? { range: [ 0, 1 ] } : {})
+      ...(withLocation ? { loc: [ 0, 1 ] } : {})
     };
     pos++;
     pushToken(token);
@@ -107,7 +107,7 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     const token = {
       type: tokenType,
       value: tokenValue,
-      ...(emitRanges ? { range: [ startPos, pos ] } : {})
+      ...(withLocation ? { loc: [ startPos, pos ] } : {})
     };
 
     // check for termination
@@ -142,9 +142,9 @@ export function getTokens (fx, tokenHandlers, options = {}) {
         ) {
           const minus = tokens.pop();
           token.value = '-' + tokenValue;
-          if (emitRanges) {
-            // ensure range offsets are up to date
-            token.range[0] = minus.range[0];
+          if (withLocation) {
+            // ensure offsets are up to date
+            token.loc[0] = minus.loc[0];
           }
           // next step tries to counter the screwing around with the tailing
           // it should be correct again once we pushToken()
@@ -157,13 +157,53 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     pushToken(token);
   }
 
-  if (mergeRanges) {
+  if (mergeRefs) {
     return mergeRefTokens(tokens);
   }
 
   return tokens;
 }
 
-export function tokenize (fx, options = {}) {
-  return getTokens(fx, lexers, options);
+/**
+ * Breaks a string formula into a list of tokens.
+ *
+ * The returned output will be an array of objects representing the tokens:
+ *
+ * ```js
+ * [
+ *   { type: FX_PREFIX, value: '=' },
+ *   { type: FUNCTION, value: 'SUM' },
+ *   { type: OPERATOR, value: '(' },
+ *   { type: REF_RANGE, value: 'A1:B2' },
+ *   { type: OPERATOR, value: ')' }
+ * ]
+ * ```
+ *
+ * Token types may be found as an Object as the
+ * [`tokenTypes` export]{@link tokenTypes} on the package
+ * (`import {tokenTypes} from '@borgar/fx';`).
+ *
+ * To support syntax highlighting as you type, `STRING` tokens are allowed to be
+ * "unterminated". For example, the incomplete formula `="Hello world` would be
+ * tokenized as:
+ *
+ * ```js
+ * [
+ *   { type: FX_PREFIX, value: '=' },
+ *   { type: STRING, value: '"Hello world', unterminated: true },
+ * ]
+ * ```
+ *
+ * @see tokenTypes
+ * @param {string} formula An Excel formula string (an Excel expression) or an array of tokens.
+ * @param {Object} [options={}]  Options
+ * @param {boolean} [options.allowTernary=false]  Enables the recognition of ternary ranges in the style of `A1:A` or `A1:1`. These are supported by Google Sheets but not Excel. See: References.md.
+ * @param {boolean} [options.negativeNumbers=true]  Merges unary minuses with their immediately following number tokens (`-`,`1`) => `-1` (alternatively these will be unary operations in the tree).
+ * @param {boolean} [options.r1c1=false]  Ranges are expected to be in the R1C1 style format rather than the more popular A1 style.
+ * @param {boolean} [options.withLocation=true]  Nodes will include source position offsets to the tokens: `{ loc: [ start, end ] }`
+ * @param {boolean} [options.mergeRefs=true]  Should ranges be returned as whole references (`Sheet1!A1:B2`) or as separate tokens for each part: (`Sheet1`,`!`,`A1`,`:`,`B2`). This is the same as calling [`mergeRefTokens`](#mergeRefTokens)
+ * @return {Array<Object>} An AST of nodes
+ */
+export function tokenize (formula, options = {}) {
+  return getTokens(formula, lexers, options);
 }