@borgar/fx 4.11.2 → 4.13.0

package/docs/API.md

@@ -383,6 +383,7 @@ The AST Abstract Syntax Tree's format is documented in [AST_format.md](./AST_for
 | [options]                   | `object`                   | `{}`    | Options                                                                                                                                             |
 | [options].allowNamed        | `boolean`                  | `true`  | Enable parsing names as well as ranges.                                                                                                             |
 | [options].allowTernary      | `boolean`                  | `false` | Enables the recognition of ternary ranges in the style of `A1:A` or `A1:1`. These are supported by Google Sheets but not Excel. See: References.md. |
+| [options].looseRefCalls     | `boolean`                  | `false` | Permits any function call where otherwise only functions that return references would be permitted.                                                 |
 | [options].negativeNumbers   | `boolean`                  | `true`  | Merges unary minuses with their immediately following number tokens (`-`,`1`) => `-1` (alternatively these will be unary operations in the tree).   |
 | [options].permitArrayCalls  | `boolean`                  | `false` | Function calls are allowed as elements of arrays. This is a feature in Google Sheets while Excel does not allow it.                                 |
 | [options].permitArrayRanges | `boolean`                  | `false` | Ranges are allowed as elements of arrays. This is a feature in Google Sheets while Excel does not allow it.                                         |

package/lib/fixRanges.spec.js

@@ -6,7 +6,8 @@ import { FUNCTION, FX_PREFIX, OPERATOR, REF_RANGE, REF_STRUCT, REF_TERNARY } fro
 
 Test.prototype.isFixed = function (expr, expected, options = {}) {
   const result = fixRanges(expr, options);
-  this.is(result, expected, expr + ' → ' + expected);
+  this.is(result, expected,
+    `\x1b[36m${expr} → ${expected} \x1b[37mopts=${JSON.stringify(options)}\x1b[0m`);
 };
 
 test('fixRanges basics', t => {

package/lib/lexer.js

@@ -4,15 +4,14 @@ import {
   NUMBER,
   OPERATOR,
   REF_NAMED,
-  STRING,
   UNKNOWN,
   WHITESPACE,
   FUNCTION,
   OPERATOR_TRIM,
   REF_RANGE
 } from './constants.js';
-import { lexers } from './lexerParts.js';
 import { mergeRefTokens } from './mergeRefTokens.js';
+import { lexers } from './lexers/sets.js';
 
 const isType = (t, type) => t && t.type === type;
 
@@ -71,7 +70,14 @@ function fixRCNames (tokens) {
 }
 
 export function getTokens (fx, tokenHandlers, options = {}) {
-  const opts = Object.assign({}, defaultOptions, options);
+  const opts = { ...defaultOptions, ...options };
+  // const opts = {
+  //   withLocation: options.withLocation ?? false,
+  //   mergeRefs: options.mergeRefs ?? true,
+  //   allowTernary: options.allowTernary ?? false,
+  //   negativeNumbers: options.negativeNumbers ?? true,
+  //   r1c1: options.r1c1 ?? false
+  // };
   const { withLocation, mergeRefs, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
@@ -103,7 +109,8 @@ export function getTokens (fx, tokenHandlers, options = {}) {
         token.type = UNKNOWN;
       }
       // push token as normally
-      tokens.push(token);
+      // tokens.push(token);
+      tokens[tokens.length] = token;
       lastToken = token;
       if (token.type !== WHITESPACE && token.type !== NEWLINE) {
         tail1 = tail0;
@@ -112,77 +119,51 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     }
   };
 
-  if (fx[0] === '=') {
-    const token = {
-      type: FX_PREFIX,
-      value: '=',
-      ...(withLocation ? { loc: [ 0, 1 ] } : {})
-    };
+  if (fx.startsWith('=')) {
+    const token = { type: FX_PREFIX, value: '=' };
+    if (withLocation) {
+      token.loc = [ 0, 1 ];
+    }
     pos++;
     pushToken(token);
   }
 
+  const numHandlers = tokenHandlers.length;
   while (pos < fx.length) {
     const startPos = pos;
-    const s = fx.slice(pos);
-    let tokenType = '';
-    let tokenValue = '';
-    for (let i = 0; i < tokenHandlers.length; i++) {
-      const t = tokenHandlers[i](s, opts);
-      if (t) {
-        tokenType = t.type;
-        tokenValue = t.value;
-        pos += tokenValue.length;
+    let token;
+    for (let i = 0; i < numHandlers; i++) {
+      token = tokenHandlers[i](fx, pos, opts);
+      if (token) {
+        pos += token.value.length;
         break;
       }
     }
 
-    if (!tokenType) {
-      tokenType = UNKNOWN;
-      tokenValue = fx[pos];
+    if (!token) {
+      token = {
+        type: UNKNOWN,
+        value: fx[pos]
+      };
       pos++;
     }
-
-    const token = {
-      type: tokenType,
-      value: tokenValue,
-      ...(withLocation ? { loc: [ startPos, pos ] } : {})
-    };
+    if (withLocation) {
+      token.loc = [ startPos, pos ];
+    }
 
     // make a note if we found a let/lambda call
-    if (lastToken && lastToken.type === FUNCTION && tokenValue === '(') {
-      const lastLC = lastToken.value.toLowerCase();
-      if (lastLC === 'lambda' || lastLC === 'let') {
+    if (lastToken && token.value === '(' && lastToken.type === FUNCTION) {
+      if (/^l(?:ambda|et)$/i.test(lastToken.value)) {
         letOrLambda++;
       }
     }
     // make a note if we found a R or C unknown
-    if (tokenType === UNKNOWN) {
-      const valLC = tokenValue.toLowerCase();
+    if (token.type === UNKNOWN && token.value.length === 1) {
+      const valLC = token.value.toLowerCase();
       unknownRC += (valLC === 'r' || valLC === 'c') ? 1 : 0;
     }
 
-    // check for termination
-    if (tokenType === STRING) {
-      const l = tokenValue.length;
-      if (tokenValue === '""') {
-        // common case that IS terminated
-      }
-      else if (tokenValue === '"' || tokenValue[l - 1] !== '"') {
-        token.unterminated = true;
-      }
-      else if (tokenValue !== '""' && tokenValue[l - 2] === '"') {
-        let p = l - 1;
-        while (tokenValue[p] === '"') { p--; }
-        const atStart = (p + 1);
-        const oddNum = ((l - p + 1) % 2 === 0);
-        if (!atStart ^ oddNum) {
-          token.unterminated = true;
-        }
-      }
-    }
-
-    if (negativeNumbers && tokenType === NUMBER) {
+    if (negativeNumbers && token.type === NUMBER) {
       const last1 = lastToken;
       // do we have a number preceded by a minus?
       if (last1 && isType(last1, OPERATOR) && last1.value === '-') {
@@ -193,8 +174,8 @@ export function getTokens (fx, tokenHandlers, options = {}) {
           !causesBinaryMinus(tail1)
         ) {
           const minus = tokens.pop();
-          token.value = '-' + tokenValue;
-          if (withLocation) {
+          token.value = '-' + token.value;
+          if (token.loc) {
             // ensure offsets are up to date
             token.loc[0] = minus.loc[0];
           }
@@ -221,7 +202,7 @@ export function getTokens (fx, tokenHandlers, options = {}) {
   // operators.
   for (const index of trimOps) {
     const before = tokens[index - 1];
-    const after = tokens[index - 1];
+    const after = tokens[index + 1];
     if (before && before.type === REF_RANGE && after && after.type === REF_RANGE) {
       tokens[index].type = OPERATOR;
     }

package/lib/lexers/advRangeOp.js

@@ -0,0 +1,18 @@
+const PERIOD = 46;
+const COLON = 58;
+
+export function advRangeOp (str, pos) {
+  const c0 = str.charCodeAt(pos);
+  if (c0 === PERIOD) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === COLON) {
+      return str.charCodeAt(pos + 2) === PERIOD ? 3 : 2;
+    }
+  }
+  else if (c0 === COLON) {
+    const c1 = str.charCodeAt(pos + 1);
+    return c1 === PERIOD ? 2 : 1;
+  }
+  return 0;
+}
+

package/lib/lexers/canEndRange.js

@@ -0,0 +1,25 @@
+// regular: [A-Za-z0-9_\u00a1-\uffff]
+export function canEndRange (str, pos) {
+  const c = str.charCodeAt(pos);
+  return !(
+    (c >= 65 && c <= 90) || // A-Z
+    (c >= 97 && c <= 122) || // a-z
+    (c >= 48 && c <= 57) || // 0-9
+    (c === 95) || // _
+    (c > 0xA0) // \u00a1-\uffff
+  );
+}
+
+// partial: [A-Za-z0-9_($.]
+export function canEndPartialRange (str, pos) {
+  const c = str.charCodeAt(pos);
+  return !(
+    (c >= 65 && c <= 90) || // A-Z
+    (c >= 97 && c <= 122) || // a-z
+    (c >= 48 && c <= 57) || // 0-9
+    (c === 95) || // _
+    (c === 40) || // (
+    (c === 36) || // $
+    (c === 46) // .
+  );
+}

package/lib/lexers/lexBoolean.js

@@ -0,0 +1,36 @@
+import { BOOLEAN } from '../constants.js';
+
+export function lexBoolean (str, pos) {
+  // "true" (case insensitive)
+  const c0 = str.charCodeAt(pos);
+  if (c0 === 84 || c0 === 116) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === 82 || c1 === 114) {
+      const c2 = str.charCodeAt(pos + 2);
+      if (c2 === 85 || c2 === 117) {
+        const c3 = str.charCodeAt(pos + 3);
+        if (c3 === 69 || c3 === 101) {
+          // non char to follow?
+          return { type: BOOLEAN, value: str.slice(pos, pos + 4) };
+        }
+      }
+    }
+  }
+  // "false" (case insensitive)
+  if (c0 === 70 || c0 === 102) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === 65 || c1 === 97) {
+      const c2 = str.charCodeAt(pos + 2);
+      if (c2 === 76 || c2 === 108) {
+        const c3 = str.charCodeAt(pos + 3);
+        if (c3 === 83 || c3 === 115) {
+          const c4 = str.charCodeAt(pos + 4);
+          if (c4 === 69 || c4 === 101) {
+            // non char to follow?
+            return { type: BOOLEAN, value: str.slice(pos, pos + 5) };
+          }
+        }
+      }
+    }
+  }
+}

package/lib/lexers/lexContext.js

@@ -0,0 +1,96 @@
+import { CONTEXT, CONTEXT_QUOTE } from '../constants.js';
+
+const QUOT_SINGLE = 39; // '
+const BR_OPEN = 91; // [
+const BR_CLOSE = 93; // ]
+const EXCL = 33; // !
+
+// xlsx xml uses a variant of the syntax that has external references in
+// bracets. Any of: [1]Sheet1!A1, '[1]Sheet one'!A1, [1]!named
+export function lexContext (str, pos, options) {
+  const c0 = str.charCodeAt(pos);
+  let br1;
+  let br2;
+  // quoted context: '(?:''|[^'])*('|$)(?=!)
+  if (c0 === QUOT_SINGLE) {
+    const start = pos;
+    pos++;
+    while (pos < str.length) {
+      const c = str.charCodeAt(pos);
+      if (c === BR_OPEN) {
+        if (br1) { return; } // only 1 allowed
+        br1 = pos;
+      }
+      else if (c === BR_CLOSE) {
+        if (br2) { return; } // only 1 allowed
+        br2 = pos;
+      }
+      else if (c === QUOT_SINGLE) {
+        pos++;
+        if (str.charCodeAt(pos) !== QUOT_SINGLE) {
+          let valid = br1 == null && br2 == null;
+          if (options.xlsx && (br1 === start + 1) && (br2 === pos - 2)) {
+            valid = true;
+          }
+          if ((br1 >= start + 1) && (br2 < pos - 2) && (br2 > br1 + 1)) {
+            valid = true;
+          }
+          if (valid && str.charCodeAt(pos) === EXCL) {
+            return { type: CONTEXT_QUOTE, value: str.slice(start, pos) };
+          }
+          return;
+        }
+      }
+      pos++;
+    }
+  }
+  // unquoted context
+  else if (c0 !== EXCL) {
+    const start = pos;
+    while (pos < str.length) {
+      const c = str.charCodeAt(pos);
+      if (c === BR_OPEN) {
+        if (br1) { return; } // only 1 allowed
+        br1 = pos;
+      }
+      else if (c === BR_CLOSE) {
+        if (br2) { return; } // only 1 allowed
+        br2 = pos;
+      }
+      else if (c === EXCL) {
+        let valid = br1 == null && br2 == null;
+        if (options.xlsx && (br1 === start) && (br2 === pos - 1)) {
+          valid = true;
+        }
+        if ((br1 >= start) && (br2 < pos - 1) && (br2 > br1 + 1)) {
+          valid = true;
+        }
+        if (valid) {
+          return { type: CONTEXT, value: str.slice(start, pos) };
+        }
+      }
+      else if (
+        (br1 == null || br2 != null) &&
+        // [0-9A-Za-z._¡¤§¨ª\u00ad¯-\uffff]
+        !(
+          (c >= 65 && c <= 90) || // A-Z
+          (c >= 97 && c <= 122) || // a-z
+          (c >= 48 && c <= 57) || // 0-9
+          (c === 46) || // .
+          (c === 95) || // _
+          (c === 161) || // ¡
+          (c === 164) || // ¤
+          (c === 167) || // §
+          (c === 168) || // ¨
+          (c === 170) || // ª
+          (c === 173) || // \u00ad
+          (c >= 175)    // ¯-\uffff
+        )
+      ) {
+        return;
+      }
+      // 0-9A-Za-z._¡¤§¨ª\u00ad¯-\uffff
+      pos++;
+    }
+  }
+}

package/lib/lexers/lexError.js

@@ -0,0 +1,15 @@
+/* eslint-disable no-mixed-operators */
+import { ERROR } from '../constants.js';
+
+const re_ERROR = /#(?:NAME\?|FIELD!|CALC!|VALUE!|REF!|DIV\/0!|NULL!|NUM!|N\/A|GETTING_DATA\b|SPILL!|UNKNOWN!|SYNTAX\?|ERROR!|CONNECT!|BLOCKED!|EXTERNAL!)/iy;
+const HASH = 35;
+
+export function lexError (str, pos) {
+  if (str.charCodeAt(pos) === HASH) {
+    re_ERROR.lastIndex = pos;
+    const m = re_ERROR.exec(str);
+    if (m) {
+      return { type: ERROR, value: m[0] };
+    }
+  }
+}

package/lib/lexers/lexFunction.js

@@ -0,0 +1,36 @@
+import { FUNCTION } from '../constants.js';
+
+const PAREN_OPEN = 40;
+
+// [A-Za-z_]+[A-Za-z\d_.]*(?=\()
+export function lexFunction (str, pos) {
+  const start = pos;
+  // starts with: a-zA-Z_
+  let c = str.charCodeAt(pos);
+  if (
+    (c < 65 || c > 90) && // A-Z
+    (c < 97 || c > 122) && // a-z
+    (c !== 95) // _
+  ) {
+    return;
+  }
+  pos++;
+  // has any number of: a-zA-Z0-9_.
+  do {
+    c = str.charCodeAt(pos);
+    if (
+      (c < 65 || c > 90) && // A-Z
+      (c < 97 || c > 122) && // a-z
+      (c < 48 || c > 57) && // 0-9
+      (c !== 95) && // _
+      (c !== 46) // .
+    ) {
+      break;
+    }
+    pos++;
+  } while (pos < str.length);
+  // followed by a (
+  if (str.charCodeAt(pos) === PAREN_OPEN) {
+    return { type: FUNCTION, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNamed.js

@@ -0,0 +1,60 @@
+/* eslint-disable max-len */
+import { REF_NAMED } from '../constants.js';
+
+// The advertized named ranges rules are a bit off from what Excel seems to do.
+// In the "extended range" of chars, it looks like it allows most things above
+// U+00B0 with the range between U+00A0-U+00AF rather random:
+// /^[a-zA-Z\\_¡¤§¨ª\u00ad¯\u00b0-\uffff][a-zA-Z0-9\\_.?¡¤§¨ª\u00ad¯\u00b0-\uffff]{0,254}/
+//
+// I've simplified to allowing everything above U+00A1:
+// /^[a-zA-Z\\_\u00a1-\uffff][a-zA-Z0-9\\_.?\u00a1-\uffff]{0,254}/
+export function lexNamed (str, pos) {
+  const start = pos;
+  // starts with: [a-zA-Z\\_\u00a1-\uffff]
+  const s = str.charCodeAt(pos);
+  if (
+    (s >= 65 && s <= 90) || // A-Z
+    (s >= 97 && s <= 122) || // a-z
+    (s === 95) || // _
+    (s === 92) || // \
+    (s > 0xA0) // \u00a1-\uffff
+  ) {
+    pos++;
+  }
+  else {
+    return;
+  }
+  // has any number of: [a-zA-Z0-9\\_.?\u00a1-\uffff]
+  let c;
+  do {
+    c = str.charCodeAt(pos);
+    if (
+      (c >= 65 && c <= 90) || // A-Z
+      (c >= 97 && c <= 122) || // a-z
+      (c >= 48 && c <= 57) || // 0-9
+      (c === 95) || // _
+      (c === 92) || // \
+      (c === 46) || // .
+      (c === 63) || // ?
+      (c > 0xA0) // \u00a1-\uffff
+    ) {
+      pos++;
+    }
+    else {
+      break;
+    }
+  } while (isFinite(c));
+
+  const len = pos - start;
+  if (len && len < 255) {
+    // names starting with \ must be at least 3 char long
+    if (s === 92 && len < 3) {
+      return;
+    }
+    // single characters R and C are forbidden as names
+    if (len === 1 && (s === 114 || s === 82 || s === 99 || s === 67)) {
+      return;
+    }
+    return { type: REF_NAMED, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNewLine.js

@@ -0,0 +1,11 @@
+import { NEWLINE } from '../constants.js';
+
+export function lexNewLine (str, pos) {
+  const start = pos;
+  while (str.charCodeAt(pos) === 10) {
+    pos++;
+  }
+  if (pos !== start) {
+    return { type: NEWLINE, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNumber.js

@@ -0,0 +1,47 @@
+import { NUMBER } from '../constants.js';
+
+function advDigits (str, pos) {
+  const start = pos;
+  do {
+    const c = str.charCodeAt(pos);
+    if (c < 48 || c > 57) { // 0-9
+      break;
+    }
+    pos++;
+  }
+  while (pos < str.length);
+  return pos - start;
+}
+
+// \d+(\.\d+)?(?:[eE][+-]?\d+)?
+export function lexNumber (str, pos) {
+  const start = pos;
+
+  // integer
+  const lead = advDigits(str, pos);
+  if (!lead) { return; }
+  pos += lead;
+
+  // optional fraction part
+  const c0 = str.charCodeAt(pos);
+  if (c0 === 46) { // .
+    pos++;
+    const frac = advDigits(str, pos);
+    if (!frac) { return; }
+    pos += frac;
+  }
+  // optional exponent part
+  const c1 = str.charCodeAt(pos);
+  if (c1 === 69 || c1 === 101) { // E e
+    pos++;
+    const sign = str.charCodeAt(pos);
+    if (sign === 43 || sign === 45) { // + -
+      pos++;
+    }
+    const exp = advDigits(str, pos);
+    if (!exp) { return; }
+    pos += exp;
+  }
+
+  return { type: NUMBER, value: str.slice(start, pos) };
+}

package/lib/lexers/lexOperator.js

@@ -0,0 +1,25 @@
+import { OPERATOR } from '../constants.js';
+
+export function lexOperator (str, pos) {
+  const c0 = str.charCodeAt(pos);
+  const c1 = str.charCodeAt(pos + 1);
+  if (
+    (c0 === 60 && c1 === 61) || // <=
+    (c0 === 62 && c1 === 61) || // >=
+    (c0 === 60 && c1 === 62)    // <>
+  ) {
+    return { type: OPERATOR, value: str.slice(pos, pos + 2) };
+  }
+  if (
+    // { } ! # % &
+    c0 === 123 || c0 === 125 || c0 === 33 || c0 === 35 || c0 === 37 || c0 === 38 ||
+    // ( ) * + , -
+    c0 === 40 || c0 === 41 || c0 === 42 || c0 === 43 || c0 === 44 || c0 === 45 ||
+    // / : ; < = >
+    c0 === 47 || c0 === 58 || c0 === 59 || c0 === 60 || c0 === 61 || c0 === 62 ||
+    // @ ^
+    c0 === 64 || c0 === 94
+  ) {
+    return { type: OPERATOR, value: str[pos] };
+  }
+}

package/lib/lexers/lexRange.js

@@ -0,0 +1,8 @@
+import { lexRangeA1 } from './lexRangeA1.js';
+import { lexRangeR1C1 } from './lexRangeR1C1.js';
+
+export function lexRange (str, pos, options) {
+  return options.r1c1
+    ? lexRangeR1C1(str, pos, options)
+    : lexRangeA1(str, pos, options);
+}