@borgar/fx 4.12.0 → 4.13.0

package/lib/fixRanges.spec.js

@@ -6,7 +6,8 @@ import { FUNCTION, FX_PREFIX, OPERATOR, REF_RANGE, REF_STRUCT, REF_TERNARY } fro
 
 Test.prototype.isFixed = function (expr, expected, options = {}) {
   const result = fixRanges(expr, options);
-  this.is(result, expected, expr + ' → ' + expected);
+  this.is(result, expected,
+    `\x1b[36m${expr} → ${expected} \x1b[37mopts=${JSON.stringify(options)}\x1b[0m`);
 };
 
 test('fixRanges basics', t => {

package/lib/lexer.js

@@ -4,15 +4,14 @@ import {
   NUMBER,
   OPERATOR,
   REF_NAMED,
-  STRING,
   UNKNOWN,
   WHITESPACE,
   FUNCTION,
   OPERATOR_TRIM,
   REF_RANGE
 } from './constants.js';
-import { lexers } from './lexerParts.js';
 import { mergeRefTokens } from './mergeRefTokens.js';
+import { lexers } from './lexers/sets.js';
 
 const isType = (t, type) => t && t.type === type;
 
@@ -71,7 +70,14 @@ function fixRCNames (tokens) {
 }
 
 export function getTokens (fx, tokenHandlers, options = {}) {
-  const opts = Object.assign({}, defaultOptions, options);
+  const opts = { ...defaultOptions, ...options };
+  // const opts = {
+  //   withLocation: options.withLocation ?? false,
+  //   mergeRefs: options.mergeRefs ?? true,
+  //   allowTernary: options.allowTernary ?? false,
+  //   negativeNumbers: options.negativeNumbers ?? true,
+  //   r1c1: options.r1c1 ?? false
+  // };
   const { withLocation, mergeRefs, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
@@ -103,7 +109,8 @@ export function getTokens (fx, tokenHandlers, options = {}) {
         token.type = UNKNOWN;
       }
       // push token as normally
-      tokens.push(token);
+      // tokens.push(token);
+      tokens[tokens.length] = token;
       lastToken = token;
       if (token.type !== WHITESPACE && token.type !== NEWLINE) {
         tail1 = tail0;
@@ -112,77 +119,51 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     }
   };
 
-  if (fx[0] === '=') {
-    const token = {
-      type: FX_PREFIX,
-      value: '=',
-      ...(withLocation ? { loc: [ 0, 1 ] } : {})
-    };
+  if (fx.startsWith('=')) {
+    const token = { type: FX_PREFIX, value: '=' };
+    if (withLocation) {
+      token.loc = [ 0, 1 ];
+    }
     pos++;
     pushToken(token);
   }
 
+  const numHandlers = tokenHandlers.length;
   while (pos < fx.length) {
     const startPos = pos;
-    const s = fx.slice(pos);
-    let tokenType = '';
-    let tokenValue = '';
-    for (let i = 0; i < tokenHandlers.length; i++) {
-      const t = tokenHandlers[i](s, opts);
-      if (t) {
-        tokenType = t.type;
-        tokenValue = t.value;
-        pos += tokenValue.length;
+    let token;
+    for (let i = 0; i < numHandlers; i++) {
+      token = tokenHandlers[i](fx, pos, opts);
+      if (token) {
+        pos += token.value.length;
         break;
       }
     }
 
-    if (!tokenType) {
-      tokenType = UNKNOWN;
-      tokenValue = fx[pos];
+    if (!token) {
+      token = {
+        type: UNKNOWN,
+        value: fx[pos]
+      };
       pos++;
     }
-
-    const token = {
-      type: tokenType,
-      value: tokenValue,
-      ...(withLocation ? { loc: [ startPos, pos ] } : {})
-    };
+    if (withLocation) {
+      token.loc = [ startPos, pos ];
+    }
 
     // make a note if we found a let/lambda call
-    if (lastToken && lastToken.type === FUNCTION && tokenValue === '(') {
-      const lastLC = lastToken.value.toLowerCase();
-      if (lastLC === 'lambda' || lastLC === 'let') {
+    if (lastToken && token.value === '(' && lastToken.type === FUNCTION) {
+      if (/^l(?:ambda|et)$/i.test(lastToken.value)) {
         letOrLambda++;
       }
     }
     // make a note if we found a R or C unknown
-    if (tokenType === UNKNOWN) {
-      const valLC = tokenValue.toLowerCase();
+    if (token.type === UNKNOWN && token.value.length === 1) {
+      const valLC = token.value.toLowerCase();
       unknownRC += (valLC === 'r' || valLC === 'c') ? 1 : 0;
     }
 
-    // check for termination
-    if (tokenType === STRING) {
-      const l = tokenValue.length;
-      if (tokenValue === '""') {
-        // common case that IS terminated
-      }
-      else if (tokenValue === '"' || tokenValue[l - 1] !== '"') {
-        token.unterminated = true;
-      }
-      else if (tokenValue !== '""' && tokenValue[l - 2] === '"') {
-        let p = l - 1;
-        while (tokenValue[p] === '"') { p--; }
-        const atStart = (p + 1);
-        const oddNum = ((l - p + 1) % 2 === 0);
-        if (!atStart ^ oddNum) {
-          token.unterminated = true;
-        }
-      }
-    }
-
-    if (negativeNumbers && tokenType === NUMBER) {
+    if (negativeNumbers && token.type === NUMBER) {
       const last1 = lastToken;
       // do we have a number preceded by a minus?
       if (last1 && isType(last1, OPERATOR) && last1.value === '-') {
@@ -193,8 +174,8 @@ export function getTokens (fx, tokenHandlers, options = {}) {
           !causesBinaryMinus(tail1)
         ) {
           const minus = tokens.pop();
-          token.value = '-' + tokenValue;
-          if (withLocation) {
+          token.value = '-' + token.value;
+          if (token.loc) {
             // ensure offsets are up to date
             token.loc[0] = minus.loc[0];
           }
@@ -221,7 +202,7 @@ export function getTokens (fx, tokenHandlers, options = {}) {
   // operators.
   for (const index of trimOps) {
     const before = tokens[index - 1];
-    const after = tokens[index - 1];
+    const after = tokens[index + 1];
     if (before && before.type === REF_RANGE && after && after.type === REF_RANGE) {
       tokens[index].type = OPERATOR;
     }

package/lib/lexers/advRangeOp.js

@@ -0,0 +1,18 @@
+const PERIOD = 46;
+const COLON = 58;
+
+export function advRangeOp (str, pos) {
+  const c0 = str.charCodeAt(pos);
+  if (c0 === PERIOD) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === COLON) {
+      return str.charCodeAt(pos + 2) === PERIOD ? 3 : 2;
+    }
+  }
+  else if (c0 === COLON) {
+    const c1 = str.charCodeAt(pos + 1);
+    return c1 === PERIOD ? 2 : 1;
+  }
+  return 0;
+}
+

package/lib/lexers/canEndRange.js

@@ -0,0 +1,25 @@
+// regular: [A-Za-z0-9_\u00a1-\uffff]
+export function canEndRange (str, pos) {
+  const c = str.charCodeAt(pos);
+  return !(
+    (c >= 65 && c <= 90) || // A-Z
+    (c >= 97 && c <= 122) || // a-z
+    (c >= 48 && c <= 57) || // 0-9
+    (c === 95) || // _
+    (c > 0xA0) // \u00a1-\uffff
+  );
+}
+
+// partial: [A-Za-z0-9_($.]
+export function canEndPartialRange (str, pos) {
+  const c = str.charCodeAt(pos);
+  return !(
+    (c >= 65 && c <= 90) || // A-Z
+    (c >= 97 && c <= 122) || // a-z
+    (c >= 48 && c <= 57) || // 0-9
+    (c === 95) || // _
+    (c === 40) || // (
+    (c === 36) || // $
+    (c === 46) // .
+  );
+}

package/lib/lexers/lexBoolean.js

@@ -0,0 +1,36 @@
+import { BOOLEAN } from '../constants.js';
+
+export function lexBoolean (str, pos) {
+  // "true" (case insensitive)
+  const c0 = str.charCodeAt(pos);
+  if (c0 === 84 || c0 === 116) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === 82 || c1 === 114) {
+      const c2 = str.charCodeAt(pos + 2);
+      if (c2 === 85 || c2 === 117) {
+        const c3 = str.charCodeAt(pos + 3);
+        if (c3 === 69 || c3 === 101) {
+          // non char to follow?
+          return { type: BOOLEAN, value: str.slice(pos, pos + 4) };
+        }
+      }
+    }
+  }
+  // "false" (case insensitive)
+  if (c0 === 70 || c0 === 102) {
+    const c1 = str.charCodeAt(pos + 1);
+    if (c1 === 65 || c1 === 97) {
+      const c2 = str.charCodeAt(pos + 2);
+      if (c2 === 76 || c2 === 108) {
+        const c3 = str.charCodeAt(pos + 3);
+        if (c3 === 83 || c3 === 115) {
+          const c4 = str.charCodeAt(pos + 4);
+          if (c4 === 69 || c4 === 101) {
+            // non char to follow?
+            return { type: BOOLEAN, value: str.slice(pos, pos + 5) };
+          }
+        }
+      }
+    }
+  }
+}

package/lib/lexers/lexContext.js

@@ -0,0 +1,96 @@
+import { CONTEXT, CONTEXT_QUOTE } from '../constants.js';
+
+const QUOT_SINGLE = 39; // '
+const BR_OPEN = 91; // [
+const BR_CLOSE = 93; // ]
+const EXCL = 33; // !
+
+// xlsx xml uses a variant of the syntax that has external references in
+// bracets. Any of: [1]Sheet1!A1, '[1]Sheet one'!A1, [1]!named
+export function lexContext (str, pos, options) {
+  const c0 = str.charCodeAt(pos);
+  let br1;
+  let br2;
+  // quoted context: '(?:''|[^'])*('|$)(?=!)
+  if (c0 === QUOT_SINGLE) {
+    const start = pos;
+    pos++;
+    while (pos < str.length) {
+      const c = str.charCodeAt(pos);
+      if (c === BR_OPEN) {
+        if (br1) { return; } // only 1 allowed
+        br1 = pos;
+      }
+      else if (c === BR_CLOSE) {
+        if (br2) { return; } // only 1 allowed
+        br2 = pos;
+      }
+      else if (c === QUOT_SINGLE) {
+        pos++;
+        if (str.charCodeAt(pos) !== QUOT_SINGLE) {
+          let valid = br1 == null && br2 == null;
+          if (options.xlsx && (br1 === start + 1) && (br2 === pos - 2)) {
+            valid = true;
+          }
+          if ((br1 >= start + 1) && (br2 < pos - 2) && (br2 > br1 + 1)) {
+            valid = true;
+          }
+          if (valid && str.charCodeAt(pos) === EXCL) {
+            return { type: CONTEXT_QUOTE, value: str.slice(start, pos) };
+          }
+          return;
+        }
+      }
+      pos++;
+    }
+  }
+  // unquoted context
+  else if (c0 !== EXCL) {
+    const start = pos;
+    while (pos < str.length) {
+      const c = str.charCodeAt(pos);
+      if (c === BR_OPEN) {
+        if (br1) { return; } // only 1 allowed
+        br1 = pos;
+      }
+      else if (c === BR_CLOSE) {
+        if (br2) { return; } // only 1 allowed
+        br2 = pos;
+      }
+      else if (c === EXCL) {
+        let valid = br1 == null && br2 == null;
+        if (options.xlsx && (br1 === start) && (br2 === pos - 1)) {
+          valid = true;
+        }
+        if ((br1 >= start) && (br2 < pos - 1) && (br2 > br1 + 1)) {
+          valid = true;
+        }
+        if (valid) {
+          return { type: CONTEXT, value: str.slice(start, pos) };
+        }
+      }
+      else if (
+        (br1 == null || br2 != null) &&
+        // [0-9A-Za-z._¡¤§¨ª\u00ad¯-\uffff]
+        !(
+          (c >= 65 && c <= 90) || // A-Z
+          (c >= 97 && c <= 122) || // a-z
+          (c >= 48 && c <= 57) || // 0-9
+          (c === 46) || // .
+          (c === 95) || // _
+          (c === 161) || // ¡
+          (c === 164) || // ¤
+          (c === 167) || // §
+          (c === 168) || // ¨
+          (c === 170) || // ª
+          (c === 173) || // \u00ad
+          (c >= 175)    // ¯-\uffff
+        )
+      ) {
+        return;
+      }
+      // 0-9A-Za-z._¡¤§¨ª\u00ad¯-\uffff
+      pos++;
+    }
+  }
+}

package/lib/lexers/lexError.js

@@ -0,0 +1,15 @@
+/* eslint-disable no-mixed-operators */
+import { ERROR } from '../constants.js';
+
+const re_ERROR = /#(?:NAME\?|FIELD!|CALC!|VALUE!|REF!|DIV\/0!|NULL!|NUM!|N\/A|GETTING_DATA\b|SPILL!|UNKNOWN!|SYNTAX\?|ERROR!|CONNECT!|BLOCKED!|EXTERNAL!)/iy;
+const HASH = 35;
+
+export function lexError (str, pos) {
+  if (str.charCodeAt(pos) === HASH) {
+    re_ERROR.lastIndex = pos;
+    const m = re_ERROR.exec(str);
+    if (m) {
+      return { type: ERROR, value: m[0] };
+    }
+  }
+}

package/lib/lexers/lexFunction.js

@@ -0,0 +1,36 @@
+import { FUNCTION } from '../constants.js';
+
+const PAREN_OPEN = 40;
+
+// [A-Za-z_]+[A-Za-z\d_.]*(?=\()
+export function lexFunction (str, pos) {
+  const start = pos;
+  // starts with: a-zA-Z_
+  let c = str.charCodeAt(pos);
+  if (
+    (c < 65 || c > 90) && // A-Z
+    (c < 97 || c > 122) && // a-z
+    (c !== 95) // _
+  ) {
+    return;
+  }
+  pos++;
+  // has any number of: a-zA-Z0-9_.
+  do {
+    c = str.charCodeAt(pos);
+    if (
+      (c < 65 || c > 90) && // A-Z
+      (c < 97 || c > 122) && // a-z
+      (c < 48 || c > 57) && // 0-9
+      (c !== 95) && // _
+      (c !== 46) // .
+    ) {
+      break;
+    }
+    pos++;
+  } while (pos < str.length);
+  // followed by a (
+  if (str.charCodeAt(pos) === PAREN_OPEN) {
+    return { type: FUNCTION, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNamed.js

@@ -0,0 +1,60 @@
+/* eslint-disable max-len */
+import { REF_NAMED } from '../constants.js';
+
+// The advertized named ranges rules are a bit off from what Excel seems to do.
+// In the "extended range" of chars, it looks like it allows most things above
+// U+00B0 with the range between U+00A0-U+00AF rather random:
+// /^[a-zA-Z\\_¡¤§¨ª\u00ad¯\u00b0-\uffff][a-zA-Z0-9\\_.?¡¤§¨ª\u00ad¯\u00b0-\uffff]{0,254}/
+//
+// I've simplified to allowing everything above U+00A1:
+// /^[a-zA-Z\\_\u00a1-\uffff][a-zA-Z0-9\\_.?\u00a1-\uffff]{0,254}/
+export function lexNamed (str, pos) {
+  const start = pos;
+  // starts with: [a-zA-Z\\_\u00a1-\uffff]
+  const s = str.charCodeAt(pos);
+  if (
+    (s >= 65 && s <= 90) || // A-Z
+    (s >= 97 && s <= 122) || // a-z
+    (s === 95) || // _
+    (s === 92) || // \
+    (s > 0xA0) // \u00a1-\uffff
+  ) {
+    pos++;
+  }
+  else {
+    return;
+  }
+  // has any number of: [a-zA-Z0-9\\_.?\u00a1-\uffff]
+  let c;
+  do {
+    c = str.charCodeAt(pos);
+    if (
+      (c >= 65 && c <= 90) || // A-Z
+      (c >= 97 && c <= 122) || // a-z
+      (c >= 48 && c <= 57) || // 0-9
+      (c === 95) || // _
+      (c === 92) || // \
+      (c === 46) || // .
+      (c === 63) || // ?
+      (c > 0xA0) // \u00a1-\uffff
+    ) {
+      pos++;
+    }
+    else {
+      break;
+    }
+  } while (isFinite(c));
+
+  const len = pos - start;
+  if (len && len < 255) {
+    // names starting with \ must be at least 3 char long
+    if (s === 92 && len < 3) {
+      return;
+    }
+    // single characters R and C are forbidden as names
+    if (len === 1 && (s === 114 || s === 82 || s === 99 || s === 67)) {
+      return;
+    }
+    return { type: REF_NAMED, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNewLine.js

@@ -0,0 +1,11 @@
+import { NEWLINE } from '../constants.js';
+
+export function lexNewLine (str, pos) {
+  const start = pos;
+  while (str.charCodeAt(pos) === 10) {
+    pos++;
+  }
+  if (pos !== start) {
+    return { type: NEWLINE, value: str.slice(start, pos) };
+  }
+}

package/lib/lexers/lexNumber.js

@@ -0,0 +1,47 @@
+import { NUMBER } from '../constants.js';
+
+function advDigits (str, pos) {
+  const start = pos;
+  do {
+    const c = str.charCodeAt(pos);
+    if (c < 48 || c > 57) { // 0-9
+      break;
+    }
+    pos++;
+  }
+  while (pos < str.length);
+  return pos - start;
+}
+
+// \d+(\.\d+)?(?:[eE][+-]?\d+)?
+export function lexNumber (str, pos) {
+  const start = pos;
+
+  // integer
+  const lead = advDigits(str, pos);
+  if (!lead) { return; }
+  pos += lead;
+
+  // optional fraction part
+  const c0 = str.charCodeAt(pos);
+  if (c0 === 46) { // .
+    pos++;
+    const frac = advDigits(str, pos);
+    if (!frac) { return; }
+    pos += frac;
+  }
+  // optional exponent part
+  const c1 = str.charCodeAt(pos);
+  if (c1 === 69 || c1 === 101) { // E e
+    pos++;
+    const sign = str.charCodeAt(pos);
+    if (sign === 43 || sign === 45) { // + -
+      pos++;
+    }
+    const exp = advDigits(str, pos);
+    if (!exp) { return; }
+    pos += exp;
+  }
+
+  return { type: NUMBER, value: str.slice(start, pos) };
+}

package/lib/lexers/lexOperator.js

@@ -0,0 +1,25 @@
+import { OPERATOR } from '../constants.js';
+
+export function lexOperator (str, pos) {
+  const c0 = str.charCodeAt(pos);
+  const c1 = str.charCodeAt(pos + 1);
+  if (
+    (c0 === 60 && c1 === 61) || // <=
+    (c0 === 62 && c1 === 61) || // >=
+    (c0 === 60 && c1 === 62)    // <>
+  ) {
+    return { type: OPERATOR, value: str.slice(pos, pos + 2) };
+  }
+  if (
+    // { } ! # % &
+    c0 === 123 || c0 === 125 || c0 === 33 || c0 === 35 || c0 === 37 || c0 === 38 ||
+    // ( ) * + , -
+    c0 === 40 || c0 === 41 || c0 === 42 || c0 === 43 || c0 === 44 || c0 === 45 ||
+    // / : ; < = >
+    c0 === 47 || c0 === 58 || c0 === 59 || c0 === 60 || c0 === 61 || c0 === 62 ||
+    // @ ^
+    c0 === 64 || c0 === 94
+  ) {
+    return { type: OPERATOR, value: str[pos] };
+  }
+}

package/lib/lexers/lexRange.js

@@ -0,0 +1,8 @@
+import { lexRangeA1 } from './lexRangeA1.js';
+import { lexRangeR1C1 } from './lexRangeR1C1.js';
+
+export function lexRange (str, pos, options) {
+  return options.r1c1
+    ? lexRangeR1C1(str, pos, options)
+    : lexRangeA1(str, pos, options);
+}

package/lib/lexers/lexRangeA1.js

@@ -0,0 +1,130 @@
+/* eslint-disable no-mixed-operators */
+import { REF_RANGE, REF_BEAM, REF_TERNARY, MAX_COLS, MAX_ROWS } from '../constants.js';
+import { advRangeOp } from './advRangeOp.js';
+import { canEndRange, canEndPartialRange } from './canEndRange.js';
+
+function advA1Col (str, pos) {
+  // [A-Z]{1,3}
+  const start = pos;
+  if (str.charCodeAt(pos) === 36) { // $
+    pos++;
+  }
+  const stop = pos + 3;
+  let col = 0;
+  do {
+    const c = str.charCodeAt(pos);
+    if (c >= 65 && c <= 90) { // A-Z
+      col = 26 * col + c - 64;
+      pos++;
+    }
+    else if (c >= 97 && c <= 122) { // a-z
+      col = 26 * col + c - 96;
+      pos++;
+    }
+    else {
+      break;
+    }
+  }
+  while (pos < stop && pos < str.length);
+  return (col && col <= MAX_COLS + 1) ? pos - start : 0;
+}
+
+function advA1Row (str, pos) {
+  // [1-9][0-9]{0,6}
+  const start = pos;
+  if (str.charCodeAt(pos) === 36) { // $
+    pos++;
+  }
+  const stop = pos + 7;
+  let row = 0;
+  let c = str.charCodeAt(pos);
+  if (c >= 49 && c <= 57) { // 1-9
+    row = row * 10 + c - 48;
+    pos++;
+    do {
+      c = str.charCodeAt(pos);
+      if (c >= 48 && c <= 57) { // 0-9
+        row = row * 10 + c - 48;
+        pos++;
+      }
+      else {
+        break;
+      }
+    }
+    while (pos < stop && pos < str.length);
+  }
+  return (row && row <= MAX_ROWS + 1) ? pos - start : 0;
+}
+
+export function lexRangeA1 (str, pos, options) {
+  let p = pos;
+  const left = advA1Col(str, p);
+  let right = 0;
+  let bottom = 0;
+  if (left) {
+    // TLBR: could be A1:A1
+    // TL R: could be A1:A (if allowTernary)
+    // TLB : could be A1:1 (if allowTernary)
+    //  LBR: could be A:A1 (if allowTernary)
+    //  L R: could be A:A
+    p += left;
+    const top = advA1Row(str, p);
+    p += top;
+    const op = advRangeOp(str, p);
+    const preOp = p;
+    if (op) {
+      p += op;
+      right = advA1Col(str, p);
+      p += right;
+      bottom = advA1Row(str, p);
+      p += bottom;
+      if (top && bottom && right) {
+        if (canEndRange(str, p) && options.mergeRefs) {
+          return { type: REF_RANGE, value: str.slice(pos, p) };
+        }
+      }
+      else if (!top && !bottom) {
+        if (canEndRange(str, p)) {
+          return { type: REF_BEAM, value: str.slice(pos, p) };
+        }
+      }
+      else if (options.allowTernary && (bottom || right)) {
+        if (canEndPartialRange(str, p)) {
+          return { type: REF_TERNARY, value: str.slice(pos, p) };
+        }
+      }
+    }
+    // LT  : this is A1
+    if (top && canEndRange(str, preOp)) {
+      return { type: REF_RANGE, value: str.slice(pos, preOp) };
+    }
+  }
+  else {
+    // T B : could be 1:1
+    // T BR: could be 1:A1 (if allowTernary)
+    const top = advA1Row(str, p);
+    if (top) {
+      p += top;
+      const op = advRangeOp(str, p);
+      if (op) {
+        p += op;
+        right = advA1Col(str, p);
+        if (right) {
+          p += right;
+        }
+        bottom = advA1Row(str, p);
+        p += bottom;
+        if (right && bottom && options.allowTernary) {
+          if (canEndPartialRange(str, p)) {
+            return { type: REF_TERNARY, value: str.slice(pos, p) };
+          }
+        }
+        if (!right && bottom) {
+          if (canEndRange(str, p)) {
+            return { type: REF_BEAM, value: str.slice(pos, p) };
+          }
+        }
+      }
+    }
+  }
+}