@borgar/fx 4.7.1 → 4.8.0

package/lib/astTypes.js

@@ -0,0 +1,96 @@
+/* eslint-disable jsdoc/require-property-description */
+
+/**
+ * @typedef {number[]} SourceLocation
+ */
+
+/**
+ * @typedef {Node} Identifier
+ * @property {"Identifier"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} name
+ */
+
+/**
+ * @typedef {Node} ReferenceIdentifier
+ * @property {"ReferenceIdentifier"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} value
+ * @property {"name" | "range" | "beam" | "table"} kind
+ */
+
+/**
+ * @typedef {Node} Literal
+ * @property {"Literal"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} raw
+ * @property {string | number | boolean} value
+ */
+
+/**
+ * @typedef {Node} ErrorLiteral
+ * @property {"ErrorLiteral"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} raw
+ * @property {string} value
+ */
+
+/**
+ * @typedef {Node} UnaryExpression
+ * @property {"UnaryExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {"+" | "-" | "%" | "#" | "@"} operator
+ * @property {AstExpression[]} arguments
+ */
+
+/**
+ * @typedef {Node} BinaryExpression
+ * @property {"BinaryExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {"=" | "<" | ">" | "<=" | ">=" | "<>" | "-" | "+" | "*" | "/" | "^" | ":" | " " | "," | "&"} operator
+ * @property {AstExpression[]} arguments
+ */
+
+/**
+ * @typedef {Node} CallExpression
+ * @property {"CallExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier} callee
+ * @property {AstExpression[]} arguments
+ */
+
+// FIXME: the awkward naming is because tooling fails, fix tooling :)
+/**
+ * @typedef {Node} MatrixExpression
+ * @property {"ArrayExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Array<Array<ReferenceIdentifier | Literal | ErrorLiteral | CallExpression>>} arguments
+ */
+
+/**
+ * @typedef {Node} LambdaExpression
+ * @property {"LambdaExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier[]} params
+ * @property {null | AstExpression} body
+ */
+
+/**
+ * @typedef {Node} LetExpression
+ * @property {"LetExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {LetDeclarator[]} declarations
+ * @property {null | AstExpression} body
+ */
+
+/**
+ * @typedef {Node} LetDeclarator
+ * @property {"LetDeclarator"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier} id
+ * @property {null | AstExpression} init
+ */
+
+/**
+ * @typedef {ReferenceIdentifier | Literal | ErrorLiteral | UnaryExpression | BinaryExpression | CallExpression | MatrixExpression | LambdaExpression | LetExpression} AstExpression
+ */

package/lib/constants.js

@@ -22,8 +22,11 @@ export const REFERENCE = 'ReferenceIdentifier';
 export const LITERAL = 'Literal';
 export const ERROR_LITERAL = 'ErrorLiteral';
 export const CALL = 'CallExpression';
+export const LAMBDA = 'LambdaExpression';
+export const LET = 'LetExpression';
 export const ARRAY = 'ArrayExpression';
 export const IDENTIFIER = 'Identifier';
+export const LET_DECL = 'LetDeclarator';
 
 export const MAX_COLS = (2 ** 14) - 1; // 16383
 export const MAX_ROWS = (2 ** 20) - 1; // 1048575

package/lib/lexer.js

@@ -38,11 +38,43 @@ const causesBinaryMinus = token => {
   );
 };
 
+function fixRCNames (tokens) {
+  let withinCall = 0;
+  let parenDepth = 0;
+  let lastToken;
+  for (const token of tokens) {
+    if (token.type === OPERATOR) {
+      if (token.value === '(') {
+        parenDepth++;
+        if (lastToken.type === FUNCTION) {
+          const v = lastToken.value.toLowerCase();
+          if (v === 'lambda' || v === 'let') {
+            withinCall = parenDepth;
+          }
+        }
+      }
+      else if (token.value === ')') {
+        parenDepth--;
+        if (parenDepth < withinCall) {
+          withinCall = 0;
+        }
+      }
+    }
+    else if (withinCall && token.type === UNKNOWN && /^[rc]$/.test(token.value)) {
+      token.type = REF_NAMED;
+    }
+    lastToken = token;
+  }
+  return tokens;
+}
+
 export function getTokens (fx, tokenHandlers, options = {}) {
   const opts = Object.assign({}, defaultOptions, options);
   const { withLocation, mergeRefs, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
+  let letOrLambda = 0;
+  let unknownRC = 0;
 
   let tail0 = null; // last non-whitespace token
   let tail1 = null; // penultimate non-whitespace token
@@ -110,6 +142,19 @@ export function getTokens (fx, tokenHandlers, options = {}) {
       ...(withLocation ? { loc: [ startPos, pos ] } : {})
     };
 
+    // make a note if we found a let/lambda call
+    if (lastToken && lastToken.type === FUNCTION && tokenValue === '(') {
+      const lastLC = lastToken.value.toLowerCase();
+      if (lastLC === 'lambda' || lastLC === 'let') {
+        letOrLambda++;
+      }
+    }
+    // make a note if we found a R or C unknown
+    if (tokenType === UNKNOWN) {
+      const valLC = tokenValue.toLowerCase();
+      unknownRC += (valLC === 'r' || valLC === 'c') ? 1 : 0;
+    }
+
     // check for termination
     if (tokenType === STRING) {
       const l = tokenValue.length;
@@ -157,6 +202,12 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     pushToken(token);
   }
 
+  // if we encountered both a LAMBDA/LET call, and unknown 'r' or 'c' tokens
+  // we'll turn the unknown tokens into names within the call.
+  if (unknownRC && letOrLambda) {
+    fixRCNames(tokens);
+  }
+
   if (mergeRefs) {
     return mergeRefTokens(tokens);
   }

package/lib/lexer.spec.js

@@ -1462,6 +1462,18 @@ test('unknowns, named ranges and functions', t => {
     { type: FX_PREFIX, value: '=' },
     { type: REF_NAMED, value: '\\foo' }
   ]);
+  t.isTokens('=\\fo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: REF_NAMED, value: '\\fo' }
+  ]);
+  t.isTokens('=\\f', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: '\\f' }
+  ]);
+  t.isTokens('=\\', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: '\\' }
+  ]);
   t.isTokens('=æði', [
     { type: FX_PREFIX, value: '=' },
     { type: REF_NAMED, value: 'æði' }
@@ -1769,3 +1781,54 @@ test('tokenize external refs syntax from XLSX files', t => {
 
   t.end();
 });
+
+test('tokenize r and c as names within LET and LAMBDA calls', t => {
+  t.isTokens('=c*(LAMBDA(r,c,r*c)+r)+r', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: 'c' },
+    { type: OPERATOR, value: '*' },
+    { type: OPERATOR, value: '(' },
+    { type: FUNCTION, value: 'LAMBDA' },
+    { type: OPERATOR, value: '(' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' }
+  ]);
+  t.isTokens('=c*(LET(r,A1,c,B2,r*c)+r)+r', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: 'c' },
+    { type: OPERATOR, value: '*' },
+    { type: OPERATOR, value: '(' },
+    { type: FUNCTION, value: 'LET' },
+    { type: OPERATOR, value: '(' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_RANGE, value: 'A1' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_RANGE, value: 'B2' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' }
+  ]);
+
+  t.end();
+});

package/lib/lexerParts.js

@@ -67,6 +67,11 @@ function lexNamed (str) {
   const m = re_NAMED.exec(str);
   if (m) {
     const lc = m[0].toLowerCase();
+    // names starting with \ must be at least 3 char long
+    if (lc[0] === '\\' && m[0].length < 3) {
+      return null;
+    }
+    // single characters R and C are forbidden as names
     if (lc === 'r' || lc === 'c') {
       return null;
     }

package/lib/parser.js

@@ -6,14 +6,17 @@
  * Beutiful Code (http://crockford.com/javascript/tdop/tdop.html).
  *
  * The parser handles most basic things Excel/Sheets do except:
- *
- * - LAMBDA expressions: =LAMBDA(x, x*x)(2)
- *     https://support.microsoft.com/en-us/office/lambda-function-bd212d27-1cd1-4321-a34a-ccbf254b8b67
- * - LET expressions: LET(x, 5, SUM(x, 1))
- *     https://support.microsoft.com/en-us/office/let-function-34842dd8-b92b-4d3f-b325-b8b8f9908999
- * - Sheet1:Sheet2!A1 references cross contexts (3D references)
+ * `Sheet1:Sheet2!A1` references cross contexts (3D references)
  */
-import { isReference, isLiteral, isFunction, isWhitespace, isFxPrefix, isOperator, isError } from './isType.js';
+import {
+  isReference,
+  isLiteral,
+  isFunction,
+  isWhitespace,
+  isFxPrefix,
+  isOperator,
+  isError
+} from './isType.js';
 import {
   UNARY,
   BINARY,
@@ -21,12 +24,18 @@ import {
   LITERAL,
   ERROR_LITERAL,
   CALL,
+  LAMBDA,
   ARRAY,
   IDENTIFIER,
   NUMBER,
   BOOLEAN,
   ERROR,
-  STRING
+  STRING,
+  LET,
+  LET_DECL,
+  REF_NAMED,
+  REF_STRUCT,
+  REF_BEAM
 } from './constants.js';
 
 import { tokenize } from './lexer.js';
@@ -53,12 +62,24 @@ const refFunctions = [
   'XLOOKUP'
 ];
 
-const isReferenceToken = token => {
+const isReferenceFunctionName = fnName => {
+  return refFunctions.includes(fnName.toUpperCase());
+};
+
+const isReferenceToken = (token, allowOperators = false) => {
   const value = (token && token.value) + '';
-  if (isReference(token)) { return true; }
-  if (isOperator(token) && (value === ':' || value === ',' || !value.trim())) { return true; } // join, union, intersection
-  if (isFunction(token) && refFunctions.includes(value.toUpperCase())) { return true; } // intersection
-  if (isError(token) && value === '#REF!') { return true; }
+  if (isReference(token)) {
+    return true;
+  }
+  if (allowOperators && isOperator(token) && (value === ':' || value === ',' || !value.trim())) {
+    return true; // join, union, intersection
+  }
+  if (isFunction(token) && isReferenceFunctionName(value)) {
+    return true; // function that yields reference
+  }
+  if (isError(token) && value === '#REF!') {
+    return true;
+  }
   return false;
 };
 
@@ -71,7 +92,8 @@ const isReferenceNode = node => {
       node.operator === ' ' ||
       node.operator === ',')
     ) ||
-    (node.type === CALL && refFunctions.includes(node.callee.name.toUpperCase()))
+    isReference(node) ||
+    (node.type === CALL && isReferenceFunctionName(node.callee.name))
   );
 };
 
@@ -82,15 +104,17 @@ let tokenIndex;
 let permitArrayRanges = false;
 let permitArrayCalls = false;
 
-function halt (message) {
+function halt (message, atIndex = null) {
   const err = new Error(message);
   err.source = tokens.map(d => d.value).join('');
-  err.sourceOffset = tokens.slice(0, tokenIndex).reduce((a, d) => a + d.value, '').length;
+  err.sourceOffset = tokens
+    .slice(0, atIndex ?? tokenIndex)
+    .reduce((a, d) => a + d.value.length, 0);
   throw err;
 }
 
 // A1 A1 | A1 (A1) | A1 ((A1)) | A1 ( (A1) ) | ...
-function refIsUpcoming () {
+function refIsUpcoming (allowOperators = false) {
   let i = tokenIndex;
   let next;
   do {
@@ -102,7 +126,7 @@ function refIsUpcoming () {
       (isOperator(next) && next.value === '(')
     )
   );
-  return isReferenceToken(next);
+  return isReferenceToken(next, allowOperators);
 }
 
 function advance (expectNext = null, leftNode = null) {
@@ -111,9 +135,11 @@ function advance (expectNext = null, leftNode = null) {
   }
   // look ahead to see if we have ( ( " ", "(" )+ REF )
   if (isWhitespace(tokens[tokenIndex])) {
-    // potential intersection operation
-    const possibleWSOp = isReferenceNode(leftNode) && refIsUpcoming();
-    if (!possibleWSOp) {
+    // potential intersection operation (so don't allow operators as upcoming)
+    const haveRef = isReferenceNode(leftNode);
+    const possibleWSOp = haveRef && refIsUpcoming(false);
+    const nextIsCall = haveRef && tokens[tokenIndex + 1].value === '(';
+    if (!possibleWSOp && !nextIsCall) {
       // ignore whitespace
       while (isWhitespace(tokens[tokenIndex])) {
         tokenIndex++;
@@ -134,7 +160,6 @@ function advance (expectNext = null, leftNode = null) {
   }
 
   let node;
-  let type = token.type;
   if (isOperator(token)) {
     node = symbolTable[token.value];
     if (!node) {
@@ -149,7 +174,6 @@ function advance (expectNext = null, leftNode = null) {
   }
   else if (isReference(token)) {
     node = symbolTable[REFERENCE];
-    type = REFERENCE;
   }
   else if (isFunction(token)) {
     node = symbolTable[FUNCTION];
@@ -159,7 +183,7 @@ function advance (expectNext = null, leftNode = null) {
   }
 
   currentNode = Object.create(node);
-  currentNode.type = type;
+  currentNode.type = token.type;
   currentNode.value = token.value;
   if (token.loc) {
     currentNode.loc = [ ...token.loc ];
@@ -289,7 +313,7 @@ const unionRefs = enable => {
 
 // arithmetic and string operations
 postfix('%'); // percent
-postfix('#', function (left) {
+postfix('#', function (left) { // spilled range (_xlfn.ANCHORARRAY)
   if (!isReferenceNode(left)) {
     halt('# expects a reference');
   }
@@ -340,6 +364,18 @@ symbol(LITERAL).nud = function () {
   return this;
 };
 symbol(REFERENCE).nud = function () {
+  if (this.type === REF_NAMED) {
+    this.kind = 'name';
+  }
+  else if (this.type === REF_STRUCT) {
+    this.kind = 'table'; // structured ?
+  }
+  else if (this.type === REF_BEAM) {
+    this.kind = 'beam';
+  }
+  else {
+    this.kind = 'range';
+  }
   this.type = REFERENCE;
   return this;
 };
@@ -359,8 +395,35 @@ symbol(FUNCTION).nud = function () {
   return this;
 };
 infix('(', 90, function (left) {
+  let callee = {
+    type: IDENTIFIER,
+    name: left.value
+  };
   if (left.id !== FUNCTION) {
-    halt('Cannot call a ' + left.type);
+    if (
+      left.type === LAMBDA ||
+      // Excel only allows calls to "names" and ref functions. Since we don't
+      // differentiate between the two (this requires a table of function names)
+      // we're overly permissive here:
+      left.type === CALL ||
+      left.type === LET ||
+      left.type === REFERENCE ||
+      (left.type === UNARY && left.value === '#') || // Because it's really SINGLE(...)()
+      (left.type === ERROR_LITERAL && left.value === '#REF!')
+    ) {
+      // in the case of REFERENCE, do we want to set the node to Identifier?
+      callee = left;
+    }
+    else {
+      halt('Unexpected call', tokenIndex - 1);
+    }
+  }
+  const lcFn = left.value.toLowerCase();
+  if (lcFn === 'lambda') {
+    return parseLambda.call(this, left);
+  }
+  if (lcFn === 'let') {
+    return parseLet.call(this, left);
   }
   const args = [];
   let lastWasComma = false;
@@ -393,10 +456,7 @@ infix('(', 90, function (left) {
   const closeParen = currentNode;
   delete this.value;
   this.type = CALL;
-  this.callee = {
-    type: IDENTIFIER,
-    name: left.value
-  };
+  this.callee = callee;
   if (left.loc) {
     this.callee.loc = [ ...left.loc ];
   }
@@ -408,6 +468,149 @@ infix('(', 90, function (left) {
   return this;
 });
 
+function parseLambda (left) {
+  const args = [];
+  const argNames = {};
+  let body;
+  let done = false;
+  const prevState = unionRefs(false);
+  if (currentNode.id !== ')') {
+    while (!done) {
+      if (isWhitespace(currentNode)) {
+        advance();
+      }
+      const argTokenIndex = tokenIndex;
+      const arg = expression(0);
+      if (currentNode.id === ',') {
+        // all but last args must be names
+        if (arg.type === REFERENCE && arg.kind === 'name') {
+          // names may not be duplicates
+          const currName = arg.value.toLowerCase();
+          if (currName in argNames) {
+            halt('Duplicate name: ' + arg.value);
+          }
+          argNames[currName] = 1;
+          const a = { type: IDENTIFIER, name: arg.value };
+          if (arg.loc) { a.loc = arg.loc; }
+          args.push(a);
+        }
+        else {
+          tokenIndex = argTokenIndex;
+          halt('LAMBDA argument is not a name');
+        }
+        advance(',');
+      }
+      else {
+        body = arg;
+        done = true;
+      }
+    }
+  }
+  unionRefs(prevState);
+  delete this.value;
+  this.type = LAMBDA;
+  this.params = args;
+  this.body = body || null;
+  if (left.loc) {
+    this.loc = [ left.loc[0], currentNode.loc[1] ];
+  }
+  advance(')', this);
+  return this;
+}
+
+function parseLet (left) {
+  const args = [];
+  const vals = [];
+  const argNames = {};
+  let body;
+  let argCounter = 0;
+  const addArgument = (arg, lastArg) => {
+    if (body) {
+      halt('Unexpected argument following calculation');
+    }
+    if (lastArg && argCounter >= 2) {
+      body = arg;
+    }
+    else {
+      const wantName = !(argCounter % 2);
+      if (wantName) {
+        if (arg && (arg.type === REFERENCE && arg.kind === 'name')) {
+          // names may not be duplicates
+          const currName = arg.value.toLowerCase();
+          if (currName in argNames) {
+            halt('Duplicate name: ' + arg.value);
+          }
+          argNames[currName] = 1;
+          args.push({ type: IDENTIFIER, name: arg.value, loc: arg.loc });
+        }
+        else if (argCounter >= 2) {
+          body = arg;
+        }
+        else {
+          halt('Argument is not a name');
+        }
+      }
+      else {
+        vals.push(arg);
+      }
+    }
+    argCounter++;
+  };
+  const prevState = unionRefs(false);
+  let lastWasComma = false;
+  if (currentNode.id !== ')') {
+    while (currentNode.id !== ')') {
+      if (isWhitespace(currentNode)) {
+        advance();
+      }
+      if (currentNode.id === ',') {
+        addArgument(null);
+        lastWasComma = true;
+        advance();
+      }
+      else {
+        const arg = expression(0);
+        addArgument(arg, currentNode.id !== ',');
+        lastWasComma = false;
+        if (currentNode.id === ',') {
+          advance(',');
+          lastWasComma = true;
+        }
+      }
+    }
+    unionRefs(prevState);
+  }
+  if (lastWasComma) {
+    addArgument(null, true);
+  }
+  // eslint-disable-next-line no-undefined
+  if (body === undefined) {
+    halt('Unexpected end of arguments');
+  }
+  unionRefs(prevState);
+  delete this.value;
+  this.type = LET;
+  this.declarations = [];
+  if (!args.length) {
+    halt('Unexpected end of arguments');
+  }
+  for (let i = 0; i < args.length; i++) {
+    const s = {
+      type: LET_DECL,
+      id: args[i],
+      init: vals[i],
+      loc: args[i].loc && [ args[i].loc[0], vals[i].loc[1] ]
+    };
+    this.declarations.push(s);
+  }
+  this.body = body;
+  if (left.loc) {
+    this.loc = [ left.loc[0], currentNode.loc[1] ];
+  }
+  advance(')', this);
+  return this;
+}
+
 // array literal
 symbol('}');
 symbol(';');
@@ -437,7 +640,6 @@ prefix('{', function () {
     else if (permitArrayCalls && isFunction(currentNode)) {
       const arg = expression(0);
       row.push(arg);
-      // FIXME: need to skip WS here?
     }
     else {
       halt(`Unexpected ${currentNode.type} in array: ${currentNode.value}`);
@@ -478,7 +680,7 @@ prefix('{', function () {
  * [AST_format.md](./AST_format.md)
  *
  * @see nodeTypes
- * @param {(string | Array<Token>)} formula An Excel formula string (an Excel expression) or an array of tokens.
+ * @param {(string | Token[])} formula An Excel formula string (an Excel expression) or an array of tokens.
  * @param {object} [options={}]  Options
  * @param {boolean} [options.allowNamed=true]  Enable parsing names as well as ranges.
  * @param {boolean} [options.allowTernary=false]  Enables the recognition of ternary ranges in the style of `A1:A` or `A1:1`. These are supported by Google Sheets but not Excel. See: References.md.
@@ -488,7 +690,7 @@ prefix('{', function () {
  * @param {boolean} [options.r1c1=false]  Ranges are expected to be in the R1C1 style format rather than the more popular A1 style.
  * @param {boolean} [options.withLocation=false]  Nodes will include source position offsets to the tokens: `{ loc: [ start, end ] }`
  * @param {boolean} [options.xlsx=false]  Switches to the `[1]Sheet1!A1` or `[1]!name` prefix syntax form for external workbooks. See: [Prefixes.md](./Prefixes.md)
- * @returns {object} An AST of nodes
+ * @returns {AstExpression} An AST of nodes
  */
 export function parse (formula, options) {
   if (typeof formula === 'string') {