@borgar/fx 4.7.1 → 4.9.0

package/docs/AST_format.md

@@ -6,7 +6,7 @@ This document specifies the core AST node types that support the Excel grammar.
 
 All AST nodes are represented by `Node` objects. They may have any prototype inheritance but implement the following basic interface:
 
-```
+```ts
 interface Node {
   type: string;
   loc?: Location | null;
@@ -17,7 +17,7 @@ The `type` field is a string representing the AST variant type. Each subtype of
 
 The `loc` field represents the source location information of the node. If the node contains no information about the source location, the field is `null`; otherwise it is an array consisting of a two numbers: A start offset (the position of the first character of the parsed source region) and an end offset (the position of the first character after the parsed source region):
 
-```
+```ts
 interface Location extends Array<number> {
   0: number;
   1: number;
@@ -26,30 +26,31 @@ interface Location extends Array<number> {
 
 ## Identifier
 
-```
+```ts
 interface Identifier extends Node {
   type: "Identifier";
   name: string;
 }
 ```
 
-An identifier. These only appear on `CallExpression` and will always be a static string representing the name of a function call.
+An identifier. These appear on `CallExpression`, `LambdaExpression`, and `LetExpression` and will always be a static string representing the name of a function call or parameter.
 
 ## ReferenceIdentifier
 
-```
+```ts
 interface ReferenceIdentifier extends Node {
   type: "ReferenceIdentifier";
   value: string;
+  kind: "name" | "range" | "beam" | "table";
 }
 ```
 
-A range identifier.
+An identifier for a range or a named. The 
 
 
 ## Literal
 
-```
+```ts
 interface Literal extends Node {
   type: "Literal";
   raw: string;
@@ -61,7 +62,7 @@ A literal token. Captures numbers, strings, and booleans. Literal errors have th
 
 ## ErrorLiteral
 
-```
+```ts
 interface ErrorLiteral extends Node {
   type: "ErrorLiteral";
   raw: string;
@@ -73,7 +74,7 @@ An Error expression.
 
 ## UnaryExpression
 
-```
+```ts
 interface UnaryExpression extends Node {
   type: "UnaryExpression";
   operator: UnaryOperator;
@@ -85,7 +86,7 @@ A unary operator expression.
 
 ### UnaryOperator 
 
-```
+```ts
 type UnaryOperator = (
   "+" | "-" | "%" | "#" | "@"
 )
@@ -95,7 +96,7 @@ A unary operator token.
 
 ## BinaryExpression
 
-```
+```ts
 interface BinaryExpression extends Node {
   type: "BinaryExpression";
   operator: BinaryOperator; 
@@ -107,7 +108,7 @@ A binary operator expression.
 
 ### BinaryOperator 
 
-```
+```ts
 type BinaryOperator = (
   "=" | "<" | ">" | "<=" | ">=" |  "<>" |
   "-" | "+" | "*" | "/" | "^" |
@@ -120,7 +121,7 @@ A binary operator token. Note that Excels union operator is whitespace so a pars
 
 ## CallExpression
 
-```
+```ts
 interface CallExpression extends Node {
   type: "CallExpression";
   callee: Identifier;
@@ -132,13 +133,42 @@ A function call expression.
 
 ## ArrayExpression
 
-```
+```ts
 interface ArrayExpression extends Node {
   type: "ArrayExpression";
-  elements: Array<Array<Literal | Error | ReferenceIdentifier>>;
+  elements: Array<Array<ReferenceIdentifier | Literal | ErrorLiteral | CallExpression>>;
 }
 ```
 
 An array expression. Excel does not have empty or sparse arrays and restricts array elements to literals. Google Sheets allows `ReferenceIdentifier`s as elements of arrays, the fx parser as an option for this but it is off by default.
 
+## LambdaExpression
+
+```ts
+interface LambdaExpression extends Node {
+  type: "LambdaExpression";
+  params: Array<Identifier>;
+  body: null | Node;
+}
+```
+
+## LetExpression
+
+```ts
+interface LetExpression extends Node {
+  type: "LetExpression";
+  declarations: Array<LetDeclarator>;
+  body: null | Node;
+}
+```
+
+## LetDeclarator
+
+```ts
+interface LetDeclarator extends Node {
+  type: "LetDeclarator";
+  id: Identifier;
+  init: null | Node;
+}
+```
 

package/lib/astTypes.js

@@ -0,0 +1,96 @@
+/* eslint-disable jsdoc/require-property-description */
+
+/**
+ * @typedef {number[]} SourceLocation
+ */
+
+/**
+ * @typedef {Node} Identifier
+ * @property {"Identifier"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} name
+ */
+
+/**
+ * @typedef {Node} ReferenceIdentifier
+ * @property {"ReferenceIdentifier"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} value
+ * @property {"name" | "range" | "beam" | "table"} kind
+ */
+
+/**
+ * @typedef {Node} Literal
+ * @property {"Literal"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} raw
+ * @property {string | number | boolean} value
+ */
+
+/**
+ * @typedef {Node} ErrorLiteral
+ * @property {"ErrorLiteral"} type
+ * @property {SourceLocation} [loc]
+ * @property {string} raw
+ * @property {string} value
+ */
+
+/**
+ * @typedef {Node} UnaryExpression
+ * @property {"UnaryExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {"+" | "-" | "%" | "#" | "@"} operator
+ * @property {AstExpression[]} arguments
+ */
+
+/**
+ * @typedef {Node} BinaryExpression
+ * @property {"BinaryExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {"=" | "<" | ">" | "<=" | ">=" | "<>" | "-" | "+" | "*" | "/" | "^" | ":" | " " | "," | "&"} operator
+ * @property {AstExpression[]} arguments
+ */
+
+/**
+ * @typedef {Node} CallExpression
+ * @property {"CallExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier} callee
+ * @property {AstExpression[]} arguments
+ */
+
+// FIXME: the awkward naming is because tooling fails, fix tooling :)
+/**
+ * @typedef {Node} MatrixExpression
+ * @property {"ArrayExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Array<Array<ReferenceIdentifier | Literal | ErrorLiteral | CallExpression>>} arguments
+ */
+
+/**
+ * @typedef {Node} LambdaExpression
+ * @property {"LambdaExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier[]} params
+ * @property {null | AstExpression} body
+ */
+
+/**
+ * @typedef {Node} LetExpression
+ * @property {"LetExpression"} type
+ * @property {SourceLocation} [loc]
+ * @property {LetDeclarator[]} declarations
+ * @property {null | AstExpression} body
+ */
+
+/**
+ * @typedef {Node} LetDeclarator
+ * @property {"LetDeclarator"} type
+ * @property {SourceLocation} [loc]
+ * @property {Identifier} id
+ * @property {null | AstExpression} init
+ */
+
+/**
+ * @typedef {ReferenceIdentifier | Literal | ErrorLiteral | UnaryExpression | BinaryExpression | CallExpression | MatrixExpression | LambdaExpression | LetExpression} AstExpression
+ */

package/lib/constants.js

@@ -22,8 +22,11 @@ export const REFERENCE = 'ReferenceIdentifier';
 export const LITERAL = 'Literal';
 export const ERROR_LITERAL = 'ErrorLiteral';
 export const CALL = 'CallExpression';
+export const LAMBDA = 'LambdaExpression';
+export const LET = 'LetExpression';
 export const ARRAY = 'ArrayExpression';
 export const IDENTIFIER = 'Identifier';
+export const LET_DECL = 'LetDeclarator';
 
 export const MAX_COLS = (2 ** 14) - 1; // 16383
 export const MAX_ROWS = (2 ** 20) - 1; // 1048575

package/lib/fixRanges.js

@@ -44,6 +44,7 @@ import { REF_STRUCT } from './constants.js';
  * @param {object} [options={}]  Options
  * @param {boolean} [options.addBounds=false]  Fill in any undefined bounds of range objects. Top to 0, bottom to 1048575, left to 0, and right to 16383.
  * @param {boolean} [options.xlsx=false]  Switches to the `[1]Sheet1!A1` or `[1]!name` prefix syntax form for external workbooks. See: [Prefixes.md](./Prefixes.md)
+ * @param {boolean} [options.thisRow=false]  Enforces using the `[#This Row]` instead of the `@` shorthand when serializing structured ranges.
  * @returns {(string | Array<Token>)} A formula string or token list (depending on which was input)
  */
 export function fixRanges (formula, options = { addBounds: false }) {
@@ -55,7 +56,7 @@ export function fixRanges (formula, options = { addBounds: false }) {
   if (!Array.isArray(formula)) {
     throw new Error('fixRanges expects an array of tokens');
   }
-  const { addBounds, r1c1, xlsx } = options;
+  const { addBounds, r1c1, xlsx, thisRow } = options;
   if (r1c1) {
     throw new Error('fixRanges does not have an R1C1 mode');
   }
@@ -68,7 +69,7 @@ export function fixRanges (formula, options = { addBounds: false }) {
     let offsetDelta = 0;
     if (token.type === REF_STRUCT) {
       const sref = parseStructRef(token.value, { xlsx });
-      const newValue = stringifyStructRef(sref, { xlsx });
+      const newValue = stringifyStructRef(sref, { xlsx, thisRow });
       offsetDelta = newValue.length - token.value.length;
       token.value = newValue;
     }

package/lib/lexer.js

@@ -38,11 +38,43 @@ const causesBinaryMinus = token => {
   );
 };
 
+function fixRCNames (tokens) {
+  let withinCall = 0;
+  let parenDepth = 0;
+  let lastToken;
+  for (const token of tokens) {
+    if (token.type === OPERATOR) {
+      if (token.value === '(') {
+        parenDepth++;
+        if (lastToken.type === FUNCTION) {
+          const v = lastToken.value.toLowerCase();
+          if (v === 'lambda' || v === 'let') {
+            withinCall = parenDepth;
+          }
+        }
+      }
+      else if (token.value === ')') {
+        parenDepth--;
+        if (parenDepth < withinCall) {
+          withinCall = 0;
+        }
+      }
+    }
+    else if (withinCall && token.type === UNKNOWN && /^[rc]$/.test(token.value)) {
+      token.type = REF_NAMED;
+    }
+    lastToken = token;
+  }
+  return tokens;
+}
+
 export function getTokens (fx, tokenHandlers, options = {}) {
   const opts = Object.assign({}, defaultOptions, options);
   const { withLocation, mergeRefs, negativeNumbers } = opts;
   const tokens = [];
   let pos = 0;
+  let letOrLambda = 0;
+  let unknownRC = 0;
 
   let tail0 = null; // last non-whitespace token
   let tail1 = null; // penultimate non-whitespace token
@@ -110,6 +142,19 @@ export function getTokens (fx, tokenHandlers, options = {}) {
       ...(withLocation ? { loc: [ startPos, pos ] } : {})
     };
 
+    // make a note if we found a let/lambda call
+    if (lastToken && lastToken.type === FUNCTION && tokenValue === '(') {
+      const lastLC = lastToken.value.toLowerCase();
+      if (lastLC === 'lambda' || lastLC === 'let') {
+        letOrLambda++;
+      }
+    }
+    // make a note if we found a R or C unknown
+    if (tokenType === UNKNOWN) {
+      const valLC = tokenValue.toLowerCase();
+      unknownRC += (valLC === 'r' || valLC === 'c') ? 1 : 0;
+    }
+
     // check for termination
     if (tokenType === STRING) {
       const l = tokenValue.length;
@@ -157,6 +202,12 @@ export function getTokens (fx, tokenHandlers, options = {}) {
     pushToken(token);
   }
 
+  // if we encountered both a LAMBDA/LET call, and unknown 'r' or 'c' tokens
+  // we'll turn the unknown tokens into names within the call.
+  if (unknownRC && letOrLambda) {
+    fixRCNames(tokens);
+  }
+
   if (mergeRefs) {
     return mergeRefTokens(tokens);
   }

package/lib/lexer.spec.js

@@ -1462,6 +1462,18 @@ test('unknowns, named ranges and functions', t => {
     { type: FX_PREFIX, value: '=' },
     { type: REF_NAMED, value: '\\foo' }
   ]);
+  t.isTokens('=\\fo', [
+    { type: FX_PREFIX, value: '=' },
+    { type: REF_NAMED, value: '\\fo' }
+  ]);
+  t.isTokens('=\\f', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: '\\f' }
+  ]);
+  t.isTokens('=\\', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: '\\' }
+  ]);
   t.isTokens('=æði', [
     { type: FX_PREFIX, value: '=' },
     { type: REF_NAMED, value: 'æði' }
@@ -1769,3 +1781,54 @@ test('tokenize external refs syntax from XLSX files', t => {
 
   t.end();
 });
+
+test('tokenize r and c as names within LET and LAMBDA calls', t => {
+  t.isTokens('=c*(LAMBDA(r,c,r*c)+r)+r', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: 'c' },
+    { type: OPERATOR, value: '*' },
+    { type: OPERATOR, value: '(' },
+    { type: FUNCTION, value: 'LAMBDA' },
+    { type: OPERATOR, value: '(' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' }
+  ]);
+  t.isTokens('=c*(LET(r,A1,c,B2,r*c)+r)+r', [
+    { type: FX_PREFIX, value: '=' },
+    { type: UNKNOWN, value: 'c' },
+    { type: OPERATOR, value: '*' },
+    { type: OPERATOR, value: '(' },
+    { type: FUNCTION, value: 'LET' },
+    { type: OPERATOR, value: '(' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_RANGE, value: 'A1' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_RANGE, value: 'B2' },
+    { type: OPERATOR, value: ',' },
+    { type: REF_NAMED, value: 'r' },
+    { type: OPERATOR, value: '*' },
+    { type: REF_NAMED, value: 'c' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' },
+    { type: OPERATOR, value: ')' },
+    { type: OPERATOR, value: '+' },
+    { type: UNKNOWN, value: 'r' }
+  ]);
+
+  t.end();
+});

package/lib/lexerParts.js

@@ -67,6 +67,11 @@ function lexNamed (str) {
   const m = re_NAMED.exec(str);
   if (m) {
     const lc = m[0].toLowerCase();
+    // names starting with \ must be at least 3 char long
+    if (lc[0] === '\\' && m[0].length < 3) {
+      return null;
+    }
+    // single characters R and C are forbidden as names
     if (lc === 'r' || lc === 'c') {
       return null;
     }