@borgar/fx 4.12.0 → 5.0.0

package/lib/lexers/sets.ts

@@ -0,0 +1,51 @@
+import { lexError } from './lexError.ts';
+import { lexRangeTrim } from './lexRangeTrim.ts';
+import { lexOperator } from './lexOperator.ts';
+import { lexBoolean } from './lexBoolean.ts';
+import { lexNewLine } from './lexNewLine.ts';
+import { lexWhitespace } from './lexWhitespace.ts';
+import { lexString } from './lexString.ts';
+import { lexContextQuoted, lexContextUnquoted } from './lexContext.ts';
+import { lexRange } from './lexRange.ts';
+import { lexStructured } from './lexStructured.ts';
+import { lexNumber } from './lexNumber.ts';
+import { lexNamed } from './lexNamed.ts';
+import { lexRefOp } from './lexRefOp.ts';
+import { lexNameFuncCntx } from './lexNameFuncCntx.ts';
+import type { Token } from '../types.ts';
+
+export type PartLexer = (
+  str: string,
+  pos: number,
+  options?: Partial<{
+    xlsx: boolean,
+    allowTerniary: boolean,
+    allowTernary: boolean,
+    mergeRefs: boolean,
+    r1c1: boolean
+  }>
+) => Token | undefined;
+
+export const lexers: PartLexer[] = [
+  lexError,
+  lexRangeTrim,
+  lexOperator,
+  lexNewLine,
+  lexWhitespace,
+  lexString,
+  lexRange,
+  lexNumber,
+  lexBoolean,
+  lexContextQuoted,
+  lexNameFuncCntx,
+  lexStructured
+];
+
+export const lexersRefs = [
+  lexRefOp,
+  lexContextQuoted,
+  lexContextUnquoted,
+  lexRange,
+  lexStructured,
+  lexNamed
+];

package/lib/mergeRefTokens.spec.ts

@@ -0,0 +1,141 @@
+import { describe, test, expect } from 'vitest';
+import { CONTEXT, FUNCTION, FX_PREFIX, OPERATOR, REF_RANGE, REF_BEAM, REF_NAMED, REF_TERNARY, UNKNOWN } from './constants.ts';
+import { mergeRefTokens } from './mergeRefTokens.ts';
+import { tokenize } from './tokenize.ts';
+
+describe('mergeRefTokens', () => {
+  test('merges reference tokens and preserves metadata', () => {
+    const list = tokenize('=SUM([Wb1]Sheet1!A1:B2)', { mergeRefs: false, withLocation: true });
+
+    expect(list).toEqual([
+      { type: FX_PREFIX, value: '=', loc: [ 0, 1 ] },
+      { type: FUNCTION, value: 'SUM', loc: [ 1, 4 ] },
+      { type: OPERATOR, value: '(', loc: [ 4, 5 ] },
+
+      { type: CONTEXT, value: '[Wb1]Sheet1', loc: [ 5, 16 ] },
+      { type: OPERATOR, value: '!', loc: [ 16, 17 ] },
+      { type: REF_RANGE, value: 'A1', loc: [ 17, 19 ] },
+      { type: OPERATOR, value: ':', loc: [ 19, 20 ] },
+      { type: REF_RANGE, value: 'B2', loc: [ 20, 22 ] },
+
+      { type: OPERATOR, value: ')', loc: [ 22, 23 ] }
+    ]);
+
+    // set IDs on all tokens about to be joined
+    list[3].id = 'id1';
+    list[4].id = 'id2';
+    list[5].id = 'id3';
+    list[6].id = 'id4';
+    list[7].id = 'id5';
+
+    const mergedList = mergeRefTokens(list);
+    expect(mergedList).toEqual([
+      { type: FX_PREFIX, value: '=', loc: [ 0, 1 ] },
+      { type: FUNCTION, value: 'SUM', loc: [ 1, 4 ] },
+      { type: OPERATOR, value: '(', loc: [ 4, 5 ] },
+      { type: REF_RANGE,
+        id: 'id5', // token has the id of the first one
+        value: '[Wb1]Sheet1!A1:B2',
+        loc: [ 5, 22 ] },
+      { type: OPERATOR, value: ')', loc: [ 22, 23 ] }
+    ]);
+  });
+
+  describe('tokenize with mergeRefs enabled', () => {
+    const opts = { mergeRefs: true, allowTernary: true };
+
+    test('basic cell references', () => {
+      expect(tokenize('A1', opts)).toEqual([
+        { type: REF_RANGE, value: 'A1' }
+      ]);
+
+      expect(tokenize('A1:A1', opts)).toEqual([
+        { type: REF_RANGE, value: 'A1:A1' }
+      ]);
+    });
+
+    test('beam references', () => {
+      expect(tokenize('A:A', opts)).toEqual([
+        { type: REF_BEAM, value: 'A:A' }
+      ]);
+    });
+
+    test('ternary references', () => {
+      expect(tokenize('A1:A', opts)).toEqual([
+        { type: REF_TERNARY, value: 'A1:A' }
+      ]);
+    });
+
+    test('quoted sheet references', () => {
+      expect(tokenize('\'Sheet1\'!A1', opts)).toEqual([
+        { type: REF_RANGE, value: '\'Sheet1\'!A1' }
+      ]);
+
+      expect(tokenize('\'Sheet1\'!A:A', opts)).toEqual([
+        { type: REF_BEAM, value: '\'Sheet1\'!A:A' }
+      ]);
+
+      expect(tokenize('\'Sheet1\'!A1:A', opts)).toEqual([
+        { type: REF_TERNARY, value: '\'Sheet1\'!A1:A' }
+      ]);
+    });
+
+    test('unquoted sheet references', () => {
+      expect(tokenize('Sheet1!A1', opts)).toEqual([
+        { type: REF_RANGE, value: 'Sheet1!A1' }
+      ]);
+
+      expect(tokenize('Sheet1!A:A', opts)).toEqual([
+        { type: REF_BEAM, value: 'Sheet1!A:A' }
+      ]);
+
+      expect(tokenize('Sheet1!A1:A', opts)).toEqual([
+        { type: REF_TERNARY, value: 'Sheet1!A1:A' }
+      ]);
+    });
+
+    test('workbook references', () => {
+      expect(tokenize('[WB]Sheet1!A1', opts)).toEqual([
+        { type: REF_RANGE, value: '[WB]Sheet1!A1' }
+      ]);
+
+      expect(tokenize('[WB]Sheet1!A:A', opts)).toEqual([
+        { type: REF_BEAM, value: '[WB]Sheet1!A:A' }
+      ]);
+
+      expect(tokenize('[WB]Sheet1!A1:A', opts)).toEqual([
+        { type: REF_TERNARY, value: '[WB]Sheet1!A1:A' }
+      ]);
+
+      expect(tokenize('[WB]Sheet1!A1.:.C3', opts)).toEqual([
+        { type: REF_RANGE, value: '[WB]Sheet1!A1.:.C3' }
+      ]);
+    });
+
+    test('named references', () => {
+      expect(tokenize('foo', opts)).toEqual([
+        { type: REF_NAMED, value: 'foo' }
+      ]);
+
+      expect(tokenize('\'quoted\'!foo', opts)).toEqual([
+        { type: REF_NAMED, value: '\'quoted\'!foo' }
+      ]);
+
+      expect(tokenize('Sheet1!foo', opts)).toEqual([
+        { type: REF_NAMED, value: 'Sheet1!foo' }
+      ]);
+    });
+
+    test('path references with different formats', () => {
+      expect(tokenize('[path]!foo', opts)).toEqual([
+        { type: UNKNOWN, value: '[path]' },
+        { type: OPERATOR, value: '!' },
+        { type: REF_NAMED, value: 'foo' }
+      ]);
+
+      expect(tokenize('[path]prefix!foo', opts)).toEqual([
+        { type: REF_NAMED, value: '[path]prefix!foo' }
+      ]);
+    });
+  });
+});

package/lib/{mergeRefTokens.js → mergeRefTokens.ts}

@@ -1,4 +1,5 @@
-import { CONTEXT, CONTEXT_QUOTE, REF_RANGE, REF_NAMED, REF_BEAM, REF_TERNARY, OPERATOR, REF_STRUCT } from './constants.js';
+import { CONTEXT, CONTEXT_QUOTE, REF_RANGE, REF_NAMED, REF_BEAM, REF_TERNARY, OPERATOR, REF_STRUCT } from './constants.ts';
+import type { Token } from './types.ts';
 
 const END = '$';
 
@@ -33,13 +34,17 @@ const validRunsMerge = [
   [ CONTEXT_QUOTE, '!', REF_NAMED, REF_STRUCT ]
 ];
 
+type TypeNode = {
+  [key: string]: TypeNode | boolean;
+};
+
 // valid token runs are converted to a tree structure
-const refPartsTree = {};
-function packList (f, node) {
+const refPartsTree: TypeNode = {};
+function packList (f: string[], node: TypeNode) {
   if (f.length) {
     const key = f[0];
-    node[key] = node[key] || {};
-    packList(f.slice(1), node[key]);
+    if (!node[key]) { node[key] = {}; }
+    packList(f.slice(1), node[key] as TypeNode);
   }
   else {
     node[END] = true;
@@ -49,20 +54,24 @@ validRunsMerge.forEach(run => packList(run.concat().reverse(), refPartsTree));
 
 // attempt to match a backwards run of tokens from a given point
 // to a path in the tree
-const matcher = (tokens, currNode, anchorIndex, index = 0) => {
-  const token = tokens[anchorIndex - index];
-  if (token) {
-    const key = (token.type === OPERATOR) ? token.value : token.type;
-    if (key in currNode) {
-      return matcher(tokens, currNode[key], anchorIndex, index + 1);
+const matcher = (tokens: Token[], currNode, anchorIndex, index = 0) => {
+  let i = index;
+  let node = currNode;
+  const max = tokens.length - index;
+  // keep walking as long as the next backward token matches a child key
+  while (i <= max) {
+    const token = tokens[anchorIndex - i];
+    if (token) {
+      const key = (token.type === OPERATOR) ? token.value : token.type;
+      if (key in node) {
+        node = node[key];
+        i += 1;
+        continue;
+      }
     }
+    // can't advance further; accept only if current node is a terminal
+    return node[END] ? i : 0;
   }
-  if (currNode[END]) {
-    // we may end here so this is a match
-    return index;
-  }
-  // no match
-  return 0;
 };
 
 /**
@@ -72,28 +81,34 @@ const matcher = (tokens, currNode, anchorIndex, index = 0) => {
  * as whole references (`Sheet1!A1:B2`) rather than separate tokens for each
  * part: (`Sheet1`,`!`,`A1`,`:`,`B2`).
  *
- * @param {Array<Token>} tokenlist An array of tokens (from `tokenize()`)
- * @returns {Array<Token>} A new list of tokens with range parts merged.
+ * @param tokenlist An array of tokens.
+ * @returns A new list of tokens with range parts merged.
  */
-export function mergeRefTokens (tokenlist) {
+export function mergeRefTokens (tokenlist: Token[]): Token[] {
   const finalTokens = [];
   // this seeks backwards because it's really the range part
   // that controls what can be joined.
   for (let i = tokenlist.length - 1; i >= 0; i--) {
     let token = tokenlist[i];
-    const valid = matcher(tokenlist, refPartsTree, i);
-    if (valid) {
-      const toMerge = tokenlist.slice(i - valid + 1, i + 1);
-      // use the meta properties from the "first" token (right-most token)
-      token = { ...token };
-      token.value = toMerge.map(d => d.value).join('');
-      // adjust the offsets to include all the text
-      if (token.loc && toMerge[0].loc) {
-        token.loc[0] = toMerge[0].loc[0];
+    const type = token.type;
+    // Quick check if token type could even start a valid run
+    if (type === REF_RANGE || type === REF_BEAM || type === REF_TERNARY ||
+        type === REF_NAMED || type === REF_STRUCT) {
+      const valid = matcher(tokenlist, refPartsTree, i);
+      if (valid > 1) {
+        token = { ...token, value: '' };
+        const start = i - valid + 1;
+        for (let j = start; j <= i; j++) {
+          token.value += tokenlist[j].value;
+        }
+        // adjust the offsets to include all the text
+        if (token.loc && tokenlist[start].loc) {
+          token.loc[0] = tokenlist[start].loc[0];
+        }
+        i -= valid - 1;
       }
-      i -= valid - 1;
     }
-    finalTokens.unshift(token);
+    finalTokens[finalTokens.length] = token;
   }
-  return finalTokens;
+  return finalTokens.reverse();
 }

package/lib/nodeTypes.ts

@@ -0,0 +1,54 @@
+import {
+  // AST types
+  UNARY,
+  BINARY,
+  REFERENCE,
+  LITERAL,
+  ERROR_LITERAL,
+  CALL,
+  ARRAY,
+  IDENTIFIER,
+  LAMBDA,
+  LET,
+  LET_DECL
+} from './constants.ts';
+
+/**
+ * A dictionary of the types used to identify AST node variants.
+ *
+ * @prop UNARY - A unary operation (`10%`)
+ * @prop BINARY - A binary operation (`10+10`)
+ * @prop REFERENCE - A range identifier (`A1`)
+ * @prop LITERAL - A literal (number, string, or boolean) (`123`, `"foo"`, `false`)
+ * @prop ERROR - An error literal (`#VALUE!`)
+ * @prop CALL - A function call expression (`SUM(1,2)`)
+ * @prop ARRAY - An array expression (`{1,2;3,4}`)
+ * @prop IDENTIFIER - A function name identifier (`SUM`)
+ * @prop LAMBDA - A LAMBDA expression (`LAMBDA(x,y,x*y)``)
+ * @prop LET - A LET expression (`LET(a,A1*10,b,SUM(F:F),a*b)`)
+ * @prop LET_DECL - A LET declaration (LET(`a,A1*10`...)
+ */
+export const nodeTypes = Object.freeze({
+  /** A unary operation (`10%`) */
+  UNARY,
+  /** A binary operation (`10+10`) */
+  BINARY,
+  /** A range identifier (`A1`) */
+  REFERENCE,
+  /** A literal (number, string, or boolean) (`123`, `"foo"`, `false`) */
+  LITERAL,
+  /** An error literal (`#VALUE!`) */
+  ERROR: ERROR_LITERAL,
+  /** A function call expression (`SUM(1,2)`) */
+  CALL,
+  /** An array expression (`{1,2;3,4}`) */
+  ARRAY,
+  /** A function name identifier (`SUM`) */
+  IDENTIFIER,
+  /** A LAMBDA expression (`LAMBDA(x,y,x*y)``) */
+  LAMBDA,
+  /** A LET expression (`LET(a,A1*10,b,SUM(F:F),a*b)`) */
+  LET,
+  /** A LET declaration (LET(`a,A1*10`...)*/
+  LET_DECL
+});