@borgar/fx 4.13.0 → 5.0.0

package/lib/astTypes.ts

@@ -0,0 +1,211 @@
+/**
+ * Represents the source location information of the node.
+ * If the node contains no information about the source location, the field is `null`;
+ * otherwise it is an array consisting of a two numbers: A start offset (the position of
+ * the first character of the parsed source region) and an end offset (the position of
+ * the first character after the parsed source region).
+ */
+export type SourceLocation = number[];
+
+/**
+ * All AST nodes are represented by `Node` objects.
+ * They may have any prototype inheritance but implement the same basic interface.
+ *
+ * The `type` field is a string representing the AST variant type.
+ * Each subtype of Node is documented below with the specific string of its `type` field.
+ * You can use this field to determine which interface a node implements.
+ */
+export type Node = {
+  /** The type of this AST node. */
+  type: string;
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+};
+
+/**
+ * An identifier. These appear on `CallExpression`, `LambdaExpression`, and `LetExpression`
+ * and will always be a static string representing the name of a function call or parameter.
+ */
+export type Identifier = {
+  /** The type of this AST node. */
+  type: 'Identifier';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The identifying name. */
+  name: string;
+} & Node;
+
+/**
+ * An identifier for a range or a name.
+ */
+export type ReferenceIdentifier = {
+  /** The type of this AST node. */
+  type: 'ReferenceIdentifier';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The untouched reference value. */
+  value: string;
+  /** The kind of reference the value holds. */
+  kind: 'name' | 'range' | 'beam' | 'table';
+} & Node;
+
+/**
+ * A literal token. Captures numbers, strings, and booleans.
+ * Literal errors have their own variant type.
+ */
+export type Literal = {
+  /** The type of this AST node. */
+  type: 'Literal';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The untouched literal source. */
+  raw: string;
+  /** The value of the literal. */
+  value: string | number | boolean;
+} & Node;
+
+/**
+ * An Error expression.
+ */
+export type ErrorLiteral = {
+  /** The type of this AST node. */
+  type: 'ErrorLiteral';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The untouched literal source. */
+  raw: string;
+  /** The value of the error. */
+  value: string;
+} & Node;
+
+/**
+ * A unary operator expression.
+ */
+export type UnaryExpression = {
+  /** The type of this AST node. */
+  type: 'UnaryExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The expression's operator. */
+  operator: UnaryOperator;
+  /** The arguments for the operator. */
+  arguments: AstExpression[];
+} & Node;
+
+/**
+ * A unary operator token.
+ */
+export type UnaryOperator = (
+  '+' | '-' | '%' | '#' | '@'
+);
+
+/**
+ * A binary operator expression.
+ */
+export type BinaryExpression = {
+  /** The type of this AST node. */
+  type: 'BinaryExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The expression's operator. */
+  operator: BinaryOperator;
+  /** The arguments for the operator. */
+  arguments: AstExpression[];
+} & Node;
+
+/**
+ * A binary operator token.
+ *
+ * Note that Excels union operator is whitespace so a parser must take care to normalize this to
+ * a single space.
+ */
+export type BinaryOperator = (
+  '=' | '<' | '>' | '<=' | '>=' | '<>' |
+  '-' | '+' | '*' | '/' | '^' |
+  ':' | ' ' | ',' |
+  '&'
+);
+
+/**
+ * A function call expression.
+ */
+export type CallExpression = {
+  /** The type of this AST node. */
+  type: 'CallExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The function being called. */
+  callee: Identifier;
+  /** The arguments for the function. */
+  arguments: AstExpression[];
+} & Node;
+
+/**
+ * An array expression. Excel does not have empty or sparse arrays and restricts array elements to
+ * literals. Google Sheets allows `ReferenceIdentifier`s and `CallExpression`s as elements of
+ * arrays, the fx parser has options for this but they are off by default.
+ */
+export type ArrayExpression = {
+  /** The type of this AST node. */
+  type: 'ArrayExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The elements of the array. */
+  elements: (ReferenceIdentifier | Literal | ErrorLiteral | CallExpression)[][];
+} & Node;
+
+/**
+ * A LAMBDA expression.
+ */
+export type LambdaExpression = {
+  /** The type of this AST node. */
+  type: 'LambdaExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The LAMBDA's parameters. */
+  params: Identifier[];
+  /** The LAMBDA's expression. */
+  body: AstExpression | null;
+} & Node;
+
+/**
+ * A LET expression.
+ */
+export type LetExpression = {
+  /** The type of this AST node. */
+  type: 'LetExpression';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The LET's variable declarations. */
+  declarations: LetDeclarator[];
+  /** The LET's scoped expression. */
+  body: AstExpression | null;
+} & Node;
+
+/**
+ * A LET parameter declaration.
+ */
+export type LetDeclarator = {
+  /** The type of this AST node. */
+  type: 'LetDeclarator';
+  /** The original source position of the node. */
+  loc?: SourceLocation;
+  /** The name of the variable. */
+  id: Identifier;
+  /** The variable's initializing expression. */
+  init: AstExpression | null;
+} & Node;
+
+/**
+ * Represents an evaluate-able expression.
+ */
+export type AstExpression =
+  ReferenceIdentifier |
+  Literal |
+  ErrorLiteral |
+  UnaryExpression |
+  BinaryExpression |
+  CallExpression |
+  ArrayExpression |
+  LambdaExpression |
+  LetExpression;

package/lib/cloneToken.ts

@@ -0,0 +1,29 @@
+import type { Token, TokenEnhanced } from './types.ts';
+
+export function cloneToken<T extends Token | TokenEnhanced> (token: T): T {
+  // Token
+  const newToken: Partial<TokenEnhanced> = {
+    type: token.type,
+    value: token.value
+  };
+  if (token.loc) {
+    newToken.loc = [ token.loc[0], token.loc[1] ];
+  }
+  if (token.unterminated != null) {
+    newToken.unterminated = token.unterminated;
+  }
+  // TokenEnhanced
+  if (typeof token.index === 'number') {
+    newToken.index = token.index;
+    if (typeof token.groupId === 'string') {
+      newToken.groupId = token.groupId;
+    }
+    if (typeof token.depth === 'number') {
+      newToken.depth = token.depth;
+    }
+    if (typeof token.error === 'boolean') {
+      newToken.error = token.error;
+    }
+  }
+  return newToken as T;
+}

package/lib/{constants.js → constants.ts}

@@ -1,5 +1,5 @@
 export const OPERATOR = 'operator';
-export const OPERATOR_TRIM = 'operator-trim';
+export const OPERATOR_TRIM = 'operator-trim'; // internal only
 export const BOOLEAN = 'bool';
 export const ERROR = 'error';
 export const NUMBER = 'number';
@@ -29,5 +29,8 @@ export const ARRAY = 'ArrayExpression';
 export const IDENTIFIER = 'Identifier';
 export const LET_DECL = 'LetDeclarator';
 
-export const MAX_COLS = (2 ** 14) - 1; // 16383
-export const MAX_ROWS = (2 ** 20) - 1; // 1048575
+/** The maximum number of columns a spreadsheet reference may hold (16383). */
+export const MAX_COLS = (2 ** 14) - 1;
+
+/** The maximum number of rows a spreadsheet reference may hold (1048575). */
+export const MAX_ROWS = (2 ** 20) - 1;

package/lib/fixRanges.spec.ts

@@ -0,0 +1,220 @@
+import { describe, test, expect } from 'vitest';
+import { tokenize } from './tokenize.ts';
+import { addTokenMeta } from './addTokenMeta.ts';
+import { fixFormulaRanges, fixFormulaRangesXlsx, fixTokenRanges, fixTokenRangesXlsx } from './fixRanges.ts';
+import { FUNCTION, FX_PREFIX, OPERATOR, REF_RANGE, REF_STRUCT, REF_TERNARY } from './constants.ts';
+
+function isFixed (expr: string, expected: string, options: any = {}) {
+  const result = options.xlsx
+    ? fixFormulaRangesXlsx(expr, options)
+    : fixFormulaRanges(expr, options);
+  expect(result).toBe(expected);
+}
+
+describe('fixRanges basics', () => {
+  test('throws on non-string inputs', () => {
+    expect(() => fixTokenRanges(123 as any)).toThrow();
+    expect(() => fixTokenRanges(null as any)).toThrow();
+    expect(() => fixFormulaRanges(123 as any)).toThrow();
+    expect(() => fixFormulaRanges(null as any)).toThrow();
+    expect(() => fixTokenRangesXlsx(123 as any)).toThrow();
+    expect(() => fixTokenRangesXlsx(null as any)).toThrow();
+    expect(() => fixFormulaRangesXlsx(123 as any)).toThrow();
+    expect(() => fixFormulaRangesXlsx(null as any)).toThrow();
+  });
+
+  test('emits new array instance and preserves meta', () => {
+    const fx = '=SUM([wb]Sheet1!B2:A1)';
+    const tokens = addTokenMeta(tokenize(fx, { mergeRefs: true }));
+    const fixedTokens = fixTokenRanges(tokens);
+
+    expect(tokens).not.toBe(fixedTokens);
+    expect(tokens[3]).not.toBe(fixedTokens[3]);
+
+    expect(tokens[3]).toEqual({
+      type: REF_RANGE,
+      value: '[wb]Sheet1!B2:A1',
+      index: 3,
+      depth: 1,
+      groupId: 'fxg1'
+    });
+
+    expect(fixedTokens[3]).toEqual({
+      type: REF_RANGE,
+      value: '[wb]Sheet1!A1:B2',
+      index: 3,
+      depth: 1,
+      groupId: 'fxg1'
+    });
+  });
+
+  test('updates token source location information', () => {
+    const tokensWithRanges = tokenize(
+      '=SUM(B2:A,table[[#This Row],[Foo]])',
+      { withLocation: true, mergeRefs: true, allowTernary: true }
+    );
+
+    expect(fixTokenRanges(tokensWithRanges, { addBounds: true })).toEqual([
+      { type: FX_PREFIX, value: '=', loc: [ 0, 1 ] },
+      { type: FUNCTION, value: 'SUM', loc: [ 1, 4 ] },
+      { type: OPERATOR, value: '(', loc: [ 4, 5 ] },
+      { type: REF_TERNARY, value: 'A2:B1048576', loc: [ 5, 16 ] },
+      { type: OPERATOR, value: ',', loc: [ 16, 17 ] },
+      { type: REF_STRUCT, value: 'table[@Foo]', loc: [ 17, 28 ] },
+      { type: OPERATOR, value: ')', loc: [ 28, 29 ] }
+    ]);
+  });
+});
+
+describe('fixRanges A1', () => {
+  const opt = { allowTernary: true };
+
+  test('doesn\'t mess with things that it doesn\'t have to', () => {
+    isFixed('=A1', '=A1', opt);
+    isFixed('=ZZ123', '=ZZ123', opt);
+    isFixed('=A1:B2', '=A1:B2', opt);
+    isFixed('=B3:OFFSET(A1,10,10)', '=B3:OFFSET(A1,10,10)', opt);
+    isFixed('=A:B', '=A:B', opt);
+    isFixed('=C:C', '=C:C', opt);
+    isFixed('=3:6', '=3:6', opt);
+    isFixed('=3:3', '=3:3', opt);
+  });
+
+  test('handles redundancy', () => {
+    isFixed('=A1:$A$1', '=A1:$A$1', opt);
+    isFixed('=A1:A1', '=A1', opt);
+  });
+
+  test('converts lowercase to uppercase', () => {
+    isFixed('=a1', '=A1', opt);
+    isFixed('=zz123', '=ZZ123', opt);
+    isFixed('=a1:b2', '=A1:B2', opt);
+  });
+
+  test('fixes flipped rectangles', () => {
+    isFixed('=B2:A1', '=A1:B2', opt);
+    isFixed('=$B$2:$A$1', '=$A$1:$B$2', opt);
+  });
+
+  test('fixes flipped beams', () => {
+    isFixed('=C:A', '=A:C', opt);
+    isFixed('=$D:B', '=B:$D', opt);
+    isFixed('=10:1', '=1:10', opt);
+    isFixed('=$5:3', '=3:$5', opt);
+  });
+
+  test('fixes flipped partials - bottom', () => {
+    isFixed('=A:A1', '=A1:A', opt);
+    isFixed('=A:A$1', '=A$1:A', opt);
+  });
+
+  test('fixes flipped partials - right', () => {
+    isFixed('=1:A1', '=A1:1', opt);
+    // $1:$A1 is rather counter intuitive case:
+    //   This range is parsed as { left=null, top=$1, right=$A, bottom=1 } but,
+    //   because left is null, right and left are flipped around, making this
+    //   end up as { left=$A, top=$1, right=null, bottom=1 } which serializes
+    //   as $A$1:1
+    isFixed('=$1:$A1', '=$A$1:1', opt);
+  });
+});
+
+describe('fixRanges A1 addBounds', () => {
+  const opt = { allowTernary: true, addBounds: true };
+
+  test('handles functions and existing bounds', () => {
+    isFixed('=B3:OFFSET(A1,10,10)', '=B3:OFFSET(A1,10,10)', opt);
+    isFixed('=A:A', '=A:A', opt);
+    isFixed('=A:A1', '=A:A', opt);
+    isFixed('=A:A$1', '=A:A', opt);
+    isFixed('=A:$A$1', '=A:$A', opt);
+    isFixed('=A.:A', '=A.:A', opt);
+  });
+
+  test('adds bounds to partials - bottom', () => {
+    isFixed('=A1:A', '=A:A', opt);
+    isFixed('=A1:Z', '=A:Z', opt);
+    isFixed('=A:A1', '=A:A', opt);
+    isFixed('=$A1:A', '=$A:A', opt);
+    isFixed('=A$1:A', '=A:A', opt);
+    isFixed('=A1:$A', '=A:$A', opt);
+    isFixed('=A2:A', '=A2:A1048576', opt);
+    isFixed('=B2:B', '=B2:B1048576', opt);
+    isFixed('=A:A2', '=A2:A1048576', opt);
+    isFixed('=B:B2', '=B2:B1048576', opt);
+    isFixed('=B.:.B2', '=B2.:.B1048576', opt);
+  });
+
+  test('adds bounds to flipped partials - bottom', () => {
+    isFixed('=A1:1', '=1:1', opt);
+    isFixed('=A1:4', '=1:4', opt);
+    isFixed('=1:A1', '=1:1', opt);
+    isFixed('=$A1:1', '=1:1', opt);
+    isFixed('=A$1:1', '=$1:1', opt);
+    isFixed('=A1:$1', '=1:$1', opt);
+    isFixed('=B1:1', '=B1:XFD1', opt);
+    isFixed('=1:B1', '=B1:XFD1', opt);
+    isFixed('=B2:20', '=B2:XFD20', opt);
+    isFixed('=2:B20', '=B2:XFD20', opt);
+    isFixed('=2:.B20', '=B2:.XFD20', opt);
+  });
+});
+
+describe('fixRanges structured references', () => {
+  test('fixes this row references', () => {
+    isFixed('=Table1[[#This Row],[Foo]]', '=Table1[@Foo]');
+    isFixed('=[[#This Row],[s:s]]', '=[@[s:s]]');
+  });
+
+  test('fixes column ranges', () => {
+    isFixed('=Table1[[#Totals],col name:Foo]', '=Table1[[#Totals],[col name]:[Foo]]');
+  });
+
+  test('fixes special identifiers order', () => {
+    isFixed('[[#data],[#headers]]', '[[#Headers],[#Data]]');
+    isFixed('[[#headers],[#data]]', '[[#Headers],[#Data]]');
+    isFixed('[[#totals],[#data]]', '[[#Data],[#Totals]]');
+    isFixed('[ [#totals], [#data] ]', '[[#Data],[#Totals]]');
+    isFixed('[[#data],[#totals]]', '[[#Data],[#Totals]]');
+  });
+
+  test('fixes column references', () => {
+    isFixed('[[#all],foo:bar]', '[[#All],[foo]:[bar]]');
+    isFixed('[[#all],[#all],[#all],[#all],[ColumnName]]', '[[#All],[ColumnName]]');
+    isFixed('[@[foo]:bar]', '[@[foo]:[bar]]');
+    isFixed('[@foo bar]', '[@[foo bar]]');
+  });
+
+  test('preserves whitespace in column headers', () => {
+    // Care must be taken with spaces in column headers.
+    // Excel considers refs only valid if they match the column name
+    // but the parser does not know the names, so it must preserve
+    // leading/trailing whitespace.
+    isFixed('[ @[foo bar] ]', '[@[foo bar]]');
+    isFixed('[ @[ foo bar ] ]', '[@[ foo bar ]]');
+    isFixed('[ @foo bar ]', '[@[foo bar ]]');
+    isFixed('[@ foo bar]', '[@[ foo bar]]');
+    isFixed('[ @ foo bar ]', '[@[ foo bar ]]');
+  });
+});
+
+describe('fixRanges works with xlsx mode', () => {
+  test('should not mess with invalid ranges in normal mode', () => {
+    isFixed("='[Workbook]'!Table[Column]", "='[Workbook]'!Table[Column]");
+    isFixed('=[Workbook]!Table[Column]', '=[Workbook]!Table[Column]');
+    isFixed("='[Foo]'!A1", "='[Foo]'!A1");
+    isFixed('=[Foo]!A1', '=[Foo]!A1');
+  });
+
+  test('should fix things in xlsx mode', () => {
+    const opts = { xlsx: true };
+    isFixed("='[Workbook]'!Table[Column]", '=[Workbook]!Table[Column]', opts);
+    isFixed('=[Workbook]!Table[Column]', '=[Workbook]!Table[Column]', opts);
+    isFixed('=[Lorem Ipsum]!Table[Column]', "='[Lorem Ipsum]'!Table[Column]", opts);
+    isFixed("='[Foo]'!A1", '=[Foo]!A1', opts);
+    isFixed('=[Foo]Bar!A1', '=[Foo]Bar!A1', opts);
+    isFixed('=[Foo Bar]Baz!A1', "='[Foo Bar]Baz'!A1", opts);
+    isFixed('=[Foo]!A1', '=[Foo]!A1', opts);
+    isFixed('=[Lorem Ipsum]!A1', "='[Lorem Ipsum]'!A1", opts);
+  });
+});