@odoo/o-spreadsheet 14.0.0 → 14.0.1

package/src/formulas/parser.ts

@@ -0,0 +1,280 @@
+import { functionRegistry } from "../functions/index";
+import { parseNumber } from "../helpers/index";
+import { _lt } from "../translation";
+import { Dependencies, NormalizedFormulaString } from "../types";
+import { Token, tokenize } from "./tokenizer";
+
+const functions = functionRegistry.content;
+
+const UNARY_OPERATORS = ["-", "+"];
+
+// -----------------------------------------------------------------------------
+// PARSER
+// -----------------------------------------------------------------------------
+interface ASTBase {
+  debug?: boolean;
+}
+
+interface ASTNumber extends ASTBase {
+  type: "NUMBER";
+  value: number;
+}
+
+interface ASTNormalizedNumber extends ASTBase {
+  type: "NORMALIZED_NUMBER";
+  value: string;
+}
+
+interface ASTReference extends ASTBase {
+  type: "REFERENCE";
+  value: number;
+}
+interface ASTString extends ASTBase {
+  type: "STRING";
+  value: string;
+}
+
+interface ASTNormalizedString extends ASTBase {
+  type: "NORMALIZED_STRING";
+  value: string;
+}
+
+interface ASTBoolean extends ASTBase {
+  type: "BOOLEAN";
+  value: boolean;
+}
+
+interface ASTUnaryOperation extends ASTBase {
+  type: "UNARY_OPERATION";
+  value: any;
+  right: AST;
+}
+
+interface ASTOperation extends ASTBase {
+  type: "BIN_OPERATION";
+  value: any;
+  left: AST;
+  right: AST;
+}
+
+export interface ASTFuncall extends ASTBase {
+  type: "FUNCALL";
+  value: string;
+  args: AST[];
+}
+
+export interface ASTAsyncFuncall extends ASTBase {
+  type: "ASYNC_FUNCALL";
+  value: string;
+  args: AST[];
+}
+
+interface ASTUnknown extends ASTBase {
+  type: "UNKNOWN";
+  value: string;
+}
+
+export type AST =
+  | ASTOperation
+  | ASTUnaryOperation
+  | ASTFuncall
+  | ASTAsyncFuncall
+  | ASTNumber
+  | ASTBoolean
+  | ASTString
+  | ASTReference
+  | ASTNormalizedNumber
+  | ASTNormalizedString
+  | ASTUnknown;
+
+const OP_PRIORITY = {
+  "^": 30,
+  "*": 20,
+  "/": 20,
+  "+": 15,
+  "-": 15,
+  "&": 13,
+  ">": 10,
+  "<>": 10,
+  ">=": 10,
+  "<": 10,
+  "<=": 10,
+  "=": 10,
+};
+
+const FUNCTION_BP = 6;
+
+function bindingPower(token: Token): number {
+  switch (token.type) {
+    case "NUMBER":
+    case "SYMBOL":
+      return 0;
+    case "COMMA":
+      return 3;
+    case "LEFT_PAREN":
+      return 5;
+    case "RIGHT_PAREN":
+      return 5;
+    case "OPERATOR":
+      return OP_PRIORITY[token.value] || 15;
+  }
+  throw new Error("?");
+}
+
+export const cellReference = new RegExp(/\$?[A-Z]+\$?[0-9]+/, "i");
+export const rangeReference = new RegExp(
+  /^\s*\$?[A-Z]+\$?[0-9]+\s*(\s*:\s*\$?[A-Z]+\$?[0-9]+\s*)?$/,
+  "i"
+);
+
+function parsePrefix(current: Token, tokens: Token[]): AST {
+  switch (current.type) {
+    case "DEBUGGER":
+      const next = parseExpression(tokens, 1000);
+      next.debug = true;
+      return next;
+    case "NUMBER":
+      return { type: "NUMBER", value: parseNumber(current.value) };
+    case "NORMALIZED_NUMBER":
+      return { type: "NORMALIZED_NUMBER", value: current.value };
+    case "STRING":
+      return { type: "STRING", value: current.value };
+    case "NORMALIZED_STRING":
+      return { type: "NORMALIZED_STRING", value: current.value };
+    case "FUNCTION":
+      if (tokens.shift()!.type !== "LEFT_PAREN") {
+        throw new Error(_lt("wrong function call"));
+      } else {
+        const args: AST[] = [];
+        if (tokens[0] && tokens[0].type !== "RIGHT_PAREN") {
+          if (tokens[0].type === "COMMA") {
+            args.push({ type: "UNKNOWN", value: "" });
+          } else {
+            args.push(parseExpression(tokens, FUNCTION_BP));
+          }
+          while (tokens[0]?.type === "COMMA") {
+            tokens.shift();
+            const token = tokens[0] as Token | undefined;
+            if (token?.type === "RIGHT_PAREN") {
+              args.push({ type: "UNKNOWN", value: "" });
+              break;
+            } else if (token?.type === "COMMA") {
+              args.push({ type: "UNKNOWN", value: "" });
+            } else {
+              args.push(parseExpression(tokens, FUNCTION_BP));
+            }
+          }
+        }
+        const closingToken = tokens.shift();
+        if (!closingToken || closingToken.type !== "RIGHT_PAREN") {
+          throw new Error(_lt("wrong function call"));
+        }
+        const isAsync = functions[current.value.toUpperCase()].async;
+        const type = isAsync ? "ASYNC_FUNCALL" : "FUNCALL";
+        return { type, value: current.value, args };
+      }
+    case "REFERENCE":
+      return {
+        type: "REFERENCE",
+        value: parseInt(current.value, 10),
+      };
+    case "SYMBOL":
+      if (["TRUE", "FALSE"].includes(current.value.toUpperCase())) {
+        return { type: "BOOLEAN", value: current.value.toUpperCase() === "TRUE" } as AST;
+      } else {
+        if (current.value) {
+          throw new Error(_lt("Invalid formula"));
+        }
+        return { type: "UNKNOWN", value: current.value };
+      }
+    case "LEFT_PAREN":
+      const result = parseExpression(tokens, 5);
+      if (!tokens.length || tokens[0].type !== "RIGHT_PAREN") {
+        throw new Error(_lt("unmatched left parenthesis"));
+      }
+      tokens.shift();
+      return result;
+    default:
+      if (current.type === "OPERATOR" && UNARY_OPERATORS.includes(current.value)) {
+        return {
+          type: "UNARY_OPERATION",
+          value: current.value,
+          right: parseExpression(tokens, OP_PRIORITY[current.value]),
+        };
+      }
+      throw new Error(_lt("nope")); //todo: provide explicit error
+  }
+}
+
+function parseInfix(left: AST, current: Token, tokens: Token[]): AST {
+  if (current.type === "OPERATOR") {
+    const bp = bindingPower(current);
+    const right = parseExpression(tokens, bp);
+
+    return {
+      type: "BIN_OPERATION",
+      value: current.value,
+      left,
+      right,
+    };
+  }
+  throw new Error(_lt("nope")); //todo: provide explicit error
+}
+
+function parseExpression(tokens: Token[], bp: number): AST {
+  const token = tokens.shift();
+  if (!token) {
+    throw new Error(_lt("invalid expression"));
+  }
+  let expr = parsePrefix(token, tokens);
+  while (tokens[0] && bindingPower(tokens[0]) > bp) {
+    expr = parseInfix(expr, tokens.shift()!, tokens);
+  }
+  return expr;
+}
+
+/**
+ * Parse an expression (as a string) into an AST.
+ */
+export function parse(str: NormalizedFormulaString): AST {
+  const tokens = tokenize(str).filter((x) => x.type !== "SPACE");
+  if (tokens[0].type === "OPERATOR" && tokens[0].value === "=") {
+    tokens.splice(0, 1);
+  }
+  const result = parseExpression(tokens, 0);
+  if (tokens.length) {
+    throw new Error(_lt("invalid expression"));
+  }
+  return result;
+}
+
+/**
+ * Converts an ast formula to the corresponding string
+ */
+export function astToFormula(ast: AST, dependencies: Dependencies): string {
+  switch (ast.type) {
+    case "FUNCALL":
+    case "ASYNC_FUNCALL":
+      const args = ast.args.map((arg) => astToFormula(arg, dependencies));
+      return `${ast.value}(${args.join(",")})`;
+    case "NUMBER":
+      return ast.value.toString();
+    case "NORMALIZED_STRING":
+      return `"${dependencies[ast.value]}"`;
+    case "NORMALIZED_NUMBER":
+    case "REFERENCE":
+      return dependencies[ast.value].toString();
+    case "STRING":
+      return ast.value;
+    case "BOOLEAN":
+      return ast.value ? "TRUE" : "FALSE";
+    case "UNARY_OPERATION":
+      return ast.value + astToFormula(ast.right, dependencies);
+    case "BIN_OPERATION":
+      return (
+        astToFormula(ast.left, dependencies) + ast.value + astToFormula(ast.right, dependencies)
+      );
+    default:
+      return ast.value;
+  }
+}

package/src/formulas/range_tokenizer.ts

@@ -0,0 +1,97 @@
+import { Token, tokenize, TokenType } from "./tokenizer";
+
+/**
+ * finds a sequence of token that represent a range and replace them with a single token
+ * The range can be
+ *  ?spaces symbol ?spaces operator: ?spaces symbol ?spaces
+ */
+export function mergeSymbolsIntoRanges(result: Token[], removeSpace = false): Token[] {
+  let operator: number | void = undefined;
+  let refStart: number | void = undefined;
+  let refEnd: number | void = undefined;
+  let startIncludingSpaces: number | void = undefined;
+
+  const reset = () => {
+    startIncludingSpaces = undefined;
+    refStart = undefined;
+    operator = undefined;
+    refEnd = undefined;
+  };
+
+  for (let i = 0; i < result.length; i++) {
+    const token = result[i];
+
+    if (startIncludingSpaces) {
+      if (refStart) {
+        if (token.type === "SPACE") {
+          continue;
+        } else if (token.type === "OPERATOR" && token.value === ":") {
+          operator = i;
+        } else if (operator && token.type === "SYMBOL") {
+          refEnd = i;
+        } else {
+          if (startIncludingSpaces && refStart && operator && refEnd) {
+            const newToken = {
+              type: <TokenType>"SYMBOL",
+              value: result
+                .slice(startIncludingSpaces, i)
+                .filter((x) => !removeSpace || x.type !== "SPACE")
+                .map((x) => x.value)
+                .join(""),
+            };
+            result.splice(startIncludingSpaces, i - startIncludingSpaces, newToken);
+            i = startIncludingSpaces + 1;
+            reset();
+          } else {
+            if (token.type === "SYMBOL") {
+              startIncludingSpaces = i;
+              refStart = i;
+              operator = undefined;
+            } else {
+              reset();
+            }
+          }
+        }
+      } else {
+        if (token.type === "SYMBOL") {
+          refStart = i;
+          operator = refEnd = undefined;
+        } else {
+          reset();
+        }
+      }
+    } else {
+      if (["SPACE", "SYMBOL"].includes(token.type)) {
+        startIncludingSpaces = i;
+        refStart = token.type === "SYMBOL" ? i : undefined;
+        operator = refEnd = undefined;
+      } else {
+        reset();
+      }
+    }
+  }
+  const i = result.length - 1;
+  if (startIncludingSpaces && refStart && operator && refEnd) {
+    const newToken = {
+      type: <TokenType>"SYMBOL",
+      value: result
+        .slice(startIncludingSpaces, i + 1)
+        .filter((x) => !removeSpace || x.type !== "SPACE")
+        .map((x) => x.value)
+        .join(""),
+    };
+    result.splice(startIncludingSpaces, i - startIncludingSpaces + 1, newToken);
+  }
+  return result;
+}
+
+/**
+ * Take the result of the tokenizer and transform it to be usable in the
+ * manipulations of range
+ *
+ * @param formula
+ */
+export function rangeTokenize(formula: string): Token[] {
+  const tokens = tokenize(formula);
+  return mergeSymbolsIntoRanges(tokens, true);
+}

package/src/formulas/tokenizer.ts

@@ -0,0 +1,242 @@
+import { functionRegistry } from "../functions/index";
+import { formulaNumberRegexp } from "../helpers/index";
+import { _lt } from "../translation";
+
+/**
+ * Tokenizer
+ *
+ * A tokenizer is a piece of code whose job is to transform a string into a list
+ * of "tokens". For example, "(12+" is converted into:
+ *   [{type: "LEFT_PAREN", value: "("},
+ *    {type: "NUMBER", value: "12"},
+ *    {type: "OPERATOR", value: "+"}]
+ *
+ * As the example shows, a tokenizer does not care about the meaning behind those
+ * tokens. It only cares about the structure.
+ *
+ * The tokenizer is usually the first step in a compilation pipeline.  Also, it
+ * is useful for the composer, which needs to be able to work with incomplete
+ * formulas.
+ */
+
+const dependencyIdentifierRegex = /^[S|N]?\d+/;
+const functions = functionRegistry.content;
+const OPERATORS = "+,-,*,/,:,=,<>,>=,>,<=,<,%,^,&".split(",");
+
+export type TokenType =
+  | "OPERATOR"
+  | "NUMBER"
+  | "STRING"
+  | "BOOLEAN"
+  | "FUNCTION"
+  | "SYMBOL"
+  | "SPACE"
+  | "DEBUGGER"
+  | "COMMA"
+  | "LEFT_PAREN"
+  | "RIGHT_PAREN"
+  | "REFERENCE"
+  | "NORMALIZED_NUMBER"
+  | "NORMALIZED_STRING"
+  | "UNKNOWN";
+
+export interface Token {
+  type: TokenType;
+  value: string;
+}
+
+export function tokenize(str: string): Token[] {
+  const chars = str.split("");
+  const result: Token[] = [];
+  let tokenCount = 0;
+
+  while (chars.length) {
+    tokenCount++;
+    if (tokenCount > 100) {
+      throw new Error(
+        _lt(
+          "This formula has over 100 parts. It can't be processed properly, consider splitting it into multiple cells"
+        )
+      );
+    }
+    let token =
+      tokenizeSpace(chars) ||
+      tokenizeMisc(chars) ||
+      tokenizeOperator(chars) ||
+      tokenizeString(chars) ||
+      tokenizeDebugger(chars) ||
+      tokenizeNormalizedReferences(chars) ||
+      tokenizeNumber(chars) ||
+      tokenizeSymbol(chars);
+
+    if (!token) {
+      token = { type: "UNKNOWN", value: chars.shift()! };
+    }
+
+    result.push(token);
+  }
+  return result;
+}
+
+export const FORMULA_REF_IDENTIFIER = "|";
+
+function tokenizeNormalizedReferences(chars: string[]): Token | null {
+  if (chars[0] === FORMULA_REF_IDENTIFIER) {
+    chars.shift(); // consume the | even if it is incorrect
+    const match = chars.join("").match(dependencyIdentifierRegex);
+    if (match) {
+      chars.splice(0, match[0].length);
+    } else {
+      return null;
+    }
+    if (chars[0] === FORMULA_REF_IDENTIFIER) {
+      chars.shift();
+    }
+    const value = match[0];
+    switch (value[0]) {
+      case "S":
+        return { type: "NORMALIZED_STRING", value: value.substring(1) };
+      case "N":
+        return { type: "NORMALIZED_NUMBER", value: value.substring(1) };
+      default:
+        return { type: "REFERENCE", value };
+    }
+  }
+  return null;
+}
+
+function tokenizeDebugger(chars: string[]): Token | null {
+  if (chars[0] === "?") {
+    chars.shift();
+    return { type: "DEBUGGER", value: "?" };
+  }
+  return null;
+}
+
+const misc = {
+  ",": "COMMA",
+  "(": "LEFT_PAREN",
+  ")": "RIGHT_PAREN",
+} as const;
+
+function tokenizeMisc(chars: string[]): Token | null {
+  if (chars[0] in misc) {
+    const value = chars.shift()!;
+    const type = misc[value];
+    return { type, value };
+  }
+  return null;
+}
+
+function startsWith(chars: string[], op: string): boolean {
+  for (let i = 0; i < op.length; i++) {
+    if (op[i] !== chars[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+function tokenizeOperator(chars: string[]): Token | null {
+  for (let op of OPERATORS) {
+    if (startsWith(chars, op)) {
+      chars.splice(0, op.length);
+      return { type: "OPERATOR", value: op };
+    }
+  }
+  return null;
+}
+
+function tokenizeNumber(chars: string[]): Token | null {
+  const match = chars.join("").match(formulaNumberRegexp);
+  if (match) {
+    chars.splice(0, match[0].length);
+    return { type: "NUMBER", value: match[0] };
+  }
+  return null;
+}
+
+function tokenizeString(chars: string[]): Token | null {
+  if (chars[0] === '"') {
+    const startChar = chars.shift();
+    const letters: any[] = [startChar];
+    while (chars[0] && (chars[0] !== startChar || letters[letters.length - 1] === "\\")) {
+      letters.push(chars.shift());
+    }
+    if (chars[0] === '"') {
+      letters.push(chars.shift());
+    }
+    return {
+      type: "STRING",
+      value: letters.join(""),
+    };
+  }
+  return null;
+}
+
+const separatorRegexp = /\w|\.|!|\$/;
+
+/**
+ * A "Symbol" is just basically any word-like element that can appear in a
+ * formula, which is not a string. So:
+ *   A1
+ *   SUM
+ *   CEILING.MATH
+ *   A$1
+ *   Sheet2!A2
+ *   'Sheet 2'!A2
+ *
+ * are examples of symbols
+ */
+function tokenizeSymbol(chars: string[]): Token | null {
+  const result: any[] = [];
+  // there are two main cases to manage: either something which starts with
+  // a ', like 'Sheet 2'A2, or a word-like element.
+  if (chars[0] === "'") {
+    let lastChar = chars.shift();
+    result.push(lastChar);
+    while (chars[0]) {
+      lastChar = chars.shift();
+      result.push(lastChar);
+      if (lastChar === "'") {
+        if (chars[0] && chars[0] === "'") {
+          lastChar = chars.shift();
+          result.push(lastChar);
+        } else {
+          break;
+        }
+      }
+    }
+
+    if (lastChar !== "'") {
+      return {
+        type: "UNKNOWN",
+        value: result.join(""),
+      };
+    }
+  }
+  while (chars[0] && chars[0].match(separatorRegexp)) {
+    result.push(chars.shift());
+  }
+  if (result.length) {
+    const value = result.join("");
+    const isFunction = value.toUpperCase() in functions;
+    const type = isFunction ? "FUNCTION" : "SYMBOL";
+    return { type, value };
+  }
+  return null;
+}
+
+const whiteSpaceRegexp = /\s/;
+
+function tokenizeSpace(chars: string[]): Token | null {
+  let length = 0;
+  while (chars[0] && chars[0].match(whiteSpaceRegexp)) {
+    length++;
+    chars.shift();
+  }
+
+  if (length) {
+    return { type: "SPACE", value: " ".repeat(length) };
+  }
+  return null;
+}