@jbrowse/jexl 4.0.1 → 5.0.1

package/esm/types.d.ts

@@ -4,20 +4,21 @@ export interface TemplatePart {
 }
 export type JexlValue = string | number | boolean | null | undefined | JexlValue[] | {
     [key: string]: JexlValue;
-};
+} | JexlFunction;
+/** What a lambda evaluates to, for a registered function to call. */
+export type JexlFunction = (...args: JexlValue[]) => JexlValue;
 export interface Token {
     type: string;
-    value: string | number | boolean | TemplatePart[];
+    value: string | number | boolean | null | TemplatePart[];
     raw: string;
 }
 export interface AstNode {
     type: string;
-    _parent?: AstNode;
     right?: AstNode;
 }
 export interface Literal extends AstNode {
     type: 'Literal';
-    value: string | number | boolean;
+    value: string | number | boolean | null;
 }
 export interface Identifier extends AstNode {
     type: 'Identifier';
@@ -79,7 +80,12 @@ export interface AssignmentExpression extends AstNode {
     operator: '=';
     left: Identifier;
 }
-export type AstNodeUnion = Literal | Identifier | BinaryExpression | UnaryExpression | ArrayLiteral | ObjectLiteral | TemplateLiteral | FunctionCall | FilterExpression | ConditionalExpression | SequenceExpression | AssignmentExpression;
+export interface Lambda extends AstNode {
+    type: 'Lambda';
+    params: string[];
+    body: AstNode;
+}
+export type AstNodeUnion = Literal | Identifier | BinaryExpression | UnaryExpression | ArrayLiteral | ObjectLiteral | TemplateLiteral | FunctionCall | FilterExpression | ConditionalExpression | SequenceExpression | AssignmentExpression | Lambda;
 export type NodeByType<T extends AstNodeUnion['type']> = Extract<AstNodeUnion, {
     type: T;
 }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/jexl",
-  "version": "4.0.1",
+  "version": "5.0.1",
   "description": "A fork of the jexl lang for jbrowse",
   "type": "module",
   "main": "dist/index.js",

package/src/Expression.ts

@@ -4,9 +4,13 @@
  */
 
 import Lexer from './Lexer.ts'
+import { analyze } from './analyze.ts'
+import { check } from './check.ts'
 import { compileAst } from './evaluator/compile.ts'
 import Parser from './parser/Parser.ts'
 
+import type { AnalyzeOptions } from './analyze.ts'
+import type { CheckOptions } from './check.ts'
 import type { CompiledNode } from './evaluator/compile.ts'
 import type { Grammar } from './grammar.ts'
 import type { AstNode } from './types.ts'
@@ -45,10 +49,7 @@ class Expression {
    * @returns {Expression} this Expression instance, for convenience
    */
   compile() {
-    const parser = new Parser(this._grammar, this._lexer)
-    const tokens = this._lexer.tokenize(this._exprStr)
-    parser.addTokens(tokens)
-    this._ast = parser.complete()
+    this._ast = new Parser(this._grammar, this._lexer).parse(this._exprStr)
     // lower the tree to closures once, here, so that eval() is just a call
     this._fn = this._ast ? compileAst(this._ast, this._grammar) : null
     this._compiled = true
@@ -72,6 +73,35 @@ class Expression {
     }
     return this._fn(context)
   }
+
+  /** The expression text this Expression was made from. */
+  get source() {
+    return this._exprStr
+  }
+
+  /** The parsed tree, or null for an expression with no tokens. */
+  get ast() {
+    if (!this._compiled) {
+      this.compile()
+    }
+    return this._ast
+  }
+
+  /**
+   * Lists what the expression reads from its context, without evaluating it.
+   * See {@link analyze}.
+   */
+  analyze(options?: AnalyzeOptions) {
+    return analyze(this.ast, options)
+  }
+
+  /**
+   * Checks the expression against a host's fields and functions, without
+   * evaluating it. See {@link check}.
+   */
+  check(options?: CheckOptions) {
+    return check(this.ast, options)
+  }
 }
 
 export default Expression

package/src/Jexl.ts

@@ -10,14 +10,29 @@ import { getGrammar } from './grammar.ts'
 import type {
   BinaryOpEval,
   BinaryOpEvalOnDemand,
+  GetMember,
   Grammar,
   GrammarElement,
   GrammarFn,
   UnaryOpEval,
-  UncheckedFn
+  UncheckedFn,
+  VariableReader
 } from './grammar.ts'
 import type { JexlValue } from './types.ts'
 
+/**
+ * Hooks for a host whose values keep their fields behind an accessor. Both are
+ * fixed at construction, since a compiled expression binds them.
+ */
+export interface JexlOptions {
+  /** Resolves `a.b` and `a[k]`. */
+  getMember?: GetMember
+  /** Resolves a bare name, such as a field of the current row. */
+  variableReader?: VariableReader
+}
+
+const MAX_COMPILED = 10_000
+
 /**
  * Jexl is the Javascript Expression Language, capable of parsing and
  * evaluating basic to complex expression strings, combined with advanced
@@ -29,9 +44,10 @@ class Jexl {
   // shared by every Expression this instance creates, so that the expensive
   // element-splitting regex is built once rather than per compile
   _lexer: Lexer
+  _compiled = new Map<string, Expression>()
 
-  constructor() {
-    this._grammar = getGrammar()
+  constructor({ getMember, variableReader }: JexlOptions = {}) {
+    this._grammar = { ...getGrammar(), getMember, variableReader }
     this._lexer = new Lexer(this._grammar)
     this.expr = this.expr.bind(this)
   }
@@ -73,6 +89,10 @@ class Jexl {
     fn: UncheckedFn | BinaryOpEvalOnDemand,
     manualEval?: boolean
   ) {
+    // replacing `-` keeps its prefix form, which is a separate operation
+    const previous = this._grammar.elements[operator]
+    const unaryEval =
+      previous?.type === 'binaryOp' ? previous.unaryEval : undefined
     // the overloads above pair `fn` with `manualEval`; the implementation
     // signature can't express that correlation, hence the assertions
     this._addGrammarElement(
@@ -81,12 +101,14 @@ class Jexl {
         ? {
             type: 'binaryOp',
             precedence,
-            evalOnDemand: fn as BinaryOpEvalOnDemand
+            evalOnDemand: fn as BinaryOpEvalOnDemand,
+            unaryEval
           }
         : {
             type: 'binaryOp',
             precedence,
-            eval: fn as unknown as BinaryOpEval
+            eval: fn as unknown as BinaryOpEval,
+            unaryEval
           }
     )
   }
@@ -132,16 +154,21 @@ class Jexl {
   }
 
   /**
-   * Creates an Expression object from the given Jexl expression string, and
-   * immediately compiles it. The returned Expression object can then be
-   * evaluated multiple times with new contexts, without generating any
-   * additional string processing overhead.
-   * @param {string} expression The Jexl expression to be compiled
-   * @returns {Expression} The compiled Expression object
+   * Compiles an expression string, or returns the Expression this instance
+   * already compiled from the same string. Adding or removing an operator
+   * empties the cache, since a compiled expression binds its operators;
+   * functions are looked up per call and leave it alone.
    */
   compile(expression: string) {
-    const exprObj = this.createExpression(expression)
-    return exprObj.compile()
+    let compiled = this._compiled.get(expression)
+    if (!compiled) {
+      compiled = this.createExpression(expression).compile()
+      if (this._compiled.size >= MAX_COMPILED) {
+        this._compiled.delete(this._compiled.keys().next().value!)
+      }
+      this._compiled.set(expression, compiled)
+    }
+    return compiled
   }
 
   /**
@@ -154,6 +181,11 @@ class Jexl {
     return new Expression(this._grammar, expression, this._lexer)
   }
 
+  /** Parses an expression string, or throws a JexlSyntaxError. */
+  parse(expression: string) {
+    return this.compile(expression).ast
+  }
+
   /**
    * Retrieves a previously set expression function.
    * @param {string} name The name of the expression function
@@ -172,8 +204,7 @@ class Jexl {
    * @throws {*} on error
    */
   eval(expression: string, context = {}) {
-    const exprObj = this.createExpression(expression)
-    return exprObj.eval(context)
+    return this.compile(expression).eval(context)
   }
 
   /**
@@ -210,7 +241,7 @@ class Jexl {
     const elem = this._grammar.elements[operator]
     if (elem?.type === 'binaryOp' || elem?.type === 'unaryOp') {
       Reflect.deleteProperty(this._grammar.elements, operator)
-      this._lexer._clearCache()
+      this._grammarChanged()
     }
   }
 
@@ -223,7 +254,12 @@ class Jexl {
    */
   _addGrammarElement(str: string, obj: GrammarElement) {
     this._grammar.elements[str] = obj
+    this._grammarChanged()
+  }
+
+  _grammarChanged() {
     this._lexer._clearCache()
+    this._compiled.clear()
   }
 }
 

package/src/Lexer.ts

@@ -3,6 +3,8 @@
  * Copyright 2020 Tom Shawver
  */
 
+import { JexlSyntaxError } from './errors.ts'
+
 import type { Grammar } from './grammar.ts'
 import type { TemplatePart, Token } from './types.ts'
 
@@ -12,9 +14,13 @@ import type { TemplatePart, Token } from './types.ts'
 // disagree on some input, and each disagreement is an "Invalid expression
 // token" for something the splitter was happy to produce.
 const identChars = String.raw`a-zA-Zа-яА-Я_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF$`
-const identPattern = `[${identChars}][${identChars}0-9]*`
+const identPart = `[${identChars}0-9]`
+const identPattern = `[${identChars}]${identPart}*`
+// a word operator such as `in`, but not part of a longer name. \b would do only
+// for ASCII names, and split `in$` or `inà` into the operator and a remainder
+const wholeWord = (word: string) => `(?<!${identPart})${word}(?!${identPart})`
 // unsigned: whether a leading '-' negates is decided separately, in getTokens
-const numberPattern = String.raw`(?:(?:[0-9]*\.[0-9]+)|[0-9]+)`
+const numberPattern = String.raw`(?:(?:[0-9]*\.[0-9]+)|[0-9]+)(?:[eE][+-]?[0-9]+)?`
 
 const numericRegex = new RegExp(`^-?${numberPattern}$`)
 const identRegex = new RegExp(`^${identPattern}$`)
@@ -35,11 +41,10 @@ const preOpRegexElems = [
   String.raw`"(?:(?:\\")|[^"])*"`,
   // Whitespace
   String.raw`\s+`,
-  // Booleans
-  String.raw`\btrue\b`,
-  String.raw`\bfalse\b`
+  // ahead of the grammar's '.', so that '.5' is a number rather than a dot
+  numberPattern
 ]
-const postOpRegexElems = [identPattern, numberPattern]
+const postOpRegexElems = [identPattern]
 const unaryMinusToken = (): Token => ({
   type: 'unaryOp',
   value: '-',
@@ -53,7 +58,8 @@ const minusNegatesAfter = new Set([
   'question',
   'colon',
   'comma',
-  'semicolon'
+  'semicolon',
+  'arrow'
 ])
 
 /**
@@ -113,6 +119,7 @@ class Lexer {
     // 1") accumulates on the minus rather than on the token before it, which
     // is what lets the parser's error messages quote the expression verbatim.
     let pendingMinus: Token | undefined
+    let offset = 0
     for (const element of elements) {
       if (this._isWhitespace(element)) {
         const last = pendingMinus ?? tokens.at(-1)
@@ -129,18 +136,19 @@ class Lexer {
       } else if (pendingMinus) {
         if (numericRegex.exec(element)) {
           // fold the sign into the number, so "-1" stays a single literal
-          const token = this._createToken('-' + element)
+          const token = this._createToken('-' + element, offset)
           token.raw = pendingMinus.raw + element
           tokens.push(token)
         } else {
           // anything else gets a standalone prefix operator, letting "-x",
           // "-(a + b)" and "-foo.bar" negate a computed value
-          tokens.push(pendingMinus, this._createToken(element))
+          tokens.push(pendingMinus, this._createToken(element, offset))
         }
         pendingMinus = undefined
       } else {
-        tokens.push(this._createToken(element))
+        tokens.push(this._createToken(element, offset))
       }
+      offset += element.length
     }
     // Catch a - at the end of the string. Let the parser handle that issue.
     if (pendingMinus) {
@@ -191,7 +199,7 @@ class Lexer {
    * @throws {Error} if the provided string is not a valid expression element.
    * @private
    */
-  _createToken(element: string): Token {
+  _createToken(element: string, offset = 0): Token {
     const token: Token = {
       type: 'literal',
       value: element,
@@ -199,7 +207,7 @@ class Lexer {
     }
     if (element.startsWith('`')) {
       token.type = 'templateString'
-      token.value = this._parseTemplateString(element)
+      token.value = this._parseTemplateString(element, offset)
       return token
     } else if (element.startsWith('"') || element.startsWith("'")) {
       token.value = this._unquote(element)
@@ -207,30 +215,30 @@ class Lexer {
       token.value = parseFloat(element)
     } else if (element === 'true' || element === 'false') {
       token.value = element === 'true'
+    } else if (element === 'null') {
+      token.value = null
     } else if (Object.hasOwn(this._grammar.elements, element)) {
       token.type = this._grammar.elements[element]!.type
     } else if (identRegex.exec(element)) {
       token.type = 'identifier'
     } else {
-      throw new Error(`Invalid expression token: ${element}`)
+      throw new JexlSyntaxError(`Invalid expression token: ${element}`, offset)
     }
     return token
   }
 
   /**
    * Escapes a string so that it can be treated as a string literal within a
-   * regular expression.
+   * regular expression. A word such as `in` also stops matching inside a
+   * longer name.
    * @param {string} str The string to be escaped
    * @returns {string} the RegExp-escaped string.
    * @see https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
    * @private
    */
   _escapeRegExp(str: string) {
-    str = str.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)
-    if (identRegex.exec(str)) {
-      str = String.raw`\b` + str + String.raw`\b`
-    }
-    return str
+    const escaped = str.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)
+    return identRegex.exec(str) ? wholeWord(escaped) : escaped
   }
 
   /**
@@ -307,7 +315,7 @@ class Lexer {
       .replaceAll(escEscRegex, '\\')
   }
 
-  _parseTemplateString(str: string) {
+  _parseTemplateString(str: string, offset = 0) {
     const parts: TemplatePart[] = []
     let current = 1
     let staticStart = 1
@@ -355,7 +363,10 @@ class Lexer {
         }
 
         if (braceDepth !== 0) {
-          throw new Error(`Unclosed interpolation in template string: ${str}`)
+          throw new JexlSyntaxError(
+            `Unclosed interpolation in template string: ${str}`,
+            offset + interpStart - '${'.length
+          )
         }
 
         parts.push({