@jbrowse/jexl 3.3.0 → 4.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/jexl",
-  "version": "3.3.0",
+  "version": "4.0.0",
   "description": "A fork of the jexl lang for jbrowse",
   "type": "module",
   "main": "dist/index.js",
@@ -15,6 +15,7 @@
   ],
   "scripts": {
     "test": "vitest",
+    "bench": "vitest bench --run",
     "coverage": "pnpm test --coverage",
     "clean": "rimraf dist esm",
     "format": "prettier --write .",

package/src/Expression.ts

@@ -4,7 +4,6 @@
  */
 
 import Lexer from './Lexer.ts'
-import Evaluator from './evaluator/Evaluator.ts'
 import { compileAst } from './evaluator/compile.ts'
 import Parser from './parser/Parser.ts'
 
@@ -71,7 +70,7 @@ class Expression {
     if (!this._fn) {
       return undefined
     }
-    return this._fn(new Evaluator(this._grammar, context))
+    return this._fn(context)
   }
 }
 

package/src/Jexl.ts

@@ -40,9 +40,9 @@ class Jexl {
    * Adds a binary operator to Jexl at the specified precedence. The higher the
    * precedence, the earlier the operator is applied in the order of operations.
    * For example, * has a higher precedence than +, because multiplication comes
-   * before division.
+   * before addition.
    *
-   * Please see grammar.js for a listing of all default operators and their
+   * Please see grammar.ts for a listing of all default operators and their
    * precedence values in order to choose the appropriate precedence for the
    * new operator.
    * @param {string} operator The operator string to be added

package/src/Lexer.ts

@@ -6,13 +6,26 @@
 import type { Grammar } from './grammar.ts'
 import type { TemplatePart, Token } from './types.ts'
 
-const numericRegex = /^-?(?:(?:[0-9]*\.[0-9]+)|[0-9]+)$/
-const identRegex =
-  /^[a-zA-Zа-яА-Я_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF$][a-zA-Zа-яА-Я0-9_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF$]*$/
+// what an identifier and a number look like, written once: the split regex
+// below carves elements out of the expression with these, and _createToken
+// then re-tests the elements it produced. Two spellings of either would
+// disagree on some input, and each disagreement is an "Invalid expression
+// token" for something the splitter was happy to produce.
+const identChars = String.raw`a-zA-Zа-яА-Я_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF$`
+const identPattern = `[${identChars}][${identChars}0-9]*`
+// unsigned: whether a leading '-' negates is decided separately, in getTokens
+const numberPattern = String.raw`(?:(?:[0-9]*\.[0-9]+)|[0-9]+)`
+
+const numericRegex = new RegExp(`^-?${numberPattern}$`)
+const identRegex = new RegExp(`^${identPattern}$`)
 const escEscRegex = /\\\\/g
 // a string literal opens with one of exactly two quote characters, so the two
 // unescaping regexes can just be named rather than built and cached per quote
 const escQuoteRegex = { "'": /\\'/g, '"': /\\"/g }
+// the escapes a template string's static text recognizes. One pass, so the
+// backslash an escaped backslash yields can't be re-read as the start of the
+// escape that follows it
+const templateEscRegex = /\\([`$\\])/g
 const whitespaceRegex = /^\s*$/
 const preOpRegexElems = [
   // Template strings
@@ -26,12 +39,7 @@ const preOpRegexElems = [
   String.raw`\btrue\b`,
   String.raw`\bfalse\b`
 ]
-const postOpRegexElems = [
-  // Identifiers
-  '[a-zA-Zа-яА-Я_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\\$][a-zA-Z0-9а-яА-Я_\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\\$]*',
-  // Numerics (without negative symbol)
-  String.raw`(?:(?:[0-9]*\.[0-9]+)|[0-9]+)`
-]
+const postOpRegexElems = [identPattern, numberPattern]
 const unaryMinusToken = (): Token => ({
   type: 'unaryOp',
   value: '-',
@@ -49,14 +57,16 @@ const minusNegatesAfter = new Set([
 ])
 
 /**
- * Lexer is a collection of stateless, statically-accessed functions for the
- * lexical parsing of a Jexl string.  Its responsibility is to identify the
- * "parts of speech" of a Jexl expression, and tokenize and label each, but
- * to do only the most minimal syntax checking; the only errors the Lexer
- * should be concerned with are if it's unable to identify the utility of
- * any of its tokens.  Errors stemming from these tokens not being in a
- * sensible configuration should be left for the Parser to handle.
- * @type {{}}
+ * Lexer handles the lexical parsing of a Jexl string. Its responsibility is to
+ * identify the "parts of speech" of a Jexl expression, and tokenize and label
+ * each, but to do only the most minimal syntax checking; the only errors the
+ * Lexer should be concerned with are if it's unable to identify the utility of
+ * any of its tokens. Errors stemming from these tokens not being in a sensible
+ * configuration should be left for the Parser to handle.
+ *
+ * An instance is bound to one grammar and memoizes the regex that splits an
+ * expression into elements, which is expensive to build; {@link #_clearCache}
+ * discards it when the grammar's elements change.
  */
 class Lexer {
   _grammar: Grammar
@@ -98,37 +108,43 @@ class Lexer {
    */
   getTokens(elements: string[]) {
     const tokens: Token[] = []
-    let negate = false
+    // a prefix minus, held back until the element it applies to is known. It
+    // carries its own raw so that whitespace arriving before that element ("-
+    // 1") accumulates on the minus rather than on the token before it, which
+    // is what lets the parser's error messages quote the expression verbatim.
+    let pendingMinus: Token | undefined
     for (const element of elements) {
       if (this._isWhitespace(element)) {
-        const last = tokens.at(-1)
+        const last = pendingMinus ?? tokens.at(-1)
         if (last) {
           last.raw += element
         }
       } else if (element === '-' && this._isNegative(tokens)) {
         // a second prefix minus in a row ("- -x"): emit the pending one as a
         // unary operator so this one can negate whatever comes next
-        if (negate) {
-          tokens.push(unaryMinusToken())
+        if (pendingMinus) {
+          tokens.push(pendingMinus)
         }
-        negate = true
-      } else if (negate) {
-        negate = false
+        pendingMinus = unaryMinusToken()
+      } else if (pendingMinus) {
         if (numericRegex.exec(element)) {
           // fold the sign into the number, so "-1" stays a single literal
-          tokens.push(this._createToken('-' + element))
+          const token = this._createToken('-' + element)
+          token.raw = pendingMinus.raw + element
+          tokens.push(token)
         } else {
           // anything else gets a standalone prefix operator, letting "-x",
           // "-(a + b)" and "-foo.bar" negate a computed value
-          tokens.push(unaryMinusToken(), this._createToken(element))
+          tokens.push(pendingMinus, this._createToken(element))
         }
+        pendingMinus = undefined
       } else {
         tokens.push(this._createToken(element))
       }
     }
     // Catch a - at the end of the string. Let the parser handle that issue.
-    if (negate) {
-      tokens.push(this._createToken('-'))
+    if (pendingMinus) {
+      tokens.push(pendingMinus)
     }
     return tokens
   }
@@ -256,10 +272,8 @@ class Lexer {
    * @private
    */
   _isNegative(tokens: Token[]) {
-    if (!tokens.length) {
-      return true
-    }
-    return minusNegatesAfter.has(tokens[tokens.length - 1]!.type)
+    const last = tokens.at(-1)
+    return !last || minusNegatesAfter.has(last.type)
   }
 
   /**
@@ -366,10 +380,7 @@ class Lexer {
   }
 
   _unescapeTemplateString(str: string) {
-    return str
-      .replaceAll('\\`', '`')
-      .replaceAll(String.raw`\$`, '$')
-      .replaceAll('\\\\', '\\')
+    return str.replaceAll(templateEscRegex, '$1')
   }
 }
 

package/src/evaluator/compile.ts

@@ -5,13 +5,16 @@
 
 import type { Grammar } from '../grammar.ts'
 import type { AstNode, AstNodeUnion, JexlValue } from '../types.ts'
-import type Evaluator from './Evaluator.ts'
+
+/** The variables an expression is evaluated against. */
+export type Context = Record<string, JexlValue>
 
 /**
- * An AST node lowered to a closure. Calling it with an Evaluator (which carries
- * the context and grammar) produces the node's value.
+ * An AST node lowered to a closure. Calling it with a context produces the
+ * node's value. The grammar is captured when the closure is built, so it is not
+ * a parameter.
  */
-export type CompiledNode = (ev: Evaluator) => JexlValue
+export type CompiledNode = (context: Context) => JexlValue
 
 /**
  * Writes a key that a plain assignment would mishandle. Storing to
@@ -20,11 +23,7 @@ export type CompiledNode = (ev: Evaluator) => JexlValue
  * re-pointed the target's prototype). Defining the property instead matches
  * what `JSON.parse('{"__proto__":1}')` produces: an ordinary own property.
  */
-function defineOwn(
-  target: Record<string, JexlValue>,
-  key: string,
-  value: JexlValue
-) {
+function defineOwn(target: Context, key: string, value: JexlValue) {
   Object.defineProperty(target, key, {
     value,
     writable: true,
@@ -33,11 +32,7 @@ function defineOwn(
   })
 }
 
-function assignOwn(
-  target: Record<string, JexlValue>,
-  key: string,
-  value: JexlValue
-) {
+function assignOwn(target: Context, key: string, value: JexlValue) {
   target[key] = value
 }
 
@@ -88,21 +83,17 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
     case 'Identifier': {
       const name = node.value
       if (!node.from) {
-        return node.relative
-          ? (ev) => ev._relContext[name]
-          : (ev) => ev._context[name]
+        return (ctx) => ctx[name]
       }
       const from = compileAst(node.from, grammar)
-      return (ev) => {
-        const context = from(ev)
-        if (context == null) {
+      return (ctx) => {
+        const subject = from(ctx)
+        if (subject == null) {
           return undefined
         }
         // an identifier chained off an array reads through its first element
-        const ctx = Array.isArray(context) ? context[0] : context
-        return ctx == null
-          ? undefined
-          : (ctx as Record<string, JexlValue>)[name]
+        const target = Array.isArray(subject) ? subject[0] : subject
+        return target == null ? undefined : (target as Context)[name]
       }
     }
 
@@ -119,18 +110,18 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       if (evalOnDemand) {
         // operands stay unevaluated behind an `eval` thunk, so operators such
         // as && and || can short-circuit
-        return (ev) =>
-          evalOnDemand({ eval: () => left(ev) }, { eval: () => right(ev) })
+        return (ctx) =>
+          evalOnDemand({ eval: () => left(ctx) }, { eval: () => right(ctx) })
       }
       const fn = op.eval
       if (!fn) {
-        return (ev) => {
-          left(ev)
-          right(ev)
+        return (ctx) => {
+          left(ctx)
+          right(ctx)
           return undefined
         }
       }
-      return (ev) => fn(left(ev), right(ev))
+      return (ctx) => fn(left(ctx), right(ctx))
     }
 
     case 'UnaryExpression': {
@@ -144,12 +135,12 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
             : undefined
       if (!fn) {
         // the operand is still evaluated for an unknown operator, as before
-        return (ev) => {
-          right(ev)
+        return (ctx) => {
+          right(ctx)
           return undefined
         }
       }
-      return (ev) => fn(right(ev))
+      return (ctx) => fn(right(ctx))
     }
 
     case 'ConditionalExpression': {
@@ -160,28 +151,22 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       const alternate = node.alternate
         ? compileAst(node.alternate, grammar)
         : undefined
-      return (ev) => {
-        const res = test(ev)
+      return (ctx) => {
+        const res = test(ctx)
         if (res) {
           // an omitted consequent ("a ?: b") yields the test result
-          return consequent ? consequent(ev) : res
+          return consequent ? consequent(ctx) : res
         }
-        return alternate ? alternate(ev) : undefined
+        return alternate ? alternate(ctx) : undefined
       }
     }
 
     case 'FilterExpression': {
-      if (node.relative) {
-        // thrown on evaluation rather than compilation, as before
-        return () => {
-          throw new Error('Relative filter expressions are not supported')
-        }
-      }
       const subject = compileAst(node.subject, grammar)
       const index = compileAst(node.expr, grammar)
-      return (ev) => {
-        const subjectVal = subject(ev)
-        const indexVal = index(ev)
+      return (ctx) => {
+        const subjectVal = subject(ctx)
+        const indexVal = index(ctx)
         if (subjectVal == null) {
           return undefined
         }
@@ -194,10 +179,10 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
     case 'ArrayLiteral': {
       const items = node.value.map((item) => compileAst(item, grammar))
       const len = items.length
-      return (ev) => {
+      return (ctx) => {
         const out: JexlValue[] = new Array(len)
         for (let i = 0; i < len; i++) {
-          out[i] = items[i]!(ev)
+          out[i] = items[i]!(ctx)
         }
         return out
       }
@@ -210,10 +195,10 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       const len = keys.length
       // resolved once, here, so the common case keeps its plain store
       const store = keys.includes('__proto__') ? defineOwn : assignOwn
-      return (ev) => {
+      return (ctx) => {
         const out: Record<string, JexlValue> = {}
         for (let i = 0; i < len; i++) {
-          store(out, keys[i]!, values[i]!(ev))
+          store(out, keys[i]!, values[i]!(ctx))
         }
         return out
       }
@@ -221,18 +206,18 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
 
     case 'TemplateLiteral': {
       // every part renders to a string, so these are narrower than CompiledNode
-      const parts = node.parts.map((part): ((ev: Evaluator) => string) => {
+      const parts = node.parts.map((part): ((ctx: Context) => string) => {
         if (part.type === 'static') {
           const { value } = part
           return () => value
         }
         const expr = compileAst(part.value, grammar)
-        return (ev) => stringify(expr(ev))
+        return (ctx) => stringify(expr(ctx))
       })
-      return (ev) => {
+      return (ctx) => {
         let out = ''
         for (const part of parts) {
-          out += part(ev)
+          out += part(ctx)
         }
         return out
       }
@@ -245,8 +230,8 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       // and `constructor` aren't callable as Jexl functions. Resolved per call
       // rather than baked in, since functions are routinely registered after
       // an expression has been compiled.
-      const lookup = (ev: Evaluator) => {
-        const { functions } = ev._grammar
+      const lookup = () => {
+        const { functions } = grammar
         if (!Object.hasOwn(functions, name)) {
           throw new Error(`Jexl Function ${name} is not defined.`)
         }
@@ -256,15 +241,15 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       // argument array and needs no spread
       switch (args.length) {
         case 0: {
-          return (ev) => lookup(ev)()
+          return () => lookup()()
         }
         case 1: {
           const [a0] = args as [CompiledNode]
-          return (ev) => lookup(ev)(a0(ev))
+          return (ctx) => lookup()(a0(ctx))
         }
         case 2: {
           const [a0, a1] = args as [CompiledNode, CompiledNode]
-          return (ev) => lookup(ev)(a0(ev), a1(ev))
+          return (ctx) => lookup()(a0(ctx), a1(ctx))
         }
         case 3: {
           const [a0, a1, a2] = args as [
@@ -272,16 +257,16 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
             CompiledNode,
             CompiledNode
           ]
-          return (ev) => lookup(ev)(a0(ev), a1(ev), a2(ev))
+          return (ctx) => lookup()(a0(ctx), a1(ctx), a2(ctx))
         }
         default: {
           const len = args.length
-          return (ev) => {
+          return (ctx) => {
             const vals: JexlValue[] = new Array(len)
             for (let i = 0; i < len; i++) {
-              vals[i] = args[i]!(ev)
+              vals[i] = args[i]!(ctx)
             }
-            return lookup(ev)(...vals)
+            return lookup()(...vals)
           }
         }
       }
@@ -290,10 +275,10 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
     case 'SequenceExpression': {
       const exprs = node.expressions.map((expr) => compileAst(expr, grammar))
       const len = exprs.length
-      return (ev) => {
+      return (ctx) => {
         let last: JexlValue
         for (let i = 0; i < len; i++) {
-          last = exprs[i]!(ev)
+          last = exprs[i]!(ctx)
         }
         return last
       }
@@ -303,9 +288,9 @@ export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
       const name = node.left.value
       const right = compileAst(node.right!, grammar)
       const store = name === '__proto__' ? defineOwn : assignOwn
-      return (ev) => {
-        const value = right(ev)
-        store(ev._context, name, value)
+      return (ctx) => {
+        const value = right(ctx)
+        store(ctx, name, value)
         return value
       }
     }

package/src/grammar.ts

@@ -231,7 +231,7 @@ export const getGrammar = (): Grammar => ({
 
   /**
    * A map of function names to javascript functions. A Jexl function
-   * takes zero ore more arguemnts:
+   * takes zero or more arguments:
    *
    *     - {*} ...args: A variable number of arguments passed to this function.
    *       All of these are pre-evaluated to their actual values before calling

package/src/parser/Parser.ts

@@ -32,7 +32,7 @@ const tokenHandlers: Record<
 /**
  * The Parser is a state machine that converts tokens from the {@link Lexer}
  * into an Abstract Syntax Tree (AST), capable of being evaluated in any
- * context by the {@link Evaluator}.  The Parser expects that all tokens
+ * context by {@link compileAst}.  The Parser expects that all tokens
  * provided to it are legal and typed properly according to the grammar, but
  * accepts that the tokens may still be in an invalid order or in some other
  * unparsable configuration that requires it to throw an Error.
@@ -52,13 +52,11 @@ class Parser {
   _state: string
   _tree: AstNode | null
   _exprStr: string
-  _relative: boolean
   _stopMap: Record<string, string>
   _cursor?: AstNode | null
   _subParser?: Parser
   _parentStop?: boolean
   _nextIdentEncapsulate?: boolean
-  _nextIdentRelative?: boolean
   _curObjKey?: string
   _sequenceExpressions?: AstNode[]
 
@@ -73,7 +71,6 @@ class Parser {
     this._state = 'expectOperand'
     this._tree = null
     this._exprStr = prefix || ''
-    this._relative = false
     this._stopMap = stopMap || {}
   }
 
@@ -143,7 +140,7 @@ class Parser {
   /**
    * Marks this Parser instance as completed and retrieves the full AST.
    * @returns {{}|null} a full expression tree, ready for evaluation by the
-   *      {@link Evaluator#eval} function, or null if no tokens were passed to
+   *      {@link compileAst} function, or null if no tokens were passed to
    *      the parser before complete was called
    * @throws {Error} if the parser is not in a state where it's legal to end
    *      the expression, indicating that the expression is incomplete
@@ -179,14 +176,6 @@ class Parser {
     return this._cursor ? this._tree : null
   }
 
-  /**
-   * Indicates whether the expression tree contains a relative path identifier.
-   * @returns {boolean} true if a relative identifier exists false otherwise.
-   */
-  isRelative() {
-    return this._relative
-  }
-
   /**
    * Ends a subexpression by completing the subParser and passing its result
    * to the subHandler configured in the current state.