@jbrowse/jexl 3.0.2 → 3.2.0

package/src/evaluator/compile.ts

@@ -0,0 +1,317 @@
+/*
+ * Jexl
+ * Copyright 2020 Tom Shawver
+ */
+
+import type { Grammar } from '../grammar.ts'
+import type { AstNode, AstNodeUnion, JexlValue } from '../types.ts'
+import type Evaluator from './Evaluator.ts'
+
+/**
+ * An AST node lowered to a closure. Calling it with an Evaluator (which carries
+ * the context and grammar) produces the node's value.
+ */
+export type CompiledNode = (ev: Evaluator) => JexlValue
+
+/**
+ * Writes a key that a plain assignment would mishandle. Storing to
+ * "__proto__" invokes the prototype setter rather than creating a property, so
+ * an object literal or assignment using that key silently lost its value (and
+ * re-pointed the target's prototype). Defining the property instead matches
+ * what `JSON.parse('{"__proto__":1}')` produces: an ordinary own property.
+ */
+function defineOwn(
+  target: Record<string, JexlValue>,
+  key: string,
+  value: JexlValue
+) {
+  Object.defineProperty(target, key, {
+    value,
+    writable: true,
+    enumerable: true,
+    configurable: true
+  })
+}
+
+function assignOwn(
+  target: Record<string, JexlValue>,
+  key: string,
+  value: JexlValue
+) {
+  target[key] = value
+}
+
+/** Renders an interpolated value for a template literal. */
+function stringify(value: JexlValue) {
+  if (value == null) {
+    return ''
+  }
+  if (
+    typeof value === 'string' ||
+    typeof value === 'number' ||
+    typeof value === 'boolean'
+  ) {
+    return String(value)
+  }
+  return JSON.stringify(value)
+}
+
+/**
+ * Lowers an expression tree into a tree of closures, resolving each node's type
+ * and its operator's implementation once, at compile time, rather than
+ * re-dispatching on `node.type` for every node on every evaluation. An
+ * expression compiled once and evaluated per-item — the usual shape for
+ * per-feature config callbacks — pays the dispatch cost once instead of N
+ * times.
+ *
+ * Operators are bound here, so a grammar change after compilation requires
+ * recompiling, which is what {@link Expression#compile} already documents.
+ * Functions are looked up per call, since they are commonly registered after
+ * an expression has been compiled.
+ *
+ * @param {{}} ast An expression tree, as produced by the Parser
+ * @param {{}} grammar The grammar to resolve operators against
+ * @returns {function} a closure returning the expression's value
+ * @throws {Error} if the tree contains an unrecognized node type
+ */
+export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
+  // AstNode types its `type` as a plain string so the Parser can build the tree
+  // loosely; narrowing to the union once, here, lets every case below see its
+  // own node type instead of repeating the same cast in each branch
+  const node = ast as AstNodeUnion
+  switch (node.type) {
+    case 'Literal': {
+      const { value } = node
+      return () => value
+    }
+
+    case 'Identifier': {
+      const name = node.value
+      if (!node.from) {
+        return node.relative
+          ? (ev) => ev._relContext[name]
+          : (ev) => ev._context[name]
+      }
+      const from = compileAst(node.from, grammar)
+      return (ev) => {
+        const context = from(ev)
+        if (context == null) {
+          return undefined
+        }
+        // an identifier chained off an array reads through its first element
+        const ctx = Array.isArray(context) ? context[0] : context
+        return ctx == null
+          ? undefined
+          : (ctx as Record<string, JexlValue>)[name]
+      }
+    }
+
+    case 'BinaryExpression': {
+      const op = grammar.elements[node.operator]
+      if (op?.type !== 'binaryOp') {
+        // matches the tree-walking behaviour: an unknown operator yields
+        // undefined without evaluating either operand
+        return () => undefined
+      }
+      const left = compileAst(node.left, grammar)
+      const right = compileAst(node.right!, grammar)
+      const { evalOnDemand } = op
+      if (evalOnDemand) {
+        // operands stay unevaluated behind an `eval` thunk, so operators such
+        // as && and || can short-circuit
+        return (ev) =>
+          evalOnDemand({ eval: () => left(ev) }, { eval: () => right(ev) })
+      }
+      const fn = op.eval
+      if (!fn) {
+        return (ev) => {
+          left(ev)
+          right(ev)
+          return undefined
+        }
+      }
+      return (ev) => fn(left(ev), right(ev))
+    }
+
+    case 'UnaryExpression': {
+      const right = compileAst(node.right!, grammar)
+      const elem = grammar.elements[node.operator]
+      const fn =
+        elem?.type === 'unaryOp'
+          ? elem.eval
+          : elem?.type === 'binaryOp'
+            ? elem.unaryEval
+            : undefined
+      if (!fn) {
+        // the operand is still evaluated for an unknown operator, as before
+        return (ev) => {
+          right(ev)
+          return undefined
+        }
+      }
+      return (ev) => fn(right(ev))
+    }
+
+    case 'ConditionalExpression': {
+      const test = compileAst(node.test, grammar)
+      const consequent = node.consequent
+        ? compileAst(node.consequent, grammar)
+        : undefined
+      const alternate = node.alternate
+        ? compileAst(node.alternate, grammar)
+        : undefined
+      return (ev) => {
+        const res = test(ev)
+        if (res) {
+          // an omitted consequent ("a ?: b") yields the test result
+          return consequent ? consequent(ev) : res
+        }
+        return alternate ? alternate(ev) : undefined
+      }
+    }
+
+    case 'FilterExpression': {
+      if (node.relative) {
+        // thrown on evaluation rather than compilation, as before
+        return () => {
+          throw new Error('Relative filter expressions are not supported')
+        }
+      }
+      const subject = compileAst(node.subject, grammar)
+      const index = compileAst(node.expr, grammar)
+      return (ev) => {
+        const subjectVal = subject(ev)
+        const indexVal = index(ev)
+        if (subjectVal == null) {
+          return undefined
+        }
+        return typeof indexVal === 'string' || typeof indexVal === 'number'
+          ? (subjectVal as Record<string | number, JexlValue>)[indexVal]
+          : undefined
+      }
+    }
+
+    case 'ArrayLiteral': {
+      const items = node.value.map((item) => compileAst(item, grammar))
+      const len = items.length
+      return (ev) => {
+        const out: JexlValue[] = new Array(len)
+        for (let i = 0; i < len; i++) {
+          out[i] = items[i]!(ev)
+        }
+        return out
+      }
+    }
+
+    case 'ObjectLiteral': {
+      const entries = Object.entries(node.value)
+      const keys = entries.map(([key]) => key)
+      const values = entries.map(([, value]) => compileAst(value, grammar))
+      const len = keys.length
+      // resolved once, here, so the common case keeps its plain store
+      const store = keys.includes('__proto__') ? defineOwn : assignOwn
+      return (ev) => {
+        const out: Record<string, JexlValue> = {}
+        for (let i = 0; i < len; i++) {
+          store(out, keys[i]!, values[i]!(ev))
+        }
+        return out
+      }
+    }
+
+    case 'TemplateLiteral': {
+      // every part renders to a string, so these are narrower than CompiledNode
+      const parts = node.parts.map((part): ((ev: Evaluator) => string) => {
+        if (part.type === 'static') {
+          const { value } = part
+          return () => value
+        }
+        const expr = compileAst(part.value, grammar)
+        return (ev) => stringify(expr(ev))
+      })
+      return (ev) => {
+        let out = ''
+        for (const part of parts) {
+          out += part(ev)
+        }
+        return out
+      }
+    }
+
+    case 'FunctionCall': {
+      const { name } = node
+      const args = node.args.map((arg) => compileAst(arg, grammar))
+      // hasOwn, so that inherited Object.prototype members such as `toString`
+      // and `constructor` aren't callable as Jexl functions. Resolved per call
+      // rather than baked in, since functions are routinely registered after
+      // an expression has been compiled.
+      const lookup = (ev: Evaluator) => {
+        const { functions } = ev._grammar
+        if (!Object.hasOwn(functions, name)) {
+          throw new Error(`Jexl Function ${name} is not defined.`)
+        }
+        return functions[name]!
+      }
+      // the small arities are spelled out so an ordinary call allocates no
+      // argument array and needs no spread
+      switch (args.length) {
+        case 0: {
+          return (ev) => lookup(ev)()
+        }
+        case 1: {
+          const [a0] = args as [CompiledNode]
+          return (ev) => lookup(ev)(a0(ev))
+        }
+        case 2: {
+          const [a0, a1] = args as [CompiledNode, CompiledNode]
+          return (ev) => lookup(ev)(a0(ev), a1(ev))
+        }
+        case 3: {
+          const [a0, a1, a2] = args as [
+            CompiledNode,
+            CompiledNode,
+            CompiledNode
+          ]
+          return (ev) => lookup(ev)(a0(ev), a1(ev), a2(ev))
+        }
+        default: {
+          const len = args.length
+          return (ev) => {
+            const vals: JexlValue[] = new Array(len)
+            for (let i = 0; i < len; i++) {
+              vals[i] = args[i]!(ev)
+            }
+            return lookup(ev)(...vals)
+          }
+        }
+      }
+    }
+
+    case 'SequenceExpression': {
+      const exprs = node.expressions.map((expr) => compileAst(expr, grammar))
+      const len = exprs.length
+      return (ev) => {
+        let last: JexlValue
+        for (let i = 0; i < len; i++) {
+          last = exprs[i]!(ev)
+        }
+        return last
+      }
+    }
+
+    case 'AssignmentExpression': {
+      const name = node.left.value
+      const right = compileAst(node.right!, grammar)
+      const store = name === '__proto__' ? defineOwn : assignOwn
+      return (ev) => {
+        const value = right(ev)
+        store(ev._context, name, value)
+        return value
+      }
+    }
+
+    default: {
+      throw new Error(`Corrupt AST: unknown node type '${ast.type}'`)
+    }
+  }
+}

package/src/grammar.ts

@@ -5,28 +5,70 @@
 
 /* eslint eqeqeq:0 */
 
+import type { JexlValue } from './types.ts'
+
+export type BinaryOpEval = (left: JexlValue, right: JexlValue) => JexlValue
+
+export type BinaryOpEvalOnDemand = (
+  left: { eval: () => JexlValue },
+  right: { eval: () => JexlValue }
+) => JexlValue
+
 export interface BinaryOp {
   type: 'binaryOp'
   precedence: number
-  eval?: (left: any, right: any) => any
-  evalOnDemand?: (left: { eval: () => any }, right: { eval: () => any }) => any
+  eval?: BinaryOpEval
+  evalOnDemand?: BinaryOpEvalOnDemand
+  /**
+   * Evaluator for the operator's prefix form, if it has one. The Lexer emits a
+   * unaryOp token when such an operator appears where an operand is expected,
+   * which is how `-x` is distinguished from `a - x`.
+   */
+  unaryEval?: (right: JexlValue) => JexlValue
 }
 
 export interface UnaryOp {
   type: 'unaryOp'
   precedence: number
-  eval: (right: any) => any
+  eval: (right: JexlValue) => JexlValue
 }
 
+/**
+ * The punctuation elements of the grammar. Unlike operators these carry no
+ * behavior of their own; the Parser's state machine gives them meaning. The
+ * type is a literal union rather than `string` so that `GrammarElement` is a
+ * discriminated union, letting operator properties be accessed without casts.
+ */
 export interface SimpleElement {
-  type: string
+  type:
+    | 'dot'
+    | 'openBracket'
+    | 'closeBracket'
+    | 'openCurl'
+    | 'closeCurl'
+    | 'colon'
+    | 'comma'
+    | 'openParen'
+    | 'closeParen'
+    | 'question'
+    | 'semicolon'
 }
 
 export type GrammarElement = BinaryOp | UnaryOp | SimpleElement
 
 export interface Grammar {
   elements: Record<string, GrammarElement>
-  functions: Record<string, (...args: any[]) => any>
+  functions: Record<string, (...args: JexlValue[]) => JexlValue>
+}
+
+/**
+ * Returns the binding power of a grammar element, or 0 for elements that
+ * aren't operators and therefore don't participate in precedence.
+ */
+export function precedenceOf(elem: GrammarElement | undefined) {
+  return elem && (elem.type === 'binaryOp' || elem.type === 'unaryOp')
+    ? elem.precedence
+    : 0
 }
 
 export const getGrammar = (): Grammar => ({
@@ -50,37 +92,38 @@ export const getGrammar = (): Grammar => ({
     '+': {
       type: 'binaryOp',
       precedence: 30,
-      eval: (left, right) => left + right
+      eval: (left, right) => (left as number) + (right as number)
     },
     '-': {
       type: 'binaryOp',
       precedence: 30,
-      eval: (left, right) => left - right
+      eval: (left, right) => (left as number) - (right as number),
+      unaryEval: (right) => -(right as number)
     },
     '*': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => left * right
+      eval: (left, right) => (left as number) * (right as number)
     },
     '/': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => left / right
+      eval: (left, right) => (left as number) / (right as number)
     },
     '//': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => Math.floor(left / right)
+      eval: (left, right) => Math.floor((left as number) / (right as number))
     },
     '%': {
       type: 'binaryOp',
       precedence: 50,
-      eval: (left, right) => left % right
+      eval: (left, right) => (left as number) % (right as number)
     },
     '^': {
       type: 'binaryOp',
       precedence: 50,
-      eval: (left, right) => Math.pow(left, right)
+      eval: (left, right) => (left as number) ** (right as number)
     },
     '==': {
       type: 'binaryOp',
@@ -95,22 +138,22 @@ export const getGrammar = (): Grammar => ({
     '>': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left > right
+      eval: (left, right) => (left as number) > (right as number)
     },
     '>=': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left >= right
+      eval: (left, right) => (left as number) >= (right as number)
     },
     '<': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left < right
+      eval: (left, right) => (left as number) < (right as number)
     },
     '<=': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left <= right
+      eval: (left, right) => (left as number) <= (right as number)
     },
     '&&': {
       type: 'binaryOp',
@@ -139,12 +182,17 @@ export const getGrammar = (): Grammar => ({
       precedence: 20,
       eval: (left, right) => {
         if (typeof right === 'string') {
-          return right.includes(left)
-        }
-        if (Array.isArray(right)) {
-          return right.includes(left)
+          // only a primitive has a meaningful substring form. An absent or
+          // structured left operand is not "in" a string, and must not be
+          // coerced to '' — every string contains the empty string, so that
+          // made `feature.missingAttr in "someString"` true for every feature.
+          return typeof left === 'string' ||
+            typeof left === 'number' ||
+            typeof left === 'boolean'
+            ? right.includes(String(left))
+            : false
         }
-        return false
+        return Array.isArray(right) ? right.includes(left) : false
       }
     },
     '!': {

package/src/parser/Parser.ts

@@ -7,10 +7,26 @@ import * as handlers from './handlers.ts'
 import { states } from './states.ts'
 
 import type Lexer from '../Lexer.ts'
-import type { AstNode, Token } from '../types.ts'
+import type { Grammar } from '../grammar.ts'
+import type { AstNode, SequenceExpression, Token } from '../types.ts'
 
-interface Grammar {
-  elements: Record<string, any>
+/**
+ * The handler to run for a token type when the state it appears in doesn't
+ * name one of its own. Token types absent here — the brackets, parens and
+ * separators — carry no behavior beyond the state transition the state machine
+ * gives them.
+ */
+const tokenHandlers: Record<
+  string,
+  ((this: Parser, token: Token) => void) | undefined
+> = {
+  binaryOp: handlers.binaryOp,
+  dot: handlers.dot,
+  identifier: handlers.identifier,
+  literal: handlers.literal,
+  semicolon: handlers.semicolon,
+  templateString: handlers.templateString,
+  unaryOp: handlers.unaryOp
 }
 
 /**
@@ -37,7 +53,7 @@ class Parser {
   _tree: AstNode | null
   _exprStr: string
   _relative: boolean
-  _stopMap: Record<string, any>
+  _stopMap: Record<string, string>
   _cursor?: AstNode | null
   _subParser?: Parser
   _parentStop?: boolean
@@ -50,7 +66,7 @@ class Parser {
     grammar: Grammar,
     lexer: Lexer,
     prefix?: string,
-    stopMap?: Record<string, any>
+    stopMap?: Record<string, string>
   ) {
     this._grammar = grammar
     this._lexer = lexer
@@ -71,18 +87,16 @@ class Parser {
    * @returns {boolean|*} the stopState value if this parser encountered a token
    *      in the stopState mapb false if tokens can continue.
    */
-  addToken(token: Token): any {
+  addToken(token: Token): string | false {
     if (this._state === 'complete') {
       throw new Error('Cannot add a new token to a completed Parser')
     }
-    const state = states[this._state]
+    const state = states[this._state]!
     const startExpr = this._exprStr
     this._exprStr += token.raw
     if (state.subHandler) {
-      if (!this._subParser) {
-        this._startSubExpression(startExpr)
-      }
-      const stopState = this._subParser!.addToken(token)
+      const subParser = this._subParser ?? this._startSubExpression(startExpr)
+      const stopState = subParser.addToken(token)
       if (stopState) {
         this._endSubExpression()
         if (stopState === '_semicolon') {
@@ -94,21 +108,18 @@ class Parser {
         }
       }
     } else if (token.type === 'semicolon' && this._stopMap[token.type]) {
-      return this._stopMap[token.type]
+      return this._stopMap[token.type]!
     } else if (state.tokenTypes?.[token.type]) {
-      const typeOpts = state.tokenTypes[token.type]
-      let handleFunc = (handlers as any)[token.type]
-      if (typeOpts.handler) {
-        handleFunc = typeOpts.handler
-      }
-      if (handleFunc) {
-        handleFunc.call(this, token)
-      }
+      const typeOpts = state.tokenTypes[token.type]!
+      // the state's own handler wins over the token type's default one, and a
+      // token type with neither only drives the transition below
+      const handleFunc = typeOpts.handler ?? tokenHandlers[token.type]
+      handleFunc?.call(this, token)
       if (typeOpts.toState) {
         this._state = typeOpts.toState
       }
     } else if (this._stopMap[token.type]) {
-      return this._stopMap[token.type]
+      return this._stopMap[token.type]!
     } else {
       throw new Error(
         `Token ${token.raw} (${token.type}) unexpected in expression: ${this._exprStr}`
@@ -124,7 +135,9 @@ class Parser {
    *      the {@link Lexer#tokenize} function.
    */
   addTokens(tokens: Token[]) {
-    tokens.forEach((token) => this.addToken(token))
+    for (const token of tokens) {
+      this.addToken(token)
+    }
   }
 
   /**
@@ -136,7 +149,14 @@ class Parser {
    *      the expression, indicating that the expression is incomplete
    */
   complete() {
-    if (this._cursor && !states[this._state].completable) {
+    // `_subParser` counts as unfinished business alongside `_cursor`: a group
+    // that opened before anything was placed in the tree, as in "(1", leaves
+    // the cursor null, and testing the cursor alone let the missing ")" pass
+    // silently and evaluate as if it were closed.
+    if (
+      (this._cursor || this._subParser) &&
+      !states[this._state]!.completable
+    ) {
       throw new Error(`Unexpected end of expression: ${this._exprStr}`)
     }
     if (this._subParser) {
@@ -147,7 +167,7 @@ class Parser {
       if (this._tree) {
         this._sequenceExpressions.push(this._tree)
       }
-      const sequence: any = {
+      const sequence: SequenceExpression = {
         type: 'SequenceExpression',
         expressions: this._sequenceExpressions
       }
@@ -173,7 +193,7 @@ class Parser {
    * @private
    */
   _endSubExpression() {
-    states[this._state].subHandler!.call(this, this._subParser!.complete())
+    states[this._state]!.subHandler!.call(this, this._subParser!.complete())
     this._subParser = undefined
   }
 
@@ -188,7 +208,7 @@ class Parser {
     if (!this._cursor) {
       this._tree = node
     } else {
-      ;(this._cursor as any).right = node
+      this._cursor.right = node
       this._setParent(node, this._cursor)
     }
     this._cursor = node
@@ -227,18 +247,20 @@ class Parser {
    * Prepares the Parser to accept a subexpression by (re)instantiating the
    * subParser.
    * @param {string} [exprStr] The expression string to prefix to the new Parser
+   * @returns {Parser} the new subParser, also stored on this instance
    * @private
    */
   _startSubExpression(exprStr?: string) {
-    let endStates = states[this._state].endStates
+    let endStates = states[this._state]!.endStates
     if (!endStates) {
       this._parentStop = true
       endStates = this._stopMap
     }
-    if (states[this._state].completable && !endStates.semicolon) {
+    if (states[this._state]!.completable && !endStates.semicolon) {
       endStates = { ...endStates, semicolon: '_semicolon' }
     }
     this._subParser = new Parser(this._grammar, this._lexer, exprStr, endStates)
+    return this._subParser
   }
 }