@jbrowse/jexl 3.0.1 → 3.1.0

package/src/evaluator/compile.ts

@@ -0,0 +1,310 @@
+/*
+ * Jexl
+ * Copyright 2020 Tom Shawver
+ */
+
+import type { Grammar } from '../grammar.ts'
+import type {
+  ArrayLiteral,
+  AssignmentExpression,
+  AstNode,
+  BinaryExpression,
+  ConditionalExpression,
+  FilterExpression,
+  FunctionCall,
+  Identifier,
+  JexlValue,
+  Literal,
+  ObjectLiteral,
+  SequenceExpression,
+  TemplateLiteral,
+  UnaryExpression
+} from '../types.ts'
+import type Evaluator from './Evaluator.ts'
+
+/**
+ * An AST node lowered to a closure. Calling it with an Evaluator (which carries
+ * the context and grammar) produces the node's value.
+ */
+export type CompiledNode = (ev: Evaluator) => JexlValue
+
+/** Renders an interpolated value for a template literal. */
+function stringify(value: JexlValue) {
+  if (value == null) {
+    return ''
+  }
+  if (
+    typeof value === 'string' ||
+    typeof value === 'number' ||
+    typeof value === 'boolean'
+  ) {
+    return String(value)
+  }
+  return JSON.stringify(value)
+}
+
+/**
+ * Lowers an expression tree into a tree of closures, resolving each node's type
+ * and its operator's implementation once, at compile time, rather than
+ * re-dispatching on `node.type` for every node on every evaluation. An
+ * expression compiled once and evaluated per-item — the usual shape for
+ * per-feature config callbacks — pays the dispatch cost once instead of N
+ * times.
+ *
+ * Operators are bound here, so a grammar change after compilation requires
+ * recompiling, which is what {@link Expression#compile} already documents.
+ * Functions are looked up per call, since they are commonly registered after
+ * an expression has been compiled.
+ *
+ * @param {{}} ast An expression tree, as produced by the Parser
+ * @param {{}} grammar The grammar to resolve operators against
+ * @returns {function} a closure returning the expression's value
+ * @throws {Error} if the tree contains an unrecognized node type
+ */
+export function compileAst(ast: AstNode, grammar: Grammar): CompiledNode {
+  switch (ast.type) {
+    case 'Literal': {
+      const { value } = ast as Literal
+      return () => value
+    }
+
+    case 'Identifier': {
+      const node = ast as Identifier
+      const name = node.value
+      if (!node.from) {
+        return node.relative
+          ? (ev) => ev._relContext[name]
+          : (ev) => ev._context[name]
+      }
+      const from = compileAst(node.from, grammar)
+      return (ev) => {
+        const context = from(ev)
+        if (context == null) {
+          return undefined
+        }
+        // an identifier chained off an array reads through its first element
+        const ctx = Array.isArray(context) ? context[0] : context
+        return ctx == null
+          ? undefined
+          : (ctx as Record<string, JexlValue>)[name]
+      }
+    }
+
+    case 'BinaryExpression': {
+      const node = ast as BinaryExpression
+      const op = grammar.elements[node.operator]
+      if (op?.type !== 'binaryOp') {
+        // matches the tree-walking behaviour: an unknown operator yields
+        // undefined without evaluating either operand
+        return () => undefined
+      }
+      const left = compileAst(node.left, grammar)
+      const right = compileAst(node.right!, grammar)
+      const { evalOnDemand } = op
+      if (evalOnDemand) {
+        // operands stay unevaluated behind an `eval` thunk, so operators such
+        // as && and || can short-circuit
+        return (ev) =>
+          evalOnDemand({ eval: () => left(ev) }, { eval: () => right(ev) })
+      }
+      const fn = op.eval
+      if (!fn) {
+        return (ev) => {
+          left(ev)
+          right(ev)
+          return undefined
+        }
+      }
+      return (ev) => fn(left(ev), right(ev))
+    }
+
+    case 'UnaryExpression': {
+      const node = ast as UnaryExpression
+      const right = compileAst(node.right!, grammar)
+      const elem = grammar.elements[node.operator]
+      const fn =
+        elem?.type === 'unaryOp'
+          ? elem.eval
+          : elem?.type === 'binaryOp'
+            ? elem.unaryEval
+            : undefined
+      if (!fn) {
+        // the operand is still evaluated for an unknown operator, as before
+        return (ev) => {
+          right(ev)
+          return undefined
+        }
+      }
+      return (ev) => fn(right(ev))
+    }
+
+    case 'ConditionalExpression': {
+      const node = ast as ConditionalExpression
+      const test = compileAst(node.test, grammar)
+      const consequent = node.consequent
+        ? compileAst(node.consequent, grammar)
+        : undefined
+      const alternate = node.alternate
+        ? compileAst(node.alternate, grammar)
+        : undefined
+      return (ev) => {
+        const res = test(ev)
+        if (res) {
+          // an omitted consequent ("a ?: b") yields the test result
+          return consequent ? consequent(ev) : res
+        }
+        return alternate ? alternate(ev) : undefined
+      }
+    }
+
+    case 'FilterExpression': {
+      const node = ast as FilterExpression
+      if (node.relative) {
+        // thrown on evaluation rather than compilation, as before
+        return () => {
+          throw new Error('Relative filter expressions are not supported')
+        }
+      }
+      const subject = compileAst(node.subject, grammar)
+      const index = compileAst(node.expr, grammar)
+      return (ev) => {
+        const subjectVal = subject(ev)
+        const indexVal = index(ev)
+        if (subjectVal == null) {
+          return undefined
+        }
+        return typeof indexVal === 'string' || typeof indexVal === 'number'
+          ? (subjectVal as Record<string | number, JexlValue>)[indexVal]
+          : undefined
+      }
+    }
+
+    case 'ArrayLiteral': {
+      const items = (ast as ArrayLiteral).value.map((item) =>
+        compileAst(item, grammar)
+      )
+      const len = items.length
+      return (ev) => {
+        const out: JexlValue[] = new Array(len)
+        for (let i = 0; i < len; i++) {
+          out[i] = items[i]!(ev)
+        }
+        return out
+      }
+    }
+
+    case 'ObjectLiteral': {
+      const entries = Object.entries((ast as ObjectLiteral).value)
+      const keys = entries.map(([key]) => key)
+      const values = entries.map(([, node]) => compileAst(node, grammar))
+      const len = keys.length
+      return (ev) => {
+        const out: Record<string, JexlValue> = {}
+        for (let i = 0; i < len; i++) {
+          out[keys[i]!] = values[i]!(ev)
+        }
+        return out
+      }
+    }
+
+    case 'TemplateLiteral': {
+      // every part renders to a string, so these are narrower than CompiledNode
+      const parts = (ast as TemplateLiteral).parts.map(
+        (part): ((ev: Evaluator) => string) => {
+          if (part.type === 'static') {
+            const { value } = part
+            return () => value
+          }
+          const expr = compileAst(part.value, grammar)
+          return (ev) => stringify(expr(ev))
+        }
+      )
+      return (ev) => {
+        let out = ''
+        for (const part of parts) {
+          out += part(ev)
+        }
+        return out
+      }
+    }
+
+    case 'FunctionCall': {
+      const node = ast as FunctionCall
+      const { name } = node
+      const args = node.args.map((arg) => compileAst(arg, grammar))
+      // hasOwn, so that inherited Object.prototype members such as `toString`
+      // and `constructor` aren't callable as Jexl functions. Resolved per call
+      // rather than baked in, since functions are routinely registered after
+      // an expression has been compiled.
+      const lookup = (ev: Evaluator) => {
+        const { functions } = ev._grammar
+        if (!Object.hasOwn(functions, name)) {
+          throw new Error(`Jexl Function ${name} is not defined.`)
+        }
+        return functions[name]!
+      }
+      // the small arities are spelled out so an ordinary call allocates no
+      // argument array and needs no spread
+      switch (args.length) {
+        case 0: {
+          return (ev) => lookup(ev)()
+        }
+        case 1: {
+          const [a0] = args as [CompiledNode]
+          return (ev) => lookup(ev)(a0(ev))
+        }
+        case 2: {
+          const [a0, a1] = args as [CompiledNode, CompiledNode]
+          return (ev) => lookup(ev)(a0(ev), a1(ev))
+        }
+        case 3: {
+          const [a0, a1, a2] = args as [
+            CompiledNode,
+            CompiledNode,
+            CompiledNode
+          ]
+          return (ev) => lookup(ev)(a0(ev), a1(ev), a2(ev))
+        }
+        default: {
+          const len = args.length
+          return (ev) => {
+            const vals: JexlValue[] = new Array(len)
+            for (let i = 0; i < len; i++) {
+              vals[i] = args[i]!(ev)
+            }
+            return lookup(ev)(...vals)
+          }
+        }
+      }
+    }
+
+    case 'SequenceExpression': {
+      const exprs = (ast as SequenceExpression).expressions.map((expr) =>
+        compileAst(expr, grammar)
+      )
+      const len = exprs.length
+      return (ev) => {
+        let last: JexlValue
+        for (let i = 0; i < len; i++) {
+          last = exprs[i]!(ev)
+        }
+        return last
+      }
+    }
+
+    case 'AssignmentExpression': {
+      const node = ast as AssignmentExpression
+      const name = node.left.value
+      const right = compileAst(node.right!, grammar)
+      return (ev) => {
+        const value = right(ev)
+        ev._context[name] = value
+        return value
+      }
+    }
+
+    default: {
+      throw new Error(`Corrupt AST: unknown node type '${ast.type}'`)
+    }
+  }
+}

package/src/grammar.ts

@@ -5,28 +5,70 @@
 
 /* eslint eqeqeq:0 */
 
+import type { JexlValue } from './types.ts'
+
+export type BinaryOpEval = (left: JexlValue, right: JexlValue) => JexlValue
+
+export type BinaryOpEvalOnDemand = (
+  left: { eval: () => JexlValue },
+  right: { eval: () => JexlValue }
+) => JexlValue
+
 export interface BinaryOp {
   type: 'binaryOp'
   precedence: number
-  eval?: (left: any, right: any) => any
-  evalOnDemand?: (left: { eval: () => any }, right: { eval: () => any }) => any
+  eval?: BinaryOpEval
+  evalOnDemand?: BinaryOpEvalOnDemand
+  /**
+   * Evaluator for the operator's prefix form, if it has one. The Lexer emits a
+   * unaryOp token when such an operator appears where an operand is expected,
+   * which is how `-x` is distinguished from `a - x`.
+   */
+  unaryEval?: (right: JexlValue) => JexlValue
 }
 
 export interface UnaryOp {
   type: 'unaryOp'
   precedence: number
-  eval: (right: any) => any
+  eval: (right: JexlValue) => JexlValue
 }
 
+/**
+ * The punctuation elements of the grammar. Unlike operators these carry no
+ * behavior of their own; the Parser's state machine gives them meaning. The
+ * type is a literal union rather than `string` so that `GrammarElement` is a
+ * discriminated union, letting operator properties be accessed without casts.
+ */
 export interface SimpleElement {
-  type: string
+  type:
+    | 'dot'
+    | 'openBracket'
+    | 'closeBracket'
+    | 'openCurl'
+    | 'closeCurl'
+    | 'colon'
+    | 'comma'
+    | 'openParen'
+    | 'closeParen'
+    | 'question'
+    | 'semicolon'
 }
 
 export type GrammarElement = BinaryOp | UnaryOp | SimpleElement
 
 export interface Grammar {
   elements: Record<string, GrammarElement>
-  functions: Record<string, (...args: any[]) => any>
+  functions: Record<string, (...args: JexlValue[]) => JexlValue>
+}
+
+/**
+ * Returns the binding power of a grammar element, or 0 for elements that
+ * aren't operators and therefore don't participate in precedence.
+ */
+export function precedenceOf(elem: GrammarElement | undefined) {
+  return elem && (elem.type === 'binaryOp' || elem.type === 'unaryOp')
+    ? elem.precedence
+    : 0
 }
 
 export const getGrammar = (): Grammar => ({
@@ -50,37 +92,38 @@ export const getGrammar = (): Grammar => ({
     '+': {
       type: 'binaryOp',
       precedence: 30,
-      eval: (left, right) => left + right
+      eval: (left, right) => (left as number) + (right as number)
     },
     '-': {
       type: 'binaryOp',
       precedence: 30,
-      eval: (left, right) => left - right
+      eval: (left, right) => (left as number) - (right as number),
+      unaryEval: (right) => -(right as number)
     },
     '*': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => left * right
+      eval: (left, right) => (left as number) * (right as number)
     },
     '/': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => left / right
+      eval: (left, right) => (left as number) / (right as number)
     },
     '//': {
       type: 'binaryOp',
       precedence: 40,
-      eval: (left, right) => Math.floor(left / right)
+      eval: (left, right) => Math.floor((left as number) / (right as number))
     },
     '%': {
       type: 'binaryOp',
       precedence: 50,
-      eval: (left, right) => left % right
+      eval: (left, right) => (left as number) % (right as number)
     },
     '^': {
       type: 'binaryOp',
       precedence: 50,
-      eval: (left, right) => Math.pow(left, right)
+      eval: (left, right) => (left as number) ** (right as number)
     },
     '==': {
       type: 'binaryOp',
@@ -95,22 +138,22 @@ export const getGrammar = (): Grammar => ({
     '>': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left > right
+      eval: (left, right) => (left as number) > (right as number)
     },
     '>=': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left >= right
+      eval: (left, right) => (left as number) >= (right as number)
     },
     '<': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left < right
+      eval: (left, right) => (left as number) < (right as number)
     },
     '<=': {
       type: 'binaryOp',
       precedence: 20,
-      eval: (left, right) => left <= right
+      eval: (left, right) => (left as number) <= (right as number)
     },
     '&&': {
       type: 'binaryOp',
@@ -139,7 +182,13 @@ export const getGrammar = (): Grammar => ({
       precedence: 20,
       eval: (left, right) => {
         if (typeof right === 'string') {
-          return right.includes(left)
+          const leftStr =
+            typeof left === 'string' ||
+            typeof left === 'number' ||
+            typeof left === 'boolean'
+              ? String(left)
+              : ''
+          return right.includes(leftStr)
         }
         if (Array.isArray(right)) {
           return right.includes(left)

package/src/parser/Parser.ts

@@ -7,11 +7,8 @@ import * as handlers from './handlers.ts'
 import { states } from './states.ts'
 
 import type Lexer from '../Lexer.ts'
-import type { AstNode, Token } from '../types.ts'
-
-interface Grammar {
-  elements: Record<string, any>
-}
+import type { Grammar } from '../grammar.ts'
+import type { AstNode, SequenceExpression, Token } from '../types.ts'
 
 /**
  * The Parser is a state machine that converts tokens from the {@link Lexer}
@@ -37,7 +34,7 @@ class Parser {
   _tree: AstNode | null
   _exprStr: string
   _relative: boolean
-  _stopMap: Record<string, any>
+  _stopMap: Record<string, string>
   _cursor?: AstNode | null
   _subParser?: Parser
   _parentStop?: boolean
@@ -50,7 +47,7 @@ class Parser {
     grammar: Grammar,
     lexer: Lexer,
     prefix?: string,
-    stopMap?: Record<string, any>
+    stopMap?: Record<string, string>
   ) {
     this._grammar = grammar
     this._lexer = lexer
@@ -71,11 +68,11 @@ class Parser {
    * @returns {boolean|*} the stopState value if this parser encountered a token
    *      in the stopState mapb false if tokens can continue.
    */
-  addToken(token: Token): any {
+  addToken(token: Token): string | false {
     if (this._state === 'complete') {
       throw new Error('Cannot add a new token to a completed Parser')
     }
-    const state = states[this._state]
+    const state = states[this._state]!
     const startExpr = this._exprStr
     this._exprStr += token.raw
     if (state.subHandler) {
@@ -85,14 +82,24 @@ class Parser {
       const stopState = this._subParser!.addToken(token)
       if (stopState) {
         this._endSubExpression()
-        if (this._parentStop) {
+        if (stopState === '_semicolon') {
+          handlers.semicolon.call(this)
+        } else if (this._parentStop) {
           return stopState
+        } else {
+          this._state = stopState
         }
-        this._state = stopState
       }
+    } else if (token.type === 'semicolon' && this._stopMap[token.type]) {
+      return this._stopMap[token.type]!
     } else if (state.tokenTypes?.[token.type]) {
-      const typeOpts = state.tokenTypes[token.type]
-      let handleFunc = (handlers as any)[token.type]
+      const typeOpts = state.tokenTypes[token.type]!
+      let handleFunc = (
+        handlers as Record<
+          string,
+          ((this: Parser, token: Token) => void) | undefined
+        >
+      )[token.type]
       if (typeOpts.handler) {
         handleFunc = typeOpts.handler
       }
@@ -103,7 +110,7 @@ class Parser {
         this._state = typeOpts.toState
       }
     } else if (this._stopMap[token.type]) {
-      return this._stopMap[token.type]
+      return this._stopMap[token.type]!
     } else {
       throw new Error(
         `Token ${token.raw} (${token.type}) unexpected in expression: ${this._exprStr}`
@@ -119,7 +126,9 @@ class Parser {
    *      the {@link Lexer#tokenize} function.
    */
   addTokens(tokens: Token[]) {
-    tokens.forEach((token) => this.addToken(token))
+    for (const token of tokens) {
+      this.addToken(token)
+    }
   }
 
   /**
@@ -131,7 +140,7 @@ class Parser {
    *      the expression, indicating that the expression is incomplete
    */
   complete() {
-    if (this._cursor && !states[this._state].completable) {
+    if (this._cursor && !states[this._state]!.completable) {
       throw new Error(`Unexpected end of expression: ${this._exprStr}`)
     }
     if (this._subParser) {
@@ -139,8 +148,10 @@ class Parser {
     }
 
     if (this._sequenceExpressions) {
-      this._sequenceExpressions.push(this._tree!)
-      const sequence: any = {
+      if (this._tree) {
+        this._sequenceExpressions.push(this._tree)
+      }
+      const sequence: SequenceExpression = {
         type: 'SequenceExpression',
         expressions: this._sequenceExpressions
       }
@@ -166,7 +177,7 @@ class Parser {
    * @private
    */
   _endSubExpression() {
-    states[this._state].subHandler!.call(this, this._subParser!.complete())
+    states[this._state]!.subHandler!.call(this, this._subParser!.complete())
     this._subParser = undefined
   }
 
@@ -181,7 +192,7 @@ class Parser {
     if (!this._cursor) {
       this._tree = node
     } else {
-      ;(this._cursor as any).right = node
+      this._cursor.right = node
       this._setParent(node, this._cursor)
     }
     this._cursor = node
@@ -223,11 +234,14 @@ class Parser {
    * @private
    */
   _startSubExpression(exprStr?: string) {
-    let endStates = states[this._state].endStates
+    let endStates = states[this._state]!.endStates
     if (!endStates) {
       this._parentStop = true
       endStates = this._stopMap
     }
+    if (states[this._state]!.completable && !endStates.semicolon) {
+      endStates = { ...endStates, semicolon: '_semicolon' }
+    }
     this._subParser = new Parser(this._grammar, this._lexer, exprStr, endStates)
   }
 }