@travetto/model-query-language 5.0.0-rc.9

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 ArcSine Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,43 @@
+<!-- This file was generated by @travetto/doc and should not be modified directly -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/model-query-language/DOC.tsx and execute "npx trv doc" to rebuild -->
+# Data Model Query Language
+
+## Datastore query language.
+
+**Install: @travetto/model-query-language**
+```bash
+npm install @travetto/model-query-language
+
+# or
+
+yarn add @travetto/model-query-language
+```
+
+This module provides a textual query language for the [Data Model Querying](https://github.com/travetto/travetto/tree/main/module/model-query#readme "Datastore abstraction for advanced query support.") interface. The language itself is fairly simple, boolean logic, with parenthetical support.The operators supported are:
+
+## Query Language
+
+   *  `<`, `<=` - Less than, and less than or equal to
+   *  `>`, `>=` - Greater than, and greater than or equal to
+   *  `!=`, `==` - Not equal to, and equal to
+   *  `~` - Matches regular expression, supports the `i` flag to trigger case insensitive searches
+   *  `!`, `not` - Negates a clause
+   *  `in`, `not-in` - Supports checking if a field is in a list of literal values
+   *  `and`, `&&` - Intersection of clauses
+   *  `or`, `||` - Union of clauses
+All sub fields are dot separated for access, e.g. `user.address.city`.A query language version of the previous query could look like:
+
+**Code: Query language with boolean checks and exists check**
+```sql
+not (age < 35) and contact != null
+```
+
+A more complex query would look like:
+
+**Code: Query language with more complex needs**
+```sql
+user.role in ['admin', 'root'] && (user.address.state == 'VA' || user.address.city == 'Springfield')
+```
+
+### Regular Expression
+When querying with regular expressions, patterns can be specified as `'strings'` or as `/patterns/`.  The latter allows for the case insensitive modifier: `/pattern/i`.  Supporting the insensitive flag is up to the underlying model implementation.

package/__index__.ts

@@ -0,0 +1 @@
+export * from './src/parser';

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@travetto/model-query-language",
+  "version": "5.0.0-rc.9",
+  "description": "Datastore query language.",
+  "keywords": [
+    "datastore",
+    "schema",
+    "typescript",
+    "travetto",
+    "query-language"
+  ],
+  "homepage": "https://travetto.io",
+  "license": "MIT",
+  "author": {
+    "email": "travetto.framework@gmail.com",
+    "name": "Travetto Framework"
+  },
+  "files": [
+    "__index__.ts",
+    "src",
+    "support"
+  ],
+  "main": "__index__.ts",
+  "repository": {
+    "url": "https://github.com/travetto/travetto.git",
+    "directory": "module/model-query"
+  },
+  "dependencies": {
+    "@travetto/model-query": "^5.0.0-rc.9",
+    "@travetto/model": "^5.0.0-rc.9",
+    "@travetto/schema": "^5.0.0-rc.9"
+  },
+  "peerDependencies": {
+    "@travetto/test": "^5.0.0-rc.9"
+  },
+  "peerDependenciesMeta": {
+    "@travetto/test": {
+      "optional": true
+    }
+  },
+  "travetto": {
+    "displayName": "Data Model Query Language"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/parser.ts

@@ -0,0 +1,198 @@
+import { castTo } from '@travetto/runtime';
+import { WhereClauseRaw } from '@travetto/model-query';
+import { QueryLanguageTokenizer } from './tokenizer';
+import { Token, Literal, GroupNode, OP_TRANSLATION, ArrayNode, AllNode } from './types';
+
+/**
+ * Determine if a token is boolean
+ */
+function isBoolean(o: unknown): o is Token & { type: 'boolean' } {
+  return !!o && typeof o === 'object' && 'type' in o && o.type === 'boolean';
+}
+
+/**
+ * Language parser
+ */
+export class QueryLanguageParser {
+
+  /**
+   * Handle all clauses
+   */
+  static handleClause(nodes: (AllNode | Token)[]): void {
+    const val: Token | ArrayNode = castTo(nodes.pop());
+    const op: Token & { value: string } = castTo(nodes.pop());
+    const ident: Token & { value: string } = castTo(nodes.pop());
+
+    // value isn't a literal or a list, bail
+    if (val.type !== 'literal' && val.type !== 'list') {
+      throw new Error(`Unexpected token: ${val.value}`);
+    }
+
+    // If operator is not an operator, bail
+    if (op.type !== 'operator') {
+      throw new Error(`Unexpected token: ${op.value}`);
+    }
+
+    // If operator is not known, bail
+    const finalOp = OP_TRANSLATION[op.value];
+
+    if (!finalOp) {
+      throw new Error(`Unexpected operator: ${op.value}`);
+    }
+
+    nodes.push({
+      type: 'clause',
+      field: ident.value,
+      op: finalOp,
+      value: val.value
+    });
+
+    // Handle unary support
+    this.unary(nodes);
+    // Simplify as we go along
+    this.condense(nodes, 'and');
+  }
+
+  /**
+   * Condense nodes to remove unnecessary groupings
+   * (a AND (b AND (c AND d))) => (a AND b AND c)
+   */
+  static condense(nodes: (AllNode | Token)[], op: 'and' | 'or'): void {
+    let second = nodes[nodes.length - 2];
+
+    while (isBoolean(second) && second.value === op) {
+      const right: AllNode = castTo(nodes.pop());
+      nodes.pop()!;
+      const left: AllNode = castTo(nodes.pop());
+      const rg: GroupNode = castTo(right);
+      if (rg.type === 'group' && rg.op === op) {
+        rg.value.unshift(left);
+        nodes.push(rg);
+      } else {
+        nodes.push({
+          type: 'group',
+          op,
+          value: [left, right]
+        });
+      }
+      second = nodes[nodes.length - 2];
+    }
+  }
+
+  /**
+   * Remove unnecessary unary nodes
+   * (((5))) => 5
+   */
+  static unary(nodes: (AllNode | Token)[]): void {
+    const second = nodes[nodes.length - 2];
+    if (second && second.type === 'unary' && second.value === 'not') {
+      const node = nodes.pop();
+      nodes.pop();
+      nodes.push({
+        type: 'unary',
+        op: 'not',
+        value: castTo(node)
+      });
+    }
+  }
+
+  /**
+   * Parse all tokens
+   */
+  static parse(tokens: Token[], pos: number = 0): AllNode {
+
+    let top: (AllNode | Token)[] = [];
+    const stack: (typeof top)[] = [top];
+    let arr: Literal[] | undefined;
+
+    let token = tokens[pos];
+    while (token) {
+      switch (token.type) {
+        case 'grouping':
+          if (token.value === 'start') {
+            stack.push(top = []);
+          } else {
+            const group = stack.pop()!;
+            top = stack[stack.length - 1];
+            this.condense(group, 'or');
+            top.push(group[0]);
+            this.unary(top);
+            this.condense(top, 'and');
+          }
+          break;
+        case 'array':
+          if (token.value === 'start') {
+            arr = [];
+          } else {
+            const arrNode: ArrayNode = { type: 'list', value: arr! };
+            top.push(arrNode);
+            arr = undefined;
+            this.handleClause(top);
+          }
+          break;
+        case 'literal':
+          if (arr !== undefined) {
+            arr.push(token.value);
+          } else {
+            top.push(token);
+            this.handleClause(top);
+          }
+          break;
+        case 'punctuation':
+          if (!arr) {
+            throw new Error(`Invalid token: ${token.value}`);
+          }
+          break;
+        default:
+          top.push(token);
+      }
+      token = tokens[++pos];
+    }
+
+    this.condense(top, 'or');
+
+    return castTo(top[0]);
+  }
+
+  /**
+   * Convert Query AST to output
+   */
+  static convert<T = unknown>(node: AllNode): WhereClauseRaw<T> {
+    switch (node.type) {
+      case 'unary': {
+        return castTo({ [`$${node.op!}`]: this.convert(node.value) });
+      }
+      case 'group': {
+        return castTo({ [`$${node.op!}`]: node.value.map(x => this.convert(x)) });
+      }
+      case 'clause': {
+        const parts = node.field!.split('.');
+        const top: WhereClauseRaw<T> = {};
+        let sub: Record<string, unknown> = top;
+        for (const p of parts) {
+          sub = sub[p] = {};
+        }
+        if (node.op === '$regex' && typeof node.value === 'string') {
+          sub[node.op!] = new RegExp(`^${node.value}`);
+        } else if ((node.op === '$eq' || node.op === '$ne') && node.value === null) {
+          sub.$exists = node.op !== '$eq';
+        } else if ((node.op === '$in' || node.op === '$nin') && !Array.isArray(node.value)) {
+          throw new Error(`Expected array literal for ${node.op}`);
+        } else {
+          sub[node.op!] = node.value;
+        }
+        return top;
+      }
+      default: throw new Error(`Unexpected node type: ${node.type}`);
+    }
+  }
+
+  /**
+   * Tokenize and parse text
+   */
+  static parseToQuery<T = unknown>(text: string): WhereClauseRaw<T> {
+    const tokens = QueryLanguageTokenizer.tokenize(text);
+    const node = this.parse(tokens);
+    return this.convert(node);
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,186 @@
+import { TimeUtil } from '@travetto/runtime';
+import { Token, TokenizeState, TokenType } from './types';
+
+const OPEN_PARENS = 0x28, CLOSE_PARENS = 0x29, OPEN_BRACKET = 0x5b, CLOSE_BRACKET = 0x5d, COMMA = 0x2c;
+const GREATER_THAN = 0x3e, LESS_THAN = 0x3c, EQUAL = 0x3d, NOT = 0x21, MODULO = 0x25, TILDE = 0x7e, AND = 0x26, OR = 0x7c;
+const SPACE = 0x20, TAB = 0x09;
+const DBL_QUOTE = 0x22, SGL_QUOTE = 0x27, FORWARD_SLASH = 0x2f, BACKSLASH = 0x5c;
+const PERIOD = 0x2e, UNDERSCORE = 0x54, DOLLAR_SIGN = 0x24, DASH = 0x2d;
+const ZERO = 0x30, NINE = 0x39, UPPER_A = 0x41, UPPER_Z = 0x5a, LOWER_A = 0x61, LOWER_Z = 0x7a;
+const LOWER_I = 0x69, LOWER_G = 0x67, LOWER_M = 0x6d, LOWER_S = 0x73;
+
+const ESCAPE: Record<string, string> = {
+  '\\n': '\n',
+  '\\r': '\r',
+  '\\t': '\t',
+  '\\"': '"',
+  "\\'": "'"
+};
+
+/**
+ * Mapping of keywords to node types and values
+ */
+const TOKEN_MAPPING: Record<string, Token> = {
+  and: { type: 'boolean', value: 'and' },
+  '&&': { type: 'boolean', value: 'and' },
+  or: { type: 'boolean', value: 'or' },
+  '||': { type: 'boolean', value: 'or' },
+  in: { type: 'operator', value: 'in' },
+  ['not-in']: { type: 'operator', value: 'not-in' },
+  not: { type: 'unary', value: 'not' },
+  '[': { type: 'array', value: 'start' },
+  ']': { type: 'array', value: 'end' },
+  '(': { type: 'grouping', value: 'start' },
+  ')': { type: 'grouping', value: 'end' },
+  null: { type: 'literal', value: null },
+  true: { type: 'literal', value: true },
+  false: { type: 'literal', value: false },
+};
+
+/**
+ * Tokenizer for the query language
+ */
+export class QueryLanguageTokenizer {
+
+  /**
+   * Process the next token.  Can specify expected type as needed
+   */
+  static #processToken(state: TokenizeState, mode?: TokenType): Token {
+    const text = state.text.substring(state.start, state.pos);
+    const res = TOKEN_MAPPING[text.toLowerCase()];
+    let value: unknown = text;
+    if (!res && state.mode === 'literal') {
+      if (/^["']/.test(text)) {
+        value = text.substring(1, text.length - 1)
+          .replace(/\\[.]/g, (a, b) => ESCAPE[a] || b);
+      } else if (/^\//.test(text)) {
+        const start = 1;
+        const end = text.lastIndexOf('/');
+        value = new RegExp(text.substring(start, end), text.substring(end + 1));
+      } else if (/^-?\d+$/.test(text)) {
+        value = parseInt(text, 10);
+      } else if (/^-?\d+[.]\d+$/.test(text)) {
+        value = parseFloat(text);
+      } else if (TimeUtil.isTimeSpan(text)) {
+        value = text;
+      } else {
+        state.mode = 'identifier';
+      }
+    }
+    return res ?? { value, type: state.mode || mode };
+  }
+
+  /**
+   * Flush state to output
+   */
+  static #flush(state: TokenizeState, mode?: TokenType): void {
+    if ((!mode || !state.mode || mode !== state.mode) && state.start !== state.pos) {
+      if (state.mode !== 'whitespace') {
+        state.out.push(this.#processToken(state, mode));
+      }
+      state.start = state.pos;
+    }
+    state.mode = mode || state.mode;
+  }
+
+  /**
+   * Determine if valid regex flag
+   */
+  static #isValidRegexFlag(ch: number): boolean {
+    return ch === LOWER_I || ch === LOWER_G || ch === LOWER_M || ch === LOWER_S;
+  }
+
+  /**
+   * Determine if valid token identifier
+   */
+  static #isValidIdentToken(ch: number): boolean {
+    return (ch >= ZERO && ch <= NINE) ||
+      (ch >= UPPER_A && ch <= UPPER_Z) ||
+      (ch >= LOWER_A && ch <= LOWER_Z) ||
+      (ch === UNDERSCORE) ||
+      (ch === DASH) ||
+      (ch === DOLLAR_SIGN) ||
+      (ch === PERIOD);
+  }
+
+  /**
+   * Read string until quote
+   */
+  static readString(text: string, pos: number): number {
+    const len = text.length;
+    const ch = text.charCodeAt(pos);
+    const q = ch;
+    pos += 1;
+    while (pos < len) {
+      if (text.charCodeAt(pos) === q) {
+        break;
+      } else if (text.charCodeAt(pos) === BACKSLASH) {
+        pos += 1;
+      }
+      pos += 1;
+    }
+    if (pos === len && text.charCodeAt(pos) !== q) {
+      throw new Error('Unterminated string literal');
+    }
+    return pos;
+  }
+
+  /**
+   * Tokenize a text string
+   */
+  static tokenize(text: string): Token[] {
+    const state: TokenizeState = {
+      out: [],
+      pos: 0,
+      start: 0,
+      text,
+      mode: undefined!
+    };
+    const len = text.length;
+    // Loop through each char
+    while (state.pos < len) {
+      // Read code as a number, more efficient
+      const ch = text.charCodeAt(state.pos);
+      switch (ch) {
+        // Handle punctuation
+        case OPEN_PARENS: case CLOSE_PARENS: case OPEN_BRACKET: case CLOSE_BRACKET: case COMMA:
+          this.#flush(state);
+          state.mode = 'punctuation';
+          break;
+        // Handle operator
+        case GREATER_THAN: case LESS_THAN: case EQUAL:
+        case MODULO: case NOT: case TILDE: case AND: case OR:
+          this.#flush(state, 'operator');
+          break;
+        // Handle whitespace
+        case SPACE: case TAB:
+          this.#flush(state, 'whitespace');
+          break;
+        // Handle quotes and slashes
+        case DBL_QUOTE: case SGL_QUOTE: case FORWARD_SLASH:
+          this.#flush(state);
+          state.mode = 'literal';
+          state.pos = this.readString(text, state.pos) + 1;
+          if (ch === FORWARD_SLASH) { // Read modifiers, not used by all, but useful in general
+            while (this.#isValidRegexFlag(text.charCodeAt(state.pos))) {
+              state.pos += 1;
+            }
+          }
+          this.#flush(state);
+          continue;
+        // Handle literal
+        default:
+          if (this.#isValidIdentToken(ch)) {
+            this.#flush(state, 'literal');
+          } else {
+            throw new Error(`Invalid character: ${text.substring(Math.max(0, state.pos - 10), state.pos + 1)}`);
+          }
+      }
+      state.pos += 1;
+    }
+
+    this.#flush(state);
+
+    return state.out;
+  }
+}

package/src/types.ts

@@ -0,0 +1,86 @@
+/**
+ * Supported token types
+ */
+export type TokenType =
+  'literal' | 'identifier' | 'boolean' |
+  'operator' | 'grouping' | 'array' |
+  'whitespace' | 'punctuation' | 'unary';
+
+/**
+ * Tokenization state
+ */
+export interface TokenizeState {
+  out: Token[];
+  pos: number;
+  start: number;
+  text: string;
+  mode: TokenType;
+}
+
+/**
+ * Literal types
+ */
+export type Literal = boolean | null | string | number | RegExp | Date;
+
+/**
+ * Token
+ */
+export interface Token {
+  type: TokenType;
+  value: Literal;
+}
+
+/**
+ * Base AST Node
+ */
+export interface Node<T extends string = string> {
+  type: T;
+}
+
+/**
+ * Simple clause
+ */
+export interface ClauseNode extends Node<'clause'> {
+  field?: string;
+  op?: string;
+  value?: Literal | Literal[];
+}
+
+/**
+ * Grouping
+ */
+export interface GroupNode extends Node<'group'> {
+  op?: 'and' | 'or';
+  value: AllNode[];
+}
+
+/**
+ * Unary node
+ */
+export interface UnaryNode extends Node<'unary'> {
+  op?: 'not';
+  value: AllNode;
+}
+
+/**
+ * Array node
+ */
+export interface ArrayNode extends Node<'list'> {
+  op?: 'not';
+  value: Literal[];
+}
+
+export type AllNode = ArrayNode | UnaryNode | GroupNode | ClauseNode;
+
+/**
+ * Translation of operators to model query keys
+ */
+export const OP_TRANSLATION: Record<string, string> = {
+  '<': '$lt', '<=': '$lte',
+  '>': '$gt', '>=': '$gte',
+  '!=': '$ne', '==': '$eq',
+  '~': '$regex', '!': '$not',
+  in: '$in', 'not-in': '$nin'
+};
+
+export const VALID_OPS = new Set(Object.keys(OP_TRANSLATION));