@dxos/echo-query 0.8.4-main.a4bbb77 → 0.8.4-main.abd8ff62ef

package/src/parser/query-builder.ts

@@ -4,10 +4,14 @@
 
 import { type Parser, type Tree, type TreeCursor } from '@lezer/common';
 
-import { Filter } from '@dxos/echo';
+import { Filter, type Tag } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
 
 import { QueryDSL } from './gen';
 
+// TODO(burdon): Return Query AST.
+export type BuildResult = { filter?: Filter.Any; name?: string };
+
 /**
  * Stateless query builder that parses DSL trees into filters.
  *
@@ -15,14 +19,16 @@ import { QueryDSL } from './gen';
  * To modify the functionality, create a minimal breaking test and direct the LLM to fix either the grammar or builder.
  */
 export class QueryBuilder {
-  constructor(private readonly _parser: Parser = QueryDSL.Parser.configure({ strict: true })) {}
+  private readonly _parser: Parser = QueryDSL.Parser.configure({ strict: true });
+
+  constructor(private readonly _tags?: Tag.Map) {}
 
   /**
    * Check valid input.
    */
   validate(input: string): boolean {
     try {
-      const tree = this._parser.parse(input);
+      const tree = this._parser.parse(normalizeInput(input));
       return tree.cursor().node.name === 'Query';
     } catch {
       return false;
@@ -32,24 +38,25 @@ export class QueryBuilder {
   /**
    * Build a query from the input string.
    */
-  build(input: string): Filter.Any | null {
+  build(input: string): BuildResult {
     try {
-      const tree = this._parser.parse(input);
-      return this.buildQuery(tree, input);
+      const normalized = normalizeInput(input);
+      const tree = this._parser.parse(normalized);
+      return this.buildQuery(tree, normalized);
     } catch {
-      return null;
+      return {};
     }
   }
 
   /**
    * Build a query from a parsed DSL tree.
    */
-  buildQuery(tree: Tree, input: string): Filter.Any {
+  buildQuery(tree: Tree, input: string): BuildResult {
     const cursor = tree.cursor();
 
     // Start at root (Query node).
     if (cursor.node.name !== 'Query') {
-      return Filter.nothing();
+      return {};
     }
 
     // Check if Query has multiple children (binary expression).
@@ -61,26 +68,76 @@ export class QueryBuilder {
       cursor.parent();
     }
 
+    // Check if this is an assignment.
+    const hasAssignment = children.some((child) => child.name === 'Assignment');
+    if (hasAssignment) {
+      return this._parseAssignment(cursor, input);
+    }
+
     // If we have an operator in the children, or multiple expressions (implicit AND), parse as binary expression.
     const hasOperator = children.some((child) => child.name === 'And' || child.name === 'Or');
     const hasMultipleExpressions =
       children.filter((child) => child.name === 'Filter' || child.name === 'Not' || child.name === '(').length > 1;
     if (hasOperator || hasMultipleExpressions) {
-      return this._parseBinaryExpression(cursor, input);
+      const filter = this._parseBinaryExpression(cursor, input);
+      return { filter };
     }
 
     // Otherwise, parse the single expression.
     if (!cursor.firstChild()) {
-      return Filter.nothing();
+      return { filter: Filter.nothing() };
     }
 
-    return this._parseExpression(cursor, input);
+    const filter = this._parseExpression(cursor, input);
+    return { filter };
+  }
+
+  /**
+   * Parse an assignment node.
+   */
+  private _parseAssignment(cursor: TreeCursor, input: string): BuildResult {
+    if (!cursor.firstChild()) {
+      return {};
+    }
+
+    let name: string | undefined;
+    let filter: Filter.Any | undefined;
+
+    // Find the Assignment node
+    do {
+      if (cursor.node.name === 'Assignment') {
+        // Get the full assignment text first
+        const assignmentText = this._getNodeText(cursor, input);
+
+        if (cursor.firstChild()) {
+          // First child should be the variable name (Identifier)
+          name = this._getNodeText(cursor, input);
+
+          // Find the parentheses in the assignment text and extract the content
+          const openParenIndex = assignmentText.indexOf('(');
+          const closeParenIndex = assignmentText.lastIndexOf(')');
+
+          if (openParenIndex !== -1 && closeParenIndex !== -1 && closeParenIndex > openParenIndex) {
+            const subInput = assignmentText.slice(openParenIndex + 1, closeParenIndex).trim();
+            const subTree = this._parser.parse(subInput);
+            const subResult = this.buildQuery(subTree, subInput);
+            filter = subResult.filter;
+          }
+
+          cursor.parent(); // Back to Assignment
+        }
+        break;
+      }
+    } while (cursor.nextSibling());
+
+    cursor.parent(); // Back to Query
+    return { filter, name };
   }
 
   /**
    * Parse an expression node.
    */
-  private _parseExpression(cursor: TreeCursor, input: string): Filter.Any {
+  private _parseExpression(cursor: TreeCursor, input: string): Filter.Any | undefined {
     const nodeName = cursor.node.name;
 
     switch (nodeName) {
@@ -91,7 +148,7 @@ export class QueryBuilder {
         // Move past NOT token to the expression.
         cursor.nextSibling();
         const notFilter = this._parseExpression(cursor, input);
-        return Filter.not(notFilter);
+        return notFilter ? Filter.not(notFilter) : undefined;
       }
 
       case 'And':
@@ -189,8 +246,9 @@ export class QueryBuilder {
             let exprEnd = cursor.to;
 
             while (cursor.nextSibling() && depth > 0) {
-              if (cursor.node.name === '(') depth++;
-              else if (cursor.node.name === ')') {
+              if (cursor.node.name === '(') {
+                depth++;
+              } else if (cursor.node.name === ')') {
                 depth--;
                 if (depth === 0) {
                   exprEnd = cursor.from;
@@ -201,15 +259,25 @@ export class QueryBuilder {
             // Parse the expression inside parentheses as a subtree.
             const subInput = input.slice(exprStart, exprEnd);
             const subTree = this._parser.parse(subInput);
-            filters.push(this.buildQuery(subTree, subInput));
+            const subResult = this.buildQuery(subTree, subInput);
+            if (subResult.filter) {
+              filters.push(subResult.filter);
+            }
           } else {
             // Simple parenthesized expression.
-            filters.push(this._parseExpression(cursor, input));
+            const subFilter = this._parseExpression(cursor, input);
+            if (subFilter) {
+              filters.push(subFilter);
+            }
+
             // Skip until we find the closing parenthesis.
             while (cursor.nextSibling() && cursor.node.name !== ')') {}
           }
         } else if (nodeName !== ')') {
-          filters.push(this._parseExpression(cursor, input));
+          const subFilter = this._parseExpression(cursor, input);
+          if (subFilter) {
+            filters.push(subFilter);
+          }
         }
       } while (cursor.nextSibling());
 
@@ -230,17 +298,18 @@ export class QueryBuilder {
   /**
    * Parse a Filter node.
    */
-  private _parseFilter(cursor: TreeCursor, input: string): Filter.Any {
+  private _parseFilter(cursor: TreeCursor, input: string): Filter.Any | undefined {
     if (!cursor.firstChild()) {
       return Filter.nothing();
     }
 
+    let result: Filter.Any | undefined = undefined;
     const filterType = cursor.node.name;
-    let result: Filter.Any;
-
     switch (filterType) {
       case 'TagFilter':
-        result = this._parseTagFilter(cursor, input);
+        if (this._tags) {
+          result = this._parseTagFilter(cursor, input);
+        }
         break;
 
       case 'TextFilter':
@@ -288,8 +357,8 @@ export class QueryBuilder {
     cursor.firstChild(); // Move to String node.
     const text = this._getNodeText(cursor, input);
     cursor.parent(); // Go back to TextFilter.
-    // Remove quotes.
-    return Filter.text(text.slice(1, -1));
+    // Remove quotes and decode escapes.
+    return Filter.text(unescapeStringLiteral(text.slice(1, -1)));
   }
 
   /**
@@ -397,9 +466,11 @@ export class QueryBuilder {
   /**
    * Parse a TagFilter node (#tag).
    */
-  private _parseTagFilter(cursor: TreeCursor, input: string): Filter.Any {
-    const tag = this._getNodeText(cursor, input);
-    return Filter.tag(tag.slice(1));
+  private _parseTagFilter(cursor: TreeCursor, input: string): Filter.Any | undefined {
+    invariant(this._tags);
+    const str = this._getNodeText(cursor, input).slice(1).toLowerCase();
+    const [key] = Object.entries(this._tags!).find(([, value]) => value.label.toLowerCase() === str) ?? [];
+    return key ? Filter.tag(key) : undefined;
   }
 
   /**
@@ -410,9 +481,9 @@ export class QueryBuilder {
 
     switch (valueType) {
       case 'String': {
-        // Remove quotes.
+        // Remove quotes and decode escapes.
         const str = this._getNodeText(cursor, input);
-        return str.slice(1, -1);
+        return unescapeStringLiteral(str.slice(1, -1));
       }
 
       case 'Number':
@@ -468,3 +539,261 @@ export class QueryBuilder {
     return input.slice(cursor.from, cursor.to);
   }
 }
+
+const KEYWORDS = new Set(['AND', 'OR', 'NOT']);
+const VALUE_LITERALS = new Set(['true', 'false', 'null']);
+const SPECIAL_CHARS = /[\s(){}\[\],"']/;
+const PROPERTY_KEY = /^[a-zA-Z_][a-zA-Z0-9_.]*$/;
+const NUMBER_LITERAL = /^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$/;
+
+// Common URL schemes — `http://...`, `mailto:rich@...` etc. should be searched as text,
+// not auto-promoted to property filters.
+const URL_SCHEMES = new Set([
+  'http',
+  'https',
+  'ftp',
+  'ftps',
+  'file',
+  'mailto',
+  'tel',
+  'sms',
+  'data',
+  'javascript',
+  'ws',
+  'wss',
+  'ssh',
+  'git',
+]);
+
+/**
+ * Normalize raw user input into a form the lezer grammar can parse.
+ * - Bare text fragments (e.g. `foo`) are wrapped in quotes so they parse as TextFilter.
+ * - Property values that aren't already quoted/numeric/boolean (e.g. `from:rich@dxos.org`) are quoted.
+ * - Tags, type filters, operators, parens, braces, quoted strings, and assignments pass through unchanged.
+ */
+export const normalizeInput = (input: string): string => {
+  const out: string[] = [];
+  let pos = 0;
+  while (pos < input.length) {
+    const currentChar = input[pos];
+
+    // Whitespace.
+    if (/\s/.test(currentChar)) {
+      out.push(currentChar);
+      pos++;
+      continue;
+    }
+
+    // Quoted string (already a valid TextFilter / Value).
+    // Single-quote only opens a string if the previous char isn't a word-char,
+    // so apostrophes inside barewords (e.g. `don't`, `O'Connor`) stay attached.
+    const prevChar = pos > 0 ? input[pos - 1] : '';
+    const isStringOpener =
+      currentChar === '"' || (currentChar === "'" && (pos === 0 || !/[a-zA-Z0-9_]/.test(prevChar)));
+    if (isStringOpener) {
+      const quoteChar = currentChar;
+      let scanIndex = pos + 1;
+      while (scanIndex < input.length && input[scanIndex] !== quoteChar) {
+        if (input[scanIndex] === '\\' && scanIndex + 1 < input.length) {
+          scanIndex += 2;
+        } else {
+          scanIndex++;
+        }
+      }
+      out.push(input.slice(pos, Math.min(scanIndex + 1, input.length)));
+      pos = scanIndex + 1;
+      continue;
+    }
+
+    // Object/array literals: pass through (contents already structured).
+    if (currentChar === '{' || currentChar === '[') {
+      const closeChar = currentChar === '{' ? '}' : ']';
+      let depth = 1;
+      let scanIndex = pos + 1;
+      while (scanIndex < input.length && depth > 0) {
+        const innerChar = input[scanIndex];
+        if (innerChar === '"' || innerChar === "'") {
+          const quoteChar = innerChar;
+          scanIndex++;
+          while (scanIndex < input.length && input[scanIndex] !== quoteChar) {
+            if (input[scanIndex] === '\\' && scanIndex + 1 < input.length) {
+              scanIndex += 2;
+            } else {
+              scanIndex++;
+            }
+          }
+          scanIndex++;
+        } else if (innerChar === currentChar) {
+          depth++;
+          scanIndex++;
+        } else if (innerChar === closeChar) {
+          depth--;
+          scanIndex++;
+        } else {
+          scanIndex++;
+        }
+      }
+      out.push(input.slice(pos, scanIndex));
+      pos = scanIndex;
+      continue;
+    }
+
+    // Single-char structural tokens. `}` / `]` only reach here when unmatched —
+    // pass them through so the lezer parser can produce a clear error rather than spinning.
+    if (
+      currentChar === '(' ||
+      currentChar === ')' ||
+      currentChar === '=' ||
+      currentChar === ',' ||
+      currentChar === '}' ||
+      currentChar === ']'
+    ) {
+      out.push(currentChar);
+      pos++;
+      continue;
+    }
+
+    // Relations.
+    if ((currentChar === '-' && input[pos + 1] === '>') || (currentChar === '<' && input[pos + 1] === '-')) {
+      out.push(input.slice(pos, pos + 2));
+      pos += 2;
+      continue;
+    }
+
+    // NOT prefix (`!`) — single token.
+    if (currentChar === '!') {
+      out.push(currentChar);
+      pos++;
+      continue;
+    }
+
+    // Tag.
+    if (currentChar === '#') {
+      let scanIndex = pos + 1;
+      while (scanIndex < input.length && /[a-zA-Z0-9_-]/.test(input[scanIndex])) {
+        scanIndex++;
+      }
+      out.push(input.slice(pos, scanIndex));
+      pos = scanIndex;
+      continue;
+    }
+
+    // Bareword: scan until next whitespace or special char.
+    let scanIndex = pos;
+    while (scanIndex < input.length) {
+      const innerChar = input[scanIndex];
+      if (innerChar === '"') {
+        break;
+      }
+      if (innerChar === "'" && scanIndex > pos && !/[a-zA-Z0-9_]/.test(input[scanIndex - 1])) {
+        break;
+      }
+      if (innerChar === "'" && scanIndex === pos) {
+        break;
+      }
+      if (innerChar !== "'" && SPECIAL_CHARS.test(innerChar)) {
+        break;
+      }
+      if (innerChar === '-' && input[scanIndex + 1] === '>') {
+        break;
+      }
+      if (innerChar === '<' && input[scanIndex + 1] === '-') {
+        break;
+      }
+      scanIndex++;
+    }
+    // Defensive: if no characters were consumed, advance one to avoid infinite loops.
+    if (scanIndex === pos) {
+      out.push(currentChar);
+      pos++;
+      continue;
+    }
+    const token = input.slice(pos, scanIndex);
+    pos = scanIndex;
+
+    // Operators.
+    if (KEYWORDS.has(token.toUpperCase())) {
+      out.push(token);
+      continue;
+    }
+
+    // Property/type filter (`key:value`).
+    const colonIdx = token.indexOf(':');
+    if (colonIdx > 0) {
+      const key = token.slice(0, colonIdx);
+      const rest = token.slice(colonIdx + 1);
+
+      // type:typename — leave for grammar's TypeFilter (Identifier value).
+      if (key === 'type') {
+        out.push(token);
+        continue;
+      }
+
+      // URLs (`http://...`, `mailto:rich@...`) and URL-paths starting with `//`
+      // are searched as text rather than auto-promoted to property filters.
+      const isUrlScheme = URL_SCHEMES.has(key.toLowerCase()) || rest.startsWith('//');
+      if (!isUrlScheme && PROPERTY_KEY.test(key)) {
+        if (rest.length === 0) {
+          // Trailing colon while typing — pass through.
+          out.push(token);
+          continue;
+        }
+        const firstValueChar = rest[0];
+        if (firstValueChar === '"' || firstValueChar === "'") {
+          // Already quoted.
+          out.push(token);
+          continue;
+        }
+        if (firstValueChar === '{' || firstValueChar === '[') {
+          // Object/array literal value.
+          out.push(token);
+          continue;
+        }
+        if (VALUE_LITERALS.has(rest) || NUMBER_LITERAL.test(rest)) {
+          // Boolean / null / number literal.
+          out.push(token);
+          continue;
+        }
+        out.push(`${key}:"${escapeStringLiteral(rest)}"`);
+        continue;
+      }
+
+      // URL or unknown key shape — fall through to text.
+    }
+
+    // Identifier followed by `=` is the LHS of an Assignment — keep as-is.
+    if (PROPERTY_KEY.test(token)) {
+      let lookahead = pos;
+      while (lookahead < input.length && /\s/.test(input[lookahead])) {
+        lookahead++;
+      }
+      if (input[lookahead] === '=') {
+        out.push(token);
+        continue;
+      }
+    }
+
+    // Bare text fragment: quote so it parses as TextFilter.
+    out.push(`"${escapeStringLiteral(token)}"`);
+  }
+
+  return out.join('');
+};
+
+/** Escape a raw value into a string literal body (without surrounding quotes). */
+const escapeStringLiteral = (value: string): string => value.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+
+/** Decode `\\` and `\"` escapes inside a string literal body. */
+const unescapeStringLiteral = (literalBody: string): string => {
+  let out = '';
+  for (let i = 0; i < literalBody.length; i++) {
+    const ch = literalBody[i];
+    if (ch === '\\' && i + 1 < literalBody.length) {
+      out += literalBody[i + 1];
+      i++;
+    } else {
+      out += ch;
+    }
+  }
+  return out;
+};

package/src/parser/query.grammar

@@ -5,6 +5,7 @@
 @top Query { expression }
 
 expression {
+  Assignment |
   Filter |
   !Not Not expression |
   expression !ImplicitAnd expression |
@@ -14,6 +15,10 @@ expression {
   "(" expression ")"
 }
 
+Assignment {
+  Identifier "=" "(" expression ")"
+}
+
 Not { 
   @specialize<Identifier, "NOT" | "not" | "!">
 }
@@ -110,7 +115,7 @@ Value {
   space { @whitespace+ }
 
   "{" "}" "[" "]" "(" ")"
-  ":" ","  "."
+  ":" ","  "." "="
 }
 
 @skip { space }
@@ -120,5 +125,6 @@ Value {
   ImplicitAnd @left,
   And @left,
   Or @left,
-  Relation @left
+  Relation @left,
+  Assignment @right
 }