@tellaro/tql 1.4.3 → 2.0.0

package/dist/src/semanticPreview.d.ts

@@ -11,6 +11,28 @@ export declare class SemanticPreview {
     private extractFromBinary;
     private extractFromUnary;
     private extractFromGroup;
+    /**
+     * `field [not] ANY value` -- the collection operator IS the operator this preview reports.
+     *
+     * Recursing straight into `expr.condition` (what this did) hands the wrapped comparison to
+     * `extractFromComparison`, which reports the INNER operator -- `eq` by default, because
+     * both grammars default the optional inner operator to `eq`. So `f any true`, `f not any
+     * true` and `f none true` all reported `eq`: the collection operator vanished and, on the
+     * negated spellings, the preview stated the logical opposite of the query.
+     */
+    private extractFromCollection;
+    /**
+     * The operator string a collection expression renders as.
+     *
+     * The shape is the one `not contains` and `not in` already use: negation lives in the
+     * operator NAME (`not_contains`, `not_in`), not in a separate flag the formatter has to
+     * remember to read. `collectionOperatorSpelling` derives `any` / `not_any` from the
+     * parser's own spelling table, so this cannot drift from what the parser accepts.
+     *
+     * An inner operator other than the implicit `eq` is appended (`any gt`), because
+     * `f any gt 5` and `f any 5` are different queries.
+     */
+    private collectionOperatorLabel;
     private extractFromComparison;
     /**
      * Get field name and mutators from field reference or mutator expression
@@ -31,6 +53,7 @@ export declare class SemanticPreview {
     private buildBinaryNode;
     private buildUnaryNode;
     private buildComparisonNode;
+    private buildCollectionNode;
     private buildGroupNode;
 }
 /**

package/dist/src/semanticPreview.js

@@ -1,11 +1,11 @@
-import { parseTql as f } from "./parser.js";
+import { parseTql as f, collectionOperatorSpelling as g } from "./parser.js";
 import { analyzeComplexity as m } from "./complexityAnalyzer.js";
-class u {
+class o {
   /**
    * Generate a complete preview for a TQL query
    */
   generatePreview(t) {
-    const r = f(t), i = m(t), n = [], s = {};
+    const r = f(t), i = m(t), n = [], c = {};
     if (!r.ast)
       return {
         fields: [],
@@ -15,15 +15,15 @@ class u {
         uniqueFields: [],
         operatorSummary: {}
       };
-    this.extractFieldUsage(r.ast, n, s);
-    const a = this.buildLogicTree(r.ast), c = [...new Set(n.map((o) => o.name))];
+    this.extractFieldUsage(r.ast, n, c);
+    const a = this.buildLogicTree(r.ast), s = [...new Set(n.map((u) => u.name))];
     return {
       fields: n,
       logicTree: a,
       complexity: i,
       fieldCount: n.length,
-      uniqueFields: c,
-      operatorSummary: s
+      uniqueFields: s,
+      operatorSummary: c
     };
   }
   /**
@@ -40,6 +40,12 @@ class u {
       case "comparison":
         this.extractFromComparison(t, r, i);
         break;
+      // `field ANY value` -- the field and operator this preview describes are inside the
+      // wrapped comparison. Without this arm the `default` below swallowed it and the
+      // preview reported a collection query as touching no fields at all.
+      case "collection":
+        this.extractFromCollection(t, r, i);
+        break;
       case "group":
         this.extractFromGroup(t, r, i);
         break;
@@ -54,16 +60,60 @@ class u {
   extractFromGroup(t, r, i) {
     this.extractFieldUsage(t.expression, r, i);
   }
+  /**
+   * `field [not] ANY value` -- the collection operator IS the operator this preview reports.
+   *
+   * Recursing straight into `expr.condition` (what this did) hands the wrapped comparison to
+   * `extractFromComparison`, which reports the INNER operator -- `eq` by default, because
+   * both grammars default the optional inner operator to `eq`. So `f any true`, `f not any
+   * true` and `f none true` all reported `eq`: the collection operator vanished and, on the
+   * negated spellings, the preview stated the logical opposite of the query.
+   */
+  extractFromCollection(t, r, i) {
+    if (t.condition.type !== "comparison") {
+      this.extractFieldUsage(t.condition, r, i);
+      return;
+    }
+    const n = t.condition, c = this.collectionOperatorLabel(t, n);
+    i[c] = (i[c] || 0) + 1;
+    const a = this.getFieldInfo(n.field);
+    r.push({
+      name: a.name,
+      operator: c,
+      value: this.extractValue(n.value),
+      // The inner operator decides HOW each element is compared; the collection operator
+      // decides WHICH elements have to match. `f any gt 5` is a range match over elements.
+      matchType: this.determineMatchType(n.operator.toLowerCase(), n.value),
+      mutators: a.mutators,
+      start: t.start,
+      end: t.end
+    });
+  }
+  /**
+   * The operator string a collection expression renders as.
+   *
+   * The shape is the one `not contains` and `not in` already use: negation lives in the
+   * operator NAME (`not_contains`, `not_in`), not in a separate flag the formatter has to
+   * remember to read. `collectionOperatorSpelling` derives `any` / `not_any` from the
+   * parser's own spelling table, so this cannot drift from what the parser accepts.
+   *
+   * An inner operator other than the implicit `eq` is appended (`any gt`), because
+   * `f any gt 5` and `f any 5` are different queries.
+   */
+  collectionOperatorLabel(t, r) {
+    const i = g(t.operator, t.negated), n = r.operator.toLowerCase();
+    return n === "eq" ? i : `${i} ${n}`;
+  }
   extractFromComparison(t, r, i) {
     const n = t.operator.toLowerCase();
     i[n] = (i[n] || 0) + 1;
-    const s = this.getFieldInfo(t.field), a = this.determineMatchType(n, t.value), c = this.extractValue(t.value);
+    const c = this.getFieldInfo(t.field), a = this.determineMatchType(n, t.value), s = this.extractValue(t.value);
     r.push({
-      name: s.name,
+      name: c.name,
       operator: n,
-      value: c,
+      value: s,
       matchType: a,
-      mutators: s.mutators,
+      mutators: c.mutators,
       start: t.start,
       end: t.end
     });
@@ -123,6 +173,12 @@ class u {
         return this.buildUnaryNode(t);
       case "comparison":
         return this.buildComparisonNode(t);
+      // `field ANY value` -- one leaf, carrying the COLLECTION operator. Returning
+      // `buildLogicTree(expr.condition)` (what this did) returned the wrapped comparison,
+      // whose operator is the inner `eq` -- so the preview rendered `f not any true` as
+      // `f eq true`, the logical opposite of the query it was describing.
+      case "collection":
+        return this.buildCollectionNode(t);
       case "group":
         return this.buildGroupNode(t);
       default:
@@ -164,6 +220,20 @@ class u {
       end: t.end
     };
   }
+  buildCollectionNode(t) {
+    if (t.condition.type !== "comparison")
+      return this.buildLogicTree(t.condition);
+    const r = t.condition, i = this.getFieldInfo(r.field);
+    return {
+      type: "comparison",
+      field: i.name,
+      operator: this.collectionOperatorLabel(t, r),
+      value: this.extractValue(r.value),
+      mutators: i.mutators,
+      start: t.start,
+      end: t.end
+    };
+  }
   buildGroupNode(t) {
     return {
       type: "group",
@@ -173,30 +243,30 @@ class u {
     };
   }
 }
+function b(e) {
+  return new o().generatePreview(e);
+}
 function T(e) {
-  return new u().generatePreview(e);
+  return new o().generatePreview(e).fields;
 }
-function v(e) {
-  return new u().generatePreview(e).fields;
+function F(e) {
+  return new o().generatePreview(e).logicTree;
 }
 function $(e) {
-  return new u().generatePreview(e).logicTree;
+  return new o().generatePreview(e).uniqueFields;
 }
-function b(e) {
-  return new u().generatePreview(e).uniqueFields;
-}
-function F(e) {
-  return new u().generatePreview(e).operatorSummary;
+function L(e) {
+  return new o().generatePreview(e).operatorSummary;
 }
 function h(e) {
   if (e.type === "comparison") {
-    const t = e.mutators?.length ? ` | ${e.mutators.join(" | ")}` : "", r = g(e.value);
+    const t = e.mutators?.length ? ` | ${e.mutators.join(" | ")}` : "", r = d(e.value);
     return `${e.field}${t} ${e.operator} ${r}`;
   }
   return e.type === "NOT" && e.children?.length === 1 ? `NOT (${h(e.children[0])})` : e.type === "group" && e.children?.length === 1 ? `(${h(e.children[0])})` : (e.type === "AND" || e.type === "OR") && e.children ? e.children.map(h).join(` ${e.type} `) : "";
 }
-function g(e) {
-  return e === null ? "null" : e === void 0 ? "" : typeof e == "string" ? `"${e}"` : Array.isArray(e) ? `[${e.map(g).join(", ")}]` : String(e);
+function d(e) {
+  return e === null ? "null" : e === void 0 ? "" : typeof e == "string" ? `"${e}"` : Array.isArray(e) ? `[${e.map(d).join(", ")}]` : String(e);
 }
 function N(e, t = "") {
   if (!e)
@@ -206,35 +276,35 @@ function N(e, t = "") {
 `);
 }
 function l(e, t, r, i) {
-  const n = i ? "└── " : "├── ", s = i ? "    " : "│   ";
+  const n = i ? "└── " : "├── ", c = i ? "    " : "│   ";
   if (e.type === "comparison") {
-    const a = e.mutators?.length ? ` | ${e.mutators.join(" | ")}` : "", c = g(e.value);
-    t.push(`${r}${n}${e.field}${a} ${e.operator} ${c}`);
-  } else e.type === "NOT" ? (t.push(`${r}${n}NOT`), e.children?.length && l(e.children[0], t, r + s, !0)) : e.type === "group" ? (t.push(`${r}${n}GROUP`), e.children?.length && l(e.children[0], t, r + s, !0)) : e.children && (t.push(`${r}${n}${e.type}`), e.children.forEach((a, c) => {
-    const o = c === e.children.length - 1;
-    l(a, t, r + s, o);
+    const a = e.mutators?.length ? ` | ${e.mutators.join(" | ")}` : "", s = d(e.value);
+    t.push(`${r}${n}${e.field}${a} ${e.operator} ${s}`);
+  } else e.type === "NOT" ? (t.push(`${r}${n}NOT`), e.children?.length && l(e.children[0], t, r + c, !0)) : e.type === "group" ? (t.push(`${r}${n}GROUP`), e.children?.length && l(e.children[0], t, r + c, !0)) : e.children && (t.push(`${r}${n}${e.type}`), e.children.forEach((a, s) => {
+    const u = s === e.children.length - 1;
+    l(a, t, r + c, u);
   }));
 }
-function d(e) {
-  return e ? e.type === "comparison" ? [e] : e.children ? e.children.flatMap(d) : [] : [];
-}
 function y(e) {
+  return e ? e.type === "comparison" ? [e] : e.children ? e.children.flatMap(y) : [] : [];
+}
+function p(e) {
   if (!e)
     return 0;
   if (!e.children || e.children.length === 0)
     return 1;
-  const t = e.children.map(y);
+  const t = e.children.map(p);
   return 1 + Math.max(...t);
 }
 export {
-  u as SemanticPreview,
-  d as flattenLogicTree,
+  o as SemanticPreview,
+  y as flattenLogicTree,
   N as formatLogicTreeAscii,
   h as formatLogicTreeNode,
-  T as generatePreview,
-  v as getFieldUsage,
-  $ as getLogicTree,
-  y as getLogicTreeDepth,
-  F as getOperatorSummary,
-  b as getUniqueFields
+  b as generatePreview,
+  T as getFieldUsage,
+  F as getLogicTree,
+  p as getLogicTreeDepth,
+  L as getOperatorSummary,
+  $ as getUniqueFields
 };

package/dist/src/tokenizer.d.ts

@@ -2,10 +2,25 @@ import { TqlAggregationFunction, TqlCollectionOperator, TqlComparisonOperator, T
 /**
  * All comparison operators supported by TQL.
  * Includes negated variants using both `not_` prefix and `!` prefix.
+ *
+ * Checked against both engines by `tests/catalogParity.spec.ts`, which derives the operator
+ * sets from `tql/src/parser/grammar.pest` and `src/tql/parser_components/grammar.py` at test
+ * time. Adding a name here that neither engine accepts fails that test.
  */
 export declare const COMPARISON_OPERATORS: TqlComparisonOperator[];
 /**
  * Symbol-based operators mapped to their word equivalents.
+ *
+ * `==` is deliberately absent. TQL's equality symbol is a SINGLE `=`
+ * (`eq_op = { ^"eq" | "=" }` in tql/src/parser/grammar.pest), and the Python parser
+ * rejects `==` outright with "Invalid operator '==' ... Use '=' for equality"
+ * (src/tql/parser_components/error_analyzer.py). Mapping it to `eq` here made the editor
+ * green-light a query that fails the moment it is run — the inverse, and worse, of a false
+ * red squiggle: the user gets no warning until execution.
+ *
+ * The tokenizer still lexes `==` as one operator token (see `scanSymbolOperator`) so the
+ * parser can name it in a single targeted error rather than splintering it into `=` + `=`
+ * and emitting a cascade.
  */
 export declare const SYMBOL_OPERATORS: Record<string, TqlComparisonOperator>;
 /**
@@ -13,23 +28,137 @@ export declare const SYMBOL_OPERATORS: Record<string, TqlComparisonOperator>;
  */
 export declare const NEGATION_SYMBOL_OPERATORS: Record<string, TqlComparisonOperator>;
 export declare const LOGICAL_OPERATORS: TqlLogicalOperator[];
+/**
+ * Symbolic spellings of the three logical operators, mapped to their canonical words.
+ *
+ * All three are declared in the engine grammars alongside the words, in the SAME rule:
+ * `and_op = @{ ^"and" ~ word_end | "&&" }`, `or_op = @{ ^"or" ~ word_end | "||" }` and
+ * `not_op = @{ ^"not" ~ word_end | "!" }` (`tql/src/parser/grammar.pest`); Python's
+ * `_setup_operators` builds the same alternations. So `a eq 1 && b eq 2` is not a
+ * convenience this package invented — it is one of two spellings of one operator, and both
+ * engines produce an IDENTICAL AST for the two.
+ *
+ * This package accepted neither. `&&` lexed as two `unknown` tokens and `||` as two `pipe`
+ * tokens, so a query both engines run was reported as `Unexpected token '&'` — a false red
+ * on syntax the product documents.
+ *
+ * Single `&` and single `|` are deliberately NOT here. Neither engine accepts them
+ * (`a eq 1 & b eq 2` raises `TQLSyntaxError at position 7` in Python, measured), and `|` is
+ * TQL's mutator pipe, so folding it into OR would silently reinterpret `f | lowercase`.
+ */
+export declare const SYMBOL_LOGICAL_OPERATORS: Record<string, TqlLogicalOperator>;
 export declare const COLLECTION_OPERATORS: TqlCollectionOperator[];
 /**
- * All mutators supported by TQL.
- * Synchronized with Python/Rust implementations.
+ * The mutator names this package offers for autocomplete.
+ *
+ * This list is NOT self-evidently in agreement with the engines, and for most of its life
+ * it was not: the docstring said "Synchronized with Python/Rust implementations" while it
+ * was synchronized with Rust only, and four names here — `geoip`, `is_loopback`,
+ * `is_multicast`, `is_link_local` — are rejected by the Python evaluator, which is what the
+ * backend query API runs.
+ *
+ * That claim is now checked rather than asserted. `tests/registryParity.spec.ts` reads the
+ * Rust dispatch table, Rust's `MUTATOR_NAMES`, Python's `ALLOWED_MUTATORS` and the
+ * `TqlMutator` union FROM SOURCE and asserts set equality across all five surfaces, with
+ * the four known divergences pinned by name so they fail if they change in either
+ * direction. Editing this list without editing the engines fails that test.
+ *
+ * Comparison operators and type hints get the same treatment in
+ * `tests/catalogParity.spec.ts`. Do not restore a bare "synchronized with X" comment
+ * anywhere here: an invariant nothing enforces reads exactly like one that is enforced.
  */
 export declare const MUTATORS: TqlMutator[];
+/**
+ * The mutators that answer a BOOLEAN, so `field | <name>` with no operator is a FILTER.
+ *
+ * ## Why this is a named set and not four characters of `startsWith("is_")`
+ *
+ * `1c1d74d` made `ip | is_private` mean `ip | is_private eq true` in both engines. Before
+ * it, the bare form meant `exists` — it matched every record that HAS an `ip` field at all,
+ * while reading as a filter. That is the worst shape in this codebase: a clause that cannot
+ * fail loudly, it just quietly stops filtering and keeps returning plausible results.
+ * `1c1d74d` retires it as a documented breaking change.
+ *
+ * Both engines DERIVE the set rather than listing it, and both got there after being burned
+ * by a list:
+ *
+ *   - **Rust** — `Mutator::returns_boolean` is a trait method defaulting to `false`; the
+ *     parser asks `mutators::returns_boolean(name)` and enumerates nothing.
+ *   - **Python** — `BOOLEAN_PREDICATE_MUTATORS` in `src/tql/mutator_classification.py`, one
+ *     frozenset. It used to be a literal `["is_private", "is_global"]` written out in THREE
+ *     places in `parser.py`, and when `is_loopback` / `is_multicast` / `is_link_local` were
+ *     added they reached none of them — so two of five predicates filtered and three
+ *     silently did not.
+ *
+ * This package cannot import either, so it holds the only hand-written copy in the product.
+ * That is a liability, and `tests/booleanPredicateSugar.spec.ts` is the mitigation: it reads
+ * Rust's `returns_boolean` overrides and Python's frozenset FROM SOURCE and asserts all
+ * three name the same five. Adding a predicate to an engine without adding it here fails
+ * that spec, and so does the reverse. Do not edit this list on its own.
+ *
+ * Note that `any` and `all` also yield booleans but are NOT here: they are collection
+ * operators, this package lexes them as `collectionOp`, and neither engine's predicate set
+ * contains them. Their output type is declared separately in `completionProvider.ts`.
+ */
+export declare const BOOLEAN_PREDICATE_MUTATORS: TqlMutator[];
+/** Membership form of {@link BOOLEAN_PREDICATE_MUTATORS}, for parser hot paths. */
+export declare const BOOLEAN_PREDICATE_MUTATOR_SET: ReadonlySet<string>;
 /**
  * All aggregation functions supported by TQL.
- * Synchronized with Python/Rust implementations.
+ *
+ * Unlike MUTATORS, COMPARISON_OPERATORS and TYPE_HINTS, this one is NOT derived from the
+ * engines: neither exposes an aggregation registry to read — the surface is spread across
+ * `stats_evaluator` branches on both sides. `tests/catalogParity.spec.ts` asserts only that
+ * this list covers a transcribed core set, which is a floor and not a parity guarantee.
  */
 export declare const AGGREGATION_FUNCTIONS: TqlAggregationFunction[];
 /**
- * Type hints for field type specification.
+ * Type hints for field type specification: `field::ip eq "10.0.0.1"`.
+ *
+ * Derived-and-checked, not transcribed. `tests/catalogParity.spec.ts` reads the Rust
+ * `type_name` rule out of `grammar.pest` and Python's `KNOWN_TYPE_HINTS` out of
+ * `src/tql/evaluator_components/field_access.py` — the set Python's grammar interpolates —
+ * and asserts exact three-way equality minus an explicit pin list. Adding a name here that
+ * neither engine accepts fails that test, and so does dropping one that both accept.
+ *
+ * The two engines now accept the SAME sixteen names, so this list is that set rather than a
+ * union papering over a disagreement. It was a union until `a12d842`: Rust's `type_name`
+ * omitted `date`, `geo`, `object` and `ip`, deliberately, because Rust parsed a type hint
+ * and read it nowhere and adding names to a layer that consumes none turns a loud parse
+ * error into a silent no-op. That commit made the Rust evaluator honour hints and then
+ * widened the rule, and the four pins recording the gap went red and were deleted.
+ *
+ * `ip` is the one to know about: a fully-implemented Python coercion branch — format
+ * validation, CIDR handling for the `cidr` operator, its own error message — that no query
+ * could reach in either engine until `b338476`, and which this package could not classify.
  */
 export declare const TYPE_HINTS: TqlTypeHint[];
 /**
- * Visualization types for stats pipeline.
+ * Chart types accepted after `=>` in a stats pipeline: `| stats count() by host => bar`.
+ *
+ * ## What enforces this list, and what does not
+ *
+ * It is a REAL contract, and against exactly one engine. The Python grammar declares
+ * `self.viz_types = one_of("bar barchart horizontal_bar ...", caseless=True)`
+ * (`src/tql/parser_components/grammar.py`) — a CLOSED set, so the backend query API rejects
+ * a chart type that is not in it (`=> zzznotaviz` raises `TQLSyntaxError`, measured). A name
+ * added here and not there is a name the editor completes and the API refuses, which is the
+ * same defect `field::geo` and `not_any` were.
+ *
+ * Rust is permissive by comparison: `viz_hint = { "=>" ~ identifier ~ viz_params? }`
+ * (`tql/src/parser/grammar.pest`) accepts ANY identifier, so it validates nothing and cannot
+ * be the authority here. That asymmetry is a live Rust/Python divergence, pinned in
+ * `tests/catalogParity.spec.ts` rather than resolved in this package.
+ *
+ * Within `js/` the list drives two things and neither validates: `classifyIdentifier` uses it
+ * for the `visualization` token class (syntax highlighting), and `getVizCompletions` for
+ * autocomplete. `parseVizHintString` in `builder.ts` casts whatever it is given to
+ * `TqlVisualizationType` without checking membership — see its own note. So a wrong name is
+ * caught by the Python parser at execution and by nothing before it.
+ *
+ * Do not restore a bare "Visualization types for stats pipeline" comment here. A list whose
+ * enforcement is unstated reads exactly like a list with none, and this one has real
+ * enforcement in a place a reader of this file would not think to look.
  */
 export declare const VISUALIZATION_TYPES: TqlVisualizationType[];
 /**