@tellaro/tql 1.5.0 → 2.0.0

package/dist/src/complexityAnalyzer.js

@@ -1,26 +1,36 @@
-import { parseTql as u } from "./parser.js";
-const d = {
+import { parseTql as d } from "./parser.js";
+import { MUTATORS as p } from "./tokenizer.js";
+const g = {
   maxDepth: 5,
   maxRegex: 3,
   maxInListSize: 100,
   maxOrChainLength: 10,
   maxLeadingWildcards: 2
-}, h = /* @__PURE__ */ new Set([
+}, h = new Set(p), c = /* @__PURE__ */ new Set([
   "geoip_lookup",
   "geoip",
   "geo",
   "nslookup"
-]), p = /\*/g, g = /^\*/;
-class c {
+]), y = /\*/g, f = /^\*/;
+function u(o, e, t) {
+  const a = c.has(o);
+  return {
+    type: "post_processing_mutator",
+    message: a ? `Mutator '${o}' requires post-processing and may be slow` : `Mutator '${o}' requires post-processing: matching happens after results return, so the filter cannot be pushed into OpenSearch`,
+    severity: a ? "warning" : "info",
+    location: { start: e, end: t }
+  };
+}
+class x {
   thresholds;
   constructor(e) {
-    this.thresholds = { ...d, ...e };
+    this.thresholds = { ...g, ...e };
   }
   /**
    * Analyze a TQL query for complexity
    */
   analyze(e) {
-    const t = u(e), a = {
+    const t = d(e), a = {
       depth: 0,
       operatorCount: 0,
       wildcardCount: 0,
@@ -59,6 +69,11 @@ class c {
       case "comparison":
         this.analyzeComparisonExpression(e, t, a);
         break;
+      // `field ANY value` -- the cost lives in the wrapped comparison (its mutators, its
+      // regex, its list size), so recurse rather than scoring the wrapper as nothing.
+      case "collection":
+        this.analyzeExpression(e.condition, t, a, n, s);
+        break;
       case "group":
         this.analyzeGroupExpression(e, t, a, n, s);
         break;
@@ -79,21 +94,11 @@ class c {
     t.operatorCount++;
     const n = e.field;
     if (n.type === "mutator")
-      t.mutatorCount++, h.has(n.mutator) && (t.postProcessingMutatorCount++, a.push({
-        type: "post_processing_mutator",
-        message: `Mutator '${n.mutator}' requires post-processing and may be slow`,
-        severity: "warning",
-        location: { start: n.start, end: n.end }
-      }));
+      t.mutatorCount++, h.has(n.mutator) && (t.postProcessingMutatorCount++, a.push(u(n.mutator, n.start, n.end)));
     else if (n.type === "field" && n.mutators) {
       t.mutatorCount += n.mutators.length;
       for (const i of n.mutators)
-        h.has(i) && (t.postProcessingMutatorCount++, a.push({
-          type: "post_processing_mutator",
-          message: `Mutator '${i}' requires post-processing and may be slow`,
-          severity: "warning",
-          location: { start: n.start, end: n.end }
-        }));
+        h.has(i) && (t.postProcessingMutatorCount++, a.push(u(i, n.start, n.end)));
     }
     const s = e.operator.toLowerCase();
     if ((s === "regexp" || s === "regex" || s === "matches" || s === "not_regexp" || s === "not_regex" || s === "not_matches") && t.regexCount++, this.analyzeValue(e.value, t, a, e), (s === "in" || s === "not_in") && Array.isArray(e.value)) {
@@ -105,10 +110,10 @@ class c {
     const s = Array.isArray(e) ? e : [e];
     for (const i of s)
       if (i.valueType === "string" && typeof i.value == "string") {
-        const l = i.value.match(p);
-        l && (t.wildcardCount += l.length, g.test(i.value) && (t.leadingWildcardCount++, t.leadingWildcardCount <= this.thresholds.maxLeadingWildcards && a.push({
+        const l = i.value.match(y);
+        l && (t.wildcardCount += l.length, f.test(i.value) && (t.leadingWildcardCount++, t.leadingWildcardCount <= this.thresholds.maxLeadingWildcards && a.push({
           type: "leading_wildcard",
-          message: `Leading wildcard in '${i.value}' is expensive - consider using 'endswith' if possible`,
+          message: `'${i.value}' contains '*', which TQL treats as a literal character, not a wildcard - use 'endswith' for a suffix match or 'matches' for a regex`,
           severity: "warning",
           location: { start: n.start, end: n.end }
         })));
@@ -173,34 +178,38 @@ class c {
    */
   generateSuggestions(e, t, a) {
     const n = new Set(e.map((s) => s.type));
-    (n.has("leading_wildcard") || n.has("multiple_wildcards")) && t.push("Leading wildcards (`*foo`) are expensive - use `endswith` if possible"), n.has("deep_nesting") && t.push(`Query depth of ${a.depth} may impact performance - consider simplifying`), n.has("multiple_regex") && t.push("Multiple regex operators are slow - consider using `contains` or `startswith` where possible"), n.has("large_in_list") && t.push("Large IN lists can be slow - consider splitting into multiple queries or using a lookup table"), n.has("complex_or_chain") && t.push("Consider using `in` instead of multiple `OR` conditions for the same field"), n.has("post_processing_mutator") && t.push("Mutators like `geoip_lookup` and `nslookup` require post-processing and cannot be optimized by the database"), n.has("nested_negation") && t.push("Simplify nested NOT expressions for better readability and potential optimization");
+    (n.has("leading_wildcard") || n.has("multiple_wildcards")) && t.push(
+      "TQL has no wildcard syntax - `*` is matched literally. Use `endswith` for a suffix match, `startswith` for a prefix, `contains` for a substring, or `matches` for a regex."
+    ), n.has("deep_nesting") && t.push(`Query depth of ${a.depth} may impact performance - consider simplifying`), n.has("multiple_regex") && t.push("Multiple regex operators are slow - consider using `contains` or `startswith` where possible"), n.has("large_in_list") && t.push("Large IN lists can be slow - consider splitting into multiple queries or using a lookup table"), n.has("complex_or_chain") && t.push("Consider using `in` instead of multiple `OR` conditions for the same field"), n.has("post_processing_mutator") && t.push("Mutators like `geoip_lookup` and `nslookup` require post-processing and cannot be optimized by the database"), n.has("nested_negation") && t.push("Simplify nested NOT expressions for better readability and potential optimization");
   }
   /**
    * Calculate overall complexity score based on metrics and warnings
    */
   calculateScore(e, t) {
     let a = 0;
-    e.depth > this.thresholds.maxDepth ? a += 2 : e.depth > 3 && (a += 1), e.regexCount > this.thresholds.maxRegex ? a += 2 : e.regexCount > 1 && (a += 1), e.leadingWildcardCount > this.thresholds.maxLeadingWildcards ? a += 2 : e.leadingWildcardCount > 0 && (a += 1), e.inListMaxSize > this.thresholds.maxInListSize ? a += 2 : e.inListMaxSize > 50 && (a += 1), e.postProcessingMutatorCount > 0 && (a += e.postProcessingMutatorCount), e.orChainLength > this.thresholds.maxOrChainLength && (a += 1);
+    e.depth > this.thresholds.maxDepth ? a += 2 : e.depth > 3 && (a += 1), e.regexCount > this.thresholds.maxRegex ? a += 2 : e.regexCount > 1 && (a += 1), e.leadingWildcardCount > this.thresholds.maxLeadingWildcards ? a += 2 : e.leadingWildcardCount > 0 && (a += 1), e.inListMaxSize > this.thresholds.maxInListSize ? a += 2 : e.inListMaxSize > 50 && (a += 1), a += t.filter(
+      (s) => s.type === "post_processing_mutator" && s.severity === "warning"
+    ).length, e.orChainLength > this.thresholds.maxOrChainLength && (a += 1);
     const n = t.filter((s) => s.severity === "error").length;
     return a += n * 2, a >= 4 ? "high" : a >= 2 ? "medium" : "low";
   }
 }
 function r(o, e) {
-  return new c(e).analyze(o);
+  return new x(e).analyze(o);
 }
-function f(o, e) {
+function L(o, e) {
   return r(o, e).score === "high";
 }
-function x(o, e) {
+function z(o, e) {
   return r(o, e).warnings;
 }
-function C(o) {
+function _(o) {
   return r(o).metrics;
 }
 export {
-  c as ComplexityAnalyzer,
+  x as ComplexityAnalyzer,
   r as analyzeComplexity,
-  C as getComplexityMetrics,
-  x as getComplexityWarnings,
-  f as isHighComplexity
+  _ as getComplexityMetrics,
+  z as getComplexityWarnings,
+  L as isHighComplexity
 };

package/dist/src/index.d.ts

@@ -7,9 +7,9 @@
  * @packageDocumentation
  */
 export type { TqlToken, TqlTokenType, TqlExpression, TqlBinaryExpression, TqlUnaryExpression, TqlComparisonExpression, TqlCollectionExpression, TqlMutatorExpression, TqlAggregationExpression, TqlStatsExpression, TqlGroupExpression, TqlLiteral, TqlFieldReference, TqlComparisonOperator, TqlLogicalOperator, TqlCollectionOperator, TqlMutator, TqlAggregationFunction, TqlTypeHint, TqlVisualizationType, TqlVizParams, TqlVizHint, TqlParseResult, TqlParseError, TqlParseWarning, TqlErrorCode, TqlWarningCode, TqlValidationResult, TqlValidationOptions, TqlFieldType, TqlCompletionItem, TqlCompletionKind, TqlCompletionContext, TqlCompletionOptions, TqlFieldSchema, TqlSnippet, TqlInputProps, TqlInputEmits, TqlInputExpose } from './types';
-export { TqlTokenizer, tokenizeTql, tokenizeTqlNoWhitespace, findTokenAtPosition, findTokensInRange, findTokenBefore, findPreviousToken, COMPARISON_OPERATORS, SYMBOL_OPERATORS, NEGATION_SYMBOL_OPERATORS, LOGICAL_OPERATORS, COLLECTION_OPERATORS, MUTATORS, AGGREGATION_FUNCTIONS, TYPE_HINTS, VISUALIZATION_TYPES, KEYWORDS } from './tokenizer';
+export { TqlTokenizer, tokenizeTql, tokenizeTqlNoWhitespace, findTokenAtPosition, findTokensInRange, findTokenBefore, findPreviousToken, COMPARISON_OPERATORS, SYMBOL_OPERATORS, NEGATION_SYMBOL_OPERATORS, LOGICAL_OPERATORS, COLLECTION_OPERATORS, MUTATORS, BOOLEAN_PREDICATE_MUTATORS, BOOLEAN_PREDICATE_MUTATOR_SET, AGGREGATION_FUNCTIONS, TYPE_HINTS, VISUALIZATION_TYPES, KEYWORDS } from './tokenizer';
 export { TqlParser, parseTql, isValidTql, getTqlErrors, formatParseError } from './parser';
-export { TqlValidator, validateTql, isValidTqlSyntax, getTqlIssues, formatValidationResult } from './validator';
+export { TqlValidator, validateTql, isValidTqlSyntax, getTqlIssues, formatValidationResult, MAX_REGEX_PATTERN_LENGTH } from './validator';
 export { getCompletions, getWordAtPosition, getCompletionTriggerCharacters, shouldTriggerCompletion } from './completionProvider';
 export { fuzzyMatch, levenshteinDistance, findClosestMatch, fuzzyFilter, highlightMatch, type FuzzyMatchResult, type FuzzyMatchOptions } from './fuzzyMatcher';
 export { SchemaCache, getSchemaCache, createSchemaCache, getCachedFieldNames, getCachedFieldType, getCachedField, type CachedSchema } from './schemaCache';

package/dist/src/parser.d.ts

@@ -1,4 +1,19 @@
-import { TqlParseError, TqlParseResult } from './types';
+import { TqlCollectionOperator, TqlParseError, TqlParseResult } from './types';
+/**
+ * The canonical WORD spelling of a collection operator, derived from the table above rather
+ * than written out a second time.
+ *
+ * `TqlCollectionExpression` normalises every accepted spelling to `operator` + `negated`, so
+ * anything that has to show a collection operator back to the user — the semantic preview is
+ * the only such consumer today — has to reverse that. Doing it by reading the table means a
+ * spelling added to or removed from the parser cannot leave a renderer describing an operator
+ * this package no longer accepts.
+ *
+ * Symbolic spellings (`!any`) are excluded on purpose: `not_any` is the form both engines
+ * accept AND the form `COMPARISON_OPERATORS` carries, which is what makes the rendered string
+ * a query the user can paste back.
+ */
+export declare function collectionOperatorSpelling(operator: TqlCollectionOperator, negated: boolean): string;
 export declare class TqlParser {
     private tokens;
     private allTokens;
@@ -42,6 +57,63 @@ export declare class TqlParser {
      * - Optional mutator chain: field | lowercase | trim eq "value"
      */
     private parseComparison;
+    /**
+     * The part of a field reference after its name token: `::type` and `| mutator` chains.
+     *
+     * Split out of `parseComparison` so the operator-first collection form
+     * (`ANY field::string | lowercase eq "x"`) reads the same field grammar rather than a
+     * reduced copy of it. A second copy is how `field::geo` came to be accepted in one place
+     * and rejected in another.
+     */
+    private parseFieldTail;
+    /**
+     * Read one `::type` slot, if the current token opens one. Returns the field with the hint
+     * applied and whether a hint name was actually consumed.
+     *
+     * Called from BOTH sides of the mutator chain so the two spellings share one definition of
+     * what may follow `::`.
+     *
+     * Match on the token's VALUE, not its type. `classifyIdentifier` tests the mutator catalog
+     * before the type-hint catalog, so a name in both — `geo` is the live case, exported by
+     * TYPE_HINTS and by MUTATORS — arrives here typed "mutator" and was rejected. The package
+     * exported `geo` as a valid type hint while its own parser refused `field::geo`. After
+     * `::` the only thing a token can be is a type name, so the catalog is the right authority
+     * and the lexical class is noise.
+     */
+    private readTypeHintSlot;
+    /**
+     * `ANY field [op] value` — the operator-first collection forms.
+     *
+     * `grammar.pest:226-231` gives `collection_comparison` four alternatives; this is the two
+     * that lead with the operator. The Python parser accepts NEITHER — probed, not inferred:
+     * `ANY f eq 'x'` and `ANY f 'x'` both raise `TQLSyntaxError at position 4`. They are
+     * therefore parsed AND flagged: see `RUST_ONLY_COLLECTION_MESSAGE`.
+     *
+     * The current token is the collection operator.
+     */
+    private parseCollectionFirst;
+    /**
+     * `field [not] ANY [op] value`, or null when the token after the field is not a
+     * collection operator at all.
+     *
+     * Only `field ANY value` — no inner comparison operator — is accepted by both engines.
+     * `field ANY eq value` is `grammar.pest`'s third alternative and Python rejects it
+     * (`f any eq 'x'` -> `TQLSyntaxError at position 9`), so it parses with the divergence
+     * warning rather than being silently accepted or silently refused.
+     */
+    private tryParseCollectionTail;
+    /**
+     * The shared tail of both collection forms: an OPTIONAL inner comparison operator
+     * (defaulting to `eq`, as both grammars do) followed by a value or a value list.
+     */
+    private finishCollection;
+    /**
+     * `<field> <comparison-op> <value>` — everything after the field reference.
+     *
+     * `fieldToken` is threaded through only for the error messages, which name the field the
+     * user typed rather than the normalised reference.
+     */
+    private parseComparisonTail;
     /**
      * Parse a value (string, number, boolean, null)
      */
@@ -50,15 +122,108 @@ export declare class TqlParser {
      * Parse a list of values for 'in' operator
      */
     private parseValueList;
+    /**
+     * Parse the optional argument list of a mutator call.
+     *
+     * `field | split(delimiter=",") eq "bob"` is valid TQL that both engines accept
+     * (`mutator = { "|" ~ !^"stats" ~ mutator_name ~ ("(" ~ mutator_args? ~ ")")? }` in
+     * tql/src/parser/grammar.pest; `mutator_with_params` in
+     * src/tql/parser_components/grammar.py). The JS parser used to stop at the mutator
+     * NAME, so the '(' fell through to the top-level "Unexpected token" arm and the editor
+     * red-squiggled a query the backend runs happily -- while this package's own completion
+     * documentation advertised that exact call form.
+     *
+     * The name token has already been consumed by the caller.
+     */
+    private parseMutatorCall;
+    /**
+     * Parse one mutator argument: `value`, or `name=value`.
+     *
+     * Returns null (after recording an error) when the current token cannot start an
+     * argument, so the caller stops rather than spinning.
+     */
+    private parseMutatorArgument;
+    /**
+     * Parse the value half of a mutator argument.
+     *
+     * Neither engine's grammar admits a list literal here, so `[...]` is an error rather
+     * than a silently-accepted superset.
+     */
+    private parseMutatorArgumentValue;
     private createFieldReference;
+    /**
+     * Consume a `not` / `!` sitting between a field and its operator, if one is there.
+     *
+     * The two-token spelling is the PRIMARY form in both grammars, not an alias: fourteen
+     * rules in `tql/src/parser/grammar.pest` are declared as
+     * `not_X_op = { neg_prefix ~ X_op | not_X_word }`, and Python builds the same shape with
+     * `(CaselessKeyword("not") | "!") + CaselessKeyword("X")`. `field not contains "x"` is
+     * how the documentation and the UI examples spell it.
+     *
+     * This package accepted only the underscore form (`not_contains`) and the glued symbol
+     * (`!contains`). Every two-token negation -- `not exists`, `not in`, `not between`,
+     * `not startswith`, `not cidr`, ten more -- was reported as
+     * `Expected operator after field 'f'`, a false red on the most readable spelling of a
+     * third of the operator surface.
+     *
+     * Only consumed when a comparison operator actually follows. Without that lookahead this
+     * would swallow the `not` in `a eq 1 and b not (c eq 2)`, where it belongs to the unary
+     * arm; leaving it in place lets the existing error recovery report on it as before.
+     */
+    private consumeNegationPrefix;
+    /**
+     * The negated form of an operator, or null when it has none.
+     *
+     * DERIVED from `COMPARISON_OPERATORS`, not transcribed: `not X` is a TQL operator exactly
+     * when `not_X` is. That catalog is itself derived from both engine grammars and pinned by
+     * `tests/catalogParity.spec.ts`, so this rule cannot drift from the grammars without that
+     * test going red first -- and `tests/negatedOperatorForms.spec.ts` re-derives the
+     * `neg_prefix ~ X_op` rules straight out of `grammar.pest` and asserts this parser accepts
+     * exactly them.
+     *
+     * Returning null rather than fabricating `not_eq` matters. Neither engine has a
+     * `neg_prefix ~ eq_op` rule, so `f not eq 1` is a syntax error in both, and inventing an
+     * operator no evaluator dispatches on would turn a parse error into an unknown-operator
+     * failure at execution -- later, and further from the user.
+     */
+    private negateOperator;
     private normalizeOperator;
     private isAtEnd;
     private peek;
+    /** Lookahead by `offset` tokens without consuming; undefined past the end. */
+    private peekAt;
     private previous;
     private advance;
     private check;
+    /**
+     * True when the current token names a mutator in `MUTATORS` but was lexed as something
+     * else because an earlier catalog claimed the name first.
+     *
+     * Deliberately consults the catalog rather than listing the seven affected names: a list
+     * would be an eighth transcription of a set this package already maintains, and it would
+     * go stale the moment a mutator is added that collides with an aggregation.
+     */
+    private isCatalogMutatorToken;
     private match;
     private matchLogical;
+    /**
+     * The canonical word a `logicalOp` token spells: `AND`, `OR` or `NOT`.
+     *
+     * `&&`, `||` and `!` are the symbolic halves of the same three grammar rules as `and`,
+     * `or` and `not` (see SYMBOL_LOGICAL_OPERATORS), so every consumer must see one name.
+     * Normalising HERE rather than in the tokenizer keeps the token's `value` equal to the
+     * source text, which the highlighting and completion surfaces read.
+     */
+    private logicalWord;
+    /**
+     * Match the WORD spelling of a logical operator only, never the symbol.
+     *
+     * Used for the `and` inside `field between low AND high`. Both engines spell that one
+     * literally -- `value ~ ^"and" ~ value` in `grammar.pest`, `CaselessKeyword("and")` in
+     * `between_comparison_natural` -- and neither accepts `between 1 && 5`. Accepting it here
+     * would make this package the only one of the three that does.
+     */
+    private matchLogicalWord;
     private addError;
     private addWarning;
     /**
@@ -71,6 +236,33 @@ export declare class TqlParser {
      * This allows the parser to continue and find additional errors.
      */
     private createPlaceholderExpression;
+    /**
+     * Flag the malformed `top N` / `bottom N` spellings on a stats aggregation.
+     *
+     * ## Why this is a scan and not a stats parser
+     *
+     * This package deliberately does not build a stats AST — the engines do. But "we do not
+     * parse it" had silently become "we accept anything after `stats`": measured at HEAD, every
+     * one of `sum(x, top 10) top 5`, `sum(x) top by y` and `count(*, top 2)` is a parse error in
+     * BOTH engines and was reported valid here.
+     *
+     * A full stats parser would have to take a side on five spellings the engines currently
+     * DISAGREE about (`count(*) top 2`, `sum(x|trim) top 3`, `sum(x::number) top 3`,
+     * `sum(x|trim, top 3)`, `by y|trim top 3` — Rust accepts all five, Python refuses all five).
+     * Resolving those is an engine question. So this fires ONLY on shapes both engines refuse,
+     * and is silent everywhere else — it can never squiggle a query an engine accepts, which is
+     * the failure mode this parser keeps being repaired for.
+     *
+     * ## Why the group-by region is excluded
+     *
+     * `top` and `bottom` are ordinary identifiers, and `| stats sum(x) by top` is valid in BOTH
+     * engines — a group-by on a field named `top`. So "a `top` not followed by a number" is only
+     * unambiguous in the two AGGREGATION modifier positions: directly after the aggregation's
+     * `)`, or after a comma at the aggregation's own paren depth. `by y top` (no count) is
+     * refused by both engines and is deliberately NOT flagged here, because distinguishing it
+     * from `by top` needs the grammar this method is declining to reimplement.
+     */
+    private checkStatsModifiers;
     /**
      * Check if the token stream represents a stats query.
      * Handles: "| stats ...", "stats ...", or starts with a filter before "| stats".