@tellaro/tql 1.5.0 → 2.0.0

package/dist/src/types.d.ts

@@ -38,9 +38,25 @@ export interface TqlComparisonExpression {
     start: number;
     end: number;
 }
+/**
+ * `field ANY value` and its relatives — a comparison applied across the ELEMENTS of an
+ * array-valued field rather than to the field as a whole.
+ *
+ * This type shipped for a long time with no parse path that could produce it: the tokenizer
+ * emitted `collectionOp` for `any` / `all` / `none`, the parser had no arm for that token
+ * type, and every collection query in the editor came back as
+ * "Expected operator after field". The type and `COLLECTION_OPERATORS` were exported the
+ * whole time, so the surface looked supported. `tests/collectionParity.spec.ts` now pins the
+ * accepted forms against both engine grammars.
+ *
+ * `negated` carries the `not any` / `!any` / `not_any` spellings, which both engines accept
+ * and which are otherwise indistinguishable from the positive form once `operator` has been
+ * normalised to its canonical upper-case name.
+ */
 export interface TqlCollectionExpression {
     type: "collection";
     operator: "ANY" | "ALL" | "NONE";
+    negated: boolean;
     field: TqlFieldReference;
     condition: TqlExpression;
     start: number;
@@ -88,21 +104,52 @@ export interface TqlLiteral {
     start: number;
     end: number;
 }
+/**
+ * One argument inside a parameterised mutator call: `split(delimiter=",")`.
+ *
+ * Both engines accept named (`delimiter=","`), positional (`","`) and bare-identifier
+ * (`old=a`) arguments; neither accepts a list literal. See `mutator_arg` in
+ * tql/src/parser/grammar.pest and `_setup_mutators` in
+ * src/tql/parser_components/grammar.py.
+ */
+export interface TqlMutatorArgument {
+    /** Present for `name=value` form; absent for a positional argument. */
+    name?: string;
+    valueType: "string" | "number" | "boolean" | "null" | "identifier";
+    value: string | number | boolean | null;
+    start: number;
+    end: number;
+}
+/**
+ * One link in a mutator chain, with the arguments it was called with.
+ *
+ * `TqlFieldReference.mutators` remains the flat list of names (unchanged for existing
+ * consumers); `mutatorCalls` carries the same chain with argument detail attached.
+ */
+export interface TqlMutatorCall {
+    name: string;
+    args: TqlMutatorArgument[];
+    /** True when the call carried an explicit `(...)` list, even an empty one. */
+    parenthesized: boolean;
+    start: number;
+    end: number;
+}
 export interface TqlFieldReference {
     type: "field";
     name: string;
     path: string[];
     mutators?: string[];
+    mutatorCalls?: TqlMutatorCall[];
     typeHint?: TqlTypeHint;
     start: number;
     end: number;
 }
-export type TqlComparisonOperator = "eq" | "ne" | "=" | "!=" | "==" | "eq_ci" | "gt" | "gte" | "lt" | "lte" | ">" | ">=" | "<" | "<=" | "contains" | "not_contains" | "!contains" | "startswith" | "not_startswith" | "!startswith" | "endswith" | "not_endswith" | "!endswith" | "contains_cs" | "not_contains_cs" | "!contains_cs" | "startswith_cs" | "not_startswith_cs" | "!startswith_cs" | "endswith_cs" | "not_endswith_cs" | "!endswith_cs" | "regexp" | "regex" | "not_regexp" | "not_regex" | "!regexp" | "!regex" | "matches" | "not_matches" | "!matches" | "between" | "not_between" | "!between" | "in" | "not_in" | "!in" | "in_cs" | "not_in_cs" | "!in_cs" | "exists" | "not_exists" | "!exists" | "is" | "is_not" | "!is" | "cidr" | "not_cidr" | "!cidr" | "any" | "all" | "none" | "not_any" | "not_all" | "not_none" | "!any" | "!all" | "!none";
+export type TqlComparisonOperator = "eq" | "ne" | "=" | "!=" | "eq_ci" | "gt" | "gte" | "lt" | "lte" | ">" | ">=" | "<" | "<=" | "contains" | "not_contains" | "!contains" | "startswith" | "not_startswith" | "!startswith" | "endswith" | "not_endswith" | "!endswith" | "contains_cs" | "not_contains_cs" | "!contains_cs" | "startswith_cs" | "not_startswith_cs" | "!startswith_cs" | "endswith_cs" | "not_endswith_cs" | "!endswith_cs" | "regexp" | "regex" | "not_regexp" | "not_regex" | "!regexp" | "!regex" | "matches" | "not_matches" | "!matches" | "between" | "not_between" | "!between" | "in" | "not_in" | "!in" | "in_cs" | "not_in_cs" | "!in_cs" | "exists" | "not_exists" | "!exists" | "is" | "is_not" | "!is" | "cidr" | "not_cidr" | "!cidr" | "any" | "all" | "none" | "not_any" | "not_all" | "not_none" | "!any" | "!all" | "!none";
 export type TqlLogicalOperator = "AND" | "OR" | "NOT";
 export type TqlCollectionOperator = "ANY" | "ALL" | "NONE";
 export type TqlMutator = "lowercase" | "uppercase" | "trim" | "split" | "length" | "replace" | "b64encode" | "b64decode" | "urldecode" | "hexencode" | "hexdecode" | "md5" | "sha256" | "refang" | "defang" | "is_private" | "is_global" | "is_multicast" | "is_loopback" | "is_link_local" | "nslookup" | "geoip_lookup" | "geoip" | "geo" | "any" | "all" | "avg" | "average" | "sum" | "min" | "max";
 export type TqlAggregationFunction = "count" | "sum" | "avg" | "average" | "mean" | "median" | "med" | "min" | "max" | "percentile" | "percentiles" | "p" | "pct" | "percentile_rank" | "percentile_ranks" | "pct_rank" | "pct_ranks" | "std" | "standard_deviation" | "unique_count" | "cardinality" | "values" | "unique" | "distinct";
-export type TqlTypeHint = "string" | "str" | "number" | "int" | "integer" | "float" | "double" | "decimal" | "date" | "bool" | "boolean" | "array" | "list" | "object" | "geo";
+export type TqlTypeHint = "string" | "str" | "number" | "int" | "integer" | "float" | "double" | "decimal" | "date" | "bool" | "boolean" | "array" | "list" | "object" | "geo" | "ip";
 export type TqlVisualizationType = "bar" | "barchart" | "horizontal_bar" | "line" | "area" | "stacked_area" | "pie" | "donut" | "scatter" | "heatmap" | "treemap" | "sunburst" | "table" | "number" | "gauge" | "map" | "grouped_bar" | "stacked_bar" | "nested_pie" | "nested_donut" | "chord" | "auto";
 /**
  * Visualization parameters (e.g., title="Events", stacked=true).
@@ -153,8 +200,43 @@ export interface TqlParseWarning {
     code: TqlWarningCode;
     suggestion?: string;
 }
-export type TqlErrorCode = "UNEXPECTED_TOKEN" | "UNEXPECTED_END" | "UNBALANCED_PARENS" | "UNBALANCED_BRACKETS" | "UNTERMINATED_STRING" | "INVALID_OPERATOR" | "INVALID_MUTATOR" | "INVALID_VALUE" | "MISSING_OPERAND" | "MISSING_OPERATOR" | "MISSING_VALUE" | "MISSING_FIELD" | "MISSING_MUTATOR" | "INVALID_SYNTAX" | "MAX_DEPTH_EXCEEDED" | "TYPE_MISMATCH";
-export type TqlWarningCode = "UNKNOWN_FIELD" | "TYPE_MISMATCH" | "DEPRECATED_OPERATOR" | "REDUNDANT_PARENS";
+export type TqlErrorCode = "UNEXPECTED_TOKEN" | "UNEXPECTED_END" | "UNBALANCED_PARENS" | "UNBALANCED_BRACKETS" | "UNTERMINATED_STRING" | "INVALID_OPERATOR" | "INVALID_MUTATOR" | "INVALID_VALUE" | "MISSING_OPERAND" | "MISSING_OPERATOR" | "MISSING_VALUE" | "MISSING_FIELD" | "MISSING_MUTATOR" | "INVALID_SYNTAX" | "MAX_DEPTH_EXCEEDED" | "TYPE_MISMATCH"
+/**
+ * `f::int | trim::str` — a type hint on BOTH sides of the mutator chain.
+ *
+ * Both engines accept the hint in either position and refuse it in both, structurally: each
+ * grammar is an ordered choice whose first alternative consumes `f::int | trim` and strands
+ * `::str`. There is no defensible answer for which hint would win.
+ */
+ | "DUPLICATE_TYPE_HINT"
+/**
+ * `::` with a token after it that can never be a type name (`f::| trim`, `f:: eq 'a'`,
+ * `f | trim:: eq 'a'`, `f::lowercase`). All four are parse errors in both engines.
+ *
+ * `f::` at END of input is NOT this — that is a hint being typed, and stays a warning.
+ */
+ | "MISSING_TYPE_HINT"
+/**
+ * A malformed `top N` / `bottom N` modifier on a stats aggregation.
+ *
+ * Reserved for the shapes BOTH engines refuse: the modifier given twice on one aggregation
+ * (`sum(x, top 10) top 5`), a modifier with no count (`sum(x) top by y`), and the in-parens
+ * form on `count(*)` / `count()` (`count(*, top 2)`). A spelling only one engine refuses is
+ * an ENGINE_DIVERGENCE warning instead — never this.
+ */
+ | "INVALID_STATS_MODIFIER";
+export type TqlWarningCode = "UNKNOWN_FIELD" | "TYPE_MISMATCH" | "DEPRECATED_OPERATOR" | "REDUNDANT_PARENS"
+/**
+ * The query parses, but only ONE of the two engines accepts it.
+ *
+ * Reserved for measured, currently-live divergences between `tql/src/parser/grammar.pest`
+ * and `src/tql/parser_components/grammar.py` — not for anything this package merely
+ * dislikes. The editor stays permissive (a red squiggle over syntax an engine accepts is
+ * the defect this parser keeps being fixed for) while still naming the engine that will
+ * refuse the query, because silently accepting it is the worse half of the same failure:
+ * the user finds out at execution instead of at the keystroke.
+ */
+ | "ENGINE_DIVERGENCE";
 export interface TqlValidationResult {
     valid: boolean;
     errors: TqlParseError[];

package/dist/src/validator.d.ts

@@ -1,4 +1,52 @@
-import { TqlParseError, TqlParseWarning, TqlValidationOptions, TqlValidationResult } from './types';
+import { TqlFieldType, TqlParseError, TqlParseWarning, TqlValidationOptions, TqlValidationResult } from './types';
+/**
+ * Maximum regex pattern length TQL will compile, in characters (code points).
+ *
+ * A pattern longer than this is refused by every TQL runtime. It is flagged here
+ * -- in the editor, as the user types -- because the alternative is discovering
+ * it after the rule has been saved and deployed: before tql#185 the Python
+ * evaluator answered "no match" for an over-long pattern while the query parsed
+ * cleanly and reported health green, so a detection rule carrying one never
+ * fired and nothing anywhere said so.
+ *
+ * Mirrored by `MAX_REGEX_PATTERN_LENGTH` in `src/tql/regex_guard.py` and
+ * `tql/src/comparator.rs`. Changing it in one place only re-creates the
+ * divergence.
+ */
+export declare const MAX_REGEX_PATTERN_LENGTH = 1000;
+/**
+ * Every field type in the `TqlFieldType` union.
+ *
+ * Declared once so an operator that genuinely applies to everything says so by NAME rather
+ * than by a transcribed twelve-element list — and so a new field type cannot be added to the
+ * union while silently falling out of the operators it belongs to.
+ * `tests/collectionOperatorTypes.spec.ts` reads the union out of `src/types.ts` and asserts
+ * this array is exactly it, so a divergence is a red rather than a quiet narrowing.
+ */
+export declare const ALL_FIELD_TYPES: TqlFieldType[];
+/**
+ * Every numeric field type in the union — all nine widths, not the five that get typed out.
+ */
+export declare const NUMERIC_FIELD_TYPES: TqlFieldType[];
+/**
+ * The types an ORDERED comparison (`gt`/`gte`/`lt`/`lte`) is defined over.
+ *
+ * Numeric and date, plus `keyword`: both evaluators fall back to a lexicographic string
+ * comparison when neither side reads as a number, and Python's own validator lists all four
+ * operators in its `keyword_ops`. `text` is excluded on that same source's reasoning — a
+ * range over an analyzed field compares tokens, not the value the user typed.
+ */
+export declare const ORDERED_FIELD_TYPES: TqlFieldType[];
+/**
+ * The types `between` / `not_between` are defined over.
+ *
+ * `ORDERED_FIELD_TYPES` minus `keyword`. The four ordered operators and `between` are NOT
+ * the same set, and the difference is Python's, not this package's: `keyword_ops` in
+ * `_validate_simple_type_compatibility` lists `gt`/`gte`/`lt`/`lte` and excludes `between`.
+ * `tests/collectionAndNullSemantics.spec.ts` pins that exclusion against the Python source,
+ * so this constant self-retires if Python ever adds it.
+ */
+export declare const BETWEEN_FIELD_TYPES: TqlFieldType[];
 export declare class TqlValidator {
     private options;
     private errors;
@@ -12,6 +60,58 @@ export declare class TqlValidator {
      * Validate an expression recursively
      */
     private validateExpression;
+    /**
+     * `f any ['a','b']` is ill-typed. Say so, and name `in`.
+     *
+     * ## The rule, as both engines implement it
+     *
+     * `any` / `none` apply a comparison to each ELEMENT of an array-valued field. The operand
+     * is therefore a SCALAR: `f any 'admin'` asks whether some element equals `'admin'`. A
+     * LIST operand asks whether some single element equals the whole two-element list, which
+     * no scalar element can. `in` is already the membership operator, and is what the author
+     * meant.
+     *
+     * Both engines now refuse it -- `ab55587` in `tql/src/opensearch/query_builder.rs`
+     * (`build_collection_op`) and `_refuse_list_operand` in
+     * `src/tql/opensearch_components/query_converter.py` -- raising a type error whose
+     * `valid_operators` are `in` / `not in`. Before that it emitted `{"term": {"f": [..]}}`,
+     * which OpenSearch answers with HTTP 400 `[term] query does not support array of values`,
+     * failing the WHOLE search including every unrelated clause; the intermediate fix emitted
+     * `match_none`, which executed cleanly and excluded nothing, silently.
+     *
+     * ## Three arities, and only the middle one is new
+     *
+     * The engines do NOT test `is_array`. They unwrap a single-element list FIRST
+     * (`unwrap_single` in Rust, `len(value) == 1` in Python) and refuse only what is still a
+     * list. Measured against the live Python translator at `ab55587`:
+     *
+     * | Query              | Python translator                                    |
+     * |--------------------|------------------------------------------------------|
+     * | `f any 'admin'`    | `{"term": {"f": "admin"}}`                            |
+     * | `f any ['a']`      | `{"term": {"f": "a"}}` -- unwrapped, NOT refused      |
+     * | `f any ['a','b']`  | `TQLTypeError`, valid operators `in`, `not in`        |
+     *
+     * A naive `Array.isArray(value)` check would flag `f any ['a']`, which both engines
+     * translate perfectly well -- a red squiggle over a working query, which is the defect
+     * this parser keeps being fixed for. Hence `length >= 2`, and hence the two arities are
+     * tested separately in `tests/collectionListOperand.spec.ts`.
+     *
+     * The empty list is not handled here: `f any []` is already an `INVALID_VALUE` error from
+     * the parser (`tests/emptyValueList.spec.ts`), and Python does not even parse it
+     * (`TQLSyntaxError`). Reporting it twice would be noise.
+     *
+     * ## Why `ALL` is excluded, and why that is not an oversight
+     *
+     * `all` / `not_all` are answered by a Painless script that compares each element to
+     * `params.value`. A list operand there emits valid DSL that matches nothing -- which is
+     * what both in-memory evaluators also answer -- so both engines dispatch `all` BEFORE the
+     * refusal and leave it untouched. `f all ['a','b']` translating cleanly is a pinned
+     * negative case here, not a gap.
+     *
+     * Negation is irrelevant: in both engines the array check precedes the quantifier wrapper,
+     * so `not any` / `not none` refuse on the same footing as the positive spellings.
+     */
+    private validateCollectionOperand;
     /**
      * Validate a comparison expression
      */