@tellaro/tql 0.1.0

package/dist/src/tokenizer.d.ts

@@ -0,0 +1,113 @@
+import { TqlAggregationFunction, TqlCollectionOperator, TqlComparisonOperator, TqlLogicalOperator, TqlMutator, TqlToken, TqlTypeHint, TqlVisualizationType } from './types';
+
+/**
+ * All comparison operators supported by TQL.
+ * Includes negated variants using both `not_` prefix and `!` prefix.
+ */
+export declare const COMPARISON_OPERATORS: TqlComparisonOperator[];
+/**
+ * Symbol-based operators mapped to their word equivalents.
+ */
+export declare const SYMBOL_OPERATORS: Record<string, TqlComparisonOperator>;
+/**
+ * Negation prefixes for operators.
+ */
+export declare const NEGATION_SYMBOL_OPERATORS: Record<string, TqlComparisonOperator>;
+export declare const LOGICAL_OPERATORS: TqlLogicalOperator[];
+export declare const COLLECTION_OPERATORS: TqlCollectionOperator[];
+/**
+ * All mutators supported by TQL.
+ * Synchronized with Python/Rust implementations.
+ */
+export declare const MUTATORS: TqlMutator[];
+/**
+ * All aggregation functions supported by TQL.
+ * Synchronized with Python/Rust implementations.
+ */
+export declare const AGGREGATION_FUNCTIONS: TqlAggregationFunction[];
+/**
+ * Type hints for field type specification.
+ */
+export declare const TYPE_HINTS: TqlTypeHint[];
+/**
+ * Visualization types for stats pipeline.
+ */
+export declare const VISUALIZATION_TYPES: TqlVisualizationType[];
+/**
+ * Keywords used in TQL.
+ */
+export declare const KEYWORDS: string[];
+export declare class TqlTokenizer {
+    private input;
+    private position;
+    private line;
+    private column;
+    private tokens;
+    constructor(input: string);
+    /**
+     * Tokenize the entire input string
+     */
+    tokenize(): TqlToken[];
+    /**
+     * Scan the next token
+     */
+    private scanToken;
+    /**
+     * Scan whitespace characters
+     */
+    private scanWhitespace;
+    /**
+     * Scan a string literal (single or double quoted)
+     */
+    private scanString;
+    /**
+     * Scan a numeric literal
+     */
+    private scanNumber;
+    /**
+     * Scan symbol-based operators (==, !=, >, >=, <, <=, !)
+     */
+    private scanSymbolOperator;
+    /**
+     * Scan an identifier (field name, operator keyword, etc.)
+     */
+    private scanIdentifier;
+    /**
+     * Classify an identifier into the appropriate token type
+     */
+    private classifyIdentifier;
+    private makeToken;
+    private isAtEnd;
+    private peek;
+    private peekNext;
+    private advance;
+    private isWhitespace;
+    private isDigit;
+    private isIdentifierStart;
+    private isIdentifierChar;
+    private isOperatorChar;
+}
+/**
+ * Tokenize a TQL query string
+ */
+export declare function tokenizeTql(query: string): TqlToken[];
+/**
+ * Get tokens without whitespace (for parsing)
+ */
+export declare function tokenizeTqlNoWhitespace(query: string): TqlToken[];
+/**
+ * Find token at a specific position
+ */
+export declare function findTokenAtPosition(tokens: TqlToken[], position: number): TqlToken | null;
+/**
+ * Find tokens in a range
+ */
+export declare function findTokensInRange(tokens: TqlToken[], start: number, end: number): TqlToken[];
+/**
+ * Get the token immediately before a position
+ */
+export declare function findTokenBefore(tokens: TqlToken[], position: number): TqlToken | null;
+/**
+ * Get the previous non-whitespace token
+ */
+export declare function findPreviousToken(tokens: TqlToken[], position: number, skipWhitespace?: boolean): TqlToken | null;

package/dist/src/tokenizer.js

@@ -0,0 +1,430 @@
+var l = Object.defineProperty;
+var u = (o, e, n) => e in o ? l(o, e, { enumerable: !0, configurable: !0, writable: !0, value: n }) : o[e] = n;
+var a = (o, e, n) => u(o, typeof e != "symbol" ? e + "" : e, n);
+const p = [
+  // Equality
+  "eq",
+  "ne",
+  // Comparison
+  "gt",
+  "gte",
+  "lt",
+  "lte",
+  // String operators
+  "contains",
+  "not_contains",
+  "startswith",
+  "not_startswith",
+  "endswith",
+  "not_endswith",
+  "regexp",
+  "regex",
+  "not_regexp",
+  "matches",
+  "not_matches",
+  // Range operators
+  "between",
+  "not_between",
+  "in",
+  "not_in",
+  // Existence operators
+  "exists",
+  "not_exists",
+  "is",
+  "is_not",
+  // IP-specific operators
+  "cidr",
+  "not_cidr",
+  // Collection operators (can be used as comparison)
+  "any",
+  "all",
+  "none",
+  "not_any",
+  "not_all",
+  "not_none"
+], c = {
+  "==": "eq",
+  "=": "eq",
+  "!=": "ne",
+  ">": "gt",
+  ">=": "gte",
+  "<": "lt",
+  "<=": "lte"
+}, d = {
+  "!in": "not_in",
+  "!contains": "not_contains",
+  "!startswith": "not_startswith",
+  "!endswith": "not_endswith",
+  "!regexp": "not_regexp",
+  "!matches": "not_matches",
+  "!between": "not_between",
+  "!exists": "not_exists",
+  "!is": "is_not",
+  "!cidr": "not_cidr",
+  "!any": "not_any",
+  "!all": "not_all",
+  "!none": "not_none"
+}, k = ["AND", "OR", "NOT"], f = ["ANY", "ALL", "NONE"], m = [
+  // String mutators
+  "lowercase",
+  "uppercase",
+  "trim",
+  "split",
+  "length",
+  "replace",
+  // Encoding mutators
+  "b64encode",
+  "b64decode",
+  "urldecode",
+  "hexencode",
+  "hexdecode",
+  "md5",
+  "sha256",
+  // Network/Security mutators
+  "refang",
+  "defang",
+  "is_private",
+  "is_global",
+  // DNS mutator
+  "nslookup",
+  // GeoIP mutators
+  "geoip_lookup",
+  "geoip",
+  "geo",
+  // List/Array mutators
+  "any",
+  "all",
+  "avg",
+  "average",
+  "sum",
+  "min",
+  "max"
+], _ = [
+  // Counting
+  "count",
+  "unique_count",
+  "cardinality",
+  // Numeric aggregations
+  "sum",
+  "min",
+  "max",
+  "avg",
+  "average",
+  "mean",
+  "median",
+  "med",
+  "std",
+  "standard_deviation",
+  // Percentile functions
+  "percentile",
+  "percentiles",
+  "p",
+  "pct",
+  "percentile_rank",
+  "percentile_ranks",
+  "pct_rank",
+  "pct_ranks",
+  // Unique values
+  "values",
+  "unique",
+  "distinct"
+], g = [
+  "string",
+  "str",
+  "number",
+  "int",
+  "integer",
+  "float",
+  "double",
+  "decimal",
+  "date",
+  "bool",
+  "boolean",
+  "array",
+  "list",
+  "object",
+  "geo"
+], T = [
+  "bar",
+  "barchart",
+  "line",
+  "area",
+  "pie",
+  "donut",
+  "scatter",
+  "heatmap",
+  "treemap",
+  "sunburst",
+  "table",
+  "number",
+  "gauge",
+  "map",
+  "grouped_bar",
+  "stacked_bar",
+  "nested_pie",
+  "nested_donut",
+  "chord"
+], w = ["by", "as", "null", "true", "false", "stats", "top", "bottom"];
+class b {
+  constructor(e) {
+    a(this, "input");
+    a(this, "position");
+    a(this, "line");
+    a(this, "column");
+    a(this, "tokens");
+    this.input = e, this.position = 0, this.line = 1, this.column = 1, this.tokens = [];
+  }
+  /**
+   * Tokenize the entire input string
+   */
+  tokenize() {
+    for (this.tokens = [], this.position = 0, this.line = 1, this.column = 1; !this.isAtEnd(); ) {
+      const e = this.scanToken();
+      e && this.tokens.push(e);
+    }
+    return this.tokens;
+  }
+  /**
+   * Scan the next token
+   */
+  scanToken() {
+    if (this.isWhitespace(this.peek()))
+      return this.scanWhitespace();
+    const e = this.position, n = this.line, i = this.column, t = this.advance();
+    switch (t) {
+      case "(":
+      case ")":
+        return this.makeToken("paren", t, e, n, i);
+      case "[":
+      case "]":
+        return this.makeToken("bracket", t, e, n, i);
+      case ",":
+        return this.makeToken("comma", t, e, n, i);
+      case "|":
+        return this.makeToken("pipe", t, e, n, i);
+    }
+    return t === "=" && this.peek() === ">" ? (this.advance(), this.makeToken("arrow", "=>", e, n, i)) : t === ":" && this.peek() === ":" ? (this.advance(), this.makeToken("doubleColon", "::", e, n, i)) : t === '"' || t === "'" ? this.scanString(t, e, n, i) : this.isDigit(t) || t === "-" && this.isDigit(this.peek()) ? this.scanNumber(e, n, i) : this.isOperatorChar(t) ? this.scanSymbolOperator(e, n, i) : this.isIdentifierStart(t) ? this.scanIdentifier(e, n, i) : this.makeToken("unknown", t, e, n, i);
+  }
+  /**
+   * Scan whitespace characters
+   */
+  scanWhitespace() {
+    const e = this.position, n = this.line, i = this.column;
+    let t = "";
+    for (; !this.isAtEnd() && this.isWhitespace(this.peek()); ) {
+      const s = this.advance();
+      t += s;
+    }
+    return this.makeToken("whitespace", t, e, n, i);
+  }
+  /**
+   * Scan a string literal (single or double quoted)
+   */
+  scanString(e, n, i, t) {
+    let s = "";
+    const r = e;
+    for (; !this.isAtEnd() && this.peek() !== e; )
+      if (this.peek() === "\\") {
+        if (this.advance(), !this.isAtEnd()) {
+          const h = this.advance();
+          switch (h) {
+            case "n":
+              s += `
+`;
+              break;
+            case "t":
+              s += "	";
+              break;
+            case "r":
+              s += "\r";
+              break;
+            case "\\":
+              s += "\\";
+              break;
+            case '"':
+              s += '"';
+              break;
+            case "'":
+              s += "'";
+              break;
+            default:
+              s += h;
+          }
+        }
+      } else
+        s += this.advance();
+    return this.isAtEnd() ? {
+      type: "error",
+      value: r + s,
+      start: n,
+      end: this.position,
+      line: i,
+      column: t,
+      raw: r + s
+    } : (this.advance(), {
+      type: "string",
+      value: s,
+      start: n,
+      end: this.position,
+      line: i,
+      column: t,
+      raw: e + s + e
+    });
+  }
+  /**
+   * Scan a numeric literal
+   */
+  scanNumber(e, n, i) {
+    this.position = e, this.column = i;
+    let t = "";
+    for (this.peek() === "-" && (t += this.advance()); !this.isAtEnd() && this.isDigit(this.peek()); )
+      t += this.advance();
+    if (this.peek() === "." && this.isDigit(this.peekNext()))
+      for (t += this.advance(); !this.isAtEnd() && this.isDigit(this.peek()); )
+        t += this.advance();
+    if (this.peek() === "e" || this.peek() === "E")
+      for (t += this.advance(), (this.peek() === "+" || this.peek() === "-") && (t += this.advance()); !this.isAtEnd() && this.isDigit(this.peek()); )
+        t += this.advance();
+    return this.makeToken("number", t, e, n, i);
+  }
+  /**
+   * Scan symbol-based operators (==, !=, >, >=, <, <=, !)
+   */
+  scanSymbolOperator(e, n, i) {
+    this.position = e, this.column = i;
+    let t = this.advance();
+    if (!this.isAtEnd()) {
+      const s = t + this.peek();
+      if (s === "=>")
+        return this.advance(), this.makeToken("arrow", "=>", e, n, i);
+      (s in c || s === "==" || s === "!=") && (t = s, this.advance());
+    }
+    if (t === "!") {
+      const s = this.position;
+      let r = "";
+      for (; !this.isAtEnd() && this.isIdentifierChar(this.peek()); )
+        r += this.advance();
+      if (r) {
+        if ("!" + r in d)
+          return this.makeToken("operator", "!" + r, e, n, i);
+        this.position = s, this.column = i + 1;
+      }
+    }
+    return t in c || t === "==" || t === "=" || t === "!=" || t === ">" || t === ">=" || t === "<" || t === "<=" ? this.makeToken("operator", t, e, n, i) : this.makeToken("unknown", t, e, n, i);
+  }
+  /**
+   * Scan an identifier (field name, operator keyword, etc.)
+   */
+  scanIdentifier(e, n, i) {
+    this.position = e, this.column = i;
+    let t = "";
+    for (; !this.isAtEnd() && this.isIdentifierChar(this.peek()); )
+      t += this.advance();
+    const s = this.classifyIdentifier(t);
+    return this.makeToken(s, t, e, n, i);
+  }
+  /**
+   * Classify an identifier into the appropriate token type
+   */
+  classifyIdentifier(e) {
+    const n = e.toUpperCase(), i = e.toLowerCase();
+    return k.includes(n) ? "logicalOp" : f.includes(n) ? "collectionOp" : i === "true" || i === "false" ? "boolean" : i === "null" ? "null" : i === "stats" ? "keyword" : p.includes(i) ? "operator" : _.includes(i) ? "aggregation" : m.includes(i) ? "mutator" : g.includes(i) ? "typeHint" : T.includes(i) ? "visualization" : w.includes(i) ? "keyword" : "field";
+  }
+  // ============================================================================
+  // Helper Methods
+  // ============================================================================
+  makeToken(e, n, i, t, s) {
+    return {
+      type: e,
+      value: n,
+      start: i,
+      end: this.position,
+      line: t,
+      column: s
+    };
+  }
+  isAtEnd() {
+    return this.position >= this.input.length;
+  }
+  peek() {
+    return this.isAtEnd() ? "\0" : this.input[this.position];
+  }
+  peekNext() {
+    return this.position + 1 >= this.input.length ? "\0" : this.input[this.position + 1];
+  }
+  advance() {
+    const e = this.input[this.position];
+    return this.position++, e === `
+` ? (this.line++, this.column = 1) : this.column++, e;
+  }
+  isWhitespace(e) {
+    return e === " " || e === "	" || e === `
+` || e === "\r";
+  }
+  isDigit(e) {
+    return e >= "0" && e <= "9";
+  }
+  isIdentifierStart(e) {
+    return e >= "a" && e <= "z" || e >= "A" && e <= "Z" || e === "_" || e === "@";
+  }
+  isIdentifierChar(e) {
+    return this.isIdentifierStart(e) || this.isDigit(e) || e === "." || // For nested field access
+    e === "-";
+  }
+  isOperatorChar(e) {
+    return e === "=" || e === "!" || e === ">" || e === "<";
+  }
+}
+function v(o) {
+  return new b(o).tokenize();
+}
+function A(o) {
+  return v(o).filter((e) => e.type !== "whitespace");
+}
+function E(o, e) {
+  for (const n of o)
+    if (e >= n.start && e < n.end)
+      return n;
+  return null;
+}
+function S(o, e, n) {
+  return o.filter((i) => i.start < n && i.end > e);
+}
+function I(o, e) {
+  let n = null;
+  for (const i of o)
+    if (i.end <= e)
+      n = i;
+    else
+      break;
+  return n;
+}
+function y(o, e, n = !0) {
+  const i = n ? o.filter((s) => s.type !== "whitespace") : o;
+  let t = null;
+  for (const s of i)
+    if (s.end <= e)
+      t = s;
+    else
+      break;
+  return t;
+}
+export {
+  _ as AGGREGATION_FUNCTIONS,
+  f as COLLECTION_OPERATORS,
+  p as COMPARISON_OPERATORS,
+  w as KEYWORDS,
+  k as LOGICAL_OPERATORS,
+  m as MUTATORS,
+  d as NEGATION_SYMBOL_OPERATORS,
+  c as SYMBOL_OPERATORS,
+  g as TYPE_HINTS,
+  b as TqlTokenizer,
+  T as VISUALIZATION_TYPES,
+  y as findPreviousToken,
+  E as findTokenAtPosition,
+  I as findTokenBefore,
+  S as findTokensInRange,
+  v as tokenizeTql,
+  A as tokenizeTqlNoWhitespace
+};