@witchcraft/expressit 0.0.2

package/dist/grammar/createTokens.js

@@ -0,0 +1,843 @@
+import { escapeRegex, isBlank } from "@alanscodelog/utils";
+import { createToken, Lexer } from "chevrotain";
+function clone(token) {
+  const t = {};
+  if (token.name !== void 0)
+    t.name = token.name;
+  if (token.CATEGORIES !== void 0)
+    t.categories = token.CATEGORIES;
+  if (token.GROUP !== void 0)
+    t.group = token.GROUP;
+  if (token.LINE_BREAKS !== void 0)
+    t.line_breaks = token.LINE_BREAKS;
+  if (token.LONGER_ALT !== void 0)
+    t.longet_alt = token.LONGER_ALT;
+  if (token.PATTERN !== void 0)
+    t.pattern = token.PATTERN;
+  if (token.PUSH_MODE !== void 0)
+    t.push_mode = token.PUSH_MODE;
+  if (token.POP_MODE !== void 0)
+    t.pop_mode = token.POP_MODE;
+  return createToken(t);
+}
+function changePushMode(tokens, mode) {
+  return tokens.map((_) => clone({
+    ..._,
+    ...mode ? { PUSH_MODE: typeof mode === "function" ? mode(_.PUSH_MODE) : mode } : {}
+  }));
+}
+function createTokens(opts) {
+  const $ = {};
+  $[
+    "ANY"
+    /* ANY */
+  ] = createToken({
+    name: "ANY",
+    pattern: Lexer.NA
+  });
+  $[
+    "QUOTE_ANY"
+    /* QUOTE_ANY */
+  ] = createToken({
+    name: "QUOTE_ANY",
+    pattern: Lexer.NA
+  });
+  $[
+    "QUOTE_SINGLE"
+    /* QUOTE_SINGLE */
+  ] = createToken({
+    name: "QUOTE_SINGLE",
+    pattern: Lexer.NA
+  });
+  $[
+    "QUOTE_DOUBLE"
+    /* QUOTE_DOUBLE */
+  ] = createToken({
+    name: "QUOTE_DOUBLE",
+    pattern: Lexer.NA
+  });
+  $[
+    "QUOTE_BACKTICK"
+    /* QUOTE_BACKTICK */
+  ] = createToken({
+    name: "QUOTE_BACKTICK",
+    pattern: Lexer.NA
+  });
+  $[
+    "REGEX_ANY"
+    /* REGEX_ANY */
+  ] = createToken({
+    name: "REGEX_ANY",
+    pattern: Lexer.NA
+  });
+  $[
+    "VALUE_FOR_SINGLE"
+    /* VALUE_FOR_SINGLE */
+  ] = createToken({
+    name: "VALUE_FOR_SINGLE",
+    pattern: Lexer.NA
+  });
+  $[
+    "VALUE_FOR_DOUBLE"
+    /* VALUE_FOR_DOUBLE */
+  ] = createToken({
+    name: "VALUE_FOR_DOUBLE",
+    pattern: Lexer.NA
+  });
+  $[
+    "VALUE_FOR_BACKTICK"
+    /* VALUE_FOR_BACKTICK */
+  ] = createToken({
+    name: "VALUE_FOR_BACKTICK",
+    pattern: Lexer.NA
+  });
+  $[
+    "OPERATOR_OR"
+    /* OPERATOR_OR */
+  ] = createToken({
+    name: "OPERATOR_OR",
+    pattern: Lexer.NA
+  });
+  $[
+    "OPERATOR_AND"
+    /* OPERATOR_AND */
+  ] = createToken({
+    name: "OPERATOR_AND",
+    pattern: Lexer.NA
+  });
+  $[
+    "OPERATOR_NOT"
+    /* OPERATOR_NOT */
+  ] = createToken({
+    name: "OPERATOR_NOT",
+    pattern: Lexer.NA
+  });
+  $[
+    "VALUE"
+    /* VALUE */
+  ] = createToken({
+    name: "VALUE",
+    pattern: Lexer.NA
+  });
+  $[
+    "_"
+    /* _ */
+  ] = createToken({
+    name: "_",
+    pattern: /\s+/,
+    group: Lexer.SKIPPED,
+    line_breaks: true
+  });
+  $[
+    "REGEX_START"
+    /* REGEX_START */
+  ] = createToken({
+    name: "REGEX_START",
+    push_mode: "notRegex",
+    pattern: /\//,
+    categories: [$[
+      "REGEX_ANY"
+      /* REGEX_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "REGEX_END"
+    /* REGEX_END */
+  ] = createToken({
+    name: "REGEX_END",
+    push_mode: "main",
+    pattern: /\/[a-z]*/,
+    categories: [$[
+      "REGEX_ANY"
+      /* REGEX_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "VALUE_REGEX"
+    /* VALUE_REGEX */
+  ] = createToken({
+    name: "VALUE_REGEX",
+    push_mode: "regexEnd",
+    line_breaks: true,
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]],
+    pattern: {
+      exec: (text, start) => {
+        let end = start;
+        let inGroup = 0;
+        let char = text[end];
+        let prevEscaped = false;
+        while (char !== void 0 && (char !== "/" || inGroup > 0 || prevEscaped)) {
+          if (char === "[")
+            inGroup++;
+          if (char === "]" && inGroup > 0)
+            inGroup--;
+          if (char === "\\") {
+            if (!prevEscaped) {
+              prevEscaped = true;
+            } else {
+              prevEscaped = false;
+            }
+          } else {
+            prevEscaped && (prevEscaped = false);
+          }
+          end++;
+          char = text[end];
+        }
+        if (start === end)
+          return null;
+        return [text.substring(start, end)];
+      }
+    }
+  });
+  $[
+    "QUOTE_SINGLE_START"
+    /* QUOTE_SINGLE_START */
+  ] = createToken({
+    name: "QUOTE_SINGLE_START",
+    pattern: /'/,
+    push_mode: "notSingle",
+    categories: [$[
+      "QUOTE_SINGLE"
+      /* QUOTE_SINGLE */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "QUOTE_DOUBLE_START"
+    /* QUOTE_DOUBLE_START */
+  ] = createToken({
+    name: "QUOTE_DOUBLE_START",
+    pattern: /"/,
+    push_mode: "notDouble",
+    categories: [$[
+      "QUOTE_DOUBLE"
+      /* QUOTE_DOUBLE */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "QUOTE_BACKTICK_START"
+    /* QUOTE_BACKTICK_START */
+  ] = createToken({
+    name: "QUOTE_BACKTICK_START",
+    pattern: /`/,
+    push_mode: "notBacktick",
+    categories: [$[
+      "QUOTE_BACKTICK"
+      /* QUOTE_BACKTICK */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "QUOTE_SINGLE_END"
+    /* QUOTE_SINGLE_END */
+  ] = createToken({
+    name: "QUOTE_SINGLE_END",
+    pattern: /'/,
+    push_mode: "main",
+    categories: [$[
+      "QUOTE_SINGLE"
+      /* QUOTE_SINGLE */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "QUOTE_DOUBLE_END"
+    /* QUOTE_DOUBLE_END */
+  ] = createToken({
+    name: "QUOTE_DOUBLE_END",
+    pattern: /"/,
+    push_mode: "main",
+    categories: [$[
+      "QUOTE_DOUBLE"
+      /* QUOTE_DOUBLE */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "QUOTE_BACKTICK_END"
+    /* QUOTE_BACKTICK_END */
+  ] = createToken({
+    name: "QUOTE_BACKTICK_END",
+    pattern: /`/,
+    push_mode: "main",
+    categories: [$[
+      "QUOTE_BACKTICK"
+      /* QUOTE_BACKTICK */
+    ], $[
+      "QUOTE_ANY"
+      /* QUOTE_ANY */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "VALUE_NOT_SINGLE"
+    /* VALUE_NOT_SINGLE */
+  ] = createToken({
+    name: "VALUE_NOT_SINGLE",
+    pattern: /(\\[\s\S]|[^'])+/,
+    push_mode: "endQuotes",
+    categories: [$[
+      "VALUE"
+      /* VALUE */
+    ], $[
+      "VALUE_FOR_SINGLE"
+      /* VALUE_FOR_SINGLE */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]],
+    line_breaks: true
+  });
+  $[
+    "VALUE_NOT_DOUBLE"
+    /* VALUE_NOT_DOUBLE */
+  ] = createToken({
+    name: "VALUE_NOT_DOUBLE",
+    pattern: /(\\[\s\S]|[^"])+/,
+    push_mode: "endQuotes",
+    categories: [$[
+      "VALUE"
+      /* VALUE */
+    ], $[
+      "VALUE_FOR_DOUBLE"
+      /* VALUE_FOR_DOUBLE */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]],
+    line_breaks: true
+  });
+  $[
+    "VALUE_NOT_BACKTICK"
+    /* VALUE_NOT_BACKTICK */
+  ] = createToken({
+    name: "VALUE_NOT_BACKTICK",
+    pattern: /(\\[\s\S]|[^`])+/,
+    push_mode: "endQuotes",
+    categories: [$[
+      "VALUE"
+      /* VALUE */
+    ], $[
+      "VALUE_FOR_BACKTICK"
+      /* VALUE_FOR_BACKTICK */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]],
+    line_breaks: true
+  });
+  const symOrs = opts.keywords.or.filter((_) => _.isSymbol).map((_) => escapeRegex(_.value));
+  const symAnds = opts.keywords.and.filter((_) => _.isSymbol).map((_) => escapeRegex(_.value));
+  const symNots = opts.keywords.not.filter((_) => _.isSymbol).map((_) => escapeRegex(_.value));
+  const wordOrs = opts.keywords.or.filter((_) => !_.isSymbol).map((_) => escapeRegex(_.value));
+  const wordAnds = opts.keywords.and.filter((_) => !_.isSymbol).map((_) => escapeRegex(_.value));
+  const wordNots = opts.keywords.not.filter((_) => !_.isSymbol).map((_) => escapeRegex(_.value));
+  let syms = [...symOrs, ...symAnds, ...symNots];
+  const customPropertyOperators = (opts.customPropertyOperators ?? []).map((_) => escapeRegex(_));
+  const expandedPropertySeparator = escapeRegex(opts.expandedPropertySeparator ?? "");
+  if (expandedPropertySeparator)
+    syms.push(expandedPropertySeparator);
+  if (customPropertyOperators)
+    syms = syms.concat(customPropertyOperators);
+  if (opts.regexValues)
+    syms.push("\\/");
+  if (opts.arrayValues) {
+    syms.push("\\[");
+  }
+  $[
+    "VALUE_UNQUOTED"
+    /* VALUE_UNQUOTED */
+  ] = createToken({
+    name: "VALUE_UNQUOTED",
+    pattern: new RegExp(`(\\\\[\\s\\S]|(${syms.length > 0 ? `(?!(${syms.join("|")}))` : ``}[^ \\t()'"\`\\\\]))+`),
+    push_mode: "endQuotes",
+    categories: [$[
+      "VALUE"
+      /* VALUE */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "BRACKET_VALUE_UNQUOTED"
+    /* BRACKET_VALUE_UNQUOTED */
+  ] = createToken({
+    name: "BRACKET_VALUE_UNQUOTED",
+    pattern: /(\\[\s\S]|([^ \]\\t'"`\\]))+/,
+    push_mode: "bracket_endQuotes",
+    categories: [$[
+      "VALUE"
+      /* VALUE */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  const operators = [];
+  $[
+    "SYM_OR"
+    /* SYM_OR */
+  ] = createToken({
+    name: "SYM_OR",
+    pattern: new RegExp(`(${symOrs.join("|")})`),
+    categories: [$[
+      "OPERATOR_OR"
+      /* OPERATOR_OR */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (symOrs.length > 0)
+    operators.push($[
+      "SYM_OR"
+      /* SYM_OR */
+    ]);
+  $[
+    "SYM_AND"
+    /* SYM_AND */
+  ] = createToken({
+    name: "SYM_AND",
+    pattern: new RegExp(`(${symAnds.join("|")})`),
+    categories: [$[
+      "OPERATOR_AND"
+      /* OPERATOR_AND */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (symAnds.length > 0)
+    operators.push($[
+      "SYM_AND"
+      /* SYM_AND */
+    ]);
+  $[
+    "SYM_NOT"
+    /* SYM_NOT */
+  ] = createToken({
+    name: "SYM_NOT",
+    pattern: new RegExp(`(${symNots.join("|")})`),
+    categories: [$[
+      "OPERATOR_NOT"
+      /* OPERATOR_NOT */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (symNots.length > 0)
+    operators.push($[
+      "SYM_NOT"
+      /* SYM_NOT */
+    ]);
+  $[
+    "WORD_OR"
+    /* WORD_OR */
+  ] = createToken({
+    name: "WORD_OR",
+    pattern: new RegExp(`(${wordOrs.join("|")})`),
+    longer_alt: $[
+      "VALUE_UNQUOTED"
+      /* VALUE_UNQUOTED */
+    ],
+    categories: [$[
+      "OPERATOR_OR"
+      /* OPERATOR_OR */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (wordOrs.length > 0)
+    operators.push($[
+      "WORD_OR"
+      /* WORD_OR */
+    ]);
+  $[
+    "WORD_AND"
+    /* WORD_AND */
+  ] = createToken({
+    name: "WORD_AND",
+    pattern: new RegExp(`(${wordAnds.join("|")})`),
+    longer_alt: $[
+      "VALUE_UNQUOTED"
+      /* VALUE_UNQUOTED */
+    ],
+    categories: [$[
+      "OPERATOR_AND"
+      /* OPERATOR_AND */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (wordAnds.length > 0)
+    operators.push($[
+      "WORD_AND"
+      /* WORD_AND */
+    ]);
+  $[
+    "WORD_NOT"
+    /* WORD_NOT */
+  ] = createToken({
+    name: "WORD_NOT",
+    pattern: new RegExp(`(${wordNots.join("|")})`),
+    longer_alt: $[
+      "VALUE_UNQUOTED"
+      /* VALUE_UNQUOTED */
+    ],
+    categories: [$[
+      "OPERATOR_NOT"
+      /* OPERATOR_NOT */
+    ], $[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (wordNots.length > 0)
+    operators.push($[
+      "WORD_NOT"
+      /* WORD_NOT */
+    ]);
+  $[
+    "EXP_PROP_OP"
+    /* EXP_PROP_OP */
+  ] = createToken({
+    name: "EXP_PROP_OP",
+    pattern: new RegExp(`${expandedPropertySeparator}`),
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  if (!isBlank(expandedPropertySeparator))
+    operators.splice(0, 0, $[
+      "EXP_PROP_OP"
+      /* EXP_PROP_OP */
+    ]);
+  $[
+    "CUSTOM_PROP_OP"
+    /* CUSTOM_PROP_OP */
+  ] = createToken({
+    name: "CUSTOM_PROP_OP",
+    pattern: new RegExp(`(${customPropertyOperators.join("|")})`),
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  const customOpEqualsExpandedOrNegationToken = [$[
+    "SYM_NOT"
+    /* SYM_NOT */
+  ].PATTERN, $[
+    "EXP_PROP_OP"
+    /* EXP_PROP_OP */
+  ].PATTERN].find((_) => {
+    var _a;
+    return (_ == null ? void 0 : _.toString()) === ((_a = $[
+      "CUSTOM_PROP_OP"
+      /* CUSTOM_PROP_OP */
+    ].PATTERN) == null ? void 0 : _a.toString());
+  }) !== void 0;
+  if (((customPropertyOperators == null ? void 0 : customPropertyOperators.length) ?? 0) > 0 && !customOpEqualsExpandedOrNegationToken)
+    operators.splice(1, 0, $[
+      "CUSTOM_PROP_OP"
+      /* CUSTOM_PROP_OP */
+    ]);
+  const expandedSepAlsoCustom = customPropertyOperators.includes(expandedPropertySeparator);
+  const customOpAlsoNegation = symNots.length > 0 && (customPropertyOperators == null ? void 0 : customPropertyOperators.find((_) => symNots.includes(_))) !== void 0;
+  $[
+    "PAREN_L"
+    /* PAREN_L */
+  ] = createToken({
+    name: "PAREN_L",
+    pattern: /\(/,
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "PAREN_R"
+    /* PAREN_R */
+  ] = createToken({
+    name: "PAREN_R",
+    pattern: /\)/,
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  const parens = [$[
+    "PAREN_L"
+    /* PAREN_L */
+  ], $[
+    "PAREN_R"
+    /* PAREN_R */
+  ]];
+  $[
+    "BRACKET_L"
+    /* BRACKET_L */
+  ] = createToken({
+    name: "BRACKET_L",
+    pattern: /\[/,
+    push_mode: "bracket_main",
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  $[
+    "BRACKET_R"
+    /* BRACKET_R */
+  ] = createToken({
+    name: "BRACKET_R",
+    pattern: /\]/,
+    push_mode: "main",
+    categories: [$[
+      "ANY"
+      /* ANY */
+    ]]
+  });
+  const quotes = [
+    $[
+      "QUOTE_SINGLE_START"
+      /* QUOTE_SINGLE_START */
+    ],
+    $[
+      "QUOTE_DOUBLE_START"
+      /* QUOTE_DOUBLE_START */
+    ],
+    $[
+      "QUOTE_BACKTICK_START"
+      /* QUOTE_BACKTICK_START */
+    ]
+  ];
+  const quotesEnds = [
+    $[
+      "QUOTE_SINGLE_END"
+      /* QUOTE_SINGLE_END */
+    ],
+    $[
+      "QUOTE_DOUBLE_END"
+      /* QUOTE_DOUBLE_END */
+    ],
+    $[
+      "QUOTE_BACKTICK_END"
+      /* QUOTE_BACKTICK_END */
+    ]
+  ];
+  const toBracket = (mode) => `bracket_${mode}`;
+  const lexerOptions = {
+    modes: {
+      main: [
+        $[
+          "_"
+          /* _ */
+        ],
+        ...parens,
+        ...opts.arrayValues ? [$[
+          "BRACKET_L"
+          /* BRACKET_L */
+        ]] : [],
+        // moves to bracket_main until a bracket pops it back out
+        ...operators,
+        ...quotes,
+        // moves to not*
+        ...opts.regexValues ? [$[
+          "REGEX_START"
+          /* REGEX_START */
+        ]] : [],
+        // moves to notRegex
+        $[
+          "VALUE_UNQUOTED"
+          /* VALUE_UNQUOTED */
+        ]
+        // moves to maybeQuoteError
+      ],
+      endQuotes: [
+        $[
+          "_"
+          /* _ */
+        ],
+        ...parens,
+        ...opts.arrayValues ? [$[
+          "BRACKET_L"
+          /* BRACKET_L */
+        ]] : [],
+        // moves to bracket_main until a bracket pops it back out to main
+        ...operators,
+        ...quotesEnds,
+        $[
+          "VALUE_UNQUOTED"
+          /* VALUE_UNQUOTED */
+        ],
+        // moves to maybeQuoteError
+        ...opts.regexValues ? [$[
+          "REGEX_START"
+          /* REGEX_START */
+        ]] : []
+        // error
+      ],
+      // we can have situations like `a"` where the left quote is missing
+      // we want the quote to match a quote end so that it pushes the state to main again, instead of shifting how everything is parsed
+      maybeQuoteError: [
+        ...quotesEnds,
+        ...opts.regexValues ? [$[
+          "REGEX_END"
+          /* REGEX_END */
+        ]] : []
+      ],
+      // all move to endQuotes
+      notSingle: [$[
+        "VALUE_NOT_SINGLE"
+        /* VALUE_NOT_SINGLE */
+      ], $[
+        "QUOTE_SINGLE_END"
+        /* QUOTE_SINGLE_END */
+      ]],
+      notDouble: [$[
+        "VALUE_NOT_DOUBLE"
+        /* VALUE_NOT_DOUBLE */
+      ], $[
+        "QUOTE_DOUBLE_END"
+        /* QUOTE_DOUBLE_END */
+      ]],
+      notBacktick: [$[
+        "VALUE_NOT_BACKTICK"
+        /* VALUE_NOT_BACKTICK */
+      ], $[
+        "QUOTE_BACKTICK_END"
+        /* QUOTE_BACKTICK_END */
+      ]],
+      ...opts.regexValues ? {
+        notRegex: [
+          $[
+            "VALUE_REGEX"
+            /* VALUE_REGEX */
+          ],
+          $[
+            "REGEX_END"
+            /* REGEX_END */
+          ]
+          // regex is empty
+        ],
+        // moves to regexEnd
+        regexEnd: [$[
+          "REGEX_END"
+          /* REGEX_END */
+        ]]
+        // moves to main
+      } : {},
+      ...opts.arrayValues ? {
+        bracket_main: [
+          ...changePushMode(quotes, toBracket),
+          $[
+            "BRACKET_R"
+            /* BRACKET_R */
+          ],
+          // move back to main
+          $[
+            "BRACKET_VALUE_UNQUOTED"
+            /* BRACKET_VALUE_UNQUOTED */
+          ]
+        ],
+        // all the following follow the same logic as the non-bracket modes, except operators and parens and regexes are not supported and are just parsed as values with BRACKET_VALUE_UNQUOTED
+        // the following tokens are also cloned to push differently: quotes (above), quote values, and quote ends
+        // they can still be matched because their parent categories are also cloned and it's those we match against
+        bracket_endQuotes: [
+          $[
+            "_"
+            /* _ */
+          ],
+          ...changePushMode(quotesEnds, toBracket),
+          $[
+            "BRACKET_R"
+            /* BRACKET_R */
+          ],
+          // move back to main
+          $[
+            "BRACKET_VALUE_UNQUOTED"
+            /* BRACKET_VALUE_UNQUOTED */
+          ]
+        ],
+        bracket_maybeQuoteError: changePushMode(quotesEnds, toBracket),
+        bracket_notSingle: changePushMode([
+          $[
+            "VALUE_NOT_SINGLE"
+            /* VALUE_NOT_SINGLE */
+          ],
+          $[
+            "QUOTE_SINGLE_END"
+            /* QUOTE_SINGLE_END */
+          ]
+        ], toBracket),
+        bracket_notDouble: changePushMode([
+          $[
+            "VALUE_NOT_DOUBLE"
+            /* VALUE_NOT_DOUBLE */
+          ],
+          $[
+            "QUOTE_DOUBLE_END"
+            /* QUOTE_DOUBLE_END */
+          ]
+        ], toBracket),
+        bracket_notBacktick: changePushMode([
+          $[
+            "VALUE_NOT_BACKTICK"
+            /* VALUE_NOT_BACKTICK */
+          ],
+          $[
+            "QUOTE_BACKTICK_END"
+            /* QUOTE_BACKTICK_END */
+          ]
+        ], toBracket)
+      } : {}
+    },
+    defaultMode: "main"
+  };
+  const lexer = new Lexer(lexerOptions);
+  return { tokens: $, lexer, info: { expandedSepAlsoCustom, customOpAlsoNegation } };
+}
+export {
+  createTokens
+};