dt-sql-parser 4.1.0-beta.1 → 4.1.0-beta.3

package/dist/parser/common/basicSQL.d.ts

@@ -40,8 +40,9 @@ export declare abstract class BasicSQL<L extends Lexer = Lexer, PRC extends Pars
      * @param candidates candidate list
      * @param allTokens all tokens from input
      * @param caretTokenIndex tokenIndex of caretPosition
+     * @param tokenIndexOffset offset of the tokenIndex in the candidates compared to the tokenIndex in allTokens
      */
-    protected abstract processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected abstract processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
     /**
      * Get a new splitListener instance.
      */
@@ -110,7 +111,6 @@ export declare abstract class BasicSQL<L extends Lexer = Lexer, PRC extends Pars
      * @param input source string
      */
     splitSQLByStatement(input: string): TextSlice[] | null;
-    private splitInputBySeparator;
     /**
      * Get suggestions of syntax and token at caretPosition
      * @param input source string

package/dist/parser/common/basicSQL.js

@@ -3,7 +3,6 @@ import { CodeCompletionCore } from 'antlr4-c3';
 import { findCaretTokenIndex } from './findCaretTokenIndex';
 import { ctxToText, tokenToWord } from './textAndWord';
 import { ErrorStrategy } from './errorStrategy';
-const SEPARATOR = ';';
 /**
  * Basic SQL class, every sql needs extends it.
  */
@@ -149,39 +148,6 @@ export class BasicSQL {
             .filter(Boolean);
         return res;
     }
-    // 通过分隔符进一步切分语句
-    splitInputBySeparator(inputSlice, caretTokenIndex) {
-        const allTokens = this.getAllTokens(inputSlice);
-        /**
-         * set startToken
-         */
-        let startToken = null;
-        for (let tokenIndex = caretTokenIndex; tokenIndex >= 0; tokenIndex--) {
-            const token = allTokens[tokenIndex];
-            if ((token === null || token === void 0 ? void 0 : token.text) === SEPARATOR) {
-                startToken = allTokens[tokenIndex + 1];
-                break;
-            }
-        }
-        if (startToken === null) {
-            startToken = allTokens[0];
-        }
-        /**
-         * set stopToken
-         */
-        let stopToken = null;
-        for (let tokenIndex = caretTokenIndex; tokenIndex < allTokens.length; tokenIndex++) {
-            const token = allTokens[tokenIndex];
-            if ((token === null || token === void 0 ? void 0 : token.text) === SEPARATOR) {
-                stopToken = token;
-                break;
-            }
-        }
-        if (stopToken === null) {
-            stopToken = allTokens[allTokens.length - 1];
-        }
-        return { startToken, stopToken };
-    }
     /**
      * Get suggestions of syntax and token at caretPosition
      * @param input source string
@@ -189,13 +155,13 @@ export class BasicSQL {
      * @returns suggestion
      */
     getSuggestionAtCaretPosition(input, caretPosition) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
         const splitListener = this.splitListener;
         this.parseWithCache(input);
         if (!this._parseTree)
             return null;
         let sqlParserIns = this._parser;
-        let allTokens = this.getAllTokens(input);
+        const allTokens = this.getAllTokens(input);
         let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens);
         let c3Context = this._parseTree;
         let tokenIndexOffset = 0;
@@ -240,8 +206,8 @@ export class BasicSQL {
                 }
             }
             // A boundary consisting of the index of the input.
-            let startIndex = (_e = (_d = startStatement === null || startStatement === void 0 ? void 0 : startStatement.start) === null || _d === void 0 ? void 0 : _d.start) !== null && _e !== void 0 ? _e : 0;
-            let stopIndex = (_g = (_f = stopStatement === null || stopStatement === void 0 ? void 0 : stopStatement.stop) === null || _f === void 0 ? void 0 : _f.stop) !== null && _g !== void 0 ? _g : input.length - 1;
+            const startIndex = (_e = (_d = startStatement === null || startStatement === void 0 ? void 0 : startStatement.start) === null || _d === void 0 ? void 0 : _d.start) !== null && _e !== void 0 ? _e : 0;
+            const stopIndex = (_g = (_f = stopStatement === null || stopStatement === void 0 ? void 0 : stopStatement.stop) === null || _f === void 0 ? void 0 : _f.stop) !== null && _g !== void 0 ? _g : input.length - 1;
             /**
              * Save offset of the tokenIndex in the range of input
              * compared to the tokenIndex in the whole input
@@ -252,26 +218,7 @@ export class BasicSQL {
              * Reparse the input fragment，
              * and c3 will collect candidates in the newly generated parseTree.
              */
-            let inputSlice = input.slice(startIndex, stopIndex);
-            /**
-             * Split the inputSlice by separator.
-             */
-            if (inputSlice.includes(SEPARATOR)) {
-                const { startToken, stopToken } = this.splitInputBySeparator(inputSlice, caretTokenIndex);
-                startIndex = (_k = startToken === null || startToken === void 0 ? void 0 : startToken.start) !== null && _k !== void 0 ? _k : 0;
-                stopIndex = (_l = (stopToken === null || stopToken === void 0 ? void 0 : stopToken.stop) + 1) !== null && _l !== void 0 ? _l : input.length;
-                /**
-                 * Save offset of the tokenIndex in the range of input
-                 * compared to the tokenIndex in the whole input
-                 */
-                tokenIndexOffset = (_m = startToken === null || startToken === void 0 ? void 0 : startToken.tokenIndex) !== null && _m !== void 0 ? _m : 0;
-                caretTokenIndex = caretTokenIndex - tokenIndexOffset;
-                /**
-                 * Get the smaller range of inputSlice
-                 */
-                inputSlice = inputSlice.slice(startIndex, stopIndex);
-                allTokens = this.getAllTokens(inputSlice);
-            }
+            const inputSlice = input.slice(startIndex, stopIndex);
             const lexer = this.createLexer(inputSlice);
             lexer.removeErrorListeners();
             const tokenStream = new CommonTokenStream(lexer);
@@ -287,7 +234,7 @@ export class BasicSQL {
         const core = new CodeCompletionCore(sqlParserIns);
         core.preferredRules = this.preferredRules;
         const candidates = core.collectCandidates(caretTokenIndex, c3Context);
-        const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex);
+        const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset);
         const syntaxSuggestions = originalSuggestions.syntax.map((syntaxCtx) => {
             const wordRanges = syntaxCtx.wordRanges.map((token) => {
                 return tokenToWord(token, this._parsedInput);

package/dist/parser/flink/index.d.ts

@@ -16,5 +16,5 @@ export declare class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, Fl
     protected get splitListener(): FlinkSqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): FlinkErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): FlinkEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/flink/index.js

@@ -38,13 +38,13 @@ export class FlinkSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new FlinkEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (let candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case FlinkSqlParser.RULE_catalogPath: {

package/dist/parser/hive/index.d.ts

@@ -16,5 +16,5 @@ export declare class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, Hive
     protected get splitListener(): HiveSqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): HiveErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): HiveEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/hive/index.js

@@ -38,13 +38,13 @@ export class HiveSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new HiveEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (let candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case HiveSqlParser.RULE_dbSchemaName: {

package/dist/parser/impala/index.d.ts

@@ -16,5 +16,5 @@ export declare class ImpalaSQL extends BasicSQL<ImpalaSqlLexer, ProgramContext,
     protected get splitListener(): ImpalaSqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): ImpalaErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): ImpalaEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/impala/index.js

@@ -37,13 +37,13 @@ export class ImpalaSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new ImpalaEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (let candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case ImpalaSqlParser.RULE_functionNameCreate: {

package/dist/parser/mysql/index.d.ts

@@ -16,5 +16,5 @@ export declare class MySQL extends BasicSQL<MySqlLexer, ProgramContext, MySqlPar
     protected get splitListener(): MysqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): MysqlErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): MySqlEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/mysql/index.js

@@ -37,13 +37,13 @@ export class MySQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new MySqlEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (const candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case MySqlParser.RULE_databaseName: {

package/dist/parser/postgresql/index.d.ts

@@ -16,5 +16,5 @@ export declare class PostgreSQL extends BasicSQL<PostgreSqlLexer, ProgramContext
     protected get splitListener(): PostgreSqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): PostgreSqlErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): PostgreSqlEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/postgresql/index.js

@@ -41,13 +41,13 @@ export class PostgreSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new PostgreSqlEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (let candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case PostgreSqlParser.RULE_table_name_create: {

package/dist/parser/spark/index.d.ts

@@ -16,5 +16,5 @@ export declare class SparkSQL extends BasicSQL<SparkSqlLexer, ProgramContext, Sp
     protected get splitListener(): SparkSqlSplitListener;
     protected createErrorListener(_errorListener: ErrorListener): SparkErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): SparkEntityCollector;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/spark/index.js

@@ -37,13 +37,13 @@ export class SparkSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new SparkEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (const candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case SparkSqlParser.RULE_namespaceName: {

package/dist/parser/trino/index.d.ts

@@ -16,5 +16,5 @@ export declare class TrinoSQL extends BasicSQL<TrinoSqlLexer, ProgramContext, Tr
     protected createErrorListener(_errorListener: ErrorListener): TrinoErrorListener;
     protected createEntityCollector(input: string, caretTokenIndex?: number): TrinoEntityCollector;
     protected preferredRules: Set<number>;
-    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number): Suggestions<Token>;
+    protected processCandidates(candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number, tokenIndexOffset: number): Suggestions<Token>;
 }

package/dist/parser/trino/index.js

@@ -39,13 +39,13 @@ export class TrinoSQL extends BasicSQL {
     createEntityCollector(input, caretTokenIndex) {
         return new TrinoEntityCollector(input, caretTokenIndex);
     }
-    processCandidates(candidates, allTokens, caretTokenIndex) {
+    processCandidates(candidates, allTokens, caretTokenIndex, tokenIndexOffset) {
         const originalSyntaxSuggestions = [];
         const keywords = [];
         for (let candidate of candidates.rules) {
             const [ruleType, candidateRule] = candidate;
-            const startTokenIndex = candidateRule.startTokenIndex;
-            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
+            const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
+            const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + tokenIndexOffset + 1);
             let syntaxContextType = void 0;
             switch (ruleType) {
                 case TrinoSqlParser.RULE_catalogRef: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dt-sql-parser",
-  "version": "4.1.0-beta.1",
+  "version": "4.1.0-beta.3",
   "authors": "DTStack Corporation",
   "description": "SQL Parsers for BigData, built with antlr4",
   "keywords": [
@@ -34,7 +34,8 @@
     "format": "prettier --write .",
     "format-g4": "antlr-format -c ./antlr.format.json -v ./src/grammar/**/*.g4",
     "cleanComment": "node ./scripts/cleanCommentCli.js",
-    "benchmark": "node --no-warnings ./scripts/benchmark.js"
+    "benchmark": "NODE_OPTIONS=--max_old_space_size=4096 && node --no-warnings ./scripts/benchmark.js",
+    "benchmark:release": "node --no-warnings ./scripts/benchmark.js --release"
   },
   "license": "MIT",
   "dependencies": {