dt-sql-parser 4.0.0-beta.4.8-no-sourcemap → 4.0.0-beta.4.9

package/README.md

@@ -47,6 +47,9 @@ Additionally, it provides auxiliary functions such as **SQL splitting** and **co
 
 We have provided [monaco-sql-languages](https://github.com/DTStack/monaco-sql-languages), it is easily to integrate with `monaco-editor`.
 
+>Tips: If you want to run `dt-sql-parser` in browser, don't forget to install the `assert` and `util` polyfills, and define the global variable `process.env`.
+  None of this is needed in the node environment, because node has them built-in.
+
 <br/>
 
 ## Installation

package/dist/index.d.ts

@@ -17,7 +17,7 @@ export * from './lib/impala/ImpalaSqlParserListener';
 export * from './lib/impala/ImpalaSqlParserVisitor';
 export { SyntaxContextType } from './parser/common/basic-parser-types';
 export type * from './parser/common/basic-parser-types';
-export type { SyntaxError, ParseError, ErrorHandler } from './parser/common/parseErrorListener';
+export type { SyntaxError, ParseError, ErrorListener } from './parser/common/parseErrorListener';
 /**
  * @deprecated legacy, will be removed.
  */

package/dist/parser/common/basic-parser-types.d.ts

@@ -4,9 +4,9 @@
  */
 export interface CaretPosition {
     /** start at 1 */
-    lineNumber: number;
+    readonly lineNumber: number;
     /** start at 1 */
-    column: number;
+    readonly column: number;
 }
 /**
  * Syntax context type at caret position
@@ -43,22 +43,22 @@ export declare enum SyntaxContextType {
 }
 export interface WordRange {
     /** content of word */
-    text: string;
+    readonly text: string;
     /** start at 0 */
-    startIndex: number;
-    stopIndex: number;
+    readonly startIndex: number;
+    readonly stopIndex: number;
     /** start at 1 */
-    line: number;
+    readonly line: number;
     /** start at 1 */
-    startColumn: number;
-    stopColumn: number;
+    readonly startColumn: number;
+    readonly stopColumn: number;
 }
 /**
  * Suggested information analyzed from the input
  */
 export interface SyntaxSuggestion<T = WordRange> {
-    syntaxContextType: SyntaxContextType;
-    wordRanges: T[];
+    readonly syntaxContextType: SyntaxContextType;
+    readonly wordRanges: T[];
 }
 /**
  * Suggested information analyzed from the input
@@ -67,21 +67,21 @@ export interface Suggestions<T = WordRange> {
     /**
      * Suggestions about syntax
      */
-    syntax: SyntaxSuggestion<T>[];
+    readonly syntax: SyntaxSuggestion<T>[];
     /**
      * Suggestions about keywords
      */
-    keywords: string[];
+    readonly keywords: string[];
 }
 export interface TextSlice {
     /** start at 0 */
-    startIndex: number;
-    endIndex: number;
+    readonly startIndex: number;
+    readonly endIndex: number;
     /** start at 1 */
-    startLine: number;
-    endLine: number;
+    readonly startLine: number;
+    readonly endLine: number;
     /** start at 1 */
-    startColumn: number;
-    endColumn: number;
-    text: string;
+    readonly startColumn: number;
+    readonly endColumn: number;
+    readonly text: string;
 }

package/dist/parser/common/basicParser.d.ts

@@ -2,7 +2,7 @@ import { Parser, Lexer, Token, CommonTokenStream, CodePointCharStream, ParserRul
 import { ParseTreeListener } from 'antlr4ts/tree';
 import { CandidatesCollection } from 'antlr4-c3';
 import { CaretPosition, Suggestions, TextSlice } from './basic-parser-types';
-import { ParseError, ErrorHandler } from './parseErrorListener';
+import { ParseError, ErrorListener } from './parseErrorListener';
 interface IParser<IParserRuleContext extends ParserRuleContext> extends Parser {
     program(): IParserRuleContext;
 }
@@ -22,7 +22,7 @@ export default abstract class BasicParser<L extends Lexer = Lexer, PRC extends P
     protected _parsedInput: string;
     protected _parseErrors: ParseError[];
     /** members for cache end */
-    private _errorHandler;
+    private _errorListener;
     /**
      * PreferredRules for antlr4-c3
      */
@@ -53,19 +53,19 @@ export default abstract class BasicParser<L extends Lexer = Lexer, PRC extends P
      * Create an antlr4 lexer from input.
      * @param input string
      */
-    createLexer(input: string, errorListener?: ErrorHandler<any>): L;
+    createLexer(input: string, errorListener?: ErrorListener<any>): L;
     /**
      * Create an antlr4 parser from input.
      * @param input string
      */
-    createParser(input: string, errorListener?: ErrorHandler<any>): P;
+    createParser(input: string, errorListener?: ErrorListener<any>): P;
     /**
      * Parse input string and return parseTree.
      * @param input string
      * @param errorListener listen parse errors and lexer errors.
      * @returns parseTree
      */
-    parse(input: string, errorListener?: ErrorHandler<any>): PRC;
+    parse(input: string, errorListener?: ErrorListener<any>): PRC;
     /**
      * Create an antlr4 parser from input.
      * And the instances will be cache.

package/dist/parser/common/basicParser.js

@@ -1,8 +1,9 @@
 import { CharStreams, CommonTokenStream, } from 'antlr4ts';
 import { ParseTreeWalker } from 'antlr4ts/tree';
 import { CodeCompletionCore } from 'antlr4-c3';
-import { findCaretTokenIndex } from '../../utils/findCaretTokenIndex';
+import { findCaretTokenIndex } from './utils/findCaretTokenIndex';
 import ParseErrorListener from './parseErrorListener';
+import { ErrorStrategy } from './errorStrategy';
 /**
  * Custom Parser class, subclass needs extends it.
  */
@@ -11,7 +12,7 @@ export default class BasicParser {
         this._parsedInput = null;
         this._parseErrors = [];
         /** members for cache end */
-        this._errorHandler = (error) => {
+        this._errorListener = (error) => {
             this._parseErrors.push(error);
         };
     }
@@ -51,6 +52,7 @@ export default class BasicParser {
     parse(input, errorListener) {
         const parser = this.createParser(input, errorListener);
         parser.buildParseTree = true;
+        parser.errorHandler = new ErrorStrategy();
         return parser.program();
     }
     /**
@@ -63,7 +65,7 @@ export default class BasicParser {
         this._charStreams = CharStreams.fromString(input.toUpperCase());
         this._lexer = this.createLexerFormCharStream(this._charStreams);
         this._lexer.removeErrorListeners();
-        this._lexer.addErrorListener(new ParseErrorListener(this._errorHandler));
+        this._lexer.addErrorListener(new ParseErrorListener(this._errorListener));
         this._tokenStream = new CommonTokenStream(this._lexer);
         /**
          * All tokens are generated in advance.
@@ -73,6 +75,7 @@ export default class BasicParser {
         this._tokenStream.fill();
         this._parser = this.createParserFromTokenStream(this._tokenStream);
         this._parser.buildParseTree = true;
+        this._parser.errorHandler = new ErrorStrategy();
         return this._parser;
     }
     /**
@@ -92,7 +95,7 @@ export default class BasicParser {
         const parser = this.createParserWithCache(input);
         this._parsedInput = input;
         parser.removeErrorListeners();
-        parser.addErrorListener(new ParseErrorListener(this._errorHandler));
+        parser.addErrorListener(new ParseErrorListener(this._errorListener));
         this._parseTree = parser.program();
         return this._parseTree;
     }
@@ -161,6 +164,7 @@ export default class BasicParser {
      * @returns suggestion
      */
     getSuggestionAtCaretPosition(input, caretPosition) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
         const splitListener = this.splitListener;
         // TODO: add splitListener to all sqlParser implements add remove following if
         if (!splitListener)
@@ -175,48 +179,63 @@ export default class BasicParser {
             return null;
         /**
          * Split sql by statement.
-         * Try to collect candidates from the caret statement only.
+         * Try to collect candidates in as small a range as possible.
          */
         this.listen(splitListener, this._parseTree);
+        const statementCount = (_a = splitListener.statementsContext) === null || _a === void 0 ? void 0 : _a.length;
+        const statementsContext = splitListener.statementsContext;
         // If there are multiple statements.
-        if (splitListener.statementsContext.length > 1) {
-            // find statement rule context where caretPosition is located.
-            const caretStatementContext = splitListener === null || splitListener === void 0 ? void 0 : splitListener.statementsContext.find((ctx) => {
-                var _a;
-                return (caretTokenIndex <= ((_a = ctx.stop) === null || _a === void 0 ? void 0 : _a.tokenIndex) &&
-                    caretTokenIndex >= ctx.start.tokenIndex);
-            });
-            if (caretStatementContext) {
-                c3Context = caretStatementContext;
-            }
-            else {
-                const lastStatementToken = splitListener.statementsContext[(splitListener === null || splitListener === void 0 ? void 0 : splitListener.statementsContext.length) - 1]
-                    .start;
+        if (statementCount > 1) {
+            /**
+             * Find a minimum valid range, reparse the fragment, and provide a new parse tree to C3.
+             * The boundaries of this range must be statements with no syntax errors.
+             * This can ensure the stable performance of the C3.
+             */
+            let startStatement;
+            let stopStatement;
+            for (let index = 0; index < statementCount; index++) {
+                const ctx = statementsContext[index];
+                const isCurrentCtxValid = !ctx.exception;
+                if (!isCurrentCtxValid)
+                    continue;
                 /**
-                 * If caretStatementContext is not found and it follows all statements.
-                 * Reparses part of the input following the penultimate statement.
-                 * And c3 will collect candidates in the new parseTreeContext.
+                 * Ensure that the statementContext before the left boundary
+                 * and the last statementContext on the right boundary are qualified SQL statements.
                  */
-                if (caretTokenIndex > (lastStatementToken === null || lastStatementToken === void 0 ? void 0 : lastStatementToken.tokenIndex)) {
-                    /**
-                     * Save offset of the tokenIndex in the partInput
-                     * compared to the tokenIndex in the whole input
-                     */
-                    tokenIndexOffset = lastStatementToken === null || lastStatementToken === void 0 ? void 0 : lastStatementToken.tokenIndex;
-                    // Correct caretTokenIndex
-                    caretTokenIndex = caretTokenIndex - tokenIndexOffset;
-                    const inputSlice = input.slice(lastStatementToken.startIndex);
-                    const lexer = this.createLexer(inputSlice);
-                    lexer.removeErrorListeners();
-                    const tokenStream = new CommonTokenStream(lexer);
-                    tokenStream.fill();
-                    const parser = this.createParserFromTokenStream(tokenStream);
-                    parser.removeErrorListeners();
-                    parser.buildParseTree = true;
-                    sqlParserIns = parser;
-                    c3Context = parser.program();
+                const isPrevCtxValid = index === 0 || !((_b = statementsContext[index - 1]) === null || _b === void 0 ? void 0 : _b.exception);
+                const isNextCtxValid = index === statementCount - 1 || !((_c = statementsContext[index + 1]) === null || _c === void 0 ? void 0 : _c.exception);
+                if (ctx.stop.tokenIndex < caretTokenIndex && isPrevCtxValid) {
+                    startStatement = ctx;
+                }
+                if (!stopStatement && ctx.start.tokenIndex > caretTokenIndex && isNextCtxValid) {
+                    stopStatement = ctx;
+                    break;
                 }
             }
+            // A boundary consisting of the index of the input.
+            const startIndex = (_e = (_d = startStatement === null || startStatement === void 0 ? void 0 : startStatement.start) === null || _d === void 0 ? void 0 : _d.startIndex) !== null && _e !== void 0 ? _e : 0;
+            const stopIndex = (_g = (_f = stopStatement === null || stopStatement === void 0 ? void 0 : stopStatement.stop) === null || _f === void 0 ? void 0 : _f.stopIndex) !== null && _g !== void 0 ? _g : input.length - 1;
+            /**
+             * Save offset of the tokenIndex in the range of input
+             * compared to the tokenIndex in the whole input
+             */
+            tokenIndexOffset = (_j = (_h = startStatement === null || startStatement === void 0 ? void 0 : startStatement.start) === null || _h === void 0 ? void 0 : _h.tokenIndex) !== null && _j !== void 0 ? _j : 0;
+            caretTokenIndex = caretTokenIndex - tokenIndexOffset;
+            /**
+             * Reparse the input fragment，
+             * and c3 will collect candidates in the newly generated parseTree.
+             */
+            const inputSlice = input.slice(startIndex, stopIndex);
+            const lexer = this.createLexer(inputSlice);
+            lexer.removeErrorListeners();
+            const tokenStream = new CommonTokenStream(lexer);
+            tokenStream.fill();
+            const parser = this.createParserFromTokenStream(tokenStream);
+            parser.removeErrorListeners();
+            parser.buildParseTree = true;
+            parser.errorHandler = new ErrorStrategy();
+            sqlParserIns = parser;
+            c3Context = parser.program();
         }
         const core = new CodeCompletionCore(sqlParserIns);
         core.preferredRules = this.preferredRules;

package/dist/parser/common/errorStrategy.d.ts

@@ -0,0 +1,12 @@
+import { DefaultErrorStrategy } from 'antlr4ts/DefaultErrorStrategy';
+import { Parser } from 'antlr4ts/Parser';
+import { RecognitionException } from 'antlr4ts/RecognitionException';
+import { Token } from 'antlr4ts/Token';
+/**
+ * Base on DefaultErrorStrategy.
+ * The difference is that it assigns exception to the context.exception when it encounters error.
+ */
+export declare class ErrorStrategy extends DefaultErrorStrategy {
+    recover(recognizer: Parser, e: RecognitionException): void;
+    recoverInline(recognizer: Parser): Token;
+}

package/dist/parser/common/errorStrategy.js

@@ -0,0 +1,51 @@
+import { DefaultErrorStrategy } from 'antlr4ts/DefaultErrorStrategy';
+import { InputMismatchException } from 'antlr4ts/InputMismatchException';
+import { IntervalSet } from 'antlr4ts/misc/IntervalSet';
+/**
+ * Base on DefaultErrorStrategy.
+ * The difference is that it assigns exception to the context.exception when it encounters error.
+ */
+export class ErrorStrategy extends DefaultErrorStrategy {
+    recover(recognizer, e) {
+        // Mark the context as an anomaly
+        for (let context = recognizer.context; context; context = context.parent) {
+            context.exception = e;
+        }
+        // Error recovery
+        if (this.lastErrorIndex === recognizer.inputStream.index &&
+            this.lastErrorStates &&
+            this.lastErrorStates.contains(recognizer.state)) {
+            recognizer.consume();
+        }
+        this.lastErrorIndex = recognizer.inputStream.index;
+        if (!this.lastErrorStates) {
+            this.lastErrorStates = new IntervalSet();
+        }
+        this.lastErrorStates.add(recognizer.state);
+        let followSet = this.getErrorRecoverySet(recognizer);
+        this.consumeUntil(recognizer, followSet);
+    }
+    recoverInline(recognizer) {
+        let e;
+        if (this.nextTokensContext === undefined) {
+            e = new InputMismatchException(recognizer);
+        }
+        else {
+            e = new InputMismatchException(recognizer, this.nextTokensState, this.nextTokensContext);
+        }
+        // Mark the context as an anomaly
+        for (let context = recognizer.context; context; context = context.parent) {
+            context.exception = e;
+        }
+        // Error recovery
+        let matchedSymbol = this.singleTokenDeletion(recognizer);
+        if (matchedSymbol) {
+            recognizer.consume();
+            return matchedSymbol;
+        }
+        if (this.singleTokenInsertion(recognizer)) {
+            return this.getMissingSymbol(recognizer);
+        }
+        throw e;
+    }
+}

package/dist/parser/common/parseErrorListener.d.ts

@@ -4,30 +4,30 @@ import { ATNSimulator } from 'antlr4ts/atn/ATNSimulator';
  * Converted from {@link SyntaxError}.
  */
 export interface ParseError {
-    startLine: number;
-    endLine: number;
-    startCol: number;
-    endCol: number;
-    message: string;
+    readonly startLine: number;
+    readonly endLine: number;
+    readonly startCol: number;
+    readonly endCol: number;
+    readonly message: string;
 }
 /**
  * The type of error resulting from lexical parsing and parsing.
  */
 export interface SyntaxError<T> {
-    recognizer: Recognizer<T, ATNSimulator>;
-    offendingSymbol: Token;
-    line: number;
-    charPositionInLine: number;
-    msg: string;
-    e: RecognitionException;
+    readonly recognizer: Recognizer<T, ATNSimulator>;
+    readonly offendingSymbol: Token;
+    readonly line: number;
+    readonly charPositionInLine: number;
+    readonly msg: string;
+    readonly e: RecognitionException;
 }
 /**
- * ErrorHandler will be invoked when it encounters a parsing error.
+ * ErrorListener will be invoked when it encounters a parsing error.
  * Includes lexical errors and parsing errors.
  */
-export type ErrorHandler<T> = (parseError: ParseError, originalError: SyntaxError<T>) => void;
+export type ErrorListener<T> = (parseError: ParseError, originalError: SyntaxError<T>) => void;
 export default class ParseErrorListener implements ANTLRErrorListener<Token> {
-    private _errorHandler;
-    constructor(errorListener: ErrorHandler<Token>);
+    private _errorListener;
+    constructor(errorListener: ErrorListener<Token>);
     syntaxError(recognizer: Recognizer<Token, ATNSimulator>, offendingSymbol: any, line: number, charPositionInLine: number, msg: string, e: RecognitionException): void;
 }

package/dist/parser/common/parseErrorListener.js

@@ -1,14 +1,14 @@
 export default class ParseErrorListener {
     constructor(errorListener) {
-        this._errorHandler = errorListener;
+        this._errorListener = errorListener;
     }
     syntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e) {
         let endCol = charPositionInLine + 1;
         if (offendingSymbol && offendingSymbol.text !== null) {
             endCol = charPositionInLine + offendingSymbol.text.length;
         }
-        if (this._errorHandler) {
-            this._errorHandler({
+        if (this._errorListener) {
+            this._errorListener({
                 startLine: line,
                 endLine: line,
                 startCol: charPositionInLine,

package/dist/{utils → parser/common/utils}/findCaretTokenIndex.d.ts

@@ -1,5 +1,5 @@
 import { Token } from 'antlr4ts';
-import { CaretPosition } from '../parser/common/basic-parser-types';
+import { CaretPosition } from '../basic-parser-types';
 /**
  * find token index via caret position (cursor position)
  * @param caretPosition

package/dist/parser/mysql.d.ts

@@ -4,7 +4,7 @@ import { MySqlLexer } from '../lib/mysql/MySqlLexer';
 import { MySqlParser, ProgramContext, SingleStatementContext } from '../lib/mysql/MySqlParser';
 import BasicParser from './common/basicParser';
 import { Suggestions } from './common/basic-parser-types';
-import { MySqlParserListener } from 'src/lib/mysql/MySqlParserListener';
+import { MySqlParserListener } from '../lib/mysql/MySqlParserListener';
 export default class MySQL extends BasicParser<MySqlLexer, ProgramContext, MySqlParser> {
     protected createLexerFormCharStream(charStreams: any): MySqlLexer;
     protected createParserFromTokenStream(tokenStream: any): MySqlParser;

package/dist/parser/spark.d.ts

@@ -4,7 +4,7 @@ import { SparkSqlLexer } from '../lib/spark/SparkSqlLexer';
 import { SparkSqlParser, ProgramContext, SingleStatementContext } from '../lib/spark/SparkSqlParser';
 import BasicParser from './common/basicParser';
 import { Suggestions } from './common/basic-parser-types';
-import { SparkSqlParserListener } from 'src/lib/spark/SparkSqlParserListener';
+import { SparkSqlParserListener } from '../lib/spark/SparkSqlParserListener';
 export default class SparkSQL extends BasicParser<SparkSqlLexer, ProgramContext, SparkSqlParser> {
     protected createLexerFormCharStream(charStreams: any): SparkSqlLexer;
     protected createParserFromTokenStream(tokenStream: any): SparkSqlParser;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dt-sql-parser",
-  "version": "4.0.0-beta.4.8-no-sourcemap",
+  "version": "4.0.0-beta.4.9",
   "authors": "DTStack Corporation",
   "description": "SQL Parsers for BigData, built with antlr4",
   "keywords": [
@@ -28,6 +28,7 @@
     "release": "node ./scripts/release.js",
     "lint": "prettier --check '**/*.ts' --config ./.prettierrc",
     "lint-fix": "prettier --write '**/*.ts' --config ./.prettierrc",
+    "format-g4": "antlr-format -c ./antlr.format.json -v ./src/grammar/**/*.g4",
     "cleanComment": "node ./scripts/cleanCommentCli.js"
   },
   "license": "MIT",
@@ -39,6 +40,7 @@
     "@swc/jest": "^0.2.26",
     "@types/jest": "^29.5.1",
     "@types/node": "^18.15.11",
+    "antlr-format-cli": "^1.2.1",
     "antlr4ts-cli": "^0.5.0-alpha.4",
     "chalk": "4.1.2",
     "commitizen": "^4.3.0",