trino_sql_parser 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 0fab1f747a3ffd1712e2047e1ae823f4a83426f0d69b297d98d5266db69fa3c0
4
+ data.tar.gz: 398bb435ddc860cb2164e001e424aa64cd16da0ac6fec394c9b43be66741947e
5
+ SHA512:
6
+ metadata.gz: 4816c2d9ed1d218a77f8fd8886b074aaebfc4cd839cb577458bcf39014bef8470c4aa8a6b8fd1093760b1b687f93f86020265993c9b7130450966fa8ac969902
7
+ data.tar.gz: e73763ede15e0be526bd1e6498fcbc46a0917d66ad33e88159381724fb4923ddc7007f0e8e13bea54373b3f2e731ff19e01565caaf17d8e21481f75c6f7ed6d6
@@ -0,0 +1,5 @@
1
+ Gemfile.lock
2
+ pkg/*
3
+ /.gradle
4
+ /build/*
5
+ /lib/trino_sql_parser/trino-sql-parser.jar
@@ -0,0 +1,31 @@
1
+
2
+ 2021-01-05 version 1.1.0:
3
+
4
+ * Update Presto 341 to Trino 351.
5
+
6
+ 2020-10-09 version 1.0.0:
7
+
8
+ * Update Presto version from 317 to 341
9
+ * Presto 341 supports (CREATE | REFRESH) MATERIALIZED VIEW statements
10
+
11
+ 2020-05-27 version 0.4.1:
12
+
13
+ * Update Presto version from 316 to 317
14
+ * Updated dependency gem versions
15
+
16
+ 2019-08-02 version 0.4.0:
17
+
18
+ * Support process running more than idle_wait was killed unexpectedly
19
+
20
+ 2019-08-02 version 0.3.0:
21
+
22
+ * Added support for PRESTO_SQL_PARSER_JAVA env var
23
+
24
+ 2019-08-02 version 0.2.0:
25
+
26
+ * Fixed PrestoSqlParser.java_args to work
27
+
28
+ 2019-08-01 version 0.1.0:
29
+
30
+ * The first release
31
+
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source "https://rubygems.org"
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2019 Sadayuki Furuhashi
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in
11
+ all copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ THE SOFTWARE.
20
+
@@ -0,0 +1,140 @@
1
+ # trino_sql_parser
2
+
3
+ Trino SQL Parser for Ruby parses a SQL using Trino's native SQL parser precisely and reports syntax errors.
4
+
5
+ Optionally, it also returns a ANTLR tokens sequence which is useful to analyze and reformat SQL statements.
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'trino_sql_parser'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install trino_sql_parser
22
+
23
+ ## Runtime dependency
24
+
25
+ `java` command must be available because this TrinoSqlParser needs to use Trino's jar file.
26
+
27
+ If `java` is not available, such as in a pre-built Docker container, you would install java using following script:
28
+
29
+ ```bash
30
+ if [[ ! -d ~/java/jre_11.0.4_11 ]]; then
31
+ mkdir -p ~/java
32
+ curl -L "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.4%2B11/OpenJDK11U-jre_x64_linux_hotspot_11.0.4_11.tar.gz" | tar zx -C ~/java
33
+ mv ~/java/jdk-11.0.4+11-jre ~/java/jre_11.0.4_11
34
+ fi
35
+ echo 'export PATH=$HOME/java/jre_11.0.4_11/bin:$PATH' >> ~/.bashrc
36
+ ```
37
+
38
+ On Circle CI 2.0, you can add following configuration to `steps` section:
39
+
40
+ ```yaml
41
+ - type: cache-restore
42
+ key: jre_11.0.4_11
43
+ - run:
44
+ name: Install java
45
+ command: |
46
+ set -xe
47
+ if [[ ! -d ~/java/jre_11.0.4_11 ]]; then
48
+ mkdir -p ~/java
49
+ curl -L "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.4%2B11/OpenJDK11U-jre_x64_linux_hotspot_11.0.4_11.tar.gz" | tar zx -C ~/java
50
+ mv ~/java/jdk-11.0.4+11-jre ~/java/jre_11.0.4_11
51
+ fi
52
+ echo 'export PATH=$HOME/java/jre_11.0.4_11/bin:$PATH' >> $BASH_ENV
53
+ - type: cache-save
54
+ key: jre_11.04_11
55
+ paths:
56
+ - ~/java
57
+ ```
58
+
59
+ ## Usage
60
+
61
+ Most typical use case is checking syntax error as following:
62
+
63
+ ```ruby
64
+ require 'trino_sql_parser'
65
+
66
+ parser = TrinoSqlParser.new
67
+ begin
68
+ parser.parse("syntax error!")
69
+ rescue TrinoSqlParser::ParseError => e
70
+ puts e.message
71
+
72
+ # Detailed error information is available in ParseError#errors
73
+ e.errors.each do |error|
74
+ line = error['line']
75
+ column = error['column']
76
+ message = error['message']
77
+ puts "Error at #{line}:#{column}: #{message}"
78
+ end
79
+ end
80
+ ```
81
+
82
+ Optionally, you can get ANTLR token list. It also supports multiple statements.
83
+
84
+ ```ruby
85
+ require 'trino_sql_parser'
86
+
87
+ sql = <<SQL
88
+ select 1;
89
+
90
+ -- this is another statement:
91
+ select profiles.id, count(*) as count from events
92
+ left join profiles on events.profile_id = profiles.id
93
+ group by 1
94
+ SQL
95
+
96
+ parser = TrinoSqlParser.new(with_tokens: true)
97
+ statements = parser.parse(sql)
98
+
99
+ # First statement's tokens
100
+ p statements[0]['tokens']
101
+ #=> [
102
+ # ["select", "'SELECT'", 1, 0],
103
+ # [" ", "WS", 1, 6],
104
+ # ["1", "INTEGER_VALUE", 1, 7], [";", "DELIMITER", 1, 8]
105
+ # ]
106
+
107
+ # Second statement's tokens
108
+ p statements[1]['tokens']
109
+ ```
110
+
111
+ ## Options
112
+
113
+ ```ruby
114
+ TrinoSqlParser.java_cmd = "java" # java command (default: TRINO_SQL_PARSER_JAVA env var or "java")
115
+ TrinoSqlParser.java_args = [] # command-line arguments of java_cmd
116
+ TrinoSqlParser.java_env = {} # environment variables given to java_cmd
117
+ ```
118
+
119
+ ## Development
120
+
121
+ ### Build
122
+
123
+ ```
124
+ bundle
125
+ bundle exec rake jar # builds jar to lib/trino_sql_parser/trino-sql-parser.jar
126
+ bundle exec rake spec # runs tests
127
+ bundle exec rake build # builds a gem file
128
+ ```
129
+
130
+ ### Release
131
+
132
+ ```
133
+ gem push pkg/trino_sql_parser-<version>.gem
134
+ ```
135
+
136
+ ### Update version and dependencies
137
+
138
+ * Gem version: `VERSION` at `lib/trino_sql_parser/version.rb`
139
+ * Trino version: dependency version at `build.gradle`
140
+
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :jar do
7
+ sh "./gradlew shadowJar"
8
+ cp "build/libs/trino-sql-parser-1.0.0-all.jar", "lib/trino_sql_parser/trino-sql-parser.jar"
9
+ end
10
+
11
+ task :default => [:jar, :spec, :build]
@@ -0,0 +1,43 @@
1
+ plugins {
2
+ id 'com.github.johnrengelman.shadow' version '5.1.0'
3
+ }
4
+ apply plugin: 'com.github.johnrengelman.shadow'
5
+ apply plugin: 'java'
6
+ apply plugin: 'idea'
7
+
8
+ group = 'trino-sql-parser-support-process'
9
+ version = '1.0.0'
10
+
11
+ dependencies {
12
+ compile 'io.trino:trino-parser:351'
13
+ compile "com.fasterxml.jackson.core:jackson-databind:2.9.9"
14
+ compile "com.fasterxml.jackson.datatype:jackson-datatype-guava:2.9.9"
15
+ compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.9"
16
+ }
17
+
18
+ repositories {
19
+ mavenCentral()
20
+ }
21
+
22
+ sourceCompatibility = 1.8
23
+ targetCompatibility = 1.8
24
+
25
+ compileJava.options.encoding = 'UTF-8'
26
+ compileTestJava.options.encoding = 'UTF-8'
27
+
28
+ tasks.withType(JavaCompile) {
29
+ options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
30
+ }
31
+
32
+ shadowJar {
33
+ manifest {
34
+ attributes 'Implementation-Title': project.name,
35
+ 'Implementation-Version': project.version,
36
+ 'Implementation-Vendor-Id': project.group,
37
+ 'Specification-Title': project.name,
38
+ 'Specification-Version': project.version,
39
+ 'Main-Class': 'TrinoSqlParserSupportProcess'
40
+ }
41
+ mergeServiceFiles {
42
+ }
43
+ }
@@ -0,0 +1,5 @@
1
+ distributionBase=GRADLE_USER_HOME
2
+ distributionPath=wrapper/dists
3
+ distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip
4
+ zipStoreBase=GRADLE_USER_HOME
5
+ zipStorePath=wrapper/dists
data/gradlew ADDED
@@ -0,0 +1,172 @@
1
+ #!/usr/bin/env sh
2
+
3
+ ##############################################################################
4
+ ##
5
+ ## Gradle start up script for UN*X
6
+ ##
7
+ ##############################################################################
8
+
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
26
+
27
+ APP_NAME="Gradle"
28
+ APP_BASE_NAME=`basename "$0"`
29
+
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
33
+ # Use the maximum available, or set MAX_FD != -1 to use that value.
34
+ MAX_FD="maximum"
35
+
36
+ warn () {
37
+ echo "$*"
38
+ }
39
+
40
+ die () {
41
+ echo
42
+ echo "$*"
43
+ echo
44
+ exit 1
45
+ }
46
+
47
+ # OS specific support (must be 'true' or 'false').
48
+ cygwin=false
49
+ msys=false
50
+ darwin=false
51
+ nonstop=false
52
+ case "`uname`" in
53
+ CYGWIN* )
54
+ cygwin=true
55
+ ;;
56
+ Darwin* )
57
+ darwin=true
58
+ ;;
59
+ MINGW* )
60
+ msys=true
61
+ ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
65
+ esac
66
+
67
+ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68
+
69
+ # Determine the Java command to use to start the JVM.
70
+ if [ -n "$JAVA_HOME" ] ; then
71
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72
+ # IBM's JDK on AIX uses strange locations for the executables
73
+ JAVACMD="$JAVA_HOME/jre/sh/java"
74
+ else
75
+ JAVACMD="$JAVA_HOME/bin/java"
76
+ fi
77
+ if [ ! -x "$JAVACMD" ] ; then
78
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79
+
80
+ Please set the JAVA_HOME variable in your environment to match the
81
+ location of your Java installation."
82
+ fi
83
+ else
84
+ JAVACMD="java"
85
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86
+
87
+ Please set the JAVA_HOME variable in your environment to match the
88
+ location of your Java installation."
89
+ fi
90
+
91
+ # Increase the maximum file descriptors if we can.
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93
+ MAX_FD_LIMIT=`ulimit -H -n`
94
+ if [ $? -eq 0 ] ; then
95
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96
+ MAX_FD="$MAX_FD_LIMIT"
97
+ fi
98
+ ulimit -n $MAX_FD
99
+ if [ $? -ne 0 ] ; then
100
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
101
+ fi
102
+ else
103
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104
+ fi
105
+ fi
106
+
107
+ # For Darwin, add options to specify how the application appears in the dock
108
+ if $darwin; then
109
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110
+ fi
111
+
112
+ # For Cygwin, switch paths to Windows format before running java
113
+ if $cygwin ; then
114
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
+
118
+ # We build the pattern for arguments to be converted via cygpath
119
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120
+ SEP=""
121
+ for dir in $ROOTDIRSRAW ; do
122
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
123
+ SEP="|"
124
+ done
125
+ OURCYGPATTERN="(^($ROOTDIRS))"
126
+ # Add a user-defined pattern to the cygpath arguments
127
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129
+ fi
130
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
131
+ i=0
132
+ for arg in "$@" ; do
133
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135
+
136
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138
+ else
139
+ eval `echo args$i`="\"$arg\""
140
+ fi
141
+ i=$((i+1))
142
+ done
143
+ case $i in
144
+ (0) set -- ;;
145
+ (1) set -- "$args0" ;;
146
+ (2) set -- "$args0" "$args1" ;;
147
+ (3) set -- "$args0" "$args1" "$args2" ;;
148
+ (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149
+ (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150
+ (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151
+ (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152
+ (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153
+ (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154
+ esac
155
+ fi
156
+
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
161
+ }
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
171
+
172
+ exec "$JAVACMD" "$@"
@@ -0,0 +1,84 @@
1
+ @if "%DEBUG%" == "" @echo off
2
+ @rem ##########################################################################
3
+ @rem
4
+ @rem Gradle startup script for Windows
5
+ @rem
6
+ @rem ##########################################################################
7
+
8
+ @rem Set local scope for the variables with windows NT shell
9
+ if "%OS%"=="Windows_NT" setlocal
10
+
11
+ set DIRNAME=%~dp0
12
+ if "%DIRNAME%" == "" set DIRNAME=.
13
+ set APP_BASE_NAME=%~n0
14
+ set APP_HOME=%DIRNAME%
15
+
16
+ @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17
+ set DEFAULT_JVM_OPTS=
18
+
19
+ @rem Find java.exe
20
+ if defined JAVA_HOME goto findJavaFromJavaHome
21
+
22
+ set JAVA_EXE=java.exe
23
+ %JAVA_EXE% -version >NUL 2>&1
24
+ if "%ERRORLEVEL%" == "0" goto init
25
+
26
+ echo.
27
+ echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28
+ echo.
29
+ echo Please set the JAVA_HOME variable in your environment to match the
30
+ echo location of your Java installation.
31
+
32
+ goto fail
33
+
34
+ :findJavaFromJavaHome
35
+ set JAVA_HOME=%JAVA_HOME:"=%
36
+ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37
+
38
+ if exist "%JAVA_EXE%" goto init
39
+
40
+ echo.
41
+ echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42
+ echo.
43
+ echo Please set the JAVA_HOME variable in your environment to match the
44
+ echo location of your Java installation.
45
+
46
+ goto fail
47
+
48
+ :init
49
+ @rem Get command-line arguments, handling Windows variants
50
+
51
+ if not "%OS%" == "Windows_NT" goto win9xME_args
52
+
53
+ :win9xME_args
54
+ @rem Slurp the command line arguments.
55
+ set CMD_LINE_ARGS=
56
+ set _SKIP=2
57
+
58
+ :win9xME_args_slurp
59
+ if "x%~1" == "x" goto execute
60
+
61
+ set CMD_LINE_ARGS=%*
62
+
63
+ :execute
64
+ @rem Setup the command line
65
+
66
+ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
67
+
68
+ @rem Execute Gradle
69
+ "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
70
+
71
+ :end
72
+ @rem End local scope for the variables with windows NT shell
73
+ if "%ERRORLEVEL%"=="0" goto mainEnd
74
+
75
+ :fail
76
+ rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
77
+ rem the _cmd.exe /c_ return code!
78
+ if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
79
+ exit /b 1
80
+
81
+ :mainEnd
82
+ if "%OS%"=="Windows_NT" endlocal
83
+
84
+ :omega
@@ -0,0 +1,73 @@
1
+ require 'json'
2
+
3
+ class TrinoSqlParser
4
+ module ClassMethods
5
+ attr_accessor :java_cmd
6
+ attr_accessor :java_args
7
+ attr_accessor :java_env
8
+ attr_accessor :jar_path
9
+ end
10
+
11
+ extend ClassMethods
12
+
13
+ self.java_cmd = ENV['TRINO_SQL_PARSER_JAVA']
14
+ if self.java_cmd == nil || self.java_cmd.strip.empty?
15
+ self.java_cmd = "java"
16
+ end
17
+
18
+ self.java_args = []
19
+
20
+ self.java_env = {}
21
+
22
+ self.jar_path = File.join(File.dirname(__FILE__), "trino_sql_parser/trino-sql-parser.jar")
23
+
24
+ class ParseError < StandardError
25
+ def initialize(errors)
26
+ @errors = errors
27
+ message = errors.map do |error|
28
+ "[#{error['line']}:#{error['column']}] #{error['message']}"
29
+ end.join(", ")
30
+ super(message)
31
+ end
32
+
33
+ attr_reader :errors
34
+ end
35
+
36
+ require 'trino_sql_parser/support_process'
37
+
38
+ def initialize(with_tokens: false)
39
+ @support_process = SupportProcess.new(with_tokens: with_tokens)
40
+ end
41
+
42
+ def parse(sql)
43
+ unless sql.is_a?(String)
44
+ raise ArgumentError, "SQL must be a String but got #{sql.class}"
45
+ end
46
+ request_line = JSON.dump({"sql" => sql})
47
+
48
+ success = false
49
+ begin
50
+ @support_process.send_line(request_line)
51
+ response_line = @support_process.receive_line
52
+ raise "Process crashed" unless response_line
53
+ response = JSON.parse(response_line)
54
+ statements = response['statements']
55
+ errors = response['errors']
56
+ success = true
57
+ rescue => e
58
+ raise "Support process failed with an error: #{e}"
59
+ ensure
60
+ @support_process.kill! unless success
61
+ end
62
+
63
+ if errors
64
+ raise ParseError.new(errors)
65
+ end
66
+
67
+ statements.map do |h|
68
+ h.reject! {|k, v| v == nil }
69
+ end
70
+
71
+ statements
72
+ end
73
+ end
@@ -0,0 +1,72 @@
1
+ require 'shellwords'
2
+
3
+ class TrinoSqlParser
4
+ class SupportProcess
5
+ def initialize(idle_wait: 2, with_tokens:)
6
+ @idle_wait = idle_wait
7
+ @with_tokens = with_tokens
8
+ @mutex = Mutex.new
9
+ @last_used_pid = nil
10
+ @pipe = nil
11
+ @pid = nil
12
+ end
13
+
14
+ def start!
15
+ return if @pipe
16
+
17
+ cmd = (
18
+ [TrinoSqlParser.java_cmd] +
19
+ TrinoSqlParser.java_args.map {|arg| Shellwords.escape(arg) } +
20
+ ["-jar", Shellwords.escape(TrinoSqlParser.jar_path)]
21
+ ).join(' ')
22
+
23
+ if @with_tokens
24
+ cmd << " --with-tokens"
25
+ end
26
+
27
+ @pipe = IO.popen(TrinoSqlParser.java_env, cmd, "r+", external_encoding: 'UTF-8')
28
+ @pid = @pipe.pid
29
+ Thread.new(@pid, &method(:monitor_thread))
30
+ end
31
+
32
+ def kill!
33
+ @mutex.synchronize do
34
+ if @pid
35
+ Process.kill("KILL", @pid)
36
+ @pipe.close rescue nil
37
+ @pipe = nil
38
+ @pid = nil
39
+ end
40
+ end
41
+ end
42
+
43
+ def send_line(line)
44
+ @mutex.synchronize do # block kill! during execution
45
+ start! unless @pipe
46
+ @pipe.puts line
47
+ @last_used_pid = @pipe.pid
48
+ end
49
+ nil
50
+ end
51
+
52
+ def receive_line
53
+ @pipe.gets
54
+ end
55
+
56
+ private
57
+
58
+ def monitor_thread(pid)
59
+ while true
60
+ done = Process.waitpid2(pid, Process::WNOHANG) rescue true
61
+ break if done
62
+
63
+ sleep @idle_wait
64
+ if @last_used_pid != pid
65
+ kill! rescue nil
66
+ end
67
+
68
+ @last_used_pid = nil # Next check kills the process
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,3 @@
1
+ class TrinoSqlParser
2
+ VERSION = "1.1.0"
3
+ end
@@ -0,0 +1 @@
1
+ rootProject.name = 'trino-sql-parser'
@@ -0,0 +1,40 @@
1
+ require 'trino_sql_parser'
2
+
3
+ RSpec.describe TrinoSqlParser do
4
+ let(:parser) do
5
+ TrinoSqlParser.new
6
+ end
7
+
8
+ it "raises ParseError::ParseError" do
9
+ expect(lambda { parser.parse("...") }).to raise_error(TrinoSqlParser::ParseError)
10
+ end
11
+
12
+ describe TrinoSqlParser::ParseError do
13
+ it "includes detailed error messages" do
14
+ e = nil
15
+ begin
16
+ parser.parse("xxx; yyy")
17
+ rescue TrinoSqlParser::ParseError
18
+ e = $!
19
+ end
20
+ expect(e.errors[0]['message']).to include("mismatched input 'xxx'")
21
+ expect(e.errors[0]['line']).to eq(1)
22
+ expect(e.errors[1]['message']).to include("mismatched input 'yyy'")
23
+ expect(e.errors[1]['line']).to eq(1)
24
+ end
25
+ end
26
+
27
+ context "with_tokens" do
28
+ let(:parser) do
29
+ TrinoSqlParser.new(with_tokens: true)
30
+ end
31
+
32
+ it "returns token list" do
33
+ statements = parser.parse("select 1; select * from t")
34
+ text, type, line, column = *statements[0]['tokens'].first
35
+ expect(text).to eq("select")
36
+ expect(line).to eq(1)
37
+ expect(column).to eq(0)
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,24 @@
1
+ import com.fasterxml.jackson.annotation.JsonProperty;
2
+
3
+ public class JsonErrorReport
4
+ {
5
+ private final String message;
6
+ private final int line;
7
+ private final int column;
8
+
9
+ public JsonErrorReport(String message, int line, int column)
10
+ {
11
+ this.message = message;
12
+ this.line= line;
13
+ this.column= column;
14
+ }
15
+
16
+ @JsonProperty
17
+ public String getMessage() { return message; }
18
+
19
+ @JsonProperty
20
+ public int getLine() { return line; }
21
+
22
+ @JsonProperty
23
+ public int getColumn() { return column; }
24
+ }
@@ -0,0 +1,14 @@
1
+ import com.fasterxml.jackson.annotation.JsonProperty;
2
+
3
+ public class JsonRequest
4
+ {
5
+ private final String sql;
6
+
7
+ public JsonRequest(@JsonProperty("sql") String sql)
8
+ {
9
+ this.sql = sql;
10
+ }
11
+
12
+ @JsonProperty
13
+ public String getSql() { return sql; }
14
+ }
@@ -0,0 +1,27 @@
1
+ import com.fasterxml.jackson.annotation.JsonProperty;
2
+
3
+ import java.util.List;
4
+
5
+ public class JsonResult
6
+ {
7
+ private final List<JsonStatement> statements;
8
+ private final List<JsonErrorReport> errors;
9
+
10
+ public JsonResult(List<JsonStatement> statements, List<JsonErrorReport> errors)
11
+ {
12
+ if (errors.isEmpty()) {
13
+ this.statements = statements;
14
+ this.errors = null;
15
+ }
16
+ else {
17
+ this.statements = null;
18
+ this.errors = errors;
19
+ }
20
+ }
21
+
22
+ @JsonProperty
23
+ public List<JsonStatement> getStatements() { return statements; }
24
+
25
+ @JsonProperty
26
+ public List<JsonErrorReport> getErrors() { return errors; }
27
+ }
@@ -0,0 +1,64 @@
1
+
2
+ import com.fasterxml.jackson.annotation.JsonFormat;
3
+ import com.fasterxml.jackson.annotation.JsonProperty;
4
+ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
5
+ import io.trino.sql.parser.StatementSplitterWithOffsetRetained.Fragment;
6
+ import io.trino.sql.parser.StatementSplitterWithOffsetRetained;
7
+ import io.trino.sql.tree.Statement;
8
+ import java.util.List;
9
+ import org.antlr.v4.runtime.Token;
10
+ import static com.google.common.collect.ImmutableList.toImmutableList;
11
+
12
+ public class JsonStatement
13
+ {
14
+ @JsonFormat(shape=JsonFormat.Shape.ARRAY)
15
+ @JsonPropertyOrder({"text", "type", "line", "column"})
16
+ public static class JsonToken
17
+ {
18
+ private final String text;
19
+ private final String type;
20
+ private final int line;
21
+ private final int column;
22
+
23
+ public JsonToken(Token token)
24
+ {
25
+ this.text = token.getText();
26
+ this.type = StatementSplitterWithOffsetRetained.getTokenName(token);
27
+ this.line= token.getLine();
28
+ this.column= token.getCharPositionInLine();
29
+ }
30
+
31
+ public String getText() { return text; }
32
+
33
+ public String getType() { return type; }
34
+
35
+ public int getLine() { return line; }
36
+
37
+ public int getColumn() { return column; }
38
+ }
39
+
40
+ private final int line;
41
+ private final int column;
42
+ private final List<JsonToken> tokens;
43
+
44
+ public JsonStatement(Fragment fragment, Statement statement, boolean withTokens)
45
+ {
46
+ this.line= fragment.getLineOffset() + 1;
47
+ this.column= fragment.getFirstLineColumnOffset();
48
+ if (withTokens) {
49
+ this.tokens = fragment.getTokens().stream().map(JsonToken::new).collect(toImmutableList());
50
+ }
51
+ else {
52
+ this.tokens = null;
53
+ }
54
+ }
55
+
56
+ @JsonProperty
57
+ public int getLine() { return line; }
58
+
59
+ @JsonProperty
60
+ public int getColumn() { return column; }
61
+
62
+ @JsonProperty
63
+ public List<JsonToken> getTokens() { return tokens; }
64
+ }
@@ -0,0 +1,82 @@
1
+
2
+ import com.fasterxml.jackson.databind.ObjectMapper;
3
+ import com.fasterxml.jackson.datatype.guava.GuavaModule;
4
+ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
5
+ import com.google.common.collect.ImmutableList;
6
+ import com.google.common.io.ByteStreams;
7
+ import io.trino.sql.parser.ParsingException;
8
+ import io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment;
9
+ import io.trino.sql.parser.ParsingOptions;
10
+ import io.trino.sql.parser.SqlParser;
11
+ import io.trino.sql.parser.StatementSplitterWithOffsetRetained.Fragment;
12
+ import io.trino.sql.parser.StatementSplitterWithOffsetRetained;
13
+ import io.trino.sql.tree.Statement;
14
+ import java.io.BufferedReader;
15
+ import java.io.InputStreamReader;
16
+ import java.util.List;
17
+ import static com.google.common.collect.ImmutableList.toImmutableList;
18
+ import static java.nio.charset.StandardCharsets.UTF_8;
19
+
20
+ public class TrinoSqlParserSupportProcess
21
+ {
22
+ public static void main(String[] args) throws Exception
23
+ {
24
+ boolean withTokens = false;
25
+
26
+ for (String arg : args) {
27
+ switch (arg) {
28
+ case "--with-tokens":
29
+ withTokens = true;
30
+ break;
31
+ default:
32
+ System.err.println("Unknown argument: " + arg);
33
+ System.exit(1);
34
+ }
35
+ }
36
+
37
+ ObjectMapper mapper = new ObjectMapper();
38
+ mapper.registerModule(new GuavaModule());
39
+ mapper.registerModule(new Jdk8Module());
40
+
41
+ BufferedReader in = new BufferedReader(new InputStreamReader(System.in, UTF_8));
42
+ while (true) {
43
+ String line = in.readLine();
44
+ if (line == null) {
45
+ return;
46
+ }
47
+
48
+ JsonRequest request = mapper.readValue(line, JsonRequest.class);
49
+ JsonResult result = parse(request.getSql(), withTokens);
50
+ System.out.println(mapper.writeValueAsString(result));
51
+ }
52
+ }
53
+
54
+ public static JsonResult parse(String sql,
55
+ boolean withTokens)
56
+ {
57
+ ImmutableList.Builder<JsonStatement> statements = ImmutableList.builder();
58
+ ImmutableList.Builder<JsonErrorReport> errors = ImmutableList.builder();
59
+
60
+ List<Fragment> fragments = StatementSplitterWithOffsetRetained.split(sql);
61
+
62
+ SqlParser parser = new SqlParser();
63
+
64
+ for (Fragment fragment : fragments) {
65
+ try {
66
+ Statement statement = parser.createStatement(
67
+ fragment.getStatement(),
68
+ new ParsingOptions(DecimalLiteralTreatment.AS_DOUBLE));
69
+ statements.add(new JsonStatement(fragment, statement, withTokens));
70
+ }
71
+ catch (ParsingException ex) {
72
+ errors.add(
73
+ new JsonErrorReport(
74
+ ex.getErrorMessage(),
75
+ ex.getLineNumber() + fragment.getLineOffset(),
76
+ ex.getColumnNumber() + (ex.getLineNumber() == 1 ? fragment.getFirstLineColumnOffset() : 0)));
77
+ }
78
+ }
79
+
80
+ return new JsonResult(statements.build(), errors.build());
81
+ }
82
+ }
@@ -0,0 +1,150 @@
1
+ // This code is in this package following classes are package-private:
2
+ // io.trino.sql.parser.DelimiterLexer
3
+ package io.trino.sql.parser;
4
+
5
+ import com.google.common.collect.ImmutableList;
6
+ import com.google.common.collect.ImmutableSet;
7
+ import java.util.ArrayList;
8
+ import java.util.List;
9
+ import java.util.Set;
10
+ import java.util.stream.Collectors;
11
+ import java.util.stream.Stream;
12
+ import org.antlr.v4.runtime.ANTLRInputStream;
13
+ import org.antlr.v4.runtime.CharStream;
14
+ import org.antlr.v4.runtime.Token;
15
+ import org.antlr.v4.runtime.TokenSource;
16
+ import org.antlr.v4.runtime.Vocabulary;
17
+
18
+ public class StatementSplitterWithOffsetRetained
19
+ {
20
+ private static final Vocabulary vocabulary = new SqlBaseParser(null).getVocabulary();
21
+
22
+ public static String getTokenName(Token token)
23
+ {
24
+ return vocabulary.getDisplayName(token.getType());
25
+ }
26
+
27
+ public static List<Fragment> split(String sql)
28
+ {
29
+ return new StatementSplitterWithOffsetRetained().run(sql);
30
+ }
31
+
32
+ private int currentLine;
33
+ private int currentCharPositionInLine;
34
+ private List<Token> currentFragmentTokens = new ArrayList<>();
35
+
36
+ private StatementSplitterWithOffsetRetained()
37
+ { }
38
+
39
+ private void resetState(int nextLine, int nextCharPositionInLine)
40
+ {
41
+ this.currentLine = nextLine;
42
+ this.currentCharPositionInLine = nextCharPositionInLine;
43
+ this.currentFragmentTokens.clear();
44
+ }
45
+
46
+ private List<Fragment> run(String sql)
47
+ {
48
+ ImmutableList.Builder<Fragment> results = ImmutableList.builder();
49
+ resetState(1, 0);
50
+
51
+ TokenSource tokens = getLexer(sql, ImmutableSet.of(";"));
52
+ while (true) {
53
+ Token token = tokens.nextToken();
54
+ if (token.getType() == Token.EOF) {
55
+ completeFragmentTo(results, null);
56
+ return results.build();
57
+ }
58
+ else if (token.getType() == SqlBaseParser.DELIMITER) {
59
+ completeFragmentTo(results, token);
60
+ resetState(token.getLine(), token.getCharPositionInLine() + token.getText().length());
61
+ }
62
+ else {
63
+ currentFragmentTokens.add(token);
64
+ }
65
+ }
66
+ }
67
+
68
+ private void completeFragmentTo(ImmutableList.Builder<Fragment> results, Token deliminatorToken)
69
+ {
70
+ String statement = currentFragmentTokens.stream().map(Token::getText).collect(Collectors.joining(""));
71
+ Token firstToken = findFirstNonWhitespaceToken(statement);
72
+ if (firstToken != null) { // skip empty statements
73
+ if (deliminatorToken != null) {
74
+ currentFragmentTokens.add(deliminatorToken);
75
+ }
76
+ int lineOffset = (currentLine - 1) + (firstToken.getLine() - 1);
77
+ int firstLineColumnOffset = firstToken.getCharPositionInLine() + (firstToken.getLine() == 1 ? currentCharPositionInLine : 0);
78
+ statement = removeLines(statement, firstToken.getLine() - 1);
79
+ statement = statement.substring(firstToken.getCharPositionInLine());
80
+ results.add(new Fragment(statement, lineOffset, firstLineColumnOffset, ImmutableList.copyOf(currentFragmentTokens)));
81
+ }
82
+ }
83
+
84
+ private static String removeLines(String string, int count)
85
+ {
86
+ if (count > 0) {
87
+ String[] lines = string.split("\\n");
88
+ return Stream.of(lines).skip(count).collect(Collectors.joining("\n"));
89
+ }
90
+ else {
91
+ return string;
92
+ }
93
+ }
94
+
95
+ private static Token findFirstNonWhitespaceToken(String statement)
96
+ {
97
+ TokenSource tokens = getLexer(statement, ImmutableSet.of());
98
+ while (true) {
99
+ Token token = tokens.nextToken();
100
+ if (token.getType() == Token.EOF) {
101
+ return null;
102
+ }
103
+ if (token.getChannel() != Token.HIDDEN_CHANNEL) {
104
+ return token;
105
+ }
106
+ }
107
+ }
108
+
109
+ private static TokenSource getLexer(String sql, Set<String> terminators)
110
+ {
111
+ CharStream stream = new CaseInsensitiveStream(new ANTLRInputStream(sql));
112
+ return new DelimiterLexer(stream, terminators);
113
+ }
114
+
115
+ public static class Fragment
116
+ {
117
+ private final String statement;
118
+ private final int lineOffset;
119
+ private final int firstLineColumnOffset;
120
+ private final List<Token> tokens;
121
+
122
+ public Fragment(String statement, int lineOffset, int firstLineColumnOffset, List<Token> tokens)
123
+ {
124
+ this.statement = statement;
125
+ this.lineOffset = lineOffset;
126
+ this.firstLineColumnOffset = firstLineColumnOffset;
127
+ this.tokens = tokens;
128
+ }
129
+
130
+ public String getStatement()
131
+ {
132
+ return statement;
133
+ }
134
+
135
+ public int getLineOffset()
136
+ {
137
+ return lineOffset;
138
+ }
139
+
140
+ public int getFirstLineColumnOffset()
141
+ {
142
+ return firstLineColumnOffset;
143
+ }
144
+
145
+ public List<Token> getTokens()
146
+ {
147
+ return tokens;
148
+ }
149
+ }
150
+ }
@@ -0,0 +1,22 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+ require "trino_sql_parser/version"
4
+
5
+ Gem::Specification.new do |gem|
6
+ gem.name = "trino_sql_parser"
7
+ gem.summary = "Trino SQL Parser for Ruby"
8
+ gem.description = "Trino SQL Parser for Ruby parses a SQL using Trino's native SQL parser precisely and reports error if given SQL has syntax errors. Optionally, it returns a ANTLR tokens sequence."
9
+ gem.version = TrinoSqlParser::VERSION
10
+ gem.authors = ["Sadayuki Furuhashi"]
11
+ gem.email = ["frsyuki@gmail.com"]
12
+ gem.homepage = "https://github.com/frsyuki/trino_sql_parser"
13
+ gem.license = "MIT"
14
+ gem.files = `git ls-files`.split("\n") + ["lib/trino_sql_parser/trino-sql-parser.jar"]
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ["lib"]
18
+
19
+ gem.add_development_dependency "bundler"
20
+ gem.add_development_dependency "rake", "~> 13"
21
+ gem.add_development_dependency "rspec", "~> 3"
22
+ end
metadata ADDED
@@ -0,0 +1,112 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: trino_sql_parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Sadayuki Furuhashi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-01-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '13'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '13'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3'
55
+ description: Trino SQL Parser for Ruby parses a SQL using Trino's native SQL parser
56
+ precisely and reports error if given SQL has syntax errors. Optionally, it returns
57
+ a ANTLR tokens sequence.
58
+ email:
59
+ - frsyuki@gmail.com
60
+ executables: []
61
+ extensions: []
62
+ extra_rdoc_files: []
63
+ files:
64
+ - ".gitignore"
65
+ - ChangeLog
66
+ - Gemfile
67
+ - LICENSE
68
+ - README.md
69
+ - Rakefile
70
+ - build.gradle
71
+ - gradle/wrapper/gradle-wrapper.jar
72
+ - gradle/wrapper/gradle-wrapper.properties
73
+ - gradlew
74
+ - gradlew.bat
75
+ - lib/trino_sql_parser.rb
76
+ - lib/trino_sql_parser/support_process.rb
77
+ - lib/trino_sql_parser/trino-sql-parser.jar
78
+ - lib/trino_sql_parser/version.rb
79
+ - settings.gradle
80
+ - spec/trino_sql_parser_spec.rb
81
+ - src/main/java/JsonErrorReport.java
82
+ - src/main/java/JsonRequest.java
83
+ - src/main/java/JsonResult.java
84
+ - src/main/java/JsonStatement.java
85
+ - src/main/java/TrinoSqlParserSupportProcess.java
86
+ - src/main/java/io/trino/sql/parser/StatementSplitterWithOffsetRetained.java
87
+ - trino_sql_parser.gemspec
88
+ homepage: https://github.com/frsyuki/trino_sql_parser
89
+ licenses:
90
+ - MIT
91
+ metadata: {}
92
+ post_install_message:
93
+ rdoc_options: []
94
+ require_paths:
95
+ - lib
96
+ required_ruby_version: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ">="
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ required_rubygems_version: !ruby/object:Gem::Requirement
102
+ requirements:
103
+ - - ">="
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ requirements: []
107
+ rubygems_version: 3.0.3
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: Trino SQL Parser for Ruby
111
+ test_files:
112
+ - spec/trino_sql_parser_spec.rb