ath 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +100 -0
- data/Rakefile +6 -0
- data/ath.gemspec +28 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/ath +90 -0
- data/lib/ath/command.rb +100 -0
- data/lib/ath/driver.rb +42 -0
- data/lib/ath/error.rb +2 -0
- data/lib/ath/query.rb +47 -0
- data/lib/ath/scanner.rb +42 -0
- data/lib/ath/shell.rb +130 -0
- data/lib/ath/version.rb +3 -0
- data/lib/ath.rb +13 -0
- metadata +126 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 134345334c460d1e50d1bea0a32bc3ec41fe122b
|
4
|
+
data.tar.gz: b2745a235f660622e72bb026a0adc8a88c124e9a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 391cf7167a4ed37c9bf2ce9f60cea79c3ecadec7053956245c92f66d59fe9a7993e7f3dd3ea72bbf45773b1bbf48eb0f0caa4ca86b895c8b1985c49bb4d5ab7f
|
7
|
+
data.tar.gz: 31ac7b442fc2f96686d676277d227ff560a46e481be39ff6285f4ac329c608d6e289299eea6da7731c8067a6804992ca7f0ff121fb4fd3bb4c9697baa3849ecf
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 winebarrel
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# ath
|
2
|
+
|
3
|
+
ath is a interactive [Amazon Athena](https://aws.amazon.com/athena/) shell.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'ath'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install ath
|
20
|
+
|
21
|
+
## Getting Started
|
22
|
+
|
23
|
+
```
|
24
|
+
$ export ATH_OUTPUT_LOCATION=s3://my-bucket
|
25
|
+
|
26
|
+
$ ath
|
27
|
+
|
28
|
+
default> show databases;
|
29
|
+
default
|
30
|
+
sampledb
|
31
|
+
|
32
|
+
default> /use sampledb
|
33
|
+
sampledb> show tables;
|
34
|
+
elb_logs
|
35
|
+
|
36
|
+
sampledb> select * from elb_logs limit 3;
|
37
|
+
"request_timestamp","elb_name","request_ip","request_port","backend_ip","backend_port","request_processing_time","backend_processing_time","client_response_time","elb_response_code","backend_response_code","received_bytes","sent_bytes","request_verb","url","protocol","user_agent","ssl_cipher","ssl_protocol"
|
38
|
+
"2015-01-01T08:00:00.516940Z","elb_demo_009","240.136.98.149","25858","172.51.67.62","8888","9.99E-4","8.11E-4","0.001561","200","200","0","428","GET","https://www.example.com/articles/746","HTTP/1.1","""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50""","DHE-RSA-AES128-SHA","TLSv1.2"
|
39
|
+
"2015-01-01T08:00:00.902953Z","elb_demo_008","244.46.184.108","27758","172.31.168.31","443","6.39E-4","0.001471","3.73E-4","200","200","0","4231","GET","https://www.example.com/jobs/688","HTTP/1.1","""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1""","DHE-RSA-AES128-SHA","TLSv1.2"
|
40
|
+
"2015-01-01T08:00:01.206255Z","elb_demo_008","240.120.203.212","26378","172.37.170.107","8888","0.001174","4.97E-4","4.89E-4","200","200","0","2075","GET","http://www.example.com/articles/290","HTTP/1.1","""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246""","-","-"
|
41
|
+
|
42
|
+
sampledb> select * from elb_logs limit 3 &
|
43
|
+
QueryExecution 2335c77b-d138-4c5d-89df-12f2781c311b
|
44
|
+
|
45
|
+
sampledb> /desc 2335c77b-d138-4c5d-89df-12f2781c311b
|
46
|
+
{
|
47
|
+
"query_execution_id": "2335c77b-d138-4c5d-89df-12f2781c311b",
|
48
|
+
"query": "select * from elb_logs limit 3",
|
49
|
+
"result_configuration": {
|
50
|
+
"output_location": "s3://sugawara-test/2335c77b-d138-4c5d-89df-12f2781c311b.csv"
|
51
|
+
},
|
52
|
+
"query_execution_context": {
|
53
|
+
"database": "sampledb"
|
54
|
+
},
|
55
|
+
"status": {
|
56
|
+
"state": "SUCCEEDED",
|
57
|
+
"submission_date_time": "2017-07-02 16:29:57 +0900",
|
58
|
+
"completion_date_time": "2017-07-02 16:29:58 +0900"
|
59
|
+
},
|
60
|
+
"statistics": {
|
61
|
+
"engine_execution_time_in_millis": 719,
|
62
|
+
"data_scanned_in_bytes": 422696
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
sampledb> /result 2335c77b-d138-4c5d-89df-12f2781c311b
|
67
|
+
"request_timestamp","elb_name","request_ip","request_port","backend_ip","backend_port","request_processing_time","backend_processing_time","client_response_time","elb_response_code","backend_response_code","received_bytes","sent_bytes","request_verb","url","protocol","user_agent","ssl_cipher","ssl_protocol"
|
68
|
+
"2015-01-01T16:00:00.516940Z","elb_demo_009","242.76.140.141","18201","172.42.159.57","80","0.001448","8.46E-4","9.97E-4","302","302","0","2911","GET","https://www.example.com/articles/817","HTTP/1.1","""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1""","DHE-RSA-AES128-SHA","TLSv1.2"
|
69
|
+
"2015-01-01T16:00:00.902953Z","elb_demo_005","246.233.91.115","1950","172.42.232.155","8888","9.59E-4","0.001703","8.93E-4","200","200","0","3027","GET","http://www.example.com/jobs/509","HTTP/1.1","""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50""","-","-"
|
70
|
+
"2015-01-01T16:00:01.206255Z","elb_demo_002","250.96.73.238","12800","172.34.87.144","80","0.001549","9.68E-4","0.001908","200","200","0","888","GET","http://www.example.com/articles/729","HTTP/1.1","""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246""","-","-"
|
71
|
+
```
|
72
|
+
|
73
|
+
## Usage
|
74
|
+
|
75
|
+
```
|
76
|
+
$ ath -h
|
77
|
+
Usage: ath [options]
|
78
|
+
-p, --profile PROFILE_NAME
|
79
|
+
--credentials-path PATH
|
80
|
+
-k, --access-key ACCESS_KEY
|
81
|
+
-s, --secret-key SECRET_KEY
|
82
|
+
-r, --region REGION
|
83
|
+
--output-location S3URI
|
84
|
+
-d, --database DATABASE
|
85
|
+
-e, --execute QUERY
|
86
|
+
-f, --file QUERY_FILR
|
87
|
+
--debug
|
88
|
+
```
|
89
|
+
|
90
|
+
```
|
91
|
+
default> /help
|
92
|
+
/debug true|false
|
93
|
+
/desc QUERY_EXECUTION_ID
|
94
|
+
/help
|
95
|
+
/list [NUM]
|
96
|
+
/pager PAGER
|
97
|
+
/result QUERY_EXECUTION_ID
|
98
|
+
/stop QUERY_EXECUTION_ID
|
99
|
+
/use DATABASE
|
100
|
+
```
|
data/Rakefile
ADDED
data/ath.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ath/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'ath'
|
8
|
+
spec.version = Ath::VERSION
|
9
|
+
spec.authors = ['winebarrel']
|
10
|
+
spec.email = ['sgwr_dts@yahoo.co.jp']
|
11
|
+
|
12
|
+
spec.summary = %q{ath is a interactive Amazon Athena shell.}
|
13
|
+
spec.description = %q{ath is a interactive Amazon Athena shell.}
|
14
|
+
spec.homepage = 'https://github.com/winebarrel/ath'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = 'exe'
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ['lib']
|
23
|
+
|
24
|
+
spec.add_dependency 'aws-sdk', '>= 2.9.21', '< 3'
|
25
|
+
spec.add_development_dependency 'bundler'
|
26
|
+
spec.add_development_dependency 'rake'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
28
|
+
end
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'ath'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require 'irb'
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/exe/ath
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$LOAD_PATH << File.expand_path('../../lib', __FILE__)
|
3
|
+
|
4
|
+
require 'logger'
|
5
|
+
require 'optparse'
|
6
|
+
require 'ath'
|
7
|
+
|
8
|
+
Version = Ath::VERSION
|
9
|
+
|
10
|
+
options = {
|
11
|
+
database: 'default',
|
12
|
+
output_location: ENV['ATH_OUTPUT_LOCATION'],
|
13
|
+
debug: false,
|
14
|
+
}
|
15
|
+
|
16
|
+
query = nil
|
17
|
+
query_file = nil
|
18
|
+
|
19
|
+
ARGV.options do |opt|
|
20
|
+
aws_opts = {}
|
21
|
+
creds_opts = {}
|
22
|
+
|
23
|
+
begin
|
24
|
+
opt.on('-p', '--profile PROFILE_NAME') {|v| creds_opts[:profile_name] = v }
|
25
|
+
opt.on('' , '--credentials-path PATH') {|v| creds_opts[:path] = v }
|
26
|
+
opt.on('-k', '--access-key ACCESS_KEY') {|v| aws_opts[:access_key] = v }
|
27
|
+
opt.on('-s', '--secret-key SECRET_KEY') {|v| aws_opts[:secret_access_key] = v }
|
28
|
+
opt.on('-r', '--region REGION') {|v| aws_opts[:region] = v }
|
29
|
+
opt.on('' , '--output-location S3URI') {|v| options[:output_location] = v }
|
30
|
+
opt.on('-d', '--database DATABASE') {|v| options[:database] = v }
|
31
|
+
opt.on('-e', '--execute QUERY') {|v| query = v }
|
32
|
+
opt.on('-f', '--file QUERY_FILR') {|v| query_file = v }
|
33
|
+
opt.on('' , '--debug') { options[:debug] = true }
|
34
|
+
opt.parse!
|
35
|
+
|
36
|
+
unless options[:output_location]
|
37
|
+
raise Ath::Error, '"--output-location" or ATH_OUTPUT_LOCATION is required'
|
38
|
+
end
|
39
|
+
|
40
|
+
if not creds_opts.empty?
|
41
|
+
creds = Aws::SharedCredentials.new(credentials_opts)
|
42
|
+
aws_opts[:credentials] = creds
|
43
|
+
end
|
44
|
+
|
45
|
+
Aws.config.update(aws_opts)
|
46
|
+
rescue Ath::Error, OptionParser::ParseError => e
|
47
|
+
$stderr.puts("[ERROR] #{e.message}")
|
48
|
+
exit 1
|
49
|
+
rescue => e
|
50
|
+
$stderr.puts("[ERROR] #{e.message}")
|
51
|
+
puts "\t" + e.backtrace.join("\n\t")
|
52
|
+
exit 1
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
if options[:debug]
|
57
|
+
Aws.config.update(
|
58
|
+
http_wire_trace: true,
|
59
|
+
logger: Logger.new($stdout).tap {|l| l.level = Logger::DEBUG },
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
begin
|
64
|
+
shell = Ath::Shell.new(database: options.delete(:database), options: options)
|
65
|
+
|
66
|
+
if query_file
|
67
|
+
if query_file == '-'
|
68
|
+
query = $stdin.read
|
69
|
+
else
|
70
|
+
query = File.read(query_file)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
if query
|
75
|
+
query.strip!
|
76
|
+
query << ';' if query !~ /[;&]\z/
|
77
|
+
shell.oneshot(query)
|
78
|
+
else
|
79
|
+
shell.start
|
80
|
+
end
|
81
|
+
rescue Interrupt
|
82
|
+
# nothing to do
|
83
|
+
rescue => e
|
84
|
+
if options[:debug]
|
85
|
+
raise e
|
86
|
+
else
|
87
|
+
$stderr.puts("[ERROR] #{[e.message, e.backtrace.first].join("\n\t")}")
|
88
|
+
exit 1
|
89
|
+
end
|
90
|
+
end
|
data/lib/ath/command.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
class Ath::Command
|
2
|
+
MAX_LIST_QUERY = 20
|
3
|
+
|
4
|
+
def initialize(shell:, command:, arg: nil)
|
5
|
+
@shell = shell
|
6
|
+
@command = command
|
7
|
+
@arg = arg
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
out = nil
|
12
|
+
|
13
|
+
case @command
|
14
|
+
when 'debug'
|
15
|
+
if @arg == 'true' or @arg == 'false'
|
16
|
+
@shell.options[:debug] = (@arg =~ /true/)
|
17
|
+
else
|
18
|
+
out = "Usage: /debug true|false"
|
19
|
+
end
|
20
|
+
when 'desc'
|
21
|
+
if @arg
|
22
|
+
query_execution = @shell.driver.get_query_execution(query_execution_id: @arg)
|
23
|
+
out = JSON.pretty_generate(query_execution.to_h)
|
24
|
+
else
|
25
|
+
out = "Usage: /desc QUERY_EXECUTION_ID"
|
26
|
+
end
|
27
|
+
when 'help'
|
28
|
+
out = usage
|
29
|
+
when 'list'
|
30
|
+
query_executions = @shell.driver.list_query_executions
|
31
|
+
query_executions.sort_by! {|qe| qe.status.submission_date_time }.reverse!
|
32
|
+
|
33
|
+
lines = query_executions.map do |qe|
|
34
|
+
line = [
|
35
|
+
qe.status.submission_date_time,
|
36
|
+
qe.query_execution_id,
|
37
|
+
qe.status.state,
|
38
|
+
]
|
39
|
+
|
40
|
+
if qe.query.length > MAX_LIST_QUERY
|
41
|
+
line << qe.query.slice(0, MAX_LIST_QUERY) + '..'
|
42
|
+
else
|
43
|
+
line << qe.query
|
44
|
+
end
|
45
|
+
|
46
|
+
line.join("\s")
|
47
|
+
end
|
48
|
+
|
49
|
+
if @arg
|
50
|
+
lines = lines.slice(0, @arg.to_i)
|
51
|
+
end
|
52
|
+
|
53
|
+
out = lines.join("\n")
|
54
|
+
when 'pager'
|
55
|
+
if @arg
|
56
|
+
@shell.pager = @arg
|
57
|
+
else
|
58
|
+
@shell.pager = nil
|
59
|
+
out = "Using stdout"
|
60
|
+
end
|
61
|
+
when 'result'
|
62
|
+
if @arg
|
63
|
+
out = @shell.driver.get_query_execution_result(query_execution_id: @arg)
|
64
|
+
else
|
65
|
+
out = "Usage: /result QUERY_EXECUTION_ID"
|
66
|
+
end
|
67
|
+
when 'stop'
|
68
|
+
if @arg
|
69
|
+
@shell.driver.stop_query_execution(query_execution_id: @arg)
|
70
|
+
else
|
71
|
+
out = "Usage: /stop QUERY_EXECUTION_ID"
|
72
|
+
end
|
73
|
+
when 'use'
|
74
|
+
if @arg
|
75
|
+
@shell.database = @arg
|
76
|
+
else
|
77
|
+
out = "Usage: /use DATABASE"
|
78
|
+
end
|
79
|
+
else
|
80
|
+
raise Ath::Error, "Unknown command: #{@command}"
|
81
|
+
end
|
82
|
+
|
83
|
+
out
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def usage
|
89
|
+
<<-EOS
|
90
|
+
/debug true|false
|
91
|
+
/desc QUERY_EXECUTION_ID
|
92
|
+
/help
|
93
|
+
/list [NUM]
|
94
|
+
/pager PAGER
|
95
|
+
/result QUERY_EXECUTION_ID
|
96
|
+
/stop QUERY_EXECUTION_ID
|
97
|
+
/use DATABASE
|
98
|
+
EOS
|
99
|
+
end
|
100
|
+
end
|
data/lib/ath/driver.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
class Ath::Driver
|
2
|
+
def initialize(athena:, s3:)
|
3
|
+
@athena = athena
|
4
|
+
@s3 = s3
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_query_execution(query_execution_id:)
|
8
|
+
|
9
|
+
@athena.get_query_execution(query_execution_id: query_execution_id).query_execution
|
10
|
+
end
|
11
|
+
|
12
|
+
def list_query_executions
|
13
|
+
query_execution_ids = @athena.list_query_executions.each_page.flat_map(&:query_execution_ids)
|
14
|
+
@athena.batch_get_query_execution(query_execution_ids: query_execution_ids.slice(0, 50)).query_executions
|
15
|
+
end
|
16
|
+
|
17
|
+
def get_query_execution_result(query_execution_id:)
|
18
|
+
query_execution = @athena.get_query_execution(query_execution_id: query_execution_id).query_execution
|
19
|
+
output_location = query_execution.result_configuration.output_location
|
20
|
+
bucket, key = output_location.sub(%r{\As3://}, '').split('/', 2)
|
21
|
+
tmp = Tempfile.create('ath')
|
22
|
+
|
23
|
+
@s3.get_object(bucket: bucket, key: key) do |chunk|
|
24
|
+
tmp.write(chunk)
|
25
|
+
end
|
26
|
+
|
27
|
+
tmp.flush
|
28
|
+
tmp
|
29
|
+
end
|
30
|
+
|
31
|
+
def start_query_execution(query_string:, database:, output_location:)
|
32
|
+
@athena.start_query_execution(
|
33
|
+
query_string: query_string,
|
34
|
+
query_execution_context: {database: database},
|
35
|
+
result_configuration: { output_location: output_location}
|
36
|
+
)
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop_query_execution(query_execution_id:)
|
40
|
+
@athena.stop_query_execution(query_execution_id: query_execution_id)
|
41
|
+
end
|
42
|
+
end
|
data/lib/ath/error.rb
ADDED
data/lib/ath/query.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
class Ath::Query
|
2
|
+
def initialize(shell:, query:, detach: false)
|
3
|
+
@shell = shell
|
4
|
+
@query = query
|
5
|
+
@detach = detach
|
6
|
+
end
|
7
|
+
|
8
|
+
def run
|
9
|
+
query_execution_id = @shell.driver.start_query_execution(
|
10
|
+
query_string: @query,
|
11
|
+
database: @shell.database,
|
12
|
+
output_location: @shell.options.fetch(:output_location)
|
13
|
+
).query_execution_id
|
14
|
+
|
15
|
+
if @detach
|
16
|
+
return "QueryExecution #{query_execution_id}"
|
17
|
+
end
|
18
|
+
|
19
|
+
abort_waiting = false
|
20
|
+
orig_handler = trap(:INT, proc { abort_waiting = true })
|
21
|
+
query_execution = nil
|
22
|
+
|
23
|
+
begin
|
24
|
+
until abort_waiting
|
25
|
+
query_execution = @shell.driver.get_query_execution(query_execution_id: query_execution_id)
|
26
|
+
|
27
|
+
if query_execution.status.completion_date_time
|
28
|
+
break
|
29
|
+
end
|
30
|
+
|
31
|
+
sleep 1
|
32
|
+
end
|
33
|
+
ensure
|
34
|
+
trap(:INT, orig_handler)
|
35
|
+
end
|
36
|
+
|
37
|
+
if abort_waiting
|
38
|
+
return "QueryExecution #{query_execution_id}: Detach query"
|
39
|
+
end
|
40
|
+
|
41
|
+
if query_execution.status.state == 'SUCCEEDED'
|
42
|
+
@shell.driver.get_query_execution_result(query_execution_id: query_execution_id)
|
43
|
+
else
|
44
|
+
"QueryExecution #{query_execution_id}: #{query_execution.status.state_change_reason}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/ath/scanner.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
class Ath::Scanner
|
2
|
+
def initialize(shell:)
|
3
|
+
@shell = shell
|
4
|
+
@buf = ''
|
5
|
+
end
|
6
|
+
|
7
|
+
def scan(line)
|
8
|
+
ss = StringScanner.new(line)
|
9
|
+
|
10
|
+
if self.empty? and (tok = ss.scan %r{/\w+(?:\s+.*)?\z})
|
11
|
+
cmd, arg = tok.split(/\s+/, 2)
|
12
|
+
cmd.slice!(0)
|
13
|
+
arg.strip! if arg
|
14
|
+
yield(Ath::Command.new(shell: @shell, command: cmd, arg: arg))
|
15
|
+
end
|
16
|
+
|
17
|
+
until ss.eos?
|
18
|
+
@buf << ' '
|
19
|
+
|
20
|
+
if (tok = ss.scan %r{[^'";&]+})
|
21
|
+
@buf << tok
|
22
|
+
elsif (tok = ss.scan /'(?:''|[^'])*'/)
|
23
|
+
@buf << tok
|
24
|
+
elsif (tok = ss.scan /"(?:""|[^"])*"/)
|
25
|
+
@buf << tok
|
26
|
+
elsif (tok = ss.scan /(?:[;&])/)
|
27
|
+
query = @buf.strip
|
28
|
+
@buf.clear
|
29
|
+
raise Ath::Error, 'No query specified' if query.empty?
|
30
|
+
yield(Ath::Query.new(shell: @shell, query: query, detach: (tok == '&')))
|
31
|
+
else
|
32
|
+
raise Ath::Error, 'You have an error in your HiveQL syntax'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
@buf.strip!
|
37
|
+
end
|
38
|
+
|
39
|
+
def empty?
|
40
|
+
@buf.empty?
|
41
|
+
end
|
42
|
+
end
|
data/lib/ath/shell.rb
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
class Ath::Shell
|
2
|
+
HISTORY_FILE = File.join(ENV.fetch('HOME', '.'), '.ath_history')
|
3
|
+
HISTSIZE = 1000
|
4
|
+
|
5
|
+
attr_reader :driver
|
6
|
+
attr_reader :options
|
7
|
+
attr_accessor :database
|
8
|
+
attr_accessor :pager
|
9
|
+
|
10
|
+
def initialize(athena: Aws::Athena::Client.new, s3: Aws::S3::Client.new, database: nil, options: {})
|
11
|
+
@driver = Ath::Driver.new(athena: athena, s3: s3)
|
12
|
+
@options = options
|
13
|
+
@database = database
|
14
|
+
@scanner = Ath::Scanner.new(shell: self)
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
load_history
|
19
|
+
|
20
|
+
while line = Readline.readline(prompt, true)
|
21
|
+
execute_query(line)
|
22
|
+
end
|
23
|
+
|
24
|
+
save_history
|
25
|
+
end
|
26
|
+
|
27
|
+
def oneshot(query)
|
28
|
+
execute_query(query)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def execute_query(line)
|
34
|
+
begin
|
35
|
+
execute_query0(line)
|
36
|
+
rescue => e
|
37
|
+
puts e.message
|
38
|
+
|
39
|
+
if @options[:debug]
|
40
|
+
puts e.backtrace.join("\n\t")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def execute_query0(line)
|
46
|
+
@scanner.scan(line) do |cmd_or_query|
|
47
|
+
out = nil
|
48
|
+
|
49
|
+
case cmd_or_query
|
50
|
+
when Ath::Command
|
51
|
+
out = cmd_or_query.run
|
52
|
+
when Ath::Query
|
53
|
+
out = cmd_or_query.run
|
54
|
+
else
|
55
|
+
raise 'must not happen'
|
56
|
+
end
|
57
|
+
|
58
|
+
print_result(out)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def prompt
|
63
|
+
database = @database || '(none)'
|
64
|
+
|
65
|
+
if @scanner.empty?
|
66
|
+
"#{database}> "
|
67
|
+
else
|
68
|
+
indent = "\s" * (database.length - 1)
|
69
|
+
"#{indent}-> "
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def print_result(out)
|
74
|
+
return unless out
|
75
|
+
|
76
|
+
if out.kind_of?(File)
|
77
|
+
begin
|
78
|
+
cmd = "cat #{out.path}"
|
79
|
+
cmd << " | #{@pager}" if @pager
|
80
|
+
system(cmd)
|
81
|
+
|
82
|
+
begin
|
83
|
+
out.seek(-1, IO::SEEK_END)
|
84
|
+
puts if out.gets !~ /\n\z/
|
85
|
+
rescue Errno::EINVAL
|
86
|
+
puts
|
87
|
+
end
|
88
|
+
ensure
|
89
|
+
out.close
|
90
|
+
end
|
91
|
+
else
|
92
|
+
if @pager
|
93
|
+
Tempfile.create('ath') do |f|
|
94
|
+
f.puts out
|
95
|
+
f.flush
|
96
|
+
system("cat #{f.path} | #{@pager}")
|
97
|
+
end
|
98
|
+
else
|
99
|
+
puts out
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def load_history
|
105
|
+
return unless File.exist?(HISTORY_FILE)
|
106
|
+
|
107
|
+
open(HISTORY_FILE) do |f|
|
108
|
+
f.each_line.map(&:strip).reject(&:empty?).each do |line|
|
109
|
+
Readline::HISTORY.push(line)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def save_history
|
115
|
+
if Readline::HISTORY.length < HISTSIZE
|
116
|
+
offset = Readline::HISTORY.length
|
117
|
+
else
|
118
|
+
offset = HISTORY
|
119
|
+
end
|
120
|
+
|
121
|
+
history = Readline::HISTORY.map(&:strip).reject(&:empty?).uniq
|
122
|
+
history = history.slice(-offset..-1) || []
|
123
|
+
|
124
|
+
return if history.empty?
|
125
|
+
|
126
|
+
open(HISTORY_FILE, 'wb') do |f|
|
127
|
+
history.each {|l| f.puts l }
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/ath/version.rb
ADDED
data/lib/ath.rb
ADDED
metadata
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ath
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- winebarrel
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 2.9.21
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 2.9.21
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: bundler
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rake
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rspec
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '3.0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '3.0'
|
75
|
+
description: ath is a interactive Amazon Athena shell.
|
76
|
+
email:
|
77
|
+
- sgwr_dts@yahoo.co.jp
|
78
|
+
executables:
|
79
|
+
- ath
|
80
|
+
extensions: []
|
81
|
+
extra_rdoc_files: []
|
82
|
+
files:
|
83
|
+
- ".gitignore"
|
84
|
+
- ".rspec"
|
85
|
+
- ".travis.yml"
|
86
|
+
- Gemfile
|
87
|
+
- LICENSE.txt
|
88
|
+
- README.md
|
89
|
+
- Rakefile
|
90
|
+
- ath.gemspec
|
91
|
+
- bin/console
|
92
|
+
- bin/setup
|
93
|
+
- exe/ath
|
94
|
+
- lib/ath.rb
|
95
|
+
- lib/ath/command.rb
|
96
|
+
- lib/ath/driver.rb
|
97
|
+
- lib/ath/error.rb
|
98
|
+
- lib/ath/query.rb
|
99
|
+
- lib/ath/scanner.rb
|
100
|
+
- lib/ath/shell.rb
|
101
|
+
- lib/ath/version.rb
|
102
|
+
homepage: https://github.com/winebarrel/ath
|
103
|
+
licenses:
|
104
|
+
- MIT
|
105
|
+
metadata: {}
|
106
|
+
post_install_message:
|
107
|
+
rdoc_options: []
|
108
|
+
require_paths:
|
109
|
+
- lib
|
110
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
requirements: []
|
121
|
+
rubyforge_project:
|
122
|
+
rubygems_version: 2.5.2
|
123
|
+
signing_key:
|
124
|
+
specification_version: 4
|
125
|
+
summary: ath is a interactive Amazon Athena shell.
|
126
|
+
test_files: []
|