ath 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +100 -0
- data/Rakefile +6 -0
- data/ath.gemspec +28 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/ath +90 -0
- data/lib/ath/command.rb +100 -0
- data/lib/ath/driver.rb +42 -0
- data/lib/ath/error.rb +2 -0
- data/lib/ath/query.rb +47 -0
- data/lib/ath/scanner.rb +42 -0
- data/lib/ath/shell.rb +130 -0
- data/lib/ath/version.rb +3 -0
- data/lib/ath.rb +13 -0
- metadata +126 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 134345334c460d1e50d1bea0a32bc3ec41fe122b
|
4
|
+
data.tar.gz: b2745a235f660622e72bb026a0adc8a88c124e9a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 391cf7167a4ed37c9bf2ce9f60cea79c3ecadec7053956245c92f66d59fe9a7993e7f3dd3ea72bbf45773b1bbf48eb0f0caa4ca86b895c8b1985c49bb4d5ab7f
|
7
|
+
data.tar.gz: 31ac7b442fc2f96686d676277d227ff560a46e481be39ff6285f4ac329c608d6e289299eea6da7731c8067a6804992ca7f0ff121fb4fd3bb4c9697baa3849ecf
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 winebarrel
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# ath
|
2
|
+
|
3
|
+
ath is a interactive [Amazon Athena](https://aws.amazon.com/athena/) shell.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'ath'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install ath
|
20
|
+
|
21
|
+
## Getting Started
|
22
|
+
|
23
|
+
```
|
24
|
+
$ export ATH_OUTPUT_LOCATION=s3://my-bucket
|
25
|
+
|
26
|
+
$ ath
|
27
|
+
|
28
|
+
default> show databases;
|
29
|
+
default
|
30
|
+
sampledb
|
31
|
+
|
32
|
+
default> /use sampledb
|
33
|
+
sampledb> show tables;
|
34
|
+
elb_logs
|
35
|
+
|
36
|
+
sampledb> select * from elb_logs limit 3;
|
37
|
+
"request_timestamp","elb_name","request_ip","request_port","backend_ip","backend_port","request_processing_time","backend_processing_time","client_response_time","elb_response_code","backend_response_code","received_bytes","sent_bytes","request_verb","url","protocol","user_agent","ssl_cipher","ssl_protocol"
|
38
|
+
"2015-01-01T08:00:00.516940Z","elb_demo_009","240.136.98.149","25858","172.51.67.62","8888","9.99E-4","8.11E-4","0.001561","200","200","0","428","GET","https://www.example.com/articles/746","HTTP/1.1","""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50""","DHE-RSA-AES128-SHA","TLSv1.2"
|
39
|
+
"2015-01-01T08:00:00.902953Z","elb_demo_008","244.46.184.108","27758","172.31.168.31","443","6.39E-4","0.001471","3.73E-4","200","200","0","4231","GET","https://www.example.com/jobs/688","HTTP/1.1","""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1""","DHE-RSA-AES128-SHA","TLSv1.2"
|
40
|
+
"2015-01-01T08:00:01.206255Z","elb_demo_008","240.120.203.212","26378","172.37.170.107","8888","0.001174","4.97E-4","4.89E-4","200","200","0","2075","GET","http://www.example.com/articles/290","HTTP/1.1","""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246""","-","-"
|
41
|
+
|
42
|
+
sampledb> select * from elb_logs limit 3 &
|
43
|
+
QueryExecution 2335c77b-d138-4c5d-89df-12f2781c311b
|
44
|
+
|
45
|
+
sampledb> /desc 2335c77b-d138-4c5d-89df-12f2781c311b
|
46
|
+
{
|
47
|
+
"query_execution_id": "2335c77b-d138-4c5d-89df-12f2781c311b",
|
48
|
+
"query": "select * from elb_logs limit 3",
|
49
|
+
"result_configuration": {
|
50
|
+
"output_location": "s3://sugawara-test/2335c77b-d138-4c5d-89df-12f2781c311b.csv"
|
51
|
+
},
|
52
|
+
"query_execution_context": {
|
53
|
+
"database": "sampledb"
|
54
|
+
},
|
55
|
+
"status": {
|
56
|
+
"state": "SUCCEEDED",
|
57
|
+
"submission_date_time": "2017-07-02 16:29:57 +0900",
|
58
|
+
"completion_date_time": "2017-07-02 16:29:58 +0900"
|
59
|
+
},
|
60
|
+
"statistics": {
|
61
|
+
"engine_execution_time_in_millis": 719,
|
62
|
+
"data_scanned_in_bytes": 422696
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
sampledb> /result 2335c77b-d138-4c5d-89df-12f2781c311b
|
67
|
+
"request_timestamp","elb_name","request_ip","request_port","backend_ip","backend_port","request_processing_time","backend_processing_time","client_response_time","elb_response_code","backend_response_code","received_bytes","sent_bytes","request_verb","url","protocol","user_agent","ssl_cipher","ssl_protocol"
|
68
|
+
"2015-01-01T16:00:00.516940Z","elb_demo_009","242.76.140.141","18201","172.42.159.57","80","0.001448","8.46E-4","9.97E-4","302","302","0","2911","GET","https://www.example.com/articles/817","HTTP/1.1","""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1""","DHE-RSA-AES128-SHA","TLSv1.2"
|
69
|
+
"2015-01-01T16:00:00.902953Z","elb_demo_005","246.233.91.115","1950","172.42.232.155","8888","9.59E-4","0.001703","8.93E-4","200","200","0","3027","GET","http://www.example.com/jobs/509","HTTP/1.1","""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50""","-","-"
|
70
|
+
"2015-01-01T16:00:01.206255Z","elb_demo_002","250.96.73.238","12800","172.34.87.144","80","0.001549","9.68E-4","0.001908","200","200","0","888","GET","http://www.example.com/articles/729","HTTP/1.1","""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246""","-","-"
|
71
|
+
```
|
72
|
+
|
73
|
+
## Usage
|
74
|
+
|
75
|
+
```
|
76
|
+
$ ath -h
|
77
|
+
Usage: ath [options]
|
78
|
+
-p, --profile PROFILE_NAME
|
79
|
+
--credentials-path PATH
|
80
|
+
-k, --access-key ACCESS_KEY
|
81
|
+
-s, --secret-key SECRET_KEY
|
82
|
+
-r, --region REGION
|
83
|
+
--output-location S3URI
|
84
|
+
-d, --database DATABASE
|
85
|
+
-e, --execute QUERY
|
86
|
+
-f, --file QUERY_FILR
|
87
|
+
--debug
|
88
|
+
```
|
89
|
+
|
90
|
+
```
|
91
|
+
default> /help
|
92
|
+
/debug true|false
|
93
|
+
/desc QUERY_EXECUTION_ID
|
94
|
+
/help
|
95
|
+
/list [NUM]
|
96
|
+
/pager PAGER
|
97
|
+
/result QUERY_EXECUTION_ID
|
98
|
+
/stop QUERY_EXECUTION_ID
|
99
|
+
/use DATABASE
|
100
|
+
```
|
data/Rakefile
ADDED
data/ath.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ath/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'ath'
|
8
|
+
spec.version = Ath::VERSION
|
9
|
+
spec.authors = ['winebarrel']
|
10
|
+
spec.email = ['sgwr_dts@yahoo.co.jp']
|
11
|
+
|
12
|
+
spec.summary = %q{ath is a interactive Amazon Athena shell.}
|
13
|
+
spec.description = %q{ath is a interactive Amazon Athena shell.}
|
14
|
+
spec.homepage = 'https://github.com/winebarrel/ath'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = 'exe'
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ['lib']
|
23
|
+
|
24
|
+
spec.add_dependency 'aws-sdk', '>= 2.9.21', '< 3'
|
25
|
+
spec.add_development_dependency 'bundler'
|
26
|
+
spec.add_development_dependency 'rake'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
28
|
+
end
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'ath'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require 'irb'
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/exe/ath
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$LOAD_PATH << File.expand_path('../../lib', __FILE__)
|
3
|
+
|
4
|
+
require 'logger'
|
5
|
+
require 'optparse'
|
6
|
+
require 'ath'
|
7
|
+
|
8
|
+
Version = Ath::VERSION
|
9
|
+
|
10
|
+
options = {
|
11
|
+
database: 'default',
|
12
|
+
output_location: ENV['ATH_OUTPUT_LOCATION'],
|
13
|
+
debug: false,
|
14
|
+
}
|
15
|
+
|
16
|
+
query = nil
|
17
|
+
query_file = nil
|
18
|
+
|
19
|
+
ARGV.options do |opt|
|
20
|
+
aws_opts = {}
|
21
|
+
creds_opts = {}
|
22
|
+
|
23
|
+
begin
|
24
|
+
opt.on('-p', '--profile PROFILE_NAME') {|v| creds_opts[:profile_name] = v }
|
25
|
+
opt.on('' , '--credentials-path PATH') {|v| creds_opts[:path] = v }
|
26
|
+
opt.on('-k', '--access-key ACCESS_KEY') {|v| aws_opts[:access_key] = v }
|
27
|
+
opt.on('-s', '--secret-key SECRET_KEY') {|v| aws_opts[:secret_access_key] = v }
|
28
|
+
opt.on('-r', '--region REGION') {|v| aws_opts[:region] = v }
|
29
|
+
opt.on('' , '--output-location S3URI') {|v| options[:output_location] = v }
|
30
|
+
opt.on('-d', '--database DATABASE') {|v| options[:database] = v }
|
31
|
+
opt.on('-e', '--execute QUERY') {|v| query = v }
|
32
|
+
opt.on('-f', '--file QUERY_FILR') {|v| query_file = v }
|
33
|
+
opt.on('' , '--debug') { options[:debug] = true }
|
34
|
+
opt.parse!
|
35
|
+
|
36
|
+
unless options[:output_location]
|
37
|
+
raise Ath::Error, '"--output-location" or ATH_OUTPUT_LOCATION is required'
|
38
|
+
end
|
39
|
+
|
40
|
+
if not creds_opts.empty?
|
41
|
+
creds = Aws::SharedCredentials.new(credentials_opts)
|
42
|
+
aws_opts[:credentials] = creds
|
43
|
+
end
|
44
|
+
|
45
|
+
Aws.config.update(aws_opts)
|
46
|
+
rescue Ath::Error, OptionParser::ParseError => e
|
47
|
+
$stderr.puts("[ERROR] #{e.message}")
|
48
|
+
exit 1
|
49
|
+
rescue => e
|
50
|
+
$stderr.puts("[ERROR] #{e.message}")
|
51
|
+
puts "\t" + e.backtrace.join("\n\t")
|
52
|
+
exit 1
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
if options[:debug]
|
57
|
+
Aws.config.update(
|
58
|
+
http_wire_trace: true,
|
59
|
+
logger: Logger.new($stdout).tap {|l| l.level = Logger::DEBUG },
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
begin
|
64
|
+
shell = Ath::Shell.new(database: options.delete(:database), options: options)
|
65
|
+
|
66
|
+
if query_file
|
67
|
+
if query_file == '-'
|
68
|
+
query = $stdin.read
|
69
|
+
else
|
70
|
+
query = File.read(query_file)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
if query
|
75
|
+
query.strip!
|
76
|
+
query << ';' if query !~ /[;&]\z/
|
77
|
+
shell.oneshot(query)
|
78
|
+
else
|
79
|
+
shell.start
|
80
|
+
end
|
81
|
+
rescue Interrupt
|
82
|
+
# nothing to do
|
83
|
+
rescue => e
|
84
|
+
if options[:debug]
|
85
|
+
raise e
|
86
|
+
else
|
87
|
+
$stderr.puts("[ERROR] #{[e.message, e.backtrace.first].join("\n\t")}")
|
88
|
+
exit 1
|
89
|
+
end
|
90
|
+
end
|
data/lib/ath/command.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
class Ath::Command
|
2
|
+
MAX_LIST_QUERY = 20
|
3
|
+
|
4
|
+
def initialize(shell:, command:, arg: nil)
|
5
|
+
@shell = shell
|
6
|
+
@command = command
|
7
|
+
@arg = arg
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
out = nil
|
12
|
+
|
13
|
+
case @command
|
14
|
+
when 'debug'
|
15
|
+
if @arg == 'true' or @arg == 'false'
|
16
|
+
@shell.options[:debug] = (@arg =~ /true/)
|
17
|
+
else
|
18
|
+
out = "Usage: /debug true|false"
|
19
|
+
end
|
20
|
+
when 'desc'
|
21
|
+
if @arg
|
22
|
+
query_execution = @shell.driver.get_query_execution(query_execution_id: @arg)
|
23
|
+
out = JSON.pretty_generate(query_execution.to_h)
|
24
|
+
else
|
25
|
+
out = "Usage: /desc QUERY_EXECUTION_ID"
|
26
|
+
end
|
27
|
+
when 'help'
|
28
|
+
out = usage
|
29
|
+
when 'list'
|
30
|
+
query_executions = @shell.driver.list_query_executions
|
31
|
+
query_executions.sort_by! {|qe| qe.status.submission_date_time }.reverse!
|
32
|
+
|
33
|
+
lines = query_executions.map do |qe|
|
34
|
+
line = [
|
35
|
+
qe.status.submission_date_time,
|
36
|
+
qe.query_execution_id,
|
37
|
+
qe.status.state,
|
38
|
+
]
|
39
|
+
|
40
|
+
if qe.query.length > MAX_LIST_QUERY
|
41
|
+
line << qe.query.slice(0, MAX_LIST_QUERY) + '..'
|
42
|
+
else
|
43
|
+
line << qe.query
|
44
|
+
end
|
45
|
+
|
46
|
+
line.join("\s")
|
47
|
+
end
|
48
|
+
|
49
|
+
if @arg
|
50
|
+
lines = lines.slice(0, @arg.to_i)
|
51
|
+
end
|
52
|
+
|
53
|
+
out = lines.join("\n")
|
54
|
+
when 'pager'
|
55
|
+
if @arg
|
56
|
+
@shell.pager = @arg
|
57
|
+
else
|
58
|
+
@shell.pager = nil
|
59
|
+
out = "Using stdout"
|
60
|
+
end
|
61
|
+
when 'result'
|
62
|
+
if @arg
|
63
|
+
out = @shell.driver.get_query_execution_result(query_execution_id: @arg)
|
64
|
+
else
|
65
|
+
out = "Usage: /result QUERY_EXECUTION_ID"
|
66
|
+
end
|
67
|
+
when 'stop'
|
68
|
+
if @arg
|
69
|
+
@shell.driver.stop_query_execution(query_execution_id: @arg)
|
70
|
+
else
|
71
|
+
out = "Usage: /stop QUERY_EXECUTION_ID"
|
72
|
+
end
|
73
|
+
when 'use'
|
74
|
+
if @arg
|
75
|
+
@shell.database = @arg
|
76
|
+
else
|
77
|
+
out = "Usage: /use DATABASE"
|
78
|
+
end
|
79
|
+
else
|
80
|
+
raise Ath::Error, "Unknown command: #{@command}"
|
81
|
+
end
|
82
|
+
|
83
|
+
out
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def usage
|
89
|
+
<<-EOS
|
90
|
+
/debug true|false
|
91
|
+
/desc QUERY_EXECUTION_ID
|
92
|
+
/help
|
93
|
+
/list [NUM]
|
94
|
+
/pager PAGER
|
95
|
+
/result QUERY_EXECUTION_ID
|
96
|
+
/stop QUERY_EXECUTION_ID
|
97
|
+
/use DATABASE
|
98
|
+
EOS
|
99
|
+
end
|
100
|
+
end
|
data/lib/ath/driver.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
class Ath::Driver
|
2
|
+
def initialize(athena:, s3:)
|
3
|
+
@athena = athena
|
4
|
+
@s3 = s3
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_query_execution(query_execution_id:)
|
8
|
+
|
9
|
+
@athena.get_query_execution(query_execution_id: query_execution_id).query_execution
|
10
|
+
end
|
11
|
+
|
12
|
+
def list_query_executions
|
13
|
+
query_execution_ids = @athena.list_query_executions.each_page.flat_map(&:query_execution_ids)
|
14
|
+
@athena.batch_get_query_execution(query_execution_ids: query_execution_ids.slice(0, 50)).query_executions
|
15
|
+
end
|
16
|
+
|
17
|
+
def get_query_execution_result(query_execution_id:)
|
18
|
+
query_execution = @athena.get_query_execution(query_execution_id: query_execution_id).query_execution
|
19
|
+
output_location = query_execution.result_configuration.output_location
|
20
|
+
bucket, key = output_location.sub(%r{\As3://}, '').split('/', 2)
|
21
|
+
tmp = Tempfile.create('ath')
|
22
|
+
|
23
|
+
@s3.get_object(bucket: bucket, key: key) do |chunk|
|
24
|
+
tmp.write(chunk)
|
25
|
+
end
|
26
|
+
|
27
|
+
tmp.flush
|
28
|
+
tmp
|
29
|
+
end
|
30
|
+
|
31
|
+
def start_query_execution(query_string:, database:, output_location:)
|
32
|
+
@athena.start_query_execution(
|
33
|
+
query_string: query_string,
|
34
|
+
query_execution_context: {database: database},
|
35
|
+
result_configuration: { output_location: output_location}
|
36
|
+
)
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop_query_execution(query_execution_id:)
|
40
|
+
@athena.stop_query_execution(query_execution_id: query_execution_id)
|
41
|
+
end
|
42
|
+
end
|
data/lib/ath/error.rb
ADDED
data/lib/ath/query.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
class Ath::Query
|
2
|
+
def initialize(shell:, query:, detach: false)
|
3
|
+
@shell = shell
|
4
|
+
@query = query
|
5
|
+
@detach = detach
|
6
|
+
end
|
7
|
+
|
8
|
+
def run
|
9
|
+
query_execution_id = @shell.driver.start_query_execution(
|
10
|
+
query_string: @query,
|
11
|
+
database: @shell.database,
|
12
|
+
output_location: @shell.options.fetch(:output_location)
|
13
|
+
).query_execution_id
|
14
|
+
|
15
|
+
if @detach
|
16
|
+
return "QueryExecution #{query_execution_id}"
|
17
|
+
end
|
18
|
+
|
19
|
+
abort_waiting = false
|
20
|
+
orig_handler = trap(:INT, proc { abort_waiting = true })
|
21
|
+
query_execution = nil
|
22
|
+
|
23
|
+
begin
|
24
|
+
until abort_waiting
|
25
|
+
query_execution = @shell.driver.get_query_execution(query_execution_id: query_execution_id)
|
26
|
+
|
27
|
+
if query_execution.status.completion_date_time
|
28
|
+
break
|
29
|
+
end
|
30
|
+
|
31
|
+
sleep 1
|
32
|
+
end
|
33
|
+
ensure
|
34
|
+
trap(:INT, orig_handler)
|
35
|
+
end
|
36
|
+
|
37
|
+
if abort_waiting
|
38
|
+
return "QueryExecution #{query_execution_id}: Detach query"
|
39
|
+
end
|
40
|
+
|
41
|
+
if query_execution.status.state == 'SUCCEEDED'
|
42
|
+
@shell.driver.get_query_execution_result(query_execution_id: query_execution_id)
|
43
|
+
else
|
44
|
+
"QueryExecution #{query_execution_id}: #{query_execution.status.state_change_reason}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/ath/scanner.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
class Ath::Scanner
|
2
|
+
def initialize(shell:)
|
3
|
+
@shell = shell
|
4
|
+
@buf = ''
|
5
|
+
end
|
6
|
+
|
7
|
+
def scan(line)
|
8
|
+
ss = StringScanner.new(line)
|
9
|
+
|
10
|
+
if self.empty? and (tok = ss.scan %r{/\w+(?:\s+.*)?\z})
|
11
|
+
cmd, arg = tok.split(/\s+/, 2)
|
12
|
+
cmd.slice!(0)
|
13
|
+
arg.strip! if arg
|
14
|
+
yield(Ath::Command.new(shell: @shell, command: cmd, arg: arg))
|
15
|
+
end
|
16
|
+
|
17
|
+
until ss.eos?
|
18
|
+
@buf << ' '
|
19
|
+
|
20
|
+
if (tok = ss.scan %r{[^'";&]+})
|
21
|
+
@buf << tok
|
22
|
+
elsif (tok = ss.scan /'(?:''|[^'])*'/)
|
23
|
+
@buf << tok
|
24
|
+
elsif (tok = ss.scan /"(?:""|[^"])*"/)
|
25
|
+
@buf << tok
|
26
|
+
elsif (tok = ss.scan /(?:[;&])/)
|
27
|
+
query = @buf.strip
|
28
|
+
@buf.clear
|
29
|
+
raise Ath::Error, 'No query specified' if query.empty?
|
30
|
+
yield(Ath::Query.new(shell: @shell, query: query, detach: (tok == '&')))
|
31
|
+
else
|
32
|
+
raise Ath::Error, 'You have an error in your HiveQL syntax'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
@buf.strip!
|
37
|
+
end
|
38
|
+
|
39
|
+
def empty?
|
40
|
+
@buf.empty?
|
41
|
+
end
|
42
|
+
end
|
data/lib/ath/shell.rb
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
class Ath::Shell
|
2
|
+
HISTORY_FILE = File.join(ENV.fetch('HOME', '.'), '.ath_history')
|
3
|
+
HISTSIZE = 1000
|
4
|
+
|
5
|
+
attr_reader :driver
|
6
|
+
attr_reader :options
|
7
|
+
attr_accessor :database
|
8
|
+
attr_accessor :pager
|
9
|
+
|
10
|
+
def initialize(athena: Aws::Athena::Client.new, s3: Aws::S3::Client.new, database: nil, options: {})
|
11
|
+
@driver = Ath::Driver.new(athena: athena, s3: s3)
|
12
|
+
@options = options
|
13
|
+
@database = database
|
14
|
+
@scanner = Ath::Scanner.new(shell: self)
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
load_history
|
19
|
+
|
20
|
+
while line = Readline.readline(prompt, true)
|
21
|
+
execute_query(line)
|
22
|
+
end
|
23
|
+
|
24
|
+
save_history
|
25
|
+
end
|
26
|
+
|
27
|
+
def oneshot(query)
|
28
|
+
execute_query(query)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def execute_query(line)
|
34
|
+
begin
|
35
|
+
execute_query0(line)
|
36
|
+
rescue => e
|
37
|
+
puts e.message
|
38
|
+
|
39
|
+
if @options[:debug]
|
40
|
+
puts e.backtrace.join("\n\t")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def execute_query0(line)
|
46
|
+
@scanner.scan(line) do |cmd_or_query|
|
47
|
+
out = nil
|
48
|
+
|
49
|
+
case cmd_or_query
|
50
|
+
when Ath::Command
|
51
|
+
out = cmd_or_query.run
|
52
|
+
when Ath::Query
|
53
|
+
out = cmd_or_query.run
|
54
|
+
else
|
55
|
+
raise 'must not happen'
|
56
|
+
end
|
57
|
+
|
58
|
+
print_result(out)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def prompt
|
63
|
+
database = @database || '(none)'
|
64
|
+
|
65
|
+
if @scanner.empty?
|
66
|
+
"#{database}> "
|
67
|
+
else
|
68
|
+
indent = "\s" * (database.length - 1)
|
69
|
+
"#{indent}-> "
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def print_result(out)
|
74
|
+
return unless out
|
75
|
+
|
76
|
+
if out.kind_of?(File)
|
77
|
+
begin
|
78
|
+
cmd = "cat #{out.path}"
|
79
|
+
cmd << " | #{@pager}" if @pager
|
80
|
+
system(cmd)
|
81
|
+
|
82
|
+
begin
|
83
|
+
out.seek(-1, IO::SEEK_END)
|
84
|
+
puts if out.gets !~ /\n\z/
|
85
|
+
rescue Errno::EINVAL
|
86
|
+
puts
|
87
|
+
end
|
88
|
+
ensure
|
89
|
+
out.close
|
90
|
+
end
|
91
|
+
else
|
92
|
+
if @pager
|
93
|
+
Tempfile.create('ath') do |f|
|
94
|
+
f.puts out
|
95
|
+
f.flush
|
96
|
+
system("cat #{f.path} | #{@pager}")
|
97
|
+
end
|
98
|
+
else
|
99
|
+
puts out
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def load_history
|
105
|
+
return unless File.exist?(HISTORY_FILE)
|
106
|
+
|
107
|
+
open(HISTORY_FILE) do |f|
|
108
|
+
f.each_line.map(&:strip).reject(&:empty?).each do |line|
|
109
|
+
Readline::HISTORY.push(line)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def save_history
|
115
|
+
if Readline::HISTORY.length < HISTSIZE
|
116
|
+
offset = Readline::HISTORY.length
|
117
|
+
else
|
118
|
+
offset = HISTORY
|
119
|
+
end
|
120
|
+
|
121
|
+
history = Readline::HISTORY.map(&:strip).reject(&:empty?).uniq
|
122
|
+
history = history.slice(-offset..-1) || []
|
123
|
+
|
124
|
+
return if history.empty?
|
125
|
+
|
126
|
+
open(HISTORY_FILE, 'wb') do |f|
|
127
|
+
history.each {|l| f.puts l }
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/ath/version.rb
ADDED
data/lib/ath.rb
ADDED
metadata
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ath
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- winebarrel
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 2.9.21
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 2.9.21
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: bundler
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rake
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rspec
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '3.0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '3.0'
|
75
|
+
description: ath is a interactive Amazon Athena shell.
|
76
|
+
email:
|
77
|
+
- sgwr_dts@yahoo.co.jp
|
78
|
+
executables:
|
79
|
+
- ath
|
80
|
+
extensions: []
|
81
|
+
extra_rdoc_files: []
|
82
|
+
files:
|
83
|
+
- ".gitignore"
|
84
|
+
- ".rspec"
|
85
|
+
- ".travis.yml"
|
86
|
+
- Gemfile
|
87
|
+
- LICENSE.txt
|
88
|
+
- README.md
|
89
|
+
- Rakefile
|
90
|
+
- ath.gemspec
|
91
|
+
- bin/console
|
92
|
+
- bin/setup
|
93
|
+
- exe/ath
|
94
|
+
- lib/ath.rb
|
95
|
+
- lib/ath/command.rb
|
96
|
+
- lib/ath/driver.rb
|
97
|
+
- lib/ath/error.rb
|
98
|
+
- lib/ath/query.rb
|
99
|
+
- lib/ath/scanner.rb
|
100
|
+
- lib/ath/shell.rb
|
101
|
+
- lib/ath/version.rb
|
102
|
+
homepage: https://github.com/winebarrel/ath
|
103
|
+
licenses:
|
104
|
+
- MIT
|
105
|
+
metadata: {}
|
106
|
+
post_install_message:
|
107
|
+
rdoc_options: []
|
108
|
+
require_paths:
|
109
|
+
- lib
|
110
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
requirements: []
|
121
|
+
rubyforge_project:
|
122
|
+
rubygems_version: 2.5.2
|
123
|
+
signing_key:
|
124
|
+
specification_version: 4
|
125
|
+
summary: ath is a interactive Amazon Athena shell.
|
126
|
+
test_files: []
|