athena-cli 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.projections.json +20 -0
- data/.ruby-version +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +75 -0
- data/LICENSE.md +20 -0
- data/README.md +128 -0
- data/Rakefile +44 -0
- data/athena-cli.gemspec +26 -0
- data/bin/athena-cli +10 -0
- data/features/athena.feature +8 -0
- data/features/step_definitions/athena_steps.rb +6 -0
- data/features/support/env.rb +15 -0
- data/lib/amazon_athena.rb +7 -0
- data/lib/amazon_athena/cli.rb +406 -0
- data/lib/amazon_athena/client.rb +114 -0
- data/lib/amazon_athena/command.rb +16 -0
- data/lib/amazon_athena/commands.rb +20 -0
- data/lib/amazon_athena/commands/alter_table_add_partition.rb +29 -0
- data/lib/amazon_athena/commands/alter_table_drop_partition.rb +29 -0
- data/lib/amazon_athena/commands/create_database.rb +23 -0
- data/lib/amazon_athena/commands/create_table.rb +23 -0
- data/lib/amazon_athena/commands/describe_table.rb +21 -0
- data/lib/amazon_athena/commands/drop_database.rb +23 -0
- data/lib/amazon_athena/commands/drop_table.rb +23 -0
- data/lib/amazon_athena/commands/repair_table.rb +23 -0
- data/lib/amazon_athena/commands/show_columns.rb +22 -0
- data/lib/amazon_athena/commands/show_create_table.rb +22 -0
- data/lib/amazon_athena/commands/show_databases.rb +17 -0
- data/lib/amazon_athena/commands/show_partitions.rb +28 -0
- data/lib/amazon_athena/commands/show_table_properties.rb +36 -0
- data/lib/amazon_athena/commands/show_tables.rb +23 -0
- data/lib/amazon_athena/partition.rb +21 -0
- data/lib/amazon_athena/transformer.rb +19 -0
- data/lib/amazon_athena/version.rb +3 -0
- data/lib/jdbc_helper/athena.rb +38 -0
- data/lib/jdbc_helper/resultset.rb +14 -0
- data/log4j.xml +18 -0
- data/script/bootstrap +5 -0
- data/script/package +8 -0
- data/script/release +16 -0
- data/script/test +6 -0
- data/test/lib/amazon_athena/commands/alter_table_add_partition_test.rb +64 -0
- data/test/lib/amazon_athena/commands/alter_table_drop_partition_test.rb +61 -0
- data/test/lib/amazon_athena/commands/create_database_test.rb +27 -0
- data/test/lib/amazon_athena/commands/describe_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/drop_database_test.rb +22 -0
- data/test/lib/amazon_athena/commands/drop_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/repair_table_test.rb +20 -0
- data/test/lib/amazon_athena/commands/show_columns_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_create_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_databases_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_partitions_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_table_properties_test.rb +25 -0
- data/test/lib/amazon_athena/commands/show_tables_test.rb +23 -0
- data/test/test_helper.rb +1 -0
- metadata +185 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ea56f38290709fb3f5b0351f88cfa946b10d820f
|
4
|
+
data.tar.gz: ab1d16e5143dde991ef413f014c60456a8bacb01
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e8b87e4a1b879ac686194e0a91a8ff4cee7b292056acaf2bf9da2cacc9a78abd44c2ebd9825f2bf51492c7495d9c20efe1ef9aff2356d9b83f22c5969cad0a88
|
7
|
+
data.tar.gz: e1109e094eb8c5c33acb55254784bce9f43db3c012e5ae36d8379db12c00f0d1cf321404a4de07a61f2e97d69e0ca4690d274c724f86a2fa593d94914e6679e1
|
data/.projections.json
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
{
|
2
|
+
"test/*_test.rb": {
|
3
|
+
"alternate": [
|
4
|
+
"{}.rb"
|
5
|
+
],
|
6
|
+
"type": "test"
|
7
|
+
},
|
8
|
+
"lib/amazon_athena/commands/*.rb": {
|
9
|
+
"alternate": [
|
10
|
+
"test/lib/amazon_athena/commands/{}_test.rb"
|
11
|
+
],
|
12
|
+
"type": "command"
|
13
|
+
},
|
14
|
+
"lib/*.rb": {
|
15
|
+
"alternate": [
|
16
|
+
"test/lib/{}_test.rb"
|
17
|
+
],
|
18
|
+
"type": "lib"
|
19
|
+
}
|
20
|
+
}
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
jruby-9.1.6.0
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
athena-cli (0.0.1)
|
5
|
+
gli (= 2.5.2)
|
6
|
+
jdbc-helper (~> 0.8.2)
|
7
|
+
table_print (~> 1.5)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
aruba (0.14.2)
|
13
|
+
childprocess (~> 0.5.6)
|
14
|
+
contracts (~> 0.9)
|
15
|
+
cucumber (>= 1.3.19)
|
16
|
+
ffi (~> 1.9.10)
|
17
|
+
rspec-expectations (>= 2.99)
|
18
|
+
thor (~> 0.19)
|
19
|
+
builder (3.2.3)
|
20
|
+
childprocess (0.5.9)
|
21
|
+
ffi (~> 1.0, >= 1.0.11)
|
22
|
+
coderay (1.1.1)
|
23
|
+
contracts (0.15.0)
|
24
|
+
cucumber (2.4.0)
|
25
|
+
builder (>= 2.1.2)
|
26
|
+
cucumber-core (~> 1.5.0)
|
27
|
+
cucumber-wire (~> 0.0.1)
|
28
|
+
diff-lcs (>= 1.1.3)
|
29
|
+
gherkin (~> 4.0)
|
30
|
+
multi_json (>= 1.7.5, < 2.0)
|
31
|
+
multi_test (>= 0.1.2)
|
32
|
+
cucumber-core (1.5.0)
|
33
|
+
gherkin (~> 4.0)
|
34
|
+
cucumber-wire (0.0.1)
|
35
|
+
diff-lcs (1.3)
|
36
|
+
ffi (1.9.18-java)
|
37
|
+
gherkin (4.0.0)
|
38
|
+
gli (2.5.2)
|
39
|
+
insensitive_hash (0.3.3)
|
40
|
+
jdbc-helper (0.8.2)
|
41
|
+
insensitive_hash (>= 0.2.4)
|
42
|
+
sql_helper (~> 0.1.1)
|
43
|
+
method_source (0.8.2)
|
44
|
+
multi_json (1.12.1)
|
45
|
+
multi_test (0.1.2)
|
46
|
+
pry (0.10.4-java)
|
47
|
+
coderay (~> 1.1.0)
|
48
|
+
method_source (~> 0.8.1)
|
49
|
+
slop (~> 3.4)
|
50
|
+
spoon (~> 0.0)
|
51
|
+
rake (10.4.2)
|
52
|
+
rdoc (5.1.0)
|
53
|
+
rspec-expectations (3.5.0)
|
54
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
55
|
+
rspec-support (~> 3.5.0)
|
56
|
+
rspec-support (3.5.0)
|
57
|
+
slop (3.6.0)
|
58
|
+
spoon (0.0.6)
|
59
|
+
ffi
|
60
|
+
sql_helper (0.1.1)
|
61
|
+
table_print (1.5.6)
|
62
|
+
thor (0.19.4)
|
63
|
+
|
64
|
+
PLATFORMS
|
65
|
+
java
|
66
|
+
|
67
|
+
DEPENDENCIES
|
68
|
+
aruba (~> 0.14)
|
69
|
+
athena-cli!
|
70
|
+
pry
|
71
|
+
rake (~> 10)
|
72
|
+
rdoc (~> 5.1)
|
73
|
+
|
74
|
+
BUNDLED WITH
|
75
|
+
1.14.6
|
data/LICENSE.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2017 Wynn Netherland
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
# athena-cli
|
2
|
+
|
3
|
+
A CLI for Amazon Athena, powered by JRuby.
|
4
|
+
|
5
|
+
[Amazon Athena](https://aws.amazon.com/athena/) allows you to define a schema and query structured data in S3 without the usual Extract-Transform-Load process. This project uses the [JDBC driver](http://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html) and JRuby to bring Athena access to your terminal.
|
6
|
+
|
7
|
+
##### _athena-cli is a rapidly evolving work in progress. Some parts of this README are [aspirational](http://tom.preston-werner.com/2010/08/23/readme-driven-development.html)._
|
8
|
+
|
9
|
+
#### Installation
|
10
|
+
|
11
|
+
Make sure you have JRuby >= 9.1.6.0.
|
12
|
+
|
13
|
+
jruby -v
|
14
|
+
|
15
|
+
Clone the repository:
|
16
|
+
|
17
|
+
git clone https://github.com/pengwynn/athena-cli
|
18
|
+
|
19
|
+
Install the dependencies:
|
20
|
+
|
21
|
+
bundle install
|
22
|
+
|
23
|
+
Build the gem:
|
24
|
+
|
25
|
+
rake gem
|
26
|
+
|
27
|
+
Install it locally
|
28
|
+
|
29
|
+
gem install pkg/athena-cli-<version>.gem
|
30
|
+
|
31
|
+
##### TODO: Install via Rubygems:
|
32
|
+
|
33
|
+
##### Setting up AWS credentials
|
34
|
+
|
35
|
+
The app will look for your AWS key and secret in the `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` environment variables. If you use multiple AWS accounts, [awsam](https://github.com/mheffner/awsam) makes that a snap. Otherwise, you'll need to supply your AWS key and secret on every command via `--key` and `--secret` flags.
|
36
|
+
|
37
|
+
##### Configuring the JDBC driver
|
38
|
+
|
39
|
+
The underlying Java code needs to know where to find your [JDBC
|
40
|
+
driver](http://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html#downloading-the-driver). Add the path to your `CLASSPATH` environment variable. Something like:
|
41
|
+
|
42
|
+
|
43
|
+
Example:
|
44
|
+
|
45
|
+
export CLASSPATH="$CLASSPATH:~/src/AthenaJDB41-1.0.0.jar"
|
46
|
+
|
47
|
+
##### Setting the scratch folder
|
48
|
+
|
49
|
+
Athena requires an S3 scratch folder for storing results. You'll need to set the `ATHENA_S3_STAGING_DIR` environment variable or pass via the `--staging-dir` command line flag.
|
50
|
+
|
51
|
+
##### Check your settings
|
52
|
+
|
53
|
+
athena-cli doctor
|
54
|
+
|
55
|
+
If you're calling it outside the repository folder, you'll want to drop a
|
56
|
+
[log4j.xml](https://github.com/pengwynn/athena-cli/blob/master/log4j.xml) file
|
57
|
+
into that folder to silence the logger.
|
58
|
+
|
59
|
+
### Hacking on athena-cli
|
60
|
+
|
61
|
+
If you're running the project form source, be sure and use
|
62
|
+
|
63
|
+
bundle exec bin/athena-cli
|
64
|
+
|
65
|
+
so that the dependencies are properly loaded.
|
66
|
+
|
67
|
+
### Usage
|
68
|
+
|
69
|
+
```
|
70
|
+
❯ athena-cli
|
71
|
+
NAME
|
72
|
+
athena-cli - CLI for Amazon Athena
|
73
|
+
|
74
|
+
SYNOPSIS
|
75
|
+
athena-cli [global options] command [command options] [arguments...]
|
76
|
+
|
77
|
+
VERSION
|
78
|
+
0.0.1
|
79
|
+
|
80
|
+
GLOBAL OPTIONS
|
81
|
+
--help - Show this message
|
82
|
+
--key=arg - AWS Access Key (default: ********)
|
83
|
+
-p, --preview - Output the SQL statement instead of running
|
84
|
+
--secret=arg - AWS Secret Key (default: ********)
|
85
|
+
--staging-dir=arg - S3 bucket for staging results (default: none)
|
86
|
+
--version -
|
87
|
+
|
88
|
+
COMMANDS
|
89
|
+
column - Manage columns in Athena tables
|
90
|
+
database - Manage Athena databases
|
91
|
+
doctor - Check for required AWS setup
|
92
|
+
help - Shows a list of commands or help for one command
|
93
|
+
partition - Manage partitions in Athena tables
|
94
|
+
query - Run a query from a file or STDIN
|
95
|
+
schema - Manage Athena schemas
|
96
|
+
table - Manage tables in Athena databases
|
97
|
+
|
98
|
+
```
|
99
|
+
|
100
|
+
### TODO
|
101
|
+
|
102
|
+
- [ ] Explore more discoverable commands. `show-tables` instead of `table
|
103
|
+
list`
|
104
|
+
- [ ] Tests for unhappy paths
|
105
|
+
- [ ] Publish to Rubygems
|
106
|
+
|
107
|
+
## LICENSE
|
108
|
+
|
109
|
+
Copyright (c) 2017 Wynn Netherland
|
110
|
+
|
111
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
112
|
+
a copy of this software and associated documentation files (the
|
113
|
+
"Software"), to deal in the Software without restriction, including
|
114
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
115
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
116
|
+
permit persons to whom the Software is furnished to do so, subject to
|
117
|
+
the following conditions:
|
118
|
+
|
119
|
+
The above copyright notice and this permission notice shall be
|
120
|
+
included in all copies or substantial portions of the Software.
|
121
|
+
|
122
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
123
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
124
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
125
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
126
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
127
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
128
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'rake/clean'
|
2
|
+
require 'rubygems'
|
3
|
+
require 'rubygems/package_task'
|
4
|
+
require 'rdoc/task'
|
5
|
+
require 'cucumber'
|
6
|
+
require 'cucumber/rake/task'
|
7
|
+
Rake::RDocTask.new do |rd|
|
8
|
+
rd.main = "README.rdoc"
|
9
|
+
rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
|
10
|
+
rd.title = 'Your application title'
|
11
|
+
end
|
12
|
+
|
13
|
+
spec = eval(File.read('athena-cli.gemspec'))
|
14
|
+
|
15
|
+
Gem::PackageTask.new(spec) do |pkg|
|
16
|
+
end
|
17
|
+
CUKE_RESULTS = 'results.html'
|
18
|
+
CLEAN << CUKE_RESULTS
|
19
|
+
desc 'Run features'
|
20
|
+
Cucumber::Rake::Task.new(:features) do |t|
|
21
|
+
opts = "features --format html -o #{CUKE_RESULTS} --format progress -x"
|
22
|
+
opts += " --tags #{ENV['TAGS']}" if ENV['TAGS']
|
23
|
+
t.cucumber_opts = opts
|
24
|
+
t.fork = false
|
25
|
+
end
|
26
|
+
|
27
|
+
desc 'Run features tagged as work-in-progress (@wip)'
|
28
|
+
Cucumber::Rake::Task.new('features:wip') do |t|
|
29
|
+
tag_opts = ' --tags ~@pending'
|
30
|
+
tag_opts = ' --tags @wip'
|
31
|
+
t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty -x -s#{tag_opts}"
|
32
|
+
t.fork = false
|
33
|
+
end
|
34
|
+
|
35
|
+
task :cucumber => :features
|
36
|
+
task 'cucumber:wip' => 'features:wip'
|
37
|
+
task :wip => 'features:wip'
|
38
|
+
require 'rake/testtask'
|
39
|
+
Rake::TestTask.new do |t|
|
40
|
+
t.libs << "test"
|
41
|
+
t.test_files = FileList['test/**/*_test.rb']
|
42
|
+
end
|
43
|
+
|
44
|
+
task :default => [:test,:features]
|
data/athena-cli.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# Ensure we require the local version and not one we might have installed already
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
require File.join([File.dirname(__FILE__),'lib','amazon_athena','version.rb'])
|
6
|
+
spec = Gem::Specification.new do |s|
|
7
|
+
s.name = 'athena-cli'
|
8
|
+
s.version = AmazonAthena::VERSION
|
9
|
+
s.author = 'Wynn Netherland'
|
10
|
+
s.email = 'wynn.netherland@gmail.com'
|
11
|
+
s.homepage = 'https://wynnnetherland.com'
|
12
|
+
s.platform = Gem::Platform::RUBY
|
13
|
+
s.summary = 'A JRuby-powered CLI for Amazon Athena'
|
14
|
+
s.licenses = ["MIT"]
|
15
|
+
s.files = `git ls-files`.split("
|
16
|
+
")
|
17
|
+
s.require_paths << 'lib'
|
18
|
+
s.bindir = 'bin'
|
19
|
+
s.executables << 'athena-cli'
|
20
|
+
s.add_development_dependency('rake', '~> 10')
|
21
|
+
s.add_development_dependency('rdoc', '~> 5.1')
|
22
|
+
s.add_development_dependency('aruba', '~> 0.14')
|
23
|
+
s.add_runtime_dependency('gli','2.5.2')
|
24
|
+
s.add_runtime_dependency('jdbc-helper', '~> 0.8.2')
|
25
|
+
s.add_runtime_dependency('table_print', '~> 1.5 ')
|
26
|
+
end
|
data/bin/athena-cli
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'aruba/cucumber'
|
2
|
+
|
3
|
+
ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
|
4
|
+
LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
|
5
|
+
|
6
|
+
Before do
|
7
|
+
# Using "announce" causes massive warnings on 1.9.2
|
8
|
+
@puts = true
|
9
|
+
@original_rubylib = ENV['RUBYLIB']
|
10
|
+
ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
|
11
|
+
end
|
12
|
+
|
13
|
+
After do
|
14
|
+
ENV['RUBYLIB'] = @original_rubylib
|
15
|
+
end
|
@@ -0,0 +1,406 @@
|
|
1
|
+
require 'gli'
|
2
|
+
require 'amazon_athena'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'table_print'
|
5
|
+
|
6
|
+
module AmazonAthena
|
7
|
+
module CLI
|
8
|
+
|
9
|
+
def self.athena
|
10
|
+
AmazonAthena::Client.new \
|
11
|
+
key: self.access_key,
|
12
|
+
secret: self.access_secret,
|
13
|
+
s3_staging_dir: self.staging_folder
|
14
|
+
end
|
15
|
+
|
16
|
+
extend GLI::App
|
17
|
+
|
18
|
+
program_desc 'CLI for Amazon Athena'
|
19
|
+
version AmazonAthena::VERSION
|
20
|
+
|
21
|
+
switch [:p, :preview],
|
22
|
+
:default_value => false,
|
23
|
+
:desc => "Output the SQL statement instead of running",
|
24
|
+
:negatable => false
|
25
|
+
|
26
|
+
flag ["key"],
|
27
|
+
:default_value => ENV["AWS_ACCESS_KEY"],
|
28
|
+
:mask => true,
|
29
|
+
:desc => "AWS Access Key"
|
30
|
+
|
31
|
+
flag ["secret"],
|
32
|
+
:default_value => ENV["AWS_SECRET_KEY"],
|
33
|
+
:mask => true,
|
34
|
+
:desc => "AWS Secret Key"
|
35
|
+
|
36
|
+
flag ["staging-dir"],
|
37
|
+
:default_value => ENV["ATHENA_S3_STAGING_DIR"],
|
38
|
+
:desc => "S3 bucket for staging results"
|
39
|
+
|
40
|
+
desc 'Check for required AWS setup'
|
41
|
+
command :doctor do |c|
|
42
|
+
c.action do |global_options,options,args|
|
43
|
+
puts "Good to go!"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
desc 'Manage Athena schemas'
|
48
|
+
command :schema do |c|
|
49
|
+
c.desc "Dump schemas to a local folder"
|
50
|
+
c.command :dump do |add|
|
51
|
+
add.flag :path,
|
52
|
+
:required => true,
|
53
|
+
:desc => "Local path for saving schemas"
|
54
|
+
add.action do |global_options,options,args|
|
55
|
+
path = options[:path]
|
56
|
+
|
57
|
+
unless File.exists?(path)
|
58
|
+
exit_now! "Path #{path} not found"
|
59
|
+
end
|
60
|
+
|
61
|
+
databases = athena.run(AmazonAthena::Commands::ShowDatabases.new)
|
62
|
+
preview = !!global_options[:p]
|
63
|
+
|
64
|
+
databases.each do |db|
|
65
|
+
folder = File.join(path, db)
|
66
|
+
prefix = File.exists?(folder) ? "Exists" : "Creating"
|
67
|
+
render "#{prefix} #{folder}"
|
68
|
+
|
69
|
+
unless preview
|
70
|
+
FileUtils.mkdir_p(folder)
|
71
|
+
end
|
72
|
+
|
73
|
+
tables = athena.run(AmazonAthena::Commands::ShowTables.new(db))
|
74
|
+
|
75
|
+
tables.each do |table|
|
76
|
+
database_table = [db, table].join(".")
|
77
|
+
file = File.join(folder, table) + ".sql"
|
78
|
+
render "Writing #{file}"
|
79
|
+
|
80
|
+
if !preview
|
81
|
+
sql = athena.run(AmazonAthena::Commands::ShowCreateTable.new(database_table))
|
82
|
+
|
83
|
+
File.open(file, 'w') { |f| f.write(sql) }
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# puts global_options, options, args
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
desc 'Manage Athena databases'
|
94
|
+
command :database do |c|
|
95
|
+
c.desc "List databases"
|
96
|
+
c.command :list do |add|
|
97
|
+
add.action do |global_options,options,args|
|
98
|
+
cmd = AmazonAthena::Commands::ShowDatabases.new
|
99
|
+
|
100
|
+
render athena.run(cmd, global_options[:p])
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
c.desc "Create a new database"
|
105
|
+
c.arg_name "database"
|
106
|
+
c.command :create do |add|
|
107
|
+
add.action do |global_options,options,args|
|
108
|
+
cmd = AmazonAthena::Commands::CreateDatabase.new(args.first)
|
109
|
+
|
110
|
+
render athena.run(cmd, global_options[:p])
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
c.desc "Drop an existing database"
|
115
|
+
c.arg_name "database"
|
116
|
+
c.command :drop do |add|
|
117
|
+
add.action do |global_options,options,args|
|
118
|
+
cmd = AmazonAthena::Commands::DropDatabase.new(args.first)
|
119
|
+
|
120
|
+
render athena.run(cmd, global_options[:p])
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
c.default_command :list
|
125
|
+
end
|
126
|
+
|
127
|
+
desc 'Manage tables in Athena databases'
|
128
|
+
command :table do |c|
|
129
|
+
c.desc "Create table"
|
130
|
+
c.arg_name "[file]"
|
131
|
+
c.command :create do |add|
|
132
|
+
add.flag [:l, :location], :desc => "S3 location - s3://bucket/path/"
|
133
|
+
add.flag [:n, :name], :desc => "Fully qualifed name as: database.table"
|
134
|
+
add.action do |global_options,options,args|
|
135
|
+
ddl = if filename = args.first
|
136
|
+
exit_now! "File not found" unless File.exists?(filename)
|
137
|
+
File.read(filename)
|
138
|
+
elsif !STDIN.tty?
|
139
|
+
$stdin.read
|
140
|
+
end
|
141
|
+
|
142
|
+
exit_now! "Must supply path to file or pass via STDIN" if ddl.to_s.empty?
|
143
|
+
|
144
|
+
|
145
|
+
opts = {
|
146
|
+
:name => options[:name],
|
147
|
+
:location => options[:location]
|
148
|
+
}
|
149
|
+
|
150
|
+
ddl = AmazonAthena::Transformer.transform_table(ddl, opts)
|
151
|
+
|
152
|
+
cmd = AmazonAthena::Commands::CreateTable.new(ddl)
|
153
|
+
|
154
|
+
render athena.run(cmd, global_options[:p])
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
c.desc "Describe a table"
|
159
|
+
c.arg_name "database.table"
|
160
|
+
c.command :describe do |add|
|
161
|
+
add.action do |global_options,options,args|
|
162
|
+
cmd = AmazonAthena::Commands::DescribeTable.new(args.first)
|
163
|
+
|
164
|
+
render athena.run(cmd, global_options[:p])
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
c.desc "List tables in a database"
|
169
|
+
c.arg_name "database"
|
170
|
+
c.command :list do |add|
|
171
|
+
add.action do |global_options,options,args|
|
172
|
+
cmd = AmazonAthena::Commands::ShowTables.new(args.first)
|
173
|
+
|
174
|
+
render athena.run(cmd, global_options[:p])
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
c.desc "Show table properties"
|
179
|
+
c.arg_name "database.table"
|
180
|
+
c.command :properties do |add|
|
181
|
+
add.action do |global_options,options,args|
|
182
|
+
cmd = AmazonAthena::Commands::ShowTableProperties.new(args.first)
|
183
|
+
|
184
|
+
render athena.run(cmd, global_options[:p])
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
c.desc "Show table create DDL"
|
189
|
+
c.arg_name "database.table"
|
190
|
+
c.command :show do |add|
|
191
|
+
add.action do |global_options,options,args|
|
192
|
+
cmd = AmazonAthena::Commands::ShowCreateTable.new(args.first)
|
193
|
+
|
194
|
+
render athena.run(cmd, global_options[:p])
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
c.desc "Refresh table partitions"
|
199
|
+
c.arg_name "database.table"
|
200
|
+
c.command :repair do |add|
|
201
|
+
add.action do |global_options,options,args|
|
202
|
+
commands = [
|
203
|
+
AmazonAthena::Commands::RepairTable.new(args.first),
|
204
|
+
AmazonAthena::Commands::ShowPartitions.new(args.first),
|
205
|
+
]
|
206
|
+
|
207
|
+
commands.each do |cmd|
|
208
|
+
render athena.run(cmd, global_options[:p])
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
c.desc "Drop table"
|
214
|
+
c.arg_name "database.table"
|
215
|
+
c.command :drop do |add|
|
216
|
+
add.action do |global_options,options,args|
|
217
|
+
cmd = AmazonAthena::Commands::DropTable.new(args.first)
|
218
|
+
|
219
|
+
render athena.run(cmd, global_options[:p])
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
desc 'Manage columns in Athena tables'
|
225
|
+
command :column do |c|
|
226
|
+
c.desc "List columns in a table"
|
227
|
+
c.arg_name "database.table"
|
228
|
+
c.command :list do |add|
|
229
|
+
add.action do |global_options,options,args|
|
230
|
+
cmd = AmazonAthena::Commands::ShowColumns.new(args.first)
|
231
|
+
|
232
|
+
render athena.run(cmd, global_options[:p])
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
c.default_command :list
|
237
|
+
end
|
238
|
+
|
239
|
+
desc 'Manage partitions in Athena tables'
|
240
|
+
command :partition do |c|
|
241
|
+
c.desc "List partitions in a table"
|
242
|
+
c.arg_name "database.table"
|
243
|
+
c.command :list do |add|
|
244
|
+
add.action do |global_options,options,args|
|
245
|
+
cmd = AmazonAthena::Commands::ShowPartitions.new(args.first)
|
246
|
+
|
247
|
+
render athena.run(cmd, global_options[:p])
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
c.desc "TODO: Add partition(s) to a table"
|
252
|
+
c.arg_name "database.table"
|
253
|
+
c.arg_name "key=value,key=value:key=value,key=value"
|
254
|
+
c.command :add do |add|
|
255
|
+
add.action do |global_options,options,args|
|
256
|
+
render "NOT IMPLEMENTED"
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
c.desc "TODO: Drop partition(s) from a table"
|
261
|
+
c.arg_name "database.table"
|
262
|
+
c.arg_name "key=value,key=value:key=value,key=value"
|
263
|
+
c.command :drop do |add|
|
264
|
+
add.action do |global_options,options,args|
|
265
|
+
render "NOT IMPLEMENTED"
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
c.default_command :list
|
270
|
+
end
|
271
|
+
|
272
|
+
desc 'Run a query from a file or STDIN'
|
273
|
+
arg_name '[file]'
|
274
|
+
command :query do |c|
|
275
|
+
c.action do |global_options,options,args|
|
276
|
+
input = args.first
|
277
|
+
sql = case input
|
278
|
+
when "", NilClass
|
279
|
+
$stdin.read if !STDIN.tty?
|
280
|
+
when String
|
281
|
+
if File.exists?(input)
|
282
|
+
File.read(input)
|
283
|
+
else
|
284
|
+
input
|
285
|
+
end
|
286
|
+
else
|
287
|
+
exit_now! "SQL string or file path required"
|
288
|
+
end
|
289
|
+
|
290
|
+
if global_options[:p]
|
291
|
+
render sql
|
292
|
+
else
|
293
|
+
begin
|
294
|
+
render athena.query(sql).map(&:to_h)
|
295
|
+
rescue Exception => e
|
296
|
+
render e.message
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
301
|
+
|
302
|
+
pre do |global,command,options,args|
|
303
|
+
@access_key = global[:key]
|
304
|
+
@access_secret = global[:secret]
|
305
|
+
@staging_folder = global[:"staging-dir"]
|
306
|
+
|
307
|
+
|
308
|
+
self.check_aws_settings
|
309
|
+
self.check_class_path
|
310
|
+
|
311
|
+
true
|
312
|
+
end
|
313
|
+
|
314
|
+
post do |global,command,options,args|
|
315
|
+
# Post logic here
|
316
|
+
# Use skips_post before a command to skip this
|
317
|
+
# block on that command only
|
318
|
+
end
|
319
|
+
|
320
|
+
on_error do |exception|
|
321
|
+
# Error logic here
|
322
|
+
# return false to skip default error handling
|
323
|
+
true
|
324
|
+
end
|
325
|
+
|
326
|
+
def self.access_key
|
327
|
+
@access_key
|
328
|
+
end
|
329
|
+
|
330
|
+
def self.access_secret
|
331
|
+
@access_secret
|
332
|
+
end
|
333
|
+
|
334
|
+
def self.staging_folder
|
335
|
+
@staging_folder
|
336
|
+
end
|
337
|
+
|
338
|
+
def self.class_path
|
339
|
+
ENV['CLASSPATH']
|
340
|
+
end
|
341
|
+
|
342
|
+
def self.check_aws_settings
|
343
|
+
if self.access_key.nil? || self.access_secret.nil?
|
344
|
+
msg = <<~MSG
|
345
|
+
athena-cli needs your AWS credentials. You can pass them via the
|
346
|
+
--key and --secret flags or set AWS_ACCESS_KEY and AWS_SECRET_KEY
|
347
|
+
environment variables.
|
348
|
+
MSG
|
349
|
+
exit_now! msg
|
350
|
+
end
|
351
|
+
|
352
|
+
if self.staging_folder.nil?
|
353
|
+
msg = <<~MSG
|
354
|
+
athena-cli requires an S3 location to use as a scratch folder.
|
355
|
+
Please provide one via the --staging-dir flag or set the
|
356
|
+
ATHENA_S3_STAGING_DIR environment variable.
|
357
|
+
MSG
|
358
|
+
exit_now! msg, 1
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
def self.check_class_path
|
363
|
+
unless self.class_path =~ /AthenaJDBC/
|
364
|
+
jar_path = File.expand_path(File.join([File.dirname(__FILE__),'..', 'jdbc','AthenaJDBC41-1.0.0.jar']))
|
365
|
+
msg = <<~MSG
|
366
|
+
JRuby requires JDBC driver to be in your Java class path.
|
367
|
+
For download instructions, see:
|
368
|
+
|
369
|
+
http://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html
|
370
|
+
|
371
|
+
After downloading, add /path/to/driver.jar to your CLASSPATH environment variable.
|
372
|
+
|
373
|
+
Example:
|
374
|
+
|
375
|
+
export CLASSPATH="$CLASSPATH:~/src/AthenaJDB41-1.0.0.jar"
|
376
|
+
MSG
|
377
|
+
exit_now! msg, 1
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
def self.details(data)
|
382
|
+
longest_key = data.keys.max_by(&:length)
|
383
|
+
data.each do |key, value|
|
384
|
+
printf "%##{longest_key.length}s %s\n", key, value
|
385
|
+
end
|
386
|
+
end
|
387
|
+
|
388
|
+
def self.render(output)
|
389
|
+
case output
|
390
|
+
when Hash
|
391
|
+
details(output)
|
392
|
+
when String
|
393
|
+
puts output
|
394
|
+
when Array
|
395
|
+
case output.first
|
396
|
+
when Hash
|
397
|
+
tp output
|
398
|
+
else
|
399
|
+
puts output
|
400
|
+
end
|
401
|
+
else
|
402
|
+
return
|
403
|
+
end
|
404
|
+
end
|
405
|
+
end
|
406
|
+
end
|