athena-cli 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.projections.json +20 -0
- data/.ruby-version +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +75 -0
- data/LICENSE.md +20 -0
- data/README.md +128 -0
- data/Rakefile +44 -0
- data/athena-cli.gemspec +26 -0
- data/bin/athena-cli +10 -0
- data/features/athena.feature +8 -0
- data/features/step_definitions/athena_steps.rb +6 -0
- data/features/support/env.rb +15 -0
- data/lib/amazon_athena.rb +7 -0
- data/lib/amazon_athena/cli.rb +406 -0
- data/lib/amazon_athena/client.rb +114 -0
- data/lib/amazon_athena/command.rb +16 -0
- data/lib/amazon_athena/commands.rb +20 -0
- data/lib/amazon_athena/commands/alter_table_add_partition.rb +29 -0
- data/lib/amazon_athena/commands/alter_table_drop_partition.rb +29 -0
- data/lib/amazon_athena/commands/create_database.rb +23 -0
- data/lib/amazon_athena/commands/create_table.rb +23 -0
- data/lib/amazon_athena/commands/describe_table.rb +21 -0
- data/lib/amazon_athena/commands/drop_database.rb +23 -0
- data/lib/amazon_athena/commands/drop_table.rb +23 -0
- data/lib/amazon_athena/commands/repair_table.rb +23 -0
- data/lib/amazon_athena/commands/show_columns.rb +22 -0
- data/lib/amazon_athena/commands/show_create_table.rb +22 -0
- data/lib/amazon_athena/commands/show_databases.rb +17 -0
- data/lib/amazon_athena/commands/show_partitions.rb +28 -0
- data/lib/amazon_athena/commands/show_table_properties.rb +36 -0
- data/lib/amazon_athena/commands/show_tables.rb +23 -0
- data/lib/amazon_athena/partition.rb +21 -0
- data/lib/amazon_athena/transformer.rb +19 -0
- data/lib/amazon_athena/version.rb +3 -0
- data/lib/jdbc_helper/athena.rb +38 -0
- data/lib/jdbc_helper/resultset.rb +14 -0
- data/log4j.xml +18 -0
- data/script/bootstrap +5 -0
- data/script/package +8 -0
- data/script/release +16 -0
- data/script/test +6 -0
- data/test/lib/amazon_athena/commands/alter_table_add_partition_test.rb +64 -0
- data/test/lib/amazon_athena/commands/alter_table_drop_partition_test.rb +61 -0
- data/test/lib/amazon_athena/commands/create_database_test.rb +27 -0
- data/test/lib/amazon_athena/commands/describe_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/drop_database_test.rb +22 -0
- data/test/lib/amazon_athena/commands/drop_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/repair_table_test.rb +20 -0
- data/test/lib/amazon_athena/commands/show_columns_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_create_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_databases_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_partitions_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_table_properties_test.rb +25 -0
- data/test/lib/amazon_athena/commands/show_tables_test.rb +23 -0
- data/test/test_helper.rb +1 -0
- metadata +185 -0
@@ -0,0 +1,36 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowTableProperties < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"SHOW TBLPROPERTIES #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
result = connection.query(statement).raw_output
|
17
|
+
|
18
|
+
data = Hash[*result.split("\n").map {|line| line.split("\t")}.flatten]
|
19
|
+
|
20
|
+
data[:name] = @database_table
|
21
|
+
|
22
|
+
if type = data.delete('EXTERNAL')
|
23
|
+
data[:external] = type
|
24
|
+
end
|
25
|
+
|
26
|
+
if last_modified = data.delete('transient_lastDdlTime')
|
27
|
+
data[:last_modified] = Time.at(last_modified.to_i)
|
28
|
+
end
|
29
|
+
|
30
|
+
data
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowTables < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_name)
|
8
|
+
@database_name = database_name.strip
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"SHOW TABLES IN #{@database_name};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement).map {|row| row.tab_name }
|
17
|
+
rescue Exception => e
|
18
|
+
e.getCause()
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module AmazonAthena
|
2
|
+
class Partition
|
3
|
+
|
4
|
+
def initialize(options: {}, location: nil)
|
5
|
+
@options = options
|
6
|
+
@location = location
|
7
|
+
end
|
8
|
+
|
9
|
+
def to_s
|
10
|
+
return nil if @options.empty?
|
11
|
+
|
12
|
+
# TODO: Sanitize and handle non-strings
|
13
|
+
opts = @options.map {|k,v| "#{k} = '#{v}'"}.join(", ")
|
14
|
+
|
15
|
+
sql = "PARTITION (#{opts})"
|
16
|
+
sql += " LOCATION '#{@location}'" if @location
|
17
|
+
|
18
|
+
sql
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module AmazonAthena
|
2
|
+
class Transformer
|
3
|
+
|
4
|
+
TABLE_NAME_PATTERN = /CREATE EXTERNAL TABLE `(?<name>\S+)`/
|
5
|
+
TABLE_LOCATION_PATTERN = /'(?<location>s3:\/\/\S+)'/
|
6
|
+
|
7
|
+
def self.transform_table(ddl, options = {})
|
8
|
+
if name = options[:name]
|
9
|
+
ddl[TABLE_NAME_PATTERN] = "CREATE EXTERNAL TABLE `#{name}`"
|
10
|
+
end
|
11
|
+
|
12
|
+
if location = options[:location]
|
13
|
+
ddl[TABLE_LOCATION_PATTERN] = "'s3://#{location}'"
|
14
|
+
end
|
15
|
+
|
16
|
+
ddl
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require "jdbc-helper"
|
2
|
+
require "jdbc_helper/resultset"
|
3
|
+
|
4
|
+
module JDBCHelper
|
5
|
+
module Athena
|
6
|
+
extend Connector
|
7
|
+
|
8
|
+
DRIVER_NAME = "com.amazonaws.athena.jdbc.AthenaDriver".freeze
|
9
|
+
JDBCHelper::Constants::Connector::DEFAULT_PARAMETERS[:athena] = {
|
10
|
+
driver: "com.amazonaws.athena.jdbc.AthenaDriver"
|
11
|
+
}
|
12
|
+
|
13
|
+
def self.connect(key: nil, secret: nil, region: "us-east-1", s3_staging_dir: nil, extra_params: {}, &block)
|
14
|
+
connect_impl :athena, {
|
15
|
+
url: "jdbc:awsathena://athena.#{region}.amazonaws.com:443",
|
16
|
+
user: key || ENV["AWS_ACCESS_KEY"],
|
17
|
+
password: secret || ENV['AWS_SECRET_KEY'],
|
18
|
+
s3_staging_dir: s3_uri(s3_staging_dir || ENV["ATHENA_S3_STAGING_DIR"])
|
19
|
+
}, {}, &block
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.configure_driver_path(class_path)
|
23
|
+
paths = ENV["CLASSPATH"].to_s.split(":")
|
24
|
+
paths << class_path
|
25
|
+
|
26
|
+
ENV["CLASSPATH"] = paths.uniq.join(":")
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.s3_uri(path)
|
30
|
+
return path if path.to_s.start_with?("s3://")
|
31
|
+
|
32
|
+
path = "s3://#{path}"
|
33
|
+
path = path + "/" unless path.end_with?("/")
|
34
|
+
|
35
|
+
path
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/log4j.xml
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" ?>
|
2
|
+
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
|
3
|
+
|
4
|
+
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
|
5
|
+
<appender name="console" class="org.apache.log4j.ConsoleAppender">
|
6
|
+
<param name="Target" value="System.out"/>
|
7
|
+
<layout class="org.apache.log4j.PatternLayout">
|
8
|
+
<param name="ConversionPattern" value="%-5p %c{1} - %m%n"/>
|
9
|
+
</layout>
|
10
|
+
</appender>
|
11
|
+
|
12
|
+
<root>
|
13
|
+
<priority value ="OFF" />
|
14
|
+
<appender-ref ref="console" />
|
15
|
+
</root>
|
16
|
+
|
17
|
+
|
18
|
+
</log4j:configuration>
|
data/script/bootstrap
ADDED
data/script/package
ADDED
data/script/release
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
# Usage: script/release
|
3
|
+
# Build the package, tag a commit, push it to origin, and then release the
|
4
|
+
# package publicly.
|
5
|
+
|
6
|
+
set -e
|
7
|
+
|
8
|
+
version="$(script/package | grep Version: | awk '{print $2}')"
|
9
|
+
[ -n "$version" ] || exit 1
|
10
|
+
|
11
|
+
git commit --allow-empty -a -m "Release $version"
|
12
|
+
git tag "v$version"
|
13
|
+
git push origin
|
14
|
+
git push origin "v$version"
|
15
|
+
gem push pkg/*-${version}.gem
|
16
|
+
|
data/script/test
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/alter_table_add_partition'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::AlterTableAddPartition do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@klass = AmazonAthena::Commands::AlterTableAddPartition
|
8
|
+
end
|
9
|
+
|
10
|
+
it "builds a ddl statement" do
|
11
|
+
partitions = []
|
12
|
+
|
13
|
+
partitions << AmazonAthena::Partition.new(
|
14
|
+
location: 's3://mystorage/path/to/INDIA_14_May_2014',
|
15
|
+
options: {
|
16
|
+
dt: '2014-05-14',
|
17
|
+
country: 'IN'
|
18
|
+
}
|
19
|
+
)
|
20
|
+
|
21
|
+
partitions << AmazonAthena::Partition.new(
|
22
|
+
location: 's3://mystorage/path/to/INDIA_15_May_2014',
|
23
|
+
options: {
|
24
|
+
dt: '2014-05-15',
|
25
|
+
country: 'IN'
|
26
|
+
}
|
27
|
+
)
|
28
|
+
|
29
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
30
|
+
expected = <<~SQL
|
31
|
+
ALTER TABLE mydb.mytable ADD
|
32
|
+
PARTITION (dt = '2014-05-14', country = 'IN') LOCATION 's3://mystorage/path/to/INDIA_14_May_2014'
|
33
|
+
PARTITION (dt = '2014-05-15', country = 'IN') LOCATION 's3://mystorage/path/to/INDIA_15_May_2014';
|
34
|
+
SQL
|
35
|
+
assert_equal expected.strip, cmd.statement
|
36
|
+
end
|
37
|
+
|
38
|
+
it "executes a query" do
|
39
|
+
partitions = []
|
40
|
+
|
41
|
+
partitions << AmazonAthena::Partition.new(
|
42
|
+
location: 's3://mystorage/path/to/INDIA_14_May_2014',
|
43
|
+
options: {
|
44
|
+
dt: '2014-05-14',
|
45
|
+
country: 'IN'
|
46
|
+
}
|
47
|
+
)
|
48
|
+
|
49
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
50
|
+
sql = <<~SQL
|
51
|
+
ALTER TABLE mydb.mytable ADD
|
52
|
+
PARTITION (dt = '2014-05-14', country = 'IN') LOCATION 's3://mystorage/path/to/INDIA_14_May_2014';
|
53
|
+
SQL
|
54
|
+
|
55
|
+
results = MiniTest::Mock.new
|
56
|
+
results.expect(:raw_output, nil)
|
57
|
+
|
58
|
+
conn = MiniTest::Mock.new
|
59
|
+
conn.expect(:query, results, [sql.strip])
|
60
|
+
|
61
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
62
|
+
cmd.run(conn)
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/alter_table_drop_partition'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::AlterTableDropPartition do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@klass = AmazonAthena::Commands::AlterTableDropPartition
|
8
|
+
end
|
9
|
+
|
10
|
+
it "builds a ddl statement" do
|
11
|
+
partitions = []
|
12
|
+
|
13
|
+
partitions << AmazonAthena::Partition.new(
|
14
|
+
options: {
|
15
|
+
dt: '2014-05-14',
|
16
|
+
country: 'IN'
|
17
|
+
}
|
18
|
+
)
|
19
|
+
|
20
|
+
partitions << AmazonAthena::Partition.new(
|
21
|
+
options: {
|
22
|
+
dt: '2014-05-15',
|
23
|
+
country: 'IN'
|
24
|
+
}
|
25
|
+
)
|
26
|
+
|
27
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
28
|
+
expected = <<~SQL
|
29
|
+
ALTER TABLE mydb.mytable DROP
|
30
|
+
PARTITION (dt = '2014-05-14', country = 'IN'),
|
31
|
+
PARTITION (dt = '2014-05-15', country = 'IN');
|
32
|
+
SQL
|
33
|
+
assert_equal expected.strip, cmd.statement
|
34
|
+
end
|
35
|
+
|
36
|
+
it "executes a query" do
|
37
|
+
partitions = []
|
38
|
+
|
39
|
+
partitions << AmazonAthena::Partition.new(
|
40
|
+
options: {
|
41
|
+
dt: '2014-05-14',
|
42
|
+
country: 'IN'
|
43
|
+
}
|
44
|
+
)
|
45
|
+
|
46
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
47
|
+
sql = <<~SQL
|
48
|
+
ALTER TABLE mydb.mytable DROP
|
49
|
+
PARTITION (dt = '2014-05-14', country = 'IN');
|
50
|
+
SQL
|
51
|
+
|
52
|
+
results = MiniTest::Mock.new
|
53
|
+
results.expect(:raw_output, nil)
|
54
|
+
|
55
|
+
conn = MiniTest::Mock.new
|
56
|
+
conn.expect(:query, results, [sql.strip])
|
57
|
+
|
58
|
+
cmd = @klass.new("mydb.mytable", partitions)
|
59
|
+
cmd.run(conn)
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/create_database'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::CreateDatabase do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@klass = AmazonAthena::Commands::CreateDatabase
|
8
|
+
end
|
9
|
+
|
10
|
+
it "provides a statement" do
|
11
|
+
cmd = @klass.new("mydb")
|
12
|
+
|
13
|
+
assert_equal "CREATE DATABASE IF NOT EXISTS mydb;", cmd.statement
|
14
|
+
end
|
15
|
+
|
16
|
+
it "executes a query" do
|
17
|
+
cmd = @klass.new("mydb")
|
18
|
+
|
19
|
+
results = MiniTest::Mock.new
|
20
|
+
results.expect(:raw_output, nil)
|
21
|
+
|
22
|
+
conn = MiniTest::Mock.new
|
23
|
+
conn.expect(:query, results, ["CREATE DATABASE IF NOT EXISTS mydb;"])
|
24
|
+
|
25
|
+
cmd.run(conn)
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/describe_table'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::DescribeTable do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@cmd = AmazonAthena::Commands::DescribeTable.new("mydb.mytable")
|
8
|
+
end
|
9
|
+
|
10
|
+
it "provides a db statement" do
|
11
|
+
assert_equal "DESCRIBE mydb.mytable;", @cmd.statement
|
12
|
+
end
|
13
|
+
|
14
|
+
it "executes a query" do
|
15
|
+
results = MiniTest::Mock.new
|
16
|
+
results.expect(:raw_output, nil)
|
17
|
+
|
18
|
+
conn = MiniTest::Mock.new
|
19
|
+
conn.expect(:query, results, ["DESCRIBE mydb.mytable;"])
|
20
|
+
|
21
|
+
@cmd.run(conn)
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/drop_database'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::DropDatabase do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@cmd = AmazonAthena::Commands::DropDatabase.new("mydb")
|
8
|
+
end
|
9
|
+
|
10
|
+
it "provides a db statement" do
|
11
|
+
assert_equal "DROP DATABASE IF EXISTS mydb;", @cmd.statement
|
12
|
+
end
|
13
|
+
|
14
|
+
it "executes a query" do
|
15
|
+
results = MiniTest::Mock.new
|
16
|
+
|
17
|
+
conn = MiniTest::Mock.new
|
18
|
+
conn.expect(:query, results, ["DROP DATABASE IF EXISTS mydb;"])
|
19
|
+
|
20
|
+
@cmd.run(conn)
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require './lib/amazon_athena/commands/drop_table'
|
3
|
+
|
4
|
+
describe AmazonAthena::Commands::DropTable do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@cmd = AmazonAthena::Commands::DropTable.new("mydb.mytable")
|
8
|
+
end
|
9
|
+
|
10
|
+
it "provides a db statement" do
|
11
|
+
assert_equal "DROP TABLE mydb.mytable;", @cmd.statement
|
12
|
+
end
|
13
|
+
|
14
|
+
it "executes a query" do
|
15
|
+
results = MiniTest::Mock.new
|
16
|
+
results.expect(:raw_output, nil)
|
17
|
+
|
18
|
+
conn = MiniTest::Mock.new
|
19
|
+
conn.expect(:query, results, ["DROP TABLE mydb.mytable;"])
|
20
|
+
|
21
|
+
@cmd.run(conn)
|
22
|
+
end
|
23
|
+
end
|