athena-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.projections.json +20 -0
- data/.ruby-version +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +75 -0
- data/LICENSE.md +20 -0
- data/README.md +128 -0
- data/Rakefile +44 -0
- data/athena-cli.gemspec +26 -0
- data/bin/athena-cli +10 -0
- data/features/athena.feature +8 -0
- data/features/step_definitions/athena_steps.rb +6 -0
- data/features/support/env.rb +15 -0
- data/lib/amazon_athena.rb +7 -0
- data/lib/amazon_athena/cli.rb +406 -0
- data/lib/amazon_athena/client.rb +114 -0
- data/lib/amazon_athena/command.rb +16 -0
- data/lib/amazon_athena/commands.rb +20 -0
- data/lib/amazon_athena/commands/alter_table_add_partition.rb +29 -0
- data/lib/amazon_athena/commands/alter_table_drop_partition.rb +29 -0
- data/lib/amazon_athena/commands/create_database.rb +23 -0
- data/lib/amazon_athena/commands/create_table.rb +23 -0
- data/lib/amazon_athena/commands/describe_table.rb +21 -0
- data/lib/amazon_athena/commands/drop_database.rb +23 -0
- data/lib/amazon_athena/commands/drop_table.rb +23 -0
- data/lib/amazon_athena/commands/repair_table.rb +23 -0
- data/lib/amazon_athena/commands/show_columns.rb +22 -0
- data/lib/amazon_athena/commands/show_create_table.rb +22 -0
- data/lib/amazon_athena/commands/show_databases.rb +17 -0
- data/lib/amazon_athena/commands/show_partitions.rb +28 -0
- data/lib/amazon_athena/commands/show_table_properties.rb +36 -0
- data/lib/amazon_athena/commands/show_tables.rb +23 -0
- data/lib/amazon_athena/partition.rb +21 -0
- data/lib/amazon_athena/transformer.rb +19 -0
- data/lib/amazon_athena/version.rb +3 -0
- data/lib/jdbc_helper/athena.rb +38 -0
- data/lib/jdbc_helper/resultset.rb +14 -0
- data/log4j.xml +18 -0
- data/script/bootstrap +5 -0
- data/script/package +8 -0
- data/script/release +16 -0
- data/script/test +6 -0
- data/test/lib/amazon_athena/commands/alter_table_add_partition_test.rb +64 -0
- data/test/lib/amazon_athena/commands/alter_table_drop_partition_test.rb +61 -0
- data/test/lib/amazon_athena/commands/create_database_test.rb +27 -0
- data/test/lib/amazon_athena/commands/describe_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/drop_database_test.rb +22 -0
- data/test/lib/amazon_athena/commands/drop_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/repair_table_test.rb +20 -0
- data/test/lib/amazon_athena/commands/show_columns_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_create_table_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_databases_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_partitions_test.rb +23 -0
- data/test/lib/amazon_athena/commands/show_table_properties_test.rb +25 -0
- data/test/lib/amazon_athena/commands/show_tables_test.rb +23 -0
- data/test/test_helper.rb +1 -0
- metadata +185 -0
@@ -0,0 +1,114 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
require "jdbc_helper/athena"
|
3
|
+
require_relative "commands"
|
4
|
+
|
5
|
+
module AmazonAthena
|
6
|
+
class Client
|
7
|
+
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
def_delegators :connection, :query, :execute
|
11
|
+
|
12
|
+
def initialize(key: nil, secret: nil, region: "us-east-1", s3_staging_dir: nil)
|
13
|
+
@key = key || ENV["AWS_ACCESS_KEY"]
|
14
|
+
@secret = secret || ENV['AWS_SECRET_KEY']
|
15
|
+
@region = region
|
16
|
+
@s3_staging_dir = s3_staging_dir || ENV["ATHENA_S3_STAGING_DIR"]
|
17
|
+
end
|
18
|
+
|
19
|
+
def databases
|
20
|
+
cmd = AmazonAthena::Commands::ShowDatabases.new
|
21
|
+
|
22
|
+
run(cmd)
|
23
|
+
end
|
24
|
+
|
25
|
+
def database_drop(database)
|
26
|
+
cmd = AmazonAthena::Commands::DropDatabase.new(database)
|
27
|
+
|
28
|
+
run(cmd)
|
29
|
+
end
|
30
|
+
|
31
|
+
def database_create(name:, location: nil, comment: nil, properties: {})
|
32
|
+
cmd = AmazonAthena::Commands::CreateDatabase.new(
|
33
|
+
name: name,
|
34
|
+
location: location,
|
35
|
+
comment: comment,
|
36
|
+
properties: properties
|
37
|
+
)
|
38
|
+
|
39
|
+
run(cmd)
|
40
|
+
end
|
41
|
+
|
42
|
+
def tables(database)
|
43
|
+
cmd = AmazonAthena::Commands::ShowTables.new(database)
|
44
|
+
|
45
|
+
run(cmd)
|
46
|
+
end
|
47
|
+
|
48
|
+
def table_drop(database_table)
|
49
|
+
cmd = AmazonAthena::Commands::DropTable.new(database)
|
50
|
+
|
51
|
+
run(cmd)
|
52
|
+
end
|
53
|
+
|
54
|
+
def table_columns(database_table)
|
55
|
+
cmd = AmazonAthena::Commands::ShowColumns.new(database_table)
|
56
|
+
|
57
|
+
run(cmd)
|
58
|
+
end
|
59
|
+
|
60
|
+
def table_show_create(database_table)
|
61
|
+
cmd = AmazonAthena::Commands::ShowCreateTable.new(database_table)
|
62
|
+
|
63
|
+
run(cmd)
|
64
|
+
end
|
65
|
+
|
66
|
+
def table_describe(database_table)
|
67
|
+
cmd = AmazonAthena::Commands::DescribeTable.new(database_table)
|
68
|
+
|
69
|
+
run(cmd)
|
70
|
+
end
|
71
|
+
|
72
|
+
def table_repair(database_table)
|
73
|
+
cmd = AmazonAthena::Commands::RepairTable.new(database_table)
|
74
|
+
run(cmd)
|
75
|
+
|
76
|
+
partitions(database_table)
|
77
|
+
end
|
78
|
+
|
79
|
+
def table_properties(database_table)
|
80
|
+
cmd = AmazonAthena::Commands::ShowTableProperties.new(database_table)
|
81
|
+
|
82
|
+
run(cmd)
|
83
|
+
end
|
84
|
+
|
85
|
+
def partitions(table)
|
86
|
+
cmd = AmazonAthena::Commands::ShowPartitions.new(database_table)
|
87
|
+
|
88
|
+
run(cmd)
|
89
|
+
end
|
90
|
+
|
91
|
+
def partitions_drop(database_table, partitions_expression)
|
92
|
+
cmd = AmazonAthena::Commands::DropPartition.new(database_table, partitions)
|
93
|
+
|
94
|
+
run(cmd)
|
95
|
+
end
|
96
|
+
|
97
|
+
def connection
|
98
|
+
return @connection if defined?(@connection) && !@connection.closed?
|
99
|
+
|
100
|
+
@connection = JDBCHelper::Athena.connect(
|
101
|
+
key: @key,
|
102
|
+
secret: @secret,
|
103
|
+
region: @region,
|
104
|
+
s3_staging_dir: @s3_staging_dir
|
105
|
+
)
|
106
|
+
end
|
107
|
+
|
108
|
+
def run(cmd, preview = false)
|
109
|
+
return cmd.preview if preview
|
110
|
+
|
111
|
+
cmd.run(connection)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative "command"
|
2
|
+
require_relative "commands/alter_table_add_partition"
|
3
|
+
require_relative "commands/alter_table_drop_partition"
|
4
|
+
require_relative "commands/create_database"
|
5
|
+
require_relative "commands/create_table"
|
6
|
+
require_relative "commands/describe_table"
|
7
|
+
require_relative "commands/drop_database"
|
8
|
+
require_relative "commands/drop_table"
|
9
|
+
require_relative "commands/repair_table"
|
10
|
+
require_relative "commands/show_columns"
|
11
|
+
require_relative "commands/show_create_table"
|
12
|
+
require_relative "commands/show_databases"
|
13
|
+
require_relative "commands/show_partitions"
|
14
|
+
require_relative "commands/show_table_properties"
|
15
|
+
require_relative "commands/show_tables"
|
16
|
+
|
17
|
+
module AmazonAthena
|
18
|
+
module Commands
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
require_relative '../partition'
|
3
|
+
|
4
|
+
module AmazonAthena
|
5
|
+
module Commands
|
6
|
+
class AlterTableAddPartition < AmazonAthena::Command
|
7
|
+
|
8
|
+
def initialize(database_table, partitions)
|
9
|
+
@database_table = database_table
|
10
|
+
@partitions = partitions
|
11
|
+
end
|
12
|
+
|
13
|
+
def partition_clauses
|
14
|
+
@partitions.map {|p| " #{p}"}.join("\n")
|
15
|
+
end
|
16
|
+
|
17
|
+
def statement
|
18
|
+
"ALTER TABLE #{@database_table} ADD\n#{partition_clauses};"
|
19
|
+
end
|
20
|
+
|
21
|
+
def run(connection)
|
22
|
+
# TODO: Map fields directly
|
23
|
+
connection.query(statement).raw_output
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
require_relative '../partition'
|
3
|
+
|
4
|
+
module AmazonAthena
|
5
|
+
module Commands
|
6
|
+
class AlterTableDropPartition < AmazonAthena::Command
|
7
|
+
|
8
|
+
def initialize(database_table, partitions)
|
9
|
+
@database_table = database_table
|
10
|
+
@partitions = partitions
|
11
|
+
end
|
12
|
+
|
13
|
+
def partition_clauses
|
14
|
+
@partitions.map {|p| " #{p}"}.join(",\n")
|
15
|
+
end
|
16
|
+
|
17
|
+
def statement
|
18
|
+
"ALTER TABLE #{@database_table} DROP\n#{partition_clauses};"
|
19
|
+
end
|
20
|
+
|
21
|
+
def run(connection)
|
22
|
+
# TODO: Map fields directly
|
23
|
+
connection.query(statement).raw_output
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class CreateDatabase < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(name)
|
8
|
+
@name = name
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"CREATE DATABASE IF NOT EXISTS #{@name};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement)
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class CreateTable < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(ddl)
|
8
|
+
@ddl = ddl
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
@ddl
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement)
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class DescribeTable < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"DESCRIBE #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement).raw_output
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class DropDatabase < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_name)
|
8
|
+
@database_name = database_name
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"DROP DATABASE IF EXISTS #{@database_name};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement)
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class DropTable < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"DROP TABLE #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement)
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class RepairTable < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"MSCK REPAIR TABLE #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement)
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowColumns < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"SHOW COLUMNS IN #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement).map {|row| row.field.strip }
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowCreateTable < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"SHOW CREATE TABLE #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
connection.query(statement).raw_output
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowDatabases < AmazonAthena::Command
|
6
|
+
|
7
|
+
def statement
|
8
|
+
"SHOW DATABASES;"
|
9
|
+
end
|
10
|
+
|
11
|
+
def run(connection)
|
12
|
+
connection.query(statement).map {|row| row.database_name }
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require_relative '../command'
|
2
|
+
|
3
|
+
module AmazonAthena
|
4
|
+
module Commands
|
5
|
+
class ShowPartitions < AmazonAthena::Command
|
6
|
+
|
7
|
+
def initialize(database_table)
|
8
|
+
@database_table = database_table
|
9
|
+
end
|
10
|
+
|
11
|
+
def statement
|
12
|
+
"SHOW PARTITIONS #{@database_table};"
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(connection)
|
16
|
+
# TODO: Map fields directly
|
17
|
+
connection.query(statement).raw_output
|
18
|
+
rescue Exception => e
|
19
|
+
case e.message
|
20
|
+
when /not a partitioned table/
|
21
|
+
"Error: #{@database_table} is not a partitioned table."
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|