dynamocli 0.1.0 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +9 -5
- data/README.md +41 -3
- data/{dynamorb.gemspec → dynamocli.gemspec} +1 -0
- data/lib/dynamocli.rb +20 -1
- data/lib/dynamocli/erase.rb +230 -0
- data/lib/dynamocli/import.rb +0 -1
- data/lib/dynamocli/version.rb +1 -1
- metadata +19 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 613ace3c66e1db3954fef903bc0f86700787448328964687ebed39520dfe2205
|
4
|
+
data.tar.gz: e0fb394d2b04fc78928fe64eb7107301c91a5edcf17719ece5decb65007ca5ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fdd626d3441fbeaf0869542d356946f954b554d8daae402a908c15b059b7405f889d408f9ad67a4dfdbc72380e13ec20738a8e16ef743b235d3f75c55c662f29
|
7
|
+
data.tar.gz: 802b9d48e4b77c5a40675234c04f7057204601bfa7077f4f45c5646157481389a36f8cd12437b922850bf8b8f533bfbb7c6d655e17e38421932f01da42884009
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# CHANGELOG
|
2
|
+
|
3
|
+
All notable changes to this project will be documented in this file.
|
4
|
+
|
5
|
+
## [0.1.2] - 2019-06-20
|
6
|
+
### Added
|
7
|
+
- Command to erase a DynamoDB table.
|
8
|
+
|
9
|
+
## [0.1.1] - 2019-05-21 [YANKED]
|
10
|
+
|
11
|
+
## [0.1.0] - 2019-05-20
|
12
|
+
### Added
|
13
|
+
- Command to import data from a CSV file to a DynamoDB table.
|
14
|
+
|
15
|
+
[0.1.0]: https://github.com/matheussilvasantos/dynamocli/releases/tag/v0.1.0
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
dynamocli (0.1.
|
4
|
+
dynamocli (0.1.2)
|
5
|
+
aws-sdk-cloudformation
|
5
6
|
aws-sdk-dynamodb (~> 1.28)
|
6
7
|
thor (~> 0.20)
|
7
8
|
|
@@ -9,14 +10,17 @@ GEM
|
|
9
10
|
remote: https://rubygems.org/
|
10
11
|
specs:
|
11
12
|
aws-eventstream (1.0.3)
|
12
|
-
aws-partitions (1.
|
13
|
-
aws-sdk-
|
13
|
+
aws-partitions (1.174.0)
|
14
|
+
aws-sdk-cloudformation (1.22.0)
|
15
|
+
aws-sdk-core (~> 3, >= 3.53.0)
|
16
|
+
aws-sigv4 (~> 1.1)
|
17
|
+
aws-sdk-core (3.54.2)
|
14
18
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
15
19
|
aws-partitions (~> 1.0)
|
16
20
|
aws-sigv4 (~> 1.1)
|
17
21
|
jmespath (~> 1.0)
|
18
|
-
aws-sdk-dynamodb (1.
|
19
|
-
aws-sdk-core (~> 3, >= 3.
|
22
|
+
aws-sdk-dynamodb (1.30.0)
|
23
|
+
aws-sdk-core (~> 3, >= 3.53.0)
|
20
24
|
aws-sigv4 (~> 1.1)
|
21
25
|
aws-sigv4 (1.1.0)
|
22
26
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Dynamocli
|
2
2
|
|
3
|
-
Utilites for interaction with your DynamoDB tables
|
3
|
+
Utilites for interaction with your DynamoDB tables.
|
4
4
|
|
5
5
|
## Installation
|
6
6
|
|
@@ -10,14 +10,52 @@ gem install dynamocli
|
|
10
10
|
|
11
11
|
## Usage
|
12
12
|
|
13
|
+
|
13
14
|
You have to configure AWS in your computer first. The program will use the AWS credentials configured in your computer.
|
14
15
|
|
15
|
-
|
16
|
+
- Import data from a CSV file to a DynamoDB table
|
17
|
+
|
18
|
+
```
|
19
|
+
Usage:
|
20
|
+
dynamocli import FILE -t, --table, --to=TABLE
|
21
|
+
|
22
|
+
Options:
|
23
|
+
-t, --table, --to=TABLE # table you want to import the data
|
24
|
+
|
25
|
+
Description:
|
26
|
+
`dynamocli import` will import the data in from a file to a table specified.
|
27
|
+
|
28
|
+
> $ dynamo import users.csv --to users
|
29
|
+
|
30
|
+
```
|
31
|
+
|
32
|
+
- Erase all the data of a DynamoDB table
|
16
33
|
|
17
34
|
```
|
18
|
-
|
35
|
+
Usage:
|
36
|
+
dynamocli erase TABLE
|
37
|
+
|
38
|
+
Options:
|
39
|
+
[--with-drift], [--no-with-drift] # drop the table and
|
40
|
+
recreate it directly instead of use deployments
|
41
|
+
|
42
|
+
Description:
|
43
|
+
`dynamocli erase` will erase all the data of the specified table.
|
44
|
+
|
45
|
+
It will drop the table and recreate it.
|
46
|
+
|
47
|
+
If the table is in a stack it will try to deploy
|
48
|
+
the stack without the table and then redeploy the
|
49
|
+
stack with the original template. You can change
|
50
|
+
this behavior passing the option --with-drift.
|
51
|
+
|
52
|
+
> $ dynamo erase users
|
19
53
|
```
|
20
54
|
|
55
|
+
From the DynamoDB Guidelines for Working with Tables documentation:
|
56
|
+
|
57
|
+
> Deleting an entire table is significantly more efficient than removing items one-by-one, which essentially doubles the write throughput as you do as many delete operations as put operations.
|
58
|
+
|
21
59
|
## Development
|
22
60
|
|
23
61
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
|
24
24
|
spec.add_dependency "thor", "~> 0.20"
|
25
25
|
spec.add_dependency "aws-sdk-dynamodb", "~> 1.28"
|
26
|
+
spec.add_dependency "aws-sdk-cloudformation"
|
26
27
|
|
27
28
|
spec.add_development_dependency "bundler", "~> 1.17"
|
28
29
|
spec.add_development_dependency "rake", "~> 10.0"
|
data/lib/dynamocli.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
|
+
require "thor"
|
1
2
|
require "dynamocli/version"
|
2
3
|
require "dynamocli/import"
|
4
|
+
require "dynamocli/erase"
|
3
5
|
|
4
6
|
module Dynamocli
|
5
7
|
class Client < Thor
|
6
8
|
desc "import FILE", "import data from FILE to dynamodb"
|
7
9
|
long_desc <<-LONGDESC
|
8
|
-
`
|
10
|
+
`dynamocli import` will import the data in from a file
|
9
11
|
to a table specified.
|
10
12
|
|
11
13
|
> $ dynamo import users.csv --to users
|
@@ -14,5 +16,22 @@ module Dynamocli
|
|
14
16
|
def import(file)
|
15
17
|
Dynamocli::Import.new(file: file, table: options[:to]).start
|
16
18
|
end
|
19
|
+
|
20
|
+
desc "erase TABLE", "erase all the data from the DynamoDB TABLE"
|
21
|
+
long_desc <<-LONGDESC
|
22
|
+
`dynamocli erase` will erase all the data of the specified table.
|
23
|
+
|
24
|
+
It will drop the table and recreate it.
|
25
|
+
|
26
|
+
If the table is in a stack it will try to deploy the stack without
|
27
|
+
the table and then redeploy the stack with the original template.
|
28
|
+
You can change this behavior passing the option --with-drift.
|
29
|
+
|
30
|
+
> $ dynamo erase users
|
31
|
+
LONGDESC
|
32
|
+
option "with-drift", desc: "drop the table and recreate it directly instead of use deployments", type: :boolean
|
33
|
+
def erase(table)
|
34
|
+
Dynamocli::Erase.new(table_name: table, with_drift: options["with-drift"]).start
|
35
|
+
end
|
17
36
|
end
|
18
37
|
end
|
@@ -0,0 +1,230 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "yaml"
|
5
|
+
require "aws-sdk-dynamodb"
|
6
|
+
require "aws-sdk-cloudformation"
|
7
|
+
|
8
|
+
class Dynamocli::Erase
|
9
|
+
def initialize(table_name:, with_drift: false)
|
10
|
+
@with_drift = with_drift
|
11
|
+
@table_name = table_name
|
12
|
+
|
13
|
+
@dynamodb = Aws::DynamoDB::Client.new
|
14
|
+
@cloudformation = Aws::CloudFormation::Client.new
|
15
|
+
@table = Aws::DynamoDB::Table.new(@table_name)
|
16
|
+
|
17
|
+
set_schema
|
18
|
+
|
19
|
+
@stack_resources = @cloudformation.describe_stack_resources(physical_resource_id: @table_name).to_h
|
20
|
+
|
21
|
+
set_stack_information
|
22
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
|
23
|
+
STDERR.puts "ERROR: #{e.message}"
|
24
|
+
exit(42)
|
25
|
+
rescue Aws::CloudFormation::Errors::ValidationError
|
26
|
+
@stack_resources = nil
|
27
|
+
end
|
28
|
+
|
29
|
+
def start
|
30
|
+
erase_table
|
31
|
+
rescue Aws::CloudFormation::Errors::ValidationError => e
|
32
|
+
STDERR.puts "ERROR: #{e.message}"
|
33
|
+
exit(42)
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def set_schema
|
39
|
+
@schema = @dynamodb.describe_table(table_name: @table_name).to_h[:table].tap do |schema|
|
40
|
+
schema.delete(:table_status)
|
41
|
+
schema.delete(:creation_date_time)
|
42
|
+
schema.delete(:table_size_bytes)
|
43
|
+
schema.delete(:item_count)
|
44
|
+
schema.delete(:table_arn)
|
45
|
+
schema.delete(:table_id)
|
46
|
+
schema[:provisioned_throughput].delete(:number_of_decreases_today)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def set_stack_information
|
51
|
+
return if @stack_resources.nil?
|
52
|
+
|
53
|
+
set_stack_name
|
54
|
+
set_stack
|
55
|
+
set_templates
|
56
|
+
rescue Aws::CloudFormation::Errors::ValidationError => e
|
57
|
+
STDERR.puts "ERROR: #{e.message}"
|
58
|
+
exit(42)
|
59
|
+
end
|
60
|
+
|
61
|
+
def set_stack_name
|
62
|
+
table_resource = @stack_resources[:stack_resources].find do |resource|
|
63
|
+
resource[:physical_resource_id] == @table_name
|
64
|
+
end
|
65
|
+
@stack_name = table_resource[:stack_name]
|
66
|
+
end
|
67
|
+
|
68
|
+
def set_stack
|
69
|
+
@stack = @cloudformation.describe_stacks(stack_name: @stack_name)[0][0]
|
70
|
+
end
|
71
|
+
|
72
|
+
def set_templates
|
73
|
+
template_body = @cloudformation.get_template(stack_name: @stack_name).to_h[:template_body]
|
74
|
+
@original_template = parse_template(template_body)
|
75
|
+
@template_without_table = parse_template(template_body)
|
76
|
+
|
77
|
+
tables = @original_template["Resources"].select { |_, v| v["Type"] == "AWS::DynamoDB::Table" }
|
78
|
+
table = tables.find { |_, v| v["Properties"]["TableName"] == @table_name }
|
79
|
+
|
80
|
+
if tables.nil?
|
81
|
+
STDERR.puts "ERROR: table #{@table_name} not found in the #{@stack_name} stack"
|
82
|
+
exit(42)
|
83
|
+
end
|
84
|
+
|
85
|
+
logical_resource_id = table.first
|
86
|
+
@template_without_table["Resources"].delete(logical_resource_id)
|
87
|
+
end
|
88
|
+
|
89
|
+
def parse_template(template)
|
90
|
+
JSON.parse(template)
|
91
|
+
rescue JSON::ParserError
|
92
|
+
YAML.load(template)
|
93
|
+
end
|
94
|
+
|
95
|
+
def erase_table
|
96
|
+
if @stack_resources.nil? || @with_drift
|
97
|
+
check_if_user_wants_to_continue_with_recreation
|
98
|
+
delete_and_recreate_the_table
|
99
|
+
else
|
100
|
+
check_if_user_wants_to_continue_with_deployment
|
101
|
+
erase_table_through_cloudformation
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def check_if_user_wants_to_continue_with_recreation
|
106
|
+
STDOUT.print(
|
107
|
+
"WARNING: You're going to drop and recreate your #{@table_name} table,\n" \
|
108
|
+
"do you really want to continue?\n" \
|
109
|
+
"(anything other than 'y' will cancel) > "
|
110
|
+
)
|
111
|
+
|
112
|
+
confirmation = STDIN.gets.strip
|
113
|
+
return if confirmation == "y"
|
114
|
+
|
115
|
+
STDOUT.puts abort_message
|
116
|
+
exit(0)
|
117
|
+
end
|
118
|
+
|
119
|
+
def abort_message
|
120
|
+
"INFO: Erase of #{@table_name} table canceled"
|
121
|
+
end
|
122
|
+
|
123
|
+
def delete_and_recreate_the_table
|
124
|
+
delete_table
|
125
|
+
wait_for_deletion_to_complete
|
126
|
+
create_table
|
127
|
+
end
|
128
|
+
|
129
|
+
def delete_table
|
130
|
+
STDOUT.puts "INFO: Deleting the #{@table_name} table"
|
131
|
+
|
132
|
+
@table.delete
|
133
|
+
|
134
|
+
STDOUT.puts "INFO: #{@table_name} table deleted"
|
135
|
+
end
|
136
|
+
|
137
|
+
def wait_for_deletion_to_complete
|
138
|
+
waiting_seconds = 0
|
139
|
+
while get_table_status == "DELETING"
|
140
|
+
STDOUT.puts "INFO: Waiting for deletion to complete"
|
141
|
+
sleep waiting_seconds += 1
|
142
|
+
end
|
143
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
144
|
+
true
|
145
|
+
end
|
146
|
+
|
147
|
+
def get_table_status
|
148
|
+
@dynamodb.describe_table(table_name: @table_name).table.table_status
|
149
|
+
end
|
150
|
+
|
151
|
+
def create_table
|
152
|
+
STDOUT.puts "INFO: Creating the #{@table_name} table"
|
153
|
+
|
154
|
+
@dynamodb.create_table(@schema)
|
155
|
+
|
156
|
+
STDOUT.puts "INFO: #{@table_name} table created"
|
157
|
+
end
|
158
|
+
|
159
|
+
def check_if_user_wants_to_continue_with_deployment
|
160
|
+
STDOUT.print(
|
161
|
+
"WARNING: You are going to deploy and redeploy your #{@stack_name} stack\n" \
|
162
|
+
"to drop and recreate the #{@table_name} table, do you really want to continue?\n" \
|
163
|
+
"(anything other than 'y' will cancel) > "
|
164
|
+
)
|
165
|
+
|
166
|
+
confirmation = STDIN.gets.strip
|
167
|
+
return if confirmation == "y"
|
168
|
+
|
169
|
+
STDOUT.puts abort_message
|
170
|
+
exit(0)
|
171
|
+
end
|
172
|
+
|
173
|
+
def erase_table_through_cloudformation
|
174
|
+
deploy_stack_without_the_table
|
175
|
+
wait_for_deployment_to_complete
|
176
|
+
deploy_stack_with_the_original_template
|
177
|
+
end
|
178
|
+
|
179
|
+
def deploy_stack_without_the_table
|
180
|
+
STDOUT.puts "INFO: Deploying the stack without the #{@table_name} table"
|
181
|
+
|
182
|
+
@cloudformation.update_stack(
|
183
|
+
stack_name: @stack_name,
|
184
|
+
template_body: @template_without_table.to_json,
|
185
|
+
parameters: @stack.parameters.map(&:to_h),
|
186
|
+
capabilities: @stack.capabilities,
|
187
|
+
role_arn: @stack.role_arn,
|
188
|
+
rollback_configuration: @stack.rollback_configuration.to_h,
|
189
|
+
stack_policy_body: get_stack_policy_body,
|
190
|
+
notification_arns: @stack.notification_arns,
|
191
|
+
tags: @stack.tags.map(&:to_h)
|
192
|
+
)
|
193
|
+
|
194
|
+
STDOUT.puts "INFO: Stack deployed without the #{@table_name} table"
|
195
|
+
end
|
196
|
+
|
197
|
+
def get_stack_policy_body
|
198
|
+
@cloudformation.get_stack_policy(stack_name: @stack_name).stack_policy_body
|
199
|
+
end
|
200
|
+
|
201
|
+
def wait_for_deployment_to_complete
|
202
|
+
waiting_seconds = 0
|
203
|
+
while get_stack_status != "UPDATE_COMPLETE"
|
204
|
+
STDOUT.puts "INFO: Waiting for deployment to complete"
|
205
|
+
sleep waiting_seconds += 1
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def get_stack_status
|
210
|
+
@cloudformation.describe_stacks(stack_name: @stack_name)[0][0].stack_status
|
211
|
+
end
|
212
|
+
|
213
|
+
def deploy_stack_with_the_original_template
|
214
|
+
STDOUT.puts "INFO: Deploying the stack with the #{@table_name} table"
|
215
|
+
|
216
|
+
@cloudformation.update_stack(
|
217
|
+
stack_name: @stack_name,
|
218
|
+
template_body: @original_template.to_json,
|
219
|
+
parameters: @stack.parameters.map(&:to_h),
|
220
|
+
capabilities: @stack.capabilities,
|
221
|
+
role_arn: @stack.role_arn,
|
222
|
+
rollback_configuration: @stack.rollback_configuration.to_h,
|
223
|
+
stack_policy_body: get_stack_policy_body,
|
224
|
+
notification_arns: @stack.notification_arns,
|
225
|
+
tags: @stack.tags.map(&:to_h)
|
226
|
+
)
|
227
|
+
|
228
|
+
STDOUT.puts "INFO: Stack deployed with the #{@table_name} table"
|
229
|
+
end
|
230
|
+
end
|
data/lib/dynamocli/import.rb
CHANGED
data/lib/dynamocli/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynamocli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matheus Silva Santos de Oliveira
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-06-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.28'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: aws-sdk-cloudformation
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bundler
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -93,6 +107,7 @@ files:
|
|
93
107
|
- ".gitignore"
|
94
108
|
- ".rspec"
|
95
109
|
- ".travis.yml"
|
110
|
+
- CHANGELOG.md
|
96
111
|
- CODE_OF_CONDUCT.md
|
97
112
|
- Gemfile
|
98
113
|
- Gemfile.lock
|
@@ -102,8 +117,9 @@ files:
|
|
102
117
|
- bin/console
|
103
118
|
- bin/dynamocli
|
104
119
|
- bin/setup
|
105
|
-
-
|
120
|
+
- dynamocli.gemspec
|
106
121
|
- lib/dynamocli.rb
|
122
|
+
- lib/dynamocli/erase.rb
|
107
123
|
- lib/dynamocli/import.rb
|
108
124
|
- lib/dynamocli/version.rb
|
109
125
|
homepage: https://github.com/matheussilvasantos/dynamocli
|