seedster 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/README.md +26 -15
- data/lib/generators/seedster/initializer_generator.rb +15 -0
- data/lib/generators/seedster/templates/initializer.rb +55 -14
- data/lib/seedster.rb +17 -9
- data/lib/seedster/data_dumper.rb +11 -10
- data/lib/seedster/data_loader.rb +25 -14
- data/lib/seedster/file_manager.rb +7 -3
- data/lib/seedster/tasks/seedster.rb +13 -8
- data/lib/seedster/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4836e8e3dd8e804c6e96d74f65c6c1d818f6962a
|
4
|
+
data.tar.gz: f260085eb875b10d583a573baead0710890b9e62
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ff5886df60737e366312a3248c8dba305c077a516dc3de230d19793c6bc079716dd7cdbfc0a9584e764f60204a9e4289b7114406b8a8f5f75d18cb654a72a97
|
7
|
+
data.tar.gz: 28fa8e49f4486531c4f7e15c7cd48e8284e41c4c31b45fd93d628640c53f0a5af239608c93c005c64400a732c373af42822e91d5d2b7b04af65e4b8d204cb0f7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## [0.1.6] - 2019-08-02
|
2
|
+
|
3
|
+
### Breaking Changes
|
4
|
+
- Changed initializer, recommended to regenerate it. Database options
|
5
|
+
names are changed to more closely match Rails.
|
6
|
+
|
7
|
+
### Added
|
8
|
+
- Feature: `skip_download` option. This option is false by default. Changing
|
9
|
+
it to true means that the data file will be available and loaded locally.
|
10
|
+
Refer to the example app [^1] for a working example of this.
|
11
|
+
|
12
|
+
[^1]: https://github.com/andyatkinson/rails-with-seedster
|
13
|
+
|
14
|
+
|
1
15
|
## [0.1.5] - 2019-07-16
|
2
16
|
|
3
17
|
### Added
|
data/README.md
CHANGED
@@ -81,22 +81,33 @@ The initializer looks like this:
|
|
81
81
|
|
82
82
|
```ruby
|
83
83
|
Seedster.configure do |c|
|
84
|
-
c.db_host = '
|
85
|
-
c.db_name = '
|
86
|
-
c.db_username = '
|
87
|
-
c.db_password = '
|
88
|
-
c.remote_host_path =
|
89
|
-
c.query_params = {
|
90
|
-
|
91
|
-
|
84
|
+
c.db_host = 'host.com' # DB host. Fill this in, or reference Rails config/development.yml database values or ar-octopus config
|
85
|
+
c.db_name = 'db_nameXXX'
|
86
|
+
c.db_username = 'db_usernameXXX'
|
87
|
+
c.db_password = 'passwordXXX'
|
88
|
+
c.remote_host_path = "/var/company/www/app/current" # where the root of the app is deployed on the host
|
89
|
+
c.query_params = { } # pass in values to interpolate into queries,
|
90
|
+
# e.g. USER_ID=XXX would be {user_id: ENV['USER_ID']}
|
91
|
+
|
92
|
+
c.ssh_user = 'ssh_userXXX' # which user will connect to the host
|
93
|
+
c.dump_host = 'app.host.com' # host where app is deployed
|
94
|
+
|
95
|
+
#
|
96
|
+
# Help:
|
97
|
+
# Comma-separated list of hashes, with keys `query` and `name`, one hash per DB table
|
98
|
+
#
|
99
|
+
# Keys:
|
100
|
+
# query: the SQL query for the table to be dumped. Add a parameter like `user_id` to be passed in.
|
101
|
+
# name: the name of the database table
|
102
|
+
#
|
92
103
|
c.tables = [
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
},
|
97
|
-
|
98
|
-
|
99
|
-
|
104
|
+
# BEGIN example ---
|
105
|
+
# {
|
106
|
+
# query: %{SELECT u.* FROM users
|
107
|
+
# where u.id = '%{user_id}'},
|
108
|
+
# name: 'users'
|
109
|
+
# }
|
110
|
+
# END example -----
|
100
111
|
]
|
101
112
|
end
|
102
113
|
```
|
@@ -1,3 +1,18 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2019 Groupon, Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
1
16
|
# https://guides.rubyonrails.org/generators.html
|
2
17
|
module Seedster
|
3
18
|
module Generators
|
@@ -1,33 +1,74 @@
|
|
1
|
+
#
|
2
|
+
# EXAMPLE APP:
|
3
|
+
#
|
4
|
+
# https://github.com/andyatkinson/rails-with-seedster
|
5
|
+
#
|
1
6
|
Seedster.configure do |c|
|
2
|
-
c.db_host = 'host.com' # DB host. Fill this in, or reference Rails config/development.yml database values or ar-octopus config
|
3
|
-
c.db_name = 'db_nameXXX'
|
4
|
-
c.db_username = 'db_usernameXXX'
|
5
|
-
c.db_password = 'passwordXXX'
|
6
7
|
|
7
|
-
|
8
|
+
# NOTE: Configured to dump and load from localhost by default,
|
9
|
+
# however more typically, the dump host would be Production, and
|
10
|
+
# the load host would be the Development environment
|
11
|
+
#
|
12
|
+
#
|
13
|
+
|
14
|
+
# DATABASE configuration
|
15
|
+
config = Rails.configuration.database_configuration
|
16
|
+
environment = "development"
|
8
17
|
|
9
|
-
c.
|
10
|
-
|
18
|
+
c.dump_host = config[environment]["host"]
|
19
|
+
c.dump_database = config[environment]["database"]
|
20
|
+
c.dump_username = config[environment]["username"]
|
21
|
+
c.dump_password = config[environment]["password"]
|
11
22
|
|
12
|
-
c.
|
23
|
+
c.load_host = config[environment]["host"]
|
24
|
+
c.load_database = config[environment]["database"]
|
25
|
+
c.load_username = config[environment]["username"]
|
26
|
+
c.load_password = config[environment]["password"]
|
13
27
|
|
14
|
-
|
28
|
+
# pass in values to substitute into queries,
|
29
|
+
# e.g. Run `rake seedster:dump USER_ID=1` to dump
|
30
|
+
# all data for User 1
|
31
|
+
c.query_params = { user_id: ENV['USER_ID'] }
|
32
|
+
|
33
|
+
# SEEDSTER options
|
34
|
+
#
|
35
|
+
# Use this option if you are provided a dump file,
|
36
|
+
# and expect to load data from the local dump file
|
37
|
+
#
|
38
|
+
# A dump file named this, will be extracted and loaded.
|
39
|
+
# `tmp/data_dumps/seedster-dump-latest.tar.gz`
|
40
|
+
#
|
41
|
+
c.skip_download = false
|
42
|
+
|
43
|
+
# REMOTE data dump download options
|
44
|
+
# NOTE: These options are ignored if `skip_download` is true
|
45
|
+
#
|
46
|
+
c.ssh_host = 'prod.host.com'
|
47
|
+
c.ssh_user = 'ssh_user' # which user will connect to the host
|
48
|
+
c.remote_host_path = "/var/company/www/app/current" # where the root of the app is deployed on the host
|
15
49
|
|
16
50
|
#
|
17
51
|
# Help:
|
18
|
-
#
|
52
|
+
# Array of hashes with keys `query` and `name`, one per table
|
19
53
|
#
|
20
54
|
# Keys:
|
21
55
|
# query: the SQL query for the table to be dumped. Add a parameter like `user_id` to be passed in.
|
22
56
|
# name: the name of the database table
|
23
57
|
#
|
24
58
|
c.tables = [
|
25
|
-
|
59
|
+
|
26
60
|
# {
|
27
|
-
# query: %{SELECT
|
28
|
-
# where u.id = '%{user_id}'},
|
61
|
+
# query: %{SELECT * FROM users WHERE id = '%{user_id}'},
|
29
62
|
# name: 'users'
|
63
|
+
# },
|
64
|
+
# {
|
65
|
+
# query: %{SELECT p.* FROM posts p JOIN users u ON p.user_id = u.id WHERE u.id = '%{user_id}'},
|
66
|
+
# name: 'posts'
|
67
|
+
# },
|
68
|
+
# {
|
69
|
+
# query: %{SELECT c.* FROM comments c JOIN users u ON c.user_id = u.id WHERE u.id = '%{user_id}'},
|
70
|
+
# name: 'comments'
|
30
71
|
# }
|
31
|
-
|
72
|
+
|
32
73
|
]
|
33
74
|
end
|
data/lib/seedster.rb
CHANGED
@@ -30,24 +30,32 @@ module Seedster
|
|
30
30
|
end
|
31
31
|
|
32
32
|
class Configuration
|
33
|
-
attr_accessor :
|
34
|
-
:
|
33
|
+
attr_accessor :dump_host, :dump_database,
|
34
|
+
:dump_username, :dump_password,
|
35
|
+
:load_host, :load_database,
|
36
|
+
:load_username, :load_password,
|
35
37
|
:remote_host_path,
|
36
38
|
:query_params,
|
37
39
|
:ssh_user,
|
38
|
-
:
|
39
|
-
:tables
|
40
|
+
:ssh_host,
|
41
|
+
:tables,
|
42
|
+
:skip_download
|
40
43
|
|
41
44
|
def initialize
|
42
|
-
@
|
43
|
-
@
|
44
|
-
@
|
45
|
-
@
|
45
|
+
@dump_host = nil
|
46
|
+
@dump_database = nil
|
47
|
+
@dump_username = nil
|
48
|
+
@dump_password = nil
|
49
|
+
@load_host = nil
|
50
|
+
@load_database = nil
|
51
|
+
@load_username = nil
|
52
|
+
@load_password = nil
|
46
53
|
@remote_host_path = nil
|
47
54
|
@query_params = {}
|
48
55
|
@ssh_user = nil
|
49
|
-
@
|
56
|
+
@ssh_host= nil
|
50
57
|
@tables = []
|
58
|
+
@skip_download = false
|
51
59
|
end
|
52
60
|
end
|
53
61
|
end
|
data/lib/seedster/data_dumper.rb
CHANGED
@@ -15,14 +15,14 @@
|
|
15
15
|
#
|
16
16
|
module Seedster
|
17
17
|
class DataDumper
|
18
|
-
|
19
|
-
attr_reader :db_host, :db_username, :db_name
|
18
|
+
attr_reader :dump_password, :dump_host, :dump_username, :dump_database
|
20
19
|
|
21
|
-
def initialize(
|
22
|
-
@
|
23
|
-
@
|
24
|
-
@
|
25
|
-
@
|
20
|
+
def initialize(dump_password:, dump_host:, dump_username:, dump_database:)
|
21
|
+
@dump_password = dump_password
|
22
|
+
@dump_host = dump_host
|
23
|
+
@dump_username = dump_username
|
24
|
+
@dump_database = dump_database
|
25
|
+
FileManager.create_dump_dir
|
26
26
|
FileManager.create_seed_file_dir
|
27
27
|
end
|
28
28
|
|
@@ -52,13 +52,14 @@ module Seedster
|
|
52
52
|
dump_file = Rails.root.join(FileManager.dump_dir, FileManager.dump_file_name)
|
53
53
|
puts "Creating dump file '#{dump_file}' from '#{FileManager.seed_file_dir}'"
|
54
54
|
tar_command = "tar -zcvf #{dump_file} -C #{FileManager.seed_file_dir} ." # use relative paths
|
55
|
-
|
55
|
+
puts "Running tar command: '#{tar_command}'"
|
56
|
+
system(tar_command)
|
56
57
|
end
|
57
58
|
|
58
59
|
def sql_results_to_file(sql:, table_name:)
|
59
60
|
filename = FileManager.get_filename(table_name: table_name)
|
60
|
-
puts "
|
61
|
-
psql_cmd = %{PGPASSWORD=#{
|
61
|
+
puts "Dumping table '#{table_name}' to file '#{filename}' with query '#{sql}'"
|
62
|
+
psql_cmd = %{PGPASSWORD=#{dump_password} psql -h #{dump_host} -d #{dump_database} -U #{dump_username} -c "\\copy (#{sql}) TO '#{filename}' WITH CSV"}
|
62
63
|
system(psql_cmd)
|
63
64
|
end
|
64
65
|
end
|
data/lib/seedster/data_loader.rb
CHANGED
@@ -15,21 +15,23 @@
|
|
15
15
|
#
|
16
16
|
module Seedster
|
17
17
|
class DataLoader
|
18
|
-
attr_reader :ssh_user, :ssh_host, :
|
19
|
-
:
|
18
|
+
attr_reader :ssh_user, :ssh_host, :remote_host_path,
|
19
|
+
:load_host, :load_database, :load_username, :load_password
|
20
20
|
|
21
|
-
def initialize(ssh_user:, ssh_host:,
|
22
|
-
|
21
|
+
def initialize(ssh_user:, ssh_host:, remote_host_path:,
|
22
|
+
load_host:, load_database:, load_username:, load_password:)
|
23
23
|
@ssh_user = ssh_user
|
24
24
|
@ssh_host = ssh_host
|
25
|
-
@local_db_name = local_db_name
|
26
25
|
@remote_host_path = remote_host_path
|
26
|
+
@load_host = load_host
|
27
|
+
@load_database = load_database
|
28
|
+
@load_username = load_username
|
29
|
+
@load_password = load_password
|
27
30
|
print_greeting
|
28
|
-
FileManager.create_seed_file_dir
|
29
31
|
end
|
30
32
|
|
31
33
|
def load!
|
32
|
-
download_and_extract_file
|
34
|
+
download_and_extract_file unless Seedster.configuration.skip_download
|
33
35
|
|
34
36
|
Seedster.configuration.tables.each do |item|
|
35
37
|
load_data(table_name: item[:name])
|
@@ -42,29 +44,38 @@ module Seedster
|
|
42
44
|
remote_host_path = Seedster.configuration.remote_host_path
|
43
45
|
remote_file = "#{remote_host_path}/#{File.join(FileManager.dump_dir, FileManager.dump_file_name)}"
|
44
46
|
scp_command = "scp -r #{ssh_user}@#{ssh_host}:#{remote_file} ."
|
45
|
-
puts "Downloading file: #{scp_command}"
|
47
|
+
puts "Downloading file: '#{scp_command}'"
|
46
48
|
system(scp_command)
|
47
49
|
|
48
50
|
untar_command = "tar -zxvf #{FileManager.dump_file_name} -C #{FileManager.seed_file_dir}"
|
49
|
-
puts "Extracting
|
51
|
+
puts "Extracting file: '#{untar_command}'"
|
50
52
|
system(untar_command)
|
51
53
|
end
|
52
54
|
|
53
55
|
def load_data(table_name:)
|
54
56
|
filename = FileManager.get_filename(table_name: table_name)
|
57
|
+
psql_load(filename: filename, table_name: table_name)
|
58
|
+
end
|
59
|
+
|
60
|
+
# TODO: this could be swapped out for MySQL equivalent by a motivated individual :)
|
61
|
+
def psql_load(filename:, table_name:)
|
55
62
|
load_command = "COPY #{table_name} FROM '#{filename}' DELIMITERS ',' CSV"
|
56
|
-
puts "
|
57
|
-
psql_cmd = %{psql
|
63
|
+
puts "Loading '#{table_name}' from '#{filename}'"
|
64
|
+
psql_cmd = %{PG_PASSWORD=#{load_password} psql --host #{load_host} --dbname #{load_database} --username #{load_username} -c "#{load_command}"}
|
58
65
|
system(psql_cmd)
|
59
66
|
end
|
60
67
|
|
61
68
|
def print_greeting
|
62
69
|
puts
|
63
70
|
puts "🌱🌱🌱🌱🌱"
|
64
|
-
puts "Hello from Seedster
|
71
|
+
puts "Hello from Seedster. A note:"
|
72
|
+
puts
|
73
|
+
puts "`rake seedster:load` expects a compatible schema version and empty tables."
|
74
|
+
puts "Run `rake db:reset` prior to running `rake seedster:load`"
|
65
75
|
puts
|
66
|
-
puts "
|
67
|
-
puts "
|
76
|
+
puts "Before loading data locally, truncate the relevant tables (or you may see constraint violation errors)."
|
77
|
+
puts "Bugs and issues: https://github.com/groupon/seedster"
|
78
|
+
puts "Thanks!"
|
68
79
|
puts "🌱🌱🌱🌱🌱"
|
69
80
|
puts
|
70
81
|
end
|
@@ -24,10 +24,9 @@ module Seedster
|
|
24
24
|
def dump_dir
|
25
25
|
# Capistrano note:
|
26
26
|
# set up as a linked_dir in config/deploy.rb
|
27
|
-
|
27
|
+
Rails.root.join(TMP_DIR, DATA_DUMPS_DIR)
|
28
28
|
end
|
29
29
|
|
30
|
-
# TODO: mock Rails.root here for test
|
31
30
|
def seed_file_dir
|
32
31
|
Rails.root.join(TMP_DIR, SEED_FILE_DIR)
|
33
32
|
end
|
@@ -40,8 +39,13 @@ module Seedster
|
|
40
39
|
"#{SEEDSTER}-dump-latest.tar.gz"
|
41
40
|
end
|
42
41
|
|
42
|
+
def create_dump_dir
|
43
|
+
puts "Creating directory: '#{FileManager.dump_dir}'"
|
44
|
+
FileUtils.mkdir_p(dump_dir)
|
45
|
+
end
|
46
|
+
|
43
47
|
def create_seed_file_dir
|
44
|
-
puts "
|
48
|
+
puts "Creating directory: '#{FileManager.seed_file_dir}'"
|
45
49
|
FileUtils.mkdir_p(seed_file_dir)
|
46
50
|
end
|
47
51
|
end
|
@@ -21,22 +21,27 @@ namespace :seedster do
|
|
21
21
|
puts "Seedster loading..."
|
22
22
|
|
23
23
|
Seedster::DataDumper.new(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
dump_host: Seedster.configuration.dump_host,
|
25
|
+
dump_database: Seedster.configuration.dump_database,
|
26
|
+
dump_username: Seedster.configuration.dump_username,
|
27
|
+
dump_password: Seedster.configuration.dump_password
|
28
28
|
).dump!
|
29
29
|
end
|
30
30
|
|
31
31
|
desc "Load application seed data from a remote dump file"
|
32
32
|
task load: :environment do
|
33
|
-
|
33
|
+
if !Rails.env.development?
|
34
|
+
puts "Exiting. Seedster Data Load is intended for development only."
|
35
|
+
return
|
36
|
+
end
|
34
37
|
|
35
|
-
config = Rails.configuration.database_configuration
|
36
38
|
Seedster::DataLoader.new(
|
37
|
-
|
39
|
+
load_host: Seedster.configuration.load_host,
|
40
|
+
load_database: Seedster.configuration.load_database,
|
41
|
+
load_username: Seedster.configuration.load_username,
|
42
|
+
load_password: Seedster.configuration.load_password,
|
38
43
|
ssh_user: Seedster.configuration.ssh_user,
|
39
|
-
ssh_host: Seedster.configuration.
|
44
|
+
ssh_host: Seedster.configuration.ssh_host,
|
40
45
|
remote_host_path: Seedster.configuration.remote_host_path
|
41
46
|
).load!
|
42
47
|
end
|
data/lib/seedster/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: seedster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Groupon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-08-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|