seedster 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43fbeb4071f54875cbbc8228d0c36b663a1dc656
-  data.tar.gz: ed258b43a623c4fb04724c91ab9c3ebbe14d6eb3
+  metadata.gz: 4836e8e3dd8e804c6e96d74f65c6c1d818f6962a
+  data.tar.gz: f260085eb875b10d583a573baead0710890b9e62
 SHA512:
-  metadata.gz: 4f55fc4569508eaeffcd84c3d7272846eaefc46467ff37476a1397f1521d7e59902cff42f9324daa4291a582ba8cf0f8e52dc0f126aea3e855c05cd0997cbcda
-  data.tar.gz: f84db4a58dca3c716b77ffa491779d89097382d1dd2c738b102368f6f29139966694fa4e105ec53279eb26a497de9af54be5bb9be8d7c0fce69dd82c10409de8
+  metadata.gz: 9ff5886df60737e366312a3248c8dba305c077a516dc3de230d19793c6bc079716dd7cdbfc0a9584e764f60204a9e4289b7114406b8a8f5f75d18cb654a72a97
+  data.tar.gz: 28fa8e49f4486531c4f7e15c7cd48e8284e41c4c31b45fd93d628640c53f0a5af239608c93c005c64400a732c373af42822e91d5d2b7b04af65e4b8d204cb0f7

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.1.6] - 2019-08-02
+
+### Breaking Changes
+  - Changed initializer, recommended to regenerate it. Database options
+    names are changed to more closely match Rails.
+
+### Added
+  - Feature: `skip_download` option. This option is false by default. Changing
+    it to true means that the data file will be available and loaded locally.
+    Refer to the example app [^1] for a working example of this.
+
+    [^1]: https://github.com/andyatkinson/rails-with-seedster
+
+
 ## [0.1.5] - 2019-07-16
 
 ### Added

data/README.md

@@ -81,22 +81,33 @@ The initializer looks like this:
 
 ```ruby
 Seedster.configure do |c|
-  c.db_host = 'XXX'
-  c.db_name = 'XXX'
-  c.db_username = 'XXX'
-  c.db_password = 'XXX'
-  c.remote_host_path = '/var/company/www/app/current'
-  c.query_params = {user_id: XXX}
-  c.ssh_user = 'XXX'
-  c.dump_host = 'XXX'
+  c.db_host = 'host.com' # DB host. Fill this in, or reference Rails config/development.yml database values or ar-octopus config
+  c.db_name = 'db_nameXXX'
+  c.db_username = 'db_usernameXXX'
+  c.db_password = 'passwordXXX'
+  c.remote_host_path = "/var/company/www/app/current" # where the root of the app is deployed on the host
+  c.query_params = { } # pass in values to interpolate into queries,
+                       # e.g. USER_ID=XXX would be {user_id: ENV['USER_ID']}
+
+  c.ssh_user = 'ssh_userXXX' # which user will connect to the host
+  c.dump_host = 'app.host.com' # host where app is deployed
+
+  #
+  # Help:
+  # Comma-separated list of hashes, with keys `query` and `name`, one hash per DB table
+  #
+  # Keys:
+  # query: the SQL query for the table to be dumped. Add a parameter like `user_id` to be passed in.
+  # name: the name of the database table
+  #
   c.tables = [
-    {
-      query: %{SELECT * FROM users WHERE id = '%{user_id}'},
-      name: 'users'
-    },
-    {
-      /* more tables here */
-    }
+    # BEGIN example ---
+    # {
+    #   query: %{SELECT u.* FROM users
+    #       where u.id = '%{user_id}'},
+    #   name: 'users'
+    # }
+    # END example -----
   ]
 end
 ```

data/lib/generators/seedster/initializer_generator.rb

@@ -1,3 +1,18 @@
+#
+# Copyright 2019 Groupon, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 # https://guides.rubyonrails.org/generators.html
 module Seedster
   module Generators

data/lib/generators/seedster/templates/initializer.rb

@@ -1,33 +1,74 @@
+#
+# EXAMPLE APP:
+#
+# https://github.com/andyatkinson/rails-with-seedster
+#
 Seedster.configure do |c|
-  c.db_host = 'host.com' # DB host. Fill this in, or reference Rails config/development.yml database values or ar-octopus config
-  c.db_name = 'db_nameXXX'
-  c.db_username = 'db_usernameXXX'
-  c.db_password = 'passwordXXX'
 
-  c.remote_host_path = "/var/company/www/app/current" # where the root of the app is deployed on the host
+  # NOTE: Configured to dump and load from localhost by default,
+  # however more typically, the dump host would be Production, and
+  # the load host would be the Development environment
+  #
+  #
+
+  # DATABASE configuration
+  config   = Rails.configuration.database_configuration
+  environment = "development"
 
-  c.query_params = { } # pass in values to interpolate into queries,
-                       # e.g. USER_ID=XXX would be {user_id: ENV['USER_ID']}
+  c.dump_host     = config[environment]["host"]
+  c.dump_database = config[environment]["database"]
+  c.dump_username = config[environment]["username"]
+  c.dump_password = config[environment]["password"]
 
-  c.ssh_user = 'ssh_userXXX' # which user will connect to the host
+  c.load_host     = config[environment]["host"]
+  c.load_database = config[environment]["database"]
+  c.load_username = config[environment]["username"]
+  c.load_password = config[environment]["password"]
 
-  c.dump_host = 'app.host.com' # host where app is deployed
+  # pass in values to substitute into queries,
+  # e.g. Run `rake seedster:dump USER_ID=1` to dump
+  # all data for User 1
+  c.query_params = { user_id: ENV['USER_ID'] }
+
+  # SEEDSTER options
+  #
+  # Use this option if you are provided a dump file,
+  # and expect to load data from the local dump file
+  #
+  # A dump file named this, will be extracted and loaded.
+  # `tmp/data_dumps/seedster-dump-latest.tar.gz`
+  #
+  c.skip_download = false
+
+  # REMOTE data dump download options
+  # NOTE: These options are ignored if `skip_download` is true
+  #
+  c.ssh_host = 'prod.host.com'
+  c.ssh_user = 'ssh_user' # which user will connect to the host
+  c.remote_host_path = "/var/company/www/app/current" # where the root of the app is deployed on the host
 
   #
   # Help:
-  # Comma-separated list of hashes, with keys `query` and `name`, one hash per DB table
+  # Array of hashes with keys `query` and `name`, one per table
   #
   # Keys:
   # query: the SQL query for the table to be dumped. Add a parameter like `user_id` to be passed in.
   # name: the name of the database table
   #
   c.tables = [
-    # BEGIN example ---
+
     # {
-    #   query: %{SELECT u.* FROM users
-    #       where u.id = '%{user_id}'},
+    #   query: %{SELECT * FROM users WHERE id = '%{user_id}'},
     #   name: 'users'
+    # },
+    # {
+    #   query: %{SELECT p.* FROM posts p JOIN users u ON p.user_id = u.id WHERE u.id = '%{user_id}'},
+    #   name: 'posts'
+    # },
+    # {
+    #   query: %{SELECT c.* FROM comments c JOIN users u ON c.user_id = u.id WHERE u.id = '%{user_id}'},
+    #   name: 'comments'
     # }
-    # END example -----
+
   ]
 end

data/lib/seedster.rb

@@ -30,24 +30,32 @@ module Seedster
   end
 
   class Configuration
-    attr_accessor :db_host, :db_name,
-      :db_username, :db_password,
+    attr_accessor :dump_host, :dump_database,
+      :dump_username, :dump_password,
+      :load_host, :load_database,
+      :load_username, :load_password,
       :remote_host_path,
       :query_params,
       :ssh_user,
-      :dump_host,
-      :tables
+      :ssh_host,
+      :tables,
+      :skip_download
 
     def initialize
-      @db_host = nil
-      @db_name = nil
-      @db_username = nil
-      @db_password = nil
+      @dump_host = nil
+      @dump_database = nil
+      @dump_username = nil
+      @dump_password = nil
+      @load_host = nil
+      @load_database = nil
+      @load_username = nil
+      @load_password = nil
       @remote_host_path = nil
       @query_params = {}
       @ssh_user = nil
-      @dump_host = nil
+      @ssh_host= nil
       @tables = []
+      @skip_download = false
     end
   end
 end

data/lib/seedster/data_dumper.rb

@@ -15,14 +15,14 @@
 #
 module Seedster
   class DataDumper
-    attr_accessor :db_password
-    attr_reader :db_host, :db_username, :db_name
+    attr_reader :dump_password, :dump_host, :dump_username, :dump_database
 
-    def initialize(db_password:, db_host:, db_username:, db_name:)
-      @db_password = db_password
-      @db_host = db_host
-      @db_username = db_username
-      @db_name = db_name
+    def initialize(dump_password:, dump_host:, dump_username:, dump_database:)
+      @dump_password  = dump_password
+      @dump_host      = dump_host
+      @dump_username  = dump_username
+      @dump_database  = dump_database
+      FileManager.create_dump_dir
       FileManager.create_seed_file_dir
     end
 
@@ -52,13 +52,14 @@ module Seedster
       dump_file = Rails.root.join(FileManager.dump_dir, FileManager.dump_file_name)
       puts "Creating dump file '#{dump_file}' from '#{FileManager.seed_file_dir}'"
       tar_command = "tar -zcvf #{dump_file} -C #{FileManager.seed_file_dir} ." # use relative paths
-      system(tar_command) # use relative paths
+      puts "Running tar command: '#{tar_command}'"
+      system(tar_command)
     end
 
     def sql_results_to_file(sql:, table_name:)
       filename = FileManager.get_filename(table_name: table_name)
-      puts "dumping '#{table_name}' to '#{filename}' with query '#{sql}'"
-      psql_cmd = %{PGPASSWORD=#{db_password} psql -h #{db_host} -d #{db_name} -U #{db_username} -c "\\copy (#{sql}) TO '#{filename}' WITH CSV"}
+      puts "Dumping table '#{table_name}' to file '#{filename}' with query '#{sql}'"
+      psql_cmd = %{PGPASSWORD=#{dump_password} psql -h #{dump_host} -d #{dump_database} -U #{dump_username} -c "\\copy (#{sql}) TO '#{filename}' WITH CSV"}
       system(psql_cmd)
     end
   end

data/lib/seedster/data_loader.rb

@@ -15,21 +15,23 @@
 #
 module Seedster
   class DataLoader
-    attr_reader :ssh_user, :ssh_host, :local_db_name,
-      :remote_host_path
+    attr_reader :ssh_user, :ssh_host, :remote_host_path,
+      :load_host, :load_database, :load_username, :load_password
 
-    def initialize(ssh_user:, ssh_host:,
-                   local_db_name:, remote_host_path:)
+    def initialize(ssh_user:, ssh_host:, remote_host_path:,
+                   load_host:, load_database:, load_username:, load_password:)
       @ssh_user = ssh_user
       @ssh_host = ssh_host
-      @local_db_name = local_db_name
       @remote_host_path = remote_host_path
+      @load_host = load_host
+      @load_database = load_database
+      @load_username = load_username
+      @load_password = load_password
       print_greeting
-      FileManager.create_seed_file_dir
     end
 
     def load!
-      download_and_extract_file
+      download_and_extract_file unless Seedster.configuration.skip_download
 
       Seedster.configuration.tables.each do |item|
         load_data(table_name: item[:name])
@@ -42,29 +44,38 @@ module Seedster
       remote_host_path = Seedster.configuration.remote_host_path
       remote_file = "#{remote_host_path}/#{File.join(FileManager.dump_dir, FileManager.dump_file_name)}"
       scp_command = "scp -r #{ssh_user}@#{ssh_host}:#{remote_file} ."
-      puts "Downloading file: #{scp_command}"
+      puts "Downloading file: '#{scp_command}'"
       system(scp_command)
 
       untar_command = "tar -zxvf #{FileManager.dump_file_name} -C #{FileManager.seed_file_dir}"
-      puts "Extracting local file: #{untar_command}"
+      puts "Extracting file: '#{untar_command}'"
       system(untar_command)
     end
 
     def load_data(table_name:)
       filename = FileManager.get_filename(table_name: table_name)
+      psql_load(filename: filename, table_name: table_name)
+    end
+
+    # TODO: this could be swapped out for MySQL equivalent by a motivated individual :)
+    def psql_load(filename:, table_name:)
       load_command = "COPY #{table_name} FROM '#{filename}' DELIMITERS ',' CSV"
-      puts "loading '#{table_name}' from '#{filename}'"
-      psql_cmd = %{psql -d '#{local_db_name}' -c "#{load_command}"}
+      puts "Loading '#{table_name}' from '#{filename}'"
+      psql_cmd = %{PG_PASSWORD=#{load_password} psql --host #{load_host} --dbname #{load_database} --username #{load_username} -c "#{load_command}"}
       system(psql_cmd)
     end
 
     def print_greeting
       puts
       puts "🌱🌱🌱🌱🌱"
-      puts "Hello from Seedster!"
+      puts "Hello from Seedster. A note:"
+      puts
+      puts "`rake seedster:load` expects a compatible schema version and empty tables."
+      puts "Run `rake db:reset` prior to running `rake seedster:load`"
       puts
-      puts "Loading data requires a compatible schema version, and empty tables."
-      puts "Before loading data locally, truncate the relevant tables (or you may see constraint errors)."
+      puts "Before loading data locally, truncate the relevant tables (or you may see constraint violation errors)."
+      puts "Bugs and issues: https://github.com/groupon/seedster"
+      puts "Thanks!"
       puts "🌱🌱🌱🌱🌱"
       puts
     end

data/lib/seedster/file_manager.rb

@@ -24,10 +24,9 @@ module Seedster
       def dump_dir
         # Capistrano note:
         # set up as a linked_dir in config/deploy.rb
-        File.join(TMP_DIR, DATA_DUMPS_DIR)
+        Rails.root.join(TMP_DIR, DATA_DUMPS_DIR)
       end
 
-      # TODO: mock Rails.root here for test
       def seed_file_dir
         Rails.root.join(TMP_DIR, SEED_FILE_DIR)
       end
@@ -40,8 +39,13 @@ module Seedster
         "#{SEEDSTER}-dump-latest.tar.gz"
       end
 
+      def create_dump_dir
+        puts "Creating directory: '#{FileManager.dump_dir}'"
+        FileUtils.mkdir_p(dump_dir)
+      end
+
       def create_seed_file_dir
-        puts "Setting up '#{FileManager.seed_file_dir}' directory"
+        puts "Creating directory: '#{FileManager.seed_file_dir}'"
         FileUtils.mkdir_p(seed_file_dir)
       end
     end

data/lib/seedster/tasks/seedster.rb

@@ -21,22 +21,27 @@ namespace :seedster do
     puts "Seedster loading..."
 
     Seedster::DataDumper.new(
-      db_host: Seedster.configuration.db_host,
-      db_name: Seedster.configuration.db_name,
-      db_username: Seedster.configuration.db_username,
-      db_password: Seedster.configuration.db_password
+      dump_host: Seedster.configuration.dump_host,
+      dump_database: Seedster.configuration.dump_database,
+      dump_username: Seedster.configuration.dump_username,
+      dump_password: Seedster.configuration.dump_password
     ).dump!
   end
 
   desc "Load application seed data from a remote dump file"
   task load: :environment do
-    return unless Rails.env.development? # only load in dev environment
+    if !Rails.env.development?
+      puts "Exiting. Seedster Data Load is intended for development only."
+      return
+    end
 
-    config = Rails.configuration.database_configuration
     Seedster::DataLoader.new(
-      local_db_name: config['development']['database'],
+      load_host: Seedster.configuration.load_host,
+      load_database: Seedster.configuration.load_database,
+      load_username: Seedster.configuration.load_username,
+      load_password: Seedster.configuration.load_password,
       ssh_user: Seedster.configuration.ssh_user,
-      ssh_host: Seedster.configuration.dump_host,
+      ssh_host: Seedster.configuration.ssh_host,
       remote_host_path: Seedster.configuration.remote_host_path
     ).load!
   end

data/lib/seedster/version.rb

@@ -14,5 +14,5 @@
 # limitations under the License.
 #
 module Seedster
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seedster
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Groupon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-16 00:00:00.000000000 Z
+date: 2019-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler