data_keeper 0.1.9 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aefdb5a66266a15337e2f65a59595a3017c744c6532d97d8c4377eb64f82d43
-  data.tar.gz: 49e39de5ecc7fbdf4355bf25c6d5300b5027c1413450aee696a51683d930065e
+  metadata.gz: 1ea02e8856f7dbdcc74cd8c82a57893d792a6a2aeb5c6963258d06b8707e9a2b
+  data.tar.gz: a6dbcdc1d8266d8a554ab6b7b1650dad9b810b5f91f8be3da1f157a877e537e9
 SHA512:
-  metadata.gz: d91d76636fd6eadab472f6442cca9484b95130209a8ba2adef6197c600ab31e43b054121991f6e96ef7200cc4a4ba4f23215ccdf5bf8d6a599d5495786832b51
-  data.tar.gz: d13e22eb926add303417d44fe1ac802b69b3839eb8349903fb10059139bd583e0d91f9e4aaa3a0e9f67ad2782d8aec4628290f48e95b92b64c1facfe1ac70a8f
+  metadata.gz: cb567b6fd35e1c68f051ee3b06e712dd9ed17e9895362ac7f025940b74cff0973255b51d95cd1effb91a67a7bd92188e50a30a07bc77e1f354008fbeab9f1d39
+  data.tar.gz: eab10951b2cf77197c6d360d2676726ea01ddfc9c6696647e96806724d7cdaf370e33f046fbc2620ee1753b7ad697200e28d932ec45bb550bbd19a655f9b5f1b

data/README.md

@@ -110,6 +110,29 @@ in your current database. It will give you an error if you try to run this in a
 Note when using raw sql, your statement is expected to return all columns for the configured table, in the default
 order (`select *`). This uses pg's COPY from/to for the full table internally. 
 
+## Docker
+
+If you're using pg under docker, you can configure also DataKeeper with the docker pg settings and then this gem
+will use the binaries under that docker container (`psql`, `pg_restore`, etc.).
+
+Since then the commands to execute are ran from within the docker instance, the port will be different
+as the one the rails app uses, so you'll need to configure what's the pg access from the docker container, use:
+
+```
+DataKeeper.docker_config = {
+  instance_name: "pg_my_app",
+  pg_host: "localhost",
+  pg_port: "5432",
+  pg_user: "myapp",
+  pg_password: "myapp"
+}
+```
+
+If the host, or user and password are the same as the ones used from the rails app, you can ignore them.
+If you configure this `docker_config` hash, then data keeper will try to use docker via `docker exec` to 
+run the pg commands.
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/data_keeper/database_config.rb

@@ -5,13 +5,41 @@ module DataKeeper
     end
 
     def psql_env
-      env = { 'PGUSER' => database_connection_config['username'] }
-      env['PGPASSWORD'] = database_connection_config['password'] if database_connection_config['password']
+      env = { 'PGUSER' => username }
+      env['PGPASSWORD'] = password if password
       env
     end
 
+    def dumper_psql_env
+      env = { 'PGUSER' => server_username }
+      env['PGPASSWORD'] = server_password if server_password
+      env
+    end
+
+    def docker_env_params
+      psql_env.map do |k, v|
+        "-e #{k}=#{v}"
+      end.join(" ")
+    end
+
+    def server_username
+      database_connection_config['username']
+    end
+
+    def server_password
+      database_connection_config['password']
+    end
+
+    def username
+      DataKeeper.docker_config[:pg_user] || database_connection_config['username']
+    end
+
+    def password
+      DataKeeper.docker_config[:pg_password] || database_connection_config['password']
+    end
+
     def host
-      database_connection_config['host'] || '127.0.0.1'
+      DataKeeper.docker_config[:pg_host] || database_connection_config['host'] || '127.0.0.1'
     end
 
     def database
@@ -19,12 +47,22 @@ module DataKeeper
     end
 
     def port
-      database_connection_config['port']
+      DataKeeper.docker_config[:pg_port] || database_connection_config['port'] || '5432'
+    end
+
+    def server_port
+      database_connection_config['port'] || '5432'
     end
 
     def connection_args
       connection_opts = '--host=:host'
-      connection_opts += ' --port=:port' if database_connection_config['port']
+      connection_opts += ' --port=:port' if port
+      connection_opts
+    end
+
+    def dumper_connection_args
+      connection_opts = '--host=:host'
+      connection_opts += ' --port=:port' if server_port
       connection_opts
     end
   end

data/lib/data_keeper/dumper.rb

@@ -24,8 +24,8 @@ module DataKeeper
       Tempfile.create do |file|
         cmd = Terrapin::CommandLine.new(
           'pg_dump',
-          "#{connection_args} -x -Fc :database > :output_path",
-          environment: psql_env
+          "#{dumper_connection_args} -x -Fc :database > :output_path",
+          environment: dumper_psql_env
         )
 
         cmd.run(
@@ -61,8 +61,8 @@ module DataKeeper
         Tempfile.create do |table_file|
           cmd = Terrapin::CommandLine.new(
             'psql',
-            "#{connection_args} -d :database -c :command > #{table_file.path}",
-            environment: psql_env
+            "#{dumper_connection_args} -d :database -c :command > #{table_file.path}",
+            environment: dumper_psql_env
           )
 
           cmd.run(
@@ -89,8 +89,8 @@ module DataKeeper
         table_args = @definition.full_tables_to_export.map { |table| "-t #{table}" }.join(' ')
         cmd = Terrapin::CommandLine.new(
           'pg_dump',
-          "#{connection_args} -x -Fc :database #{table_args} > :output_path",
-          environment: psql_env
+          "#{dumper_connection_args} -x -Fc :database #{table_args} > :output_path",
+          environment: dumper_psql_env
         )
 
         cmd.run(
@@ -116,8 +116,8 @@ module DataKeeper
 
         cmd = Terrapin::CommandLine.new(
           'pg_dump',
-          "#{connection_args} -x --schema-only -Fc :database > :output_path",
-          environment: psql_env
+          "#{dumper_connection_args} -x --schema-only -Fc :database > :output_path",
+          environment: dumper_psql_env
         )
 
         cmd.run(
@@ -144,8 +144,8 @@ module DataKeeper
         sequences_args = all_sequences_to_export.map { |table| "-t #{table}" }.join(' ')
         cmd = Terrapin::CommandLine.new(
           'pg_dump',
-          "#{connection_args} -x -Fc :database #{sequences_args} > :output_path",
-          environment: psql_env
+          "#{dumper_connection_args} -x -Fc :database #{sequences_args} > :output_path",
+          environment: dumper_psql_env
         )
 
         cmd.run(database: database, host: host, port: port, output_path: sequences_dump_file.path)
@@ -167,8 +167,8 @@ module DataKeeper
     def all_sequences_to_export
       cmd = Terrapin::CommandLine.new(
         'psql',
-        "#{connection_args} -d :database -c :sql -A -R ',' -t",
-        environment: psql_env
+        "#{dumper_connection_args} -d :database -c :sql -A -R ',' -t",
+        environment: dumper_psql_env
       )
 
       sequences = cmd.run(

data/lib/data_keeper/loader.rb

@@ -7,9 +7,9 @@ module DataKeeper
     def initialize(dump, file)
       @dump = dump
       @file = file
-      @psql_version = Terrapin::CommandLine.new('psql', '--version').run
-                                           .match(/[0-9]{1,}\.[0-9]{1,}/)
-                                           .to_s.to_f
+      @psql_version = build_terrapin_command('psql', '--version').run
+                                                                 .match(/[0-9]{1,}\.[0-9]{1,}/)
+                                                                 .to_s.to_f
     end
 
     def load!
@@ -35,12 +35,23 @@ module DataKeeper
       end
     end
 
+    def build_terrapin_command(binary, args, docker_args = args)
+      if DataKeeper.docker_config.any?
+        Terrapin::CommandLine.new(
+          'docker',
+          "exec #{docker_env_params} -i #{DataKeeper.docker_config[:instance_name]} #{binary} #{docker_args}"
+        )
+      else
+        Terrapin::CommandLine.new(
+          binary,
+          args,
+          environment: psql_env
+        )
+      end
+    end
+
     def set_ar_internal_metadata!
-      cmd = Terrapin::CommandLine.new(
-        'psql',
-        "#{connection_args} -d :database -c :sql",
-        environment: psql_env
-      )
+      cmd = build_terrapin_command("psql", "#{connection_args} -d :database -c :sql")
 
       cmd.run(
         database: database,
@@ -58,12 +69,10 @@ module DataKeeper
     end
 
     def load_full_database!
-      ensure_schema_compatibility!
-
-      pg_restore = Terrapin::CommandLine.new(
-        'pg_restore',
+      pg_restore = build_terrapin_command(
+        "pg_restore",
         "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path}#{log_redirect}",
-        environment: psql_env
+        "#{connection_args} --no-owner --dbname #{database} < #{@file.path}#{log_redirect}"
       )
 
       pg_restore.run(
@@ -79,10 +88,10 @@ module DataKeeper
       ensure_schema_compatibility!
 
       inflate(@file.path) do |schema_path, tables_path, sql_files, sequences_path|
-        pg_restore = Terrapin::CommandLine.new(
-          'pg_restore',
+        pg_restore = build_terrapin_command(
+          "pg_restore",
           "#{connection_args} -j 4 --no-owner -s --dbname :database #{schema_path}#{log_redirect}",
-          environment: psql_env
+          "#{connection_args} --no-owner -s --dbname :database < #{schema_path}#{log_redirect}"
         )
 
         pg_restore.run(
@@ -91,10 +100,10 @@ module DataKeeper
           port: port
         )
 
-        pg_restore = Terrapin::CommandLine.new(
-          'pg_restore',
+        pg_restore = build_terrapin_command(
+          "pg_restore",
           "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path}#{log_redirect}",
-          environment: psql_env
+          "#{connection_args} --data-only --no-owner --disable-triggers --dbname :database < #{tables_path}#{log_redirect}"
         )
 
         pg_restore.run(
@@ -104,11 +113,7 @@ module DataKeeper
         )
 
         sql_files.each do |table, csv_path|
-          cmd = Terrapin::CommandLine.new(
-            'psql',
-            "#{connection_args} -d :database -c :command < :csv_path",
-            environment: psql_env
-          )
+          cmd = build_terrapin_command("psql", "#{connection_args} -d :database -c :command < :csv_path")
 
           cmd.run(
             database: database,
@@ -119,10 +124,10 @@ module DataKeeper
           )
         end
 
-        pg_restore = Terrapin::CommandLine.new(
-          'pg_restore',
+        pg_restore = build_terrapin_command(
+          "pg_restore",
           "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{sequences_path}#{log_redirect}",
-          environment: psql_env
+          "#{connection_args} --data-only --no-owner --disable-triggers --dbname :database < #{sequences_path}#{log_redirect}"
         )
 
         pg_restore.run(
@@ -136,11 +141,7 @@ module DataKeeper
     end
 
     def ensure_schema_compatibility!
-      cmd = Terrapin::CommandLine.new(
-        'psql',
-        "#{connection_args} -d :database -c :command",
-        environment: psql_env
-      )
+      cmd = build_terrapin_command("psql", "#{connection_args} -d :database -c :command")
 
       if @psql_version >= 11.0
         cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
@@ -207,9 +208,9 @@ module DataKeeper
 
           yield(
             inflated_files.schema_path,
-            inflated_files.tables_path,
-            inflated_files.sql_dumps,
-            inflated_files.sequences_path
+              inflated_files.tables_path,
+              inflated_files.sql_dumps,
+              inflated_files.sequences_path
           )
         end
       end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.9"
+  VERSION = "0.2.1"
 end

data/lib/data_keeper.rb

@@ -17,6 +17,7 @@ module DataKeeper
 
   @dump_definition_builders = {}
   @storage = nil
+  @docker_config = {}
   @database_config = -> { Rails.configuration.database_configuration[Rails.env] }
 
   def self.define_dump(name, type = :partial, &block)
@@ -69,6 +70,10 @@ module DataKeeper
     @storage = value
   end
 
+  def self.docker_config=(value)
+    @docker_config = value
+  end
+
   def self.database_config=(value)
     @database_config = value
   end
@@ -77,6 +82,10 @@ module DataKeeper
     @database_config
   end
 
+  def self.docker_config
+    @docker_config
+  end
+
   def self.clear_dumps!
     @dump_definition_builders = {}
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.1
 platform: ruby
 authors:
 - Roger Campos
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-28 00:00:00.000000000 Z
+date: 2022-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -100,7 +100,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/rogercampos/data_keeper
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -116,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Easy management of database dumps for dev env
 test_files: []