data_keeper 0.1.3 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dc4571443a29e3b6e5ef297eaf58bcc904245f2fa03e9afdbcb8004156163a3
-  data.tar.gz: 8eeb45101080f53b64365b28a3d5b03201ff22bf57399682dfeeac439df59496
+  metadata.gz: c525131692e6a1de920301cbc4b55360fc7dd089ccfc1b1db1068ef328b78e0c
+  data.tar.gz: ee646b314fc7203cadab9c366559c060c91ae9a45cea906f4ee4f4409c5bcd76
 SHA512:
-  metadata.gz: 6736489028cef881ae0c24617d87f54ac67d727121350e6a6c6bccbd1a13cc5c0eff21809b4a7705e34740a72270bebd7a65a6042dd253d98ee5cc4bc4455c8b
-  data.tar.gz: 30b58fe74b9b2c4beda31b7fb345bf2be6907c3f60e448cf7634e1458cf7ed6cff48662798b91dae4bd443b68ae680b01813595b0a95632a4a51f9e17e9f4b3b
+  metadata.gz: 2b7ae402b492cd31c8d82b0481f0c8088ef09f28d9fb09b37f86040e14c4ce6f1d2b4d2b75dca6f5bf78d18306844ed26657bcfcadfda0c0001523e8fac88534
+  data.tar.gz: 38c8bb4d0fcdb05cd43f93f7a0d70f69f33067e2e40dea1c0e8ddc4e6b4910cc7f635e5291059801a5cecade692ca4e714deb841b152b9fe792785c9435408ca

data/.gitignore

@@ -6,5 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+/lib/tmp
 
 Gemfile.lock

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "minitest", "~> 5.0"
+gem 'pg'

data/lib/data_keeper/database_config.rb

@@ -1,7 +1,7 @@
 module DataKeeper
   module DatabaseConfig
     def database_connection_config
-      Rails.configuration.database_configuration[Rails.env]
+      @database_connection_config ||= DataKeeper.database_config.call
     end
 
     def psql_env

data/lib/data_keeper/definition.rb

@@ -33,6 +33,7 @@ module DataKeeper
       @tables = []
       @raw_sqls = {}
       @definition_block = definition_block
+      @on_after_load_block = nil
     end
 
     def evaluate!

data/lib/data_keeper/dumper.rb

@@ -48,6 +48,7 @@ module DataKeeper
             dump_schema(tar)
             dump_partial_tables(tar)
             dump_sqls(tar)
+            dump_sequences(tar)
           end
         end
 
@@ -136,8 +137,47 @@ module DataKeeper
       end
     end
 
+    def dump_sequences(tar)
+      Tempfile.create do |sequences_dump_file|
+        sequences_dump_file.binmode
+
+        sequences_args = all_sequences_to_export.map { |table| "-t #{table}" }.join(' ')
+        cmd = Terrapin::CommandLine.new(
+          'pg_dump',
+          "#{connection_args} -x -Fc :database #{sequences_args} > :output_path",
+          environment: psql_env
+        )
+
+        cmd.run(database: database, host: host, port: port, output_path: sequences_dump_file.path)
+
+        tar.add_file_simple("sequences.dump", 0644, File.size(sequences_dump_file.path)) do |io|
+          sequences_dump_file.reopen(sequences_dump_file)
+
+          while !sequences_dump_file.eof?
+            io.write(sequences_dump_file.read(2048))
+          end
+        end
+      end
+    end
+
     def filename
       "#{@dump_name}-#{Time.now.strftime("%Y%m%d-%H%M")}"
     end
+
+    def all_sequences_to_export
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :sql -A -R ',' -t",
+        environment: psql_env
+      )
+
+      sequences = cmd.run(
+        database: database,
+        host: host,
+        port: port,
+        sql: "SELECT sequencename from pg_sequences;"
+      )
+      sequences.split(',').map{|x| x.strip}
+    end
   end
 end

data/lib/data_keeper/loader.rb

@@ -7,6 +7,9 @@ module DataKeeper
     def initialize(dump, file)
       @dump = dump
       @file = file
+      @psql_version = Terrapin::CommandLine.new('psql', '--version').run
+                                           .match(/[0-9]{1,}\.[0-9]{1,}/)
+                                           .to_s.to_f
     end
 
     def load!
@@ -17,51 +20,68 @@ module DataKeeper
       end
 
       if @dump.on_after_load_block
+        ActiveRecord::Base.establish_connection
         @dump.on_after_load_block.call
       end
     end
 
     private
 
-    def load_full_database!
-      pg_restore = Terrapin::CommandLine.new(
-        'pg_restore',
-        "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path} 2>/dev/null",
+    def log_redirect
+      if Terrapin::CommandLine.logger
+        ""
+      else
+        "  2>/dev/null"
+      end
+    end
+
+    def set_ar_internal_metadata!
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :sql",
         environment: psql_env
       )
 
-      pg_restore.run(
+      cmd.run(
         database: database,
         host: host,
-        port: port
+        port: port,
+        sql: "DELETE from ar_internal_metadata"
       )
 
-      cmd = Terrapin::CommandLine.new(
-        'psql',
-        "#{connection_args} -d :database -c :sql",
+      cmd.run(
+        database: database,
+        host: host,
+        port: port,
+        sql: "INSERT into ar_internal_metadata (key, value, created_at, updated_at) VALUES ('environment', 'development', '2020-04-03 12:25:54.094209', '2020-04-03 12:25:54.094209')"
+      )
+    end
+
+    def load_full_database!
+      ensure_schema_compatibility!
+
+      pg_restore = Terrapin::CommandLine.new(
+        'pg_restore',
+        "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path}#{log_redirect}",
         environment: psql_env
       )
 
-      cmd.run(
+      pg_restore.run(
         database: database,
         host: host,
-        port: port,
-        sql: "UPDATE ar_internal_metadata SET value = 'development'"
+        port: port
       )
+
+      set_ar_internal_metadata!
     end
 
     def load_partial_database!
-      inflate(@file.path) do |schema_path, tables_path, sql_files|
-        cmd = Terrapin::CommandLine.new(
-          'psql',
-          "#{connection_args} -d :database -c :command",
-          environment: psql_env
-        )
-        cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+      ensure_schema_compatibility!
 
+      inflate(@file.path) do |schema_path, tables_path, sql_files, sequences_path|
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} -j 4 --no-owner --dbname :database #{schema_path} 2>/dev/null",
+          "#{connection_args} -j 4 --no-owner -s --dbname :database #{schema_path}#{log_redirect}",
           environment: psql_env
         )
 
@@ -73,7 +93,7 @@ module DataKeeper
 
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path} 2>/dev/null",
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path}#{log_redirect}",
           environment: psql_env
         )
 
@@ -99,7 +119,33 @@ module DataKeeper
           )
         end
 
-        Rake::Task['db:environment:set'].invoke
+        pg_restore = Terrapin::CommandLine.new(
+          'pg_restore',
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{sequences_path}#{log_redirect}",
+          environment: psql_env
+        )
+
+        pg_restore.run(
+          database: database,
+          host: host,
+          port: port
+        )
+
+        set_ar_internal_metadata!
+      end
+    end
+
+    def ensure_schema_compatibility!
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :command",
+        environment: psql_env
+      )
+
+      if @psql_version >= 11.0
+        cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+      else
+        cmd.run(database: database, host: host, port: port, command: "create schema if not exists public")
       end
     end
 
@@ -117,9 +163,8 @@ module DataKeeper
 
         validate("Schema file is missing") { !!schema_path } &&
           validate("Tables file is missing") { !!tables_path } &&
-          validate("Not all sql custom dumps are present") do
-            sql_dumps.size == @dump.sqls.keys.size
-          end
+          validate("Not all sql custom dumps are present") { sql_dumps.size == @dump.sqls.keys.size } &&
+          validate("Sequences file is missing") { !!sequences_path }
       end
 
       def schema_path
@@ -130,6 +175,10 @@ module DataKeeper
         @tables_path ||= @paths.find { |x| File.basename(x) == "tables.dump" }
       end
 
+      def sequences_path
+        @sequences_path ||= @paths.find { |x| File.basename(x) == "sequences.dump" }
+      end
+
       def sql_dumps
         @sql_dumps ||= @dump.sqls.map do |name, (table, _proc)|
           path = @paths.find { |x| File.basename(x) == "#{name}.csv" }
@@ -159,7 +208,8 @@ module DataKeeper
           yield(
             inflated_files.schema_path,
             inflated_files.tables_path,
-            inflated_files.sql_dumps
+            inflated_files.sql_dumps,
+            inflated_files.sequences_path
           )
         end
       end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.3"
+  VERSION = "0.1.7"
 end

data/lib/data_keeper.rb

@@ -17,6 +17,7 @@ module DataKeeper
 
   @dump_definition_builders = {}
   @storage = nil
+  @database_config = -> { Rails.configuration.database_configuration[Rails.env] }
 
   def self.define_dump(name, type = :partial, &block)
     @dump_definition_builders[name.to_sym] = DefinitionBuilder.new(type, block)
@@ -59,6 +60,14 @@ module DataKeeper
     @storage = value
   end
 
+  def self.database_config=(value)
+    @database_config = value
+  end
+
+  def self.database_config
+    @database_config
+  end
+
   def self.clear_dumps!
     @dump_definition_builders = {}
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.7
 platform: ruby
 authors:
 - Roger Campos
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-06 00:00:00.000000000 Z
+date: 2021-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -115,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Easy management of database dumps for dev env