data_keeper 0.1.4 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0442b3f4ba1b60c62570ca379d4a5d6f2a2181ec9cb7561107d52a3492b62ec5
-  data.tar.gz: e7594ccf321cd010c5cf74926ff0dd881d4aee7e6c142b204a58c9f9df137ebd
+  metadata.gz: 0666762022ca90ffac33878de1b342661885b9aeecfdac775a729f3de2b48cc3
+  data.tar.gz: 0c17de29ea41f6023579bd8e8b6bb1e5bf34e682ff934291686d3c5721af02c2
 SHA512:
-  metadata.gz: 12a6f3303abd4528976c51e49361f183b306fc352df0d9bc8cb3c2e44845f61920f8b9bf4e70c438cd42a8a33703dd98693a71c10f0a79e90d5015c8ac2977bb
-  data.tar.gz: a5b9ecb78feecd034287fd86e34bdbd151d028a61d0be6ea58494b35102c3080d2c7c97eb3c5816d470a66a2f52a44c773a64a96ee2e3d3a01562ad5b8fc101c
+  metadata.gz: 6369301be6b94b87e5a8ea26d0e56926d560c9229c3ed7dba6a12e70193324584d7eec1e10ca7f3d02a4de5f5473a2b4fb0a4109094dd7176af5e8f2a5729995
+  data.tar.gz: c6ef2e6e5d521def03e06e3eb8c7113f9a8f61376d6b7d3f3e3d2b26fdc4c8ebc5cfba615d84224517fb5955c15d16e89e6a7a170f21bd6eb751aac810fdc975

data/.gitignore

@@ -6,5 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+/lib/tmp
 
 Gemfile.lock

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "minitest", "~> 5.0"
+gem 'pg'

data/lib/data_keeper/database_config.rb

@@ -1,7 +1,7 @@
 module DataKeeper
   module DatabaseConfig
     def database_connection_config
-      Rails.configuration.database_configuration[Rails.env]
+      @database_connection_config ||= DataKeeper.database_config.call
     end
 
     def psql_env

data/lib/data_keeper/definition.rb

@@ -33,6 +33,7 @@ module DataKeeper
       @tables = []
       @raw_sqls = {}
       @definition_block = definition_block
+      @on_after_load_block = nil
     end
 
     def evaluate!

data/lib/data_keeper/dumper.rb

@@ -48,6 +48,7 @@ module DataKeeper
             dump_schema(tar)
             dump_partial_tables(tar)
             dump_sqls(tar)
+            dump_sequences(tar)
           end
         end
 
@@ -136,8 +137,47 @@ module DataKeeper
       end
     end
 
+    def dump_sequences(tar)
+      Tempfile.create do |sequences_dump_file|
+        sequences_dump_file.binmode
+
+        sequences_args = all_sequences_to_export.map { |table| "-t #{table}" }.join(' ')
+        cmd = Terrapin::CommandLine.new(
+          'pg_dump',
+          "#{connection_args} -x -Fc :database #{sequences_args} > :output_path",
+          environment: psql_env
+        )
+
+        cmd.run(database: database, host: host, port: port, output_path: sequences_dump_file.path)
+
+        tar.add_file_simple("sequences.dump", 0644, File.size(sequences_dump_file.path)) do |io|
+          sequences_dump_file.reopen(sequences_dump_file)
+
+          while !sequences_dump_file.eof?
+            io.write(sequences_dump_file.read(2048))
+          end
+        end
+      end
+    end
+
     def filename
       "#{@dump_name}-#{Time.now.strftime("%Y%m%d-%H%M")}"
     end
+
+    def all_sequences_to_export
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :sql -A -R ',' -t",
+        environment: psql_env
+      )
+
+      sequences = cmd.run(
+        database: database,
+        host: host,
+        port: port,
+        sql: "SELECT sequencename from pg_sequences;"
+      )
+      sequences.split(',').map{|x| x.strip}
+    end
   end
 end

data/lib/data_keeper/loader.rb

@@ -7,6 +7,9 @@ module DataKeeper
     def initialize(dump, file)
       @dump = dump
       @file = file
+      @psql_version = Terrapin::CommandLine.new('psql', '--version').run
+                                           .match(/[0-9]{1,}\.[0-9]{1,}/)
+                                           .to_s.to_f
     end
 
     def load!
@@ -17,58 +20,68 @@ module DataKeeper
       end
 
       if @dump.on_after_load_block
+        ActiveRecord::Base.establish_connection
         @dump.on_after_load_block.call
       end
     end
 
     private
 
-    def load_full_database!
+    def log_redirect
+      if Terrapin::CommandLine.logger
+        ""
+      else
+        "  2>/dev/null"
+      end
+    end
+
+    def set_ar_internal_metadata!
       cmd = Terrapin::CommandLine.new(
         'psql',
-        "#{connection_args} -d :database -c :command",
+        "#{connection_args} -d :database -c :sql",
         environment: psql_env
       )
-      cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
 
-      pg_restore = Terrapin::CommandLine.new(
-        'pg_restore',
-        "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path} 2>/dev/null",
-        environment: psql_env
+      cmd.run(
+        database: database,
+        host: host,
+        port: port,
+        sql: "DELETE from ar_internal_metadata"
       )
 
-      pg_restore.run(
+      cmd.run(
         database: database,
         host: host,
-        port: port
+        port: port,
+        sql: "INSERT into ar_internal_metadata (key, value, created_at, updated_at) VALUES ('environment', 'development', '2020-04-03 12:25:54.094209', '2020-04-03 12:25:54.094209')"
       )
+    end
 
-      cmd = Terrapin::CommandLine.new(
-        'psql',
-        "#{connection_args} -d :database -c :sql",
+    def load_full_database!
+      ensure_schema_compatibility!
+
+      pg_restore = Terrapin::CommandLine.new(
+        'pg_restore',
+        "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path}#{log_redirect}",
         environment: psql_env
       )
 
-      cmd.run(
+      pg_restore.run(
         database: database,
         host: host,
-        port: port,
-        sql: "UPDATE ar_internal_metadata SET value = 'development'"
+        port: port
       )
+
+      set_ar_internal_metadata!
     end
 
     def load_partial_database!
-      inflate(@file.path) do |schema_path, tables_path, sql_files|
-        cmd = Terrapin::CommandLine.new(
-          'psql',
-          "#{connection_args} -d :database -c :command",
-          environment: psql_env
-        )
-        cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+      ensure_schema_compatibility!
 
+      inflate(@file.path) do |schema_path, tables_path, sql_files, sequences_path|
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} -j 4 --no-owner --dbname :database #{schema_path} 2>/dev/null",
+          "#{connection_args} -j 4 --no-owner -s --dbname :database #{schema_path}#{log_redirect}",
           environment: psql_env
         )
 
@@ -80,7 +93,7 @@ module DataKeeper
 
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path} 2>/dev/null",
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path}#{log_redirect}",
           environment: psql_env
         )
 
@@ -106,7 +119,33 @@ module DataKeeper
           )
         end
 
-        Rake::Task['db:environment:set'].invoke
+        pg_restore = Terrapin::CommandLine.new(
+          'pg_restore',
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{sequences_path}#{log_redirect}",
+          environment: psql_env
+        )
+
+        pg_restore.run(
+          database: database,
+          host: host,
+          port: port
+        )
+
+        set_ar_internal_metadata!
+      end
+    end
+
+    def ensure_schema_compatibility!
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :command",
+        environment: psql_env
+      )
+
+      if @psql_version >= 11.0
+        cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+      else
+        cmd.run(database: database, host: host, port: port, command: "create schema if not exists public")
       end
     end
 
@@ -124,9 +163,8 @@ module DataKeeper
 
         validate("Schema file is missing") { !!schema_path } &&
           validate("Tables file is missing") { !!tables_path } &&
-          validate("Not all sql custom dumps are present") do
-            sql_dumps.size == @dump.sqls.keys.size
-          end
+          validate("Not all sql custom dumps are present") { sql_dumps.size == @dump.sqls.keys.size } &&
+          validate("Sequences file is missing") { !!sequences_path }
       end
 
       def schema_path
@@ -137,6 +175,10 @@ module DataKeeper
         @tables_path ||= @paths.find { |x| File.basename(x) == "tables.dump" }
       end
 
+      def sequences_path
+        @sequences_path ||= @paths.find { |x| File.basename(x) == "sequences.dump" }
+      end
+
       def sql_dumps
         @sql_dumps ||= @dump.sqls.map do |name, (table, _proc)|
           path = @paths.find { |x| File.basename(x) == "#{name}.csv" }
@@ -166,7 +208,8 @@ module DataKeeper
           yield(
             inflated_files.schema_path,
             inflated_files.tables_path,
-            inflated_files.sql_dumps
+            inflated_files.sql_dumps,
+            inflated_files.sequences_path
           )
         end
       end

data/lib/data_keeper/local_storage.rb

@@ -21,14 +21,15 @@ module DataKeeper
     def retrieve(dump_name)
       tempfile = Tempfile.new
       local_store_dir = @local_store_dir
+      last_dump_filename = nil
 
       on complete_host do
-        last_dump = capture :ls, "-1t #{File.join(local_store_dir, dump_name.to_s)} | head -n 1"
+        last_dump_filename = capture :ls, "-1t #{File.join(local_store_dir, dump_name.to_s)} | head -n 1"
 
-        download! File.join(local_store_dir, dump_name.to_s, last_dump), tempfile.path
+        download! File.join(local_store_dir, dump_name.to_s, last_dump_filename), tempfile.path
       end
 
-      yield(tempfile)
+      yield(tempfile, last_dump_filename)
     ensure
       tempfile.delete
     end

data/lib/data_keeper/s3_storage.rb

@@ -78,7 +78,7 @@ module DataKeeper
     end
 
     def retrieve(dump_name)
-      prefix = "#{@store_dir}#{dump_name.to_s}"
+      prefix = "#{@store_dir}#{dump_name.to_s}/"
       last_dump = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse.first
 
       Tempfile.create do |tmp_file|
@@ -86,7 +86,7 @@ module DataKeeper
         s3_client.stream_to_io(last_dump.key, tmp_file)
         tmp_file.flush
 
-        yield(tmp_file)
+        yield(tmp_file, File.basename(last_dump.key))
       end
     end
 

data/lib/data_keeper/tasks/data_keeper.rake

@@ -35,4 +35,15 @@ namespace :data_keeper do
 
     DataKeeper.load_dump!(name, path)
   end
+
+  desc "Downloads the given dump raw file in the current directory"
+  task :download, [:name] => [:environment] do |_t, args|
+    name = args[:name]
+
+    if name.blank? || !DataKeeper.dump?(name)
+      raise "Please use this rake task giving a name of a configured dump. Ex: bin/rake data_keeper:pull[full]"
+    end
+
+    DataKeeper.download_dump!(name, ".")
+  end
 end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.4"
+  VERSION = "0.1.8"
 end

data/lib/data_keeper.rb

@@ -17,6 +17,7 @@ module DataKeeper
 
   @dump_definition_builders = {}
   @storage = nil
+  @database_config = -> { Rails.configuration.database_configuration[Rails.env] }
 
   def self.define_dump(name, type = :partial, &block)
     @dump_definition_builders[name.to_sym] = DefinitionBuilder.new(type, block)
@@ -43,6 +44,15 @@ module DataKeeper
     end
   end
 
+  def self.download_dump!(name, path)
+    raise DumpDoesNotExist unless dump?(name)
+    raise NoStorageDefined if @storage.nil?
+
+    @storage.retrieve(name) do |file, filename|
+      FileUtils.cp file.path, File.join(path, filename)
+    end
+  end
+
   def self.load_dump!(name, path)
     raise DumpDoesNotExist unless File.file?(path)
     raise NoStorageDefined if @storage.nil?
@@ -59,6 +69,14 @@ module DataKeeper
     @storage = value
   end
 
+  def self.database_config=(value)
+    @database_config = value
+  end
+
+  def self.database_config
+    @database_config
+  end
+
   def self.clear_dumps!
     @dump_definition_builders = {}
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.8
 platform: ruby
 authors:
 - Roger Campos
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-08 00:00:00.000000000 Z
+date: 2021-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -100,7 +100,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/rogercampos/data_keeper
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -115,8 +115,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Easy management of database dumps for dev env
 test_files: []