data_keeper 0.1.6 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 281f71c099696e1d51c88dd57e2cd0edf5f028bbbf4340c4457923e3a17b553a
-  data.tar.gz: 1592db229f9407735630803bff1a97bd85ed051ec30b878029952ffcfb272759
+  metadata.gz: 8aefdb5a66266a15337e2f65a59595a3017c744c6532d97d8c4377eb64f82d43
+  data.tar.gz: 49e39de5ecc7fbdf4355bf25c6d5300b5027c1413450aee696a51683d930065e
 SHA512:
-  metadata.gz: cf10459525a1cd3f210c6ef926b2b6d87a217505c8d8c8c155e99021d7a8837312fbf0038302987a683bf2a66f335d7507071451eff213ce004536ea42d26ca9
-  data.tar.gz: 30ff5218bb48851d938b96afba015e9076226b25342171e35ac47b5d78ecc8d8c3308afc2ffd6a94ded725495303da9a65f4394530a63cb193fcf005a90d4952
+  metadata.gz: d91d76636fd6eadab472f6442cca9484b95130209a8ba2adef6197c600ab31e43b054121991f6e96ef7200cc4a4ba4f23215ccdf5bf8d6a599d5495786832b51
+  data.tar.gz: d13e22eb926add303417d44fe1ac802b69b3839eb8349903fb10059139bd583e0d91f9e4aaa3a0e9f67ad2782d8aec4628290f48e95b92b64c1facfe1ac70a8f

data/lib/data_keeper/definition.rb

@@ -37,9 +37,12 @@ module DataKeeper
     end
 
     def evaluate!
-      instance_eval(&@definition_block) if @definition_block
+      @definition ||=
+        begin
+          instance_eval(&@definition_block) if @definition_block
 
-      Definition.new(@type, @tables, @raw_sqls, @on_after_load_block)
+          Definition.new(@type, @tables, @raw_sqls, @on_after_load_block)
+        end
     end
 
     def table(name)

data/lib/data_keeper/dumper.rb

@@ -48,6 +48,7 @@ module DataKeeper
             dump_schema(tar)
             dump_partial_tables(tar)
             dump_sqls(tar)
+            dump_sequences(tar)
           end
         end
 
@@ -136,8 +137,47 @@ module DataKeeper
       end
     end
 
+    def dump_sequences(tar)
+      Tempfile.create do |sequences_dump_file|
+        sequences_dump_file.binmode
+
+        sequences_args = all_sequences_to_export.map { |table| "-t #{table}" }.join(' ')
+        cmd = Terrapin::CommandLine.new(
+          'pg_dump',
+          "#{connection_args} -x -Fc :database #{sequences_args} > :output_path",
+          environment: psql_env
+        )
+
+        cmd.run(database: database, host: host, port: port, output_path: sequences_dump_file.path)
+
+        tar.add_file_simple("sequences.dump", 0644, File.size(sequences_dump_file.path)) do |io|
+          sequences_dump_file.reopen(sequences_dump_file)
+
+          while !sequences_dump_file.eof?
+            io.write(sequences_dump_file.read(2048))
+          end
+        end
+      end
+    end
+
     def filename
       "#{@dump_name}-#{Time.now.strftime("%Y%m%d-%H%M")}"
     end
+
+    def all_sequences_to_export
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :sql -A -R ',' -t",
+        environment: psql_env
+      )
+
+      sequences = cmd.run(
+        database: database,
+        host: host,
+        port: port,
+        sql: "SELECT sequencename from pg_sequences;"
+      )
+      sequences.split(',').map{|x| x.strip}
+    end
   end
 end

data/lib/data_keeper/loader.rb

@@ -76,7 +76,9 @@ module DataKeeper
     end
 
     def load_partial_database!
-      inflate(@file.path) do |schema_path, tables_path, sql_files|
+      ensure_schema_compatibility!
+
+      inflate(@file.path) do |schema_path, tables_path, sql_files, sequences_path|
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
           "#{connection_args} -j 4 --no-owner -s --dbname :database #{schema_path}#{log_redirect}",
@@ -117,6 +119,18 @@ module DataKeeper
           )
         end
 
+        pg_restore = Terrapin::CommandLine.new(
+          'pg_restore',
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{sequences_path}#{log_redirect}",
+          environment: psql_env
+        )
+
+        pg_restore.run(
+          database: database,
+          host: host,
+          port: port
+        )
+
         set_ar_internal_metadata!
       end
     end
@@ -149,9 +163,8 @@ module DataKeeper
 
         validate("Schema file is missing") { !!schema_path } &&
           validate("Tables file is missing") { !!tables_path } &&
-          validate("Not all sql custom dumps are present") do
-            sql_dumps.size == @dump.sqls.keys.size
-          end
+          validate("Not all sql custom dumps are present") { sql_dumps.size == @dump.sqls.keys.size } &&
+          validate("Sequences file is missing") { !!sequences_path }
       end
 
       def schema_path
@@ -162,6 +175,10 @@ module DataKeeper
         @tables_path ||= @paths.find { |x| File.basename(x) == "tables.dump" }
       end
 
+      def sequences_path
+        @sequences_path ||= @paths.find { |x| File.basename(x) == "sequences.dump" }
+      end
+
       def sql_dumps
         @sql_dumps ||= @dump.sqls.map do |name, (table, _proc)|
           path = @paths.find { |x| File.basename(x) == "#{name}.csv" }
@@ -191,7 +208,8 @@ module DataKeeper
           yield(
             inflated_files.schema_path,
             inflated_files.tables_path,
-            inflated_files.sql_dumps
+            inflated_files.sql_dumps,
+            inflated_files.sequences_path
           )
         end
       end

data/lib/data_keeper/local_storage.rb

@@ -21,14 +21,15 @@ module DataKeeper
     def retrieve(dump_name)
       tempfile = Tempfile.new
       local_store_dir = @local_store_dir
+      last_dump_filename = nil
 
       on complete_host do
-        last_dump = capture :ls, "-1t #{File.join(local_store_dir, dump_name.to_s)} | head -n 1"
+        last_dump_filename = capture :ls, "-1t #{File.join(local_store_dir, dump_name.to_s)} | head -n 1"
 
-        download! File.join(local_store_dir, dump_name.to_s, last_dump), tempfile.path
+        download! File.join(local_store_dir, dump_name.to_s, last_dump_filename), tempfile.path
       end
 
-      yield(tempfile)
+      yield(tempfile, last_dump_filename)
     ensure
       tempfile.delete
     end

data/lib/data_keeper/s3_storage.rb

@@ -78,7 +78,7 @@ module DataKeeper
     end
 
     def retrieve(dump_name)
-      prefix = "#{@store_dir}#{dump_name.to_s}"
+      prefix = "#{@store_dir}#{dump_name.to_s}/"
       last_dump = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse.first
 
       Tempfile.create do |tmp_file|
@@ -86,7 +86,7 @@ module DataKeeper
         s3_client.stream_to_io(last_dump.key, tmp_file)
         tmp_file.flush
 
-        yield(tmp_file)
+        yield(tmp_file, File.basename(last_dump.key))
       end
     end
 

data/lib/data_keeper/tasks/data_keeper.rake

@@ -35,4 +35,15 @@ namespace :data_keeper do
 
     DataKeeper.load_dump!(name, path)
   end
+
+  desc "Downloads the given dump raw file in the current directory"
+  task :download, [:name] => [:environment] do |_t, args|
+    name = args[:name]
+
+    if name.blank? || !DataKeeper.dump?(name)
+      raise "Please use this rake task giving a name of a configured dump. Ex: bin/rake data_keeper:pull[full]"
+    end
+
+    DataKeeper.download_dump!(name, ".")
+  end
 end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.6"
+  VERSION = "0.1.9"
 end

data/lib/data_keeper.rb

@@ -44,6 +44,15 @@ module DataKeeper
     end
   end
 
+  def self.download_dump!(name, path)
+    raise DumpDoesNotExist unless dump?(name)
+    raise NoStorageDefined if @storage.nil?
+
+    @storage.retrieve(name) do |file, filename|
+      FileUtils.cp file.path, File.join(path, filename)
+    end
+  end
+
   def self.load_dump!(name, path)
     raise DumpDoesNotExist unless File.file?(path)
     raise NoStorageDefined if @storage.nil?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.9
 platform: ruby
 authors:
 - Roger Campos
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-20 00:00:00.000000000 Z
+date: 2022-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -100,7 +100,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/rogercampos/data_keeper
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -116,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Easy management of database dumps for dev env
 test_files: []