data_keeper 0.1.0 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c4e8095729b6927bc11ae296fc30571577d1dfd5f5d910b1092f924981efa69
-  data.tar.gz: bb646294c8847fd637b765fd6a2f16af605dabe656b663d2cb0e6f54ff396f79
+  metadata.gz: 0442b3f4ba1b60c62570ca379d4a5d6f2a2181ec9cb7561107d52a3492b62ec5
+  data.tar.gz: e7594ccf321cd010c5cf74926ff0dd881d4aee7e6c142b204a58c9f9df137ebd
 SHA512:
-  metadata.gz: cd3696d94b9ef70bf108e8c77ee880e1b97e68989955c6c7975b4f9cea9196b95417a17e6b5bf6e3a9baf2047cd0a1db524e2bc8dd660d2576f10385f38a038d
-  data.tar.gz: 1ecab9088e179625113bb3a490a83c60a16405924f527caa059ed1dc05c68527049f12cc985aa1280f6e094597903f5bf280420a21c98a1670eb527b4e766c52
+  metadata.gz: 12a6f3303abd4528976c51e49361f183b306fc352df0d9bc8cb3c2e44845f61920f8b9bf4e70c438cd42a8a33703dd98693a71c10f0a79e90d5015c8ac2977bb
+  data.tar.gz: a5b9ecb78feecd034287fd86e34bdbd151d028a61d0be6ea58494b35102c3080d2c7c97eb3c5816d470a66a2f52a44c773a64a96ee2e3d3a01562ad5b8fc101c

data/README.md

@@ -36,23 +36,40 @@ order to download these dumps later. Ex:
 DataKeeper.storage = DataKeeper::LocalStorage.new(
   local_store_dir: "/users/fredy/backups/...",
   remote_access: {
-    type: "scp",
-    host: "141.12.241.22",
-    port: "8622",
-    user: "fredy"
+    host: "10.10.10.10",
+    port: "22",
+    user: "user"
   }
 )
 ```
 
-Other storages, like S3, could be implemented, but currently this gem only ships with local storage.
-If you want to do your own, you can assign as an storage whatever object that responds to:
+There's also support for storing the dumps in s3, using `DataKeeper::S3Storage` like in this example:
 
-- `#save(file, filename, dump_name)`, where file is a File object and filename a string. This method should save the given
-  dump file. 
+```ruby
+# Explicit require is necessary
+require 'data_keeper/s3_storage'
+
+DataKeeper.storage = DataKeeper::S3Storage.new(
+  bucket: 'bucket-name',
+  store_dir: 'dumps/',
+  acl: "private",
+  remote_access: {
+          access_key_id: Rails.application.credentials.access_key_id,
+          secret_access_key: Rails.application.credentials.secret_access_key,
+          region: 'eu-central-1'
+  }
+)
+```
+
+
+Other storages can be implemented. An storage can be any object that responds to those two methods:
+
+- `#save(file, filename, dump_name)`, where file is a File object and filename and dump_name are strings. 
+  This method should save the given dump file in the store. 
 
 - `#retrieve(dump_name) { |file| (...) }`, which should retrieve the latest stored dump with the given dump_name.
-  It should yield the given block passing the File object pointing to the retrieved dump file in the local filesystem,
-  which is expected to be cleaned up on block termination.
+  It should yield the given block passing the `File` or `Tempfile` object pointing to the retrieved dump 
+  file in the local filesystem, which is expected to be cleaned up on block termination.
 
 
 Then, declare some dumps to work with:
@@ -101,7 +118,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/data_keeper.
+Bug reports and pull requests are welcome on GitHub at https://github.com/rogercampos/data_keeper.
 
 
 ## License

data/data_keeper.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.description   = %q{Easy management of database dumps for dev env}
   spec.homepage      = "https://github.com/rogercampos/data_keeper"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -27,5 +27,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "terrapin", ">= 0.5.0"
   spec.add_dependency "sshkit", ">= 1.20.0"
   spec.add_dependency "rails", ">= 5.0.0"
-
 end

data/lib/data_keeper.rb

@@ -15,18 +15,20 @@ module DataKeeper
   DumpDoesNotExist = Class.new(Error)
   NoStorageDefined = Class.new(Error)
 
-  @dumps = {}
+  @dump_definition_builders = {}
   @storage = nil
 
   def self.define_dump(name, type = :partial, &block)
-    @dumps[name.to_sym] = DefinitionBuilder.build(type, block)
+    @dump_definition_builders[name.to_sym] = DefinitionBuilder.new(type, block)
   end
 
   def self.create_dump!(name)
     raise DumpDoesNotExist unless dump?(name)
     raise NoStorageDefined if @storage.nil?
 
-    Dumper.new(name, @dumps[name.to_sym]).run! do |file, filename|
+    definition = @dump_definition_builders[name.to_sym].evaluate!
+
+    Dumper.new(name, definition).run! do |file, filename|
       @storage.save(file, filename, name)
     end
   end
@@ -34,9 +36,10 @@ module DataKeeper
   def self.fetch_and_load_dump!(name)
     raise DumpDoesNotExist unless dump?(name)
     raise NoStorageDefined if @storage.nil?
+    definition = @dump_definition_builders[name.to_sym].evaluate!
 
     @storage.retrieve(name) do |file|
-      Loader.new(@dumps[name.to_sym], file).load!
+      Loader.new(definition, file).load!
     end
   end
 
@@ -44,11 +47,12 @@ module DataKeeper
     raise DumpDoesNotExist unless File.file?(path)
     raise NoStorageDefined if @storage.nil?
 
-    Loader.new(@dumps[name.to_sym], File.open(path)).load!
+    definition = @dump_definition_builders[name.to_sym].evaluate!
+    Loader.new(definition, File.open(path)).load!
   end
 
   def self.dump?(name)
-    @dumps.key?(name.to_sym)
+    @dump_definition_builders.key?(name.to_sym)
   end
 
   def self.storage=(value)
@@ -56,6 +60,6 @@ module DataKeeper
   end
 
   def self.clear_dumps!
-    @dumps = {}
+    @dump_definition_builders = {}
   end
 end

data/lib/data_keeper/definition.rb

@@ -26,21 +26,19 @@ module DataKeeper
   end
 
   class DefinitionBuilder
-    attr_reader :tables, :raw_sqls, :on_after_load_block
+    def initialize(type, definition_block)
+      raise InvalidDumpType, "Invalid type! use :partial or :full" unless [:partial, :full].include?(type)
 
-    def initialize(definition_block)
+      @type = type
       @tables = []
       @raw_sqls = {}
-      instance_eval(&definition_block) if definition_block
+      @definition_block = definition_block
     end
 
-    def self.build(type, block)
-      @type = type
-      raise InvalidDumpType, "Invalid type! use :partial or :full" unless [:partial, :full].include?(type)
-
-      builder = new(block)
+    def evaluate!
+      instance_eval(&@definition_block) if @definition_block
 
-      Definition.new(type, builder.tables, builder.raw_sqls, builder.on_after_load_block)
+      Definition.new(@type, @tables, @raw_sqls, @on_after_load_block)
     end
 
     def table(name)

data/lib/data_keeper/dumper.rb

@@ -35,7 +35,7 @@ module DataKeeper
           output_path: file.path
         )
 
-        yield file, "#{filename}.dump"
+        yield File.open(file.path), "#{filename}.dump"
       end
     end
 
@@ -51,7 +51,7 @@ module DataKeeper
           end
         end
 
-        yield file, "#{filename}.tar.gz"
+        yield File.open(file.path), "#{filename}.tar.gz"
       end
     end
 

data/lib/data_keeper/loader.rb

@@ -24,6 +24,13 @@ module DataKeeper
     private
 
     def load_full_database!
+      cmd = Terrapin::CommandLine.new(
+        'psql',
+        "#{connection_args} -d :database -c :command",
+        environment: psql_env
+      )
+      cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+
       pg_restore = Terrapin::CommandLine.new(
         'pg_restore',
         "#{connection_args} -j 4 --no-owner --dbname #{database} #{@file.path} 2>/dev/null",
@@ -52,9 +59,16 @@ module DataKeeper
 
     def load_partial_database!
       inflate(@file.path) do |schema_path, tables_path, sql_files|
+        cmd = Terrapin::CommandLine.new(
+          'psql',
+          "#{connection_args} -d :database -c :command",
+          environment: psql_env
+        )
+        cmd.run(database: database, host: host, port: port, command: "drop schema if exists public")
+
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} -j 4 --no-owner --dbname #{database} #{schema_path} 2>/dev/null",
+          "#{connection_args} -j 4 --no-owner --dbname :database #{schema_path} 2>/dev/null",
           environment: psql_env
         )
 
@@ -66,7 +80,7 @@ module DataKeeper
 
         pg_restore = Terrapin::CommandLine.new(
           'pg_restore',
-          "#{connection_args} -c -j 4 --no-owner --dbname #{database} #{tables_path} 2>/dev/null",
+          "#{connection_args} --data-only -j 4 --no-owner --disable-triggers --dbname :database #{tables_path} 2>/dev/null",
           environment: psql_env
         )
 
@@ -88,7 +102,7 @@ module DataKeeper
             host: host,
             port: port,
             csv_path: csv_path,
-            command: "COPY #{table} FROM stdin DELIMITER ',' CSV HEADER"
+            command: "ALTER TABLE #{table} DISABLE TRIGGER all; COPY #{table} FROM stdin DELIMITER ',' CSV HEADER"
           )
         end
 

data/lib/data_keeper/s3_storage.rb

@@ -0,0 +1,103 @@
+begin
+  require 'aws-sdk-s3'
+rescue LoadError
+  raise "You must include the 'aws-sdk-s3' gem in your Gemfile in order to use this s3 storage."
+end
+
+module DataKeeper
+  class S3Storage
+    class Client
+      NoSuchKey = Class.new(StandardError)
+
+      def initialize(client_options:, bucket: nil)
+        @client_options = client_options
+        @client = Aws::S3::Client.new(client_options)
+        @bucket = bucket
+      end
+
+      def delete_files(file_paths)
+        @client.delete_objects(
+          bucket: @bucket,
+          delete: {
+            objects: file_paths.map { |key| { key: key } }
+          }
+        )
+      end
+
+      def list_contents(prefix = '')
+        @client.list_objects(bucket: @bucket, prefix: prefix).contents
+      rescue Aws::S3::Errors::NoSuchKey
+        raise NoSuchKey, prefix
+      end
+
+      # Streams all contents from `path` into the provided io object, calling #write to it.
+      # io can be a File, or any other IO-like object.
+      def stream_to_io(path, io, opts = {})
+        @client.get_object(opts.merge(
+          bucket: @bucket,
+          key: path
+        ), target: io)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise NoSuchKey, path
+      end
+
+      # Uploads the given file into the target_path in the s3 bucket.
+      # `file` must be a file stored locally. Can be either a raw string (path),
+      # or a File/Tempfile object (close is up to you).
+      def put_file(target_path, file, options = {})
+        file.rewind if file.respond_to?(:rewind)
+
+        s3 = Aws::S3::Resource.new(@client_options)
+        obj = s3.bucket(@bucket).object(target_path)
+        obj.upload_file(file, options)
+      end
+    end
+
+    def initialize(bucket:, store_dir:, remote_access:, acl: "public-read", keep_amount: 3)
+      @bucket = bucket
+      @store_dir = store_dir
+      @remote_access = remote_access
+      @acl = acl
+      @keep_amount = keep_amount
+    end
+
+    def save(file, filename, dump_name)
+      path = dump_path(dump_name, filename)
+
+      s3_client.put_file(path, file, acl: @acl)
+
+      prefix = "#{@store_dir}#{dump_name.to_s}"
+
+      keys_to_delete = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse[@keep_amount..-1]
+
+      return unless keys_to_delete
+
+      s3_client.delete_files(keys_to_delete.map(&:key))
+
+      true
+    end
+
+    def retrieve(dump_name)
+      prefix = "#{@store_dir}#{dump_name.to_s}"
+      last_dump = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse.first
+
+      Tempfile.create do |tmp_file|
+        tmp_file.binmode
+        s3_client.stream_to_io(last_dump.key, tmp_file)
+        tmp_file.flush
+
+        yield(tmp_file)
+      end
+    end
+
+    private
+
+    def s3_client
+      @s3_client ||= Client.new(bucket: @bucket, client_options: @remote_access)
+    end
+
+    def dump_path(dump_name, filename)
+      File.join(@store_dir, dump_name.to_s, "#{SecureRandom.alphanumeric(40)}-#{filename}")
+    end
+  end
+end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.0"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.4
 platform: ruby
 authors:
 - Roger Campos
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-09 00:00:00.000000000 Z
+date: 2021-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -91,6 +91,7 @@ files:
 - lib/data_keeper/loader.rb
 - lib/data_keeper/local_storage.rb
 - lib/data_keeper/railtie.rb
+- lib/data_keeper/s3_storage.rb
 - lib/data_keeper/tasks/data_keeper.rake
 - lib/data_keeper/version.rb
 - todo.md
@@ -107,7 +108,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="