data_keeper 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c4e8095729b6927bc11ae296fc30571577d1dfd5f5d910b1092f924981efa69
-  data.tar.gz: bb646294c8847fd637b765fd6a2f16af605dabe656b663d2cb0e6f54ff396f79
+  metadata.gz: 268e784853f728ad8c4cefd36ca8cc2f09b2892ab8751f1eb0f16dc17bc72900
+  data.tar.gz: 62ea9b6a3bf57ae952417db4fd149365f56f4baf63f67839bde26d3d2f352464
 SHA512:
-  metadata.gz: cd3696d94b9ef70bf108e8c77ee880e1b97e68989955c6c7975b4f9cea9196b95417a17e6b5bf6e3a9baf2047cd0a1db524e2bc8dd660d2576f10385f38a038d
-  data.tar.gz: 1ecab9088e179625113bb3a490a83c60a16405924f527caa059ed1dc05c68527049f12cc985aa1280f6e094597903f5bf280420a21c98a1670eb527b4e766c52
+  metadata.gz: 8ea7c0182fb8698c7d9c2be973cf7d3def1845222027feb90107adda8d333488ef01318268f8ae1356c2bd45d89419cb908443b5a9037df359cf5fc63376fca7
+  data.tar.gz: 7110186baac099fe5ce341186b25594c4ad730a811bc4270c4cd8e1d14d3cf4858bce5eb8bed0e621152da6d5729223c4d47c4f74b8e558c9539bb5bfcecd0c1

data/README.md

@@ -36,23 +36,37 @@ order to download these dumps later. Ex:
 DataKeeper.storage = DataKeeper::LocalStorage.new(
   local_store_dir: "/users/fredy/backups/...",
   remote_access: {
-    type: "scp",
-    host: "141.12.241.22",
-    port: "8622",
-    user: "fredy"
+    host: "10.10.10.10",
+    port: "22",
+    user: "user"
   }
 )
 ```
 
-Other storages, like S3, could be implemented, but currently this gem only ships with local storage.
-If you want to do your own, you can assign as an storage whatever object that responds to:
+There's also support for storing the dumps in s3, using `DataKeeper::S3Storage` like in this example:
 
-- `#save(file, filename, dump_name)`, where file is a File object and filename a string. This method should save the given
-  dump file. 
+```ruby
+DataKeeper.storage = DataKeeper::S3Storage.new(
+  bucket: 'bucket-name',
+  store_dir: 'dumps/',
+  acl: "private",
+  remote_access: {
+          access_key_id: Rails.application.credentials.access_key_id,
+          secret_access_key: Rails.application.credentials.secret_access_key,
+          region: 'eu-central-1'
+  }
+)
+```
+
+
+Other storages can be implemented. An storage can be any object that responds to those two methods:
+
+- `#save(file, filename, dump_name)`, where file is a File object and filename and dump_name are strings. 
+  This method should save the given dump file in the store. 
 
 - `#retrieve(dump_name) { |file| (...) }`, which should retrieve the latest stored dump with the given dump_name.
-  It should yield the given block passing the File object pointing to the retrieved dump file in the local filesystem,
-  which is expected to be cleaned up on block termination.
+  It should yield the given block passing the `File` or `Tempfile` object pointing to the retrieved dump 
+  file in the local filesystem, which is expected to be cleaned up on block termination.
 
 
 Then, declare some dumps to work with:
@@ -101,7 +115,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/data_keeper.
+Bug reports and pull requests are welcome on GitHub at https://github.com/rogercampos/data_keeper.
 
 
 ## License

data/data_keeper.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.description   = %q{Easy management of database dumps for dev env}
   spec.homepage      = "https://github.com/rogercampos/data_keeper"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -27,5 +27,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "terrapin", ">= 0.5.0"
   spec.add_dependency "sshkit", ">= 1.20.0"
   spec.add_dependency "rails", ">= 5.0.0"
-
 end

data/lib/data_keeper/s3_storage.rb

@@ -0,0 +1,103 @@
+begin
+  require 'aws-sdk-s3'
+rescue LoadError
+  raise "You must include the 'aws-sdk-s3' gem in your Gemfile in order to use this s3 storage."
+end
+
+module DataKeeper
+  class S3Storage
+    class Client
+      NoSuchKey = Class.new(StandardError)
+
+      def initialize(client_options:, bucket: nil)
+        @client_options = client_options
+        @client = Aws::S3::Client.new(client_options)
+        @bucket = bucket
+      end
+
+      def delete_files(file_paths)
+        @client.delete_objects(
+          bucket: @bucket,
+          delete: {
+            objects: file_paths.map { |key| { key: key } }
+          }
+        )
+      end
+
+      def list_contents(prefix = '')
+        @client.list_objects(bucket: @bucket, prefix: prefix).contents
+      rescue Aws::S3::Errors::NoSuchKey
+        raise NoSuchKey, prefix
+      end
+
+      # Streams all contents from `path` into the provided io object, calling #write to it.
+      # io can be a File, or any other IO-like object.
+      def stream_to_io(path, io, opts = {})
+        @client.get_object(opts.merge(
+          bucket: @bucket,
+          key: path
+        ), target: io)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise NoSuchKey, path
+      end
+
+      # Uploads the given file into the target_path in the s3 bucket.
+      # `file` must be a file stored locally. Can be either a raw string (path),
+      # or a File/Tempfile object (close is up to you).
+      def put_file(target_path, file, options = {})
+        file.rewind if file.respond_to?(:rewind)
+
+        s3 = Aws::S3::Resource.new(@client_options)
+        obj = s3.bucket(@bucket).object(target_path)
+        obj.upload_file(file, options)
+      end
+    end
+
+    def initialize(bucket:, store_dir:, remote_access:, acl: "public-read", keep_amount: 3)
+      @bucket = bucket
+      @store_dir = store_dir
+      @remote_access = remote_access
+      @acl = acl
+      @keep_amount = keep_amount
+    end
+
+    def save(file, filename, dump_name)
+      path = dump_path(dump_name, filename)
+
+      s3_client.put_file(path, file, acl: @acl)
+
+      prefix = "#{@store_dir}#{dump_name.to_s}"
+
+      keys_to_delete = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse[@keep_amount..-1]
+
+      return unless keys_to_delete
+
+      s3_client.delete_files(keys_to_delete.map(&:key))
+
+      true
+    end
+
+    def retrieve(dump_name)
+      prefix = "#{@store_dir}#{dump_name.to_s}"
+      last_dump = s3_client.list_contents(prefix).sort_by(&:last_modified).reverse.first
+
+      Tempfile.create do |tmp_file|
+        tmp_file.binmode
+        s3_client.stream_to_io(last_dump.key, tmp_file)
+        tmp_file.flush
+
+        yield(tmp_file)
+      end
+    end
+
+    private
+
+    def s3_client
+      @s3_client ||= Client.new(bucket: @bucket, client_options: @remote_access)
+    end
+
+    def dump_path(dump_name, filename)
+      File.join(@store_dir, dump_name.to_s, "#{SecureRandom.alphanumeric(40)}-#{filename}")
+    end
+  end
+end

data/lib/data_keeper/version.rb

@@ -1,3 +1,3 @@
 module DataKeeper
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_keeper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Roger Campos
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-09 00:00:00.000000000 Z
+date: 2021-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -91,6 +91,7 @@ files:
 - lib/data_keeper/loader.rb
 - lib/data_keeper/local_storage.rb
 - lib/data_keeper/railtie.rb
+- lib/data_keeper/s3_storage.rb
 - lib/data_keeper/tasks/data_keeper.rake
 - lib/data_keeper/version.rb
 - todo.md
@@ -107,7 +108,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="