riserva 0.1.2 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: db30408186529f3b54b2351fcfda5f20474f78a3
-  data.tar.gz: da8cc2d981e0ea05801ad07a9fa42eae79a50e4b
+SHA256:
+  metadata.gz: 9d9c1baf7002404750e398f00ffd3c65f66aac5f9405de22a4976bfdc8d6d44e
+  data.tar.gz: b93e96cebf3e341dc1b3f40e47efeadc0ec81931ab30774d3da3ade4bed4c20c
 SHA512:
-  metadata.gz: 818540fa6bb016d026122b544387c60d2060e52f598d5488735fb2fe88a75d2726fbd5aabf6141635323c1d9251727eee1a21335f7a16d0b153a567321b68288
-  data.tar.gz: de7b3add993e1329095c3b39eadb3c220c0f632d4d945a6981fe82449fb8a91e916642080958bbad5440fc902f63b24b60aa1a65a5f72718231958872e07c9d2
+  metadata.gz: a1def65971f0715dde4fc92b4cede1e995a59631f555889093e62e303d7c26accc7735b05ffc9c4d018113478da61e9ebdf09981fd23dccc045982bde2a68de7
+  data.tar.gz: 894fc37b635f4deb25bad08b50db64420c6fb470b76c80d5e9225832c05d2f04fa73371f92a47fc4dd468d4a6d1589b9ae95dfab823d377b65a6be6556b7f492

data/.rspec

@@ -1,2 +1,3 @@
+--require spec_helper
 --format documentation
 --color

data/TODO.md

@@ -1,5 +1,6 @@
 ## TODO
 
-* delete old files
+* encrypt backups
 * restore from backup
+* implement deduplication
 * create archives in `/tmp` folder if location is not specified

data/bin/riserva

@@ -10,8 +10,10 @@ class RiservaExecutable
 
   def call
     case Choice[:perform]
-    when 'backup' then
+    when 'backup'
       Riserva::Commands::Backup.new.call
+    when 'clean'
+      Riserva::Commands::Clean.new.call
     end
   end
 end

data/config/samples/riserva.yml.sample

@@ -8,6 +8,8 @@ folders:
 storage:
   google_drive:
     secrets: 'config/google_drive_secrets.json'
+    days_to_keep: 30
 
   dropbox:
     secrets: 'config/dropbox_secrets.json'
+    days_to_keep: 30

data/lib/riserva.rb

@@ -2,7 +2,9 @@
 
 require 'riserva/version'
 require 'active_support/core_ext/string/inflections'
+require 'active_support/time'
 require 'pathname'
+require 'parallel'
 
 module Riserva
   autoload :Config, 'riserva/config'
@@ -11,6 +13,7 @@ module Riserva
   module Commands
     autoload :ApplicationCommand, 'riserva/commands/application_command'
     autoload :Backup, 'riserva/commands/backup'
+    autoload :Clean, 'riserva/commands/clean'
     autoload :CreateArchive, 'riserva/commands/create_archive'
     autoload :UploadFile, 'riserva/commands/upload_file'
   end
@@ -18,6 +21,7 @@ module Riserva
   module Listeners
     autoload :ApplicationListener, 'riserva/listeners/application_listener'
     autoload :Backup, 'riserva/listeners/backup'
+    autoload :Clean, 'riserva/listeners/clean'
     autoload :CreateArchive, 'riserva/listeners/create_archive'
     autoload :UploadFile, 'riserva/listeners/upload_file'
   end
@@ -31,4 +35,8 @@ module Riserva
   def self.logger
     Riserva::Log.new.logger
   end
+
+  def self.version
+    "Riserva v#{Riserva::VERSION}"
+  end
 end

data/lib/riserva/command_line.rb

@@ -4,8 +4,8 @@ Choice.options do
   option :perform do
     short '-p'
     long '--perform=OPERATION'
-    desc 'Operation to perform, valid options: backup (default)'
-    valid %w[backup]
+    desc 'Operation to perform, valid options: backup (default), clean'
+    valid %w[backup clean]
     default 'backup'
   end
 
@@ -31,7 +31,7 @@ Choice.options do
     long '--version'
     desc 'Show version'
     action do
-      puts "Riserva backup tool v#{Riserva::VERSION}"
+      puts Riserva.version
       exit
     end
   end

data/lib/riserva/commands/backup.rb

@@ -1,46 +1,44 @@
 module Riserva::Commands
   class Backup < ApplicationCommand
-    attr_reader :uploaders
+    MAX_THREADS = 5
 
     def initialize
       super
-      @uploaders = {}
     end
 
     def call
-      create_archives
-      push_to_cloud
+      perform
 
-      success? ? broadcast(:ok) : broadcast(:failed)
+      broadcast(:ok)
+    rescue StandardError => e
+      broadcast(:failed)
+      raise e
     end
 
     private
 
-    def success?
-      @uploaders.values.map { |uploader| uploader.files == archivator.files }.all?
-    end
-
-    def create_archives
-      Riserva::Config.folders { |folder| archivator.call(folder) }
+    def perform
+      Riserva::Config.folders.each do |folder|
+        push_to_cloud archivator.call(folder).files
+      end
     end
 
     def archivator
       @archivator ||= Riserva::Commands::CreateArchive.new
     end
 
-    def push_to_cloud
-      Riserva::Config.storages do |storage|
-        archivator.files.each { |file| upload_file(storage, file) }
+    def push_to_cloud(files)
+      Parallel.map(Riserva::Config.storages, in_threads: MAX_THREADS) do |storage|
+        files.map { |file| upload_file(storage, file) }
       end
     end
 
     def upload_file(storage, file)
-      Riserva.logger.info("Uploading file #{file} to #{storage.title}...")
-
       uploader(storage).call(file)
     end
 
     def uploader(storage)
+      @uploaders ||= {}
       @uploaders[storage.title] ||= Riserva::Commands::UploadFile.new(storage)
     end
   end

data/lib/riserva/commands/clean.rb

@@ -0,0 +1,21 @@
+module Riserva::Commands
+  class Clean < ApplicationCommand
+    def call
+      storages.each do |storage|
+        broadcast(:start_cleaning, storage)
+
+        if storage.clean
+          broadcast(:ok)
+        else
+          broadcast(:failed, storage)
+        end
+      end
+    end
+
+    private
+
+    def storages
+      Riserva::Config.storages
+    end
+  end
+end

data/lib/riserva/commands/create_archive.rb

@@ -1,7 +1,8 @@
 module Riserva::Commands
   class CreateArchive < ApplicationCommand
     def call(path)
-      return broadcast(:invalid) unless super
+      broadcast(:start, "folder: #{path}")
+      return broadcast(:invalid, "`#{@path}` is not a directory") unless super
 
       file = archive_name
       create_archive(file) ? broadcast(:ok, file) : broadcast(:failed)

data/lib/riserva/commands/upload_file.rb

@@ -6,9 +6,10 @@ module Riserva::Commands
     end
 
     def call(path)
+      broadcast(:start, "file: #{path} to #{@storage.title}...")
       return broadcast(:invalid) unless super
 
-      success? ? broadcast(:ok, @path) : broadcast(:failed, @path)
+      success? ? broadcast(:ok, @path, @storage.title) : broadcast(:failed, @path)
     end
 
     private

data/lib/riserva/config.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require 'safe_yaml'
+require 'yaml'
 
 module Riserva
   class Config
     DEFAULT_PATH = 'config/riserva.yml'
 
     def initialize
-      SafeYAML::OPTIONS[:default_mode] = :safe
-
       @path = ENV['RISERVA_CONFIG'] || DEFAULT_PATH
     end
 
@@ -22,14 +20,14 @@ module Riserva
     end
 
     def self.folders
-      read('folders').each { |folder| yield Pathname.new(folder) }
+      read('folders').map { |folder| Pathname.new(folder) }
     end
 
     def self.storages
-      read('storage').keys.each do |storage|
+      read('storage').keys.map do |storage|
         klass = "Riserva::Storage::#{storage.camelize}"
-        yield(klass.safe_constantize.new) unless klass.nil?
-      end
+        klass.safe_constantize&.new
+      end.compact
     end
 
     private

data/lib/riserva/listeners/application_listener.rb

@@ -8,20 +8,31 @@ module Riserva::Listeners
       @files = Set.new
     end
 
-    def ok(file)
+    def start(*args)
+      Riserva.logger.info(progname) { ['Starting', *args].join(': ') }
+    end
+
+    def ok(file, *args)
       @files << Pathname.new(file)
 
-      Riserva.logger.info(progname) { "OK: #{file}" }
+      Riserva.logger.info(progname) { ["OK: #{file}", *args].join(': ') }
     end
 
-    def failed
-      Riserva.logger.error(progname) { 'Failed' }
+    def invalid(message = nil)
+      Riserva.logger.debug(progname) { ['Invalid', message].compact.join(': ') }
+    end
+
+    def failed(file = nil)
+      Riserva.logger.error(progname) { ['Failed', file].compact.join(': ') }
     end
 
     protected
 
     def progname
-      self.class.name.split('::').last
+      command = self.class.name.split('::').last
+      worker = Parallel.worker_number.presence
+
+      [command, worker].compact.join(' ')
     end
   end
 end

data/lib/riserva/listeners/backup.rb

@@ -6,7 +6,7 @@ module Riserva::Listeners
   class Backup < ApplicationListener
     def initialize
       notify('Starting backup...')
-      Riserva.logger.info('Starting backup...')
+      Riserva.logger.info(Riserva.version) { 'Starting backup...' }
     end
 
     def ok

data/lib/riserva/listeners/clean.rb

@@ -0,0 +1,19 @@
+module Riserva::Listeners
+  class Clean < ApplicationListener
+    def initialize
+      Riserva.logger.info('Starting cleanup...')
+    end
+
+    def ok
+      Riserva.logger.info('OK')
+    end
+
+    def failed(storage)
+      Riserva.logger.info("Failed: #{storage.title}")
+    end
+
+    def start_cleaning(storage)
+      Riserva.logger.info("Deleting old files on #{storage.title}...")
+    end
+  end
+end

data/lib/riserva/storage/application_storage.rb

@@ -17,5 +17,12 @@ module Riserva::Storage
     def config_secrets
       Riserva::Config.read([:storage, title, :secrets].join('.'))
     end
+
+    def time_to_keep
+      value = Riserva::Config.read([:storage, title, :days_to_keep].join('.'))
+      return unless value
+
+      value.days
+    end
   end
 end

data/lib/riserva/storage/dropbox.rb

@@ -26,6 +26,16 @@ module Riserva::Storage
       data.content_hash == checksum
     end
 
+    def clean
+      return unless time_to_keep
+
+      session.list_folder('').entries.each do |file|
+        next unless file.client_modified < time_to_keep.ago
+
+        session.delete file.path_lower
+      end
+    end
+
     private
 
     def session

data/lib/riserva/storage/google_drive.rb

@@ -20,6 +20,14 @@ module Riserva::Storage
       file.md5_checksum == checksum
     end
 
+    def clean
+      return unless time_to_keep
+
+      session.files(q: ['createdTime < ?', time_to_keep.ago]).each do |file|
+        file.delete(true)
+      end
+    end
+
     private
 
     def session

data/lib/riserva/version.rb

@@ -1,3 +1,3 @@
 module Riserva
-  VERSION = '0.1.2'
+  VERSION = '0.1.7'
 end

data/riserva.gemspec

@@ -21,10 +21,10 @@ Gem::Specification.new do |spec|
   spec.executables << 'riserva'
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'guard-rspec', '~> 4.7', '>= 4.7.3'
   spec.add_development_dependency 'wisper-rspec', '~> 0.0.3'
 
   spec.add_runtime_dependency 'activesupport', '~> 4.0'
@@ -33,6 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'dropbox_content_hasher', '~> 0.1.0'
   spec.add_runtime_dependency 'google_drive', '~> 2.1', '>= 2.1.5'
   spec.add_runtime_dependency 'notifier', '~> 0.5.2'
-  spec.add_runtime_dependency 'safe_yaml', '~> 1.0', '>= 1.0.4'
+  spec.add_runtime_dependency 'parallel', '~> 1.19', '>= 1.19.2'
   spec.add_runtime_dependency 'wisper', '~> 2.0'
 end

metadata

@@ -1,77 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: riserva
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.7
 platform: ruby
 authors:
 - Igor Malinovskiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-12 00:00:00.000000000 Z
+date: 2020-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: wisper-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -177,25 +171,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.5.2
 - !ruby/object:Gem::Dependency
-  name: safe_yaml
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.19'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.4
+        version: 1.19.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.19'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.4
+        version: 1.19.2
 - !ruby/object:Gem::Dependency
   name: wisper
   requirement: !ruby/object:Gem::Requirement
@@ -223,7 +217,6 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
-- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -238,11 +231,13 @@ files:
 - lib/riserva/command_line.rb
 - lib/riserva/commands/application_command.rb
 - lib/riserva/commands/backup.rb
+- lib/riserva/commands/clean.rb
 - lib/riserva/commands/create_archive.rb
 - lib/riserva/commands/upload_file.rb
 - lib/riserva/config.rb
 - lib/riserva/listeners/application_listener.rb
 - lib/riserva/listeners/backup.rb
+- lib/riserva/listeners/clean.rb
 - lib/riserva/listeners/create_archive.rb
 - lib/riserva/listeners/upload_file.rb
 - lib/riserva/log.rb
@@ -271,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Backup files to cloud drive

data/Guardfile

@@ -1,70 +0,0 @@
-# A sample Guardfile
-# More info at https://github.com/guard/guard#readme
-
-## Uncomment and set this to only include directories you want to watch
-# directories %w(app lib config test spec features) \
-#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
-
-## Note: if you are using the `directories` clause above and you are not
-## watching the project directory ('.'), then you will want to move
-## the Guardfile to a watched dir and symlink it back, e.g.
-#
-#  $ mkdir config
-#  $ mv Guardfile config/
-#  $ ln -s config/Guardfile .
-#
-# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
-
-# Note: The cmd option is now required due to the increasing number of ways
-#       rspec may be run, below are examples of the most common uses.
-#  * bundler: 'bundle exec rspec'
-#  * bundler binstubs: 'bin/rspec'
-#  * spring: 'bin/rspec' (This will use spring if running and you have
-#                          installed the spring binstubs per the docs)
-#  * zeus: 'zeus rspec' (requires the server to be started separately)
-#  * 'just' rspec: 'rspec'
-
-guard :rspec, cmd: "bundle exec rspec" do
-  require "guard/rspec/dsl"
-  dsl = Guard::RSpec::Dsl.new(self)
-
-  # Feel free to open issues for suggestions and improvements
-
-  # RSpec files
-  rspec = dsl.rspec
-  watch(rspec.spec_helper) { rspec.spec_dir }
-  watch(rspec.spec_support) { rspec.spec_dir }
-  watch(rspec.spec_files)
-
-  # Ruby files
-  ruby = dsl.ruby
-  dsl.watch_spec_files_for(ruby.lib_files)
-
-  # Rails files
-  rails = dsl.rails(view_extensions: %w(erb haml slim))
-  dsl.watch_spec_files_for(rails.app_files)
-  dsl.watch_spec_files_for(rails.views)
-
-  watch(rails.controllers) do |m|
-    [
-      rspec.spec.call("routing/#{m[1]}_routing"),
-      rspec.spec.call("controllers/#{m[1]}_controller"),
-      rspec.spec.call("acceptance/#{m[1]}")
-    ]
-  end
-
-  # Rails config changes
-  watch(rails.spec_helper)     { rspec.spec_dir }
-  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
-  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
-
-  # Capybara features specs
-  watch(rails.view_dirs)     { |m| rspec.spec.call("features/#{m[1]}") }
-  watch(rails.layouts)       { |m| rspec.spec.call("features/#{m[1]}") }
-
-  # Turnip features and steps
-  watch(%r{^spec/acceptance/(.+)\.feature$})
-  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
-    Dir[File.join("**/#{m[1]}.feature")][0] || "spec/acceptance"
-  end
-end