webtranslateit-safe 0.4.0

data/lib/webtranslateit/safe/config/node.rb

@@ -0,0 +1,79 @@
+require 'webtranslateit/safe/config/builder'
+module WebTranslateIt
+
+  module Safe
+
+    module Config
+
+      class Node
+
+        attr_reader :parent, :data
+
+        def initialize(parent = nil, data = {}, &)
+          @parent = parent
+          @data = {}
+          merge(data, &)
+        end
+
+        def merge(data = {}, &block)
+          builder = Builder.new(self, data)
+          builder.instance_eval(&block) if block
+          self
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if !value.nil? && path.empty?
+
+          value&.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent&.find(*path)
+        end
+        alias [] find
+
+        def set_multi(key, value)
+          @data[key.to_s] ||= []
+          @data[key.to_s].push(*value)
+        end
+
+        def set(key, value)
+          @data[key.to_s] = value
+        end
+        alias []= set
+
+        def each(&)
+          @data.each(&)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.each_with_object({}) do |key, res|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+          end
+        end
+
+        def dump(indent = '')
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump("#{indent}    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/ftp.rb

@@ -0,0 +1,85 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Ftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:ftp, :path] || config[:local, :path] || ':kind/:id')
+      end
+
+      def save
+        raise 'pipe-streaming not supported for FTP.' unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
+
+        return if dry_run? || local_only?
+
+        port ||= 21
+        Net::FTP.open(host) do |ftp|
+          ftp.connect(host, port)
+          ftp.login(user, password)
+          puts "Sending #{@backup.path} to #{full_path}" if verbose?
+          begin
+            ftp.put(@backup.path, full_path)
+          rescue Net::FTPPermError
+            puts "Ensuring remote path (#{path}) exists" if verbose?
+          end
+        end
+        puts '...done' if verbose?
+      end
+
+      def cleanup
+        return if local_only? || dry_run?
+
+        return unless (keep = config[:keep, :ftp])
+
+        puts "listing files: #{host}:#{base}*" if verbose?
+        port ||= 21
+        Net::FTP.open(host) do |ftp|
+          ftp.connect(host, port)
+          ftp.login(user, password)
+          files = ftp.nlst(path)
+          pattern = File.basename(base.to_s)
+          files = files.select { |x| x.start_with?(pattern) }
+          puts(files.collect { |x| x }) if verbose?
+
+          files = files
+                  .collect { |x| x }
+                  .sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
+            ftp.delete(file) unless dry_run? || local_only?
+          end
+        end
+      end
+
+      def host
+        config[:ftp, :host]
+      end
+
+      def user
+        config[:ftp, :user]
+      end
+
+      def password
+        config[:ftp, :password]
+      end
+
+      def port
+        config[:ftp, :port]
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/gpg.rb

@@ -0,0 +1,52 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Gpg < Pipe
+
+      def active?
+        raise "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        command = config[:gpg, :command] || 'gpg'
+        if key
+          "|#{command} #{config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|#{command} #{config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        '.gpg'
+      end
+
+      private
+
+      def password
+        @password ||= config[:gpg, :password]
+      end
+
+      def key
+        @key ||= config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return 'TEMP_GENERATED_FILENAME' if dry_run?
+
+        WebTranslateIt::Safe::TmpFile.create('gpg-pass') { |file| file.write(pass) }
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/gzip.rb

@@ -0,0 +1,29 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        '|gzip'
+      end
+
+      def extension
+        '.gz'
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/local.rb

@@ -0,0 +1,55 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Local < Sink
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH/FTP sink
+        true
+      end
+
+      protected
+
+      def path
+        @path ||= File.expand_path(expand(config[:local, :path] || raise('missing :local/:path')))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if verbose?
+
+        # FIXME: probably need to change this to smth like @backup.finalize!
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+
+        return if dry_run?
+
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+        benchmark = Benchmark.realtime do
+          system "#{@backup.command}>#{@backup.path}"
+        end
+        puts("command took #{format('%.2f', benchmark)} second(s).") if verbose?
+      end
+
+      def cleanup
+        return unless (keep = config[:keep, :local])
+
+        puts "listing files #{base}" if verbose?
+
+        # TODO: cleanup ALL zero-length files
+
+        files = Dir["#{base}*"]
+                .select { |f| File.file?(f) && File.size(f).positive? }
+                .sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if dry_run? || verbose?
+          File.unlink(f) unless dry_run?
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/mongodump.rb

@@ -0,0 +1,30 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Mongodump < Source
+
+
+      def command
+        opts = []
+        opts << "--host #{config[:host]}" if config[:host]
+        opts << "-u #{config[:user]}" if config[:user]
+        opts << "-p #{config[:password]}" if config[:password]
+        opts << "--out #{output_directory}"
+
+        "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(' ')} && cd #{output_directory} && tar cf - ."
+      end
+
+      def extension = '.tar'
+
+      protected
+
+      def output_directory
+        File.join(TmpFile.tmproot, 'mongodump')
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/mysqldump.rb

@@ -0,0 +1,36 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Mysqldump < Source
+
+      def command
+        "mysqldump --defaults-extra-file=#{mysql_password_file} #{config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension = '.sql'
+
+      protected
+
+      def mysql_password_file
+        WebTranslateIt::Safe::TmpFile.create('mysqldump') do |file|
+          file.puts '[mysqldump]'
+          %w[user password socket host port].each do |k|
+            v = config[k]
+            # values are quoted if needed
+            file.puts "#{k} = #{v.inspect}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        return unless (skip_tables = config[:skip_tables])
+
+        [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" }.join(' ')
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/pgdump.rb

@@ -0,0 +1,36 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Pgdump < Source
+
+      def command
+        ENV['PGPASSWORD'] = (config['password'] || nil)
+        "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
+      end
+
+      def extension = '.sql'
+
+      protected
+
+      def postgres_options
+        config[:options]
+      end
+
+      def postgres_host
+        config['host'] && "--host='#{config['host']}'"
+      end
+
+      def postgres_port
+        config['port'] && "--port='#{config['port']}'"
+      end
+
+      def postgres_username
+        config['user'] && "--username='#{config['user']}'"
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/pipe.rb

@@ -0,0 +1,23 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Pipe < Stream
+
+      # process adds required commands to the current
+      # shell command string
+      # :active?, :pipe, :extension and :post_process are
+      # defined in inheriting pipe classes
+      def process
+        return unless active?
+
+        @backup.command << pipe
+        @backup.extension << extension
+        post_process
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/s3.rb

@@ -0,0 +1,80 @@
+module WebTranslateIt
+  module Safe
+    class S3 < Sink
+      MAX_S3_FILE_SIZE = 5368709120
+
+      def active?
+        bucket && key && secret
+      end
+
+      protected
+
+      def path
+        @path ||= expand(config[:s3, :path] || config[:local, :path] || ':kind/:id')
+      end
+
+      def save
+        # FIXME: user friendly error here :)
+        raise RuntimeError, 'pipe-streaming not supported for S3.' unless @backup.path
+
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only?
+
+        puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run?
+        unless dry_run? || local_only?
+          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
+            File.open(@backup.path) do |file|
+              AWS::S3::S3Object.store(full_path, file, bucket)
+            end
+          end
+          puts '...done' if verbose?
+          puts('Upload took ' + sprintf('%.2f', benchmark) + ' second(s).') if verbose?
+        end
+      end
+
+      def cleanup
+        return if local_only?
+
+        return unless keep = config[:keep, :s3]
+
+        puts "listing files: #{bucket}:#{base}*" if verbose?
+        files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if verbose?
+
+        files = files.
+          collect {|x| x.key}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose?
+          AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only?
+        end
+      end
+
+      def bucket
+        config[:s3, :bucket]
+      end
+
+      def key
+        config[:s3, :key]
+      end
+
+      def secret
+        config[:s3, :secret]
+      end
+
+      private
+      
+      def bucket_exists?(bucket)
+        true if AWS::S3::Bucket.find(bucket)
+      rescue AWS::S3::NoSuchBucket
+        false
+      end
+    end
+  end
+end

data/lib/webtranslateit/safe/sftp.rb

@@ -0,0 +1,96 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ':kind/:id')
+      end
+
+      def save
+        raise 'pipe-streaming not supported for SFTP.' unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
+
+        return if dry_run? || local_only?
+
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          puts "Sending #{@backup.path} to #{full_path}" if verbose?
+          begin
+            sftp.upload! @backup.path, full_path
+          rescue Net::SFTP::StatusException
+            puts "Ensuring remote path (#{path}) exists" if verbose?
+            # mkdir -p
+            folders = path.split('/')
+            folders.each_index do |i|
+              folder = folders[0..i].join('/')
+              puts "Creating #{folder} on remote" if verbose?
+              begin
+                sftp.mkdir!(folder)
+              rescue StandardError
+                Net::SFTP::StatusException
+              end
+            end
+            retry
+          end
+        end
+        puts '...done' if verbose?
+      end
+
+      def cleanup
+        return if local_only? || dry_run?
+
+        return unless (keep = config[:keep, :sftp])
+
+        puts "listing files: #{host}:#{base}*" if verbose?
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts(files.collect(&:name)) if verbose?
+
+          files = files
+                  .collect(&:name)
+                  .sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
+            sftp.remove!(file) unless dry_run? || local_only?
+          end
+        end
+      end
+
+      def host
+        config[:sftp, :host]
+      end
+
+      def user
+        config[:sftp, :user]
+      end
+
+      def password
+        config[:sftp, :password]
+      end
+
+      def port
+        config[:sftp, :port]
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/sink.rb

@@ -0,0 +1,40 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Sink < Stream
+
+      def process
+        return unless active?
+
+        save
+        cleanup
+      end
+
+      protected
+
+      # path is defined in subclass
+      # base is used in 'cleanup' to find all files that begin with base. the '.'
+      # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
+      def base
+        @base ||= File.join(path, "#{File.basename(@backup.filename).split('.').first}.")
+      end
+
+      def full_path
+        @full_path ||= File.join(path, @backup.filename) + @backup.extension
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&)
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/source.rb

@@ -0,0 +1,51 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Source < Stream
+
+      attr_accessor :id
+
+      def initialize(id, config)
+        @id = id.to_s
+        @config = config
+      end
+
+      def timestamp
+        Time.now.strftime('%y%m%d-%H%M')
+      end
+
+      def kind
+        self.class.human_name
+      end
+
+      def filename
+        @filename ||= expand(':kind-:id.:timestamp')
+      end
+
+      def backup
+        return @backup if @backup
+
+        @backup = Backup.new(
+          id: @id,
+          kind: kind,
+          extension: extension,
+          command: command,
+          timestamp: timestamp
+        )
+        # can't do this in the initializer hash above since
+        # filename() calls expand() which requires @backup
+        # FIXME: move expansion to the backup (last step in ctor) assign :tags here
+        @backup.filename = filename
+        @backup
+      end
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+    end
+
+  end
+
+end