rest-ftp-daemon-remote-s3 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31cf8655302c112d563c71b02e8ec94c2a61b801
+  data.tar.gz: 055ed157321fc8b564b0daa2c4db4b36740dd508
+SHA512:
+  metadata.gz: 2abd8ecb845d5725357cf2dc637474dfe6f2fccaa4112d3dda8b0886567ea478f87033f8e221c6047f0046e76f07302fc0ca20928583864e9a6a56baa0cc4c5e
+  data.tar.gz: 4e0ccea3de1e1f5e9f675395d69335efa3c09fd472e6af45897374e7fe710818f42d32f04bcb029acf9c7882db999f5d98a712d30ccfd5a336b1d25bd609e6b5

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1 @@
+# rest-ftp-daemon-remote-s3

data/Rakefile

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+require "rubygems"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.fail_on_error = false
+end
+
+# Load my own tasks
+Dir.glob('lib/tasks/*.rake').each { |r| import r }
+
+# Run specs by default
+desc "Run all tests"
+task default: [:spec, :rubocop]

data/lib/plugins/rest-ftp-daemon/remote/remote_s3.rb

@@ -0,0 +1,175 @@
+# Dependencies
+require 'aws-sdk-resources'
+
+# Register this handler
+module URI
+  class S3 < Generic; end
+  @@schemes["S3"]     = S3
+end
+
+# Handle sFTP transfers for Remote class
+module RestFtpDaemon::Remote
+  class RemoteS3 < RemoteBase
+
+    # Defaults
+    S3_MIN_PART         = 5*MB
+    S3_MAX_COUNT        = 10_000
+
+    # Class options
+    attr_reader :client
+    attr_reader :target
+
+    # URI schemes handled by this plugin
+    def self.handles
+      [URI::S3]
+    end
+
+    def initialize target, job, config
+      super
+    end
+
+    def connect
+      super
+
+      # Connect remote server
+      @client = Aws::S3::Client.new(
+        region: @target.aws_region,
+        credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
+        http_wire_trace: debug_enabled
+        )
+
+    rescue Exception => exception
+      raise RemoteConnectError, "#{exception.class}: #{exception.message}"
+    end
+
+    def size_if_exists target
+      log_debug "size_if_exists rel[#{target.path_rel}]"
+      object = @client.get_object(bucket: target.aws_bucket, key: target.path_rel)
+    rescue Aws::S3::Errors::NotFound => e
+      return false
+    else
+      return object.content_length
+    end
+
+    def push source, target, &callback
+      # Push init
+      raise RestFtpDaemon::AssertionFailed, "push/client" if @client.nil?
+
+      # Do the transfer, passing the file to the best method
+      File.open(source.path_abs, 'r', encoding: 'BINARY') do |file|
+        if file.size >= S3_MIN_PART
+          upload_multipart  file, target.aws_bucket, target.path_rel, target.name, &callback
+        else
+          upload_onefile    file, target.aws_bucket, target.path_rel, target.name, &callback
+        end
+      end
+    end
+
+    def move source, target
+      # Identify the source object
+      # obj = @client.bucket(source.aws_bucket).object(source.path_rel)
+      # raise RestFtpDaemon::AssertionFailed, "move: object not found" unless obj
+
+      # Move the file
+      # log_debug "move: copy bucket[#{source.aws_bucket}] source[#{source.path_rel}] target[#{target.path_rel}]"
+      @client.copy_object(bucket: source.aws_bucket, key: target.path_rel, copy_source: "#{source.aws_bucket}/#{source.path_rel}")
+      # log_debug "move: delete bucket[#{source.aws_bucket}] source[#{source.path_rel}]"
+      @client.delete_object(bucket: source.aws_bucket, key: source.path_rel)
+      # log_debug "move: done"
+
+      # Move the file
+      # obj.move_to(target.path_rel, :bucket_name => target.aws_bucket)
+    end
+
+    def connected?
+      !@client.nil?
+    end
+
+  private
+
+    def upload_onefile file, s3_bucket, s3_path, s3_name, &callback
+      log_debug "push: put_object", {
+        s3_bucket:    s3_bucket,
+        s3_path:      s3_path,
+        }
+      @client.put_object(bucket: s3_bucket, key: s3_path, body: file)
+    end
+
+    def upload_multipart file, s3_bucket, s3_path, s3_name, &callback
+      # Init
+      current_part  = 1
+
+      # Compute parameters
+      file_size     = file.size
+      parts_size    = compute_parts_size(file_size)
+      parts_count   = (file_size.to_f / parts_size).ceil
+
+      # Prepare basic opts
+      options = {
+        bucket: s3_bucket,
+        key:    s3_path,
+        }
+
+      # Declare multipart upload
+      mpu_create_response = @client.create_multipart_upload(options)
+      options[:upload_id] = mpu_create_response.upload_id
+      log_debug "push: create_multipart_upload", {
+        s3_bucket:    s3_bucket,
+        s3_path:      s3_path,
+        upload_id:    options[:upload_id],
+        file_size:    format_bytes(file_size, "B"),
+        parts_size:   format_bytes(parts_size, "B"),
+        parts_count:  parts_count
+        }
+
+      # Upload each part
+      file.each_part(parts_size) do |part|
+        # Prepare part upload
+        opts = options.merge({
+          body:        part,
+          part_number: current_part,
+          })
+        part_size = part.bytesize
+        log_debug "upload_part [#{current_part}/#{parts_count}] part_size[#{part_size}]"
+
+        # Push this over there
+        resp = @client.upload_part(opts)
+        
+        # Send progress info upwards
+        yield part_size, s3_name
+
+        # Increment part number
+        current_part += 1
+      end
+
+      # Retrieve parts and complete upload
+      parts_resp = @client.list_parts(options)
+      those_parts = parts_resp.parts.map do |part| 
+        { part_number: part.part_number, etag: part.etag }
+      end
+      opts = options.merge({
+        multipart_upload: {
+          parts: those_parts
+          } 
+        })
+      log_debug "complete_multipart_upload"
+      mpu_complete_response = @client.complete_multipart_upload(opts)
+    end  
+
+    def compute_parts_size filesize
+      # Initial part size is minimal
+      partsize_mini = S3_MIN_PART
+
+      # Other partsize if too many blocks
+      partsize_bigf = (filesize.to_f / S3_MAX_COUNT).ceil
+
+      # Decide
+      return [partsize_mini, partsize_bigf].max
+    end
+
+    def debug_enabled
+      @config[:debug_s3]
+    end
+
+  end
+end

data/rest-ftp-daemon-remote-s3.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+
+  # Project version
+  spec.version                    = "0.0.1"
+
+  # Project description
+  spec.name                       = "rest-ftp-daemon-remote-s3"
+  spec.authors                    = ["Bruno MEDICI"]
+  spec.email                      = "rftpd-project@bmconseil.com"
+  spec.description                = "rest-ftp-daemon plugin: S3"
+  spec.summary                    = "rest-ftp-daemon plugin: transfer files from/to Amazon S3"
+  spec.homepage                   = "http://github.com/bmedici/#{spec.name}"
+  spec.licenses                   = ["MIT"]
+  spec.date                       = Time.now.strftime("%Y-%m-%d")
+
+  # List files and executables
+  spec.files                      = `git ls-files -z`.split("\x0")
+  spec.executables                = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths              = ["lib"]
+  spec.required_ruby_version      = ">= 2.3"
+
+  # Runtime dependencies
+  spec.add_runtime_dependency     "aws-sdk-resources", '~> 2.6'
+
+  # Development dependencies
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: rest-ftp-daemon-remote-s3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Bruno MEDICI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-resources
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'rest-ftp-daemon plugin: S3'
+email: rftpd-project@bmconseil.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- lib/plugins/rest-ftp-daemon/remote/remote_s3.rb
+- rest-ftp-daemon-remote-s3.gemspec
+homepage: http://github.com/bmedici/rest-ftp-daemon-remote-s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: 'rest-ftp-daemon plugin: transfer files from/to Amazon S3'
+test_files: []