activestorage-sftp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0c1cad83fd43e3324294dd2c0081f8b61a76c003
+  data.tar.gz: 99d47be472b87031a8d506d0630b35e45b5d1061
+SHA512:
+  metadata.gz: 4e73d5a932d849c9a901a4065950ec1600fe1a2cb1446bd80f6f99239a3c59f4c53e81054eeb57f86f43e1f81897e60830d8cccd74abfa14380eaaa7b31781f8
+  data.tar.gz: 20732912bfbf3332da9a2264d9bd7179cc94dee9533be29f3c71402bc82afa4f25b69d67c948f34bb33cd1cc070f31653eac80ecb5026827375eb1c6b86a85a6

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in activestorage-sftp.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 treenewbee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+Remote DiskService through SFTP, for ActiveStorage. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'activestorage-sftp'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install activestorage-sftp
+
+## Usage
+
+
+Each application server saves blobs to file server through SFTP:
+```yml
+# config/storage.yml
+sftp:
+  service: SFTP
+  user: user
+  root: /var/www/proj/shared/storage
+  host: file.intranet
+  public_host: https://file.internet
+```
+File server serves blobs using DiskService:
+```yml
+# config/storage.yml
+local:
+  service: Disk
+  root: <%= Rails.root.join("storage") %>
+```
+
+Or use it as backup for your primary service:
+```yml
+# config/storage.yml
+mirrored:
+  service: Mirror
+  primary: local #/S3/AzureStorage/GCS
+  mirrors:
+    - sftp
+sftp:
+  service: SFTP
+  user: user
+  root: /etc/backup/proj
+  host: secure.backup
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/treenewbee/activestorage-sftp.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/activestorage-sftp.gemspec

@@ -0,0 +1,31 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "active_storage/sftp/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "activestorage-sftp"
+  spec.version       = ActiveStorage::SFTP::VERSION
+  spec.authors       = ["treenewbee"]
+  spec.email         = ["yangguchen@gmail.com"]
+
+  spec.summary       = %q{SFTP Service for ActiveStorage}
+  spec.description   = %q{SFTP Service for ActiveStorage}
+  spec.homepage      = "https://github.com/treenewbee/activestorage-sftp"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", "> 5.2.0"
+  spec.add_dependency "net-sftp", "~> 2.1.2"
+
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "activestorage/sftp"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_storage/service/sftp_service.rb

@@ -0,0 +1,229 @@
+require "net/sftp"
+require "digest/md5"
+require "active_support/core_ext/numeric/bytes"
+
+module ActiveStorage
+  # Wraps a remote path as an Active Storage service. See ActiveStorage::Service for the generic API
+  # documentation that applies to all services.
+  class Service::SFTPService < Service
+
+    MAX_CHUNK_SIZE = 64.kilobytes.freeze
+
+    attr_reader :host, :user, :root, :public_host
+
+    def initialize(host:, user:, public_host: nil, root: './')
+      @host = host
+      @user = user
+      @root = root
+      @public_host = public_host
+    end
+
+    def upload(key, io, checksum: nil, **)
+      instrument :upload, key: key, checksum: checksum do
+        ensure_integrity_of(io, checksum) if checksum
+        mkdir_for(key)
+        through_sftp do |sftp|
+          sftp.upload!(io.path, path_for(key))
+        end
+      end
+    end
+
+    def download(key, chunk_size: MAX_CHUNK_SIZE, &block)
+      if chunk_size > MAX_CHUNK_SIZE
+        # TODO Error
+        raise "Maximum chunk size: #{MAX_CHUNK_SIZE}"
+      end
+      if block_given?
+        instrument :streaming_download, key: key do
+          through_sftp do |sftp|
+            file = sftp.open!(path_for(key))
+            buf = StringIO.new
+            pos = 0
+            eof = false
+            until eof do
+              request = sftp.read(file, pos, chunk_size) do |response|
+                if response.eof?
+                  eof = true
+                elsif !response.ok?
+                  # TODO Error
+                  raise "SFTP Response Error"
+                else
+                  chunk = response[:data]
+                  block.call(chunk)
+                  buf << chunk
+                  pos += chunk.size
+                end
+              end
+              request.wait
+            end
+            sftp.close(file)
+            buf.string
+          end
+        end
+      else
+        instrument :download, key: key do
+          io = StringIO.new
+          through_sftp do |sftp|
+            sftp.download!(path_for(key), io)
+          end
+          io.string
+        rescue Errno::ENOENT
+          raise ActiveStorage::FileNotFoundError
+        end
+      end
+    end
+
+    def download_chunk(key, range)
+      instrument :download_chunk, key: key, range: range do
+        if range.size > MAX_CHUNK_SIZE
+          # TODO Error
+          raise "Maximun chunk size: #{MAX_CHUNK_SIZE}"
+        end
+        chunk = StringIo.new
+        through_sftp do |sftp|
+          sftp.open(path_for(key)) do |file|
+            chunk << sftp.read(file, range.begin, ranage.size).response[:data]
+          end
+        end
+        chunk.string
+      rescue Errno::ENOENT
+        raise ActiveStorage::FileNotFoundError
+      end
+    end
+
+    def delete(key)
+      instrument :delete, key: key do
+        through_sftp do |sftp|
+          sftp.remove!(path_for(key))
+        end
+      rescue Net::SFTP::StatusException
+        # Ignore files already deleted
+      end
+    end
+
+    def delete_prefixed(prefix)
+      instrument :delete_prefixed, prefix: prefix do
+        through_sftp do |sftp|
+          sftp.dir.glob(root, "#{prefix}*") do |entry|
+            begin
+              sftp.remove!(entry.path)
+            rescue Net::SFTP::StatusException
+              # Ignore files already deleted
+            end
+          end
+        end
+      end
+    end
+
+    def exist?(key)
+      instrument :exist, key: key do |payload|
+        answer = false
+        through_sftp do |sftp|
+          answer = sftp.stat(path_for(key)).present?
+        end
+        payload[:exist] = answer
+        answer
+      end
+    end
+
+    def url(key, expires_in:, filename:, disposition:, content_type:)
+        instrument :url, key: key do |payload|
+          raise "public_host not defined." unless public_host
+          content_disposition = content_disposition_with(type: disposition, filename: filename)
+          verified_key_with_expiration = ActiveStorage.verifier.generate(
+              {
+                  key: key,
+                  disposition: content_disposition,
+                  content_type: content_type
+              },
+              { expires_in: expires_in,
+                purpose: :blob_key }
+          )
+
+          generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration,
+                                                             host: public_host,
+                                                             disposition: content_disposition,
+                                                             content_type: content_type,
+                                                             filename: filename
+          )
+          payload[:url] = generated_url
+
+          generated_url
+      end
+    end
+
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+      instrument :url, key: key do |payload|
+        verified_token_with_expiration = ActiveStorage.verifier.generate(
+            {
+                key: key,
+                content_type: content_type,
+                content_length: content_length,
+                checksum: checksum
+            },
+            { expires_in: expires_in,
+              purpose: :blob_token }
+        )
+
+        generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration,
+                                                                  host: public_host
+        )
+
+        payload[:url] = generated_url
+
+        generated_url
+      end
+    end
+
+    def headers_for_direct_upload(key, content_type:, **)
+      { "Content-Type" => content_type }
+    end
+
+    protected
+      def through_sftp(&block)
+        Net::SFTP.start(@host, @user) do |sftp|
+          block.call(sftp)
+        end
+      end
+
+      def path_for(key)
+        File.join folder_for(key), key
+      end
+
+      def folder_for(key)
+        File.join root, relative_folder_for(key)
+      end
+
+      def relative_folder_for(key)
+        [ key[0..1], key[2..3] ].join("/")
+      end
+
+      def mkdir_for(key)
+        through_sftp do |sftp|
+          sub_folder = File.join root, key[0..1]
+          begin
+            sftp.opendir!(sub_folder)
+          rescue => e
+            sftp.mkdir!(sub_folder)
+          end
+          sub_folder = File.join(sub_folder, key[2..3])
+          begin
+            sftp.opendir!(sub_folder)
+          rescue => e
+            sftp.mkdir!(sub_folder)
+          end
+        end
+      end
+
+      def ensure_integrity_of(io, checksum)
+        unless Digest::MD5.new.update(io.read).base64digest == checksum
+          delete key
+          raise ActiveStorage::IntegrityError
+        end
+      end
+
+      def url_helpers
+        @url_helpers ||= Rails.application.routes.url_helpers
+      end
+  end
+end

data/lib/active_storage/sftp.rb

@@ -0,0 +1,9 @@
+require "active_storage/sftp/version"
+require "active_storage/service/sftp_service"
+
+module ActiveStorage
+  module SFTP
+    class Error < StandardError; end
+    # Your code goes here...
+  end
+end

data/lib/active_storage/sftp/version.rb

@@ -0,0 +1,5 @@
+module ActiveStorage
+  module SFTP
+    VERSION = "0.1.0"
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: activestorage-sftp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- treenewbee
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: SFTP Service for ActiveStorage
+email:
+- yangguchen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- activestorage-sftp.gemspec
+- bin/console
+- bin/setup
+- lib/active_storage/service/sftp_service.rb
+- lib/active_storage/sftp.rb
+- lib/active_storage/sftp/version.rb
+homepage: https://github.com/treenewbee/activestorage-sftp
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.4
+signing_key: 
+specification_version: 4
+summary: SFTP Service for ActiveStorage
+test_files: []