activestorage-sftp 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 53fb9c020a024d48787fccfbdd311b88f1eef5b6
-  data.tar.gz: 396a0b2ffc2cde6bb0f88bf169cdbf2932110855
+  metadata.gz: 86f61c010b69d4972dbd32fd3eeec9ad415d5051
+  data.tar.gz: 3d82a1f0208e565c98a791506c8c11619faa7a4e
 SHA512:
-  metadata.gz: 930f62d8856cfda94bfa90285ab6c59fc5d3d4ccdd14ce851a2dece838e5f78114438c87195d2c467738aa01031be7eea591e1f7244f40222507aaa52f30ffbe
-  data.tar.gz: 5a7f980c04c712750beec2f4a328bc676d0be570cc9c52854c46c2e053d0f99fe69f6a47b955fb3157eb554d5efff98579d9bf0b1cb14099ce8718d2b57689b1
+  metadata.gz: 6ccd930d3aa5c2cd6f26dd6476b64000e935e1984f01536efd7abd46191f587ed48defdeac2f74efe3ee7155d1a45fe79ce14034613ac3b6c84002192bbca61c
+  data.tar.gz: cd1c9b1f1983508558e833af339105e2089adb47e23f70b4903c775beac5d75a2bc67c06d844908c15fafebfa183debce548b0df38d60b05f5459e83d81d286b

data/README.md

@@ -20,8 +20,8 @@ Or install it yourself as:
 
 ## Usage
 
-
 Each application server saves blobs to file server through SFTP:
+
 ```yml
 # config/storage.yml
 sftp:
@@ -30,7 +30,9 @@ sftp:
   root: /var/www/proj/shared/storage
   host: file.intranet
   public_host: https://file.internet
+  password: <%= ENV['PASSWORD'] %> # optional
 ```
+
 File server serves blobs using DiskService:
 ```yml
 # config/storage.yml
@@ -54,6 +56,36 @@ sftp:
   host: secure.backup
 ```
 
+### Further configuration options:
+
+#### use_public_url: Generate plain ("dumb") URLs of upload server
+
+By default the generated URLs will include parameters for `content_disposition`, expiration hints etc.  A generated blobs URL might thus look like:
+
+    https://publichost/PATH/rails/active_storage/disk/hash-hash/name.JPG?content_type=image%2Fjpeg&disposition=inline%3B+filename%3D
+
+If you prefer simple URLs like
+
+    https://publichost/PATH/hash
+
+you can set a configuration option:
+
+```yml
+# config/storage.yml
+sftp:
+  simple_public_urls: true # defaults to false
+```
+
+
+#### verify_via_http_get: Faster existence verification via HTTP GET
+
+The default way of verifying that a blob does exist is to login to the sftp server and stat() the relevant file.  This is done e.g. before re-transforming and uploading an image variant.  While other "caching" solutions exist to speed up that process, a simple and efficient way of verifying the existence of a file is to query the relevant server with an HTTP HEAD request.  Depending on the setup this might not always be a viable way, so it can be switched on with a configuration option.
+
+```yml
+# config/storage.yml
+sftp:
+  verify_via_http_get: true # defaults to false
+```
 
 ## Contributing
 

data/activestorage-sftp.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "rails", "> 5.2.0"
-  spec.add_dependency "net-sftp", "~> 2.1.2"
+  spec.add_dependency "rails", ">= 5.2.0"
+  spec.add_dependency "net-sftp", ">= 2.1.2"
 
   spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/active_storage/service/sftp_service.rb

@@ -11,12 +11,15 @@ module ActiveStorage
 
     attr_reader :host, :user, :root, :public_host, :public_root
 
-    def initialize(host:, user:, public_host: nil, root: './', public_root: nil)
+    def initialize(host:, user:, public_host: nil, root: './', public_root: './', password: nil, simple_public_urls: false, verify_via_http_get: false)
       @host = host
       @user = user
       @root = root
       @public_host = public_host
       @public_root = public_root
+      @password = password
+      @simple_public_urls = simple_public_urls
+      @verify_via_http_get = verify_via_http_get
     end
 
     def upload(key, io, checksum: nil, **)
@@ -116,37 +119,76 @@ module ActiveStorage
     def exist?(key)
       instrument :exist, key: key do |payload|
         answer = false
-        through_sftp do |sftp|
-          answer = sftp.stat(path_for(key)).present?
+
+        if @verify_via_http_get
+          uri = URI([public_host, relative_folder_for(key), key].join('/'))
+          request = Net::HTTP.new uri.host
+          response = request.request_head uri.path
+
+          answer = (response.code.to_i == 200)
+        else
+          through_sftp do |sftp|
+            # TODO Probably adviseable to let some more exceptions go through
+            begin
+              sftp.stat!(path_for(key)) do |response|
+                answer = response.ok?
+              end
+            rescue Net::SFTP::StatusException => e
+              answer = false
+            end
+          end
         end
+
         payload[:exist] = answer
         answer
       end
     end
 
     def url(key, expires_in:, filename:, disposition:, content_type:)
-        instrument :url, key: key do |payload|
-          raise "public_host not defined." unless public_host
-          content_disposition = content_disposition_with(type: disposition, filename: filename)
-          verified_key_with_expiration = ActiveStorage.verifier.generate(
-              {
-                  key: key,
-                  disposition: content_disposition,
-                  content_type: content_type
-              },
-              { expires_in: expires_in,
-                purpose: :blob_key }
-          )
-
-          generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration,
-                                                             host: public_host,
-                                                             disposition: content_disposition,
-                                                             content_type: content_type,
-                                                             filename: filename
-          )
-          payload[:url] = generated_url
-
-          generated_url
+      if @simple_public_urls
+        public_url(key)
+      else
+        classic_url(key,
+                    expires_in: expires_in,
+                    filename: filename,
+                    disposition: disposition,
+                    content_type: content_type)
+      end
+    end
+
+    def classic_url(key, expires_in:, filename:, disposition:, content_type:)
+      instrument :url, key: key do |payload|
+        raise NotConfigured, "public_host not defined." unless public_host
+        content_disposition = content_disposition_with(type: disposition, filename: filename)
+        verified_key_with_expiration = ActiveStorage.verifier.generate(
+          {
+            key: key,
+            disposition: content_disposition,
+            content_type: content_type
+          },
+          {
+            expires_in: expires_in,
+            purpose: :blob_key
+          }
+        )
+
+        generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration,
+                                                           host: public_host,
+                                                           disposition: content_disposition,
+                                                           content_type: content_type,
+                                                           filename: filename
+        )
+        payload[:url] = generated_url
+        generated_url
+      end
+    end
+
+    def public_url(key)
+      instrument :url, key: key do |payload|
+        raise NotConfigured, "public_host not defined." unless public_host
+        generated_url = File.join(public_host, public_root, path_for(key), key)
+        payload[:url] = generated_url
+        generated_url
       end
     end
 
@@ -183,7 +225,8 @@ module ActiveStorage
 
     protected
       def through_sftp(&block)
-        Net::SFTP.start(@host, @user) do |sftp|
+        opts = @password.present? ? {password: @password} : {}
+        Net::SFTP.start(@host, @user, opts) do |sftp|
           block.call(sftp)
         end
       end
@@ -197,18 +240,20 @@ module ActiveStorage
       end
 
       def mkdir_for(key)
+        mkdir_p_for(path_for key)
+      end
+
+      def mkdir_p_for(abs_path)
         through_sftp do |sftp|
-          sub_folder = File.join root, key[0..1]
-          begin
-            sftp.opendir!(sub_folder)
-          rescue => e
-            sftp.mkdir!(sub_folder)
-          end
-          sub_folder = File.join(sub_folder, key[2..3])
-          begin
-            sftp.opendir!(sub_folder)
-          rescue => e
-            sftp.mkdir!(sub_folder)
+          base_path = ''
+          abs_path.split('/')[0...-1].each do |path|
+            sub_folder = File.join(base_path, path)
+            begin
+              sftp.opendir!(sub_folder)
+            rescue => e
+              sftp.mkdir!(sub_folder)
+            end
+            base_path = sub_folder
           end
         end
       end

data/lib/active_storage/sftp.rb

@@ -6,5 +6,8 @@ module ActiveStorage
     class Error < StandardError; end
     class ChunkSizeError < Error; end
     class SFTPResponseError < Error; end
+    class NotConfigured < Error; end
   end
 end
+
+

data/lib/active_storage/sftp/version.rb

@@ -1,5 +1,5 @@
 module ActiveStorage
   module SFTP
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: activestorage-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - treenewbee
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-16 00:00:00.000000000 Z
+date: 2019-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: net-sftp
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.2
 - !ruby/object:Gem::Dependency