paperclip-storage-ftp 1.2.6 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52dd89bd22677f466873efbf00464db5185f89e7
-  data.tar.gz: cc7bf20e77a50e009cf45b2feca86c311d3451aa
+  metadata.gz: de0a8d8ea7ed9dc2328bf8c6b9d5ff9eaae40f35
+  data.tar.gz: 7da4635ee11a19c26c051e4a5c23229efa00521a
 SHA512:
-  metadata.gz: 854d6376410b4c7e7dbed7d0ad0ecc35aa349c2974618aeef3340c4795e0bcc1a237c68dd4dae433a559cfd69b85860afdd63618dd8cf459c91355dbe9d57eb5
-  data.tar.gz: 7463b456f29a1bc116a4ff64ae34626a90c9682e043f6c3046c613ad7e8d8c6053c833eaee6c466330873e64cc593fccc9beda548a225b052505be4fcb48af5a
+  metadata.gz: 7947b3eb940d332765d567b71c6f548e5ed1ef8bdba0654801a46496981d7a8f8d89af5ff3803889192cc3ab2d6be29d200a1937c117942cbaf1e24531a49436
+  data.tar.gz: 267ed5502e2193a0acc7357d64b6a33c501eac55fc35101046da96405ea0f1fb01b773c04961e494a8c3d2a2ea6225355f4dcad6555285f1b55dcceb3e300b44

data/.gitignore

@@ -24,6 +24,8 @@ tmp
 vendor/apache-ftpserver/res/ftpd.pid*
 vendor/apache-ftpserver/res/home/*
 vendor/apache-ftpserver/res/user1/*
+!vendor/apache-ftpserver/res/user1/.gitkeep
 vendor/apache-ftpserver/res/user2/*
+!vendor/apache-ftpserver/res/user2/.gitkeep
 vendor/apache-ftpserver/res/log/*
 .jrubyrc

data/.travis.yml

@@ -8,3 +8,5 @@ gemfile:
 matrix:
   allow_failures:
     - rvm: jruby-19mode
+sudo: false
+cache: bundler

data/README.md

@@ -87,6 +87,10 @@ end
 
 ## Changelog
 
+### 1.2.7
+
+* Reduce number of FTP commands for creating directories [#27](https://github.com/xing/paperclip-storage-ftp/pull/27)
+
 ### 1.2.6
 
 * New option `:ftp_keep_empty_directories` to disable the removal of empty parent directories when deleting files (introduced in 1.2.2). See usage example above.

data/lib/paperclip/storage/ftp.rb

@@ -39,11 +39,14 @@ module Paperclip
           with_ftp_servers do |servers|
             servers.map do |server|
               run_thread do
+                write_queue = {}
                 @queued_for_write.each do |style_name, file|
                   remote_path = path(style_name)
                   log("saving ftp://#{server.user}@#{server.host}:#{remote_path}")
-                  server.put_file(file.path, remote_path)
+                  write_queue[file.path] = remote_path
                 end
+
+                server.put_files(write_queue)
               end
             end.each(&:join)
           end

data/lib/paperclip/storage/ftp/server.rb

@@ -60,10 +60,29 @@ module Paperclip
 
         def put_file(local_file_path, remote_file_path)
           pathname = Pathname.new(remote_file_path)
-          mkdir_p(pathname.dirname.to_s)
           connection.putbinaryfile(local_file_path, remote_file_path)
         end
 
+        def put_files(file_paths)
+          tree = directory_tree(file_paths.values)
+          mktree(tree)
+
+          file_paths.each do |local_file_path, remote_file_path|
+            put_file(local_file_path, remote_file_path)
+          end
+        end
+
+        def directory_tree(file_paths)
+          directories = file_paths.map do |path|
+            Pathname.new(path).dirname.to_s.split("/").reject(&:empty?)
+          end
+          tree = Hash.new {|h, k| h[k] = Hash.new(&h.default_proc)}
+          directories.each do |directory|
+            directory.inject(tree){|h,k| h[k]}
+          end
+          tree
+        end
+
         def delete_file(remote_file_path)
           connection.delete(remote_file_path)
         rescue Net::FTPPermError
@@ -79,16 +98,20 @@ module Paperclip
           # Stop trying to remove parent directories
         end
 
-        def mkdir_p(dirname)
-          pathname = Pathname.new(dirname)
-          pathname.descend do |p|
+        def mktree(tree, base = "/")
+          return unless tree.any?
+          list = connection.nlst(base)
+          tree.reject{|k,_| list.include?(k)}.each do |directory, sub_directories|
             begin
-              connection.mkdir(p.to_s)
+              connection.mkdir(base + directory)
             rescue Net::FTPPermError
-              # This error can be caused by an existing directory.
-              # Ignore, and keep on trying to create child directories.
+              # This error can be caused by an already existing directory,
+              # maybe it was created in the meantime.
             end
           end
+          tree.each do |directory, sub_directories|
+            mktree(sub_directories, base + directory + "/")
+          end
         end
 
         private

data/paperclip-storage-ftp.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.name          = "paperclip-storage-ftp"
   gem.require_paths = ["lib"]
-  gem.version       = "1.2.6"
+  gem.version       = "1.2.7"
 
   gem.add_dependency("paperclip")
 

data/spec/integration_spec.rb

@@ -137,4 +137,18 @@ describe "paperclip-storage-ftp", :integration => true do
       end
     end
   end
+
+  context "performance" do
+    let(:user) { UserWithOneServerAndDeepPath.new }
+    let(:padded_user_id) { user.id.to_s.rjust(3, "0") }
+    let(:uploaded_file_deep_path) { FtpServer::USER1_PATH + "/img/user_with_one_server_and_deep_paths/avatars/000/000/#{padded_user_id}/original/avatar.jpg" }
+
+    it "triggers minimal amount of ftp commands" do
+      expect_any_instance_of(Net::FTP).to receive(:nlst).exactly(7).times.and_call_original
+      expect_any_instance_of(Net::FTP).to receive(:mkdir).exactly(9).times.and_call_original
+      user.avatar = file
+      user.save!
+      File.exist?(uploaded_file_deep_path).should be true
+    end
+  end
 end

data/spec/paperclip/storage/ftp/server_spec.rb

@@ -2,6 +2,7 @@ require "spec_helper"
 
 describe Paperclip::Storage::Ftp::Server do
   let(:server) { Paperclip::Storage::Ftp::Server.new }
+  let(:connection) { double("connection") }
 
   context "initialize" do
     it "accepts options to initialize attributes" do
@@ -25,7 +26,7 @@ describe Paperclip::Storage::Ftp::Server do
 
   context "#file_exists?" do
     before do
-      server.stub(:connection).and_return(double("connection"))
+      server.stub(:connection).and_return(connection)
     end
 
     it "returns true if the file exists on the server" do
@@ -56,7 +57,7 @@ describe Paperclip::Storage::Ftp::Server do
 
   context "#get_file" do
     before do
-      server.stub(:connection).and_return(double("connection"))
+      server.stub(:connection).and_return(connection)
     end
 
     it "returns the file object" do
@@ -67,19 +68,163 @@ describe Paperclip::Storage::Ftp::Server do
 
   context "#put_file" do
     before do
-      server.stub(:connection).and_return(double("connection"))
+      server.stub(:connection).and_return(connection)
     end
 
     it "stores the file on the server" do
-      server.should_receive(:mkdir_p).with("/files")
       server.connection.should_receive(:putbinaryfile).with("/tmp/original.jpg", "/files/original.jpg")
       server.put_file("/tmp/original.jpg", "/files/original.jpg")
     end
   end
 
+  context "#put_files" do
+    before do
+      server.stub(:connection).and_return(connection)
+    end
+
+    shared_examples "proper handling" do
+      it "passes files to #put_file" do
+        server.should_receive(:mktree).with(tree)
+        server.should_receive(:put_file).with(files.keys.first, files.values.first).ordered
+        server.should_receive(:put_file).with(files.keys.last, files.values.last).ordered
+        server.put_files files
+      end
+    end
+
+    context "common directories" do
+      let(:files) do
+        {
+          "/tmp/foo1.jpg" => "/bar/foo1.jpg",
+          "/tmp/foo2.jpg" => "/bar/foo2.jpg"
+        }
+      end
+      let(:tree) { { "bar"=>{} } }
+
+      include_examples "proper handling"
+    end
+
+    context "no common directories" do
+      let(:files) do
+        {
+          "/tmp/foo1.jpg" => "/bar/foo1.jpg",
+          "/tmp/foo2.jpg" => "/baz/foo2.jpg"
+        }
+      end
+      let(:tree) { { "bar"=>{}, "baz"=>{} } }
+
+      include_examples "proper handling"
+    end
+
+    context "exactly one file" do
+      let(:files) do
+        { "/tmp/foo1.jpg" => "/bar/foo1.jpg" }
+      end
+      let(:tree) { { "bar"=>{} } }
+
+      it "passes file to #put_file" do
+        server.should_receive(:mktree).with(tree)
+        server.should_receive(:put_file).with(files.keys.first, files.values.first)
+        server.put_files files
+      end
+    end
+
+    context "no files" do
+      let(:files) { {} }
+
+      it "does not to anything" do
+        server.should_not_receive(:put_file)
+        connection.should_not_receive(:mkdir)
+        server.put_files files
+      end
+    end
+  end
+
+  context "#directory_tree" do
+    let(:files) {
+      %w(/foo/bar1.jpg /foo/bar2.jpg /foo/bar/baz.jpg /foo/foo/bar.jpg /foobar/foobar.jpg /root.jpg)
+    }
+
+    it "handles empty file list" do
+      expect(server.directory_tree([])).to eq({})
+    end
+
+    it "extracts nested directory structure" do
+      expect(server.directory_tree(files)).to eq(
+        {
+          "foo" => {
+            "bar" => {},
+            "foo" => {}
+          },
+          "foobar"=>{}
+        }
+      )
+    end
+  end
+
+  context "#mktree" do
+    before do
+      server.stub(:connection).and_return(connection)
+    end
+    let(:tree) do
+      {
+        "foo"=>{
+          "bar"=>{},
+          "baz"=>{"qux"=>{}}},
+        "foobar"=>{}
+      }
+    end
+
+    it "handles empty tree" do
+      server.mktree({})
+    end
+
+    context "empty ftp tree" do
+      it "creates entire nested tree" do
+        connection.should_receive(:nlst).with("/").ordered.and_return([])
+        connection.should_receive(:mkdir).with("/foo").ordered
+        connection.should_receive(:mkdir).with("/foobar").ordered
+        connection.should_receive(:nlst).with("/foo/").ordered.and_return([])
+        connection.should_receive(:mkdir).with("/foo/bar").ordered
+        connection.should_receive(:mkdir).with("/foo/baz").ordered
+        connection.should_receive(:nlst).with("/foo/baz/").ordered.and_return([])
+        connection.should_receive(:mkdir).with("/foo/baz/qux").ordered
+        server.mktree(tree)
+      end
+    end
+
+    context "partially existent ftp tree" do
+      it "creates only the missing directories" do
+        connection.should_receive(:nlst).with("/").ordered.and_return(["foo"])
+        connection.should_receive(:mkdir).with("/foobar").ordered
+        connection.should_receive(:nlst).with("/foo/").ordered.and_return(["baz"])
+        connection.should_receive(:mkdir).with("/foo/bar").ordered
+        connection.should_receive(:nlst).with("/foo/baz/").ordered.and_return(["qux"])
+        server.mktree(tree)
+      end
+    end
+
+    context "intermittent creation of directories" do
+      let(:tree) do
+        {
+          "foo"=>{},
+          "bar"=>{"foobar"=>{}}
+        }
+      end
+
+      it "handles Net::FTPPermError" do
+        connection.should_receive(:nlst).with("/").ordered.and_return([])
+        connection.should_receive(:mkdir).with("/foo").ordered.and_raise(Net::FTPPermError)
+        connection.should_receive(:mkdir).with("/bar").ordered.and_raise(Net::FTPPermError)
+        connection.should_receive(:nlst).with("/bar/").ordered.and_return([])
+        connection.should_receive(:mkdir).with("/bar/foobar").ordered.and_raise(Net::FTPPermError)
+        server.mktree(tree)
+      end
+    end
+  end
+
   context "#delete_file" do
     before do
-      server.stub(:connection).and_return(double("connection"))
+      server.stub(:connection).and_return(connection)
     end
 
     it "deletes the file on the server" do
@@ -96,7 +241,7 @@ describe Paperclip::Storage::Ftp::Server do
 
   context "#rmdir_p" do
     before do
-      server.stub(:connection).and_return(double("connection"))
+      server.stub(:connection).and_return(connection)
     end
 
     it "deletes the directory and all parent directories" do
@@ -119,24 +264,4 @@ describe Paperclip::Storage::Ftp::Server do
       server.connection.should == ftp
     end
   end
-
-  context "mkdir_p" do
-    before do
-      server.stub(:connection).and_return(double("connection"))
-    end
-
-    it "creates the directory and all its parent directories" do
-      server.connection.should_receive(:mkdir).with("/").ordered
-      server.connection.should_receive(:mkdir).with("/files").ordered
-      server.connection.should_receive(:mkdir).with("/files/foo").ordered
-      server.connection.should_receive(:mkdir).with("/files/foo/bar").ordered
-      server.mkdir_p("/files/foo/bar")
-    end
-
-    it "does not stop on Net::FTPPermError" do
-      server.connection.should_receive(:mkdir).with("/").and_raise(Net::FTPPermError)
-      server.connection.should_receive(:mkdir).with("/files")
-      server.mkdir_p("/files")
-    end
-  end
 end

data/spec/paperclip/storage/ftp_spec.rb

@@ -100,10 +100,13 @@ describe Paperclip::Storage::Ftp do
         :thumb    => thumb_file
       })
 
-      first_server.should_receive(:put_file).with("/tmp/original/foo.jpg", "/files/original/foo.jpg")
-      first_server.should_receive(:put_file).with("/tmp/thumb/foo.jpg", "/files/thumb/foo.jpg")
-      second_server.should_receive(:put_file).with("/tmp/original/foo.jpg", "/files/original/foo.jpg")
-      second_server.should_receive(:put_file).with("/tmp/thumb/foo.jpg", "/files/thumb/foo.jpg")
+      write_queue = {
+        "/tmp/original/foo.jpg" => "/files/original/foo.jpg",
+        "/tmp/thumb/foo.jpg" => "/files/thumb/foo.jpg"
+      }
+
+      first_server.should_receive(:put_files).with(write_queue)
+      second_server.should_receive(:put_files).with(write_queue)
 
       attachment.should_receive(:with_ftp_servers).and_yield([first_server, second_server])
 

data/spec/support/integration/user.rb

@@ -88,6 +88,20 @@ class UserWithInvalidPort < UserBase
   end
 end
 
+class UserWithOneServerAndDeepPath < UserBase
+  setup_avatar_attachment(avatar_options.merge(
+    :path     => "/img/:class/:attachment/:id_partition/:style/:filename",
+    :ftp_servers => [
+      {
+        :host     => "127.0.0.1",
+        :user     => "user1",
+        :password => "secret1",
+        :port     => 2121
+      }
+    ]
+  ))
+end
+
 class UserIgnoringFailingConnection < UserWithInvalidPort
   setup_avatar_attachment(avatar_options.merge(
     :ftp_ignore_failing_connections => true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: paperclip-storage-ftp
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.7
 platform: ruby
 authors:
 - Sebastian Röbke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-22 00:00:00.000000000 Z
+date: 2015-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paperclip
@@ -135,6 +135,8 @@ files:
 - vendor/apache-ftpserver/res/conf/users.properties
 - vendor/apache-ftpserver/res/ftp-db.sql
 - vendor/apache-ftpserver/res/ftpserver.jks
+- vendor/apache-ftpserver/res/user1/.gitkeep
+- vendor/apache-ftpserver/res/user2/.gitkeep
 homepage: https://github.com/xing/paperclip-storage-ftp
 licenses:
 - MIT
@@ -155,7 +157,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Allow Paperclip attachments to be stored on FTP servers