ftpd 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d52205c73f9747677423670210f0b7eff28659e7
-  data.tar.gz: 352b8761bf5518f2a41b549c64b57f21d989d93e
+  metadata.gz: 15d6cd56c8148e7961d07ebb86246fb3c1fe7a25
+  data.tar.gz: 54b088a8bee0cfe945377d9dd97b5ef1bcf97650
 SHA512:
-  metadata.gz: 190fe51abf6cdc5b7a8122cb32bd31d476d43a55d40034f5eaea3b0e5d4ceb640ff0be3472b6791efac335b601cec7adb4a51beb23a07ad3022befe58cc0b2da
-  data.tar.gz: 639d4e7024ec277a57bdc77c985dfa3068df523660038412e4e341b228f7354f373049250700461f57dbeaa7e291c43528d08ff0fcb340e4da70dadebeb7a31f
+  metadata.gz: 4978b27e90b70d04d670f2e60db440dc7e2b5686eda9674bf539db822f873e7210d446a640b95600f88f7dba89b2dbf41b01b3594ec1f28ee81ae670abdaa8d8
+  data.tar.gz: dfcbb64e17744d9dd2016e2d7572da2194e4b5e24cd3efe986a7f1ca71f2ae76876039ffcb8001cc49ba1ab235348cff7838c48238c2afca941dab17c0026ac6

data/Changelog.md

@@ -2,6 +2,29 @@ This is the change log for the main branch of ftpd, which supports
 Ruby 1.9 and greater.  For ruby 1.8.7, please use the latest version
 before 0.8.0.
 
+### 0.17.0
+
+This release is brought to you by Mike Ragalie.  Thanks, Mike!
+
+Features
+
+* Streaming file transfers (issues #12 and #26)
+
+API Changes
+
+These breaking API changes are for streaming file transfers.  Custom
+file systems will notice these changes.  Nobody else should.
+
+* The FileSystemErrorTranslator has been removed.  Instead, the file
+  system may raise FtpServerError with a message and an optional error
+  code.
+
+* DiskFileSystem#write now takes an `Ftpd::Stream` object instead of a
+  `String`.
+
+* DiskFileSystem#read expects to receive a block and yields an `IO`
+  object to the block.
+
 ### 0.16.0
 
 Bug fixes
@@ -12,7 +35,7 @@ Bug fixes
 
 Bug fixes
 
-* Ignore Errno::ENOTCONN on socket shutdown (issue #24)	
+* Ignore Errno::ENOTCONN on socket shutdown (issue #24)
 
 Administration
 

data/README.md

@@ -357,6 +357,7 @@ Among those who have improved ftpd are:
 * Joshua Rutherford
 * Larry. W. Cashdollar
 * Michael de Silva
+* Mike Ragalie
 * cransom
 
 If I've forgotten to add you, please remind me, or submit a merge

data/VERSION

@@ -1 +1 @@
-0.16.0
+0.17.0

data/doc/benchmarks.md

@@ -5,78 +5,67 @@ Benchmarks using pyftpd's benchmark script and
 
 ## Results
 
-### ftpd 0.4.0
-
-    (starting with 13.6M of RSS memory being used)
-    STOR (client -> server)                              845.55 MB/sec  13.6M RSS
-    RETR (server -> client)                             1234.80 MB/sec  859.5M RSS
-    200 concurrent clients (connect, login)                0.14 secs    863.0M RSS
-    STOR (1 file with 200 idle clients)                  848.56 MB/sec  863.0M RSS
-    RETR (1 file with 200 idle clients)                 1227.46 MB/sec  866.4M RSS
-    200 concurrent clients (RETR 10.0M file)               5.36 secs    2.0G RSS
-    200 concurrent clients (STOR 10.0M file)               4.65 secs    2.0G RSS
-    200 concurrent clients (QUIT)                          0.01 secs
-
-### pyftpdlib 1.0.1
-
-    (starting with 8.9M of RSS memory being used)
-    STOR (client -> server)                               81.73 MB/sec  8.9M RSS
-    RETR (server -> client)                             1001.05 MB/sec  8.9M RSS
-    200 concurrent clients (connect, login)                2.58 secs    9.6M RSS
-    STOR (1 file with 200 idle clients)                  108.46 MB/sec  9.6M RSS
-    RETR (1 file with 200 idle clients)                 1134.18 MB/sec  9.6M RSS
-    200 concurrent clients (RETR 10.0M file)               2.32 secs    10.3M RSS
-    200 concurrent clients (STOR 10.0M file)               4.12 secs    10.4M RSS
+### ftpd 0.17.0
+
+    (starting with 14.9M of memory being used)
+    STOR (client -> server)                              120.75 MB/sec  14.9M
+    RETR (server -> client)                              741.77 MB/sec  22.7M
+    200 concurrent clients (connect, login)                0.20 secs    19.2M
+    STOR (1 file with 200 idle clients)                  144.77 MB/sec  19.2M
+    RETR (1 file with 200 idle clients)                  542.86 MB/sec  27.1M
+    200 concurrent clients (RETR 10.0M file)               6.38 secs    32.1M
+    200 concurrent clients (STOR 10.0M file)               2.76 secs    21.7M
+    200 concurrent clients (QUIT)                          0.03 secs
+
+### pyftpd 1.4.0
+
+    (starting with 4.2M of memory being used)
+    STOR (client -> server)                              127.62 MB/sec  4.2M
+    RETR (server -> client)                             1170.82 MB/sec  4.3M
+    200 concurrent clients (connect, login)                2.53 secs    4.8M
+    STOR (1 file with 200 idle clients)                  113.38 MB/sec  4.9M
+    RETR (1 file with 200 idle clients)                 1139.89 MB/sec  4.9M
+    200 concurrent clients (RETR 10.0M file)               2.55 secs    5.6M
+    200 concurrent clients (STOR 10.0M file)               2.03 secs    5.7M
     200 concurrent clients (QUIT)                          0.02 secs
 
-### proftpd 1.3.4a-3
+### proftpd 1.3.5rc4
 
-    (starting with 2.3M of RSS memory being used)
-    STOR (client -> server)                               93.06 MB/sec  6.6M RSS
-    RETR (server -> client)                             1267.63 MB/sec  6.6M RSS
-    200 concurrent clients (connect, login)               14.21 secs    868.0M RSS
-    STOR (1 file with 200 idle clients)                   76.04 MB/sec  872.3M RSS
-    RETR (1 file with 200 idle clients)                 1289.75 MB/sec  872.3M RSS
-    200 concurrent clients (RETR 10.0M file)               2.04 secs    868.0M RSS
-    200 concurrent clients (STOR 10.0M file)               4.51 secs    868.2M RSS
+    (starting with 1.4M of memory being used)
+    STOR (client -> server)                              117.59 MB/sec  3.2M
+    RETR (server -> client)                             1318.32 MB/sec  3.2M
+    200 concurrent clients (connect, login)               12.20 secs    366.4M
+    STOR (1 file with 200 idle clients)                  123.82 MB/sec  368.3M
+    RETR (1 file with 200 idle clients)                 1302.86 MB/sec  368.3M
+    200 concurrent clients (RETR 10.0M file)               2.33 secs    366.4M
+    200 concurrent clients (STOR 10.0M file)               2.76 secs    366.4M
     200 concurrent clients (QUIT)                          0.00 secs
 
-### Discussion
+### Notes
 
-ftpd's STOR results seen anomalous.  I suspect that proftpd and
-pyftpdlib aren't getting a fair shake here.  proftpd and pyftpdlib are
-serving my home directory, whereas ftpd is serving a temporary
-directory, but I don't know what difference that could make.
+* Ftpd's fast time on the login test, compared to proftpd and
+  pyftpdlib, is probably a result of it not doing PAM authentication
+  whereas the other ftpd servers are.  It is not an apples-to-apples
+  comparison.
 
-Ftpd is a memory hog.  During a STOR or RETR, it loads the entire
-contents of a file into memory.  This limits the number of concurrent
-file transfers it can handle.  The pyftpd team uses -n 300 (300
-concurrent connections) when benchmarking, but Ftpd can't handle that
-many at the moment.
-
-Ftpd's fast time on the login test, compared to proftpd and pyftpdlib,
-is a result of it not doing PAM authentication.  It is an unfair
-comparison and should be disregarded.
-
-pyftpd's memory footprint is impressive.
-
-ftpd is less performant with many concurrent connections than either
-proftpd or pyftpdlib.
+* The only benchmark for which ftpd beats the competition is the
+  single-client STOR.
 
 ## Setup
 
 ### Machine
 
 * Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz (4 cores)
-* Python 2.7.3rc2 (default, Apr 22 2012, 22:35:38)
-* ruby 2.0.0p0 (2013-02-24 revision 39474) [i686-linux]
+* Python 3.4.1rc1
+* pyftpd 1.4.0
+* ruby 2.1.1p76 (2014-02-24 revision 45161) [i686-linux]
 * Linux 3.0.0-1-686-pae #1 SMP Sat Aug 27 16:41:03 UTC 2011 i686 GNU/Linux
 
 ### Benchmark command
 
 bench.py needs psutil.  Under Debian, install _python-psutil_.
 
-    pyftpdlib-1.0.1/test$ python bench.py  -u <USER> -p <PASS> -H localhost -P <PORT> -b all -n 200 -k <PID>
+    pyftpdlib-1.0.1/test$ python3 bench.py  -u <USER> -p <PASS> -H localhost -P <PORT> -b all -n 200 -k <PID>
 
 ### Proftpd
 
@@ -90,4 +79,4 @@ bench.py needs psutil.  Under Debian, install _python-psutil_.
 
 ### Pyftpd
 
-    pyftpdlib-1.0.0/demo$ sudo python unix_daemon.py 2>/dev/null
+    pyftpdlib-1.4.0/demo$ sudo python ./unix_daemon.py 2>/dev/null

data/features/support/test_server.rb

@@ -98,21 +98,21 @@ class TestServer
 
       def raise_on_file_system_error(method_name)
         original_method = instance_method(method_name)
-        define_method method_name do |*args|
+        define_method method_name do |*args, &block|
           ftp_path = args.first
           if force_file_system_error?(ftp_path)
             raise Ftpd::PermanentFileSystemError, 'Unable to do it'
           end
-          original_method.bind(self).call *args
+          original_method.bind(self).call *args, &block
         end
       end
 
       def return_true_on_file_system_error(method_name)
         original_method = instance_method(method_name)
-        define_method method_name do |*args|
+        define_method method_name do |*args, &block|
           ftp_path = args.first
           return true if force_file_system_error?(ftp_path)
-          original_method.bind(self).call *args
+          original_method.bind(self).call *args, &block
         end
       end
 

data/ftpd.gemspec

@@ -2,15 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: ftpd 0.16.0 ruby lib
+# stub: ftpd 0.17.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "ftpd"
-  s.version = "0.16.0"
+  s.version = "0.17.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Wayne Conrad"]
-  s.date = "2014-05-19"
+  s.date = "2014-06-29"
   s.description = "ftpd is a pure Ruby FTP server library.  It supports implicit and explicit TLS, IPV6, passive and active mode, and is unconditionally compliant per RFC-1123.  It can be used as part of a test fixture or embedded in a program."
   s.email = "wconrad@yagni.com"
   s.extra_rdoc_files = [
@@ -140,7 +141,6 @@ Gem::Specification.new do |s|
     "ftpd.gemspec",
     "insecure-test-cert.pem",
     "lib/ftpd.rb",
-    "lib/ftpd/ascii_helper.rb",
     "lib/ftpd/auth_levels.rb",
     "lib/ftpd/cmd_abor.rb",
     "lib/ftpd/cmd_allo.rb",
@@ -195,7 +195,6 @@ Gem::Specification.new do |s|
     "lib/ftpd/exception_translator.rb",
     "lib/ftpd/exceptions.rb",
     "lib/ftpd/file_info.rb",
-    "lib/ftpd/file_system_error_translator.rb",
     "lib/ftpd/file_system_helper.rb",
     "lib/ftpd/ftp_server.rb",
     "lib/ftpd/insecure_certificate.rb",
@@ -208,6 +207,7 @@ Gem::Specification.new do |s|
     "lib/ftpd/server.rb",
     "lib/ftpd/session.rb",
     "lib/ftpd/session_config.rb",
+    "lib/ftpd/stream.rb",
     "lib/ftpd/telnet.rb",
     "lib/ftpd/temp_dir.rb",
     "lib/ftpd/tls_server.rb",
@@ -224,7 +224,7 @@ Gem::Specification.new do |s|
     "spec/disk_file_system_spec.rb",
     "spec/exception_translator_spec.rb",
     "spec/file_info_spec.rb",
-    "spec/file_system_error_translator_spec.rb",
+    "spec/ftp_server_error_spec.rb",
     "spec/list_format/eplf_spec.rb",
     "spec/list_format/ls_spec.rb",
     "spec/list_path_spec.rb",
@@ -237,8 +237,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/wconrad/ftpd"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.1.11"
+  s.rubygems_version = "2.2.2"
   s.summary = "Pure Ruby FTP server library"
 
   if s.respond_to? :specification_version then

data/lib/ftpd.rb

@@ -13,10 +13,7 @@ require 'tmpdir'
 # Gems
 require 'memoizer'
 
-
 # Ftpd
-
-require_relative 'ftpd/ascii_helper'
 require_relative 'ftpd/auth_levels'
 require_relative 'ftpd/cmd_abor'
 require_relative 'ftpd/cmd_allo'
@@ -70,7 +67,6 @@ require_relative 'ftpd/error'
 require_relative 'ftpd/exception_translator'
 require_relative 'ftpd/exceptions'
 require_relative 'ftpd/file_info'
-require_relative 'ftpd/file_system_error_translator'
 require_relative 'ftpd/file_system_helper'
 require_relative 'ftpd/ftp_server'
 require_relative 'ftpd/insecure_certificate'
@@ -83,6 +79,7 @@ require_relative 'ftpd/read_only_disk_file_system'
 require_relative 'ftpd/server'
 require_relative 'ftpd/session'
 require_relative 'ftpd/session_config'
+require_relative 'ftpd/stream'
 require_relative 'ftpd/telnet'
 require_relative 'ftpd/temp_dir'
 require_relative 'ftpd/tls_server'

data/lib/ftpd/cmd_appe.rb

@@ -12,8 +12,9 @@ module Ftpd
         syntax_error unless path
         path = File.expand_path(path, name_prefix)
         ensure_accessible path
-        contents = receive_file
-        file_system.append path, contents
+        receive_file do |data_socket|
+          file_system.append path, data_socket
+        end
         reply "226 Transfer complete"
       end
     end

data/lib/ftpd/cmd_auth.rb

@@ -7,10 +7,10 @@ module Ftpd
     def cmd_auth(security_scheme)
       ensure_tls_supported
       if socket.encrypted?
-        error "503 AUTH already done"
+        error "AUTH already done", 503
       end
       unless security_scheme =~ /^TLS(-C)?$/i
-        error "504 Security scheme not implemented: #{security_scheme}"
+        error "Security scheme not implemented: #{security_scheme}", 504
       end
       reply "234 AUTH #{security_scheme} OK."
       socket.encrypt

data/lib/ftpd/cmd_dele.rb

@@ -8,7 +8,7 @@ module Ftpd
       ensure_logged_in
       ensure_file_system_supports :delete
       path = argument
-      error "501 Path required" unless path
+      error "Path required", 501 unless path
       path = File.expand_path(path, name_prefix)
       ensure_accessible path
       ensure_exists path

data/lib/ftpd/cmd_list.rb

@@ -11,7 +11,7 @@ module Ftpd
         ensure_file_system_supports :file_info
         path = list_path(argument)
         path = File.expand_path(path, name_prefix)
-        transmit_file(list(path), 'A')
+        transmit_file(StringIO.new(list(path)), 'A')
       end
     end
 

data/lib/ftpd/cmd_mode.rb

@@ -8,8 +8,8 @@ module Ftpd
       syntax_error unless argument
       ensure_logged_in
       name, implemented = TRANSMISSION_MODES[argument]
-      error "504 Invalid mode code" unless name
-      error "504 Mode not implemented" unless implemented
+      error "Invalid mode code", 504 unless name
+      error "Mode not implemented", 504 unless implemented
       self.mode = argument
       reply "200 Mode set to #{name}"
     end

data/lib/ftpd/cmd_nlst.rb

@@ -10,7 +10,7 @@ module Ftpd
         ensure_file_system_supports :dir
         path = list_path(argument)
         path = File.expand_path(path, name_prefix)
-        transmit_file(name_list(path), 'A')
+        transmit_file(StringIO.new(name_list(path)), 'A')
       end
     end
 

data/lib/ftpd/cmd_opts.rb

@@ -6,7 +6,7 @@ module Ftpd
 
     def cmd_opts(argument)
       syntax_error unless argument
-      error '501 Unsupported option'
+      error 'Unsupported option', 501
     end
 
   end

data/lib/ftpd/cmd_pbsz.rb

@@ -9,10 +9,10 @@ module Ftpd
       syntax_error unless buffer_size =~ /^\d+$/
       buffer_size = buffer_size.to_i
       unless socket.encrypted?
-        error "503 PBSZ must be preceded by AUTH"
+        error "PBSZ must be preceded by AUTH", 503
       end
       unless buffer_size == 0
-        error "501 PBSZ=0"
+        error "PBSZ=0", 501
       end
       reply "200 PBSZ=0"
       self.protection_buffer_size_set = true

data/lib/ftpd/cmd_prot.rb

@@ -7,14 +7,14 @@ module Ftpd
     def cmd_prot(level_arg)
       level_code = level_arg.upcase
       unless protection_buffer_size_set
-        error "503 PROT must be preceded by PBSZ"
+        error "PROT must be preceded by PBSZ", 503
       end
       level = DATA_CHANNEL_PROTECTION_LEVELS[level_code]
       unless level
-        error "504 Unknown protection level"
+        error "Unknown protection level", 504
       end
       unless level == :private
-        error "536 Unsupported protection level #{level}"
+        error "Unsupported protection level #{level}", 536
       end
       self.data_channel_protection_level = level
       reply "200 Data protection level #{level_code}"

data/lib/ftpd/cmd_retr.rb

@@ -13,8 +13,9 @@ module Ftpd
         path = File.expand_path(path, name_prefix)
         ensure_accessible path
         ensure_exists path
-        contents = file_system.read(path)
-        transmit_file contents
+        file_system.read(path) do |file|
+          transmit_file file
+        end
       end
     end
 

data/lib/ftpd/cmd_size.rb

@@ -11,9 +11,17 @@ module Ftpd
       path = File.expand_path(path, name_prefix)
       ensure_accessible(path)
       ensure_exists(path)
-      contents = file_system.read(path)
-      contents = (data_type == 'A') ? unix_to_nvt_ascii(contents) : contents
-      reply "213 #{contents.bytesize}"
+      file_system.read(path) do |file|
+        if data_type == 'A'
+          output = StringIO.new
+          io = Ftpd::Stream.new(output, data_type)
+          io.write(file)
+          size = output.size
+        else
+          size = file.size
+        end
+        reply "213 #{size}"
+      end
     end
 
   end