net-sftp 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d42976ec0ca9fa6131be80a2d84e8e4b375a95dc410aa20f987ebf56b5717c3
-  data.tar.gz: 8b3b507190fed78c32dedd040e4d5d814ca36a7ecc0b72a6de402459f9f8b999
+  metadata.gz: 4d3d330529ea03b189dc54e18be90d35bb2bfb79374917e9e22a2fcd3d7d79ca
+  data.tar.gz: feaf13c15e37ea24c5ed45f52a842b0a8d4413cc2a27f1ee6de5a03a0975d527
 SHA512:
-  metadata.gz: 1ac5c4215ede73ce6f16b1db405647cc545d420c2e4ab87ba64e7db5efd4535a32549217a34b8a3989978dd47af1dd274f28a953c5f28c8d16f3f6948047e458
-  data.tar.gz: b61e54caed650be40e5b8a3d89f9d6cb52ffcb0c4fbe37ebfd2c2f20e5fca15405e7c06bd579b71d3458c6182fdc2e247caae4c1d5e0a5c6ed0adab73c205e6c
+  metadata.gz: 3da2ef173bd6e6ac212cf3ae73243a887ae0d136fc01f03a06b383f00a6ea19a55d2f461f5fadff10d6991dd295c6fb9c826edf6557f1d2a831fe32f2782dfdf
+  data.tar.gz: e12c6bdf4efc55e51fbf6dbe3ddb12494b8f3b090e9786bdf6deb002a6966505e9a6a63440fdb43fb23da9c20fce7b0c40d2b2e5425dd095a97dfe1648102dfb

data/.github/pull_request_template.md

@@ -0,0 +1 @@
+<!-- After opening the PR, add a changelog.d/<PR number>.<type>.txt fragment (none, change, security, breaking or breaking-security) instead of editing CHANGES.txt; see CONTRIBUTING.md#changelog. -->

data/.github/workflows/changelog.yml

@@ -0,0 +1,18 @@
+name: Changelog
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+
+permissions:
+  contents: read
+
+jobs:
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # HEAD is the PR merged into the base branch; HEAD^1 is the base it was merged into.
+          fetch-depth: 2
+      - uses: net-ssh/changelogd@v1

data/.github/workflows/ci.yml

@@ -6,7 +6,8 @@ permissions:
   contents: read
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # setup-ruby has no builds of Ruby 3.1 and older for ubuntu-latest (26.04).
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -21,3 +22,16 @@ jobs:
           bundler-cache: true
       - name: Run Tests
         run: bundle exec rake test
+  frozen-string-literal:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '4.0'
+          bundler-cache: true
+      - name: Run Tests with frozen string literals
+        run: bundle exec rake test
+        env:
+          RUBYOPT: --enable-frozen-string-literal

data/CHANGES.txt

@@ -1,3 +1,39 @@
+=== 4.1.0
+
+* BREAKING: Require Ruby 2.6 or later. Bundler on older Rubies resolves to 4.0.2. [#207]
+* BREAKING: `upload` and `download` no longer close an IO object passed in as the local side; `download` flushes it instead. Callers that relied on the close must close it themselves. [#202]
+* Security: An `FXP_VERSION` packet after the session is open now raises `Net::SFTP::Exception` instead of switching protocol version mid-session and dropping pending requests. [#193]
+* Security: A server can no longer make the client allocate without bound, or hang, by sending an ACL, extended attribute or directory listing count larger than the packet holds; such packets now raise `Net::SFTP::Exception`. [#192]
+* Single-file downloads keep up to `:requests` reads in flight (default 64) instead of waiting for each 32 KB reply, so large downloads are no longer bound by round-trip time. [#204]
+* `upload!` and `download!` now give new files and directories the source's permissions (masked to 0777, subject to the umask) like OpenSSH's sftp client, so executables stay executable and private files are no longer created world-readable. The new `:preserve` option also copies access and modification times and sets permissions exactly. As with OpenSSH, attributes are best effort: if they can't be read the transfer uses default permissions, and if they can't be set a warning is logged and the transfer still succeeds. [#203]
+* Added `Net::SFTP::Session#extensions`, the extensions the server advertised in its `FXP_VERSION` packet (such as `posix-rename@openssh.com`). [#195]
+* The `:version` option now caps the negotiated SFTP protocol version; servers that always reply with their own version (OpenSSH replies 3) used to be negotiated at that version instead of the one requested. [#194]
+
+=== 4.0.2
+
+* Security: Recursive downloads refuse to follow an existing local symlink at a server-named path instead of writing through it. The local path passed by the caller may still be a symlink. Suggested in #162 by srosenhamer [#186]
+* Security: Recursive downloads reject file names from the server that are not a single path component (such as `../x` or `a/b`) instead of writing outside the target directory. Reported in #162 by srosenhamer [#185, #162]
+* Net::SFTP now works with frozen string literals enabled: `File` reads and `Dir#glob` no longer mutate frozen strings, and `glob` no longer modifies entry names in place (adds `Name#name=`). Based on #157 by @frederikspang [#198, #157]
+* `File#read` and `File#gets` no longer loop forever when the server returns an empty data packet, and `read` with a negative length raises `ArgumentError` like `IO#read`. Based on #174 by @OskarEichler [#189, #174]
+* Show fields an SFTP v4+ server omitted as `?` in directory listings (`longname`), like `ls -l` does, instead of blank, `-` or `---------` [#188]
+* Fixed `File#write` hanging on data of 256 KiB or more by sending it in 32 KiB write requests [#99, #187]
+* SFTP request ids now wrap around at the uint32 limit, skipping ids still awaiting a response, instead of growing past it so that responses on very long-lived sessions could no longer be matched to their requests. Thanks @OskarEichler [#179]
+* Uploads keep exactly `:requests` writes in flight (one more was sent before, so the default is now 2 rather than 3), and a zero, negative or non-numeric `:requests` or `:read_size` on `upload` or `download` now raises `ArgumentError` when the transfer is set up instead of stalling. Thanks @OskarEichler [#178]
+* Reconnecting a closed `Net::SFTP::Session` now discards partially received packet data from the old channel, so stale bytes are no longer parsed as the start of the new channel's first packet. Thanks @OskarEichler [#177]
+* Don't crash formatting a directory listing (`longname`) when an SFTP v4+ server omits the modification time, size or permissions, and don't show omitted permissions as `rwxrwxrwx`. Thanks @OskarEichler [#175, #146]
+* `Dir#glob` no longer lists every subdirectory for a pattern without `/` (which can only match top-level names), saving a round trip per directory and no longer failing on unreadable subdirectories. Thanks @OskarEichler [#173, #180]
+* `File#gets` no longer rescans the whole line after each chunk, so reading very long lines is no longer quadratic (a 16 MiB line ending in `"\r\n"` went from about 5 s to 0.01 s). Thanks @OskarEichler [#172]
+* `File#gets` treats a nil or negative limit as no limit, like `IO#gets`, instead of returning nil and corrupting the position (or raising for nil). Thanks @OskarEichler [#171]
+* Fixed SFTP v5/v6 `open` ignoring `IO::TRUNC` and `IO::CREAT` in numeric flags: `IO::WRONLY | IO::TRUNC` now requests `TRUNCATE_EXISTING` instead of opening without truncation, and read-only opens with `IO::CREAT` now request `OPEN_OR_CREATE`. Thanks @OskarEichler [#170]
+* Include the server's error message in `StatusException` and `Response#to_s` for SFTP protocol v3 and later, instead of only the status code. Thanks @OskarEichler [#169, #156]
+* Fixed SFTP v4+ attribute serialization raising `TypeError` when nanoseconds were given for only some timestamps (e.g. `atime_nseconds` with `mtime` but no `mtime_nseconds`); the missing subsecond fields are now sent as 0. Thanks @OskarEichler [#168]
+* `Net::SFTP.start` with a block now closes the SSH session when the block exits with `break`, `return` or `throw`, instead of leaking it. Thanks @OskarEichler [#167]
+* Attributes objects now copy the hash they are given, so a frozen hash no longer raises `FrozenError` and the caller's hash is no longer modified by the default `:type` or by cached owner/group/uid/gid lookups. Thanks @OskarEichler [#166]
+* `File#gets(limit)` no longer makes an extra remote read when the buffered data already satisfies the limit, and `gets(0)` returns an empty string like `IO#gets`. Thanks @OskarEichler [#165]
+* `File#puts` now behaves like `IO#puts`: with no arguments it writes a newline (it used to write nothing), `nil` and `false` are written instead of raising, objects with `to_ary` are expanded, and a recursive array no longer overflows the stack. Thanks @OskarEichler [#164]
+* Fixed `Net::SFTP::Operations::File#write` writing at the read-ahead offset instead of the current position after a buffered read (e.g. `read(2)` on an `r+` file), which corrupted data. Thanks @OskarEichler [#163]
+* Accept a `Pathname` as the local or remote path for uploads and downloads. Thanks @Tietew and @til [#153, #102]
+
 === 4.0.1
 
 * Allow net-ssh 8 [#181]

data/CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# Contributing to net-sftp
+
+Run the tests with `bundle exec rake test`.
+
+## Changelog
+
+Don't edit `CHANGES.txt`. Instead, add a fragment file to `changelog.d/` in your PR; the release's version bump (`rake vbump:*`) moves all fragments into a new `=== <version>` section at the top of `CHANGES.txt` and deletes them.
+
+A fragment is named `<number>.<type>.txt`, where `<number>` is the PR number (so open the PR first, then add the fragment), and contains a single UTF-8 paragraph: the entry as it should read in `CHANGES.txt`, without the bullet or prefix. Lines are joined, so it can be wrapped. If it doesn't end with a reference list such as `[#106, #107]` or `[GHSA-xxxx-xxxx-xxxx]`, ` [#<number>]` is appended.
+
+| Type                | Written as                     | Use for                                                      |
+| ------------------- | ------------------------------ | ------------------------------------------------------------ |
+| `breaking-security` | `* BREAKING: Security: <text>` | an incompatible change made for security                     |
+| `breaking`          | `* BREAKING: <text>`           | an incompatible change                                       |
+| `security`          | `* Security: <text>`           | a security fix or hardening                                  |
+| `change`            | `* <text>`                     | any other user-visible fix or feature                        |
+| `none`              | nothing                        | PRs with no user-visible change (tests, CI, docs, refactors) |
+
+Examples:
+
+```sh
+rake "changelog:new[184,change,Fixed downloading files with multibyte names]"
+# changelog.d/184.change.txt -> * Fixed downloading files with multibyte names [#184]
+
+echo 'Refuse symlinks that point outside the download directory [GHSA-xxxx-xxxx-xxxx]' > changelog.d/185.security.txt
+touch changelog.d/186.none.txt
+```
+
+For several entries from one PR add a suffix: `184.change.txt`, `184.change.2.txt` (`changelog:new` picks the next free one).
+
+`rake changelog:check` validates the fragments and `rake changelog:preview` prints the section the next release would add. CI fails a PR that edits `CHANGES.txt` or doesn't add or modify a `changelog.d/<PR number>.<type>.txt` fragment, unless it only touches `changelog.d/` or a maintainer adds the `skip-changelog` label; Dependabot PRs are skipped.
+
+The full rules, including how a release orders the entries, are in the [changelogd README](https://github.com/net-ssh/changelogd#readme).

data/Gemfile

@@ -9,6 +9,9 @@ gem "rake", ">= 12.0"
 gem "rdoc"
 gem "logger" # no longer a default gem in Ruby 4.0+
 
+# git: rather than github:, which Bundler 1.x (the default on Ruby 2.6) expands to an unsupported git:// URL.
+gem "changelogd", git: "https://github.com/net-ssh/changelogd.git", tag: "v1", group: :development
+
 gem 'byebug', group: %i[development test] if !Gem.win_platform? && RUBY_ENGINE == "ruby"
 
 if ENV["CI"]

data/README.rdoc

@@ -35,6 +35,9 @@ In a nutshell:
     # download a file or directory from the remote host
     sftp.download!("/path/to/remote", "/path/to/local")
 
+    # also copy access and modification times (permissions are always copied)
+    sftp.download!("/path/to/remote", "/path/to/local", :preserve => true)
+
     # grab data off the remote host directly to a buffer
     data = sftp.download!("/path/to/remote")
 

data/Rakefile

@@ -19,6 +19,16 @@ name = "net-sftp"
 require_relative "lib/net/sftp/version"
 version = Net::SFTP::Version::CURRENT
 
+begin
+  require "changelogd/rake"
+rescue LoadError
+  # A git gem isn't on the load path under plain `rake`, as the release workflow runs it; bundler/setup could clash with the running rake.
+  require "bundler"
+  $LOAD_PATH.unshift(*Bundler.definition.specs["changelogd"].flat_map(&:full_require_paths))
+  require "changelogd/rake"
+end
+Changelogd::RakeTasks.new(version_file: "lib/net/sftp/version.rb")
+
 namespace :cert do
   desc "Update public cert from private - only run if public is expired"
   task :update_public_when_expired do
@@ -53,16 +63,19 @@ RDoc::Task.new do |rdoc|
 end
 
 def change_version(&block)
+  Rake::Task["changelog:check"].invoke
   version_file = 'lib/net/sftp/version.rb'
   require_relative version_file
   pre = Net::SFTP::Version::PRE
   tiny = Net::SFTP::Version::TINY
-  result = block[pre: pre, tiny: Net::SFTP::Version::TINY]
+  minor = Net::SFTP::Version::MINOR
+  result = block[pre: pre, tiny: tiny, minor: minor]
   raise ArgumentError, "Version change logic should always return a pre" unless result.key?(:pre)
 
   new_pre = result[:pre]
   new_tiny = result[:tiny] || tiny
-  found = { pre: false, tiny: false }
+  new_minor = result[:minor] || minor
+  found = { pre: false, tiny: false, minor: false }
   File.open("#{version_file}.new", "w") do |f|
     File.readlines(version_file).each do |line|
       match =
@@ -94,20 +107,34 @@ def change_version(&block)
         end
       end
 
+      if new_minor != minor
+        match = /^(\s+MINOR\s+=\s+)#{minor}(\s*)$/.match(line)
+        if match
+          prefix = match[1]
+          postfix = match[2]
+          new_line = "#{prefix}#{new_minor}#{postfix}"
+          puts "Changing:\n  - #{line}  + #{new_line}"
+          line = new_line
+          found[:minor] = true
+        end
+      end
+
       f.write(line)
     end
     raise ArgumentError, "Cound not find line: PRE = \"#{pre}\" in #{version_file}" unless found[:pre]
     raise ArgumentError, "Cound not find line: TINY = \"#{tiny}\" in #{version_file}" unless found[:tiny] || new_tiny == tiny
+    raise ArgumentError, "Cound not find line: MINOR = \"#{minor}\" in #{version_file}" unless found[:minor] || new_minor == minor
   end
 
   FileUtils.mv version_file, "#{version_file}.old"
   FileUtils.mv "#{version_file}.new", version_file
+  Rake::Task["changelog:release"].invoke
 end
 
 namespace :vbump do
   desc "Final release"
   task :final do
-    change_version do |pre:, tiny:|
+    change_version do |pre:, tiny:, **|
       _ = tiny
       if pre.nil?
         { tiny: tiny + 1, pre: nil }
@@ -119,9 +146,18 @@ namespace :vbump do
     end
   end
 
+  desc "Minor release"
+  task :minor do
+    change_version do |pre:, minor:, **|
+      raise ArgumentError, "PRE is #{pre.inspect}; release it with vbump:final first" unless pre.nil?
+
+      { minor: minor + 1, tiny: 0, pre: nil }
+    end
+  end
+
   desc "Increment prerelease"
   task :pre, [:type] do |_t, args|
-    change_version do |pre:, tiny:|
+    change_version do |pre:, tiny:, **|
       puts " PRE => #{pre.inspect}"
       match = /^([a-z]+)(\d+)/.match(pre)
       raise ArgumentError, "Unexpected pre: #{pre}" if match.nil? && args[:type].nil?

data/changelog.d/README.md

@@ -0,0 +1 @@
+Changelog fragments: one `<PR number>.<type>.txt` file per user-visible change, assembled into `CHANGES.txt` when the version is bumped. See [CONTRIBUTING.md](../CONTRIBUTING.md#changelog).

data/lib/net/sftp/operations/dir.rb

@@ -58,17 +58,18 @@ module Net; module SFTP; module Operations
     def glob(path, pattern, flags=0)
       flags |= ::File::FNM_PATHNAME
       path = path.chop if path.end_with?('/') && path != '/'
+      recursive = !pattern.is_a?(String) || pattern.include?('/')
 
       results = [] unless block_given?
       queue = entries(path).reject { |e| %w(. ..).include?(e.name) }
       while queue.any?
         entry = queue.shift
 
-        if entry.directory? && !%w(. ..).include?(::File.basename(entry.name))
-          queue += entries("#{path}/#{entry.name}").map do |e|
-            e.name.replace("#{entry.name}/#{e.name}")
+        if recursive && entry.directory? && !%w(. ..).include?(::File.basename(entry.name))
+          queue.concat(entries("#{path}/#{entry.name}").map do |e|
+            e.name = "#{entry.name}/#{e.name}"
             e
-          end
+          end)
         end
 
         if ::File.fnmatch(pattern, entry.name, flags)

data/lib/net/sftp/operations/download.rb

@@ -1,3 +1,4 @@
+require 'pathname'
 require 'net/ssh/loggable'
 
 module Net; module SFTP; module Operations
@@ -48,12 +49,26 @@ module Net; module SFTP; module Operations
   # * <tt>:progress</tt> - either a block or an object to act as a progress
   #   callback. See the discussion of "progress monitoring" below.
   # * <tt>:requests</tt> - the number of pending SFTP requests to allow at
-  #   any given time. When downloading an entire directory tree recursively,
-  #   this will default to 16. Setting this higher might improve throughput.
-  #   Reducing it will reduce throughput.
+  #   any given time. When downloading a single file, this is the number of
+  #   reads kept in flight and defaults to 64. When downloading an entire
+  #   directory tree recursively, it is the number of files and directories
+  #   transferred at once and defaults to 16. Setting this higher might
+  #   improve throughput, particularly on high-latency links. Reducing it
+  #   will reduce throughput.
   # * <tt>:read_size</tt> - the maximum number of bytes to read at a time
   #   from the source. Increasing this value might improve throughput. It
   #   defaults to 32,000 bytes.
+  # * <tt>:preserve</tt> - if true, the local files and directories get the
+  #   access and modification times of the remote ones, and their permissions
+  #   are set exactly, even on files that already exist and regardless of the
+  #   umask.
+  #
+  # Like OpenSSH's sftp client, new local files and directories are created
+  # with the permissions of the remote ones, without setuid, setgid and sticky
+  # bits and subject to the umask. Nothing is set when +local+ is an IO object.
+  # If the remote attributes can't be read, the download goes ahead with
+  # default permissions; if they can't be applied locally, a warning is logged
+  # and the download still succeeds.
   #
   # == Progress Monitoring
   #
@@ -145,13 +160,19 @@ module Net; module SFTP; module Operations
     # This will return immediately, and requires that the SSH event loop be
     # run in order to effect the download. (See #wait.)
     def initialize(sftp, local, remote, options={}, &progress)
+      local = local.to_path if local.is_a?(Pathname)
+      remote = remote.to_path if remote.is_a?(Pathname)
+
       @sftp = sftp
       @local = local
       @remote = remote
       @progress = progress || options[:progress]
       @options = options
       @active = 0
+      @directories = []
       @properties = options[:properties] || {}
+      requests
+      read_size
 
       self.logger = sftp.logger
 
@@ -204,7 +225,8 @@ module Net; module SFTP; module Operations
 
       # A simple struct for encapsulating information about a single remote
       # file or directory that needs to be downloaded.
-      Entry = Struct.new(:remote, :local, :directory, :size, :handle, :offset, :sink)
+      Entry = Struct.new(:remote, :local, :directory, :size, :handle, :offset, :sink,
+                         :next_read, :pending, :window, :chunks, :eof, :attributes)
 
       #--
       # "ruby -w" hates private attributes, so we have to do these longhand
@@ -222,13 +244,29 @@ module Net; module SFTP; module Operations
 
       # The number of bytes to read at a time from remote files.
       def read_size
-        options[:read_size] || DEFAULT_READ_SIZE
+        @read_size ||= positive_option(:read_size, DEFAULT_READ_SIZE)
       end
 
-      # The number of simultaneou SFTP requests to use to effect the download.
-      # Defaults to 16 for recursive downloads.
+      # The number of simultaneous SFTP requests to use to effect the download.
+      # Defaults to 16 for recursive downloads and 64 for single files.
       def requests
-        options[:requests] || (recursive? ? 16 : 2)
+        @requests ||= positive_option(:requests, recursive? ? 16 : 64)
+      end
+
+      # The most reads a single file may have outstanding at once.
+      def read_window_limit
+        recursive? ? 1 : requests
+      end
+
+      def preserve?
+        options[:preserve]
+      end
+
+      def positive_option(name, default)
+        value = (options[name] || default).to_i
+        raise ArgumentError, ":#{name} must be positive" unless value > 0
+
+        value
       end
 
       # Enqueues as many files and directories from the stack as possible
@@ -240,7 +278,13 @@ module Net; module SFTP; module Operations
 
           if entry.directory
             update_progress(:mkdir, entry.local)
-            ::Dir.mkdir(entry.local) unless ::File.directory?(entry.local)
+            refuse_symlink!(entry.local) if server_named?(entry) && ::File.symlink?(entry.local)
+            created = !::File.directory?(entry.local)
+            ::Dir.mkdir(entry.local, local_permissions(entry, 0777) | 0300) if created
+            if created || preserve?
+              @directories << entry
+              stat_directory(entry) unless entry.attributes
+            end
             request = sftp.opendir(entry.remote, &method(:on_opendir))
             request[:entry] = entry
           else
@@ -248,7 +292,64 @@ module Net; module SFTP; module Operations
           end
         end
 
-        update_progress(:finish) if !active?
+        if !active?
+          set_directory_attributes
+          update_progress(:finish)
+        end
+      end
+
+      # Only entries listed by readdir come with attributes.
+      def stat_directory(entry)
+        @active += 1
+        sftp.stat(entry.remote) do |response|
+          @active -= 1
+          entry.attributes = remote_attributes(response, "stat #{entry.remote}")
+          process_next_entry
+        end
+      end
+
+      # Without the remote attributes the download goes ahead with default
+      # permissions and no times, as it would for a server that never sent any.
+      def remote_attributes(response, operation)
+        return response[:attrs] if response.ok?
+
+        debug { "#{operation} failed, using default attributes: #{response}" }
+        nil
+      end
+
+      # Deepest directories first, after their contents; setting attributes on
+      # a directory doesn't change its parent's modification time.
+      def set_directory_attributes
+        @directories.reverse_each do |entry|
+          next unless entry.attributes
+
+          permissions = local_permissions(entry, 0777)
+          permissions &= ~::File.umask unless preserve?
+          set_local_attributes(entry, permissions)
+        end
+        @directories.clear
+      end
+
+      # setuid, setgid and sticky bits are never applied, matching OpenSSH.
+      def local_permissions(entry, default)
+        permissions = entry.attributes && entry.attributes.permissions
+        permissions ? permissions & 0777 : default
+      end
+
+      # Like OpenSSH's sftp client, failing to set attributes only warns; the
+      # data has already been transferred.
+      def set_local_attributes(entry, permissions)
+        attempt_local("chmod", entry) { ::File.chmod(permissions, entry.local) } if permissions
+
+        attributes = entry.attributes
+        return unless preserve? && attributes && attributes.atime && attributes.mtime
+        attempt_local("utime", entry) { ::File.utime(attributes.atime, attributes.mtime, entry.local) }
+      end
+
+      def attempt_local(operation, entry)
+        yield
+      rescue SystemCallError => e
+        lwarn { "#{operation} #{entry.local} failed: #{e.message}" }
       end
 
       # Called when a remote directory is "opened" for reading, e.g. to
@@ -275,7 +376,10 @@ module Net; module SFTP; module Operations
         else
           response[:names].each do |item|
             next if item.name == "." || item.name == ".."
-            stack << Entry.new(::File.join(entry.remote, item.name), ::File.join(entry.local, item.name), item.directory?, item.attributes.size)
+            validate_entry_name!(entry, item.name)
+            child = Entry.new(::File.join(entry.remote, item.name), ::File.join(entry.local, item.name), item.directory?, item.attributes.size)
+            child.attributes = item.attributes
+            stack << child
           end
 
           # take this opportunity to enqueue more requests
@@ -286,6 +390,38 @@ module Net; module SFTP; module Operations
         end
       end
 
+      # The name comes from the server, so anything but a single path component
+      # would let a malicious server write outside the target directory.
+      def validate_entry_name!(entry, name)
+        if name.empty? || name.include?("/") || name.include?("\0") ||
+           (::File::ALT_SEPARATOR && name.include?(::File::ALT_SEPARATOR))
+          raise Net::SFTP::Exception, "readdir #{entry.remote} returned an invalid file name: #{name.inspect}"
+        end
+      end
+
+      # The caller's own +local+ may be a symlink on purpose; anything below it
+      # was named by the server (net-sftp#162).
+      def server_named?(entry)
+        entry.local != local
+      end
+
+      # Like OpenSSH, the file is always created writable by its owner.
+      def open_local_file(entry)
+        permissions = local_permissions(entry, 0666) | 0200
+        return ::File.open(entry.local, "wb", permissions) unless server_named?(entry) && defined?(::File::NOFOLLOW)
+
+        begin
+          ::File.open(entry.local, "wb", permissions, :flags => ::File::NOFOLLOW)
+        rescue SystemCallError
+          raise unless ::File.symlink?(entry.local)
+          refuse_symlink!(entry.local)
+        end
+      end
+
+      def refuse_symlink!(path)
+        raise Net::SFTP::Exception, "#{path} is a symlink, refusing to follow it"
+      end
+
       # Called when a file is to be opened for reading from the remote server.
       def open_file(entry)
         update_progress(:open, entry)
@@ -301,45 +437,116 @@ module Net; module SFTP; module Operations
         process_next_entry
       end
 
-      # Called when a file has been opened. This will call #download_next_chunk
+      # Called when a file has been opened. This will call #request_reads
       # to initiate the data transfer.
       def on_open(response)
         entry = response.request[:entry]
         raise StatusException.new(response, "open #{entry.remote}") unless response.ok?
 
         entry.handle = response[:handle]
-        entry.sink = entry.local.respond_to?(:write) ? entry.local : ::File.open(entry.local, "wb")
+        if entry.attributes || entry.local.respond_to?(:write)
+          start_download(entry)
+        else
+          request = sftp.fstat(entry.handle, &method(:on_fstat))
+          request[:entry] = entry
+        end
+      end
+
+      # Called when the attributes of a file opened by #open_file arrive.
+      def on_fstat(response)
+        entry = response.request[:entry]
+        entry.attributes = remote_attributes(response, "fstat #{entry.remote}")
+        start_download(entry)
+      end
+
+      def start_download(entry)
+        entry.sink = entry.local.respond_to?(:write) ? entry.local : open_local_file(entry)
         entry.offset = 0
+        entry.next_read = 0
+        entry.pending = 0
+        entry.window = 1
+        entry.chunks = {}
+
+        request_reads(entry)
+      end
 
-        download_next_chunk(entry)
+      # Keeps up to entry.window reads of #read_size bytes outstanding until
+      # the server reports EOF. The window starts at one and grows with each
+      # data reply, so small files don't cost a burst of reads past their end.
+      def request_reads(entry)
+        while !entry.eof && entry.pending < entry.window
+          read_range(entry, entry.next_read, read_size)
+          entry.next_read += read_size
+        end
       end
 
-      # Initiates a read of the next #read_size bytes from the file.
-      def download_next_chunk(entry)
-        request = sftp.read(entry.handle, entry.offset, read_size, &method(:on_read))
+      def read_range(entry, offset, length)
+        entry.pending += 1
+        request = sftp.read(entry.handle, offset, length, &method(:on_read))
         request[:entry] = entry
-        request[:offset] = entry.offset
+        request[:offset] = offset
+        request[:length] = length
       end
 
-      # Called when a read from a file finishes. If the read was successful
-      # and returned data, this will call #download_next_chunk to read the
-      # next bit from the file. Otherwise the file will be closed.
+      # Called when a read from a file finishes. Replies can arrive out of
+      # order, so data is buffered until everything before it has been
+      # written. Once the server has reported EOF and no reads are left
+      # outstanding, the file is closed.
       def on_read(response)
         entry = response.request[:entry]
+        entry.pending -= 1
 
         if response.eof?
-          update_progress(:close, entry)
-          entry.sink.close
-          request = sftp.close(entry.handle, &method(:on_close))
-          request[:entry] = entry
+          entry.eof = true
         elsif !response.ok?
           raise StatusException.new(response, "read #{entry.remote}")
         else
-          entry.offset += response[:data].bytesize
-          update_progress(:get, entry, response.request[:offset], response[:data])
-          entry.sink.write(response[:data])
-          download_next_chunk(entry)
+          offset = response.request[:offset]
+          length = response.request[:length]
+          data = response[:data]
+
+          if data.empty? || data.bytesize > length
+            raise Net::SFTP::Exception, "read #{entry.remote} returned #{data.bytesize} bytes for a #{length} byte request"
+          end
+
+          # Servers may return less than asked for without being at EOF, and
+          # the reads after this one have already been sent.
+          read_range(entry, offset + data.bytesize, length - data.bytesize) if data.bytesize < length
+
+          entry.chunks[offset] = data
+          write_buffered_chunks(entry)
+          entry.window += 1 if entry.window < read_window_limit
         end
+
+        request_reads(entry)
+        close_file(entry) if entry.pending.zero?
+      end
+
+      def write_buffered_chunks(entry)
+        while (data = entry.chunks.delete(entry.offset))
+          update_progress(:get, entry, entry.offset, data)
+          entry.sink.write(data)
+          entry.offset += data.bytesize
+        end
+      end
+
+      def close_file(entry)
+        if entry.chunks.any?
+          raise Net::SFTP::Exception, "read #{entry.remote} hit EOF at #{entry.offset} but the server returned data beyond it"
+        end
+
+        update_progress(:close, entry)
+        # The caller's IO stays open, but is flushed so its data is written when download! returns.
+        if entry.sink.equal?(local)
+          entry.sink.flush if entry.sink.respond_to?(:flush)
+        else
+          entry.sink.close
+        end
+        unless entry.local.respond_to?(:write)
+          set_local_attributes(entry, preserve? ? local_permissions(entry, nil) : nil)
+        end
+        request = sftp.close(entry.handle, &method(:on_close))
+        request[:entry] = entry
       end
 
       # Called when a file handle is closed.