winrm-fs 0.4.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7cb6a97dc554ab9436a28969f3ea70c8771c0858
-  data.tar.gz: 4578906a4b74cffeb2f00bfe362dd059a7b33db6
+  metadata.gz: 3e54c394c912274542248ea943f17cbf8b5801b7
+  data.tar.gz: 8fa0f2fead6f018f6f22cde2b1109c6fcda72fde
 SHA512:
-  metadata.gz: babcce6f4e58ad60de51a34844149c551edcc61201b1f358472b2796b2d4a91759ae7d0ae878afb88f20a500f1e4928132dcbf0997ddcadd9299c514ee2a0811
-  data.tar.gz: c92f3204daef1eb4d47d56f174651adb1640b57bd17664518f05ffd7d795738942fd8c1f8748d3c1fbb98a3f745dd289844fe8dfeed7d7198696ac3b1f38e84c
+  metadata.gz: 3d11d2a9ea8715736c3dd9caedfc01ef96a954043896f15e9cd7e164f45b2d8222fe6d0151895c39969e1864347a788009cdd4f6ae83e2041dab02f57deca55d
+  data.tar.gz: 74dda7732bd2094e698a6651ef1cfc83fd64969c9cba111e8c75dc536133209d8c453a2ebe7eba321a9d049ea9e44d04904b89ba27a14bb1439391149abd997e

data/.travis.yml

@@ -1,5 +1,9 @@
 language: ruby
 rvm:
-  - 1.9.3
   - 2.0.0
-  - 2.1.0
+  - 2.1.0
+
+# This prevents testing branches that are created just for PRs
+branches:
+  only:
+  - master

data/Gemfile

@@ -1,3 +1,5 @@
 # encoding: UTF-8
 source 'https://rubygems.org'
 gemspec
+
+gem 'rb-readline'

data/README.md

@@ -8,8 +8,8 @@ Files may be copied from the local machine to the winrm endpoint. Individual fil
 ```ruby
 require 'winrm-fs'
 
-service = WinRM::WinRMWebService.new(...
-file_manager = WinRM::FS::FileManager.new(service)
+connection = WinRM::Connection.new(...
+file_manager = WinRM::FS::FileManager.new(connection)
 
 # upload file.txt from the current working directory
 file_manager.upload('file.txt', 'c:/file.txt')
@@ -25,6 +25,9 @@ file_manager.upload([
 ], '$env:ProgramData')
 ```
 
+### Optimizing WinRM settings
+Since winrm-fs 1.0/winrm 2.0, files are uploaded using the PSRP protocol and transfer speeds are dramatically improved from previous versions. This is largely due to the fact that the size of chunks that can be transferred at one time are now governed by the `MaxEnvelopeSizekb` winrm configuration setting on the endpoint. This default to 500 on Windows 2012 R2 and 150 on Windows 2008 R2. You may experience much faster transfer rates on 2008 R2 by increasing this setting.
+
 ### Handling progress events
 If you want to implement your own custom progress handling, you can pass a code
 block and use the proggress data that `upload` yields to this block:
@@ -40,16 +43,6 @@ If you're having trouble, first of all its most likely a network or WinRM config
 issue. Take a look at the [WinRM gem troubleshooting](https://github.com/WinRb/WinRM#troubleshooting)
 first.
 
-The most [common error](https://github.com/WinRb/winrm-fs/issues/1) with this gem is getting a 500 error because your maxConcurrentOperationsPerUser limit has been reached.
-
-```
-The WS-Management service cannot process the request. This user is allowed a
-maximum number of 1500 concurrent operations, which has been exceeded. Close
-existing operations for this user, or raise the quota for this user.
-```
-
-You can workaround this by increasing your operations per user quota.
-
 ## Contributing
 
 1. Fork it.

data/VERSION

@@ -1 +1 @@
-0.4.3
+1.0.0

data/Vagrantfile

@@ -5,5 +5,5 @@
 VAGRANTFILE_API_VERSION = '2'
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = 'mwrock/Windows2012R2'
+  config.vm.box = 'mwrock/WindowsNano'
 end

data/appveyor.yml

@@ -6,7 +6,7 @@ platform:
 
 environment:
   winrm_user: test_user
-  winrm_pass: Pass@word1
+  winrm_password: Pass@word1
 
   matrix:
     - ruby_version: "21"
@@ -19,7 +19,7 @@ branches:
     - master
 
 install:
-  - ps: net user /add $env:winrm_user $env:winrm_pass
+  - ps: net user /add $env:winrm_user $env:winrm_password
   - ps: net localgroup administrators $env:winrm_user /add
   - ps: winrm set winrm/config/client/auth '@{Basic="true"}'
   - ps: winrm set winrm/config/service/auth '@{Basic="true"}'

data/changelog.md

@@ -1,4 +1,6 @@
 # WinRM-fs Gem Changelog
+# 1.0.0
+- Using winrm v2. File uploads just got a whole lot faster!
 
 # 0.4.3
 - Fix error handling with wmf5, filtering out progress output from inspected stderr.

data/lib/winrm-fs/core/file_transporter.rb

@@ -41,8 +41,8 @@ module WinRM
       # sessions being invoked which can be 2 orders of magnitude more
       # expensive than vanilla CMD commands.
       #
-      # This object is supported by either a `CommandExecutor` instance as it
-      # depends on the `#run_cmd` and `#run_powershell_script` API contracts.
+      # This object is supported by a `PowerShell` instance as it
+      # depends on the `#run` API contract.
       #
       # An optional logger can be supplied, assuming it can respond to the
       # `#debug` and `#debug?` messages.
@@ -50,12 +50,12 @@ module WinRM
       # @author Fletcher Nichol <fnichol@nichol.ca>
       # @author Matt Wrock <matt@mattwrock.com>
       class FileTransporter
-        # Creates a FileTransporter given a CommandExecutor object.
+        # Creates a FileTransporter given a PowerShell object.
         #
-        # @param executor [CommandExecutor] a winrm CommandExecutor object
-        def initialize(executor, opts = {})
-          @executor  = executor
-          @logger = executor.service.logger
+        # @param shell [PowerShell] a winrm PowerShell object
+        def initialize(shell, opts = {})
+          @shell  = shell
+          @logger = shell.logger
           @id_generator = opts.fetch(:id_generator) { -> { SecureRandom.uuid } }
         end
 
@@ -74,6 +74,7 @@ module WinRM
         def upload(locals, remote)
           files = nil
           report = nil
+          remote = remote.to_s
 
           elapsed1 = Benchmark.measure do
             files = make_files_hash(Array(locals), remote)
@@ -91,7 +92,7 @@ module WinRM
           end
 
           elapsed3 = Benchmark.measure do
-            report = decode_files(files)
+            report = extract_files(files)
             merge_with_report!(files, report)
             cleanup(files)
           end
@@ -100,7 +101,7 @@ module WinRM
             "Uploaded #{files.keys.size} items " \
             "dirty_check: #{duration(elapsed1.real)} " \
             "stream_files: #{duration(elapsed2.real)} " \
-            "decode: #{duration(elapsed3.real)} " \
+            "extract: #{duration(elapsed3.real)} " \
           )
 
           [total_size, files]
@@ -108,12 +109,6 @@ module WinRM
 
         private
 
-        # @return [Integer] the maximum number of bytes that can be supplied on
-        #   a Windows CMD prompt without exceeded the maximum command line
-        #   length
-        # @api private
-        MAX_ENCODED_WRITE = 8000
-
         # @return [String] the Array pack template for Base64 encoding a stream
         #   of data
         # @api private
@@ -128,9 +123,24 @@ module WinRM
         # @api private
         attr_reader :logger
 
-        # @return [Winrm::CommandExecutor] a WinRM CommandExecutor
+        # @return [Winrm::Shells::Powershell] a WinRM Powershell shell
+        # @api private
+        attr_reader :shell
+
+        # @return [Integer] the maximum number of bytes to send per request
+        #   when streaming a file. This is optimized to send as much data
+        #   as allowed in a single PSRP fragment
         # @api private
-        attr_reader :executor
+        def max_encoded_write
+          @max_encoded_write ||= begin
+            empty_command = WinRM::PSRP::MessageFactory.create_pipeline_message(
+              '00000000-0000-0000-0000-000000000000',
+              '00000000-0000-0000-0000-000000000000',
+              stream_command('')
+            )
+            shell.max_fragment_blob_size - empty_command.bytes.length
+          end
+        end
 
         # Examines the files and corrects the file destination if it is
         # targeting an existing folder. In this case, the destination path
@@ -198,9 +208,9 @@ module WinRM
         # @api private
         def check_files(files)
           logger.debug 'Running check_files.ps1'
-          hash_file = create_remote_hash_file(check_files_ps_hash(files))
+          hash_file = check_files_ps_hash(files)
           script = WinRM::FS::Scripts.render('check_files', hash_file: hash_file)
-          parse_response(executor.run_powershell_script(script))
+          parse_response(shell.run(script))
         end
 
         # Constructs a collection of destination path/MD5 checksum pairs as a
@@ -236,63 +246,44 @@ module WinRM
           end
         end
 
-        # Creates a remote Base64-encoded temporary file containing a
-        # PowerShell hash table.
-        #
-        # @param hash [String] a String representation of a PowerShell hash
-        #   table
-        # @return [String] the remote path to the temporary file
-        # @api private
-        def create_remote_hash_file(hash)
-          hash_file = "$env:TEMP\\hash-#{@id_generator.call}.txt"
-          hash.lines.each { |line| logger.debug line.chomp }
-          StringIO.open(hash) { |io| stream_upload(io, hash_file) }
-          hash_file
-        end
-
-        # Runs the decode_files PowerShell script against a collection of
+        # Runs the extract_files PowerShell script against a collection of
         # temporary file/destination path pairs. The PowerShell script returns
         # its results as a CSV-formatted report which is converted into a Ruby
-        # Hash. The script will not be invoked if there are no "dirty" files
+        # Hash. The script will not be invoked if there are no zip files
         # present in the incoming files Hash.
         #
         # @param files [Hash] files hash, keyed by the local MD5 digest
         # @return [Hash] a report hash, keyed by the local MD5 digest
         # @api private
-        def decode_files(files)
-          decoded_files = decode_files_ps_hash(files)
+        def extract_files(files)
+          extracted_files = extract_files_ps_hash(files)
 
-          if decoded_files == ps_hash({})
-            logger.debug 'No remote files to decode, skipping'
+          if extracted_files == ps_hash({})
+            logger.debug 'No remote files to extract, skipping'
             {}
           else
-            logger.debug 'Running decode_files.ps1'
-            hash_file = create_remote_hash_file(decoded_files)
-            script = WinRM::FS::Scripts.render('decode_files', hash_file: hash_file)
+            logger.debug 'Running extract_files.ps1'
+            script = WinRM::FS::Scripts.render('extract_files', hash_file: extracted_files)
 
-            parse_response(executor.run_powershell_script(script))
+            parse_response(shell.run(script))
           end
         end
 
         # Constructs a collection of temporary file/destination path pairs for
-        # all "dirty" files as a String representation of the contents of a
-        # PowerShell Hash Table. A "dirty" file is one which has the
-        # `"chk_dirty"` option set to `"True"` in the incoming files Hash.
+        # all zipped folders as a String representation of the contents of a
+        # PowerShell Hash Table.
         #
         # @param files [Hash] files hash, keyed by the local MD5 digest
         # @return [String] the inner contents of a PowerShell Hash Table
         # @api private
-        def decode_files_ps_hash(files)
-          file_data = files.select do |_, data|
-            data['chk_dirty'] == 'True' || data.key?('tmpzip')
-          end
+        def extract_files_ps_hash(files)
+          file_data = files.select { |_, data| data.key?('tmpzip') }
 
-          i = 0
-          result = file_data.map do |_, data|
+          result = file_data.map do |md5, data|
             val = { 'dst' => data['dst'] }
             val['tmpzip'] = data['tmpzip'] if data['tmpzip']
 
-            [data['tmpfile'] || "clean#{i += 1}", val]
+            [md5, val]
           end
 
           ps_hash(Hash[result])
@@ -321,6 +312,7 @@ module WinRM
         def make_files_hash(locals, remote)
           hash = {}
           locals.each do |local|
+            local = local.to_s
             expanded = File.expand_path(local)
             expanded += local[-1] if local.end_with?('/', '\\')
 
@@ -358,20 +350,6 @@ module WinRM
           ' ' * depth
         end
 
-        # Parses CLIXML String into regular String (without any XML syntax).
-        # Inspired by https://github.com/WinRb/WinRM/issues/106.
-        #
-        # @param  clixml [String] clixml text
-        # @return [String] parsed clixml into String
-        def clixml_to_s(clixml)
-          doc = REXML::Document.new(clixml)
-          text = doc.get_elements('//S').map(&:text).join
-          text.gsub(/_x(\h\h\h\h)_/) do
-            code = Regexp.last_match[1]
-            code.hex.chr
-          end
-        end
-
         # Parses response of a PowerShell script or CMD command which contains
         # a CSV-formatted document in the standard output stream.
         #
@@ -380,22 +358,15 @@ module WinRM
         # @return [Hash] report hash, keyed by the local MD5 digest
         # @api private
         def parse_response(output)
-          exitcode = output[:exitcode]
+          exitcode = output.exitcode
           stderr = output.stderr
-          if stderr.include?('The command line is too long')
-            # The powershell script which should result in `output` parameter
-            # is too long, remove some newlines, comments, etc from it.
-            fail StandardError, 'The command line is too long' \
-              ' (powershell script is too long)'
-          end
-          pretty_stderr = clixml_to_s(stderr)
 
           if exitcode != 0
             fail FileTransporterFailed, "[#{self.class}] Upload failed " \
-              "(exitcode: #{exitcode})\n#{pretty_stderr}"
-          elsif pretty_stderr != '\r\n' && pretty_stderr != ''
+              "(exitcode: #{exitcode})\n#{stderr}"
+          elsif stderr != '\r\n' && stderr != ''
             fail FileTransporterFailed, "[#{self.class}] Upload failed " \
-              "(exitcode: 0), but stderr present\n#{pretty_stderr}"
+              "(exitcode: 0), but stderr present\n#{stderr}"
           end
 
           logger.debug 'Parsing CSV Response'
@@ -441,42 +412,59 @@ module WinRM
         #   the number of bytes transferred to the remote host
         # @api private
         def stream_upload(input_io, dest)
-          dest_cmd = dest.sub('$env:TEMP', '%TEMP%')
-          read_size = (MAX_ENCODED_WRITE.to_i / 4) * 3
+          read_size = ((max_encoded_write - dest.length) / 4) * 3
           chunk, bytes = 1, 0
           buffer = ''
-          executor.run_cmd(%(echo|set /p=>"#{dest_cmd}")) # truncate empty file
+          shell.run(<<-EOS
+            $to = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("#{dest}")
+            $parent = Split-Path $to
+            if(!(Test-path $parent)) { mkdir $parent | Out-Null }
+            $fileStream = New-Object -TypeName System.IO.FileStream -ArgumentList @(
+                $to,
+                [system.io.filemode]::Create,
+                [System.io.FileAccess]::Write,
+                [System.IO.FileShare]::ReadWrite
+            )
+            EOS
+          )
+
           while input_io.read(read_size, buffer)
             bytes += (buffer.bytesize / 3 * 4)
-            executor.run_cmd([buffer].pack(BASE64_PACK)
-              .insert(0, 'echo ').concat(%( >> "#{dest_cmd}")))
+            shell.run(stream_command([buffer].pack(BASE64_PACK)))
             logger.debug "Wrote chunk #{chunk} for #{dest}" if chunk % 25 == 0
             chunk += 1
             yield bytes if block_given?
           end
+          shell.run('$fileStream.Dispose()')
           buffer = nil # rubocop:disable Lint/UselessAssignment
 
           [chunk - 1, bytes]
         end
 
-        # Uploads a local file to a Base64-encoded temporary file.
+        def stream_command(encoded_bytes)
+          <<-EOS
+            $bytes=[Convert]::FromBase64String('#{encoded_bytes}')
+            $fileStream.Write($bytes, 0, $bytes.length)
+          EOS
+        end
+
+        # Uploads a local file.
         #
         # @param src [String] path to a local file
-        # @param tmpfile [String] path to the temporary file on the remote
-        #   host
+        # @param dest [String] path to the file on the remote host
         # @return [Integer,Integer] the number of resulting upload chunks and
         #   the number of bytes transferred to the remote host
         # @api private
-        def stream_upload_file(src, tmpfile, &block)
-          logger.debug "Uploading #{src} to encoded tmpfile #{tmpfile}"
+        def stream_upload_file(src, dest, &block)
+          logger.debug "Uploading #{src} to #{dest}"
           chunks, bytes = 0, 0
           elapsed = Benchmark.measure do
             File.open(src, 'rb') do |io|
-              chunks, bytes = stream_upload(io, tmpfile, &block)
+              chunks, bytes = stream_upload(io, dest, &block)
             end
           end
           logger.debug(
-            "Finished uploading #{src} to encoded tmpfile #{tmpfile} " \
+            "Finished uploading #{src} to #{dest} " \
             "(#{bytes.to_f / 1000} KB over #{chunks} chunks) " \
             "in #{duration(elapsed.real)}"
           )
@@ -496,9 +484,8 @@ module WinRM
           files.each do |md5, data|
             src = data.fetch('src_zip', data['src'])
             if data['chk_dirty'] == 'True'
-              tmpfile = "$env:TEMP\\b64-#{md5}.txt"
-              response[md5] = { 'tmpfile' => tmpfile }
-              chunks, bytes = stream_upload_file(src, tmpfile) do |xfered|
+              response[md5] = { 'dest' => data['tmpzip'] || data['dst'] }
+              chunks, bytes = stream_upload_file(src, data['tmpzip'] || data['dst']) do |xfered|
                 yield data['src'], xfered
               end
               response[md5]['chunks'] = chunks

data/lib/winrm-fs/file_manager.rb

@@ -23,10 +23,10 @@ module WinRM
     # Perform file transfer operations between a local machine and winrm endpoint
     class FileManager
       # Creates a new FileManager instance
-      # @param [WinRMWebService] WinRM web service client
-      def initialize(service)
-        @service = service
-        @logger = service.logger
+      # @param [WinRM::Connection] WinRM web connection client
+      def initialize(connection)
+        @connection = connection
+        @logger = connection.logger
       end
 
       # Gets the MD5 checksum of the specified file if it exists,
@@ -35,7 +35,7 @@ module WinRM
       def checksum(path)
         @logger.debug("checksum: #{path}")
         script = WinRM::FS::Scripts.render('checksum', path: path)
-        @service.create_executor { |e| e.run_powershell_script(script).stdout.chomp }
+        @connection.shell(:powershell) { |e| e.run(script).stdout.chomp }
       end
 
       # Create the specifed directory recursively
@@ -44,7 +44,7 @@ module WinRM
       def create_dir(path)
         @logger.debug("create_dir: #{path}")
         script = WinRM::FS::Scripts.render('create_dir', path: path)
-        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
+        @connection.shell(:powershell) { |e| e.run(script).exitcode == 0 }
       end
 
       # Deletes the file or directory at the specified path
@@ -53,7 +53,7 @@ module WinRM
       def delete(path)
         @logger.debug("deleting: #{path}")
         script = WinRM::FS::Scripts.render('delete', path: path)
-        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
+        @connection.shell(:powershell) { |e| e.run(script).exitcode == 0 }
       end
 
       # Downloads the specified remote file to the specified local path
@@ -62,8 +62,8 @@ module WinRM
       def download(remote_path, local_path)
         @logger.debug("downloading: #{remote_path} -> #{local_path}")
         script = WinRM::FS::Scripts.render('download', path: remote_path)
-        output = @service.create_executor { |e| e.run_powershell_script(script) }
-        return false if output[:exitcode] != 0
+        output = @connection.shell(:powershell) { |e| e.run(script) }
+        return false if output.exitcode != 0
         contents = output.stdout.gsub('\n\r', '')
         out = Base64.decode64(contents)
         IO.binwrite(local_path, out)
@@ -76,7 +76,7 @@ module WinRM
       def exists?(path)
         @logger.debug("exists?: #{path}")
         script = WinRM::FS::Scripts.render('exists', path: path)
-        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
+        @connection.shell(:powershell) { |e| e.run(script).exitcode == 0 }
       end
 
       # Gets the current user's TEMP directory on the remote system, for example
@@ -84,7 +84,7 @@ module WinRM
       # @return [String] Full path to the temp directory
       def temp_dir
         @guest_temp ||= begin
-          (@service.create_executor { |e| e.run_cmd('echo %TEMP%') }).stdout.chomp.gsub('\\', '/')
+          (@connection.shell(:powershell) { |e| e.run('$env:TEMP') }).stdout.chomp.gsub('\\', '/')
         end
       end
 
@@ -107,8 +107,8 @@ module WinRM
       # @yieldparam [String] Target path on the winrm endpoint
       # @return [Fixnum] The total number of bytes copied
       def upload(local_path, remote_path, &block)
-        @service.create_executor do |executor|
-          file_transporter ||= WinRM::FS::Core::FileTransporter.new(executor)
+        @connection.shell(:powershell) do |shell|
+          file_transporter ||= WinRM::FS::Core::FileTransporter.new(shell)
           file_transporter.upload(local_path, remote_path, &block)[0]
         end
       end