winrm-fs 0.2.3 → 0.3.0

data/lib/winrm-fs/core/tmp_zip.rb

@@ -0,0 +1,166 @@
+# -*- encoding: utf-8 -*-
+#
+# Author:: Fletcher (<fnichol@nichol.ca>)
+#
+# Copyright (C) 2015, Fletcher Nichol
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'delegate'
+require 'pathname'
+require 'tempfile'
+require 'zip'
+
+module WinRM
+  module FS
+    module Core
+      # A temporary Zip file for a given directory.
+      #
+      # @author Fletcher Nichol <fnichol@nichol.ca>
+      class TmpZip
+        # Contructs a new Zip file for the given directory.
+        #
+        # There are 2 ways to interpret the directory path:
+        #
+        # * If the directory has no path separator terminator, then the
+        #   directory basename will be used as the base directory in the
+        #   resulting zip file.
+        # * If the directory has a path separator terminator (such as `/` or
+        #   `\\`), then the entries under the directory will be added to the
+        #   resulting zip file.
+        #
+        # The following emaples assume a directory tree structure of:
+        #
+        #     src
+        #     |-- alpha.txt
+        #     |-- beta.txt
+        #     \-- sub
+        #         \-- charlie.txt
+        #
+        # @example Including the base directory in the zip file
+        #
+        #   TmpZip.new("/path/to/src")
+        #   # produces a zip file with entries:
+        #   # - src/alpha.txt
+        #   # - src/beta.txt
+        #   # - src/sub/charlie.txt
+        #
+        # @example Excluding the base directory in the zip file
+        #
+        #   TmpZip.new("/path/to/src/")
+        #   # produces a zip file with entries:
+        #   # - alpha.txt
+        #   # - beta.txt
+        #   # - sub/charlie.txt
+        #
+        # @param dir [String,Pathname,#to_s] path to the directory
+        # @param logger [#debug,#debug?] an optional logger/ui object that
+        #   responds to `#debug` and `#debug?` (default `nil`)
+        def initialize(dir, logger = nil)
+          @logger = logger || Logging.logger[self]
+          @dir = Pathname.new(dir)
+          @zip_io = Tempfile.open(['tmpzip-', '.zip'], binmode: true)
+          write_zip
+          @zip_io.close
+        end
+
+        # @return [Pathname] path to zip file
+        def path
+          Pathname.new(zip_io.path) if zip_io.path
+        end
+
+        # Unlinks (deletes) the zip file from the filesystem.
+        def unlink
+          zip_io.unlink
+        end
+
+        private
+
+        # @return [Pathname] the directory used to create the Zip file
+        # @api private
+        attr_reader :dir
+
+        # @return [#debug] the logger
+        # @api private
+        attr_reader :logger
+
+        # @return [IO] the Zip file IO
+        # @api private
+        attr_reader :zip_io
+
+        # @return [Array<Pathname] all recursive files under the base
+        #   directory, excluding directories
+        # @api private
+        def entries
+          Pathname.glob(dir.join('**/*')).delete_if(&:directory?).sort
+        end
+
+        # (see Logging.log_subject)
+        # @api private
+        def log_subject
+          @log_subject ||= [self.class.to_s.split('::').last, path].join('::')
+        end
+
+        # Adds all file entries to the Zip output stream.
+        #
+        # @param zos [Zip::OutputStream] zip output stream
+        # @api private
+        def produce_zip_entries(zos)
+          entries.each do |entry|
+            entry_path = entry.sub(/#{dir}\//i, '')
+            logger.debug "+++ Adding #{entry_path}"
+            zos.put_next_entry(
+              zip_entry(entry_path),
+              nil, nil, ::Zip::Entry::DEFLATED, Zlib::BEST_COMPRESSION)
+            entry.open('rb') { |src| IO.copy_stream(src, zos) }
+          end
+          logger.debug '=== All files added.'
+        end
+
+        # Writes out a temporary Zip file.
+        #
+        # @api private
+        def write_zip
+          logger.debug 'Populating files'
+          Zip::OutputStream.write_buffer(NoDupIO.new(zip_io)) do |zos|
+            produce_zip_entries(zos)
+          end
+        end
+
+        def zip_entry(entry_path)
+          Zip::Entry.new(
+            zip_io.path,
+            entry_path.to_s,
+            nil, nil, nil, nil, nil, nil,
+            ::Zip::DOSTime.new(2000)
+          )
+        end
+
+        # Simple delegate wrapper to prevent `#dup` calls being made on IO
+        # objects. This is used to bypass an issue in the `Zip::Outputstream`
+        # constructor where an incoming IO is duplicated, leading to races
+        # on flushing the final stream to disk.
+        #
+        # @author Fletcher Nichol <fnichol@nichol.ca>
+        # @api private
+        class NoDupIO < SimpleDelegator
+          # @return [self] returns self and does *not* return a duplicate
+          #   object
+          def dup
+            self
+          end
+        end
+      end
+    end
+  end
+end

data/lib/winrm-fs/file_manager.rb

@@ -14,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'winrm'
 require_relative 'scripts/scripts'
-require_relative 'core/upload_orchestrator'
+require_relative 'core/file_transporter'
 
 module WinRM
   module FS
@@ -25,7 +26,7 @@ module WinRM
       # @param [WinRMWebService] WinRM web service client
       def initialize(service)
         @service = service
-        @logger = Logging.logger[self]
+        @logger = service.logger
       end
 
       # Gets the MD5 checksum of the specified file if it exists,
@@ -34,7 +35,7 @@ module WinRM
       def checksum(path)
         @logger.debug("checksum: #{path}")
         script = WinRM::FS::Scripts.render('checksum', path: path)
-        @service.powershell(script).stdout.chomp
+        @service.create_executor { |e| e.run_powershell_script(script).stdout.chomp }
       end
 
       # Create the specifed directory recursively
@@ -43,7 +44,7 @@ module WinRM
       def create_dir(path)
         @logger.debug("create_dir: #{path}")
         script = WinRM::FS::Scripts.render('create_dir', path: path)
-        @service.powershell(script)[:exitcode] == 0
+        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
       end
 
       # Deletes the file or directory at the specified path
@@ -52,7 +53,7 @@ module WinRM
       def delete(path)
         @logger.debug("deleting: #{path}")
         script = WinRM::FS::Scripts.render('delete', path: path)
-        @service.powershell(script)[:exitcode] == 0
+        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
       end
 
       # Downloads the specified remote file to the specified local path
@@ -61,7 +62,7 @@ module WinRM
       def download(remote_path, local_path)
         @logger.debug("downloading: #{remote_path} -> #{local_path}")
         script = WinRM::FS::Scripts.render('download', path: remote_path)
-        output = @service.powershell(script)
+        output = @service.create_executor { |e| e.run_powershell_script(script) }
         return false if output[:exitcode] != 0
         contents = output.stdout.gsub('\n\r', '')
         out = Base64.decode64(contents)
@@ -75,14 +76,16 @@ module WinRM
       def exists?(path)
         @logger.debug("exists?: #{path}")
         script = WinRM::FS::Scripts.render('exists', path: path)
-        @service.powershell(script)[:exitcode] == 0
+        @service.create_executor { |e| e.run_powershell_script(script)[:exitcode] == 0 }
       end
 
       # Gets the current user's TEMP directory on the remote system, for example
       # 'C:/Windows/Temp'
       # @return [String] Full path to the temp directory
       def temp_dir
-        @guest_temp ||= (@service.cmd('echo %TEMP%')).stdout.chomp.gsub('\\', '/')
+        @guest_temp ||= begin
+          (@service.create_executor { |e| e.run_cmd('echo %TEMP%') }).stdout.chomp.gsub('\\', '/')
+        end
       end
 
       # Upload one or more local files and directories to a remote directory
@@ -104,13 +107,9 @@ module WinRM
       # @yieldparam [String] Target path on the winrm endpoint
       # @return [Fixnum] The total number of bytes copied
       def upload(local_path, remote_path, &block)
-        @logger.debug("uploading: #{local_path} -> #{remote_path}")
-
-        upload_orchestrator = WinRM::FS::Core::UploadOrchestrator.new(@service)
-        if File.file?(local_path)
-          upload_orchestrator.upload_file(local_path, remote_path, &block)
-        else
-          upload_orchestrator.upload_directory(local_path, remote_path, &block)
+        @service.create_executor do |executor|
+          file_transporter ||= WinRM::FS::Core::FileTransporter.new(executor)
+          file_transporter.upload(local_path, remote_path, &block)[0]
         end
       end
     end

data/lib/winrm-fs/scripts/check_files.ps1.erb

@@ -0,0 +1,48 @@
+$hash_file = "<%= hash_file %>"
+
+Function Cleanup($o) { if (($o -ne $null) -and ($o.GetType().GetMethod("Dispose") -ne $null)) { $o.Dispose() } }
+
+Function Decode-Base64File($src, $dst) {
+  Try {
+    $in = (Get-Item $src).OpenRead()
+    $b64 = New-Object -TypeName System.Security.Cryptography.FromBase64Transform
+    $m = [System.Security.Cryptography.CryptoStreamMode]::Read
+    $d = New-Object -TypeName System.Security.Cryptography.CryptoStream $in,$b64,$m
+    Copy-Stream $d ($out = [System.IO.File]::OpenWrite($dst))
+  } Finally { Cleanup $in; Cleanup $out; Cleanup $d }
+}
+
+Function Copy-Stream($src, $dst) { $b = New-Object Byte[] 4096; while (($i = $src.Read($b, 0, $b.Length)) -ne 0) { $dst.Write($b, 0, $i) } }
+
+Function Check-Files($h) {
+  return $h.GetEnumerator() | ForEach-Object {
+    $dst = Unresolve-Path $_.Key
+    New-Object psobject -Property @{
+      chk_exists = ($exists = Test-Path $dst -PathType Leaf)
+      src_md5 = ($sMd5 = $_.Value)
+      dst_md5 = ($dMd5 = if ($exists) { Get-MD5Sum $dst } else { $null })
+      chk_dirty = ($dirty = if ($sMd5 -ne $dMd5) { $true } else { $false })
+      verifies = if ($dirty -eq $false) { $true } else { $false }
+    }
+  } | Select-Object -Property chk_exists,src_md5,dst_md5,chk_dirty,verifies
+}
+
+Function Get-MD5Sum($src) {
+  Try {
+    $c = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider
+    $bytes = $c.ComputeHash(($in = (Get-Item $src).OpenRead()))
+    return ([System.BitConverter]::ToString($bytes)).Replace("-", "").ToLower()
+  } Finally { Cleanup $c; Cleanup $in }
+}
+
+Function Invoke-Input($in) {
+  $in = Unresolve-Path $in
+  Decode-Base64File $in ($decoded = "$($in).ps1")
+  $expr = Get-Content $decoded | Out-String
+  Remove-Item $in,$decoded -Force
+  return Invoke-Expression "$expr"
+}
+
+Function Unresolve-Path($p) { if ($p -eq $null) { return $null } else { return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($p) } }
+
+Check-Files (Invoke-Input $hash_file) | ConvertTo-Csv -NoTypeInformation

data/lib/winrm-fs/scripts/decode_files.ps1.erb

@@ -0,0 +1,59 @@
+trap {$e = $_.Exception; $e.InvocationInfo.ScriptName; do {$e.Message; $e = $e.InnerException} while ($e); break;}
+$progresspreference = 'SilentlyContinue'
+function Decode-Base64File($src, $dst) {folder (split-path $dst);sc -force -Encoding Byte -Path $dst -Value ([Convert]::FromBase64String([IO.File]::ReadAllLines($src)))}
+function Copy-Stream($src, $dst) {$b = New-Object Byte[] 4096; while (($i = $src.Read($b, 0, $b.Length)) -ne 0) { $dst.Write($b, 0, $i) } }
+function Resolve-ProviderPath{ $input | % {if ($_){(Resolve-Path $_).ProviderPath} else{$null}} }
+function Get-FrameworkVersion { "Full", "Client" | % {([version](gp "HKLM:\Software\Microsoft\NET Framework Setup\NDP\v4\$_").Version)} | select -first 1}
+function Test-NETStack($Version){ Get-FrameworkVersion -ge $Version }
+function Test-IOCompression {($PSVersionTable.PSVersion.Major -ge 3) -and (Test-NETStack '4.5')}
+function folder($path){ $path | ? {-not (test-path $_)} | % {$null = mkdir $_}}
+function disposable($o){($o -is [IDisposable]) -and (($o | gm | %{$_.name}) -contains 'Dispose')}
+function use($obj, [scriptblock]$sb){try {& $sb} catch [exception]{throw $_} finally {if (disposable $obj) {$obj.Dispose()}} }
+set-alias RPP -Value Resolve-ProviderPath
+
+Function Decode-Files($hash) {
+  foreach ($key in $hash.keys) {
+    $value = $hash[$key]
+    $tmp, $tzip, $dst = $Key, $Value["tmpzip"], $Value["dst"]
+    if($tmp -ne "") {
+      $sMd5 = (gi $tmp).BaseName.Replace("b64-", "")
+      $decoded = if ($tzip -ne $null) { $tzip } else { $dst }
+      Decode-Base64File $tmp $decoded
+      rm $tmp -Force
+      $dMd5 = Get-MD5Sum $decoded
+      $verifies = $sMd5 -like $dMd5
+    }
+    if ($tzip) {Unzip-File $tzip $dst}
+    New-Object psobject -Property @{dst=$dst;verifies=$verifies;src_md5=$sMd5;dst_md5=$dMd5;tmpfile=$tmp;tmpzip=$tzip}
+  }
+}
+
+Function Get-MD5Sum($src) {
+  if ($src -and (test-path $src)) {
+  use ($c = New-Object -TypeName Security.Cryptography.MD5CryptoServiceProvider) {
+    use ($in = (gi $src).OpenRead()) {([BitConverter]::ToString($c.ComputeHash($in))).Replace("-", "").ToLower()}}
+  }
+}
+
+Function Invoke-Input($in) {
+  $in = $in | rpp
+  $d = "$($in).ps1"
+  Decode-Base64File $in $d
+  $expr = gc $d | Out-String
+  rm $in,$d -Force
+  iex "$expr"
+}
+
+Function Unzip-File($src, $dst) {
+  $unpack = $src -replace '\.zip'
+  $dst_parent = Split-Path -Path $dst -Parent
+  if(!(Test-Path $dst_parent)) { $dst = $dst_parent }
+  folder $unpack, $dst
+  if (Test-IOCompression) {Add-Type -AN System.IO.Compression.FileSystem; [IO.Compression.ZipFile]::ExtractToDirectory($src, $unpack)}
+  else {Try {$s = New-Object -ComObject Shell.Application; ($s.NameSpace($unpack)).CopyHere(($s.NameSpace($src)).Items(), 0x610)} Finally {[void][Runtime.Interopservices.Marshal]::ReleaseComObject($s)}}
+  dir $unpack | cp -dest "$dst/" -force -recurse
+  rm $unpack -recurse -force
+}
+
+$hash_file = "<%= hash_file %>"
+Decode-Files (Invoke-Input $hash_file) | ConvertTo-Csv -NoTypeInformation

data/lib/winrm-fs/scripts/download.ps1.erb

@@ -1,7 +1,8 @@
 $p = $ExecutionContext.SessionState.Path
 $path = $p.GetUnresolvedProviderPathFromPSPath("<%= path %>")
 if (Test-Path $path -PathType Leaf) {
-  [System.convert]::ToBase64String([System.IO.File]::ReadAllBytes($path))
+  $bytes = [System.convert]::ToBase64String([System.IO.File]::ReadAllBytes($path))
+  Write-Host $bytes
   exit 0
 }
 exit 1

data/spec/config-example.yml

@@ -1,5 +1,5 @@
 auth_type: plaintext
-endpoint: "http://localhost:55985/wsman"
+endpoint: "http://192.168.137.20:5985/wsman"
 options:
   user: vagrant
   pass: vagrant

data/spec/{file_manager_spec.rb → integration/file_manager_spec.rb}

@@ -1,10 +1,10 @@
 # encoding: UTF-8
 require 'pathname'
 
-describe WinRM::FS::FileManager, integration: true do
+describe WinRM::FS::FileManager do
   let(:dest_dir) { File.join(subject.temp_dir, "winrm_#{rand(2**16)}") }
   let(:temp_upload_dir) { '$env:TEMP/winrm-upload' }
-  let(:this_dir) { File.expand_path(File.dirname(__FILE__)) }
+  let(:spec_dir) { File.expand_path(File.dirname(File.dirname(__FILE__))) }
   let(:this_file) { __FILE__ }
   let(:service) { winrm_connection }
 
@@ -35,7 +35,7 @@ describe WinRM::FS::FileManager, integration: true do
 
   context 'temp_dir' do
     it 'should return the remote users temp dir' do
-      expect(subject.temp_dir).to match(%r{C:/Users/\w+/AppData/Local/Temp})
+      expect(subject.temp_dir).to match(%r{C:/Users/\S+/AppData/Local/Temp})
     end
   end
 
@@ -58,7 +58,7 @@ describe WinRM::FS::FileManager, integration: true do
     end
 
     it 'should upload using relative file path' do
-      subject.upload('./spec/file_manager_spec.rb', dest_file)
+      subject.upload('./spec/integration/file_manager_spec.rb', dest_file)
       expect(subject).to have_created(dest_file).with_content(this_file)
     end
 
@@ -88,14 +88,16 @@ describe WinRM::FS::FileManager, integration: true do
 
     it 'yields progress data' do
       block_called = false
+      total_bytes_copied = 0
       total = subject.upload(this_file, dest_file) do \
         |bytes_copied, total_bytes, local_path, remote_path|
         expect(total_bytes).to be > 0
-        expect(bytes_copied).to eq(total_bytes)
+        total_bytes_copied = bytes_copied
         expect(local_path).to eq(this_file)
         expect(remote_path).to eq(dest_file)
         block_called = true
       end
+      expect(total_bytes_copied).to eq(total)
       expect(block_called).to be true
       expect(total).to be > 0
     end
@@ -107,7 +109,7 @@ describe WinRM::FS::FileManager, integration: true do
     end
 
     it 'should upload when content differs' do
-      matchers_file = File.join(this_dir, 'matchers.rb')
+      matchers_file = File.join(spec_dir, 'matchers.rb')
       subject.upload(matchers_file, dest_file)
       bytes_uploaded = subject.upload(this_file, dest_file)
       expect(bytes_uploaded).to be > 0
@@ -119,32 +121,48 @@ describe WinRM::FS::FileManager, integration: true do
   end
 
   context 'upload empty file' do
-    let(:empty_src_file) { Tempfile.new('empty').path }
+    let(:empty_src_file) { Tempfile.new('empty') }
     let(:dest_file) { File.join(dest_dir, 'emptyfile.txt') }
 
     it 'creates a new empty file' do
-      expect(subject.upload(empty_src_file, dest_file)).to be 0
+      expect(subject.upload(empty_src_file.path, dest_file)).to be 0
       expect(subject).to have_created(dest_file).with_content('')
     end
 
     it 'overwrites an existing file' do
       expect(subject.upload(this_file, dest_file)).to be > 0
-      expect(subject.upload(empty_src_file, dest_file)).to be 0
+      expect(subject.upload(empty_src_file.path, dest_file)).to be 0
       expect(subject).to have_created(dest_file).with_content('')
     end
   end
 
   context 'upload directory' do
-    let(:root_dir) { File.expand_path('../', File.dirname(__FILE__)) }
+    let(:root_dir) { File.expand_path('../../', File.dirname(__FILE__)) }
     let(:winrm_fs_dir) { File.join(root_dir, 'lib/winrm-fs') }
     let(:core_dir) { File.join(root_dir, 'lib/winrm-fs/core') }
 
-    it 'copies the entire directory recursively' do
+    it 'copies the directory contents recursively when directory does not exist' do
       bytes_uploaded = subject.upload(winrm_fs_dir, dest_dir)
       expect(bytes_uploaded).to be > 0
 
       Dir.glob(winrm_fs_dir + '/**/*.rb').each do |host_file|
-        host_file_rel = Pathname.new(host_file).relative_path_from(Pathname.new(winrm_fs_dir)).to_s
+        host_file_rel = Pathname.new(host_file).relative_path_from(
+          Pathname.new(winrm_fs_dir)
+        ).to_s
+        remote_file = File.join(dest_dir, host_file_rel)
+        expect(subject).to have_created(remote_file).with_content(host_file)
+      end
+    end
+
+    it 'copies the directory recursively when directory does exist' do
+      subject.create_dir(dest_dir)
+      bytes_uploaded = subject.upload(winrm_fs_dir, dest_dir)
+      expect(bytes_uploaded).to be > 0
+
+      Dir.glob(winrm_fs_dir + '/**/*.rb').each do |host_file|
+        host_file_rel = Pathname.new(host_file).relative_path_from(
+          Pathname.new(winrm_fs_dir).dirname
+        ).to_s
         remote_file = File.join(dest_dir, host_file_rel)
         expect(subject).to have_created(remote_file).with_content(host_file)
       end
@@ -156,6 +174,14 @@ describe WinRM::FS::FileManager, integration: true do
       expect(bytes_uploaded).to eq 0
     end
 
+    it 'unzips the directory when cached content is the same' do
+      subject.upload(winrm_fs_dir, dest_dir)
+      subject.delete(dest_dir)
+      expect(subject.exists?(dest_dir)).to be false
+      subject.upload(winrm_fs_dir, dest_dir)
+      expect(subject.exists?(dest_dir)).to be true
+    end
+
     it 'copies the directory when content differs' do
       subject.upload(winrm_fs_dir, dest_dir)
       bytes_uploaded = subject.upload(core_dir, dest_dir)