odisk 0.2.2 → 0.2.3

data/README.md

@@ -25,9 +25,11 @@ which utilizes an alternative approach to dealing with multiple threads.
 
 ## <a name="release">Release Notes</a>
 
-### Release 0.2.2
+### Release 0.2.3
 
- - Added odisk_forget which removes remote entries causing a file or directory to look like a new file or directory.
+ - Added odisk_forget which clears the local and remote digest of a specific file or directory.
+
+ - Added odisk_remove which marks a file or directory as removed so that all file or directory is removed from all local copies.
 
 # Plans and Notes
 
@@ -46,12 +48,6 @@ know when you run into bugs.
   - pass array of ignores to digest creation
    - loosen up current restriction on any file that begins with .
 
- - Support file removal
-  - Detect file removal based on previous digest or by calling a remove script
-  - Keep record of removals in digest
-  - Note conflicts if modifications are more recent that removal
-   - Use a script to pick remove or keep
-
  - Handle changes in mode, owner, and group
   - Compare to previous digest to detect changes
    - File modification times are not changed by mode, owner, or group changes

data/bin/odisk

@@ -33,12 +33,18 @@ $plain = false
 $remote = ::ODisk::Remote.new()
 $crypter_count = 2
 $copier_count = 4
+$careful = false
 
-opts = OptionParser.new('Usage: odisk [options] <local_directory>')
+opts = OptionParser.new(%{Usage: odisk [options] <local_directory>
+
+Synchronizes a local directory with a remote directory. The remote directory
+is encrypted and compressed.
+})
 opts.on('-s', 'decrease verbosity')                 { $verbose += 1 unless 5 == $verbose }
 opts.on('-v', 'increase verbosity')                 { $verbose -= 1 unless 0 == $verbose }
 opts.on('-d', 'generate digests only')              { $digests_only = true }
 opts.on('-u', 'unencrypted or plain remote files')  { $plain = true }
+opts.on('-c', 'careful mode, no local removes')     { $careful = true }
 opts.on('-n', 'dry run / no modifications')         { $dry_run = true }
 opts.on('-r', '--remote [user@host:dir:pass_file]',
         String, 'remote user, host, directory, passphrase file for gpg')  { |r|

data/bin/odisk_forget

@@ -59,7 +59,7 @@ unless ODisk.gather_remote_info($local_top, $remote)
 end
 $remote.pass_file = nil if $plain
 
-Thread.current[:name] = 'main'
+Thread.current[:name] = 'forget'
 ::Opee::Env.logger.formatter = proc { |s,t,p,m|
   s = '' if s.nil?
   "#{s[0]} [#{t.strftime('%Y-%m-%dT%H:%M:%S.%6N')} ##{p}]: #{m}\n"
@@ -67,13 +67,7 @@ Thread.current[:name] = 'main'
 ::Opee::Env.logger.severity = $verbose
 
 if Logger::INFO >= $verbose
-  if $digests_only
-    ::Opee::Env.info(%{
-  Generate Local Digests
-    local directory:  #{::File.expand_path($local_top)}
-})
-  else
-    ::Opee::Env.info(%{
+  ::Opee::Env.info(%{
   Forget "#{$forget_me}"
     remote host:      #{$remote.host}
     remote user:      #{$remote.user}
@@ -81,10 +75,9 @@ if Logger::INFO >= $verbose
     local:            #{::File.expand_path($local_top)}
     dry run:          #{$dry_run}
 })
-  end
 end
 
-# If $local_top/.odisk/remote does not exist or is different that what is in $remote, replace it.
+# If $local_top/.odisk/remote does not exist or is different than what is in $remote, replace it.
 remote_str = $remote.to_s
 top_remote_path = ::File.join($local_top, '.odisk', 'remote')
 if !::File.file?(top_remote_path) || ::File.read(top_remote_path).strip() != remote_str

data/bin/odisk_remove

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+while (i = ARGV.index('-I'))
+  x,path = ARGV.slice!(i, 2)
+  $: << path
+end
+
+# TBD tmp for testing
+$: << ::File.join(::File.dirname(__FILE__), "../../oj/lib")
+$: << ::File.join(::File.dirname(__FILE__), "../../oj/ext")
+$: << ::File.join(::File.dirname(__FILE__), "../../opee/lib")
+$: << ::File.join(::File.dirname(__FILE__), "../lib")
+
+require 'optparse'
+begin
+  v = $VERBOSE
+  $VERBOSE = false
+  require 'net/ssh'
+  require 'net/sftp'
+  $VERBOSE = v
+end
+require 'opee'
+require 'oj'
+require 'odisk'
+
+$verbose = Logger::WARN
+$dry_run = false
+$dir = '.'
+$master = nil
+$remote = ::ODisk::Remote.new()
+
+opts = OptionParser.new(%{Usage: odisk_remove [options] <local_directory> <relative_remove_path>
+
+Will remove data for the file or directory specified from the local file system as well as on the
+remote server. The path will also be marked indication that the file or directory should be removed
+from other local directories if that directory is synced with the remote site. To get rid of the
+remove flag for the path use the oDisk_cleanup command or sync with a local as master.
+})
+opts.on('-s', 'decrease verbosity')                 { $verbose += 1 unless 5 == $verbose }
+opts.on('-v', 'increase verbosity')                 { $verbose -= 1 unless 0 == $verbose }
+opts.on('-n', 'dry run / no modifications')         { $dry_run = true }
+opts.on('-r', '--remote [user@host:dir:pass_file]',
+        String, 'remote user, host, directory, passphrase file for gpg')  { |r|
+  $remote.update(r)
+}
+opts.on('-h', '--help', 'Show this display')        { puts opts.help; Process.exit!(0) }
+dirs = opts.parse(ARGV)
+
+if 2 != dirs.size
+  puts opts.help
+  Process.exit!(0)
+end
+$local_top = ::File.expand_path(dirs[0])
+$remove_me = dirs[1] # relative path from top
+
+#  TBD move this to odisk.rb
+unless ODisk.gather_remote_info($local_top, $remote)
+  puts opts.help
+  Process.exit!(0)
+end
+$remote.pass_file = nil if $plain
+
+Thread.current[:name] = 'remove'
+::Opee::Env.logger.formatter = proc { |s,t,p,m|
+  s = '' if s.nil?
+  "#{s[0]} [#{t.strftime('%Y-%m-%dT%H:%M:%S.%6N')} ##{p}]: #{m}\n"
+}
+::Opee::Env.logger.severity = $verbose
+
+if Logger::INFO >= $verbose
+  ::Opee::Env.info(%{
+  Remove "#{$remove_me}"
+    remote host:      #{$remote.host}
+    remote user:      #{$remote.user}
+    remote directory: #{$remote.dir}
+    local:            #{::File.expand_path($local_top)}
+    dry run:          #{$dry_run}
+})
+end
+
+# If $local_top/.odisk/remote does not exist or is different than what is in $remote, replace it.
+remote_str = $remote.to_s
+top_remote_path = ::File.join($local_top, '.odisk', 'remote')
+if !::File.file?(top_remote_path) || ::File.read(top_remote_path).strip() != remote_str
+  ::Opee::Env.info("Writing #{top_remote_path}")
+  unless $dry_run
+  `mkdir -p #{::File.join($local_top, '.odisk')}`
+  ::File.open(top_remote_path, 'w') { |f| f.write(remote_str + "\n") }
+  end
+end
+
+path = ::File.join($local_top, $remove_me)
+local_digest_path = ::File.join(::File.dirname(path), '.odisk', 'digest.json')
+remove_name = ::File.basename(path)
+remote_path = ::File.join($remote.dir, $remove_me)
+remote_digest_path = ::File.join(::File.dirname(remote_path), '.odisk', 'digest.json')
+
+# Removed the local file.
+`rm -rf "#{path}"`
+
+# Remove remove_me from local digest.
+digest = Oj.load_file(local_digest_path, mode: :object)
+digest.delete(remove_name)
+Oj.to_file(local_digest_path, digest, indent: 2) unless $dry_run
+
+# Removed the remote file or directory.
+Net::SSH.start($remote.host, $remote.user) do |ssh|
+  ssh.exec!(%{rm -rf "#{remote_path}.gpg"})
+  ssh.exec!(%{rm -rf "#{remote_path}"})
+end
+
+# Set the removed flag on the remove_me entry in the remote digest.
+Net::SFTP.start($remote.host, $remote.user) do |ftp|
+  json = ftp.download!(remote_digest_path)
+  digest = Oj.load(json, mode: :object)
+  unless (entry = digest[remove_name]).nil?
+    entry.removed = true
+  end
+  json = Oj.dump(digest, indent: 2)
+  ftp.open!(remote_digest_path, 'w') { |res| ftp.write!(res[:handle], 0, json) }
+end

data/lib/odisk/copier.rb

@@ -1,3 +1,10 @@
+begin
+  v = $VERBOSE
+  $VERBOSE = false
+  require 'net/ssh'
+  require 'net/sftp'
+  $VERBOSE = v
+end
 
 module ODisk
   # Provides upload and download functionality using sftp and ssh on a single connection.
@@ -36,8 +43,12 @@ module ODisk
           ::Opee::Env.warn("Uploaded \"#{local}\"")
         rescue Net::SFTP::StatusException => e
           if Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE == e.code
-            assure_dirs_exist(::File.dirname(remote))
-            retry
+            begin
+              assure_dirs_exist(::File.dirname(remote))
+              retry
+            rescue Exception => e2
+              ::Opee::Env.error("Upload of \"#{local}\" failed: #{e2.class}: #{e2.message}")
+            end
           else
             ::Opee::Env.error("Upload of \"#{local}\" failed: #{e.class}: (#{e.code}) #{e.description}\n #{e.text}\n #{e.response}")
           end
@@ -64,11 +75,27 @@ module ODisk
       @copy_queue.ask(:ready, self)
     end
 
+    def remove_local(path)
+      `rm -rf "#{path}"` unless $dry_run
+      ::Opee::Env.warn("Removed local \"#{path}\"")
+    end
+
+    def remove_remote(path)
+      unless $dry_runb
+        @ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
+        out = @ssh.exec!(%{rm -rf "#{path}" "#{path}.gpg"})
+        raise out unless out.nil? || out.strip().empty?
+      end
+      ::Opee::Env.warn("Removed remote \"#{path}\"")
+    end
+
     def assure_dirs_exist(dir)
       ::Opee::Env.info("creating remote dir \"#{dir}\"")
-      @ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
-      out = @ssh.exec!(%{mkdir -p "#{dir}"})
-      raise out unless out.nil? || out.strip().empty?
+      unless $dry_run
+        @ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
+        out = @ssh.exec!(%{mkdir -p "#{dir}"})
+        raise 'Remote ' + out unless out.nil? || out.strip().empty?
+      end
     end
 
   end # Copier

data/lib/odisk/fetcher.rb

@@ -42,7 +42,10 @@ module ODisk
         files = @ftp.dir.entries(top).map {|e| e.name }
         job.remote_digest = Oj.load(json, mode: :object)
         missing = []
-        job.remote_digest.entries.each { |e| missing << e.name unless files.include?(e.name + '.gpg') || files.include?(e.name) || e.is_a?(::ODisk::Link) }
+        job.remote_digest.entries.each { |e| missing << e.name unless (files.include?(e.name + '.gpg') ||
+                                                                       files.include?(e.name) ||
+                                                                       e.is_a?(::ODisk::Link) ||
+                                                                       e.removed) }
         unless ::ODisk::Planner::Step::REMOTE == $master
           missing.each { |name| job.remote_digest.delete(name) }
         end

data/lib/odisk/planner.rb

@@ -54,16 +54,16 @@ module ODisk
           if le.class != re.class
             ::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Local and remote types do not match.")
             steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
+          elsif le.removed
+            ::Opee::Env.error("Unexpected digest entry for #{ld.top_path}/#{name}. The removed flag is sent in the local digest.")
+          elsif re.removed
+            steps[name] = Step.new(name, Step::LOCAL, Step::REMOVE)
           elsif le.is_a?(::ODisk::File) || le.is_a?(::ODisk::Link)
             op = le.is_a?(::ODisk::File) ? Step::COPY : Step::LINK
             if Step::LOCAL == master
               steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
             elsif Step::REMOTE == master
               steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
-            elsif le.removed
-              # TBD
-            elsif re.removed
-              # TBD
             elsif le.mtime > re.mtime
               pe = ph[name]
               if pe.nil? || pe.mtime == re.mtime
@@ -196,8 +196,21 @@ module ODisk
 
     def process_sync(job, odisk_dir)
       dirs = []
-      job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
-      job.remote_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
+      if Step::REMOTE == $master
+        job.remote_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
+      elsif Step::LOCAL == $master
+        job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
+      else
+        job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
+        job.remote_digest.entries.each do |e|
+          next unless e.is_a?(::ODisk::Dir)
+          if e.removed
+            dirs.delete(e.name)
+          else
+            dirs << e.name unless dirs.include?(e.name)
+          end
+        end
+      end
       dirs.each do |dir|
         path = job.path.empty? ? dir : ::File.join(job.path, dir)
         local = ::File.join($local_top, path)
@@ -250,7 +263,16 @@ module ODisk
           nrh[e.name] = e
           nlh[e.name] = e
         when Step::REMOVE
-          # TBD
+          path = job.path.empty? ? s.name : ::File.join(job.path, s.name)
+          if Step::REMOTE == s.master
+            remote = ::File.join($remote.dir, path)
+            @copy_queue.add_method(:remove_remote, remote)
+            nrh.delete(s.name)
+          else
+            local = ::File.join($local_top, path)
+            @copy_queue.add_method(:remove_local, local)
+            nlh.delete(s.name)
+          end
         when Step::DIGEST
           e = (Step::REMOTE == s.master) ? job.remote_digest[s.name] : job.current_digest[s.name]
           nrh[e.name] = e

data/lib/odisk/version.rb

@@ -1,5 +1,5 @@
 
 module ODisk
   # Current version of the module. 
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: odisk
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-16 00:00:00.000000000 Z
+date: 2012-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opee
@@ -80,6 +80,7 @@ email: peter@ohler.com
 executables:
 - odisk
 - odisk_forget
+- odisk_remove
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -105,8 +106,12 @@ files:
 - lib/odisk.rb
 - LICENSE
 - README.md
-- bin/odisk
-- bin/odisk_forget
+- !binary |-
+  YmluL29kaXNr
+- !binary |-
+  YmluL29kaXNrX2ZvcmdldA==
+- !binary |-
+  YmluL29kaXNrX3JlbW92ZQ==
 homepage: http://www.ohler.com/odisk
 licenses: []
 post_install_message: 
@@ -129,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: odisk
-rubygems_version: 1.8.23
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Remote Encrypted File Synchronization, oDisk