dbox 0.6.15 → 0.7.0

data/History.txt

@@ -1,3 +1,8 @@
+== 0.7.0 / 2012-11-05
+  * Major Enhancements
+    * Changed to be case insensitive, like Dropbox.
+    * Removed support for concurrent uploads/downloads, as it was often encountering throttling from Dropbox APIs.
+
 == 0.6.15 / 2012-07-26
   * Bug Fixes
     * Fixed issue with gem in 0.6.14.

data/README.md

@@ -263,12 +263,3 @@ $ export DROPBOX_AUTH_SECRET=pqej9rmnj0i1gcxr4
 > File.read("#{ENV['HOME']}/Dropbox/Public/hello.txt")
 => "Oh, Hello"
 ```
-
-Advanced
---------
-
-To speed up your syncs, you can manually set the number concurrent dropbox operations to execute. (The default is 2.)
-
-```sh
-$ export DROPBOX_CONCURRENCY=5
-```

data/Rakefile

@@ -18,7 +18,7 @@ Jeweler::Tasks.new do |gem|
   gem.add_dependency "oauth", ">= 0.4.5"
   gem.add_dependency "json", ">= 1.5.3"
   gem.add_dependency "sqlite3", ">= 1.3.3"
-  gem.add_dependency "activesupport", ">= 3.0.1"
+  gem.add_dependency "insensitive_hash", ">= 0.3.0"
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/TODO.txt

@@ -1,3 +1,8 @@
+* Upgrade to newest Dropbox Ruby SDK.
+* Look into using the new /chunked_upload support instead of custom streaming upload.
+* Look into using /delta api.
+* Try to get rid of extra updated dirs on pull after push (have push update all the mtimes and such that it affects, so the pull is clean). -- This will clean up the tests more than anything.
+* Ensure Dropbox API best practices are being followed as closely as possible (https://www.dropbox.com/developers/reference/bestpractice).
 * Look into occasional hanging on pull operations.
 * Look into memory leak when syncing a large amount of files.
 * Look into 10000 file limit on metadata operations.

data/VERSION

@@ -1 +1 @@
-0.6.15
+0.7.0

data/dbox.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "dbox"
-  s.version = "0.6.15"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ken Pratt"]
-  s.date = "2012-07-26"
+  s.date = "2012-11-06"
   s.description = "An easy-to-use Dropbox client with fine-grained control over syncs."
   s.email = "ken@kenpratt.net"
   s.executables = ["dbox"]
@@ -32,7 +32,6 @@ Gem::Specification.new do |s|
     "lib/dbox/database.rb",
     "lib/dbox/db.rb",
     "lib/dbox/loggable.rb",
-    "lib/dbox/parallel_tasks.rb",
     "lib/dbox/syncer.rb",
     "lib/dbox/utils.rb",
     "sample_polling_script.rb",
@@ -54,7 +53,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/kenpratt/dbox"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.16"
+  s.rubygems_version = "1.8.24"
   s.summary = "Dropbox made easy."
 
   if s.respond_to? :specification_version then
@@ -65,20 +64,20 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<oauth>, [">= 0.4.5"])
       s.add_runtime_dependency(%q<json>, [">= 1.5.3"])
       s.add_runtime_dependency(%q<sqlite3>, [">= 1.3.3"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 3.0.1"])
+      s.add_runtime_dependency(%q<insensitive_hash>, [">= 0.3.0"])
     else
       s.add_dependency(%q<multipart-post>, [">= 1.1.2"])
       s.add_dependency(%q<oauth>, [">= 0.4.5"])
       s.add_dependency(%q<json>, [">= 1.5.3"])
       s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
-      s.add_dependency(%q<activesupport>, [">= 3.0.1"])
+      s.add_dependency(%q<insensitive_hash>, [">= 0.3.0"])
     end
   else
     s.add_dependency(%q<multipart-post>, [">= 1.1.2"])
     s.add_dependency(%q<oauth>, [">= 0.4.5"])
     s.add_dependency(%q<json>, [">= 1.5.3"])
     s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
-    s.add_dependency(%q<activesupport>, [">= 3.0.1"])
+    s.add_dependency(%q<insensitive_hash>, [">= 0.3.0"])
   end
 end
 

data/lib/dbox.rb

@@ -9,14 +9,13 @@ require "yaml"
 require "logger"
 require "cgi"
 require "sqlite3"
-require "active_support/core_ext/hash/indifferent_access"
+require "insensitive_hash/minimal"
 
 require "dbox/loggable"
 require "dbox/utils"
 require "dbox/api"
 require "dbox/database"
 require "dbox/db"
-require "dbox/parallel_tasks"
 require "dbox/syncer"
 
 module Dbox

data/lib/dbox/api.rb

@@ -89,7 +89,7 @@ module Dbox
     def handle_response(path, res, &else_proc)
       case res
       when Hash
-        HashWithIndifferentAccess.new(res)
+        InsensitiveHash[res]
       when String
         res
       when Net::HTTPNotFound

data/lib/dbox/database.rb

@@ -55,20 +55,21 @@ module Dbox
       @db.execute_batch(%{
         CREATE TABLE IF NOT EXISTS metadata (
           id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-          remote_path  varchar(255) NOT NULL,
+          remote_path  text COLLATE NOCASE UNIQUE NOT NULL,
           version      integer NOT NULL
         );
         CREATE TABLE IF NOT EXISTS entries (
           id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-          path         varchar(255) UNIQUE NOT NULL,
+          path         text COLLATE NOCASE UNIQUE NOT NULL,
           is_dir       boolean NOT NULL,
           parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
-          local_hash   varchar(255),
-          remote_hash  varchar(255),
+          local_hash   text,
+          remote_hash  text,
           modified     datetime,
-          revision     varchar(255)
+          revision     text
         );
         CREATE INDEX IF NOT EXISTS entry_parent_ids ON entries(parent_id);
+        CREATE INDEX IF NOT EXISTS entry_path ON entries(path);
       })
     end
 
@@ -82,7 +83,7 @@ module Dbox
           ALTER TABLE metadata RENAME TO metadata_old;
           CREATE TABLE metadata (
             id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-            remote_path  varchar(255) NOT NULL,
+            remote_path  text NOT NULL,
             version      integer NOT NULL
           );
           INSERT INTO metadata SELECT id, remote_path, version FROM metadata_old;
@@ -121,12 +122,12 @@ module Dbox
           ALTER TABLE entries RENAME TO entries_old;
           CREATE TABLE entries (
             id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-            path         varchar(255) UNIQUE NOT NULL,
+            path         text UNIQUE NOT NULL,
             is_dir       boolean NOT NULL,
             parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
-            hash         varchar(255),
+            hash         text,
             modified     datetime,
-            revision     varchar(255)
+            revision     text
           );
           INSERT INTO entries SELECT id, path, is_dir, parent_id, hash, modified, null FROM entries_old;
         })
@@ -153,13 +154,13 @@ module Dbox
           ALTER TABLE entries RENAME TO entries_old;
           CREATE TABLE entries (
             id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-            path         varchar(255) UNIQUE NOT NULL,
+            path         text UNIQUE NOT NULL,
             is_dir       boolean NOT NULL,
             parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
-            local_hash   varchar(255),
-            remote_hash  varchar(255),
+            local_hash   text,
+            remote_hash  text,
             modified     datetime,
-            revision     varchar(255)
+            revision     text
           );
           INSERT INTO entries SELECT id, path, is_dir, parent_id, null, hash, modified, revision FROM entries_old;
         })
@@ -181,6 +182,50 @@ module Dbox
           COMMIT;
         })
       end
+
+      if metadata[:version] < 5
+        log.info "Migrating to database schema v5"
+
+        # make path be case insensitive
+        @db.execute_batch(%{
+          BEGIN TRANSACTION;
+
+          -- migrate metadata table
+          ALTER TABLE metadata RENAME TO metadata_old;
+          CREATE TABLE IF NOT EXISTS metadata (
+            id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+            remote_path  text COLLATE NOCASE UNIQUE NOT NULL,
+            version      integer NOT NULL
+          );
+          INSERT INTO metadata SELECT id, remote_path, version FROM metadata_old;
+          DROP TABLE metadata_old;
+
+          -- migrate entries table
+          ALTER TABLE entries RENAME TO entries_old;
+          CREATE TABLE entries (
+            id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+            path         text COLLATE NOCASE UNIQUE NOT NULL,
+            is_dir       boolean NOT NULL,
+            parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
+            local_hash   text,
+            remote_hash  text,
+            modified     datetime,
+            revision     text
+          );
+          INSERT INTO entries SELECT id, path, is_dir, parent_id, local_hash, remote_hash, modified, revision FROM entries_old;
+          DROP TABLE entries_old;
+
+          -- recreate indexes
+          DROP INDEX IF EXISTS entry_parent_ids;
+          DROP INDEX IF EXISTS entry_path;
+          CREATE INDEX entry_parent_ids ON entries(parent_id);
+          CREATE INDEX entry_path ON entries(path);
+
+          -- update version
+          UPDATE metadata SET version = 5;
+          COMMIT;
+        })
+      end
     end
 
     METADATA_COLS = [ :remote_path, :version ] # don't need to return id
@@ -189,7 +234,7 @@ module Dbox
     def bootstrap(remote_path)
       @db.execute(%{
         INSERT INTO metadata (remote_path, version) VALUES (?, ?);
-      }, remote_path, 4)
+      }, remote_path, 5)
       @db.execute(%{
         INSERT INTO entries (path, is_dir) VALUES (?, ?)
       }, "", 1)
@@ -341,6 +386,8 @@ module Dbox
         h = make_fields(entry_cols, res)
         h[:is_dir] = (h[:is_dir] == 1)
         h[:modified]  = Time.at(h[:modified]) if h[:modified]
+        h[:local_path] = relative_to_local_path(h[:path])
+        h[:remote_path] = relative_to_remote_path(h[:path])
         h
       else
         nil

data/lib/dbox/syncer.rb

@@ -1,6 +1,5 @@
 module Dbox
   class Syncer
-    DEFAULT_CONCURRENCY = 2
     MIN_BYTES_TO_STREAM_DOWNLOAD = 1024 * 100 # 100kB
 
     include Loggable
@@ -36,11 +35,6 @@ module Dbox
       @@_api ||= API.connect
     end
 
-    def self.concurrency
-      n = ENV["DROPBOX_CONCURRENCY"].to_i
-      n > 0 ? n : DEFAULT_CONCURRENCY
-    end
-
     class Operation
       include Loggable
       include Utils
@@ -53,11 +47,7 @@ module Dbox
       end
 
       def api
-        Thread.current[:api] || @api
-      end
-
-      def clone_api_into_current_thread
-        Thread.current[:api] = api.clone()
+        @api
       end
 
       def metadata
@@ -78,7 +68,7 @@ module Dbox
 
       def current_dir_entries_as_hash(dir)
         if dir[:id]
-          out = {}
+          out = InsensitiveHash.new
           database.contents(dir[:id]).each {|e| out[e[:path]] = e }
           out
         else
@@ -99,17 +89,20 @@ module Dbox
       end
 
       def saving_parent_timestamp(entry, &proc)
-        local_path = relative_to_local_path(entry[:path])
-        parent = File.dirname(local_path)
+        parent = File.dirname(entry[:local_path])
         saving_timestamp(parent, &proc)
       end
 
       def update_file_timestamp(entry)
-        File.utime(Time.now, entry[:modified], relative_to_local_path(entry[:path]))
+        begin
+          File.utime(Time.now, entry[:modified], entry[:local_path])
+        rescue Errno::ENOENT
+          nil
+        end
       end
 
       def gather_remote_info(entry)
-        res = api.metadata(relative_to_remote_path(entry[:path]), entry[:remote_hash])
+        res = api.metadata(entry[:remote_path], entry[:remote_hash])
         case res
         when Hash
           out = process_basic_remote_props(res)
@@ -133,6 +126,8 @@ module Dbox
       def process_basic_remote_props(res)
         out = {}
         out[:path]        = remote_to_relative_path(res[:path])
+        out[:local_path]  = relative_to_local_path(out[:path])
+        out[:remote_path] = relative_to_remote_path(out[:path])
         out[:modified]    = parse_time(res[:modified])
         out[:is_dir]      = res[:is_dir]
         out[:remote_hash] = res[:hash] if res[:hash]
@@ -188,62 +183,57 @@ module Dbox
         parent_ids_of_failed_entries = []
         changelist = { :created => [], :deleted => [], :updated => [], :failed => [] }
 
-        # spin up a parallel task queue
-        ptasks = ParallelTasks.new(Syncer.concurrency) { clone_api_into_current_thread() }
-        ptasks.start
-
         changes.each do |op, c|
           case op
           when :create
             c[:parent_id] ||= lookup_id_by_path(c[:parent_path])
             if c[:is_dir]
-              # directory creation cannot go in a thread, since later
-              # operations might depend on the directory being there
+              # create the local directory
               create_dir(c)
               database.add_entry(c[:path], true, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], nil)
               changelist[:created] << c[:path]
             else
-              ptasks.add do
-                begin
-                  res = create_file(c)
-                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
-                  database.add_entry(c[:path], false, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], local_hash)
-                  changelist[:created] << c[:path]
-                  if res.kind_of?(Array) && res[0] == :conflict
-                    changelist[:conflicts] ||= []
-                    changelist[:conflicts] << res[1]
-                  end
-                rescue Exception => e
-                  log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-                  parent_ids_of_failed_entries << c[:parent_id]
-                  changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
+              # download the new file
+              begin
+                res = create_file(c)
+                local_hash = calculate_hash(c[:local_path])
+                database.add_entry(c[:path], false, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], local_hash)
+                changelist[:created] << c[:path]
+                if res.kind_of?(Array) && res[0] == :conflict
+                  changelist[:conflicts] ||= []
+                  changelist[:conflicts] << res[1]
                 end
+              rescue Exception => e
+                log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+                parent_ids_of_failed_entries << c[:parent_id]
+                changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
               end
             end
           when :update
             if c[:is_dir]
+              # update the local directory
               update_dir(c)
               database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash])
               changelist[:updated] << c[:path]
             else
-              ptasks.add do
-                begin
-                  res = update_file(c)
-                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
-                  database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash], :local_hash => local_hash)
-                  changelist[:updated] << c[:path]
-                  if res.kind_of?(Array) && res[0] == :conflict
-                    changelist[:conflicts] ||= []
-                    changelist[:conflicts] << res[1]
-                  end
-                rescue Exception => e
-                  log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-                  parent_ids_of_failed_entries << c[:parent_id]
-                  changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
+              # download updates to the file
+              begin
+                res = update_file(c)
+                local_hash = calculate_hash(c[:local_path])
+                database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash], :local_hash => local_hash)
+                changelist[:updated] << c[:path]
+                if res.kind_of?(Array) && res[0] == :conflict
+                  changelist[:conflicts] ||= []
+                  changelist[:conflicts] << res[1]
                 end
+              rescue Exception => e
+                log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+                parent_ids_of_failed_entries << c[:parent_id]
+                changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
               end
             end
           when :delete
+            # delete the local directory/file
             c[:is_dir] ? delete_dir(c) : delete_file(c)
             database.delete_entry_by_path(c[:path])
             changelist[:deleted] << c[:path]
@@ -255,9 +245,6 @@ module Dbox
           end
         end
 
-        # wait for operations to finish
-        ptasks.finish
-
         # clear hashes on any dirs with children that failed so that
         # they are processed again on next pull
         parent_ids_of_failed_entries.uniq.each do |id|
@@ -320,24 +307,20 @@ module Dbox
           end
 
           # add any deletions
-          out += (existing_entries.keys.sort - found_paths.sort).map do |p|
+          out += case_insensitive_difference(existing_entries.keys, found_paths).map do |p|
             [:delete, existing_entries[p]]
           end
         end
 
         # recursively process new & existing subdirectories in parallel
-        threads = recur_dirs.map do |operation, dir|
-          Thread.new do
-            begin
-              clone_api_into_current_thread()
-              Thread.current[:out] = calculate_changes(dir, operation)
-            rescue Exception => e
-              log.error "Error while caclulating changes for #{operation} on #{dir[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-              Thread.current[:out] = [[:failed, dir.merge({ :operation => operation, :error => e })]]
-            end
+        recur_dirs.each do |operation, dir|
+          begin
+            out += calculate_changes(dir, operation)
+          rescue Exception => e
+            log.error "Error while caclulating changes for #{operation} on #{dir[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+            out += [[:failed, dir.merge({ :operation => operation, :error => e })]]
           end
         end
-        threads.each {|t| t.join; out += t[:out] }
 
         out
       end
@@ -351,7 +334,7 @@ module Dbox
       end
 
       def create_dir(dir)
-        local_path = relative_to_local_path(dir[:path])
+        local_path = dir[:local_path]
         log.info "Creating #{local_path}"
         saving_parent_timestamp(dir) do
           FileUtils.mkdir_p(local_path)
@@ -364,7 +347,7 @@ module Dbox
       end
 
       def delete_dir(dir)
-        local_path = relative_to_local_path(dir[:path])
+        local_path = dir[:local_path]
         log.info "Deleting #{local_path}"
         saving_parent_timestamp(dir) do
           FileUtils.rm_r(local_path)
@@ -382,7 +365,7 @@ module Dbox
       end
 
       def delete_file(file)
-        local_path = relative_to_local_path(file[:path])
+        local_path = file[:local_path]
         log.info "Deleting file: #{local_path}"
         saving_parent_timestamp(file) do
           FileUtils.rm_rf(local_path)
@@ -390,8 +373,8 @@ module Dbox
       end
 
       def download_file(file)
-        local_path = relative_to_local_path(file[:path])
-        remote_path = relative_to_remote_path(file[:path])
+        local_path = file[:local_path]
+        remote_path = file[:remote_path]
 
         # check to ensure we aren't overwriting an untracked file or a
         # file with local modifications
@@ -449,41 +432,34 @@ module Dbox
         log.debug "Executing changes:\n" + changes.map {|c| c.inspect }.join("\n")
         changelist = { :created => [], :deleted => [], :updated => [], :failed => [] }
 
-        # spin up a parallel task queue
-        ptasks = ParallelTasks.new(Syncer.concurrency) { clone_api_into_current_thread() }
-        ptasks.start
-
         changes.each do |op, c|
           case op
           when :create
             c[:parent_id] ||= lookup_id_by_path(c[:parent_path])
 
             if c[:is_dir]
-              # directory creation cannot go in a thread, since later
-              # operations might depend on the directory being there
+              # create the remote directiory
               create_dir(c)
               database.add_entry(c[:path], true, c[:parent_id], nil, nil, nil, nil)
               force_metadata_update_from_server(c)
               changelist[:created] << c[:path]
             else
-              # spin up a thread to upload the file
-              ptasks.add do
-                begin
-                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
-                  res = upload_file(c)
-                  database.add_entry(c[:path], false, c[:parent_id], nil, nil, nil, local_hash)
-                  if c[:path] == res[:path]
-                    force_metadata_update_from_server(c)
-                    changelist[:created] << c[:path]
-                  else
-                    log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
-                    changelist[:conflicts] ||= []
-                    changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
-                  end
-                rescue Exception => e
-                  log.error "Error while uploading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-                  changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
+              # upload a new file
+              begin
+                local_hash = calculate_hash(c[:local_path])
+                res = upload_file(c)
+                database.add_entry(c[:path], false, c[:parent_id], nil, nil, nil, local_hash)
+                if case_insensitive_equal(c[:path], res[:path])
+                  force_metadata_update_from_server(c)
+                  changelist[:created] << c[:path]
+                else
+                  log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
+                  changelist[:conflicts] ||= []
+                  changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
                 end
+              rescue Exception => e
+                log.error "Error while uploading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+                changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
               end
             end
           when :update
@@ -494,46 +470,41 @@ module Dbox
 
             # only update files -- nothing to do to update a dir
             if !c[:is_dir]
-
-              # spin up a thread to upload the file
-              ptasks.add do
-                begin
-                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
-                  res = upload_file(c)
-                  database.update_entry_by_path(c[:path], :local_hash => local_hash)
-                  if c[:path] == res[:path]
-                    force_metadata_update_from_server(c)
-                    changelist[:updated] << c[:path]
-                  else
-                    log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
-                    changelist[:conflicts] ||= []
-                    changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
-                  end
-                rescue Exception => e
-                  log.error "Error while uploading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-                  changelist[:failed] << { :operation => :update, :path => c[:path], :error => e }
+              # upload changes to a file
+              begin
+                local_hash = calculate_hash(c[:local_path])
+                res = upload_file(c)
+                database.update_entry_by_path(c[:path], :local_hash => local_hash)
+                if case_insensitive_equal(c[:path], res[:path])
+                  force_metadata_update_from_server(c)
+                  changelist[:updated] << c[:path]
+                else
+                  log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
+                  changelist[:conflicts] ||= []
+                  changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
                 end
+              rescue Exception => e
+                log.error "Error while uploading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+                changelist[:failed] << { :operation => :update, :path => c[:path], :error => e }
               end
             end
           when :delete
-            # spin up a thread to delete the file/dir
-            ptasks.add do
+            # delete a remote file/directory
+            begin
               begin
-                begin
-                  if c[:is_dir]
-                    delete_dir(c)
-                  else
-                    delete_file(c)
-                  end
-                rescue Dbox::RemoteMissing
-                  # safe to delete even if remote is already gone
+                if c[:is_dir]
+                  delete_dir(c)
+                else
+                  delete_file(c)
                 end
-                database.delete_entry_by_path(c[:path])
-                changelist[:deleted] << c[:path]
-              rescue Exception => e
-                log.error "Error while deleting #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
-                changelist[:failed] << { :operation => :delete, :path => c[:path], :error => e }
+              rescue Dbox::RemoteMissing
+                # safe to delete even if remote is already gone
               end
+              database.delete_entry_by_path(c[:path])
+              changelist[:deleted] << c[:path]
+            rescue Exception => e
+              log.error "Error while deleting #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
+              changelist[:failed] << { :operation => :delete, :path => c[:path], :error => e }
             end
           when :failed
             changelist[:failed] << { :operation => c[:operation], :path => c[:path], :error => c[:error] }
@@ -542,9 +513,6 @@ module Dbox
           end
         end
 
-        # wait for operations to finish
-        ptasks.finish
-
         # sort & return output
         sort_changelist(changelist)
       end
@@ -559,7 +527,17 @@ module Dbox
         child_paths = list_contents(dir).sort
 
         child_paths.each do |p|
-          c = { :path => p, :modified => mtime(p), :is_dir => is_dir(p), :parent_path => dir[:path], :local_hash => calculate_hash(relative_to_local_path(p)) }
+          local_path = relative_to_local_path(p)
+          remote_path = relative_to_remote_path(p)
+          c = {
+            :path => p,
+            :local_path => local_path,
+            :remote_path => remote_path,
+            :modified => mtime(local_path),
+            :is_dir => is_dir(local_path),
+            :parent_path => dir[:path],
+            :local_hash => calculate_hash(local_path)
+          }
           if entry = existing_entries[p]
             c[:id] = entry[:id]
             recur_dirs << c if c[:is_dir] # queue dir for later
@@ -572,7 +550,7 @@ module Dbox
         end
 
         # add any deletions
-        out += (existing_entries.keys.sort - child_paths).map do |p|
+        out += case_insensitive_difference(existing_entries.keys, child_paths).map do |p|
           [:delete, existing_entries[p]]
         end
 
@@ -585,11 +563,11 @@ module Dbox
       end
 
       def mtime(path)
-        File.mtime(relative_to_local_path(path))
+        File.mtime(path)
       end
 
       def is_dir(path)
-        File.directory?(relative_to_local_path(path))
+        File.directory?(path)
       end
 
       def modified?(entry, res)
@@ -607,30 +585,30 @@ module Dbox
       end
 
       def list_contents(dir)
-        local_path = relative_to_local_path(dir[:path])
+        local_path = dir[:local_path]
         paths = Dir.entries(local_path).reject {|s| s == "." || s == ".." || s.start_with?(".") }
         paths.map {|p| local_to_relative_path(File.join(local_path, p)) }
       end
 
       def create_dir(dir)
-        remote_path = relative_to_remote_path(dir[:path])
+        remote_path = dir[:remote_path]
         log.info "Creating #{remote_path}"
         api.create_dir(remote_path)
       end
 
       def delete_dir(dir)
-        remote_path = relative_to_remote_path(dir[:path])
+        remote_path = dir[:remote_path]
         api.delete_dir(remote_path)
       end
 
       def delete_file(file)
-        remote_path = relative_to_remote_path(file[:path])
+        remote_path = file[:remote_path]
         api.delete_file(remote_path)
       end
 
       def upload_file(file)
-        local_path = relative_to_local_path(file[:path])
-        remote_path = relative_to_remote_path(file[:path])
+        local_path = file[:local_path]
+        remote_path = file[:remote_path]
         db_entry = database.find_by_path(file[:path])
         last_revision = db_entry ? db_entry[:revision] : nil
         res = api.put_file(remote_path, local_path, last_revision)

data/lib/dbox/utils.rb

@@ -25,8 +25,8 @@ module Dbox
 
     # assumes local_path is defined
     def local_to_relative_path(path)
-      if path.include?(local_path)
-        path.sub(local_path, "").sub(/^\//, "")
+      if path =~ /^#{local_path}\/?(.*)$/i
+        $1
       else
         raise BadPath, "Not a local path: #{path}"
       end
@@ -34,8 +34,8 @@ module Dbox
 
     # assumes remote_path is defined
     def remote_to_relative_path(path)
-      if path.include?(remote_path)
-        path.sub(remote_path, "").sub(/^\//, "")
+      if path =~ /^#{remote_path}\/?(.*)$/i
+        $1
       else
         raise BadPath, "Not a remote path: #{path}"
       end
@@ -44,9 +44,9 @@ module Dbox
     # assumes local_path is defined
     def relative_to_local_path(path)
       if path && path.length > 0
-        File.join(local_path, path)
+        case_insensitive_join(local_path, path)
       else
-        local_path
+        case_insensitive_resolve(local_path)
       end
     end
 
@@ -59,6 +59,37 @@ module Dbox
       end
     end
 
+    def case_insensitive_resolve(path)
+      if File.exists?(path)
+        path
+      else
+        matches = Dir.glob(path, File::FNM_CASEFOLD)
+        case matches.size
+        when 0 then path
+        when 1 then matches.first
+        else raise(RuntimeError, "Oops, you have multiple files with the same case. Please delete one of them, as Dropbox is case insensitive. (#{matches.join(', ')})")
+        end
+      end
+    end
+
+    def case_insensitive_join(path, *rest)
+      if rest.length == 0
+        case_insensitive_resolve(path)
+      else
+        rest = rest.map {|s| s.split(File::SEPARATOR) }.flatten
+        case_insensitive_join(File.join(case_insensitive_resolve(path), rest[0]), *rest[1..-1])
+      end
+    end
+
+    def case_insensitive_difference(a, b)
+      b = b.map(&:downcase).sort
+      a.reject {|s| b.include?(s.downcase) }
+    end
+
+    def case_insensitive_equal(a, b)
+      a && b && a.downcase == b.downcase
+    end
+
     def calculate_hash(filepath)
       begin
         Digest::MD5.file(filepath).to_s
@@ -71,7 +102,7 @@ module Dbox
 
     def find_nonconflicting_path(filepath)
       proposed = filepath
-      while File.exists?(proposed)
+      while File.exists?(case_insensitive_resolve(proposed))
         dir, p = File.split(proposed)
         p = p.sub(/^(.*?)( \((\d+)\))?(\..*?)?$/) { "#{$1} (#{$3 ? $3.to_i + 1 : 1})#{$4}" }
         proposed = File.join(dir, p)

data/spec/dbox_spec.rb

@@ -460,6 +460,134 @@ describe Dbox do
       Dbox.sync(@local).should eql(:pull => { :created => ["au_revoir.txt", "farewell.txt", "hello (1).txt"], :deleted => [], :updated => ["", "goodbye.txt"], :failed => [] },
                                    :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
     end
+
+    it "should be able to handle a file that has changed case" do
+      Dbox.create(@remote, @local)
+      make_file "#{@local}/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      rename_file "#{@local}/hello.txt", "#{@local}/HELLO.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [""], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle a file that has changed case remotely" do
+      Dbox.create(@remote, @local)
+      @alternate = "#{ALTERNATE_LOCAL_TEST_PATH}/#{@name}"
+      Dbox.clone(@remote, @alternate)
+      make_file "#{@local}/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["hello.txt"], :deleted => [], :updated => [""], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+      rename_file "#{@local}/hello.txt", "#{@local}/HELLO.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [""], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle a folder that has changed case" do
+      Dbox.create(@remote, @local)
+      mkdir "#{@local}/foo"
+      make_file "#{@local}/foo/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      rename_file "#{@local}/foo", "#{@local}/FOO"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "foo"], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+      make_file "#{@local}/FOO/hello2.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["FOO/hello2.txt"], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle a folder that has changed case remotely" do
+      Dbox.create(@remote, @local)
+      @alternate = "#{ALTERNATE_LOCAL_TEST_PATH}/#{@name}"
+      Dbox.clone(@remote, @alternate)
+      mkdir "#{@local}/foo"
+      make_file "#{@local}/foo/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [""], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+      rename_file "#{@local}/foo", "#{@local}/FOO"
+      make_file "#{@local}/FOO/hello2.txt"
+      make_file "#{@alternate}/foo/hello3.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "foo"], :failed => [] },
+                                   :push => { :created => ["FOO/hello2.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["foo/hello2.txt"], :deleted => [], :updated => ["foo"], :failed => [] },
+                                       :push => { :created => ["foo/hello3.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@local).should eql(:pull => { :created => ["foo/hello3.txt"], :deleted => [], :updated => ["foo"], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle creating a new file of a different case from a deleted file" do
+      Dbox.create(@remote, @local)
+      mkdir "#{@local}/foo"
+      make_file "#{@local}/foo/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      rm_rf "#{@local}/foo"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "foo"], :failed => [] },
+                                   :push => { :created => [], :deleted => ["foo"], :updated => [], :failed => [] })
+      mkdir "#{@local}/FOO"
+      make_file "#{@local}/FOO/HELLO.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [""], :failed => [] },
+                                   :push => { :created => ["FOO", "FOO/HELLO.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "FOO"], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle creating a new file of a different case from a deleted file remotely" do
+      Dbox.create(@remote, @local)
+      @alternate = "#{ALTERNATE_LOCAL_TEST_PATH}/#{@name}"
+      Dbox.clone(@remote, @alternate)
+
+      mkdir "#{@local}/foo"
+      make_file "#{@local}/foo/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [""], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+      rm_rf "#{@alternate}/foo"
+      Dbox.sync(@alternate).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                       :push => { :created => [], :deleted => ["foo"], :updated => [], :failed => [] })
+      mkdir "#{@alternate}/FOO"
+      make_file "#{@alternate}/FOO/HELLO.txt"
+      make_file "#{@alternate}/FOO/HELLO2.txt"
+      Dbox.sync(@alternate).should eql(:pull => { :created => [], :deleted => [], :updated => [""], :failed => [] },
+                                       :push => { :created => ["FOO", "FOO/HELLO.txt", "FOO/HELLO2.txt"], :deleted => [], :updated => [], :failed => [] })
+
+      rename_file "#{@alternate}/FOO", "#{@alternate}/Foo"
+      make_file "#{@alternate}/Foo/Hello3.txt"
+      Dbox.sync(@alternate).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "FOO"], :failed => [] },
+                                       :push => { :created => ["Foo/Hello3.txt"], :deleted => [], :updated => [], :failed => [] })
+
+      Dbox.sync(@local).should eql(:pull => { :created => ["foo/HELLO2.txt", "foo/Hello3.txt"], :deleted => [], :updated => ["", "FOO", "foo/HELLO.txt"], :failed => [] },
+                                   :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
+
+    it "should be able to handle nested directories with case changes" do
+      Dbox.create(@remote, @local)
+      @alternate = "#{ALTERNATE_LOCAL_TEST_PATH}/#{@name}"
+      Dbox.clone(@remote, @alternate)
+
+      mkdir "#{@local}/foo"
+      make_file "#{@local}/foo/hello.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => [], :failed => [] },
+                                   :push => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["foo", "foo/hello.txt"], :deleted => [], :updated => [""], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+
+      rename_file "#{@local}/foo", "#{@local}/FOO"
+      mkdir "#{@local}/FOO/BAR"
+      make_file "#{@local}/FOO/BAR/hello2.txt"
+      Dbox.sync(@local).should eql(:pull => { :created => [], :deleted => [], :updated => ["", "foo"], :failed => [] },
+                                   :push => { :created => ["FOO/BAR", "FOO/BAR/hello2.txt"], :deleted => [], :updated => [], :failed => [] })
+      Dbox.sync(@alternate).should eql(:pull => { :created => ["FOO/BAR/hello2.txt", "foo/BAR"], :deleted => [], :updated => ["foo"], :failed => [] },
+                                       :push => { :created => [], :deleted => [], :updated => [], :failed => [] })
+    end
   end
 
   describe "#move" do

data/spec/spec_helper.rb

@@ -40,6 +40,11 @@ def make_file(filepath, size_in_kb=1)
   `dd if=/dev/urandom of="#{filepath.gsub('"','\"')}" bs=1024 count=#{size_in_kb} 1>/dev/null 2>/dev/null`
 end
 
+def rename_file(oldpath, newpath)
+  FileUtils.mv oldpath, "#{oldpath}-tmp"
+  FileUtils.mv "#{oldpath}-tmp", newpath
+end
+
 RSpec::Matchers.define :exist do
   match do |actual|
     File.exists?(actual) == true

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dbox
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 6
-  - 15
-  version: 0.6.15
+  - 7
+  - 0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Ken Pratt
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-26 00:00:00 Z
+date: 2012-11-06 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: multipart-post
@@ -82,19 +82,19 @@ dependencies:
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: activesupport
+  name: insensitive_hash
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 19
         segments: 
+        - 0
         - 3
         - 0
-        - 1
-        version: 3.0.1
+        version: 0.3.0
   type: :runtime
   version_requirements: *id005
 description: An easy-to-use Dropbox client with fine-grained control over syncs.
@@ -121,7 +121,6 @@ files:
 - lib/dbox/database.rb
 - lib/dbox/db.rb
 - lib/dbox/loggable.rb
-- lib/dbox/parallel_tasks.rb
 - lib/dbox/syncer.rb
 - lib/dbox/utils.rb
 - sample_polling_script.rb
@@ -168,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.16
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Dropbox made easy.

data/lib/dbox/parallel_tasks.rb

@@ -1,80 +0,0 @@
-require "thread"
-
-#
-# Usage:
-#
-#  puts "Creating task queue with 5 concurrent workers"
-#  tasks = ParallelTasks.new(5) { puts "Worker thread starting up" }
-#
-#  puts "Starting workers"
-#  tasks.start
-#
-#  puts "Making some work"
-#  20.times do
-#    tasks.add do
-#      x = rand(5)
-#      puts "Sleeping for #{x}s"
-#      sleep x
-#    end
-#  end
-#
-#  puts "Waiting for workers to finish"
-#  tasks.finish
-#
-#  puts "Done"
-#
-class ParallelTasks
-  def initialize(num_workers, &initialization_proc)
-    @num_workers = num_workers
-    @initialization_proc = initialization_proc
-    @workers = []
-    @work_queue = Queue.new
-    @semaphore = Mutex.new
-    @done_making_tasks = false
-  end
-
-  def start
-    @num_workers.times do
-      @workers << Thread.new do
-        @initialization_proc.call if @initialization_proc
-        done = false
-        while !done
-          task = nil
-          @semaphore.synchronize do
-            unless @work_queue.empty?
-              task = @work_queue.pop()
-            else
-              if @done_making_tasks
-                done = true
-              else
-                sleep 0.1
-              end
-            end
-          end
-          if task
-            begin
-              task.call
-            rescue Exception => e
-              log.error e.inspect
-            end
-          end
-        end
-      end
-    end
-  end
-
-  def add(&proc)
-    @work_queue << proc
-  end
-
-  def finish
-    @done_making_tasks = true
-    begin
-      @workers.each {|t| t.join }
-    rescue Exception => e
-      log.error "Error waiting for workers to complete tasks: #{e.inspect}"
-      @workers.each {|t| t.kill }
-      raise e
-    end
-  end
-end