dbox 0.5.3 → 0.6.0

data/lib/dbox/database.rb

@@ -3,6 +3,7 @@ module Dbox
 
   class Database
     include Loggable
+    include Utils
 
     DB_FILENAME = ".dbox.sqlite3"
 
@@ -61,16 +62,20 @@ module Dbox
           path         varchar(255) UNIQUE NOT NULL,
           is_dir       boolean NOT NULL,
           parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
-          hash         varchar(255),
+          local_hash   varchar(255),
+          remote_hash  varchar(255),
           modified     datetime,
-          revision     integer
+          revision     varchar(255)
         );
         CREATE INDEX IF NOT EXISTS entry_parent_ids ON entries(parent_id);
       })
     end
 
     def migrate
+      # removing local_path from metadata
       if metadata[:version] < 2
+        log.info "Migrating to database schema v2"
+
         @db.execute_batch(%{
           BEGIN TRANSACTION;
           ALTER TABLE metadata RENAME TO metadata_old;
@@ -85,15 +90,105 @@ module Dbox
           COMMIT;
         })
       end
+
+      # migrating to new Dropbox API 1.0 (from integer revisions to
+      # string revisions)
+      if metadata[:version] < 3
+        log.info "Migrating to database schema v3"
+
+        api = API.connect
+        new_revisions = {}
+
+        # fetch the new revision IDs from dropbox
+        find_entries().each do |entry|
+          path = relative_to_remote_path(entry[:path])
+          begin
+            data = api.metadata(path, nil, false)
+            # record nev revision ("rev") iff old revisions ("revision") match
+            if entry[:revision] == data["revision"]
+              new_revisions[entry[:id]] = data["rev"]
+            end
+          rescue Dbox::ServerError => e
+            log.error e
+          end
+        end
+
+        # modify the table to have a string for revision (blanked out
+        # for each entry)
+        @db.execute_batch(%{
+          BEGIN TRANSACTION;
+          ALTER TABLE entries RENAME TO entries_old;
+          CREATE TABLE entries (
+            id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+            path         varchar(255) UNIQUE NOT NULL,
+            is_dir       boolean NOT NULL,
+            parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
+            hash         varchar(255),
+            modified     datetime,
+            revision     varchar(255)
+          );
+          INSERT INTO entries SELECT id, path, is_dir, parent_id, hash, modified, null FROM entries_old;
+        })
+
+        # copy in the new revision IDs
+        new_revisions.each do |id, revision|
+          update_entry_by_id(id, :revision => revision)
+        end
+
+        # drop old table and commit
+        @db.execute_batch(%{
+          DROP TABLE entries_old;
+          UPDATE metadata SET version = 3;
+          COMMIT;
+        })
+      end
+
+      if metadata[:version] < 4
+        log.info "Migrating to database schema v4"
+
+        # add local_hash column, rename hash to remote_hash
+        @db.execute_batch(%{
+          BEGIN TRANSACTION;
+          ALTER TABLE entries RENAME TO entries_old;
+          CREATE TABLE entries (
+            id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+            path         varchar(255) UNIQUE NOT NULL,
+            is_dir       boolean NOT NULL,
+            parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
+            local_hash   varchar(255),
+            remote_hash  varchar(255),
+            modified     datetime,
+            revision     varchar(255)
+          );
+          INSERT INTO entries SELECT id, path, is_dir, parent_id, null, hash, modified, revision FROM entries_old;
+        })
+
+        # calculate hashes on files with same timestamp as we have (as that was the previous mechanism used to check freshness)
+        find_entries().each do |entry|
+          unless entry[:is_dir]
+            path = relative_to_local_path(entry[:path])
+            if times_equal?(File.mtime(path), entry[:modified])
+              update_entry_by_id(entry[:id], :local_hash => calculate_hash(path))
+            end
+          end
+        end
+
+        # drop old table and commit
+        @db.execute_batch(%{
+          DROP TABLE entries_old;
+          UPDATE metadata SET version = 4;
+          COMMIT;
+        })
+      end
     end
 
     METADATA_COLS = [ :remote_path, :version ] # don't need to return id
-    ENTRY_COLS    = [ :id, :path, :is_dir, :parent_id, :hash, :modified, :revision ]
+    ENTRY_COLS    = [ :id, :path, :is_dir, :parent_id, :local_hash, :remote_hash, :modified, :revision ]
 
     def bootstrap(remote_path)
       @db.execute(%{
         INSERT INTO metadata (remote_path, version) VALUES (?, ?);
-      }, remote_path, 2)
+      }, remote_path, 4)
       @db.execute(%{
         INSERT INTO entries (path, is_dir) VALUES (?, ?)
       }, "", 1)
@@ -116,6 +211,10 @@ module Dbox
       out
     end
 
+    def remote_path
+      metadata()[:remote_path]
+    end
+
     def update_metadata(fields)
       set_str = fields.keys.map {|k| "#{k}=?" }.join(",")
       @db.execute(%{
@@ -142,8 +241,8 @@ module Dbox
       find_entries("WHERE parent_id=? AND is_dir=1", dir_id)
     end
 
-    def add_entry(path, is_dir, parent_id, modified, revision, hash)
-      insert_entry(:path => path, :is_dir => is_dir, :parent_id => parent_id, :modified => modified, :revision => revision, :hash => hash)
+    def add_entry(path, is_dir, parent_id, modified, revision, remote_hash, local_hash)
+      insert_entry(:path => path, :is_dir => is_dir, :parent_id => parent_id, :modified => modified, :revision => revision, :remote_hash => remote_hash, :local_hash => local_hash)
     end
 
     def update_entry_by_id(id, fields)
@@ -172,7 +271,7 @@ module Dbox
 
     def migrate_entry_from_old_db_format(entry, parent = nil)
       # insert entry into sqlite db
-      add_entry(entry.path, entry.dir?, (parent ? parent[:id] : nil), entry.modified_at, entry.revision, nil)
+      add_entry(entry.path, entry.dir?, (parent ? parent[:id] : nil), entry.modified_at, entry.revision, nil, nil)
 
       # recur on children
       if entry.dir?
@@ -233,7 +332,6 @@ module Dbox
         h = make_fields(ENTRY_COLS, res)
         h[:is_dir] = (h[:is_dir] == 1)
         h[:modified]  = Time.at(h[:modified]) if h[:modified]
-        h.delete(:hash) unless h[:is_dir]
         h
       else
         nil

data/lib/dbox/syncer.rb

@@ -1,6 +1,7 @@
 module Dbox
   class Syncer
     MAX_PARALLEL_DBOX_OPS = 5
+    MIN_BYTES_TO_STREAM_DOWNLOAD = 1024 * 100 # 100kB
 
     include Loggable
 
@@ -37,6 +38,7 @@ module Dbox
 
     class Operation
       include Loggable
+      include Utils
 
       attr_reader :database
 
@@ -65,38 +67,6 @@ module Dbox
         metadata[:remote_path]
       end
 
-      def local_to_relative_path(path)
-        if path.include?(local_path)
-          path.sub(local_path, "").sub(/^\//, "")
-        else
-          raise BadPath, "Not a local path: #{path}"
-        end
-      end
-
-      def remote_to_relative_path(path)
-        if path.include?(remote_path)
-          path.sub(remote_path, "").sub(/^\//, "")
-        else
-          raise BadPath, "Not a remote path: #{path}"
-        end
-      end
-
-      def relative_to_local_path(path)
-        if path && path.length > 0
-          File.join(local_path, path)
-        else
-          local_path
-        end
-      end
-
-      def relative_to_remote_path(path)
-        if path && path.length > 0
-          File.join(remote_path, path)
-        else
-          remote_path
-        end
-      end
-
       def remove_dotfiles(contents)
         contents.reject {|c| File.basename(c[:path]).start_with?(".") }
       end
@@ -116,29 +86,11 @@ module Dbox
         @_ids[path] ||= database.find_by_path(path)[:id]
       end
 
-      def time_to_s(t)
-        case t
-        when Time
-          # matches dropbox time format
-          t.utc.strftime("%a, %d %b %Y %H:%M:%S +0000")
-        when String
-          t
-        end
-      end
-
-      def parse_time(t)
-        case t
-        when Time
-          t
-        when String
-          Time.parse(t)
-        end
-      end
-
       def saving_timestamp(path)
         mtime = File.mtime(path)
-        yield
+        res = yield
         File.utime(Time.now, mtime, path)
+        res
       end
 
       def saving_parent_timestamp(entry, &proc)
@@ -152,7 +104,7 @@ module Dbox
       end
 
       def gather_remote_info(entry)
-        res = api.metadata(relative_to_remote_path(entry[:path]), entry[:hash])
+        res = api.metadata(relative_to_remote_path(entry[:path]), entry[:remote_hash])
         case res
         when Hash
           out = process_basic_remote_props(res)
@@ -175,11 +127,12 @@ module Dbox
 
       def process_basic_remote_props(res)
         out = {}
-        out[:path]     = remote_to_relative_path(res[:path])
-        out[:modified] = parse_time(res[:modified])
-        out[:is_dir]   = res[:is_dir]
-        out[:hash]     = res[:hash] if res[:hash]
-        out[:revision] = res[:revision] if res[:revision]
+        out[:path]        = remote_to_relative_path(res[:path])
+        out[:modified]    = parse_time(res[:modified])
+        out[:is_dir]      = res[:is_dir]
+        out[:remote_hash] = res[:hash] if res[:hash]
+        out[:revision]    = res[:rev] if res[:rev]
+        out[:size]        = res[:bytes] if res[:bytes]
         out
       end
 
@@ -228,14 +181,19 @@ module Dbox
               # directory creation cannot go in a thread, since later
               # operations might depend on the directory being there
               create_dir(c)
-              database.add_entry(c[:path], true, c[:parent_id], c[:modified], c[:revision], c[:hash])
+              database.add_entry(c[:path], true, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], nil)
               changelist[:created] << c[:path]
             else
               ptasks.add do
                 begin
-                  create_file(c)
-                  database.add_entry(c[:path], false, c[:parent_id], c[:modified], c[:revision], c[:hash])
+                  res = create_file(c)
+                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
+                  database.add_entry(c[:path], false, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], local_hash)
                   changelist[:created] << c[:path]
+                  if res.kind_of?(Array) && res[0] == :conflict
+                    changelist[:conflicts] ||= []
+                    changelist[:conflicts] << res[1]
+                  end
                 rescue Dbox::ServerError => e
                   log.error "Error while downloading #{c[:path]}: #{e.inspect}"
                   parent_ids_of_failed_entries << c[:parent_id]
@@ -246,14 +204,19 @@ module Dbox
           when :update
             if c[:is_dir]
               update_dir(c)
-              database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :hash => c[:hash])
+              database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash])
               changelist[:updated] << c[:path]
             else
               ptasks.add do
                 begin
-                  update_file(c)
-                  database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :hash => c[:hash])
+                  res = update_file(c)
+                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
+                  database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash], :local_hash => local_hash)
                   changelist[:updated] << c[:path]
+                  if res.kind_of?(Array) && res[0] == :conflict
+                    changelist[:conflicts] ||= []
+                    changelist[:conflicts] << res[1]
+                  end
                 rescue Dbox::ServerError => e
                   log.error "Error while downloading #{c[:path]}: #{e.inspect}"
                   parent_ids_of_failed_entries << c[:parent_id]
@@ -276,11 +239,11 @@ module Dbox
         # clear hashes on any dirs with children that failed so that
         # they are processed again on next pull
         parent_ids_of_failed_entries.uniq.each do |id|
-          database.update_entry_by_id(id, :hash => nil)
+          database.update_entry_by_id(id, :remote_hash => nil)
         end
 
         # sort & return output
-        changelist.keys.each {|k| changelist[k].sort! }
+        changelist.keys.each {|k| k == :conflicts ? changelist[k].sort! {|c1, c2| c1[:original] <=> c2[:original] } : changelist[k].sort! }
         changelist
       end
 
@@ -317,7 +280,7 @@ module Dbox
               c[:modified] = parse_time(c[:modified])
               if c[:is_dir]
                 # queue dir for later
-                c[:hash] = entry[:hash]
+                c[:remote_hash] = entry[:remote_hash]
                 recur_dirs << [:update, c]
               else
                 # update iff modified
@@ -355,9 +318,9 @@ module Dbox
 
       def modified?(entry, res)
         out = (entry[:revision] != res[:revision]) ||
-              (time_to_s(entry[:modified]) != time_to_s(res[:modified]))
-        out ||= (entry[:hash] != res[:hash]) if res.has_key?(:hash)
-        log.debug "#{entry[:path]} modified? r#{entry[:revision]} vs. r#{res[:revision]}, h#{entry[:hash]} vs. h#{res[:hash]}, t#{time_to_s(entry[:modified])} vs. t#{time_to_s(res[:modified])} => #{out}"
+              !times_equal?(entry[:modified], res[:modified])
+        out ||= (entry[:remote_hash] != res[:remote_hash]) if res.has_key?(:remote_hash)
+        log.debug "#{entry[:path]} modified? r#{entry[:revision]} vs. r#{res[:revision]}, h#{entry[:remote_hash]} vs. h#{res[:remote_hash]}, t#{time_to_s(entry[:modified])} vs. t#{time_to_s(res[:modified])} => #{out}"
         out
       end
 
@@ -404,15 +367,43 @@ module Dbox
         local_path = relative_to_local_path(file[:path])
         remote_path = relative_to_remote_path(file[:path])
 
-        # stream download to temp file, then atomic move to real path
+        # check to ensure we aren't overwriting an untracked file or a
+        # file with local modifications
+        clobbering = false
+        if entry = database.find_by_path(file[:path])
+          clobbering = calculate_hash(local_path) != entry[:local_hash]
+        else
+          clobbering = File.exists?(local_path)
+        end
+
+        # stream files larger than the minimum
+        stream = file[:size] && file[:size] > MIN_BYTES_TO_STREAM_DOWNLOAD
+
+        # download to temp file
         tmp = generate_tmpfilename(file[:path])
         File.open(tmp, "w") do |f|
-          api.get_file(remote_path, f)
+          api.get_file(remote_path, f, stream)
+        end
+
+        # rename old file if clobbering
+        if clobbering && File.exists?(local_path)
+          backup_path = find_nonconflicting_path(local_path)
+          FileUtils.mv(local_path, backup_path)
+          backup_relpath = local_to_relative_path(backup_path)
+          log.warn "#{file[:path]} had a conflict and the existing copy was renamed to #{backup_relpath} locally"
         end
-        FileUtils.mv(tmp, local_path)
 
+        # atomic move over to the real file, and update the timestamp
+        FileUtils.mv(tmp, local_path)
         update_file_timestamp(file)
+
+        if backup_relpath
+          [:conflict, { :original => file[:path], :renamed => backup_relpath }]
+        else
+          true
+        end
       end
+
     end
 
     class Push < Operation
@@ -445,17 +436,24 @@ module Dbox
               # directory creation cannot go in a thread, since later
               # operations might depend on the directory being there
               create_dir(c)
-              database.add_entry(c[:path], true, c[:parent_id], nil, nil, nil)
+              database.add_entry(c[:path], true, c[:parent_id], nil, nil, nil, nil)
               force_metadata_update_from_server(c)
               changelist[:created] << c[:path]
             else
               # spin up a thread to upload the file
               ptasks.add do
                 begin
-                  upload_file(c)
-                  database.add_entry(c[:path], false, c[:parent_id], nil, nil, nil)
-                  force_metadata_update_from_server(c)
-                  changelist[:created] << c[:path]
+                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
+                  res = upload_file(c)
+                  database.add_entry(c[:path], false, c[:parent_id], nil, nil, nil, local_hash)
+                  if c[:path] == res[:path]
+                    force_metadata_update_from_server(c)
+                    changelist[:created] << c[:path]
+                  else
+                    log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
+                    changelist[:conflicts] ||= []
+                    changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
+                  end
                 rescue Dbox::ServerError => e
                   log.error "Error while uploading #{c[:path]}: #{e.inspect}"
                   changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
@@ -474,9 +472,17 @@ module Dbox
               # spin up a thread to upload the file
               ptasks.add do
                 begin
-                  upload_file(c)
-                  force_metadata_update_from_server(c)
-                  changelist[:updated] << c[:path]
+                  local_hash = calculate_hash(relative_to_local_path(c[:path]))
+                  res = upload_file(c)
+                  database.update_entry_by_path(c[:path], :local_hash => local_hash)
+                  if c[:path] == res[:path]
+                    force_metadata_update_from_server(c)
+                    changelist[:updated] << c[:path]
+                  else
+                    log.warn "#{c[:path]} had a conflict and was renamed to #{res[:path]} on the server"
+                    changelist[:conflicts] ||= []
+                    changelist[:conflicts] << { :original => c[:path], :renamed => res[:path] }
+                  end
                 rescue Dbox::ServerError => e
                   log.error "Error while uploading #{c[:path]}: #{e.inspect}"
                   changelist[:failed] << { :operation => :update, :path => c[:path], :error => e }
@@ -526,7 +532,7 @@ module Dbox
         child_paths = list_contents(dir).sort
 
         child_paths.each do |p|
-          c = { :path => p, :modified => mtime(p), :is_dir => is_dir(p), :parent_path => dir[:path] }
+          c = { :path => p, :modified => mtime(p), :is_dir => is_dir(p), :parent_path => dir[:path], :local_hash => calculate_hash(relative_to_local_path(p)) }
           if entry = existing_entries[p]
             c[:id] = entry[:id]
             recur_dirs << c if c[:is_dir] # queue dir for later
@@ -560,8 +566,16 @@ module Dbox
       end
 
       def modified?(entry, res)
-        out = time_to_s(entry[:modified]) != time_to_s(res[:modified])
-        log.debug "#{entry[:path]} modified? t#{time_to_s(entry[:modified])} vs. t#{time_to_s(res[:modified])} => #{out}"
+        out = true
+        if entry[:is_dir]
+          out = !times_equal?(entry[:modified], res[:modified])
+          log.debug "#{entry[:path]} modified? t#{time_to_s(entry[:modified])} vs. t#{time_to_s(res[:modified])} => #{out}"
+        else
+          eh = entry[:local_hash]
+          rh = res[:local_hash]
+          out = !(eh && rh && eh == rh)
+          log.debug "#{entry[:path]} modified? #{eh} vs. #{rh} => #{out}"
+        end
         out
       end
 
@@ -591,14 +605,17 @@ module Dbox
         local_path = relative_to_local_path(file[:path])
         remote_path = relative_to_remote_path(file[:path])
         File.open(local_path) do |f|
-          api.put_file(remote_path, f)
+          db_entry = database.find_by_path(file[:path])
+          last_revision = db_entry ? db_entry[:revision] : nil
+          res = api.put_file(remote_path, f, last_revision)
+          process_basic_remote_props(res)
         end
       end
 
       def force_metadata_update_from_server(entry)
         res = gather_remote_info(entry)
         unless res == :not_modified
-          database.update_entry_by_path(entry[:path], :modified => res[:modified], :revision => res[:revision], :hash => res[:hash])
+          database.update_entry_by_path(entry[:path], :modified => res[:modified], :revision => res[:revision], :remote_hash => res[:remote_hash])
         end
         update_file_timestamp(database.find_by_path(entry[:path]))
       end