dbox 0.4.4 → 0.5.0

data/README.md

@@ -3,7 +3,7 @@ dbox
 
 An easy way to push and pull your Dropbox folders, with fine-grained control over what folder you are syncing, where you are syncing it to, and when you are doing it.
 
-**IMPORTANT:** This is **not** an automated Dropbox client. It will exit after sucessfully pushing/pulling, so if you want regular updates, you can run it in cron, a loop, etc.
+**IMPORTANT:** This is **not** an automated Dropbox client. It will exit after sucessfully pushing/pulling, so if you want regular updates, you can run it in cron, a loop, etc. If you do want to run it in a loop, take a look at [sample_polling_script.rb](http://github.com/kenpratt/dbox/blob/master/sample_polling_script.rb).
 
 
 Installation

data/Rakefile

@@ -17,6 +17,8 @@ Jeweler::Tasks.new do |gem|
   gem.add_dependency "multipart-post", ">= 1.1.2"
   gem.add_dependency "oauth", ">= 0.4.5"
   gem.add_dependency "json", ">= 1.5.3"
+  gem.add_dependency "sqlite3", ">= 1.3.3"
+  gem.add_dependency "activesupport", ">= 3.0.1"
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/TODO.txt

@@ -1,3 +1,6 @@
-* Look down directory tree until you hit a .dropbox.db file
+* Refactor threading in change detection in pull to use ParallelTasks?
+* Saving SQLite db changes dir timestamp -- try to preserve it?
 * Add a "sync" command that pushes and pulls in one go
-* Add support for partial push/pull
+* See if prepared statements speed up operations on large repos much
+* Look down directory tree until you hit a .dbox.sqlite3 file so you can use commands from anywhere inside a tree (like git)
+* Add support for partial push/pull (subtree push/pull)?

data/VERSION

@@ -1 +1 @@
-0.4.4
+0.5.0

data/dbox.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{dbox}
-  s.version = "0.4.4"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = [%q{Ken Pratt}]
-  s.date = %q{2011-06-28}
+  s.date = %q{2011-09-19}
   s.description = %q{An easy-to-use Dropbox client with fine-grained control over syncs.}
   s.email = %q{ken@kenpratt.net}
   s.executables = [%q{dbox}]
@@ -28,8 +28,12 @@ Gem::Specification.new do |s|
     "dbox.gemspec",
     "lib/dbox.rb",
     "lib/dbox/api.rb",
+    "lib/dbox/database.rb",
     "lib/dbox/db.rb",
     "lib/dbox/loggable.rb",
+    "lib/dbox/parallel_tasks.rb",
+    "lib/dbox/syncer.rb",
+    "sample_polling_script.rb",
     "spec/dbox_spec.rb",
     "spec/spec_helper.rb",
     "vendor/dropbox-client-ruby/LICENSE",
@@ -45,7 +49,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/kenpratt/dbox}
   s.licenses = [%q{MIT}]
   s.require_paths = [%q{lib}]
-  s.rubygems_version = %q{1.8.1}
+  s.rubygems_version = %q{1.8.5}
   s.summary = %q{Dropbox made easy.}
 
   if s.respond_to? :specification_version then
@@ -55,15 +59,21 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<multipart-post>, [">= 1.1.2"])
       s.add_runtime_dependency(%q<oauth>, [">= 0.4.5"])
       s.add_runtime_dependency(%q<json>, [">= 1.5.3"])
+      s.add_runtime_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 3.0.1"])
     else
       s.add_dependency(%q<multipart-post>, [">= 1.1.2"])
       s.add_dependency(%q<oauth>, [">= 0.4.5"])
       s.add_dependency(%q<json>, [">= 1.5.3"])
+      s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_dependency(%q<activesupport>, [">= 3.0.1"])
     end
   else
     s.add_dependency(%q<multipart-post>, [">= 1.1.2"])
     s.add_dependency(%q<oauth>, [">= 0.4.5"])
     s.add_dependency(%q<json>, [">= 1.5.3"])
+    s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+    s.add_dependency(%q<activesupport>, [">= 3.0.1"])
   end
 end
 

data/lib/dbox.rb

@@ -8,10 +8,15 @@ require "time"
 require "yaml"
 require "logger"
 require "cgi"
+require "sqlite3"
+require "active_support/core_ext/hash/indifferent_access"
 
 require "dbox/loggable"
 require "dbox/api"
+require "dbox/database"
 require "dbox/db"
+require "dbox/parallel_tasks"
+require "dbox/syncer"
 
 module Dbox
   def self.authorize
@@ -19,41 +24,47 @@ module Dbox
   end
 
   def self.create(remote_path, local_path)
+    log.debug "Creating (remote: #{remote_path}, local: #{local_path})"
     remote_path = clean_remote_path(remote_path)
     local_path = clean_local_path(local_path)
-    Dbox::DB.create(remote_path, local_path)
+    migrate_dbfile(local_path)
+    Dbox::Syncer.create(remote_path, local_path)
   end
 
   def self.clone(remote_path, local_path)
+    log.debug "Cloning (remote: #{remote_path}, local: #{local_path})"
     remote_path = clean_remote_path(remote_path)
     local_path = clean_local_path(local_path)
-    Dbox::DB.clone(remote_path, local_path)
+    migrate_dbfile(local_path)
+    Dbox::Syncer.clone(remote_path, local_path)
   end
 
   def self.pull(local_path)
+    log.debug "Pulling (local: #{local_path})"
     local_path = clean_local_path(local_path)
-    Dbox::DB.pull(local_path)
+    migrate_dbfile(local_path)
+    Dbox::Syncer.pull(local_path)
   end
 
   def self.push(local_path)
+    log.debug "Pushing (local: #{local_path})"
     local_path = clean_local_path(local_path)
-    Dbox::DB.push(local_path)
+    migrate_dbfile(local_path)
+    Dbox::Syncer.push(local_path)
   end
 
   def self.move(new_remote_path, local_path)
+    log.debug "Moving (new remote: #{new_remote_path}, local: #{local_path})"
     new_remote_path = clean_remote_path(new_remote_path)
     local_path = clean_local_path(local_path)
-    Dbox::DB.move(new_remote_path, local_path)
+    migrate_dbfile(local_path)
+    Dbox::Syncer.move(new_remote_path, local_path)
   end
 
   def self.exists?(local_path)
     local_path = clean_local_path(local_path)
-    Dbox::DB.exists?(local_path)
-  end
-
-  def self.corrupt?(local_path)
-    local_path = clean_local_path(local_path)
-    Dbox::DB.corrupt?(local_path)
+    migrate_dbfile(local_path)
+    Dbox::Database.exists?(local_path)
   end
 
   private
@@ -68,4 +79,13 @@ module Dbox
     raise(ArgumentError, "Missing local path") unless path
     File.expand_path(path)
   end
+
+  def self.migrate_dbfile(path)
+    if Dbox::DB.exists?(path)
+      log.warn "Old database file format found -- migrating to new database format"
+      Dbox::Database.migrate_from_old_db_format(Dbox::DB.load(path))
+      Dbox::DB.destroy!(path)
+      log.warn "Migration complete"
+    end
+  end
 end

data/lib/dbox/api.rb

@@ -30,12 +30,18 @@ module Dbox
       api
     end
 
+    attr_reader :client
+
     # IMPORTANT: API.new is private. Please use API.authorize or API.connect as the entry point.
     private_class_method :new
     def initialize
       @conf = self.class.conf
     end
 
+    def initialize_copy(other)
+      @client = other.client.clone()
+    end
+
     def connect
       auth_key = ENV["DROPBOX_AUTH_KEY"]
       auth_secret = ENV["DROPBOX_AUTH_SECRET"]
@@ -52,13 +58,17 @@ module Dbox
         res = yield
         case res
         when Hash
-          res
+          HashWithIndifferentAccess.new(res)
         when String
           res
         when Net::HTTPNotFound
           raise RemoteMissing, "#{path} does not exist on Dropbox"
         when Net::HTTPForbidden
           raise RequestDenied, "Operation on #{path} denied"
+        when Net::HTTPNotModified
+          :not_modified
+        when true
+          true
         else
           raise RuntimeError, "Unexpected result: #{res.inspect}"
         end
@@ -68,10 +78,10 @@ module Dbox
       end
     end
 
-    def metadata(path = "/")
+    def metadata(path = "/", hash = nil)
       log.debug "Fetching metadata for #{path}"
       run(path) do
-        res = @client.metadata(@conf["root"], escape_path(path))
+        res = @client.metadata(@conf["root"], escape_path(path), 10000, hash)
         log.debug res.inspect
         res
       end
@@ -96,10 +106,10 @@ module Dbox
       end
     end
 
-    def get_file(path)
+    def get_file(path, output_file_obj)
       log.info "Downloading #{path}"
       run(path) do
-        @client.get_file(@conf["root"], escape_path(path))
+        @client.get_file(@conf["root"], escape_path(path), output_file_obj)
       end
     end
 

data/lib/dbox/database.rb

@@ -0,0 +1,216 @@
+module Dbox
+  class DatabaseError < RuntimeError; end
+
+  class Database
+    include Loggable
+
+    DB_FILENAME = ".dbox.sqlite3"
+
+    def self.create(remote_path, local_path)
+      db = new(local_path)
+      if db.bootstrapped?
+        raise DatabaseError, "Database already initialized -- please use 'dbox pull' or 'dbox push'."
+      end
+      db.bootstrap(remote_path, local_path)
+      db
+    end
+
+    def self.load(local_path)
+      db = new(local_path)
+      unless db.bootstrapped?
+        raise DatabaseError, "Database not initialized -- please run 'dbox create' or 'dbox clone'."
+      end
+      db
+    end
+
+    def self.exists?(local_path)
+      File.exists?(File.join(local_path, DB_FILENAME))
+    end
+
+    def self.migrate_from_old_db_format(old_db)
+      new_db = create(old_db.remote_path, old_db.local_path)
+      new_db.delete_entry_by_path("") # clear out root record
+      new_db.migrate_entry_from_old_db_format(old_db.root)
+    end
+
+    # IMPORTANT: Database.new is private. Please use Database.create
+    # or Database.load as the entry point.
+    private_class_method :new
+    def initialize(local_path)
+      FileUtils.mkdir_p(local_path)
+      @db = SQLite3::Database.new(File.join(local_path, DB_FILENAME))
+      @db.trace {|sql| log.debug sql.strip }
+      @db.execute("PRAGMA foreign_keys = ON;")
+      ensure_schema_exists
+    end
+
+    def ensure_schema_exists
+      @db.execute_batch(%{
+        CREATE TABLE IF NOT EXISTS metadata (
+          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+          local_path   varchar(255) NOT NULL,
+          remote_path  varchar(255) NOT NULL,
+          version      integer NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS entries (
+          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+          path         varchar(255) UNIQUE NOT NULL,
+          is_dir       boolean NOT NULL,
+          parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
+          hash         varchar(255),
+          modified     datetime,
+          revision     integer
+        );
+        CREATE INDEX IF NOT EXISTS entry_parent_ids ON entries(parent_id);
+      })
+    end
+
+    METADATA_COLS = [ :local_path, :remote_path, :version ] # don't need to return id
+    ENTRY_COLS    = [ :id, :path, :is_dir, :parent_id, :hash, :modified, :revision ]
+
+    def bootstrap(remote_path, local_path)
+      @db.execute(%{
+        INSERT INTO metadata (local_path, remote_path, version) VALUES (?, ?, ?);
+      }, local_path, remote_path, 1)
+      @db.execute(%{
+        INSERT INTO entries (path, is_dir) VALUES (?, ?)
+      }, "", 1)
+    end
+
+    def bootstrapped?
+      n = @db.get_first_value(%{
+        SELECT count(id) FROM metadata LIMIT 1;
+      })
+      n && n > 0
+    end
+
+    def metadata
+      cols = METADATA_COLS
+      res = @db.get_first_row(%{
+        SELECT #{cols.join(',')} FROM metadata LIMIT 1;
+      })
+      make_fields(cols, res) if res
+    end
+
+    def update_metadata(fields)
+      set_str = fields.keys.map {|k| "#{k}=?" }.join(",")
+      @db.execute(%{
+        UPDATE metadata SET #{set_str};
+      }, *fields.values)
+    end
+
+    def root_dir
+      find_entry("WHERE parent_id is NULL")
+    end
+
+    def find_by_path(path)
+      raise(ArgumentError, "path cannot be null") unless path
+      find_entry("WHERE path=?", path)
+    end
+
+    def contents(dir_id)
+      raise(ArgumentError, "dir_id cannot be null") unless dir_id
+      find_entries("WHERE parent_id=?", dir_id)
+    end
+
+    def subdirs(dir_id)
+      raise(ArgumentError, "dir_id cannot be null") unless dir_id
+      find_entries("WHERE parent_id=? AND is_dir=1", dir_id)
+    end
+
+    def add_entry(path, is_dir, parent_id, modified, revision, hash)
+      insert_entry(:path => path, :is_dir => is_dir, :parent_id => parent_id, :modified => modified, :revision => revision, :hash => hash)
+    end
+
+    def update_entry_by_path(path, fields)
+      raise(ArgumentError, "path cannot be null") unless path
+      update_entry(["WHERE path=?", path], fields)
+    end
+
+    def delete_entry_by_path(path)
+      raise(ArgumentError, "path cannot be null") unless path
+      delete_entry("WHERE path=?", path)
+    end
+
+    def migrate_entry_from_old_db_format(entry, parent = nil)
+      # insert entry into sqlite db
+      add_entry(entry.path, entry.dir?, (parent ? parent[:id] : nil), entry.modified_at, entry.revision, nil)
+
+      # recur on children
+      if entry.dir?
+        new_parent = find_by_path(entry.path)
+        entry.contents.each {|child_path, child| migrate_entry_from_old_db_format(child, new_parent) }
+      end
+    end
+
+    private
+
+    def find_entry(conditions = "", *args)
+      res = @db.get_first_row(%{
+        SELECT #{ENTRY_COLS.join(",")} FROM entries #{conditions} LIMIT 1;
+      }, *args)
+      entry_res_to_fields(res)
+    end
+
+    def find_entries(conditions = "", *args)
+      out = []
+      @db.execute(%{
+        SELECT #{ENTRY_COLS.join(",")} FROM entries #{conditions} ORDER BY path ASC;
+      }, *args) do |res|
+        out << entry_res_to_fields(res)
+      end
+      out
+    end
+
+    def insert_entry(fields)
+      log.debug "Inserting entry: #{fields.inspect}"
+      h = fields.clone
+      h[:modified]  = h[:modified].to_i if h[:modified]
+      h[:is_dir] = (h[:is_dir] ? 1 : 0) unless h[:is_dir].nil?
+      @db.execute(%{
+        INSERT INTO entries (#{h.keys.join(",")})
+        VALUES (#{(["?"] * h.size).join(",")});
+      }, *h.values)
+    end
+
+    def update_entry(where_clause, fields)
+      log.debug "Updating entry: #{where_clause}, #{fields.inspect}"
+      h = fields.clone
+      h[:modified]  = h[:modified].to_i if h[:modified]
+      conditions, *args = *where_clause
+      set_str = h.keys.map {|k| "#{k}=?" }.join(",")
+      @db.execute(%{
+        UPDATE entries SET #{set_str} #{conditions};
+      }, *(h.values + args))
+    end
+
+    def delete_entry(conditions = "", *args)
+      @db.execute(%{
+        DELETE FROM entries #{conditions};
+      }, *args)
+    end
+
+    def entry_res_to_fields(res)
+      if res
+        h = make_fields(ENTRY_COLS, res)
+        h[:is_dir] = (h[:is_dir] == 1)
+        h[:modified]  = Time.at(h[:modified]) if h[:modified]
+        h.delete(:hash) unless h[:is_dir]
+        h
+      else
+        nil
+      end
+    end
+
+    def make_fields(keys, vals)
+      if keys && vals
+        raise ArgumentError.new("Can't make a fields hash with #{keys.size} keys and #{vals.size} vals") unless keys.size == vals.size
+        out = {}
+        keys.each_with_index {|k, i| out[k] = vals[i] }
+        out
+      else
+        nil
+      end
+    end
+  end
+end