datapimp 1.2.7 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee1c9e42c2e0221ca734f17b15f0265597e74f90
-  data.tar.gz: 03bc06e371e3ec0fee16682b39e51a4d48bf4623
+  metadata.gz: b303a51b6bbca4a93af57cfaa463562e17fa8758
+  data.tar.gz: 640333b946ff814751d2077fbaea43e5b1af0c10
 SHA512:
-  metadata.gz: 15b3917ea6d396ac439c514f29dd1040b3942afe19cb33cb6fb32935f146b13844132bd24f8b17816ad730327d8e1e3a23f1b5f06535790f16ed3920ffbc898a
-  data.tar.gz: 3447b34fe02b2c3750295e604bee6a9b71060abd106e9e0417c7ea2ee90f8bf08ed52dedcb2b8a10a409bb7b524bc91d27f05f20d021b400b121dd9b312bffdf
+  metadata.gz: 8d830340d599872cfd2f0efd11bd9f8a8fcb9d45a6d5918a5f45cb27959b2e45ca90366f67dabb758e5fe94477cdca87f868ba1736517127efb7cf6bbe7de7b4
+  data.tar.gz: b480ce5379e3474a7c2f6175762b10c7e065f879b05e5ce6353b635a9a3684da908adc4edde70939f9bdbaadd5fe36f489a0545760ddbcd4951228ffd09cc91a

data/lib/datapimp/cli/create.rb

@@ -82,10 +82,11 @@ command 'create s3 bucket' do |c|
 
   c.option '--setup-website', 'Setup the bucket for website hosting'
   c.option '--create-redirect-bucket', 'Setup a redirect bucket'
+  c.option '--private', nil, 'Make this bucket private.'
 
   c.action do |args, options|
     raise 'Must specify bucket name' unless args.first
-    Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action()
+    Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action(make_private: !!options.private)
   end
 end
 

data/lib/datapimp/sync/dropbox_folder.rb

@@ -8,21 +8,58 @@ module Datapimp
       log "DropboxFolder run:#{action}"
 
       if action == :push
-        run_push_action
+        run_push_action(options)
       elsif action == :pull
-        run_pull_action
+        run_pull_action(options)
       elsif action == :create
-        run_create_action
+        run_create_action(options)
+      elsif action == :delta
+        run_delta_action(options)
       end
     end
 
-    def run_create_action
-      dropbox.mkdir(remote)
+    def run_delta_action(options={})
+      delta.entries.each do |entry|
+        remote_dropbox_path = entry.path
+
+        begin
+          reg = Regexp.new("#{remote}\/?", "i")
+          relative_local_path = remote_dropbox_path.gsub(reg, '')
+          target = local_path.join(relative_local_path)
+
+          if entry.is_dir
+            log "Creating directory: #{ target }" unless target.exist?
+            FileUtils.mkdir_p(target)
+          elsif entry.is_deleted && target.exist?
+              log "Deleting #{ target }"
+              target.unlink
+          elsif entry.is_deleted && !target.exist?
+            log "Skipping #{ entry.path }"
+          elsif entry.bytes == target.size
+            log "Skipping #{ entry.path }"
+          elsif !entry.is_deleted && target.exist? && entry.bytes != target.size
+            log "Downloading #{ target }"
+            target.open("wb") {|fh| fh.write(entry.download) }
+          else
+            log "Found something we can't handle"
+            binding.pry
+          end
+        rescue
+          nil
+        end
+      end
+
+      log "Processed #{ delta.entries.length } in cursor: #{ delta.cursor }"
+      cursor_path.open("w+") {|fh| fh.write(delta.cursor) }
+
+      delta.entries.length
     end
 
-    def run_pull_action
-      binding.pry
+    def run_create_action(options={})
+      dropbox.mkdir(remote)
+    end
 
+    def run_pull_action(options={})
       remote_path_entries.each do |entry|
         remote_dropbox_path = entry.path
 
@@ -31,7 +68,7 @@ module Datapimp
           relative_local_path = remote_dropbox_path.gsub("#{remote}/",'')
           target = local_path.join(relative_local_path)
 
-          next if target.exist? && target.size == entry.bytes
+          next if !entry.is_dir && target.exist? && target.size == entry.bytes
 
           if entry.is_dir
             log "Creating folder #{ relative_local_path }"
@@ -48,7 +85,7 @@ module Datapimp
       end
     end
 
-    def run_push_action
+    def run_push_action(options={})
       if remote_path_missing?
         run_create_action()
       end
@@ -73,7 +110,7 @@ module Datapimp
     # how to modify the state of `local_path` to match what exists
     # on Dropbox.
     def delta
-      @delta ||= dropbox.delta(cursor, remote_path)
+      @delta ||= dropbox.delta(cursor, path_prefix: remote.with_leading_slash)
     end
 
     # A Pointer to the local path we will be syncing with the Dropbox remote
@@ -89,7 +126,7 @@ module Datapimp
     end
 
     def cursor_path
-      local_path.join('.dropbox-cursor')
+      local_path.join('.dropbox_cursor')
     end
 
     def remote_path
@@ -123,4 +160,3 @@ module Datapimp
 
   end
 end
-

data/lib/datapimp/sync/s3_bucket.rb

@@ -121,6 +121,11 @@ module Datapimp
         deploy_manifest_path.open("w+") {|fh| fh.write(deploy_manifest.to_json) }
       end
 
+      def run_reset_action(options={})
+        bucket = directories.get(remote)
+        bucket.files.each {|f| key = f.key; f.delete rescue nil; f.destroy rescue nil; log "Deleting #{ key }"}
+      end
+
       def run_pull_action(options={})
         directories = Datapimp::Sync.amazon.storage.directories
         bucket = directories.get(remote)
@@ -133,21 +138,27 @@ module Datapimp
 
         bucket.files.each do |file|
           local_file = local_path.join(file.key)
-          next if local_file.exist? && file.etag == Digest::MD5.hexdigest(local_file.read)
+
+          if local_file.exist? && file.etag == Digest::MD5.hexdigest(local_file.read)
+            log "Skipping #{ file.key }"
+            next
+          end
 
           FileUtils.mkdir_p(local_file.dirname)
 
-          local_file.open("w+") {|fh| log("Updated #{ file.key }"); fh.write(file.body) }
+          local_file.open("w+") {|fh| fh.write(file.body); log("Updated #{ file.key }"); }
         end
       end
 
       def run_create_action(options={})
         directories = Datapimp::Sync.amazon.storage.directories
 
+        make_private = !!options[:make_private]
+
         bucket = if existing = directories.get(remote)
           existing
         else
-          directories.create(key:remote, public: true)
+          directories.create(key:remote, public: !make_private)
         end
 
         storage.put_bucket_website(remote, :IndexDocument => 'index.html', :ErrorDocument => 'error.html')
@@ -166,6 +177,8 @@ module Datapimp
           run_update_acl_action(options)
         elsif action == :pull
           run_pull_action(options)
+        elsif action == :reset
+          run_reset_options(options)
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datapimp
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.2.8
 platform: ruby
 authors:
 - Jonathan Soeder
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2015-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -508,7 +508,6 @@ files:
 - lib/datapimp/sources/pivotal.rb
 - lib/datapimp/sync.rb
 - lib/datapimp/sync/cloudfront_distribution.rb
-- lib/datapimp/sync/dropbox_delta.rb
 - lib/datapimp/sync/dropbox_folder.rb
 - lib/datapimp/sync/google_drive_folder.rb
 - lib/datapimp/sync/s3_bucket.rb

data/lib/datapimp/sync/dropbox_delta.rb

@@ -1,67 +0,0 @@
-module Datapimp
-  module Sync
-    class DropboxDelta
-
-      attr_accessor :client,
-                    :data,
-                    :cursor,
-                    :entries,
-                    :path_prefix
-
-      def initialize(client, cursor, path_prefix=nil)
-        @client = client
-        @cursor = cursor
-        @path_prefix = path_prefix
-      end
-
-      def processed!
-        # TODO
-        # Should update cursor
-      end
-
-      def entries
-        return @entries if @entries
-        fetch
-        @entries
-      end
-
-      def _dropbox_delta at=nil
-        at ||= cursor
-        response = client.delta(at, path_prefix)
-        self.cursor = response["cursor"]
-        response
-      end
-
-      def data
-        @data ||= fetch
-      end
-
-      def on_reset path_prefix, cursor
-      end
-
-      def fetch
-        return @response if @response
-
-        response = _dropbox_delta
-
-        if response["reset"] == true
-          on_reset(path_prefix, cursor)
-        end
-
-        self.entries = {}.to_mash
-
-        response["entries"].each do |entry|
-          path, meta = entry
-          self.entries[path] = meta
-        end
-
-        if response["has_more"] == true
-          # TODO Implement
-        end
-
-        @response = response
-      end
-
-    end
-  end
-end