datapimp 1.0.6 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f0a7816f33ece7bc79ca55d86e01ffafb666ae1a
-  data.tar.gz: 2a56cb9f675a9f5ea532f163cdf841a3a9e612a8
+  metadata.gz: 34acb2d1cc3f7c113a962128faa64f3d0ae8d382
+  data.tar.gz: 9b4cf19a026f35bb3c978a7eafef314ccebe2487
 SHA512:
-  metadata.gz: 398f2a75829491e5ff4823eb5fc1acc98cbdb94d87c481845b431bb8533afc62946819fdccdc83ff2e9be4372ce4af11eab29f0fda466fd0553113ffc669e57f
-  data.tar.gz: de27174ba144c621e18eccf32993beae4f9dce4857d0014413fbfcd572b35d6764d116356c10db740998aba7c4c5ae57ce8957cec0e7bb69b2203cedf112147a
+  metadata.gz: 738b6bd20b2be6ef71a228406dd36b20197427b1afcc6ffe963df23d34c71b36180bcd0e037753e36698238a5209011f0a86afaf64f70e9a068be2248893f29b
+  data.tar.gz: 76e95ba024481ade64e27edf5ebebfcc592c551ed9e62edd40ee2c88be87bc416345090350396befd4cdae01f4b8001cb59775bd43e444b6e48f233a56f27649

data/lib/datapimp/cli/sync.rb

@@ -13,16 +13,18 @@ command "sync folder" do |c|
     local, remote = args
 
     folder = case
-      when options.type == "dropbox"
-        Datapimp::Sync::DropboxFolder.new(local: local, remote: remote)
-      when options.type == "google"
-        # Return the folders
-        # collection = Datapimp::Sync.google.api.collections.first
-        #
-        # svg = collection.files.first
-        # svg.export_as_file(/download/path, "image/svg+xml")
-        Datapimp::Sync::GoogleDriveFolder.new(local: local, remote: remote)
-    end
+             when options.type == "dropbox"
+               Datapimp::Sync::DropboxFolder.new(local: local, remote: remote)
+             when options.type == "google"
+               # Return the folders
+               # collection = Datapimp::Sync.google.api.collections.first
+               #
+               # svg = collection.files.first
+               # svg.export_as_file(/download/path, "image/svg+xml")
+               Datapimp::Sync::GoogleDriveFolder.new(local: local, remote: remote)
+             when options.type == "aws" || options.type == "s3"
+               Datapimp::Sync::S3Bucket.new(local: local, remote: remote)
+             end
 
     folder.run(options.action, options.to_hash.to_mash)
   end

data/lib/datapimp/configuration.rb

@@ -58,6 +58,12 @@ module Datapimp
       end
     end
 
+    def deploy_manifests_path
+      Pathname(home_config_path.dirname).join("deploy-manifests").tap do |dir|
+        FileUtils.mkdir_p(dir) unless dir.exist?
+      end
+    end
+
     def manifest_filename
       "datapimp.json"
     end

data/lib/datapimp/core_ext.rb

@@ -3,3 +3,23 @@ class Hash
     Hashie::Mash.new(self)
   end
 end
+
+class Pathname
+  def to_pathname
+    self
+  end
+end
+
+class String
+  def to_pathname
+    Pathname(self)
+  end
+
+  def without_leading_slash
+    gsub(/^\//,'')
+  end
+
+  def with_leading_slash
+    "/#{without_leading_slash}"
+  end
+end

data/lib/datapimp/logging.rb

@@ -0,0 +1,15 @@
+module Datapimp
+  module Logging
+    def log(*args)
+      logger.send(:info, *args)
+    end
+
+    def logger= value
+      @logger = value
+    end
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+end

data/lib/datapimp/sync/dropbox_folder.rb

@@ -1,5 +1,7 @@
 module Datapimp
   class Sync::DropboxFolder < Hashie::Mash
+    include Datapimp::Logging
+
     # Provides easy access to the Dropbox client
     def dropbox
       @dropbox ||= Datapimp::Sync.dropbox
@@ -48,23 +50,33 @@ module Datapimp
     def run(action, options={})
       action = action.to_sym
 
-      if action == :push
-        if remote_path_missing?
-          dropbox.mkdir(remote)
-        end
-
-        Dir[local_path.join("**/*")].each do |f|
-          # Upload the file
-          binding.pry
-        end
+      log "DropboxFolder run:#{action}"
 
+      if action == :push
+        run_push_action
       elsif action == :pull
-        # TODO
-        # Implement the Delta call
+        run_pull_action
       end
     end
 
-    def ensure_remote_folder_exists
+    def run_pull_action
+      binding.pry
+    end
+
+    def run_push_action
+      if remote_path_missing?
+        dropbox.mkdir(remote)
+      end
+
+      Dir[local_path.join("**/*")].each do |f|
+        f = Pathname(f)
+        base = f.relative_path_from(local_path).to_s
+        target_path = File.join(remote, base)
+
+        log "Uploading #{ f } to #{target_path}"
+
+        dropbox.upload(target_path, f.read, :overwrite => false)
+      end
     end
   end
 end

data/lib/datapimp/sync/s3_bucket.rb

@@ -0,0 +1,120 @@
+module Datapimp
+  module Sync
+    class S3Bucket < Hashie::Mash
+      include Datapimp::Logging
+
+      # returns the s3 bucket via fog
+      def s3
+        @s3 ||= Datapimp::Sync.amazon.storage.directories.get(remote)
+      end
+
+      def local_path
+        Pathname(local)
+      end
+
+      def deploy_manifest_path
+        Datapimp.config.deploy_manifests_path
+          .tap {|p| FileUtils.mkdir_p(p) }
+          .join(remote.to_s.parameterize + '.json')
+      end
+
+      def deploy_manifest
+        @deploy_manifest ||= (JSON.parse(deploy_manifest_path.read) || {} rescue {})
+      end
+
+      def build_deploy_manifest_from_remote
+        # TODO
+        # Implement
+      end
+
+      # builds a manifest of MD5 hashes for each file
+      # so that we aren't deploying stuff which is the
+      # same since last time we deployed
+      def prepare_manifest_for(entries)
+        m = deploy_manifest
+
+        entries.each do |entry|
+          destination = Pathname(entry).relative_path_from(local_path).to_s.without_leading_slash
+
+          deploy_manifest.fetch(destination) do
+            next unless destination.match(/\w+\.\w+/)
+            m[destination] = nil
+          end
+        end
+      end
+
+      def run_update_acl_action(options={})
+        s3.files.each do |file|
+          file.acl = 'public-read'
+          file.save
+          log "Updated acl for #{ file.key } to public-read"
+        end
+      end
+
+      def asset_fingerprints
+        deploy_manifest['asset_fingerprints'] ||= {}
+      end
+
+      def run_push_action(options={})
+        entries = Dir[local_path.join('**/*')].map(&:to_pathname)
+        prepare_manifest_for(entries)
+
+        entries.reject! { |entry| entry.to_s.match(/\.DS_Store/) }
+        entries.reject!(&:directory?)
+
+        uploaded = deploy_manifest['uploaded'] = []
+
+        entries.each do |entry|
+          destination = entry.relative_path_from(local_path).to_s.without_leading_slash
+          fingerprint = Digest::MD5.hexdigest(entry.read)
+
+          if asset_fingerprints[destination] == fingerprint
+            #log "Skipping #{ destination }: found in manifest"
+            next
+          end
+
+          if existing = s3.files.get(destination)
+            if existing.etag == fingerprint
+              log "Skipping #{ destination }: similar etag"
+            else
+              existing.body = entry.read
+              existing.acl = 'public-read'
+              log "Updated #{ destination }"
+              uploaded << destination
+              existing.save
+            end
+          else
+            log "Uploaded #{ destination }"
+            s3.files.create(key: destination, body: entry.read, acl: 'public-read')
+            uploaded << destination
+          end
+
+          asset_fingerprints[destination] = fingerprint
+        end
+
+        if count == 0
+          return
+        end
+
+        log "Saving deploy manifest. #{ deploy_manifest.keys.length } entries"
+        deploy_manifest_path.open("w+") {|fh| fh.write(deploy_manifest.to_json) }
+      end
+
+      def run_pull_action(options={})
+
+      end
+
+      def run(action, options={})
+        action = action.to_sym
+
+        if action == :push
+          run_push_action(options)
+        elsif action == :update_acl
+          run_update_acl_action(options={})
+        elsif action == :pull
+          run_pull_action
+        end
+      end
+    end
+  end
+end

data/lib/datapimp/version.rb

@@ -1,3 +1,3 @@
 module Datapimp
-  VERSION = "1.0.6"
+  VERSION = "1.0.8"
 end

data/lib/datapimp.rb

@@ -30,7 +30,9 @@ end
 
 require 'datapimp/version'
 require 'datapimp/configuration'
+require 'datapimp/logging'
 require 'datapimp/sources'
 require 'datapimp/sync'
 require 'datapimp/sync/dropbox_folder'
 require 'datapimp/sync/google_drive_folder'
+require 'datapimp/sync/s3_bucket'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datapimp
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.8
 platform: ruby
 authors:
 - Jonathan Soeder
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-22 00:00:00.000000000 Z
+date: 2015-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -380,6 +380,7 @@ files:
 - lib/datapimp/clients/google.rb
 - lib/datapimp/configuration.rb
 - lib/datapimp/core_ext.rb
+- lib/datapimp/logging.rb
 - lib/datapimp/sources.rb
 - lib/datapimp/sources/dropbox.rb
 - lib/datapimp/sources/excel.rb
@@ -391,6 +392,7 @@ files:
 - lib/datapimp/sync/dropbox_delta.rb
 - lib/datapimp/sync/dropbox_folder.rb
 - lib/datapimp/sync/google_drive_folder.rb
+- lib/datapimp/sync/s3_bucket.rb
 - lib/datapimp/version.rb
 - packaging/wrapper.sh
 - spec/spec_helper.rb