aws_one_click_staging 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3e3cdc6ae9700edde65741d6ff8e797d3b260e45
-  data.tar.gz: 21a64e404ca0f783c34d102b858cea2dcf8ec3b0
+  metadata.gz: d779231c0e0dc99fe5f3c7ed558c953517b52eb0
+  data.tar.gz: 6bad4ad285f0cc504c48b32efcecb0ed00660692
 SHA512:
-  metadata.gz: 97712b3d3f4bccc0dc6c7a9e1813c43d6bcae612328091a07e7e5ea6cd9e71e11056fabfa4efa59518b098c9a53e0786b0af2d67ca2e98b342f6656d69378891
-  data.tar.gz: 6bb7d555055c86dff31d8c814f9b2570b4d3afc116fdfc5e2d52bc71edf1e579a3976b8bbd0166bbc0119ceda0d601bd19998c05aeec173309ed5c527bfe671f
+  metadata.gz: 20456486a2357d797831905748a5352a8ad32aed06e11682c2071c3957066ed1341b744f3b5831076f374b1cc3f15085dd5517e0d7f4f2e443573efb0e5c7a01
+  data.tar.gz: 2c3e1a8eb4829d92be2edab97e39fff8edc018a6f51e3c5f6712ad7fe003e1db97f3e0ee4b487e2a36466c30d10041650337e07b68a18781a1a1f1726c01778b

data/lib/aws_one_click_staging/bucket_sync_service.rb

@@ -1,36 +1,46 @@
 # props to bantic
 # https://gist.github.com/bantic/4080793
 require 'aws/s3' # gem name is 'aws-sdk'
+require "thwait"
 
 class BucketSyncService
   attr_reader :from_bucket, :to_bucket, :logger
   attr_accessor :debug
 
-  DEFAULT_ACL = :public_read
-
   # from_credentials and to_credentials are both hashes with these keys:
   #  * :aws_access_key_id
   #  * :aws_secret_access_key
   #  * :bucket
   def initialize(from_credentials, to_credentials)
-    @from_bucket = bucket_from_credentials(from_credentials)
-    @to_bucket   = bucket_from_credentials(to_credentials)
+    @from_bucket = get_bucket_reference(from_credentials)
+    @to_bucket   = get_bucket_reference(to_credentials)
   end
 
   def perform(output=STDOUT)
+    AWS.eager_autoload! # this makes threads even more happy
+
     object_counts = {sync:0, skip:0}
     create_logger(output)
 
+    threads = []
+
     logger.info "Starting sync."
     from_bucket.objects.each do |object|
-      if object_needs_syncing?(object)
-        sync(object)
-        object_counts[:sync] += 1
-      else
-        logger.debug "Skipped #{pp object}"
-        object_counts[:skip] += 1
-      end
+      threads << Thread.new {
+        if object_needs_syncing?(object)
+          sync(object)
+          object_counts[:sync] += 1
+        else
+          logger.debug "Skipped #{pp object}"
+          object_counts[:skip] += 1
+        end
+      }
+      sleep 0.01 while threads.select {|t| t.alive?}.count > 16
     end
+
+
+    ThreadsWait.all_waits(threads)
+
     logger.info "Done. Synced #{object_counts[:sync]}, " +
       "skipped #{object_counts[:skip]}."
   end
@@ -45,7 +55,7 @@ class BucketSyncService
 
   def sync(object)
     logger.debug "Syncing #{pp object}"
-    object.copy_to( to_bucket.objects[object.key], acl:DEFAULT_ACL)
+    object.copy_to(to_bucket.objects[object.key])
   end
 
   def pp(object)
@@ -56,25 +66,28 @@ class BucketSyncService
 
   def object_needs_syncing?(object)
     to_object = to_bucket.objects[object.key]
-    return true if !to_object.exists?
+    return true if !to_object.exists? # object isn't even present in the dst bucket
 
-    return to_object.etag != object.etag
+    return to_object.etag != object.etag # does the etag on the dst object differ from src?
   end
 
-  def bucket_from_credentials(credentials)
+  def get_bucket_reference(credentials)
     s3 = AWS::S3.new(access_key_id:      credentials[:aws_access_key_id],
                      secret_access_key:  credentials[:aws_secret_access_key])
 
     bucket = s3.buckets[ credentials[:bucket] ]
-    if !bucket.exists?
-      bucket = s3.buckets.create( credentials[:bucket] )
-    end
+    create_bucket_if_needed!(bucket)
+  end
+
+  def create_bucket_if_needed!(bucket)
+    return s3.buckets.create( credentials[:bucket] ) if !bucket.exists?
     bucket
   end
 end
 
 
 
+
 =begin
 Example usage:
  from_creds = {aws_access_key_id:"XXX", aws_secret_access_key:"YYY", bucket:"first-bucket"}

data/lib/aws_one_click_staging/version.rb

@@ -1,3 +1,3 @@
 module AwsOneClickStaging
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws_one_click_staging
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - TheNotary
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-26 00:00:00.000000000 Z
+date: 2015-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk