dribbble-bucket-sync 0.0.1 → 0.0.2

data/README.md

@@ -33,8 +33,12 @@ Output should be like this:
     $ dribbble_bucket_sync ryantownsend ~/dribbble
     Starting synchronisation...
     ryantownsend has 29 buckets.
-    Timelines has 2 shots.
-    Colour Palettes has 1 shots.
+    Timelines has 20 shots.
+    Connecting to dribbble.s3.amazonaws.com:80
+    Downloading 20 files:
+    ....................
+    Infographics has 12 shots.
+    [...]
     Synchronisation complete.
 
 The syntax to run within your own app is fairly straight-forward:

data/dribbble_bucket_sync.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.version       = DribbbleBucketSync::VERSION
   
-  s.add_dependency "dribbble-bucket-api"
+  s.add_dependency "dribbble-bucket-api", "~> 0.0.6"
   s.add_development_dependency "rspec"
   s.add_development_dependency "simplecov"
 end

data/lib/dribbble_bucket_sync/folder.rb

@@ -1,6 +1,7 @@
 require "fileutils"
 require "uri"
 require "net/http"
+require "timeout"
 
 class DribbbleBucketSync
 	class Folder
@@ -17,23 +18,35 @@ class DribbbleBucketSync
 		end
 		
 		def add_shot(shot)
-			url = shot.image_url
-			# add the file to the shots list
-			filename = "#{shot.id}#{File.extname(url)}"
-			@shots << filename
+			# calculate the filename
+			filename = "#{shot.id}.#{shot.ext}"
 			# calculate the path
 			path = File.join(@path, filename)
-			# download if it doesn't exist
-			unless File.exist?(path)
-				download(url, path)
+			# add the shot to the storage
+			@shots << { filename: filename, destination: path, url: shot.image_url }
+		end
+		
+		def sync
+			download_new_shots
+			remove_old_shots
+		end
+		
+		# downloads any shots missing from the folder
+		def download_new_shots
+			# select all the shots that don't exist
+			new_shots = @shots.select do |s|
+				!File.exist?(s[:destination])
 			end
+			# download them all
+			download_files(new_shots)
 		end
 		
+		# deletes any shots in the folder that no longer exist in the bucket
 		def remove_old_shots
 			# loop through all shots in the folder
 			current_shot_files.each do |file|
 				# delete file if the shot isn't in the array
-				unless @shots.include?(File.basename(file))
+				unless @shots.map { |s| s[:filename] }.include?(File.basename(file))
 					FileUtils.rm(file)
 				end
 			end
@@ -45,12 +58,39 @@ class DribbbleBucketSync
 		end
 		
 		private
-		# downloads a URL to a local path
-		def download(url, destination)
-			# parse the URL
-			uri = URI.parse(url)
-			# connect to the site
-			Net::HTTP.start(uri.host, uri.port) do |http|
+		# download URLs to a local path
+		def download_files(files)
+			# map the urls into groups by hostname
+			groups = files.inject({}) do |result,file|
+				uri = URI.parse(file[:url])
+				result["#{uri.host}:#{uri.port}"] ||= []
+				result["#{uri.host}:#{uri.port}"] << {
+					destination: file[:destination],
+					uri: uri
+				}
+				result
+			end
+			# loop through each group
+			groups.each do |key, files|
+				host = files.first[:uri].host
+				port = files.first[:uri].port
+				puts "Connecting to #{host}:#{port}"
+				# connect to the site
+				Net::HTTP.start(host, files.first[:uri].port) do |http|
+					puts "Downloading #{files.size} files:"
+					# loop through each uri
+					files.each do |file|
+						# download the file
+						download_file(http, file[:uri], file[:destination])
+						print "."
+					end
+					puts ""
+				end
+			end
+		end
+		
+		def download_file(http, uri, destination)
+			Timeout::timeout(30) do
 				# download the file
 				resp = http.get(uri.request_uri)
 				# open the local file
@@ -59,6 +99,10 @@ class DribbbleBucketSync
 					file.write(resp.body)
 				end
 			end
+		rescue Timeout::Error => e
+			puts "Request timed out: #{uri.request_uri}, moving on to next... (#{e.class.name})"
+		rescue EOFError => e
+			puts "Request failed: #{uri.request_uri}, moving on to next... (#{e.class.name})"
 		end
 		
 		# ensure the folder exists

data/lib/dribbble_bucket_sync/version.rb

@@ -1,3 +1,3 @@
 class DribbbleBucketSync
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/dribbble_bucket_sync.rb

@@ -31,6 +31,9 @@ class DribbbleBucketSync
 			buckets.each do |bucket|
 				sync_bucket(bucket, directory)
 			end
+			# being nice to Dribbble site
+			# sleep for 1 - 3 seconds every 5 buckets
+			sleep(3) + 1
 		end
 	end
 	
@@ -54,9 +57,12 @@ class DribbbleBucketSync
 			shots.each do |shot|
 				folder.add_shot(shot)
 			end
+			# being nice to Dribbble site
+			# sleep for 0 - 2 seconds every 15 shots
+			sleep(3)
 		end
-		# remove old shots
-		folder.remove_old_shots
+		# process downloads / deletions
+		folder.sync
 	end
 	
 	def connection

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dribbble-bucket-sync
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -16,17 +16,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.6
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement