outatime 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9413cb4ec7dfd1657acd3b8f031fc03b74c9ca9c
-  data.tar.gz: d880faa841f6e506d602c22cbb5ad76027f2f1ff
+  metadata.gz: ed16328dea0d2e300283d1982a891c17a23341bb
+  data.tar.gz: b2bd09379a2fe1ef252ab34894f88fbcb33d468a
 SHA512:
-  metadata.gz: d258fd21d762e66e887507f5630af69e86cd1532633b64b115e1c6d78767e94db640c5bbf13923ad8f924ba3afd4a5032d26ad637bf8eec09b01961f02420326
-  data.tar.gz: f8044e58e7f1ca9a4acec8551a663b48612f72837bed2d3d112557bd7cc765758673a171b329ed73d229f0988114c89943195cf014ae60a5c002ec2536be134e
+  metadata.gz: 30b90838d71c9d13f5dde1b1e1b512c324b7c5e9b349b6af99593d8e943834a84ebc1d13bb570dd549f21e0698bf8e9209d00cc12860c49d3a05fb1228e5955c
+  data.tar.gz: bb2152657dd2c26d0501b9814b97e5242f433aae73ea955a96e43218f0ecd1b6063b4deb1e3a3c2695ccf134fe1daf4ed34b11e47eafc53651c68c17d04ad2c1

data/lib/outatime/cli.rb

@@ -22,7 +22,7 @@ module Outatime
     def run
       fetcher = Outatime::Fetcher.new(options)
 
-      pb = ProgressBar.create(total: fetcher.total_size,
+      pb = ProgressBar.create(total: nil,
                               format: "%t: |%B| %f %c/%C %R MB/sec",
                               rate_scale: lambda { |rate| rate / 1024 / 1024 },
                               throttle_rate: 0.5)

data/lib/outatime/fetcher.rb

@@ -1,3 +1,45 @@
+require 'thread/pool'
+
+# Outatime::Fetcher is responsible for finding the exact revision for each file
+# for a given time.
+#
+# AWS S3 API lists all file revisions in a very particular order and this
+# class takes advantage of that to quickly parse and find the revision.
+#
+# The 'GET Bucket Object Versions'
+# (http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html)
+# returns all file revisions ordered by key (path + filename) and last
+# modified time.
+#
+# For example:
+#
+# id: 12456, key: "src/file1", last_modified: 7 Feb 11:38
+# id: 12357, key: "src/file1", last_modified: 7 Feb 11:37
+# id: 12357, key: "src/file1", last_modified: 7 Feb 11:00
+# id: 22222, key: "src/file2", last_modified: 7 Feb 11:39
+# ...
+#
+# Keep in mind that this response is paginated, where the max amount of
+# revisions per page is 1000.
+#
+# When a versioned bucket contains a huge amount of files and revisions,
+# fetching all data may take a long time. So Outatime::Fetcher starts
+# downloading the correct revision even when not all file revisions are
+# already fetched, i.e. when it is still downloading the revisions and their
+# data. That accelerates the process.
+#
+# So how do we know when we have all information we need before start downloading
+# a specific file revision? As the response is ordered by key (filename) and
+# timestamps, fetcher can start downloading if the actual response page
+# contains all possible revisions for a file, i.e. its revisions aren't
+# paginated.
+#
+# When the response for a given key (filename) is paginated, the file is not
+# downloaded until all revisions are fetched in the next response page.
+#
+# This algorithm removes the need to fetch all file revisions (which may take
+# several requests) for a versioned bucket before starting to download its files,
+# but acts on each individual file instead.
 module Outatime
   class Fetcher
     attr_accessor :options
@@ -15,10 +57,10 @@ module Outatime
     #
     def initialize(options = {})
       @options           = options
-      @files_mutex       = Mutex.new
       @fetch_block_mutex = Mutex.new
       @s3_client         = options[:s3_client] if options[:s3_client]
-      @from = ::Chronic.parse(@options[:from]) if @options[:from]
+      @from              = ::Chronic.parse(@options[:from]) if @options[:from]
+      @pool              = Thread.pool(@options.fetch(:threads, 20))
 
       # raise if the date/time was not parsed
       raise ArgumentError, "The from time was not parseable." if @from.nil?
@@ -31,7 +73,11 @@ module Outatime
     #
     # Returns nothing.
     def fetch!(&block)
-      fetch_objects(object_versions, &block)
+      object_versions do |object_version|
+        fetch_object(object_version, &block)
+      end
+
+      @pool.wait(:done)
     end
 
     # Public: Returns the objects total size.
@@ -45,34 +91,29 @@ module Outatime
     #
     # Returns an Array of Aws::S3::Types::ObjectVersion.
     def object_versions
-      puts "fetching object versions from #{@from}" if verbose?
-      @files ||= begin
-        versions = []
-        delete_markers = []
-
-        s3_client.list_object_versions(bucket: @options[:bucket],
-          prefix: @options[:prefix]).each do |response|
-
-          versions.concat(filter_future_items(response.versions))
-          delete_markers.concat(filter_future_items(response.delete_markers))
-        end
-
-        # keep only the latest versions
-        # AWS lists the latest versions first, so it should be OK to use uniq here.
-        versions.uniq!       { |obj| obj.key }
-        delete_markers.uniq! { |obj| obj.key }
-
-        delete_marker_keys = delete_markers.map { |dm| dm.key }
-
-        # check versions to see if we have newer delete_markers
-        # if so, delete those versions
-        versions.delete_if do |version|
-          if dm_index = delete_marker_keys.index(version.key)
-            if version.last_modified <= delete_markers[dm_index].last_modified
-              true
+      remaining_versions       = []
+      remaining_delete_markers = []
+
+      s3_client.list_object_versions(bucket: @options[:bucket],
+        prefix: @options[:prefix]).each do |response|
+
+        versions              = remaining_versions.concat(response.versions)
+        versions_by_key       = versions.group_by {|v| v.key }
+        delete_markers        = remaining_delete_markers.concat(response.delete_markers)
+        delete_markers_by_key = delete_markers.group_by {|v| v.key }
+
+        versions_by_key.each do |key, versions|
+          next if response.next_key_marker == key
+          filter_items(versions).each do |version|
+            dl_marker = filter_items(Array(delete_markers_by_key[version.key])).first
+            if dl_marker.nil? || (version.last_modified > dl_marker.last_modified)
+              yield version
             end
           end
         end
+
+        remaining_versions       = Array(versions_by_key[response.next_key_marker])
+        remaining_delete_markers = Array(delete_markers_by_key[response.next_key_marker])
       end
     end
 
@@ -90,33 +131,24 @@ module Outatime
     # files - an Array of Aws::S3::Types::ObjectVersion.
     #
     # Returns nothing.
-    def fetch_objects(files)
-      threads = []
-
-      @options[:threads].times do
-        threads << Thread.new do
-          while !(file = @files_mutex.synchronize { files.pop }).nil? do
-            dest = Pathname.new("#{@options[:destination]}/#{file.key}")
-
-            if file.key.end_with?("/")
-              puts "Creating s3 subdirectory #{file.key} - #{Time.now}" if verbose?
-              dest.mkpath
-            else
-              dest.dirname.mkpath
-
-              puts "Copying from s3 #{file.key} - #{Time.now}" if verbose?
-              s3_client.get_object(response_target: "#{dest}",
-                            bucket: @options[:bucket],
-                            key: file.key,
-                            version_id: file.version_id)
-            end
-
-            @fetch_block_mutex.synchronize { yield file } if block_given?
-          end
+    def fetch_object(file)
+      @pool.process do
+        dest = Pathname.new("#{@options[:destination]}/#{file.key}")
+        if file.key.end_with?("/")
+          puts "Creating s3 subdirectory #{file.key} - #{Time.now}" if verbose?
+          dest.mkpath
+        else
+          dest.dirname.mkpath
+
+          puts "Copying from s3 #{file.key} - #{Time.now}" if verbose?
+          s3_client.get_object(response_target: "#{dest}",
+                        bucket: @options[:bucket],
+                        key: file.key,
+                        version_id: file.version_id)
         end
-      end
 
-      threads.map(&:join)
+        @fetch_block_mutex.synchronize { yield file } if block_given?
+      end
     end
 
     # Private: Creates the S3 client instance.
@@ -132,8 +164,8 @@ module Outatime
     # items - An Array of objects. Object must respond to #last_modified.
     #
     # Returns Array.
-    def filter_future_items(items)
-      items.keep_if { |obj| obj.last_modified <= @from }
+    def filter_items(items)
+      items.keep_if { |obj| obj.last_modified <= @from }.uniq {|obj| obj.key }
     end
   end
 end

data/lib/outatime/version.rb

@@ -1,3 +1,3 @@
 module Outatime
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: outatime
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Justin Mazzi
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-01 00:00:00.000000000 Z
+date: 2017-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk