bramble 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 138b78b6cd5ccf7de287e447453cb010de725eb7
-  data.tar.gz: 554476911d84da387cc7017b4bb0041810507fe7
+  metadata.gz: 61f945ee023bed4fa6464c90b8568060a1768824
+  data.tar.gz: 3d2b138f12b2fe5421914732cbbaf04b94a8b40a
 SHA512:
-  metadata.gz: c78a8f5a1eaf795df61ef62439ee77d2de522edea38f9c8ee77538662b91410615b362ea25e7b387f333cc2461d8b32a631130c1c277b1852befdd3062bcef9d
-  data.tar.gz: f112df8555c290fbf037ec7c401014f4d5c6ec1794a0351317baf568632c211b0ca794c75f7ac6505ae40f2213b40c2603df59394de5b223796ab8bb3e429f27
+  metadata.gz: 151e9f40aec216edbadbec3ea225633957c4f294a145ced9fd2c90ac70ba02f52d2014eb09f83011b29eba0224ad0085a2a059a3b297d460ea102325f4068cca
+  data.tar.gz: a13af709e45714aba91eee42ab8a6600f02c4c76574f39b7df9e5591fcf6c6f2234883a68eff619bd22a6e60bd9b4294aebc10cf9366930c750b53e151b81b9c

data/README.md

@@ -44,6 +44,16 @@ We have some staff-only views that expose stats about how people use our app. Ev
       # observations => [1, 1, 1, 1, 1]
       observations.length
     end
+
+    # If a .map or .reduce hits an error,
+    # it will be rescued and passed here.
+    # To cause the job to fail, raise it again.
+    # Otherwise, let it pass
+    def self.on_error(err)
+      Bugsnag.notify(err)
+      # Or, to trigger a faiure:
+      # raise(err)
+    end
   end
   ```
 
@@ -83,7 +93,8 @@ We have some staff-only views that expose stats about how people use our app. Ev
 
 ## Todo
 
-- Adapters: Memcached, ActiveRecord
+- Write some more adapters: Memcached, ActiveRecord
+- Warn if no storage is configured
 - Do we have atomicity issues? Occasional test failures
 - Consolidate storage in Redis to a single key? (Could some keys be evicted while others remain?)
 

data/lib/bramble.rb

@@ -2,6 +2,7 @@ require "ostruct"
 require "active_job"
 require "bramble/base_job"
 require "bramble/begin_job"
+require "bramble/error_handling"
 require "bramble/keys"
 require "bramble/map"
 require "bramble/map_job"

data/lib/bramble/base_job.rb

@@ -1,4 +1,6 @@
 module Bramble
+  # This is the parent class for all Bramble jobs.
+  # It sets the queue based on your config.
   class BaseJob < ActiveJob::Base
     queue_as { Bramble.config.queue_as }
   end

data/lib/bramble/begin_job.rb

@@ -4,6 +4,7 @@ module Bramble
   #
   # Then it starts the map-reduce job.
   class BeginJob < Bramble::BaseJob
+    # Call the defined `.items(options)` function
     def perform(handle, implementation_name, items_options)
       implementation = implementation_name.constantize
       all_items = implementation.items(items_options)

data/lib/bramble/error_handling.rb

@@ -0,0 +1,11 @@
+module Bramble
+  module ErrorHandling
+    # If an error is raised during the block,
+    # pass it to the implementation's `on_error` function.
+    def self.rescuing(implementation)
+      yield
+    rescue StandardError => err
+      implementation.on_error(err)
+    end
+  end
+end

data/lib/bramble/keys.rb

@@ -1,4 +1,5 @@
 module Bramble
+  # A collection of helpers for getting string keys in the storage adapter.
   module Keys
     module_function
 

data/lib/bramble/map.rb

@@ -4,6 +4,7 @@ module Bramble
 
     module_function
 
+    # For each of `values`, queue up a job to call the map function
     def perform(handle, implementation, values)
       Bramble::State.running?(handle) do
         storage.set(map_total_count_key(handle), values.length)
@@ -13,18 +14,25 @@ module Bramble
       end
     end
 
+    # Call `.map` on `value`, storing the result for `.reduce` and handling any error.
     def perform_map(handle, implementation, value)
       Bramble::State.running?(handle) do
         impl_keys_key = keys_key(handle)
-        implementation.map(value) do |map_key, map_val|
-          Bramble::State.running?(handle) do
-            raw_key = Bramble::Serialize.dump(map_key)
-            raw_value = Bramble::Serialize.dump(map_val)
-            storage.map_keys_push(impl_keys_key, raw_key)
-            storage.map_result_push(data_key(handle, raw_key), raw_value)
+
+        Bramble::ErrorHandling.rescuing(implementation) do
+          # Execute the provided map function
+          implementation.map(value) do |map_key, map_val|
+            Bramble::State.running?(handle) do
+              raw_key = Bramble::Serialize.dump(map_key)
+              raw_value = Bramble::Serialize.dump(map_val)
+              # Push the result to be reduced
+              storage.map_keys_push(impl_keys_key, raw_key)
+              storage.map_result_push(data_key(handle, raw_key), raw_value)
+            end
           end
         end
 
+        # Mark this item as mapped (even if there was an error)
         Bramble::State.running?(handle) do
           finished = storage.increment(map_finished_count_key(handle))
           total = storage.get(map_total_count_key(handle)).to_i

data/lib/bramble/map_job.rb

@@ -1,4 +1,5 @@
 module Bramble
+  # Calls the `.map(value)` function
   class MapJob < Bramble::BaseJob
     def perform(handle, mapper_name, raw_value)
       mapper = mapper_name.constantize

data/lib/bramble/reduce.rb

@@ -4,24 +4,38 @@ module Bramble
 
     module_function
 
+    # Queue up a job to call `.reduce` on keys for `handle`
     def perform(handle, implementation)
       Bramble::State.running?(handle) do
+        # Set how many reduce call we expect
         all_raw_keys = storage.map_keys_get(keys_key(handle))
         storage.set(reduce_total_count_key(handle), all_raw_keys.length)
+        # Enqueue a job for each reduce call
         all_raw_keys.each do |raw_key|
           Bramble::ReduceJob.perform_later(handle, implementation.name, raw_key)
         end
       end
     end
 
+    # Perform `.reduce` on `raw_key`, handling errors and saving the result
     def perform_reduce(handle, implementation, raw_key)
       if Bramble::State.running?(handle)
         raw_values = storage.map_result_get(data_key(handle, raw_key))
         values = Bramble::Serialize.load(raw_values)
         key = Bramble::Serialize.load(raw_key)
-        reduced_value = implementation.reduce(key, values)
+        reduced_value = nil
+
+        Bramble::ErrorHandling.rescuing(implementation) do
+          # Run the defined .reduce function
+          reduced_value = implementation.reduce(key, values)
+          # Store the result
+          Bramble::State.running?(handle) do
+            storage.reduce_result_set(result_key(handle), raw_key, Bramble::Serialize.dump(reduced_value))
+          end
+        end
+
+        # Mark this key as reduced, check if we're finished
         Bramble::State.running?(handle) do
-          storage.reduce_result_set(result_key(handle), raw_key, Bramble::Serialize.dump(reduced_value))
           storage.increment(reduce_finished_count_key(handle))
           if Bramble::State.percent_reduced(handle) >= 1
             storage.set(finished_at_key(handle), Time.now.to_i)

data/lib/bramble/reduce_job.rb

@@ -1,4 +1,5 @@
 module Bramble
+  # Call `.reduce` for `key`
   class ReduceJob < Bramble::BaseJob
     def perform(handle, implementation_name, key)
       implementation = implementation_name.constantize

data/lib/bramble/result.rb

@@ -4,6 +4,9 @@ module Bramble
 
     attr_reader :handle, :percent_mapped, :percent_reduced, :finished_at
 
+    # Read the state for `handle` and store it in this object
+    # The state for `handle` may change during this time, but you won't
+    # see the changes until you get a new result.
     def initialize(handle)
       job_id = storage.get(Bramble::Keys.job_id_key(handle))
       @handle = "#{handle}:#{job_id}"
@@ -17,6 +20,7 @@ module Bramble
       end
     end
 
+    # @return [Hash<Any, Any>] The `key => value` results of `.reduce`
     def data
       @data ||= begin
         key = Bramble::Keys.result_key(handle)
@@ -25,15 +29,20 @@ module Bramble
       end
     end
 
+    # @return [Boolean] True if all data has been mapped and reduced
     def finished?
       # Possible to be greater than 1 because of floating-point arithmetic
       percent_finished >= 1
     end
 
+    # @return [Boolean] True if the job has been started but it isn't finished yet
     def running?
       started? && !finished?
     end
 
+    # How far along is this job?
+    # `.map` is considered 50%, `.reduce` is considered 50%
+    # @return [Float] Percent progress for this job
     def percent_finished
       (percent_mapped + percent_reduced) / 2
     end

data/lib/bramble/state.rb

@@ -1,4 +1,5 @@
 module Bramble
+  # Helpers for detecting and managing the state of a given `handle` while the job is (or isn't) running
   module State
     extend Bramble::Keys
     SEPARATOR = ":"
@@ -14,6 +15,7 @@ module Bramble
       is_running
     end
 
+    # Mark `handle` as started
     def start_job(handle)
       handle_name, job_id = handle.split(SEPARATOR)
       previous_job_id = storage.get(job_id_key(handle_name))
@@ -24,6 +26,7 @@ module Bramble
       storage.set(job_id_key(handle_name), job_id)
     end
 
+    # Clear the state of `handle`
     def clear_job(handle)
       handle_name, job_id = handle.split(SEPARATOR)
       storage.delete(job_id_key(handle_name))
@@ -32,6 +35,7 @@ module Bramble
       clear_map(handle)
     end
 
+    # How many values of `handle` have been sent to `.map`?
     def percent_mapped(handle)
       percent_between_keys(
         map_total_count_key(handle),
@@ -39,6 +43,7 @@ module Bramble
       )
     end
 
+    # How many values of `handle` have been sent to `.reduce?`
     def percent_reduced(handle)
       percent_between_keys(
         reduce_total_count_key(handle),
@@ -46,6 +51,7 @@ module Bramble
       )
     end
 
+    # Clear all traces of the `.map` operation for `handle`
     def clear_map(handle)
       map_group_keys = storage.map_keys_get(keys_key(handle))
       map_group_keys.each do |group_key|
@@ -56,6 +62,7 @@ module Bramble
       storage.delete(map_finished_count_key(handle))
     end
 
+    # Clear all traces of the `.reduce` operation for `handle`
     def clear_reduce(handle)
       storage.delete(reduce_total_count_key(handle))
       storage.delete(reduce_finished_count_key(handle))

data/lib/bramble/storage.rb

@@ -3,6 +3,8 @@ require "bramble/storage/redis_storage"
 require "bramble/storage/memory_storage"
 
 module Bramble
+  # A single access point to the storage selected by `Bramble.config.storage`.
+  # All methods are delegated to that storage adapter
   module Storage
     extend SingleForwardable
 

data/lib/bramble/version.rb

@@ -1,3 +1,3 @@
 module Bramble
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: bramble
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Robert Mosolgo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-16 00:00:00.000000000 Z
+date: 2016-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,7 @@ files:
 - lib/bramble/base_job.rb
 - lib/bramble/begin_job.rb
 - lib/bramble/conf.rb
+- lib/bramble/error_handling.rb
 - lib/bramble/keys.rb
 - lib/bramble/map.rb
 - lib/bramble/map_job.rb