bramble 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 044266e293f683cdeabe0d4f9d7789d49324195c
-  data.tar.gz: 6b516203cd8fc97eb20810245a05d08a96c3e898
+  metadata.gz: c78abeda625df09bdbbfbaa059768757a54380f9
+  data.tar.gz: b641d188443171568abe011b22fc5b3ccf7083e6
 SHA512:
-  metadata.gz: 77f06d4145ce1ad5622cca85c687531ca3b76bcb9c94831d0015e15118cf34a110828de0cd0c2e706f403503f44b16f3ffb858713380090ad2bd99a08025b27c
-  data.tar.gz: d7b4d8f7c716d1a7b70a1fd4b57ac6be24c166494f2be03e0d86b1b25336bd542c56b5ad0adcc03bf359bacd6197764322f206e5575842213294d8f3d1ffc564
+  metadata.gz: 3d5e01178d77d2cd06c99634318835143dbdcb3ba203ecc6477cb7bb60d32019521d531660bc1a8f6e7eab258d0d55ec617113ee61e112c3e31a74a9094b655a
+  data.tar.gz: a54391b20995074b7b16382d75823d247f9fb267f27bae2804db4ed5e97c3fc2fbd331cbf666e8a5d42817aafabe8a26c3f194148bf29cc6010876cd2763c4d4

data/README.md

@@ -15,10 +15,15 @@ Map-reduce with ActiveJob
   end
   ```
 
-- Define a module with `map` and `reduce` functions:
+- Define a module with `map`, `reduce` and `items(options = {})` functions:
 
   ```ruby
   module LetterCount
+    # Generate a list of items based on some input
+    def self.items(filepath)
+      File.read(filepath).split(" ")
+    end
+
     # .map is called with each item in the input
     def self.map(word)
       letters = word.upcase.each_char
@@ -38,24 +43,26 @@ Map-reduce with ActiveJob
   end
   ```
 
-- Start a job with a handle, module, and some data:
+- Start a job with a handle, module, and an (optional) argument for finding data:
 
   ```ruby
   # used for fetching the result later:
   handle = "shakespeare-letter-count"
 
-  # Something that responds to #each:
-  data = hamlet.split(" ")
+  # will be sent to `.items(filepath)`
+  hamlet_path = "./shakespeare/hamlet.txt"
 
   # Begin the process:
-  Bramble.map_reduce(handle, LetterCount, words_in_hamlet)
+  Bramble.map_reduce(handle, LetterCount, hamlet_path)
   ```
 
 - Later, fetch the result using the handle:
 
   ```ruby
-  result = Bramble.read("shakespeare-letter-count")
-  # { "A" => 100, "B" => 100, ... }
+  result = Bramble.get("shakespeare-letter-count")
+  result.running?   # => false
+  result.finished?  # => true
+  result.data       # => { "A" => 100, "B" => 100, ... }
   ```
 
 - Delete the saved result:
@@ -66,8 +73,6 @@ Map-reduce with ActiveJob
 
 ## Todo
 
-- Use `Storage` as gateway to `config.storage`
-- Job convenience class?
 - `.fetch` to find-or-calculate?
 - Adapters: Memcache, ActiveRecord
 

data/lib/bramble/base_job.rb

@@ -0,0 +1,5 @@
+module Bramble
+  class BaseJob < ActiveJob::Base
+    queue_as { Bramble.config.queue_as }
+  end
+end

data/lib/bramble/begin_job.rb

@@ -0,0 +1,13 @@
+module Bramble
+  # This exists to call `implementation.items` in the background.
+  # It might take a long time to fetch, so let's background it.
+  #
+  # Then it starts the map-reduce job.
+  class BeginJob < Bramble::BaseJob
+    def perform(handle, implementation_name, items_options)
+      implementation = implementation_name.constantize
+      all_items = implementation.items(items_options)
+      Bramble::Map.perform(handle, implementation, all_items)
+    end
+  end
+end

data/lib/bramble/keys.rb

@@ -1,5 +1,7 @@
 module Bramble
   module Keys
+    module_function
+
     def namespace(handle)
       "#{Bramble.config.namespace}:#{handle}"
     end
@@ -12,8 +14,12 @@ module Bramble
       "#{namespace(handle)}:keys"
     end
 
-    def finished_count_key(handle)
-      "#{namespace(handle)}:finished_count"
+    def map_finished_count_key(handle)
+      "#{namespace(handle)}:map_finished_count"
+    end
+
+    def reduce_finished_count_key(handle)
+      "#{namespace(handle)}:reduce_finished_count"
     end
 
     def total_count_key(handle)
@@ -23,5 +29,13 @@ module Bramble
     def result_key(handle)
       "#{namespace(handle)}:result"
     end
+
+    def job_id_key(handle)
+      "#{namespace(handle)}:job_id"
+    end
+
+    def status_key(handle)
+      "#{namespace(handle)}:status"
+    end
   end
 end

data/lib/bramble/map.rb

@@ -5,25 +5,32 @@ module Bramble
     module_function
 
     def perform(handle, implementation, values)
-      # TODO: make sure there isn't one going on right now
-      Bramble::Storage.delete(handle)
-      storage.set(total_count_key(handle), values.length)
-      values.each do |value|
-        Bramble::MapJob.perform_later(handle, implementation.name, value)
+      Bramble::State.running?(handle) do
+        storage.set(total_count_key(handle), values.length)
+        values.each do |value|
+          Bramble::MapJob.perform_later(handle, implementation.name, Bramble::Serialize.dump(value))
+        end
       end
     end
 
     def perform_map(handle, implementation, value)
-      impl_keys_key = keys_key(handle)
-      implementation.map(value) do |map_key, map_val|
-        raw_key = Bramble::Storage.dump(map_key)
-        storage.map_keys_push(impl_keys_key, raw_key)
-        storage.map_result_push(data_key(handle, raw_key), Bramble::Storage.dump(map_val))
-      end
-      finished = storage.increment(finished_count_key(handle))
-      total = storage.get(total_count_key(handle)).to_i
-      if finished == total
-        Bramble::Reduce.perform(handle, implementation)
+      Bramble::State.running?(handle) do
+        impl_keys_key = keys_key(handle)
+        implementation.map(value) do |map_key, map_val|
+          Bramble::State.running?(handle) do
+            raw_key = Bramble::Serialize.dump(map_key)
+            storage.map_keys_push(impl_keys_key, raw_key)
+            storage.map_result_push(data_key(handle, raw_key), Bramble::Serialize.dump(map_val))
+          end
+        end
+        Bramble::State.running?(handle) do
+          finished = storage.increment(map_finished_count_key(handle))
+          total = storage.get(total_count_key(handle)).to_i
+          if finished == total
+            Bramble::Reduce.perform(handle, implementation)
+            Bramble::State.clear_map(handle)
+          end
+        end
       end
     end
 
@@ -32,7 +39,7 @@ module Bramble
     module_function
 
     def storage
-      Bramble.config.storage
+      Bramble::Storage
     end
   end
 end

data/lib/bramble/map_job.rb

@@ -1,8 +1,8 @@
 module Bramble
-  class MapJob < ActiveJob::Base
-    queue_as { Bramble.config.queue_as }
-    def perform(handle, mapper_name, value)
+  class MapJob < Bramble::BaseJob
+    def perform(handle, mapper_name, raw_value)
       mapper = mapper_name.constantize
+      value = Bramble::Serialize.load(raw_value)
       Bramble::Map.perform_map(handle, mapper, value)
     end
   end

data/lib/bramble/reduce.rb

@@ -12,10 +12,18 @@ module Bramble
     end
 
     def perform_reduce(handle, implementation, raw_key)
-      values = storage.map_result_get(data_key(handle, raw_key))
-      values = Bramble::Storage.load(values)
-      reduced_value = implementation.reduce(Bramble::Storage.load(raw_key), values)
-      storage.reduce_result_set(result_key(handle), raw_key, Bramble::Storage.dump(reduced_value))
+      if Bramble::State.running?(handle)
+        raw_values = storage.map_result_get(data_key(handle, raw_key))
+        values = Bramble::Serialize.load(raw_values)
+        key = Bramble::Serialize.load(raw_key)
+        reduced_value = implementation.reduce(key, values)
+        Bramble::State.running?(handle) do
+          storage.reduce_result_set(result_key(handle), raw_key, Bramble::Serialize.dump(reduced_value))
+          storage.increment(reduce_finished_count_key(handle))
+        end
+      else
+        Bramble::State.clear_reduce(handle)
+      end
     end
 
     private
@@ -23,7 +31,7 @@ module Bramble
     module_function
 
     def storage
-      Bramble.config.storage
+      Bramble::Storage
     end
   end
 end

data/lib/bramble/reduce_job.rb

@@ -1,9 +1,8 @@
 module Bramble
-  class ReduceJob < ActiveJob::Base
-    queue_as { Bramble.config.queue_as }
-    def perform(handle, reducer_name, key)
-      reducer = reducer_name.constantize
-      Bramble::Reduce.perform_reduce(handle, reducer, key)
+  class ReduceJob < Bramble::BaseJob
+    def perform(handle, implementation_name, key)
+      implementation = implementation_name.constantize
+      Bramble::Reduce.perform_reduce(handle, implementation, key)
     end
   end
 end

data/lib/bramble/result.rb

@@ -0,0 +1,46 @@
+module Bramble
+  # This class exposes the data and some info about the state of the task
+  class Result
+
+    attr_reader :handle
+
+    def initialize(handle)
+      job_id = storage.get(Bramble::Keys.job_id_key(handle))
+      @handle = "#{handle}:#{job_id}"
+    end
+
+    def data
+      @data ||= begin
+        key = Bramble::Keys.result_key(handle)
+        results = storage.reduce_result_get(key)
+        Bramble::Serialize.load(results)
+      end
+    end
+
+    def finished?
+      @finished ||= finished_count > 0 && total_count == finished_count
+    end
+
+    def running?
+      @running ||= started? && !finished?
+    end
+
+    private
+
+    def total_count
+      @total_count ||= storage.get(Bramble::Keys.total_count_key(handle)).to_i
+    end
+
+    def finished_count
+      @finished_count ||= storage.get(Bramble::Keys.reduce_finished_count_key(handle)).to_i
+    end
+
+    def started?
+      @started ||= !!storage.get(Bramble::Keys.status_key(handle))
+    end
+
+    def storage
+      Bramble::Storage
+    end
+  end
+end

data/lib/bramble/serialize.rb

@@ -0,0 +1,24 @@
+module Bramble
+  # eg, Redis uses strings only, so use this module to freeze and thaw values from storage
+  module Serialize
+    # prepare an object for storage
+    def self.dump(obj)
+      Marshal.dump(obj)
+    end
+
+    # reload an object from storage
+    def self.load(stored_obj)
+      case stored_obj
+      when Array
+        stored_obj.map { |obj| load(obj) }
+      when Hash
+        stored_obj.inject({}) do |memo, (k, v)|
+          memo[load(k)] = load(v)
+          memo
+        end
+      else
+        Marshal.load(stored_obj)
+      end
+    end
+  end
+end

data/lib/bramble/state.rb

@@ -0,0 +1,56 @@
+module Bramble
+  module State
+    extend Bramble::Keys
+    SEPARATOR = ":"
+    module_function
+
+    # Run the block and return true if the `job_id` is still active
+    def running?(handle)
+      handle_name, job_id = handle.split(SEPARATOR)
+      is_running = storage.get(job_id_key(handle_name)) == job_id
+      if block_given?
+        yield
+      end
+      is_running
+    end
+
+    def start_job(handle)
+      handle_name, job_id = handle.split(SEPARATOR)
+      previous_job_id = storage.get(job_id_key(handle_name))
+      if previous_job_id
+        clear_job("#{handle_name}:#{previous_job_id}")
+      end
+      storage.set(status_key(handle), "started")
+      storage.set(job_id_key(handle_name), job_id)
+    end
+
+    def clear_job(handle)
+      handle_name, job_id = handle.split(SEPARATOR)
+      storage.delete(job_id_key(handle_name))
+      storage.delete(status_key(handle))
+      clear_reduce(handle)
+      clear_map(handle)
+    end
+
+    def clear_map(handle)
+      map_group_keys = storage.map_keys_get(keys_key(handle))
+      map_group_keys.each do |group_key|
+        storage.delete(data_key(handle, group_key))
+      end
+      storage.delete(keys_key(handle))
+      storage.delete(map_finished_count_key(handle))
+    end
+
+    def clear_reduce(handle)
+      storage.delete(total_count_key(handle))
+      storage.delete(reduce_finished_count_key(handle))
+      storage.delete(result_key(handle))
+    end
+
+    private
+
+    def self.storage
+      Bramble::Storage
+    end
+  end
+end

data/lib/bramble/storage/memory_storage.rb

@@ -9,6 +9,10 @@ module Bramble
 
       module_function
 
+      def transaction
+        yield
+      end
+
       def set(key, value)
         STORAGE[key] = value
       end

data/lib/bramble/storage.rb

@@ -1,56 +1,20 @@
+require "forwardable"
 require "bramble/storage/redis_storage"
 require "bramble/storage/memory_storage"
 
 module Bramble
   module Storage
-    extend Bramble::Keys
+    extend SingleForwardable
 
-    def self.read(handle)
-      key = result_key(handle)
-      results = storage.reduce_result_get(key)
-      load(results)
-    end
-
-    # Wipe out the results for this handle
-    def self.delete(handle)
-      # Reset counts
-      storage.delete(total_count_key(handle))
-      storage.delete(finished_count_key(handle))
-      # Reset result
-      storage.delete(result_key(handle))
-
-      # Reset dangling map data
-      map_group_keys = storage.map_keys_get(keys_key(handle))
-      map_group_keys.each do |group_key|
-        storage.delete(data_key(handle, group_key))
-      end
-      storage.delete(keys_key(handle))
-    end
-
-
-    # prepare an object for storage
-    def self.dump(obj)
-      Marshal.dump(obj)
-    end
-
-    # reload an object from storage
-    def self.load(stored_obj)
-      case stored_obj
-      when Array
-        stored_obj.map { |obj| load(obj) }
-      when Hash
-        stored_obj.inject({}) do |memo, (k, v)|
-          memo[load(k)] = load(v)
-          memo
-        end
-      else
-        Marshal.load(stored_obj)
-      end
-    end
+    def_delegators :storage_instance,
+      :set, :get, :delete, :increment,
+      :map_result_push, :map_result_get,
+      :reduce_result_set, :reduce_result_get,
+      :map_keys_push, :map_keys_get
 
     private
 
-    def self.storage
+    def self.storage_instance
       Bramble.config.storage
     end
   end

data/lib/bramble/version.rb

@@ -1,3 +1,3 @@
 module Bramble
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/bramble.rb

@@ -1,10 +1,15 @@
 require "ostruct"
 require "active_job"
+require "bramble/base_job"
+require "bramble/begin_job"
 require "bramble/keys"
 require "bramble/map"
 require "bramble/map_job"
 require "bramble/reduce"
 require "bramble/reduce_job"
+require "bramble/result"
+require "bramble/serialize"
+require "bramble/state"
 require "bramble/storage"
 require "bramble/version"
 require "bramble/conf"
@@ -19,19 +24,27 @@ module Bramble
   end
 
   # @param handle [String] This string will be used to store the result
-  # @param implementation [#map, #reduce, #name] The container of map and reduce methods
-  # @param items [Array] List of items to map over
-  def self.map_reduce(handle, implementation, items)
-    Bramble::Map.perform(handle, implementation, items)
+  # @param implementation [.map, .reduce, .name, .items(options)] The container of map and reduce methods
+  # @param items_options [Object] will be passed to .items
+  def self.map_reduce(handle, implementation, items_options = {})
+    # Secret feature: the implementation can provide a job_id
+    job_id = if implementation.respond_to?(:job_id)
+      implementation.job_id
+    else
+      Time.now.strftime("%s%6N")
+    end
+    handle = "#{handle}:#{job_id}"
+    Bramble::State.start_job(handle)
+    Bramble::BeginJob.perform_later(handle, implementation.name, items_options)
   end
 
-  # Get results for `handle`, if they exist
-  def self.read(handle)
-    Bramble::Storage.read(handle)
+  # @return [Bramble::Result] Status & data for this handle
+  def self.get(handle)
+    Bramble::Result.new(handle)
   end
 
   # Remove results for `handle`, if there are any
   def self.delete(handle)
-    Bramble::Storage.delete(handle)
+    Bramble::State.clear_job(handle)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bramble
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Robert Mosolgo
@@ -103,12 +103,17 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/bramble.rb
+- lib/bramble/base_job.rb
+- lib/bramble/begin_job.rb
 - lib/bramble/conf.rb
 - lib/bramble/keys.rb
 - lib/bramble/map.rb
 - lib/bramble/map_job.rb
 - lib/bramble/reduce.rb
 - lib/bramble/reduce_job.rb
+- lib/bramble/result.rb
+- lib/bramble/serialize.rb
+- lib/bramble/state.rb
 - lib/bramble/storage.rb
 - lib/bramble/storage/memory_storage.rb
 - lib/bramble/storage/redis_storage.rb