bramble 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 044266e293f683cdeabe0d4f9d7789d49324195c
+  data.tar.gz: 6b516203cd8fc97eb20810245a05d08a96c3e898
+SHA512:
+  metadata.gz: 77f06d4145ce1ad5622cca85c687531ca3b76bcb9c94831d0015e15118cf34a110828de0cd0c2e706f403503f44b16f3ffb858713380090ad2bd99a08025b27c
+  data.tar.gz: d7b4d8f7c716d1a7b70a1fd4b57ac6be24c166494f2be03e0d86b1b25336bd542c56b5ad0adcc03bf359bacd6197764322f206e5575842213294d8f3d1ffc564

data/README.md

@@ -0,0 +1,76 @@
+# Bramble [![Build Status](https://travis-ci.org/rmosolgo/bramble.svg?branch=master)](https://travis-ci.org/rmosolgo/bramble)
+
+Map-reduce with ActiveJob
+
+## Usage
+
+- Setup ActiveJob with a queue named `:bramble`
+
+- Setup Redis and give Bramble a connection object:
+
+  ```ruby
+  my_redis_connection = Redis.new # Your connection settings here!
+  Bramble.config do |conf|
+    conf.redis_conn = my_redis_connection
+  end
+  ```
+
+- Define a module with `map` and `reduce` functions:
+
+  ```ruby
+  module LetterCount
+    # .map is called with each item in the input
+    def self.map(word)
+      letters = word.upcase.each_char
+
+      # call `yield` to emit a key-value pair for processing
+      letters.each { |letter| yield(letter, 1) }
+    end
+
+    # .reduce is called with
+    # - `yield` key (first argument)
+    # - array of `yield` values (second argument)
+    def self.reduce(letter, observations)
+      # letter => "A"
+      # observations => [1, 1, 1, 1, 1]
+      observations.length
+    end
+  end
+  ```
+
+- Start a job with a handle, module, and some data:
+
+  ```ruby
+  # used for fetching the result later:
+  handle = "shakespeare-letter-count"
+
+  # Something that responds to #each:
+  data = hamlet.split(" ")
+
+  # Begin the process:
+  Bramble.map_reduce(handle, LetterCount, words_in_hamlet)
+  ```
+
+- Later, fetch the result using the handle:
+
+  ```ruby
+  result = Bramble.read("shakespeare-letter-count")
+  # { "A" => 100, "B" => 100, ... }
+  ```
+
+- Delete the saved result:
+
+  ```ruby
+  Bramble.delete("shakespeare-letter-count")
+  ```
+
+## Todo
+
+- Use `Storage` as gateway to `config.storage`
+- Job convenience class?
+- `.fetch` to find-or-calculate?
+- Adapters: Memcache, ActiveRecord
+
+## Development
+
+- `rake test`

data/lib/bramble.rb

@@ -0,0 +1,37 @@
+require "ostruct"
+require "active_job"
+require "bramble/keys"
+require "bramble/map"
+require "bramble/map_job"
+require "bramble/reduce"
+require "bramble/reduce_job"
+require "bramble/storage"
+require "bramble/version"
+require "bramble/conf"
+
+module Bramble
+  def self.config
+    if block_given?
+      yield(Bramble::CONF)
+    else
+      Bramble::CONF
+    end
+  end
+
+  # @param handle [String] This string will be used to store the result
+  # @param implementation [#map, #reduce, #name] The container of map and reduce methods
+  # @param items [Array] List of items to map over
+  def self.map_reduce(handle, implementation, items)
+    Bramble::Map.perform(handle, implementation, items)
+  end
+
+  # Get results for `handle`, if they exist
+  def self.read(handle)
+    Bramble::Storage.read(handle)
+  end
+
+  # Remove results for `handle`, if there are any
+  def self.delete(handle)
+    Bramble::Storage.delete(handle)
+  end
+end

data/lib/bramble/conf.rb

@@ -0,0 +1,8 @@
+module Bramble
+  CONF = OpenStruct.new(
+    redis_conn: nil,
+    namespace: "Bramble",
+    queue_as: :bramble,
+    storage: Bramble::Storage::RedisStorage
+  )
+end

data/lib/bramble/keys.rb

@@ -0,0 +1,27 @@
+module Bramble
+  module Keys
+    def namespace(handle)
+      "#{Bramble.config.namespace}:#{handle}"
+    end
+
+    def data_key(handle, key)
+      "#{namespace(handle)}:data:#{key}"
+    end
+
+    def keys_key(handle)
+      "#{namespace(handle)}:keys"
+    end
+
+    def finished_count_key(handle)
+      "#{namespace(handle)}:finished_count"
+    end
+
+    def total_count_key(handle)
+      "#{namespace(handle)}:total_count"
+    end
+
+    def result_key(handle)
+      "#{namespace(handle)}:result"
+    end
+  end
+end

data/lib/bramble/map.rb

@@ -0,0 +1,38 @@
+module Bramble
+  module Map
+    extend Bramble::Keys
+
+    module_function
+
+    def perform(handle, implementation, values)
+      # TODO: make sure there isn't one going on right now
+      Bramble::Storage.delete(handle)
+      storage.set(total_count_key(handle), values.length)
+      values.each do |value|
+        Bramble::MapJob.perform_later(handle, implementation.name, value)
+      end
+    end
+
+    def perform_map(handle, implementation, value)
+      impl_keys_key = keys_key(handle)
+      implementation.map(value) do |map_key, map_val|
+        raw_key = Bramble::Storage.dump(map_key)
+        storage.map_keys_push(impl_keys_key, raw_key)
+        storage.map_result_push(data_key(handle, raw_key), Bramble::Storage.dump(map_val))
+      end
+      finished = storage.increment(finished_count_key(handle))
+      total = storage.get(total_count_key(handle)).to_i
+      if finished == total
+        Bramble::Reduce.perform(handle, implementation)
+      end
+    end
+
+    private
+
+    module_function
+
+    def storage
+      Bramble.config.storage
+    end
+  end
+end

data/lib/bramble/map_job.rb

@@ -0,0 +1,9 @@
+module Bramble
+  class MapJob < ActiveJob::Base
+    queue_as { Bramble.config.queue_as }
+    def perform(handle, mapper_name, value)
+      mapper = mapper_name.constantize
+      Bramble::Map.perform_map(handle, mapper, value)
+    end
+  end
+end

data/lib/bramble/reduce.rb

@@ -0,0 +1,29 @@
+module Bramble
+  module Reduce
+    extend Bramble::Keys
+
+    module_function
+
+    def perform(handle, implementation)
+      all_raw_keys = storage.map_keys_get(keys_key(handle))
+      all_raw_keys.each do |raw_key|
+        Bramble::ReduceJob.perform_later(handle, implementation.name, raw_key)
+      end
+    end
+
+    def perform_reduce(handle, implementation, raw_key)
+      values = storage.map_result_get(data_key(handle, raw_key))
+      values = Bramble::Storage.load(values)
+      reduced_value = implementation.reduce(Bramble::Storage.load(raw_key), values)
+      storage.reduce_result_set(result_key(handle), raw_key, Bramble::Storage.dump(reduced_value))
+    end
+
+    private
+
+    module_function
+
+    def storage
+      Bramble.config.storage
+    end
+  end
+end

data/lib/bramble/reduce_job.rb

@@ -0,0 +1,9 @@
+module Bramble
+  class ReduceJob < ActiveJob::Base
+    queue_as { Bramble.config.queue_as }
+    def perform(handle, reducer_name, key)
+      reducer = reducer_name.constantize
+      Bramble::Reduce.perform_reduce(handle, reducer, key)
+    end
+  end
+end

data/lib/bramble/storage.rb

@@ -0,0 +1,57 @@
+require "bramble/storage/redis_storage"
+require "bramble/storage/memory_storage"
+
+module Bramble
+  module Storage
+    extend Bramble::Keys
+
+    def self.read(handle)
+      key = result_key(handle)
+      results = storage.reduce_result_get(key)
+      load(results)
+    end
+
+    # Wipe out the results for this handle
+    def self.delete(handle)
+      # Reset counts
+      storage.delete(total_count_key(handle))
+      storage.delete(finished_count_key(handle))
+      # Reset result
+      storage.delete(result_key(handle))
+
+      # Reset dangling map data
+      map_group_keys = storage.map_keys_get(keys_key(handle))
+      map_group_keys.each do |group_key|
+        storage.delete(data_key(handle, group_key))
+      end
+      storage.delete(keys_key(handle))
+    end
+
+
+    # prepare an object for storage
+    def self.dump(obj)
+      Marshal.dump(obj)
+    end
+
+    # reload an object from storage
+    def self.load(stored_obj)
+      case stored_obj
+      when Array
+        stored_obj.map { |obj| load(obj) }
+      when Hash
+        stored_obj.inject({}) do |memo, (k, v)|
+          memo[load(k)] = load(v)
+          memo
+        end
+      else
+        Marshal.load(stored_obj)
+      end
+    end
+
+    private
+
+    def self.storage
+      Bramble.config.storage
+    end
+  end
+end

data/lib/bramble/storage/memory_storage.rb

@@ -0,0 +1,57 @@
+require "set"
+
+module Bramble
+  module Storage
+    # ☠ This is for single-threaded, single-process Ruby only!
+    # If you try to use this in production, you're going to have a bad time.
+    module MemoryStorage
+      STORAGE = {}
+
+      module_function
+
+      def set(key, value)
+        STORAGE[key] = value
+      end
+
+      def get(key)
+        STORAGE[key]
+      end
+
+      def delete(key)
+        STORAGE.delete(key)
+      end
+
+      def increment(key)
+        STORAGE[key] ||= 0
+        STORAGE[key] += 1
+      end
+
+      def map_result_push(key, value)
+        STORAGE[key] ||= []
+        STORAGE[key] << value
+      end
+
+      def map_result_get(key)
+        STORAGE[key] || []
+      end
+
+      def reduce_result_set(storage_key, reduce_key, value)
+        STORAGE[storage_key] ||= {}
+        STORAGE[storage_key][reduce_key] = value
+      end
+
+      def reduce_result_get(storage_key)
+        STORAGE[storage_key] || {}
+      end
+
+      def map_keys_push(key, value)
+        STORAGE[key] ||= Set.new
+        STORAGE[key] << value
+      end
+
+      def map_keys_get(key)
+        STORAGE[key] || Set.new
+      end
+    end
+  end
+end

data/lib/bramble/storage/redis_storage.rb

@@ -0,0 +1,55 @@
+module Bramble
+  module Storage
+    module RedisStorage
+      module_function
+
+      def set(key, value)
+        redis_conn.set(key, value)
+      end
+
+      def get(key)
+        redis_conn.get(key)
+      end
+
+      def delete(key)
+        redis_conn.del(key)
+      end
+
+      def increment(key)
+        redis_conn.incr(key)
+      end
+
+      def map_result_push(key, value)
+        redis_conn.rpush(key, value)
+      end
+
+      def map_result_get(key)
+        redis_conn.lrange(key, 0, -1)
+      end
+
+      def reduce_result_set(storage_key, reduce_key, value)
+        redis_conn.hset(storage_key, reduce_key, value)
+      end
+
+      def reduce_result_get(storage_key)
+        redis_conn.hgetall(storage_key)
+      end
+
+      def map_keys_push(key, value)
+        redis_conn.sadd(key, value)
+      end
+
+      def map_keys_get(key)
+        redis_conn.smembers(key)
+      end
+
+      private
+
+      module_function
+
+      def redis_conn
+        Bramble.config.redis_conn
+      end
+    end
+  end
+end

data/lib/bramble/version.rb

@@ -0,0 +1,3 @@
+module Bramble
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: bramble
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Robert Mosolgo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Distribute map-reduce tasks with ActiveJob, storing the results in Redis
+  (or another backend)
+email: rdmosolgo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/bramble.rb
+- lib/bramble/conf.rb
+- lib/bramble/keys.rb
+- lib/bramble/map.rb
+- lib/bramble/map_job.rb
+- lib/bramble/reduce.rb
+- lib/bramble/reduce_job.rb
+- lib/bramble/storage.rb
+- lib/bramble/storage/memory_storage.rb
+- lib/bramble/storage/redis_storage.rb
+- lib/bramble/version.rb
+homepage: https://github.com/rmosolgo/bramble
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Map-reduce, backed by ActiveJob
+test_files: []