bramble 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2baaf8e5894d3d7673a6888e6eab091883520c7d
-  data.tar.gz: 55f0b6791f770a1a6fe710d3a98972677cdfed31
+  metadata.gz: 138b78b6cd5ccf7de287e447453cb010de725eb7
+  data.tar.gz: 554476911d84da387cc7017b4bb0041810507fe7
 SHA512:
-  metadata.gz: e081ada613296ab5d5a410bf0759b2a425610a3618ae2d0762b762ee25689e344b10d8a2943bdf45a9b9e22534ffc78c4aac36f22fa09db9fa4d957534e78e78
-  data.tar.gz: 4196394526521889d7d56a4f981fad5a7152847cdf62ee1146e0d5ac4411b392fc46b6578dafac648918320ec4af0750ef50d7f4499f1fe0be38e4baf10e859d
+  metadata.gz: c78a8f5a1eaf795df61ef62439ee77d2de522edea38f9c8ee77538662b91410615b362ea25e7b387f333cc2461d8b32a631130c1c277b1852befdd3062bcef9d
+  data.tar.gz: f112df8555c290fbf037ec7c401014f4d5c6ec1794a0351317baf568632c211b0ca794c75f7ac6505ae40f2213b40c2603df59394de5b223796ab8bb3e429f27

data/README.md

@@ -1,6 +1,10 @@
-# Bramble [![Build Status](https://travis-ci.org/rmosolgo/bramble.svg?branch=master)](https://travis-ci.org/rmosolgo/bramble)
+# Bramble [![Build Status](https://travis-ci.org/rmosolgo/bramble.svg?branch=master)](https://travis-ci.org/rmosolgo/bramble) [![Gem Version](https://badge.fury.io/rb/bramble.svg)](https://badge.fury.io/rb/bramble)
 
-Map-reduce with ActiveJob
+Map-reduce with ActiveJob + database
+
+## Rationale
+
+We have some staff-only views that expose stats about how people use our app. Eventually, our tables grew so large that MySQL wouldn't aggregate them all at once. So we can use this to generate those stats over time.
 
 ## Usage
 
@@ -43,6 +47,8 @@ Map-reduce with ActiveJob
   end
   ```
 
+  Inputs and outputs are serialized with __JSON__, so some Ruby types will be lost (eg, Symbols).
+
 - Start a job with a handle, module, and an (optional) argument for finding data:
 
   ```ruby
@@ -60,9 +66,13 @@ Map-reduce with ActiveJob
 
   ```ruby
   result = Bramble.get("shakespeare-letter-count")
-  result.running?   # => false
-  result.finished?  # => true
-  result.data       # => { "A" => 100, "B" => 100, ... }
+  result.running?         # => false
+  result.finished?        # => true
+  result.data             # => { "A" => 100, "B" => 100, ... }
+  result.percent_finished # 1.0
+  result.percent_mapped   # 1.0
+  result.percent_reduced  # 1.0
+  result.finished_at      # 2016-05-16 12:31:00 UTC
   ```
 
 - Delete the saved result:
@@ -73,8 +83,9 @@ Map-reduce with ActiveJob
 
 ## Todo
 
-- `.fetch` to find-or-calculate?
-- Adapters: Memcache, ActiveRecord
+- Adapters: Memcached, ActiveRecord
+- Do we have atomicity issues? Occasional test failures
+- Consolidate storage in Redis to a single key? (Could some keys be evicted while others remain?)
 
 ## Development
 

data/lib/bramble/keys.rb

@@ -41,5 +41,9 @@ module Bramble
     def status_key(handle)
       "#{namespace(handle)}:status"
     end
+
+    def finished_at_key(handle)
+      "#{namespace(handle)}:finished_at"
+    end
   end
 end

data/lib/bramble/map.rb

@@ -19,10 +19,12 @@ module Bramble
         implementation.map(value) do |map_key, map_val|
           Bramble::State.running?(handle) do
             raw_key = Bramble::Serialize.dump(map_key)
+            raw_value = Bramble::Serialize.dump(map_val)
             storage.map_keys_push(impl_keys_key, raw_key)
-            storage.map_result_push(data_key(handle, raw_key), Bramble::Serialize.dump(map_val))
+            storage.map_result_push(data_key(handle, raw_key), raw_value)
           end
         end
+
         Bramble::State.running?(handle) do
           finished = storage.increment(map_finished_count_key(handle))
           total = storage.get(map_total_count_key(handle)).to_i

data/lib/bramble/reduce.rb

@@ -23,6 +23,9 @@ module Bramble
         Bramble::State.running?(handle) do
           storage.reduce_result_set(result_key(handle), raw_key, Bramble::Serialize.dump(reduced_value))
           storage.increment(reduce_finished_count_key(handle))
+          if Bramble::State.percent_reduced(handle) >= 1
+            storage.set(finished_at_key(handle), Time.now.to_i)
+          end
         end
       else
         Bramble::State.clear_reduce(handle)

data/lib/bramble/result.rb

@@ -2,11 +2,19 @@ module Bramble
   # This class exposes the data and some info about the state of the task
   class Result
 
-    attr_reader :handle
+    attr_reader :handle, :percent_mapped, :percent_reduced, :finished_at
 
     def initialize(handle)
       job_id = storage.get(Bramble::Keys.job_id_key(handle))
       @handle = "#{handle}:#{job_id}"
+      @percent_mapped = Bramble::State.percent_mapped(@handle)
+      @percent_reduced = Bramble::State.percent_reduced(@handle)
+      if finished?
+        finished_at_ms = storage.get(Bramble::Keys.finished_at_key(@handle)).to_i
+        @finished_at = Time.at(finished_at_ms)
+      else
+        @finished_at = nil
+      end
     end
 
     def data
@@ -18,22 +26,19 @@ module Bramble
     end
 
     def finished?
-      @finished ||= finished_count > 0 && total_count == finished_count
+      # Possible to be greater than 1 because of floating-point arithmetic
+      percent_finished >= 1
     end
 
     def running?
-      @running ||= started? && !finished?
+      started? && !finished?
     end
 
-    private
-
-    def total_count
-      @total_count ||= storage.get(Bramble::Keys.reduce_total_count_key(handle)).to_i
+    def percent_finished
+      (percent_mapped + percent_reduced) / 2
     end
 
-    def finished_count
-      @finished_count ||= storage.get(Bramble::Keys.reduce_finished_count_key(handle)).to_i
-    end
+    private
 
     def started?
       @started ||= !!storage.get(Bramble::Keys.status_key(handle))

data/lib/bramble/serialize.rb

@@ -1,9 +1,10 @@
+require "json"
 module Bramble
   # eg, Redis uses strings only, so use this module to freeze and thaw values from storage
   module Serialize
     # prepare an object for storage
     def self.dump(obj)
-      Marshal.dump(obj)
+      JSON.dump(obj)
     end
 
     # reload an object from storage
@@ -16,8 +17,10 @@ module Bramble
           memo[load(k)] = load(v)
           memo
         end
+      when nil
+        nil
       else
-        Marshal.load(stored_obj)
+        JSON.load(stored_obj)
       end
     end
   end

data/lib/bramble/state.rb

@@ -32,6 +32,20 @@ module Bramble
       clear_map(handle)
     end
 
+    def percent_mapped(handle)
+      percent_between_keys(
+        map_total_count_key(handle),
+        map_finished_count_key(handle)
+      )
+    end
+
+    def percent_reduced(handle)
+      percent_between_keys(
+        reduce_total_count_key(handle),
+        reduce_finished_count_key(handle)
+      )
+    end
+
     def clear_map(handle)
       map_group_keys = storage.map_keys_get(keys_key(handle))
       map_group_keys.each do |group_key|
@@ -46,6 +60,7 @@ module Bramble
       storage.delete(reduce_total_count_key(handle))
       storage.delete(reduce_finished_count_key(handle))
       storage.delete(result_key(handle))
+      storage.delete(finished_at_key(handle))
     end
 
     private
@@ -53,5 +68,15 @@ module Bramble
     def self.storage
       Bramble::Storage
     end
+
+    def self.percent_between_keys(total_key, finished_key)
+      total = storage.get(total_key).to_f
+      if total == 0
+        0
+      else
+        finished = storage.get(finished_key).to_i
+        finished / total
+      end
+    end
   end
 end

data/lib/bramble/version.rb

@@ -1,3 +1,3 @@
 module Bramble
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bramble
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Robert Mosolgo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-13 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob