perforated 0.8.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88f1b011129d5fe9d05a47fbbfde518bcbdc271a
-  data.tar.gz: c8d932fadb905e4b95c7daaff47e7c6e2a406dee
+  metadata.gz: af43ff72681ac3aa03057419afb02db11d89dcec
+  data.tar.gz: 52f9abfc9c9239e34fbde54958e22f8bdbf3d9e7
 SHA512:
-  metadata.gz: 21f8ba0179ca2b0a8748e5dc565017878c900fb4eff8b84e3b91960d9112cbbf55e8ae6b76f8c03bb1211e573b76c401be6a926170c6e9a246dcd5c7e247ea1d
-  data.tar.gz: 7633062dd1634963e018fff974059fb373614a396b088cda8bac13ed2ba1defe6e6d478e21ac12835828f1f3480fc54d46855f1c267afec8c66f85aae7ba2900
+  metadata.gz: 4c84edc00c26382975abd7a028687c54e9cc2f99c1717324f95a60aa2fc3bb9471acff847a1b65cead2aaa62a03509c4af0a95f7083b6c4eecf49d46b201cec3
+  data.tar.gz: 01eba26e7cb2e63d9ac66cedebed88b2a2b7943aacbf9a928d3bfef8d6ebeaed53db168f748581cececc235c5f51643d86e1aac845c3a71ff9275ccd0204da8e

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 vendor
 bin
+TODO

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## Version 0.9.0
+
+* `as_json` and `to_json` now take a block. The block will be applied to each
+  model as it is being cached, allowing for serializers or presenters without
+  much additional memory overhead.
+* `as_json` has been removed. The overhead from marshalling/unmarshalling had
+  enough overhead that it negated the caching benefits.
+* Strategies no longer expect a `suffix` property, as anything cached is
+  expected to be a JSON string.
+* Caching is performed in batches, which can be controlled by passing
+  `batch_size` through to `to_json`. Arrays are refined to support
+  `find_in_batches` in order to maintain compatibility with ActiveRecord
+  Relations. For especially large collections this can provide significant
+  memory savings (particularly when paired with passing a block through for
+  custom serialization).
+
 ## Version 0.8.2
 
 * Really force the use of custom `fetch_multi` when using `NullStore`, not just

data/Gemfile

@@ -2,6 +2,6 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'dalli'
+gem 'oj'
 gem 'redis'
-gem 'redis-activesupport'
+gem 'redis-activesupport', github: 'redis-store/redis-activesupport'

data/README.md

@@ -117,12 +117,12 @@ cache method you can provide a custom key caching strategy.
 
 ```ruby
 module CustomStrategy
-  def self.expand_cache_key(object, suffix)
-    [object.id, object.updated_at, suffix].join('/')
+  def self.expand_cache_key(object)
+    [object.id, object.updated_at].join('/')
   end
 end
 
-perforated = Perforated::Cache.new(array, CustomStrategy)
+perforated = Perforated.new(array, CustomStrategy)
 ```
 
 ## Installation

data/bench/giant.rb

@@ -3,10 +3,8 @@ require 'bundler'
 Bundler.setup
 
 require 'benchmark'
-require 'dalli'
 require 'perforated'
 require 'active_support/core_ext/object'
-require 'active_support/cache/dalli_store'
 require 'redis'
 require 'redis-activesupport'
 
@@ -17,7 +15,7 @@ Structure = Struct.new(:id) do
 end
 
 module Strategy
-  def self.expand_cache_key(object, suffix)
+  def self.expand_cache_key(object)
     "perf-#{object}"
   end
 end
@@ -25,24 +23,26 @@ end
 perforated = Perforated::Cache.new((0..20_000).map { |i| Structure.new(i) }, Strategy)
 
 Benchmark.bm do |x|
+  GC.disable
+  puts "Total Objects: #{ObjectSpace.count_objects[:TOTAL]}"
+
   x.report('memory-1') { perforated.to_json }
   x.report('memory-2') { perforated.to_json }
 
+  puts "Total Objects: #{ObjectSpace.count_objects[:TOTAL]}"
+
   Perforated.configure do |config|
     config.cache = ActiveSupport::Cache::RedisStore.new(host: 'localhost', db: 5)
   end
 
   Perforated.cache.clear
 
+  GC.enable
+  GC.start
+  GC.disable
+
   x.report('redis-1') { perforated.to_json }
   x.report('redis-2') { perforated.to_json }
 
-  Perforated.configure do |config|
-    config.cache = ActiveSupport::Cache::DalliStore.new('localhost')
-  end
-
-  Perforated.cache.clear
-
-  x.report('dalli-1') { perforated.to_json }
-  x.report('dalli-2') { perforated.to_json }
+  puts "Total Objects: #{ObjectSpace.count_objects[:TOTAL]}"
 end

data/bench/rebuilder.rb

@@ -0,0 +1,23 @@
+require 'bundler'
+
+Bundler.setup
+
+require 'benchmark'
+require 'perforated'
+require 'oj'
+
+keys = %w[notes tags comments]
+objects = (0..10_000).map do |num|
+  keys = keys.rotate
+
+  { keys.first => num }
+end
+
+strings   = objects.map { |obj| JSON.dump(obj) }
+rebuilder = Perforated::Rebuilder.new(strings, Oj)
+
+N = 100
+
+Benchmark.bmbm do |x|
+  x.report('rebuild') { N.times { rebuilder.rebuild(rooted: true) } }
+end

data/lib/perforated.rb

@@ -1,16 +1,12 @@
 require 'active_support/cache'
 require 'json'
 require 'perforated/cache'
-require 'perforated/compatibility/fetch_multi'
-require 'perforated/rooted'
-require 'perforated/strategy/default'
-require 'perforated/version'
 
 module Perforated
   extend self
 
   def new(*args)
-    Perforated::Cache.new(args)
+    Perforated::Cache.new(*args)
   end
 
   def cache=(new_cache)

data/lib/perforated/cache.rb

@@ -1,40 +1,42 @@
+require 'perforated/rebuilder'
+require 'perforated/strategy'
+require 'perforated/compatibility/find_in_batches'
+require 'perforated/compatibility/fetch_multi'
+
 module Perforated
   class Cache
-    attr_reader :enumerable, :key_strategy
-
-    def initialize(enumerable, key_strategy = Perforated::Strategy::Default)
-      @enumerable   = enumerable
-      @key_strategy = key_strategy
-    end
+    using Perforated::Compatibility::FindInBatches
 
-    def as_json(options = {})
-      keyed   = keyed_enumerable('as-json')
-      objects = fetch_multi(keyed) { |key| keyed[key].as_json }.values
+    attr_accessor :enumerable, :strategy
 
-      if options[:rooted]
-        Perforated::Rooted.merge(objects)
-      else
-        objects
-      end
+    def initialize(enumerable, strategy = Perforated::Strategy)
+      @enumerable = enumerable
+      @strategy   = strategy
     end
 
-    def to_json(options = {})
-      keyed   = keyed_enumerable('to-json')
-      objects = fetch_multi(keyed) { |key| keyed[key].to_json }
-      concat  = concatenate(objects)
+    def to_json(rooted: false, batch_size: 1000, &block)
+      results = []
 
-      if options[:rooted]
-        Perforated::Rooted.reconstruct(concat)
-      else
-        concat
+      enumerable.find_in_batches(batch_size: batch_size) do |subset|
+        keyed = key_mapped(subset)
+
+        results << fetch_multi(keyed) do |key|
+          if block_given?
+            (yield keyed[key]).to_json
+          else
+            keyed[key].to_json
+          end
+        end.values
       end
+
+      Perforated::Rebuilder.new(results).rebuild(rooted: rooted)
     end
 
     private
 
-    def keyed_enumerable(suffix = '')
-      enumerable.each_with_object({}) do |object, memo|
-        memo[key_strategy.expand_cache_key(object, suffix)] = object
+    def key_mapped(subset)
+      subset.each_with_object({}) do |object, memo|
+        memo[strategy.expand_cache_key(object)] = object
       end
     end
 
@@ -43,9 +45,5 @@ module Perforated
 
       Perforated::Compatibility.fetch_multi(*keys, &block)
     end
-
-    def concatenate(objects)
-      "[#{objects.values.join(',')}]"
-    end
   end
 end

data/lib/perforated/compatibility/find_in_batches.rb

@@ -0,0 +1,11 @@
+module Perforated
+  module Compatibility
+    module FindInBatches
+      refine Array do
+        def find_in_batches(batch_size: 1000, &block)
+          each_slice(batch_size, &block)
+        end
+      end
+    end
+  end
+end

data/lib/perforated/rebuilder.rb

@@ -0,0 +1,50 @@
+require 'set'
+
+module Perforated
+  class Rebuilder
+    attr_reader :parser, :strings
+
+    def initialize(strings, parser = Perforated.json)
+      @strings = strings
+      @parser  = parser
+    end
+
+    def rebuild(rooted: false)
+      if rooted
+        parser.dump(merge(parser.load(concatenated)))
+      else
+        concatenated
+      end
+    end
+
+    def concatenated
+      "[#{strings.join(',')}]"
+    end
+
+    private
+
+    def merge(objects)
+      merged = objects.each_with_object({}) do |object, memo|
+        object.each do |key, value|
+          memo[key] ||= Set.new
+
+          if value.is_a?(Array)
+            memo[key].merge(value)
+          else
+            memo[key].add(value)
+          end
+        end
+      end
+
+      sets_to_arrays(merged)
+    end
+
+    def sets_to_arrays(merged)
+      merged.each do |key, value|
+        merged[key] = value.to_a
+      end
+
+      merged
+    end
+  end
+end

data/lib/perforated/strategy.rb

@@ -0,0 +1,9 @@
+require 'active_support/cache'
+
+module Perforated
+  module Strategy
+    def self.expand_cache_key(object)
+      ActiveSupport::Cache.expand_cache_key(object.cache_key)
+    end
+  end
+end

data/lib/perforated/version.rb

@@ -1,3 +1,3 @@
 module Perforated
-  VERSION = '0.8.2'
+  VERSION = '0.9.0'
 end

data/perforated.gemspec

@@ -26,5 +26,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec', '~> 2.14'
+  spec.add_development_dependency 'rspec', '~> 3'
 end

data/spec/perforated/cache_spec.rb

@@ -4,12 +4,8 @@ describe Perforated::Cache do
   after { Perforated.cache.clear }
 
   Language = Struct.new(:name) do
-    def as_json
-      { name: name }
-    end
-
     def to_json
-      as_json.to_json
+      { name: name }.to_json
     end
 
     def cache_key
@@ -18,12 +14,8 @@ describe Perforated::Cache do
   end
 
   Family = Struct.new(:name, :languages) do
-    def as_json
-      { languages: languages }
-    end
-
     def to_json
-      as_json.to_json
+      { languages: languages }.to_json
     end
 
     def cache_key
@@ -31,54 +23,51 @@ describe Perforated::Cache do
     end
   end
 
-  describe '#as_json' do
-    it 'constructs automatically cached serialized output' do
+  describe '#to_json' do
+    it 'constructs a stringified json array of underlying values' do
       ruby   = Language.new('Ruby')
       elixir = Language.new('Elixir')
       cache  = Perforated::Cache.new([ruby, elixir])
 
-      expect(cache.as_json).to eq([{ name: 'Ruby' }, { name: 'Elixir' }])
-
-      expect(Perforated.cache.read('Language/Ruby/as-json')).to eq(ruby.as_json)
-      expect(Perforated.cache.read('Language/Elixir/as-json')).to eq(elixir.as_json)
+      expect(cache.to_json).to eq(%([{"name":"Ruby"},{"name":"Elixir"}]))
+      expect(Perforated.cache.exist?('Language/Ruby')).to   be_truthy
+      expect(Perforated.cache.exist?('Language/Elixir')).to be_truthy
     end
 
     it 'does not overwrite existing key values' do
       erlang = Language.new('Erlang')
-      Perforated.cache.write('Language/Erlang/as-json', { name: 'Elixir' })
-
-      Perforated::Cache.new([erlang]).as_json
-
-      expect(Perforated.cache.read('Language/Erlang/as-json')).to eq(name: 'Elixir')
-    end
+      Perforated.cache.write('Language/Erlang', JSON.dump(name: 'Elixir'))
 
-    it 'merges objects comprised of rooted arrays' do
-      lisps   = Family.new('Lisp', ['scheme', 'clojure'])
-      scripts = Family.new('Script', ['perl', 'ruby'])
-      cache   = Perforated::Cache.new([lisps, scripts])
+      Perforated::Cache.new([erlang]).to_json
 
-      expect(cache.as_json(rooted: true)).to eq(
-        languages: %w[scheme clojure perl ruby]
-      )
+      expect(Perforated.cache.read('Language/Erlang')).to eq(JSON.dump(name: 'Elixir'))
     end
 
     it 'safely returns an empty enumerable when empty' do
       cache = Perforated::Cache.new([])
 
-      expect(cache.as_json).to eq([])
-      expect(cache.as_json(rooted: true)).to eq({})
+      expect(cache.to_json).to eq('[]')
+      expect(cache.to_json(rooted: true)).to eq('{}')
     end
-  end
 
-  describe '#to_json' do
-    it 'constructs a stringified json array of underlying values' do
-      cache = Perforated::Cache.new([Language.new('Ruby'), Language.new('Elixir')])
+    it 'applies a provided block to the object before caching' do
+      ruby  = Language.new('Ruby')
+      cache = Perforated::Cache.new([ruby])
 
-      expect(cache.to_json).to eq(%([{"name":"Ruby"},{"name":"Elixir"}]))
-      expect(Perforated.cache.exist?('Language/Ruby/to-json')).to   be_truthy
-      expect(Perforated.cache.exist?('Language/Elixir/to-json')).to be_truthy
+      serializer = Struct.new(:lang) do
+        def to_json
+          { name: lang.name.upcase }.to_json
+        end
+      end
+
+      results = cache.to_json do |lang|
+        serializer.new(lang)
+      end
+
+      expect(results).to eq(JSON.dump([{ name: 'RUBY' }]))
     end
 
+
     it 'reconstructs rooted objects into a single merged object' do
       lisps   = Family.new('Lisp', ['scheme', 'clojure'])
       scripts = Family.new('Script', ['perl', 'ruby'])

data/spec/perforated/compatibility/find_in_batches_spec.rb

@@ -0,0 +1,25 @@
+require 'perforated/compatibility/find_in_batches'
+
+describe Perforated::Compatibility::FindInBatches do
+  FindBatches = Struct.new(:array) do
+    using Perforated::Compatibility::FindInBatches
+
+    def perform(&block)
+      array.find_in_batches(batch_size: 50, &block)
+    end
+  end
+
+  describe '#find_in_batches' do
+    it 'iterates over an enumerable in batches' do
+      enumerable = [1] * 100
+      eachable   = FindBatches.new(enumerable)
+      results    = []
+
+      eachable.perform do |arr|
+        results << arr.reduce(&:+)
+      end
+
+      expect(results).to eq([50, 50])
+    end
+  end
+end

data/spec/perforated/rebuilder_spec.rb

@@ -0,0 +1,18 @@
+require 'json'
+require 'perforated/rebuilder'
+
+describe Perforated::Rebuilder do
+  describe '#rebuild' do
+    it 'merges stringified json' do
+      string_a = JSON.dump(families: { name: 'lang' }, languages: [{ name: 'scheme' }])
+      string_b = JSON.dump(families: { name: 'lang' }, languages: [{ name: 'clojure' }])
+      string_c = JSON.dump(families: { name: 'lang' })
+
+      rooted = Perforated::Rebuilder.new([string_a, string_b], JSON)
+
+      expect(rooted.rebuild(rooted: true)).to eq(
+        '{"families":[{"name":"lang"}],"languages":[{"name":"scheme"},{"name":"clojure"}]}'
+      )
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perforated
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors:
 - Parker Selbert
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-27 00:00:00.000000000 Z
+date: 2014-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: '3'
 description: Intellgent json collection caching
 email:
 - parker@sorentwo.com
@@ -82,16 +82,19 @@ files:
 - README.md
 - Rakefile
 - bench/giant.rb
+- bench/rebuilder.rb
 - lib/perforated.rb
 - lib/perforated/cache.rb
 - lib/perforated/compatibility/fetch_multi.rb
-- lib/perforated/rooted.rb
-- lib/perforated/strategy/default.rb
+- lib/perforated/compatibility/find_in_batches.rb
+- lib/perforated/rebuilder.rb
+- lib/perforated/strategy.rb
 - lib/perforated/version.rb
 - perforated.gemspec
 - spec/perforated/cache_spec.rb
 - spec/perforated/compatibility/fetch_multi_spec.rb
-- spec/perforated/rooted_spec.rb
+- spec/perforated/compatibility/find_in_batches_spec.rb
+- spec/perforated/rebuilder_spec.rb
 - spec/perforated_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/sorentwo/perforated
@@ -124,6 +127,7 @@ summary: 'The most expensive part of serving a JSON request is converting the se
 test_files:
 - spec/perforated/cache_spec.rb
 - spec/perforated/compatibility/fetch_multi_spec.rb
-- spec/perforated/rooted_spec.rb
+- spec/perforated/compatibility/find_in_batches_spec.rb
+- spec/perforated/rebuilder_spec.rb
 - spec/perforated_spec.rb
 - spec/spec_helper.rb

data/lib/perforated/rooted.rb

@@ -1,33 +0,0 @@
-require 'set'
-
-module Perforated
-  module Rooted
-    def self.merge(objects)
-      merged = objects.each_with_object({}) do |object, memo|
-        object.each do |key, value|
-          memo[key] ||= Set.new
-
-          if value.is_a?(Array)
-            memo[key].merge(value)
-          else
-            memo[key].add(value)
-          end
-        end
-      end
-
-      sets_to_arrays(merged)
-    end
-
-    def self.reconstruct(objects, parser = Perforated.json)
-      parser.dump(merge(parser.load(objects)))
-    end
-
-    def self.sets_to_arrays(object)
-      object.each do |key, value|
-        object[key] = value.to_a
-      end
-
-      object
-    end
-  end
-end

data/lib/perforated/strategy/default.rb

@@ -1,11 +0,0 @@
-module Perforated
-  module Strategy
-    class Default
-      def self.expand_cache_key(object, suffix = '')
-        args = object.cache_key + [suffix]
-
-        ActiveSupport::Cache.expand_cache_key(args)
-      end
-    end
-  end
-end

data/spec/perforated/rooted_spec.rb

@@ -1,26 +0,0 @@
-require 'perforated'
-
-describe Perforated::Rooted do
-  describe '.merge' do
-    it 'merges all nested objects' do
-      obj_a = { languages: [{ name: 'scheme' }] }
-      obj_b = { languages: [{ name: 'clojure' }] }
-
-      expect(Perforated::Rooted.merge([obj_a, obj_b])).to eq(
-        languages: [{ name: 'scheme' }, { name: 'clojure' }]
-      )
-    end
-  end
-
-  describe '.reconstruct' do
-    it 'merges stringified json' do
-      obj_a  = JSON.dump({ languages: [{ name: 'scheme' }] })
-      obj_b  = JSON.dump({ languages: [{ name: 'clojure' }] })
-      concat = "[#{obj_a},#{obj_b}]"
-
-      expect(Perforated::Rooted.reconstruct(concat)).to eq(
-        '{"languages":[{"name":"scheme"},{"name":"clojure"}]}'
-      )
-    end
-  end
-end