mongoid-cached-json 1.3.0 → 1.4.0

data/README.md

@@ -10,9 +10,8 @@ Using `Mongoid::CachedJson` we were able to cut our JSON API average response ti
 Resources
 ---------
 
-* [Need Help?](http://groups.google.com/group/mongoid-cached-json)
+* [Need Help? Google Group](http://groups.google.com/group/mongoid-cached-json)
 * [Source Code](http://github.com/dblock/mongoid-cached-json)
-* [Travis CI](https://secure.travis-ci.org/dblock/mongoid-cached-json)
 
 Quickstart
 ----------
@@ -231,10 +230,26 @@ describe "updating a person" do
 end
 ```
 
+Performance
+-----------
+
+This implements two interesting optimizations.
+
+### Bulk Reference Resolving w/ Local Store
+
+Consider an array of Mongoid instances, each with numerous references to other objects. It's typical to see such instances reference the same object. `Mongoid::CachedJson` first collects all JSON references, then resolves them after suppressing duplicates. This significantly reduces the number of cache queries.
+
+### Fetching Cache Data in Bulk
+
+Various cache stores, including Memcached, support bulk read operations. The [Dalli](https://github.com/mperham/dalli) gem exposes this via the `read_multi` method. `Mongoid::CachedJson` will always invoke `read_multi` where available, which significantly reduces the number of network roundtrips to the cache servers.
+
 Contributing
 ------------
 
-Fork the project. Make your feature addition or bug fix with tests. Send a pull request. Bonus points for topic branches.
+* Fork the project. 
+* Make your feature addition or bug fix with tests. 
+* Don't forget to update `CHANGELOG.md`. 
+* Send a pull request. Bonus points for topic branches.
 
 Copyright and License
 ---------------------

data/lib/mongoid-cached-json.rb

@@ -1,6 +1,10 @@
 # encoding: utf-8
-require 'mongoid'
 require 'active_support/concern'
+require 'mongoid'
+require 'mongoid-cached-json/key_references'
+require 'mongoid-cached-json/array'
+require 'mongoid-cached-json/hash'
+require 'mongoid-cached-json/mongoid_criteria'
 require 'mongoid-cached-json/version'
 require 'mongoid-cached-json/config'
 require 'mongoid-cached-json/cached_json'

data/lib/mongoid-cached-json/array.rb

@@ -0,0 +1,22 @@
+class Array
+
+  def as_json_partial(options = {})
+    json_keys = nil
+    json = map do |i|
+      if i.respond_to?(:as_json_partial)
+        partial_json_keys, json = i.as_json_partial(options)
+        json_keys = json_keys ? json_keys.merge_set(partial_json_keys) : partial_json_keys
+        json
+      else
+        i.as_json(options)
+      end
+    end
+    [ json_keys, json ]
+  end
+
+  def as_json(options = {})
+    json_keys, json = as_json_partial(options)
+    Mongoid::CachedJson.materialize_json_references_with_read_multi(json_keys, json)
+  end
+
+end

data/lib/mongoid-cached-json/cached_json.rb

@@ -2,7 +2,7 @@
 module Mongoid
   module CachedJson
     extend ActiveSupport::Concern
-  
+
     included do
       class_attribute :all_json_properties
       class_attribute :all_json_versions
@@ -10,9 +10,9 @@ module Mongoid
       class_attribute :cached_json_reference_defs
       class_attribute :hide_as_child_json_when
     end
-  
+
     module ClassMethods
-    
+
       # Define JSON fields for a class.
       #
       # @param [ hash ] defs JSON field definition.
@@ -44,7 +44,27 @@ module Mongoid
         end
         before_save :expire_cached_json
       end
-  
+
+      # Materialize a cached JSON within a cache block.
+      def materialize_cached_json(clazz, id, object_reference, options)
+        is_top_level_json = options[:is_top_level_json] || false
+        object_reference = clazz.where({ :_id => id }).first if !object_reference
+        if !object_reference || (!is_top_level_json && options[:properties] != :all && clazz.hide_as_child_json_when.call(object_reference))
+          nil
+        else
+          Hash[clazz.cached_json_field_defs[options[:properties]].map do |field, definition|
+            # version match
+            versions = ([definition[:version] ] | Array(definition[:versions])).compact
+            next unless versions.empty? or versions.include?(options[:version])
+            json_value = (definition[:definition].is_a?(Symbol) ? object_reference.send(definition[:definition]) : definition[:definition].call(object_reference))
+            Mongoid::CachedJson.config.transform.each do |t|
+              json_value = t.call(field, definition, json_value)
+            end
+            [field, json_value]
+          end]
+        end
+      end
+
       # Given an object definition in the form of either an object or a class, id pair,
       # grab the as_json representation from the cache if possible, otherwise create
       # the as_json representation by loading the object from the database. For any
@@ -61,36 +81,29 @@ module Mongoid
           object_reference = nil
           clazz, id = object_def[:clazz], object_def[:id]
         end
-        json = Mongoid::CachedJson.config.cache.fetch(self.cached_json_key(options, clazz, id), { :force => !! Mongoid::CachedJson.config.disable_caching }) do
-          object_reference = clazz.where({ :_id => id }).first if !object_reference
-          if !object_reference or (!is_top_level_json and options[:properties] != :all and clazz.hide_as_child_json_when.call(object_reference))
-            nil
-          else
-            Hash[clazz.cached_json_field_defs[options[:properties]].map do |field, definition|
-              # version match
-              versions = ([definition[:version] ] | Array(definition[:versions])).compact
-              next unless versions.empty? or versions.include?(options[:version])
-              json_value = (definition[:definition].is_a?(Symbol) ? object_reference.send(definition[:definition]) : definition[:definition].call(object_reference))
-              Mongoid::CachedJson.config.transform.each do |t|
-                json_value = t.call(field, definition, json_value)
-              end
-              [field, json_value]
-            end]
-          end
-        end
+        key = self.cached_json_key(options, clazz, id)
+        json = { :_ref => { :_clazz => self, :_key => key, :_materialize_cached_json => [ clazz, id, object_reference, options ] }}
+        keys = KeyReferences.new
+        keys.set_and_add(key, json)
         reference_defs = clazz.cached_json_reference_defs[options[:properties]]
         if !reference_defs.empty?
           object_reference = clazz.where({ :_id => id }).first if !object_reference
           if object_reference and (is_top_level_json or options[:properties] == :all or !clazz.hide_as_child_json_when.call(object_reference))
-            json = json.merge(Hash[reference_defs.map do |field, definition|
+            json.merge!(Hash[reference_defs.map do |field, definition|
               json_properties_type = (options[:properties] == :all) ? :all : :short
-              [field, clazz.resolve_json_reference(options.merge({ :properties => json_properties_type, :is_top_level_json => false}), object_reference, field, definition)]
+              reference_keys, reference = clazz.resolve_json_reference(options.merge({ :properties => json_properties_type, :is_top_level_json => false}), object_reference, field, definition)
+              if (reference.is_a?(Hash) && ref = reference[:_ref])
+                ref[:_parent] = json
+                ref[:_field] = field
+              end
+              keys.merge_set(reference_keys)
+              [field, reference]
             end])
           end
         end
-        json
+        [ keys, json ]
       end
-  
+
       # Cache key.
       def cached_json_key(options, cached_class, cached_id)
         base_class_name = cached_class.collection_name.to_s.singularize.camelize
@@ -102,6 +115,7 @@ module Mongoid
       # be able to load the as_json representation from the cache without even getting the
       # model from the database and materializing it through Mongoid. We'll try to do this first.
       def resolve_json_reference(options, object, field, reference_def)
+        keys = nil
         reference_json = nil
         if reference_def[:metadata]
           key = reference_def[:metadata].key.to_sym
@@ -112,10 +126,10 @@ module Mongoid
           end
           if reference_def[:metadata].relation == Mongoid::Relations::Referenced::ManyToMany
             reference_json = object.send(key).map do |id|
-              materialize_json(options, { :clazz => clazz, :id => id })
+              materialize_keys, json = materialize_json(options, { :clazz => clazz, :id => id })
+              keys = keys ? keys.merge_set(materialize_keys) : materialize_keys
+              json
             end.compact
-          elsif reference_def[:metadata].relation == Mongoid::Relations::Referenced::In
-            reference_json = materialize_json(options, { :clazz => clazz, :id => object.send(key) })
           end
         end
         # If we get to this point and reference_json is still nil, there's no chance we can
@@ -123,23 +137,73 @@ module Mongoid
         if ! reference_json
           reference_def_definition = reference_def[:definition]
           reference = reference_def_definition.is_a?(Symbol) ? object.send(reference_def_definition) : reference_def_definition.call(object)
-          reference_json = reference.as_json(options) if reference
+          reference_json = nil
+          if reference
+            if reference.respond_to?(:as_json_partial)
+              reference_keys, reference_json = reference.as_json_partial(options)
+              keys = keys ? keys.merge_set(reference_keys) : reference_keys
+            else
+              reference_json = reference.as_json(options)
+            end
+          end
         end
-        reference_json
+        [ keys, reference_json ]
       end
-  
+
     end
-  
-    def as_json(options = {})
+
+    # Check whether the cache supports :read_multi and prefetch the data if it does.
+    def self.materialize_json_references_with_read_multi(key_refs, partial_json)
+      unfrozen_keys = key_refs.keys.to_a.map(&:dup) if key_refs # see https://github.com/mperham/dalli/pull/320
+      local_cache = unfrozen_keys && Mongoid::CachedJson.config.cache.respond_to?(:read_multi) ? Mongoid::CachedJson.config.cache.read_multi(unfrozen_keys) : {}
+      Mongoid::CachedJson.materialize_json_references(key_refs, local_cache) if key_refs
+      partial_json
+    end
+
+    # Materialize all the JSON references in place.
+    def self.materialize_json_references(key_refs, local_cache = {})
+      key_refs.each_pair do |key, refs|
+        refs.each do |ref|
+          _ref = ref.delete(:_ref)
+          key = _ref[:_key]
+          fetched_json = local_cache[key] if local_cache.has_key?(key)
+          fetched_json ||= (local_cache[key] = Mongoid::CachedJson.config.cache.fetch(key, { :force => !! Mongoid::CachedJson.config.disable_caching }) do
+            _ref[:_clazz].materialize_cached_json(* _ref[:_materialize_cached_json])
+          end)
+          if fetched_json
+            ref.merge! fetched_json
+          elsif _ref[:_parent]
+            # a single _ref that resolved to a nil
+            _ref[:_parent][_ref[:_field]] = nil
+          end
+        end
+      end
+    end
+
+    # Return a partial JSON without resolved references and all the keys.
+    def as_json_partial(options = {})
       options ||= {}
       if options[:properties] and ! self.all_json_properties.member?(options[:properties])
         raise ArgumentError.new("Unknown properties option: #{options[:properties]}")
       end
-      self.class.materialize_json({
+      # partial, unmaterialized JSON
+      keys, partial_json = self.class.materialize_json({
         :properties => :short, :is_top_level_json => true, :version => Mongoid::CachedJson.config.default_version
       }.merge(options), { :object => self })
+      [ keys, partial_json ]
     end
-  
+
+    # Fetch the partial JSON and materialize all JSON references.
+    def as_json_cached(options = {})
+      keys, json = as_json_partial(options)
+      Mongoid::CachedJson.materialize_json_references_with_read_multi(keys, json)
+    end
+
+    # Return the JSON representation of the object.
+    def as_json(options = {})
+      as_json_cached(options)
+    end
+
     # Expire all JSON entries for this class.
     def expire_cached_json
       self.all_json_properties.each do |properties|
@@ -152,9 +216,9 @@ module Mongoid
         end
       end
     end
-  
+
     class << self
-    
+
       # Set the configuration options. Best used by passing a block.
       #
       # @example Set up configuration options.
@@ -168,6 +232,6 @@ module Mongoid
       end
       alias :config :configure
     end
-      
+
   end
 end

data/lib/mongoid-cached-json/hash.rb

@@ -0,0 +1,23 @@
+class Hash
+
+  def as_json_partial(options = {})
+    json_keys = nil
+    json = inject({}) do |h, (k, v)|
+      if v.respond_to?(:as_json_partial)
+        partial_json_keys, partial_json = v.as_json_partial(options)
+        json_keys = json_keys ? json_keys.merge_set(partial_json_keys) : partial_json_keys
+        h[k] = partial_json
+      else
+        h[k] = v.as_json(options)
+      end
+      h
+    end
+    [ json_keys, json ]
+  end
+
+  def as_json(options = {})
+    json_keys, json = as_json_partial(options)
+    Mongoid::CachedJson.materialize_json_references_with_read_multi(json_keys, json)
+  end
+
+end

data/lib/mongoid-cached-json/key_references.rb

@@ -0,0 +1,26 @@
+# Keep key references to be replaced once the entire JSON is available.
+# encoding: utf-8
+module Mongoid
+  module CachedJson
+    class KeyReferences < Hash
+
+      def merge_set(keys)
+        if keys
+          keys.each_pair do |k, jsons|
+            self[k] ||= []
+            self[k].concat(jsons)
+          end
+        end
+        self
+      end
+
+      def set_and_add(key, json)
+        self[key] ||= []
+        self[key] << json
+        self
+      end
+
+    end
+  end
+end
+

data/lib/mongoid-cached-json/mongoid_criteria.rb

@@ -0,0 +1,24 @@
+module Mongoid
+  class Criteria
+
+    def as_json_partial(options = {})
+      json_keys = nil
+      json = map do |i|
+        if i.respond_to?(:as_json_partial)
+          partial_json_keys, partial_json = i.as_json_partial(options)
+          json_keys = json_keys ? json_keys.merge_set(partial_json_keys) : partial_json_keys
+          partial_json
+        else
+          i.as_json(options)
+        end
+      end
+      [ json_keys, json ]
+    end
+
+    def as_json(options = {})
+      json_keys, json = as_json_partial(options)
+      Mongoid::CachedJson.materialize_json_references_with_read_multi(json_keys, json)
+    end
+
+  end
+end

data/lib/mongoid-cached-json/version.rb

@@ -1,6 +1,6 @@
 module Mongoid
   module CachedJson
-    VERSION = '1.3.0'
+    VERSION = '1.4.0'
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid-cached-json
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-12 00:00:00.000000000 Z
+date: 2013-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -125,6 +125,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: dalli
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
 description: Cached-json is a DSL for describing JSON representations of Mongoid models.
 email: dblock@dblock.org
 executables: []
@@ -134,8 +150,12 @@ extra_rdoc_files:
 - README.md
 files:
 - lib/mongoid-cached-json.rb
+- lib/mongoid-cached-json/array.rb
 - lib/mongoid-cached-json/cached_json.rb
 - lib/mongoid-cached-json/config.rb
+- lib/mongoid-cached-json/hash.rb
+- lib/mongoid-cached-json/key_references.rb
+- lib/mongoid-cached-json/mongoid_criteria.rb
 - lib/mongoid-cached-json/version.rb
 - LICENSE.md
 - README.md
@@ -154,7 +174,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1107312364045411622
+      hash: -1533991509568558766
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: