jason-rails 0.5.1 → 0.6.4

data/lib/jason/graph_helper.rb

@@ -0,0 +1,165 @@
+class Jason::GraphHelper
+  attr_reader :id, :includes_helper
+
+  def initialize(id, includes_helper)
+    @id = id
+    @includes_helper = includes_helper
+  end
+
+  def add_edge(parent_model, parent_id, child_model, child_id)
+    edge = "#{parent_model}:#{parent_id}/#{child_model}:#{child_id}"
+    $redis_jason.sadd("jason:subscriptions:#{id}:graph", edge)
+  end
+
+  def remove_edge(parent_model, parent_id, child_model, child_id)
+    edge = "#{parent_model}:#{parent_id}/#{child_model}:#{child_id}"
+    $redis_jason.srem("jason:subscriptions:#{id}:graph", edge)
+  end
+
+  def add_edges(all_models, instance_ids)
+    edges = build_edges(all_models, instance_ids)
+    $redis_jason.sadd("jason:subscriptions:#{id}:graph", edges)
+  end
+
+  def remove_edges(all_models, instance_ids)
+    edges = build_edges(all_models, instance_ids)
+    $redis_jason.srem("jason:subscriptions:#{id}:graph", edges)
+  end
+
+  def apply_remove_node(node)
+    edges = $redis_jason.smembers("jason:subscriptions:#{id}:graph")
+    edges = find_edges_with_node(edges, node)
+    diff_edges_from_graph(remove_edges: edges)
+  end
+
+  # Add and remove edges, return graph before and after
+  def apply_update(add: nil, remove: nil)
+    add_edges = []
+    remove_edges = []
+
+    if add.present?
+      add.each do |edge_set|
+        add_edges += build_edges(edge_set[:model_names], edge_set[:instance_ids])
+      end
+    end
+
+    if remove.present?
+      remove.each do |edge_set|
+        remove_edges += build_edges(edge_set[:model_names], edge_set[:instance_ids], include_root: false)
+      end
+    end
+    diff_edges_from_graph(add_edges: add_edges, remove_edges: remove_edges)
+  end
+
+  def diff_edges_from_graph(add_edges: [], remove_edges: [])
+    old_edges, new_edges = Jason::LuaGenerator.new.update_set_with_diff("jason:subscriptions:#{id}:graph", add_edges.flatten, remove_edges.flatten)
+
+    old_graph = build_graph_from_edges(old_edges)
+    new_graph = build_graph_from_edges(new_edges)
+
+    old_nodes = (old_graph.values + old_graph.keys).flatten.uniq - ['root']
+    new_nodes = (new_graph.values + new_graph.keys).flatten.uniq - ['root']
+    orphan_nodes = find_orphans_in_graph(new_graph)
+
+    added_nodes = new_nodes - old_nodes - orphan_nodes
+    removed_nodes = old_nodes - new_nodes + orphan_nodes
+
+    orphaned_edges = orphan_nodes.map do |node|
+      find_edges_with_node(new_edges, node)
+    end.flatten
+
+    if orphaned_edges.present?
+      $redis_jason.srem("jason:subscriptions:#{id}:graph", orphaned_edges)
+    end
+
+    ids_to_add = {}
+    ids_to_remove = {}
+
+    added_nodes.each do |node|
+      model_name, instance_id = node.split(':')
+      ids_to_add[model_name] ||= []
+      ids_to_add[model_name].push(instance_id)
+    end
+
+    removed_nodes.each do |node|
+      model_name, instance_id = node.split(':')
+      ids_to_remove[model_name] ||= []
+      ids_to_remove[model_name].push(instance_id)
+    end
+
+    { ids_to_remove: ids_to_remove, ids_to_add: ids_to_add }
+  end
+
+  def find_edges_with_node(edges, node)
+    edges.select do |edge|
+      parent, child = edge.split('/')
+      parent == node || child == node
+    end
+  end
+
+  def find_orphans
+    edges = $redis_jason.smembers("jason:subscriptions:#{id}:graph")
+    graph = build_graph_from_edges(edges)
+    find_orphans_in_graph(graph)
+  end
+
+  def find_orphans_in_graph(graph)
+    reachable_nodes = get_reachable_nodes(graph)
+    all_nodes = (graph.values + graph.keys).flatten.uniq - ['root']
+    all_nodes - reachable_nodes
+  end
+
+  def get_reachable_nodes(graph, parent = 'root')
+    reached_nodes = graph[parent] || []
+    reached_nodes.each do |child|
+      reached_nodes += get_reachable_nodes(graph, child)
+    end
+    reached_nodes
+  end
+
+  def build_graph_from_edges(edges)
+    graph = {}
+    edges.each do |edge|
+      parent, child = edge.split('/')
+      graph[parent] ||= []
+      graph[parent].push(child)
+    end
+    graph
+  end
+
+  private
+
+  def build_edges(all_models, instance_ids, include_root: true)
+    # Build the tree
+    # Find parent -> child model relationships
+    edges = []
+
+    all_models.each_with_index do |parent_model, parent_idx|
+      all_models.each_with_index do |child_model, child_idx|
+        next if parent_model == child_model
+        next if !includes_helper.in_sub(parent_model, child_model)
+
+        pairs = instance_ids.map { |row| [row[parent_idx], row[child_idx]] }
+          .uniq
+          .reject{ |pair| pair[0].blank? || pair[1].blank? }
+
+        edges += pairs.map.each do |pair|
+          "#{parent_model}:#{pair[0]}/#{child_model}:#{pair[1]}"
+        end
+      end
+    end
+
+    root_model = includes_helper.root_model
+
+    if include_root && all_models.include?(root_model)
+      root_idx = all_models.find_index(root_model)
+      root_ids = instance_ids.map { |row| row[root_idx] }.uniq.compact
+
+      edges += root_ids.map do |id|
+        "root/#{root_model}:#{id}"
+      end
+    end
+
+    edges
+  end
+end

data/lib/jason/includes_helper.rb

@@ -0,0 +1,108 @@
+# Helper to provide other modules with information about the includes of a subscription
+
+class Jason::IncludesHelper
+  attr_accessor :main_tree
+
+  def initialize(main_tree)
+    raise "Root must be hash" if !main_tree.is_a?(Hash)
+    raise "Only one root key allowed" if main_tree.keys.size != 1
+    @main_tree = main_tree
+  end
+
+  def all_models_recursive(tree)
+    sub_models = if tree.is_a?(Hash)
+      tree.map do |k,v|
+        [k, all_models_recursive(v)]
+      end
+    elsif tree.is_a?(Array)
+      tree.map do |v|
+        all_models_recursive(v)
+      end
+    else
+      tree
+    end
+  end
+
+  def all_models(model_name = nil)
+    model_name = model_name.presence || root_model
+    assoc_name = get_assoc_name(model_name)
+    tree = get_tree_for(assoc_name)
+    [model_name, all_models_recursive(tree)].flatten.uniq.map(&:to_s).map(&:singularize)
+  end
+
+  def root_model
+    main_tree.keys[0]
+  end
+
+  # assoc could be plural or not, so need to scan both.
+  def get_assoc_name(model_name, haystack = main_tree)
+    if haystack.is_a?(Hash)
+      haystack.each do |assoc_name, includes_tree|
+        if model_name.pluralize == assoc_name.to_s.pluralize
+          return assoc_name
+        else
+          found_assoc = get_assoc_name(model_name, includes_tree)
+          return found_assoc if found_assoc
+        end
+      end
+    elsif haystack.is_a?(Array)
+      haystack.each do |element|
+        if element.is_a?(String)
+          if model_name.pluralize == element.pluralize
+            return element
+          end
+        else
+          found_assoc = get_assoc_name(model_name, element)
+          return found_assoc if found_assoc
+        end
+      end
+    else
+      if model_name.pluralize == haystack.to_s.pluralize
+        return haystack
+      end
+    end
+
+    return nil
+  end
+
+  def get_tree_for(needle, assoc_name = nil, haystack = main_tree)
+    return haystack if needle.to_s.pluralize == assoc_name.to_s.pluralize
+
+    if haystack.is_a?(Hash)
+      haystack.each do |assoc_name, includes_tree|
+        found_haystack = get_tree_for(needle, assoc_name, includes_tree)
+        return found_haystack if found_haystack.present?
+      end
+    elsif haystack.is_a?(Array)
+      haystack.each do |includes_tree|
+        found_haystack = get_tree_for(needle, nil, includes_tree)
+        return found_haystack if found_haystack.present?
+      end
+    elsif haystack.is_a?(String)
+      found_haystack = get_tree_for(needle, haystack, nil)
+      return found_haystack if found_haystack.present?
+    end
+
+    return []
+  end
+
+  def in_sub(parent_model, child_model)
+    tree = get_tree_for(parent_model)
+
+    if tree.is_a?(Hash)
+      return tree.keys.map(&:singularize).include?(child_model)
+    elsif tree.is_a?(Array)
+      tree.each do |element|
+        if element.is_a?(String)
+          return true if element.singularize == child_model
+        elsif element.is_a?(Hash)
+          return true if element.keys.map(&:singularize).include?(child_model)
+        end
+      end
+    elsif tree.is_a?(String)
+      return tree.singularize == child_model
+    end
+
+    return false
+  end
+end

data/lib/jason/lua_generator.rb

@@ -44,6 +44,28 @@ class Jason::LuaGenerator
       end
     LUA
 
-    result = $redis_jason.eval cmd, [], [model_name, id, sub_id, gidx]
+    $redis_jason.eval cmd, [], [model_name, id, sub_id, gidx]
+  end
+
+  def update_set_with_diff(key, add_members, remove_members)
+    cmd = <<~LUA
+      local old_members = redis.call('smembers', KEYS[1])
+      local add_size = ARGV[1]
+
+      for k, m in pairs({unpack(ARGV, 2, add_size + 1)}) do
+        redis.call('sadd', KEYS[1], m)
+      end
+
+      for k, m in pairs({unpack(ARGV, add_size + 2, #ARGV)}) do
+        redis.call('srem', KEYS[1], m)
+      end
+
+      return old_members
+    LUA
+
+    args = [add_members.size, add_members, remove_members].flatten
+
+    old_members = $redis_jason.eval cmd, [key], args
+    return [old_members, (old_members + add_members - remove_members).uniq]
   end
 end

data/lib/jason/publisher.rb

@@ -25,8 +25,15 @@ module Jason::Publisher
     end
   end
 
-  def publish_json
+  def force_publish_json
+    # As-if newly created
+    publish_json(self.attributes.map { |k,v| [k, [nil, v]] }.to_h)
+  end
+
+  def publish_json(previous_changes = {})
     payload, gidx = cache_json
+
+    return if skip_publish_json
     subs = jason_subscriptions # Get this first, because could be changed
 
     # Situations where IDs may need to change and this can't be immediately determined
@@ -37,20 +44,19 @@ module Jason::Publisher
     # TODO: Optimize this, by caching associations rather than checking each time instance is saved
     jason_assocs = self.class.reflect_on_all_associations(:belongs_to).select { |assoc| assoc.klass.respond_to?(:has_jason?) }
     jason_assocs.each do |assoc|
-      if self.previous_changes[assoc.foreign_key].present?
-
+      if previous_changes[assoc.foreign_key].present?
         Jason::Subscription.update_ids(
           self.class.name.underscore,
           id,
-          assoc.klass.name.underscore,
-          self.previous_changes[assoc.foreign_key][0],
-          self.previous_changes[assoc.foreign_key][1]
+          assoc.name.to_s.singularize,
+          previous_changes[assoc.foreign_key][0],
+          previous_changes[assoc.foreign_key][1]
         )
       elsif (persisted? && @was_a_new_record && send(assoc.foreign_key).present?)
         Jason::Subscription.update_ids(
           self.class.name.underscore,
           id,
-          assoc.klass.name.underscore,
+          assoc.name.to_s.singularize,
           nil,
           send(assoc.foreign_key)
         )
@@ -65,32 +71,30 @@ module Jason::Publisher
     end
 
     # - An instance is created where it belongs_to an _all_ subscription
-    if self.previous_changes['id'].present?
+    if previous_changes['id'].present?
       Jason::Subscription.add_id(self.class.name.underscore, id)
     end
 
-    return if skip_publish_json
-
-    if self.persisted?
+    if persisted?
       jason_subscriptions.each do |sub_id|
         Jason::Subscription.new(id: sub_id).update(self.class.name.underscore, id, payload, gidx)
       end
-    else
-      subs.each do |sub_id|
-        Jason::Subscription.new(id: sub_id).destroy(self.class.name.underscore, id)
-      end
     end
   end
 
   def publish_json_if_changed
     subscribed_fields = api_model.subscribed_fields
-    publish_json if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
+    publish_json(self.previous_changes) if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
   end
 
   def jason_subscriptions
     Jason::Subscription.for_instance(self.class.name.underscore, id)
   end
 
+  def jason_cached_value
+    JSON.parse($redis_jason.hget("jason:cache:#{self.class.name.underscore}", id) || '{}')
+  end
+
   class_methods do
     def cache_all
       all.each(&:cache_json)

data/lib/jason/subscription.rb

@@ -1,17 +1,32 @@
 class Jason::Subscription
   attr_accessor :id, :config
+  attr_reader :includes_helper, :graph_helper
 
   def initialize(id: nil, config: nil)
     if id
       @id = id
       raw_config = $redis_jason.hgetall("jason:subscriptions:#{id}").map { |k,v| [k, JSON.parse(v)] }.to_h
+      raise "Subscription ID #{id} does not exist" if raw_config.blank?
       set_config(raw_config)
     else
       @id = Digest::MD5.hexdigest(config.sort_by { |key| key }.to_h.to_json)
-      pp config.sort_by { |key| key }.to_h.to_json
       configure(config)
     end
-    pp @id
+    @includes_helper = Jason::IncludesHelper.new({ model => self.config['includes'] })
+    @graph_helper = Jason::GraphHelper.new(self.id, @includes_helper)
+
+    check_for_missing_keys
+  end
+
+  def broadcaster
+    @broadcaster ||= Jason::Broadcaster.new(channel)
+  end
+
+  def check_for_missing_keys
+    missing_keys = includes_helper.all_models - Jason.schema.keys.map(&:to_s)
+    if missing_keys.present?
+      raise "#{missing_keys.inspect} are not in the schema. Only models in the Jason schema can be subscribed."
+    end
   end
 
   def self.upsert_by_config(model, conditions: {}, includes: {})
@@ -26,6 +41,13 @@ class Jason::Subscription
     self.new(id: id)
   end
 
+  def self.for_instance_with_child(model_name, id, child_model_name, include_all = true)
+    sub_ids = for_instance(model_name, id, include_all = true)
+    sub_ids.select do |sub_id|
+      find_by_id(sub_id).includes_helper.in_sub(model_name, child_model_name)
+    end
+  end
+
   def self.for_instance(model_name, id, include_all = true)
     subs = $redis_jason.smembers("jason:models:#{model_name}:#{id}:subscriptions")
     if include_all
@@ -41,37 +63,117 @@ class Jason::Subscription
 
   # Find and update subscriptions affected by a model changing foreign key
   # comment, comment_id, post, old_post_id, new_post_id
-  def self.update_ids(model_name, id, parent_model_name, old_foreign_id, new_foreign_id)
-    # Check if this change means it needs to be removed
-    # First find subscriptions that reference this model
-
-    if old_foreign_id
-      old_model_subscriptions = for_instance(parent_model_name, old_foreign_id, false)
-      new_model_subscriptions = for_instance(parent_model_name, new_foreign_id, false)
-    else
-      # If this is a new instance, we need to include _all_ subscriptions
-      old_model_subscriptions = []
-      new_model_subscriptions = for_instance(parent_model_name, new_foreign_id, true)
+  def self.update_ids(changed_model_name, changed_model_id, foreign_model_name, old_foreign_id, new_foreign_id)
+    # There are 4 cases to consider.
+    # changed_instance ---/--- foreign_instance
+    #                  \--+--- new_foreign_instance
+    #
+    # foreign instance can either be parent or child for a given subscription
+    # 1. Child swap/add: foreign is child
+    # 2. Stay in the family: foreign is parent + both old and new foreign instances are part of the sub
+    # 3. Join the family: foreign is parent + only new foreign instance are part of the sub
+    # 4. Leave the family: foreign is parent + only the old foreign instance is part of the sub
+
+    #########
+    # Subs where changed is parent
+    sub_ids = for_instance_with_child(changed_model_name, changed_model_id, foreign_model_name, true)
+    sub_ids.each do |sub_id|
+      subscription = find_by_id(sub_id)
+
+      # If foreign key has been nulled, nothing to add
+      add = new_foreign_id.present? ? [
+        {
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, new_foreign_id]]
+        },
+        # Add IDs of child models
+        subscription.load_ids_for_sub_models(foreign_model_name, new_foreign_id)
+      ] : nil
+
+      id_changeset = subscription.graph_helper.apply_update({
+        remove: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, old_foreign_id]]
+        }],
+        add: add
+      })
+
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
     end
 
-    # To add
-    (new_model_subscriptions - old_model_subscriptions).each do |sub_id|
-      # add the current ID to the subscription, then add the tree below it
-      find_by_id(sub_id).set_id(model_name, id)
+    old_sub_ids = for_instance_with_child(foreign_model_name, old_foreign_id, changed_model_name, true)
+    new_sub_ids = for_instance_with_child(foreign_model_name, new_foreign_id, changed_model_name, true)
+
+    #########
+    # Subs where changed is child
+    # + parent in both old + new
+    # this is simple, only the edges need to change - no IDs can be changed
+    (old_sub_ids & new_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      subscription.graph_helper.apply_update({
+        remove: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, old_foreign_id]]
+        }],
+        add: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, new_foreign_id]]
+        }]
+      })
     end
 
-    # To remove
-    (old_model_subscriptions - new_model_subscriptions).each do |sub_id|
-      find_by_id(sub_id).remove_ids(model_name, [id])
+    #########
+    # Subs where changed is child
+    # + old parent wasn't in the sub, but new parent is
+    # IE the changed instance is joining the sub
+    # No edges are removed, just added
+    (new_sub_ids - old_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      id_changeset = subscription.graph_helper.apply_update({
+        add: [
+          {
+            model_names: [changed_model_name, foreign_model_name],
+            instance_ids: [[changed_model_id, new_foreign_id]]
+          },
+          # Add IDs of child models
+          subscription.load_ids_for_sub_models(changed_model_name, changed_model_id)
+        ]
+      })
+
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
     end
 
-    # TODO changes to sub models - e.g. post -> comment -> user
+    #########
+    # --> Leaving the family
+    # Subs where changed is child
+    # + old parent was in the sub, but new parent isn't
+    # Just need to remove the link, orphan detection will do the rest
+    (old_sub_ids - new_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      id_changeset = subscription.graph_helper.apply_update({
+        remove: [
+          {
+            model_names: [changed_model_name, foreign_model_name],
+            instance_ids: [[changed_model_id, old_foreign_id]]
+          }
+        ]
+      })
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
+    end
   end
 
   def self.remove_ids(model_name, ids)
+    # td: finish this
     ids.each do |instance_id|
       for_instance(model_name, instance_id, false).each do |sub_id|
-        find_by_id(sub_id).remove_ids(model_name, [instance_id])
+        subscription = find_by_id(sub_id)
+
+        id_changeset = subscription.graph_helper.apply_remove_node("#{model_name}:#{instance_id}")
+        subscription.apply_id_changeset(id_changeset)
+        subscription.broadcast_id_changeset(id_changeset)
       end
     end
   end
@@ -82,93 +184,25 @@ class Jason::Subscription
   end
 
   def self.all
-    $redis_jason.keys('jason:subscriptions:*')
+    $redis_jason.smembers('jason:subscriptions').map { |id| Jason::Subscription.find_by_id(id) }
   end
 
   def set_config(raw_config)
-    @config =  raw_config.with_indifferent_access
-  end
-
-  # E.g. add comment#123, and then sub models
-  def set_id(model_name, id)
-    commit_ids(model_name, [id])
-    assoc_name = get_assoc_name(model_name)
-    set_ids_for_sub_models(assoc_name, [id])
+    @config =  raw_config.deep_stringify_keys.deep_transform_values { |v| v.is_a?(Symbol) ? v.to_s : v }
   end
 
   def clear_id(model_name, id, parent_model_name)
     remove_ids(model_name, [id])
   end
 
-  # Set the instance IDs for the subscription
-  # Add an entry to the subscription list for each instance
-  def set_ids(assoc_name = model, referrer_model_name = nil, referrer_ids = nil, enforce: false)
-    model_name = assoc_name.to_s.singularize
-
-    if referrer_model_name.blank? && conditions.blank?
-      $redis_jason.sadd("jason:models:#{model_name}:all:subscriptions", id)
-      ids = model_klass(model_name).all.pluck(:id)
-      set_ids_for_sub_models(assoc_name, ids, enforce: enforce)
-      return
-    end
-
-    if referrer_model_name.blank?
-      ids = model_klass(model_name).where(conditions).pluck(:id)
-    else
-      assoc = model_klass(referrer_model_name).reflect_on_association(assoc_name.to_sym)
-
-      if assoc.is_a?(ActiveRecord::Reflection::HasManyReflection)
-        ids = model_klass(model_name).where(assoc.foreign_key => referrer_ids).pluck(:id)
-      elsif assoc.is_a?(ActiveRecord::Reflection::BelongsToReflection)
-        ids = model_klass(referrer_model_name).where(id: referrer_ids).pluck(assoc.foreign_key)
-      end
-    end
-    return if ids.blank?
-
-    enforce ? enforce_ids(model_name, ids) : commit_ids(model_name, ids)
-    set_ids_for_sub_models(assoc_name, ids, enforce: enforce)
-  end
-
-  def refresh_ids(assoc_name = model, referrer_model_name = nil, referrer_ids)
-
-  end
-
   # Add IDs that aren't present
   def commit_ids(model_name, ids)
-    pp 'COMMIT'
-    pp model_name
-    pp ids
     $redis_jason.sadd("jason:subscriptions:#{id}:ids:#{model_name}", ids)
     ids.each do |instance_id|
       $redis_jason.sadd("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  # Ensure IDs are _only_ the ones passed
-  def enforce_ids(model_name, ids)
-    old_ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
-
-    # Remove
-    ids_to_remove = old_ids - ids
-    if ids_to_remove.present?
-      $redis_jason.srem("jason:subscriptions:#{id}:ids:#{model_name}", ids_to_remove)
-    end
-
-    ids_to_remove.each do |instance_id|
-      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
-    end
-
-    # Add
-    ids_to_add = ids - old_ids
-    if ids_to_add.present?
-      $redis_jason.sadd("jason:subscriptions:#{id}:ids:#{model_name}", ids_to_add)
-    end
-
-    ids_to_add.each do |instance_id|
-      $redis_jason.sadd("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
-    end
-  end
-
   def remove_ids(model_name, ids)
     $redis_jason.srem("jason:subscriptions:#{id}:ids:#{model_name}", ids)
     ids.each do |instance_id|
@@ -176,108 +210,81 @@ class Jason::Subscription
     end
   end
 
-  # 'posts', [post#1, post#2,...]
-  def set_ids_for_sub_models(assoc_name, ids, enforce: false)
-    model_name = assoc_name.to_s.singularize
-    # Limitation: Same association can't appear twice
-    includes_tree = get_tree_for(assoc_name)
+  def apply_id_changeset(changeset)
+    changeset[:ids_to_add].each do |model_name, ids|
+      commit_ids(model_name, ids)
+    end
 
-    if includes_tree.is_a?(Hash)
-      includes_tree.each do |assoc_name, includes_tree|
-        set_ids(assoc_name, model_name, ids, enforce: enforce)
-      end
-    # [:likes, :user]
-    elsif includes_tree.is_a?(Array)
-      includes_tree.each do |assoc_name|
-        set_ids(assoc_name, model_name, ids, enforce: enforce)
-      end
-    elsif includes_tree.is_a?(String)
-      set_ids(includes_tree, model_name, ids, enforce: enforce)
+    changeset[:ids_to_remove].each do |model_name, ids|
+      remove_ids(model_name, ids)
     end
   end
 
-  # assoc could be plural or not, so need to scan both.
-  def get_assoc_name(model_name, haystack = includes)
-    return model_name if model_name == model
-
-    if haystack.is_a?(Hash)
-      haystack.each do |assoc_name, includes_tree|
-        if model_name.pluralize == assoc_name.to_s.pluralize
-          return assoc_name
-        else
-          found_assoc = get_assoc_name(model_name, includes_tree)
-          return found_assoc if found_assoc
-        end
-      end
-    elsif haystack.is_a?(Array)
-      haystack.each do |assoc_name|
-        if model_name.pluralize == assoc_name.to_s.pluralize
-          return assoc_name
-        end
-      end
-    else
-      if model_name.pluralize == haystack.to_s.pluralize
-        return haystack
-      end
+  def broadcast_id_changeset(changeset)
+    changeset[:ids_to_add].each do |model_name, ids|
+      ids.each { |id| add(model_name, id) }
     end
 
-    return nil
+    changeset[:ids_to_remove].each do |model_name, ids|
+      ids.each { |id| destroy(model_name, id) }
+    end
   end
 
-  def get_tree_for(needle, assoc_name = nil, haystack = includes)
-    return includes if needle == model
-    return haystack if needle.to_s == assoc_name.to_s
+  # Take a model name and IDs and return an edge set of all the models that appear and
+  # their instance IDs
+  def load_ids_for_sub_models(model_name, ids)
+    # Limitation: Same association can't appear twice
+    includes_tree = includes_helper.get_tree_for(model_name)
+    all_models = includes_helper.all_models(model_name)
 
-    if haystack.is_a?(Hash)
-      haystack.each do |assoc_name, includes_tree|
-        found_haystack = get_tree_for(needle, assoc_name, includes_tree)
-        return found_haystack if found_haystack
+    relation = model_name.classify.constantize.all.eager_load(includes_tree)
+
+    if model_name == model
+      if conditions.blank?
+        $redis_jason.sadd("jason:models:#{model_name}:all:subscriptions", id)
+        all_models -= [model_name]
+      else
+        relation = relation.where(conditions)
       end
+    else
+      raise "Must supply IDs for sub models" if ids.nil?
+      return if ids.blank?
+      relation = relation.where(id: ids)
     end
 
-    return nil
-  end
+    pluck_args = all_models.map { |m| "#{m.pluralize}.id" }
+    instance_ids = relation.pluck(*pluck_args)
 
-  def all_models(tree = includes)
-    sub_models = if tree.is_a?(Hash)
-      tree.map do |k,v|
-        [k, all_models(v)]
-      end
-    else
-      tree
+    # pluck returns only a 1D array if only 1 arg passed
+    if all_models.size == 1
+      instance_ids = [instance_ids]
     end
 
-    pp ([model] + [sub_models]).flatten.uniq.map(&:to_s).map(&:singularize)
-    ([model] + [sub_models]).flatten.uniq.map(&:to_s).map(&:singularize)
+    return { model_names: all_models, instance_ids: instance_ids }
   end
 
-  def clear_all_ids(assoc_name = model)
-    model_name = assoc_name.to_s.singularize
-    includes_tree = model_name == model ? includes : get_tree_for(assoc_name)
-
-    if model_name == model && conditions.blank?
-      $redis_jason.srem("jason:models:#{model_name}:all:subscriptions", id)
-    end
+  # 'posts', [post#1, post#2,...]
+  def set_ids_for_sub_models(model_name = model, ids = nil, enforce: false)
+    edge_set = load_ids_for_sub_models(model_name, ids)
 
-    ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
-    ids.each do |instance_id|
-      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
-    end
-    $redis_jason.del("jason:subscriptions:#{id}:ids:#{model_name}")
+    # Build the tree
+    id_changeset = graph_helper.apply_update({
+      add: [edge_set]
+    })
+    apply_id_changeset(id_changeset)
+  end
 
-    # Recursively clear IDs
-    # { comments: [:like] }
-    if includes_tree.is_a?(Hash)
-      includes_tree.each do |assoc_name, includes_tree|
-        clear_all_ids(assoc_name)
+  def clear_all_ids
+    includes_helper.all_models.each do |model_name|
+      if model_name == model && conditions.blank?
+        $redis_jason.srem("jason:models:#{model_name}:all:subscriptions", id)
       end
-    # [:likes, :user]
-    elsif includes_tree.is_a?(Array)
-      includes_tree.each do |assoc_name|
-        clear_all_ids(assoc_name)
+
+      ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+      ids.each do |instance_id|
+        $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
       end
-    elsif includes_tree.is_a?(String)
-      clear_all_ids(includes_tree)
+      $redis_jason.del("jason:subscriptions:#{id}:ids:#{model_name}")
     end
   end
 
@@ -297,12 +304,9 @@ class Jason::Subscription
     @config['conditions']
   end
 
-  def includes
-    @config['includes']
-  end
-
   def configure(raw_config)
     set_config(raw_config)
+    $redis_jason.sadd("jason:subscriptions", id)
     $redis_jason.hmset("jason:subscriptions:#{id}", *config.map { |k,v| [k, v.to_json] }.flatten)
   end
 
@@ -316,7 +320,7 @@ class Jason::Subscription
     $redis_jason.hset("jason:consumers", consumer_id, Time.now.utc)
 
     if before_consumer_count == 0
-      set_ids
+      set_ids_for_sub_models
     end
   end
 
@@ -334,11 +338,17 @@ class Jason::Subscription
   end
 
   def channel
-    "jason:#{id}"
+    "jason-#{id}"
+  end
+
+  def user_can_access?(user)
+    # td: implement the authorization logic here
+    return true if Jason.authorization_service.blank?
+    Jason.authorization_service.call(user, model, conditions, includes_helper.all_models - [model])
   end
 
   def get
-    all_models.map { |model_name| get_for_model(model_name) }
+    includes_helper.all_models.map { |model_name| [model_name, get_for_model(model_name)] }.to_h
   end
 
   def get_for_model(model_name)
@@ -352,8 +362,6 @@ class Jason::Subscription
       instance_jsons, idx = Jason::LuaGenerator.new.get_payload(model_name, id)
     end
 
-    return if instance_jsons.blank?
-
     payload = instance_jsons.map do |instance_json|
       instance_json ? JSON.parse(instance_json) : {}
     end
@@ -367,6 +375,21 @@ class Jason::Subscription
     }
   end
 
+  def add(model_name, instance_id)
+    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}:idx")
+    payload = JSON.parse($redis_jason.hget("jason:cache:#{model_name}", instance_id) || '{}')
+
+    payload = {
+      id: instance_id,
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
+
+    broadcaster.broadcast(payload)
+  end
+
   def update(model_name, instance_id, payload, gidx)
     idx = Jason::LuaGenerator.new.get_subscription(model_name, instance_id, id, gidx)
     return if idx.blank?
@@ -379,11 +402,11 @@ class Jason::Subscription
       idx: idx.to_i
     }
 
-    ActionCable.server.broadcast(channel, payload)
+    broadcaster.broadcast(payload)
   end
 
   def destroy(model_name, instance_id)
-    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}idx")
+    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}:idx")
 
     payload = {
       id: instance_id,
@@ -393,6 +416,6 @@ class Jason::Subscription
       idx: idx.to_i
     }
 
-    ActionCable.server.broadcast(channel, payload)
+    broadcaster.broadcast(payload)
   end
 end