jason-rails 0.3.0 → 0.6.0

data/lib/jason/lua_generator.rb

@@ -0,0 +1,71 @@
+class Jason::LuaGenerator
+  ## TODO load these scripts and evalsha
+  def cache_json(model_name, id, payload)
+    cmd = <<~LUA
+      local gidx = redis.call('INCR', 'jason:gidx')
+      redis.call( 'set', 'jason:cache:' .. ARGV[1] .. ':' .. ARGV[2] .. ':gidx', gidx )
+      redis.call( 'hset', 'jason:cache:' .. ARGV[1], ARGV[2], ARGV[3] )
+      return gidx
+    LUA
+
+    result = $redis_jason.eval cmd, [], [model_name, id, payload.to_json]
+  end
+
+  def get_payload(model_name, sub_id)
+    # If value has changed, return old value and new idx. Otherwise do nothing.
+    cmd = <<~LUA
+      local t = {}
+      local models = {}
+      local ids = redis.call('smembers', 'jason:subscriptions:' .. ARGV[2] .. ':ids:' .. ARGV[1])
+
+      for k,id in pairs(ids) do
+        models[#models+1] = redis.call( 'hget', 'jason:cache:' .. ARGV[1], id)
+      end
+
+      t[#t+1] = models
+      t[#t+1] = redis.call( 'get', 'jason:subscription:' .. ARGV[2] .. ':' .. ARGV[1] .. ':idx' )
+
+      return t
+    LUA
+
+    $redis_jason.eval cmd, [], [model_name, sub_id]
+  end
+
+  def get_subscription(model_name, id, sub_id, gidx)
+    # If value has changed, return old value and new idx. Otherwise do nothing.
+    cmd = <<~LUA
+      local last_gidx = redis.call('get', 'jason:cache:' .. ARGV[1] .. ':' .. ARGV[2] .. ':gidx') or 0
+
+      if (ARGV[4] >= last_gidx) then
+        local sub_idx = redis.call( 'incr', 'jason:subscription:' .. ARGV[3] .. ':' .. ARGV[1] .. ':idx' )
+        return sub_idx
+      else
+        return false
+      end
+    LUA
+
+    $redis_jason.eval cmd, [], [model_name, id, sub_id, gidx]
+  end
+
+  def update_set_with_diff(key, add_members, remove_members)
+    cmd = <<~LUA
+      local old_members = redis.call('smembers', KEYS[1])
+      local add_size = ARGV[1]
+
+      for k, m in pairs({unpack(ARGV, 2, add_size + 1)}) do
+        redis.call('sadd', KEYS[1], m)
+      end
+
+      for k, m in pairs({unpack(ARGV, add_size + 2, #ARGV)}) do
+        redis.call('srem', KEYS[1], m)
+      end
+
+      return old_members
+    LUA
+
+    args = [add_members.size, add_members, remove_members].flatten
+
+    old_members = $redis_jason.eval cmd, [key], args
+    return [old_members, (old_members + add_members - remove_members).uniq]
+  end
+end

data/lib/jason/publisher.rb

@@ -1,73 +1,146 @@
 module Jason::Publisher
   extend ActiveSupport::Concern
 
+  # Warning: Could be expensive. Mainly useful for rebuilding cache after changing Jason config or on deploy
+  def self.cache_all
+    Rails.application.eager_load!
+    ActiveRecord::Base.descendants.each do |klass|
+      klass.cache_all if klass.respond_to?(:cache_all)
+    end
+  end
+
   def cache_json
     as_json_config = api_model.as_json_config
     scope = api_model.scope
 
+    # Exists
     if self.persisted? && (scope.blank? || self.class.unscoped.send(scope).exists?(self.id))
       payload = self.reload.as_json(as_json_config)
-      $redis.hset("jason:#{self.class.name.underscore}:cache", self.id, payload.to_json)
+      gidx = Jason::LuaGenerator.new.cache_json(self.class.name.underscore, self.id, payload)
+      return [payload, gidx]
+    # Has been destroyed
     else
-      $redis.hdel("jason:#{self.class.name.underscore}:cache", self.id)
+      $redis_jason.hdel("jason:cache:#{self.class.name.underscore}", self.id)
+      return []
     end
   end
 
-  def publish_json
-    cache_json
+  def force_publish_json
+    # As-if newly created
+    publish_json(self.attributes.map { |k,v| [k, [nil, v]] }.to_h)
+  end
+
+  def publish_json(previous_changes = {})
+    payload, gidx = cache_json
+
     return if skip_publish_json
-    subscriptions = $redis.hgetall("jason:#{self.class.name.underscore}:subscriptions")
-    subscriptions.each do |id, config_json|
-      config = JSON.parse(config_json)
+    subs = jason_subscriptions # Get this first, because could be changed
+
+    # Situations where IDs may need to change and this can't be immediately determined
+    # - An instance is created where it belongs_to an instance under a subscription
+    # - An instance belongs_to association changes - e.g. comment.post_id changes to/from one with a subscription
+    # - TODO: The value of an instance changes so that it enters/leaves a subscription
 
-      if (config['conditions'] || {}).all? { |field, value| self.send(field) == value }
-        Jason::Subscription.new(id: id).update(self.class.name.underscore)
+    # TODO: Optimize this, by caching associations rather than checking each time instance is saved
+    jason_assocs = self.class.reflect_on_all_associations(:belongs_to).select { |assoc| assoc.klass.respond_to?(:has_jason?) }
+    jason_assocs.each do |assoc|
+      if previous_changes[assoc.foreign_key].present?
+        Jason::Subscription.update_ids(
+          self.class.name.underscore,
+          id,
+          assoc.name.to_s.singularize,
+          previous_changes[assoc.foreign_key][0],
+          previous_changes[assoc.foreign_key][1]
+        )
+      elsif (persisted? && @was_a_new_record && send(assoc.foreign_key).present?)
+        Jason::Subscription.update_ids(
+          self.class.name.underscore,
+          id,
+          assoc.name.to_s.singularize,
+          nil,
+          send(assoc.foreign_key)
+        )
+      end
+    end
+
+    if !persisted? # Deleted
+      Jason::Subscription.remove_ids(
+        self.class.name.underscore,
+        [id]
+       )
+    end
+
+    # - An instance is created where it belongs_to an _all_ subscription
+    if previous_changes['id'].present?
+      Jason::Subscription.add_id(self.class.name.underscore, id)
+    end
+
+    if persisted?
+      jason_subscriptions.each do |sub_id|
+        Jason::Subscription.new(id: sub_id).update(self.class.name.underscore, id, payload, gidx)
       end
     end
   end
 
   def publish_json_if_changed
     subscribed_fields = api_model.subscribed_fields
-    publish_json if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
+    publish_json(self.previous_changes) if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
+  end
+
+  def jason_subscriptions
+    Jason::Subscription.for_instance(self.class.name.underscore, id)
   end
 
   class_methods do
-    def subscriptions
-      $redis.hgetall("jason:#{self.name.underscore}:subscriptions")
+    def cache_all
+      all.each(&:cache_json)
     end
 
-    def publish_all(instances)
-      instances.each(&:cache_json)
-
-      subscriptions.each do |id, config_json|
-        Jason::Subscription.new(id: id).update(self.name.underscore)
-      end
+    def has_jason?
+      true
     end
 
     def flush_cache
-      $redis.del("jason:#{self.name.underscore}:cache")
+      $redis_jason.del("jason:cache:#{self.name.underscore}")
     end
 
     def setup_json
+      self.before_save -> {
+        @was_a_new_record = new_record?
+      }
       self.after_initialize -> {
         @api_model = Jason::ApiModel.new(self.class.name.underscore)
       }
       self.after_commit :publish_json_if_changed
+    end
+
+    def find_or_create_by_id(params)
+      object = find_by(id: params[:id])
+
+      if object
+        object.update(params)
+      elsif params[:hidden]
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
+      end
 
-      include_models = Jason::ApiModel.new(self.name.underscore).include_models
+      object
+    end
 
-      include_models.map do |assoc|
-        puts assoc
-        reflection = self.reflect_on_association(assoc.to_sym)
-        reflection.klass.after_commit -> {
-          subscribed_fields = Jason::ApiModel.new(self.class.name.underscore).subscribed_fields
-          puts subscribed_fields.inspect
+    def find_or_create_by_id!(params)
+      object = find_by(id: params[:id])
 
-          if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present?
-            self.send(reflection.inverse_of.name)&.publish_json
-          end
-        }
+      if object
+        object.update!(params)
+      elsif params[:hidden]
+        ## TODO: We're diverging from semantics of the Rails bang! methods here, which would normally either raise or return an object. Find a way to make this better.
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
       end
+
+      object
     end
   end
 

data/lib/jason/publisher_old.rb

@@ -0,0 +1,112 @@
+module Jason::PublisherOld
+  extend ActiveSupport::Concern
+
+  def cache_json
+    as_json_config = api_model.as_json_config
+    scope = api_model.scope
+
+    if self.persisted? && (scope.blank? || self.class.unscoped.send(scope).exists?(self.id))
+      payload = self.reload.as_json(as_json_config)
+      $redis_jason.hset("jason:#{self.class.name.underscore}:cache", self.id, payload.to_json)
+    else
+      $redis_jason.hdel("jason:#{self.class.name.underscore}:cache", self.id)
+    end
+  end
+
+  def publish_json
+    cache_json
+    return if skip_publish_json
+
+    self.class.jason_subscriptions.each do |id, config_json|
+      config = JSON.parse(config_json)
+
+      if (config['conditions'] || {}).all? { |field, value| self.send(field) == value }
+        Jason::Subscription.new(id: id).update(self.class.name.underscore)
+      end
+    end
+  end
+
+  def publish_json_if_changed
+    subscribed_fields = api_model.subscribed_fields
+    publish_json if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
+  end
+
+  class_methods do
+    def subscriptions
+      $redis_jason.hgetall("jason:#{self.name.underscore}:subscriptions")
+    end
+
+    def jason_subscriptions
+      $redis_jason.hgetall("jason:#{self.name.underscore}:subscriptions")
+    end
+
+    def publish_all(instances)
+      instances.each(&:cache_json)
+
+      subscriptions.each do |id, config_json|
+        Jason::Subscription.new(id: id).update(self.name.underscore)
+      end
+    end
+
+    def flush_cache
+      $redis_jason.del("jason:#{self.name.underscore}:cache")
+    end
+
+    def setup_json
+      self.after_initialize -> {
+        @api_model = Jason::ApiModel.new(self.class.name.underscore)
+      }
+      self.after_commit :publish_json_if_changed
+
+      include_models = Jason::ApiModel.new(self.name.underscore).include_models
+
+      include_models.map do |assoc|
+        puts assoc
+        reflection = self.reflect_on_association(assoc.to_sym)
+        reflection.klass.after_commit -> {
+          subscribed_fields = Jason::ApiModel.new(self.class.name.underscore).subscribed_fields
+          puts subscribed_fields.inspect
+
+          if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present?
+            self.send(reflection.inverse_of.name)&.publish_json
+          end
+        }
+      end
+    end
+
+    def find_or_create_by_id(params)
+      object = find_by(id: params[:id])
+
+      if object
+        object.update(params)
+      elsif params[:hidden]
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
+      end
+
+      object
+    end
+
+    def find_or_create_by_id!(params)
+      object = find_by(id: params[:id])
+
+      if object
+        object.update!(params)
+      elsif params[:hidden]
+        ## TODO: We're diverging from semantics of the Rails bang! methods here, which would normally either raise or return an object. Find a way to make this better.
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
+      end
+
+      object
+    end
+  end
+
+  included do
+    attr_accessor :skip_publish_json, :api_model
+
+    setup_json
+  end
+end

data/lib/jason/subscription.rb

@@ -1,172 +1,423 @@
 class Jason::Subscription
   attr_accessor :id, :config
+  attr_reader :includes_helper, :graph_helper
 
   def initialize(id: nil, config: nil)
     if id
       @id = id
-      raw_config = $redis.hgetall("jason:subscriptions:#{id}").map { |k,v| [k, JSON.parse(v)] }.to_h.with_indifferent_access
+      raw_config = $redis_jason.hgetall("jason:subscriptions:#{id}").map { |k,v| [k, JSON.parse(v)] }.to_h
+      raise "Subscription ID #{id} does not exist" if raw_config.blank?
       set_config(raw_config)
     else
-      @id = Digest::MD5.hexdigest(config.to_json)
+      @id = Digest::MD5.hexdigest(config.sort_by { |key| key }.to_h.to_json)
       configure(config)
     end
+    @includes_helper = Jason::IncludesHelper.new({ model => self.config['includes'] })
+    @graph_helper = Jason::GraphHelper.new(self.id, @includes_helper)
+
+    check_for_missing_keys
   end
 
-  def set_config(raw_config)
-    @config =  raw_config.with_indifferent_access.map { |k,v| [k.underscore.to_s, v] }.to_h
+  def broadcaster
+    @broadcaster ||= Jason::Broadcaster.new(channel)
   end
 
-  def configure(raw_config)
-    set_config(raw_config)
-    $redis.hmset("jason:subscriptions:#{id}", *config.map { |k,v| [k, v.to_json]}.flatten)
+  def check_for_missing_keys
+    missing_keys = includes_helper.all_models - Jason.schema.keys.map(&:to_s)
+    if missing_keys.present?
+      raise "#{missing_keys.inspect} are not in the schema. Only models in the Jason schema can be subscribed."
+    end
   end
 
-  def destroy
-    config.each do |model, value|
-      $redis.srem("jason:#{model.to_s.underscore}:subscriptions", id)
+  def self.upsert_by_config(model, conditions: {}, includes: {})
+    self.new(config: {
+      model: model,
+      conditions: conditions || {},
+      includes: includes || {}
+    })
+  end
+
+  def self.find_by_id(id)
+    self.new(id: id)
+  end
+
+  def self.for_instance_with_child(model_name, id, child_model_name, include_all = true)
+    sub_ids = for_instance(model_name, id, include_all = true)
+    sub_ids.select do |sub_id|
+      find_by_id(sub_id).includes_helper.in_sub(model_name, child_model_name)
     end
-    $redis.del("jason:subscriptions:#{id}")
   end
 
-  def add_consumer(consumer_id)
-    before_consumer_count = consumer_count
-    $redis.sadd("jason:subscriptions:#{id}:consumers", consumer_id)
-    $redis.hset("jason:consumers", consumer_id, Time.now.utc)
+  def self.for_instance(model_name, id, include_all = true)
+    subs = $redis_jason.smembers("jason:models:#{model_name}:#{id}:subscriptions")
+    if include_all
+      subs += $redis_jason.smembers("jason:models:#{model_name}:all:subscriptions")
+    end
 
-    if before_consumer_count == 0
-      add_subscriptions
-      publish_all
+    subs
+  end
+
+  def self.for_model(model_name)
+
+  end
+
+  # Find and update subscriptions affected by a model changing foreign key
+  # comment, comment_id, post, old_post_id, new_post_id
+  def self.update_ids(changed_model_name, changed_model_id, foreign_model_name, old_foreign_id, new_foreign_id)
+    # There are 4 cases to consider.
+    # changed_instance ---/--- foreign_instance
+    #                  \--+--- new_foreign_instance
+    #
+    # foreign instance can either be parent or child for a given subscription
+    # 1. Child swap/add: foreign is child
+    # 2. Stay in the family: foreign is parent + both old and new foreign instances are part of the sub
+    # 3. Join the family: foreign is parent + only new foreign instance are part of the sub
+    # 4. Leave the family: foreign is parent + only the old foreign instance is part of the sub
+
+    #########
+    # Subs where changed is parent
+    sub_ids = for_instance_with_child(changed_model_name, changed_model_id, foreign_model_name, true)
+    sub_ids.each do |sub_id|
+      subscription = find_by_id(sub_id)
+
+      # If foreign key has been nulled, nothing to add
+      add = new_foreign_id.present? ? [
+        {
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, new_foreign_id]]
+        },
+        # Add IDs of child models
+        subscription.load_ids_for_sub_models(foreign_model_name, new_foreign_id)
+      ] : nil
+
+      id_changeset = subscription.graph_helper.apply_update({
+        remove: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, old_foreign_id]]
+        }],
+        add: add
+      })
+
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
+    end
+
+    old_sub_ids = for_instance_with_child(foreign_model_name, old_foreign_id, changed_model_name, true)
+    new_sub_ids = for_instance_with_child(foreign_model_name, new_foreign_id, changed_model_name, true)
+
+    #########
+    # Subs where changed is child
+    # + parent in both old + new
+    # this is simple, only the edges need to change - no IDs can be changed
+    (old_sub_ids & new_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      subscription.graph_helper.apply_update({
+        remove: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, old_foreign_id]]
+        }],
+        add: [{
+          model_names: [changed_model_name, foreign_model_name],
+          instance_ids: [[changed_model_id, new_foreign_id]]
+        }]
+      })
+    end
+
+    #########
+    # Subs where changed is child
+    # + old parent wasn't in the sub, but new parent is
+    # IE the changed instance is joining the sub
+    # No edges are removed, just added
+    (new_sub_ids - old_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      id_changeset = subscription.graph_helper.apply_update({
+        add: [
+          {
+            model_names: [changed_model_name, foreign_model_name],
+            instance_ids: [[changed_model_id, new_foreign_id]]
+          },
+          # Add IDs of child models
+          subscription.load_ids_for_sub_models(changed_model_name, changed_model_id)
+        ]
+      })
+
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
+    end
+
+    #########
+    # --> Leaving the family
+    # Subs where changed is child
+    # + old parent was in the sub, but new parent isn't
+    # Just need to remove the link, orphan detection will do the rest
+    (old_sub_ids - new_sub_ids).each do |sub_id|
+      subscription = find_by_id(sub_id)
+      id_changeset = subscription.graph_helper.apply_update({
+        remove: [
+          {
+            model_names: [changed_model_name, foreign_model_name],
+            instance_ids: [[changed_model_id, old_foreign_id]]
+          }
+        ]
+      })
+      subscription.apply_id_changeset(id_changeset)
+      subscription.broadcast_id_changeset(id_changeset)
     end
   end
 
-  def remove_consumer(consumer_id)
-    $redis.srem("jason:subscriptions:#{id}:consumers", consumer_id)
-    $redis.hdel("jason:consumers", consumer_id)
+  def self.remove_ids(model_name, ids)
+    # td: finish this
+    ids.each do |instance_id|
+      for_instance(model_name, instance_id, false).each do |sub_id|
+        subscription = find_by_id(sub_id)
 
-    if consumer_count == 0
-      remove_subscriptions
+        id_changeset = subscription.graph_helper.apply_remove_node("#{model_name}:#{instance_id}")
+        subscription.apply_id_changeset(id_changeset)
+        subscription.broadcast_id_changeset(id_changeset)
+      end
     end
   end
 
-  def consumer_count
-    $redis.scard("jason:subscriptions:#{id}:consumers")
+  # Add ID to any _all_ subscriptions
+  def self.add_id(model_name, id)
+
   end
 
-  def channel
-    "jason:#{id}"
+  def self.all
+    $redis_jason.smembers('jason:subscriptions').map { |id| Jason::Subscription.find_by_id(id) }
   end
 
-  def publish_all
-    config.each do |model, model_config|
-      klass = model.to_s.classify.constantize
-      conditions = model_config['conditions'] || {}
-      klass.where(conditions).find_each(&:cache_json)
-      update(model)
+  def set_config(raw_config)
+    @config =  raw_config.deep_stringify_keys.deep_transform_values { |v| v.is_a?(Symbol) ? v.to_s : v }
+  end
+
+  def clear_id(model_name, id, parent_model_name)
+    remove_ids(model_name, [id])
+  end
+
+  # Add IDs that aren't present
+  def commit_ids(model_name, ids)
+    $redis_jason.sadd("jason:subscriptions:#{id}:ids:#{model_name}", ids)
+    ids.each do |instance_id|
+      $redis_jason.sadd("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  def add_subscriptions
-    config.each do |model, value|
-      $redis.hset("jason:#{model.to_s.underscore}:subscriptions", id, value.to_json)
-      update(model)
+  def remove_ids(model_name, ids)
+    $redis_jason.srem("jason:subscriptions:#{id}:ids:#{model_name}", ids)
+    ids.each do |instance_id|
+      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  def remove_subscriptions
-    config.each do |model, _|
-      $redis.hdel("jason:#{model.to_s.underscore}:subscriptions", id)
+  def apply_id_changeset(changeset)
+    changeset[:ids_to_add].each do |model_name, ids|
+      commit_ids(model_name, ids)
+    end
+
+    changeset[:ids_to_remove].each do |model_name, ids|
+      remove_ids(model_name, ids)
     end
   end
 
-  def self.publish_all
-    JASON_API_MODEL.each do |model, _v|
-      klass = model.to_s.classify.constantize
-      klass.publish_all(klass.all) if klass.respond_to?(:publish_all)
+  def broadcast_id_changeset(changeset)
+    changeset[:ids_to_add].each do |model_name, ids|
+      ids.each { |id| add(model_name, id) }
+    end
+
+    changeset[:ids_to_remove].each do |model_name, ids|
+      ids.each { |id| destroy(model_name, id) }
     end
   end
 
-  def get(model)
-    value = JSON.parse($redis.get("#{channel}:#{model}:value") || '{}')
-    idx = $redis.get("#{channel}:#{model}:idx").to_i
+  # Take a model name and IDs and return an edge set of all the models that appear and
+  # their instance IDs
+  def load_ids_for_sub_models(model_name, ids)
+    # Limitation: Same association can't appear twice
+    includes_tree = includes_helper.get_tree_for(model_name)
+    all_models = includes_helper.all_models(model_name)
 
-    {
-      type: 'payload',
-      md5Hash: id,
-      model: model,
-      value: value,
-      idx: idx
-    }
+    relation = model_name.classify.constantize.all.eager_load(includes_tree)
+
+    if model_name == model
+      if conditions.blank?
+        $redis_jason.sadd("jason:models:#{model_name}:all:subscriptions", id)
+        all_models -= [model_name]
+      else
+        relation = relation.where(conditions)
+      end
+    else
+      raise "Must supply IDs for sub models" if ids.nil?
+      return if ids.blank?
+      relation = relation.where(id: ids)
+    end
+
+    pluck_args = all_models.map { |m| "#{m.pluralize}.id" }
+    instance_ids = relation.pluck(*pluck_args)
+
+    # pluck returns only a 1D array if only 1 arg passed
+    if all_models.size == 1
+      instance_ids = [instance_ids]
+    end
+
+    return { model_names: all_models, instance_ids: instance_ids }
   end
 
-  def get_diff(old_value, value)
-    JsonDiff.generate(old_value, value)
+  # 'posts', [post#1, post#2,...]
+  def set_ids_for_sub_models(model_name = model, ids = nil, enforce: false)
+    edge_set = load_ids_for_sub_models(model_name, ids)
+
+    # Build the tree
+    id_changeset = graph_helper.apply_update({
+      add: [edge_set]
+    })
+    apply_id_changeset(id_changeset)
   end
 
-  def deep_stringify(value)
-    if value.is_a?(Hash)
-      value.deep_stringify_keys
-    elsif value.is_a?(Array)
-      value.map { |x| x.deep_stringify_keys }
+  def clear_all_ids
+    includes_helper.all_models.each do |model_name|
+      if model_name == model && conditions.blank?
+        $redis_jason.srem("jason:models:#{model_name}:all:subscriptions", id)
+      end
+
+      ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+      ids.each do |instance_id|
+        $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
+      end
+      $redis_jason.del("jason:subscriptions:#{id}:ids:#{model_name}")
     end
   end
 
-  def get_throttle
-    if !$throttle_rate || !$throttle_timeout || Time.now.utc > $throttle_timeout
-      $throttle_timeout = Time.now.utc + 5.seconds
-      $throttle_rate = (Sidekiq.redis { |r| r.get 'global_throttle_rate' } || 0).to_i
-    else
-      $throttle_rate
+  def ids(model_name = model)
+    $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+  end
+
+  def model
+    @config['model']
+  end
+
+  def model_klass(model_name)
+    model_name.to_s.classify.constantize
+  end
+
+  def conditions
+    @config['conditions']
+  end
+
+  def configure(raw_config)
+    set_config(raw_config)
+    $redis_jason.sadd("jason:subscriptions", id)
+    $redis_jason.hmset("jason:subscriptions:#{id}", *config.map { |k,v| [k, v.to_json] }.flatten)
+  end
+
+  def destroy
+    raise
+  end
+
+  def add_consumer(consumer_id)
+    before_consumer_count = consumer_count
+    $redis_jason.sadd("jason:subscriptions:#{id}:consumers", consumer_id)
+    $redis_jason.hset("jason:consumers", consumer_id, Time.now.utc)
+
+    if before_consumer_count == 0
+      set_ids_for_sub_models
     end
   end
 
-  # Atomically update and return patch
-  def update(model)
-    start_time = Time.now.utc
-    conditions = config[model]['conditions']
+  def remove_consumer(consumer_id)
+    $redis_jason.srem("jason:subscriptions:#{id}:consumers", consumer_id)
+    $redis_jason.hdel("jason:consumers", consumer_id)
 
-    value = $redis.hgetall("jason:#{model}:cache")
-      .values.map { |v| JSON.parse(v) }
-      .select { |v| (conditions || {}).all? { |field, value| v[field] == value } }
-      .sort_by { |v| v['id'] }
+    if consumer_count == 0
+      clear_all_ids
+    end
+  end
 
-    # lfsa = last finished, started at
-    # If another job that started after this one, finished before this one, skip sending this state update
-    if Time.parse(Sidekiq.redis { |r| r.get("jason:#{channel}:lfsa") || '1970-01-01 00:00:00 UTC' } ) < start_time
-      Sidekiq.redis { |r| r.set("jason:#{channel}:lfsa", start_time) }
+  def consumer_count
+    $redis_jason.scard("jason:subscriptions:#{id}:consumers")
+  end
+
+  def channel
+    "jason-#{id}"
+  end
+
+  def user_can_access?(user)
+    # td: implement the authorization logic here
+    return true if Jason.authorization_service.blank?
+    Jason.authorization_service.call(user, model, conditions, includes_helper.all_models - [model])
+  end
+
+  def get
+    includes_helper.all_models.map { |model_name| [model_name, get_for_model(model_name)] }.to_h
+  end
+
+  def get_for_model(model_name)
+    if $redis_jason.sismember("jason:models:#{model_name}:all:subscriptions", id)
+      instance_jsons_hash, idx = $redis_jason.multi do |r|
+        r.hgetall("jason:cache:#{model_name}")
+        r.get("jason:subscription:#{id}:#{model_name}:idx")
+      end
+      instance_jsons = instance_jsons_hash.values
     else
-      return
+      instance_jsons, idx = Jason::LuaGenerator.new.get_payload(model_name, id)
     end
 
-    value = deep_stringify(value)
+    return if instance_jsons.blank?
 
-    # If value has changed, return old value and new idx. Otherwise do nothing.
-    cmd = <<~LUA
-      local old_val=redis.call('get', ARGV[1] .. ':value')
-      if old_val ~= ARGV[2] then
-        redis.call('set', ARGV[1] .. ':value', ARGV[2])
-        local new_idx = redis.call('incr', ARGV[1] .. ':idx')
-        return { new_idx, old_val }
-      end
-    LUA
+    payload = instance_jsons.map do |instance_json|
+      instance_json ? JSON.parse(instance_json) : {}
+    end
 
-    result = $redis.eval cmd, [], ["#{channel}:#{model}", value.to_json]
-    return if result.blank?
+    {
+      type: 'payload',
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
+  end
 
-    idx = result[0]
-    old_value = JSON.parse(result[1] || '{}')
+  def add(model_name, instance_id)
+    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}:idx")
+    payload = JSON.parse($redis_jason.hget("jason:cache:#{model_name}", instance_id) || '{}')
 
-    diff = get_diff(old_value, value)
+    payload = {
+      id: instance_id,
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
 
-    end_time = Time.now.utc
+    broadcaster.broadcast(payload)
+  end
+
+  def update(model_name, instance_id, payload, gidx)
+    idx = Jason::LuaGenerator.new.get_subscription(model_name, instance_id, id, gidx)
+    return if idx.blank?
 
     payload = {
-      model: model,
+      id: instance_id,
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
+
+    broadcaster.broadcast(payload)
+  end
+
+  def destroy(model_name, instance_id)
+    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}:idx")
+
+    payload = {
+      id: instance_id,
+      model: model_name,
+      destroy: true,
       md5Hash: id,
-      diff: diff,
-      idx: idx.to_i,
-      latency: ((end_time - start_time)*1000).round
+      idx: idx.to_i
     }
 
-    ActionCable.server.broadcast("jason:#{id}", payload)
+    broadcaster.broadcast(payload)
   end
 end