jason-rails 0.4.1 → 0.5.0

data/lib/jason/publisher_old.rb

@@ -0,0 +1,112 @@
+module Jason::PublisherOld
+  extend ActiveSupport::Concern
+
+  def cache_json
+    as_json_config = api_model.as_json_config
+    scope = api_model.scope
+
+    if self.persisted? && (scope.blank? || self.class.unscoped.send(scope).exists?(self.id))
+      payload = self.reload.as_json(as_json_config)
+      $redis_jason.hset("jason:#{self.class.name.underscore}:cache", self.id, payload.to_json)
+    else
+      $redis_jason.hdel("jason:#{self.class.name.underscore}:cache", self.id)
+    end
+  end
+
+  def publish_json
+    cache_json
+    return if skip_publish_json
+
+    self.class.jason_subscriptions.each do |id, config_json|
+      config = JSON.parse(config_json)
+
+      if (config['conditions'] || {}).all? { |field, value| self.send(field) == value }
+        Jason::Subscription.new(id: id).update(self.class.name.underscore)
+      end
+    end
+  end
+
+  def publish_json_if_changed
+    subscribed_fields = api_model.subscribed_fields
+    publish_json if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present? || !self.persisted?
+  end
+
+  class_methods do
+    def subscriptions
+      $redis_jason.hgetall("jason:#{self.name.underscore}:subscriptions")
+    end
+
+    def jason_subscriptions
+      $redis_jason.hgetall("jason:#{self.name.underscore}:subscriptions")
+    end
+
+    def publish_all(instances)
+      instances.each(&:cache_json)
+
+      subscriptions.each do |id, config_json|
+        Jason::Subscription.new(id: id).update(self.name.underscore)
+      end
+    end
+
+    def flush_cache
+      $redis_jason.del("jason:#{self.name.underscore}:cache")
+    end
+
+    def setup_json
+      self.after_initialize -> {
+        @api_model = Jason::ApiModel.new(self.class.name.underscore)
+      }
+      self.after_commit :publish_json_if_changed
+
+      include_models = Jason::ApiModel.new(self.name.underscore).include_models
+
+      include_models.map do |assoc|
+        puts assoc
+        reflection = self.reflect_on_association(assoc.to_sym)
+        reflection.klass.after_commit -> {
+          subscribed_fields = Jason::ApiModel.new(self.class.name.underscore).subscribed_fields
+          puts subscribed_fields.inspect
+
+          if (self.previous_changes.keys.map(&:to_sym) & subscribed_fields).present?
+            self.send(reflection.inverse_of.name)&.publish_json
+          end
+        }
+      end
+    end
+
+    def find_or_create_by_id(params)
+      object = find_by(id: params[:id])
+
+      if object
+        object.update(params)
+      elsif params[:hidden]
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
+      end
+
+      object
+    end
+
+    def find_or_create_by_id!(params)
+      object = find_by(id: params[:id])
+
+      if object
+        object.update!(params)
+      elsif params[:hidden]
+        ## TODO: We're diverging from semantics of the Rails bang! methods here, which would normally either raise or return an object. Find a way to make this better.
+        return false ## If an object is passed with hidden = true but didn't already exist, it's safe to never create it
+      else
+        object = create!(params)
+      end
+
+      object
+    end
+  end
+
+  included do
+    attr_accessor :skip_publish_json, :api_model
+
+    setup_json
+  end
+end

data/lib/jason/subscription.rb

@@ -4,166 +4,389 @@ class Jason::Subscription
   def initialize(id: nil, config: nil)
     if id
       @id = id
-      raw_config = $redis_jason.hgetall("jason:subscriptions:#{id}").map { |k,v| [k, JSON.parse(v)] }.to_h.with_indifferent_access
+      raw_config = $redis_jason.hgetall("jason:subscriptions:#{id}").map { |k,v| [k, JSON.parse(v)] }.to_h
       set_config(raw_config)
     else
-      @id = Digest::MD5.hexdigest(config.to_json)
+      @id = Digest::MD5.hexdigest(config.sort_by { |key| key }.to_h.to_json)
+      pp config.sort_by { |key| key }.to_h.to_json
       configure(config)
     end
+    pp @id
   end
 
-  def set_config(raw_config)
-    @config =  raw_config.with_indifferent_access.map { |k,v| [k.underscore.to_s, v] }.to_h
+  def self.upsert_by_config(model, conditions: {}, includes: {})
+    self.new(config: {
+      model: model,
+      conditions: conditions || {},
+      includes: includes || {}
+    })
   end
 
-  def configure(raw_config)
-    set_config(raw_config)
-    $redis_jason.hmset("jason:subscriptions:#{id}", *config.map { |k,v| [k, v.to_json]}.flatten)
+  def self.find_by_id(id)
+    self.new(id: id)
   end
 
-  def destroy
-    config.each do |model, value|
-      $redis_jason.srem("jason:#{model.to_s.underscore}:subscriptions", id)
+  def self.for_instance(model_name, id, include_all = true)
+    subs = $redis_jason.smembers("jason:models:#{model_name}:#{id}:subscriptions")
+    if include_all
+      subs += $redis_jason.smembers("jason:models:#{model_name}:all:subscriptions")
     end
-    $redis_jason.del("jason:subscriptions:#{id}")
+
+    subs
   end
 
-  def add_consumer(consumer_id)
-    before_consumer_count = consumer_count
-    $redis_jason.sadd("jason:subscriptions:#{id}:consumers", consumer_id)
-    $redis_jason.hset("jason:consumers", consumer_id, Time.now.utc)
+  def self.for_model(model_name)
 
-    add_subscriptions
-    publish_all
   end
 
-  def remove_consumer(consumer_id)
-    $redis_jason.srem("jason:subscriptions:#{id}:consumers", consumer_id)
-    $redis_jason.hdel("jason:consumers", consumer_id)
+  # Find and update subscriptions affected by a model changing foreign key
+  # comment, comment_id, post, old_post_id, new_post_id
+  def self.update_ids(model_name, id, parent_model_name, old_foreign_id, new_foreign_id)
+    # Check if this change means it needs to be removed
+    # First find subscriptions that reference this model
 
-    if consumer_count == 0
-      remove_subscriptions
+    if old_foreign_id
+      old_model_subscriptions = for_instance(parent_model_name, old_foreign_id, false)
+      new_model_subscriptions = for_instance(parent_model_name, new_foreign_id, false)
+    else
+      # If this is a new instance, we need to include _all_ subscriptions
+      old_model_subscriptions = []
+      new_model_subscriptions = for_instance(parent_model_name, new_foreign_id, true)
+    end
+
+    # To add
+    (new_model_subscriptions - old_model_subscriptions).each do |sub_id|
+      # add the current ID to the subscription, then add the tree below it
+      find_by_id(sub_id).set_id(model_name, id)
     end
+
+    # To remove
+    (old_model_subscriptions - new_model_subscriptions).each do |sub_id|
+      find_by_id(sub_id).remove_ids(model_name, [id])
+    end
+
+    # TODO changes to sub models - e.g. post -> comment -> user
   end
 
-  def consumer_count
-    $redis_jason.scard("jason:subscriptions:#{id}:consumers")
+  def self.remove_ids(model_name, ids)
+    ids.each do |instance_id|
+      for_instance(model_name, instance_id, false).each do |sub_id|
+        find_by_id(sub_id).remove_ids(model_name, [instance_id])
+      end
+    end
   end
 
-  def channel
-    "jason:#{id}"
+  # Add ID to any _all_ subscriptions
+  def self.add_id(model_name, id)
+
+  end
+
+  def self.all
+    $redis_jason.keys('jason:subscriptions:*')
+  end
+
+  def set_config(raw_config)
+    @config =  raw_config.with_indifferent_access
+  end
+
+  # E.g. add comment#123, and then sub models
+  def set_id(model_name, id)
+    commit_ids(model_name, [id])
+    assoc_name = get_assoc_name(model_name)
+    set_ids_for_sub_models(assoc_name, [id])
+  end
+
+  def clear_id(model_name, id, parent_model_name)
+    remove_ids(model_name, [id])
   end
 
-  def publish_all
-    config.each do |model, model_config|
-      klass = model.to_s.classify.constantize
-      conditions = model_config['conditions'] || {}
-      klass.where(conditions).find_each(&:cache_json)
-      update(model)
+  # Set the instance IDs for the subscription
+  # Add an entry to the subscription list for each instance
+  def set_ids(assoc_name = model, referrer_model_name = nil, referrer_ids = nil, enforce: false)
+    model_name = assoc_name.to_s.singularize
+
+    if referrer_model_name.blank? && conditions.blank?
+      $redis_jason.sadd("jason:models:#{model_name}:all:subscriptions", id)
+      ids = model_klass(model_name).all.pluck(:id)
+      set_ids_for_sub_models(assoc_name, ids, enforce: enforce)
+      return
     end
+
+    if referrer_model_name.blank?
+      ids = model_klass(model_name).where(conditions).pluck(:id)
+    else
+      assoc = model_klass(referrer_model_name).reflect_on_association(assoc_name.to_sym)
+
+      if assoc.is_a?(ActiveRecord::Reflection::HasManyReflection)
+        ids = model_klass(model_name).where(assoc.foreign_key => referrer_ids).pluck(:id)
+      elsif assoc.is_a?(ActiveRecord::Reflection::BelongsToReflection)
+        ids = model_klass(referrer_model_name).where(id: referrer_ids).pluck(assoc.foreign_key)
+      end
+    end
+    return if ids.blank?
+
+    enforce ? enforce_ids(model_name, ids) : commit_ids(model_name, ids)
+    set_ids_for_sub_models(assoc_name, ids, enforce: enforce)
   end
 
-  def add_subscriptions
-    config.each do |model, value|
-      $redis_jason.hset("jason:#{model.to_s.underscore}:subscriptions", id, value.to_json)
-      update(model)
+  def refresh_ids(assoc_name = model, referrer_model_name = nil, referrer_ids)
+
+  end
+
+  # Add IDs that aren't present
+  def commit_ids(model_name, ids)
+    pp 'COMMIT'
+    pp model_name
+    pp ids
+    $redis_jason.sadd("jason:subscriptions:#{id}:ids:#{model_name}", ids)
+    ids.each do |instance_id|
+      $redis_jason.sadd("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  def remove_subscriptions
-    config.each do |model, _|
-      $redis_jason.hdel("jason:#{model.to_s.underscore}:subscriptions", id)
+  # Ensure IDs are _only_ the ones passed
+  def enforce_ids(model_name, ids)
+    old_ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+
+    # Remove
+    $redis_jason.srem("jason:subscriptions:#{id}:ids:#{model_name}", (old_ids - ids))
+
+    (old_ids - ids).each do |instance_id|
+      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
+    end
+
+    # Add
+    $redis_jason.sadd("jason:subscriptions:#{id}:ids:#{model_name}", (ids - old_ids))
+
+    (ids - old_ids).each do |instance_id|
+      $redis_jason.sadd("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  def self.publish_all
-    JASON_API_MODEL.each do |model, _v|
-      klass = model.to_s.classify.constantize
-      klass.publish_all(klass.all) if klass.respond_to?(:publish_all)
+  def remove_ids(model_name, ids)
+    $redis_jason.srem("jason:subscriptions:#{id}:ids:#{model_name}", ids)
+    ids.each do |instance_id|
+      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
     end
   end
 
-  def get(model)
-    value = JSON.parse($redis_jason.get("#{channel}:#{model}:value") || '[]')
-    idx = $redis_jason.get("#{channel}:#{model}:idx").to_i
+  # 'posts', [post#1, post#2,...]
+  def set_ids_for_sub_models(assoc_name, ids, enforce: false)
+    model_name = assoc_name.to_s.singularize
+    # Limitation: Same association can't appear twice
+    includes_tree = get_tree_for(assoc_name)
 
-    {
-      type: 'payload',
-      md5Hash: id,
-      model: model,
-      value: value,
-      idx: idx
-    }
+    if includes_tree.is_a?(Hash)
+      includes_tree.each do |assoc_name, includes_tree|
+        set_ids(assoc_name, model_name, ids, enforce: enforce)
+      end
+    # [:likes, :user]
+    elsif includes_tree.is_a?(Array)
+      includes_tree.each do |assoc_name|
+        set_ids(assoc_name, model_name, ids, enforce: enforce)
+      end
+    elsif includes_tree.is_a?(String)
+      set_ids(includes_tree, model_name, ids, enforce: enforce)
+    end
   end
 
-  def get_diff(old_value, value)
-    JsonDiff.generate(old_value, value)
+  # assoc could be plural or not, so need to scan both.
+  def get_assoc_name(model_name, haystack = includes)
+    return model_name if model_name == model
+
+    if haystack.is_a?(Hash)
+      haystack.each do |assoc_name, includes_tree|
+        if model_name.pluralize == assoc_name.to_s.pluralize
+          return assoc_name
+        else
+          found_assoc = get_assoc_name(model_name, includes_tree)
+          return found_assoc if found_assoc
+        end
+      end
+    elsif haystack.is_a?(Array)
+      haystack.each do |assoc_name|
+        if model_name.pluralize == assoc_name.to_s.pluralize
+          return assoc_name
+        end
+      end
+    else
+      if model_name.pluralize == haystack.to_s.pluralize
+        return haystack
+      end
+    end
+
+    return nil
   end
 
-  def deep_stringify(value)
-    if value.is_a?(Hash)
-      value.deep_stringify_keys
-    elsif value.is_a?(Array)
-      value.map { |x| x.deep_stringify_keys }
+  def get_tree_for(needle, assoc_name = nil, haystack = includes)
+    return includes if needle == model
+    return haystack if needle.to_s == assoc_name.to_s
+
+    if haystack.is_a?(Hash)
+      haystack.each do |assoc_name, includes_tree|
+        found_haystack = get_tree_for(needle, assoc_name, includes_tree)
+        return found_haystack if found_haystack
+      end
     end
+
+    return nil
   end
 
-  def get_throttle
-    if !$throttle_rate || !$throttle_timeout || Time.now.utc > $throttle_timeout
-      $throttle_timeout = Time.now.utc + 5.seconds
-      $throttle_rate = ($redis_jason.get('global_throttle_rate') || 0).to_i
+  def all_models(tree = includes)
+    sub_models = if tree.is_a?(Hash)
+      tree.map do |k,v|
+        [k, all_models(v)]
+      end
     else
-      $throttle_rate
+      tree
     end
+
+    pp ([model] + [sub_models]).flatten.uniq.map(&:to_s).map(&:singularize)
+    ([model] + [sub_models]).flatten.uniq.map(&:to_s).map(&:singularize)
   end
 
-  # Atomically update and return patch
-  def update(model)
-    start_time = Time.now.utc
-    conditions = config[model]['conditions']
+  def clear_all_ids(assoc_name = model)
+    model_name = assoc_name.to_s.singularize
+    includes_tree = model_name == model ? includes : get_tree_for(assoc_name)
 
-    value = $redis_jason.hgetall("jason:#{model}:cache")
-      .values.map { |v| JSON.parse(v) }
-      .select { |v| (conditions || {}).all? { |field, value| v[field] == value } }
-      .sort_by { |v| v['id'] }
+    if model_name == model && conditions.blank?
+      $redis_jason.srem("jason:models:#{model_name}:all:subscriptions", id)
+    end
 
-    # lfsa = last finished, started at
-    # If another job that started after this one, finished before this one, skip sending this state update
-    if Time.parse($redis_jason.get("jason:#{channel}:lfsa") || '1970-01-01 00:00:00 UTC') < start_time
-      $redis_jason.set("jason:#{channel}:lfsa", start_time)
-    else
-      return
+    ids = $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+    ids.each do |instance_id|
+      $redis_jason.srem("jason:models:#{model_name}:#{instance_id}:subscriptions", id)
+    end
+    $redis_jason.del("jason:subscriptions:#{id}:ids:#{model_name}")
+
+    # Recursively clear IDs
+    # { comments: [:like] }
+    if includes_tree.is_a?(Hash)
+      includes_tree.each do |assoc_name, includes_tree|
+        clear_all_ids(assoc_name)
+      end
+    # [:likes, :user]
+    elsif includes_tree.is_a?(Array)
+      includes_tree.each do |assoc_name|
+        clear_all_ids(assoc_name)
+      end
+    elsif includes_tree.is_a?(String)
+      clear_all_ids(includes_tree)
     end
+  end
 
-    value = deep_stringify(value)
+  def ids(model_name = model)
+    $redis_jason.smembers("jason:subscriptions:#{id}:ids:#{model_name}")
+  end
 
-    # If value has changed, return old value and new idx. Otherwise do nothing.
-    cmd = <<~LUA
-      local old_val=redis.call('get', ARGV[1] .. ':value')
-      if old_val ~= ARGV[2] then
-        redis.call('set', ARGV[1] .. ':value', ARGV[2])
-        local new_idx = redis.call('incr', ARGV[1] .. ':idx')
-        return { new_idx, old_val }
+  def model
+    @config['model']
+  end
+
+  def model_klass(model_name)
+    model_name.to_s.classify.constantize
+  end
+
+  def conditions
+    @config['conditions']
+  end
+
+  def includes
+    @config['includes']
+  end
+
+  def configure(raw_config)
+    set_config(raw_config)
+    $redis_jason.hmset("jason:subscriptions:#{id}", *config.map { |k,v| [k, v.to_json] }.flatten)
+  end
+
+  def destroy
+    raise
+  end
+
+  def add_consumer(consumer_id)
+    before_consumer_count = consumer_count
+    $redis_jason.sadd("jason:subscriptions:#{id}:consumers", consumer_id)
+    $redis_jason.hset("jason:consumers", consumer_id, Time.now.utc)
+
+    if before_consumer_count == 0
+      set_ids
+    end
+  end
+
+  def remove_consumer(consumer_id)
+    $redis_jason.srem("jason:subscriptions:#{id}:consumers", consumer_id)
+    $redis_jason.hdel("jason:consumers", consumer_id)
+
+    if consumer_count == 0
+      clear_all_ids
+    end
+  end
+
+  def consumer_count
+    $redis_jason.scard("jason:subscriptions:#{id}:consumers")
+  end
+
+  def channel
+    "jason:#{id}"
+  end
+
+  def get
+    all_models.map { |model_name| get_for_model(model_name) }
+  end
+
+  def get_for_model(model_name)
+    if $redis_jason.sismember("jason:models:#{model_name}:all:subscriptions", id)
+      instance_jsons_hash, idx = $redis_jason.multi do |r|
+        r.hgetall("jason:cache:#{model_name}")
+        r.get("jason:subscription:#{id}:#{model_name}:idx")
       end
-    LUA
+      instance_jsons = instance_jsons_hash.values
+    else
+      instance_jsons, idx = Jason::LuaGenerator.new.get_payload(model_name, id)
+    end
 
-    result = $redis_jason.eval cmd, [], ["#{channel}:#{model}", value.to_json]
-    return if result.blank?
+    return if instance_jsons.blank?
 
-    idx = result[0]
-    old_value = JSON.parse(result[1] || '[]')
-    diff = get_diff(old_value, value)
+    payload = instance_jsons.map do |instance_json|
+      instance_json ? JSON.parse(instance_json) : {}
+    end
+
+    {
+      type: 'payload',
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
+  end
 
-    end_time = Time.now.utc
+  def update(model_name, instance_id, payload, gidx)
+    idx = Jason::LuaGenerator.new.get_subscription(model_name, instance_id, id, gidx)
+    return if idx.blank?
 
     payload = {
-      model: model,
+      id: instance_id,
+      model: model_name,
+      payload: payload,
+      md5Hash: id,
+      idx: idx.to_i
+    }
+
+    ActionCable.server.broadcast(channel, payload)
+  end
+
+  def destroy(model_name, instance_id)
+    idx = $redis_jason.incr("jason:subscription:#{id}:#{model_name}idx")
+
+    payload = {
+      id: instance_id,
+      model: model_name,
+      destroy: true,
       md5Hash: id,
-      diff: diff,
-      idx: idx.to_i,
-      latency: ((end_time - start_time)*1000).round
+      idx: idx.to_i
     }
 
-    ActionCable.server.broadcast("jason:#{id}", payload)
+    ActionCable.server.broadcast(channel, payload)
   end
 end