pakyow-data 1.0.0.rc1

data/lib/pakyow/data/subscribers/adapters/memory.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+
+require "concurrent/array"
+require "concurrent/hash"
+require "concurrent/timer_task"
+
+require "pakyow/support/deep_dup"
+require "pakyow/support/deep_freeze"
+
+module Pakyow
+  module Data
+    class Subscribers
+      module Adapters
+        # Manages data subscriptions in memory.
+        #
+        # Great for development, not for use in production!
+        #
+        # @api private
+        class Memory
+          class << self
+            def generate_subscription_id(subscription)
+              Digest::SHA1.hexdigest(Marshal.dump(subscription))
+            end
+          end
+
+          using Support::DeepDup
+
+          extend Support::DeepFreeze
+          unfreezable :subscriptions_by_id, :subscription_ids_by_source, :subscribers_by_subscription_id, :subscription_ids_by_subscriber, :expirations_for_subscriber
+
+          def initialize(*)
+            @subscriptions_by_id = Concurrent::Hash.new
+            @subscription_ids_by_source = Concurrent::Hash.new
+            @subscribers_by_subscription_id = Concurrent::Hash.new
+            @subscription_ids_by_subscriber = Concurrent::Hash.new
+            @expirations_for_subscriber = Concurrent::Hash.new
+
+            Concurrent::TimerTask.new(execution_interval: 10, timeout_interval: 10) {
+              @expirations_for_subscriber.each do |subscriber, timeout|
+                if timeout < Time.now
+                  unsubscribe(subscriber)
+                end
+              end
+            }.execute
+          end
+
+          def register_subscriptions(subscriptions, subscriber: nil)
+            subscriptions.map { |subscription|
+              self.class.generate_subscription_id(subscription).tap do |subscription_id|
+                register_subscription_with_subscription_id(subscription, subscription_id)
+                register_subscription_id_for_source(subscription_id, subscription[:source])
+                register_subscriber_for_subscription_id(subscriber, subscription_id)
+              end
+            }
+          end
+
+          def subscriptions_for_source(source)
+            subscription_ids_for_source(source).map { |subscription_id|
+              subscription_with_id(subscription_id)
+            }
+          end
+
+          def unsubscribe(subscriber)
+            subscription_ids_for_subscriber(subscriber).dup.each do |subscription_id|
+              unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
+            end
+          end
+
+          def expire(subscriber, seconds)
+            @expirations_for_subscriber[subscriber] = Time.now + seconds
+          end
+
+          def persist(subscriber)
+            @expirations_for_subscriber.delete(subscriber)
+          end
+
+          def expiring?(subscriber)
+            @expirations_for_subscriber.key?(subscriber)
+          end
+
+          def subscribers_for_subscription_id(subscription_id)
+            @subscribers_by_subscription_id[subscription_id] || []
+          end
+
+          SERIALIZABLE_IVARS = %i(
+            @subscriptions_by_id
+            @subscription_ids_by_source
+            @subscribers_by_subscription_id
+            @expirations_for_subscriber
+          ).freeze
+
+          def serialize
+            SERIALIZABLE_IVARS.each_with_object({}) do |ivar, hash|
+              hash[ivar] = instance_variable_get(ivar)
+            end
+          end
+
+          protected
+
+          def subscription_ids_for_source(source)
+            (@subscription_ids_by_source[source] || []).select { |subscription_id|
+              subscribers_for_subscription_id(subscription_id).any? { |subscriber|
+                !expiring?(subscriber)
+              }
+            }
+          end
+
+          def subscription_with_id(subscription_id)
+            subscription = @subscriptions_by_id[subscription_id].deep_dup
+            subscription[:id] = subscription_id
+            subscription
+          end
+
+          def subscription_ids_for_subscriber(subscriber)
+            @subscription_ids_by_subscriber[subscriber] || []
+          end
+
+          def register_subscription_with_subscription_id(subscription, subscription_id)
+            @subscriptions_by_id[subscription_id] = subscription
+          end
+
+          def register_subscription_id_for_source(subscription_id, source)
+            @subscription_ids_by_source[source] ||= Concurrent::Array.new
+            (@subscription_ids_by_source[source] << subscription_id).uniq!
+          end
+
+          def register_subscriber_for_subscription_id(subscriber, subscription_id)
+            @subscribers_by_subscription_id[subscription_id] ||= Concurrent::Array.new
+            (@subscribers_by_subscription_id[subscription_id] << subscriber).uniq!
+
+            @subscription_ids_by_subscriber[subscriber] ||= Concurrent::Array.new
+            (@subscription_ids_by_subscriber[subscriber] << subscription_id).uniq!
+          end
+
+          def unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
+            subscribers_for_subscription_id(subscription_id).delete(subscriber)
+            subscription_ids_for_subscriber(subscriber).delete(subscription_id)
+
+            if subscribers_for_subscription_id(subscription_id).empty?
+              @subscriptions_by_id.delete(subscription_id)
+
+              @subscription_ids_by_source.each do |_, ids|
+                ids.delete(subscription_id)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pakyow/data/subscribers/adapters/redis/pipeliner.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Pakyow
+  module Data
+    class Subscribers
+      module Adapters
+        class Redis
+          # @api private
+          class Pipeliner
+            TIMEOUT = 1.0 / 100.0
+
+            def initialize(redis)
+              @redis = redis
+              @commands = []
+            end
+
+            def enqueue(future)
+              @commands << { future: future, callback: Proc.new }
+            end
+
+            def wait
+              @commands.map { |command|
+                while command[:future].value.is_a?(::Redis::FutureNotReady)
+                  sleep TIMEOUT
+                end
+
+                command[:callback].call(command[:future].value)
+              }
+            end
+
+            def self.pipeline(redis)
+              pipeliner = Pipeliner.new(redis)
+
+              redis.pipelined do
+                yield pipeliner
+              end
+
+              pipeliner.wait
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pakyow/data/subscribers/adapters/redis/scripts/_shared.lua

@@ -0,0 +1,73 @@
+local function massive_redis_command(command, key, t)
+  local i = 1
+  local temp = {}
+
+  while(i <= #t) do
+    table.insert(temp, t[i + 1])
+    table.insert(temp, t[i])
+
+    if #temp >= 1000 then
+      redis.call(command, key, unpack(temp))
+      temp = {}
+    end
+
+    i = i + 2
+  end
+
+  if #temp > 0 then
+    redis.call(command, key, unpack(temp))
+  end
+end
+
+local function key_subscription_id(key_prefix, key_delimeter, subscription_id)
+  return key_prefix .. key_delimeter .. "subscription:" .. subscription_id
+end
+
+local function key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
+  return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "subscribers"
+end
+
+local function key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id)
+  return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "source"
+end
+
+local function key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber)
+  return key_prefix .. key_delimeter .. "subscriber:" .. subscriber
+end
+
+local function key_subscription_ids_by_source(key_prefix, key_delimeter, source)
+  return key_prefix .. key_delimeter .. "source:" .. source
+end
+
+local function subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, min, max)
+  return redis.call("zrangebyscore", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), min, max)
+end
+
+local function expire_subscription(key_prefix, key_delimeter, subscription_id)
+  local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
+
+  -- only expire the subscription if it is not related to a non-expiring subscriber
+  if redis.call("zcount", subscription_key, "+inf", "+inf") == 0 then
+    local time_expire = redis.call("zrevrangebyscore", subscription_key, "+inf", 0, "WITHSCORES", "LIMIT", 0, 1)[2]
+
+    if time_expire ~= "inf" then
+      local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
+      redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), time_expire, subscription_id)
+
+      redis.call("expireat", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
+      redis.call("expireat", subscription_key, time_expire + 1)
+      redis.call("expireat", key_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
+    end
+  end
+end
+
+local function persist_subscription(key_prefix, key_delimeter, subscription_id)
+  local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
+
+  local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
+  redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
+
+  redis.call("persist", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
+  redis.call("persist", subscription_key)
+  redis.call("persist", key_subscription_id(key_prefix, key_delimeter, subscription_id))
+end

data/lib/pakyow/data/subscribers/adapters/redis/scripts/expire.lua

@@ -0,0 +1,16 @@
+local key_prefix = ARGV[1]
+local key_delimeter = ARGV[2]
+local subscriber = ARGV[3]
+local time_expire = ARGV[4]
+
+redis.call("expireat", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), time_expire + 1)
+
+local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, "+inf", "+inf")
+
+local i = 1
+while(i <= #subscription_ids) do
+  local subscription_id = subscription_ids[i]
+  redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire, subscriber)
+  expire_subscription(key_prefix, key_delimeter, subscription_id)
+  i = i + 1
+end

data/lib/pakyow/data/subscribers/adapters/redis/scripts/persist.lua

@@ -0,0 +1,15 @@
+local key_prefix = ARGV[1]
+local key_delimeter = ARGV[2]
+local subscriber = ARGV[3]
+
+redis.call("persist", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
+
+local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, 0, "+inf")
+
+local i = 1
+while(i <= #subscription_ids) do
+  local subscription_id = subscription_ids[i]
+  redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), "+inf", subscriber)
+  persist_subscription(key_prefix, key_delimeter, subscription_id)
+  i = i + 1
+end

data/lib/pakyow/data/subscribers/adapters/redis/scripts/register.lua

@@ -0,0 +1,37 @@
+local key_prefix = ARGV[1]
+local key_delimeter = ARGV[2]
+local subscriber = ARGV[3]
+local subscription_id = ARGV[4]
+local subscription_string = ARGV[5]
+local source = ARGV[6]
+local now = ARGV[7]
+
+local expiry = "+inf"
+
+-- determine if the subscriber is expiring
+local ttl = redis.call("ttl", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
+
+if ttl > -1 then
+  expiry = now + ttl
+end
+
+-- store the subscription
+redis.call("set", key_subscription_id(key_prefix, key_delimeter, subscription_id), subscription_string)
+
+-- add the subscription to the subscriber's set
+redis.call("zadd", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), expiry, subscription_id)
+
+-- add the subscriber to the subscription's set
+redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), expiry, subscriber)
+
+-- add the subscription to the source's set
+redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
+
+-- define what source the subscription is for
+redis.call("set", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), source)
+
+if ttl > -1 then
+  expire_subscription(key_prefix, key_delimeter, subscription_id)
+else
+  persist_subscription(key_prefix, key_delimeter, subscription_id)
+end

data/lib/pakyow/data/subscribers/adapters/redis.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+
+require "redis"
+require "concurrent/timer_task"
+require "connection_pool"
+
+require "pakyow/support/deep_freeze"
+
+module Pakyow
+  module Data
+    class Subscribers
+      module Adapters
+        # Manages data subscriptions in redis.
+        #
+        # Use this in production.
+        #
+        # @api private
+        class Redis
+          class << self
+            def stringify_subscription(subscription)
+              Marshal.dump(subscription)
+            end
+
+            def generate_subscription_id(subscription_string)
+              Digest::SHA1.hexdigest(subscription_string)
+            end
+          end
+
+          SCRIPTS = %i(register expire persist).freeze
+          KEY_PART_SEPARATOR = "/"
+          KEY_PREFIX = "data"
+          INFINITY = "+inf"
+
+          extend Support::DeepFreeze
+          unfreezable :redis
+
+          def initialize(config)
+            @redis = ConnectionPool.new(**config[:pool]) {
+              ::Redis.new(config[:connection])
+            }
+
+            @prefix = [config[:key_prefix], KEY_PREFIX].join(KEY_PART_SEPARATOR)
+
+            @scripts = {}
+            load_scripts
+
+            Concurrent::TimerTask.new(execution_interval: 300, timeout_interval: 300) {
+              cleanup
+            }.execute
+          end
+
+          def register_subscriptions(subscriptions, subscriber:)
+            [].tap do |subscription_ids|
+              subscriptions.each do |subscription|
+                subscription_string = self.class.stringify_subscription(subscription)
+                subscription_id = self.class.generate_subscription_id(subscription_string)
+                source = subscription[:source]
+
+                @redis.with do |redis|
+                  redis.evalsha(@scripts[:register], argv: [
+                    @prefix,
+                    KEY_PART_SEPARATOR,
+                    subscriber.to_s,
+                    subscription_id,
+                    subscription_string,
+                    source.to_s,
+                    Time.now.to_i
+                  ])
+                end
+
+                subscription_ids << subscription_id
+              end
+            end
+          end
+
+          def subscriptions_for_source(source)
+            subscriptions_for_subscription_ids(subscription_ids_for_source(source)).compact
+          end
+
+          def unsubscribe(subscriber)
+            expire(subscriber, 0)
+          end
+
+          def expire(subscriber, seconds)
+            @redis.with do |redis|
+              redis.evalsha(@scripts[:expire], argv: [
+                @prefix,
+                KEY_PART_SEPARATOR,
+                subscriber.to_s,
+                Time.now.to_i + seconds
+              ])
+            end
+          end
+
+          def persist(subscriber)
+            @redis.with do |redis|
+              redis.evalsha(@scripts[:persist], argv: [
+                @prefix,
+                KEY_PART_SEPARATOR,
+                subscriber.to_s
+              ])
+            end
+          end
+
+          def expiring?(subscriber)
+            @redis.with do |redis|
+              redis.ttl(key_subscription_ids_by_subscriber(subscriber)) > -1
+            end
+          end
+
+          def subscribers_for_subscription_id(subscription_id)
+            @redis.with do |redis|
+              redis.zrangebyscore(
+                key_subscribers_by_subscription_id(
+                  subscription_id
+                ), INFINITY, INFINITY
+              ).map(&:to_sym)
+            end
+          end
+
+          def subscription_ids_for_source(source)
+            @redis.with do |redis|
+              redis.zrangebyscore(
+                key_subscription_ids_by_source(
+                  source
+                ), INFINITY, INFINITY
+              )
+            end
+          end
+
+          # FIXME: Refactor this into a lua script. We'll want to stop using SCAN and instead store
+          # known sources in a set. Cleanup should then be based off the set of known sources and
+          # return the number of values that were removed.
+          #
+          def cleanup
+            @redis.with do |redis|
+              redis.scan_each(match: key_subscription_ids_by_source("*")) do |key|
+                Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Cleaning up expired subscriptions for #{key}"
+                removed_count = redis.zremrangebyscore(key, 0, Time.now.to_i)
+                Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Removed #{removed_count} members for #{key}"
+              end
+            end
+          end
+
+          private
+
+          def subscriptions_for_subscription_ids(subscription_ids)
+            return [] if subscription_ids.empty?
+
+            @redis.with do |redis|
+              redis.mget(subscription_ids.map { |subscription_id|
+                key_subscription_id(subscription_id)
+              }).zip(subscription_ids).map { |subscription_string, subscription_id|
+                begin
+                  Marshal.restore(subscription_string).tap do |subscription|
+                    subscription[:id] = subscription_id
+                  end
+                rescue TypeError
+                  Pakyow.logger.error "could not find subscription for #{subscription_id}"
+                  {}
+                end
+              }
+            end
+          end
+
+          def build_key(*parts)
+            [@prefix].concat(parts).join(KEY_PART_SEPARATOR)
+          end
+
+          def key_subscription_id(subscription_id)
+            build_key("subscription:#{subscription_id}")
+          end
+
+          def key_subscribers_by_subscription_id(subscription_id)
+            build_key("subscription:#{subscription_id}", "subscribers")
+          end
+
+          def key_subscription_ids_by_subscriber(subscriber)
+            build_key("subscriber:#{subscriber}")
+          end
+
+          def key_subscription_ids_by_source(source)
+            build_key("source:#{source}")
+          end
+
+          def key_source_for_subscription_id(subscription_id)
+            build_key("subscription:#{subscription_id}", "source")
+          end
+
+          def load_scripts
+            @redis.with do |redis|
+              SCRIPTS.each do |script|
+                script_content = File.read(
+                  File.expand_path("../redis/scripts/_shared.lua", __FILE__)
+                ) + File.read(
+                  File.expand_path("../redis/scripts/#{script}.lua", __FILE__)
+                )
+
+                @scripts[script] = redis.script(:load, script_content)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pakyow/data/subscribers.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "concurrent/executor/thread_pool_executor"
+
+require "pakyow/support/core_refinements/method/introspection"
+require "pakyow/support/deep_freeze"
+
+module Pakyow
+  module Data
+    # @api private
+    class Subscribers
+      attr_accessor :lookup, :adapter
+
+      extend Support::DeepFreeze
+      unfreezable :executor
+
+      using Support::Refinements::Method::Introspection
+
+      def initialize(app, adapter = :memory, adapter_config = {})
+        @app = app
+
+        require "pakyow/data/subscribers/adapters/#{adapter}"
+        @adapter = Pakyow::Data::Subscribers::Adapters.const_get(
+          adapter.to_s.capitalize
+        ).new(
+          adapter_config.to_h.merge(
+            app.config.data.subscriptions.adapter_settings.to_h
+          )
+        )
+
+        @executor = Concurrent::ThreadPoolExecutor.new(
+          auto_terminate: false,
+          min_threads: 1,
+          max_threads: 10,
+          max_queue: 0
+        )
+      rescue LoadError, NameError => error
+        raise UnknownSubscriberAdapter.build(error, adapter: adapter)
+      end
+
+      def shutdown
+        @executor.shutdown
+        @executor.wait_for_termination(30)
+      end
+
+      def register_subscriptions(subscriptions, subscriber: nil, &block)
+        @executor << Proc.new {
+          subscriptions.each do |subscription|
+            subscription[:version] = @app.config.data.subscriptions.version
+          end
+
+          @adapter.register_subscriptions(subscriptions, subscriber: subscriber).tap do |ids|
+            yield ids if block_given?
+          end
+        }
+      end
+
+      def did_mutate(source_name, changed_values = nil, result_source = nil)
+        @executor << Proc.new {
+          begin
+            @adapter.subscriptions_for_source(source_name).select { |subscription|
+              process?(subscription, changed_values, result_source)
+            }.uniq.each do |subscription|
+              if subscription[:version] == @app.config.data.subscriptions.version
+                process(subscription, result_source)
+              end
+            end
+          rescue => error
+            Pakyow.logger.error "[Pakyow::Data::Subscribers] did_mutate failed: #{error}"
+          end
+        }
+      end
+
+      def unsubscribe(subscriber)
+        @adapter.unsubscribe(subscriber)
+      end
+
+      def expire(subscriber, seconds)
+        @adapter.expire(subscriber, seconds)
+      end
+
+      def persist(subscriber)
+        @adapter.persist(subscriber)
+      end
+
+      private
+
+      def process?(subscription, changed_values, result_source)
+        subscription[:handler] && qualified_subscription?(subscription, changed_values, result_source)
+      end
+
+      def process(subscription, mutated_source)
+        callback = subscription[:handler].new(@app)
+        arguments = {}
+
+        if callback.method(:call).keyword_argument?(:id)
+          arguments[:id] = subscription[:id]
+        end
+
+        if callback.method(:call).keyword_argument?(:result)
+          arguments[:result] = if subscription[:ephemeral]
+            mutated_source
+          else
+            subscription[:proxy]
+          end
+        end
+
+        if callback.method(:call).keyword_argument?(:subscription)
+          arguments[:subscription] = subscription
+        end
+
+        callback.call(subscription[:payload], **arguments)
+      end
+
+      def qualified_subscription?(subscription, changed_values, result_source)
+        if subscription[:ephemeral]
+          result_source.qualifications == subscription[:qualifications]
+        else
+          original_results = if result_source
+            result_source.original_results
+          else
+            []
+          end
+
+          qualified?(
+            subscription.delete(:qualifications).to_a,
+            changed_values,
+            result_source.to_a,
+            original_results.to_a
+          )
+        end
+      end
+
+      QUALIFIABLE_TYPES = [Hash, Support::IndifferentHash]
+      def qualified?(qualifications, changed_values, changed_results, original_results)
+        qualifications.all? do |key, value|
+          (QUALIFIABLE_TYPES.include?(changed_values.class) && changed_values.to_h[key] == value) || qualified_result?(key, value, changed_results, original_results)
+        end
+      end
+
+      def qualified_result?(key, value, changed_results, original_results)
+        original_results.concat(changed_results).any? do |result|
+          result[key] == value
+        end
+      end
+    end
+  end
+end