simple_master 1.0.0

data/lib/simple_master/storage/dataset.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "simple_master/storage/table"
+require "simple_master/storage/test_table"
+
+module SimpleMaster
+  module Storage
+    class Dataset
+      require "objspace"
+
+      class << self
+        def after_load_procs
+          @after_load_procs ||= []
+        end
+
+        def after_load(&proc)
+          after_load_procs << proc
+        end
+
+        def run_after_load
+          after_load_procs.each(&:call)
+        end
+      end
+
+      DEFAULT_TABLE = Table
+      DEFAULT_LOADER = Loader::QueryLoader
+
+      attr_reader :table_class
+      attr_reader :loader
+      attr_reader :diff
+      attr_accessor :tables
+      attr_accessor :cache
+      attr_accessor :load_targets
+
+      def initialize(table_class: DEFAULT_TABLE, loader: nil)
+        @table_class = table_class
+        @loader = loader || DEFAULT_LOADER.new
+        @diff = {}
+
+        initialize_cache
+      end
+
+      def initialize_cache
+        self.tables = Hash.new { |hash, klass| hash[klass] = table_class.new(klass, self, loader) }.compare_by_identity
+        self.cache = {}
+      end
+
+      def table(klass)
+        @tables[klass]
+      end
+
+      def reload
+        if table_class <= SimpleMaster::Storage::OndemandTable
+          unload
+        else
+          load
+        end
+      end
+
+      def load
+        memsize do
+          cache.clear
+          targets = @load_targets || SimpleMaster.targets
+
+          tables = targets.map(&:base_class).uniq.map { table(_1) }
+
+          timer("MasterData load") do
+            tables.each(&:load_records)
+          end
+
+          timer("MasterData cache update") do
+            SimpleMaster.use_dataset(self) do
+              tables.each(&:update_class_method_cache)
+              tables.each(&:tap_instance_methods)
+              tables.each(&:freeze_all)
+            end
+          end
+
+          timer("Cache update") do
+            SimpleMaster.use_dataset(self) do
+              self.class.run_after_load
+            end
+          end
+        end
+        self
+      end
+
+      def unload
+        cache.clear
+        tables.clear
+      end
+
+      # NOTE: Pass a empty hash to duplicate with empty diff.
+      def duplicate(diff: nil)
+        diff ||= @diff
+        new_dataset = self.class.new(table_class: table_class, loader: loader)
+        new_dataset.diff = diff.deep_dup
+        tables.each do |klass, table|
+          new_dataset.tables[klass] = table.duplicate_for(new_dataset)
+        end
+
+        new_dataset
+      end
+
+      def reload_klass(klass)
+        fail NotImplementedError unless table_class == SimpleMaster::Storage::OndemandTable
+
+        tables.delete(klass)
+      end
+      alias reload_class reload_klass
+
+      # Cache helper for other data sources.
+      def cache_read(key)
+        cache[key]
+      end
+
+      def cache_fetch(key)
+        cache.fetch(key) do
+          cache[key] = yield
+        end
+      end
+
+      def cache_write(key, value)
+        cache[key] = value
+      end
+
+      def cache_delete(key)
+        cache.delete(key)
+      end
+
+      # NOTE: set diff before records loaded.
+      def diff=(diff_json)
+        @diff = if diff_json.is_a?(String)
+                  JSON.parse(diff_json)
+                else
+                  diff_json || {}
+                end
+      end
+
+      private
+
+      def memsize
+        GC.compact
+        before = ObjectSpace.memsize_of_all
+
+        res = yield
+
+        GC.compact
+        after = ObjectSpace.memsize_of_all
+
+        SimpleMaster.logger.info { "Consumed memory size: #{after - before}" }
+
+        res
+      end
+
+      def timer(text)
+        t1 = Time.zone.now
+        @timer_index ||= 0
+        @timer_index += 1
+
+        res = yield
+
+        t2 = Time.zone.now
+        @timer_index -= 1
+
+        SimpleMaster.logger.info { "#{'  ' * @timer_index}#{text}: #{t2 - t1}s" }
+
+        res
+      end
+    end
+  end
+end

data/lib/simple_master/storage/ondemand_table.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "simple_master/storage/table"
+
+module SimpleMaster
+  module Storage
+    class OndemandTable < Table
+      def sub_table(sub_klass)
+        (@sub_tables ||= klass.descendants.reject(&:abstract_class).index_with { |k| self.class.new(k, dataset, loader) })[sub_klass]
+      end
+
+      def load_records
+        @class_method_cache = nil
+        super
+      end
+
+      def all
+        return @all if @all
+
+        if klass.sti_sub_class?
+          klass.sti_base_class.all
+          @all
+        else
+          load_records
+          SimpleMaster.use_dataset(dataset) do
+            tap_instance_methods
+            freeze_all
+          end
+        end
+      end
+
+      def id_hash
+        return @id_hash if @id_hash
+
+        update_id_hash
+      end
+
+      def grouped_hash
+        return @grouped_hash if @grouped_hash
+
+        update_grouped_hash
+      end
+
+      def class_method_cache
+        return @class_method_cache if @class_method_cache
+
+        # make sure @all is loaded to prevent errors
+        all
+
+        update_class_method_cache
+      end
+
+      def update_sub_tables
+        sub_klasses = klass.descendants.reject(&:abstract_class)
+        return if sub_klasses.empty?
+
+        grouped = all.group_by(&:class)
+        sub_klasses.each do |sub_klass|
+          sub_sub_klasses = [sub_klass, *sub_klass.descendants].reject(&:abstract_class)
+
+          sub_table = self.sub_table(sub_klass)
+          sub_table.all = sub_sub_klasses.flat_map { grouped[_1] || EMPTY_ARRAY }.freeze
+          sub_table.digest = digest
+        end
+      end
+    end
+  end
+end

data/lib/simple_master/storage/table.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module SimpleMaster
+  module Storage
+    class Table
+      METADATA_PREFIX = "__"
+
+      def initialize(klass, dataset, loader)
+        @klass = klass
+        @dataset = dataset
+        @loader = loader
+        @method_cache = Hash.new { |k, v| k[v] = {}.compare_by_identity }.compare_by_identity
+
+        @sub_tables = nil
+      end
+
+      def sub_table(klass)
+        @sub_tables[klass]
+      end
+
+      def duplicate_for(dataset, table_klass = self.class)
+        table = table_klass.new(klass, dataset, loader)
+
+        if table.diff == applied_diff
+          table.all = all
+          table.applied_diff = applied_diff
+          table.digest = digest
+
+          # If already loaded, copy existing id_hash and grouped_hash
+          table.id_hash = id_hash if @id_hash
+          table.grouped_hash = grouped_hash if @grouped_hash
+          table.sub_tables = sub_tables&.transform_values { |sub_table|
+            sub_table.duplicate_for(dataset)
+          }
+        end
+
+        table
+      end
+
+      def load_records
+        @method_cache.clear
+
+        loader.load_records(self)
+      end
+
+      attr_accessor :klass
+      attr_accessor :dataset
+      attr_accessor :loader
+      attr_accessor :sub_tables
+      attr_accessor :all
+      attr_accessor :id_hash
+      attr_accessor :grouped_hash
+      attr_accessor :class_method_cache
+      attr_accessor :method_cache
+      attr_accessor :applied_diff
+      attr_accessor :digest
+
+      def freeze_all
+        all.each(&:freeze).tap { run_on_sub_tables(:freeze_all) }
+      end
+
+      def update_id_hash
+        self.id_hash = all.index_by(&:id).freeze
+      end
+
+      def update_grouped_hash
+        grouped_hash = {}.compare_by_identity
+
+        klass.group_keys.each do |group_key|
+          grouped_hash[group_key] = all.group_by(&group_key).freeze.each_value(&:freeze)
+        end
+        grouped_hash.freeze
+
+        self.grouped_hash = grouped_hash
+      end
+
+      def update_class_method_cache
+        self.class_method_cache = class_method_cache = {}.compare_by_identity
+        # Use instance_eval so a block runs in the declaring class context
+        klass.all_class_method_cache_info.each do |args, initializer|
+          result = klass.instance_eval(&initializer)
+          if args.length == 1
+            result = [result]
+          end
+
+          args.zip(result).each do |arg, value|
+            class_method_cache[arg] = value
+          end
+        end
+        class_method_cache.freeze.tap {
+          run_on_sub_tables(:update_class_method_cache)
+        }
+      end
+
+      def tap_instance_methods
+        klass.instance_methods_need_tap&.each do |method_name|
+          all.each(&method_name)
+        end
+        run_on_sub_tables(:tap_instance_methods)
+      end
+
+      def update_sub_tables
+        sub_klasses = klass.descendants.reject(&:abstract_class)
+        return if sub_klasses.empty?
+
+        @sub_tables = sub_klasses.index_with { |sub_klass| self.class.new(sub_klass, dataset, loader) }
+
+        grouped = all.group_by(&:class)
+
+        sub_klasses.each do |sub_klass|
+          sub_sub_klasses = [sub_klass, *sub_klass.descendants].reject(&:abstract_class)
+
+          sub_table = self.sub_table(sub_klass)
+          sub_table.all = sub_sub_klasses.flat_map { grouped[_1] || EMPTY_ARRAY }.freeze
+          sub_table.digest = digest
+          sub_table.applied_diff = applied_diff
+          sub_table.update_id_hash
+          sub_table.update_grouped_hash
+        end
+      end
+
+      def run_on_sub_tables(method)
+        sub_tables&.each_value(&method)
+      end
+
+      def diff
+        dataset.diff[klass.table_name]
+      end
+
+      def apply_diff
+        update_id_hash
+        if diff.present?
+          apply_diff_to_id_hash
+          self.all = id_hash.values.freeze
+        end
+        update_grouped_hash
+
+        self
+      end
+
+      def apply_diff_to_id_hash
+        id_hash = self.id_hash.dup
+        diff&.each do |key, record_diff|
+          next if metadata_key?(key)
+          id = key.to_i
+
+          if record_diff.nil?
+            id_hash.delete(id)
+            next
+          end
+
+          original_record = id_hash[id]
+
+          record =
+            if original_record.nil?
+              new_klass = klass.sti_column && record_diff[klass.sti_column.to_s]&.constantize || klass
+              new_klass.new(id: id)
+            elsif klass.sti_column && record_diff[klass.sti_column.to_s]
+              record_diff[klass.sti_column.to_s].constantize.new(original_record.attributes)
+            else
+              original_record.create_copy
+            end
+
+          record_diff.each do |k, v|
+            next if metadata_key?(k)
+            # Handle expanded JSON columns by converting keys back to JSON strings before assignment
+            if v.is_a?(Hash) || v.is_a?(Array)
+              v = v.to_json
+            end
+            record.send(:"#{k}=", v)
+          rescue NoMethodError => _e
+            raise ColumnNotExist, "Column #{k} does not exist on #{klass}."
+          rescue => e
+            raise AssignmentError, "Failed to assign data: #{klass}.#{k} = #{v.inspect}. Message: #{e.message}."
+          end
+
+          id_hash[id] = record
+        end
+
+        self.applied_diff = diff
+        self.id_hash = id_hash.freeze
+      end
+
+      class ColumnNotExist < StandardError
+      end
+
+      class AssignmentError < StandardError
+      end
+
+      private
+
+      def metadata_key?(key)
+        key.to_s.start_with?(METADATA_PREFIX)
+      end
+    end
+  end
+end

data/lib/simple_master/storage/test_table.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "simple_master/storage/table"
+
+module SimpleMaster
+  module Storage
+    # Ondemand table based on updated id_hash.
+    class TestTable < Table
+      def initialize(_klass, _dataset, _loader)
+        super
+
+        @all = []
+        @id_hash = {}
+        @grouped_hash = {}
+        @class_method_cache = {}
+        @method_cache = Hash.new { |k, v| k[v] = {}.compare_by_identity }.compare_by_identity
+
+        @all_need_update = true
+        @grouped_hash_need_update = true
+        @class_method_cache_need_update = true
+      end
+
+      def sub_table(sub_klass)
+        (@sub_tables ||= klass.descendants.reject(&:abstract_class).index_with { |k| self.class.new(k, dataset, loader) })[sub_klass]
+      end
+
+      def update(id, record)
+        id_hash[id] = record
+        @all_need_update = true
+        @grouped_hash_need_update = true
+        @class_method_cache_need_update = true
+      end
+
+      def record_updated
+        @grouped_hash_need_update = true
+        @class_method_cache_need_update = true
+      end
+
+      def all=(records)
+        @all_need_update = false
+        super.tap { update_id_hash }
+      end
+
+      def all
+        if @all_need_update
+          @all_need_update = false
+          @all = id_hash.values
+        end
+        @all
+      end
+
+      def grouped_hash
+        if @grouped_hash_need_update
+          @grouped_hash_need_update = false
+          update_grouped_hash
+        end
+        @grouped_hash
+      end
+
+      def class_method_cache
+        if @class_method_cache_need_update
+          @class_method_cache_need_update = false
+          update_class_method_cache
+        end
+        @class_method_cache
+      end
+    end
+  end
+end

data/lib/simple_master/storage.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SimpleMaster
+  module Storage
+  end
+end
+
+require "simple_master/storage/table"
+require "simple_master/storage/ondemand_table"
+require "simple_master/storage/test_table"
+require "simple_master/storage/dataset"

data/lib/simple_master/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SimpleMaster
+  VERSION = "1.0.0"
+end

data/lib/simple_master.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "active_support/dependencies"
+require "active_record"
+require "request_store"
+require "logger"
+require "simple_master/version"
+
+module SimpleMaster
+  EMPTY_ARRAY = [].freeze
+  EMPTY_HASH = {}.freeze
+
+  def self.logger
+    if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      Rails.logger
+    else
+      @logger ||= Logger.new($stdout)
+    end
+  end
+
+  def self.init(for_test: false)
+    is_database_available = database_available?
+    unless is_database_available
+      SimpleMaster.logger.warn "DB not connected. SimpleMaster will not initialize associations to ActiveRecord."
+    end
+
+    yield if block_given?
+
+    targets.each { |klass| klass.init(is_database_available, for_test: for_test) }
+  end
+
+  def self.targets
+    Master.descendants.reject(&:abstract_class)
+  end
+
+  def self.database_available?
+    # Raises an error if the DB is missing
+    ::ActiveRecord::Base.connection.verify!
+
+    true
+  rescue
+    false
+  end
+
+  def self.use_dataset(dataset)
+    former_dataset = $current_dataset
+    $current_dataset = dataset
+    yield
+  ensure
+    $current_dataset = former_dataset
+  end
+end
+
+require "simple_master/active_record"
+require "simple_master/loader"
+require "simple_master/schema"
+require "simple_master/storage"
+require "simple_master/master"
+
+class ActiveRecord::Associations::Preloader::Association
+  prepend SimpleMaster::ActiveRecord::PreloaderAssociationExtension
+end