panko_serializer 0.8.5 → 1.0.0

data/lib/panko/code_gen/serializer_cache.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require_relative "../code_gen"
+require_relative "../config"
+require_relative "descriptor_builder"
+require_relative "instance_pool"
+
+module Panko
+  module CodeGen
+    # Per-serializer-class cache of compiled Generated Classes and their Descriptor. The engine
+    # keeps no cache between compile calls.
+    #
+    # Keyed by class identity, so a code reload (a new class object) starts empty. Reopening a
+    # live serializer or record class after its first serialize is not picked up: a variant's
+    # column-or-method choice for each source is fixed when it first sees the record class.
+    module SerializerCache
+      class Slots
+        attr_accessor :compiled, :pool, :variants
+      end
+
+      class State
+        attr_accessor :descriptor, :capacity_warned
+        attr_reader :json, :hash
+
+        def initialize
+          @json = Slots.new
+          @hash = Slots.new
+        end
+
+        def slot(output)
+          case output
+          when :json then @json
+          when :hash then @hash
+          else raise ArgumentError, "unknown output mode: #{output.inspect}"
+          end
+        end
+      end
+
+      # Guards the Descriptor and Slots writes and reset!. Reads skip it: under the GVL an ivar
+      # read sees nil or the finished value. Not reentrant, so a locked section must not lock again.
+      COMPILE_MUTEX = Mutex.new
+
+      EMPTY_VARIANTS = {}.freeze
+
+      def self.fetch(serializer_class, output:)
+        state = serializer_class._cg_state
+        compiled = state&.slot(output)&.compiled
+        return compiled if compiled
+
+        # Outside the lock: descriptor_for takes the mutex itself.
+        descriptor = descriptor_for(serializer_class)
+
+        COMPILE_MUTEX.synchronize do
+          slot = state_for!(serializer_class).slot(output)
+          slot.compiled ||= Panko::CodeGen.compile(descriptor, output: output, config: Config.new)
+        end
+      end
+
+      def self.instance_pool(serializer_class, output)
+        state = serializer_class._cg_state
+        pool = state&.slot(output)&.pool
+        return pool if pool
+
+        compiled = fetch(serializer_class, output: output)
+        descriptor = descriptor_for(serializer_class)
+
+        COMPILE_MUTEX.synchronize do
+          # Catches a +filters_for+ added some other way (e.g. +extend+) before the first
+          # serialize. ||= never downgrades a +true+ set by the hooks.
+          serializer_class._cg_has_filters_for ||= serializer_class.respond_to?(:filters_for)
+          slot = state_for!(serializer_class).slot(output)
+          slot.pool ||= InstancePool.new(
+            :"_panko_cg_pool_#{output}_#{serializer_class.object_id}",
+            compiled, descriptor
+          )
+        end
+      end
+
+      # Stores only compiled variants (bounded by capacity) and +CompileError+ base pins, so a
+      # failing compile does not repeat on every inline-cache miss. Ineligible classes, capacity
+      # overflow and other errors get the base pool with no entry, so per-call classes cannot
+      # grow the map. The compile runs outside the mutex; when two threads compile the same
+      # variant, the insert keeps one.
+      def self.variant_pool(serializer_class, output, model)
+        state = serializer_class._cg_state
+        variants = state&.slot(output)&.variants
+        pool = variants && variants[model]
+
+        unless pool
+          base = instance_pool(serializer_class, output)
+          pool, admissible = auto_variant_pool(serializer_class, output, model, base)
+          if admissible
+            COMPILE_MUTEX.synchronize do
+              # state_for!, not a bare _cg_state read: a reset! can land during the unlocked
+              # variant compile above.
+              slot = state_for!(serializer_class).slot(output)
+              current = slot.variants || EMPTY_VARIANTS
+              if (existing = current[model])
+                pool = existing
+              elsif (admitted = admit_variant(serializer_class, model, pool, base, current))
+                slot.variants = current.merge(model => admitted).freeze
+                pool = admitted
+              else
+                pool = base
+              end
+            end
+          end
+        end
+
+        remember_last(serializer_class, output, model, pool)
+        pool
+      end
+
+      def self.descriptor_for(serializer_class)
+        state = serializer_class._cg_state
+        cached = state&.descriptor
+        return cached if cached
+
+        COMPILE_MUTEX.synchronize do
+          state = state_for!(serializer_class)
+          state.descriptor ||=
+            DescriptorBuilder.uniquify_names(DescriptorBuilder.build(serializer_class))
+        end
+      end
+
+      # Clears every cache for +serializer_class+ so the next serialize rebuilds from its DSL
+      # declarations. Used by tests.
+      def self.reset!(serializer_class)
+        COMPILE_MUTEX.synchronize do
+          serializer_class._cg_state = nil
+          serializer_class._cg_last_json = nil
+          serializer_class._cg_last_hash = nil
+          serializer_class._cg_public_descriptor = nil
+          # Not nil: Serializer#descriptor reads the flag without going through instance_pool,
+          # so nil would skip +filters_for+ there.
+          serializer_class._cg_has_filters_for = serializer_class.respond_to?(:filters_for)
+        end
+      end
+
+      # False for unseen classes and for classes pinned to the base pool.
+      def self.specialized?(serializer_class, output, model)
+        state = serializer_class._cg_state
+        return false unless state
+        slot = state.slot(output)
+        pool = slot.variants && slot.variants[model]
+        return false unless pool
+        !pool.equal?(slot.pool)
+      end
+
+      # Includes classes pinned to the base pool.
+      def self.variant_models(serializer_class, output)
+        state = serializer_class._cg_state
+        variants = state&.slot(output)&.variants
+        variants ? variants.keys : []
+      end
+
+      # Callers must hold +COMPILE_MUTEX+: creating the State outside the lock could lose a
+      # concurrent writer's cells.
+      def self.state_for!(serializer_class)
+        serializer_class._cg_state ||= State.new
+      end
+      private_class_method :state_for!
+
+      # One frozen pair, so racing writers can never leave the model of one call with the pool
+      # of another. A direct class ivar keeps the hit path at one ivar read and one compare.
+      def self.remember_last(serializer_class, output, model, pool)
+        pair = [model, pool].freeze
+        case output
+        when :json then serializer_class._cg_last_json = pair
+        when :hash then serializer_class._cg_last_hash = pair
+        else raise ArgumentError, "unknown output mode: #{output.inspect}"
+        end
+      end
+      private_class_method :remember_last
+
+      # Returns +[pool, admissible]+. Errors other than +CompileError+ can be temporary (lost
+      # connection, schema not loaded), so they are not stored, and a serializer that works
+      # on the Generic path never raises here.
+      def self.auto_variant_pool(serializer_class, output, model, base)
+        return [base, false] unless auto_specialize?(model)
+        if specialized_count(serializer_class, output, base) >= Panko::Config.auto_specialization.capacity
+          warn_capacity_once(serializer_class, model)
+          return [base, false]
+        end
+
+        descriptor = DescriptorBuilder.uniquify_names(DescriptorBuilder.specialize(descriptor_for(serializer_class), model))
+        compiled = Panko::CodeGen.compile(descriptor, output: output, config: Config.new(guarded_model: true))
+        pool = InstancePool.new(
+          :"_panko_cg_pool_#{output}_#{serializer_class.object_id}_#{model.object_id}",
+          compiled, descriptor
+        )
+        [pool, true]
+      rescue CompileError
+        [base, true]
+      rescue
+        [base, false]
+      end
+      private_class_method :auto_variant_pool
+
+      def self.auto_specialize?(model)
+        Panko::Config.auto_specialization.enabled &&
+          model.respond_to?(:columns_hash) &&
+          model.respond_to?(:attribute_methods_generated?) &&
+          DescriptorBuilder.resolvable_name?(model)
+      end
+      private_class_method :auto_specialize?
+
+      # Pinned entries share the base pool object, so only non-base entries count.
+      def self.specialized_count(serializer_class, output, base)
+        state = serializer_class._cg_state
+        variants = state&.slot(output)&.variants || EMPTY_VARIANTS
+        variants.count { |_, pool| !pool.equal?(base) }
+      end
+      private_class_method :specialized_count
+
+      # Runs under +COMPILE_MUTEX+ and checks capacity again, since the check in
+      # {auto_variant_pool} ran without the lock. +nil+ means over capacity.
+      def self.admit_variant(serializer_class, model, candidate, base, current)
+        return candidate if candidate.equal?(base)
+        return candidate if current.count { |_, pool| !pool.equal?(base) } < Panko::Config.auto_specialization.capacity
+
+        warn_capacity_once(serializer_class, model)
+        nil
+      end
+      private_class_method :admit_variant
+
+      # Reads the State without creating it: both callers run after instance_pool built it, and
+      # state_for! needs +COMPILE_MUTEX+, which one caller already holds. The flag write can race;
+      # at worst the warning prints more than once.
+      def self.warn_capacity_once(serializer_class, model)
+        state = serializer_class._cg_state
+        return if state.nil? || state.capacity_warned
+        state.capacity_warned = true
+        warn "#{serializer_class} auto-specialization capacity " \
+          "(#{Panko::Config.auto_specialization.capacity}) reached at #{model}; " \
+          "further record classes use the generic path. Raise " \
+          "Panko::Config.auto_specialization.capacity if this is intentional."
+      end
+      private_class_method :warn_capacity_once
+    end
+  end
+end

data/lib/panko/code_gen/validators/callable_arity.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module Validators
+    # Raises ArityError when a MethodAttribute body or an Association +if:+
+    # has an arity outside 0..3.
+    module CallableArity
+      # The generator's call shapes cover only 0..3; it treats any other arity as 3.
+      ALLOWED_ARITIES = 0..3
+
+      # Walks by object identity, so a shared or self-recursive Descriptor is checked once.
+      def self.validate(descriptor, output:, config:)
+        walk(descriptor, {})
+        nil
+      end
+
+      class << self
+        private
+
+        # A Symbol body names a method and has no arity to check.
+        def walk(descriptor, seen)
+          return if seen[descriptor.__id__]
+          seen[descriptor.__id__] = true
+          descriptor.method_attributes.each do |ma|
+            next if ma.body.is_a?(Symbol)
+            check_arity!(descriptor.name, ma.name, "MethodAttribute#body", ma.body.arity)
+          end
+          descriptor.associations.each do |assoc|
+            check_arity!(descriptor.name, assoc.name, "Association#if", assoc.if.arity) if assoc.if
+            assoc.descriptors.each { |child| walk(child, seen) }
+          end
+        end
+
+        def check_arity!(descriptor_name, field_name, callable_label, arity)
+          return if ALLOWED_ARITIES.include?(arity)
+          raise ArityError,
+            "#{descriptor_name}##{field_name}: #{callable_label} has arity #{arity}; must be 0, 1, 2, or 3."
+        end
+      end
+    end
+  end
+end

data/lib/panko/code_gen/validators/name_uniqueness.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module Validators
+    # Raises NameCollisionError when two Fields at the same level share a name:
+    # each Field writes its output under its name. The same name at different levels is fine.
+    module NameUniqueness
+      def self.validate(descriptor, output:, config:)
+        walk(descriptor, {})
+        nil
+      end
+
+      class << self
+        private
+
+        def walk(descriptor, seen_descriptors)
+          return if seen_descriptors[descriptor.__id__]
+          seen_descriptors[descriptor.__id__] = true
+          check_level!(descriptor)
+          descriptor.associations.each { |assoc| assoc.descriptors.each { |child| walk(child, seen_descriptors) } }
+        end
+
+        def check_level!(descriptor)
+          seen_names = {}
+          fields_with_kinds(descriptor).each do |field, kind|
+            previous_kind = seen_names[field.name]
+            if previous_kind
+              raise NameCollisionError,
+                "#{descriptor.name}##{field.name}: #{previous_kind} and #{kind} share name; " \
+                "every Field at the same level must have a unique name."
+            end
+            seen_names[field.name] = kind
+          end
+        end
+
+        def fields_with_kinds(descriptor)
+          descriptor.attributes.map { |a| [a, "Attribute"] } +
+            descriptor.method_attributes.map { |m| [m, "MethodAttribute"] } +
+            descriptor.associations.map { |a| [a, "Association"] }
+        end
+      end
+    end
+  end
+end

data/lib/panko/code_gen/validators/source_resolution.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module Validators
+    # Raises UnknownSourceError when an Attribute source on an Active Record +model:+
+    # is neither a column nor an instance method. Other models are not checked.
+    module SourceResolution
+      def self.validate(descriptor, output:, config:)
+        walk(descriptor, {})
+        nil
+      end
+
+      class << self
+        private
+
+        def walk(descriptor, seen)
+          return if seen[descriptor.__id__]
+          seen[descriptor.__id__] = true
+          classify_attributes!(descriptor) if descriptor.model
+          descriptor.associations.each { |assoc| assoc.descriptors.each { |child| walk(child, seen) } }
+        end
+
+        def classify_attributes!(descriptor)
+          model = descriptor.model
+          return unless ar_class?(model)
+          ActiveRecord::DefineAttributeMethods.ensure!(model)
+          descriptor.attributes.each do |attribute|
+            classify_or_raise!(model, attribute, descriptor.name)
+          end
+        end
+
+        # The classifier knows only the class and the source; add the Descriptor and Field names.
+        def classify_or_raise!(model, attribute, descriptor_name)
+          ActiveRecord::AccessClassifier.classify(model, attribute.source)
+          nil
+        rescue UnknownSourceError
+          raise UnknownSourceError,
+            "#{descriptor_name}##{attribute.name}: Attribute#source :#{attribute.source} " \
+            "is not a column or instance method on #{model.name}."
+        end
+
+        def ar_class?(klass)
+          klass.respond_to?(:columns_hash) && klass.respond_to?(:attribute_methods_generated?)
+        end
+      end
+    end
+  end
+end

data/lib/panko/code_gen/validators/validator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module Validators
+    class Validator
+      # Order matters: an arity error is reported before a source or name error.
+      DEFAULT_RULES = [CallableArity, SourceResolution, NameUniqueness].freeze
+
+      def initialize(rules: DEFAULT_RULES)
+        @rules = rules
+      end
+
+      def validate(descriptor, output:, config:)
+        @rules.each { |rule| rule.validate(descriptor, output: output, config: config) }
+        nil
+      end
+    end
+  end
+end

data/lib/panko/code_gen/writers_pool.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "oj"
+require_relative "../config"
+
+module Panko::CodeGen
+  # A stack, not one writer per fiber: a nested serialize call (from a method attribute)
+  # pops a different writer, or creates one when the stack is empty.
+  # +Oj::StringWriter#reset+ keeps the grown buffer, so +checkin+ drops a writer
+  # whose output was larger than +Panko::Config.writer_pool_max_bytes+.
+  class WritersPool
+    STORAGE_KEY = :_panko_writers
+
+    def checkout
+      storage.pop || Oj::StringWriter.new(mode: :rails)
+    end
+
+    # A +nil+ +result+ means the call raised; the writer is dropped then too.
+    def checkin(writer, result)
+      return if result.nil? || result.bytesize > Panko::Config.writer_pool_max_bytes
+
+      writer.reset
+      storage.push(writer)
+    end
+
+    private
+
+    def storage
+      raise NotImplementedError, "#{self.class} must override #storage"
+    end
+
+    # +Thread.current[]+ is fiber-local in MRI. Needs no Rails.
+    class ThreadLocal < WritersPool
+      private
+
+      def storage
+        Thread.current[STORAGE_KEY] ||= []
+      end
+    end
+
+    # Follows ActiveRecord's connection pool: per thread, or per fiber when
+    # +ActiveSupport::IsolatedExecutionState.isolation_level+ is +:fiber+.
+    class IsolatedExecutionState < WritersPool
+      private
+
+      def storage
+        ActiveSupport::IsolatedExecutionState[STORAGE_KEY] ||= []
+      end
+    end
+  end
+end

data/lib/panko/code_gen.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "oj"
+
+module Panko
+  module CodeGen
+    # Shared by the generated file banner and the synthetic backtrace path.
+    GENERATOR_NAME = "Panko::CodeGen"
+  end
+end
+
+require_relative "code_gen/errors"
+require_relative "code_gen/config"
+require_relative "code_gen/descriptor"
+require_relative "code_gen/code_builder"
+require_relative "code_gen/filter"
+require_relative "code_gen/datetime_format"
+require_relative "code_gen/active_record/access_classifier"
+require_relative "code_gen/active_record/define_attribute_methods"
+require_relative "code_gen/validators/callable_arity"
+require_relative "code_gen/validators/source_resolution"
+require_relative "code_gen/validators/name_uniqueness"
+require_relative "code_gen/validators/validator"
+require_relative "code_gen/generators/generated_names"
+require_relative "code_gen/generators/sink"
+require_relative "code_gen/generators/json_sink"
+require_relative "code_gen/generators/hash_sink"
+require_relative "code_gen/generators/field_walk"
+require_relative "code_gen/generators/record_access/generic"
+require_relative "code_gen/generators/record_access/specialized"
+require_relative "code_gen/generators/cycle_membership"
+require_relative "code_gen/generators/descriptor_walk"
+require_relative "code_gen/generators/field_index"
+require_relative "code_gen/generators/release"
+require_relative "code_gen/generators/banner"
+require_relative "code_gen/generators/class_emitter"
+require_relative "code_gen/generators/fanout"
+require_relative "code_gen/generator"
+require_relative "code_gen/compile_cache"
+require_relative "code_gen/compiler"
+require_relative "code_gen/dump"
+require_relative "code_gen/writers_pool"
+
+# Internal code generator: turns an immutable Descriptor into a class that
+# emits JSON or a Ruby Hash.
+module Panko::CodeGen
+  # Responds to no Oj callback, so +Oj.sc_parse+ only checks the JSON is valid
+  # without building any Ruby objects.
+  JSON_NOOP_PARSER = Object.new.freeze
+
+  # Passed positionally: a +mode: :strict+ keyword would allocate a Hash per record.
+  JSON_STRICT_PARSE_OPTS = {mode: :strict}.freeze
+
+  # Returns a new Generated Class on every call. Raises CompileError when +descriptor+
+  # cannot be compiled, ArgumentError when +output:+ is not +:json+ or +:hash+.
+  def self.compile(descriptor, output:, config: Config.new)
+    Compiler.new(descriptor, output: output, config: config).compile
+  end
+
+  # Writes one file per Descriptor (root at +path:+, children beside it) and returns
+  # +path:+. Accepts a public +Panko::Descriptor+ too.
+  def self.dump(descriptor, output:, path:, config: Config.new)
+    # +defined?+ guard: the engine also loads standalone, without +Panko::Descriptor+.
+    if defined?(::Panko::Descriptor) && descriptor.is_a?(::Panko::Descriptor)
+      descriptor = SerializerCache.descriptor_for(descriptor.serializer)
+    end
+    Dump.new(descriptor, output: output, config: config, path: path).dump
+  end
+
+  # Hash mode only: converts a value the way Oj +:rails+ mode does in JSON mode (+#as_json+,
+  # non-finite Float to nil). Without ActiveSupport there is no +#as_json+, so the value passes raw.
+  def self.cast_datetime(value)
+    # Runs on many Hash-mode field writes, so types whose +#as_json+ returns self skip the
+    # call. Symbol is not one of them: its +#as_json+ is a String.
+    case value
+    when String, Integer, NilClass, TrueClass, FalseClass then value
+    when Float then value.finite? ? value : nil
+    else
+      value.respond_to?(:as_json) ? value.as_json : value
+    end
+  end
+end

data/lib/panko/compile_all.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative "serializer"
+require_relative "compile_all_result"
+require_relative "code_gen/serializer_cache"
+
+module Panko
+  COMPILE_ALL_MODES = %i[json hash].freeze
+
+  # Compiles every loaded Panko::Serializer subclass that declares a field:
+  # the base for each mode, plus a specialized variant per {Serializer.models}
+  # model. Call it at boot, after eager load and before forking workers.
+  # Run warm-up calls after it: a compile defines constants, which drops YJIT
+  # code that reads the same constant names. Safe to repeat.
+  #
+  # @param modes [Array<Symbol>] a non-empty subset of +[:json, :hash]+
+  # @return [Panko::CompileAllResult]
+  # @raise [ArgumentError] when +modes+ is empty or holds an unknown mode
+  def self.compile_all(modes: COMPILE_ALL_MODES)
+    if modes.empty? || !(modes - COMPILE_ALL_MODES).empty?
+      raise ArgumentError, "compile_all modes: expected a non-empty subset of #{COMPILE_ALL_MODES.inspect}, got #{modes.inspect}"
+    end
+
+    compiled = []
+    without_models = []
+    not_specialized = {}
+    errors = {}
+    cache = CodeGen::SerializerCache
+
+    compile_all_serializers.each do |serializer|
+      models = serializer._cg_models
+      missed = []
+      begin
+        modes.each do |mode|
+          cache.instance_pool(serializer, mode)
+          models.each do |model|
+            cache.variant_pool(serializer, mode, model)
+            missed << model unless cache.specialized?(serializer, mode, model)
+          end
+        end
+      rescue => error
+        errors[serializer] = error
+        next
+      end
+
+      if models.empty?
+        without_models << serializer
+      elsif missed.empty?
+        compiled << serializer
+      else
+        not_specialized[serializer] = missed.uniq
+      end
+    end
+
+    CompileAllResult.new(
+      compiled: compiled.freeze,
+      without_models: without_models.freeze,
+      not_specialized: not_specialized.freeze,
+      errors: errors.freeze
+    )
+  end
+
+  # Collected before any compile, so the Generated Classes it adds are not
+  # visited. They subclass the user serializer; only they define +serialize_one+.
+  def self.compile_all_serializers
+    found = []
+    pending = Panko::Serializer.subclasses
+    until pending.empty?
+      serializer = pending.shift
+      next if serializer.method_defined?(:serialize_one)
+      pending.concat(serializer.subclasses)
+      next if serializer._cg_attributes.empty? &&
+        serializer._cg_method_attributes.empty? &&
+        serializer._cg_associations.empty?
+      found << serializer
+    end
+    found
+  end
+  private_class_method :compile_all_serializers
+end

data/lib/panko/compile_all_result.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Panko
+  # What one {Panko.compile_all} call compiled. Every serializer it visited
+  # is in exactly one of the four fields.
+  #
+  # @!attribute [r] compiled
+  #   @return [Array<Class>] serializers with the base and a specialized
+  #     variant for every declared model, in every requested mode
+  # @!attribute [r] without_models
+  #   @return [Array<Class>] serializers that declare no {Serializer.models}:
+  #     only the base is compiled; ActiveRecord record classes can still
+  #     specialize on first serialize
+  # @!attribute [r] not_specialized
+  #   @return [Hash{Class => Array<Class>}] serializer to the declared models
+  #     that got no specialized variant
+  # @!attribute [r] errors
+  #   @return [Hash{Class => StandardError}] serializers whose base compile
+  #     raised; their first serialize compiles again
+  CompileAllResult = Data.define(:compiled, :without_models, :not_specialized, :errors)
+end