panko_serializer 0.8.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f713e7a7ea669bf0f3fa26ed2780c3f84a56a0faa0013f5514f6e732c49c86c3
-  data.tar.gz: b3eace69707162028e2ea379687cd7e3ee0d1bf8b2d1e68dda0402df3da9166e
+  metadata.gz: 933667f4549f885acffbbd822422f568a0b5ba737b05bc2f333299e9d2ba3085
+  data.tar.gz: 20f6302ef51561dce29e0608ad2a22334c0d8625f62cf815d7d31fb9a30e49f6
 SHA512:
-  metadata.gz: 0a12aa0fd1d42cd6b19867c87e278f5d0df045b25f2f93580a6c7e939d44d5859f7f3b983655d828cacafd20574e4278b620c89eabbccfdf387f17bc201dfcad
-  data.tar.gz: 5f0d4103ab1382af51d22278006a938bc013829878a709bded8e0564d87ee5f2d945ef3d93c5eadf9c9ee8d309adf416d1b6bea90c72ffac729af6a5b62e9ffe
+  metadata.gz: a520312057f8f41687f6051faff37eb2061ca2b4d4939d54631b52250e120277c920b455480ca0977d4112a7f80200b960948c85895377574f0b324c1be976ac
+  data.tar.gz: '0779d69d34164e6d6fd361909bc1a4395c047f80190f61753dbcff817bef8ddd415278dfccd863bfff53ff0d5558dce6b6137d8c47e3cd8a989ec7cac9860761'

data/README.md

@@ -4,14 +4,16 @@
 
 Panko is a library which is inspired by ActiveModelSerializers 0.9 for serializing ActiveRecord/Ruby objects to JSON strings, fast.
 
-To achieve its [performance](https://panko.dev/performance):
+Panko is pure Ruby: no native extension, nothing to compile at install. It needs Ruby 3.4+ and ActiveSupport 7.2+. To achieve its [performance](https://panko.dev/performance):
 
-* Oj - Panko relies on Oj since it's fast and allows for incremental serialization using `Oj::StringWriter`
-* Serialization Descriptor - Panko computes most of the metadata ahead of time, to save time later in serialization.
-* Type casting — Panko does type casting by itself, instead of relying on ActiveRecord.
+* Code generation: each serializer is compiled once into specialized, straight-line Ruby, so serialization runs no per-record introspection and YJIT optimizes it well.
+* Auto-specialization: the first time a serializer meets an ActiveRecord class, Panko compiles a variant hard-wired to that model, reading values straight from its attribute storage.
+* Oj: JSON is written incrementally using `Oj::StringWriter`.
 
 To dig deeper about the performance choices, read [Design Choices](https://panko.dev/design-choices).
 
+Coming from 0.8 (the C extension)? Read [Upgrading from 0.8](https://panko.dev/upgrading).
+
 
 Support
 -------

data/lib/panko/array_serializer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "code_gen/runtime"
+
 module Panko
   class ArraySerializer
     attr_accessor :subjects
@@ -15,38 +17,75 @@ Please pass valid each_serializer to ArraySerializer, for example:
         }
       end
 
-      serializer_options = {
-        only: options.fetch(:only, []),
-        except: options.fetch(:except, []),
-        context: options[:context],
-        scope: options[:scope]
-      }
-
-      @serialization_context = SerializationContext.create(options)
-      @descriptor = Panko::SerializationDescriptor.build(@each_serializer, serializer_options, @serialization_context)
+      @context = options[:context]
+      @scope = options[:scope]
+      @only = options[:only]
+      @except = options[:except]
     end
 
     def to_json
-      serialize_to_json @subjects
+      serialize_to_json(@subjects)
+    end
+
+    # Same contract as Panko::Serializer#descriptor.
+    def descriptor
+      each_serializer = @each_serializer
+      filters = if @only || @except || each_serializer._cg_has_filters_for
+        Panko::CodeGen::Runtime.runtime_filters(each_serializer, @context, @scope, @only, @except)
+      end
+      base = Panko::Descriptor.for(each_serializer)
+      filters ? Panko::Descriptor::Filtered.new(base, filters) : base
     end
 
     def serialize(subjects)
-      serialize_with_writer(subjects, Panko::ObjectWriter.new).output
+      serialize_batch(subjects, :hash) do |instance, records, filters|
+        instance.serialize_many(records, context: @context, scope: @scope, filters: filters)
+      end
     end
 
     def to_a
-      serialize_with_writer(@subjects, Panko::ObjectWriter.new).output
+      serialize(@subjects)
     end
 
     def serialize_to_json(subjects)
-      serialize_with_writer(subjects, Oj::StringWriter.new(mode: :rails)).to_s
+      serialize_batch(subjects, :json) do |instance, records, filters|
+        instance.serialize_many(records, context: @context, scope: @scope, filters: filters)
+      end
+    end
+
+    # Writes the array into +writer+ (an +Oj::StringWriter+) as the value of +key+, or as the
+    # next element when +key+ is nil. The bytes match what +to_json+ returns.
+    def write_json(writer, key = nil)
+      serialize_batch(@subjects, :json) do |instance, records, filters|
+        instance.serialize_many_to(writer, records, key, context: @context, scope: @scope, filters: filters)
+      end
     end
 
     private
 
-    def serialize_with_writer(subjects, writer)
-      Panko.serialize_objects(subjects.to_a, writer, @descriptor)
-      writer
+    # The pool is picked by the first record's class. A mixed-class array is
+    # safe: a specialized variant hands records of another class to its generic twin.
+    def serialize_batch(subjects, mode)
+      each_serializer = @each_serializer
+      records = subjects.to_a
+      model = records.first.class
+      cached = (mode == :hash) ? each_serializer._cg_last_hash : each_serializer._cg_last_json
+      pool = if cached && model.equal?(cached[0])
+        cached[1]
+      else
+        Panko::CodeGen::SerializerCache.variant_pool(each_serializer, mode, model)
+      end
+      filters = if @only || @except || each_serializer._cg_has_filters_for
+        Panko::CodeGen::Runtime.runtime_filters(each_serializer, @context, @scope, @only, @except)
+      end
+      stack = pool.stack
+      instance = stack.pop || pool.build
+      begin
+        yield instance, records, filters
+      ensure
+        instance._release
+        stack.push(instance)
+      end
     end
   end
 end

data/lib/panko/code_gen/active_record/access_classifier.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module ActiveRecord
+    # Columns read through AR's own generated reader classify as +:column+.
+    # A user override of a column reader classifies as +:method+, so the
+    # Specialized path never bypasses it. Call +DefineAttributeMethods.ensure!+
+    # first, or non-column readers (+attribute+, +alias_attribute+) raise +UnknownSourceError+.
+    module AccessClassifier
+      def self.classify(klass, source)
+        if klass.columns_hash.key?(source.to_s)
+          return user_override?(klass, source) ? :method : :column
+        end
+        return :method if klass.method_defined?(source)
+        raise UnknownSourceError,
+          "#{klass.name}: source :#{source} is not a column or instance method."
+      end
+
+      def self.user_override?(klass, source)
+        return false unless klass.method_defined?(source)
+        !generated_reader?(klass.instance_method(source).owner)
+      end
+      private_class_method :user_override?
+
+      # The name check matches the fake model classes the unit specs use; +is_a?+ matches real AR.
+      # +defined?+ lets this run when Active Record is not loaded.
+      def self.generated_reader?(owner)
+        return true if owner.name.to_s.end_with?("::GeneratedAttributeMethods")
+        defined?(::ActiveRecord::AttributeMethods::GeneratedAttributeMethods) &&
+          owner.is_a?(::ActiveRecord::AttributeMethods::GeneratedAttributeMethods)
+      end
+      private_class_method :generated_reader?
+
+      # +is_a?+ rather than the +#type+ symbol: +Type::Serialized+ and encrypted
+      # attribute types share the symbol but do not inherit from +Type::Json+.
+      def self.json_typed?(model, attribute_name)
+        model.type_for_attribute(attribute_name.to_s).is_a?(::ActiveRecord::Type::Json)
+      end
+
+      # +:date+ / +:time+ are absent: their raw shapes differ from the
+      # "YYYY-MM-DD HH:MM:SS" shape {Panko::CodeGen::DateTimeFormat} splices.
+      DATETIME_TYPES = %i[datetime timestamptz].freeze
+
+      def self.datetime_typed?(model, attribute_name)
+        DATETIME_TYPES.include?(model.type_for_attribute(attribute_name.to_s).type)
+      end
+
+      # Cast values that +cast_datetime+ returns unchanged, so Hash mode may skip it.
+      # +:decimal+, +:float+, +:inet+, +:cidr+ are absent: +cast_datetime+ can change their value.
+      # +:json+ / +:jsonb+ are listed although +#as_json+ builds a new object: parsed JSON holds
+      # only primitives, so the result is ==-equal and skipping the cast saves only allocations.
+      PLAIN_TYPES = %i[
+        string text integer bigint boolean uuid binary
+        json jsonb macaddr
+      ].freeze
+
+      # Wrapper types (+Type::Serialized+, PG's +OID::Array+) report their +#subtype+'s +#type+
+      # but cast to something else.
+      def self.plain_typed?(model, attribute_name)
+        type = model.type_for_attribute(attribute_name.to_s)
+        PLAIN_TYPES.include?(type.type) && !type.respond_to?(:subtype)
+      end
+    end
+  end
+end

data/lib/panko/code_gen/active_record/define_attribute_methods.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  module ActiveRecord
+    module DefineAttributeMethods
+      def self.ensure!(klass)
+        return if klass.attribute_methods_generated?
+        klass.define_attribute_methods
+        nil
+      end
+    end
+  end
+end

data/lib/panko/code_gen/code_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  class CodeBuilder
+    INDENT_UNIT = "  "
+
+    def initialize
+      @lines = []
+      @indent = 0
+    end
+
+    def line(str = "")
+      @lines << (INDENT_UNIT * @indent) + str
+    end
+
+    # Unlike +line("")+, adds no indent, so the output has no trailing whitespace.
+    def blank
+      @lines << ""
+    end
+
+    def indent
+      @indent += 1
+      yield
+    ensure
+      @indent -= 1
+    end
+
+    def to_s
+      @lines.join("\n")
+    end
+  end
+end

data/lib/panko/code_gen/compile_cache.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  # Descriptor -> Generated Class, one entry per unique Descriptor in a tree.
+  class CompileCache
+    # Keyed by +__id__+, so equal but distinct Descriptors get separate entries.
+    def initialize
+      @store = {}
+    end
+
+    def get(descriptor)
+      @store[descriptor.__id__]
+    end
+
+    def set(descriptor, generated_class)
+      @store[descriptor.__id__] = generated_class
+    end
+
+    # The block must call +#set+ before it descends into children, so a recursive
+    # call on the same Descriptor hits the entry and the cycle ends.
+    def lookup_or_compile(descriptor)
+      cached = @store[descriptor.__id__]
+      return cached if cached
+      yield
+      @store[descriptor.__id__]
+    end
+  end
+end

data/lib/panko/code_gen/compiler.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  class Compiler
+    # A table, not +to_s.upcase+: the +:hash+ suffix is +Hash+, not +HASH+.
+    OUTPUT_SUFFIXES = {json: "JSON", hash: "Hash"}.freeze
+
+    def initialize(descriptor, output:, config:, validator: Validators::Validator.new,
+      generator: Generator.new)
+      @descriptor = descriptor
+      @output = output
+      @config = config
+      @validator = validator
+      @generator = generator
+      @cache = CompileCache.new
+    end
+
+    def compile
+      @validator.validate(@descriptor, output: @output, config: @config)
+      source = @generator.emit(@descriptor, output: @output, config: @config)
+      namespace = Class.new
+      bind_anonymous_parents(namespace)
+      namespace.module_eval(source, synthetic_path, 1)
+      cache_descendants(@descriptor, namespace)
+      @cache.get(@descriptor)
+    end
+
+    private
+
+    # A Hash, not one constant per class: +const_set+ gives an anonymous class a name,
+    # which would change +parent_class.name+ for the next compile.
+    def bind_anonymous_parents(namespace)
+      anonymous = {}
+      Generators::DescriptorWalk.in_emit_order(@descriptor).each do |descriptor|
+        parent = descriptor.parent_class
+        next if parent.name
+        anonymous[descriptor.name] = parent
+      end
+      namespace.const_set(:ANON_PARENTS, anonymous.freeze) unless anonymous.empty?
+    end
+
+    def synthetic_path
+      "(#{GENERATOR_NAME}: #{@descriptor.name}/#{@output})"
+    end
+
+    def cache_descendants(descriptor, namespace)
+      @cache.lookup_or_compile(descriptor) do
+        klass = namespace.const_get(:"#{descriptor.name}_#{OUTPUT_SUFFIXES.fetch(@output)}")
+        @cache.set(descriptor, klass)
+        descriptor.associations.each do |assoc|
+          assoc.descriptors.each { |child| cache_descendants(child, namespace) }
+        end
+      end
+    end
+  end
+end

data/lib/panko/code_gen/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  # Compile-time settings baked into a Generated Class; omitted fields take DEFAULTS.
+  #
+  # - +null_for_missing_has_one+: a has_one source returning +nil+ writes +null+
+  #   (+nil+ in Hash mode) instead of omitting the key.
+  # - +supports_root_key+: when +false+, passing +root_key:+ raises +ArgumentError+.
+  # - +json_column_emit+ (JSON mode, Specialized path only): +:wire_format+ writes the
+  #   stored bytes after a well-formedness check, +:html_safe+ the typed value, escaped.
+  # - +pool_writer+: no effect in Hash output mode.
+  # - +guarded_model+: a record of another class goes to the generic body.
+  #   Needs a named +descriptor.model+.
+  Config = Data.define(
+    :null_for_missing_has_one,
+    :supports_root_key,
+    :hash_record_key_type,
+    :hash_output_key_type,
+    :json_column_emit,
+    :pool_writer,
+    :guarded_model
+  )
+
+  Config::DEFAULTS = {
+    null_for_missing_has_one: true,
+    supports_root_key: false,
+    hash_record_key_type: :string,
+    hash_output_key_type: :string,
+    json_column_emit: :wire_format,
+    pool_writer: true,
+    guarded_model: false
+  }.freeze
+
+  Config::HASH_KEY_TYPES = %i[string symbol].freeze
+
+  Config::JSON_COLUMN_EMIT_MODES = %i[wire_format html_safe].freeze
+
+  # Prepended because +Data.define+ defines +new+ on Config itself, so a
+  # plain +def self.new+ would replace it and leave no +super+ to call.
+  module Config::ClassMethods
+    def new(**kwargs)
+      merged = Config::DEFAULTS.merge(kwargs)
+      validate!(merged)
+      super(**merged)
+    end
+
+    private
+
+    def validate!(values)
+      validate_enum!(:hash_record_key_type, values[:hash_record_key_type])
+      validate_enum!(:hash_output_key_type, values[:hash_output_key_type])
+      validate_json_column_emit!(values[:json_column_emit])
+      validate_boolean!(:pool_writer, values[:pool_writer])
+      validate_boolean!(:guarded_model, values[:guarded_model])
+    end
+
+    def validate_enum!(field, value)
+      return if Config::HASH_KEY_TYPES.include?(value)
+      raise DescriptorError,
+        "Config##{field}: invalid value #{value.inspect}; must be :string or :symbol."
+    end
+
+    # +ArgumentError+, not +DescriptorError+: this field picks how code is
+    # written, it is not part of the Descriptor's shape.
+    def validate_json_column_emit!(value)
+      return if Config::JSON_COLUMN_EMIT_MODES.include?(value)
+      raise ArgumentError,
+        "Config#json_column_emit: invalid value #{value.inspect}; must be :wire_format or :html_safe."
+    end
+
+    def validate_boolean!(field, value)
+      return if value.equal?(true) || value.equal?(false)
+      raise ArgumentError,
+        "Config##{field}: invalid value #{value.inspect}; must be true or false."
+    end
+  end
+
+  Config.singleton_class.prepend(Config::ClassMethods)
+end

data/lib/panko/code_gen/datetime_format.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  # Formats a raw DB datetime String ("YYYY-MM-DD HH:MM:SS[.fraction]") as ISO-8601 without
+  # building a Time. Used only when +ActiveRecord.default_timezone+ is +:utc+, so the value is UTC.
+  module DateTimeFormat
+    TEMPLATE = "0000-00-00T00:00:00.000Z"
+
+    module_function
+
+    # Returns +nil+ when +raw+ is not a DB datetime String (a dirty attribute holding
+    # a Time, a nil column, another adapter format); callers then use the type-cast read.
+    def format_raw(raw)
+      return nil unless raw.is_a?(String)
+      len = raw.bytesize
+      # Some adapters already return ISO-8601 UTC ("...T...Z").
+      return raw if len >= 20 && raw.getbyte(len - 1) == 90 && raw.getbyte(10) == 84
+      return nil if len < 19
+      return nil unless raw.getbyte(10) == 32
+
+      out = TEMPLATE.dup
+      out.bytesplice(0, 10, raw, 0, 10)
+      out.bytesplice(11, 8, raw, 11, 8)
+      if len > 20 && raw.getbyte(19) == 46
+        # Truncate (never round) to milliseconds, like +#xmlschema(3)+. Copy only the
+        # digits: PG trims trailing zeros and appends "+00" ("...15.5+00").
+        fraction_digits = 0
+        while fraction_digits < 3
+          byte = raw.getbyte(20 + fraction_digits)
+          break if byte.nil? || byte < 48 || byte > 57
+          fraction_digits += 1
+        end
+        out.bytesplice(20, fraction_digits, raw, 20, fraction_digits) if fraction_digits > 0
+      end
+      out
+    end
+  end
+end

data/lib/panko/code_gen/descriptor.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Panko::CodeGen
+  # Returned by a Method Attribute body to omit its Field. Compared by
+  # identity, so no caller value can be mistaken for it.
+  SKIP = Object.new.freeze
+
+  # Each Descriptor-family +Data+ type validates its own fields at +.new+,
+  # so +Descriptor+ checks only the type of each array element, not its fields.
+  module StructuralValidation
+    module_function
+
+    def validate_symbol!(field, value)
+      return if value.is_a?(Symbol)
+      raise DescriptorError, "#{field}: must be a Symbol; got #{value.inspect}:#{value.class}"
+    end
+
+    def validate_callable!(field, value)
+      return if value.is_a?(Symbol)
+      if value.is_a?(UnboundMethod)
+        raise DescriptorError, "#{field}: must be a bound Callable, not an UnboundMethod; got #{value.inspect}:#{value.class}"
+      end
+      return if value.respond_to?(:call)
+      raise DescriptorError, "#{field}: must respond to #call; got #{value.inspect}:#{value.class}"
+    end
+
+    def validate_optional_class!(field, value)
+      return if value.nil?
+      return if value.is_a?(Class)
+      raise DescriptorError, "#{field}: must be a Class or nil; got #{value.inspect}:#{value.class}"
+    end
+
+    def validate_class!(field, value)
+      return if value.is_a?(Class)
+      raise DescriptorError, "#{field}: must be a Class; got #{value.inspect}:#{value.class}"
+    end
+
+    def validate_non_empty_string!(field, value)
+      return if value.is_a?(String) && !value.empty?
+      raise DescriptorError, "#{field}: must be a non-empty String; got #{value.inspect}:#{value.class}"
+    end
+
+    # +kind_label+ is passed apart from +element_class+ so the namespace
+    # does not show in the error message.
+    def validate_array_of!(field, value, element_class, kind_label)
+      unless value.is_a?(Array)
+        raise DescriptorError, "#{field}: must be an Array of #{kind_label}; got #{value.inspect}:#{value.class}"
+      end
+      value.each do |el|
+        next if el.is_a?(element_class)
+        raise DescriptorError, "#{field}: must contain only #{kind_label} elements; got #{el.inspect}:#{el.class}"
+      end
+    end
+
+    def validate_descriptor!(field, value)
+      return if value.is_a?(Descriptor)
+      raise DescriptorError, "#{field}: must be a Descriptor; got #{value.inspect}:#{value.class}"
+    end
+
+    def validate_kind!(field, value)
+      return if Association::KINDS.include?(value)
+      raise DescriptorError, "#{field}: must be :has_one or :has_many; got #{value.inspect}:#{value.class}"
+    end
+  end
+
+  # A Field read from the Record's +source+ (defaults to +name+).
+  Attribute = Data.define(:name, :source) do
+    def initialize(name:, source: name)
+      StructuralValidation.validate_symbol!("Attribute#name", name)
+      StructuralValidation.validate_symbol!("Attribute#source", source)
+      super
+    end
+  end
+
+  # +body+ is a Callable (passed record, context, scope up to its arity) or a Symbol
+  # naming a +parent_class+ method; returning +SKIP+ omits the Field.
+  MethodAttribute = Data.define(:name, :body) do
+    def initialize(name:, body:)
+      StructuralValidation.validate_symbol!("MethodAttribute#name", name)
+      StructuralValidation.validate_callable!("MethodAttribute#body", body)
+      super
+    end
+  end
+
+  # +descriptor+ may be the parent itself for self-recursive shapes.
+  Association = Data.define(:name, :kind, :descriptor, :source, :if, :variants) do
+    # +variants+ serve a source returning several record classes: a record uses the
+    # variant whose +model+ is its exact class, else +descriptor+.
+    def initialize(name:, kind:, descriptor:, source: nil, if: nil, variants: [])
+      # +if+ is a keyword, so its value cannot be read as a local by name.
+      if_callable = binding.local_variable_get(:if)
+      source = name if source.nil?
+      StructuralValidation.validate_symbol!("Association#name", name)
+      StructuralValidation.validate_kind!("Association#kind", kind)
+      StructuralValidation.validate_descriptor!("Association#descriptor", descriptor)
+      StructuralValidation.validate_symbol!("Association#source", source)
+      # +if+ must be a Callable: a Symbol passes this check but fails at compile
+      # (Symbol has no +arity+).
+      StructuralValidation.validate_callable!("Association#if", if_callable) unless if_callable.nil?
+      StructuralValidation.validate_array_of!("Association#variants", variants, Descriptor, "Descriptor")
+      validate_variants!(variants)
+      super(name: name, kind: kind, descriptor: descriptor, source: source, if: if_callable, variants: variants.frozen? ? variants : variants.dup.freeze)
+    end
+
+    def descriptors
+      variants.empty? ? [descriptor] : [descriptor, *variants]
+    end
+
+    private
+
+    def validate_variants!(variants)
+      models = variants.map(&:model)
+      if models.any? { |model| model.nil? || model.name.nil? }
+        raise DescriptorError, "Association#variants: every variant needs a named model; got #{models.inspect}"
+      end
+      return if models.uniq.size == models.size
+      raise DescriptorError, "Association#variants: models must be distinct; got #{models.inspect}"
+    end
+  end
+
+  Association::KINDS = %i[has_one has_many].freeze
+
+  # +model+ nil uses the generic path, a Class allows specialization; the Generated
+  # Class subclasses +parent_class+.
+  Descriptor = Data.define(:name, :model, :attributes, :method_attributes, :associations, :parent_class) do
+    def initialize(name:, model:, attributes:, method_attributes:, associations:, parent_class:)
+      StructuralValidation.validate_non_empty_string!("Descriptor#name", name)
+      StructuralValidation.validate_optional_class!("Descriptor#model", model)
+      StructuralValidation.validate_array_of!("Descriptor#attributes", attributes, Attribute, "Attribute")
+      StructuralValidation.validate_array_of!("Descriptor#method_attributes", method_attributes, MethodAttribute, "MethodAttribute")
+      StructuralValidation.validate_array_of!("Descriptor#associations", associations, Association, "Association")
+      StructuralValidation.validate_class!("Descriptor#parent_class", parent_class)
+      super
+    end
+  end
+end