export_kit 0.1.0

data/lib/export_kit/model_rows.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+require "date"
+require "time"
+
+module ExportKit
+  class ModelRows
+    def initialize(record_set:, input:, batch_size:)
+      @record_set = record_set
+      @input = input
+      @batch_size = batch_size
+      raise ArgumentError, "batch_size must be positive" unless batch_size.is_a?(Integer) && batch_size.positive?
+    end
+
+    def each
+      index = 0
+      batches do |rows|
+        rows.each do |row|
+          index += 1
+          yield present(row.drop(1))
+        end
+      end
+    rescue Renderers::InvalidDocument => e
+      raise e.at(record_set: record_set.name, row: index)
+    end
+
+    private
+
+    attr_reader :batch_size, :input, :record_set
+
+    def batches
+      relation = record_set.scope.call(input)
+      validate_relation!(relation)
+      remaining = relation.limit_value
+      cursor = nil
+      while remaining != 0 && (rows = read_batch(cursor, remaining)).any?
+        yield rows
+        cursor = rows.last.first
+        remaining -= rows.size if remaining
+      end
+    end
+
+    def read_batch(cursor, remaining)
+      limit = remaining ? [remaining, batch_size].min : batch_size
+      record_set.page(input, after: cursor, limit:).pluck(*selections)
+    end
+
+    def validate_relation!(relation)
+      unless valid_relation?(relation)
+        raise ArgumentError, "export requires an unordered, ungrouped relation without offset or distinct"
+      end
+      return unless record_set.model.primary_key.is_a?(Array) || !record_set.model.primary_key
+
+      raise ArgumentError, "export requires a single primary key"
+    end
+
+    def valid_relation?(relation)
+      relation.is_a?(ActiveRecord::Relation) && relation.klass == record_set.model &&
+        !relation.offset_value && relation.group_values.empty? &&
+        !relation.distinct_value && relation.order_values.empty?
+    end
+
+    def selections
+      [record_set.model.arel_table[record_set.model.primary_key], *record_set.selections(input)]
+    end
+
+    def present(values)
+      row = record_set.output_keys.zip(values).to_h do |key, value|
+        [key, scalar_field(key, value)]
+      end
+      record_set.time_of_day_fields.each do |key|
+        value = values[record_set.output_keys.index(key)]
+        row[key] = value.strftime("%H:%M:%S") if value
+      end
+      record_set.present(row, input:)
+    end
+
+    def scalar_field(key, value)
+      scalar(value)
+    rescue Renderers::InvalidDocument => e
+      raise e.at(column: key)
+    end
+
+    def scalar(value)
+      case value
+      when Time, ActiveSupport::TimeWithZone then value.utc.iso8601(6)
+      when Date then value.iso8601
+      when BigDecimal then value.to_s("F")
+      when Array then value.map { scalar(it) }
+      when *Renderers::Base::SCALARS then value
+      else raise Renderers::InvalidDocument, "unsupported model value"
+      end
+    end
+  end
+end

data/lib/export_kit/model_schema.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module ExportKit
+  # Lets an application model explicitly describe the columns available to any export.
+  # Completeness is checked only when the application calls validate_export_schema!.
+  module ModelSchema
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :defined_export_schema, instance_writer: false, default: nil
+    end
+
+    class_methods do
+      def export_schema(&definition)
+        return defined_export_schema unless definition
+
+        raise ArgumentError, "Export schema already defined for #{name}" if defined_export_schema
+
+        self.defined_export_schema = DefinitionBuilder.new.tap { it.instance_eval(&definition) }.build
+      end
+
+      def validate_export_schema!
+        schema = export_schema
+        raise ArgumentError, "export schema is missing" unless schema
+
+        classified = schema.allowed + schema.denied.keys
+        raise ArgumentError, "export schema does not classify every column" unless classified.sort == column_names.sort
+
+        schema
+      end
+    end
+
+    Definition = Data.define(:allowed, :denied, :formatters) do
+      alias_method :export, :allowed
+      alias_method :omit, :denied
+    end
+
+    class DefinitionBuilder
+      def initialize
+        @allowed = []
+        @denied = {}
+        @formatters = {}
+      end
+
+      def allow(*declarations)
+        each_field_declaration(declarations) do |key, formatter|
+          key = key.to_s
+          allowed << key
+          formatters[key] = formatter if formatter
+        end
+      end
+
+      def timestamps
+        allow :created_at, :updated_at
+      end
+
+      def omit(*columns, reason:)
+        raise ArgumentError, "omission reason is required" if reason.to_s.strip.empty?
+        raise ArgumentError, "omitted fields must be unique" if denied.keys.intersect?(columns.map(&:to_s))
+
+        columns.each { |column| denied[column.to_s] = reason.to_s }
+      end
+
+      alias deny omit
+
+      def build
+        validate!
+
+        Definition.new(allowed: allowed.freeze, denied: denied.freeze, formatters: formatters.freeze).freeze
+      end
+
+      private
+
+      attr_reader :allowed, :denied, :formatters
+
+      def each_field_declaration(declarations)
+        items = declarations.one? && declarations.first.is_a?(Array) ? declarations.first.dup : declarations.dup
+        until items.empty?
+          key = items.shift
+
+          unless key.is_a?(String) || key.is_a?(Symbol)
+            raise ArgumentError, "An export field must be a String or Symbol"
+          end
+
+          yield key, (items.shift if items.first.respond_to?(:call))
+        end
+      end
+
+      def validate!
+        duplicates = allowed & denied.keys
+        raise ArgumentError, "Fields cannot be both exported and omitted: #{duplicates.join(', ')}" if duplicates.any?
+        raise ArgumentError, "Export fields must be unique" unless allowed.uniq == allowed
+      end
+    end
+  end
+end

data/lib/export_kit/model_serializer.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+module ExportKit
+  # Generic DSL for selecting model rows and a subset of fields allowed by `export_schema`.
+  # It knows Active Record, but not a concrete export, document shape or output format.
+  # Non-model rows deliberately bypass this class and are created by an export presenter.
+  class ModelSerializer
+    Field = Data.define(:key, :label_key)
+
+    Category = Data.define(:category, :version, :record_sets, :omitted_tables, :omission_reason) do
+      def included? = record_sets.any?
+    end
+
+    RecordSet = Data.define(
+      :name, :model, :scope, :fields, :expressions, :export_fields, :field_presenters, :policy
+    ) do
+      def output_keys = export_fields.map(&:key)
+
+      def selections(subject)
+        table = model.arel_table
+        fields.map { table[it] } + expressions.values.map { Arel.sql(it.call(subject)) }
+      end
+
+      def page(subject, after:, limit:)
+        id = model.arel_table[model.primary_key]
+        relation = scope.call(subject)
+        relation = relation.where(id.gt(after)) unless after.nil?
+        relation.reorder(id).limit(limit)
+      end
+
+      def each_row(input: nil, batch_size: 500, &block)
+        return enum_for(__method__, input:, batch_size:) unless block
+
+        ModelRows.new(record_set: self, input:, batch_size:).each(&block)
+      end
+
+      def table(input: nil, labels: {}, batch_size: 500)
+        Table.new(columns: output_keys, labels:, rows: ->(&emit) { each_row(input:, batch_size:, &emit) })
+      end
+
+      def time_of_day_fields
+        fields.select { model.type_for_attribute(it).type == :time }
+      end
+
+      def present(row, input:)
+        row.to_h do |key, value|
+          formatter = field_presenters[key]
+          [key, formatter ? present_field(key, formatter, value, input) : value]
+        end
+      end
+
+      private
+
+      def present_field(key, formatter, value, input)
+        present_value(formatter, value, input)
+      rescue ArgumentError, TypeError, EncodingError
+        raise Renderers::InvalidDocument.new(error_details: { record_set: name, column: key })
+      end
+
+      def present_value(formatter, value, input)
+        formatter.arity == 1 ? formatter.call(value) : formatter.call(value, input)
+      end
+    end
+
+    # Read-only table index derived from the model schemas used by this serializer.
+    class ModelSchemas
+      def initialize(serializer:)
+        @schemas = serializer.all.flat_map(&:record_sets).each_with_object({}) do |record_set, index|
+          table_name = record_set.model.table_name
+          existing = index[table_name]
+          if existing && existing != record_set.policy
+            raise ArgumentError, "Conflicting export schemas for #{table_name}"
+          end
+
+          index[table_name] = record_set.policy
+        end.freeze
+        freeze
+      end
+
+      def all = schemas
+      def each(&) = schemas.each(&)
+      def fetch(table_name) = schemas.fetch(table_name.to_s)
+      def key?(table_name) = schemas.key?(table_name.to_s)
+
+      private
+
+      attr_reader :schemas
+    end
+
+    def initialize(record_set_builder: RecordSetBuilder)
+      @record_set_builder = record_set_builder
+      @categories = {}
+    end
+
+    def category(key, version: 1, &definition)
+      key = normalize(key)
+      raise ArgumentError, "Export category already registered: #{key}" if categories.key?(key)
+
+      categories[key] = CategoryBuilder.new(key, version:, record_set_builder:, &definition).build
+    end
+
+    def exclude(key, reason:)
+      key = normalize(key)
+      raise ArgumentError, "Export category already registered: #{key}" if categories.key?(key)
+
+      categories[key] = Category.new(
+        category: key, version: 1, record_sets: [].freeze, omitted_tables: {}.freeze,
+        omission_reason: reason
+      )
+    end
+
+    def all = categories.values.freeze
+    def each(&) = categories.each_value(&)
+    def fetch(key) = categories.fetch(normalize(key))
+
+    def freeze
+      categories.freeze
+      super
+    end
+
+    private
+
+    attr_reader :categories, :record_set_builder
+
+    def normalize(key)
+      key.to_s.tap do |value|
+        raise ArgumentError, "Invalid export category: #{value.inspect}" unless value.match?(/\A[a-z][a-z0-9_]*\z/)
+      end
+    end
+
+    # DSL for one category and its model-backed record sets.
+    class CategoryBuilder
+      def initialize(category, version:, record_set_builder:, &definition)
+        @record_set_builder = record_set_builder
+        @category = category.to_s
+        @version = version
+        @record_sets = []
+        @omitted_tables = {}
+        instance_eval(&definition)
+      end
+
+      def records(name, model:, scope:, fields:, schema: nil, &definition)
+        record_sets << record_set_builder.new(name, model:, scope:, fields:, schema:, &definition).build
+      end
+
+      def omit_table(table_name, reason:)
+        omitted_tables[table_name.to_s] = reason
+      end
+
+      def build
+        Category.new(category:, version:, record_sets: record_sets.freeze,
+                     omitted_tables: omitted_tables.freeze, omission_reason: nil)
+      end
+
+      private
+
+      attr_reader :category, :omitted_tables, :record_sets, :version, :record_set_builder
+    end
+
+    # DSL for selecting model fields, SQL expressions and per-export formatter overrides.
+    class RecordSetBuilder
+      def initialize(name, model:, scope:, fields:, schema:, &definition)
+        @name = name.to_s
+        @model = model
+        @scope = scope
+        @configured_fields, inline_formatters = parse_fields(fields)
+        @policy = schema || (model.export_schema if model.respond_to?(:export_schema))
+
+        @expressions = {}
+        @label_keys = {}
+        @presenters = inline_formatters
+        instance_eval(&definition) if definition
+      end
+
+      def computed(key, sql:, label: key, presenter: nil)
+        key = key.to_s
+        raise ArgumentError, "computed SQL must be callable" unless sql.respond_to?(:call)
+        raise ArgumentError, "computed fields must be unique" if expressions.key?(key)
+
+        expressions[key] = sql
+        label_keys[key] = label.to_s
+        presenters[key] = presenter if presenter
+      end
+
+      def format(key, formatter = nil, &block)
+        key = key.to_s
+        callable = formatter || block
+        raise ArgumentError, "#{key} formatter must be callable" unless callable.respond_to?(:call)
+
+        presenters[key] = callable
+      end
+
+      def build
+        fields = chosen_fields.freeze
+        validate_presenters!(fields)
+        RecordSet.new(**record_set_attributes(fields))
+      end
+
+      private
+
+      attr_reader :configured_fields, :expressions, :label_keys, :model, :name, :policy, :presenters, :scope
+
+      def chosen_fields
+        allowed = allowed_fields
+        chosen = configured_fields == :all ? allowed : Array(configured_fields).map(&:to_s)
+        raise ArgumentError, "#{name} reads a field outside the export schema" unless (chosen - allowed).empty?
+        if chosen.uniq != chosen || chosen.intersect?(expressions.keys)
+          raise ArgumentError,
+                "export fields must be unique"
+        end
+
+        chosen
+      end
+
+      def allowed_fields
+        return policy.allowed if policy
+        raise ArgumentError, "fields: :all requires an export schema" if configured_fields == :all
+
+        model.column_names
+      end
+
+      def registered_fields(keys)
+        labels = keys.index_with { it }.merge(label_keys)
+        keys.map { Field.new(key: it, label_key: labels.fetch(it)) }
+      end
+
+      def field_presenters_for(fields)
+        defaults = policy ? policy.formatters.slice(*fields) : {}
+        defaults.merge(presenters).freeze
+      end
+
+      def record_set_attributes(fields)
+        {
+          name:, model:, scope:, fields:, expressions: expressions.freeze,
+          export_fields: registered_fields(fields + expressions.keys).freeze,
+          field_presenters: field_presenters_for(fields), policy:
+        }
+      end
+
+      def validate_presenters!(fields)
+        unknown = presenters.keys - fields - expressions.keys
+        raise ArgumentError, "#{name} formats a field it does not export" if unknown.any?
+      end
+
+      def parse_fields(declarations)
+        return [:all, {}] if declarations == :all
+
+        items = Array(declarations).dup
+        fields = []
+        formatters = {}
+        until items.empty?
+          key = items.shift
+          unless key.is_a?(String) || key.is_a?(Symbol)
+            raise ArgumentError, "An export field must be a String or Symbol"
+          end
+
+          key = key.to_s
+          fields << key
+          formatters[key] = items.shift if items.first.respond_to?(:call)
+        end
+        [fields, formatters]
+      end
+    end
+  end
+end

data/lib/export_kit/output.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ExportKit
+  class Output
+    def initialize(io:, max_bytes:)
+      @io = io
+      @max_bytes = max_bytes
+      @written = 0
+    end
+
+    def write(*strings)
+      bytes = strings.sum(&:bytesize)
+      position = io.respond_to?(:pos) ? io.pos : written
+      raise Renderers::TooLarge if position + bytes > max_bytes
+
+      @written += bytes
+      io.write(*strings)
+    end
+
+    def <<(string)
+      write(string)
+      self
+    end
+
+    def copy_from(source)
+      while (chunk = source.read(16_384))
+        write(chunk)
+      end
+    end
+
+    def reopen(other)
+      raise ArgumentError, "cannot replace export output" unless other.is_a?(Output) && io.equal?(other.send(:io))
+
+      self
+    end
+
+    def seek(*) = io.seek(*)
+    def pos = io.pos
+
+    def pos=(position)
+      raise Renderers::TooLarge if position > max_bytes
+
+      io.pos = position
+    end
+
+    def size = io.size
+    def flush = io.flush
+    def rewind = io.rewind
+    def tell = io.tell
+    def binmode = self
+    def close = flush
+    def closed? = io.closed?
+
+    private
+
+    attr_reader :io, :max_bytes, :written
+  end
+end

data/lib/export_kit/render.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module ExportKit
+  class Render
+    FORMAT_UNSUPPORTED = :export_format_unsupported
+    TOO_LARGE = :export_too_large
+    RENDER_FAILED = :export_render_failed
+
+    def initialize(data:, format:, max_bytes:, renderer: nil, content_type: nil, extension: nil,
+                   font: nil, bold_font: nil, **options)
+      @data = data
+      @format = format.to_s
+      @max_bytes = max_bytes
+      @renderer = renderer
+      @content_type = content_type
+      @extension = extension
+      @fonts = { normal: font, bold: bold_font }
+      @options = options.merge(font:, bold_font:).freeze
+    end
+
+    def call
+      validate_limit!
+      definition = format_definition
+      return Result.failure(FORMAT_UNSUPPORTED) unless definition
+      return Result.failure(RENDER_FAILED) unless supported_data?
+
+      render_file(definition)
+    rescue Renderers::TooLarge
+      Result.failure(TOO_LARGE)
+    rescue JSON::GeneratorError, EncodingError, Renderers::InvalidDocument,
+           Prawn::Errors::IncompatibleStringEncoding, Prawn::Errors::CannotFit, Zip::Error => e
+      Result.failure(RENDER_FAILED, error_details: e.respond_to?(:error_details) ? e.error_details : {})
+    end
+
+    private
+
+    attr_reader :content_type, :data, :extension, :fonts, :format, :max_bytes, :renderer, :options
+
+    def render_file(definition)
+      file = Tempfile.new(["export-kit", ".#{definition.extension}"], binmode: true)
+      output = Output.new(io: file, max_bytes:)
+      manifest = render_to(output)
+      output.flush
+      file.rewind
+      artifact = build_artifact(file, definition, manifest)
+      Result.success(artifact)
+    ensure
+      file&.close! unless artifact
+    end
+
+    def validate_limit!
+      return if max_bytes.is_a?(Integer) && max_bytes.positive?
+
+      raise Renderers::InvalidDocument, "max_bytes must be a positive integer"
+    end
+
+    def format_definition
+      return FormatRegistry.find(format) unless renderer
+
+      registered = FormatRegistry.find(format)
+      custom_extension = extension || registered&.extension
+      custom_type = content_type || registered&.content_type
+      validate_descriptor!(custom_extension, custom_type)
+      FormatRegistry::Definition.new(key: format, extension: custom_extension, content_type: custom_type)
+    end
+
+    def validate_descriptor!(custom_extension, custom_type)
+      unless valid_descriptor?(custom_extension, /\A[a-z0-9]{1,12}\z/) &&
+             valid_descriptor?(custom_type, %r{\A[\w.+-]+/[\w.+-]+\z})
+        raise Renderers::InvalidDocument, "custom renderers require an extension and content type"
+      end
+    end
+
+    def valid_descriptor?(value, pattern)
+      value.is_a?(String) && value.match?(pattern)
+    end
+
+    def supported_data?
+      return true if renderer || format == "json"
+      return data.is_a?(Table) if format == "csv"
+
+      data.is_a?(Document)
+    end
+
+    def render_to(output)
+      if renderer
+        renderer.call(input: data, output:, options:)
+        return nil
+      end
+      return render_document(output) if data.is_a?(Document)
+      return render_table(output) if data.is_a?(Table)
+
+      write_json(data, output)
+      nil
+    end
+
+    def render_document(output)
+      implementation = case format
+                       when "json" then Renderers::Json.new
+                       when "csv_zip" then Renderers::CsvArchive.new
+                       when "pdf" then Renderers::Pdf.new(fonts:)
+                       end
+      implementation.render(document: data, output:, max_bytes:)
+    end
+
+    def render_table(output)
+      format == "json" ? write_json_rows(output) : write_csv_rows(output)
+      nil
+    end
+
+    def write_json_rows(output)
+      output.write("[")
+      separator = ""
+      each_table_row do |row|
+        output.write(separator)
+        write_json(row, output)
+        separator = ","
+      end
+      output.write("]")
+    end
+
+    def write_csv_rows(output)
+      csv = CSV.new(output)
+      csv << data.headers.map { Renderers::CsvArchive.csv_value(it) }
+      each_table_row do |row|
+        csv << data.columns.map { Renderers::CsvArchive.csv_value(row.fetch(it)) }
+      end
+    end
+
+    def each_table_row(&)
+      Renderers::Base.new.send(:each_row, data, &)
+    end
+
+    def write_json(value, output, depth = 0)
+      raise Renderers::InvalidDocument, "JSON nesting exceeds limit" if depth > 100
+
+      case value
+      when Hash then write_json_hash(value, output, depth)
+      when Array then write_json_array(value, output, depth)
+      when *Renderers::Base::SCALARS then output.write(JSON.generate(value))
+      else raise Renderers::InvalidDocument, "unsupported JSON value"
+      end
+    end
+
+    def write_json_hash(value, output, depth)
+      validate_json_keys!(value.keys)
+      output.write("{")
+      value.each_with_index do |(key, item), index|
+        output.write(",") if index.positive?
+        output.write(JSON.generate(key.to_s), ":")
+        write_json(item, output, depth + 1)
+      end
+      output.write("}")
+    end
+
+    def validate_json_keys!(keys)
+      unless keys.all? { it.is_a?(String) || it.is_a?(Symbol) }
+        raise Renderers::InvalidDocument, "JSON object keys must be strings or symbols"
+      end
+
+      normalized = keys.map(&:to_s)
+      raise Renderers::InvalidDocument, "duplicate JSON keys" unless normalized.uniq == normalized
+    end
+
+    def write_json_array(value, output, depth)
+      output.write("[")
+      value.each_with_index do |item, index|
+        output.write(",") if index.positive?
+        write_json(item, output, depth + 1)
+      end
+      output.write("]")
+    end
+
+    def build_artifact(file, definition, manifest)
+      document = data if data.is_a?(Document) || data.is_a?(Table)
+      Artifact.new(file:, byte_size: file.size, sha256: Digest::SHA256.file(file.path).hexdigest,
+                   manifest:, format:, content_type: definition.content_type, extension: definition.extension,
+                   **document_metadata, document:)
+    end
+
+    def document_metadata
+      return { identifier: nil, schema_version: nil, metadata: {} } unless data.is_a?(Document)
+
+      { identifier: data.identifier, schema_version: data.schema_version, metadata: data.metadata }
+    end
+  end
+end