yaml_exporter 0.1.0 → 0.2.0

data/RELEASING.md

@@ -0,0 +1,45 @@
+# Releasing
+
+Releases are automated: **publish a GitHub Release and CI pushes the gem to
+RubyGems** via [Trusted Publishing](https://guides.rubygems.org/trusted-publishing/)
+(OIDC — no API key stored anywhere). See `.github/workflows/gem-push.yml`.
+
+## One-time setup
+
+On [rubygems.org](https://rubygems.org) → the `yaml_exporter` gem → **Trusted
+Publishers** → *Add* a GitHub Actions publisher:
+
+- Repository: `itadventurer/yaml_exporter`
+- Workflow: `gem-push.yml`
+
+The gem already exists on RubyGems, so use the normal *Add* flow above (the
+"pending trusted publisher" flow is only for a gem that doesn't exist yet).
+
+Once the publisher is added, the old `RUBYGEMS_AUTH_TOKEN` secret is no longer
+used and can be deleted.
+
+## Cutting a release
+
+1. **Bump the version and close the changelog.** Set the new version in
+   `lib/yaml_exporter/version.rb`, and in `CHANGELOG.md` rename the
+   `## Unreleased` heading to `## vX.Y.Z`, adding a fresh empty `## Unreleased`
+   above it. Open as a normal PR; merge to `main`.
+
+2. **Publish the release** for the matching tag — the workflow expects `vX.Y.Z`
+   to equal the version in `version.rb` — using that version's changelog section
+   as the notes:
+
+   ```bash
+   gh release create v0.2.0 --title v0.2.0 --notes "…"
+   ```
+
+   (or use the GitHub UI → *Draft a new release* → pick the tag). Publishing
+   fires the workflow, which runs the tests, builds the gem, attaches a
+   build-provenance attestation, and pushes it.
+
+That's it — `gem-push.yml` does the build + push; you never touch credentials.
+
+### Versioning
+
+[SemVer](https://semver.org): new backward-compatible features → minor (`0.x`),
+bug-fixes → patch. Breaking changes → bump the minor while pre-`1.0`.

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Defines the `build`/`install`/`release` tasks from the gemspec. The
+# rubygems/release-gem CI action runs `rake release` to push the gem.
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.warning = false
+end
+
+task default: :test

data/lib/yaml_exporter/builder.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module YamlExporter
+  # Evaluates a `yaml_structure do ... end` block via instance_eval and emits
+  # a Structure. All declaration-time argument validation happens here (or
+  # inside the constructors of the Node classes it delegates to).
+  #
+  # Accepts either a class or a callable that resolves to a class. The
+  # callable form lets node constructors defer AR reflection lookups until
+  # the inner block actually references the class (it doesn't, for
+  # attribute-only blocks) — this matters for anonymous ActiveRecord
+  # classes.
+  class Builder
+    def initialize(klass, &block)
+      @klass_resolver = klass.respond_to?(:call) ? klass : -> { klass }
+      @nodes = []
+      instance_eval(&block) if block
+    end
+
+    def build
+      Structure.new(klass: @klass_resolver, nodes: @nodes.freeze)
+    end
+
+    # ---- DSL ----------------------------------------------------------
+
+    def attributes(*names)
+      names.each { |n| @nodes << Nodes::Attribute.new(name: n, owner_class: @klass_resolver) }
+    end
+
+    def one(name, find_by: nil, of: nil, &block)
+      if block && find_by
+        raise ArgumentError,
+              "`one #{name.inspect}`: cannot combine a block (owned) with find_by: (reference). Pick one."
+      end
+      if of && !find_by
+        raise ArgumentError,
+              "`one #{name.inspect}`: of: requires find_by:."
+      end
+
+      if block
+        @nodes << Nodes::OneOwned.new(name: name, owner_class: klass, &block)
+      elsif find_by && of
+        @nodes << Nodes::OneReferenceOf.new(name: name, owner_class: klass, find_by: find_by, of: of)
+      elsif find_by
+        @nodes << Nodes::OneReference.new(name: name, owner_class: klass, find_by: find_by)
+      else
+        raise ArgumentError,
+              "`one #{name.inspect}`: must pass either find_by: (reference) or a block (owned)."
+      end
+    end
+
+    def many(name, find_by: nil, through: nil, positioned_by: nil, &block)
+      if through
+        unless find_by
+          raise ArgumentError,
+                "`many #{name.inspect}`: through: requires find_by: to resolve the target."
+        end
+        # A block is optional: with one, its attributes describe the join row;
+        # without one, the association is a bare reference list (and
+        # positioned_by:, when given, derives the join's position column from
+        # the YAML order). The join row itself is always managed by the DSL.
+        @nodes << Nodes::ManyThrough.new(
+          name: name, owner_class: klass, through: through, find_by: find_by,
+          positioned_by: positioned_by, &block
+        )
+      elsif positioned_by && !block
+        raise ArgumentError,
+              "`many #{name.inspect}`: positioned_by: requires a block — the column lives on the owned record."
+      elsif block && find_by
+        @nodes << Nodes::ManyFindBy.new(
+          name: name, owner_class: klass, find_by: find_by,
+          positioned_by: positioned_by, &block
+        )
+      elsif block
+        @nodes << Nodes::ManyPositional.new(
+          name: name, owner_class: klass, positioned_by: positioned_by, &block
+        )
+      elsif find_by
+        @nodes << Nodes::ManyReference.new(name: name, owner_class: klass, find_by: find_by)
+      else
+        raise ArgumentError,
+              "`many #{name.inspect}`: pass either find_by: (reference list), a block (owned), " \
+              'or both / through: + find_by: + block.'
+      end
+    end
+
+    private
+
+    def klass
+      @klass ||= @klass_resolver.call
+    end
+  end
+end

data/lib/yaml_exporter/exporter.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module YamlExporter
+  # Marker subclass: a String that must be emitted as a YAML literal block
+  # scalar (`|`). Attribute nodes wrap the values of `text` columns in this so
+  # the Exporter's visitor can render them as block scalars regardless of
+  # length, while `string`/varchar columns stay inline.
+  #
+  # Trailing whitespace on a line (a space/tab before a newline) makes
+  # libyaml's emitter refuse block style and silently fall back to a
+  # double-quoted inline scalar — which is exactly what breaks the "text
+  # columns are always block scalars" promise. Such whitespace is virtually
+  # always an accidental typo, so we strip it per line on construction. This
+  # also normalizes CR (`\r\n`/`\r`), which trips the same fallback.
+  class LiteralString < ::String
+    def self.new(value)
+      super(normalize(value.to_s))
+    end
+
+    def self.normalize(value)
+      value.split("\n", -1).map(&:rstrip).join("\n")
+    end
+  end
+
+  # Walks a record + structure and emits the YAML document.
+  #
+  # Key order follows declaration order of the structure's nodes. List order
+  # inside a `many` association is decided by the node itself.
+  #
+  # `omit_nil` (an export-time option, passed from `yaml_export`) drops keys
+  # whose exported value is "empty": nil, or an empty list. This is round-trip
+  # safe because import treats a missing key, an explicit `null`, and an empty
+  # list identically. An owned child that exists but has only nil attributes
+  # exports as `{}` and is kept — omitting it would mean "destroy" on
+  # re-import.
+  class Exporter
+    def initialize(record, structure, omit_nil: true)
+      @root_record = record
+      @root_structure = structure
+      @omit_nil = omit_nil
+    end
+
+    def to_yaml
+      hash = build_hash(@root_record, @root_structure)
+      dump(hash)
+    end
+
+    # Called recursively by nodes that own nested structures.
+    def build_hash(record, structure)
+      result = {}
+      structure.nodes.each do |node|
+        key, value = node.export(record, exporter: self)
+        next if @omit_nil && omit?(value)
+
+        result[key] = value
+      end
+      result
+    end
+
+    private
+
+    def omit?(value)
+      value.nil? || (value.is_a?(Array) && value.empty?)
+    end
+
+    def dump(hash)
+      visitor = BlockScalarTree.create
+      visitor << hash
+      visitor.restore_astral(visitor.tree.yaml)
+    end
+
+    # Renders any LiteralString value as a literal block scalar (`|`). Plain
+    # Strings and every other type fall through to Psych's default handling,
+    # so the output is byte-identical to `YAML.dump` whenever no LiteralString
+    # is present.
+    #
+    # Astral-plane characters (codepoints >= U+10000 — emoji, CJK Ext B, …)
+    # are the one thing that defeats block style: libyaml's emitter treats any
+    # 4-byte UTF-8 character as non-printable, and a non-printable character
+    # can only be written escaped (`\U0001F4A1`), which exists only in
+    # double-quoted style. So a single emoji silently drops the whole value
+    # back to an inline quoted scalar. BMP characters (umlauts, accents, ✓, →)
+    # are unaffected.
+    #
+    # To keep block style we swap each astral character out for a Private Use
+    # Area sentinel — which libyaml *does* consider printable — before
+    # emitting, then swap the real characters back into the finished document.
+    # The sentinels never survive in the output, and the round-trip is exact:
+    # the parser reads literal astral characters in block scalars without
+    # trouble.
+    class BlockScalarTree < Psych::Visitors::YAMLTree
+      ASTRAL = /[\u{10000}-\u{10FFFF}]/
+      # U+E000/U+E001 are Private Use Area: printable to libyaml, and they
+      # never carry meaning in real text, so a `<open>digits<close>` token
+      # cannot collide with exported content.
+      SENTINEL_OPEN  = "\u{E000}"
+      SENTINEL_CLOSE = "\u{E001}"
+      SENTINEL = /#{SENTINEL_OPEN}\d+#{SENTINEL_CLOSE}/.freeze
+
+      def accept(target)
+        if target.is_a?(LiteralString)
+          return @emitter.scalar(escape_astral(target.to_s), nil, nil, true, true, Psych::Nodes::Scalar::LITERAL)
+        end
+
+        super
+      end
+
+      # Replaces each astral character with a sentinel token, remembering the
+      # reverse mapping so #restore_astral can put the real characters back.
+      def escape_astral(text)
+        return text unless text.match?(ASTRAL)
+
+        text.gsub(ASTRAL) do |char|
+          forward[char] ||= begin
+            token = "#{SENTINEL_OPEN}#{substitutions.size}#{SENTINEL_CLOSE}"
+            substitutions[token] = char
+            token
+          end
+        end
+      end
+
+      def restore_astral(yaml)
+        return yaml if substitutions.empty?
+
+        yaml.gsub(SENTINEL) { |token| substitutions.fetch(token) }
+      end
+
+      private
+
+      def substitutions
+        @substitutions ||= {}
+      end
+
+      def forward
+        @forward ||= {}
+      end
+    end
+  end
+end

data/lib/yaml_exporter/importer.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module YamlExporter
+  # Drives the import of a YAML document into a tree of ActiveRecord records.
+  #
+  # Works in a single transaction on the root record's class. For every
+  # record it touches (root + any owned child) it runs two phases:
+  #
+  #   1. pre_save  — nodes that mutate columns on *this* record
+  #                  (Attribute, OneReference).
+  #   2. save!     — the record is now persisted and has an id.
+  #   3. post_save — nodes that need parent.id to do their work
+  #                  (OneOwned, ManyPositional, ManyFindBy,
+  #                   ManyReference, ManyThrough).
+  #
+  # Each node is responsible for its own persistence; this class just
+  # orchestrates phases and validates unknown keys.
+  class Importer
+    def initialize(record, structure)
+      @root_record = record
+      @root_structure = structure
+    end
+
+    def import(yaml_string)
+      @root_record.class.transaction do
+        data = parse(yaml_string)
+        apply(@root_record, @root_structure, data, path: '')
+      end
+      @root_record
+    end
+
+    # Called recursively by nodes that own child records (OneOwned,
+    # ManyBase flavors) once they've built or found the child row.
+    #
+    # `extra_keys` lets the caller (typically a `many` node) declare keys
+    # that belong to *it*, not to `structure`. Example: `find_by: :slug`
+    # means `slug:` is legal at the entry level but isn't a declared
+    # Attribute on the child — it's the discriminator the parent node
+    # consumes. The child's own Attribute setters never see it because
+    # no Attribute node declares it; it simply passes validation.
+    def apply(record, structure, data, path:, extra_keys: [])
+      data = {} if data.nil?
+      unless data.is_a?(Hash)
+        raise UnknownAttributeError, "#{describe_path(path)}: expected a mapping, got #{data.class}"
+      end
+
+      validate_keys!(structure, data, path: path, extra_keys: extra_keys)
+
+      pre, post = structure.nodes.partition { |n| n.phase == :pre_save }
+      pre.each { |n| n.import(record, data, path: path, importer: self) }
+      record.save!
+      post.each { |n| n.import(record, data, path: path, importer: self) }
+    end
+
+    private
+
+    def parse(yaml_string)
+      data = YAML.safe_load(
+        yaml_string,
+        permitted_classes: [Date, Time, Symbol],
+        aliases: true
+      )
+      data = {} if data.nil?
+      unless data.is_a?(Hash)
+        raise UnknownAttributeError, "root: expected a mapping, got #{data.class}"
+      end
+      data
+    end
+
+    def validate_keys!(structure, data, path:, extra_keys: [])
+      declared = structure.nodes.flat_map(&:yaml_keys).to_set
+      extra_keys.each { |k| declared << k.to_s }
+      unknown = data.keys.map(&:to_s).reject { |k| declared.include?(k) }
+      return if unknown.empty?
+
+      location = describe_path(path)
+      unknown_list = unknown.map { |k| "'#{k}'" }.join(', ')
+      raise UnknownAttributeError,
+            "#{location}: unknown key#{unknown.size == 1 ? '' : 's'} #{unknown_list}"
+    end
+
+    def describe_path(path)
+      path.empty? ? 'root' : path
+    end
+  end
+end

data/lib/yaml_exporter/nodes/attribute.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module YamlExporter
+  module Nodes
+    # A single declared column on the current record.
+    #
+    # Phase: :pre_save (assigns a value on `record` before `record.save!`).
+    class Attribute
+      attr_reader :name
+
+      # `owner_class` may be a Class or a 0-arity callable. The callable form
+      # matches how the rest of the node tree defers reflection lookups —
+      # needed by the anonymous AR classes in dsl_validation_test.
+      def initialize(name:, owner_class: nil)
+        @name = name.to_sym
+        @owner_class_resolver =
+          if owner_class.nil?
+            nil
+          elsif owner_class.respond_to?(:call)
+            owner_class
+          else
+            -> { owner_class }
+          end
+      end
+
+      def phase
+        :pre_save
+      end
+
+      def yaml_keys
+        [@name.to_s]
+      end
+
+      # Missing key, explicit null, and provided value all normalize to the
+      # same AR setter call — so "no author" and "author: null" mean the
+      # same thing downstream.
+      def import(record, data, path:, importer:)
+        value = data.key?(@name.to_s) ? data[@name.to_s] : nil
+        record.public_send("#{@name}=", value)
+      end
+
+      # `text` columns export as YAML literal block scalars (`|`); we signal
+      # that by wrapping the string value in LiteralString. `string`/varchar
+      # columns stay inline regardless of length.
+      def export(record, exporter:)
+        value = record.public_send(@name)
+        value = LiteralString.new(value) if value.is_a?(::String) && text_column?
+        [@name.to_s, value]
+      end
+
+      def schema_fragment
+        { @name => { type: TypeInference.schema_type_for(owner_class, @name) } }
+      end
+
+      private
+
+      def text_column?
+        return @text_column if defined?(@text_column)
+
+        @text_column = TypeInference.text_column?(owner_class, @name)
+      end
+
+      def owner_class
+        return nil unless @owner_class_resolver
+
+        @owner_class ||= @owner_class_resolver.call
+      end
+    end
+  end
+end

data/lib/yaml_exporter/nodes/many_base.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module YamlExporter
+  module Nodes
+    # Shared base for `many` flavors that take a block:
+    # - ManyPositional (no find_by, no through)
+    # - ManyFindBy    (find_by, no through)
+    # - ManyThrough   (through + find_by)
+    #
+    # Holds the block's sub-structure and the common accessors. Subclasses
+    # override how each YAML entry maps to an existing-or-new child record.
+    # ManyReference is NOT a subclass — it has no block and different
+    # semantics.
+    #
+    # Target-class resolution is deferred: we store the owner class and the
+    # association name, and resolve the reflected class on demand. Eager
+    # resolution breaks anonymous ActiveRecord classes whose inverse
+    # associations (often referenced via string `class_name:`) can't be
+    # resolved at declaration time.
+    class ManyBase
+      attr_reader :name, :owner_class, :find_by, :positioned_by, :sub_structure
+
+      def initialize(name:, owner_class:, find_by: nil, positioned_by: nil, &block)
+        raise ArgumentError, "`many #{name.inspect}`: block required" unless block
+
+        @name = name.to_sym
+        @owner_class = owner_class
+        @find_by = find_by&.to_sym
+        @positioned_by = positioned_by&.to_sym
+
+        reflection = owner_class.reflect_on_association(@name)
+        unless reflection
+          raise ArgumentError,
+                "`many #{name.inspect}`: #{owner_class} has no association `#{name}`"
+        end
+
+        # Pass a lazy class resolver into the inner Builder: attribute-only
+        # blocks never trigger it.
+        @sub_structure = Builder.new(-> { entry_class }, &block).build
+
+        validate_positioned_by!
+      end
+
+      def phase
+        :post_save
+      end
+
+      def yaml_keys
+        [@name.to_s]
+      end
+
+      def target_class
+        @target_class ||= @owner_class.reflect_on_association(@name).klass
+      end
+
+      # Class against which the block is evaluated. For vanilla `many ... do
+      # end` this is the target class. ManyThrough overrides to return the
+      # join class (the block's attributes attach to the join row).
+      def entry_class
+        target_class
+      end
+
+      def import(parent, data, path:, importer:)
+        entries = data.key?(@name.to_s) ? data[@name.to_s] : nil
+        entries = [] if entries.nil?
+
+        unless entries.is_a?(Array)
+          raise UnknownAttributeError,
+                "#{list_path(path)}: expected a list, got #{entries.class}"
+        end
+
+        parent.association(@name).reset
+        existing = current_entries(parent)
+
+        # Object-identity set: `find_or_build_child` returns the exact same
+        # Ruby object from `existing` when it matches, so we don't have to
+        # invent a primary-key-based identifier (keeps this working for
+        # composite PKs, custom primary keys, and other non-id models).
+        kept = Set.new.compare_by_identity
+
+        entries.each_with_index do |entry, index|
+          unless entry.is_a?(Hash)
+            raise UnknownAttributeError,
+                  "#{entry_path(path, index)}: expected a mapping, got #{entry.class}"
+          end
+
+          child = find_or_build_child(parent, entry, index, existing: existing)
+          # positioned_by is DSL-owned: the column value is derived from the
+          # array index (1-based), never read from the YAML entry. We set it
+          # on the child before #apply so the pre-save phase persists it in
+          # the same round-trip as the block's attributes.
+          child.public_send("#{@positioned_by}=", index + 1) if @positioned_by
+
+          importer.apply(child, @sub_structure, entry,
+                         path: entry_path(path, index),
+                         extra_keys: extra_entry_keys)
+          kept << child
+        end
+
+        destroy_missing(existing, kept)
+      end
+
+      def export(parent, exporter:)
+        records = sort_for_export(Array(parent.public_send(@name)))
+        list = records.map { |child| exporter.build_hash(child, @sub_structure) }
+        [@name.to_s, list]
+      end
+
+      def schema_fragment
+        props = @sub_structure.nodes.each_with_object({}) { |n, acc| acc.merge!(n.schema_fragment) }
+        extra_entry_schema.each { |k, v| props[k.to_sym] = v }
+        { @name => { type: 'array', items: { type: 'object', properties: props } } }
+      end
+
+      # --- Subclass hooks --------------------------------------------------
+
+      # Return the child records currently associated with `parent`.
+      # Order is whatever the association's default scope produces — if
+      # you care (positional semantics), set `-> { order(...) }` on the
+      # `has_many`. ManyFindBy / ManyThrough match by column or
+      # association object, so order doesn't affect their correctness.
+      def current_entries(parent)
+        parent.public_send(@name).to_a
+      end
+
+      # Build a new child instance bound to `parent`. We delegate to the
+      # collection's `.build`, which is the PK-agnostic path: AR fills in the
+      # FK (and any polymorphic `*_type` column) by consulting the
+      # association's `association_primary_key`, so composite/custom primary
+      # keys keep working. Subclasses may override for join-table semantics.
+      def build_child(parent)
+        parent.public_send(@name).build
+      end
+
+      # Destroy children from `existing` that weren't touched this round.
+      # `kept` is a `compare_by_identity` Set of Ruby objects; no PK columns
+      # are consulted.
+      def destroy_missing(existing, kept)
+        existing.each do |child|
+          child.destroy! unless kept.include?(child)
+        end
+      end
+
+      # Used by Exporter to produce a stable list order. positioned_by,
+      # when present, wins over every other ordering rule. The `to_key`
+      # fallback is PK-agnostic (returns `[pk_value]` for surrogate-id rows
+      # and the composite array for composite-PK rows); nil (unsaved) folds
+      # to [] and leaves Ruby's stable sort to preserve incoming order.
+      def sort_for_export(records)
+        return records.sort_by { |r| [positioned_value(r), r.to_key || []] } if @positioned_by
+
+        default_export_order(records)
+      end
+
+      # Subclasses override to express their "natural" order (find_by for
+      # reference-style flavors). Default uses the record's primary key so
+      # we don't bake in `:id`.
+      def default_export_order(records)
+        records.sort_by { |r| r.to_key || [] }
+      end
+
+      # Additional entry-level keys the subclass considers legal but that
+      # aren't declared in the block (e.g. the `find_by` discriminator).
+      def extra_entry_keys
+        []
+      end
+
+      # Schema fragment for those extra entry keys. Default: infer the type
+      # from the `entry_class` (the class the block is evaluated against) —
+      # correct for ManyFindBy (entry_class == target_class). ManyThrough
+      # overrides because its extra key lives on target_class, not the join.
+      # TypeInference gracefully falls back to 'string' for unknown columns.
+      def extra_entry_schema
+        extra_entry_keys.each_with_object({}) do |k, acc|
+          acc[k.to_sym] = { type: TypeInference.schema_type_for(entry_class, k) }
+        end
+      end
+
+      # Child lookup/construction for a single YAML entry. Must return a
+      # record (new or existing) that will be passed through Importer#apply.
+      # Abstract here; subclasses must implement.
+      def find_or_build_child(_parent, _entry, _index, existing:) # rubocop:disable Lint/UnusedMethodArgument
+        raise NotImplementedError, "#{self.class} must implement #find_or_build_child"
+      end
+
+      private
+
+      # Nil positions sort last — treat them as the largest possible index so
+      # drifted DB rows still land in a deterministic spot when exporting.
+      def positioned_value(record)
+        value = record.public_send(@positioned_by)
+        value.nil? ? Float::INFINITY : value
+      end
+
+      def list_path(path)
+        path.empty? ? @name.to_s : "#{path}.#{@name}"
+      end
+
+      def entry_path(path, index)
+        "#{list_path(path)}[#{index}]"
+      end
+
+      def validate_positioned_by!
+        return if @positioned_by.nil?
+
+        declared = @sub_structure.nodes.grep(Nodes::Attribute).map(&:name)
+        return unless declared.include?(@positioned_by)
+
+        raise ArgumentError,
+              "`many #{@name.inspect}`: positioned_by column #{@positioned_by.inspect} " \
+              "cannot also appear in the block's `attributes` list — the DSL owns it"
+      end
+    end
+  end
+end