herringbone 0.4.0 → 0.5.0

data/lib/herringbone/redaction.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # Rewrites a Parquet file with rows removed or column values replaced, for GDPR erasure ("forget
+  # me") and pseudonymization. A Redaction is built once and applied to any number of files, one
+  # file in and one file out:
+  #
+  #   forget = Herringbone::Redaction.new do |r|
+  #     r.where(user_id: 42).delete
+  #     r.where(email: "anna@example.com").replace(email: nil, name: nil)
+  #     r.replace(:phone) { |phone| phone&.gsub(/\d(?=\d{3})/, "#") }
+  #     r.drop :ssn
+  #   end
+  #   File.open("in.parquet", "rb") do |input|
+  #     File.open("out.parquet", "wb") { |output| forget.apply(input, output) }
+  #   end
+  #
+  # Statements apply in declared order, per row: a deleted row is gone for later statements, and
+  # later statements (their conditions and their blocks) see what earlier ones replaced.
+  #
+  # Row groups no statement touches are copied byte for byte. A row group where only leaf columns
+  # are replaced gets just those column chunks re-encoded. A row group with deleted rows, or with a
+  # nested field replaced as a whole, is rewritten. Either way no deleted or replaced value is
+  # left in data pages, dictionary pages, statistics, the page index or bloom filters.
+  class Redaction
+    # One statement of a Redaction
+    #
+    # @!attribute kind
+    #   @return [Symbol] +:delete+ or +:replace+
+    # @!attribute where
+    #   @return [Hash{String, Symbol => Object}, nil] conditions as given to #where, nil for every row
+    # @!attribute targets
+    #   @return [Array<String>] columns to replace (top-level names or dotted struct member paths)
+    # @!attribute constants
+    #   @return [Hash{String => Object}] column => value, for a replace without a block
+    # @!attribute block
+    #   @return [Proc, nil] computes the replacement values
+    Statement = Struct.new(:kind, :where, :targets, :constants, :block)
+
+    # What #apply did, for the audit trail an erasure needs
+    #
+    # @!attribute rows_read
+    #   @return [Integer] rows decoded (rows of row groups that were copied unread are not counted)
+    # @!attribute rows_deleted
+    #   @return [Integer] rows removed
+    # @!attribute rows_changed
+    #   @return [Integer] rows kept where at least one replace produced a different value
+    # @!attribute row_groups
+    #   @return [Hash{Symbol => Integer}] +{copied:, rewritten:}+ row group counts
+    Report = Struct.new(:rows_read, :rows_deleted, :rows_changed, :row_groups, keyword_init: true)
+
+    # Rows selected with Redaction#where, waiting for a verb: #delete or #replace
+    class Scope
+      # @return [Hash{String, Symbol => Object}] the conditions given to Redaction#where
+      attr_reader :conditions
+
+      # @param redaction [Redaction] the redaction the statement is added to
+      # @param conditions [Hash{String, Symbol => Object}] column => condition, as Reader#read(where:)
+      def initialize(redaction, conditions)
+        @redaction = redaction
+        @conditions = conditions
+        @used = false
+      end
+
+      # @return [Boolean] whether a verb was called on this scope
+      def used? = @used
+
+      # Removes the matching rows
+      #
+      # @return [Redaction] the redaction, for chaining
+      def delete
+        @used = true
+        @redaction.add_statement(Statement.new(:delete, @conditions, [], {}, nil))
+      end
+
+      # Replaces values in the matching rows, see Redaction#replace
+      #
+      # @param columns [Array<String, Symbol>] columns whose values the block computes
+      # @param constants [Hash{String, Symbol => Object}] column => value to set
+      # @option constants [Object] :any_column value to store in that column (keys are column names)
+      # @yield [value, row] once per matching row and named column
+      # @yieldparam value [Object] the current value
+      # @yieldparam row [Hash{String => Object}] the whole row, only when the block takes two parameters
+      # @yieldreturn [Object] the value to store
+      # @return [Redaction] the redaction, for chaining
+      # @raise [ArgumentError] when given both or neither of +constants+ and a block
+      def replace(*columns, **constants, &block)
+        statement = Redaction.replace_statement(@conditions, columns, constants, block)
+        @used = true
+        @redaction.add_statement(statement)
+      end
+
+      # Short summary for the console: the columns the conditions name, not their values
+      #
+      # @return [String]
+      def inspect
+        "#<#{self.class.name} where #{@conditions.keys.join(", ")}#{" (no verb yet)" unless @used}>"
+      end
+    end
+
+    # @return [Array<Statement>] the statements, in declared order
+    attr_reader :statements
+
+    # @return [Array<String>] fields and struct members to remove from the schema
+    attr_reader :drops
+
+    # Builds a redaction. The block receives the redaction and declares the statements on it,
+    # see the class description.
+    #
+    # @yield [r] declares the statements
+    # @yieldparam r [Redaction] the redaction being built
+    # @yieldreturn [void]
+    # @raise [ArgumentError] for a block that takes no parameter, an invalid statement, or a
+    #   #where left without a verb
+    def initialize(&block)
+      @statements = []
+      @drops = []
+      @scopes = []
+      return unless block
+      if block.arity.zero?
+        raise ArgumentError, "The block receives the redaction: Redaction.new { |r| r.where(user_id: 42).delete }"
+      end
+      yield self
+      check_scopes!
+    end
+
+    # Selects the rows the next verb applies to. Takes what Reader#read(where:) takes: values,
+    # Arrays (IN), Ranges, nil (IS NULL), callables and dotted struct member paths, all of which
+    # must hold.
+    #
+    #   where(user_id: 42).delete
+    #   where("address.city" => "Amsterdam", created_at: ..cutoff).replace(name: nil)
+    #
+    # @param conditions [Hash{String, Symbol => Object}, nil] column => condition
+    # @param more [Hash{String, Symbol => Object}] more conditions, as keyword arguments
+    # @option more [Object] :any_column condition on that column (keys are column names)
+    # @return [Scope] call #delete or #replace on it
+    # @raise [ArgumentError] when there are no conditions
+    def where(conditions = nil, **more)
+      raise ArgumentError, "where expects a Hash of column => condition" unless conditions.nil? || conditions.is_a?(Hash)
+      conditions = (conditions || {}).merge(more)
+      raise ArgumentError, "where needs at least one condition" if conditions.empty?
+      scope = Scope.new(self, conditions)
+      @scopes << scope
+      scope
+    end
+
+    # Replaces values in every row (or, after #where, in the matching rows). Either constants:
+    #
+    #   replace(email: nil, name: "[deleted]")
+    #
+    # or column names and a block computing each value from the current one, and from the whole
+    # row (a Hash with String keys, as Reader returns it) when the block takes two parameters:
+    #
+    #   replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", key, email) }
+    #   replace(:name) { |name, row| row["consent"] ? name : nil }
+    #
+    # A column is a top-level field or a struct member by dotted path. Values inside a list or map
+    # are replaced by replacing the whole field with a block that receives the Array or Hash.
+    # A struct member of a row whose struct is null is left alone.
+    #
+    # @param columns [Array<String, Symbol>] columns whose values the block computes
+    # @param constants [Hash{String, Symbol => Object}] column => value to set
+    # @option constants [Object] :any_column value to store in that column (keys are column names)
+    # @yield [value, row] once per row and named column
+    # @yieldparam value [Object] the current value
+    # @yieldparam row [Hash{String => Object}] the whole row, only when the block takes two parameters
+    # @yieldreturn [Object] the value to store
+    # @return [Redaction] self
+    # @raise [ArgumentError] when given both or neither of +constants+ and a block
+    def replace(*columns, **constants, &block)
+      add_statement(Redaction.replace_statement(nil, columns, constants, block))
+    end
+
+    # Removes columns from the schema: top-level fields, or struct members by dotted path
+    #
+    # @param columns [Array<String, Symbol>] columns to remove
+    # @return [Redaction] self
+    # @raise [ArgumentError] when no column is given
+    def drop(*columns)
+      raise ArgumentError, "drop needs at least one column" if columns.empty?
+      @drops.concat(columns.map(&:to_s))
+      self
+    end
+
+    # Internal (used by Scope): appends a statement
+    #
+    # @param statement [Statement] the statement to add
+    # @return [Redaction] self
+    def add_statement(statement)
+      @statements << statement
+      self
+    end
+
+    # Whether #apply would change +input_io+: true when a column is dropped, when a replace without
+    # #where meets a non-empty file, or when some row matches a #where. Reads as little as it can:
+    # row groups are ruled out with statistics, bloom filters and the page index first, and only
+    # the columns the conditions name are read for the rest.
+    #
+    # @param input_io [IO, StringIO] the Parquet file, read with #seek and #read
+    # @return [Boolean]
+    # @raise [ArgumentError] when the redaction does not fit the file's schema
+    def affects?(input_io)
+      check_scopes!
+      Rewriter.new(self, input_io).affects?
+    end
+
+    # Writes the redacted copy of +input_io+ to +output_io+. The redaction is checked against the
+    # file's schema first, so a missing column, a nil for a required column or a #where without a
+    # verb raise before anything is written. The output is left unfinished (no footer) when an
+    # error happens later, for instance a block returning a value its column cannot store.
+    #
+    # @param input_io [IO, StringIO] the Parquet file, read with #seek and #read; not closed
+    # @param output_io [IO, #write] destination, written sequentially; not closed
+    # @param writer_options [Hash{Symbol => Object}] Writer options for the re-encoded column chunks
+    #   (+compression:+, +bloom_filters:+, +page_rows:+, +dictionary:+...), and +metadata:+ to
+    #   replace the footer key/value metadata instead of copying it
+    # @option writer_options [Symbol] :compression (codec of each source chunk) codec for re-encoded chunks
+    # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+    #   columns whose re-encoded chunks get a bloom filter, besides those whose source chunk had one
+    # @option writer_options [Hash{String => String}] :metadata (the input's) footer key/value metadata
+    # @option writer_options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+    # @option writer_options [Integer] :page_rows (20_000) maximum rows per data page
+    # @option writer_options [Integer] :data_page_version (1) 1 or 2
+    # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
+    # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+    # @return [Report] what was done
+    # @raise [ArgumentError] when the redaction does not fit the file's schema, or for a writer
+    #   option that does not apply (+row_group_bytes:+, +row_group_rows:+)
+    # @raise [EncodeError] when a replacement value cannot be written to its column
+    def apply(input_io, output_io, **writer_options)
+      check_scopes!
+      Rewriter.new(self, input_io).apply(output_io, **writer_options)
+    end
+
+    # Short summary for the console: one clause per statement, naming the columns but not the
+    # condition values, since a where may hold a long list of ids
+    #
+    # @return [String]
+    def inspect
+      clauses = @statements.map do |s|
+        clause = (s.kind == :delete) ? "delete" : "replace #{s.targets.join(", ")}"
+        s.where ? "#{clause} where #{s.where.keys.join(", ")}" : clause
+      end
+      clauses << "drop #{@drops.join(", ")}" unless @drops.empty?
+      "#<#{self.class.name} #{clauses.empty? ? "(no statements)" : clauses.join("; ")}>"
+    end
+
+    # Builds a replace statement from the arguments of #replace
+    #
+    # @param where [Hash{String, Symbol => Object}, nil] conditions, nil for every row
+    # @param columns [Array<String, Symbol>] columns for the block
+    # @param constants [Hash{String, Symbol => Object}] column => value
+    # @param block [Proc, nil] computes the values
+    # @return [Statement]
+    # @raise [ArgumentError] when given both or neither of +constants+ and a block
+    def self.replace_statement(where, columns, constants, block)
+      if block
+        unless constants.empty?
+          raise ArgumentError, "replace takes column names and a block, or column => value pairs, not both"
+        end
+        raise ArgumentError, "replace with a block needs the names of the columns to replace" if columns.empty?
+        Statement.new(:replace, where, columns.map(&:to_s).uniq, {}, block)
+      else
+        unless columns.empty?
+          raise ArgumentError, "replace(#{columns.map(&:inspect).join(", ")}) needs a block computing the " \
+            "values; use replace(column: value) to set constants"
+        end
+        raise ArgumentError, "replace needs columns: replace(email: nil) or replace(:email) { |v| ... }" if constants.empty?
+        constants = constants.transform_keys(&:to_s)
+        Statement.new(:replace, where, constants.keys, constants, nil)
+      end
+    end
+
+    private
+
+    # @return [void]
+    # @raise [ArgumentError] when a #where was not followed by #delete or #replace
+    def check_scopes!
+      idle = @scopes.reject(&:used?)
+      return if idle.empty?
+      raise ArgumentError, "where(#{idle.first.conditions.inspect}) needs a verb: .delete or .replace(...)"
+    end
+  end
+end
+
+require_relative "redaction/rewriter"

data/lib/herringbone/schema.rb

@@ -248,12 +248,19 @@ module Herringbone
 
     # Builds a schema with the DSL, see Schema::Builder
     #
-    # @yield block evaluated with +instance_eval+ on a Builder, declaring the top-level fields
+    #   Herringbone::Schema.define do |s|
+    #     s.int64 :id, null: false
+    #     s.string :name
+    #   end
+    #
+    # @yield [s] declares the top-level fields
+    # @yieldparam s [Builder] the builder to declare fields on
+    # @yieldreturn [void]
     # @return [Schema]
-    # @raise [ArgumentError] when no field is declared, or a declaration is invalid
+    # @raise [ArgumentError] when no field is declared, a declaration is invalid, or the block takes
+    #   no parameter
     def self.define(&block)
-      builder = Builder.new
-      builder.instance_eval(&block) if block
+      builder = Builder.build("Herringbone::Schema.define { |s| s.int64 :id }", &block)
       raise ArgumentError, "A schema needs at least one field" if builder.nodes.empty?
       new(Node.new(name: "schema", repetition: :required, children: builder.nodes))
     end
@@ -267,19 +274,19 @@ module Herringbone
     # Time/DateTime -> timestamp(micros), Date -> date, BigDecimal -> decimal(38, max scale seen),
     # Hash -> struct, Array -> list. Columns that are nil in every sampled row become strings.
     # Fields declared in the block (Builder DSL) replace the inferred ones of the same name:
-    #   Schema.infer(rows) { json :payload }
+    #   Schema.infer(rows) { |s| s.json :payload }
     #
     # @param rows [Enumerable<Hash, Object>] rows to sample; only the first INFER_SAMPLE are read
-    # @yield optional block evaluated with +instance_eval+ on a Builder, declaring fields that
-    #   replace inferred ones (or are added after them)
+    # @yield [s] optional, declares fields that replace inferred ones (or are added after them)
+    # @yieldparam s [Builder] the builder to declare fields on
+    # @yieldreturn [void]
     # @return [Schema]
-    # @raise [ArgumentError] when there are no rows, a row is not Hash-like, or a column mixes
-    #   values that map to no single Parquet type
+    # @raise [ArgumentError] when there are no rows, a row is not Hash-like, a column mixes values
+    #   that map to no single Parquet type, or the block takes no parameter
     def self.infer(rows, &block)
+      overrides = Builder.build("Herringbone::Schema.infer(rows) { |s| s.json :payload }", &block)
       sample_rows = rows.first(INFER_SAMPLE).map { |r| Inference.row_hash(r) }
       raise ArgumentError, "Cannot infer a schema from zero rows" if sample_rows.empty?
-      overrides = Builder.new
-      overrides.instance_eval(&block) if block
       declared = overrides.nodes.to_h { |n| [n.name, n] }
       names = sample_rows.flat_map { |r| r.keys.map(&:to_s) }.uniq
       nodes = names.map do |name|
@@ -355,7 +362,7 @@ module Herringbone
         else
           classes = values.map(&:class).uniq
           raise ArgumentError, "Cannot infer a Parquet type for #{name} from #{classes.map(&:name).join(", ")}; " \
-            "declare it in a block: Schema.infer(rows) { string :#{name} }"
+            "declare it in a block: Schema.infer(rows) { |s| s.string :#{name} }"
         end
       end
     end
@@ -511,30 +518,58 @@ module Herringbone
 
     # DSL for defining schemas:
     #
-    #   Herringbone::Schema.define do
-    #     int64 :id, null: false
-    #     string :name
-    #     list :tags, :string
-    #     map :scores, :string, :double
-    #     struct :address do
-    #       string :city
+    #   Herringbone::Schema.define do |s|
+    #     s.int64 :id, null: false
+    #     s.string :name
+    #     s.list :tags, :string
+    #     s.map :scores, :string, :double
+    #     s.struct :address do |address|
+    #       address.string :city
     #     end
-    #     decimal :price, precision: 12, scale: 2
-    #     timestamp :created_at, unit: :micros
+    #     s.decimal :price, precision: 12, scale: 2
+    #     s.timestamp :created_at, unit: :micros
     #   end
     #
-    # Fields are nullable unless null: false is given.
+    # Fields are nullable unless null: false is given. The blocks of #struct, #list and #map get
+    # a Builder of their own.
     class Builder
       # @return [Array<Node>] fields declared so far, in declaration order
       attr_reader :nodes
 
-      # Starts with no fields; Schema.define evaluates the block against this builder
+      # Yields a new Builder to the block.
+      #
+      # @param usage [String] how the entry point is called with a block, for the error message
+      # @yield [s] declares fields
+      # @yieldparam s [Builder] the new builder
+      # @yieldreturn [void]
+      # @return [Builder] the builder, with no fields when there is no block
+      # @raise [ArgumentError] when the block takes no parameter
+      def self.build(usage, &block)
+        check_block!(block, usage)
+        builder = new
+        block&.call(builder)
+        builder
+      end
+
+      # A block without a parameter was most likely written for the +instance_eval+ DSL of
+      # earlier versions, and would fail on its first declaration with a NoMethodError
+      #
+      # @param block [Proc, nil] the block given to the entry point
+      # @param usage [String] how the entry point is called with a block, for the error message
+      # @return [void]
+      # @raise [ArgumentError] when the block takes no parameter
+      def self.check_block!(block, usage)
+        return if block.nil? || !block.parameters.empty?
+        raise ArgumentError, "The block receives the schema builder as a parameter: #{usage}"
+      end
+
+      # Starts with no fields
       def initialize
         @nodes = []
       end
 
       # Types that need no options; each gets a DSL method taking a name and the options of #column,
-      # e.g. +int64 :id, null: false+
+      # e.g. +s.int64 :id, null: false+
       PRIMITIVES = %i[
         boolean int8 int16 int32 int64 uint8 uint16 uint32 uint64 float double float16
         string binary json bson uuid date int96
@@ -595,17 +630,21 @@ module Herringbone
       # @param name [String, Symbol] field name
       # @param null [Boolean] whether the struct as a whole may be null
       # @param field_id [Integer, nil] field id to store in the schema
-      # @yield block evaluated with +instance_eval+ on a new Builder, declaring the struct's fields
+      # @yield [struct] declares the struct's fields
+      # @yieldparam struct [Builder] a new builder for the struct's fields
+      # @yieldreturn [void]
       # @return [Node] the added group node
-      # @raise [ArgumentError] when the block declares no fields (or is missing), or for a duplicate name
+      # @raise [ArgumentError] when the block is missing, takes no parameter or declares no fields,
+      #   or for a duplicate name
       def struct(name, null: true, field_id: nil, &block)
-        add Node.new(name: name, repetition: rep(null), children: struct_fields(name, &block), field_id: field_id)
+        children = struct_fields(name, usage(name, "struct :#{name}", "string :city"), &block)
+        add Node.new(name: name, repetition: rep(null), children: children, field_id: field_id)
       end
 
-      # list :tags, :string
-      # list :tags, :string, element_null: false
-      # list :points, :struct do double :x; double :y; end
-      # list :matrix do list :element, :double end   (block declares the element)
+      #   s.list :tags, :string
+      #   s.list :tags, :string, element_null: false
+      #   s.list :points, :struct do |points| points.double :x; points.double :y end
+      #   s.list :matrix do |matrix| matrix.list :element, :double end # block declares the element
       #
       # Written as the standard 3-level LIST: an optional (or required) group holding a repeated
       # group "list" whose single child is "element".
@@ -623,21 +662,28 @@ module Herringbone
       # @option type_opts [Symbol] :unit time or timestamp unit
       # @option type_opts [Boolean] :utc time or timestamp UTC adjustment
       # @option type_opts [Integer] :length FIXED_LEN_BYTE_ARRAY width
-      # @yield block evaluated with +instance_eval+ on a new Builder: the struct's fields for a
-      #   +:struct+ element, or exactly one field (renamed to "element") when +type+ is nil
+      # @yield [list] declares the struct's fields for a +:struct+ element, or exactly one field
+      #   (renamed to "element") when +type+ is nil
+      # @yieldparam list [Builder] a new builder for the element
+      # @yieldreturn [void]
       # @return [Node] the added LIST group node
-      # @raise [ArgumentError] when neither a type nor a block is given, the block declares the wrong
-      #   number of fields, or for a duplicate name
+      # @raise [ArgumentError] when neither a type nor a block is given, the block takes no parameter
+      #   or declares the wrong number of fields, or for a duplicate name
       def list(name, type = nil, null: true, element_null: true, field_id: nil, **type_opts, &block)
-        element = element_node("element", type, element_null, type_opts, &block)
+        example = if type
+          usage(name, "list :#{name}, :#{type}", "double :x")
+        else
+          usage(name, "list :#{name}", "list :element, :double")
+        end
+        element = element_node("element", type, element_null, type_opts, example, &block)
         repeated = Node.new(name: "list", repetition: :repeated, children: [element])
         add Node.new(name: name, repetition: rep(null), children: [repeated],
           logical_type: Format::LogicalType.new(list: Format::ListType.new),
           converted_type: Format::ConvertedType::LIST, field_id: field_id)
       end
 
-      # map :scores, :string, :double
-      # map :things, :string, :struct do int32 :a end
+      #   s.map :scores, :string, :double
+      #   s.map :things, :string, :struct do |things| things.int32 :a end
       #
       # Written as the standard MAP: a group holding a repeated group "key_value" with a required
       # "key" and a "value". Keys are never null.
@@ -655,20 +701,24 @@ module Herringbone
       # @option type_opts [Symbol] :unit time or timestamp unit
       # @option type_opts [Boolean] :utc time or timestamp UTC adjustment
       # @option type_opts [Integer] :length FIXED_LEN_BYTE_ARRAY width
-      # @yield block evaluated with +instance_eval+ on a new Builder: the struct's fields for a
-      #   +:struct+ value, or exactly one field (renamed to "value") when +value_type+ is nil
+      # @yield [map] declares the struct's fields for a +:struct+ value, or exactly one field
+      #   (renamed to "value") when +value_type+ is nil
+      # @yieldparam map [Builder] a new builder for the value
+      # @yieldreturn [void]
       # @return [Node] the added MAP group node
-      # @raise [ArgumentError] for an invalid key or value declaration, or a duplicate name
+      # @raise [ArgumentError] for an invalid key or value declaration, a block that takes no
+      #   parameter, or a duplicate name
       def map(name, key_type, value_type = nil, null: true, value_null: true, field_id: nil, **type_opts, &block)
-        key = element_node("key", key_type, false, {})
-        value = element_node("value", value_type, value_null, type_opts, &block)
+        head = ["map :#{name}", ":#{key_type}", (":#{value_type}" if value_type)].compact.join(", ")
+        key = element_node("key", key_type, false, {}, nil)
+        value = element_node("value", value_type, value_null, type_opts, usage(name, head, "int32 :a"), &block)
         kv = Node.new(name: "key_value", repetition: :repeated, children: [key, value])
         add Node.new(name: name, repetition: rep(null), children: [kv],
           logical_type: Format::LogicalType.new(map: Format::MapType.new),
           converted_type: Format::ConvertedType::MAP, field_id: field_id)
       end
 
-      # Generic column declaration: column :name, :int32, null: false
+      # Generic column declaration: s.column :name, :int32, null: false
       #
       # @param name [String, Symbol] field name
       # @param type [Symbol, String] DSL type: one of PRIMITIVES, or +:time+, +:timestamp+,
@@ -729,40 +779,54 @@ module Herringbone
       # @param type [Symbol, String, nil] DSL type, +:struct+, or nil to take the field the block declares
       # @param nullable [Boolean] whether the node may be null (not applied to a block-declared field)
       # @param type_opts [Hash{Symbol => Object}] type options passed to Types.physical_attributes
-      # @yield block evaluated with +instance_eval+ on a new Builder
+      # @param example [String, nil] the declaration called with a block, for the error message
+      # @yield [inner] declares the element, or the fields of a +:struct+
+      # @yieldparam inner [Builder] a new builder
+      # @yieldreturn [void]
       # @return [Node] the element node, not added to #nodes
-      # @raise [ArgumentError] when the block is missing, declares the wrong number of fields, or
-      #   declares no fields for a +:struct+
-      def element_node(name, type, nullable, type_opts, &block)
+      # @raise [ArgumentError] when the block is missing or takes no parameter, declares the wrong
+      #   number of fields, or declares no fields for a +:struct+
+      def element_node(name, type, nullable, type_opts, example, &block)
         if type.nil?
           raise ArgumentError, "Give an element type or a block declaring the element" unless block
-          inner = Builder.new
-          inner.instance_eval(&block)
+          inner = Builder.build(example, &block)
           raise ArgumentError, "The element block must declare exactly one field" unless inner.nodes.size == 1
           node = inner.nodes.first
           node.name = name
           node
         elsif type.to_sym == :struct
-          Node.new(name: name, repetition: rep(nullable), children: struct_fields(name, &block))
+          Node.new(name: name, repetition: rep(nullable), children: struct_fields(name, example, &block))
         else
           leaf_node(name, type, rep(nullable), type_opts)
         end
       end
 
-      # Evaluates a struct's block on a new Builder; Parquet groups need at least one child.
+      # Yields a new Builder for a struct's fields; Parquet groups need at least one child.
       #
       # @param name [String, Symbol] struct name, for the error message
-      # @yield block evaluated with +instance_eval+ on a new Builder, declaring the struct's fields
+      # @param example [String] the declaration called with a block, for the error message
+      # @yield [inner] declares the struct's fields
+      # @yieldparam inner [Builder] a new builder
+      # @yieldreturn [void]
       # @return [Array<Node>] the declared fields
-      # @raise [ArgumentError] when the block is missing or declares no fields
-      def struct_fields(name, &block)
+      # @raise [ArgumentError] when the block is missing, takes no parameter or declares no fields
+      def struct_fields(name, example, &block)
         raise ArgumentError, "struct #{name} needs a block declaring its fields" unless block
-        inner = Builder.new
-        inner.instance_eval(&block)
+        inner = Builder.build(example, &block)
         raise ArgumentError, "struct #{name} has no fields" if inner.nodes.empty?
         inner.nodes
       end
 
+      # @param name [String, Symbol] field name, which names the block parameter when it can
+      # @param head [String] the declaration without its block, e.g. "struct :address"
+      # @param declaration [String] a declaration for the block's body, e.g. "string :city"
+      # @return [String] the declaration with a block taking a parameter, e.g.
+      #   "s.struct :address do |address| address.string :city end"
+      def usage(name, head, declaration)
+        var = name.to_s.match?(/\A[a-z_][a-z0-9_]*\z/) ? name : "inner"
+        "s.#{head} do |#{var}| #{var}.#{declaration} end"
+      end
+
       # @param name [String, Symbol] field name
       # @param type [Symbol, String] DSL type
       # @param repetition [Symbol] +:optional+ or +:required+

data/lib/herringbone/simple_writer.rb

@@ -23,7 +23,7 @@ module Herringbone
   # Columns declared in a block (Schema::Builder DSL) replace inferred ones, e.g. for a column that
   # holds both numbers and text:
   #
-  #   Herringbone::SimpleWriter.new(io) { string :code }
+  #   Herringbone::SimpleWriter.new(io) { |s| s.string :code }
   class SimpleWriter
     # Opens a writer, yields it and closes it, finishing the file. If the block raises, the file is
     # left unfinished (no footer), as with Writer.open. To declare columns, use #initialize and #close.
@@ -60,13 +60,16 @@ module Herringbone
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     #   (other Writer options are passed on as well)
-    # @yield optional block evaluated with +instance_eval+ on a Schema::Builder, declaring columns
-    #   (named in #headers!) that replace inferred ones
+    # @yield [s] optional, declares columns (named in #headers!) that replace inferred ones
+    # @yieldparam s [Schema::Builder] the builder to declare columns on
+    # @yieldreturn [void]
     # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    # @raise [ArgumentError] when the block takes no parameter
     def initialize(io, **options, &overrides)
+      Schema::Builder.check_block!(overrides, "Herringbone::SimpleWriter.new(io) { |s| s.string :code }")
       @headers = nil
       @overrides = overrides
-      fix = "Herringbone::SimpleWriter.new(io) { %s }"
+      fix = "Herringbone::SimpleWriter.new(io) { |s| s.%s }"
       @writer = InferringWriter.new(io, fix: fix, **options) { |sample| schema_for(sample) }
     end
 
@@ -117,6 +120,13 @@ module Herringbone
     # @return [void]
     def abort = @writer.abort
 
+    # Short summary for the console, without the rows
+    #
+    # @return [String] the number of columns and the underlying writer's summary
+    def inspect
+      "#<#{self.class.name} columns=#{@headers&.size.inspect} writer=#{@writer.inspect}>"
+    end
+
     private
 
     # @param row [Array, Hash] row given to #<<

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end