emf 0.1.2 → 0.2.0

data/lib/emf/emr/binary/thin_shell.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Emf
+  module Emr
+    module Binary
+      # The lazy and thin shells of a Record (see FastFill). Included by
+      # Record, so every method here is BinData's with its @field_objs
+      # settled first.
+      module ThinShell
+        # A record `FastFill` returns is a thin shell: the record's bytes and
+        # offset, no @field_objs. Its fields are copied from the filler's
+        # template, with the unpacked value assigned, the first time anything
+        # asks for them (every BinData path to a field goes through
+        # instantiate_obj_at); that first ask also makes @field_objs, in
+        # place, so the record keeps its identity and every field object
+        # made afterwards is the one every later read answers. Nothing read
+        # straight from @field_objs may see an unmade field as a default, so
+        # clear and clear? settle the shell first.
+        LAZY_IVARS = %i[@lazy_filler @lazy_data @lazy_base @lazy_built @lazy_patched].freeze
+
+        def initialize_instance
+          @lazy_filler = @lazy_data = @lazy_base = @lazy_built = @lazy_patched = nil
+          super
+        end
+
+        def clear
+          hydrate
+          @lazy_filler = nil
+          super
+        end
+
+        def clear?
+          instantiate_all_objs if @lazy_filler
+          super
+        end
+
+        # A frozen record cannot make @field_objs later, so it is made now;
+        # its fields still fill in lazily, as a BinData record's do.
+        def freeze
+          hydrate if @lazy_filler
+          super
+        end
+
+        # BinData's dup is shallow: a copy shares @field_objs, so a field
+        # either record makes or edits is the other's too. A thin source has
+        # no @field_objs yet, so it is made first and shared. The two also
+        # share their bytes, so neither is untouched any more.
+        def initialize_copy(source)
+          super
+          return unless @lazy_filler
+
+          source.send(:hydrate)
+          @field_objs = source.instance_variable_get(:@field_objs)
+          @lazy_patched = true
+          source.instance_variable_set(:@lazy_patched, true) unless source.frozen?
+        end
+
+        # The filler is generated code Marshal cannot dump, so a record is
+        # dumped as the plain BinData record it reads as: every field made,
+        # no lazy source.
+        def marshal_dump
+          instantiate_all_objs if @lazy_filler
+          @field_objs&.each { |field| field.settle_points if field.is_a?(FastFill::FlatPoints) }
+          (instance_variables - LAZY_IVARS).to_h { |name| [name, instance_variable_get(name)] }
+        end
+
+        def marshal_load(ivars)
+          ivars.each { |name, value| instance_variable_set(name, value) }
+          LAZY_IVARS.each { |name| instance_variable_set(name, nil) }
+        end
+
+        # Fixed-layout records serialize with one Array#pack, as WireAdapter
+        # already does; a block or a layout with floats is BinData's writer.
+        def to_binary_s(&block)
+          packed = built_binary_s unless block
+          return packed if packed
+
+          return thin_binary_s if !block && thin_parsed?
+
+          packed = FastFill.for_class(self.class)&.pack(self) unless block
+          packed || super
+        end
+
+        # [data, offset, untouched] of the bytes a shell is filled from, nil
+        # for any other record. `untouched` is true while no field has been
+        # made: a field not made is still its bytes, a field made may have
+        # been edited and is read from the wire.
+        def lazy_source
+          return unless @lazy_filler
+
+          [@lazy_data, @lazy_base, !@lazy_patched && (@field_objs.nil? || @field_objs.all?(&:nil?))]
+        end
+
+        # Makes a fresh record the thin shell of the record at `base` of
+        # `data`: no field made and no @field_objs. `params` and `names` are
+        # the template's, shared as a copy of it would share them. A built
+        # shell's data is the whole record, packed from the attributes.
+        def thin(data, base, filler, params, names, built)
+          @params = params
+          @field_names = names
+          @lazy_data = data
+          @lazy_base = base
+          @lazy_filler = filler
+          @lazy_built = built
+          self
+        end
+
+        # The size of a built shell no field of which, bar n_size, has been made
+        # (fixed layouts never change size, a point array's size is its bytes').
+        def num_bytes
+          return @lazy_data.bytesize if built_clean?
+          return @lazy_filler.record_size(@lazy_data, @lazy_base) if thin_parsed?
+
+          super
+        end
+
+        # The record's bytes while a built shell still is them (n_size patched
+        # in), else nil.
+        def built_binary_s
+          return unless built_clean?
+
+          bytes = @lazy_data.dup
+          size = patched_size
+          bytes[4, 4] = [size].pack("V") if size
+          bytes
+        end
+
+        # Appends `built_binary_s` to `out` and answers true, without the copy
+        # when n_size needs no patch; false (nothing appended) for any other
+        # record.
+        def append_built_to?(out)
+          return false unless built_clean?
+
+          out << (patched_size ? built_binary_s : @lazy_data)
+          true
+        end
+
+        # Appends the record's bytes to `out`: the built fast path, else
+        # `to_binary_s`.
+        def append_to(out)
+          return out if append_built_to?(out)
+
+          out << to_binary_s
+        end
+
+        private
+
+        # `n_size=` on a built shell whose n_size is not made: the new value
+        # goes into the shell's own bytes (only n_size is there) and the
+        # field is made, from them, if it is asked for. A value the bytes
+        # already hold writes nothing: a write would copy the buffer a
+        # `to_binary_s` dup still shares. Anything that is not
+        # a uint32 Integer is BinData's to assign (or to refuse); nil then,
+        # the assigned value otherwise.
+        def packed_assign(index, value)
+          return unless index == N_SIZE_INDEX && @lazy_built && @lazy_filler &&
+                        value.instance_of?(Integer) && value >= 0 && value <= 0xFFFF_FFFF
+
+          @lazy_patched = true
+          data = @lazy_data
+          low = value & 0xFFFF
+          high = value >> 16
+          return value if data.getbyte(4) | (data.getbyte(5) << 8) == low &&
+                          data.getbyte(6) | (data.getbyte(7) << 8) == high
+
+          data.setbyte(4, low & 0xFF)
+          data.setbyte(5, low >> 8)
+          data.setbyte(6, high & 0xFF)
+          data.setbyte(7, high >> 8)
+          value
+        end
+
+        N_SIZE_INDEX = 1
+        private_constant :N_SIZE_INDEX
+
+        # The n_size to patch into a built shell's bytes, nil when the bytes
+        # already hold it.
+        def patched_size
+          size = @field_objs&.[](N_SIZE_INDEX)&.value
+          size if size && size != @lazy_data.unpack1("V", offset: 4)
+        end
+
+        # A built shell whose bytes are still the record: only n_size, which
+        # is patched into them, may have been made.
+        def built_clean?
+          return false unless @lazy_built && @lazy_filler
+          return true if @field_objs.nil?
+
+          made = @field_objs.size - @field_objs.count(nil)
+          return true if made.zero?
+
+          # A value that is not an Integer is BinData's to write (or refuse).
+          made == 1 && @field_objs[N_SIZE_INDEX]&.value.instance_of?(Integer)
+        end
+
+        # A thin shell of a parsed record: its bytes are in the input.
+        def thin_parsed?
+          @field_objs.nil? && @lazy_filler && !@lazy_built
+        end
+
+        def thin_binary_s
+          @lazy_data.byteslice(@lazy_base, @lazy_filler.record_size(@lazy_data, @lazy_base))
+        end
+
+        # Makes @field_objs of a thin shell: every slot unmade.
+        def hydrate
+          @field_objs = Array.new(@lazy_filler.field_count) if @field_objs.nil?
+        end
+
+        def find_index_of(obj)
+          hydrate
+          super
+        end
+
+        def instantiate_obj_at(index)
+          return super unless @lazy_filler
+
+          hydrate
+          return unless @field_objs[index].nil?
+
+          @field_objs[index] = @lazy_filler.lazy_field(index, self, @lazy_data, @lazy_base)
+        end
+      end
+    end
+  end
+end

data/lib/emf/emr/binary/value_view/layout.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Emf
+  module Emr
+    module Binary
+      class ValueView
+        # The leaf and array layout of one wire class, from a fresh instance.
+        class Layout
+          # Unpack directive and width of each leaf class.
+          LEAVES = {
+            "BinData::Uint32le" => ["V", 4],
+            "BinData::Int32le" => ["l<", 4],
+            "BinData::Uint8" => ["C", 1],
+            "Emf::Binary::Float32Le" => ["e", 4]
+          }.freeze
+          # Unpack directive and element width of each point coordinate class.
+          POINTS = {
+            "BinData::Int32le" => ["l<", 8],
+            "BinData::Int16le" => ["s<", 4]
+          }.freeze
+          FLOAT = "Emf::Binary::Float32Le"
+          private_constant :LEAVES, :POINTS, :FLOAT
+
+          # tree: [value class, [[name, node], ...]]; a leaf node is [:leaf, index], the array [:array].
+          # leaves: { directive:, at:, float:, klass: } in wire order.
+          # array: { name:, directive:, bytes:, count:, klass: } or nil.
+          attr_reader :tree, :leaves, :array, :size, :classes
+
+          def initialize(wire_class)
+            @wire_class = wire_class
+            @leaves = []
+            @classes = []
+            @array = nil
+            @size = 0
+            sample = wire_class.new
+            @tree = struct_of(sample)
+            raise Unsupported, "layout does not tile" unless sample.num_bytes == @size
+          end
+
+          def leaf_paths(node = @tree, prefix = [])
+            return [prefix] if node.first == :leaf
+            return [] if node.first == :array
+
+            node.last.flat_map { |name, child| leaf_paths(child, prefix + [name]) }
+          end
+
+          # Index of the leaf holding the point count.
+          def count_index
+            slot = @tree.last.assoc(@array[:count])
+            raise Unsupported, "count field" unless slot && slot.last.first == :leaf
+
+            slot.last.last
+          end
+
+          private
+
+          def struct_of(struct)
+            children = struct.field_names.map { |name| [name, child_of(struct.public_send(name), name, struct)] }
+            klass = ValueView.value_class(struct.class, children.map(&:first))
+            @classes << klass
+            [klass, children]
+          end
+
+          def child_of(field, name, parent)
+            if field.respond_to?(:field_names)
+              struct_of(field)
+            elsif field.is_a?(BinData::Array)
+              array_of(field, name, parent)
+            else
+              leaf_of(field)
+            end
+          end
+
+          def leaf_of(field)
+            name = field.class.name
+            directive, width = LEAVES.fetch(name) { raise Unsupported, "leaf #{field.class}" }
+            @leaves << { directive: directive, at: @size, float: FLOAT.eql?(name), klass: name }
+            @size += width
+            [:leaf, @leaves.size - 1]
+          end
+
+          def array_of(field, name, parent)
+            raise Unsupported, "second array" if @array || !parent.instance_of?(@wire_class)
+            raise Unsupported, "array is not last" unless parent.field_names.last == name
+
+            point = field.send(:new_element)
+            directive, bytes = POINTS.fetch(point.x.class.name) { raise Unsupported, "point #{point.x.class}" }
+            raise Unsupported, "point members" unless point.field_names == %i[x y] && point.x.instance_of?(point.y.class)
+
+            count = field.send(:get_parameter, :initial_length)
+            raise Unsupported, "array count" unless count.is_a?(Symbol)
+
+            @array = { name: name, directive: directive, bytes: bytes, count: count,
+                       klass: ValueView.value_class(point.class, %i[x y]) }
+            [:array]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emf/emr/binary/value_view/reader_source.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Emf
+  module Emr
+    module Binary
+      class ValueView
+        # Source of the readers of a view: `read_bytes(data, base)`,
+        # `read_wire(wire)` and `read_mixed(data, base, wire)`.
+        class ReaderSource
+          def initialize(layout)
+            @layout = layout
+          end
+
+          # Source of `read_bytes(data, base)`, `read_wire(wire)` and
+          # `read_mixed(data, base, wire)`; each builds the value object from
+          # the locals `v0..vN` (unpacked leaves) and `points`, or from the
+          # wire's accessors.
+          def to_s
+            unpack = "#{locals} = data.unpack(#{@layout.leaves.map { |leaf| leaf[:directive] }.join.inspect}, offset: base)"
+            <<~RUBY
+              def read_bytes(data, base)
+                #{unpack}
+                #{array_source}
+                #{construct(@layout.tree, :bytes, 'wire')}
+              end
+
+              def read_wire(wire)
+                #{construct(@layout.tree, :wire, 'wire')}
+              end
+
+              def read_mixed(data, base, wire)
+                #{unpack}
+                #{array_source}
+                objs = wire.instance_variable_get(:@field_objs)
+                #{mixed_construct}
+              end
+            RUBY
+          end
+
+          private
+
+          def locals
+            @layout.leaves.each_index.map { |index| "v#{index}" }.join(", ")
+          end
+
+          # Source for the local `points` of an array record (nothing for
+          # others); the reader returns nil when the count overruns the record.
+          def array_source
+            array = @layout.array
+            return "" unless array
+
+            count = "v#{@layout.count_index}"
+            size = @layout.size
+            <<~RUBY
+              return if #{size} + (#{count} * #{array[:bytes]}) > data.unpack1("V", offset: base + 4) ||
+                        base + #{size} + (#{count} * #{array[:bytes]}) > data.bytesize
+
+              flat = data.unpack(#{array[:directive].inspect} + (#{count} * 2).to_s, offset: base + #{size})
+              points = Array.new(#{count}) { |i| POINT.new(flat[i * 2], flat[(i * 2) + 1]) }.freeze
+            RUBY
+          end
+
+          def construct(node, mode, chain)
+            return (mode == :bytes ? "v#{node.last}" : "#{chain}.value") if node.first == :leaf
+            if node.first == :array
+              return (mode == :bytes ? "points" : "#{chain}.map { |point| POINT.new(point.x.value, point.y.value) }.freeze")
+            end
+
+            klass, children = node
+            args = children.map { |name, child| construct(child, mode, "#{chain}.#{name}") }
+            "CLASSES[#{@layout.classes.index(klass)}].new(#{args.join(', ')})"
+          end
+
+          # `construct` for the top struct with each field taken from the wire
+          # when the shell has made it (its slot in @field_objs is set; a thin shell
+          # has no @field_objs and has made none).
+          def mixed_construct
+            klass, children = @layout.tree
+            args = children.each_with_index.map do |(name, child), slot|
+              "(objs&.[](#{slot}) ? #{construct(child, :wire, "wire.#{name}")} : #{construct(child, :bytes, '')})"
+            end
+            "CLASSES[#{@layout.classes.index(klass)}].new(#{args.join(', ')})"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emf/emr/binary/value_view.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Emf
+  module Emr
+    module Binary
+      # Field values of a fixed-layout or point-array wire record as plain,
+      # frozen Ruby objects, with no BinData field object made.
+      #
+      # A view is built once per wire class from the class's own field list
+      # and proved against BinData on a record with a distinct value in every
+      # leaf (the bytes reader, the wire reader and BinData's own read must
+      # agree, signedness and float values included). A class that cannot be
+      # proved has no view: `ValueView.for_class` returns nil.
+      #
+      # The value object mirrors the wire: one `Data` member per field name,
+      # nested structs (`rcl_bounds`, `xform`, `color`) as nested `Data`
+      # objects with the wire's member names, and a counted point array as a
+      # frozen Array of `Data` points (`x`, `y`). Integers are Integers
+      # (signed or unsigned as the wire declares); float32 leaves are Floats,
+      # NaN payloads included (the same float32 widening BinData applies).
+      #
+      # Three readers produce equal objects: `from_bytes` unpacks a record's
+      # bytes with one String#unpack, `from_wire` reads the wire's values, and
+      # `from_mixed` reads a lazy shell, taking each field the shell has made
+      # from the wire and the others from the bytes.
+      class ValueView
+        # Namespace of the generated value classes.
+        module Values; end
+
+        # A layout the view does not support.
+        class Unsupported < StandardError; end
+
+        autoload :Layout, "emf/emr/binary/value_view/layout"
+        autoload :ReaderSource, "emf/emr/binary/value_view/reader_source"
+
+        BINDATA_WRITER = BinData::Base.instance_method(:to_binary_s)
+        private_constant :BINDATA_WRITER
+
+        @views = {}.compare_by_identity
+        # Marks every value class, whichever base class it is built on.
+        module Value; end
+
+        @value_classes = {}.compare_by_identity
+        @lock = Mutex.new
+
+        class << self
+          # The view of a wire class, or nil when the class has none.
+          def for_class(wire_class)
+            @views.fetch(wire_class) { @lock.synchronize { @views[wire_class] = build(wire_class) } }
+          end
+
+          # The shared value class of a BinData struct class with these members.
+          def value_class(struct_class, members)
+            @value_classes[struct_class] ||= define_value_class(members).tap do |klass|
+              name = struct_class.name&.split("::")&.last
+              Values.const_set(name, klass) if name&.match?(/\A[A-Z]\w*\z/) && !Values.const_defined?(name, false)
+            end
+          end
+
+          private
+
+          # Data (Ruby 3.2+) where it exists; on Ruby 3.1 a Struct whose
+          # instances freeze themselves, so values stay frozen everywhere.
+          def define_value_class(members)
+            klass = defined?(::Data.define) ? ::Data.define(*members) : frozen_struct(members)
+            klass.include(Value)
+          end
+
+          def frozen_struct(members)
+            Struct.new(*members) do
+              def initialize(*)
+                super
+                freeze
+              end
+            end
+          end
+
+          def build(wire_class)
+            filler = FastFill.for_class(wire_class)
+            return unless filler && !filler.is_a?(FastFill::TextFill)
+
+            new(wire_class).tap(&:prove)
+          rescue StandardError
+            nil
+          end
+        end
+
+        attr_reader :wire_class, :value_class
+
+        def initialize(wire_class)
+          @wire_class = wire_class
+          @layout = Layout.new(wire_class)
+          @value_class = @layout.tree.first
+          generate
+        end
+
+        # The values of the record at `base` of the binary `data`; nil when
+        # its point count overruns the record.
+        def from_bytes(data, base) = read_bytes(data, base)
+
+        # The values of a lazy shell over the record at `base` of `data`:
+        # each top-level field the shell has made from the wire, the others
+        # from the bytes. Nil when the point count overruns the record.
+        def from_mixed(data, base, wire) = read_mixed(data, base, wire)
+
+        # The values the wire holds now.
+        def from_wire(wire) = read_wire(wire)
+
+        # Raises unless the readers agree with BinData on a distinct-valued record.
+        def prove
+          sample = distinct_sample
+          bytes = BINDATA_WRITER.bind_call(sample)
+          values = read_bytes("\xAA\xBB".b + bytes + "\xCC".b, 2)
+          reference = read_wire(@wire_class.read(bytes))
+          raise Unsupported, "bytes and wire differ" unless values == read_wire(sample) && values == reference
+          raise Unsupported, "a count past the record was read" unless overrun_rejected?(bytes)
+
+          self
+        end
+
+        private
+
+        def generate
+          mod = Module.new
+          mod.const_set(:CLASSES, @layout.classes.freeze)
+          mod.const_set(:POINT, @layout.array[:klass]) if @layout.array
+          mod.module_eval(ReaderSource.new(@layout).to_s, __FILE__, __LINE__)
+          extend(mod)
+        end
+
+        # A record with a distinct value in every leaf: negative signed, high
+        # unsigned, fractional floats.
+        def distinct_sample
+          wire = @wire_class.new
+          @layout.leaf_paths.each_with_index do |path, index|
+            target = path.inject(wire) { |obj, name| obj.public_send(name) }
+            target.assign(distinct(@layout.leaves.fetch(index), index))
+          end
+          fill_array(wire)
+        end
+
+        def distinct(leaf, index)
+          return index + 1.5 if leaf[:float]
+
+          case leaf[:klass]
+          when "BinData::Int32le" then -(index + 1) * 1000
+          when "BinData::Uint32le" then 0xFFFF_FF00 - index
+          else 0x80 + index
+          end
+        end
+
+        def fill_array(wire)
+          array = @layout.array
+          return wire unless array
+
+          limit = array[:bytes] == 8 ? 2**31 : 2**15
+          wire.public_send(array[:count]).assign(3)
+          wire.public_send(array[:name]).assign([{ x: -1, y: limit - 1 }, { x: 5, y: -limit }, { x: 0, y: 7 }])
+          wire
+        end
+
+        # A count claiming more points than the record holds reads as nil.
+        def overrun_rejected?(bytes)
+          return true unless @layout.array
+
+          bad = bytes.dup
+          bad[@layout.leaves.fetch(@layout.count_index)[:at], 4] = [1_000_000].pack("V")
+          read_bytes(bad, 0).nil?
+        end
+      end
+    end
+  end
+end

data/lib/emf/emr/binary/variable_buffers.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Emf
+  module Emr
+    module Binary
+      # Reads the offset/size-addressed variable parts of a record straight from
+      # its bytes, so they are visible even when a wire `body` or `trailing`
+      # field absorbed them. Offsets are from record start. A part that is
+      # absent, empty, starts inside the fixed layout or leaves the record is
+      # nil, never an error; nothing is computed at parse time.
+      #
+      #   fixed_layout 100                     # bytes of the fixed layout
+      #   variable_buffer :bmi_bytes, offset: :off_bmi_src, length: :cb_bmi_src
+      #
+      # offset, length and the fixed size are an Integer, a Symbol naming a
+      # reader on the record, or a lambda run on the record.
+      module VariableBuffers
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        # Class macros.
+        module ClassMethods
+          attr_reader :fixed_layout_size
+
+          def fixed_layout(size)
+            @fixed_layout_size = size
+          end
+
+          def buffer_specs
+            @buffer_specs ||= {}
+          end
+
+          # Declare a part and, unless reader is false, a reader named for it.
+          def variable_buffer(name, offset:, length:, reader: true)
+            buffer_specs[name] = [offset, length]
+            define_method(name) { buffer_range(name)&.then { |range| record_bytes.byteslice(range) } } if reader
+          end
+        end
+
+        # The bytes inside nSize after the fixed layout and after every declared
+        # part that lies inside the record, or "" when there are none. nil when
+        # nSize is smaller than the fixed layout or larger than the record.
+        def layout_surplus_bytes
+          bytes = record_bytes
+          size = n_size
+          fixed = resolve(self.class.fixed_layout_size)
+          return unless fixed && size >= fixed && size <= bytes.bytesize
+
+          finish = self.class.buffer_specs.each_key.filter_map { |name| buffer_range(name)&.end }.max
+          start = [fixed, finish.to_i].max
+          start >= size ? "".b : bytes.byteslice(start, size - start)
+        end
+
+        private
+
+        def record_bytes
+          to_binary_s
+        end
+
+        def buffer_range(name)
+          offset, length = self.class.buffer_specs.fetch(name).map { |part| resolve(part) }
+          return unless offset && length&.positive?
+          return if offset < resolve(self.class.fixed_layout_size)
+
+          finish = offset + length
+          (offset...finish) if finish <= record_bytes.bytesize
+        end
+
+        def resolve(part)
+          case part
+          when Integer then part
+          when Symbol then public_send(part)
+          else instance_exec(&part)
+          end
+        end
+
+        # Unsigned 32-bit value at a record offset, 0 past the end (so a part
+        # sized by it is empty).
+        def u32_at(offset)
+          raw = record_bytes.byteslice(offset, 4)
+          raw&.bytesize == 4 ? raw.unpack1("V") : 0
+        end
+      end
+    end
+  end
+end

data/lib/emf/emr/binary.rb

@@ -7,6 +7,10 @@ module Emf
       autoload :Record, "emf/emr/binary/record"
       autoload :WithBounds, "emf/emr/binary/with_bounds"
       autoload :TypeCodes, "emf/emr/binary/type_codes"
+      autoload :FastFill, "emf/emr/binary/fast_fill"
+      autoload :PackPlan, "emf/emr/binary/pack_plan"
+      autoload :WireCopy, "emf/emr/binary/fast_fill"
+      autoload :ValueView, "emf/emr/binary/value_view"
       autoload :Records, "emf/emr/binary/records"
     end
   end