yeptris 0.1.0

data/lib/yeptris/psych.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require "yeptris"
+require "yeptris/psych/handler"
+require "yeptris/psych/parser"
+require "yeptris/psych/coder_shim"
+require "yeptris/psych/visitors"
+
+# The Psych drop-in namespace (TODO.impl/15 phase C).
+#
+# `require "yeptris/psych"` rebinds the top-level Psych constant to
+# this module (the original, if any, stays reachable as
+# ::Psych::ORIGINAL). Semantics follow the Psych suite: load is
+# SAFE by default (Psych 5 behavior — plain data only; anything
+# tagged raises), unsafe_load materializes everything the yeptris
+# loader understands, and parse returns the Nodes tree over the
+# document without materializing.
+module Yeptris
+  module Psych
+    class Error < StandardError; end
+    class SyntaxError < Error
+      attr_reader :line, :column
+
+      def initialize(message, line = 0, column = 0)
+        super(message)
+        @line = line
+        @column = column
+      end
+    end
+    class BadAlias < Error; end
+    class DisallowedClass < Error
+      attr_reader :name
+
+      def initialize(name)
+        super("Tried to load unspecified class: #{name}")
+        @name = name
+      end
+    end
+    class AliasNotEnabled < Error; end
+
+    class << self
+      # Psych 5: load is safe — plain data structures only. Tagged
+      # nodes raise DisallowedClass unless their class is permitted
+      # (Date/Time/Symbol are built in; they are plain data here).
+      def load(yaml, permitted_classes: [], aliases: false, **)
+        safe_load(yaml, permitted_classes: permitted_classes, aliases: aliases)
+      end
+
+      # Full revival over the Nodes tree: !ruby/object, !ruby/struct,
+      # !ruby/set, encode_with/init_with, alias identity. Plain-data
+      # loads should use load/safe_load (the Materializer fast path).
+      def unsafe_load(yaml, **)
+        tree = parse(yaml)
+        return nil if tree.nil?
+
+        Visitors::ToRuby.visit(tree.children.first)
+      end
+
+      def safe_load(yaml, permitted_classes: [], aliases: false, **)
+        doc = Yeptris::Document.parse(yaml, schema: :compat_11)
+        begin
+          walk_safe(doc.root(0), permitted_classes, aliases)
+        ensure
+          doc.free
+        end
+      end
+
+      # The first document's node tree (no Ruby materialization).
+      def parse(yaml)
+        doc = Yeptris::Document.parse(yaml, schema: :compat_11)
+        return nil if doc.document_count.zero?
+
+        Nodes::Builder.document(doc)
+      rescue Yeptris::ParseError => e
+        raise SyntaxError, e.message
+      end
+
+      def parse_stream(yaml)
+        doc = Yeptris::Document.parse(yaml, schema: :compat_11)
+        return nil if doc.document_count.zero?
+
+        stream = Nodes::Stream.new
+        (0...doc.document_count).each do |i|
+          stream.children << Nodes::Builder.document_stream_child(doc, i)
+        end
+        stream.instance_variable_set(:@owner, doc)
+        ObjectSpace.define_finalizer(
+          stream, proc { doc.free unless doc.freed? }
+        )
+        stream
+      rescue Yeptris::ParseError => e
+        raise SyntaxError, e.message
+      ensure
+        doc&.free if doc && !stream
+      end
+
+      # Arbitrary object graphs through the YAMLTree visitor
+      # (anchors, !ruby/ tags); plain data rides the fast builder.
+      def dump(obj, io = nil)
+        # scalars take the fast builder; EVERYTHING else (including
+        # plain containers — they may nest custom objects) goes
+        # through the visitor, which builds the same DOM for plain
+        # data anyway
+        out =
+          case obj
+          when nil, true, false, String, Integer, Float, Symbol, Date, Time
+            Yeptris::YAML.dump(obj)
+          else
+            Visitors::YAMLTree.new.push(obj).finish
+          end
+        return out unless io
+
+        io.write(out)
+        io
+      end
+
+      private
+
+      # yeptris materializes plain data only — there is nothing
+      # unsafe it COULD load. The safety walk enforces what Psych
+      # enforces on such documents: aliases need opt-in, and explicit
+      # non-core tags raise DisallowedClass (the only "classes" the
+      # loader can produce are core-schema types, all permitted).
+      def walk_safe(root, permitted, aliases_enabled)
+        check(root, aliases_enabled) if root
+        root.to_ruby
+      end
+
+      def check(node, aliases_enabled)
+        case node.kind
+        when :alias
+          raise AliasNotEnabled, "Unknown alias" unless aliases_enabled
+        when :scalar, :mapping, :sequence
+          tag = node.tag
+          unless tag.nil?
+            name = tag.split(":").last
+            unless %w[str int float bool null timestamp seq map merge value
+                      binary].include?(name)
+              raise DisallowedClass, name
+            end
+          end
+          case node.kind
+          when :mapping
+            node.each_pair do |k, v|
+              check(k, aliases_enabled)
+              check(v, aliases_enabled)
+            end
+          when :sequence
+            node.each { |e| check(e, aliases_enabled) }
+          end
+        end
+      end
+    end
+
+    # Psych::Nodes over the yeptris document: the tree IS the parsed
+    # document (children are node handles, not copies) — parse cost
+    # is the parse, and to_ruby reuses the Materializer.
+    module Nodes
+      class Node
+        include Enumerable
+
+        attr_reader :children
+        attr_reader :handle # @api private — the Yeptris::Node
+
+        def initialize(handle = nil, children = [])
+          @handle = handle
+          @children = children
+        end
+
+        def each(&block)
+          @children.each(&block)
+        end
+
+        # The Ruby object for this subtree (Materializer semantics).
+        def to_ruby
+          @handle.to_ruby
+        end
+      end
+
+      class Stream < Node
+        def free
+          @owner&.free
+        end
+      end
+
+      # Owns the underlying Yeptris::Document: node handles in the
+      # tree stay valid while the tree is reachable; a GC finalizer
+      # releases the C memory when it is not.
+      class Document < Node
+        attr_reader :version, :tags
+
+        def initialize(version = [], tags = {})
+          super(nil)
+          @version = version
+          @tags = tags
+        end
+
+        # @api private — transfer of C ownership to this tree
+        def own(yeptris_doc)
+          @owner = yeptris_doc
+          ObjectSpace.define_finalizer(
+            self, proc { yeptris_doc.free unless yeptris_doc.freed? }
+          )
+          self
+        end
+
+        def free
+          @owner&.free
+        end
+      end
+
+      class Scalar < Node
+        attr_reader :value, :tag, :anchor, :plain, :quoted, :style
+
+        def initialize(value = nil, anchor: nil, tag: nil, plain: true,
+                       quoted: false, style: :plain)
+          super(nil)
+          @value = value
+          @anchor = anchor
+          @tag = tag
+          @plain = plain
+          @quoted = quoted
+          @style = style
+        end
+      end
+
+      class Sequence < Node
+        attr_reader :anchor, :tag, :style
+
+        def initialize(anchor: nil, tag: nil, style: :block)
+          super(nil, [])
+          @anchor = anchor
+          @tag = tag
+          @style = style
+        end
+      end
+
+      class Mapping < Node
+        attr_reader :anchor, :tag, :style
+
+        def initialize(anchor: nil, tag: nil, style: :block)
+          super(nil, [])
+          @anchor = anchor
+          @tag = tag
+          @style = style
+        end
+      end
+
+      class Alias < Node
+        attr_reader :anchor
+
+        def initialize(anchor)
+          super(nil)
+          @anchor = anchor
+        end
+      end
+
+      # Builds the Nodes tree from a parsed document.
+      module Builder
+        module_function
+
+        # parse(): one document, owning the yeptris document
+        def document(doc, index = 0)
+          document_stream_child(doc, index).own(doc)
+        end
+
+        # parse_stream(): a child document sharing one owner (the
+        # stream owns the yeptris document)
+        def document_stream_child(doc, index)
+          root = doc.root(index)
+          d = Document.new
+          d.instance_variable_set(:@handle, root&.document)
+          d.children << node(root) if root
+          d
+        end
+
+        def node(n)
+          case n.kind
+          when :mapping
+            m = Mapping.new(anchor: n.anchor, tag: n.tag)
+            n.each_pair do |k, v|
+              m.children << node(k)
+              m.children << node(v)
+            end
+            m.instance_variable_set(:@handle, n)
+            m
+          when :sequence
+            s = Sequence.new(anchor: n.anchor, tag: n.tag)
+            n.each { |e| s.children << node(e) }
+            s.instance_variable_set(:@handle, n)
+            s
+          when :alias
+            a = Alias.new(n.value)
+            a.instance_variable_set(:@handle, n)
+            a
+          else
+            sc = Scalar.new(n.value, anchor: n.anchor, tag: n.tag,
+                                  plain: n.style == :plain, style: n.style)
+            sc.instance_variable_set(:@handle, n)
+            sc
+          end
+        end
+      end
+    end
+  end
+end
+
+# The drop-in: rebind the top-level constant ( Psych-stdlib, if
+# already loaded, stays reachable as Yeptris::Psych::ORIGINAL).
+if defined?(::Psych) && !::Psych.equal?(Yeptris::Psych) &&
+   !Yeptris::Psych.const_defined?(:ORIGINAL, false)
+  Yeptris::Psych.const_set(:ORIGINAL, ::Psych)
+end
+Object.send(:remove_const, :Psych) if defined?(::Psych)
+::Psych = Yeptris::Psych

data/lib/yeptris/valueml.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module Yeptris
+  # Value-stream materialization (TODO.impl/15 phase F): one drain of
+  # PRE-CONVERTED typed values from the C side, then a minimal Ruby
+  # walk — no per-scalar parsing (the number kernels already ran),
+  # key/value pairing rides the entries' is_key bit, anchors arrive
+  # as uniform entries decorating the value they bind. The Psych
+  # quirks re-decide from tag_id + the raw bytes every entry carries.
+  module ValueML
+    DOC = 0
+    V_NULL = 1
+    V_BOOL = 2
+    V_INT = 3
+    V_FLOAT = 4
+    V_STR = 5
+    V_TS = 6
+    SEQ_OPEN = 7
+    MAP_OPEN = 8
+    CLOSE = 9
+    ALIAS = 10
+    ANCHOR = 11
+
+    FIELDS = 7
+    VALUE_SIZE = 24
+    # kind, tag, is_key, b | off, len | pad4 | payload (INT/FLOAT bits)
+    UNPACK = "C4V2x4q<"
+
+    module_function
+
+    def load_all(yaml, schema: :compat_11)
+      yaml = yaml.read if yaml.respond_to?(:read)
+      yaml = yaml.to_s
+      vals_p = ::FFI::MemoryPointer.new(:pointer)
+      count_p = ::FFI::MemoryPointer.new(:uint64)
+      arena_p = ::FFI::MemoryPointer.new(:pointer)
+      alen_p = ::FFI::MemoryPointer.new(:uint64)
+      st = FFI.yeptris_value_drain(
+        yaml, yaml.bytesize,
+        schema == :compat_11 ? FFI::SCHEMA_11_COMPAT : FFI::SCHEMA_12_CORE,
+        vals_p, count_p, arena_p, alen_p
+      )
+      raise ParseError, FFI.last_error_message if st != FFI::OK
+
+      vals = vals_p.read_pointer
+      arena = arena_p.read_pointer
+      begin
+        count = count_p.read_uint64
+        flat = vals.read_bytes(count * VALUE_SIZE).unpack(UNPACK * count)
+        arena_bytes = arena.read_bytes(alen_p.read_uint64)
+        arena_bytes.force_encoding(Encoding::UTF_8)
+        walk(flat, arena_bytes)
+      ensure
+        FFI.yeptris_value_free(vals, arena)
+      end
+    end
+
+    def load(yaml, schema: :compat_11)
+      docs = load_all(yaml, schema: schema)
+      docs.empty? ? nil : docs.first
+    end
+
+    # field offsets in the unpacked 7-tuple
+    KIND = 0
+    TAG = 1
+    IS_KEY = 2
+    B = 3
+    OFF = 4
+    LEN = 5
+    P64 = 6
+
+    def walk(flat, arena)
+      docs = []
+      stack = []
+      anchors = {}
+      pending_key = []
+      pending_key_tag = []
+      pending_anchor = nil
+      merge_target = []
+
+      i = 0
+      n = flat.length
+      while i < n
+        kind = flat[i + KIND]
+        case kind
+        when V_STR
+          text = arena.byteslice(flat[i + OFF], flat[i + LEN])
+          # implicit-plain ':name' scans to a Symbol (Psych's
+          # ScalarScanner); quoted ':x' stays a String
+          v = if flat[i + B] == 1 && text.length > 1 &&
+                text.start_with?(":") && !text.start_with?("::")
+                text[1..].to_sym
+              else
+                text
+              end
+          if pending_anchor
+            anchors[pending_anchor] = v
+            pending_anchor = nil
+          end
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when V_INT
+          v = flat[i + P64]
+          if pending_anchor
+            anchors[pending_anchor] = v
+            pending_anchor = nil
+          end
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when V_FLOAT
+          text = arena.byteslice(flat[i + OFF], flat[i + LEN])
+          # Psych's float grammar requires the dot (or an inf/nan
+          # word, or sexagesimal ':') — exponent-only forms are
+          # Strings even when the compat tag says FLOAT
+          v = if text.include?(".") || text.include?(":") || text.start_with?(".")
+                [flat[i + P64]].pack("q<").unpack1("E")
+              else
+                text
+              end
+          if pending_anchor
+            anchors[pending_anchor] = v
+            pending_anchor = nil
+          end
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when V_BOOL
+          text = arena.byteslice(flat[i + OFF], flat[i + LEN])
+          # Psych quirk: single-char y/n stay Strings
+          v = text.length == 1 ? text : flat[i + B] == 1
+          if pending_anchor
+            anchors[pending_anchor] = v
+            pending_anchor = nil
+          end
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when V_NULL
+          slot(docs, stack, pending_key, pending_key_tag, nil, merge_target, flat, i)
+        when V_TS
+          v = Materializer.parse_timestamp(arena.byteslice(flat[i + OFF], flat[i + LEN]))
+          if pending_anchor
+            anchors[pending_anchor] = v
+            pending_anchor = nil
+          end
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when MAP_OPEN
+          h = {}
+          if pending_anchor
+            anchors[pending_anchor] = h
+            pending_anchor = nil
+          end
+          merge_target.push(slot(docs, stack, pending_key, pending_key_tag, h, merge_target,
+                                 flat, i))
+          stack.push(h)
+          pending_key.push(nil)
+          pending_key_tag.push(nil)
+        when SEQ_OPEN
+          a = []
+          if pending_anchor
+            anchors[pending_anchor] = a
+            pending_anchor = nil
+          end
+          merge_target.push(slot(docs, stack, pending_key, pending_key_tag, a, merge_target,
+                                 flat, i))
+          stack.push(a)
+          pending_key.push(nil)
+          pending_key_tag.push(nil)
+        when CLOSE
+          closed = stack.pop
+          pending_key.pop
+          pending_key_tag.pop
+          target = merge_target.pop
+          Materializer.merge_into(target, closed) if target
+        when DOC
+          docs.push(nil)
+        when ALIAS
+          v = anchors[arena.byteslice(flat[i + OFF], flat[i + LEN])]
+          slot(docs, stack, pending_key, pending_key_tag, v, merge_target, flat, i)
+        when ANCHOR
+          pending_anchor = arena.byteslice(flat[i + OFF], flat[i + LEN])
+        end
+        i += FIELDS
+      end
+      docs
+    end
+
+    # Places a completed value: document root, sequence entry, or a
+    # map's key (is_key) / value (completing the pending pair).
+    # Returns the merge TARGET when the value is a still-empty
+    # container placed under a '<<' key — the open-site caller pushes
+    # it so the matching CLOSE merges into it (an inline map's
+    # contents arrive after its open); scalar merges apply now.
+    def slot(docs, stack, pending_key, pending_key_tag, v, _merge_target, flat, i)
+      if stack.empty?
+        docs[-1] = v
+        return nil
+      end
+      parent = stack.last
+      if parent.is_a?(Array)
+        parent.push(v)
+        return nil
+      end
+      if flat[i + IS_KEY] == 1
+        pending_key[-1] = v
+        pending_key_tag[-1] = flat[i + TAG]
+        return nil
+      end
+      key = pending_key[-1]
+      pending_key[-1] = nil
+      if key == "<<" && pending_key_tag[-1] == 9 # TAG_MERGE
+        if (v.is_a?(Hash) || v.is_a?(Array)) && v.empty?
+          parent # deferred: contents arrive after the open
+        else
+          Materializer.merge_into(parent, v)
+          nil
+        end
+      else
+        parent[key] = v
+        nil
+      end
+    end
+  end
+end

data/lib/yeptris/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Yeptris
+  VERSION = "0.1.0"
+end