statifier 0.0.1 → 0.1.0

data/lib/statifier/xml/parser.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # One parse of one document. The element tree is built with an explicit
+    # stack, so nesting depth never touches the call stack.
+    #
+    # @api private
+    class Parser
+      include CharacterData
+      include Markup
+      include Tags
+
+      # The constructs inside the root element that open with `<` and are
+      # not tags.
+      OTHER_MARKUP = /<(?:!--|!\[CDATA\[|\?)/
+
+      # @param text [String] the document, valid UTF-8 with no byte order mark
+      def initialize(text)
+        @cursor = Cursor.new(text)
+        @stack = []
+        @run = nil
+        @doctype = false
+      end
+
+      # @return [Element] the root element
+      # @raise [Refused] when the document is malformed
+      def parse
+        skip_misc(prolog: true)
+        @cursor.refuse(:missing_root, "no root element") if @cursor.eos?
+        @cursor.refuse(:content_before_root, "character data before the root element") unless @cursor.at?(START_TAG)
+        root = read_root
+        skip_misc(prolog: false)
+        @cursor.refuse(:content_after_root, "content after the root element") unless @cursor.eos?
+        root
+      end
+
+      private
+
+      def read_root
+        root = open_tag
+        root = step until root
+        root
+      end
+
+      # Reads one construct inside the root element; answers the root once
+      # its end tag is read.
+      def step
+        refuse_unclosed if @cursor.eos?
+        return close_tag if @cursor.at?("</")
+        return open_tag if start_tag?
+
+        read_content
+        nil
+      end
+
+      # Whether a start tag begins at the read position: a `<` that opens no
+      # other construct.
+      def start_tag? = @cursor.at?("<") && !@cursor.at?(OTHER_MARKUP)
+
+      # Reads a comment, a CDATA section, a processing instruction or a run of
+      # character data.
+      def read_content
+        if @cursor.at?("<!--") then skip_comment
+        elsif @cursor.at?("<![CDATA[") then read_cdata(run_buffer)
+        elsif @cursor.at?("<?") then skip_processing_instruction
+        else read_character_data(run_buffer)
+        end
+      end
+
+      # Reads a start tag; answers the root when the root closes itself.
+      def open_tag
+        flush
+        element, closed = read_start_tag(@stack.last&.scope || Scope::EMPTY)
+        @tag_end = @cursor.mark
+        return adopt(element.finish(@cursor.span(element.start))) if closed
+
+        @stack.push(element)
+        nil
+      end
+
+      # Reads an end tag; answers the root when it closes the root.
+      def close_tag
+        flush
+        element = read_end_tag(@stack.pop)
+        @tag_end = @cursor.mark
+        adopt(element)
+      end
+
+      # Appends a finished element to its parent, or answers it when it is
+      # the root.
+      def adopt(element)
+        return element if @stack.empty?
+
+        @stack.last.children << element
+        nil
+      end
+
+      # The text run open since the last tag. It starts where that tag ends, as
+      # the reference's spans do, so a comment or processing instruction
+      # before the first character data is inside the span.
+      def run_buffer
+        @run ||= [@tag_end, +""]
+        @run.last
+      end
+
+      # Closes the open text run, if any, at the read position: the `<` of the
+      # next tag.
+      def flush
+        return unless @run
+
+        @stack.last.children << Text.new(value: @run.last.freeze, location: @cursor.span(@run.first))
+        @run = nil
+      end
+
+      def refuse_unclosed
+        @cursor.refuse(:unclosed_element, "element <#{@stack.last.name}> is never closed")
+      end
+    end
+  end
+end

data/lib/statifier/xml/scope.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # The namespace declarations in scope at an element: the default
+    # namespace (nil when none is declared) and the bound prefixes. Each
+    # element folds its own `xmlns` and `xmlns:prefix` attributes over the
+    # scope it inherits, so a declaration reaches its element's subtree and
+    # no further. A value binds as written, the empty string included: an
+    # empty declaration (`xmlns=""`, `xmlns:p=""`) binds the empty URI and
+    # undeclares nothing, as the reference's namespace scope does.
+    #
+    # @api private
+    Scope = Data.define(:default, :prefixes) do
+      # @param attributes [Array<Attribute>] an element's own attributes
+      # @return [Scope] the scope for that element's subtree
+      def declare(attributes)
+        declared = attributes.select { |attribute| attribute.name.start_with?("xmlns") }
+        return self if declared.empty?
+
+        declared.reduce(self) { |scope, attribute| scope.bind(attribute.name, attribute.value) }
+      end
+
+      # @param name [String] an attribute name, perhaps a declaration
+      # @param uri [String] its value
+      # @return [Scope] this scope with the declaration folded in, if it is one
+      def bind(name, uri)
+        return with(default: uri) if name == "xmlns"
+        return self unless name.start_with?("xmlns:") && name.length > 6
+
+        with(prefixes: prefixes.merge(name.delete_prefix("xmlns:") => uri).freeze)
+      end
+
+      # @param name [String] a qualified element name, prefix included
+      # @return [String, nil] the namespace it resolves to, or nil when no
+      #   declaration is in scope for it
+      def resolve(name)
+        prefix, local = name.split(":", 2)
+        local.nil? ? default : prefixes[prefix]
+      end
+    end
+
+    # The scope outside the root element: nothing declared.
+    Scope::EMPTY = Scope.new(default: nil, prefixes: {}.freeze)
+  end
+end

data/lib/statifier/xml/tags.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # Reading start tags, end tags and attributes.
+    #
+    # @api private
+    module Tags
+      # XML 1.0's NameStartChar and NameChar productions.
+      NAME_START = "A-Za-z_:\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF" \
+                   "\\u200C\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD" \
+                   "\\u{10000}-\\u{EFFFF}"
+      NAME = /[#{NAME_START}][#{NAME_START}\-.0-9\u00B7\u0300-\u036F\u203F\u2040]*/
+      WHOLE_NAME = /\A#{NAME}\z/
+      START_TAG = /<[#{NAME_START}]/
+      TAG_END = %r{/?>}
+
+      # An element whose end tag is still to come.
+      OpenElement = Struct.new(:name, :namespace, :attributes, :children, :start, :scope) do
+        # @param location [Location] the element's whole span
+        # @return [Element] the element, its children frozen
+        def finish(location) = Element.new(name:, namespace:, attributes:, children: children.freeze, location:)
+      end
+
+      private
+
+      # Reads a name at the read position.
+      def read_name
+        name = @cursor.scan(NAME)
+        return name.freeze if name
+
+        refuse_unexpected_end if @cursor.eos?
+        @cursor.refuse(:expected_name, "expected a name")
+      end
+
+      # Reads a start tag, `<name (S attribute)* S? (> | />)`, under the
+      # namespace scope it inherits. Answers the open element and whether
+      # the tag closed itself.
+      def read_start_tag(inherited)
+        start = @cursor.mark
+        @cursor.skip("<")
+        name = read_name
+        attributes = read_attributes
+        closed = !@cursor.skip("/>").nil?
+        @cursor.skip(">") unless closed
+        scope = inherited.declare(attributes)
+        [OpenElement.new(name, scope.resolve(name), attributes, [], start, scope), closed]
+      end
+
+      def read_attributes
+        attributes = []
+        loop do
+          spaced = @cursor.skip_whitespace
+          return attributes.freeze if @cursor.at?(TAG_END)
+
+          refuse_unexpected_end if @cursor.eos?
+          @cursor.refuse(:expected_whitespace, "expected whitespace before an attribute") unless spaced
+          attributes << read_attribute
+        end
+      end
+
+      # Reads `name S? = S? quoted-value`.
+      def read_attribute
+        start = @cursor.mark
+        name = read_name
+        expect_equals(name)
+        quote = expect_quote(name)
+        value_start = @cursor.mark
+        value = read_value(quote)
+        value_location = @cursor.span(value_start)
+        @cursor.skip(quote)
+        Attribute.new(name:, value:, location: @cursor.span(start), value_location:)
+      end
+
+      def expect_equals(name)
+        @cursor.skip_whitespace
+        unless @cursor.skip("=")
+          refuse_unexpected_end if @cursor.eos?
+          @cursor.refuse(:expected_equals, "expected '=' after attribute #{name}")
+        end
+        @cursor.skip_whitespace
+      end
+
+      def expect_quote(name)
+        quote = @cursor.scan(/["']/)
+        return quote if quote
+
+        refuse_unexpected_end if @cursor.eos?
+        @cursor.refuse(:expected_quote, "expected a quoted value for attribute #{name}")
+      end
+
+      # Reads `</name S? >` and closes `open` with it.
+      def read_end_tag(open)
+        start = @cursor.mark
+        @cursor.skip("</")
+        name = read_name
+        @cursor.skip_whitespace
+        expect_tag_end
+        return open.finish(@cursor.span(open.start)) if name == open.name
+
+        @cursor.refuse(:mismatched_end_tag, "end tag </#{name}> where </#{open.name}> was due", start)
+      end
+
+      def expect_tag_end
+        return if @cursor.skip(">")
+
+        refuse_unexpected_end if @cursor.eos?
+        @cursor.refuse(:expected_tag_end, "expected '>' to end the tag")
+      end
+    end
+  end
+end

data/lib/statifier/xml.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+require_relative "xml/nodes"
+require_relative "xml/cursor"
+require_relative "xml/scope"
+require_relative "xml/character_data"
+require_relative "xml/markup"
+require_relative "xml/tags"
+require_relative "xml/parser"
+
+module Statifier
+  # XML source in, a generic element tree out: the first step of reading a
+  # chart. This layer knows XML and nothing else; a later one gives the names
+  # meaning. It is a small hand-written reader of the XML that SCXML
+  # documents use, and needs no XML library.
+  #
+  # Beyond splitting the markup, it normalizes attribute values (XML 1.0
+  # section 3.3.3), folds line breaks in character data (section 2.11),
+  # expands the five predefined entities and character references, and gives
+  # every element the namespace its name resolves to under the declarations
+  # in scope. It validates no name against any vocabulary, keeps duplicate
+  # attributes and whitespace-only text, and reads no document type
+  # definition. Comments, processing instructions, the XML declaration and a
+  # document type declaration produce no nodes, and anything outside the
+  # root element is discarded.
+  #
+  # @api private
+  module XML
+    # The encodings whose strings are read from their bytes as UTF-8.
+    READ_AS_BYTES = [Encoding::BINARY, Encoding::US_ASCII].freeze
+    private_constant :READ_AS_BYTES
+
+    # Parses `source` into its root element, or answers why it cannot.
+    # Malformed input is never raised: the result carries the refusal, with
+    # a message and the line and column where reading stopped.
+    #
+    # The document is read as UTF-8. A string tagged binary or US-ASCII is
+    # read from its bytes; a string in any other encoding is transcoded
+    # first, and the byte offsets in every {Location} are then offsets into
+    # the transcoded text. A leading byte order mark is refused, as the
+    # reference refuses it.
+    #
+    # @param source [String] the document
+    # @return [Result] the root {Element} as `value`, or a {ParseError} as
+    #   `error`
+    # @raise [TypeError] when `source` is not a String
+    def self.parse(source)
+      raise TypeError, "expected a String, got #{source.class}" unless source.is_a?(String)
+
+      Result.new(value: Parser.new(utf8(source)).parse, error: nil)
+    rescue Refused => e
+      Result.new(value: nil, error: e.error)
+    end
+
+    # @api private
+    def self.utf8(source)
+      text = transcode(source)
+      unless text.valid_encoding?
+        Cursor.new(text).tap { _1.move_to(first_invalid_byte(text)) }
+              .refuse(:invalid_encoding, "the document is not valid UTF-8")
+      end
+      Cursor.new(text).refuse(:byte_order_mark, "a byte order mark is not read") if text.start_with?("\uFEFF")
+      text
+    end
+
+    # @api private
+    def self.transcode(source)
+      return source if source.encoding == Encoding::UTF_8
+      return source.dup.force_encoding(Encoding::UTF_8) if READ_AS_BYTES.include?(source.encoding)
+
+      source.encode(Encoding::UTF_8)
+    rescue EncodingError
+      Cursor.new("").refuse(:invalid_encoding, "the document cannot be read as UTF-8")
+    end
+
+    # @api private
+    def self.first_invalid_byte(text)
+      text.each_char.take_while(&:valid_encoding?).sum(&:bytesize)
+    end
+
+    private_class_method :utf8, :transcode, :first_invalid_byte
+  end
+end