statifier 0.0.1 → 0.1.0

data/lib/statifier/validator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative "document"
+require_relative "lowering"
+require_relative "validator/error"
+require_relative "validator/context"
+require_relative "validator/states"
+require_relative "validator/history"
+require_relative "validator/data"
+require_relative "validator/executable"
+require_relative "validator/attributes"
+
+module Statifier
+  # The typed chart in, the chart or every error it carries out: the third
+  # step of reading a chart, a port of the reference's validator, and the
+  # gate in front of the compiler. A chart with an error is not compiled.
+  #
+  # Lowering keeps what was written, including shapes the specification
+  # forbids; the validator holds that chart to the rules lowering leaves to
+  # it. It also answers warnings, findings that do not gate compilation.
+  # Four contracts, the reference's own:
+  #
+  # - Collect-all. Every check runs over the whole chart on every call; a
+  #   chart that breaks five rules answers five errors.
+  # - Document order. The errors, and separately the warnings, are sorted by
+  #   where they start; findings at one offset keep the order the checks
+  #   ran in.
+  # - Never a partial result. The chart is answered only when there is no
+  #   error, and it is the caller's own chart, unchanged.
+  # - Both channels always. The warnings are answered whether or not an
+  #   error is.
+  #
+  # No check here parses an expression: a `cond` or an `expr` is checked
+  # when the chart is compiled, as the reference checks it there.
+  #
+  # @api private
+  module Validator
+    # The checks that gate compilation, in the reference's order: it decides
+    # the order of errors at one offset.
+    CHECKS = [
+      [States, :ids], [States, :targets], [States, :initial_targets], [States, :initial_elements],
+      [History, :histories], [History, :finals], [History, :final_parents], [History, :default_entries],
+      [DataChecks, :donedata], [DataChecks, :contents], [DataChecks, :params], [Enums, :boilerplate],
+      [Enums, :enums], [DataChecks, :data], [Executable, :assigns], [Executable, :ifs], [Executable, :scripts],
+      [Executable, :invokes], [Executable, :sends_check], [Executable, :cancels], [Attributes, :duplicates]
+    ].freeze
+
+    # The checks whose findings are warnings.
+    WARNING_CHECKS = [[Executable, :finalize_warnings]].freeze
+
+    # What {Validator.validate} answers: the chart as `value` when no check
+    # found an error (nil otherwise), every {Error} as `errors` and every
+    # {Warning} as `warnings`, each frozen and in document order.
+    Result = ::Data.define(:value, :errors, :warnings) do
+      # @return [Boolean] true when the chart may be compiled
+      def ok? = errors.empty?
+    end
+
+    # The encodings whose strings are read from their bytes.
+    READ_AS_BYTES = [Encoding::BINARY, Encoding::US_ASCII].freeze
+
+    # Holds a lowered chart to every rule that gates compilation.
+    #
+    # @param document [Document] the chart {Lowering.lower} answered
+    # @param source [String] the document it was read from: the enumerated
+    #   attributes and the attributes of each tag are read back out of it
+    # @param invoke_content_markup [Boolean] accept a root that declares no
+    #   namespace, as inline invoke content markup is placed in the SCXML
+    #   namespace; a root declaring another namespace is still refused
+    # @return [Result] the chart, or its errors, and its warnings
+    def self.validate(document, source, invoke_content_markup: false)
+      context = Index.build(document, utf8(source), invoke_content_markup)
+      errors = run(CHECKS, document, context)
+      Result.new(value: errors.empty? ? document : nil, errors:, warnings: run(WARNING_CHECKS, document, context))
+    end
+
+    # Every finding of `checks`, sorted by start offset, the checks' order
+    # kept at one offset.
+    def self.run(checks, document, context)
+      findings = checks.flat_map { |owner, check| owner.public_send(check, document, context) }
+      findings.each_with_index.sort_by { |finding, index| [finding.location.start_offset, index] }.map(&:first).freeze
+    end
+
+    # The source as the reader read it, so that its byte offsets slice it.
+    def self.utf8(source)
+      return source if source.encoding == Encoding::UTF_8
+      return source.dup.force_encoding(Encoding::UTF_8) if READ_AS_BYTES.include?(source.encoding)
+
+      source.encode(Encoding::UTF_8)
+    end
+
+    private_class_method :run, :utf8
+  end
+end

data/lib/statifier/version.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
 module Statifier
-  VERSION = "0.0.1"
+  # The gem's version.
+  VERSION = "0.1.0"
 end

data/lib/statifier/xml/character_data.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # Reading the text of a document: character data, CDATA sections,
+    # attribute values and the references inside them.
+    #
+    # Character data folds line breaks per XML 1.0 section 2.11: a literal
+    # CR LF pair and a literal lone CR each become one LF, inside a CDATA
+    # section too, while a CR decoded from `
` stays a CR. An attribute
+    # value is normalized per section 3.3.3: a literal TAB, LF or CR becomes a
+    # space, a literal CR LF pair becomes one space, a character reference
+    # keeps its decoded character, and nothing is trimmed or collapsed. Both
+    # read the raw source, so a CR followed by a comment is a lone CR.
+    #
+    # The five predefined entities and decimal and hex character references
+    # expand in text and in attribute values. Any other entity reference is
+    # refused in a document with no document type declaration, where it can
+    # name no declared entity. In a document with one, a reference to any
+    # other name is kept as its literal text (`&name;`), as the reference
+    # keeps it: the declarations are not read, so nothing expands it.
+    #
+    # @api private
+    module CharacterData
+      PREDEFINED = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'" }.freeze
+
+      # A reference body runs to the first `;` and never spans whitespace,
+      # markup or a quote; one that does means the `&` began no reference.
+      REFERENCE = /&([^;\x20\t\r\n<&"']+);/
+      HEX = /\A#x([0-9A-Fa-f]+)\z/
+      DECIMAL = /\A#([0-9]+)\z/
+
+      # The characters XML 1.0's Char production leaves out that a valid
+      # UTF-8 string can still hold (a surrogate cannot be valid UTF-8).
+      EXCLUDED = "\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F\\uFFFE\\uFFFF"
+      INVALID = /[#{EXCLUDED}]/
+      TEXT_RUN = /[^<&\r\]#{EXCLUDED}]+/
+      CDATA_RUN = /[^\]\r#{EXCLUDED}]+/
+      VALUE_RUNS = {
+        '"' => /[^"<&\t\n\r#{EXCLUDED}]+/,
+        "'" => /[^'<&\t\n\r#{EXCLUDED}]+/
+      }.freeze
+
+      private
+
+      # Reads character data up to the next `<` or the end into `buffer`.
+      def read_character_data(buffer)
+        buffer << (@cursor.scan(TEXT_RUN) || text_special) until @cursor.eos? || @cursor.at?("<")
+      end
+
+      def text_special
+        return read_reference if @cursor.at?("&")
+        return line_break("\n") if @cursor.at?("\r")
+
+        @cursor.refuse(:cdata_end_in_text, "']]>' is not allowed in character data") if @cursor.at?("]]>")
+        return @cursor.getch if @cursor.at?("]")
+
+        refuse_character
+      end
+
+      # Reads `<![CDATA[ ... ]]>` into `buffer`: the content verbatim except
+      # that line breaks fold.
+      def read_cdata(buffer)
+        @cursor.skip("<![CDATA[")
+        buffer << (@cursor.scan(CDATA_RUN) || cdata_special) until @cursor.skip("]]>")
+      end
+
+      def cdata_special
+        @cursor.refuse_at_end(:unterminated_cdata, "a CDATA section is never closed") if @cursor.eos?
+        return line_break("\n") if @cursor.at?("\r")
+        return @cursor.getch if @cursor.at?("]")
+
+        refuse_character
+      end
+
+      # Reads an attribute value up to its closing `quote`, which it leaves
+      # unread, and answers it normalized.
+      def read_value(quote)
+        value = +""
+        value << (@cursor.scan(VALUE_RUNS.fetch(quote)) || value_special) until @cursor.at?(quote)
+        value.freeze
+      end
+
+      def value_special
+        refuse_unexpected_end if @cursor.eos?
+        @cursor.refuse(:lt_in_attribute_value, "'<' is not allowed in an attribute value") if @cursor.at?("<")
+        return read_reference if @cursor.at?("&")
+        return line_break(" ") if @cursor.at?("\r")
+        return @cursor.getch && " " if @cursor.at?(/[\t\n]/)
+
+        refuse_character
+      end
+
+      # Consumes a CR and the LF after it, if any, and answers `replacement`.
+      def line_break(replacement)
+        @cursor.skip(/\r\n?/)
+        replacement
+      end
+
+      # Reads `&name;`, `&#digits;` or `&#xhex;` and answers the character it
+      # stands for.
+      def read_reference
+        at = @cursor.mark
+        @cursor.refuse(:malformed_reference, "'&' does not start a reference", at) unless @cursor.skip(REFERENCE)
+        body = @cursor[1]
+        return character_reference(body, at) if body.start_with?("#")
+
+        PREDEFINED.fetch(body) { entity_reference(body, at) }
+      end
+
+      def entity_reference(body, at)
+        return "&#{body};" if @doctype && body.match?(Tags::WHOLE_NAME)
+
+        @cursor.refuse(:unknown_entity, "&#{body}; is not a predefined entity", at)
+      end
+
+      def character_reference(body, at)
+        digits = body[HEX, 1]
+        code = digits ? digits.to_i(16) : body[DECIMAL, 1]&.to_i
+        return code.chr(Encoding::UTF_8) if code && xml_char?(code)
+
+        @cursor.refuse(:invalid_character_reference, "&#{body}; is not a valid character reference", at)
+      end
+
+      # XML 1.0's Char production.
+      def xml_char?(code)
+        [0x9, 0xA, 0xD].include?(code) || (0x20..0xD7FF).cover?(code) ||
+          (0xE000..0xFFFD).cover?(code) || (0x10000..0x10FFFF).cover?(code)
+      end
+
+      def refuse_character
+        code = @cursor.scan(INVALID).ord
+        @cursor.move_to(@cursor.offset - code.chr(Encoding::UTF_8).bytesize)
+        @cursor.refuse(:invalid_character, format("the character U+%04X is not allowed in XML", code))
+      end
+
+      def refuse_unexpected_end
+        @cursor.refuse(:unexpected_end, "the document ends inside a tag")
+      end
+    end
+  end
+end

data/lib/statifier/xml/cursor.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # How a refusal leaves the reader: raised inside one parse and rescued by
+    # {XML.parse}, which answers it as a failed {Result}. It never reaches a
+    # caller.
+    #
+    # @api private
+    class Refused < StandardError
+      # @return [ParseError] the refusal
+      attr_reader :error
+
+      # @param error [ParseError] the refusal to carry
+      def initialize(error)
+        super(error.message)
+        @error = error
+      end
+    end
+
+    # The read position in one document: a byte scanner, and the line and
+    # column of the last position asked for, counted forward from there so
+    # that the whole document is counted once.
+    #
+    # @api private
+    class Cursor
+      # A position: the byte offset and the line and column it sits at.
+      Mark = Data.define(:offset, :line, :column)
+
+      WHITESPACE = /[\x20\t\r\n]+/
+
+      # @return [String] the document, as UTF-8
+      attr_reader :text
+
+      # @param text [String] the document, valid UTF-8
+      def initialize(text)
+        @text = text
+        @scanner = StringScanner.new(text)
+        @known = Mark.new(offset: 0, line: 1, column: 1)
+      end
+
+      # @return [Integer] the byte offset of the read position
+      def offset = @scanner.pos
+
+      # @return [Boolean] whether the whole document has been read
+      def eos? = @scanner.eos?
+
+      # @param pattern [String, Regexp] what to look for at the read position
+      # @return [Boolean] whether it is there; nothing is consumed
+      def at?(pattern) = !@scanner.match?(pattern).nil?
+
+      # @param pattern [String, Regexp] what to consume
+      # @return [String, nil] the text consumed, or nil when it is not there
+      def scan(pattern) = @scanner.scan(pattern)
+
+      # @param pattern [String, Regexp] what to consume
+      # @return [Integer, nil] the bytes consumed, or nil when it is not there
+      def skip(pattern) = @scanner.skip(pattern)
+
+      # @param pattern [Regexp] what to consume up to and through
+      # @return [Integer, nil] the bytes consumed, or nil, consuming nothing,
+      #   when it does not occur
+      def skip_until(pattern) = @scanner.skip_until(pattern)
+
+      # @param group [Integer] a group of the last match
+      # @return [String, nil] its text
+      def [](group) = @scanner[group]
+
+      # @return [String] the next character, consumed
+      def getch = @scanner.getch
+
+      # @return [Integer, nil] the bytes of whitespace consumed, or nil when
+      #   there is none
+      def skip_whitespace = @scanner.skip(WHITESPACE)
+
+      # Moves the read position.
+      #
+      # @param offset [Integer] the byte offset to read from next
+      # @return [void]
+      def move_to(offset)
+        @scanner.pos = offset
+      end
+
+      # @return [Mark] the read position with its line and column
+      def mark
+        pos = @scanner.pos
+        @known = Mark.new(offset: 0, line: 1, column: 1) if pos < @known.offset
+        @known = count_to(pos) if pos > @known.offset
+        @known
+      end
+
+      # @param from [Mark] where the span starts
+      # @param to [Mark] where it ends, exclusive; the read position when omitted
+      # @return [Location]
+      def span(from, to = mark)
+        Location.new(start_line: from.line, start_column: from.column, start_offset: from.offset,
+                     end_line: to.line, end_column: to.column, end_offset: to.offset)
+      end
+
+      # Refuses the document.
+      #
+      # @param reason [Symbol] the reason token
+      # @param message [String] what is wrong, for a person
+      # @param at [Mark] where; the read position when omitted
+      # @raise [Refused] always
+      def refuse(reason, message, at = mark)
+        raise Refused, ParseError.new(reason:, message: "#{message} (line #{at.line}, column #{at.column})",
+                                      location: span(at, at))
+      end
+
+      # Refuses the document at its end.
+      #
+      # @param (see #refuse)
+      # @raise [Refused] always
+      def refuse_at_end(reason, message)
+        @scanner.terminate
+        refuse(reason, message)
+      end
+
+      private
+
+      def count_to(pos)
+        chunk = @text.byteslice(@known.offset, pos - @known.offset)
+        newline = chunk.rindex("\n")
+        return Mark.new(offset: pos, line: @known.line, column: @known.column + chunk.length) unless newline
+
+        Mark.new(offset: pos, line: @known.line + chunk.count("\n"), column: chunk.length - newline)
+      end
+    end
+  end
+end

data/lib/statifier/xml/markup.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # The constructs that produce no node: comments, processing
+    # instructions, the XML declaration and the document type declaration.
+    #
+    # @api private
+    module Markup
+      S = "[\\x20\\t\\r\\n]"
+      EQ = "#{S}*=#{S}*".freeze
+
+      # What follows `<?xml` in an XML declaration (XML 1.0 production 23).
+      # Group 1 or 2 is the declared encoding.
+      DECLARATION = /
+        #{S}+version#{EQ}(?:"1\.[0-9]+"|'1\.[0-9]+')
+        (?:#{S}+encoding#{EQ}(?:"([A-Za-z][A-Za-z0-9._-]*)"|'([A-Za-z][A-Za-z0-9._-]*)'))?
+        (?:#{S}+standalone#{EQ}(?:"(?:yes|no)"|'(?:yes|no)'))?
+        #{S}*\?>
+      /x
+
+      private
+
+      # Skips whitespace, comments and processing instructions outside the
+      # root element; before it (`prolog`) also one document type
+      # declaration. Stops at the first other character.
+      def skip_misc(prolog:)
+        doctype = prolog
+        loop do
+          @cursor.skip_whitespace
+          next skip_comment if @cursor.at?("<!--")
+          next skip_processing_instruction if @cursor.at?("<?")
+          return unless doctype && @cursor.at?("<!DOCTYPE")
+
+          skip_doctype
+          doctype = false
+        end
+      end
+
+      # Skips `<!-- ... -->`, refusing `--` inside it as XML 1.0 section 2.5
+      # does.
+      def skip_comment
+        @cursor.skip("<!--")
+        @cursor.refuse_at_end(:unterminated_comment, "a comment is never closed") unless @cursor.skip_until(/--/)
+        return if @cursor.skip(">")
+
+        @cursor.move_to(@cursor.offset - 2)
+        @cursor.refuse(:double_hyphen_in_comment, "'--' is not allowed inside a comment")
+      end
+
+      # Skips `<? ... ?>`. The target `xml` is the XML declaration, read only
+      # at the very start of the document; any other spelling of it is
+      # reserved.
+      def skip_processing_instruction
+        start = @cursor.mark
+        @cursor.skip("<?")
+        target = read_name
+        return read_declaration(start) if target == "xml" && start.offset.zero?
+
+        refuse_reserved(target, start) if target.casecmp?("xml")
+        return if @cursor.skip_until(/\?>/)
+
+        @cursor.refuse_at_end(:unterminated_processing_instruction, "a processing instruction is never closed")
+      end
+
+      def read_declaration(start)
+        @cursor.refuse(:malformed_xml_declaration, "the XML declaration is malformed", start) \
+          unless @cursor.skip(DECLARATION)
+        encoding = @cursor[1] || @cursor[2]
+        return if encoding.nil? || encoding.casecmp?("UTF-8")
+
+        @cursor.refuse(:unsupported_encoding, "the document declares the encoding #{encoding}; only UTF-8 is read",
+                       start)
+      end
+
+      def refuse_reserved(target, start)
+        if target == "xml"
+          @cursor.refuse(:misplaced_xml_declaration,
+                         "the XML declaration is allowed only at the start of the document", start)
+        end
+        @cursor.refuse(:reserved_target, "the processing instruction target #{target} is reserved", start)
+      end
+
+      # Skips `<!DOCTYPE ... >` as the reference reads one: the declaration
+      # ends at the `>` that brings the count of open angle brackets,
+      # `<!DOCTYPE` included, to zero. The reference also finds the end a
+      # second way, at the first `>` outside square brackets, and reads the
+      # document only when the two ends agree or nothing between them is
+      # markup. Neither reading treats a quote specially, and the
+      # declarations inside are not read: only the predefined entities
+      # expand.
+      def skip_doctype
+        start = @cursor.mark
+        @cursor.skip("<!DOCTYPE")
+        angle, bracket = DoctypeEnds.new(@cursor.text, @cursor.offset).ends
+        unless angle && bracket
+          @cursor.refuse_at_end(:unterminated_doctype, "a document type declaration is never closed")
+        end
+        low, high = [angle, bracket].minmax
+        refuse_ambiguous_doctype(start) if @cursor.text.byteslice(low, high - low).include?("<")
+        @cursor.move_to(angle)
+        @doctype = true
+      end
+
+      def refuse_ambiguous_doctype(start)
+        @cursor.refuse(:ambiguous_doctype,
+                       "the document type declaration has two different ends with markup between them", start)
+      end
+    end
+
+    # The two ends of a document type declaration, read from just after
+    # `<!DOCTYPE`: the byte offset after the `>` that balances the angle
+    # brackets, and after the first `>` outside square brackets (a `]`
+    # never taking the depth below zero). Either is nil when the document
+    # ends first.
+    #
+    # @api private
+    class DoctypeEnds
+      # @param text [String] the document
+      # @param offset [Integer] the byte offset just after `<!DOCTYPE`
+      def initialize(text, offset)
+        @scanner = StringScanner.new(text)
+        @scanner.pos = offset
+        @angles = 1
+        @depth = 0
+      end
+
+      # @return [Array(Integer, Integer)] the angle-bracket end and the
+      #   square-bracket end, each nil when the document ends first
+      def ends
+        step(@scanner.matched) while (@angle.nil? || @bracket.nil?) && @scanner.skip_until(/[<>\[\]]/)
+        [@angle, @bracket]
+      end
+
+      private
+
+      def step(char)
+        case char
+        when "<" then @angles += 1
+        when "[" then @depth += 1
+        when "]" then @depth = [@depth - 1, 0].max
+        else close
+        end
+      end
+
+      def close
+        @bracket ||= @scanner.pos if @depth.zero?
+        @angle ||= @scanner.pos if @angles == 1
+        @angles -= 1
+      end
+    end
+  end
+end

data/lib/statifier/xml/nodes.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Statifier
+  module XML
+    # A source span: 1-based lines and columns, 0-based byte offsets into the
+    # document read as UTF-8, and an exclusive end. Columns count code
+    # points, and only a line feed starts a new line. An element's span runs
+    # from the `<` of its start tag to the byte after the `>` of its end tag
+    # (the tag itself when it closes itself), so its start line and column
+    # are those of its `<`.
+    Location = Data.define(:start_line, :start_column, :start_offset, :end_line, :end_column, :end_offset) do
+      # The raw text this span covers.
+      #
+      # @param source [String] the UTF-8 document the span was read from
+      # @return [String] the bytes from the start offset up to the end offset
+      def slice(source) = source.byteslice(start_offset, end_offset - start_offset)
+    end
+
+    # One attribute, in document order. `value` is normalized (XML 1.0
+    # section 3.3.3) and its references expanded; `location` covers
+    # `name="value"` with its quotes, and `value_location` only the raw text
+    # inside the quotes.
+    Attribute = Data.define(:name, :value, :location, :value_location)
+
+    # An element. `name` is the qualified name as written, prefix included;
+    # `namespace` is the URI that name resolves to under the declarations in
+    # scope (the empty string when an empty declaration is in scope for it),
+    # or nil when none is. `attributes` keeps document order and every
+    # duplicate; `children` interleaves elements and text runs.
+    Element = Data.define(:name, :namespace, :attributes, :children, :location)
+
+    # A run of character data between two tags, coalesced across any
+    # comment, processing instruction or CDATA section inside it. `value` is
+    # expanded and line-break folded (XML 1.0 section 2.11); `location`
+    # covers the raw run.
+    Text = Data.define(:value, :location)
+
+    # Why a document was refused: a reason symbol, a message for a person
+    # that ends with the line and column, and the zero-width location where
+    # reading stopped.
+    ParseError = Data.define(:reason, :message, :location)
+
+    # What {XML.parse} answers: the root element as `value`, or the refusal
+    # as `error`.
+    Result = Data.define(:value, :error) do
+      # @return [Boolean] true when the document was read
+      def ok? = error.nil?
+    end
+  end
+end