flexr 1.0.0

data/lib/flexr/unicode/reference_regexp.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Unicode
+    module ReferenceRegexp
+      # rubocop:disable Style/MutableConstant
+      CACHE = {}
+      # rubocop:enable Style/MutableConstant
+      module_function
+
+      def match(pattern, subject, encoding:, options: 0, unicode: false)
+        regexp = compiled(pattern, encoding: encoding, options: options, unicode: unicode)
+        subject = subject.dup.force_encoding(regexp.encoding)
+        regexp.match(subject, 0)
+      rescue RegexpError, ArgumentError, EncodingError
+        nil
+      end
+
+      def compiled(pattern, encoding:, options: nil, unicode: false)
+        effective_options = options.nil? ? pattern.options : options
+        key = [pattern.source, effective_options, encoding, unicode]
+        return CACHE[key] if CACHE.key?(key)
+
+        parser = Regexp::Parser.new(pattern.source, options: effective_options, encoding: encoding, unicode: unicode)
+        source = source_for(parser.parse, ignorecase: effective_options.anybits?(::Regexp::IGNORECASE))
+        regexp_options = effective_options & ~::Regexp::IGNORECASE
+        CACHE[key] = ::Regexp.new(source, regexp_options).freeze
+      end
+
+      def source_for(node, ignorecase: false)
+        case node
+        when Regexp::AST::Empty, Regexp::AST::Anchor then ""
+        when Regexp::AST::ByteRange then byte_class([[node.lo, node.hi]])
+        when Regexp::AST::CodepointRange then codepoint_class([[node.lo, node.hi]])
+        when Regexp::AST::CharClass
+          ranges = node.ranges.flat_map do |range|
+            if range.first == Regexp::AST::Property
+              property_ranges = Unicode::Property.ranges(range[2])
+              property_ranges = casefold_ranges(property_ranges) if ignorecase
+              range[1] ? complement(property_ranges) : property_ranges
+            else
+              [range]
+            end
+          end
+          ranges = complement(ranges) if node.negated
+          codepoint_class(ranges)
+        when Regexp::AST::Seq then node.children.map { |child| source_for(child, ignorecase: ignorecase) }.join
+        when Regexp::AST::Alt
+          "(?:#{node.children.map { |child| source_for(child, ignorecase: ignorecase) }.join('|')})"
+        when Regexp::AST::Star then "(?:#{source_for(node.child, ignorecase: ignorecase)})*"
+        else
+          raise CompileError, "unsupported reference AST node: #{node.class}"
+        end
+      end
+
+      def byte_class(ranges)
+        "[#{ranges.map { |lo, hi| byte_escape(lo, hi) }.join}]"
+      end
+
+      def byte_escape(lo, hi)
+        lo == hi ? format("\\x%<byte>02X", byte: lo) : format("\\x%<lo>02X-\\x%<hi>02X", lo: lo, hi: hi)
+      end
+
+      def codepoint_class(ranges)
+        ranges = scalar_ranges(ranges)
+        return "(?!)" if ranges.empty?
+
+        "[#{ranges.sort_by(&:first).map { |lo, hi| codepoint_escape(lo, hi) }.join}]"
+      end
+
+      def scalar_ranges(ranges)
+        ranges.flat_map do |lo, hi|
+          result = []
+          result << [lo, [hi, 0xd7ff].min] if lo <= 0xd7ff
+          result << [[lo, 0xe000].max, hi] if hi >= 0xe000
+          result
+        end
+      end
+
+      def codepoint_escape(lo, hi)
+        first = ::Regexp.escape([lo].pack("U"))
+        last = ::Regexp.escape([hi].pack("U"))
+        lo == hi ? first : "#{first}-#{last}"
+      end
+
+      def complement(ranges)
+        result = []
+        cursor = 0
+        ranges.sort_by(&:first).each do |lo, hi|
+          result << [cursor, lo - 1] if cursor < lo
+          cursor = [cursor, hi + 1].max
+        end
+        result << [cursor, 0x10ffff] if cursor <= 0x10ffff
+        result
+      end
+
+      def casefold_ranges(ranges)
+        CaseFold.merge(ranges.flat_map { |lo, hi| CaseFold.ranges(lo, hi) })
+      end
+    end
+  end
+end

data/lib/flexr/unicode/utf8_splitter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Unicode
+    module Utf8Splitter
+      module_function
+
+      def split(lo, hi)
+        raise ArgumentError, "invalid codepoint range" if lo > hi || lo.negative? || hi > 0x10ffff
+        return [] if lo.between?(0xd800, 0xdfff) && hi.between?(0xd800, 0xdfff)
+        return [encoded(lo).map { |byte| [byte, byte] }] if lo == hi
+
+        ranges = []
+        [[0, 0x7f, 1], [0x80, 0x7ff, 2], [0x800, 0xffff, 3], [0x10000, 0x10ffff, 4]].each do |min, max, length|
+          lower = [lo, min].max
+          upper = [hi, max].min
+          next if lower > upper
+
+          walk(lower, upper, length, [], ranges)
+        end
+        ranges
+      end
+
+      def walk(lo, hi, length, prefix, output)
+        if prefix.length == length
+          output << prefix.map { |byte| [byte, byte] }
+          return
+        end
+
+        if prefix.length == length - 1
+          first_byte = prefix.empty? ? encoded(lo)[0] : 0x80
+          last_byte = prefix.empty? ? encoded(hi)[0] : 0xbf
+          group_start = nil
+          first_byte.upto(last_byte) do |byte|
+            min_cp, max_cp = prefix_bounds(prefix + [byte], length)
+            allowed = min_cp && lo <= min_cp && max_cp <= hi
+            if allowed
+              group_start ||= byte
+            elsif group_start
+              output << (prefix.map { |value| [value, value] } + [[group_start, byte - 1]])
+              group_start = nil
+            end
+          end
+          output << (prefix.map { |value| [value, value] } + [[group_start, last_byte]]) if group_start
+          return
+        end
+
+        first_byte = prefix.empty? ? encoded(lo)[0] : 0x80
+        last_byte = prefix.empty? ? encoded(hi)[0] : 0xbf
+        first_byte.upto(last_byte) do |byte|
+          next_prefix = prefix + [byte]
+          min_cp, max_cp = prefix_bounds(next_prefix, length)
+          next if min_cp.nil? || max_cp < lo || min_cp > hi
+
+          if lo <= min_cp && max_cp <= hi
+            rest = length - next_prefix.length
+            output << (next_prefix.map { |value| [value, value] } + Array.new(rest) { [0x80, 0xbf] })
+          else
+            walk(lo, hi, length, next_prefix, output)
+          end
+        end
+      end
+
+      def encoded(codepoint)
+        [codepoint].pack("U").bytes
+      end
+
+      def prefix_bounds(prefix, length)
+        return [nil, nil] if prefix.empty?
+
+        bytes = prefix + Array.new(length - prefix.length, 0x80)
+        high = prefix + Array.new(length - prefix.length, 0xbf)
+        if prefix.length == 1
+          bytes[1] = 0xa0 if length == 3 && prefix.first == 0xe0
+          high[1] = 0x9f if length == 3 && prefix.first == 0xed
+          bytes[1] = 0x90 if length == 4 && prefix.first == 0xf0
+          high[1] = 0x8f if length == 4 && prefix.first == 0xf4
+        end
+        return [nil, nil] unless valid_prefix?(bytes, length) && valid_prefix?(high, length)
+
+        [decode(bytes), decode(high)]
+      end
+
+      def valid_prefix?(bytes, length)
+        first = bytes.first
+        expected = if first <= 0x7f then 1 elsif first.between?(0xc2, 0xdf) then 2
+                   elsif first.between?(0xe0, 0xef) then 3
+                   elsif first.between?(0xf0, 0xf4) then 4 end
+        return false unless expected == length
+        return false if bytes[1..].any? { |byte| !byte.between?(0x80, 0xbf) }
+        return false if length == 3 && first == 0xe0 && bytes[1] < 0xa0
+        return false if length == 3 && first == 0xed && bytes[1] > 0x9f
+        return false if length == 4 && first == 0xf0 && bytes[1] < 0x90
+        return false if length == 4 && first == 0xf4 && bytes[1] > 0x8f
+        true
+      end
+
+      def decode(bytes)
+        return bytes.first if bytes.length == 1
+        case bytes.length
+        when 2 then ((bytes[0] & 0x1f) << 6) | (bytes[1] & 0x3f)
+        when 3 then ((bytes[0] & 0x0f) << 12) | ((bytes[1] & 0x3f) << 6) | (bytes[2] & 0x3f)
+        when 4 then ((bytes[0] & 0x07) << 18) | ((bytes[1] & 0x3f) << 12) |
+          ((bytes[2] & 0x3f) << 6) | (bytes[3] & 0x3f)
+        end
+      end
+    end
+  end
+end

data/lib/flexr/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Flexr
+  VERSION = "1.0.0"
+end

data/lib/flexr.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+require_relative "flexr/version"
+require_relative "flexr/errors"
+require_relative "flexr/diagnostics"
+require_relative "flexr/ir"
+require_relative "flexr/regexp/ast"
+require_relative "flexr/regexp/parser"
+require_relative "flexr/regexp/normalizer"
+require_relative "flexr/regexp/unsupported"
+require_relative "flexr/regexp/char_class"
+require_relative "flexr/unicode/utf8_splitter"
+require_relative "flexr/unicode/data/properties"
+require_relative "flexr/unicode/data/case_folding"
+require_relative "flexr/unicode/property"
+require_relative "flexr/unicode/reference_regexp"
+require_relative "flexr/unicode/case_fold"
+require_relative "flexr/automaton/byte_class_set"
+require_relative "flexr/automaton/nfa"
+require_relative "flexr/automaton/dfa"
+require_relative "flexr/automaton/compiler"
+require_relative "flexr/automaton/analysis"
+require_relative "flexr/automaton/minimizer"
+require_relative "flexr/automaton/accel"
+require_relative "flexr/codegen/base"
+require_relative "flexr/codegen/table"
+require_relative "flexr/codegen/direct"
+require_relative "flexr/codegen/firstmatch"
+require_relative "flexr/codegen/table_packer"
+require_relative "flexr/runtime/location"
+require_relative "flexr/runtime/token"
+require_relative "flexr/runtime/buffer"
+require_relative "flexr/runtime/errors"
+require_relative "flexr/runtime/interpreter"
+require_relative "flexr/runtime/core"
+require_relative "flexr/dsl"
+require_relative "flexr/lexer"
+require_relative "flexr/generated"
+require_relative "flexr/source/static_eval"
+require_relative "flexr/source/prism_reader"
+require_relative "flexr/source/passthrough"
+require_relative "flexr/importer"
+require_relative "flexr/generator"
+require_relative "flexr/options"
+require_relative "flexr/rake_task"
+require_relative "flexr/cli"
+
+module Flexr
+  class << self
+    def compile_pattern(pattern, options: {})
+      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
+      return Automaton::ReferenceDFA.new(regexp, unicode: options[:unicode] == true) if
+        reference_pattern?(regexp, unicode: options[:unicode] == true)
+      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
+      rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
+      state = IR::State.new(name: :initial, inclusive: true, id: 0)
+      spec = IR::Spec.new(
+        class_name: "Pattern", backend: :table, token_kind: :array,
+        encoding: encoding, options: options, declared_tokens: [],
+        states: { initial: state }, rules: [rule], eof_rules: {}, verbatim: nil
+      )
+      Automaton::Compiler.new(spec).compile.machines.fetch(:initial).dfa
+    end
+
+    def reference_pattern?(regexp, unicode: false)
+      return true if regexp.source.match?(/\\[pP]\{/) || regexp.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+
+      unicode && regexp.encoding != Encoding::BINARY && regexp.source.match?(/\\[dDwWsS]/)
+    end
+
+    def parse_pattern(pattern, options: {})
+      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
+      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
+      ast = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
+                               unicode: options[:unicode] == true).parse
+      Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
+    end
+  end
+end

data/site/README.md

@@ -0,0 +1,22 @@
+# flexr product site
+
+This directory contains the Astro/Starlight product site for flexr. It is a static GitHub Pages site with a custom landing page, documentation routes, and a fixture-backed playground preview.
+
+## Local development
+
+```sh
+cd site
+pnpm install
+pnpm dev
+```
+
+Run the same checks used by the Pages workflow:
+
+```sh
+pnpm verify
+pnpm build
+```
+
+The current playground does not execute arbitrary Ruby. It demonstrates the matching decision model from fixed fixtures and keeps `--eval` out of the browser. A future Ruby WASM worker must preserve that security boundary and add runtime/generated equivalence tests before the UI claims full execution.
+
+The repository examples, CLI definitions, and Ruby implementation remain the behavioral source of truth. Site copy links to those artifacts where a summary would otherwise drift.

data/site/astro.config.mjs

@@ -0,0 +1,57 @@
+import { defineConfig } from 'astro/config';
+import starlight from '@astrojs/starlight';
+
+export default defineConfig({
+  site: 'https://ydah.github.io',
+  base: '/flexr',
+  integrations: [
+    starlight({
+      title: 'flexr',
+      description: 'A Ruby-native lexer generator for parser authors.',
+      customCss: ['./src/styles/custom.css'],
+      sidebar: [
+        {
+          label: 'Learn',
+          items: [
+            { label: 'Getting started', slug: 'learn/getting-started' },
+            { label: 'Runtime mode', slug: 'learn/runtime-mode' },
+            { label: 'Generation', slug: 'learn/generation' },
+            { label: 'Parser integration', slug: 'learn/parser-integration' }
+          ]
+        },
+        {
+          label: 'Concepts',
+          items: [
+            { label: 'Matching semantics', slug: 'concepts/matching-semantics' },
+            { label: 'Runtime vs generated', slug: 'concepts/runtime-vs-generated' },
+            { label: 'Regexp model', slug: 'concepts/regexp-model' },
+            { label: 'Security model', slug: 'concepts/security-model' }
+          ]
+        },
+        {
+          label: 'Reference',
+          items: [
+            { label: 'DSL', slug: 'reference/dsl' },
+            { label: 'Action context', slug: 'reference/action-context' },
+            { label: 'Runtime', slug: 'reference/runtime' },
+            { label: 'Tokens and locations', slug: 'reference/tokens-and-locations' },
+            { label: 'CLI', slug: 'reference/cli' },
+            { label: 'Regexp compatibility', slug: 'reference/regexp' },
+            { label: 'Diagnostics', slug: 'reference/diagnostics' },
+            { label: 'Public API', slug: 'reference/public-api' }
+          ]
+        },
+        {
+          label: 'Project',
+          items: [
+            { label: 'Examples', slug: 'examples' },
+            { label: 'Benchmarks', slug: 'benchmarks' }
+          ]
+        }
+      ],
+      social: [
+        { icon: 'github', label: 'GitHub', href: 'https://github.com/ydah/flexr' }
+      ]
+    })
+  ]
+});

data/site/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "flexr-site",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "astro dev",
+    "build": "astro check && astro build",
+    "preview": "astro preview",
+    "verify": "node scripts/verify-site.mjs"
+  },
+  "devDependencies": {
+    "@astrojs/check": "^0.9.4",
+    "typescript": "^5.7.3"
+  },
+  "dependencies": {
+    "@astrojs/starlight": "^0.36.0",
+    "astro": "^5.14.1"
+  }
+}