flexr 1.0.0 → 1.1.0

data/lib/flexr/unicode/data.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Unicode
+    module Data
+      autoload :VERSION, File.expand_path("data/properties", __dir__)
+      autoload :PROPERTIES, File.expand_path("data/properties", __dir__)
+      autoload :CASE_FOLD, File.expand_path("data/case_folding", __dir__)
+    end
+  end
+end

data/lib/flexr/unicode/property.rb

@@ -2,8 +2,6 @@
 
 module Flexr
   module Unicode
-    VERSION = Data::VERSION
-
     module Property
       ALIASES = {
         "L" => "L", "Letter" => "L", "N" => "N", "Number" => "N", "Nd" => "Nd",
@@ -21,17 +19,17 @@ module Flexr
       module_function
 
       def ranges(name, negate: false)
-        key = [name.to_s, negate]
+        canonical = canonical_name(name)
+        key = [canonical, negate]
         return CACHE[key] if CACHE.key?(key)
 
-        canonical = canonical_name(name)
         ranges = if canonical.start_with?("POSIX_")
           posix_ranges(canonical.delete_prefix("POSIX_"))
         else
           property_ranges(canonical, name)
         end
         ranges = complement(ranges) if negate
-        CACHE[key] = ranges.map(&:dup).freeze
+        CACHE[key] = ranges.map { |range| range.dup.freeze }.freeze
       end
 
       def canonical_name(name)

data/lib/flexr/unicode/reference_regexp.rb

@@ -30,6 +30,7 @@ module Flexr
       def source_for(node, ignorecase: false)
         case node
         when Regexp::AST::Empty, Regexp::AST::Anchor then ""
+        when Regexp::AST::Fail then "(?!)"
         when Regexp::AST::ByteRange then byte_class([[node.lo, node.hi]])
         when Regexp::AST::CodepointRange then codepoint_class([[node.lo, node.hi]])
         when Regexp::AST::CharClass
@@ -48,6 +49,9 @@ module Flexr
         when Regexp::AST::Alt
           "(?:#{node.children.map { |child| source_for(child, ignorecase: ignorecase) }.join('|')})"
         when Regexp::AST::Star then "(?:#{source_for(node.child, ignorecase: ignorecase)})*"
+        when Regexp::AST::Repeat
+          maximum = node.maximum.nil? ? "" : node.maximum
+          "(?:#{source_for(node.child, ignorecase: ignorecase)}){#{node.minimum},#{maximum}}"
         else
           raise CompileError, "unsupported reference AST node: #{node.class}"
         end

data/lib/flexr/unicode/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Unicode
+    VERSION = "15.1.0"
+  end
+end

data/lib/flexr/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Flexr
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/flexr.rb

@@ -1,81 +1,14 @@
 # frozen_string_literal: true
 
-require "json"
-require "digest"
-require_relative "flexr/version"
-require_relative "flexr/errors"
-require_relative "flexr/diagnostics"
-require_relative "flexr/ir"
-require_relative "flexr/regexp/ast"
-require_relative "flexr/regexp/parser"
-require_relative "flexr/regexp/normalizer"
-require_relative "flexr/regexp/unsupported"
-require_relative "flexr/regexp/char_class"
-require_relative "flexr/unicode/utf8_splitter"
-require_relative "flexr/unicode/data/properties"
-require_relative "flexr/unicode/data/case_folding"
-require_relative "flexr/unicode/property"
-require_relative "flexr/unicode/reference_regexp"
-require_relative "flexr/unicode/case_fold"
-require_relative "flexr/automaton/byte_class_set"
-require_relative "flexr/automaton/nfa"
-require_relative "flexr/automaton/dfa"
-require_relative "flexr/automaton/compiler"
-require_relative "flexr/automaton/analysis"
-require_relative "flexr/automaton/minimizer"
-require_relative "flexr/automaton/accel"
-require_relative "flexr/codegen/base"
-require_relative "flexr/codegen/table"
-require_relative "flexr/codegen/direct"
-require_relative "flexr/codegen/firstmatch"
-require_relative "flexr/codegen/table_packer"
-require_relative "flexr/runtime/location"
-require_relative "flexr/runtime/token"
-require_relative "flexr/runtime/buffer"
-require_relative "flexr/runtime/errors"
-require_relative "flexr/runtime/interpreter"
-require_relative "flexr/runtime/core"
-require_relative "flexr/dsl"
-require_relative "flexr/lexer"
-require_relative "flexr/generated"
-require_relative "flexr/source/static_eval"
-require_relative "flexr/source/prism_reader"
-require_relative "flexr/source/passthrough"
-require_relative "flexr/importer"
-require_relative "flexr/generator"
-require_relative "flexr/options"
-require_relative "flexr/rake_task"
-require_relative "flexr/cli"
+require_relative "flexr/runtime"
 
 module Flexr
-  class << self
-    def compile_pattern(pattern, options: {})
-      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
-      return Automaton::ReferenceDFA.new(regexp, unicode: options[:unicode] == true) if
-        reference_pattern?(regexp, unicode: options[:unicode] == true)
-      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
-      rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
-      state = IR::State.new(name: :initial, inclusive: true, id: 0)
-      spec = IR::Spec.new(
-        class_name: "Pattern", backend: :table, token_kind: :array,
-        encoding: encoding, options: options, declared_tokens: [],
-        states: { initial: state }, rules: [rule], eof_rules: {}, verbatim: nil
-      )
-      Automaton::Compiler.new(spec).compile.machines.fetch(:initial).dfa
-    end
-
-    def reference_pattern?(regexp, unicode: false)
-      return true if regexp.source.match?(/\\[pP]\{/) || regexp.source.match?(/\[:(?:\^)?[a-z]+:\]/)
-
-      unicode && regexp.encoding != Encoding::BINARY && regexp.source.match?(/\\[dDwWsS]/)
-    end
-
-    def parse_pattern(pattern, options: {})
-      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
-      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
-      ast = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
-                               unicode: options[:unicode] == true).parse
-      Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
-    end
-  end
+  autoload :ArtifactWriter, File.expand_path("flexr/artifact_writer", __dir__)
+  autoload :Codegen, File.expand_path("flexr/codegen", __dir__)
+  autoload :Source, File.expand_path("flexr/source", __dir__)
+  autoload :Generator, File.expand_path("flexr/generator", __dir__)
+  autoload :Importer, File.expand_path("flexr/importer", __dir__)
+  autoload :Options, File.expand_path("flexr/options", __dir__)
+  autoload :RakeTask, File.expand_path("flexr/rake_task", __dir__)
+  autoload :CLI, File.expand_path("flexr/cli", __dir__)
 end

data/tools/coverage.rb

@@ -7,7 +7,12 @@ Coverage.start(lines: true)
 
 status = RSpec::Core::Runner.run(["spec", "--format", "progress", "--no-color"], File::NULL, $stdout)
 result = Coverage.result
-files = Dir[File.expand_path("../lib/flexr/**/*.rb", __dir__)]
+files = Dir[File.expand_path("../lib/flexr/**/*.rb", __dir__)].reject do |file|
+  # Generated artifacts duplicate the scanner template's executable lines.
+  # Their behavior is exercised by the suite, while the template is counted
+  # once as the maintainable source of those lines.
+  File.foreach(file).first(3).any? { |line| line.include?("Generated by flexr") }
+end
 totals = files.map do |file|
   lines = result.fetch(file, {}).fetch(:lines, [])
   executable = lines.count { |count| !count.nil? }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flexr
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Yudai Takada
@@ -77,6 +77,7 @@ files:
 - docs/reference/regexp.md
 - docs/reference/runtime.md
 - docs/reference/tokens-and-locations.md
+- docs/releases/v1.1.0.md
 - docs/tutorial/build-a-calculator-lexer.md
 - examples/calculator/README.md
 - examples/calculator/lexer.flexr.rb
@@ -92,19 +93,25 @@ files:
 - examples/with_racc/lexer.flexr.rb
 - exe/flexr
 - lib/flexr.rb
+- lib/flexr/action_resolver.rb
+- lib/flexr/artifact_writer.rb
 - lib/flexr/automaton/accel.rb
 - lib/flexr/automaton/analysis.rb
+- lib/flexr/automaton/backend_cost_model.rb
 - lib/flexr/automaton/byte_class_set.rb
 - lib/flexr/automaton/compiler.rb
 - lib/flexr/automaton/dfa.rb
 - lib/flexr/automaton/minimizer.rb
 - lib/flexr/automaton/nfa.rb
+- lib/flexr/automaton/types.rb
 - lib/flexr/cli.rb
+- lib/flexr/codegen.rb
 - lib/flexr/codegen/base.rb
 - lib/flexr/codegen/direct.rb
 - lib/flexr/codegen/firstmatch.rb
 - lib/flexr/codegen/table.rb
 - lib/flexr/codegen/table_packer.rb
+- lib/flexr/configuration.rb
 - lib/flexr/diagnostics.rb
 - lib/flexr/dsl.rb
 - lib/flexr/errors.rb
@@ -122,16 +129,19 @@ files:
 - lib/flexr/regexp/tokenizer.flexr.rb
 - lib/flexr/regexp/tokenizer.rb
 - lib/flexr/regexp/unsupported.rb
+- lib/flexr/runtime.rb
 - lib/flexr/runtime/buffer.rb
 - lib/flexr/runtime/core.rb
 - lib/flexr/runtime/errors.rb
 - lib/flexr/runtime/interpreter.rb
 - lib/flexr/runtime/location.rb
 - lib/flexr/runtime/token.rb
+- lib/flexr/source.rb
 - lib/flexr/source/passthrough.rb
 - lib/flexr/source/prism_reader.rb
 - lib/flexr/source/static_eval.rb
 - lib/flexr/unicode/case_fold.rb
+- lib/flexr/unicode/data.rb
 - lib/flexr/unicode/data/LICENSE-UNICODE.txt
 - lib/flexr/unicode/data/UNICODE_VERSION
 - lib/flexr/unicode/data/case_folding.rb
@@ -139,6 +149,7 @@ files:
 - lib/flexr/unicode/property.rb
 - lib/flexr/unicode/reference_regexp.rb
 - lib/flexr/unicode/utf8_splitter.rb
+- lib/flexr/unicode/version.rb
 - lib/flexr/version.rb
 - site/README.md
 - site/astro.config.mjs