resyma 0.1.1

data/README.md

@@ -0,0 +1,167 @@
+# A DSL Engine
+
+## Introduction
+
+```ruby
+require "resyma"
+require "date"
+
+#
+# Define a new language by creating a subclass of Resyma::Language and
+# specifying the syntax in method 'syntax'
+#
+class LangDate < Resyma::Language
+  # syntax of 'syntax': regex >> action
+  def syntax
+    # e.g. today
+    id("today") >> Date.today
+
+    # e.g. 2023/1/1
+    (int; id("/"); int; id("/"); int) >> begin
+      year = nodes[0].to_ruby
+      month = nodes[2].to_ruby
+      day = nodes[4].to_ruby
+      Date.new(year, month, day)
+    end
+
+    # e.g. +1.year
+    (numop; numbase; "."; [id("day"), id("month"), id("year")]) >> begin
+      op, num, _, unit = nodes
+      sig = op.to_literal == "+" ? 1 : -1
+      val = num.to_literal.to_i * sig
+      case unit.to_literal
+      when "day" then Date.today.next_day(val)
+      when "month" then Date.today.next_month(val)
+      when "year" then Date.today.next_year(val)
+      end
+    end
+
+    # Recursively interpret
+    id("yesterday") >> LangDate.load { -1.day }
+    id("tomorrow") >> LangDate.load { +1.day }
+  end
+end
+
+def date(&block)
+  LangDate.load(&block) # Interpret a block as DSL
+end
+
+date { today }    #=> #<Date: 2023-02-09 (...)>
+date { tomorrow } #=> #<Date: 2023-02-10 (...)>
+date { 2024/2/9 } #=> #<Date: 2024-02-09 (...)>
+date { +7.day }   #=> #<Date: 2023-02-16 (...)>
+date { -3.month } #=> #<Date: 2022-11-09 (...)>
+```
+
+`Resyma` is a draft of a DSL engine. We prevent blocks containing DSL from evaluating, apply our matching algorithm to the parse tree, and pass matched nodes to libraries to implement the specific semantics of their DSL. Since semantic restrictions like method definiton are unimportant, the syntax of your DSL can be quite free.
+
+Note that this library is unstable and experimental. Several severe limitations will be described in following sections.
+
+## Define your DSL
+
+Define a new DSL by defining a subclass of `Resyma::Language`.
+
+```ruby
+class MyLang << Resyma::Language
+  def syntax
+    regex >> action
+    more...
+  end
+end
+```
+
+`regex` is a DSL defining syntax of your language, and `action` is an arbitrary ruby expression defining semantics of your language. In particular, `regex` is one of following:
+
+- `type`, `type("value")`, `"value"`: match a node by type, value, or both.
+- `(a; b; c)`: match a sequence of nodes in order of `a`, `b`, `c`, where every component is a `regex`
+- `[a, b, c]`: match one of `a`, `b`, `c`, where every component is a `regex`
+- `a..`, `a...`: match `a` zero or more time, or one or more time, where `a` is a `regex`
+- `[a]`: optionally match `a`
+
+Comprehensive document is in the plan.
+
+## Limitations
+
+- Parse tree, not AST
+  Our algorithm works on parse trees, namely concrete syntax trees, but not AST. However, most of Ruby libraries function only at the AST level. Currently, we derive AST by [parser](https://github.com/whitequark/parser) and convert it to parse tree. It is an unacceptable solution because AST of `parser` describes abstract structures of codes and disregards details like parenthesis or semicolons, which in turn causes malfunction of our algorithm.
+- Capturing group
+  In regular expression of string, we capture key components by grouping (e.g., `/Hi, (\w+)!/`) for further processing. Without this feature, regular expression is just a boolean function and almost useless. Currently, `Resyma` does not support capturing group, but we can provide users with a complete list of nodes matched with the regular expression. So users can process matched nodes but cannot choose specific nodes.
+
+## More examples
+
+### Nise-TOML
+
+[TOML](https://toml.io/en/) is a configuraton language.
+
+```ruby
+require "resyma/nise/toml"
+
+LangTOML.load do
+
+    # This is a nise-TOML document
+
+    title = "TOML Example"
+
+    [owner]
+    name = "Tom Preston-Werner"
+
+    [database]
+    enabled = true
+    ports = [ 8000, 8001, 8002 ]
+    data = [ ["delta", "phi"], [3.14] ]
+    temp_targets = { cpu: 79.5, case: 72.0 }
+
+    [servers]
+
+    [servers.alpha]
+    ip = "10.0.0.1"
+    role = "frontend"
+
+    [servers.beta]
+    ip = "10.0.0.2"
+    role = "backend"
+end
+
+#=>   {:title    => "TOML Example",
+#      :owner    => {:name => "Tom Preston-Werner"},
+#      :database => {:enabled => true,
+#                    :ports   => [8000, 8001, 8002],
+#                    :data    => [["delta", "phi"], [3.14]],
+#                    :temp_targets =>{:cpu => 79.5, :case => 72.0}},
+#      :servers  => {:alpha => {:ip => "10.0.0.1", :role => "frontend"},
+#                    :beta  => {:ip => "10.0.0.2", :role => "backend"}}}
+```
+
+### Timeline
+
+`Timeline` uses DSL defined by the example at the top.
+
+```ruby
+require "resyma/nise/date"
+
+LangTimeline.load do
+  [2020/8/15] - "First day of class"
+  [2020/10/9] - "Test #1"
+  [yesterday] - "Research paper due"
+  [today]     - "Zzz..."
+  [+7.day]    - "Test #2"
+  [+2.month]  - "Final project due"
+end
+
+#=> [[#<Date: 2020-08-15 (...)>, "First day of class"],
+#    [#<Date: 2020-10-09 (...)>, "Test #1"],
+#    [#<Date: 2023-02-08 (...)>, "Research paper due"],
+#    [#<Date: 2023-02-09 (...)>, "Zzz..."],
+#    [#<Date: 2023-02-16 (...)>, "Test #2"],
+#    [#<Date: 2023-04-09 (...)>, "Final project due"]]
+```
+
+### Rubymoji
+
+```ruby
+require "resyma/nise/rubymoji"
+
+rumoji { o^o }         #=> 🙃
+rumoji { O.O ?? }      #=> 🤔
+rumoji { Zzz.. (x.x) } #=> 😴
+```

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec]

data/lib/resyma/core/algorithm/engine.rb

@@ -0,0 +1,189 @@
+require "resyma/core/utilities"
+require "resyma/core/automaton"
+require "resyma/core/algorithm/tuple"
+
+module Resyma
+  module Core
+    #
+    # The engine of the matching algorithm
+    #
+    class Engine
+      #
+      # Create an instance of the algorithmic engine
+      #
+      # @param [Array<Resyma::Core::Automaton>, Resyma::Core::Automaton]
+      #   automata A list of automata
+      #
+      def initialize(automata)
+        automata = [automata] if automata.is_a?(Automaton)
+        raise TypeError, "Need a list of automata" unless automata.is_a?(Array)
+
+        @automata = automata
+      end
+
+      #
+      # Determine the type of the node corresponding step 2 of the algorithm
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A node of parse tree
+      #
+      # @return [:a, :b, :c] Type of the node
+      #
+      def node_type(parsetree)
+        if parsetree.root?          then :a
+        elsif parsetree.index.zero? then :b
+        else
+          :c
+        end
+      end
+
+      #
+      # Right-most path of the tree
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A parse tree
+      #
+      # @return [Set<Resyma::Core::ParseTree>] A set of nodes on the RMP
+      #
+      def RMP(parsetree)
+        if parsetree.leaf?
+          Set[parsetree]
+        else
+          Set[parsetree] | RMP(parsetree.children.last)
+        end
+      end
+
+      #
+      # Step 2 of the algorithm
+      #
+      # @param [Resyma::Core::ParseTree] node A parse tree node
+      #
+      # @return [nil] Nothing
+      #
+      def assign_start!(node)
+        @automata.each_with_index do |automaton, idx|
+          node.field.start[idx] =
+            case node_type(node)
+            when :a
+              Set[Tuple2.new(-1, automaton.start, belongs_to: idx)]
+            when :b
+              node.parent.field.start[idx]
+            when :c
+              # @type [Resyma::Core::ParseTree]
+              brother = node.parent.children[node.index - 1]
+              Utils.big_union(RMP(brother).map do |node_|
+                node_.field.trans[idx].map do |tuple4|
+                  Tuple2.new(tuple4.p_, tuple4.q_, belongs_to: idx)
+                end
+              end)
+            end
+        end
+      end
+
+      #
+      # Step 3 of the algorithm
+      #
+      # @param [Resyma::Core::ParseTree] node A parse tree
+      #
+      # @return [nil] Nothing
+      #
+      def assign_trans!(node)
+        @automata.each_with_index do |automaton, idx|
+          node.field.trans[idx] = node.field.start[idx].map do |tuple2|
+            next_q = automaton.destination(tuple2.q, node)
+            next_q and Tuple4.new(tuple2.p, tuple2.q, node.field.id, next_q,
+                                  belongs_to: idx)
+          end.compact.to_set
+        end
+      end
+
+      CACHE_INDEX_TO_NODE = "ALGO_IDX_NODE_MAP".freeze
+
+      #
+      # Find the parse tree through its ID
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A parse tree processed by
+      #   `#traverse!`
+      # @param [Integer] id Depth-first ordered ID, based on 0
+      #
+      # @return [Resyma::Core::ParseTree, nil] Result, or nil if node with `id`
+      #   does not exist
+      #
+      def node_of(parsetree, id)
+        parsetree.cache[Engine::CACHE_INDEX_TO_NODE][id]
+      end
+
+      #
+      # Traverse the parse tree once and modify fields for every nodes
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A parse tree
+      #
+      # @return [nil] Nothing
+      #
+      def traverse!(parsetree)
+        id = 0
+        parsetree.cache[Engine::CACHE_INDEX_TO_NODE] = {}
+        parsetree.depth_first_each do |tree|
+          tree.field.id = id
+          assign_start! tree
+          assign_trans! tree
+          parsetree.cache[Engine::CACHE_INDEX_TO_NODE][id] = tree
+          id += 1
+        end
+      end
+
+      #
+      # Compute accepted 4-tuples in the processed tree
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A parse tree processed by
+      #   `#traverse!`
+      #
+      # @return [Set<Resyma::Core::Tuple4>] A set of 4-tuples
+      #
+      def accepted_tuples(parsetree)
+        Utils.big_union(RMP(parsetree).map do |node|
+          Utils.big_union(node.field.trans.values.map do |set_of_tuple4|
+            set_of_tuple4.select do |tuple4|
+              @automata[tuple4.belongs_to].accept?(tuple4.q_)
+            end.to_set
+          end)
+        end)
+      end
+
+      #
+      # Backtrack the derivational node sequence terminating at the 4-tuple
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A processed parse tree
+      # @param [Resyma::Core::Tuple4] accepted_tuple4 A 4-tuple derived from
+      #   `#accepted_tuples`
+      #
+      # @return [Array<Resyma::Core::Tuple4>] A derivational node sequence
+      #
+      def backtrack_for(parsetree, tuple4)
+        if tuple4.p == -1
+          [tuple4]
+        else
+          # @type [Resyma::Core::ParseTree]
+          prev = parsetree.cache[Engine::CACHE_INDEX_TO_NODE][tuple4.p]
+          prev_tuple4 = prev.field.trans[tuple4.belongs_to].find do |candidate|
+            candidate.p_ == tuple4.p && candidate.q_ == tuple4.q
+          end
+          backtrack_for(parsetree, prev_tuple4) + [tuple4]
+        end
+      end
+
+      #
+      # Backtrack the derivational node sequence(s) of a parse tree processed by
+      #   the algorithm
+      #
+      # @param [Resyma::Core::ParseTree] parsetree A parse tree processed by
+      #   `#traverse!`
+      # @param [Set<Resyma::Core::Tuple4>] terminal_tuples Accepted sets derived
+      #   from `#accepted_tuples`
+      #
+      # @return [Array<Array<Resyma::Core::Tuple4>] Derivational node sequences
+      #
+      def backtrack(parsetree, terminal_tuples = accepted_tuples(parsetree))
+        terminal_tuples.map { |tuple4| backtrack_for(parsetree, tuple4) }
+      end
+    end
+  end
+end

data/lib/resyma/core/algorithm/matcher.rb

@@ -0,0 +1,48 @@
+require "resyma/core/automaton/matchable"
+
+module Resyma
+  module Core
+    class PTNodeMatcher
+      include Matchable
+      #
+      # Create a instance of parse tree node matcher
+      #
+      # @param [Symbol] type Symbol of the node
+      # @param [Object] value Value of the node, indicating that the node is a
+      #   token. Currently, `nil` means that the node is an non-leaf node or it
+      #   is a leaf node but its value is unimportant
+      #
+      def initialize(type, value = nil)
+        @type = type
+        @value = value
+      end
+
+      attr_reader :type, :value
+
+      def ==(other)
+        other.is_a?(PTNodeMatcher) &&
+          other.type == @type &&
+          other.value == @value
+      end
+
+      #
+      # Whether the matcher matches with the parse tree
+      #
+      # @param [Resyma::Core::ParseTree] parsetree Node of parse tree
+      #
+      # @return [true, false] Result
+      #
+      def match_with_value?(parsetree)
+        if parsetree.is_a?(Resyma::Core::ParseTree)
+          parsetree.symbol == @type &&
+            (@value.nil? || (parsetree.leaf? &&
+                             parsetree.children[0] == @value))
+        elsif parsetree.is_a?(PTNodeMatcher)
+          self == parsetree
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/resyma/core/algorithm/tuple.rb

@@ -0,0 +1,25 @@
+module Resyma
+  module Core
+    class Tuple4
+      def initialize(p, q, p_, q_, belongs_to: 0)
+        @p = p
+        @q = q
+        @p_ = p_
+        @q_ = q_
+        @belongs_to = belongs_to
+      end
+
+      attr_accessor :p, :q, :p_, :q_, :belongs_to
+    end
+
+    class Tuple2
+      def initialize(p, q, belongs_to: 0)
+        @p = p
+        @q = q
+        @belongs_to = belongs_to
+      end
+
+      attr_accessor :p, :q, :belongs_to
+    end
+  end
+end

data/lib/resyma/core/algorithm.rb

@@ -0,0 +1,5 @@
+require "resyma/core/algorithm/tuple"
+require "resyma/core/algorithm/matcher"
+require "resyma/core/algorithm/engine"
+
+__END__

data/lib/resyma/core/automaton/builder.rb

@@ -0,0 +1,78 @@
+require "set"
+
+require "resyma/core/automaton/state"
+require "resyma/core/automaton/transition"
+
+module Resyma
+  module Core
+    class AutomatonBuilder
+      class NoStartError < Resyma::Error; end
+
+      def initialize
+        @next_id = 0
+        @start = nil
+        @accept_set = Set[]
+        @transition_table = TransitionTable.new
+      end
+
+      #
+      # Adds a new state to the automaton and returns it
+      #
+      # @return [Resyma::Core::State] A new state
+      #
+      def new_state!(start: false, accept: false)
+        inst = State.with_id(@next_id)
+        @next_id += 1
+        start! inst if start
+        accept! inst if accept
+        inst
+      end
+
+      #
+      # Adds a new transition to the automaton
+      #
+      # @param [Resyma::Core::State] from_state Starting state
+      # @param [Resyma::Core::Matchable] matchable Condition
+      # @param [Resyma::Core::State] to_state Destination
+      #
+      # @return [nil] Nothing
+      #
+      def add_transition!(from_state, matchable, to_state)
+        @transition_table.add_transition!(from_state, matchable, to_state)
+      end
+
+      #
+      # Specify a starting state for the automaton
+      #
+      # @param [Resyma::Core::State] state The starting state
+      #
+      # @return [nil] Nothing
+      #
+      def start!(state)
+        @start = state
+        nil
+      end
+
+      #
+      # Add a state to the accept set of the automaton
+      #
+      # @param [Resyma::Core::State] state An acceptable state
+      #
+      # @return [nil] Nothing
+      #
+      def accept!(state)
+        @accept_set.add(state)
+        nil
+      end
+
+      def build
+        if @start.nil?
+          raise NoStartError,
+                "Cannot build a automaton without a start state"
+        end
+
+        Automaton.new(@start, @accept_set, @transition_table)
+      end
+    end
+  end
+end

data/lib/resyma/core/automaton/definition.rb

@@ -0,0 +1,32 @@
+require "set"
+
+module Resyma
+  module Core
+    class Automaton
+      #
+      # Returns a new instance of Resyma::Core::Automaton
+      #
+      # @param [Resyma::Core::State] start Starting state of the automaton
+      # @param [Set<Resyma::Core::State>] accept_set A set of states accepted by
+      #   the automaton
+      # @param [Resyma::Core::TransitionTable] transition_table Transitions of
+      #   the automaton
+      #
+      def initialize(start, accept_set, transition_table = TransitionTable.new)
+        @start = start
+        @accept_set = accept_set
+        @transition_table = transition_table
+      end
+
+      attr_reader :start, :accept_set, :transition_table
+
+      def accept?(state)
+        @accept_set.include? state
+      end
+
+      def destination(state, value)
+        @transition_table.destination(state, value)
+      end
+    end
+  end
+end

data/lib/resyma/core/automaton/epsilon_NFA.rb

@@ -0,0 +1,115 @@
+require "set"
+require "resyma/core/utilities"
+require "resyma/core/automaton/definition"
+require "resyma/core/automaton/builder"
+
+module Resyma
+  module Core
+    class EpsilonClass
+      include Matchable
+    end
+
+    Epsilon = EpsilonClass.new
+
+    class Automaton
+      #
+      # Computes the epsilon-closure of `state`
+      #
+      # @param [Set<Resyma::Core::State>] state_set Starting set of state
+      #
+      # @return [Set<Resyma::Core::State>] The epsilon-closure
+      #
+      def eclose(state_set)
+        queue = state_set.to_a
+        closure = queue.to_set
+        until queue.empty?
+          next_state = queue.shift
+          transition_table.candidates(next_state).each do |can|
+            next unless can.condition.equal?(Epsilon) &&
+                        !closure.include?(can.destination)
+
+            closure.add(can.destination)
+            queue.push(can.destination)
+          end
+        end
+        closure
+      end
+
+      def possible_nonepsilon_conditions(state)
+        transition_table.candidates(state).map(&:condition).select do |cond|
+          !cond.equal?(Epsilon)
+        end.to_set
+      end
+
+      #
+      # Computes next epsilon-closures connecting with `epsilon_closure`
+      #
+      # @param [Set<Resyma::Core::State>] epsilon_closure Current closure
+      # @param [Hash] closure_map Hash map from closures to states, may be
+      #   modified
+      # @param [Resyma::Core::AutomatonBuilder] ab Builder of the new DFA, may
+      #   be modified
+      #
+      # @return [Array<Set>] New unrecorded epsilon-closures
+      #
+      def generate_reachable_epsilon_closures(epsilon_closure, closure_map, ab)
+        current_state = closure_map[epsilon_closure]
+        condition_sets = epsilon_closure.map do |state|
+          possible_nonepsilon_conditions(state)
+        end
+        new_closures = []
+        Utils.big_union(condition_sets).each do |cond|
+          new_closure = Set[]
+          epsilon_closure.each do |state|
+            # [WARN] We are using a `matchable` as a `matched value, which may
+            # cause unexpected consequence
+            dst = destination(state, cond)
+            new_closure.add dst unless dst.nil?
+          end
+          raise "Internal error: No destination states" if new_closure.empty?
+
+          new_closure = eclose(new_closure)
+          if closure_map.include?(new_closure)
+            recorded_state = closure_map[new_closure]
+            ab.add_transition!(current_state, cond, recorded_state)
+          else
+            new_state = ab.new_state!
+            closure_map[new_closure] = new_state
+            ab.add_transition!(current_state, cond, new_state)
+            new_closures.push new_closure
+          end
+        end
+        new_closures
+      end
+
+      def to_DFA
+        ab = AutomatonBuilder.new
+        start_closure = eclose(Set[start])
+        start_state = ab.new_state!
+        ab.start! start_state
+        closure_map = { start_closure => start_state }
+        queue = [start_closure]
+        until queue.empty?
+          next_closure = queue.shift
+          unrecorded_closures = generate_reachable_epsilon_closures(
+            next_closure, closure_map, ab
+          )
+          queue += unrecorded_closures
+        end
+        closure_map.each do |closure, state|
+          ab.accept! state if closure.any? { |s| accept? s }
+        end
+        ab.build
+      end
+
+      def has_epsilon?
+        transition_table.table.values.each do |cans|
+          cans.each do |can|
+            return true if can.condition.equal?(Epsilon)
+          end
+        end
+        false
+      end
+    end
+  end
+end

data/lib/resyma/core/automaton/matchable.rb

@@ -0,0 +1,16 @@
+module Resyma
+  module Core
+    module Matchable
+      #
+      # Matches with an object
+      #
+      # @param [Object] _other Value to be matched
+      #
+      # @return [true,false] Result
+      #
+      def match_with_value?(_other)
+        false
+      end
+    end
+  end
+end