resyma 0.1.1

data/lib/resyma/language.rb

@@ -0,0 +1,290 @@
+require "parser"
+require "resyma/parsetree"
+require "resyma/core/automaton"
+require "resyma/core/parsetree"
+require "resyma/core/algorithm"
+
+module Resyma
+  class IllegalLanguageDefinitionError < Error; end
+
+  class IllegalRegexError < Error; end
+
+  class LanguageSyntaxError < Error; end
+
+  #
+  # An visitor which builds automaton. The particular syntax is defined as
+  # follow:
+  #
+  # regex: char | seq | rep | or | opt
+  # char: STRING | ID | ID '(' STRING ')'
+  # seq: '(' regex (';' regex)+ ')'
+  # rep: regex '..' | regex '...'
+  # or: '[' regex (',' regex)+ ']'
+  # opt: '[' regex ']'
+  #
+  class RegexBuildVistor
+    #
+    # Build a Resyma::Core::Automaton from an AST
+    #
+    # @param [Parser::AST::Node] ast An abstract syntax tree
+    #
+    # @return [Resyma::Core::Regexp] A regular expression
+    #
+    def visit(ast)
+      case ast.type
+      when :array
+        if ast.children.length > 1
+          build_or ast
+        elsif ast.children.empty?
+          raise IllegalRegexError, "Empty array is illegal"
+        else
+          build_opt ast
+        end
+      when :begin
+        build_seq(ast)
+      when :irange
+        build_rep(ast, false)
+      when :erange
+        build_rep(ast, true)
+      when :str
+        value = ast.children.first
+        type = Core::CONST_TOKEN_TABLE[value]
+        if type.nil?
+          raise IllegalRegexError,
+                "Unknown constant token [#{value}]"
+        end
+        build_char(type, value)
+      when :send
+        rec, type, *args = ast.children
+        raise IllegalRegexError, "Reciever #{rec} is illegal" unless rec.nil?
+
+        if args.length > 1
+          raise IllegalRegexError,
+                "Two or more arguments is illegal"
+        end
+
+        return build_char(type, nil) if args.empty?
+
+        value = args.first
+        if value.type == :str
+          build_char(type, value.children.first)
+        else
+          raise IllegalRegexError,
+                "Character regex only accepts static string as value, got " +
+                value.type.to_s
+        end
+      end
+    end
+
+    include Core::RegexpOp
+    def build_char(type, value)
+      rchr(Core::PTNodeMatcher.new(type, value))
+    end
+
+    def build_seq(ast)
+      regex_lst = ast.children.map { |sub| visit(sub) }
+      rcat(*regex_lst)
+    end
+
+    def build_rep(ast, one_or_more)
+      left, right = ast.children
+      raise IllegalRegexError, "Beginless range is illegal" if left.nil?
+      raise IllegalRegexError, "Only endless range is legal" unless right.nil?
+
+      regex = visit(left)
+      if one_or_more
+        rcat(regex, rrep(regex))
+      else
+        rrep(regex)
+      end
+    end
+
+    def build_or(ast)
+      n = ast.children.length
+      unless n > 1
+        raise IllegalRegexError,
+              "Or-regex must contain two or more branches, but found #{n}"
+      end
+
+      regex_lst = ast.children.map { |sub| visit(sub) }
+      ror(*regex_lst)
+    end
+
+    def build_opt(ast)
+      value = ast.children[0]
+      regex = visit(value)
+      ror(reps, regex)
+    end
+  end
+
+  class ActionEnvironment
+    def initialize(nodes, binding, filename, lineno)
+      @nodes = nodes
+      @src_binding = binding
+      @src_filename = filename
+      @src_lineno = lineno
+    end
+
+    attr_reader :nodes, :src_binding, :src_filename, :src_lineno
+  end
+
+  #
+  # Language created from single automaton and a associated action. Syntax:
+  #
+  #   regex >> expr
+  #
+  # where `regex` should adhere to syntax described in Resyma::RegexBuildVistor,
+  # and `expr` is an arbitrary ruby expression. Note that
+  #
+  #  - Readonly variable `nodes` is visible in the `expr`, which is a `Array` of
+  #    Resyma::Core::ParseTree and denotes the derivational node sequence
+  #  - Readonly variables `src_binding`, `src_filename` and `src_lineno` are
+  #    visible in the `expr`, which describe the environment surrounding the DSL
+  #  - Variables above can be shadowed by local variables
+  #  - Multiple expressions can be grouped to atom by `begin; end`
+  #
+  class MonoLanguage
+    def initialize(automaton, action)
+      @automaton = automaton
+      @action = action
+    end
+
+    attr_accessor :automaton, :action
+
+    def self.node(type, children)
+      Parser::AST::Node.new(type, children)
+    end
+
+    def self.from(ast, bd, filename, _lino)
+      if ast.type != :send
+        raise IllegalLanguageDefinitionError,
+              "AST with type #{ast} cannot define a language"
+      elsif ast.children[1] != :>>
+        raise IllegalLanguageDefinitionError,
+              "Only AST whose operator is '>>' can define a language"
+      elsif ast.children.length != 3
+        raise IllegalLanguageDefinitionError,
+              "Language definition should be 'regex >> expr'"
+      end
+
+      regex_ast, _, action_ast = ast.children
+
+      automaton = RegexBuildVistor.new.visit(regex_ast).to_automaton
+      action_proc_ast = node :block, [
+        node(:send, [nil, :lambda]),
+        node(:args, [node(:arg, [:__ae__])]),
+        node(:block, [
+               node(:send, [node(:lvar, [:__ae__]), :instance_eval]),
+               node(:args, []),
+               action_ast
+             ])
+      ]
+      action_str = Unparser.unparse(action_proc_ast)
+      # lambda generated by unparser will add two lines before our code
+      # The following adjustment allow us to locate `proc`s in the action
+      # properly
+      action = eval(action_str, bd, filename,
+                    action_ast.loc.expression.line - 2)
+      new automaton, action
+    end
+  end
+
+  class Language
+    def initialize
+      # @type [Array<Resyma::MonoLanguage>]
+      @mono_languages = nil
+      # @type [Resyma::Core::Engine]
+      @engine = nil
+    end
+
+    attr_reader :engine
+
+    def syntax; end
+
+    def build_language(procedure)
+      ast, bd, filename, lino = Resyma.source_of(procedure)
+      body_ast = Resyma.extract_body(ast)
+      if body_ast.nil?
+        raise LanguageSyntaxError,
+              "Define your language by override method syntax"
+      end
+      if body_ast.type == :begin
+        @mono_languages = body_ast.children.map do |stm_ast|
+          MonoLanguage.from(stm_ast, bd, filename, lino)
+        end
+        @engine = Resyma::Core::Engine.new(@mono_languages.map(&:automaton))
+      else
+        @mono_languages = [MonoLanguage.from(body_ast, bd, filename, lino)]
+        @engine = Resyma::Core::Engine.new(@mono_languages.first.automaton)
+      end
+    end
+
+    def built?
+      @mono_languages && @engine
+    end
+
+    #
+    # Interpret the parse tree as current language. Note that the original
+    # evaluation result in the tree will be overlapped.
+    #
+    # @param [Resyma::Core::ParseTree] parsetree A parse tree
+    # @param [Binding] binding Environment
+    # @param [String] filename Source location
+    # @param [Integer] lineno Source location
+    #
+    # @return [Object] Evaluation result
+    #
+    def load_parsetree!(parsetree, binding, filename, lineno, need_clean = true)
+      build_language(method(:syntax)) unless built?
+      parsetree.clear! if need_clean
+      parsetree.parent = nil
+      @engine.traverse!(parsetree)
+      accepted_set = @engine.accepted_tuples(parsetree)
+      tuple4 = accepted_set.min_by(&:belongs_to)
+      if tuple4.nil?
+        raise LanguageSyntaxError,
+              "The code does not adhere to syntax defined by #{self.class}"
+      end
+      dns = @engine.backtrack_for(parsetree, tuple4)
+      nodes = dns.map { |t| @engine.node_of(parsetree, t.p_) }
+      action = @mono_languages[tuple4.belongs_to].action
+      ae = ActionEnvironment.new(nodes, binding, filename, lineno)
+      action.call(ae)
+    end
+
+    #
+    # Interpret the AST as current language
+    #
+    # @param [Parser::AST::Node] ast An abstract syntax tree
+    #
+    # @return [Object] Result returned by action, see Resyma::MonoLanguage for
+    #   more information
+    #
+    def load_ast(ast, binding, filename, lineno)
+      parsetree = Core::DEFAULT_CONVERTER.convert(ast)
+      load_parsetree!(parsetree, binding, filename, lineno, false)
+    end
+
+    #
+    # Load a block as DSL. Note that argument of the block will be ignored.
+    #
+    # @return [Object] Result of the evaluation, defined by current DSL
+    #
+    def load(&block)
+      ast, bd, filename, = Resyma.source_of(block)
+      body_ast = Resyma.extract_body(ast)
+      lino = body_ast.loc.expression.line
+      load_ast(body_ast, bd, filename, lino)
+    end
+
+    #
+    # Initialize a new instance without argument and call `#load`
+    #
+    # @return [Object] Result of the evaluation.
+    # @see See #load
+    #
+    def self.load(&block)
+      new.load(&block)
+    end
+  end
+end

data/lib/resyma/nise/date.rb

@@ -0,0 +1,53 @@
+require "resyma"
+require "date"
+
+#
+# DSL reading dates
+#
+class LangDate < Resyma::Language
+  def syntax
+    id("today") >> Date.today
+
+    (int; id("/"); int; id("/"); int) >> begin
+      year = nodes[0].to_ruby
+      month = nodes[2].to_ruby
+      day = nodes[4].to_ruby
+      Date.new(year, month, day)
+    end
+
+    (numop; numbase; "."; [id("day"), id("month"), id("year")]) >> begin
+      op, num, _, unit = nodes
+      sig = op.to_literal == "+" ? 1 : -1
+      val = num.to_literal.to_i * sig
+      case unit.to_literal
+      when "day" then Date.today.next_day(val)
+      when "month" then Date.today.next_month(val)
+      when "year" then Date.today.next_year(val)
+      end
+    end
+
+    id("yesterday") >> LangDate.load { -1.day }
+    id("tomorrow") >> LangDate.load { +1.day }
+  end
+end
+
+def date(&block)
+  LangDate.load(&block)
+end
+
+class LangTimeline < Resyma::Language
+  def syntax
+    (array; id("-"); str)... >> begin
+      items = []
+      until nodes.empty?
+        date, _, text = (1..3).map { nodes.shift }
+        raise SyntaxError if date.children.length != 3 # '[' DATE ']'
+
+        date = LangDate.new.load_parsetree!(date.children[1], src_binding,
+                                            src_filename, src_lineno)
+        items.push [date, text.to_ruby]
+      end
+      items
+    end
+  end
+end

data/lib/resyma/nise/rubymoji.rb

@@ -0,0 +1,13 @@
+require "resyma"
+
+class LangRubymoji < Resyma::Language
+  def syntax
+    (id("O"); "."; id("O"); str("??")) >> "🤔"
+    (id("o"); id("^"); id("o")) >> "🙃"
+    (id("Zzz"); ".."; "("; id("x"); "."; id("x"); ")") >> "😴"
+  end
+end
+
+def rumoji(&block)
+  LangRubymoji.load(&block)
+end

data/lib/resyma/nise/toml.rb

@@ -0,0 +1,63 @@
+require "resyma"
+
+class LangTOMLBuilder
+  def initialize
+    @root = make_hash
+    @prefix = []
+  end
+
+  def make_hash
+    Hash.new { |hash, key| hash[key] = make_hash }
+  end
+
+  attr_accessor :root, :prefix
+
+  def add!(path, value)
+    raise SyntaxError if path.empty?
+    abs_path = @prefix + path
+    cur = @root
+    abs_path[...-1].each do |name|
+      cur = cur[name.to_sym]
+    end
+    cur[abs_path.last.to_sym] = value
+  end
+end
+
+class LangTOMLNamespace < Resyma::Language
+  def syntax
+    ("["; id; ("."; id)..; "]") >> begin
+      nodes.select { |n| n.symbol == :id }.map { |n| n.to_literal }
+    end
+  end
+end
+
+class LangTOML < Resyma::Language
+  def syntax
+    [array, 
+     (id; ("."; id)..; "="; [int, str, "true", "false", array, hash])]... >>
+    begin
+      bdr = LangTOMLBuilder.new
+      until nodes.empty?
+        car = nodes.shift
+        if car.symbol == :array
+          namespace = 
+            LangTOMLNamespace.new.load_parsetree!(
+              car, src_binding, src_filename, src_lineno
+            )
+          bdr.prefix = namespace
+        else
+          path = [car]
+          car = nodes.shift
+          until car.symbol == :eq
+            path.push car if car.symbol == :id
+            car = nodes.shift
+          end
+          value = nodes.shift
+          bdr.add!(path.map(&:to_literal), 
+                   value.to_ruby(src_binding, src_filename, src_lineno))
+        end
+      end
+      bdr.root
+    end
+  end
+end

data/lib/resyma/parsetree.rb

@@ -0,0 +1,163 @@
+require "resyma/core/parsetree"
+require "unparser"
+
+module Resyma
+  class UnsupportedError < Error; end
+  class NoASTError < Error; end
+
+  #
+  # Derive an AST from a procedure-like object
+  #
+  # @param [#to_proc] procedure Typically Proc or Method
+  #
+  # @return [[Parser::AST::Node, Binding, filename, lino]] An abstract syntax
+  #   tree, the environment surrounding the procedure, and its source location
+  #
+  def self.source_of(procedure)
+    procedure = procedure.to_proc
+    ast = Core.locate(procedure)
+    raise UnsupportedError, "Cannot locate the source of #{ast}" if ast.nil?
+
+    [ast, procedure.binding, *procedure.source_location]
+  end
+
+  #
+  # Extract body part from AST of a procedure-like object
+  #
+  # @param [Parser::AST::Node] procedure_ast AST of a Proc or a Method
+  #
+  # @return [Parser::AST::Node] AST of function body
+  #
+  def self.extract_body(procedure_ast)
+    case procedure_ast.type
+    when :block then procedure_ast.children[2]
+    when :def then procedure_ast.children[2]
+    when :defs then procedure_ast.children[3]
+    else
+      raise UnsupportedError,
+            "Not a supported type of procedure: #{procedure_ast.type}"
+    end
+  end
+
+  #
+  # Derive the parse tree of a function body
+  #
+  # @param [#to_proc] procedure A procedure-like object, typically Proc and
+  #   Method
+  #
+  # @return [[Resyma::Core::ParseTree, Binding, filename, lino]] A parse tree,
+  #   the environment surrounding the procedure, and its source location
+  #
+  def self.body_parsetree_of(procedure)
+    ast, bd, filename, lino = source_of(procedure)
+    body_ast = extract_body(ast)
+    [Core::DEFAULT_CONVERTER.convert(body_ast), bd, filename, lino]
+  end
+
+  #
+  # Evaluator for Resyma::Core::ParseTree
+  #
+  class Evaluator
+    def initialize
+      @rules = {}
+    end
+
+    #
+    # Define a evaluation rule
+    #
+    # @param [Symbol, Array<Symbol>] type Type(s) assocating to the rule
+    # @yieldparam [Parser::AST::Node] AST of the node
+    # @yieldparam [Binding] The environment surrounding the DSL
+    # @yieldparam [String] filename Source location
+    # @yieldparam [Integer] lino Source location
+    #
+    # @return [nil] Nothing
+    #
+    def def_rule(type, &block)
+      types = type.is_a?(Array) ? type : [type]
+      types.each { |sym| @rules[sym] = block }
+      nil
+    end
+
+    #
+    # Evaluate AST of the parse tree
+    #
+    # @param [Resyma::Core::ParseTree] parsetree A parse tree whose `ast` is not
+    #   `nil`
+    # @param [Binding] bd Environment
+    # @param [String] filename Source location
+    # @param [Integer] lino Source location
+    #
+    # @return [Object] Reterning value of corresponding evaluating rule
+    #
+    def evaluate(parsetree, bd, filename, lino)
+      if parsetree.ast.nil?
+        raise NoASTError,
+              "AST of parse trees is necessary for evaluation"
+      end
+
+      evaluate_ast(parsetree.ast, bd, filename, lino)
+    end
+
+    #
+    # Evaluate the AST by defined rules
+    #
+    # @param [Parser::AST::Node] ast An abstract syntax tree
+    # @param [Binding] bd Environment
+    # @param [String] filename Source location
+    # @param [Integer] lino Source location
+    #
+    # @return [Object] Returning value of corresponding evaluating rule
+    #
+    def evaluate_ast(ast, bd, filename, lino)
+      evaluator = @rules[ast.type]
+      if evaluator.nil?
+        fallback ast, bd, filename, lino
+      else
+        evaluator.call(ast, bd, filename, lino)
+      end
+    end
+
+    #
+    # Fallback evaluating method. AST whose type is not defined by current
+    #   evaluator will be passed to this method. The default action is unparse
+    #   the AST by `unparser` and evaluate the string by `eval`
+    #
+    # @param [Parser::AST::Node] ast An abstract syntax tree
+    # @param [Binding] bd The environment
+    # @param [String] filename Source location
+    # @param [Integer] lino Source location
+    #
+    # @return [Object] Evaluating result
+    #
+    def fallback(ast, bd, filename, lino)
+      string = Unparser.unparse(ast)
+      eval(string, bd, filename, lino)
+    end
+  end
+
+  module Core
+    class ParseTree
+      #
+      # Evaluate current parse tree using default evaluator
+      #
+      # @param [Binding] bd Environment
+      # @param [String] filename Source location
+      # @param [Integer] lino Source location
+      #
+      # @return [Object] Evaluation result
+      #
+      def to_ruby(bd = binding, filename = "(resyma)", lino = 1)
+        Evaluator.new.evaluate(self, bd, filename, lino)
+      end
+
+      def to_literal
+        unless leaf?
+          raise TypeError,
+                "Cannot convert a non-leaf node(i.e. non-token) to literal"
+        end
+        children.first
+      end
+    end
+  end
+end

data/lib/resyma/program/automaton.rb

@@ -0,0 +1,84 @@
+require "resyma"
+require "ruby-graphviz"
+require "resyma/core/automaton"
+require "parser/current"
+
+class Visualizer
+  def initialize(automaton)
+    @viz = GraphViz.new(:G, type: :graph)
+    @viz[:rankdir] = "LR"
+    @id_pool = 0
+    # @type [Resyma::Core::Automaton]
+    @automaton = automaton
+    @node_cache = {}
+  end
+
+  def def_state(state)
+    @id_pool += 1
+    label = state.id.to_s
+    shape = @automaton.accept?(state) ? "doublecircle" : "circle"
+    @viz.add_node(@id_pool.to_s, label: label, shape: shape)
+  end
+
+  def shorten(str, limit = 10)
+    str = str.to_s
+    if str.length > limit
+      str[0...limit] + "..."
+    else
+      str
+    end
+  end
+
+  def node_of(state)
+    node = @node_cache[state]
+    return node if node
+
+    node = def_state(state)
+    @node_cache[state] = node
+    node
+  end
+
+  def viz_matcher(node_matcher)
+    rez = node_matcher.type.to_s
+    rez += "(#{shorten(node_matcher.value)})" if node_matcher.value
+    rez
+  end
+
+  def viz!
+    @automaton.transition_table.table.each do |src, value|
+      src_node = node_of src
+      value.each do |can|
+        node_matcher = can.condition
+        dest = can.destination
+        dest_node = node_of(dest)
+        @viz.add_edge(src_node, dest_node, { label: viz_matcher(node_matcher) })
+      end
+    end
+  end
+
+  def output(filename)
+    @viz.output(png: filename)
+  end
+end
+
+def launch
+  output_filename = "./resyma-automaton.png"
+
+  OptionParser.new do |opts|
+    opts.on("-o", "--output FILE",
+            "PNG containing the resulting tree") do |file|
+      output_filename = file
+    end
+    opts.on_tail("-h", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  source = $stdin.read
+  ast = Parser::CurrentRuby.parse(source)
+  automaton = Resyma::RegexBuildVistor.new.visit(ast).to_automaton
+  viz = Visualizer.new(automaton)
+  viz.viz!
+  viz.output(output_filename)
+end