bel_parser 1.0.0.alpha.1

data/lib/bel_parser/parsers/expression/term.rl

@@ -0,0 +1,157 @@
+# begin: ragel
+=begin
+%%{
+  machine bel;
+
+  include 'parameter.rl';
+
+  action start_function {
+    @buffers[:function] = []
+  }
+
+  action append_function {
+    @buffers[:function] << fc
+  }
+
+  action finish_function {
+    @buffers[:function] = identifier(utf8_string(@buffers[:function]))
+  }
+
+  action term_init {
+    @buffers[:term_stack] = [ term() ]
+  }
+
+  action inner_term_init {
+    @buffers[:term_stack] << term()
+  }
+
+  action term_fx {
+    fx                        = @buffers[:function]
+    @buffers[:term_stack][-1] = @buffers[:term_stack][-1] << function(fx)
+  }
+
+  action term_argument {
+    @buffers[:term_stack][-1] = @buffers[:term_stack][-1] << argument(@buffers[:parameter])
+    @buffers[:parameter]      = nil
+  }
+
+  action fxbt {
+    fpc -= @buffers[:function].length + 1
+    fcall inner_term;
+  }
+
+  action fxret {
+    inner_term = @buffers[:term_stack].pop
+    @buffers[:term_stack][-1] = @buffers[:term_stack][-1] << argument(inner_term)
+    fret;
+  }
+
+  action yield_term_ast  {
+    yield @buffers[:term_stack][-1]
+  }
+
+  inner_term :=
+    IDENT >inner_term_init >start_function $append_function %finish_function
+    SP*
+    '(' @term_fx
+      (
+        BEL_PARAMETER %term_argument |
+        IDENT >start_function $append_function '(' @fxbt
+      )
+      (
+        SP* ',' SP*
+        (
+          BEL_PARAMETER %term_argument |
+          IDENT >start_function $append_function '(' @fxbt
+        )
+      )*
+    ')' @fxret;
+
+  outer_term =
+    IDENT >term_init >start_function $append_function %finish_function
+    SP*
+    '(' @term_fx
+      (
+        BEL_PARAMETER %term_argument |
+        IDENT >start_function $append_function '(' @fxbt
+      )
+      (
+        SP* ',' SP*
+        (
+          BEL_PARAMETER %term_argument |
+          IDENT >start_function $append_function '(' @fxbt
+        )
+      )*
+    ')';
+
+  term :=
+    outer_term %yield_term_ast NL;
+}%%
+=end
+# end: ragel
+
+require_relative '../ast/node'
+require_relative '../mixin/buffer'
+require_relative '../nonblocking_io_wrapper'
+
+module BELParser
+  module Parsers
+    module Expression
+      module Term
+
+        class << self
+
+          MAX_LENGTH = 1024 * 128 # 128K
+
+          def parse(content)
+            return nil unless content
+
+            Parser.new(content).each do |obj|
+              yield obj
+            end
+          end
+        end
+
+        private
+
+        class Parser
+          include Enumerable
+          include BELParser::Parsers::Buffer
+          include BELParser::Parsers::AST::Sexp
+
+          def initialize(content)
+            @content = content
+      # begin: ragel        
+            %% write data;
+      # end: ragel        
+          end
+
+          def each
+            @buffers = {}
+            stack    = []
+            data     = @content.unpack('C*')
+            p        = 0
+            pe       = data.length
+            eof      = data.length
+
+      # begin: ragel        
+            %% write init;
+            %% write exec;
+      # end: ragel        
+          end
+        end
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  $stdin.each_line do |line|
+    BELParser::Parsers::Expression::Term.parse(line) { |obj|
+      puts obj.inspect
+    }
+  end
+end
+
+# vim: ft=ruby ts=2 sw=2:
+# encoding: utf-8

data/lib/bel_parser/parsers/line_parser.rb

@@ -0,0 +1,92 @@
+require_relative 'common'
+require_relative 'expression'
+require_relative 'bel_script'
+
+# Top-level LINE module.
+module LINE
+  include BELParser::Parsers::Common
+  include BELParser::Parsers::Expression
+  include BELParser::Parsers::BELScript
+
+  BEL_PARSERS = [
+    BELParser::Parsers::Common::BlankLine,
+    BELParser::Parsers::Common::CommentLine,
+    BELParser::Parsers::Common::Identifier,
+    BELParser::Parsers::Common::String,
+    BELParser::Parsers::Common::List,
+    BELParser::Parsers::Expression::Parameter,
+    BELParser::Parsers::Expression::Term,
+    BELParser::Parsers::Expression::Relationship,
+    BELParser::Parsers::Expression::Comment,
+    BELParser::Parsers::Expression::StatementObservedTerm,
+    BELParser::Parsers::Expression::StatementSimple,
+    BELParser::Parsers::Expression::StatementNested,
+    BELParser::Parsers::BELScript::DefineAnnotation,
+    BELParser::Parsers::BELScript::DefineNamespace,
+    BELParser::Parsers::BELScript::Set,
+    BELParser::Parsers::BELScript::Unset
+  ].freeze
+
+  # rubocop:disable MethodLength, AbcSize
+  def self.parse(io)
+    # single line transform
+    line_enum = io
+                .each_line
+                .lazy
+                .map { |line| LINE.normalize_line_terminators(line) }
+
+    # multi-line transform
+    loop do
+      begin
+        line = line_enum.next
+
+        while line.end_with?("\\\n")
+          line.chomp!("\\\n")
+          line += line_enum.next
+        end
+
+        BEL_PARSERS.each do |parser|
+          # rubocop:disable BlockDelimiters
+          parser.parse(line) { |obj|
+            puts "parser: #{parser.inspect},"\
+                 "line: #{line.strip},"\
+                 "object: \n#{obj.inspect}"
+          }
+        end
+      rescue StopIteration
+        return
+      end
+    end
+  end
+
+  def self.normalize_line_terminators(string)
+    return nil unless string
+    string.strip + "\n"
+  end
+
+  def self.map_empty_line(string)
+    if LINE.match_empty_line(string)
+      nil
+    else
+      string
+    end
+  end
+
+  def self.map_comment_line(string)
+    if LINE.match_comment_line(string)
+      nil
+    else
+      string
+    end
+  end
+
+  def self.match_empty_line(string)
+    string =~ /^\s*$/
+  end
+
+  def self.match_comment_line(string)
+    string =~ /^\s*#/
+  end
+end
+
+LINE.parse($stdin) if FILE == $PROGRAM_NAME

data/lib/bel_parser/parsers/mixin/buffer.rb

@@ -0,0 +1,10 @@
+module BELParser
+  module Parsers
+    # Buffer module.
+    module Buffer
+      def utf8_string(buffer)
+        buffer.pack('C*').force_encoding('utf-8')
+      end
+    end
+  end
+end

data/lib/bel_parser/parsers/nonblocking_io_wrapper.rb

@@ -0,0 +1,50 @@
+# Provides a platform-independent, non-blocking wrapper for reading an
+# {http://ruby-doc.org/core-2.2.2/IO.html IO}-like object. This wrapper
+# object must be enumerated using the {#each} method.
+class NonblockingIOWrapper
+  # Initialize this wrapper around the +io+ object and read at most
+  # +read_length+ bytes in a non-blocking manner.
+  #
+  # @param [IO]     io          an IO-like object
+  # @param [Fixnum] read_length the buffer length to read
+  def initialize(io, read_length = (128 * 1024))
+    @io          = io
+    @read_length = read_length
+    @read_method = nonblocking_read(@io)
+  end
+
+  # Yields each buffer read from the wrapped IO-like object to the provided
+  # block (e.g. +{ |block| ... }+). The read length is set on {#initialize}.
+  #
+  # @yield the buffers read from the IO-like object
+  # @yieldparam [String] buffer the read buffer as uninterpreted bytes
+  def each
+    while (buffer = @read_method.call(@read_length))
+      yield buffer
+    end
+  rescue IO::WaitReadable
+    IO.select([@io])
+    retry
+  # rubocop:disable HandleExceptions
+  rescue EOFError
+    # end of stream; parsing complete
+  end
+
+  private
+
+  # Returns the method, appropriate for your platform, to read IO in a
+  # non-blocking manner.
+  #
+  # @example Call directly.
+  #   nonblocking_read(StringIO.new('hello')).call(4)
+  #
+  # @param  [IO] io an IO-like object
+  # @return [Method] a non-blocking read method
+  def nonblocking_read(io)
+    if Gem.win_platform?
+      io.method(:sysread)
+    else
+      io.method(:read_nonblock)
+    end
+  end
+end

data/lib/bel_parser/script/parser.rb

@@ -0,0 +1,49 @@
+require_relative '../ast_filter'
+require_relative '../ast_generator'
+require_relative '../parsers/common'
+require_relative '../parsers/expression'
+require_relative '../parsers/bel_script'
+
+module BELParser
+  module Script
+    # Parser for BEL Script.
+    class Parser
+      include BELParser::Parsers::Common
+      include BELParser::Parsers::Expression
+      include BELParser::Parsers::BELScript
+
+      FILTER = BELParser::ASTFilter.new(
+        :statement_simple,
+        :observed_term,
+        :nested_statement,
+        :define_annotation,
+        :define_namespace,
+        :set,
+        :unset,
+        :blank_line,
+        :comment_line
+      )
+
+      def each(io)
+        if block_given?
+          filtered_ast = FILTER.each(BELParser::ASTGenerator.new.each(io))
+          filtered_ast.each do |results|
+            yield results
+          end
+        else
+          enum_for(:each, io)
+        end
+      end
+    end
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  BELParser::Script::Parser.new.each($stdin) do |line_result|
+    line_number, line, ast_results = line_result
+    puts "#{line_number}: #{line}"
+    ast_results.each do |ast|
+      puts ast.to_s(1)
+    end
+  end
+end

data/lib/bel_parser/vendor/ast.rb

@@ -0,0 +1,17 @@
+# {AST} is a library for manipulating abstract syntax trees.
+#
+# It embraces immutability; each AST node is inherently frozen at
+# creation, and updating a child node requires recreating that node
+# and its every parent, recursively.
+# This is a design choice. It does create some pressure on
+# garbage collector, but completely eliminates all concurrency
+# and aliasing problems.
+#
+# See also {AST::Node}, {AST::Processor::Mixin} and {AST::Sexp} for
+# additional recommendations and design patterns.
+#
+module AST
+  require_relative 'ast/node'
+  require_relative 'ast/processor'
+  require_relative 'ast/sexp'
+end

data/lib/bel_parser/vendor/ast/node.rb

@@ -0,0 +1,254 @@
+module AST
+  # Node is an immutable class, instances of which represent abstract
+  # syntax tree nodes. It combines semantic information (i.e. anything
+  # that affects the algorithmic properties of a program) with
+  # meta-information (line numbers or compiler intermediates).
+  #
+  # Notes on inheritance
+  # ====================
+  #
+  # The distinction between semantics and metadata is important. Complete
+  # semantic information should be contained within just the {#type} and
+  # {#children} of a Node instance; in other words, if an AST was to be
+  # stripped of all meta-information, it should remain a valid AST which
+  # could be successfully processed to yield a result with the same
+  # algorithmic properties.
+  #
+  # Thus, Node should never be inherited in order to define methods which
+  # affect or return semantic information, such as getters for `class_name`,
+  # `superclass` and `body` in the case of a hypothetical `ClassNode`. The
+  # correct solution is to use a generic Node with a {#type} of `:class`
+  # and three children. See also {Processor} for tips on working with such
+  # ASTs.
+  #
+  # On the other hand, Node can and should be inherited to define
+  # application-specific metadata (see also {#initialize}) or customize the
+  # printing format. It is expected that an application would have one or two
+  # such classes and use them across the entire codebase.
+  #
+  # The rationale for this pattern is extensibility and maintainability.
+  # Unlike static ones, dynamic languages do not require the presence of a
+  # predefined, rigid structure, nor does it improve dispatch efficiency,
+  # and while such a structure can certainly be defined, it does not add
+  # any value but incurs a maintaining cost.
+  # For example, extending the AST even with a transformation-local
+  # temporary node type requires making globally visible changes to
+  # the codebase.
+  #
+  class Node
+    # Returns the type of this node.
+    # @return [Symbol]
+    attr_reader :type
+
+    # Returns the children of this node.
+    # The returned value is frozen.
+    # @return [Array]
+    attr_reader :children
+
+    # Returns the precomputed hash value for this node
+    # @return [Fixnum]
+    attr_reader :hash
+
+    # Constructs a new instance of Node.
+    #
+    # The arguments `type` and `children` are converted with `to_sym` and
+    # `to_a` respectively. Additionally, the result of converting `children`
+    # is frozen. While mutating the arguments is generally considered harmful,
+    # the most common case is to pass an array literal to the constructor. If
+    # your code does not expect the argument to be frozen, use `#dup`.
+    #
+    # The `properties` hash is passed to {#assign_properties}.
+    def initialize(type, children=[], properties={})
+      @type, @children = type.to_sym, children.to_a.freeze
+
+      assign_properties(properties)
+
+      @hash = [@type, @children, self.class].hash
+
+      freeze
+    end
+
+    # Test if other object is equal to
+    # @param [Object] other
+    # @return [Boolean]
+    def eql?(other)
+      self.class.eql?(other.class)   &&
+      @type.eql?(other.type)         &&
+      @children.eql?(other.children)
+    end
+
+    # By default, each entry in the `properties` hash is assigned to
+    # an instance variable in this instance of Node. A subclass should define
+    # attribute readers for such variables. The values passed in the hash
+    # are not frozen or whitelisted; such behavior can also be implemented
+    # by subclassing Node and overriding this method.
+    #
+    # @return [nil]
+    def assign_properties(properties)
+      properties.each do |name, value|
+        instance_variable_set :"@#{name}", value
+      end
+
+      nil
+    end
+    protected :assign_properties
+
+    alias   :original_dup :dup
+    private :original_dup
+
+    # Nodes are already frozen, so there is no harm in returning the
+    # current node as opposed to initializing from scratch and freezing
+    # another one.
+    #
+    # @return self
+    def dup
+      self
+    end
+    alias :clone :dup
+
+    # Returns a new instance of Node where non-nil arguments replace the
+    # corresponding fields of `self`.
+    #
+    # For example, `Node.new(:foo, [ 1, 2 ]).updated(:bar)` would yield
+    # `(bar 1 2)`, and `Node.new(:foo, [ 1, 2 ]).updated(nil, [])` would
+    # yield `(foo)`.
+    #
+    # If the resulting node would be identical to `self`, does nothing.
+    #
+    # @param  [Symbol, nil] type
+    # @param  [Array, nil]  children
+    # @param  [Hash, nil]   properties
+    # @return [AST::Node]
+    def updated(type=nil, children=nil, properties=nil)
+      new_type       = type       || @type
+      new_children   = children   || @children
+      new_properties = properties || {}
+
+      if @type == new_type &&
+          @children == new_children &&
+          properties.nil?
+        self
+      else
+        # Maybe change call?
+        original_dup.send :initialize, new_type, new_children, new_properties
+      end
+    end
+
+    # Compares `self` to `other`, possibly converting with `to_ast`. Only
+    # `type` and `children` are compared; metadata is deliberately ignored.
+    #
+    # @return [Boolean]
+    def ==(other)
+      if equal?(other)
+        true
+      elsif other.respond_to? :to_ast
+        other = other.to_ast
+        other.type == self.type &&
+          other.children == self.children
+      else
+        false
+      end
+    end
+
+    # Concatenates `array` with `children` and returns the resulting node.
+    #
+    # @return [AST::Node]
+    def concat(array)
+      updated(nil, @children + array.to_a)
+    end
+
+    alias + concat
+
+    # Appends `element` to `children` and returns the resulting node.
+    #
+    # @return [AST::Node]
+    def append(element)
+      updated(nil, @children + [element])
+    end
+
+    alias << append
+
+    # Returns {#children}. This is very useful in order to decompose nodes
+    # concisely. For example:
+    #
+    #     node = s(:gasgn, :$foo, s(:integer, 1))
+    #     s
+    #     var_name, value = *node
+    #     p var_name # => :$foo
+    #     p value    # => (integer 1)
+    #
+    # @return [Array]
+    def to_a
+      children
+    end
+
+    # Converts `self` to a pretty-printed s-expression.
+    #
+    # @param  [Integer] indent Base indentation level.
+    # @return [String]
+    def to_sexp(indent=0)
+      indented = "  " * indent
+      sexp = "#{indented}(#{fancy_type}"
+
+      first_node_child = children.index do |child|
+        child.is_a?(Node) || child.is_a?(Array)
+      end || children.count
+
+      children.each_with_index do |child, idx|
+        if child.is_a?(Node) && idx >= first_node_child
+          sexp << "\n#{child.to_sexp(indent + 1)}"
+        else
+          sexp << " #{child.inspect}"
+        end
+      end
+
+      sexp << ")"
+
+      sexp
+    end
+
+    alias to_s to_sexp
+
+    # Converts `self` to a s-expression ruby string.
+    # The code return will recreate the node, using the sexp module s()
+    #
+    # @param  [Integer] indent Base indentation level.
+    # @return [String]
+    def inspect(indent=0)
+      indented = "  " * indent
+      sexp = "#{indented}s(:#{@type}"
+
+      first_node_child = children.index do |child|
+        child.is_a?(Node) || child.is_a?(Array)
+      end || children.count
+
+      children.each_with_index do |child, idx|
+        if child.is_a?(Node) && idx >= first_node_child
+          sexp << ",\n#{child.inspect(indent + 1)}"
+        else
+          sexp << ", #{child.inspect}"
+        end
+      end
+
+      sexp << ")"
+
+      sexp
+    end
+
+    # @return [AST::Node] self
+    def to_ast
+      self
+    end
+
+    protected
+
+    # Returns `@type` with all underscores replaced by dashes. This allows
+    # to write symbol literals without quotes in Ruby sources and yet have
+    # nicely looking s-expressions.
+    #
+    # @return [String]
+    def fancy_type
+      @type.to_s.gsub('_', '-')
+    end
+  end
+end