myco 0.1.0.dev

data/lib/myco/parser/lexer.rl

@@ -0,0 +1,393 @@
+
+%%machine lexer; # %
+
+%%{
+# %
+  constant   = c_upper c_alnum* ;
+  identifier = c_lower c_alnum* ;
+  
+  comment    = '#' (any - c_eol)*; # end-of-line comment
+  
+  integer    = [0-9]+ ;
+  float      = [0-9]+ '.' [0-9]+ ;
+  
+  strbody_norm = ^('\\' | '"');
+  strbody      = strbody_norm* ('\\' c_any strbody_norm*)*;
+  
+  # "foo bar"
+  #
+  string     = (
+    zlen     % { note_begin :string }
+    '"'      % { note :string, :T_STRING_BEGIN; note :string }
+    strbody  % { note :string, :T_STRING_BODY;  note :string }
+    '"'      % { note :string, :T_STRING_END }
+  );
+  
+  # id: foo
+  #
+  declid = (
+    zlen       % { note_begin :declid }
+    'id'       % { note :declid, :T_DECLID_TAG }
+    c_space*
+    ':'
+    c_space*   % { note :declid }
+    identifier % { note :declid, :T_DECLID_VALUE }
+  );
+  
+  # [foo]
+  #
+  category = (
+    zlen       % { note_begin :category }
+    '['        % { note :category, :T_CATEGORY_BEGIN }
+    c_space*   % { note :category }
+    identifier % { note :category, :T_CATEGORY_BODY }
+    c_space*   % { note :category }
+    ']'        % { note :category, :T_CATEGORY_END }
+  );
+  
+  # :foo
+  # :"bar baz"
+  #
+  symbol = (
+    ':'            % { note_begin :symbol }
+    (
+      (
+        identifier % { note :symbol, :T_SYMBOL; }
+      )
+    | (
+        '"'        % { note :symbol, :T_SYMSTR_BEGIN; note :symbol }
+        strbody    % { note :symbol, :T_SYMSTR_BODY;  note :symbol }
+        '"'        % { note :symbol, :T_SYMSTR_END; }
+      )
+    )
+  );
+  
+  # Foo
+  # ::Bar
+  # Foo::Bar
+  # ::Foo::Bar::Baz
+  #
+  sconstant = (
+    zlen         % { note_begin :sconstant }
+    (
+      '::'       % { note :sconstant, :T_SCOPE;    note :sconstant }
+    )? (
+      constant   % { note :sconstant, :T_CONSTANT; note :sconstant }
+      '::'       % { note :sconstant, :T_SCOPE;    note :sconstant }
+    )*
+    constant     % { note :sconstant, :T_CONSTANT }
+  );
+  
+  # Foo,Bar,Baz
+  #
+  constant_list = (
+    zlen           % { note_begin :constant_list, nil }
+    sconstant      % { xfer_notes :sconstant, :constant_list }
+    (
+      c_space*     % { note :constant_list }
+      ','          % { note :constant_list, :T_CONST_SEP }
+      c_space_nl*
+      sconstant    % { xfer_notes :sconstant, :constant_list }
+    )*
+  );
+  
+  # Foo <
+  #
+  cdefn_begin = (
+    zlen      % { note_begin :cdefn_begin }
+    constant  % { note :cdefn_begin, :T_CONSTANT }
+    c_space*  % { note :cdefn_begin }
+    '<'       % { note :cdefn_begin, :T_DEFINE }
+  ) % {
+    emit_notes :cdefn_begin
+  };
+  
+  # Object {
+  #
+  decl_begin = (
+    (cdefn_begin c_space_nl*)?
+    constant_list
+    c_space_nl*  % { note_begin :decl_begin }
+    '{'          % { note :decl_begin, :T_DECLARE_BEGIN }
+  ) % {
+    emit_notes :constant_list
+    emit_notes :decl_begin
+  };
+  
+  # Starting delimiter for a string declaration
+  #
+  # Can be any string of characters following a
+  # constant name + whitespace that is not ambiguous
+  # with some other construction
+  #
+  # The ending delimiter will be calculated from as follows:
+  # The string of characters is reversed.
+  # If there are groups of "alphabetical" characters,
+  # the intra-group order remains intact.
+  # If there are non-alphabetical characters with "directionality",
+  # the "opposite" characters are substituted.
+  #
+  dstr_delim = (
+    ^(c_space_nl|'{'|':'|',')
+    ^(c_space_nl)+
+  );
+  
+  # Object @@@
+  #   ...
+  # @@@
+  #
+  dstr_begin = (
+    constant_list
+    c_space+    % { mark :space }
+    dstr_delim  % { grab :delim, kram(:space) }
+  ) % {
+    emit_notes :constant_list
+    
+    start, stop = @stored[:delim]
+    emit :T_DECLSTR_BEGIN, start, stop
+    
+    # Table of replacement characters to use when calculating
+    # the ending delimiter from the starting delimiter.
+    # Directional characters are replaced with their opposite.
+    @dstr_replace_table ||= %w{
+      < > ( ) { } [ ]
+    }
+    
+    # Calculate the ending delimiter to look for and store it
+    @dstr_destrlim = text(start, stop) \
+      .split(/(?<=[^a-zA-Z])|(?=[^a-zA-Z])/)
+      .map { |str|
+        idx = @dstr_replace_table.find_index(str)
+        idx.nil? ? str : 
+          (idx.odd? ? @dstr_replace_table[idx-1] : @dstr_replace_table[idx+1])
+      }
+      .reverse
+      .join ''
+  };
+  
+  # identifier (
+  #
+  args_begin = (
+    zlen               % { note_begin :args_begin }
+    identifier         % { note :args_begin, :T_IDENTIFIER }
+    c_space_nl*        % { note :args_begin }
+    '('                % { note :args_begin, :T_ARGS_BEGIN }
+  );
+  
+  ##
+  # Top level machine
+  
+  main := |*
+    c_space;
+    comment;
+    
+    decl_begin  => { fcall decl_body; };
+    dstr_begin  => { fcall dstr_body; };
+    
+    string     => { emit_notes :string };
+    declid     => { emit_notes :declid };
+    category   => { emit_notes :category };
+    identifier => { emit :T_IDENTIFIER };
+    constant   => { emit :T_CONSTANT };
+    '::'       => { emit :T_SCOPE };
+    
+    ':' => { fcall pre_meme; };
+    
+    ';'  => { emit :T_EXPR_SEP };
+    c_nl => { emit :T_EXPR_SEP };
+    
+    c_eof => { emit :T_DECLARE_END };
+    any => { error :main };
+  *|;
+  
+  ##
+  # Declarative body machine
+  
+  decl_body := |*
+    c_space;
+    comment;
+    
+    (c_eol|';') => { emit :T_EXPR_SEP };
+    
+    decl_begin  => { fcall decl_body; };
+    dstr_begin  => { fcall dstr_body; };
+    
+    string     => { emit_notes :string };
+    declid     => { emit_notes :declid };
+    category   => { emit_notes :category };
+    identifier => { emit :T_IDENTIFIER };
+    constant   => { emit :T_CONSTANT };
+    '::'       => { emit :T_SCOPE };
+    
+    ':' => { fcall pre_meme; };
+    
+    '}' => { emit :T_DECLARE_END; fret; };
+    
+    any => { error :decl_body };
+  *|;
+  
+  ##
+  # Pre-meme body sub-machines
+  
+  pre_meme := |*
+    c_space_nl+;
+    comment;
+    
+    # Parameters are specified within '|'s
+    '|'   => { emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body; };
+    
+    # A meme begins with either a '{' or some other item for inline
+    ^(c_space_nl|'{'|'|') =>
+      { fhold; emit :T_MEME_BEGIN, @ts, @ts; bpush :meml; fgoto meme_body; };
+    '{'   => { emit :T_MEME_BEGIN;           bpush :meme; fgoto meme_body; };
+    
+    any => { error :pre_meme };
+  *|;
+  
+  ##
+  # Declarative string machine
+  
+  dstr_body := |*
+    (
+      c_nl     % { mark :newline }
+      (^c_nl)* % { grab :line, kram(:newline) }
+    ) => {
+      start, stop = @stored[:line];
+      line_text = text start, stop
+      
+      raise "No known delimiter for string declaration." \
+        if @dstr_delim.nil?
+      
+      if (line_text =~ /^(\s*)(\S+)/; $2==@dstr_delim)
+        emit :T_DECLSTR_BODY, *@dstr_body_start, start
+        @dstr_body_start = nil
+        @dstr_delim = nil
+        
+        emit :T_DECLSTR_END, start+$1.size, stop
+        fret;
+      else
+        @dstr_body_start ||= start
+      end
+    };
+  *|;
+  
+  ##
+  # Meme body machine
+  
+  meme_body := |*
+    c_space+;
+    comment;
+    
+    decl_begin => { fcall decl_body; };
+    dstr_begin => { fcall dstr_body; };
+    
+    args_begin => { emit_notes :args_begin; bpush :args;  fcall meme_body; };
+    '('        => { emit :T_PAREN_BEGIN;    bpush :paren; fcall meme_body; };
+    '['        => { emit :T_ARRAY_BEGIN;    bpush :array; fcall meme_body; };
+    '{'        => { emit :T_MEME_BEGIN;     bpush :meme;  fcall meme_body; };
+    
+    'self'     => { emit :T_SELF };
+    'null'     => { emit :T_NULL };
+    'void'     => { emit :T_VOID };
+    'true'     => { emit :T_TRUE };
+    'false'    => { emit :T_FALSE };
+    integer    => { emit :T_INTEGER };
+    float      => { emit :T_FLOAT };
+    constant   => { emit :T_CONSTANT };
+    identifier => { emit :T_IDENTIFIER };
+    '.'        => { emit :T_DOT };
+    '?'        => { emit :T_QUEST };
+    '::'       => { emit :T_SCOPE };
+    '='        => { emit :T_ASSIGN };
+    '+'        => { emit :T_OP_PLUS };
+    '-'        => { emit :T_OP_MINUS };
+    '*'        => { emit :T_OP_MULT };
+    '/'        => { emit :T_OP_DIV };
+    '%'        => { emit :T_OP_MOD };
+    '**'       => { emit :T_OP_EXP };
+    ('<'|'>'|'<='|'>='|'=='|'==='|'<=>'|'=~')
+               => { emit :T_OP_COMPARE };
+    '&&'       => { emit :T_OP_AND };
+    '||'       => { emit :T_OP_OR };
+    
+    symbol     => { emit_notes :symbol };
+    string     => { emit_notes :string };
+    
+    '\\\n';    # Escaped newline - ignore
+    
+    
+    '&' => {
+      case bthis
+      when :param; emit :T_OP_TOPROC
+      when :args;  emit :T_OP_TOPROC
+      else;        error :meme_body
+      end
+    };
+    
+    ',' => {
+      case bthis
+      when :args;  emit :T_ARG_SEP
+      when :param; emit :T_ARG_SEP
+      when :array; emit :T_ARG_SEP
+      else;        error :meme_body
+      end
+    };
+    
+    ';' => {
+      case bthis
+      when :meme;  emit :T_EXPR_SEP
+      when :meml;  emit :T_EXPR_SEP
+      when :paren; emit :T_EXPR_SEP
+      else;        error :meme_body
+      end
+    };
+    
+    c_eol => {
+      case bthis
+      when :meme;  emit :T_EXPR_SEP
+      when :meml;  emit :T_MEME_END, @ts, @ts; fhold; bpop; fret;
+      when :paren; emit :T_EXPR_SEP
+      when :args;  emit :T_ARG_SEP
+      when :param; emit :T_ARG_SEP
+      when :array; emit :T_ARG_SEP
+      else;        error :meme_body
+      end
+    };
+    
+    '}' => {
+      case bthis
+      when :meme;  emit :T_MEME_END; bpop; fret;
+      else;        error :meme_body
+      end
+    };
+    
+    ')' => {
+      case bthis
+      when :args;  emit :T_ARGS_END;  bpop; fret;
+      when :paren; emit :T_PAREN_END; bpop; fret;
+      else;        error :meme_body
+      end
+    };
+    
+    ']' => {
+      case bthis
+      when :array; emit :T_ARRAY_END; bpop; fret;
+      else;        error :meme_body
+      end
+    };
+    
+    '|' => {
+      case bthis
+      when :param; emit :T_PARAMS_END; bpop; fret;
+      when :meme;  emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
+      when :meml;  emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
+      when :paren; emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
+      else;        error :meme_body
+      end
+    };
+    
+    any => { error :meme_body };
+  *|;
+  
+}%%
+# %

data/lib/myco/parser/lexer_char_classes.rl

@@ -0,0 +1,56 @@
+
+%%machine lexer; # %
+
+%%{
+# %
+  ## 
+  # Basic character types - taken from:
+  #   https://github.com/whitequark/parser/blob/master/lib/parser/lexer.rl
+  #
+  # License for whitequark/parser reproduced below.
+
+  # Copyright (c) 2013 Peter Zotov  <whitequark@whitequark.org>
+  #
+  # Parts of the source are derived from ruby_parser:
+  # Copyright (c) Ryan Davis, seattle.rb
+  #
+  # MIT License
+  #
+  # Permission is hereby granted, free of charge, to any person obtaining
+  # a copy of this software and associated documentation files (the
+  # "Software"), to deal in the Software without restriction, including
+  # without limitation the rights to use, copy, modify, merge, publish,
+  # distribute, sublicense, and/or sell copies of the Software, and to
+  # permit persons to whom the Software is furnished to do so, subject to
+  # the following conditions:
+  #
+  # The above copyright notice and this permission notice shall be
+  # included in all copies or substantial portions of the Software.
+  #
+  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  
+  c_nl       = '\n' $ do_nl;
+  c_space    = [ \t\r\f\v];
+  c_space_nl = c_space | c_nl;
+  
+  c_eof      = 0x04 | 0x1a | 0 | zlen; # ^D, ^Z, \0, EOF
+  c_eol      = c_nl | c_eof;
+  c_any      = any - c_eof;
+  
+  c_nl_zlen  = c_nl | zlen;
+  c_line     = any - c_nl_zlen;
+  
+  c_unicode  = c_any - 0x00..0x7f;
+  c_upper    = [A-Z];
+  c_lower    = [a-z_]  | c_unicode;
+  c_alpha    = c_lower | c_upper;
+  c_alnum    = c_alpha | [0-9];
+  
+}%%
+# %

data/lib/myco/parser/lexer_common.rb

@@ -0,0 +1,95 @@
+
+class CodeTools::Parser
+  class Lexer
+    
+    def reset_common
+      warn "Lexer still has items on @bstack: #{@bstack.inspect}" \
+        if @bstack and !@bstack.empty?
+      
+      @newlines = [0]
+      @marks  = {}
+      @stored = {}
+      @bstack = []
+    end
+    
+    def mark name, pos=@p
+      @marks[name] = pos
+    end
+    
+    def kram name
+      @marks.delete name
+    end
+    
+    def grab name, start=@ts, stop=@p
+      @stored[name] = [start, stop]
+    end
+    
+    def note_begin queue_name, pos=@p
+      queue = @marks[queue_name] = (pos ? [pos] : [])
+    end
+    
+    def note queue_name, type=nil, pos=@p
+      queue = (@marks[queue_name] ||= [])
+      queue << pos
+      queue << type if type
+      queue
+    end
+    
+    def unnote queue_name, count=1
+      queue = (@marks[queue_name] ||= [])
+      queue.pop count
+      queue
+    end
+    
+    def emit_notes queue_name
+      queue = (@marks[queue_name] || [])
+      queue.each_slice(3) { |a,b,c| emit c,a,b if a && b && c }
+      queue.clear
+    end
+    
+    def xfer_notes queue_name_a, queue_name_b
+      queue_a = (@marks[queue_name_a] || [])
+      queue_b = (@marks[queue_name_b] ||= [])
+      queue_b << queue_a.shift until queue_a.empty?
+      queue_b
+    end
+    
+    def bpush name
+      @bstack << name
+    end
+    
+    def bthis
+      @bstack.last
+    end
+    
+    def bpop
+      @bstack.pop
+    end
+    
+    def error(location, hint=nil)
+      str = "Lexer met unexpected character(s) in #{location.inspect}: #{text.inspect}"
+      str += "; "+hint.to_s if hint
+      str += "\n@marks  = #{@marks }"
+      str += "\n@stored = #{@stored}"
+      str += "\n@bstack = #{@bstack}"
+      str += "\n@stack  = #{@stack}"
+      str += "\n"
+      warn str
+    end
+    
+    
+    def do_nl
+      @newlines << @p unless @newlines.include? @p
+    end
+    
+    def emit(type, start = @ts, stop = @te)
+      @tokens << [type, text(start,stop), locate(start)]
+    end
+    
+    def locate index
+      ary = @newlines.take_while { |i| i <= index }
+      row, col = ary.size, index-ary.last+1
+    end
+    
+  end
+end

data/lib/myco/parser/lexer_skeleton.rl

@@ -0,0 +1,154 @@
+
+## 
+# Basic Lexer skeleton - taken from gist:
+#   https://gist.github.com/YorickPeterse/10658884
+# Originally from:
+#   https://github.com/YorickPeterse/oga/blob/master/lib/oga/xml/lexer.rl
+#
+# License for the source gist reproduced below.
+
+# Copyright (c) 2014, Yorick Peterse
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+%%machine lexer; # %
+
+class CodeTools::Parser
+  class Lexer
+    %% write data; # %
+    
+    ##
+    # @param [String] data The data to lex.
+    #
+    def initialize(data)
+      @data = data.unpack('U*') << 0
+      
+      reset
+    end
+    
+    ##
+    # Resets the internal state of the lexer. Typically you don't need to
+    # call this method yourself as its called by #lex after lexing a given
+    # String.
+    #
+    def reset
+      @line     = 1
+      @ts       = nil
+      @te       = nil
+      @tokens   = []
+      @stack    = []
+      @top      = 0
+      @cs       = self.class.lexer_start
+      @act      = 0
+      @elements = []
+      @eof      = @data.length
+      @p        = 0
+      @pe       = @eof
+      reset_common
+    end
+    
+    ##
+    # Lexes the supplied String and returns an Array of tokens. Each token is
+    # an Array in the following format:
+    #
+    #     [TYPE, VALUE]
+    #
+    # The type is a symbol, the value is either nil or a String.
+    #
+    # This method resets the internal state of the lexer after consuming the
+    # input.
+    #
+    # @param [String] data The string to consume.
+    # @return [Array]
+    # @see #advance
+    #
+    def lex
+      tokens = []
+      
+      while token = advance
+        tokens << token
+      end
+      
+      reset
+      
+      return tokens
+    end
+    
+    ##
+    # Advances through the input and generates the corresponding tokens.
+    #
+    # This method does *not* reset the internal state of the lexer.
+    #
+    # @param [String] data The String to consume.
+    # @return [Array]
+    #
+    def advance
+      _lexer_actions             = self.class.send :_lexer_actions
+      _lexer_range_lengths       = self.class.send :_lexer_range_lengths
+      _lexer_trans_actions       = self.class.send :_lexer_trans_actions
+      _lexer_key_offsets         = self.class.send :_lexer_key_offsets
+      _lexer_index_offsets       = self.class.send :_lexer_index_offsets
+      _lexer_to_state_actions    = self.class.send :_lexer_to_state_actions
+      _lexer_trans_keys          = self.class.send :_lexer_trans_keys
+      _lexer_from_state_actions  = self.class.send :_lexer_from_state_actions
+      _lexer_single_lengths      = self.class.send :_lexer_single_lengths
+      _lexer_trans_targs         = self.class.send :_lexer_trans_targs
+      _lexer_eof_trans           = self.class.send :_lexer_eof_trans
+      _lexer_indicies            = self.class.send :_lexer_indicies
+      
+      %% write exec;
+      # %
+      
+      return @tokens.shift
+    end
+    
+    private
+    
+    ##
+    # Returns the text of the current buffer based on the supplied start and
+    # stop position.
+    #
+    # By default `@ts` and `@te` are used as the start/stop position.
+    #
+    # @param [Fixnum] start
+    # @param [Fixnum] stop
+    # @return [String]
+    #
+    def text(start = @ts, stop = @te)
+      return @data[start...stop].pack('U*')
+    end
+    
+    
+    %%{
+    # %
+      # Use instance variables for `ts` and friends.
+      access @;
+      getkey (@data[@p] || 0);
+      variable p @p;
+      variable pe @pe;
+      variable eof @eof;
+      
+      action do_nl { do_nl }
+      
+      include "lexer_char_classes.rl"; # Basic character classes
+      include "lexer.rl";              # Main rules file
+    }%%
+    # %
+  end
+end