myco 0.1.0.dev → 0.1.0

data/lib/myco/parser/lexer.rl

@@ -1,393 +0,0 @@
-
-%%machine lexer; # %
-
-%%{
-# %
-  constant   = c_upper c_alnum* ;
-  identifier = c_lower c_alnum* ;
-  
-  comment    = '#' (any - c_eol)*; # end-of-line comment
-  
-  integer    = [0-9]+ ;
-  float      = [0-9]+ '.' [0-9]+ ;
-  
-  strbody_norm = ^('\\' | '"');
-  strbody      = strbody_norm* ('\\' c_any strbody_norm*)*;
-  
-  # "foo bar"
-  #
-  string     = (
-    zlen     % { note_begin :string }
-    '"'      % { note :string, :T_STRING_BEGIN; note :string }
-    strbody  % { note :string, :T_STRING_BODY;  note :string }
-    '"'      % { note :string, :T_STRING_END }
-  );
-  
-  # id: foo
-  #
-  declid = (
-    zlen       % { note_begin :declid }
-    'id'       % { note :declid, :T_DECLID_TAG }
-    c_space*
-    ':'
-    c_space*   % { note :declid }
-    identifier % { note :declid, :T_DECLID_VALUE }
-  );
-  
-  # [foo]
-  #
-  category = (
-    zlen       % { note_begin :category }
-    '['        % { note :category, :T_CATEGORY_BEGIN }
-    c_space*   % { note :category }
-    identifier % { note :category, :T_CATEGORY_BODY }
-    c_space*   % { note :category }
-    ']'        % { note :category, :T_CATEGORY_END }
-  );
-  
-  # :foo
-  # :"bar baz"
-  #
-  symbol = (
-    ':'            % { note_begin :symbol }
-    (
-      (
-        identifier % { note :symbol, :T_SYMBOL; }
-      )
-    | (
-        '"'        % { note :symbol, :T_SYMSTR_BEGIN; note :symbol }
-        strbody    % { note :symbol, :T_SYMSTR_BODY;  note :symbol }
-        '"'        % { note :symbol, :T_SYMSTR_END; }
-      )
-    )
-  );
-  
-  # Foo
-  # ::Bar
-  # Foo::Bar
-  # ::Foo::Bar::Baz
-  #
-  sconstant = (
-    zlen         % { note_begin :sconstant }
-    (
-      '::'       % { note :sconstant, :T_SCOPE;    note :sconstant }
-    )? (
-      constant   % { note :sconstant, :T_CONSTANT; note :sconstant }
-      '::'       % { note :sconstant, :T_SCOPE;    note :sconstant }
-    )*
-    constant     % { note :sconstant, :T_CONSTANT }
-  );
-  
-  # Foo,Bar,Baz
-  #
-  constant_list = (
-    zlen           % { note_begin :constant_list, nil }
-    sconstant      % { xfer_notes :sconstant, :constant_list }
-    (
-      c_space*     % { note :constant_list }
-      ','          % { note :constant_list, :T_CONST_SEP }
-      c_space_nl*
-      sconstant    % { xfer_notes :sconstant, :constant_list }
-    )*
-  );
-  
-  # Foo <
-  #
-  cdefn_begin = (
-    zlen      % { note_begin :cdefn_begin }
-    constant  % { note :cdefn_begin, :T_CONSTANT }
-    c_space*  % { note :cdefn_begin }
-    '<'       % { note :cdefn_begin, :T_DEFINE }
-  ) % {
-    emit_notes :cdefn_begin
-  };
-  
-  # Object {
-  #
-  decl_begin = (
-    (cdefn_begin c_space_nl*)?
-    constant_list
-    c_space_nl*  % { note_begin :decl_begin }
-    '{'          % { note :decl_begin, :T_DECLARE_BEGIN }
-  ) % {
-    emit_notes :constant_list
-    emit_notes :decl_begin
-  };
-  
-  # Starting delimiter for a string declaration
-  #
-  # Can be any string of characters following a
-  # constant name + whitespace that is not ambiguous
-  # with some other construction
-  #
-  # The ending delimiter will be calculated from as follows:
-  # The string of characters is reversed.
-  # If there are groups of "alphabetical" characters,
-  # the intra-group order remains intact.
-  # If there are non-alphabetical characters with "directionality",
-  # the "opposite" characters are substituted.
-  #
-  dstr_delim = (
-    ^(c_space_nl|'{'|':'|',')
-    ^(c_space_nl)+
-  );
-  
-  # Object @@@
-  #   ...
-  # @@@
-  #
-  dstr_begin = (
-    constant_list
-    c_space+    % { mark :space }
-    dstr_delim  % { grab :delim, kram(:space) }
-  ) % {
-    emit_notes :constant_list
-    
-    start, stop = @stored[:delim]
-    emit :T_DECLSTR_BEGIN, start, stop
-    
-    # Table of replacement characters to use when calculating
-    # the ending delimiter from the starting delimiter.
-    # Directional characters are replaced with their opposite.
-    @dstr_replace_table ||= %w{
-      < > ( ) { } [ ]
-    }
-    
-    # Calculate the ending delimiter to look for and store it
-    @dstr_destrlim = text(start, stop) \
-      .split(/(?<=[^a-zA-Z])|(?=[^a-zA-Z])/)
-      .map { |str|
-        idx = @dstr_replace_table.find_index(str)
-        idx.nil? ? str : 
-          (idx.odd? ? @dstr_replace_table[idx-1] : @dstr_replace_table[idx+1])
-      }
-      .reverse
-      .join ''
-  };
-  
-  # identifier (
-  #
-  args_begin = (
-    zlen               % { note_begin :args_begin }
-    identifier         % { note :args_begin, :T_IDENTIFIER }
-    c_space_nl*        % { note :args_begin }
-    '('                % { note :args_begin, :T_ARGS_BEGIN }
-  );
-  
-  ##
-  # Top level machine
-  
-  main := |*
-    c_space;
-    comment;
-    
-    decl_begin  => { fcall decl_body; };
-    dstr_begin  => { fcall dstr_body; };
-    
-    string     => { emit_notes :string };
-    declid     => { emit_notes :declid };
-    category   => { emit_notes :category };
-    identifier => { emit :T_IDENTIFIER };
-    constant   => { emit :T_CONSTANT };
-    '::'       => { emit :T_SCOPE };
-    
-    ':' => { fcall pre_meme; };
-    
-    ';'  => { emit :T_EXPR_SEP };
-    c_nl => { emit :T_EXPR_SEP };
-    
-    c_eof => { emit :T_DECLARE_END };
-    any => { error :main };
-  *|;
-  
-  ##
-  # Declarative body machine
-  
-  decl_body := |*
-    c_space;
-    comment;
-    
-    (c_eol|';') => { emit :T_EXPR_SEP };
-    
-    decl_begin  => { fcall decl_body; };
-    dstr_begin  => { fcall dstr_body; };
-    
-    string     => { emit_notes :string };
-    declid     => { emit_notes :declid };
-    category   => { emit_notes :category };
-    identifier => { emit :T_IDENTIFIER };
-    constant   => { emit :T_CONSTANT };
-    '::'       => { emit :T_SCOPE };
-    
-    ':' => { fcall pre_meme; };
-    
-    '}' => { emit :T_DECLARE_END; fret; };
-    
-    any => { error :decl_body };
-  *|;
-  
-  ##
-  # Pre-meme body sub-machines
-  
-  pre_meme := |*
-    c_space_nl+;
-    comment;
-    
-    # Parameters are specified within '|'s
-    '|'   => { emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body; };
-    
-    # A meme begins with either a '{' or some other item for inline
-    ^(c_space_nl|'{'|'|') =>
-      { fhold; emit :T_MEME_BEGIN, @ts, @ts; bpush :meml; fgoto meme_body; };
-    '{'   => { emit :T_MEME_BEGIN;           bpush :meme; fgoto meme_body; };
-    
-    any => { error :pre_meme };
-  *|;
-  
-  ##
-  # Declarative string machine
-  
-  dstr_body := |*
-    (
-      c_nl     % { mark :newline }
-      (^c_nl)* % { grab :line, kram(:newline) }
-    ) => {
-      start, stop = @stored[:line];
-      line_text = text start, stop
-      
-      raise "No known delimiter for string declaration." \
-        if @dstr_delim.nil?
-      
-      if (line_text =~ /^(\s*)(\S+)/; $2==@dstr_delim)
-        emit :T_DECLSTR_BODY, *@dstr_body_start, start
-        @dstr_body_start = nil
-        @dstr_delim = nil
-        
-        emit :T_DECLSTR_END, start+$1.size, stop
-        fret;
-      else
-        @dstr_body_start ||= start
-      end
-    };
-  *|;
-  
-  ##
-  # Meme body machine
-  
-  meme_body := |*
-    c_space+;
-    comment;
-    
-    decl_begin => { fcall decl_body; };
-    dstr_begin => { fcall dstr_body; };
-    
-    args_begin => { emit_notes :args_begin; bpush :args;  fcall meme_body; };
-    '('        => { emit :T_PAREN_BEGIN;    bpush :paren; fcall meme_body; };
-    '['        => { emit :T_ARRAY_BEGIN;    bpush :array; fcall meme_body; };
-    '{'        => { emit :T_MEME_BEGIN;     bpush :meme;  fcall meme_body; };
-    
-    'self'     => { emit :T_SELF };
-    'null'     => { emit :T_NULL };
-    'void'     => { emit :T_VOID };
-    'true'     => { emit :T_TRUE };
-    'false'    => { emit :T_FALSE };
-    integer    => { emit :T_INTEGER };
-    float      => { emit :T_FLOAT };
-    constant   => { emit :T_CONSTANT };
-    identifier => { emit :T_IDENTIFIER };
-    '.'        => { emit :T_DOT };
-    '?'        => { emit :T_QUEST };
-    '::'       => { emit :T_SCOPE };
-    '='        => { emit :T_ASSIGN };
-    '+'        => { emit :T_OP_PLUS };
-    '-'        => { emit :T_OP_MINUS };
-    '*'        => { emit :T_OP_MULT };
-    '/'        => { emit :T_OP_DIV };
-    '%'        => { emit :T_OP_MOD };
-    '**'       => { emit :T_OP_EXP };
-    ('<'|'>'|'<='|'>='|'=='|'==='|'<=>'|'=~')
-               => { emit :T_OP_COMPARE };
-    '&&'       => { emit :T_OP_AND };
-    '||'       => { emit :T_OP_OR };
-    
-    symbol     => { emit_notes :symbol };
-    string     => { emit_notes :string };
-    
-    '\\\n';    # Escaped newline - ignore
-    
-    
-    '&' => {
-      case bthis
-      when :param; emit :T_OP_TOPROC
-      when :args;  emit :T_OP_TOPROC
-      else;        error :meme_body
-      end
-    };
-    
-    ',' => {
-      case bthis
-      when :args;  emit :T_ARG_SEP
-      when :param; emit :T_ARG_SEP
-      when :array; emit :T_ARG_SEP
-      else;        error :meme_body
-      end
-    };
-    
-    ';' => {
-      case bthis
-      when :meme;  emit :T_EXPR_SEP
-      when :meml;  emit :T_EXPR_SEP
-      when :paren; emit :T_EXPR_SEP
-      else;        error :meme_body
-      end
-    };
-    
-    c_eol => {
-      case bthis
-      when :meme;  emit :T_EXPR_SEP
-      when :meml;  emit :T_MEME_END, @ts, @ts; fhold; bpop; fret;
-      when :paren; emit :T_EXPR_SEP
-      when :args;  emit :T_ARG_SEP
-      when :param; emit :T_ARG_SEP
-      when :array; emit :T_ARG_SEP
-      else;        error :meme_body
-      end
-    };
-    
-    '}' => {
-      case bthis
-      when :meme;  emit :T_MEME_END; bpop; fret;
-      else;        error :meme_body
-      end
-    };
-    
-    ')' => {
-      case bthis
-      when :args;  emit :T_ARGS_END;  bpop; fret;
-      when :paren; emit :T_PAREN_END; bpop; fret;
-      else;        error :meme_body
-      end
-    };
-    
-    ']' => {
-      case bthis
-      when :array; emit :T_ARRAY_END; bpop; fret;
-      else;        error :meme_body
-      end
-    };
-    
-    '|' => {
-      case bthis
-      when :param; emit :T_PARAMS_END; bpop; fret;
-      when :meme;  emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
-      when :meml;  emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
-      when :paren; emit :T_PARAMS_BEGIN; bpush :param; fcall meme_body;
-      else;        error :meme_body
-      end
-    };
-    
-    any => { error :meme_body };
-  *|;
-  
-}%%
-# %

data/lib/myco/parser/lexer_char_classes.rl

@@ -1,56 +0,0 @@
-
-%%machine lexer; # %
-
-%%{
-# %
-  ## 
-  # Basic character types - taken from:
-  #   https://github.com/whitequark/parser/blob/master/lib/parser/lexer.rl
-  #
-  # License for whitequark/parser reproduced below.
-
-  # Copyright (c) 2013 Peter Zotov  <whitequark@whitequark.org>
-  #
-  # Parts of the source are derived from ruby_parser:
-  # Copyright (c) Ryan Davis, seattle.rb
-  #
-  # MIT License
-  #
-  # Permission is hereby granted, free of charge, to any person obtaining
-  # a copy of this software and associated documentation files (the
-  # "Software"), to deal in the Software without restriction, including
-  # without limitation the rights to use, copy, modify, merge, publish,
-  # distribute, sublicense, and/or sell copies of the Software, and to
-  # permit persons to whom the Software is furnished to do so, subject to
-  # the following conditions:
-  #
-  # The above copyright notice and this permission notice shall be
-  # included in all copies or substantial portions of the Software.
-  #
-  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-  # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-  # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-  
-  c_nl       = '\n' $ do_nl;
-  c_space    = [ \t\r\f\v];
-  c_space_nl = c_space | c_nl;
-  
-  c_eof      = 0x04 | 0x1a | 0 | zlen; # ^D, ^Z, \0, EOF
-  c_eol      = c_nl | c_eof;
-  c_any      = any - c_eof;
-  
-  c_nl_zlen  = c_nl | zlen;
-  c_line     = any - c_nl_zlen;
-  
-  c_unicode  = c_any - 0x00..0x7f;
-  c_upper    = [A-Z];
-  c_lower    = [a-z_]  | c_unicode;
-  c_alpha    = c_lower | c_upper;
-  c_alnum    = c_alpha | [0-9];
-  
-}%%
-# %

data/lib/myco/parser/lexer_common.rb

@@ -1,95 +0,0 @@
-
-class CodeTools::Parser
-  class Lexer
-    
-    def reset_common
-      warn "Lexer still has items on @bstack: #{@bstack.inspect}" \
-        if @bstack and !@bstack.empty?
-      
-      @newlines = [0]
-      @marks  = {}
-      @stored = {}
-      @bstack = []
-    end
-    
-    def mark name, pos=@p
-      @marks[name] = pos
-    end
-    
-    def kram name
-      @marks.delete name
-    end
-    
-    def grab name, start=@ts, stop=@p
-      @stored[name] = [start, stop]
-    end
-    
-    def note_begin queue_name, pos=@p
-      queue = @marks[queue_name] = (pos ? [pos] : [])
-    end
-    
-    def note queue_name, type=nil, pos=@p
-      queue = (@marks[queue_name] ||= [])
-      queue << pos
-      queue << type if type
-      queue
-    end
-    
-    def unnote queue_name, count=1
-      queue = (@marks[queue_name] ||= [])
-      queue.pop count
-      queue
-    end
-    
-    def emit_notes queue_name
-      queue = (@marks[queue_name] || [])
-      queue.each_slice(3) { |a,b,c| emit c,a,b if a && b && c }
-      queue.clear
-    end
-    
-    def xfer_notes queue_name_a, queue_name_b
-      queue_a = (@marks[queue_name_a] || [])
-      queue_b = (@marks[queue_name_b] ||= [])
-      queue_b << queue_a.shift until queue_a.empty?
-      queue_b
-    end
-    
-    def bpush name
-      @bstack << name
-    end
-    
-    def bthis
-      @bstack.last
-    end
-    
-    def bpop
-      @bstack.pop
-    end
-    
-    def error(location, hint=nil)
-      str = "Lexer met unexpected character(s) in #{location.inspect}: #{text.inspect}"
-      str += "; "+hint.to_s if hint
-      str += "\n@marks  = #{@marks }"
-      str += "\n@stored = #{@stored}"
-      str += "\n@bstack = #{@bstack}"
-      str += "\n@stack  = #{@stack}"
-      str += "\n"
-      warn str
-    end
-    
-    
-    def do_nl
-      @newlines << @p unless @newlines.include? @p
-    end
-    
-    def emit(type, start = @ts, stop = @te)
-      @tokens << [type, text(start,stop), locate(start)]
-    end
-    
-    def locate index
-      ary = @newlines.take_while { |i| i <= index }
-      row, col = ary.size, index-ary.last+1
-    end
-    
-  end
-end

data/lib/myco/parser/lexer_skeleton.rl

@@ -1,154 +0,0 @@
-
-## 
-# Basic Lexer skeleton - taken from gist:
-#   https://gist.github.com/YorickPeterse/10658884
-# Originally from:
-#   https://github.com/YorickPeterse/oga/blob/master/lib/oga/xml/lexer.rl
-#
-# License for the source gist reproduced below.
-
-# Copyright (c) 2014, Yorick Peterse
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-%%machine lexer; # %
-
-class CodeTools::Parser
-  class Lexer
-    %% write data; # %
-    
-    ##
-    # @param [String] data The data to lex.
-    #
-    def initialize(data)
-      @data = data.unpack('U*') << 0
-      
-      reset
-    end
-    
-    ##
-    # Resets the internal state of the lexer. Typically you don't need to
-    # call this method yourself as its called by #lex after lexing a given
-    # String.
-    #
-    def reset
-      @line     = 1
-      @ts       = nil
-      @te       = nil
-      @tokens   = []
-      @stack    = []
-      @top      = 0
-      @cs       = self.class.lexer_start
-      @act      = 0
-      @elements = []
-      @eof      = @data.length
-      @p        = 0
-      @pe       = @eof
-      reset_common
-    end
-    
-    ##
-    # Lexes the supplied String and returns an Array of tokens. Each token is
-    # an Array in the following format:
-    #
-    #     [TYPE, VALUE]
-    #
-    # The type is a symbol, the value is either nil or a String.
-    #
-    # This method resets the internal state of the lexer after consuming the
-    # input.
-    #
-    # @param [String] data The string to consume.
-    # @return [Array]
-    # @see #advance
-    #
-    def lex
-      tokens = []
-      
-      while token = advance
-        tokens << token
-      end
-      
-      reset
-      
-      return tokens
-    end
-    
-    ##
-    # Advances through the input and generates the corresponding tokens.
-    #
-    # This method does *not* reset the internal state of the lexer.
-    #
-    # @param [String] data The String to consume.
-    # @return [Array]
-    #
-    def advance
-      _lexer_actions             = self.class.send :_lexer_actions
-      _lexer_range_lengths       = self.class.send :_lexer_range_lengths
-      _lexer_trans_actions       = self.class.send :_lexer_trans_actions
-      _lexer_key_offsets         = self.class.send :_lexer_key_offsets
-      _lexer_index_offsets       = self.class.send :_lexer_index_offsets
-      _lexer_to_state_actions    = self.class.send :_lexer_to_state_actions
-      _lexer_trans_keys          = self.class.send :_lexer_trans_keys
-      _lexer_from_state_actions  = self.class.send :_lexer_from_state_actions
-      _lexer_single_lengths      = self.class.send :_lexer_single_lengths
-      _lexer_trans_targs         = self.class.send :_lexer_trans_targs
-      _lexer_eof_trans           = self.class.send :_lexer_eof_trans
-      _lexer_indicies            = self.class.send :_lexer_indicies
-      
-      %% write exec;
-      # %
-      
-      return @tokens.shift
-    end
-    
-    private
-    
-    ##
-    # Returns the text of the current buffer based on the supplied start and
-    # stop position.
-    #
-    # By default `@ts` and `@te` are used as the start/stop position.
-    #
-    # @param [Fixnum] start
-    # @param [Fixnum] stop
-    # @return [String]
-    #
-    def text(start = @ts, stop = @te)
-      return @data[start...stop].pack('U*')
-    end
-    
-    
-    %%{
-    # %
-      # Use instance variables for `ts` and friends.
-      access @;
-      getkey (@data[@p] || 0);
-      variable p @p;
-      variable pe @pe;
-      variable eof @eof;
-      
-      action do_nl { do_nl }
-      
-      include "lexer_char_classes.rl"; # Basic character classes
-      include "lexer.rl";              # Main rules file
-    }%%
-    # %
-  end
-end