sourcify 0.1.2 → 0.2.0

data/lib/sourcify/proc/scanner.rl

@@ -0,0 +1,285 @@
+Sourcify.require_rb('proc', 'scanner', 'extensions')
+
+module Sourcify
+  module Proc
+    module Scanner #:nodoc:all
+
+%%{
+  machine proc_scanner;
+
+  kw_class  = 'class';
+  kw_module = 'module';
+  kw_def    = 'def';
+  kw_begin  = 'begin';
+  kw_case   = 'case';
+  kw_if     = 'if';
+  kw_unless = 'unless';
+  kw_do     = 'do';
+  kw_then   = 'then';
+  kw_for    = 'for';
+  kw_while  = 'while';
+  kw_until  = 'until';
+  kw_end    = 'end';
+
+  vchar   = alnum | '_';
+  const   = upper . vchar* . ('::' . alpha . vchar*)*;
+  var     = (lower | '_') . vchar*;
+  meth    = (alpha | '_') . vchar*;
+  symbol  = ':' . (var | const);
+  assoc   = '=>';
+  assgn   = '=';
+  comma   = ',';
+  label   = (var | const) . ':';
+
+  lparen  = '(';
+  rparen  = ')';
+  lbrace  = '{';
+  rbrace  = '}';
+  smcolon = ';';
+  newline = '\n';
+  ospaces = space* -- newline;
+  mspaces = space+ -- newline;
+
+  ## MACHINE >> New statement
+  new_statement := |*
+
+    (kw_for | kw_while | kw_until) . ^vchar => {
+      push(:kw_do_alias2, ts, te)
+      increment_counter(:do_end, 0..1)
+      fgoto main;
+    };
+
+    (
+      ((kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless) . ^vchar) |
+      (kw_class . ospaces . '<<' . ospaces . ^newline+)
+    ) => {
+      push(:kw_do_alias1, ts, te)
+      increment_counter(:do_end, 1)
+      fgoto main;
+    };
+
+    ospaces => { push(:space, ts, te) };
+    any => { fhold; fgoto main; };
+
+  *|;
+
+  ## MACHINE >> One-liner comment
+  per_line_comment := |*
+    ^newline* => {
+      push(:comment, ts.pred, te)
+      fgoto main;
+    };
+  *|;
+
+  ## MACHINE >> Block comment
+  block_comment := |*
+    any* . newline . '=end' . ospaces . ^newline* => {
+      unless push_comment(ts, te)
+        fgoto main;
+      end
+    };
+  *|;
+
+  ## MACHINE >> Heredoc
+  heredoc := |*
+    ^newline* . newline . ospaces . ^newline+ => {
+      unless push_heredoc(ts, te)
+        fgoto main;
+      end
+    };
+  *|;
+
+  ## MACHINE >> Double quote string
+  double_quote_str := |*
+    delimiter = [\~\`\!\@\#\$\%\^\&\*\)\-\_\+\=\}\]\:\;\'\"\>\,\.\?\/\|\\];
+    (^delimiter* . delimiter) -- ('\\' . delimiter) => {
+      unless push_dstring(ts, te)
+        fgoto main;
+      end
+    };
+  *|;
+
+  ## MACHINE >> Main
+  main := |*
+
+    ## == Start/end of do..end block
+
+    kw_do => {
+      push(:kw_do, ts, te)
+      increment_counter(:do_end)
+      fgoto new_statement;
+    };
+
+    kw_end => {
+      push(:kw_end, ts, te)
+      decrement_counter(:do_end)
+    };
+
+    ## == Start/end of {...} block
+
+    lbrace => {
+      push(:lbrace, ts, te)
+      increment_counter(:brace)
+    };
+
+    rbrace => {
+      push(:rbrace, ts, te)
+      decrement_counter(:brace)
+    };
+
+    assoc => {
+      push(:assoc, ts, te)
+      fix_counter_false_start(:brace)
+    };
+
+    label => {
+      push_label(ts, te)
+      fix_counter_false_start(:brace)
+    };
+
+    ## == New statement
+
+    newline => {
+      push(:newline, ts, te)
+      increment_lineno
+      fgoto new_statement;
+    };
+
+    smcolon | lparen | assgn | kw_then | comma => {
+      push(:newline_alias, ts, te)
+      fgoto new_statement;
+    };
+
+    ## == Comment
+
+    '#' => {
+      fgoto per_line_comment;
+    };
+
+    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
+      push_comment(ts, te)
+      increment_lineno
+      fgoto block_comment;
+    };
+
+    ## == Misc
+
+    var         => { push(:var, ts, te) };
+    const       => { push(:const, ts, te) };
+    symbol      => { push(:symbol, ts, te) };
+    mspaces     => { push(:space, ts, te) };
+    any         => { push(:any, ts, te) };
+    '&:' . meth => { push(:to_proc, ts, te) };
+
+    ## == Heredoc
+
+    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
+      push_heredoc(ts, te)
+      increment_lineno
+      fgoto heredoc;
+    };
+
+    ## == Single quote strings
+
+    sqs1  = "'" . ([^\']* | ([^\'\\]*[\\][\'][^\']*)*) . '\\\\'* . "'";
+    sqs2  = '~' . ([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) . '\\\\'* . '~';
+    sqs3  = '`' . ([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) . '\\\\'* . '`';
+    sqs4  = '!' . ([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) . '\\\\'* . '!';
+    sqs5  = '@' . ([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) . '\\\\'* . '@';
+    sqs6  = '#' . ([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) . '\\\\'* . '#';
+    sqs7  = '$' . ([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) . '\\\\'* . '$';
+    sqs8  = '%' . ([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) . '\\\\'* . '%';
+    sqs9  = '^' . ([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) . '\\\\'* . '^';
+    sqs10 = '&' . ([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) . '\\\\'* . '&';
+    sqs11 = '*' . ([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) . '\\\\'* . '*';
+    sqs12 = '-' . ([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) . '\\\\'* . '-';
+    sqs13 = '_' . ([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) . '\\\\'* . '_';
+    sqs14 = '+' . ([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) . '\\\\'* . '+';
+    sqs15 = '=' . ([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) . '\\\\'* . '=';
+    sqs16 = '<' . ([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) . '\\\\'* . '>';
+    sqs17 = '|' . ([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) . '\\\\'* . '|';
+    sqs18 = ':' . ([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) . '\\\\'* . ':';
+    sqs19 = ';' . ([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) . '\\\\'* . ';';
+    sqs20 = '"' . ([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) . '\\\\'* . '"';
+    sqs21 = ',' . ([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) . '\\\\'* . ',';
+    sqs22 = '.' . ([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) . '\\\\'* . '.';
+    sqs23 = '?' . ([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) . '\\\\'* . '?';
+    sqs24 = '/' . ([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) . '\\\\'* . '/';
+    sqs25 = '{' . ([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) . '\\\\'* . '}';
+    sqs26 = '[' . ([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) . '\\\\'* . ']';
+    sqs27 = '(' . ([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) . '\\\\'* . ')';
+    sqs28 = '\\' . ([^\\]* | ([^\)]*[\\][\\][^\\]*)*) . '\\';
+
+    sqm = ('%q' | '%w'); (
+      sqs1        | sqm . sqs1  | sqm . sqs2  | sqm . sqs3  | sqm . sqs4  |
+      sqm . sqs5  | sqm . sqs6  | sqm . sqs7  | sqm . sqs8  | sqm . sqs9  |
+      sqm . sqs10 | sqm . sqs11 | sqm . sqs12 | sqm . sqs13 | sqm . sqs14 |
+      sqm . sqs15 | sqm . sqs16 | sqm . sqs17 | sqm . sqs18 | sqm . sqs19 |
+      sqm . sqs20 | sqm . sqs21 | sqm . sqs22 | sqm . sqs23 | sqm . sqs24 |
+      sqm . sqs25 | sqm . sqs26 | sqm . sqs27 | sqm . sqs28
+    ) => {
+      push(:sstring, ts, te)
+    };
+
+    ## == Double quote strings
+
+    dqs1  = '"' . (([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) -- '#{') . '\\\\'* . ('"' | '#{');
+    dqs2  = '`' . (([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) -- '#{') . '\\\\'* . ('`' | '#{');
+    dqs3  = '/' . (([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) -- '#{') . '\\\\'* . ('/' | '#{');
+    dqs4  = "'" . (([^\']* | ([^\'\\]*[\\][\'][^\']*)*) -- '#{') . '\\\\'* . ("'" | '#{');
+    dqs5  = '~' . (([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) -- '#{') . '\\\\'* . ('~' | '#{');
+    dqs6  = '!' . (([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) -- '#{') . '\\\\'* . ('!' | '#{');
+    dqs7  = '@' . (([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) -- '#{') . '\\\\'* . ('@' | '#{');
+    dqs8  = '#' . (([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) -- '#{') . '\\\\'* . ('#' | '#{');
+    dqs9  = '$' . (([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) -- '#{') . '\\\\'* . ('$' | '#{');
+    dqs10 = '%' . (([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) -- '#{') . '\\\\'* . ('%' | '#{');
+    dqs11 = '^' . (([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) -- '#{') . '\\\\'* . ('^' | '#{');
+    dqs12 = '&' . (([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) -- '#{') . '\\\\'* . ('&' | '#{');
+    dqs13 = '*' . (([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) -- '#{') . '\\\\'* . ('*' | '#{');
+    dqs14 = '-' . (([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) -- '#{') . '\\\\'* . ('-' | '#{');
+    dqs15 = '_' . (([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) -- '#{') . '\\\\'* . ('_' | '#{');
+    dqs16 = '+' . (([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) -- '#{') . '\\\\'* . ('+' | '#{');
+    dqs17 = '=' . (([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) -- '#{') . '\\\\'* . ('=' | '#{');
+    dqs18 = '<' . (([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) -- '#{') . '\\\\'* . ('>' | '#{');
+    dqs19 = '|' . (([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) -- '#{') . '\\\\'* . ('|' | '#{');
+    dqs20 = ':' . (([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) -- '#{') . '\\\\'* . (':' | '#{');
+    dqs21 = ';' . (([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) -- '#{') . '\\\\'* . (';' | '#{');
+    dqs22 = ',' . (([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) -- '#{') . '\\\\'* . (',' | '#{');
+    dqs23 = '.' . (([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) -- '#{') . '\\\\'* . ('.' | '#{');
+    dqs24 = '?' . (([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) -- '#{') . '\\\\'* . ('?' | '#{');
+    dqs25 = '{' . (([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) -- '#{') . '\\\\'* . ('}' | '#{');
+    dqs26 = '[' . (([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) -- '#{') . '\\\\'* . (']' | '#{');
+    dqs27 = '(' . (([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) -- '#{') . '\\\\'* . (')' | '#{');
+    dqs28 = '\\' . (([^\\]* | ([^\\]*[\\][\\][^\\]*)*) -- '#{') . ('\\' | '#{');
+
+    dqm = ('%Q' | '%W' | '%x' | '%r' | '%'); (
+      dqs1        | dqs2        | dqs3        | dqm . dqs1  | dqm . dqs2  |
+      dqm . dqs3  | dqm . dqs4  | dqm . dqs5  | dqm . dqs6  | dqm . dqs7  |
+      dqm . dqs8  | dqm . dqs9  | dqm . dqs10 | dqm . dqs11 | dqm . dqs12 |
+      dqm . dqs13 | dqm . dqs14 | dqm . dqs15 | dqm . dqs16 | dqm . dqs17 |
+      dqm . dqs18 | dqm . dqs19 | dqm . dqs20 | dqm . dqs21 | dqm . dqs22 |
+      dqm . dqs23 | dqm . dqs24 | dqm . dqs25 | dqm . dqs26 | dqm . dqs27 |
+      dqm . dqs28
+    ) => {
+      if push_dstring(ts, te)
+        fgoto double_quote_str;
+      end
+    };
+
+  *|;
+
+}%%
+%% write data;
+
+      extend Scanner::Extensions
+
+      def self.execute!
+        data = @data
+        eof = data.length
+        %% write init;
+        %% write exec;
+      end
+
+    end
+  end
+end

data/lib/sourcify/proc/scanner/comment.rb

@@ -0,0 +1,21 @@
+module Sourcify
+  module Proc
+    module Scanner #:nodoc:all
+      class Comment
+
+        def <<(content)
+          (@contents ||= []) << content
+        end
+
+        def to_s
+          @contents.join
+        end
+
+        def closed?
+          @contents[-1].split("\n")[-1].strip == '=end'
+        end
+
+      end
+    end
+  end
+end

data/lib/sourcify/proc/scanner/counter.rb

@@ -0,0 +1,44 @@
+module Sourcify
+  module Proc
+    module Scanner #:nodoc:all
+      class Counter
+
+        attr_reader :counts
+
+        def initialize
+          @counts = [0,0]
+        end
+
+        def started?
+          @counts.any?(&:nonzero?)
+        end
+
+        def just_started?
+          @counts.any?{|count| count == 1 }
+        end
+
+        def balanced?
+          @counts.any?(&:zero?)
+        end
+
+        def decrement
+          (0..1).each{|i| @counts[i] -= 1 unless @counts[i].zero? }
+        end
+
+        def increment(val = 1)
+          if val.is_a?(Range)
+            @counts[0] += val.first
+            @counts[1] += val.last
+          else
+            (0..1).each{|i| @counts[i] += 1 }
+          end
+        end
+
+      end
+
+      class DoEndBlockCounter < Counter; end
+      class BraceBlockCounter < Counter; end
+
+    end
+  end
+end

data/lib/sourcify/proc/scanner/dstring.rb

@@ -0,0 +1,58 @@
+module Sourcify
+  module Proc
+    module Scanner #:nodoc:all
+      class DString < Struct.new(:tag)
+
+        # To suppress 'warning: Object#type is deprecated; use Object#class' when
+        # evaluating string
+        attr_reader :type
+
+        def <<(content)
+          (@contents ||= []) << content
+        end
+
+        def to_s
+          @contents.join
+        end
+
+        def closed?
+          # NOTE: The only real error is SyntaxError, other errors are due
+          # to undefined variable or watever, which are perfectly ok when
+          # testing for validity of the string.
+          begin
+            instance_eval(safe_contents) if evaluable?
+          rescue SyntaxError
+            false
+          rescue Exception
+            true
+          end
+        end
+
+        private
+
+          CLOSING_TAGS = {'(' => ')', '[' => ']', '<' => '>', '{' => '}'}
+
+          def evaluable?
+            @contents[-1][-1].chr == end_tag
+          end
+
+          def safe_contents
+            # NOTE: %x & ` strings are dangerous to eval cos they execute shell commands,
+            # thus we convert them to normal strings 1st
+            @contents.join.gsub(/(%x)(\W|\_)/, '%Q\2').gsub(/.{0,2}(`)/) do |s|
+              s =~ /^(%Q|%W|%r|%x|.?%|.?\\)/ ? s : s.sub(/`$/,'%Q`')
+            end
+          end
+
+          def start_tag
+            @start_tag ||= tag[-1].chr
+          end
+
+          def end_tag
+            @end_tag ||= (CLOSING_TAGS[start_tag] || start_tag)
+          end
+
+      end
+    end
+  end
+end

data/lib/sourcify/proc/scanner/extensions.rb

@@ -0,0 +1,135 @@
+%w{heredoc comment dstring counter}.each do |f|
+  Sourcify.require_rb('proc', 'scanner', f)
+end
+
+module Sourcify
+  module Proc
+    module Scanner #:nodoc:all
+      module Extensions
+
+        class Escape < Exception; end
+
+        def process(data, stop_on_newline = false)
+          begin
+            @stop_on_newline = stop_on_newline
+            @results, @data = [], data.unpack("c*")
+            reset_attributes
+            execute!
+          rescue Escape
+            @results
+          end
+        end
+
+        def data_frag(range)
+          @data[range].pack('c*')
+        end
+
+        def push(key, ts, te)
+          @results << :symbol_to_proc if key == :to_proc && @lineno == 1
+          @tokens << [key, data_frag(ts .. te.pred)]
+        end
+
+        def push_dstring(ts, te)
+          data = data_frag(ts .. te.pred)
+          unless @dstring
+            @dstring = DString.new(data[%r{^("|`|/|%(?:Q|W|r|x|)(?:\W|_))},1])
+          end
+          @dstring << data
+          return true unless @dstring.closed?
+          @tokens << [:dstring, @dstring.to_s]
+          @dstring = nil
+        end
+
+        def push_comment(ts, te)
+          data = data_frag(ts .. te.pred)
+          @comment ||= Comment.new
+          @comment << data
+          return true unless @comment.closed?
+          @tokens << [:comment, @comment.to_s]
+          @comment = nil
+        end
+
+        def push_heredoc(ts, te)
+          data = data_frag(ts .. te.pred)
+          unless @heredoc
+            indented, tag = data.match(/\<\<(\-?)['"]?(\w+)['"]?$/)[1..3]
+            @heredoc = Heredoc.new(tag, !indented.empty?)
+          end
+          @heredoc << data
+          return true unless @heredoc.closed?(data_frag(te .. te))
+          @tokens << [:heredoc, @heredoc.to_s]
+          @heredoc = nil
+        end
+
+        def push_label(ts, te)
+          # NOTE: 1.9.* supports label key, which RubyParser cannot handle, thus
+          # conversion is needed.
+          data = data_frag(ts .. te.pred)
+          @tokens << [:symbol, data.sub(/^(.*)\:$/, ':\1')]
+          @tokens << [:space, ' ']
+          @tokens << [:assoc, '=>']
+        end
+
+        def increment_lineno
+          @lineno += 1
+          raise Escape if @stop_on_newline || !@results.empty?
+        end
+
+        def increment_counter(key, count = 1)
+          return if other_counter(key).started?
+          counter = this_counter(key)
+          offset_attributes unless counter.started?
+          counter.increment(count)
+        end
+
+        def decrement_counter(key)
+          return unless (counter = this_counter(key)).started?
+          counter.decrement
+          construct_result_code if counter.balanced?
+        end
+
+        def fix_counter_false_start(key)
+          reset_attributes if this_counter(key).just_started?
+        end
+
+        def other_counter(type)
+          {:do_end => @brace_counter, :brace => @do_end_counter}[type]
+        end
+
+        def this_counter(type)
+          {:brace => @brace_counter, :do_end => @do_end_counter}[type]
+        end
+
+        def construct_result_code
+          begin
+            code = 'proc ' + @tokens.map(&:last).join
+            eval(code) # TODO: any better way to check for completeness of proc code ??
+            @results << code
+            raise Escape if @stop_on_newline or @lineno != 1
+            reset_attributes
+          rescue Exception
+            raise if $!.is_a?(Escape)
+          end
+        end
+
+        def reset_attributes
+          @tokens = []
+          @lineno = 1
+          @heredoc, @dstring, @comment = nil
+          @do_end_counter = DoEndBlockCounter.new
+          @brace_counter = BraceBlockCounter.new
+        end
+
+        def offset_attributes
+          @lineno = 1 # Fixing JRuby's lineno bug (see http://jira.codehaus.org/browse/JRUBY-5014)
+          unless @tokens.empty?
+            last = @tokens[-1]
+            @tokens.clear
+            @tokens << last
+          end
+        end
+
+      end
+    end
+  end
+end