luobo 0.0.2 → 0.0.5

data/README.rdoc

@@ -1,9 +1,36 @@
 = Luobo
 
-Luobo is a simple, easy to extend code generator.
+Luobo is a code-generator that can expands macros defined in a source file.
 
-When you feed <tt>tuzi</tt> (bunny) with some luobo (carrot) script, it will produce sources base on your predefined rules.
+Basically you use luobo by extends a Converter:
 
-= Examples
+    require "luobo"
+    class LuaSpecConverter < Luobo::Converter
+      def regex_comment_header; '^\s*--' end
 
-= Build-in Drivers
+      def do_hello token
+        "-- Hello, " + token.line_code + "!"
+      end
+    end
+
+You write a lua file (`test.lua`) like:
+
+    require "some_code"
+
+    -- HELLO: world
+
+You run:
+
+    LuaSpecConverter.new('test.lua', STDOUT).convert
+
+Then you will get those in stdout:
+
+    require "some_code"
+
+    -- Hello, world!
+
+TODO:
+
+- block inside comments
+- more robust tests
+- remove bin file

data/{example → backup/example}/parser.lub

@@ -12,10 +12,10 @@ Raw line contains ->
 AST: Raw line contains -> ->
 AST: 2,fake block
 
-## SPEC: a spec blog ->
+SPEC: a spec blog ->
   do that
   do this
-  ## SPEC: nested spec ->
+  SPEC: nested spec ->
     redo that
     redo this
   doing outside

data/backup/lib/luobo.rb

@@ -0,0 +1,5 @@
+require "yaml"
+
+require "luobo/token"
+require "luobo/driver"
+require "luobo/base"

data/{lib → backup/lib}/luobo/base.rb

@@ -55,48 +55,6 @@ module Luobo
       @driver.dump(@output, contents) 
     end
 
-    # travel up through the token stack, close all token with
-    # indent level larger or equal to the parameter
-    def close_stack indent_level
-      source = ''
-      while @token_stack.size > 0 and @token_stack[-1].indent_level >= indent_level
-        if @token_stack.size > 1 
-          @token_stack[-2].add_block_code @driver.convert(@token_stack[-1])
-        else # this is the last token in the stack
-          self.dump(@driver.convert(@token_stack[-1]))
-        end
-        @token_stack.pop
-      end
-    end
-
-    # add a new line to the existing token if it requires block codes
-    # or write the last token out before a new token
-    # this function invokes after a loop expansion.
-    def process_line line, ln, loop_n = 0
-      indent_level = 0
-      if /^(?<head_space_>\s+)/ =~ line
-        indent_level = head_space_.size
-      end
-      
-      # try to close stack if applicable
-      self.close_stack indent_level
-
-      # starts a named_processor or starts a raw_processor 
-      processor_name = '_raw'
-      line_code = line.gsub(/^\s*/,"")
-      block_code = nil
-      if matches = /#{regex_proc_line}/.match(line) 
-        processor_name = matches["proc_name_"]
-        line_code = matches["line_code_"]
-        block_code = '' if line_code.gsub!(/#{regex_block_start}/, '')
-      end
-      
-      @token_stack << Token.new(ln, line, indent_level, processor_name, line_code, block_code)
-
-      # unless it opens for block code close it soon, 
-      # (dedicate the dump function to close_stack())
-      self.close_stack indent_level unless block_code
-    end
 
     # add a new line to the example, separate examples to array
     def add_example_line line
@@ -112,8 +70,12 @@ module Luobo
 
     # regex configurations
     # match characters before a processor keyword
+    def regex_proc_head_without_space
+      "(?<proc_head_>\s*(\#{1,}|\-{2,}|\/{2,}))?"
+    end
+
     def regex_proc_head
-      "\s*(\#{2,}|\-{2,}|\/{2,})?\s*"
+      regex_proc_head_without_space + '\s*'
     end
 
     def regex_proc_name
@@ -128,9 +90,6 @@ module Luobo
       "^" + regex_proc_head + regex_proc_name + regex_proc_end + "(?<line_code_>.+)"
     end
 
-    def regex_block_start
-      "\s*(?<block_start_>\-\>)?\s*$"
-    end
 
     def regex_loop_line
       "^" + regex_proc_head + "\%\s*\=\=\=\=+\s*"

data/backup/lib/luobo/token.rb

@@ -0,0 +1,32 @@
+module Luobo
+  ## this class holds a block of carrot source code tokenized by the parser.
+  class Token
+    attr_accessor :ln, :line, :indent_level, :processor_name, :line_code, :block_code
+
+    def initialize ln, line, indent_level, processor_name, line_code, block_code, proc_head = ''
+      @ln, @line, @indent_level, @processor_name, @line_code, @block_code, @proc_head = ln, line, indent_level, processor_name, line_code, block_code, proc_head
+    end
+  
+    # add a line to current block args, separate each line with "\n"
+    def add_block_code line
+      line.chomp!
+      unless processor_name == '_raw'
+        line.gsub!(/^#{@proc_head}/, '')
+      end
+
+      if self.block_code 
+        self.block_code += "\n" + line
+      else
+        self.block_code = line
+      end
+    end
+
+    def block_args
+      YAML.load(block_code)
+    end
+
+    def line_args
+      YAML.load(line_code)
+    end
+  end
+end

data/backup/lib/luobo/version.rb

@@ -0,0 +1,4 @@
+module Luobo
+  VERSION = '0.0.3'
+end
+

data/backup/spec/token_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+describe Luobo::Token do
+  subject (:token) do
+    Luobo::Token.new(1, "line", 4, "Echo", "same: hash", "['block', 'in', 'array']") 
+  end
+
+  its :ln do 
+    should eq(1) 
+  end
+
+  its :line do
+    should eq("line")
+  end
+
+  its :indent_level do
+    should eq(4)
+  end
+
+  its :processor_name do
+    should eq("Echo")
+  end
+
+  its :line_args do
+    token.line_args["same"].should eq("hash")
+  end
+
+  its :block_args do
+    token.block_args[2].should eq("array")
+  end
+
+end

data/examples/loop_example.rb

@@ -0,0 +1,8 @@
+# This is a loop example.
+
+#% ====
+var = <%= variable %>
+#% ----
+#% variable: good!
+#% ----
+#% variable: also good!

data/examples/lua_loop.lua

@@ -0,0 +1,11 @@
+
+-- % ====
+-- SPEC: "<%= test_name %>" ->
+   local name = "<%= name %>"
+   do_test()
+-- % ----
+-- % test_name: first test
+-- % name: first
+-- % ----
+-- % test_name: second test
+-- % name: last

data/lib/luobo.rb

@@ -1,5 +1,221 @@
 require "yaml"
-
+require "erubis"
 require "luobo/token"
-require "luobo/driver"
-require "luobo/base"
+
+# converter class
+class Luobo
+
+  def initialize file, output
+    @source_file = file
+      
+    # handle output file or output IO 
+    if output.is_a?(IO)
+      @output = output
+    elsif output.is_a?(String)
+      @output_file = output
+      @output = File.open(output, "w")
+    end
+
+    # initialize a array to hold tokens waiting for process:
+    @token_stack = Array.new
+
+    # initialize loop template and examples
+    self.reset_loop
+  end
+
+  # initialize the holders for a example based loop, 
+  # or reset after a loop expansion
+  def reset_loop
+    @loop_start_ln = 0
+    @loop_template = nil
+    @loop_examples = Array.new
+  end
+
+  # extend a loop be examples
+  def expand_loop
+    raise "no examples found for loop start on line #{@loop_start_ln}" unless @loop_examples.size > 0
+    loop_n = 0
+    @loop_examples.each do |exa|
+      loop_n += 1
+      # treat each loop example as a yaml var definition for the template
+      rslt = Erubis::Eruby.new(@loop_template).result(YAML.load(exa))
+      li = 0
+      rslt.split("\n").each do |line|
+        li += 1
+        self.process_line @loop_start_ln + li, line
+      end
+    end
+    
+    # clear up holders
+    self.reset_loop
+  end
+
+  # add a new line to the example, separate examples to array
+  def add_example_line line
+    if /#{regex_new_example}/.match(line)
+      # start a new loop example with a placeholder nil        
+      @loop_examples << nil
+    else
+      raise "you need use '#{regex_new_example}' to start a loop example" unless @loop_examples.size > 0
+      line.gsub!(/#{regex_example_head}/, '')
+      @loop_examples[-1] = @loop_examples[-1] ? @loop_examples[-1] + "\n" + line : line
+    end
+  end
+
+  # handle convert for each token
+  def convert token
+    pname = "do_" + token.processor_name.downcase
+    if self.respond_to?(pname)
+      self.send(pname.to_sym, token)
+    else
+      self.do__missing(token)
+    end
+  end
+
+  # processor converters and dump callbacks
+  def do_setup; "" end     # call before all tokens
+  def do_cleanup; "" end   # call after all tokens
+
+  def do__raw token
+    if token.line and token.line.size > 0
+      token.line#.gsub(/^\s*/, "") + "\n"
+    else
+      ""
+    end
+  end
+
+  def do__missing token
+    src = token.line
+    src += token.block_code + "\n" if token.block_code
+    src
+  end
+
+  def dump contents
+    @output.print contents
+  end
+
+  # regex settings
+  def regex_line_comment; "" end
+  def regex_proc_head; "(#{regex_line_comment})?(?<leading_spaces_>\s*)" end
+  def regex_proc_name; "(?<processor_name_>[A-Z][_A-Z0-9]*)" end
+  def regex_proc_line; "^" + regex_proc_head + regex_proc_name + regex_proc_end + "(?<line_code_>.+)" end
+  def regex_proc_end; "\s*\:?\s+" end
+  def regex_block_start; "\-\>" end
+
+  # example loop related:
+  def regex_loop_line; "^" + regex_proc_head + "\%\s*\=\=\=\=+\s*" end
+  def regex_new_example; "^" + regex_proc_head + "\%\s*\-\-\-\-+\s*" end
+  def regex_example_head; "^" + regex_proc_head + "\%\s*" end
+  def regex_example_line; "^" + regex_proc_head + "\%\s*(?<example_>.+)\s*" end
+
+  # create a token from line
+  def tokenize ln, line
+    pure_line = line.gsub(/^#{regex_line_comment}/, '') # trim line comment marker
+
+    indent_level = 0
+    processor_name = '_raw'
+    line_code = ''
+    block_open = false
+    if matches = /#{regex_proc_line}/.match(pure_line) 
+      processor_name = matches["processor_name_"]
+      indent_level = matches["leading_spaces_"].size
+      line_code = matches["line_code_"]
+      block_open = true if /#{regex_block_start}\s*$/ =~ line_code
+      line_code.gsub!(/\s*(#{regex_block_start})?\s*$/, '')  
+    elsif matches = /^#{regex_proc_head}[^\s]/.match(pure_line) 
+      indent_level = matches["leading_spaces_"].size
+    end
+
+    Token.new ln, line, indent_level, processor_name, line_code, block_open 
+  end
+
+  # travel up through the token stack, close all token with
+  # indent level larger or equal to the parameter
+  def close_stack indent_level
+    while @token_stack.size > 0 and @token_stack[-1].indent_level >= indent_level
+      if @token_stack.size > 1 # if this is not the last token, add to parents
+        @token_stack[-2].add_block_code self.convert(@token_stack[-1])
+      else # this is the last token in the stack
+        dump self.convert(@token_stack[-1])
+      end
+      @token_stack.pop
+    end
+  end
+
+  # add a new line to the existing token if it requires block codes
+  # or write the last token out before a new token
+  # this function invokes after a loop expansion or if the line
+  # is not in any examples and loop templates.
+  def process_line ln, line
+    # create a token, with processor name
+    token = self.tokenize(ln, line)
+    # based on the the token indent level, close token stack if any
+    self.close_stack token.indent_level
+    
+    # add token and then close it unless it opens a block
+    @token_stack << token
+    self.close_stack token.indent_level unless token.block_open?
+
+    self # for methods chain
+  end
+
+  # process 
+  def process!    
+    fh = File.open(@source_file, 'r:utf-8')
+    in_loop = false
+    in_example = false
+    fh.each do |line|
+      line.chomp!
+      line.gsub!("\t", "  ") # convert tab to 2 spaces
+
+      # interrupt for loops
+      if /#{regex_loop_line}/.match(line) 
+        in_loop = true
+        @loop_start_ln = $.
+        next # break further processing if detecting a loop
+      end
+
+      # expand and end current loop 
+      if in_example
+        unless /#{regex_example_head}/.match(line) # if the line not marked as an example
+          self.expand_loop
+          in_example = false
+          in_loop = false
+        end
+      end
+
+      # end a loop template and start a loop example
+      if in_loop
+        in_example = true if /#{regex_example_head}/.match(line)
+      end
+
+      # dispatch the current line to different holders
+      if in_example
+        self.add_example_line line
+      elsif in_loop
+        @loop_template = @loop_template ? @loop_template + "\n" + line : line
+      else
+        self.process_line $., line
+      end
+      
+    end
+    fh.close
+
+    # do not forget expand last loop if it reaches over EOF.
+    self.expand_loop if @loop_start_ln > 0
+
+    # close all remain stacks (if any)
+    self.close_stack 0
+
+    self.dump(self.do_cleanup)
+    @output.close if @output_file
+ 
+    self # return self for method chains
+  end
+
+  # command handler
+  def self.convert! file, output
+    self.new(file, output).process!
+  end
+
+end

data/lib/luobo/token.rb

@@ -1,28 +1,42 @@
-module Luobo
-  ## this class holds a block of carrot source code tokenized by the parser.
-  class Token
-    attr_accessor :ln, :line, :indent_level, :processor_name, :line_code, :block_code
+## this class holds a block of carrot source code tokenized by the parser.
+class Token
+  attr_accessor :ln, :line, :indent_level, # raw line and line number
+    :processor_name, :line_code, :blocks, :indent_level
 
-    def initialize ln, line, indent_level, processor_name, line_code, block_code
-      @ln, @line, @indent_level, @processor_name, @line_code, @block_code = ln, line, indent_level, processor_name, line_code, block_code
-    end
-  
-    # add a line to current block args, separate each line with "\n"
-    def add_block_code line
-      line.chomp!
-      if self.block_code 
-        self.block_code += "\n" + line
-      else
-        self.block_code = line
-      end
-    end
+  def initialize ln, line, indent_level, processor_name, line_code = '', block_open = false
+    @ln, @line, @indent_level, @processor_name, @line_code = ln, line, indent_level, processor_name, line_code
+    @blocks = Array.new if block_open
+  end
 
-    def block_args
-      YAML.load(block_code)
-    end
+  def indent
+    " " * indent_level
+  end
 
-    def line_args
-      YAML.load(line_code)
+  # add a line to current block args, separate each line with "\n"
+  def add_block_code line
+    raise "block not opened in line #{:ln}" unless block_open?
+    line.chomp.split("\n").each do |ln| 
+      @blocks << ln
     end
   end
+
+  def has_block? 
+    (@blocks and @blocks.size > 0) ? true : false
+  end
+
+  def block_open?
+    @blocks.is_a?(Array)
+  end
+
+  def block_code
+    has_block? ? @blocks.join("\n") : ""
+  end
+
+  def block_args
+    YAML.load(block_code)
+  end
+
+  def line_args
+    YAML.load(line_code)
+  end
 end

data/lib/luobo/version.rb

@@ -1,4 +1,4 @@
 module Luobo
-  VERSION = '0.0.2'
+  VERSION = '0.0.5'
 end
 

data/luobo.gemspec

@@ -2,12 +2,11 @@ $LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
 require 'luobo/version'
 
 Gem::Specification.new 'luobo', Luobo::VERSION do |s|
-  s.description       = "Luobo is a simple, easy to extend code generator."
-  s.summary           = "Easy to extend code generator"
+  s.description       = "Luobo is an easy to extend code generator."
+  s.summary           = "Easy to extend macroes inside you soure code"
   s.authors           = ["Huang Wei"]
   s.email             = "huangw@pe-po.com"
   s.homepage          = "https://github.com/huangw/luobo-gem"
-  s.executables       << "tuzi"
   s.files             = `git ls-files`.split("\n") - %w[.gitignore]
   s.test_files        = Dir.glob("{spec,test}/**/*.rb")
   s.rdoc_options      = %w[--line-numbers --inline-source --title Luobo --main README.rdoc --encoding=UTF-8]

data/spec/block_spec.rb

@@ -0,0 +1,79 @@
+require "spec_helper"
+
+class OpLuobo < Luobo
+  attr_accessor :token_stack, :dumps
+
+  def dump contents;
+    # p contents
+    @dumps = Array.new unless @dumps 
+    @dumps << contents 
+  end
+
+  def last_dump; @dumps[-1] if @dumps end
+  
+  def stack_size; @token_stack.size end
+
+  def do_send token
+    "send_out: " + token.line_code + "\n" + token.block_code 
+  end
+end
+
+describe OpLuobo do
+  
+  context "no open block and no indent" do
+    subject {OpLuobo.new('-', STDOUT).process_line(1, "Send out an output")}
+      
+    its(:stack_size) { should eq(0) }
+    its(:last_dump) { should eq("Send out an output") }
+  end
+
+  context "with an open block and no indent" do
+    subject(:lb) {OpLuobo.new('-', STDOUT).process_line(1, "SEND: out an output ->")}
+      
+    its(:stack_size) { should eq(1) }
+    its(:last_dump) { should be_nil }
+
+    it "holds a block line" do
+      lb.process_line(2, "  block with indent")
+      lb.stack_size.should eq(1)
+      lb.last_dump.should be_nil
+
+      lb.process_line(3, "This is in the outside")
+      lb.stack_size.should eq(0)
+      lb.last_dump.should eq("This is in the outside")
+      lb.dumps[-2].should eq("send_out: out an output\n  block with indent")
+    end
+  end
+
+  context "with two indented block" do
+    subject(:lb) {OpLuobo.new('-', STDOUT).process_line(1, "SEND: out an output ->")}
+    
+    it "holds all block lines" do
+      lb.process_line(2, "  block with indent")
+      lb.process_line(3, "    block with more indent")
+      lb.process_line(4, "This is in the outside")
+      lb.stack_size.should eq(0)
+      lb.last_dump.should eq("This is in the outside")
+      lb.dumps[-2].should eq("send_out: out an output\n  block with indent\n    block with more indent")
+    end
+      
+  end
+
+  context "with nested indented block" do
+    subject(:lb) {OpLuobo.new('-', STDOUT).process_line(1, "SEND: out an output ->")}
+
+    it "process from the last block" do
+      lb.process_line(2, "  block with indent")
+      lb.process_line(3, "    SEND: more lines to send ->")
+      lb.process_line(4, "      an inside send line")
+      lb.process_line(5, "    still in the first block")      
+      lb.process_line(6, "close all stack")
+
+      lb.stack_size.should eq(0)
+      lb.last_dump.should eq("close all stack")
+      lb.dumps[-2].should eq("send_out: out an output\n  block with indent\nsend_out: more lines to send\n      an inside send line\n    still in the first block")
+    end
+
+  end
+
+end

data/spec/command_spec.rb

@@ -0,0 +1,14 @@
+require "spec_helper"
+
+class CommandConverter < Luobo
+  def process!
+    "OK"
+  end
+end
+
+describe Luobo do
+  it "exports a class method convert!" do
+    CommandConverter.convert!('examples/hello_processor.rb', STDOUT).should eq("OK")
+  end
+end
+

data/spec/loop_spec.rb

@@ -0,0 +1,35 @@
+require "spec_helper"
+
+class LoopLuobo < Luobo
+  attr_accessor :token_stack, :dumps
+
+  def regex_line_comment; "\s*#+\s?" end # use # as line comments
+
+  def dump contents;
+    # p contents
+    @dumps = Array.new unless @dumps 
+    @dumps << contents 
+  end
+
+  def last_dump; @dumps[-1] if @dumps end
+  
+  def stack_size; @token_stack.size end
+
+  def do_send token
+    "send_out: " + token.line_code + "\n" + token.block_code 
+  end
+end
+
+describe LoopLuobo do
+
+  context "Simple examples" do
+    subject(:lb){ LoopLuobo.new('examples/loop_example.rb', STDOUT) }
+    it "expands variables inside" do
+      lb.process!
+      lb.dumps[-3].should eq("var = good!")
+      lb.dumps[-2].should eq("var = also good!")
+    end
+
+  end
+
+end

data/spec/lua_loop_spec.rb

@@ -0,0 +1,35 @@
+require "spec_helper"
+
+class LuaLoopLuobo < Luobo
+  attr_accessor :token_stack, :dumps
+
+  def regex_line_comment; "\s*--+\s?" end # use # as line comments
+
+  def dump contents;
+    # p contents
+    @dumps = Array.new unless @dumps 
+    @dumps << contents 
+  end
+
+  def last_dump; @dumps[-1] if @dumps end
+  
+  def stack_size; @token_stack.size end
+
+  def do_spec token
+    "spec (#{token.line_code}, function()\n" + token.block_code + "\nend)"
+  end
+end
+
+describe LuaLoopLuobo do
+
+  context "Simple examples" do
+    subject(:lb){ LuaLoopLuobo.new('examples/lua_loop.lua', STDOUT) }
+    it "expands variables inside" do
+      lb.process!
+      lb.dumps[-3].should eq("spec (\"first test\", function()\n   local name = \"first\"\n   do_test()\nend)")
+      lb.dumps[-2].should eq("spec (\"second test\", function()\n   local name = \"last\"\n   do_test()\nend)")
+    end
+
+  end
+
+end

data/spec/token_spec.rb

@@ -1,32 +1,123 @@
 require "spec_helper"
 
-describe Luobo::Token do
-  subject (:token) do
-    Luobo::Token.new(1, "line", 4, "Echo", "same: hash", "['block', 'in', 'array']") 
-  end
+class LuaToken < Luobo
+  def regex_line_comment; "\s*\-\-+\s" end
+  def dump contents; contents end # directly return
+end
+
+class RubyToken < Luobo
+  def regex_line_comment; "\s*#+\s" end
+  def dump contents; contents end
+end
+
+describe Luobo do
+  context "without line comment marker" do
+    subject (:lb) do
+      Luobo.new('-', STDOUT)
+    end
+
+    it "return a token without indent" do
+      token = lb.tokenize(1, "PROCNAME: proc line code")
+      token.is_a?(Token).should be_true
+      token.processor_name.should eq("PROCNAME")
+      token.indent_level.should eq(0)
+      token.ln.should eq(1)
+      token.line.should eq("PROCNAME: proc line code")
+      token.line_code.should eq("proc line code")
+      token.has_block?.should be_false
+      token.block_code.should eq("")
+      token.block_open?.should be_false
+    end
+
+    it "return a token with indent" do
+      token = lb.tokenize(1, "  PROCNAME: proc line code")
+      token.processor_name.should eq("PROCNAME")
+      token.indent_level.should eq(2)
+      token.line_code.should eq("proc line code")
+      token.has_block?.should be_false
+      token.block_code.should eq("")
+      token.block_open?.should be_false
+    end
 
-  its :ln do 
-    should eq(1) 
+    it "return a token with block open" do
+      token = lb.tokenize(1, "PROCNAME2: proc line code ->")
+      token.processor_name.should eq("PROCNAME2")
+      token.line.should eq("PROCNAME2: proc line code ->")
+      token.line_code.should eq("proc line code")
+      token.block_code.should eq("")
+      token.block_open?.should be_true
+      token.has_block?.should be_false # no block added yet
+      token.add_block_code("Should\nBe True\n")
+      token.has_block?.should be_true
+      token.blocks.size.should eq(2)
+    end
   end
 
-  its :line do
-    should eq("line")
+  context "lua comment with no indent" do
+    subject do
+      LuaToken.new('-', STDOUT).tokenize(2, " -- SPEC: what ever ->")
+    end
+  
+    its :processor_name do
+      should eq("SPEC")
+    end
+
+    its :line_code do
+      should eq("what ever")
+    end
+
+    its :block_open? do
+      should be_true
+    end
+
+    its :indent_level do
+      should eq(0)
+    end
   end
 
-  its :indent_level do
-    should eq(4)
+
+  context "lua comment with indent" do
+    subject do
+      LuaToken.new('-', STDOUT).tokenize(2, " --   SPEC: what ever ->")
+    end
+  
+    its :indent_level do
+      should eq(2)
+    end
   end
 
-  its :processor_name do
-    should eq("Echo")
+  context "ruby comment with indent" do
+    subject do
+      RubyToken.new('-', STDOUT).tokenize(2, " #     INSIDE: what ever ->")
+    end
+  
+    its :processor_name do
+      should eq("INSIDE")
+    end
+
+    its :indent_level do
+      should eq(4)
+    end
   end
 
-  its :line_args do
-    token.line_args["same"].should eq("hash")
+  context "plain raw line" do
+    subject do
+      Luobo.new('-', STDOUT).tokenize(20, "What a wonderful spirit")
+    end
+
+    its :processor_name do
+      should eq("_raw")
+    end
   end
 
-  its :block_args do
-    token.block_args[2].should eq("array")
+  context "ruby comment with raw line" do
+    subject do
+      Luobo.new('-', STDOUT).tokenize(20, "# What a wonderful spirit")
+    end
+
+    its :processor_name do
+      should eq("_raw")
+    end
   end
 
 end

data/temp.rb

@@ -0,0 +1,142 @@
+
+
+  
+  
+  def dump contents
+    @output.print contents if contents
+  end
+
+  # travel up through the token stack, close all token with
+  # indent level larger or equal to the parameter
+  def close_stack indent_level
+    source = ''
+    while @token_stack.size > 0 and @token_stack[-1].indent_level >= indent_level
+      if @token_stack.size > 1 
+        @token_stack[-2].add_block_code self.convert(@token_stack[-1])
+      else # this is the last token in the stack
+        self.dump(self.convert(@token_stack[-1]))
+      end
+      @token_stack.pop
+    end
+  end
+
+  # add a new line to the existing token if it requires block codes
+  # or write the last token out before a new token
+  # this function invokes after a loop expansion.
+  def process_line line, ln, loop_n = 0
+    indent_level = 0
+    nline = line.gsub(/[\-#\/]/, '')
+    if /^(?<head_space_>\s+)/ =~ nline
+      indent_level = head_space_.size
+    end
+
+    # try to close stack if applicable
+    self.close_stack indent_level
+
+    # starts a named_processor or starts a raw_processor 
+    processor_name = '_raw'
+    line_code = line.gsub(/^\s*/,"")
+    block_code = nil
+    if matches = /#{regex_proc_line}/.match(line) 
+      proc_head = matches["proc_head_"]
+      processor_name = matches["proc_name_"]
+      line_code = matches["line_code_"]
+      block_code = '' if line_code.gsub!(/#{regex_block_start}/, '')  
+    end
+    
+    @token_stack << Token.new(ln, line, indent_level, processor_name, line_code, block_code, proc_head)
+
+    # unless it opens for block code close it soon, 
+    # (dedicate the dump function to close_stack())
+    self.close_stack indent_level unless block_code
+  end
+
+  # regex definition
+  # -------------------------------
+  def regex_line_comments; "" end # if defined, all 
+
+  def regex_proc_head
+    regex_line_comments + '\s*'
+  end
+
+  def regex_proc_name
+    "(?<proc_name_>[A-Z][A-Z0-9_]+)"
+  end
+
+  def regex_proc_end
+    "\s*\:?\s*"
+  end
+
+  def regex_block_start
+    "\s*(?<block_start_>\-\>)?\s*$"
+  end
+
+
+  # ===============================
+
+  # handle convert for each token
+  def convert token
+    pname = "do_" + token.processor_name.downcase
+    if self.respond_to?(pname)
+      self.send(pname.to_sym, token)
+    else
+      self.do__missing(token)
+    end
+  end
+
+  # ===============================
+
+  # parse the file, whenever found a token, convert
+  def process
+
+    fh = File.open(@source_file, 'r:utf-8')
+    self.dump(self.do_setup)
+
+    fh.each do |line|
+      line.chomp!
+      line.gsub!("\t", "  ") # convert tab to 2 spaces
+      
+      # interrupt for loops
+      if /#{regex_loop_line}/.match(line) 
+        in_loop = true
+        @loop_start_ln = $.
+        next # break further processing if detecting a loop
+      end
+
+      # expand and end current loop 
+      if in_example
+        unless /#{regex_example_head}/.match(line) # if the line not marked as an example
+          self.expand_loop
+          in_example = false
+          in_loop = false
+        end
+      end
+
+      # end a loop template and start a loop example
+      if in_loop
+        in_example = true if /#{regex_example_head}/.match(line)
+      end
+
+      # dispatch the current line to different holders
+      if in_example
+        self.add_example_line line
+      elsif in_loop
+        @loop_template = @loop_template ? @loop_template + "\n" + line : line
+      else
+        self.process_line line, $.
+      end
+    end
+    fh.close
+
+    # do not forget expand last loop if it reaches over EOF.
+    self.expand_loop if @loop_start_ln > 0
+
+    # close all remain stacks (if any)
+    self.close_stack 0
+
+    self.dump(self.do_cleanup)
+    @output.close if @output_file
+    self
+  end
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: luobo
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -43,27 +43,37 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Luobo is a simple, easy to extend code generator.
+description: Luobo is an easy to extend code generator.
 email: huangw@pe-po.com
-executables:
-- tuzi
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - README.rdoc
 - Rakefile
-- bin/tuzi
-- example/parser.lub
+- backup/bin/tuzi
+- backup/example/parser.lub
+- backup/lib/luobo.rb
+- backup/lib/luobo/base.rb
+- backup/lib/luobo/driver.rb
+- backup/lib/luobo/token.rb
+- backup/lib/luobo/version.rb
+- backup/spec/driver_spec.rb
+- backup/spec/parser_spec.rb
+- backup/spec/token_spec.rb
+- examples/loop_example.rb
+- examples/lua_loop.lua
 - lib/luobo.rb
-- lib/luobo/base.rb
-- lib/luobo/driver.rb
 - lib/luobo/token.rb
 - lib/luobo/version.rb
 - luobo.gemspec
-- spec/driver_spec.rb
-- spec/parser_spec.rb
+- spec/block_spec.rb
+- spec/command_spec.rb
+- spec/loop_spec.rb
+- spec/lua_loop_spec.rb
 - spec/spec_helper.rb
 - spec/token_spec.rb
+- temp.rb
 homepage: https://github.com/huangw/luobo-gem
 licenses: []
 post_install_message: 
@@ -94,9 +104,11 @@ rubyforge_project:
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Easy to extend code generator
+summary: Easy to extend macroes inside you soure code
 test_files:
-- spec/driver_spec.rb
-- spec/parser_spec.rb
+- spec/block_spec.rb
+- spec/command_spec.rb
+- spec/loop_spec.rb
+- spec/lua_loop_spec.rb
 - spec/spec_helper.rb
 - spec/token_spec.rb