rsec 0.3.2 → 0.3.6

data/examples/little_markdown.rb

@@ -0,0 +1,167 @@
+# a markdown translator
+# 
+# The differences between this and original markdown:
+# - markdown in inline tags are not processed
+# - every line-break in non-tag parts is translated into <br/>
+# - nested list elements are not supported
+
+require "rsec"
+
+class LittleMarkdown
+  include Rsec::Helper
+
+  def initialize
+    @markdown_line_translator = make_markdown_line_translator
+    @parser = (make_xml_tag_parser | make_char_parser).star.eof
+  end
+
+  def translate src
+    @stack = []
+    @charsbuf = ''
+    @out = ''
+    @parser.parse! src
+    flush_chars
+    @out
+  end
+
+  def flush_chars
+    @out.<< translate_markdown @charsbuf
+    @charsbuf = ''
+  end
+
+  def make_char_parser
+    # care stringscanner's bug, see issues
+    (/./.r | /\n/).fail('char'){|c| @charsbuf << c}
+  end
+
+  # make a single-line markdown parser
+  def make_markdown_line_translator
+    line_text = lazy{line}.map{|tokens|
+      tokens.empty? ? Rsec::INVALID : tokens.join # filter out empty
+    }
+
+    title = /"[^"]*"|'[^']*'/.r._?{|(s)|
+      s ? "title=#{s}" : ''
+    }
+    img = seq('!['.r >> /[^\]]+/ << '](', /[^\)"']+/, title, ')'){|(txt, path, title)|
+      "<img src='#{path}' #{title}>#{txt}</img>"
+    }
+    link = seq(('['.r >> /[^\]]+/ << ']('), /[^\)"']+/, title, ')'){|(txt, path, title)|
+      "<a href='#{path}' #{title}>#{txt}</a>"
+    }
+    # NOTE strong should be left of em
+    strong = ('**'.r >> line_text << '**').map{|s|
+      "<strong>#{s}</strong>"
+    }
+    em = ('*'.r >> line_text << '*').map{|s|
+      "<em>#{s}</em>"
+    }
+    code = ('`'.r >> /[^`]+/ << '`').map{|s|
+      "<code>#{s}</code>"
+    }
+    escape = '<'.r{'&lt;'} | '&'.r{'&amp;'} | /\\[\!\`\*\[\]]/.r{|s|s[1]}
+    text = /[^\!\`\*\[\]]+/
+    id = seq_(('['.r >> /[^\]]+/ << ']:'), text){|(id, text)|
+      "<span id='#{id}'>#{text}</span>"
+    }
+    line = (img | link | strong | em | code | escape | id | text).star
+    line.eof.map &:join
+  end
+  
+  # pseudo xml tag parser, except <br> and <hr> and <script>
+  def make_xml_tag_parser
+    name  = /[\w-]+/ # greedy, no need to worry space between first attr
+    value = /"[^"]*"|'[^']*'/
+    attr  = seq_(name, seq_('=', value)._?)
+    attrs = /\s*/.r.join(attr)
+
+    # use a stack to ensure tag matching
+    tag_start = seq('<', name, attrs){|res|
+      @stack.push res[1].downcase
+      res
+    }
+    tag_empty_end = '/>'.r{|res|
+      @stack.pop
+      res
+    }
+    tag_non_empty_end = seq('>', lazy{content}, '</', name, /\s*\>/){|res|
+      if @stack.pop == res[3].downcase
+        res
+      else
+        Rsec::INVALID
+      end
+    }
+    special_tag = /\<[bh]r\s*\>/i.r | seq_('<script', attrs, /\>.*?\<\/script\>/)
+    tag = special_tag | seq(tag_start, (tag_empty_end | tag_non_empty_end))
+
+    # xml content
+    comment = /<!--([^-]|-[^-])*-->/
+    cdata   = /<!\[CDATA\[.*?\]\]>/x
+    entity  = /&(nbsp|lt|gt|amp|cent|pound|yen|euro|sect|copy|reg|trade|#[a-f0-9]{2,4});/i
+    text    = /[^<&]+/
+    content = (cdata.r | comment | entity | tag | text).star
+    tag.fail('tag'){|res|
+      if @charsbuf.end_with? "\n"
+        flush_chars
+        @out << res.join
+      else
+        @charsbuf << res.join # inline tags
+      end
+    }
+  end
+
+  # translate markdown
+  def translate_markdown str
+    lines = str.split("\n").chunk{|line|
+      line[/^(\ {4}|\#{1,6}\ |[\+\-\>]\ |)/]
+    }.map{|(leading, lines)|
+      case leading
+      when '    '                                              # code
+        "<pre><code>#{lines.join "\n"}</code></pre>"
+      when /\#{1,6}/                                           # headings
+        hn = "h#{leading.strip.size}"
+        lines.map! do |line|
+          line = line.sub(/\#{1,6}/, '')
+          "<#{hn}>#{@markdown_line_translator.parse! line}</#{hn}>"
+        end
+        lines.join
+      when '> '                                                # block quote
+        # TODO nested
+        lines.map! do |line|
+          @markdown_line_translator.parse! line[2..-1]
+        end
+        "<blockquote>#{lines.join '<br/>'}</blockquote>"
+      when '+ '                                                # numbered list
+        # TODO nested
+        lines.map! do |line|
+          "<li>#{@markdown_line_translator.parse! line[2..-1]}</li>"
+        end
+        "<ol>#{lines.join}</ol>"
+      when '- '                                                # unordered list
+        # TODO nested
+        lines.map! do |line|
+          "<li>#{@markdown_line_translator.parse! line[2..-1]}</li>"
+        end
+        "<ul>#{lines.join}</ul>"
+      else
+        lines.map! do |line|
+          @markdown_line_translator.parse! line
+        end
+        lines.join "<br/>"
+      end
+    }
+    # add trailing '\n' s
+    lines.join('<br/>') << ('<br/>' * str[/\n*\Z/].size)
+  end
+
+end
+
+if __FILE__ == $PROGRAM_NAME
+  lm = LittleMarkdown.new
+  puts lm.translate <<-MD
+## *a *
+<pre a="3">123afd</pre>
+  ** b **
+  MD
+end
+

data/examples/nasm_manual.rb

@@ -0,0 +1,136 @@
+# Parse NASM manual [nasm.txt] and generate a list of opcodes.
+# Results are saved in [nasm_codes.txt], undocumented codes are printed.
+# Further: extend the parser to generate an X86 assembler.
+require "rsec"
+
+module NASMManualParser
+  include Rsec::Helper
+  extend self
+
+  Instructions = {}
+
+  class UnSupportedError < RuntimeError
+  end
+
+  class Instruction < Struct.new(:nemonic, :operands, :code, :archs)
+  end
+
+  def debug parser, *strs
+    return parser unless $debug
+    strs.each do |str|
+      parser.eof.parse! str
+    end
+    parser
+  end
+
+  def reg_parser
+    gp_reg  = /E?[ABCD]X|E?(SP|BP|SI|DI)/
+    gp_reg8 = /[ABCD][HL]/
+    seg_reg = /ES|CS|SS|DS|FS|GS/
+    fpu_reg = /ST[0-7]/
+    mmx_reg = /MM[0-7]/
+    xr_reg  = /CR[0234]|DR[012367]|TR[34567]/
+    reg = gp_reg.r | gp_reg8 | seg_reg | fpu_reg | mmx_reg | xr_reg
+    debug reg, 'AX'
+  end
+
+  def operands_parser
+    imm_class     = /imm:imm(32|16)|imm(32|16|8)?/
+    mem_class     = /mem(80|64|32|16|8)?/ # be ware of the order
+    reg_class     = /reg(32|16|8)|(fpu|mmx|seg)reg/
+    memoffs_class = /memoffs(32|16|8)/
+    tr_class      = 'TR3/4/5/6/7'
+    classes       = (imm_class.r | memoffs_class | mem_class | reg_class | tr_class).fail 'operand class'
+    reg           = reg_parser.fail 'register'
+    num           = /\d/.r(&:to_i).fail 'num'
+    # memoffs should be left of mem
+    operand       = classes | reg | num
+    operands      = operand.join('/').even.join(',').even
+    debug operands, 'reg32', 'AX,memoffs16'
+  end
+
+  def code_parser
+    plus_cc     = /[0-9A-F][0-9A-F]\+cc/
+    plus_r      = /[0-9A-F][0-9A-F]\+r/
+    hex         = /[0-9A-F][0-9A-F]/.r {|s| s.to_i 16}
+    slash       = /\/[\dr]/
+    imm_code    = /i[bwd]/
+    reg_code    = /rw\/rd|r[bwd]/
+    ref_code    = /ow\/od|o[wd]/
+    prefix_code = /[oa](32|16)/
+    code =\
+      (plus_cc.r | plus_r | hex | slash |
+      imm_code | reg_code | ref_code | prefix_code).join(/\s+/).even
+    debug code, 'o32 0F C8+r', 'o32 6B /r ib', 'o16 A1 ow/od'
+  end
+
+  def archs_parser
+    arch = symbol(/8086|186|286|386|486|PENT|P6|CYRIX|FPU|MMX|PRIV|UNDOC/)
+    archs = ('['.r >> arch.join(',').even << ']').map do |archs|
+      # map to set
+      archs.inject({}){|h, arch|
+        raise UnSupportedError, 'not implemented' if arch == 'UNDOC'
+        h[arch] = true
+        h
+      }
+    end
+    debug archs, '[386,FPU]'
+  end
+
+  def instruction_parser
+    nemonic = /[A-Z]\w+|xxSAR/
+    operands = operands_parser._?
+    code = ';'.r >> code_parser
+    archs = archs_parser
+    instruction = seq_ nemonic, operands, code, archs do |nemonic, (operands), code, archs|
+      Instruction.new nemonic, operands, code, archs
+    end
+    debug instruction, 'FISUBR mem32 ; DA /5 [8086,FPU]', 'BSWAP reg32 ; o32 0F C8+r [486]'
+  end
+
+  def desugar line
+    # r/m short hands
+    line = line.gsub /r\/m(32|16|8)/, 'reg\1/mem\1'
+    line.gsub! 'r/m64', 'mmxreg/mem64'
+    # compress space
+    line.sub! /\s(TO|NEAR|FAR|SHORT)/, '_\1'
+    line
+  end
+
+  def parse_line parser, line
+    parser.parse! desugar line
+  rescue Rsec::SyntaxError
+  rescue UnSupportedError
+  end
+
+  def parse filename
+    parsed = ''
+    parser = instruction_parser.eof
+    src = File.read filename
+    src.lines.with_index do |raw_line, idx|
+      line = raw_line.strip
+      # this shapy shows the line is something defining an nemonic
+      if line =~ /^\w+\s+[^;\[]+;\ [^;\[]+\[.+\]$/
+        if (parse_line parser, line)
+          parsed << raw_line
+        else
+          puts "unparsed:#{idx}\t#{line}"
+        end
+      end
+    end
+    parsed
+  end
+
+end
+
+if __FILE__ == $PROGRAM_NAME
+  $debug = true
+  manual = "#{File.dirname __FILE__}/nasm_manual.txt"
+  codes  = "#{File.dirname __FILE__}/nasm_codes.txt"
+  File.open codes, 'w' do |file|
+    file.<< NASMManualParser.parse manual
+  end
+  puts '-' * 80
+  puts "X86 asm codes are saved to #{codes}"
+end
+

data/lib/rsec/helpers.rb

@@ -212,7 +212,7 @@ module Rsec #:nodoc:
 
     # @ desc
     #   Repeat n or in a range.
-    #   If range.end < 0, repeat at least range.begin
+    #   If range.end &lt; 0, repeat at least range.begin
     #   (Infinity and -Infinity are considered)
     def * n, &p
       # FIXME if self is an epsilon parser, will cause infinite loop
@@ -329,8 +329,8 @@ module Rsec #:nodoc:
     # @ desc
     #   Think about "innerHTML"!
     # @ example
-    #   parser = seq('<b>', /[\w\s]+/, '</b>').inner
-    #   parser.parse('<b>the inside</b>')
+    #   parser = seq('&lt;b&gt;', /[\w\s]+/, '&lt;/b&gt;').inner
+    #   parser.parse('&lt;b&gt;the inside&lt;/b&gt;')
     def inner &p
       Inner[self].map p
     end

data/lib/rsec/parsers/misc.rb

@@ -14,8 +14,6 @@ module Rsec #:nodoc
   class Fail < Binary
     def Fail.[] left, tokens
       # TODO mutex
-      @mask_bit ||= 0
-      @token_table ||= []
       if @mask_bit > 1000
         raise "You've created too many fail parsers, If it is your intention, call Rsec::Fail.reset when previous expect settings can be thrown away."
       end
@@ -29,6 +27,7 @@ module Rsec #:nodoc
       @mask_bit = 0
       @token_table = []
     end
+    Fail.reset
 
     def Fail.get_tokens mask
       res = []
@@ -116,6 +115,7 @@ module Rsec #:nodoc
   end
 
   # should be end-of-file after parsing
+  # FIXME seems parser keeps a state when using parse!, see nasm manual parse
   class Eof < Unary
     def _parse ctx
       ret = some()._parse ctx

data/lib/rsec/utils.rb

@@ -92,6 +92,7 @@ module Rsec #:nodoc:
     # the text is 80 at most
     def line_text pos
       from = string.rindex "\n", pos
+      (from = string.rindex "\n", pos - 1) if from == pos
       from = from ? from + 1 : 0
       from = pos - 40 if (from < pos - 40)
 

data/readme.rdoc

@@ -16,7 +16,7 @@ The pure Ruby gem is fast enough (about 10+x faster than treetop generated code)
 
 For extreme performance under C Ruby:
 
-	gem in rsec-ext
+    gem in rsec-ext
 
 It is about 30% faster than Haskell Parsec in the benchmark.
 

data/test/test_misc.rb

@@ -20,6 +20,10 @@ class TestMisc < TC
     p = ''.r.eof
     asp '', p
     ase INVALID, p.parse('a')
+
+    p = seq('a', 'b').eof
+    ase INVALID, p.parse('abc')
+    ase ['a', 'b'], p.parse('ab')
   end
 
   def test_cache

metadata

@@ -1,24 +1,28 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: rsec
-version: !ruby/object:Gem::Version
-  version: 0.3.2
+version: !ruby/object:Gem::Version 
   prerelease: 
+  version: 0.3.6
 platform: ruby
-authors:
+authors: 
 - NS
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-02-24 00:00:00.000000000 +08:00
+
+date: 2011-03-08 00:00:00 +08:00
 default_executable: 
 dependencies: []
+
 description: Easy and extreme fast dynamic PEG parser combinator.
 email: 
 executables: []
+
 extensions: []
-extra_rdoc_files:
+
+extra_rdoc_files: 
 - readme.rdoc
-files:
+files: 
 - license.txt
 - readme.rdoc
 - lib/rsec/helpers.rb
@@ -33,6 +37,8 @@ files:
 - examples/arithmetic.rb
 - examples/bnf.rb
 - examples/c_minus.rb
+- examples/little_markdown.rb
+- examples/nasm_manual.rb
 - examples/scheme.rb
 - examples/slow_json.rb
 - examples/s_exp.rb
@@ -55,26 +61,30 @@ files:
 has_rdoc: true
 homepage: http://rsec.heroku.com
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
       version: 1.9.1
-required_rubygems_version: !ruby/object:Gem::Requirement
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 1.6.1
 signing_key: 
 specification_version: 3
 summary: Extreme Fast Parser Combinator for Ruby
 test_files: []
+