mucgly 0.0.1

data/lib/mucgly.rb

@@ -0,0 +1,627 @@
+module Mucgly
+
+
+    def Mucgly::error( str )
+        if Opt['warn_only'].given
+            puts "*** Mucgly Error: #{str}"
+        else
+            raise RuntimeError, "*** Mucgly Error: #{str}"
+        end
+    end
+    
+    def Mucgly::debug( str )
+        puts "\nMucglyDebug: #{str}" if Opt['debug'].given
+    end
+    
+
+    # Execution environment for MucglyFile.
+    class Env
+
+        @@fileStartHook = nil
+        @@fileEndHook = nil
+
+        attr_accessor :_separators
+        attr_accessor :_inIO
+        attr_accessor :_outIO
+        
+        def initialize
+            @_outIO = STDOUT
+        end
+
+        def Env.fileStartHook=( val )
+            @@fileStartHook = val
+        end
+
+        def Env.fileEndHook=( val )
+            @@fileEndHook = val
+        end
+
+        def Env.fileStartHook
+            @@fileStartHook
+        end
+
+        def Env.fileEndHook
+            @@fileEndHook
+        end
+
+        def fileStartHook
+            @@fileStartHook
+        end
+
+        def fileEndHook
+            @@fileEndHook
+        end
+
+        def write( str )
+            @_outIO.write str
+        end
+
+        def puts( str )
+            @_outIO.puts str
+        end
+
+        def source( file )
+            instance_eval( File.read( file ), file )
+        end
+
+
+        # ------------------------------------------------------------
+        # Interface methods:
+
+
+        def _processFilePair( filePair )
+            _openInput( filePair[ 0 ] )
+            _openOutput( filePair[ 1 ] )
+            Mucgly::MucglyFile.new( self )
+        end
+
+        def _processFilePairs( filePairs )
+            filePairs.each do |pair|
+                _processFilePair( pair )
+            end
+        end
+
+        def _openInput( filename )
+            @_inIO = EasyFile::ReadStack.new( filename )
+        end
+
+        def _openOutput( filename )
+            @_outIO = EasyFile::WriteStack.new( filename )
+        end
+
+        def _openString( filename )
+            @_outIO = EasyFile::String.open( filename )
+        end
+
+        def _pushInput( name )
+            @_inIO.push( name )
+        end
+
+        def _pushOutput( name )
+            @_outIO.push( name )
+        end
+
+        def _closeInput
+            @_inIO.close
+        end
+
+        def _closeOutput
+            @_outIO.close
+        end
+
+        def _ofilename
+            @_outIO.filename
+        end
+
+        def _olinenumber
+            @_outIO.line
+        end
+
+        def _ifilename
+            @_inIO.filename
+        end
+
+        def _ilinenumber
+            @_inIO.line
+        end
+
+        def _eval( cmd )
+            instance_eval cmd            
+        end
+
+    end
+
+
+    # MucglyFile includes processing and status for a file that is treated
+    # as Mucgly file. Separators are copied from the upper level
+    # context i.e. lower level settings does NOT affect higher level
+    # settings. Status of parsing is kept within the class.
+    class MucglyFile
+        
+        attr_accessor :env, :parseState
+
+        def initialize( env )
+
+            @env = env
+
+            # Process fileStartHook callback:
+            @env._eval( @env.fileStartHook ) if @env.fileStartHook
+            
+            # Copy upper level separators.
+            @env._separators = @env._separators.copy
+
+            # Process the Mucgly file.
+            parse
+        end
+        
+
+        # Input is a stream of characters or control values.
+        #
+        # Supported types: char, hookEnd, hookBeg, escape, eof, and
+        # empty
+        class Token
+            
+            attr_accessor :type, :value
+
+            def initialize( type, value = nil )
+                @type = type
+                @value = value
+            end
+
+            def isPunct
+                !isChar
+            end
+            
+            def isChar
+                @type == :char
+            end
+
+            def to_s
+                "#{@type.to_s}:#{value}"
+            end
+
+        end
+
+
+        # Parse state regarding buffered tokens.
+        class ParseState
+
+            def initialize( host )
+                @host = host
+                @stack = [[]]
+                @level = 0
+                @active = false
+            end
+
+            # Add token.
+            def shift( token )
+                if @stack[-1][-1] && @stack[-1][-1].type == token.type
+                    @stack[-1][-1].value += token.value
+                else
+                    @stack[-1].push token
+                end                    
+            end
+
+            # Push new parse context.
+            def push
+                unless inside?
+                    flush
+                end
+                inc
+                @stack.push []
+            end
+
+            # Pop top parse context.
+            def pop
+                if @active
+                    # Execute.
+                    eval( @stack[-1] )
+                    @stack.pop
+                else
+                    # Output literal.
+                    output( @stack[-1] )
+                    @stack.pop
+                end
+
+                dec
+            end
+
+            # Output shifted tokens.
+            def flush
+                output( @stack[-1] )
+                @stack[-1] = []
+            end
+
+            # Output list of tokens.
+            def output( list )
+                list.each do |t|
+                    @host.env.write t.value
+                end
+            end
+
+            # Inside macro def.
+            def inside?
+                @level > 0
+            end
+
+            # Inc macro level.
+            def inc
+                @active = true
+                @level += 1
+            end
+
+            # Dec macro level.
+            def dec
+                @level -= 1
+                @active = false
+            end
+
+            # Evaluate shifted tokens.
+            def eval( list )
+                idx = 0
+
+                # Skip hookbeg.
+                idx += 1
+                
+                # Check if macro is internal command.
+                if list[idx].value[0] == "."
+
+                    # Variable reference.
+
+                    varname = list[idx].value[1..-1]
+
+                    # Skip hookend.
+                    idx += 2
+
+                    raise RuntimeError, "Garbage after hookend." if list[ idx ]
+
+                    @host.env._eval( "write @#{varname}" )
+
+                elsif list[idx].value[0] == ":"
+
+                    # Command.
+
+                    cmd, rest = list[idx].value[1..-1].scan( /([a-z]+) (.*)/ )[0]
+
+                    # Skip hookend.
+                    idx += 2
+
+                    raise RuntimeError, "Garbage after hookend." if list[ idx ]
+                    
+                    case cmd
+
+                    when 'include'
+                        @host.env._pushInput( rest )
+                        @host.env._inIO.automode = false
+                        @host.parse
+
+                    when 'output'
+                        @host.env._pushOutput( rest )
+
+                    when 'close'
+                        @host.env._closeOutput
+
+                    when 'comment'
+                        # ' mode coloring fix comment.
+                        true
+
+                    when 'source'
+                        @host.env.source( rest )
+
+                    when 'hook'
+                        if rest.split.length == 2
+                            @host.env._separators.hookBegChars = rest.split[0]
+                            @host.env._separators.hookEndChars = rest.split[1]
+                        else
+                            @host.env._separators.hookBegChars = rest
+                            @host.env._separators.hookEndChars = rest
+                        end
+
+                    when 'hookbeg'
+                        @host.env._separators.hookBegChars = rest
+
+                    when 'hookend'
+                        @host.env._separators.hookEndChars = rest
+
+                    when 'escape'
+                        @host.env._separators.escapeChar = rest
+
+                    when 'exit'
+                        exit
+
+                    else
+                        Mucgly::error "unkown internal command: \"#{cmd}\""
+
+                    end
+
+                elsif list[idx].value[0] == "#"
+
+                    # Remove macro cancel char from macro and output
+                    # the macro again.
+                    
+                    @host.env.write( list[idx-1].value )
+                    @host.env.write( list[idx].value[1..-1] )
+                    @host.env.write( list[idx+1].value )
+
+                    # Skip hookend.
+                    idx += 2
+
+                    raise RuntimeError, "Garbage after hookend." if list[ idx ]
+
+                else
+
+                    # Ruby code to evaluate.
+
+                    code = list[idx].value
+
+                    # Skip hookend.
+                    idx += 2
+
+                    raise RuntimeError, "Garbage after hookend." if list[ idx ]
+
+                    @host.env._eval code
+
+                end
+
+            end
+
+
+            # Return char either from character buffer (String) or file stream.
+            def getChars( cnt = 1 )
+                c = @host.env._inIO.read( cnt )
+                Mucgly::debug( "Read char: \"#{c}\"" )
+                c
+            end
+
+            # Put character back.
+            def putBack( c )
+                @host.env._inIO.putback( c )
+                Mucgly::debug( "Putback char: \"#{c}\"" )
+            end
+
+            # Find a given string from input (true/false).
+            def findInInput( this )
+
+                str = getChars( this.length )
+
+                if str == this
+                    # Found requested string from input.
+                    true
+                else
+                    # Mismatch to requested string, put back to input.
+                    putBack( str )
+                    false
+                end
+            end
+
+
+            # Create a token based on the current characters. The
+            # token returned is effected by parsing context and
+            # separator settings.
+            def getTokenRaw
+
+                escapeChar = @host.env._separators.escapeChar
+                hookEndChars = @host.env._separators.hookEndChars
+                hookBegChars = @host.env._separators.hookBegChars
+
+
+                c = getChars
+
+                if c == nil
+
+                    Token.new( :eof )
+
+                else
+
+                    # Handle escape characters before hooks.
+                    if c == escapeChar
+
+                        c = getChars
+
+                        if c == "\n"
+                            # Escaped newline.
+                            return Token.new( :empty, "" )
+
+                        elsif c == escapeChar
+                            # Escaped escape.
+                            return Token.new( :char, escapeChar )
+
+                        elsif inside?
+
+                            # Escape inside first level macro.
+
+                            if ( c == " " || c == "\n" ) and
+                                    ( escapeChar ==
+                                      hookEndChars )
+
+                                # Hookend when same as escape is
+                                # "hookend+space".
+                                return Token.new( :hookEnd, escapeChar + c )
+
+                            elsif c == hookEndChars[0]
+
+                                # Escaped hookend.
+                                return Token.new( :char, c )
+
+                            else
+
+                                # Ineffective escape.
+                                return Token.new( :char,
+                                                  escapeChar + c )
+                            end
+
+                        else
+
+                            # Escape outside macro.
+
+                            if escapeChar == hookEndChars
+
+                                putBack( c )
+                                return Token.new( :hookBeg,
+                                                  hookBegChars )
+
+                            elsif c == hookEndChars[0]
+
+                                return Token.new( :char, c )
+
+                            elsif ( c == " " || c == "\n" )
+
+                                # Remove escaped space and newlines.
+                                return Token.new( :empty )
+
+                            elsif c == hookBegChars[0]
+
+                                return Token.new( :char, c )
+
+                            else
+
+                                # Ineffective escape.
+                                return Token.new( :char,
+                                                  escapeChar + c )
+
+                            end
+
+                        end
+
+                    else
+                        
+                        # No escape, putback and search for hooks.
+                        putBack c
+
+                    end
+
+
+                    # Look for hooks or return a character.
+                    if inside?
+
+                        # Inside macro.
+
+                        if findInInput( hookEndChars )
+                            return Token.new( :hookEnd, hookEndChars )
+                        end
+
+                    else
+
+                        # Outside macro.
+
+                        if findInInput( hookBegChars )
+                            return Token.new( :hookBeg, hookBegChars )
+                        end
+
+                    end
+
+
+                    # No escapes, no hooks, so return a char.
+                    c = getChars
+
+                    if c == nil
+                        return Token.new( :eof )
+                    else
+                        return Token.new( :char, c )
+                    end
+
+                end
+            end
+
+
+            def getToken
+                t = getTokenRaw
+                Mucgly::debug( "Got token: #{t.type}:\"#{t.value}\"" )
+                t
+            end
+
+        end
+
+
+        # Process a Mucgly file.
+        def parse
+
+            parseState = ParseState.new( self )
+
+            while true
+
+                t = parseState.getToken
+
+                if t.type == :eof
+
+                    parseState.flush
+
+                    # Process fileEndHook callback:
+                    @env._eval( @env.fileEndHook ) if @env.fileEndHook
+
+                    # End of stream.
+                    # @fp.close
+
+                    @env._inIO.close
+
+                    return
+
+                elsif t.type == :hookBeg
+
+                    parseState.push
+                    parseState.shift( t )
+
+                elsif t.type == :hookEnd
+
+                    parseState.shift( t )
+                    parseState.pop
+
+                else
+
+                    parseState.shift( t )
+
+                end
+
+                
+            end
+
+        end
+                    
+    end
+
+
+
+    # Separators are collection of special characters used by Mucgly.
+    #
+    # These are:
+    # [escape] Escape char is used to convert special chars into their
+    #          literal form or to remove the following char. Special
+    #          characters are: escape char, hookbeg, hookend. Escape
+    #          can remove the following newline, and space in special
+    #          circumtances. 
+    # [hookbeg] Hookbeg char (or string) starts macros. If user wants
+    #           a literal hookbeg, then <escape> should be put before
+    #           it.
+    # [hookend] Hookend char (or string) terminates macros. Hookend
+    #           can also be escaped. If hookend end is same character
+    #           as escape, then hookend followed by a space is
+    #           considered as hookend.
+    class Separators
+        
+        # Set of control characters for hook and escape indentification.
+        attr_reader :escapeChar
+        attr_accessor :hookBegChars, :hookEndChars
+        
+        def initialize
+            @escapeChar = "\\"
+            @hookBegChars = "-<"
+            @hookEndChars = ">-"
+        end
+        
+        def escapeChar=( value )
+            if value.length != 1
+                Mucgly::error( "Escape must be 1 char long, got \"#{value}\"" )
+            end
+            @escapeChar = value
+        end
+
+        def copy
+            c = Separators.new
+            c.escapeChar = @escapeChar
+            c.hookBegChars = @hookBegChars
+            c.hookEndChars = @hookEndChars
+            return c
+        end
+    end
+
+end

data/test/test_basic.rx.txt

@@ -0,0 +1,19 @@
+Line 1 from file "-<write _ifilename>-".
+-<:hook \\>-\
+Test escapes (backslash, empty): "\\" "\ \ " \
+ ... and the rest of the line. 
+Next the test file "test_include.txt" is included.
+\:comment :include test_include.txt\ \
+\:include test_include.txt\ \
+ This is again in file "\write _ifilename\ ".
+Next line should be three times "bar" in a row.
+\@foo="bar";\ \.foo\ \write @foo\ \.foo\ 
+A line of text.
+\puts "Just some text from Mucgly."\ \
+Source "test_include.rb" and instantiate the class defined in there.
+\:source test_include.rb\ \@obj = MyClass.new\ \
+Print out a sum using the previously instantiated class.
+\puts "Sum of 1+2 is: #{@obj.sum(1,2)}..."\ \
+The last line in "\write _ifilename\ "
+The name of this file is: "\write _ifilename\ ".
+We are outputting to file \write _ofilename\  and \write _olinenumber.to_s\ 

data/test/test_include.rb

@@ -0,0 +1,6 @@
+# Normal Ruby code.
+class MyClass
+    def sum( a, b )
+        a + b
+    end
+end

data/test/test_include.txt

@@ -0,0 +1,3 @@
+Line 1 in test file test_included.txt.
+Line 2 in test file test_included.txt.
+...which is not terminated with a NL.

data/test/test_mucgly.rb

@@ -0,0 +1,32 @@
+require 'test/unit'
+
+
+# Test execution routine.
+def runTest( cmdopts, test, sep = "" )
+    Dir.chdir( 'test' )
+    FileUtils.mkdir_p "result"
+    rf = "result/#{test}#{sep}.txt"
+    gf = "golden/#{test}#{sep}.txt"
+
+    system( "export RUBYLIB=#{ENV['RUBYLIB']}:../lib; ../bin/mucgly #{cmdopts} -i #{test}.rx.txt -o #{rf}" )
+
+    if false
+        # Populate golden files after inspection.
+        system( "cp #{rf} #{gf}" )
+    end
+
+    assert( system( "diff #{rf} #{gf}" ), "FAILED: diff #{rf} #{gf}" )
+
+    Dir.chdir( '..' )
+end
+
+
+class MucglyTest < Test::Unit::TestCase
+
+    def test_basic() runTest( "", "test_basic" ); end 
+    def test_specials_cli1() runTest( "-s @ -e /", "test_specials_cli", "_1" ); end 
+    def test_specials_cli2() runTest( "-sb @ -se @ -e /", "test_specials_cli", "_2" ); end 
+    def test_specials_cmd() runTest( "", "test_specials_cmd" ); end 
+    def test_multi() runTest( "-m", "test_multi" ); end 
+
+end

data/test/test_multi.rx.txt

@@ -0,0 +1,4 @@
+-<@counter = 0>-\
+-<write "Counter value is #{@counter} at round 1">-
+-<#@counter += 1>-\
+-<#write  "Counter value is #{@counter} at round 2">-

data/test/test_specials_cli.rx.txt

@@ -0,0 +1,11 @@
+Line @write (_ilinenumber+1).to_s@ from file "@write _ifilename@".
+Test escapes (backslash, empty): "//" "@ @" /
+ ... and the rest of the line. 
+Next the test file "test_include.txt" is included.
+@:include test_include.txt@/
+ This is again in file "@write _ifilename@ ".
+Next line should be three times "bar" in a row.
+@/@foo="bar"@@.foo@@.foo@@puts /@foo@/
+A line of text.
+@puts  "Just some text from Mucgly."@/
+The last line in "@write _ifilename@"