deplate 0.8 → 0.8.1

data/lib/deplate/guesslanguage.rb

@@ -0,0 +1,57 @@
+# guesslanguage.rb
+# @Author:      Thomas Link (micathom AT gmail com)
+# @License:     GPL (see http://www.gnu.org/licenses/gpl.txt)
+# @Created:     2006-12-29.
+# @Last Change: 2007-01-06.
+# @Revision:    0.1.22
+
+require 'zlib'
+
+# This is ported form/based on:
+# - Title: Guess language of text using ZIP
+# - Submitter: Dirk Holtwick
+# - Last Updated: 2004/12/07
+# - Version no: 1.2
+# - Category: Algorithms 
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/355807
+# http://www.heise.de/newsticker/data/wst-28.01.02-003/
+# http://xxx.uni-augsburg.de/format/cond-mat/0108530
+class Guesslanguage
+    def initialize
+        @data = []
+    end
+
+    def zip(text)
+        Zlib::Deflate.new.deflate(text, Zlib::FINISH)
+    end
+
+    # register a text as corpus for a language or author.
+    # <name> may also be a function or whatever you need
+    # to handle the result.
+    def register(name, corpus)
+        ziplen = zip(corpus).size
+        @data << [name, corpus, ziplen]
+    end
+
+    # <part> is a text that will be compared with the registered
+    # corpora and the function will return what you defined as
+    # <name> in the registration process.
+    def guess_with_diff(part)
+        what = nil
+        diff = nil
+        for name, corpus, ziplen in @data
+            nz = zip(corpus + part).size - ziplen
+            if diff.nil? or nz < diff
+                what = name
+                diff = nz
+            end
+        end
+        return [diff.to_f/part.size, what]
+    end
+
+    def guess(part)
+        diff, lang = guess_with_diff(part)
+        lang
+    end
+end
+

data/lib/deplate/input.rb

@@ -1,10 +1,10 @@
 # input.rb
-# @Author:      Thomas Link (samul AT web.de)
+# @Author:      Thomas Link (micathom AT gmail com)
 # @Website:     http://deplate.sf.net/
 # @License:     GPL (see http://www.gnu.org/licenses/gpl.txt)
 # @Created:     10-M�r-2005.
-# @Last Change: 07-Jun-2006.
-# @Revision:    0.467
+# @Last Change: 2007-09-30.
+# @Revision:    0.809
 #
 # = Description
 # = Usage
@@ -34,32 +34,87 @@ class Deplate::Input < Deplate::CommonObject
         @deplate  = deplate
         @options  = deplate.options
         @args     = args
-        @elements = args[:elements] || Deplate::Element.elements
-        @commands = args[:commands] || Deplate::Command.commands
-        @regions  = args[:regions ] || Deplate::Region.regions
-        @macros   = args[:macros]   || Deplate::Macro.macros
-        @paragraph_class = args[:paragraph_class] || Deplate::Element::Paragraph
-        @comment_class   = args[:comment_class]   || Deplate::Element::Comment
-        @command_class   = args[:command_class]   || Deplate::Element::Command
-        @allow_onthefly_particles = args[:onthefly_particles] || true
+        @elements = select_value(args[:elements], Deplate::Element.elements)
+        @commands = select_value(args[:commands], Deplate::Command.commands)
+        @regions  = select_value(args[:regions ], Deplate::Region.regions)
+        @macros   = select_value(args[:macros], Deplate::Macro.macros)
+        @paragraph_class = select_value(args[:paragraph_class], Deplate::Element::Paragraph)
+        @comment_class   = select_value(args[:comment_class], Deplate::Element::Comment)
+        @command_class   = select_value(args[:command_class], Deplate::Element::Command)
+        @allow_onthefly_particles = select_value(args[:onthefly_particles], true)
         if @options.skeletons.empty?
             @skeleton_expander = nil
         else
-            expander = args[:expander_class] || Deplate::SkeletonExpander
+            expander = select_value(args[:expander_class], Deplate::SkeletonExpander)
             @skeleton_expander = expander.new(deplate)
         end
         initialize_particles
     end
-    
+   
+    def select_value(arg, default)
+        case arg
+        when :default
+            default
+        # when Array
+        #     p "DBG", arg
+        #     # arg.collect {|x| x == :default ? default.dup : x}.flatten
+        #     rv = arg.collect {|x| x == :default ? [] : x}.flatten
+        #     p "DBG", rv
+        #     rv
+        else
+            arg || default
+        end
+    end
+    private :select_value
+
+    def args_fill_with_default(args)
+        # args[:elements]           = nil
+        # args[:particles]          = nil
+        # args[:particles_ext]      = nil
+        # args[:commands]           = nil
+        # args[:regions]            = nil
+        # args[:macros]             = nil
+        # args[:onthefly_particles] = nil
+        # args[:paragraph_class]    = nil
+        # args[:comment_class]      = nil
+        # args[:command_class]      = nil
+
+        args[:elements]           = :default
+        args[:particles]          = nil
+        args[:particles_ext]      = nil
+        args[:commands]           = :default
+        args[:regions]            = :default
+        args[:macros]             = :default
+        args[:onthefly_particles] = :default
+        args[:paragraph_class]    = :default
+        args[:comment_class]      = :default
+        args[:command_class]      = :default
+
+        # args[:elements]           = Deplate::Element.elements
+        # args[:particles]          = Deplate::Particle.particles.dup
+        # args[:particles_ext]      = Deplate::Particle.particles_ext.dup
+        # args[:commands]           = Deplate::Command.commands
+        # args[:regions]            = Deplate::Region.regions
+        # args[:macros]             = Deplate::Macro.macros
+        # args[:onthefly_particles] = true
+        # args[:paragraph_class]    = Deplate::Element::Paragraph
+        # args[:comment_class]      = Deplate::Element::Comment
+        # args[:command_class]      = Deplate::Element::Command
+        # args[:expander_class]     = Deplate::SkeletonExpander
+    end
+
     def initialize_particles(alt=false, args=@args)
         always = args[:always]
         if always or @allow_onthefly_particles or !@rx_particles
-            @particles_custom ||= args[:particles]
+            # TBD: What was these _custom variables good for?
+            @particles_custom ||= select_value(args[:particles], Deplate::Particle.particles)
             if @particles_custom
-                @particles = args[:particles].dup
+                # @particles = args[:particles].dup
+                @particles = @particles_custom.dup
             else
                 @particles = Deplate::Particle.particles.dup
             end
+            # @particles = select_value(@particles_custom, Deplate::Particle.particles).dup
             for p in @options.disabled_particles
                 disable_particle_class(p)
             end
@@ -67,12 +122,14 @@ class Deplate::Input < Deplate::CommonObject
         end
         
         if always or !defined?(@particles_ext) or (@allow_onthefly_particles and alt)
-            @particles_ext_custom ||= args[:particles_ext]
+            @particles_ext_custom ||= select_value(args[:particles_ext], Deplate::Particle.particles_ext)
             if @particles_ext_custom
-                @particles_ext = args[:particles_ext].dup
+                # @particles_ext = args[:particles_ext].dup
+                @particles_ext = @particles_ext_custom.dup
             else
                 @particles_ext = Deplate::Particle.particles_ext.dup
             end
+            # @particles = select_value(@particles_ext_custom, Deplate::Particle.particles_ext).dup
             initialize_rx(:extended)
         else
             @particles_all    = @particles
@@ -126,8 +183,10 @@ class Deplate::Input < Deplate::CommonObject
     def register_particle(particle, args={})
         arr = args[:extended] ? @particles_ext : @particles
         id  = args[:id] || particle.name
+        rpl = args[:replace]
+        oid = rpl ? rpl.name : id
         catch(:cont) do
-            old = @@custom_particles[id]
+            old = @@custom_particles[oid]
             if old
                 idx = arr.index(old)
                 if idx
@@ -135,12 +194,46 @@ class Deplate::Input < Deplate::CommonObject
                     throw :cont
                 end
             end
-            arr << particle
+            if rpl and (rpli = arr.index(rpl))
+                arr[rpli] = particle
+            elsif args[:unshift]
+                arr.unshift(particle)
+            else
+                arr << particle
+            end
         end
         @@custom_particles[id] = particle
         initialize_rx(:both)
     end
     
+    def remove_named_elements(hash, field, default, elements)
+        arr = hash[field] ||= default
+        for e in elements
+            case e
+            when String
+                arr.delete(e)
+            else
+                i = arr.index(e)
+                arr.delete(i) if i
+            end
+        end
+    end
+
+    def replace_named_elements(hash, field, default, elements)
+        arr = hash[field] ||= default
+        arr.collect! do |elt|
+            elements.each do |from, to|
+                case from
+                when String
+                    elt.myname == from ? to : elt
+                else
+                    elt == from ? to : elt
+                end
+            end
+        end
+        arr
+    end
+
     def get_rx_particles(particles)
         return Regexp.new(particles.collect do |e|
             e.rx.source[1..-1]
@@ -181,7 +274,7 @@ class Deplate::Input < Deplate::CommonObject
             unless accum.empty?
                 line.gsub!(/^\s+/, '')
             end
-            if !unfinished_region?(acc_array.last) and comment_class.match(line)
+            if !unfinished_region?(acc_array.last) and (comment_class.match(line) and !comment_class.show_comment?(@deplate, line))
                 next
             else
                 use_line_continuation = acc_array.last ? acc_array.last.line_cont : true
@@ -198,6 +291,7 @@ class Deplate::Input < Deplate::CommonObject
     end
     
     def handle_line(array, line, lineNumber=-1)
+        # p "DBG line=#{line.inspect}"
         last = array.last
         if line
             src = Deplate::Source.new(@deplate.current_source, @deplate.current_source_stats,
@@ -247,12 +341,14 @@ class Deplate::Input < Deplate::CommonObject
                 e, m = match_elements(line)
                 if last
                     # so, there is a element in the queue
-                    if e == comment_class
-                        c = nil
-                    elsif last.finished?
+                    # if e == comment_class
+                    #     # and !e.show_comment?(@deplate, line)
+                    #     # c = nil
+                    # elsif last.finished?
+                    if last.finished?
                         # last is finished, so we start a new element
                         c = e || paragraph_class
-                    elsif last.to_be_continued?(line, e, m)
+                    elsif last.to_be_continued?(line, e, m) and (e != comment_class or last.is_a?(comment_class))
                         # this line is something else, so we ask last if it wants 
                         # to be continued
                         case last.multiliner
@@ -285,8 +381,10 @@ class Deplate::Input < Deplate::CommonObject
                     c.do_accumulate(src, array, @deplate, line, m)
                 end
             end
-        elsif last and !last.finished?
-            last = finish_last(array, last, lineNumber)
+        else
+            if last and !last.finished?
+                last = finish_last(array, last, lineNumber)
+            end
         end
     end
     private :handle_line
@@ -294,11 +392,13 @@ class Deplate::Input < Deplate::CommonObject
     def unfinished_region?(last)
         return last && !last.finished? && last.endRx
     end
-                   
+
     def finish_last(array, last, lineNumber=-1)
-        last.source.end   = lineNumber if lineNumber > 0
+        last.source.end  = lineNumber if lineNumber > 0
         # finished_element  = last.finish
-        finished_element = array.pop.finish
+        finished_element = array.pop
+        finished_element = finished_element.finish
+        # finished_element = array.pop.finish
         if finished_element.kind_of?(Array)
             for e in finished_element
                 handle_finished_element(array, e)
@@ -311,14 +411,21 @@ class Deplate::Input < Deplate::CommonObject
     private :finish_last
 
     def handle_finished_element(array, element)
-        unless element.nil? or element.drop?
+        filter = @deplate.variables['efilter']
+        unless element.nil? or element.drop?(filter)
             pred = array.last
-            unless pred and pred.unify(element)
+            if pred and pred.unify(element)
+                if pred.exclude?(filter)
+                    pred.pop(array)
+                end
+            else
                 if !@deplate.labels_floating.empty? and element.can_be_labelled
                     element.put_label(@deplate.labels_floating)
                     @deplate.labels_floating = []
                 end
-                array << element
+                unless element.exclude?(filter)
+                    array << element
+                end
             end
         end
     end
@@ -336,7 +443,12 @@ class Deplate::Input < Deplate::CommonObject
                 return [e, m]
             end
         end
-        return [nil, @paragraph_class.match(line)]
+        m = comment_class.match(line)
+        if m
+            return [comment_class, m]
+        else
+            return [nil, @paragraph_class.match(line)]
+        end
     end
     private :match_elements
 
@@ -356,10 +468,10 @@ class Deplate::Input < Deplate::CommonObject
     #
     # Anyway, you should call this method directly unless you have to, but use 
     # the more specialized ones.
-    def parse_using(container, text, rx, particles, alt=true)
+    def parse_using(container, text, rx, particles, alt=true, args={})
         rest = text
         rt   = []
-        last = ""
+        last = ''
         src  = container ? container.source : nil
         while !rest.empty?
             begin
@@ -370,7 +482,7 @@ class Deplate::Input < Deplate::CommonObject
                 else
                     sample = rest
                 end
-                log_error(container, ["Regexp error when parsing", sample], :error, src)
+                log_error(container, ['Regexp error when parsing', sample], :error, src)
             rescue Exception => e
                 puts e.backtrace[0..10].join("\n")
                 raise e
@@ -392,6 +504,7 @@ class Deplate::Input < Deplate::CommonObject
                                     rt << e.new(@deplate, container, rt, m, alt, txt, rest)
                                 rescue Exception => exc
                                     puts exc
+                                    # puts exc.backtrace[0..10].join("\n")
                                     # puts "#{exc}\nInternal error when initializing %s: %s" % [e.name, txt]
                                     # raise exc
                                 end
@@ -408,6 +521,12 @@ class Deplate::Input < Deplate::CommonObject
                 break
             end
         end
+        if (filter = @deplate.variables['pfilter'])
+            # c = container.kind_of?(Deplate::BaseParticle) ? container : nil
+            rt.delete_if do |p|
+                p.exclude?(filter, args[:pcontainer])
+            end
+        end
         return rt
     end
 
@@ -428,7 +547,8 @@ class Deplate::Input < Deplate::CommonObject
         parse(container, text, alt, excluded)
     end
 
-    def parse(container, text, alt=true, excluded=[])
+    def parse(container, text, alt=true, excluded=nil, args={})
+        excluded ||= []
         rx, particles = get_particles(:both, alt)
         for p in excluded
             case p
@@ -438,7 +558,7 @@ class Deplate::Input < Deplate::CommonObject
                 particles.delete(p)
             end
         end
-        parse_using(container, text, rx, particles, alt)
+        parse_using(container, text, rx, particles, alt, args)
     end
 
     ### Arguments
@@ -447,13 +567,32 @@ class Deplate::Input < Deplate::CommonObject
     # @@rxsrc_argval       = %{("(\\\\"|[^"])*?"|(\\\\=|\\\\:|\\\\!|[^=!]+)+?)}
     # @@rxsrc_argval       = %{(\\\\=|\\\\:|[^=:]*)+?}
     # @@rxsrc_argval = %{("(\\\\"|[^"])*?"|(\\\\ |\\\\=|\\\\:|\\\\!|[%s]+)+?)}
-    @@rxsrc_argval = %{("(\\\\"|[^"])*"|\\([^)]+\\)|\\\\.|[%s"]+)+?}
     # @@rxsrc_argval = %{("(\\\\"|[^"])*"|\\\\.|[%s"]+)+?}
-    @@rxsrc_key    = %{[@\\w]+(\\[\\S*?\\])?}
-    @@rxsrc_args1  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "^=!:"
-    @@rxsrc_args2  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "^=!"
+    @@rxsrc_argval = %{("(\\\\"|[^"])*"|\\([^)]+\\)|\\\\.|[^%s"]+)+?}
+
+    # @@rxsrc_key    = %{[$@\\w]+(\\[\\S*?\\])?}
+    # @@rxsrc_args1  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "=!:"
+    # @@rxsrc_args2  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "=!"
+    # @@rx_args      = /^\s*(#{@@rxsrc_args2})(?=(\s*$|\s+#{@@rxsrc_key}(!|=)))/
+    # @@rx_argstext  = /^\s*((#{@@rxsrc_args1})*?)(:\s*(.+))?$/
+    # @@ri_text = 9
+
+    @@rxsrc_key    = %{[$@\\w]+(\\[\\S*?\\])?}
+    # @@rxsrc_args1  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "=!:"
+    # @@rxsrc_args2  = %{(#{@@rxsrc_key})(!|=(#{@@rxsrc_argval}))\\s*} % "=!"
+    @@rxsrc_args1  = %{(#{@@rxsrc_key})((?>!)|(?>=)(#{@@rxsrc_argval}))\\s*} % "=!:"
+    @@rxsrc_args2  = %{(#{@@rxsrc_key})((?>!)|(?>=)(#{@@rxsrc_argval}))\\s*} % "=!"
     @@rx_args      = /^\s*(#{@@rxsrc_args2})(?=(\s*$|\s+#{@@rxsrc_key}(!|=)))/
     @@rx_argstext  = /^\s*((#{@@rxsrc_args1})*?)(:\s*(.+))?$/
+    @@ri_text = 9
+
+    # @@rxsrc_key    = %{\\s+([$@[:alnum:]_]+(\\[[^\\]]*\\])?)}
+    # # @@rxsrc_key    = %{\\s+([$@[:alnum:]_]+(\\[[^\\]]*\\])?)}
+    # @@rxsrc_args1  = %{#{@@rxsrc_key}(!|=(#{@@rxsrc_argval}))} % "=!:"
+    # @@rxsrc_args2  = %{#{@@rxsrc_key}(!|=(#{@@rxsrc_argval}))} % "=!"
+    # @@rx_args      = /^(#{@@rxsrc_args2})(?=(\s*$|#{@@rxsrc_key}(!|=)))/
+    # @@rx_argstext  = /^((#{@@rxsrc_args1})*?)\s*(:\s*(.+))?$/
+    # @@ri_text = 9
 
     def parse_args(argText, container=nil, firstPass=true, parseText=false)
         unless argText
@@ -462,28 +601,44 @@ class Deplate::Input < Deplate::CommonObject
         argText.strip!
         if argText.empty?
             return [{}, '']
+        else
+            # argText.insert(0, ' ')
         end
+        # p "DBG 0 #{argText.inspect}"
         if firstPass
-            m  = @@rx_argstext.match(argText)
+            m = @@rx_argstext.match(argText)
+            m &&= m.captures
         else
-            m = [nil, argText]
+            m = [argText]
         end
         accum = {}
         if m
-            args = m[1]
-            text = m[10]
-            while !args.empty?
+            args = m[0]
+            # args.insert(0, ' ')
+            text = m[@@ri_text]
+            # p "DBG 1.1 m=#{m.inspect}"
+            # p "DBG 1.2 args=#{args.inspect} => #{(args =~ /\S/).inspect}"
+            # p "DBG 1.3 text=#{text.inspect}"
+            while args =~ /\S/
+                if args =~ /^[^=[:space:]]+=[^"([:space:]]*?[^\\"(]([!=])/
+                    log_error(container, ['Argument parse error', $1, args])
+                    return [{}, '']
+                end
+                # p "DBG 2.1 #@@rx_args"
+                # p "DBG 2.2 #{args.inspect}"
                 m = @@rx_args.match(args)
+                # p "DBG 3"
                 if m
                     key = m[2]
                     val = m[5]
-                    Deplate::Core.canonic_args(accum, key, val)
+                    # p "DBG #{key.inspect}=#{val.inspect}"
+                    Deplate::Core.canonic_args(accum, key, val, container && container.source)
                     args = m.post_match
                     if args.empty?
                         break
                     end
                 else
-                    log_error(@container, ['Argument parse error', args, argText])
+                    log_error(container, ['Argument parse error', args, argText])
                     break
                 end
             end
@@ -503,7 +658,7 @@ class Deplate::Input < Deplate::CommonObject
             accum['@id'] = id
             accum['id']  = @deplate.formatter.encode_id(id)
         end
-        fmt   = accum['fmt']
+        fmt = accum['fmt']
         accum.delete('fmt') if fmt
         ifOpt0 = ifOpt = accum['if']
         if ifOpt