rbpar 0.1.0

data/README

@@ -0,0 +1,58 @@
+*** RBPAR README ***
+
+
+INTRODUCTION
+
+rbpar is a program and an accompanying library suite meant 
+for formatting text paragraphs. The library includes an 
+implementation of the optimal line breaking algorithm by 
+Donald E. Knuth and a semi-intelligent paragraph parser that 
+is not confused by email quotes. This is due to the fact 
+that rbpar was first designed for email writing. 
+
+The program executable 'rbpar.rb' is installed to the 
+executable path by default. The program read input from 
+STDIN and outputs formatted paragraphs to STDOUT. 
+
+
+INSTALLATION
+
+If you received rbpar gem, you can install the gem with 
+"gem install rbpar-0.1.0.gem". An easier alternative is to 
+use rubygems to download and install it directly with "gem 
+install rbpar". The file rbpar.rb should be automaticly 
+added to your path. 
+
+
+USAGE
+
+    rbpar.rb [-v] [-w <number>]
+
+rbpar.rb accepts the following parameters:
+
+    -v, --vim :     Use vim-style line endings (with an ending space)
+    -w, --width:    Set the desired line width
+
+Example usage to format a text file:
+
+    cat unprocessed.txt | rbpar.rb > processed.txt
+
+
+VIM INTEGRATION
+
+To make vim 'gq' operator use rbpar, use the following 
+command either in vim command menu or in your vimrc file: 
+
+set formatprg=rbpar.rb\ -w\ 63\ -v
+
+This sets the desired line width to be 62 characters. 
+One extra space is inserted after every line that does 
+not end a paragraph. This enables vim (with format-option 
+'w') to understand the paragraph lines. This is handy when 
+using the 'auto-format' format option. See vim help for 
+"formatoptions", "fo-table" and "auto-format" for details. 
+
+
+CONTACT
+
+Please email comments and suggestions to <ismo@iki.fi>.

data/bin/rbpar.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/ruby
+
+# The command line utility for using the rbpar library.
+#
+# Copyright Ismo Puustinen 2007.
+
+require 'rbpar_main'
+require 'getoptlong'
+
+# This is the main program for the rbpar paragraph processing library. 
+# The idea is to use the program as a stdin->stdout processor and pipe
+# the input data to it. For example, this would work:
+#
+#  $ cat inputfile.txt | ruby rbpar.rb -w 63 > outputfile.txt
+#
+# Thus, the basic usage is identical to 'fmt' and 'par'.
+
+def main
+
+    # get the options from the command line
+
+    parser = GetoptLong.new()
+    parser.set_options(
+        ["-h", "--help", GetoptLong::NO_ARGUMENT],
+        ["-w", "--width", GetoptLong::OPTIONAL_ARGUMENT],
+        ["-v", "--vim", GetoptLong::NO_ARGUMENT]
+    )
+
+    # --vim means that an extra space should be left after each
+    # non-empty line to signify that the next line is also part of the
+    # same paragraph
+
+    # set default values
+    width = 68
+    vim = false
+
+    loop do
+        begin
+            opt, arg = parser.get
+            break if not opt
+
+            case opt
+                when "-h"
+                    puts "Usage:\n\trbpar.rb [-w width] [-v] [-h]"
+                    exit(0)
+                when "-w"
+                    width = arg.to_i
+                when "-v"
+                    vim = true
+            end
+        end
+    end
+
+    # reserve room for the extra space
+    width -= 1 if vim
+
+    # check the parameters
+    unless width > 0
+        error_text = "Negative or zero paragraph width!"
+        error_text += " (Note: using the -v option causes the resevation of one character)" if vim
+        raise ArgumentError, error_text
+    end
+
+    rbpar = RbParIterator.new(width)
+    first = true
+
+    # add lines to the parser object ...
+    rbpar << $stdin.readlines
+
+    # ... and read back ready paragraphs
+    rbpar.each do |paragraph|
+        # add a line break before each paragraph (except the first)
+        unless first
+            #$stdout.write("\n") unless first
+        else
+            first = false
+        end
+        # write the lines to stdout
+        paragraph.each do |line|
+            $stdout.write(line.rstrip)
+            $stdout.write(" ") if vim
+            $stdout.write("\n")
+        end
+    end
+
+end
+
+# run the program!
+
+# this if-statement is for non-gem testing
+#if __FILE__ == $PROGRAM_NAME
+    main
+#end
+

data/lib/rbpar_engine.rb

@@ -0,0 +1,196 @@
+#!/usr/bin/ruby
+
+# Implementation of the optimal line breaking algorithm.
+#
+# Copyright Ismo Puustinen 2007.
+
+class ResultHash < Hash
+
+    END_VALUE = "end"
+   
+    def initialize(words)
+        super(END_VALUE)
+        @words = words
+    end
+
+    def each()
+
+        # each successive call to each yields another line
+
+        i = 0
+        while (nextIndex = self[i.to_s]) != END_VALUE
+            yield @words[i...nextIndex].join(" ")
+            i = nextIndex
+        end
+        # also the last line
+        yield @words[i...@words.length].join(" ")
+    end
+end
+
+class DynamicBreaker
+
+    # DynamicBreaker implements Donald Knuth's algorithm for allocating
+    # words to lines in optimal way. The algorithm uses dynamic programming
+    # to accomplish this in reasonable time.
+
+    def initialize
+        @charLengthCache = Hash.new
+        @cost = Hash.new(0)
+    end
+    
+    def charLength(words, beginIndex, endIndex)
+
+        # see if the result is already cached
+
+        token = beginIndex.to_s+ "_" + endIndex.to_s
+        cachedResult = @charLengthCache[token]
+        if cachedResult.nil?
+            length = realCharLength(words, beginIndex, endIndex)
+            return @charLengthCache[token] = length
+        end
+        return cachedResult
+    end
+
+    def realCharLength(words, beginIndex, endIndex)
+
+        # sum the lenghts of the words and add 1 for the space after the
+        # word
+
+        return (beginIndex...endIndex).inject(0) do |sum, i|
+            sum + words[i].length + 1
+        end - 1 # remove the last space
+    end
+
+    def calculatePenalty(words, availableLength, beginIndex, endIndex)
+
+        # get the line length
+        
+        actualLength = charLength(words, beginIndex, endIndex)
+
+        if availableLength < actualLength
+            print "Error: too long line"
+        end
+        
+        # The optimum value should be bit less than the line width
+
+        optimumLength = availableLength
+
+        if availableLength > 20
+            optimumLength = (availableLength*0.95).to_i
+        end
+
+        # penalty is the whitespace length at the end of the line to 
+        # second power
+        
+        lineScore = (optimumLength - actualLength)**2 + @cost[endIndex.to_s]
+    end
+
+    def findBreak(words, availableLength, beginIndex)
+
+        # find the best way from here to the end of the input string
+
+        first = true
+        smallestPenalty = 0
+        smallestPenaltyIndex = beginIndex + 1
+        
+        ((beginIndex+1)...words.length).each do |i|
+            unless isLegal(words, availableLength, beginIndex, i)
+                # the next word is no longer in the line length range
+                if i == beginIndex + 1
+                    # The first word is too long to fit on one line. 
+                    # This is a special case, and means that the word 
+                    # should not be part of any route. This is why 
+                    # we set the penalty to be maximum (so that the 
+                    # previous word would be the preferred endpoint 
+                    # for a line) and the smallestPenaltyIndex to be 
+                    # the next word (so that a new line would begin 
+                    # there). 
+                    penalty = 100000000 # FIXME: infinite (MAXINT)
+                end
+                break
+            end
+            
+            if first
+                first = false
+                smallestPenalty = calculatePenalty(words, availableLength, beginIndex, i)
+                smallestPenaltyIndex = i
+            else
+                penalty = calculatePenalty(words, availableLength, beginIndex, i)
+                if penalty < smallestPenalty
+                    smallestPenalty = penalty
+                    smallestPenaltyIndex = i
+                end
+            end
+        end
+
+        return smallestPenaltyIndex, smallestPenalty
+    end
+            
+    def isLegal(words, availableLength, beginIndex, endIndex)
+        charLength(words, beginIndex, endIndex) <= availableLength
+    end
+    
+    def parse(words, availableLength)
+        
+        # check the parameters, since this is the public entrypoint to
+        # the algorithm
+        
+        if words.nil? or availableLength <= 0
+            if words.nil?
+                puts "Nil words!"
+            else
+                puts "available length <= 0"
+            end
+            raise ArgumentError, "Incorrect parameters to parse function!"
+        elsif words.empty?
+            # no words
+            return []
+        elsif words.length == 1
+            # only one word
+            return words
+        end
+        
+        n = words.length
+
+        # one initialization is enough
+        @charLengthCache.clear
+        @cost.clear
+
+        # initialize with the words. Note: not as the default value
+        nextBreak = ResultHash.new(words)
+
+        # index n-1 is the last word, always on last line
+        (n-2).downto(0) do |i|
+            unless charLength(words, i, n) <= availableLength
+                # normal case, not the last line
+
+                # The last line needs to have cost to next break as 0
+                # and the next break as end state. This is done with
+                # implicit Hash default values.
+
+                r, penalty = findBreak(words, availableLength, i)
+                @cost[i.to_s] = @cost[r.to_s] + penalty
+                nextBreak[i.to_s] = r
+            end
+        end
+
+        # get the lines from the ResultHash
+
+        return nextBreak.collect
+    end
+end
+
+if __FILE__ == $PROGRAM_NAME
+
+    # test code
+
+    breaker = DynamicBreaker.new()
+
+    words = "Sed eget ligula. Nunc fringilla. In ullamcorper turpis quis tortor. Maecenas fringilla dui aliquet leo. Nulla nec mi ut mauris ultrices sollicitudin. Mauris feugiat ornare massa. Ut vitae dolor sed urna blandit imperdiet. Cras tempus, orci sollicitudin pulvinar ultricies, sapien urna fringilla risus, eu rhoncus metus nisi a risus. Aliquam erat volutpat. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Duis iaculis lorem sit amet neque. Cras quis tellus. Ut molestie eros sit amet nibh blandit luctus. Quisque ac sem. Nulla condimentum eros rhoncus ipsum. Duis enim. Phasellus mattis posuere augue.".split(" ")
+
+    results = breaker.parse(words, 70)
+    results.each do |line|
+        puts line.length.to_s + " " + line
+    end
+
+end

data/lib/rbpar_main.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/ruby
+
+# Implementation of a paragraph parsing library API for the rbpar suite.
+# Use this class as the main entry point if you require high-level
+# functions.
+#
+# Copyright Ismo Puustinen 2007.
+
+require 'rbpar_engine'
+require 'rbpar_paragraph'
+
+class RbParIterator
+
+    # RbParIterator provides the API for accessing the parsing and line
+    # breaking system.
+
+    include Enumerable
+
+    def initialize(width)
+        unless width > 0
+            raise ArgumentError, "Negative or zero paragraph width!"
+        end
+        @breaker = DynamicBreaker.new()
+        @width = width
+        @readlines = Array.new()
+    end
+
+    def <<(lines)
+        @readlines = @readlines + lines
+    end
+
+    def process_paragraph(lines)
+
+        # the paragraph is (at this point) only an array of lines
+
+        paragraph = Paragraph.new(lines)
+        
+        # split the paragraph to split paragraphs according to e-mail
+        # quoting etc.
+        paragraphs = paragraph.get_paragraphs
+
+        # results has the processed sub-paragraphs
+        results = paragraphs.collect do |sub_paragraph|
+            # process removes the line breaks: it doesn't make sense to
+            # send pure line breaks there
+            if sub_paragraph[0] != "\n"
+                sub_paragraph.process!(@breaker, @width)
+            else
+                [""]
+            end
+        end
+
+        # combine the result paragraphs into one
+        return results.inject([]) do |total, paragraph|
+            total + paragraph
+        end
+
+    end
+
+    def each
+        @readlines.join.each('') do |paragraph|
+            lines = paragraph.collect
+            yield process_paragraph(lines)
+        end
+    end
+
+end
+