treat 0.1.3 → 0.1.4

data/LICENSE

@@ -18,6 +18,7 @@ Author: Louis-Antoine Mullie (louis.mullie@gmail.com). Copyright 2012.
 Non-trivial amount of code has been incorporated and modified from
 other libraries, specifically for the following files:
 
+- formatters/readers/odt.rb - Mark Watson (GPL license)
 - processors/tokenizers/macintyre.rb - Utiyama Masao (Ruby License)
 - processors/tokenizers/tactful.rb - Matthew Bunday (GPL license)
 - processors/tokenizers/perl.rb - Todd A. Fisher (MIT License)

data/lib/treat.rb

@@ -84,7 +84,7 @@ module Treat
   end
   
   # The current version of Treat.
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 
   # $LOAD_PATH << '/ruby/treat/lib/' # Remove for release
   
@@ -105,10 +105,8 @@ module Treat
     attr_accessor :lib
     # String - folder for tests.
     attr_accessor :test
-    # String - folder for temp files.
-    attr_accessor :tmp
   end
-  
+
   # Set the default language to english.
   self.default_language = :eng
   # Set the default encoding to utf-8.
@@ -119,15 +117,16 @@ module Treat
   self.language_detection_level = :section
   # Set the lib path to that of this file.
   self.lib = File.dirname(__FILE__)
-  # Set the paths to the bin, test and tmp folders.
+  # Set the paths to the bin folder.
   self.bin = self.lib + '/../bin'
+  # Set the paths to the test folder.
   self.test = self.lib + '/../test'
-  self.tmp = self.lib + '/../tmp'
-  
+
   # Require modified core classes.
   require 'treat/object'
   require 'treat/kernel'
-  
+  require 'treat/string'
+
   # Require all files for the Treat library.
   require 'treat/exception'
   require 'treat/languages'
@@ -138,5 +137,4 @@ module Treat
   
   # Make sugar available when needed.
   extend Sugar
-
 end

data/lib/treat/detectors.rb

@@ -9,14 +9,14 @@ module Treat
     module Encoding 
       extend Group
       self.type = :annotator
-      self.targets = [:entity]
-      self.default = :native
+      self.targets = [:document]
+      self.default = :r_chardet19
     end
     # Group for algorithms that support format detection.
     module Format
       extend Group
       self.type = :annotator
-      self.targets = [:entity]
+      self.targets = [:document]
       self.default = :file
     end
     # Group for algorithms that do language detection.

data/lib/treat/detectors/encoding/r_chardet19.rb

@@ -6,16 +6,20 @@ module Treat
       # A wrapper for the 'rchardet19' gem, which
       # detects the encoding of a file.
       class RChardet19
-        # Returns the encoding of the entity according
+        # Returns the encoding of the document according
         # to the 'rchardet19' gem.
         #
         # Options: none.
-        def self.encoding(entity, options={})
-          r = CharDet.detect(entity.to_s)
-          Treat::Feature.new({
-            r.encoding.
-            gsub('-', '_').intern => 
+        def self.encoding(document, options={})
+          r = CharDet.detect(document.file)
+          if r.encoding
+            Treat::Feature.new({
+              r.encoding.
+              gsub('-', '_').downcase.intern =>
             r.confidence}).best
+          else
+            :unknown
+          end
         end
       end
     end

data/lib/treat/extractors/topic_words/lda.rb

@@ -38,7 +38,7 @@ module Treat
           options = DefaultOptions.merge(options)
           # Create a corpus with the collection
           sections = collection.sections.collect do |t|
-            t.to_s.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?")
+            t.to_s.encode_compliant('UTF-8')    # fix
           end
           corpus = Lda::TextCorpus.new(sections)
 

data/lib/treat/formatters/readers/abw.rb

@@ -5,9 +5,9 @@ module Treat
         require 'rexml/document'
         require 'rexml/streamlistener'
         def self.read(document, options = {})
-          xml_h = AbiWordXmlHandler.new(
-          REXML::Document.parse_stream((IO.read(document.file)), xml_h))
-          document << xml_h.plain_text
+          xml_h = AbiWordXmlHandler.new
+          REXML::Document.parse_stream(IO.read(document.file), xml_h)
+          document << Treat::Entities::Entity.from_string(xml_h.plain_text)
           document
         end
         class AbiWordXmlHandler
@@ -16,14 +16,10 @@ module Treat
           def initialize
             @plain_text = ""
           end
-          def text s
-            begin
-              s = s.strip
-              if s.length > 0
-                @plain_text << s
-                @plain_text << "\n"
-              end
-            end if s != 'AbiWord' && s != 'application/x-abiword'
+          def text(s)
+            if s != 'AbiWord' && s != 'application/x-abiword'
+              @plain_text << s if s.strip.length > 0
+            end
           end
         end
       end

data/lib/treat/formatters/readers/odt.rb

@@ -1,18 +1,27 @@
 module Treat
   module Formatters
     module Readers
+      # A reader for the ODT (Open Office) document format.
+      #
+      # Based on work by Mark Watson, licensed under the GPL.
+      # Original project website: http://www.markwatson.com/opensource/
       class Odt
-        # Build an entity from a string in plain text format.
+        # Require the 'zip' gem to unarchive the ODT files
+        silence_warnings { require 'zip' }
+        # Build an entity from an ODT file.
         def self.read(document, options = {})
-          f = File.read(document.file)
-          f = f.force_encoding("UTF-8")
-          xml_h = OOXmlHandler.new(
-            REXML::Document.parse_stream(f, xml_h)
-          )
-          document << xml_h.plain_text
+          f = nil
+          Zip::ZipFile.open(document.file, Zip::ZipFile::CREATE) do |zipfile|
+            f = zipfile.read('content.xml')
+          end
+          raise "Couldn't unzip dot file #{document.file}!" unless f
+          xml_h = OOXmlHandler.new
+          REXML::Document.parse_stream(f, xml_h)
+          puts xml_h.plain_text
+          document << Treat::Entities::Entity.from_string(xml_h.plain_text)
           document
         end
-
+        # Xml listener for the parsing of the ODT file.
         class OOXmlHandler
           require 'rexml/document'
           require 'rexml/streamlistener'
@@ -20,6 +29,7 @@ module Treat
           attr_reader :plain_text
           def initialize
             @plain_text = ""
+            @last_name = ""
           end
           def tag_start(name, attrs)
             @last_name = name
@@ -35,7 +45,7 @@ module Treat
           end
         end
       end
-      
+
     end
   end
 end

data/lib/treat/formatters/serializers/xml.rb

@@ -9,8 +9,8 @@ module Treat
         def self.serialize(entity, options = {})
           options = {:indent => 0} if options[:indent].nil?
           if options[:indent] == 0
-            enc = entity.encoding(:r_chardet19).to_s.gsub('_', '-').upcase
-            string = "<?xml version=\"1.0\" encoding=\"#{enc}\" standalone=\"no\" ?>"
+            # enc = entity.encoding(:r_chardet19).to_s.gsub('_', '-').upcase
+            string = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\" ?>"  # Fix
           else
             string = ''
           end

data/lib/treat/formatters/unserializers/xml.rb

@@ -1,12 +1,12 @@
 module Treat
   module Formatters
     module Unserializers
-      # Recreates the entity tree corresponding to 
+      # Recreates the entity tree corresponding to
       # a serialized XML file.
       class XML
         require 'nokogiri'
         # Unserialize an entity stored in XML format.
-        # 
+        #
         # Options: none.
         def self.unserialize(document, options = {})
           # Read in the XML file.
@@ -14,7 +14,7 @@ module Treat
           xml_reader = Nokogiri::XML::Reader.from_memory(xml)
           current_element = nil
           previous_depth = 0
-          
+
           # Read the XML file entity by entity.
           while xml_reader.read
             # The depth in the XML tree.
@@ -31,18 +31,20 @@ module Treat
               previous_depth = current_depth
               next
             end
-            
+
             id = nil; value = ''
             attributes = {}; edges = {}
-            xml_reader.attributes.each_pair do |k,v|
-              if k == 'id'
-                id = v
-              elsif k == 'edges'
-                edges = v
-              elsif k == 'value'
-                value = v
-              else
-                attributes[k.intern] = v
+            unless xml_reader.attributes.empty?
+              xml_reader.attributes.each_pair do |k,v|
+                if k == 'id'
+                  id = v
+                elsif k == 'edges'
+                  edges = v
+                elsif k == 'value'
+                  value = v
+                else
+                  attributes[k.intern] = v
+                end
               end
             end
 
@@ -66,18 +68,18 @@ module Treat
                 current_element.register_token(current_element)
               end
             end
-  
+
             previous_depth = current_depth
           end
           document << current_element
           document
         end
-        
+
         def self.revive(type, value, id)
           klass = Treat::Entities.const_get(cc(type))
           klass.new(value, id)
         end
-        
+
       end
     end
   end

data/lib/treat/kernel.rb

@@ -32,9 +32,13 @@ module Kernel
   # Create a temporary file which is deleted
   # after execution of the block.
   def create_temp_file(ext, value = nil, &block)
-    tmp = Tempfile.new(['', ".#{ext.to_s}"], Treat.tmp)
-    tmp.puts(value) if value
-    block.call(tmp.path)
+    fname = "../tmp/#{Random.rand(10000000).to_s}.#{ext}"
+    File.open(fname, 'w') do |f| 
+      f.write(value) if value 
+      block.call(f.path)
+    end
+  ensure
+    File.delete(fname)
   end
   # Convert un_camel_case to CamelCase.
   def camel_case(o_phrase)

data/lib/treat/string.rb

@@ -0,0 +1,5 @@
+class String
+  def encode_compliant(encoding)
+    encode!(encoding, :invalid => :replace, :undef => :replace, :replace => "?")
+  end
+end

data/lib/treat/sugar.rb

@@ -18,6 +18,7 @@ module Treat
         end
       end
     end
+    alias :sweeten :edulcorate
     # Uninstalls syntactic sugar.
     def unedulcorate
       return unless @@edulcorated
@@ -30,6 +31,7 @@ module Treat
         end
       end
     end
+    alias :unsweeten :unedulcorate
     # Boolean - whether syntactic sugar is
     # enabled or not.
     def edulcorated?; @@edulcorated; end

data/test/tc_detectors.rb

@@ -11,7 +11,6 @@ module Treat
       end
       
       def test_encoding_detectors
-        assert_equal :utf_8, @doc.encoding(:native)
         assert_equal :utf_8, @doc.encoding(:r_chardet19)
       end
       

data/test/tc_entity.rb

@@ -57,7 +57,7 @@ module Treat
           @text.encoding(:nonexistent)
         end
         assert_nothing_raised do
-          @text.format
+          @text.language
         end
       end
       

data/test/tc_treat.rb

@@ -51,7 +51,6 @@ module Treat
         assert_not_nil Treat.lib
         assert_not_nil Treat.bin
         assert_not_nil Treat.test
-        assert_not_nil Treat.tmp
       end
       
       def test_file_permissions

data/test/tests.rb

@@ -2,13 +2,13 @@ require 'test/unit'
 
 $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 
-# $LOAD_PATH << '/ruby/treat/test'
-
 require 'treat'
-require 'texts'
 
+# $LOAD_PATH << '/ruby/treat/test' # Remove for release
 # Treat.bin = '/ruby/nat/bin' # Remove for release
 
+require 'texts'
+
 require 'tc_treat'
 require 'tc_tree'
 require 'tc_entity'

data/test/texts/english/half_cocked_basel.txt

@@ -0,0 +1,16 @@
+Half-cocked Basel
+Stop-gap rules on banks’ trading books may add perilous complexity
+
+THE NEW-YEAR hangover throbbed agonisingly for investment bankers this year. Blame Basel 2.5, a new set of international rules which charges banks higher capital for the risks they run in their trading books (as opposed to their banking books, where they keep assets that they intend to hold to maturity). Those charges were too low before. And heaping higher costs on banks should please politicians and Joe Public. But they add another layer of complexity to banks’ risk management.
+
+Basel 2.5 came into force on December 31st in most European and major world financial jurisdictions. Switzerland applied the rules a year early, and the costs are substantial. Third-quarter figures for Credit Suisse show a 28% increase in risk-weighted assets, and hence capital charges, for its investment-banking activities purely because of Basel 2.5.
+
+The most notable laggard is America. US financial regulators do not oppose Basel 2.5, but it clashes with the Dodd-Frank act, America’s big wet blanket of a financial reform. Basel 2.5 uses credit ratings from recognised agencies such as Moody’s and Standard & Poor’s to calibrate capital charges. Dodd-Frank expressly forbids the use of such ratings agencies, whose poor judgments are held partly responsible for the crisis. Instead American regulators are working on their own cocktail of credit-risk calibrations for Basel 2.5, using market data and country-risk ratings from the OECD. Their solution is still months away from application (though not as distant as implementation by the Russians or Argentines).
+
+Basel 2.5 for the first time charges banks extra capital for the credit risk of what they hold in their trading portfolio (because the crisis showed that markets are not always liquid enough to be able to offload assets). That includes a charge for the risk that a counterparty goes bust. It also imposes heavy charges on securitised bundles of assets unless the credit risk of each piece of the bundle has an identifiable market price. Banks that have portfolios of trading positions which they reckon offset each other have to convince regulators that their risk models work or face being charged at a cruder, standardised rate.
+
+The problem with Basel 2.5, recognised by regulators and bankers alike, is its complexity. The risk of a trading portfolio must now be broken down into five “buckets”—value at risk (VaR), a measure of how much could be lost in an average trading day; stressed VaR (how much could be lost in extreme conditions); plus three types of credit risk ranging from the risk of single credits to those of securitised loans. Traders are understandably confused. For some banks, developing risk models and getting them approved is just too expensive: more complex businesses will be shut down. That will please those who want banks to be more boring.
+
+But unintended consequences will doubtless follow. Useful products may become less tradable. Trading of riskier products could migrate to unregulated entities. Banks may be tempted into new forms of regulatory arbitrage, by juggling assets between their trading book and their banking book. Worst of all, perhaps, is the increased risk of back-office bungling because of the extra complexity.
+
+Regulators recognise this risk. The Basel Committee on Banking Supervision, which drew up the rules and is also responsible for the full Basel 3 regime that will come into force in 2019, is still conducting what it calls a “fundamental review” of capital rules for banks’ trading books. Publication is not expected before March. Those sore heads will not soon clear.

data/test/texts/english/hungarys_troubles.abw

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE abiword PUBLIC "-//ABISOURCE//DTD AWML 1.0 Strict//EN" "http://www.abisource.com/awml.dtd">
+<abiword template="false" styles="unlocked" xmlns:fo="http://www.w3.org/1999/XSL/Format" xmlns:math="http://www.w3.org/1998/Math/MathML" xid-max="47" xmlns:dc="http://purl.org/dc/elements/1.1/" fileformat="1.0" xmlns:svg="http://www.w3.org/2000/svg" xmlns:awml="http://www.abisource.com/awml.dtd" xmlns="http://www.abisource.com/awml.dtd" xmlns:xlink="http://www.w3.org/1999/xlink" version="0.99.2" xml:space="preserve" props="dom-dir:ltr; document-footnote-restart-section:0; document-endnote-type:numeric; document-endnote-place-enddoc:1; document-endnote-initial:1; lang:en-US; document-endnote-restart-section:0; document-footnote-restart-page:0; document-footnote-type:numeric; document-footnote-initial:1; document-endnote-place-endsection:0">
+<!-- ======================================================================== -->
+<!-- This file is an AbiWord document.                                        -->
+<!-- AbiWord is a free, Open Source word processor.                           -->
+<!-- More information about AbiWord is available at http://www.abisource.com/ -->
+<!-- You should not edit this file by hand.                                   -->
+<!-- ======================================================================== -->
+
+<metadata>
+<m key="dc.format">application/x-abiword</m>
+<m key="abiword.generator">AbiWord</m>
+</metadata>
+<history version="1" edit-time="49" last-saved="1327547309" uid="e83e3112-47ca-11e1-8326-faa5dd588a80">
+<version id="1" started="1327547309" uid="0575e6ee-47cb-11e1-8326-faa5dd588a80" auto="0" top-xid="47"/>
+</history>
+<styles>
+<s type="P" name="Normal" followedby="Current Settings" props="font-family:Times New Roman; margin-top:0pt; color:000000; margin-left:0pt; text-position:normal; widows:2; font-style:normal; text-indent:0in; font-variant:normal; margin-right:0pt; lang:en-US; line-height:1.0; font-size:12pt; text-decoration:none; margin-bottom:0pt; font-weight:normal; bgcolor:transparent; text-align:left; font-stretch:normal"/>
+</styles>
+<pagesize pagetype="Letter" orientation="portrait" width="8.500000" height="11.000000" units="in" page-scale="1.000000"/>
+<section xid="46" props="page-margin-footer:0.5in; page-margin-header:0.5in">
+<p style="Normal" xid="47" props="text-align:left; dom-dir:ltr">Hungary's troubles</p>
+<p style="Normal" xid="1" props="text-align:left; dom-dir:ltr">Not just a rap on the knuckles</p>
+<p style="Normal" xid="2" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="3" props="text-align:left; dom-dir:ltr">THE pressure is piling up on the beleaguered Hungarian government. Today the European Commission threatened it with legal action over several new "cardinal" laws that would require a two-thirds majority in parliament to overturn.</p>
+<p style="Normal" xid="4" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="5" props="text-align:left; dom-dir:ltr">The commission is still considering the laws, but today it highlighted concerns over three issues:</p>
+<p style="Normal" xid="6" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="7" props="text-align:left; dom-dir:ltr"> - The independence of the central bank. Late last year the Hungarian parliament passed a law which expands the monetary council and takes the power to nominate deputies away from the governor and hands it to the prime minister. A separate law opens the door to a merger between the bank and the financial regulator.</p>
+<p style="Normal" xid="8" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="9" props="text-align:left; dom-dir:ltr"> - The judiciary. More than 200 judges over the age of 62 have been forced into retirement and hundreds more face the sack. The new National Judicial Authority is headed by Tünde Handó, a friend of the family of Viktor Orban, the prime minister.</p>
+<p style="Normal" xid="10" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="11" props="text-align:left; dom-dir:ltr"> - The independence of the national data authority.</p>
+<p style="Normal" xid="12" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="13" props="text-align:left; dom-dir:ltr">That wasn't all the commission had to say today. Hungary also received a ticking-off from Olli Rehn (pictured), the economic-affairs commissioner, for not doing enough to tackle its budget deficit. It may now lose access to EU funds.</p>
+<p style="Normal" xid="14" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="15" props="text-align:left; dom-dir:ltr">Slammed in Brussels, the Hungarian government is also under pressure at home. Earlier this week Gordon Bajnai, who served as Socialist prime minister from 2009-10, fired off a broadside that sent shockwaves through the political and media establishments.</p>
+<p style="Normal" xid="16" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="17" props="text-align:left; dom-dir:ltr">After a year and a half of government by the right-wing Fidesz party, wrote Mr Bajnai in a lengthy article on the website of the Patriotism and Progress Public Policy Foundation, democracy has been destroyed in Hungary. The country, he warned, is scarred by division and is drifting towards bankruptcy and away from Europe.</p>
+<p style="Normal" xid="18" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="19" props="text-align:left; dom-dir:ltr">Mr Bajnai called for a radical change of government and a complete political re-orientation. “A new government must have a programme readily at hand that can be applied without delay: a programme that promotes the republic, reconciliation, and recovery.” </p>
+<p style="Normal" xid="20" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="21" props="text-align:left; dom-dir:ltr">Fidesz is rattled by Mr Bajnai, who since leaving office has been teaching at Columbia University in New York. Understandably so. He headed a technocratic administration which stabilised the economy. Unlike his Socialist predecessor, Ferenc Gyurcsany, he was neither part of the old Communist elite nor connected to it by marriage, and so cannot be smeared as a "Komcsi". He is modern in outlook and well regarded internationally.</p>
+<p style="Normal" xid="22" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="23" props="text-align:left; dom-dir:ltr">Moreover, say those how know him, Mr Bajnai has little patience for the narcissistic exceptionalism that shapes Fidesz’s worldview. Exhibit A: the plaintive cry of Janos Martonyi, the foreign minister, who lamented recently: “The world will never understand our pains and spiritual wounds.” Such self-pity is unlikely to endear the Hungarian government to Brussels or Washington DC (to where it has sent an envoy this week to negotiate with the IMF).</p>
+<p style="Normal" xid="24" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="25" props="text-align:left; dom-dir:ltr">Fidesz won a two-thirds majority in 2010. But its support is evaporating, and analysts say there is a gap in the political market for a centrist pro-business party committed to democratic norms. Mr Bajnai, who has not ruled out a return to politics, would be an obvious candidate to lead it.</p>
+<p style="Normal" xid="26" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="27" props="text-align:left; dom-dir:ltr">Meanwhile, as Hungarians watch the value of their assets vaporise, in large part thanks to the government’s increasingly erratic policies, Mr Orban smirks his way through press conferences. Here he is dodging questions from a reporter from HVG, an economics weekly, about his responsibility for the crisis and trying to shift the blame to his old enemy Andras Simor, president of the central bank. The interview ran as follows:</p>
+<p style="Normal" xid="28" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="29" props="text-align:left; dom-dir:ltr">hvg.hu: Do you feel responsible for the falling/weakening forint?</p>
+<p style="Normal" xid="30" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="31" props="text-align:left; dom-dir:ltr">Mr Orban: You mean the president of the central bank? He did not comment on it.</p>
+<p style="Normal" xid="32" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="33" props="text-align:left; dom-dir:ltr">hvg.hu: No, you, Mr prime minister!</p>
+<p style="Normal" xid="34" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="35" props="text-align:left; dom-dir:ltr">Mr Orban: The personal responsibility of the president of the central bank was not discussed over the meeting.</p>
+<p style="Normal" xid="36" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="37" props="text-align:left; dom-dir:ltr">hvg.hu: You, your personal…!</p>
+<p style="Normal" xid="38" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="39" props="text-align:left; dom-dir:ltr">Mr Orban: That neither.</p>
+<p style="Normal" xid="40" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="41" props="text-align:left; dom-dir:ltr">Surrounded by yes-men and grinning flunkies, Mr Orban seems increasingly out of touch. His future will likely be decided not in the gilded corridors of the Hungarian parliament, but in Brussels and Washington DC.</p>
+<p style="Normal" xid="42" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="43" props="text-align:left; dom-dir:ltr">What happens next? If his hand is forced Mr Orban can probably endure policy reversals on the independence of the central bank and the data ombudsman. Sorry, he would say to his loyal followers: national crisis, what can you do.</p>
+<p style="Normal" xid="44" props="text-align:left; dom-dir:ltr"></p>
+<p style="Normal" xid="45" props="text-align:left; dom-dir:ltr">The dismantling of the judiciary would be another matter. If outsiders keep up the pressure and the judicial changes are judged to be in breach of the EU treaty, Mr Orban would be in a tricky spot. It’s hard to see how he could declare the 200-plus judges his government has forced into retirement ready for office after all, and still sit in his own.</p>
+</section>
+</abiword>

data/test/texts/english/zero_sum.html

@@ -0,0 +1,111 @@
+<!DOCTYPE html>
+<html class="no-flash" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr" xmlns:fb="http://www.facebook.com/2008/fbml">
+
+<head>
+  <title>Trade: The zero-sum president | The Economist</title>
+</head>
+
+<body class="not-front not-logged-in page-node node-type-mtblog one-sidebar sidebar-right world-menu business-menu economics-menu printedition-menu science-technology-menu culture-menu">
+      <div id="fb-root"></div>
+   <div id="page" class="container">
+    <a name="top" id="navigation-top"></a>
+
+      <div id="leaderboard" class="clearfix">
+    <div id="block-ec_ads-leaderboard_ad" class="block block-ec_ads">
+    <div class="content clearfix">
+    <div id="leaderboard-ad"><!-- Site: Commerce.  Zone: Opinion | Blogs/Free Exchange |  --> <script language="JavaScript" type="text/javascript">document.write('<script language="JavaScript" src="//ad.doubleclick.net/adj/teg.lasn/blo6;subs=n;wsub=n;sdn=n;dcopt=ist;pos=ldr_top;sz=728x90;tile=1;ord=' + random_ad_nr + '?" type="text/javascript"><\/script>')</script><noscript><a href="//ad.doubleclick.net/jump/teg.lasn/blo6;subs=n;wsub=n;sdn=n;dcopt=ist;pos=ldr_top;sz=728x90;tile=1;ord=595534706?"><img src="//ad.doubleclick.net/ad/teg.lasn/blo6;subs=n;wsub=n;sdn=n;dcopt=ist;pos=ldr_top;sz=728x90;tile=1;ord=595534706?" width="728" height="90" border="0" alt=""></a></noscript></div>  </div>
+</div><div id="block-ec_ads-subscription_ad" class="block block-ec_ads">
+    <div class="content clearfix">
+    <div id="subslug-ad"><!-- Site: Commerce.  Zone: Opinion | Blogs/Free Exchange |  --> <script language="JavaScript" type="text/javascript">document.write('<a href="//ad.doubleclick.net/jump/teg.lasn/blo6;subs=n;wsub=n;sdn=n;pos=sub_top;sz=223x90;tile=2;ord=' + random_ad_nr + '?" target="_self"><img src="//ad.doubleclick.net/ad/teg.lasn/blo6;subs=n;wsub=n;sdn=n;pos=sub_top;sz=223x90;tile=2;ord=' + random_ad_nr + '?"><\/a>')</script><noscript><a href="//ad.doubleclick.net/jump/teg.lasn/blo6;subs=n;wsub=n;sdn=n;pos=sub_top;sz=223x90;tile=2;ord=595534706?"><img src="//ad.doubleclick.net/ad/teg.lasn/blo6;subs=n;wsub=n;sdn=n;pos=sub_top;sz=223x90;tile=2;ord=595534706?" width="223" height="90" border="0" alt=""></a></noscript></div>  </div>
+</div>  </div>
+
+
+<div id="header" class="clearfix">
+
+  <div id="header-topstripe" class="clearfix">
+          <div id="header-logo">
+                  <h2><a href="/"><img src="//media.economist.com/sites/all/themes/econfinal/images/the-economist-logo.gif" alt="The Economist" /></a></h2>
+              </div>
+    
+    <div id="header-main">
+
+          <div id="header-links" class="clearfix">
+          <div id="header-links-user">
+            <div id="header-login">
+  <div class="item-list"><ul><li class="first"><a href="/user/login?destination=node%2F21543439" class="show-login" data-ec-omniture="masthead|act_prod|login">Log in</a></li>
+<li class="even"><a href="/user/register?destination=node%2F21543439&amp;rp=masthead" data-ec-omniture="masthead|act_prod|register">Register</a></li>
+<li><a href="/user" data-ec-omniture="masthead|act_prod|my_account">My account</a></li>
+<li class="even last"><a href="/products/subscribe" data-ec-omniture="masthead|act_prod|subscribe">Subscribe</a></li>
+</ul></div></div> <!-- /#header-login -->
+          </div><!-- /#header-links-user -->
+
+          <div id="header-links-general">
+            <ul>
+              <li><a data-ec-omniture="masthead|act_prod|digital_mobile" href="/digital" title="Digital &amp; mobile">Digital &amp; mobile</a></li>
+              <li class="pipe"><a data-ec-omniture="masthead|act_prod|events" href="/events" title="Events">Events</a></li>
+              <li class="pipe"><a data-ec-omniture="masthead|act_prod|newsletters" href="/newsletters" title="Newsletters">Newsletters</a></li>
+              <li class="pipe"><a data-ec-omniture="masthead|act_prod|rss" class="rss-link" href="/rss" title="RSS">RSS</a></li>
+              <li class="pipe"><a data-ec-omniture="masthead|act_prod|jobs" href="http://jobs.economist.com" title="Jobs">Jobs</a></li>
+              <li class="pipe last"><a data-ec-omniture="masthead|act_prod|help" href="/help" title="Help">Help</a></li>
+            </ul>
+          </div><!-- /#header-links-general -->
+      </div><!-- /#header-links -->
+
+      <div id="header-timestamp" class="clearfix">
+        Wednesday January 25th 2012      </div><!-- /#header-timestamp -->
+
+      <div id="header-title-search" class="clearfix">
+
+        
+                  <div id="header-search" class="clearfix">
+            <form action="/blogs/freeexchange/2012/01/trade"  accept-charset="UTF-8" method="post" id="search-theme-form">
+<div><div id="search" class="container-inline">
+  <div class="form-item clearfix" id="edit-search-theme-form-1-wrapper">
+ <label for="edit-search-theme-form-1">Search this site:</label>
+<input type="text" maxlength="128" name="search_theme_form" id="edit-search-theme-form-1" size="15" value="" title="Enter the terms you wish to search for." class="form-text search-field" placeholder="Search" />
+</div>
+<input type="submit" name="op" id="edit-submit-1" value="Search"  class="form-submit" />
+<input type="hidden" name="form_build_id" id="form-95221799fc500ba9ac651bab969c1a14" value="form-95221799fc500ba9ac651bab969c1a14"  />
+<input type="hidden" name="form_id" id="edit-search-theme-form" value="search_theme_form"  />
+</div>
+
+</div></form>
+          </div><!-- /#header-search -->
+        
+      </div><!-- /#header-title-search -->    
+    
+    </div><!-- /#header-main -->
+  </div><!-- /#header-topstripe -->
+
+</div> <!-- /#header -->
+
+    
+        <div id="columns" class="clearfix">
+                  <div id="column-content" class="grid-10 grid-first clearfix">
+                                <!-- Create left column on search pages -->
+        <div class="node-blog-tpl" class="clearfix">
+  <div class="blog-title freeexchange">
+  <img  class="imagefield imagefield-field_blog_logo" width="50" height="50" alt="" src="http://media.economist.com/sites/default/files/BlogFreeEx.jpg?1274391686" />  <div class="blog-post-header">
+    <h2>Economics</h4>
+    <span><a href="/blogs/freeexchange">Free exchange</a></span>
+  </div>
+</div>
+      <h1 class="ec-blog-fly-title">Trade</h1>
+    <h3 class="ec-blog-headline">
+    The zero-sum president  </h3>
+  <p class="ec-blog-info">
+    Jan 25th 2012, 9:31 by R.A. | LONDON  </p>
+  <div id="block-ec_components-share_inline_header" class="block block-ec_components">
+    <div class="content clearfix">
+    <div class="share_inline_header"><ul class="clearfix"><li class="share-inline-header-facebook first" data-ec-omniture-frame="top_fb"><fb:like href="http://www.economist.com/blogs/freeexchange/2012/01/trade" send="false" layout="button_count" show_faces="false" font=""></fb:like></li>
+<li class="share-inline-header-twitter even last" data-ec-omniture-frame="top_twitter"><a href="http://twitter.com/share" class="twitter-share-button" data-count="horizontal" data-related="theeconomist" data-url="http://econ.st/AC5wZo" data-counturl="http://www.economist.com/blogs/freeexchange/2012/01/trade">Tweet</a></li>
+</ul></div>  </div>
+</div>
+  <div class="ec-blog-body">
+    <p>STATE of the Union addresses tend to be long, winding affairs, filled with a grab bag of policy ideas that will altenatively appeal to and irk people across the political spectrum. Barack Obama's <a href="http://www.telegraph.co.uk/news/worldnews/barackobama/9037296/State-of-the-Union-speech-2012-speech-in-full.html">latest address</a> had plenty of sensible ideas in it: tax reform, including reductions in corporate rates; more spending and accountability on education and infrastructure investment; streamlining of the regulatory environment; and so on. He led off, however, with a call for a reshoring of manufacturing jobs seemingly calculated to cost him <em>The Economist</em>'s endorsement. Granted, annoying <em>The Economist</em> is, almost definitionally, good politics. For a president whose hallmark has been soaring orations promising hope, however, Mr Obama's take on the global economy is strikingly bleak and depressing.</p><p>The president was not so unreasonable as to suggest that the American economy could recapture all of its lost manufacturing jobs. Nor was he wrong to point out that countries like China have used direct subsidies, financial shenanigans and currency manipulation to give their exporters a leg up. Yet at no point did he attempt to justify the unstated assumption that what America ought really to do is develop an economy like China's—a place, recall, scarcely one-sixth as rich as America, riddled with potentially debilitating economic imbalances, and governed by an unaccountable monopoly of a communist party. Perhaps more distressing, he implied in several places that the reason to become more like China was that only by doing so could America <em>defeat</em> China, and others, at economics. Consider the line:</p><blockquote><p>Our workers are the most productive on Earth, and if the playing field is level, I promise you – America will always win.</p></blockquote><p>Leaving others, one is forced to conclude, to lose—not once, not occasionally, but <em>always</em>. And what is likely to be the outcome of unending defeat? Destitution? Are we to hope that other countries are left with no gainful employment opportunities at all? If that means dreadful poverty, then Mr Obama ought to be dragged before an international tribunal. But maybe it's not so bad, in which case we have to wonder why it's so damned important to "win" whatever contest it is we're having. Is the implication that it's possible to get by all right, to not be poor, without having lots of demanding manufacturing jobs? That doesn't sound so bad, actually; are we sure America doesn't want to sign up for that? Of course, if this is the nature of economic activity, and if America is determined to defeat other countries, it's worth asking whether it wouldn't make sense to deliberately <em>sabotage</em> other places, or bomb them; after all, it's hard to lose to a country whose people are dead. On the other hand, if victory is so important, we might expect other countries to retaliate, or preemptively attack. Maybe it would be better if the world divided itself into two competing but fairly isolated factions locked in a sort of "cold war".</p><p>Later, the president added:</p><blockquote><p>Don’t let other countries win the race for the future.</p></blockquote><p>The context, innocuously enough, was in calling for greater support for American research and development efforts. But the language of this statement is either daft or ghastly, depending on how charitably one is willing to read it. Is Mr Obama so dense as to miss that when America invents things other countries benefit, and vice versa? If a German discovers a cure for cancer, shouldn't we be ecstatic about that, rather than angry? Indeed, shouldn't we be quite happy and interested in ensuring that Germans and Britons and Indians have the capability and opportunity to develop fantastic new technologies? In the more nefarious reading, Mr Obama seems to accept that only relative standing really matters. A sick, poor world in which America always triumphs is preferable in all cases to one in which America maybe doesn't "win" the race to discover every last little thing that's out there to be discovered. And hell, one has to ask again whether the easiest way to prevent other countries from winning the race for the future isn't simply to blow up their labs.</p><p>Look, I understand the forgiving interpretation of these remarks. Americans are motivated by competition and patriotism, and if that's the only way to rally the country behind fundamentally sound policies like subsidies for basic research, then that's the card you play. And, in practice, Mr Obama's reforms will probably not do much more than offset the crummy, mercantilist choices made by other governments elsewhere. No one is talking about going back to the early 19th century, or to the days of communist containment.</p><p>I don't see that that's an acceptable excuse. People who live outside of America are people just like Americans, and we should all rejoice in their rising prosperity, the more so when it occurs through additions to the stock of human knowledge that will benefit people everywhere. If an American president can't communicate that simple idea to his citizenry, out of fear that he'll be drummed out of office on a wave of nationalistic outrage, then he doesn't deserve to be president and his country doesn't deserve to win a damned thing, least of all the right to call itself "exceptional", a beacon of hope and freedom. A zero-sum world is a world <em>without</em> hope, and if Mr Obama is convinced that's what we're in then I don't see much need for him to stick around.</p>  </div>
+  
+    </div> <!-- /#columns -->
+
+  </div> <!-- /#page -->
+  </body>
+</html>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: treat
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-25 00:00:00.000000000 Z
+date: 2012-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rjb
-  requirement: &70232866225980 !ruby/object:Gem::Requirement
+  requirement: &70243259830940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866225980
+  version_requirements: *70243259830940
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70232866224920 !ruby/object:Gem::Requirement
+  requirement: &70243259829420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866224920
+  version_requirements: *70243259829420
 - !ruby/object:Gem::Dependency
   name: chronic
-  requirement: &70232866240260 !ruby/object:Gem::Requirement
+  requirement: &70243259828180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866240260
+  version_requirements: *70243259828180
 - !ruby/object:Gem::Dependency
   name: hpricot
-  requirement: &70232866239080 !ruby/object:Gem::Requirement
+  requirement: &70243259826580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866239080
+  version_requirements: *70243259826580
 - !ruby/object:Gem::Dependency
   name: psych
-  requirement: &70232866237980 !ruby/object:Gem::Requirement
+  requirement: &70243259824560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866237980
+  version_requirements: *70243259824560
 - !ruby/object:Gem::Dependency
   name: rchardet19
-  requirement: &70232866236800 !ruby/object:Gem::Requirement
+  requirement: &70243259839300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866236800
+  version_requirements: *70243259839300
 - !ruby/object:Gem::Dependency
   name: whatlanguage
-  requirement: &70232866235660 !ruby/object:Gem::Requirement
+  requirement: &70243259837640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866235660
+  version_requirements: *70243259837640
 - !ruby/object:Gem::Dependency
   name: wordnet
-  requirement: &70232866234360 !ruby/object:Gem::Requirement
+  requirement: &70243259835240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866234360
+  version_requirements: *70243259835240
 - !ruby/object:Gem::Dependency
   name: rbtagger
-  requirement: &70232866232840 !ruby/object:Gem::Requirement
+  requirement: &70243259832320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866232840
+  version_requirements: *70243259832320
 - !ruby/object:Gem::Dependency
   name: engtagger
-  requirement: &70232866247500 !ruby/object:Gem::Requirement
+  requirement: &70243259839180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866247500
+  version_requirements: *70243259839180
 - !ruby/object:Gem::Dependency
   name: punkt-segmenter
-  requirement: &70232866246340 !ruby/object:Gem::Requirement
+  requirement: &70243259836880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +131,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866246340
+  version_requirements: *70243259836880
 - !ruby/object:Gem::Dependency
   name: tokenizer
-  requirement: &70232866245320 !ruby/object:Gem::Requirement
+  requirement: &70243259834100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,10 +142,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866245320
+  version_requirements: *70243259834100
 - !ruby/object:Gem::Dependency
   name: tactful_tokenizer
-  requirement: &70232866242520 !ruby/object:Gem::Requirement
+  requirement: &70243259830340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -153,10 +153,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866242520
+  version_requirements: *70243259830340
 - !ruby/object:Gem::Dependency
   name: english
-  requirement: &70232866257200 !ruby/object:Gem::Requirement
+  requirement: &70243259828860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -164,10 +164,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866257200
+  version_requirements: *70243259828860
 - !ruby/object:Gem::Dependency
   name: linguistics
-  requirement: &70232866255600 !ruby/object:Gem::Requirement
+  requirement: &70243259827140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -175,10 +175,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866255600
+  version_requirements: *70243259827140
 - !ruby/object:Gem::Dependency
   name: ruby-stemmer
-  requirement: &70232866254220 !ruby/object:Gem::Requirement
+  requirement: &70243259824840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -186,10 +186,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866254220
+  version_requirements: *70243259824840
 - !ruby/object:Gem::Dependency
   name: uea-stemmer
-  requirement: &70232866252360 !ruby/object:Gem::Requirement
+  requirement: &70243259812140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -197,10 +197,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866252360
+  version_requirements: *70243259812140
 - !ruby/object:Gem::Dependency
   name: lda-ruby
-  requirement: &70232866250960 !ruby/object:Gem::Requirement
+  requirement: &70243259810000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -208,10 +208,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866250960
+  version_requirements: *70243259810000
 - !ruby/object:Gem::Dependency
   name: nickel
-  requirement: &70232866249680 !ruby/object:Gem::Requirement
+  requirement: &70243259808300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -219,10 +219,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232866249680
+  version_requirements: *70243259808300
 - !ruby/object:Gem::Dependency
   name: unprof
-  requirement: &70232866265000 !ruby/object:Gem::Requirement
+  requirement: &70243259805800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -230,7 +230,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70232866265000
+  version_requirements: *70243259805800
 description: ! ' Treat is a toolkit for text retrieval, information extraction and
   natural language processing. '
 email:
@@ -243,7 +243,6 @@ files:
 - lib/treat/categories.rb
 - lib/treat/category.rb
 - lib/treat/delegatable.rb
-- lib/treat/detectors/encoding/native.rb
 - lib/treat/detectors/encoding/r_chardet19.rb
 - lib/treat/detectors/format/file.rb
 - lib/treat/detectors/language/language_detector.rb
@@ -348,6 +347,7 @@ files:
 - lib/treat/processors.rb
 - lib/treat/proxies.rb
 - lib/treat/registrable.rb
+- lib/treat/string.rb
 - lib/treat/sugar.rb
 - lib/treat/tree.rb
 - lib/treat/visitable.rb
@@ -364,10 +364,16 @@ files:
 - test/tc_treat.rb
 - test/tc_tree.rb
 - test/tests.rb
+- test/texts/english/half_cocked_basel.txt
+- test/texts/english/hose_and_dry.doc
+- test/texts/english/hungarys_troubles.abw
 - test/texts/english/long.html
 - test/texts/english/long.txt
 - test/texts/english/medium.txt
+- test/texts/english/republican_nomination.pdf
+- test/texts/english/saving_the_euro.odt
 - test/texts/english/short.txt
+- test/texts/english/zero_sum.html
 - test/texts.rb
 - examples/benchmark.rb
 - examples/keywords.rb

data/lib/treat/detectors/encoding/native.rb

@@ -1,17 +0,0 @@
-module Treat
-  module Detectors
-    module Encoding
-      # A wrapper class for Ruby's native encoding detector.
-      class Native
-        # Return the encoding of the entity according
-        # to the Ruby interpreter.
-        #
-        # Options: none.
-        def self.encoding(entity, options={})
-          entity.value.encoding.name.
-          gsub('-', '_').downcase.intern
-        end
-      end
-    end
-  end
-end