iudex-core 1.2.1-java → 1.3.0-java

data/History.rdoc

@@ -1,3 +1,14 @@
+=== 1.3.0 (2012-11-8)
+* Add DomainKey with optional :type to support configuring of
+  a :domain,:type specific HostQueue
+* Add VisitQueue.config( options ) extensions for cleaner
+  configuration of HostQueue with type, rate, etc.
+* Update TLDSets based on upstream c61f326ad19f 2012-10-25
+* Upgrade to gravitext-util ~> 1.7.0
+* Upgrade to slf4j ~> 1.7.0, logback ~> 1.5 (dev)
+* Misc java logging simplifications with slf4j 1.7 varargs
+* Expose GenericWorkPollStrategy.log()
+
 === 1.2.1 (2012-9-15)
 * Upgrade/narrow to gravitext-util ~> 1.6.1
 * Upgrade to slf4j [1.6.5,1.8), logback ~> 1.2 (dev)

data/Manifest.txt

@@ -14,6 +14,7 @@ lib/iudex-core/base.rb
 lib/iudex-core.rb
 lib/iudex-core/config.rb
 lib/iudex-core/mojibake.rb
+lib/iudex-core/visit_queue.rb
 test/setup.rb
 test/test_charsets.rb
 test/test_content_fetcher.rb
@@ -24,4 +25,4 @@ test/test_redirect_handler.rb
 test/test_visit_manager.rb
 test/test_visit_queue.rb
 test/test_visit_url.rb
-lib/iudex-core/iudex-core-1.2.1.jar
+lib/iudex-core/iudex-core-1.3.0.jar

data/Rakefile

@@ -18,6 +18,7 @@ task :clean do
   rm_f 'src/main/java/iudex/core/TLDSets.java'
 end
 
+desc "Download and install latest effective_tld_name.dat"
 task :refresh_tld_dat do
   sh( "curl http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/effective_tld_names.dat?raw=1" +
       " -o build/effective_tld_name.dat" )

data/build/effective_tld_name.dat

@@ -219,7 +219,6 @@ net.au
 org.au
 edu.au
 gov.au
-csiro.au
 asn.au
 id.au
 // Historic 2LDs (closed to new registration, but sites still exist)
@@ -950,9 +949,15 @@ gov.gr
 // gs : http://en.wikipedia.org/wiki/.gs
 gs
 
-// gt : http://www.gt/politicas.html
-*.gt
-!www.gt
+// gt : http://www.gt/politicas_de_registro.html
+gt
+com.gt
+edu.gt
+gob.gt
+ind.gt
+mil.gt
+net.gt
+org.gt
 
 // gu : http://gadao.gov.gu/registration.txt
 *.gu
@@ -4166,6 +4171,7 @@ name.my
 
 // mz : http://www.gobin.info/domainname/mz-template.doc
 *.mz
+!teledata.mz
 
 // na : http://www.na-nic.com.na/
 // http://www.info.na/domain/
@@ -5359,8 +5365,17 @@ ed.pw
 go.pw
 belau.pw
 
-// py : http://www.nic.py/faq_a.html#faq_b
-*.py
+// py : http://www.nic.py/pautas.html#seccion_9
+// Confirmed by registry 2012-10-03
+com.py
+coop.py
+edu.py
+gov.py
+mil.py
+net.py
+org.py
+!nic.py
+!una.py
 
 // qa : http://domains.qa/en/
 qa
@@ -5999,20 +6014,20 @@ com.ug
 org.ug
 
 // uk : http://en.wikipedia.org/wiki/.uk
+// Submitted by registry <noc@nominet.org.uk> 2012-10-02
 *.uk
+*.nhs.uk
+*.police.uk
 *.sch.uk
 !bl.uk
 !british-library.uk
-!icnet.uk
 !jet.uk
 !mod.uk
+!national-library-scotland.uk
 !nel.uk
-!nhs.uk
 !nic.uk
 !nls.uk
-!national-library-scotland.uk
 !parliament.uk
-!police.uk
 
 // us : http://en.wikipedia.org/wiki/.us
 us
@@ -6288,8 +6303,19 @@ gov.vc
 mil.vc
 edu.vc
 
-// ve : http://registro.nic.ve/nicve/registro/index.html
-*.ve
+// ve : https://registro.nic.ve/
+// Confirmed by registry 2012-10-04
+ve
+co.ve
+com.ve
+e12.ve
+edu.ve
+gov.ve
+info.ve
+mil.ve
+net.ve
+org.ve
+web.ve
 
 // vg : http://en.wikipedia.org/wiki/.vg
 vg
@@ -6529,15 +6555,20 @@ priv.at
 co.ca
 
 // CentralNic : http://www.centralnic.com/names/domains
-// Confirmed by registry <gavin.brown@centralnic.com> 2008-06-09
+// Confirmed by registry <gavin.brown@centralnic.com> 2012-09-27
+ae.org
 ar.com
 br.com
 cn.com
+com.de
 de.com
 eu.com
 gb.com
+gb.net
 gr.com
 hu.com
+hu.net
+jp.net
 jpn.com
 kr.com
 no.com
@@ -6545,25 +6576,68 @@ qc.com
 ru.com
 sa.com
 se.com
+se.net
 uk.com
+uk.net
 us.com
+us.org
 uy.com
 za.com
-gb.net
-jp.net
-se.net
-uk.net
-ae.org
-us.org
-com.de
 
 // Opera Software, A.S.A.
 // Requested by Yngve Pettersen <yngve@opera.com> 2009-11-26
 operaunite.com
 
 // Google, Inc.
-// Requested by Eduardo Vela <evn@google.com> 2010-09-06
+// Requested by Eduardo Vela <evn@google.com> 2012-10-24
 appspot.com
+blogspot.be
+blogspot.bj
+blogspot.ca
+blogspot.cf
+blogspot.ch
+blogspot.co.at
+blogspot.co.il
+blogspot.co.nz
+blogspot.co.uk
+blogspot.com
+blogspot.com.ar
+blogspot.com.au
+blogspot.com.br
+blogspot.com.es
+blogspot.cv
+blogspot.cz
+blogspot.de
+blogspot.dk
+blogspot.fi
+blogspot.fr
+blogspot.gr
+blogspot.hk
+blogspot.hu
+blogspot.ie
+blogspot.in
+blogspot.it
+blogspot.jp
+blogspot.kr
+blogspot.mr
+blogspot.mx
+blogspot.nl
+blogspot.no
+blogspot.pt
+blogspot.re
+blogspot.ro
+blogspot.se
+blogspot.sg
+blogspot.sk
+blogspot.td
+blogspot.tw
+codespot.com
+googleapis.com
+googlecode.com
+
+// DreamHost : http://www.dreamhost.com/
+// Requested by Andrew Farmer <andrew.farmer@dreamhost.com> 2012-10-02
+dreamhosters.com
 
 // iki.fi : Submitted by Hannu Aronsson <haa@iki.fi> 2009-11-05
 iki.fi
@@ -6865,4 +6939,12 @@ webhop.org
 worse-than.tv
 writesthisblog.com
 
+// BetaInABox
+// Requested by adrian@betainabox.com 2012-09-13
+betainabox.com
+
+// Red Hat, Inc. OpenShift : https://openshift.redhat.com/
+// Requested by Tim Kramer <tkramer@rhcloud.com> 2012-10-24
+rhcloud.com
+
 // ===END PRIVATE DOMAINS===

data/lib/iudex-core/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module Core
-    VERSION = '1.2.1'
+    VERSION = '1.3.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-core/visit_queue.rb

@@ -0,0 +1,86 @@
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+
+module Iudex::Core
+
+  # Configuration extensions for Java::iudex.core.VisitQueue.
+  class VisitQueue
+
+    # Configure defaults, a specific domain or domain,type pair via an
+    # options Hash.
+    #
+    # ==== Options
+    #
+    # :domain:: Registration level domain String. If not specified,
+    #           :type is ignored and other options apply as general
+    #           defaults for all (otherwise un-configured
+    #           domains/types).
+    #
+    # :type:: An optional type (i.e. PAGE). If specified, this
+    #         :domain,:type pair will be given its own HostQueue with
+    #         other the options applying exclusively to it.
+    #
+    # :rate:: Target maximum rate of crawl as a Float requests/second
+    #         for this :domain(,:type) or the default for any not
+    #         otherwise configured. Resource limits including :cons
+    #         and HTTP client connections may further inhibit rate
+    #         below this value. (Initial default is 2.0 req/second)
+    #
+    # :delay:: Alternative inverse to :rate as Integer milliseconds to
+    #          delay between scheduling visits. If specifies, takes
+    #          precedence over rate.
+    #
+    # :cons:: Maximum number of concurrent requests for this
+    #         :domain(,:type) or the default for any not otherwise
+    #         configured. Note that the HTTP clients have their own
+    #         per *host:port* destination connection limit which
+    #         should generally be set higher than this value.
+    #         (Initial default: 1)
+    #
+    def config( opts = {} )
+
+      if opts[ :domain ]
+        opts = { :rate => delay_to_rate( default_min_host_delay ),
+                 :cons => default_max_access_per_host }.merge( opts )
+        configure_host( opts[ :domain ],
+                        opts[ :type ], # includes nil
+                        opts[ :delay ] || rate_to_delay( opts[ :rate ] ),
+                        opts[ :cons ] )
+      else
+        if opts[ :rate ]
+          self.default_min_host_delay = rate_to_delay( opts[ :rate ] )
+        end
+        self.default_min_host_delay = opts[ :delay ] if opts[ :delay ]
+        self.default_max_access_per_host = opts[ :cons ] if opts[ :cons ]
+      end
+
+    end
+
+    private
+
+    def rate_to_delay( r )
+      ( 1_000.0 / r ).round
+    end
+
+    def delay_to_rate( d )
+      ( 1_000.0 / d )
+    end
+
+  end
+
+end

data/lib/iudex-core.rb

@@ -53,3 +53,4 @@ module Iudex
 end
 
 require 'iudex-core/mojibake'
+require 'iudex-core/visit_queue'

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-core</artifactId>
   <packaging>jar</packaging>
-  <version>1.2.1</version>
+  <version>1.3.0</version>
   <name>Iudex Core System</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.2.1</version>
+    <version>1.3.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -30,19 +30,19 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-filter</artifactId>
-      <version>[1.2.1,1.2.999)</version>
+      <version>[1.3.0,1.3.999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-http</artifactId>
-      <version>[1.2.1,1.2.999)</version>
+      <version>[1.3.0,1.3.999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-barc</artifactId>
-      <version>[1.2.1,1.2.999)</version>
+      <version>[1.3.0,1.3.999)</version>
     </dependency>
 
     <dependency>

data/test/test_visit_manager.rb

@@ -73,6 +73,11 @@ class TestVisitManager < MiniTest::Unit::TestCase
       self.max_check_interval = 21
       self.max_poll_interval = 130 #ms
       @batch = 0
+      @log = RJack::SLF4J[ self.class ]
+    end
+
+    def log
+      @log.java_logger
     end
 
     def pollWorkImpl( visit_q )

data/test/test_visit_queue.rb

@@ -34,7 +34,7 @@ class TestVisitQueue < MiniTest::Unit::TestCase
 
   def setup
     @visit_q = VisitQueue.new
-    @visit_q.default_min_host_delay = 50 #ms
+    @visit_q.config( :delay => 50 ) #ms
     @scheduler = Executors::new_scheduled_thread_pool( 2 )
   end
 
@@ -96,7 +96,7 @@ class TestVisitQueue < MiniTest::Unit::TestCase
   end
 
   def test_configure
-    @visit_q.configure_host( 'h2.com', 75, 2 )
+    @visit_q.config( :domain => 'h2.com', :delay => 75, :cons => 2 )
 
     [ %w[   h2 a 2.2 ],
       %w[ w.h2 b 2.1 ],
@@ -125,6 +125,39 @@ class TestVisitQueue < MiniTest::Unit::TestCase
     assert_queue_empty
   end
 
+  def test_configure_type
+    @visit_q.config( :domain => 'h2.com',
+                     :delay => 75, :cons => 2 )
+    @visit_q.config( :domain => 'h2.com', :type => 'ALT',
+                     :delay => 50, :cons => 1 )
+
+    [ %w[   h2     a 2.2 ],
+      %w[ w.h2     b 2.1 ],
+      %w[   h2:ALT c 3.2 ],
+      %w[   h2:ALT d 3.1 ],
+      %w[   h1     a 1.2 ],
+      %w[   h1     b 1.1 ] ].each do |oinp|
+
+      @visit_q.add( order( oinp ) )
+
+    end
+    assert_equal( 3, @visit_q.host_count, "host count" )
+
+    expected = [ %w[   h2:ALT c 3.2 ],
+                 %w[   h2     a 2.2 ],
+                 %w[   h1     a 1.2 ],
+                 %w[   h2:ALT d 3.1 ],
+                 %w[   h1     b 1.1 ],
+                 %w[ w.h2     b 2.1 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    assert_queue_empty
+  end
+
   def test_multi_access_2
     @visit_q.default_max_access_per_host = 2
     add_common_orders
@@ -170,7 +203,7 @@ class TestVisitQueue < MiniTest::Unit::TestCase
   def test_interleaved
     @visit_q.default_max_access_per_host = 2
     @visit_q.default_min_host_delay = 3 #ms
-    @visit_q.configure_host( 'h2.com', 1, 4 )
+    @visit_q.config( :domain => 'h2.com', :delay => 1, :cons => 4 )
 
     512.times do |i|
       @visit_q.add( order( [ %w[ h1 h2 ][rand( 2 )], i, 5 * rand ] ) )
@@ -222,10 +255,13 @@ class TestVisitQueue < MiniTest::Unit::TestCase
 
   def order( args )
     host, c, p = args
+    host, t = host.split( ':' )
+
     UniMap.new.tap do |o|
       o.url = visit_url( "http://#{host}.com/#{c}" )
       o.priority = p.to_f
       o.vtest_input = args
+      o.type = t || 'PAGE'
     end
   end
 

metadata

@@ -1,191 +1,230 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: iudex-core
-version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 1.2.1
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 1.3.0
 platform: java
-authors: 
-  - David Kellum
-autorequire: 
+authors:
+- David Kellum
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2012-09-15 00:00:00 Z
-dependencies: 
-  - !ruby/object:Gem::Dependency 
-    name: rjack-slf4j
-    version_requirements: &id001 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.6.5
-        - - <
-          - !ruby/object:Gem::Version 
-            version: "1.8"
-    requirement: *id001
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: hooker
-    version_requirements: &id002 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.0.0
-    requirement: *id002
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: gravitext-util
-    version_requirements: &id003 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.6.1
-    requirement: *id003
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: iudex-filter
-    version_requirements: &id004 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.2.1
-    requirement: *id004
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: iudex-http
-    version_requirements: &id005 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.2.1
-    requirement: *id005
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: iudex-barc
-    version_requirements: &id006 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.2.1
-    requirement: *id006
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: minitest
-    version_requirements: &id007 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "2.3"
-    requirement: *id007
-    prerelease: false
-    type: :development
-  - !ruby/object:Gem::Dependency 
-    name: rjack-logback
-    version_requirements: &id008 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "1.2"
-    requirement: *id008
-    prerelease: false
-    type: :development
-  - !ruby/object:Gem::Dependency 
-    name: rjack-tarpit
-    version_requirements: &id009 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "2.0"
-    requirement: *id009
-    prerelease: false
-    type: :development
+date: 2012-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rjack-slf4j
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: hooker
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: gravitext-util
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: iudex-filter
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: iudex-http
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: iudex-barc
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: minitest
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rjack-logback
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rjack-tarpit
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    none: false
+  prerelease: false
+  type: :development
 description: Iudex is a general purpose web crawler and feed processor in ruby/java. The iudex-core gem contains core facilities and notably, does not contain such facilities as database-backed state management.
-email: 
-  - dek-oss@gravitext.com
-executables: 
-  - iudex-test-config
-  - iudex-url-norm
+email:
+- dek-oss@gravitext.com
+executables:
+- iudex-test-config
+- iudex-url-norm
 extensions: []
-
-extra_rdoc_files: 
-  - History.rdoc
-  - README.rdoc
-files: 
-  - History.rdoc
-  - Manifest.txt
-  - README.rdoc
-  - Rakefile
-  - pom.xml
-  - bin/iudex-test-config
-  - bin/iudex-url-norm
-  - build/TLDSets.java.erb
-  - build/effective_tld_name.dat
-  - build/tld_set_generator.rb
-  - config/config.rb
-  - config/mojibake
-  - lib/iudex-core/base.rb
-  - lib/iudex-core.rb
-  - lib/iudex-core/config.rb
-  - lib/iudex-core/mojibake.rb
-  - test/setup.rb
-  - test/test_charsets.rb
-  - test/test_content_fetcher.rb
-  - test/test_content_source.rb
-  - test/test_log_writer.rb
-  - test/test_mojibake.rb
-  - test/test_redirect_handler.rb
-  - test/test_visit_manager.rb
-  - test/test_visit_queue.rb
-  - test/test_visit_url.rb
-  - lib/iudex-core/iudex-core-1.2.1.jar
+extra_rdoc_files:
+- History.rdoc
+- README.rdoc
+files:
+- History.rdoc
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- pom.xml
+- bin/iudex-test-config
+- bin/iudex-url-norm
+- build/TLDSets.java.erb
+- build/effective_tld_name.dat
+- build/tld_set_generator.rb
+- config/config.rb
+- config/mojibake
+- lib/iudex-core/base.rb
+- lib/iudex-core.rb
+- lib/iudex-core/config.rb
+- lib/iudex-core/mojibake.rb
+- lib/iudex-core/visit_queue.rb
+- test/setup.rb
+- test/test_charsets.rb
+- test/test_content_fetcher.rb
+- test/test_content_source.rb
+- test/test_log_writer.rb
+- test/test_mojibake.rb
+- test/test_redirect_handler.rb
+- test/test_visit_manager.rb
+- test/test_visit_queue.rb
+- test/test_visit_url.rb
+- lib/iudex-core/iudex-core-1.3.0.jar
 homepage: http://iudex.gravitext.com
 licenses: []
-
-post_install_message: 
-rdoc_options: 
-  - --main
-  - README.rdoc
-require_paths: 
-  - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+post_install_message:
+rdoc_options:
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2
   none: false
-  requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 2
-        segments: 
-          - 0
-        version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2
   none: false
-  requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 2
-        segments: 
-          - 0
-        version: "0"
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.8.15
-signing_key: 
+rubyforge_project:
+rubygems_version: 1.8.24
+signing_key:
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java.
 test_files: []
-
+...