iudex-core 1.3.1-java → 1.4.0-java

data/History.rdoc

@@ -1,3 +1,29 @@
+=== 1.4.0 (2013-10-29)
+* Work order reservation and experimental concurrent (no replace) work
+  polling support:
+  * Add WorkPollStrategy.discard hook in support of un-reserving a
+    replaced VisitQueue.
+  * VisitManager and GenericWorkPollStrategy changes for discard and
+    support of false shouldReplaceQueue
+* VisitManager.doWaitOnGeneration default true as this is a safety
+  requirement when replacing the queue. When not replacing the queue
+  no wait occurs.
+* Add VisitQueue.maxAccessTotal and associated waiting to avoid over
+  committing to too many concurrent HTML requests when using an
+  asynchronous HTTP client and many independent domains are
+  available. By default there is no limit, but recommended to set this
+  to some multiple of the VisitManager threads.  For an example of the
+  problem this avoids: full HTML parsing can saturate a single CPU
+  and will cause some number of, say, 600 concurrent HTTP requests to
+  timeout given insufficient CPU/threads to handle pending reads into
+  memory.
+* Add pre-set, dynamic REQUEST_HEADERS support to ContentFetcher. Any
+  request headers found here override the staticly set headers at
+  initialization.
+* Update TLDSets based on upstream 06c405ba54b3 2013-09-17
+* Upgrade to iudex-* ~> 1.4.0 dependencies
+* Upgrade to minitest ~> 4.7.4 (dev)
+
 === 1.3.1 (2012-11-15)
 * Fix bugs in using/preserving configuration with explicit :type
   specified (new in 1.3.0)

data/Manifest.txt

@@ -25,4 +25,4 @@ test/test_redirect_handler.rb
 test/test_visit_manager.rb
 test/test_visit_queue.rb
 test/test_visit_url.rb
-lib/iudex-core/iudex-core-1.3.1.jar
+lib/iudex-core/iudex-core-1.4.0.jar

data/README.rdoc

@@ -11,7 +11,7 @@ does not contain such facilities as database-backed state management.
 
 == License
 
-Copyright (c) 2008-2012 David Kellum
+Copyright (c) 2008-2013 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You

data/bin/iudex-test-config

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-url-norm

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/build/TLDSets.java.erb

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2012 David Kellum
+ * Copyright (c) 2008-2013 David Kellum
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License.  You may

data/build/effective_tld_name.dat

@@ -175,17 +175,16 @@ it.ao
 // aq : http://en.wikipedia.org/wiki/.aq
 aq
 
-// ar : http://en.wikipedia.org/wiki/.ar
-*.ar
-!congresodelalengua3.ar
-!educ.ar
-!gobiernoelectronico.ar
-!mecon.ar
-!nacion.ar
-!nic.ar
-!promocion.ar
-!retina.ar
-!uba.ar
+// ar : https://nic.ar/normativa-vigente.xhtml
+ar
+com.ar
+edu.ar
+gob.ar
+int.ar
+mil.ar
+net.ar
+org.ar
+tur.ar
 
 // arpa : http://en.wikipedia.org/wiki/.arpa
 // Confirmed by registry <iana-questions@icann.org> 2008-06-18
@@ -221,6 +220,7 @@ edu.au
 gov.au
 asn.au
 id.au
+csiro.au
 // Historic 2LDs (closed to new registration, but sites still exist)
 info.au
 conf.au
@@ -691,6 +691,14 @@ inf.cu
 // cv : http://en.wikipedia.org/wiki/.cv
 cv
 
+// cw : http://www.una.cw/cw_registry/
+// Confirmed by registry <registry@una.net> 2013-03-26
+cw
+com.cw
+edu.cw
+net.cw
+org.cw
+
 // cx : http://en.wikipedia.org/wiki/.cx
 // list of other 2nd level tlds ?
 cx
@@ -1071,13 +1079,14 @@ tozsde.hu
 utazas.hu
 video.hu
 
-// id : http://en.wikipedia.org/wiki/.id
-// see also: https://register.pandi.or.id/
+// id : https://register.pandi.or.id/
 id
 ac.id
+biz.id
 co.id
 go.id
 mil.id
+my.id
 net.id
 or.id
 sch.id
@@ -4221,13 +4230,16 @@ other.nf
 store.nf
 
 // ng : http://psg.com/dns/ng/
-// Submitted by registry <randy@psg.com> 2008-06-17
-ac.ng
+ng
 com.ng
 edu.ng
-gov.ng
+name.ng
 net.ng
 org.ng
+sch.ng
+gov.ng
+mil.ng
+mobi.ng
 
 // ni : http://www.nic.ni/dominios.htm
 *.ni
@@ -5028,7 +5040,16 @@ nu
 *.nz
 
 // om : http://en.wikipedia.org/wiki/.om
-*.om
+om
+co.om
+com.om
+edu.om
+gov.om
+med.om
+museum.om
+net.om
+org.om
+pro.om
 !mediaphone.om
 !nawrastelecom.om
 !nawras.om
@@ -5307,6 +5328,9 @@ org.pn
 edu.pn
 net.pn
 
+// post : http://en.wikipedia.org/wiki/.post
+post
+
 // pr : http://www.nic.pr/index.asp?f=1
 pr
 com.pr
@@ -5367,6 +5391,7 @@ belau.pw
 
 // py : http://www.nic.py/pautas.html#seccion_9
 // Confirmed by registry 2012-10-03
+py
 com.py
 coop.py
 edu.py
@@ -5374,8 +5399,6 @@ gov.py
 mil.py
 net.py
 org.py
-!nic.py
-!una.py
 
 // qa : http://domains.qa/en/
 qa
@@ -5730,8 +5753,13 @@ store.st
 // su : http://en.wikipedia.org/wiki/.su
 su
 
-// sv : http://www.svnet.org.sv/svpolicy.html
-*.sv
+// sv : http://www.svnet.org.sv/niveldos.pdf
+sv
+com.sv
+edu.sv
+gob.sv
+org.sv
+red.sv
 
 // sx : http://en.wikipedia.org/wiki/.sx
 // Confirmed by registry <jcvignes@openregistry.com> 2012-05-31
@@ -5904,16 +5932,20 @@ club.tw
 組織.tw
 商業.tw
 
-// tz : http://en.wikipedia.org/wiki/.tz
-// Submitted by registry <randy@psg.com> 2008-06-17
-// Updated from http://www.tznic.or.tz/index.php/domains.html 2010-10-25
+// tz : http://www.tznic.or.tz/index.php/domains
+// Confirmed by registry <manager@tznic.or.tz> 2013-01-22
 ac.tz
 co.tz
 go.tz
+hotel.tz
+info.tz
+me.tz
 mil.tz
+mobi.tz
 ne.tz
 or.tz
 sc.tz
+tv.tz
 
 // ua : https://hostmaster.ua/policy/?ua
 // Submitted by registry <dk@cctld.ua> 2012-04-27
@@ -6015,9 +6047,8 @@ org.ug
 
 // uk : http://en.wikipedia.org/wiki/.uk
 // Submitted by registry <noc@nominet.org.uk> 2012-10-02
+// and tweaked by us pending further consultation.
 *.uk
-*.nhs.uk
-*.police.uk
 *.sch.uk
 !bl.uk
 !british-library.uk
@@ -6543,19 +6574,62 @@ xxx
 // ===END ICANN DOMAINS===
 // ===BEGIN PRIVATE DOMAINS===
 
-// info.at : http://www.info.at/
-biz.at
-info.at
-
-// priv.at : http://www.nic.priv.at/
-// Submitted by registry <lendl@nic.at> 2008-06-09
-priv.at
+// Amazon CloudFront : https://aws.amazon.com/cloudfront/
+// Requested by Donavan Miller <donavanm@amazon.com> 2013-03-22
+cloudfront.net
+
+// Amazon Elastic Compute Cloud: https://aws.amazon.com/ec2/
+// Requested by Osman Surkatty <osmans@amazon.com> 2013-04-02
+compute.amazonaws.com
+us-east-1.amazonaws.com
+compute-1.amazonaws.com
+z-1.compute-1.amazonaws.com
+z-2.compute-1.amazonaws.com
+ap-northeast-1.compute.amazonaws.com
+ap-southeast-1.compute.amazonaws.com
+ap-southeast-2.compute.amazonaws.com
+eu-west-1.compute.amazonaws.com
+sa-east-1.compute.amazonaws.com
+us-gov-west-1.compute.amazonaws.com
+us-west-1.compute.amazonaws.com
+us-west-2.compute.amazonaws.com
+
+// Amazon Elastic Beanstalk : https://aws.amazon.com/elasticbeanstalk/
+// Requested by Adam Stein <astein@amazon.com> 2013-04-02
+elasticbeanstalk.com
+
+// Amazon Elastic Load Balancing : https://aws.amazon.com/elasticloadbalancing/
+// Requested by Scott Vidmar <svidmar@amazon.com> 2013-03-27
+elb.amazonaws.com
+
+// Amazon S3 : https://aws.amazon.com/s3/
+// Requested by Courtney Eckhardt <coec@amazon.com> 2013-03-22
+s3.amazonaws.com
+s3-us-west-2.amazonaws.com
+s3-us-west-1.amazonaws.com
+s3-eu-west-1.amazonaws.com
+s3-ap-southeast-1.amazonaws.com
+s3-ap-southeast-2.amazonaws.com
+s3-ap-northeast-1.amazonaws.com
+s3-sa-east-1.amazonaws.com
+s3-us-gov-west-1.amazonaws.com
+s3-fips-us-gov-west-1.amazonaws.com
+s3-website-us-east-1.amazonaws.com
+s3-website-us-west-2.amazonaws.com
+s3-website-us-west-1.amazonaws.com
+s3-website-eu-west-1.amazonaws.com
+s3-website-ap-southeast-1.amazonaws.com
+s3-website-ap-southeast-2.amazonaws.com
+s3-website-ap-northeast-1.amazonaws.com
+s3-website-sa-east-1.amazonaws.com
+s3-website-us-gov-west-1.amazonaws.com
 
-// co.ca : http://registry.co.ca
-co.ca
+// BetaInABox
+// Requested by adrian@betainabox.com 2012-09-13
+betainabox.com
 
 // CentralNic : http://www.centralnic.com/names/domains
-// Confirmed by registry <gavin.brown@centralnic.com> 2012-09-27
+// Requested by registry <gavin.brown@centralnic.com> 2012-09-27
 ae.org
 ar.com
 br.com
@@ -6584,79 +6658,24 @@ us.org
 uy.com
 za.com
 
-// Opera Software, A.S.A.
-// Requested by Yngve Pettersen <yngve@opera.com> 2009-11-26
-operaunite.com
-
-// Google, Inc.
-// Requested by Eduardo Vela <evn@google.com> 2012-10-24
-appspot.com
-blogspot.be
-blogspot.bj
-blogspot.ca
-blogspot.cf
-blogspot.ch
-blogspot.co.at
-blogspot.co.il
-blogspot.co.nz
-blogspot.co.uk
-blogspot.com
-blogspot.com.ar
-blogspot.com.au
-blogspot.com.br
-blogspot.com.es
-blogspot.cv
-blogspot.cz
-blogspot.de
-blogspot.dk
-blogspot.fi
-blogspot.fr
-blogspot.gr
-blogspot.hk
-blogspot.hu
-blogspot.ie
-blogspot.in
-blogspot.it
-blogspot.jp
-blogspot.kr
-blogspot.mr
-blogspot.mx
-blogspot.nl
-blogspot.no
-blogspot.pt
-blogspot.re
-blogspot.ro
-blogspot.se
-blogspot.sg
-blogspot.sk
-blogspot.td
-blogspot.tw
-codespot.com
-googleapis.com
-googlecode.com
-
-// DreamHost : http://www.dreamhost.com/
-// Requested by Andrew Farmer <andrew.farmer@dreamhost.com> 2012-10-02
-dreamhosters.com
-
-// iki.fi : Submitted by Hannu Aronsson <haa@iki.fi> 2009-11-05
-iki.fi
-
 // c.la : http://www.c.la/
 c.la
 
-// ZaNiC : http://www.za.net/
-// Confirmed by registry <hostmaster@nic.za.net> 2009-10-03
-za.net
-za.org
+// cloudControl : https://www.cloudcontrol.com/
+// Requested by Tobias Wilken <tw@cloudcontrol.com> 2013-07-23
+cloudcontrolled.com
+cloudcontrolapp.com
+
+// co.ca : http://registry.co.ca/
+co.ca
 
 // CoDNS B.V.
-// Added 2010-05-23.
 co.nl
 co.no
 
-// Mainseek Sp. z o.o. : http://www.co.pl/
-co.pl
+// DreamHost : http://www.dreamhost.com/
+// Requested by Andrew Farmer <andrew.farmer@dreamhost.com> 2012-10-02
+dreamhosters.com
 
 // DynDNS.com : http://www.dyndns.com/services/dns/dyndns/
 dyndns-at-home.com
@@ -6939,12 +6958,104 @@ webhop.org
 worse-than.tv
 writesthisblog.com
 
-// BetaInABox
-// Requested by adrian@betainabox.com 2012-09-13
-betainabox.com
+// Fastly Inc. http://www.fastly.com/
+// Requested by Vladimir Vuksan <vladimir@fastly.com> 2013-05-31
+a.ssl.fastly.net
+b.ssl.fastly.net
+global.ssl.fastly.net
+a.prod.fastly.net
+global.prod.fastly.net
+
+// GitHub, Inc.
+// Requested by Ben Toews <btoews@github.com> 2013-04-18
+github.io
+
+// GlobeHosting, Inc.
+// Requested by Zoltan Egresi <egresi@globehosting.com> 2013-07-12
+ro.com
+
+// Google, Inc.
+// Requested by Eduardo Vela <evn@google.com> 2012-10-24
+appspot.com
+blogspot.be
+blogspot.bj
+blogspot.ca
+blogspot.cf
+blogspot.ch
+blogspot.co.at
+blogspot.co.il
+blogspot.co.nz
+blogspot.co.uk
+blogspot.com
+blogspot.com.ar
+blogspot.com.au
+blogspot.com.br
+blogspot.com.es
+blogspot.cv
+blogspot.cz
+blogspot.de
+blogspot.dk
+blogspot.fi
+blogspot.fr
+blogspot.gr
+blogspot.hk
+blogspot.hu
+blogspot.ie
+blogspot.in
+blogspot.it
+blogspot.jp
+blogspot.kr
+blogspot.mr
+blogspot.mx
+blogspot.nl
+blogspot.no
+blogspot.pt
+blogspot.re
+blogspot.ro
+blogspot.se
+blogspot.sg
+blogspot.sk
+blogspot.td
+blogspot.tw
+codespot.com
+googleapis.com
+googlecode.com
+
+// Heroku : https://www.heroku.com/
+// Requested by Tom Maher <tmaher@heroku.com> 2013-05-02
+herokuapp.com
+herokussl.com
+
+// iki.fi
+// Requested by Hannu Aronsson <haa@iki.fi> 2009-11-05
+iki.fi
+
+// info.at : http://www.info.at/
+biz.at
+info.at
+
+// Michau Enterprises Limited : http://www.co.pl/
+co.pl
+
+// NYC.mn : http://www.information.nyc.mn
+// Requested by Matthew Brown <mattbrown@nyc.mn> 2013-03-11
+nyc.mn
+
+// Opera Software, A.S.A.
+// Requested by Yngve Pettersen <yngve@opera.com> 2009-11-26
+operaunite.com
 
 // Red Hat, Inc. OpenShift : https://openshift.redhat.com/
 // Requested by Tim Kramer <tkramer@rhcloud.com> 2012-10-24
 rhcloud.com
 
+// priv.at : http://www.nic.priv.at/
+// Requested by registry <lendl@nic.at> 2008-06-09
+priv.at
+
+// ZaNiC : http://www.za.net/
+// Requested by registry <hostmaster@nic.za.net> 2009-10-03
+za.net
+za.org
+
 // ===END PRIVATE DOMAINS===

data/build/tld_set_generator.rb

@@ -2,7 +2,7 @@
 # -*- ruby -*-
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-core.rb

@@ -1,6 +1,6 @@
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-core/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,7 +16,7 @@
 
 module Iudex
   module Core
-    VERSION = '1.3.1'
+    VERSION = '1.4.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-core/config.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-core/mojibake.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-core/visit_queue.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-core</artifactId>
   <packaging>jar</packaging>
-  <version>1.3.1</version>
+  <version>1.4.0</version>
   <name>Iudex Core System</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -30,19 +30,19 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-filter</artifactId>
-      <version>[1.3.0,1.3.999)</version>
+      <version>[1.4.0,1.4.999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-http</artifactId>
-      <version>[1.3.0,1.3.999)</version>
+      <version>[1.4.0,1.4.999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-barc</artifactId>
-      <version>[1.3.0,1.3.999)</version>
+      <version>[1.4.0,1.4.999)</version>
     </dependency>
 
     <dependency>

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_charsets.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_content_fetcher.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -45,10 +45,15 @@ module TestHTTPMocks
     def initialize
       super()
       @status = 200
+      @request_headers = []
+    end
+
+    def addRequestHeader( h )
+      @request_headers << h
     end
 
     def requestHeaders
-      [ ]
+      @request_headers
     end
 
     def responseHeaders
@@ -114,6 +119,7 @@ module TestHTTPMocks
 end
 
 class TestContentFetcher < MiniTest::Unit::TestCase
+  include Iudex::HTTP
   include Iudex::Core
   include Iudex::Core::Filters
   include Iudex::Filter::Core
@@ -135,10 +141,25 @@ class TestContentFetcher < MiniTest::Unit::TestCase
       assert_equal( DEFAULT_URL, out.url.to_s )
       assert_equal( 200, out.status )
       assert_equal( WEAK_ETAG, out.etag )
+      assert_equal( "User-Agent", out.request_headers.first.name )
       assert( out.source )
     end
   end
 
+  def test_dynamic_request_headers
+    inp = create_content
+    inp.request_headers = [ Header.new( "Dynamic", "value" ),
+                            Header.new( "User-Agent", "override" ) ]
+    fetch( inp ) do |out|
+      hhash = out.request_headers.inject( {} ) do |m,h|
+        m[ h.name ] = h.value
+        m
+      end
+      assert_equal( { "Dynamic" => "value",
+                      "User-Agent" => "override" }, hhash )
+    end
+  end
+
   def test_304
     client = MockHTTPClient.new
     def client.request( session, handler )
@@ -182,6 +203,7 @@ class TestContentFetcher < MiniTest::Unit::TestCase
     cf = ContentFetcher.new( client,
                              counter,
                              FilterChain.new( "test-rec", [ rec ] ) )
+    cf.request_headers = [ Header.new( "User-Agent", "default" ) ]
     cf.filter( content )
   end
 

data/test/test_content_source.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_log_writer.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_mojibake.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_redirect_handler.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_visit_manager.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -38,6 +38,9 @@ class TestVisitManager < MiniTest::Unit::TestCase
     @latch = CountDownLatch.new( 20 )
 
     @manager = VisitManager.new( TestWorkPoller.new )
+    @manager.do_wait_on_generation = false
+
+    @scheduler = Executors::new_scheduled_thread_pool( 1 )
 
     test_filter = fltr do |order|
       @scheduler.schedule( proc { @manager.release( order, nil ) },
@@ -47,7 +50,6 @@ class TestVisitManager < MiniTest::Unit::TestCase
 
     @manager.filter_chain = FilterChain.new( "test", [ test_filter ] )
 
-    @scheduler = Executors::new_scheduled_thread_pool( 1 )
   end
 
   def teardown

data/test/test_visit_queue.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -171,6 +171,48 @@ class TestVisitQueue < MiniTest::Unit::TestCase
     assert_queue_empty
   end
 
+  def test_configure_type_2
+    @visit_q.config( :delay => 50, :cons => 1 )
+    @visit_q.config( :domain => 'h2.com',
+                     :delay => 75, :cons => 2 )
+    @visit_q.config( :domain => 'h2.com', :type => 'ALT',
+                     :rate => 20, :cons => 1 )
+    @visit_q = @visit_q.clone
+
+    LOG.debug { "As configured:\n" + @visit_q.dump }
+
+    [ %w[   h2     a 2.2 ],
+      %w[ w.h2:AL2 b 2.1 ],
+      %w[   h2:ALT c 3.2 ],
+      %w[   h2:ALT d 3.1 ],
+      %w[   h1:AL2 a 1.2 ],
+      %w[   h1     b 1.1 ] ].each do |oinp|
+
+      @visit_q.add( order( oinp ) )
+
+    end
+
+    LOG.debug { "After add:\n" + @visit_q.dump }
+
+    assert_equal( 3, @visit_q.host_count, "host count" )
+
+    expected = [ %w[   h2:ALT c 3.2 ],
+                 %w[   h2     a 2.2 ],
+                 %w[   h1:AL2 a 1.2 ],
+                 %w[   h2:ALT d 3.1 ],
+                 %w[   h1     b 1.1 ],
+                 %w[ w.h2:AL2 b 2.1 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    LOG.debug { "After consumed:\n" + @visit_q.dump }
+
+    assert_queue_empty
+  end
+
   def test_multi_access_2
     @visit_q.default_max_access_per_host = 2
     add_common_orders
@@ -206,10 +248,63 @@ class TestVisitQueue < MiniTest::Unit::TestCase
                  %w[   h2 c 2.0 ] ]
 
     p = 0
+    concurs = []
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+      concurs << @visit_q.acquired_count
+    end
+
+    assert_operator( 2, :<, concurs.max )
+    assert_queue_empty
+  end
+
+  def test_max_access_total_2
+    @visit_q.default_max_access_per_host = 999
+    @visit_q.max_access_total = 2
+    add_common_orders
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h3 b 3.1 ],
+                 %w[ w.h2 b 2.1 ],
+                 %w[   h1 b 1.1 ],
+                 %w[ m.h3 c 3.0 ],
+                 %w[   h2 c 2.0 ] ]
+
+    p = 0
+    concurs = []
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+      concurs << @visit_q.acquired_count
+    end
+
+    assert_equal( 2, concurs.max )
+    assert_queue_empty
+  end
+
+  def test_max_access_total_1
+    @visit_q.default_max_access_per_host = 999
+    @visit_q.max_access_total = 1
+    add_common_orders
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h3 b 3.1 ],
+                 %w[ w.h2 b 2.1 ],
+                 %w[ m.h3 c 3.0 ],
+                 %w[   h1 b 1.1 ],
+                 %w[   h2 c 2.0 ] ]
+
+    p = 0
+    concurs = []
     expected.each do |o|
       assert_equal( o, acquire_order, p += 1 )
+      concurs << @visit_q.acquired_count
     end
 
+    assert_equal( 1, concurs.max )
     assert_queue_empty
   end
 

data/test/test_visit_url.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

metadata

@@ -2,14 +2,14 @@
 name: iudex-core
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.1
+  version: 1.4.0
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-15 00:00:00.000000000 Z
+date: 2013-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rjack-slf4j
@@ -65,13 +65,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -81,13 +81,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -97,13 +97,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -113,13 +113,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   prerelease: false
   type: :development
@@ -193,7 +193,7 @@ files:
 - test/test_visit_manager.rb
 - test/test_visit_queue.rb
 - test/test_visit_url.rb
-- lib/iudex-core/iudex-core-1.3.1.jar
+- lib/iudex-core/iudex-core-1.4.0.jar
 homepage: http://iudex.gravitext.com
 licenses: []
 post_install_message: