iudex-filter 1.0.0-java → 1.1.0-java

data/History.rdoc

@@ -1,2 +1,12 @@
+=== 1.1.0 (2011-11-13)
+* Filter chain factory ergonomics:
+  * flatten.compact filters in create_chain, top level filters
+  * main_summary_period, main_by_filter_period on by default
+  * Add listener param to create_chain, create_switch for nested listener
+* New SubListFilter
+* Add FilterException to imports (Siddharth Raghavan)
+* Update to minitest ~> 2.3
+* Update to gravitext-util ~> 1.5.1 (for UniMap.toString)
+
 === 1.0.0 (2011-04-04)
 * Initial release.

data/Manifest.txt

@@ -14,4 +14,5 @@ test/setup.rb
 test/test_filter_base.rb
 test/test_filter_chain_factory.rb
 test/test_proc_filter.rb
-lib/iudex-filter/iudex-filter-1.0.0.jar
+test/test_sublist_filter.rb
+lib/iudex-filter/iudex-filter-1.1.0.jar

data/Rakefile

@@ -4,7 +4,7 @@ $LOAD_PATH << './lib'
 require 'iudex-filter/base'
 
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.2'
+gem     'rjack-tarpit', '~> 1.4'
 require 'rjack-tarpit'
 
 t = RJack::TarPit.new( 'iudex-filter',
@@ -14,10 +14,10 @@ t = RJack::TarPit.new( 'iudex-filter',
 t.specify do |h|
   h.developer( "David Kellum", "dek-oss@gravitext.com" )
   h.extra_deps += [ [ 'rjack-slf4j',    '~> 1.6.1' ],
-                    [ 'gravitext-util', '~> 1.5.0' ] ]
+                    [ 'gravitext-util', '~> 1.5.1' ] ]
 
   h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',      '>= 1.7.1', '< 2.1' ],
+  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
                         [ 'rjack-logback', '>= 1.0' ] ]
 end
 

data/lib/iudex-filter/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module Filter
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-filter/filter_chain_factory.rb

@@ -25,6 +25,9 @@ module Iudex
       class FilterChainFactory
         attr_reader :description
 
+        attr_accessor :main_summary_period
+        attr_accessor :main_by_filter_period
+
         include KeyHelper
 
         def initialize( description = "default" )
@@ -33,20 +36,22 @@ module Iudex
           @log = RJack::SLF4J[ [ RJack::SLF4J.to_log_name( self.class ),
                                  description ].join('.') ]
 
-          @summary_period = nil
-          @by_filter_period = nil
+          @main_summary_period = 10.0
+          @main_by_filter_period = 60.0
 
           @index = nil
           @chain = nil
           @listener = nil
         end
 
+        # Deprecated: Use main_summary_period accessor
         def add_summary_reporter( period_s = 10.0 )
-          @summary_period = period_s
+          @main_summary_period = period_s
         end
 
+        # Deprecated: Use main_by_filter_period accessor
         def add_by_filter_reporter( period_s = 60 * 10.0 )
-          @by_filter_period = period_s
+          @main_by_filter_period = period_s
         end
 
         def open
@@ -54,12 +59,20 @@ module Iudex
 
           @index = FilterIndex.new
 
-          flts = filters
+          # Temp setup of empty listener, since full listeners setup
+          # requires filters, log_and_register which itself requires
+          # listeners via create_chain
+          @listener = place_holder = NoOpListener.new
+
+          flts = filters.flatten.compact
           log_and_register( flts )
 
+          @chain = create_chain( @description, flts, :main )
           @listener = ListenerChain.new( listeners )
-          @chain = create_chain( @description, flts )
-          @chain.listener = @listener
+
+          # Now replace the temp listener with the final listener
+          # chain
+          replace_listeners( @chain, place_holder, @listener )
 
           # With all filters loaded and thus key references, make sure
           # UniMap accessors are defined (for ruby filters)
@@ -101,52 +114,47 @@ module Iudex
           []
         end
 
-        def log_listener( desc )
-          LogListener.new( "iudex.filter.core.FilterChain.#{desc}", @index )
-        end
-
         def listeners
-          ll = [ log_listener( @description ) ]
-
-          if @summary_period
-            ll << SummaryReporter.new( @description, @summary_period )
-          end
-
-          if @by_filter_period
-            ll << ByFilterReporter.new( @index,
-                                        ByFilterLogger.new( @description, @index ),
-                                        @by_filter_period )
-          end
-          ll
+          create_listeners( @description,
+                            @main_summary_period,
+                            @main_by_filter_period )
         end
 
         # Create, yield to optional block, and return FilterChain if
         # flts is not empty. Otherwise return a NoOpFilter and don't
-        # yield. If passed a single Symbol argument, will use both
+        # yield. If passed a Symbol desc and nil flts, will use both
         # as description and method to obtain flts array from.
-        def create_chain( desc, flts = nil )
+        def create_chain( desc, flts = nil, listener = nil )
 
           if desc.is_a?( Symbol )
             flts = send( desc ) unless flts
             desc = desc.to_s.gsub( /_/, '-' )
           end
 
+          flts = flts.flatten.compact if flts
+
           if flts.nil? || flts.empty?
             NoOpFilter.new
           else
             c = FilterChain.new( desc, flts )
-            c.listener = log_listener( desc )
+            if listener.nil?
+              c.listener = log_listener( desc )
+            elsif listener == :main
+              c.listener = @listener
+            else
+              c.listener = listener
+            end
             yield c if block_given?
             c
           end
         end
 
         # Create a new Switch given selector key and map of values to
-        # filters.
+        # filters, or values to [filters,listener]
         def create_switch( key, value_filters_map )
           switch = Switch.new
-          value_filters_map.each do |value, filters|
-            create_chain( value.to_s.downcase, filters ) do |chain|
+          value_filters_map.each do |value, (filters, listener)|
+            create_chain( value.to_s.downcase, filters, listener ) do |chain|
               switch.add_proposition( Selector.new( key, value ), chain )
             end
           end
@@ -155,6 +163,19 @@ module Iudex
 
         private
 
+        def create_listeners( desc, summary_period, by_filter_period )
+          [ log_listener( desc ),
+            ( SummaryReporter.new( desc, summary_period ) if summary_period ),
+            ( ByFilterReporter.new( @index,
+                                    ByFilterLogger.new( desc, @index ),
+                                    by_filter_period ) if by_filter_period )
+          ].compact
+        end
+
+        def log_listener( desc )
+          LogListener.new( "iudex.filter.core.FilterChain.#{desc}", @index )
+        end
+
         def log_and_register( filters, depth = 0 )
           filters.each do |filter|
             name = @index.register( filter )
@@ -164,6 +185,20 @@ module Iudex
             end
           end
         end
+
+        def replace_listeners( filter, place_holder, listener )
+          if filter.kind_of?( FilterContainer )
+            if filter.kind_of?( FilterChain )
+              if filter.listener == place_holder
+                filter.listener = listener
+              end
+            end
+            filter.children.each do |c|
+              replace_listeners( c, place_holder, listener )
+            end
+          end
+        end
+
       end
 
     end

data/lib/iudex-filter.rb

@@ -27,9 +27,11 @@ module Iudex
 
     import 'iudex.filter.Filter'
     import 'iudex.filter.FilterContainer'
+    import 'iudex.filter.FilterException'
     import 'iudex.filter.Described'
     import 'iudex.filter.Named'
     import 'iudex.filter.NoOpFilter'
+    import 'iudex.filter.NoOpListener'
 
     module Core
       import 'iudex.filter.core.ByFilterReporter'
@@ -42,6 +44,7 @@ module Iudex
       import 'iudex.filter.core.MDCUnsetter'
       import 'iudex.filter.core.Selector'
       import 'iudex.filter.core.Setter'
+      import 'iudex.filter.core.SubListFilter'
       import 'iudex.filter.core.SummaryReporter'
       import 'iudex.filter.core.Switch'
     end

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-filter</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.0</version>
+  <version>1.1.0</version>
   <name>Iudex Filter</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <relativePath>..</relativePath>
   </parent>
 

data/test/test_filter_base.rb

@@ -29,6 +29,8 @@ class TestFilterBase < MiniTest::Unit::TestCase
   include Iudex::Filter
   include Gravitext::HTMap
 
+  import 'iudex.filter.FilterListener'
+
   def test_base_name
     f = FilterBase.new
     assert_equal( "i.f.FilterBase", f.name )
@@ -39,4 +41,32 @@ class TestFilterBase < MiniTest::Unit::TestCase
     assert_equal( "TestFilter", f.name )
   end
 
+  # Filter exception may be raised from Ruby, and is handled as per
+  # Java: caught by chain, forwarded to listener as failure
+  def test_raise_filter_exception
+
+    test_filter = FilterBase.new
+    def test_filter.filter( map )
+      raise FilterException.new( "Expected" )
+    end
+
+    listener = FilterListener.new
+    class << listener
+      attr_accessor :fail
+      def failed( filter, map, x )
+        @fail = [ filter, map, x ]
+      end
+    end
+
+    fc = Core::FilterChain.new( "test", [ test_filter ] )
+    fc.listener = listener
+
+    map = UniMap.new
+    refute( fc.filter( map ) )
+
+    assert_equal( test_filter, listener.fail[0] )
+    assert_equal( map,         listener.fail[1] )
+    assert_instance_of( FilterException, listener.fail[2] )
+  end
+
 end

data/test/test_filter_chain_factory.rb

@@ -54,8 +54,8 @@ class TestFilterChainFactory < MiniTest::Unit::TestCase
 
   def test_filter_chain
     fcf = FilterChainFactory.new( "test" )
-    fcf.add_summary_reporter( 1.0 )
-    fcf.add_by_filter_reporter( 2.5 )
+    fcf.main_summary_period = 1.0
+    fcf.main_by_filter_period = 2.5
 
     def fcf.filters
       [ MDCSetter.new( TKEY ) ] + super +
@@ -67,7 +67,7 @@ class TestFilterChainFactory < MiniTest::Unit::TestCase
     end
 
     2.times do |r|
-      assert( ! fcf.open? )
+      refute( fcf.open? )
 
       fcf.filter do |chain|
         1000.times do |t|
@@ -78,9 +78,39 @@ class TestFilterChainFactory < MiniTest::Unit::TestCase
         end
       end
 
-      assert( ! fcf.open? )
+      refute( fcf.open? )
     end
 
   end
 
+  def test_nested_reporting
+    fcf = FilterChainFactory.new( "test" )
+    class << fcf
+      attr_accessor :summary_reporter
+      def filters
+        [ create_chain( :sub_filters, nil, :main ) ]
+      end
+      def listeners
+        super.tap do |ll|
+          @summary_reporter = ll[1] #FIXME: Brittle
+        end
+      end
+
+      def sub_filters
+        [ 6, 4, 6, 6 ].map { |p| RandomFilter.new( p ) }
+      end
+    end
+
+    fcf.filter do |chain|
+      100.times do |t|
+        map = UniMap.new
+        map.tkey = t
+        chain.filter( map )
+      end
+    end
+
+    assert_equal( 100, fcf.summary_reporter.total_count )
+
+  end
+
 end

data/test/test_sublist_filter.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-filter'
+require 'iudex-filter/proc_filter'
+
+class TestSubListFilter < MiniTest::Unit::TestCase
+  include Iudex::Filter
+  include Iudex::Filter::Core
+  include Gravitext::HTMap
+
+  SLIST = UniMap.create_key( 'slist', Java::java.util.List )
+  SVAL  = UniMap.create_key( 'sval' )
+
+  UniMap.define_accessors
+
+  def test_remove_rejects
+    ( -1 .. 2 ).each do |c|
+      map = UniMap.new
+      map.slist = ( [ UniMap.new ] * c if c >= 0 )
+      f = sublist_filter( [ fltr { :reject } ] )
+      f.filter( map )
+
+      refute map.slist
+    end
+  end
+
+  def test_do_not_remove_rejects_when_set
+    [ -1, 1, 2 ].each do |c|
+      map = UniMap.new
+      map.slist = pre = ( [ UniMap.new ] * c if c >= 0 )
+      f = sublist_filter( [ fltr { :reject } ], false )
+      f.filter( map )
+
+      assert_equal( pre, map.slist )
+    end
+  end
+
+  def test_do_not_remove_accepted
+    [ -1, 1, 2 ].each do |c|
+      map = UniMap.new
+      map.slist = pre = ( [ UniMap.new ] * c if c >= 0 )
+      f = sublist_filter( [] ) #accept all
+      f.filter( map )
+
+      assert_equal( pre, map.slist )
+    end
+  end
+
+  def test_modify
+    ( -1 .. 2 ).each do |c|
+      map = UniMap.new
+      map.slist = pre = ( [ UniMap.new ] * c if c >= 0 )
+      f = sublist_filter( [ fltr { |s| s.sval = :mod } ] )
+      f.filter( map )
+
+      assert_equal( Array( pre ).length,
+                    Array( map.slist ).length )
+
+      Array( map.slist ).each do |s|
+        assert_equal( :mod, s.sval )
+      end
+    end
+  end
+
+  def sublist_filter( sfilters, remove_rejects = true )
+     SubListFilter.new( SLIST,
+                        FilterChain.new( "sub", sfilters ),
+                        remove_rejects )
+  end
+
+end

metadata

@@ -2,7 +2,7 @@
 name: iudex-filter
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.0
+  version: 1.1.0
 platform: java
 authors: 
   - David Kellum
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-04 00:00:00 -07:00
-default_executable: 
+date: 2011-11-13 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: rjack-slf4j
@@ -32,7 +31,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.5.0
+            version: 1.5.1
     type: :runtime
     version_requirements: *id002
   - !ruby/object:Gem::Dependency 
@@ -41,12 +40,9 @@ dependencies:
     requirement: &id003 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.7.1
-        - - <
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "2.1"
+            version: "2.3"
     type: :development
     version_requirements: *id003
   - !ruby/object:Gem::Dependency 
@@ -68,7 +64,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.3.0
+            version: 1.4.0
     type: :development
     version_requirements: *id005
 description: |-
@@ -102,8 +98,9 @@ files:
   - test/test_filter_base.rb
   - test/test_filter_chain_factory.rb
   - test/test_proc_filter.rb
-  - lib/iudex-filter/iudex-filter-1.0.0.jar
-has_rdoc: true
+  - test/test_sublist_filter.rb
+  - lib/iudex-filter/iudex-filter-1.1.0.jar
+  - .gemtest
 homepage: http://github.com/dekellum/iudex
 licenses: []
 
@@ -128,11 +125,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: iudex-filter
-rubygems_version: 1.5.1
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java
 test_files: 
   - test/test_proc_filter.rb
+  - test/test_sublist_filter.rb
   - test/test_filter_base.rb
   - test/test_filter_chain_factory.rb