iudex-simhash 1.0.0-java

data/test/setup.rb

@@ -0,0 +1,35 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+
+require 'rjack-logback'
+RJack::Logback.config_console( :stderr => true )
+
+require 'minitest/unit'
+require 'minitest/autorun'
+
+# Make test output logging compatible: no partial lines.
+# class TestOut
+#   def print( *a ); $stdout.puts( *a ); end
+#   def puts( *a );  $stdout.puts( *a ); end
+# end
+# MiniTest::Unit.output = TestOut.new

data/test/test_fuzzy_set.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-simhash'
+
+class TestFuzzySet < MiniTest::Unit::TestCase
+  include Iudex::SimHash::BruteFuzzy
+
+  # Series that will allow all but last at 3 bit threshold, all at 2
+  # bit threshold.
+  TEST_SERIES = [ %w[ FFFF_FFFF_FFFF_FFFF
+                      7FFF_7FFF_7FFF_7FFF
+                      F7FF_F7FF_F7FF_F7FF
+                      FF7F_FF7F_FF7F_FFFF ],
+
+                  %w[ 0000_0000_0000_0000
+                      0100_1000_1000_0010
+                      1000_0100_0100_1000
+                      0010_0010_0010_0001
+                      0001_0001_0001_0000 ],
+
+                  %w[ 0000_0000_0000_0000
+                      0010_0100_0100_0100
+                      0001_0000_1000_1001
+                      0100_1001_0001_0000
+                      0000_0010_0010_0010 ] ]
+
+  def test_hex
+    assert_equal( 0x1000_0000, hex( "1000_0000" ) )
+    assert_equal( 0x7FFF_FFFF_FFFF_FFFF, hex( "7FFF_FFFF_FFFF_FFFF" ) )
+    assert_equal( -1, hex( "FFFF_FFFF_FFFF_FFFF" ) )
+  end
+
+  def test_match
+    m = FuzzyList64.new( 100, 4 )
+    assert(   m.fuzzy_match( 0, 0 ) )
+    assert(   m.fuzzy_match( hex( '7FFF_FFFF_FFFF_FFFF' ),
+                             hex( '7FFF_FFFF_7777_FFFF' ) ) )
+
+    assert(   m.fuzzy_match( hex( 'FFFF_FFFF_FFFF_FFFF' ),
+                             hex( 'FFFF_FFFF_7777_FFFF' ) ) )
+
+    assert( ! m.fuzzy_match( hex( '7FFF_FFFF_FFFF_FFFF' ),
+                             hex( '7FFF_FFFF_EFFF_7777' ) ) )
+
+    assert( ! m.fuzzy_match( hex( 'FFFF_FFFF_FFFF_FFFF' ),
+                             hex( 'FFFF_FFFF_EFFF_7777' ) ) )
+  end
+
+  def test_add
+    m = FuzzyList64.new( 100, 4 )
+    assert(   m.addIfNotFound(  0x0 ) )
+    assert(   m.addIfNotFound( 0xFF ) )
+    assert( ! m.addIfNotFound( 0xFE ) )
+    assert( ! m.addIfNotFound(  0x1 ) )
+  end
+
+  def test_series_list
+    assert_series( FuzzyList64 )
+  end
+
+  def test_series_tree
+    assert_series( FuzzyTree64 )
+  end
+
+  def assert_series( fclz )
+    TEST_SERIES.each do |s|
+      assert_series_last( fclz.new( 5, 3 ), s )
+      assert_series_all(  fclz.new( 5, 2 ), s )
+    end
+  end
+
+  def assert_series_last( fset, s )
+    s = s.dup
+    last = s.pop # Remove last for now
+    assert_series_all( fset, s )
+    assert( ! fset.addIfNotFound( hex( last ) ), last )
+  end
+
+  def assert_series_all( fset, s )
+    s.each { |k| assert( fset.addIfNotFound( hex( k ) ), k ) }
+  end
+
+  def test_find_series_list
+    assert_find_series( FuzzyList64 )
+  end
+
+  def test_find_series_tree
+    assert_find_series( FuzzyTree64 )
+  end
+
+  def assert_find_series( fclz )
+    TEST_SERIES.each do |s|
+      assert_find_series_last( fclz.new( 5, 3 ), s )
+      assert_find_series_all(  fclz.new( 5, 2 ), s )
+    end
+  end
+
+  def assert_find_series_last( fset, s )
+    s = s.dup
+    last = s.pop # Remove last for now
+    assert_find_series_all( fset, s )
+    l = Java::java.util.ArrayList.new;
+    assert( ! fset.addFindAll( hex( last ), l ) )
+    assert( l.size(), 1 );
+
+    # Remove the match and try again.
+    assert( fset.remove( l.get( 0 ) ), "remove match" )
+    assert( fset.remove( hex( last ) ), "remove last" )
+    l.clear
+    assert( ! fset.addFindAll( hex( last ), l ) )
+    assert( l.empty? )
+  end
+
+  def assert_find_series_all( fset, s )
+    s.each do |k|
+      l = Java::java.util.ArrayList.new;
+      assert( ! fset.addFindAll( hex( k ), l ) )
+      assert( l.empty? )
+    end
+  end
+
+  def hex( h )
+    BruteFuzzy::unsignedHexToLong( h.gsub( /_/, '' ) )
+  end
+end

data/test/test_simhash_generator.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-simhash'
+require 'iudex-simhash/factory_helper'
+
+class TestSimhashGenerator < MiniTest::Unit::TestCase
+  include Gravitext::HTMap
+  include Iudex::Core
+  include Iudex::Core::Filters
+  include Iudex::HTML
+  include Iudex::HTML::Filters
+  include Iudex::HTML::Tree
+  include Iudex::HTML::Tree::Filters
+  include Iudex::Filter::Core
+  include Iudex::SimHash::Filters
+
+  include Iudex::SimHash::Filters::FactoryHelper
+
+  import 'iudex.html.HTMLUtils'
+
+  UniMap.define_accessors
+
+  Order = HTMLTreeFilter::Order
+
+  def test_default_stopwords
+    stopwords = simhash_stopwords
+    assert( stopwords.contains( 'from' ) )
+  end
+
+  def test_generate
+    html = <<HTML
+<html>
+ <head>
+  <title>Title</title>
+ </head>
+ <body>
+  <p>We are talking about the same thing here.</p>
+  <p>Really this is the same exact thing I was telling you last time.</p>
+  <p>cruft</p> <!-- Ignored by default 0.3 wordy ratio -->
+ </body>
+</html>
+HTML
+
+    map = content( html )
+    assert( filter_chain.filter( map ) )
+    assert_equal( 'Title', map.title.to_s )
+    assert_equal( 'eaa4172924c0bf6e', hex( map.simhash ) )
+
+    html.gsub!( /the/, "\t" )       # Removing stop words doesn't matter
+    html.gsub!( /cruft/, "xcruft" ) # cruft by any other name...
+    map = content( html )
+    assert( filter_chain.filter( map ) )
+    assert_equal( 'eaa4172924c0bf6e', hex( map.simhash ) )
+  end
+
+  def content( html, charset = "UTF-8" )
+    map = UniMap.new
+    map.source = HTMLUtils::source( html.to_java_bytes, "UTF-8" )
+    map
+  end
+
+  def filter_chain
+    filters = []
+    filters << HTMLParseFilter.new( :source.to_k, nil, :source_tree.to_k )
+    filters << TitleExtractor.new
+    filters << TextCtrlWSFilter.new( :title.to_k )
+
+    tfc = TreeFilterChain.new( [ MetaSkipFilter.new,
+                                 CharactersNormalizer.new,
+                                 WordCounter.new,
+                                 WordyCounter.new ] )
+
+    filters << HTMLTreeFilter.new( :source_tree.to_k, tfc, Order::DEPTH_FIRST )
+
+    filters << simhash_generator
+
+    FilterChain.new( "test", filters )
+  end
+
+  def hex( l )
+    Java::java.lang.Long::toHexString( l )
+  end
+
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-simhash
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-html
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.7.1
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+    type: :development
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :development
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :development
+    version_requirements: *id004
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-simhash gem contains support for generation and
+  searching over simhash fingerprints
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - iudex-simhash-perftest
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - bin/iudex-simhash-perftest
+  - config/stopwords.en
+  - lib/iudex-simhash/base.rb
+  - lib/iudex-simhash.rb
+  - lib/iudex-simhash/factory_helper.rb
+  - lib/iudex-simhash/sim_hash_gen_perf_test_factory.rb
+  - test/setup.rb
+  - test/test_fuzzy_set.rb
+  - test/test_simhash_generator.rb
+  - test/html/gentest.html
+  - lib/iudex-simhash/iudex-simhash-1.0.0.jar
+has_rdoc: true
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-simhash
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_fuzzy_set.rb
+  - test/test_simhash_generator.rb