iudex-simhash 1.0.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-04-04)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+bin/iudex-simhash-perftest
+config/stopwords.en
+lib/iudex-simhash/base.rb
+lib/iudex-simhash.rb
+lib/iudex-simhash/factory_helper.rb
+lib/iudex-simhash/sim_hash_gen_perf_test_factory.rb
+test/setup.rb
+test/test_fuzzy_set.rb
+test/test_simhash_generator.rb
+test/html/gentest.html
+lib/iudex-simhash/iudex-simhash-1.0.0.jar

data/README.rdoc

@@ -0,0 +1,33 @@
+= iudex-simhash
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-simhash gem contains support for generation and
+searching over simhash fingerprints
+
+== Dependencies
+
+* Java 1.5+
+
+For tests:
+
+* JRuby 1.3+
+
+== License
+
+Copyright (c) 2010-2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,39 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-simhash/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.2'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-simhash',
+                       Iudex::SimHash::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+  h.extra_deps += [ [ 'iudex-html',     '~> 1.0.0' ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',      '>= 1.7.1', '< 2.1' ],
+                        [ 'rjack-logback', '~> 1.0' ] ]
+end
+
+file 'Manifest.txt' => [ 'pom.xml' ]
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+t.define_tasks

data/bin/iudex-simhash-perftest

@@ -0,0 +1,104 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#.hashdot.vm.options += -Xmx1g
+# For 64b add: -XX:+UseCompressedOops
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'rjack-logback'
+
+require 'iudex-simhash'
+require 'iudex-simhash/sim_hash_gen_perf_test_factory'
+
+require 'optparse'
+
+require 'gravitext-util'
+require 'gravitext-util/perftest'
+
+module PerfTest
+  include Iudex::SimHash::BruteFuzzy
+  import 'iudex.simhash.brutefuzzy.FuzzySetPerfTest'
+
+  MD = FuzzySetPerfTest::Mode
+
+  def self.run( args = ARGV )
+
+    options = {}
+    oparser = OptionParser.new do |opts|
+      opts.banner = <<USAGE
+Usage: iudex-simhash-perftest (fuzzy|gen) [options] <length> <threshold-bits>
+USAGE
+      opts.on( "-t", "--threads N", Integer, "Test with thread count" ) do |n|
+        options[ :threads ] = n
+      end
+      opts.on( "--tree-bits=N", Integer,
+               "Test TREE only, with segments from specified minimum bits" ) do |n|
+        options[ :tree_bits ] = n
+      end
+      opts.on( "--capacity=N", Integer,
+               "Set initial capacity to N (default: length)" ) do |n|
+        options[ :capacity ] = n
+      end
+
+    end
+
+    oparser.parse!
+
+    tests = case( ARGV.shift )
+            when 'fuzzy'
+              fuzzy_tests( options )
+            when 'gen'
+              gen_test
+            else
+              puts oparser
+              exit 1
+            end
+
+    harness = Gravitext::PerfTest::Harness.new( tests )
+
+    harness.thread_count = options[ :threads ] if options[ :threads ]
+    harness.execute
+  end
+
+  def self.fuzzy_tests( options )
+    length = ARGV.shift.to_i || 1000
+    threshold = ARGV.shift.to_i || 3
+
+    min_bits = options[ :tree_bits ] || 0
+    segbits = [ 0, 4, 8, 16 ].select do |b|
+      ( b == 0 || ( ( 64 / b ) > threshold ) ) && ( b >= min_bits )
+    end
+
+    segbits.map do |bits|
+      mode = ( bits == 0 ) ? MD::LIST : MD::TREE
+      pt = FuzzySetPerfTest.new( mode, length, threshold )
+      pt.max_bits = bits if bits > 0
+      pt.initial_capacity = options[ :capacity ] if options[ :capacity ]
+      pt
+    end
+  end
+
+  def self.gen_test
+    [ SimHashGenPerfTestFactory.new.perf_test ]
+  end
+
+end
+
+PerfTest.run

data/config/stopwords.en

@@ -0,0 +1,35 @@
+# Common 3-4 character english words to be dropped.
+# Note that 1-2 character words are already dropped by the Tokenizer.
+all
+and
+are
+been
+but
+can
+did
+does
+for
+from
+get
+had
+have
+isn
+its
+not
+one
+our
+out
+said
+say
+that
+the
+they
+this
+was
+were
+what
+when
+will
+with
+you
+your

data/lib/iudex-simhash.rb

@@ -0,0 +1,44 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-html'
+
+require 'iudex-simhash/base.rb'
+
+require 'java'
+
+module Iudex
+  module SimHash
+
+    require "iudex-simhash/iudex-simhash-#{VERSION}.jar"
+
+    import 'iudex.simhash.SimHashKeys'
+
+    module BruteFuzzy
+      import 'iudex.simhash.brutefuzzy.BruteFuzzy'
+      import 'iudex.simhash.brutefuzzy.FuzzyList64'
+      import 'iudex.simhash.brutefuzzy.FuzzyTree64'
+    end
+
+    module Filters
+      import 'iudex.simhash.filters.SimHashGenerator'
+    end
+
+    module Gen
+      import 'iudex.simhash.gen.StopWordSet'
+    end
+  end
+end

data/lib/iudex-simhash/base.rb

@@ -0,0 +1,21 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module SimHash
+    VERSION = '1.0.0'
+  end
+end

data/lib/iudex-simhash/factory_helper.rb

@@ -0,0 +1,65 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-simhash'
+require 'iudex-filter/key_helper'
+
+module Iudex
+  module SimHash
+    module Filters
+      module FactoryHelper
+        include Iudex::Core
+        include Iudex::HTML
+
+        DEFAULT_WORDS = File.join( File.dirname( __FILE__ ), '..', '..',
+                                   'config', 'stopwords.en' )
+
+        def simhash_stopwords( wfile = DEFAULT_WORDS )
+          words = File.open( wfile ) { |fin| fin.readlines }
+          words.map! { |w| w.strip }
+          words.reject! { |w| w =~ /^#/ }
+
+          Gen::StopWordSet.new( words )
+        end
+
+        Element = Java::com.gravitext.xml.tree.Element
+
+        def simhash_generator( input = :simhash_generator_inputs,
+                               stopwords = simhash_stopwords )
+
+          inputs = send( input ).map { |r| r.to_a }.map do | key, ratio |
+            key = key.to_k
+            i = if( key.value_type == Element.java_class )
+                  SimHashGenerator::Input.forTree( key )
+                else
+                  SimHashGenerator::Input.forText( key )
+                end
+            i.wordy_ratio = ratio if ratio
+            i
+          end
+
+          SimHashGenerator.new( inputs, stopwords )
+        end
+
+        def simhash_generator_inputs
+          [ [ :title ],
+            [ :source_tree, 0.30 ] ]
+        end
+
+      end
+    end
+  end
+end

data/lib/iudex-simhash/sim_hash_gen_perf_test_factory.rb

@@ -0,0 +1,80 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-simhash'
+require 'iudex-simhash/factory_helper'
+
+class SimHashGenPerfTestFactory
+  include Gravitext::HTMap
+  include Iudex::Core
+  include Iudex::Core::Filters
+  include Iudex::HTML
+  include Iudex::HTML::Filters
+  include Iudex::HTML::Tree
+  include Iudex::HTML::Tree::Filters
+  include Iudex::Filter::Core
+  include Iudex::SimHash::Filters
+
+  include Iudex::SimHash::Filters::FactoryHelper
+
+  import 'iudex.html.HTMLUtils'
+
+  Order = HTMLTreeFilter::Order
+
+  import 'iudex.simhash.filters.SimHashGenPerfTest'
+
+  def initialize
+    UniMap.define_accessors
+  end
+
+  def perf_test
+
+    # Initial parse
+    map = content
+    filter_chain.filter( map )
+
+    SimHashGenPerfTest.new( map, simhash_generator )
+  end
+
+  def content
+    map = UniMap.new
+
+    html = File.read( File.join( File.dirname( __FILE__ ),  '..', '..',
+                      'test', 'html', 'gentest.html' ) )
+
+    map.source = HTMLUtils::source( html.to_java_bytes, "UTF-8" )
+    map
+  end
+
+  def filter_chain
+    filters = []
+    filters << HTMLParseFilter.new( ContentKeys::SOURCE,
+                                    nil, HTMLKeys::SOURCE_TREE )
+    filters << TitleExtractor.new
+    filters << TextCtrlWSFilter.new( ContentKeys::TITLE )
+
+    tfc = TreeFilterChain.new( [ MetaSkipFilter.new,
+                                 CharactersNormalizer.new,
+                                 WordCounter.new,
+                                 WordyCounter.new ] )
+
+    filters << HTMLTreeFilter.new( HTMLKeys::SOURCE_TREE,
+                                   tfc, Order::DEPTH_FIRST )
+
+    FilterChain.new( "perf_test", filters )
+  end
+
+end