RubyGems - rdig - Versions diffs - 0.1.0 - Mend

rdig 0.1.0

Files changed (28) hide show

data/CHANGES +2 -0
data/LICENSE +20 -0
data/README +61 -0
data/TODO +0 -0
data/bin/rdig +32 -0
data/doc/examples/config.rb +53 -0
data/install.rb +89 -0
data/lib/htmlentities/CHANGES +21 -0
data/lib/htmlentities/COPYING +7 -0
data/lib/htmlentities/README +15 -0
data/lib/htmlentities/htmlentities.rb +281 -0
data/lib/rdig.rb +243 -0
data/lib/rdig/content_extractors.rb +145 -0
data/lib/rdig/crawler.rb +176 -0
data/lib/rdig/highlight.rb +24 -0
data/lib/rdig/http_client.rb +22 -0
data/lib/rdig/index.rb +39 -0
data/lib/rdig/search.rb +77 -0
data/lib/rdig/url_filters.rb +171 -0
data/rakefile +325 -0
data/test/fixtures/html/custom_tag_selectors.html +25 -0
data/test/fixtures/html/entities.html +15 -0
data/test/fixtures/html/simple.html +17 -0
data/test/test_helper.rb +18 -0
data/test/unit/etag_filter_test.rb +23 -0
data/test/unit/html_content_extractor_test.rb +64 -0
data/test/unit/url_filters_test.rb +96 -0
metadata +102 -0

data/test/fixtures/html/custom_tag_selectors.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+	<head>
+    <meta http-equiv="content-type" content="text/html;charset=iso-8859-1">
+    <meta name="DC.title" content="Title from DC meta data" />
+    <title>Sample Title</title>
+  </head>
+  <body>
+    <h1 class="title">Sample Title in h1</h1>
+    <a href="http://test.host/outside.html">Affe</a>
+    <h1>Some sample <span>text</span></h1>
+    <div id="content">
+      <a href="/inside.html">Affe</a>
+      <!-- some comment -->
+      <p>
+      Real content is here.</p>
+    </div>
+    <!-- another comment
+    here -->
+    <p>Some footer</p>
+    <a href="#foo">Top</a>
+    <a href="/footer.html">Affe</a>
+  </body>
+</html>

data/test/fixtures/html/entities.html ADDED Viewed

@@ -0,0 +1,15 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+	<head>
+    <meta http-equiv="content-type" content="text/html;charset=iso-8859-1">
+    <title>Sample &amp; Title</title>
+  </head>
+  <body>
+    <h1>Some &gt; Links</h1>
+    <p>don't&nbsp;break me!</p>
+    <a href="http://test.host/affe.html?b=a&amp;c=d">Affe</a>
+    <a href="http://test.host/affe2.html?b=a&c=d">Affe</a>
+    <h1>&Uuml;ml&auml;uts</h1>
+    <p>hei&szlig; hier &#223;</p>
+  </body>
+</html>

data/test/fixtures/html/simple.html ADDED Viewed

@@ -0,0 +1,17 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+	<head>
+    <meta http-equiv="content-type" content="text/html;charset=iso-8859-1">
+    <title>Sample Title</title>
+  </head>
+  <body>
+    <h1>A Link</h1>
+    <a href="http://test.host/affe.html">Affe</a>
+    <h1>Some sample <span>text</span></h1>
+    <!-- invalid markup follows -->
+    <p>Lorem<br>
+    <!-- another comment
+    here -->
+    ipsum
+  </body>
+</html>

data/test/test_helper.rb ADDED Viewed

@@ -0,0 +1,18 @@
+require 'test/unit'
+require 'rdig'
+#File.expand_path(File.dirname(__FILE__) + "/../init.rb")
+# require File.expand_path(File.dirname(__FILE__) + "/../init.rb")
+module TestHelper
+  include RDig
+  def read_fixture(path)
+    File.open("#{File.expand_path(File.dirname(__FILE__))}/fixtures/#{path}") { |f|
+      f.read
+    }
+  end
+  def html_doc(name)
+    read_fixture("html/#{name}.html")
+  end
+end

data/test/unit/etag_filter_test.rb ADDED Viewed

@@ -0,0 +1,23 @@
+require 'test_helper'
+class ETagFilterTest < Test::Unit::TestCase
+  include TestHelper, RDig
+  def setup
+    @filter = ETagFilter.new
+  end
+  def test_add
+    d0 = OpenStruct.new(:etag => nil)
+    assert @filter.apply(d0)
+    d1 = OpenStruct.new(:etag => 'abc1234')
+    assert @filter.apply(d1)
+    assert !@filter.apply(d1)
+    d2 = OpenStruct.new(:etag => 'abc1235')
+    assert @filter.apply(d2)
+    assert !@filter.apply(d2)
+    assert !@filter.apply(d1)
+  end
+end

data/test/unit/html_content_extractor_test.rb ADDED Viewed

@@ -0,0 +1,64 @@
+require 'test_helper'
+class HtmlContentExtractorTest < Test::Unit::TestCase
+  include TestHelper
+  def setup
+    @extractor = ContentExtractors::HtmlContentExtractor
+    @nbsp = [160].pack('U') # non breaking space
+    @config_backup = RDig.config.content_extraction.html.clone
+  end
+  def teardown
+    RDig.config.content_extraction.html = @config_backup
+  end
+  def test_simple
+    result = @extractor.process(html_doc('simple'))
+    assert_not_nil result
+    assert_equal 'Sample Title', result[:title]
+    assert_not_nil result[:content]
+    assert_not_nil result[:links]
+    assert_equal 1, result[:links].size
+    assert_equal 'A Link Affe Some sample text Lorem ipsum', result[:content]
+    assert_equal 'http://test.host/affe.html', result[:links].first
+  end
+  def test_entities
+    result = @extractor.process(html_doc('entities'))
+    assert_equal 'Sample & Title', result[:title]
+    assert_equal 'http://test.host/affe.html?b=a&c=d', result[:links].first
+    assert_equal 'http://test.host/affe2.html?b=a&c=d', result[:links].last
+    assert_equal "Some > Links don't#{@nbsp}break me! Affe Affe Ümläuts heiß hier ß", result[:content]
+  end
+  def test_custom_content_element
+    RDig.configuration do |config|
+      config.content_extraction.html.title_tag_selector = lambda do |tagsoup|
+        tagsoup.find('h1', :attrs => { 'class', 'title' })
+      end
+      config.content_extraction.html.content_tag_selector = lambda do |tagsoup|
+        tagsoup.find('div', :attrs => { 'id', 'content' })
+      end
+    end
+    result = @extractor.process(html_doc('custom_tag_selectors'))
+    assert_equal 'Sample Title in h1', result[:title]
+    assert_equal 'Affe Real content is here.', result[:content]
+    # check if links are collected outside the content tag, too:
+    assert_equal 3, result[:links].size
+    assert_equal 'http://test.host/outside.html', result[:links].first
+    assert_equal '/inside.html', result[:links][1]
+    assert_equal '/footer.html', result[:links][2]
+  end
+  def test_title_from_dcmeta
+    RDig.configuration do |config|
+      config.content_extraction.html.title_tag_selector = lambda do |tagsoup|
+        tagsoup.find('meta', :attrs => { 'name', 'DC.title' })['content']
+      end
+    end
+    result = @extractor.process(html_doc('custom_tag_selectors'))
+    assert_equal 'Title from DC meta data', result[:title]
+  end
+end

data/test/unit/url_filters_test.rb ADDED Viewed

@@ -0,0 +1,96 @@
+require 'test_helper'
+class UrlFilterTest < Test::Unit::TestCase
+  include TestHelper, RDig
+  def setup
+  end
+  # test a chain configured with direct parameters
+  def test_filterchain
+    cfg = [
+      { UrlFilters::UrlInclusionFilter => /.+html$/ },
+      { :hostname_filter => 'test.host' }
+    ]
+    chain = UrlFilters::FilterChain.new(cfg)
+    assert_nil chain.apply(Document.new("http://test.host/affe.htm"))
+    assert_not_nil chain.apply(Document.new("http://test.host/affe.html"))
+    assert_nil chain.apply(Document.new("http://test.host.com/affe.html"))
+  end
+  # test default chain config
+  def test_default_filterchain
+    chain = UrlFilters::FilterChain.new(RDig.filter_chain)
+    assert_nil chain.apply(Document.new("http://www.example.com/affe.htm"))
+    assert_not_nil chain.apply(Document.new("http://localhost:3000/affe.html"))
+    assert_nil chain.apply(Document.new("http://localhost.com/affe.html"))
+  end
+  # check lookup of chain parameters from config
+  def test_filterchain_config
+    RDig.configuration do |conf|
+      conf.crawler.include_patterns = /.+html$/
+      conf.crawler.include_hosts = 'test.host'
+    end
+    cfg = [
+      { UrlFilters::UrlInclusionFilter => :include_patterns },
+      { :hostname_filter => :include_hosts }
+    ]
+    chain = UrlFilters::FilterChain.new(cfg)
+    assert_nil chain.apply(Document.new("http://test.host/affe.htm"))
+    assert_not_nil chain.apply(Document.new("http://test.host/affe.html"))
+    assert_nil chain.apply(Document.new("http://test.host.com/affe.html"))
+  end
+  def test_urlpattern_filter
+    f = UrlFilters::UrlInclusionFilter.new(/.*\.html$/)
+    assert_nil f.apply(Document.new("http://test.host/affe.htm"))
+    assert_not_nil f.apply(Document.new("http://test.host/affe.html"))
+    f = UrlFilters::UrlExclusionFilter.new([ /.*\.html$/, /.*\.aspx/ ])
+    assert_not_nil f.apply(Document.new("http://test.host/affe.htm"))
+    assert_nil f.apply(Document.new("http://test.host/affe.html"))
+    assert_nil f.apply(Document.new("http://test.host/affe.aspx"))
+    f = UrlFilters::UrlExclusionFilter.new([ /http:\/\/[^\/]+\/dir1/ ])
+    assert_nil f.apply(Document.new("http://test.host/dir1/affe.aspx"))
+    assert_not_nil f.apply(Document.new("http://test.host/dir2/dir1/affe.htm"))
+    assert_not_nil f.apply(Document.new("http://test.host/affe.htm"))
+    assert_not_nil f.apply(Document.new("http://test.host/dir2/affe.htm"))
+    f = UrlFilters::UrlExclusionFilter.new([ /\/dir1/ ])
+    assert_nil f.apply(Document.new("http://test.host/dir1/affe.aspx"))
+    assert_nil f.apply(Document.new("http://test.host/dir2/dir1/affe.htm"))
+    assert_not_nil f.apply(Document.new("http://test.host/affe.htm"))
+    assert_not_nil f.apply(Document.new("http://test.host/dir2/affe.htm"))
+  end
+  def test_hostname_filter
+    include_hosts = [ 'test.host', 'localhost' ]
+    assert_nil UrlFilters.hostname_filter(Document.new('http://google.com/'), include_hosts)
+    assert_not_nil UrlFilters.hostname_filter(Document.new('http://test.host/file.html'), include_hosts)
+    assert_not_nil UrlFilters.hostname_filter(Document.new('http://localhost/file.html'), include_hosts)
+  end
+  def test_fix_relative_uri
+    doc = Document.new('http://test.host/dir/file.html')
+    assert_equal('http://test.host/dir/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('another.html', doc.uri)).uri.to_s)
+    assert_equal('http://test.host/dir/../another.html',
+                  UrlFilters.fix_relative_uri(Document.new('../another.html', doc.uri)).uri.to_s)
+    assert_equal('http://test.host/dir/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('/dir/another.html', doc.uri)).uri.to_s)
+    assert_equal('http://test.host/dir/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('http://test.host/dir/another.html', doc.uri)).uri.to_s)
+    assert_equal('HTTP://test.host/dir/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('HTTP://test.host/dir/another.html', doc.uri)).uri.to_s)
+    doc = Document.new('https://test.host/dir/')
+    assert_equal('https://test.host/dir/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('another.html', doc.uri)).uri.to_s)
+    doc = Document.new('https://test.host/')
+    assert_equal('https://test.host/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('another.html', doc.uri)).uri.to_s)
+    doc = Document.new('https://test.host')
+    assert_equal('https://test.host/another.html',
+                  UrlFilters.fix_relative_uri(Document.new('another.html', doc.uri)).uri.to_s)
+  end
+end

metadata ADDED Viewed

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+rubygems_version: 0.8.10
+specification_version: 1
+name: rdig
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+date: 2006-03-25
+summary: Ruby based web site indexing and searching library.
+require_paths:
+  - lib
+email: jk@jkraemer.net
+homepage: http://rdig.rubyforge.org/
+rubyforge_project: rdig
+description: "RDig provides an HTTP crawler and content extraction utilities to help building
+  a site search for web sites or intranets. Internally, Ferret is used for the
+  full text indexing. After creating a config file  for your site, the index can
+  be built with a single call to rdig."
+autorequire:
+default_executable: rdig
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement
+  requirements:
+    -
+      - ">"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+  version:
+platform: ruby
+authors:
+  - Jens Kraemer
+files:
+  - bin/rdig
+  - lib/rdig
+  - lib/htmlentities
+  - lib/rdig.rb
+  - lib/rdig/http_client.rb
+  - lib/rdig/crawler.rb
+  - lib/rdig/search.rb
+  - lib/rdig/highlight.rb
+  - lib/rdig/index.rb
+  - lib/rdig/url_filters.rb
+  - lib/rdig/content_extractors.rb
+  - lib/htmlentities/CHANGES
+  - lib/htmlentities/COPYING
+  - lib/htmlentities/README
+  - lib/htmlentities/htmlentities.rb
+  - test/unit
+  - test/fixtures
+  - test/test_helper.rb
+  - test/unit/etag_filter_test.rb
+  - test/unit/url_filters_test.rb
+  - test/unit/html_content_extractor_test.rb
+  - test/fixtures/html
+  - test/fixtures/html/entities.html
+  - test/fixtures/html/simple.html
+  - test/fixtures/html/custom_tag_selectors.html
+  - doc/examples
+  - doc/examples/config.rb
+  - LICENSE
+  - TODO
+  - CHANGES
+  - README
+  - install.rb
+  - rakefile
+test_files: []
+rdoc_options:
+  - "--title"
+  - "Rake -- Ruby Make"
+  - "--main"
+  - README
+  - "--line-numbers"
+extra_rdoc_files:
+  - README
+  - CHANGES
+  - LICENSE
+  - TODO
+executables:
+  - rdig
+extensions: []
+requirements: []
+dependencies:
+  - !ruby/object:Gem::Dependency
+    name: ferret
+    version_requirement:
+    version_requirements: !ruby/object:Gem::Version::Requirement
+      requirements:
+        -
+          - ">="
+          - !ruby/object:Gem::Version
+            version: 0.3.2
+      version:
+  - !ruby/object:Gem::Dependency
+    name: rubyful_soup
+    version_requirement:
+    version_requirements: !ruby/object:Gem::Version::Requirement
+      requirements:
+        -
+          - ">="
+          - !ruby/object:Gem::Version
+            version: 1.0.4
+      version: