ariel 0.0.1

data/test/test_rule.rb

@@ -0,0 +1,38 @@
+require 'ariel'
+require 'ariel_test_case'
+
+class TestRule < Ariel::TestCase
+  def setup
+    @labeled=Ariel::TokenStream.new
+    @labeled.tokenize("90 Colfax, <b> Palms </b>, Phone: (818) 508-1570")
+    @labeled.set_label_at 35
+    @perfect_rule=Ariel::Rule.new(:forward, [["Phone"], ["("]])
+    @early_rule=Ariel::Rule.new(:forward, [[:anything]])
+    @late_rule=Ariel::Rule.new(:forward, [["508"]])
+    @unlabeled=Ariel::TokenStream.new
+    @unlabeled.tokenize("Robot 9753 reporting for duty. BEEP BEEP")
+  end
+
+  def test_apply_to
+    md=nil
+    @perfect_rule.apply_to(@labeled) {|md|}
+    assert_equal :perfect, md.type
+    @early_rule.apply_to(@labeled) {|md|}
+    assert_equal :early, md.type
+    assert_equal 1, md.token_loc
+    @late_rule.apply_to(@labeled) {|md|}
+    assert_equal :late, md.type
+    assert_nil (@perfect_rule.apply_to(@unlabeled))
+  end
+
+  def test_matches
+    assert(@early_rule.matches(@labeled, :early))
+    assert(@late_rule.matches(@labeled, :early, :late))
+    assert(@perfect_rule.matches(@unlabeled, :fail))
+  end
+
+  def test_wildcard_count
+    assert_equal 0, @perfect_rule.wildcard_count
+    assert_equal 1, @early_rule.wildcard_count
+  end
+end

data/test/test_structure_node.rb

@@ -0,0 +1,81 @@
+require 'ariel'
+require 'ariel_test_case'
+
+class TestStructureNode < Ariel::TestCase
+  def setup
+    @tree=Ariel::StructureNode.new do |r|
+      r.item :item_info do |i|
+        i.item :title
+        i.item :price
+        i.item :stock_level
+      end
+    end
+  end
+  def test_unnested
+    t=Ariel::StructureNode.new {|r| r.item :picture; r.item :title; r.item :description; r.item :url}
+    assert t
+    assert_equal Ariel::StructureNode, t.picture.class
+    assert_equal :root, t.meta.name
+  end
+
+  def test_nested
+    assert @tree.item_info.children.has_key?(:title)
+  end
+
+  def test_nested_with_list
+    doc_tree=Ariel::StructureNode.new do |r|
+      r.item :restaurant_list do |r|
+        r.list_item :restaurant do |r|
+          r.item :name
+          r.item :address
+          r.item :phone
+          r.item :review
+          r.item :credit_card_list do |c|
+            c.item :credit_card
+          end
+        end
+      end
+    end
+    assert doc_tree
+    assert_equal :list, doc_tree.restaurant_list.restaurant.meta.node_type
+  end
+
+  def test_extend_structure
+    assert (@tree.extend_structure {|r| r.item :site_copyright; r.item :logo;})
+    assert @tree.children.has_key?(:site_copyright)
+    assert @tree.children.has_key?(:logo)
+    assert (@tree.item_info.extend_structure {|i| i.item :picture})
+    assert @tree.item_info.children.has_key?(:picture)
+  end
+
+#   def test_apply_extraction_tree_on
+# #     t = Ariel::StructureNode.new do |r|
+# # #       r.title
+# # # #       r.content do |c|
+# # # # #         c.excerpt
+# # # # # #         c.body
+# # # # # # #       end
+# # # # # # # #     end
+# # # # # # # # #     str = %q{Title: The test of the Century
+# # # # # # # # # # <b>Excerpt</b>: <i>A look back at what could be considered the greatest ever test.</i>
+# # # # # # # # # # # There was once a test designed to assess whether apply_extraction_tree_on worked.}
+# # # # # # # # # # # #     tokenstream = Ariel::TokenStream.new
+# # # # # # # # # # # # #     tokenstream.tokenize(str)
+# # # # # # # # # # # # # #     root = Ariel::ExtractedNode.new(tokenstream, :structure=>t, :name=>:root)
+# # # # # # # # # # # # # # #     t.title.meta.start_rule = Ariel::Rule.new(["Title", ":"])
+# # # # # # # # # # # # # # # #     t.title.meta.end_rule = Ariel::Rule.new(["<b>"])
+# # # # # # # # # # # # # # # # #     t.title.meta.end_rule.direction = :back
+# # # # # # # # # # # # # # # # # #     t.content.meta.start_rule = Ariel::Rule.new(["Century"]) #later implementation might use skip_until("<b>")
+# # # # # # # # # # # # # # # # # # #     t.content.meta.end_rule = Ariel::Rule.new()
+# # # # # # # # # # # # # # # # # # # #     t.content.meta.end_rule.direction = :back
+# # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.start_rule = Ariel::Rule.new(["<i>"])
+# # # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.end_rule = Ariel::Rule.new([".</"])
+# # # # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.end_rule.direction = :back
+# # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.start_rule = Ariel::Rule.new(["i", ">"])
+# # # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.end_rule = Ariel::Rule.new()
+# # # # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.end_rule.direction = :back
+# # # # # # # # # # # # # # # # # # # # # # # # # # #     t.apply_extraction_tree_on root
+#  end
+
+
+end

data/test/test_token.rb

@@ -0,0 +1,16 @@
+require 'ariel'
+require 'ariel_test_case'
+
+class TestToken < Ariel::TestCase
+  def setup
+    @t=Ariel::Token.new('Test', 0, 4)
+  end
+
+  def test_matches?
+    assert @t.matches?('Test')
+    assert_equal false, @t.matches?('test')
+    assert_equal false, @t.matches?('te')
+    assert @t.matches?(:alpha)
+    assert_equal false, @t.matches?(:html_tag)
+  end
+end

data/test/test_token_stream.rb

@@ -0,0 +1,82 @@
+require 'ariel'
+require 'ariel_test_case'
+
+class TestTokenStream < Ariel::TestCase
+  include Fixtures
+  
+  def setup
+    @stream=Ariel::TokenStream.new
+    @text = "This is test101. See below:"
+    @stream.tokenize(@text)
+    
+    @labeled_stream = Ariel::TokenStream.new
+    @labeled_stream.tokenize(@@labeled_document, true)
+  end
+  
+  def test_advance
+    assert_equal Ariel::Token.new("This", 0, 4), @stream.advance
+  end
+
+  def test_cur_pos
+    assert_equal 0, @stream.cur_pos
+    @stream.advance
+    assert_equal 1, @stream.cur_pos
+  end
+
+  def test_each
+    i=0
+    @stream.each {i=i+1}
+    assert_equal 8, i
+    assert_equal 9, @stream.cur_pos
+  end
+
+  def test_rewind
+    @stream.each {}
+    @stream.rewind
+    assert_equal 0, @stream.cur_pos
+  end
+
+  def test_skip_to
+    assert @stream.skip_to("This")
+    assert_equal 1, @stream.cur_pos  #Test the matched token has been consumed
+    assert_nil @stream.skip_to("Ruby")
+    assert_equal 1, @stream.cur_pos  #Stream's position remains unchanged by a failed match
+    assert @stream.skip_to("See", "below")
+    assert_equal 7, @stream.cur_pos
+    @stream.rewind
+    @stream.skip_to(:anything, "below")
+    assert_equal 7, @stream.cur_pos
+  end
+
+  def test_tokenize
+    assert_equal 8, @stream.tokens.length
+    @stream.each do |token|
+      assert_equal @text[token.start_loc...token.end_loc], token.text
+    end
+    @labeled_stream.each do |token|
+      assert_equal @@labeled_document[token.start_loc...token.end_loc], token.text
+    end
+  end
+
+  def test_set_label_at
+    assert_raise(ArgumentError) {@stream.set_label_at 1}
+    assert_nil @stream.label_index
+    assert(@labeled_stream.set_label_at(16))
+    assert_equal("The", @labeled_stream.tokens[@labeled_stream.label_index].text)
+  end
+
+  def test_raw_text
+    assert_equal @text, @stream.raw_text
+    assert_equal @@labeled_document.chomp, @labeled_stream.raw_text
+  end
+
+  def test_text
+    assert_equal @text, @stream.text
+    assert_equal @@unlabeled_document.chomp, @labeled_stream.text
+  end
+
+  def test_slice_by_token_index
+    assert sliced=@stream.slice_by_token_index(1,3)
+    assert_equal @text[sliced.tokens.first.start_loc...sliced.tokens.last.end_loc], sliced.text
+  end
+end

data/test/test_wildcards.rb

@@ -0,0 +1,18 @@
+require 'ariel'
+require 'ariel_test_case'
+
+class TestWildcards < Ariel::TestCase
+  
+  def test_list
+    assert (wildcards=Ariel::Wildcards.list)
+    assert (wildcards.kind_of? Hash)
+  end
+
+  def test_matching
+    assert matches=Ariel::Wildcards.matching("123")
+    assert (matches.include? :alpha_numeric)
+    assert (matches.include? :numeric)
+    assert (matches.include? :anything)
+    assert_equal 3, matches.size
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: ariel
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+date: 2006-08-09 00:00:00 +01:00
+summary: A Ruby Information Extraction Library
+require_paths: 
+- lib
+email: asbradbury@gmail.com
+homepage: http://ariel.rubyforge.org
+rubyforge_project: ariel
+description: Ariel uses machine learning to assist in extracting information from semi-structured  documents including (but not in any way limited to) web pages
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+- A. S. Bradbury
+files: 
+- lib/ariel
+- lib/ariel.rb
+- lib/ariel/extracted_node.rb
+- lib/ariel/learner.rb
+- lib/ariel/example_document_loader.rb
+- lib/ariel/rule_set.rb
+- lib/ariel/rule.rb
+- lib/ariel/wildcards.rb
+- lib/ariel/token_stream.rb
+- lib/ariel/label_utils.rb
+- lib/ariel/structure_node.rb
+- lib/ariel/token.rb
+- lib/ariel/candidate_selector.rb
+- lib/ariel/node_like.rb
+- test/test_learner.rb
+- test/specs
+- test/test_rule.rb
+- test/ariel_test_case.rb
+- test/fixtures.rb
+- test/test_token_stream.rb
+- test/test_example_document_loader.rb
+- test/test_token.rb
+- test/test_structure_node.rb
+- test/test_label_utils.rb
+- test/test_candidate_selector.rb
+- test/test_wildcards.rb
+- test/specs/token_stream_spec.rb
+- test/specs/wildcards_spec.rb
+- test/specs/token_spec.rb
+- README
+- LICENSE
+- examples/raa
+- examples/google_calculator
+- examples/raa/structure.rb
+- examples/raa/labeled
+- examples/raa/unlabeled
+- examples/raa/structure.yaml
+- examples/raa/labeled/mongrel.html
+- examples/raa/labeled/highline.html
+- examples/raa/unlabeled/pdf-writer.html
+- examples/google_calculator/structure.rb
+- examples/google_calculator/labeled
+- examples/google_calculator/unlabeled
+- examples/google_calculator/structure.yaml
+- examples/google_calculator/labeled/1
+- examples/google_calculator/labeled/2
+- examples/google_calculator/labeled/3
+- examples/google_calculator/unlabeled/1
+- examples/google_calculator/unlabeled/2
+- bin/ariel
+test_files: 
+- test/test_learner.rb
+- test/test_rule.rb
+- test/test_token_stream.rb
+- test/test_example_document_loader.rb
+- test/test_token.rb
+- test/test_structure_node.rb
+- test/test_label_utils.rb
+- test/test_candidate_selector.rb
+- test/test_wildcards.rb
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+- LICENSE
+executables: 
+- ariel
+extensions: []
+
+requirements: []
+
+dependencies: []
+