ariel 0.0.1 → 0.1.0

data/test/test_example_document_loader.rb

@@ -1,7 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestExampleDocumentLoader < Ariel::TestCase
-  include Fixtures
-
-end

data/test/test_label_utils.rb

@@ -1,15 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestLabelUtils < Ariel::TestCase
-  include Fixtures
-
-  def test_label_regex
-    assert_equal 2, Ariel::LabelUtils.label_regex.uniq.size
-    assert_kind_of Regexp, Ariel::LabelUtils.label_regex[0]
-  end
-  
-  def test_clean_string
-    assert_equal @@unlabeled_document, Ariel::LabelUtils.clean_string(@@labeled_document)
-  end
-end

data/test/test_learner.rb

@@ -1,38 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestLearner < Ariel::TestCase
-  include Fixtures
-
-  def setup
-    #Examples stolen from the STALKER paper. Target to extract is the area
-    #codes.
-    @e=@@labeled_addresses
-    @learner=Ariel::Learner.new(*@e)
-  end
-
-  def test_set_seed
-    assert_equal @e[1], @learner.current_seed # LabeledStream with smallest label_index
-  end
-
-  def test_generate_initial_candidates
-    @learner.direction=:forward
-    @learner.generate_initial_candidates
-    c=@learner.candidates
-    assert (c.include? Ariel::Rule.new(:forward, [["("]]))
-    assert (c.include? Ariel::Rule.new(:forward, [[:anything]]))
-    assert (c.include? Ariel::Rule.new(:forward, [[:punctuation]]))
-  end
-
-  def test_refine
-    @learner.current_rule=Ariel::Rule.new(:forward, [["<b>"]])
-    assert @learner.refine
-    @learner.current_rule=Ariel::Rule.new(:forward, [["<b>", "Palms"], ["Phone"]])
-    assert @learner.refine
-  end
-
-  def test_learn_rule
-    rule=@learner.learn_rule :forward
-    p rule
-  end
-end

data/test/test_rule.rb

@@ -1,38 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestRule < Ariel::TestCase
-  def setup
-    @labeled=Ariel::TokenStream.new
-    @labeled.tokenize("90 Colfax, <b> Palms </b>, Phone: (818) 508-1570")
-    @labeled.set_label_at 35
-    @perfect_rule=Ariel::Rule.new(:forward, [["Phone"], ["("]])
-    @early_rule=Ariel::Rule.new(:forward, [[:anything]])
-    @late_rule=Ariel::Rule.new(:forward, [["508"]])
-    @unlabeled=Ariel::TokenStream.new
-    @unlabeled.tokenize("Robot 9753 reporting for duty. BEEP BEEP")
-  end
-
-  def test_apply_to
-    md=nil
-    @perfect_rule.apply_to(@labeled) {|md|}
-    assert_equal :perfect, md.type
-    @early_rule.apply_to(@labeled) {|md|}
-    assert_equal :early, md.type
-    assert_equal 1, md.token_loc
-    @late_rule.apply_to(@labeled) {|md|}
-    assert_equal :late, md.type
-    assert_nil (@perfect_rule.apply_to(@unlabeled))
-  end
-
-  def test_matches
-    assert(@early_rule.matches(@labeled, :early))
-    assert(@late_rule.matches(@labeled, :early, :late))
-    assert(@perfect_rule.matches(@unlabeled, :fail))
-  end
-
-  def test_wildcard_count
-    assert_equal 0, @perfect_rule.wildcard_count
-    assert_equal 1, @early_rule.wildcard_count
-  end
-end

data/test/test_structure_node.rb

@@ -1,81 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestStructureNode < Ariel::TestCase
-  def setup
-    @tree=Ariel::StructureNode.new do |r|
-      r.item :item_info do |i|
-        i.item :title
-        i.item :price
-        i.item :stock_level
-      end
-    end
-  end
-  def test_unnested
-    t=Ariel::StructureNode.new {|r| r.item :picture; r.item :title; r.item :description; r.item :url}
-    assert t
-    assert_equal Ariel::StructureNode, t.picture.class
-    assert_equal :root, t.meta.name
-  end
-
-  def test_nested
-    assert @tree.item_info.children.has_key?(:title)
-  end
-
-  def test_nested_with_list
-    doc_tree=Ariel::StructureNode.new do |r|
-      r.item :restaurant_list do |r|
-        r.list_item :restaurant do |r|
-          r.item :name
-          r.item :address
-          r.item :phone
-          r.item :review
-          r.item :credit_card_list do |c|
-            c.item :credit_card
-          end
-        end
-      end
-    end
-    assert doc_tree
-    assert_equal :list, doc_tree.restaurant_list.restaurant.meta.node_type
-  end
-
-  def test_extend_structure
-    assert (@tree.extend_structure {|r| r.item :site_copyright; r.item :logo;})
-    assert @tree.children.has_key?(:site_copyright)
-    assert @tree.children.has_key?(:logo)
-    assert (@tree.item_info.extend_structure {|i| i.item :picture})
-    assert @tree.item_info.children.has_key?(:picture)
-  end
-
-#   def test_apply_extraction_tree_on
-# #     t = Ariel::StructureNode.new do |r|
-# # #       r.title
-# # # #       r.content do |c|
-# # # # #         c.excerpt
-# # # # # #         c.body
-# # # # # # #       end
-# # # # # # # #     end
-# # # # # # # # #     str = %q{Title: The test of the Century
-# # # # # # # # # # <b>Excerpt</b>: <i>A look back at what could be considered the greatest ever test.</i>
-# # # # # # # # # # # There was once a test designed to assess whether apply_extraction_tree_on worked.}
-# # # # # # # # # # # #     tokenstream = Ariel::TokenStream.new
-# # # # # # # # # # # # #     tokenstream.tokenize(str)
-# # # # # # # # # # # # # #     root = Ariel::ExtractedNode.new(tokenstream, :structure=>t, :name=>:root)
-# # # # # # # # # # # # # # #     t.title.meta.start_rule = Ariel::Rule.new(["Title", ":"])
-# # # # # # # # # # # # # # # #     t.title.meta.end_rule = Ariel::Rule.new(["<b>"])
-# # # # # # # # # # # # # # # # #     t.title.meta.end_rule.direction = :back
-# # # # # # # # # # # # # # # # # #     t.content.meta.start_rule = Ariel::Rule.new(["Century"]) #later implementation might use skip_until("<b>")
-# # # # # # # # # # # # # # # # # # #     t.content.meta.end_rule = Ariel::Rule.new()
-# # # # # # # # # # # # # # # # # # # #     t.content.meta.end_rule.direction = :back
-# # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.start_rule = Ariel::Rule.new(["<i>"])
-# # # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.end_rule = Ariel::Rule.new([".</"])
-# # # # # # # # # # # # # # # # # # # # # # #     t.content.excerpt.meta.end_rule.direction = :back
-# # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.start_rule = Ariel::Rule.new(["i", ">"])
-# # # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.end_rule = Ariel::Rule.new()
-# # # # # # # # # # # # # # # # # # # # # # # # # #     t.content.body.meta.end_rule.direction = :back
-# # # # # # # # # # # # # # # # # # # # # # # # # # #     t.apply_extraction_tree_on root
-#  end
-
-
-end

data/test/test_token.rb

@@ -1,16 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestToken < Ariel::TestCase
-  def setup
-    @t=Ariel::Token.new('Test', 0, 4)
-  end
-
-  def test_matches?
-    assert @t.matches?('Test')
-    assert_equal false, @t.matches?('test')
-    assert_equal false, @t.matches?('te')
-    assert @t.matches?(:alpha)
-    assert_equal false, @t.matches?(:html_tag)
-  end
-end

data/test/test_token_stream.rb

@@ -1,82 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestTokenStream < Ariel::TestCase
-  include Fixtures
-  
-  def setup
-    @stream=Ariel::TokenStream.new
-    @text = "This is test101. See below:"
-    @stream.tokenize(@text)
-    
-    @labeled_stream = Ariel::TokenStream.new
-    @labeled_stream.tokenize(@@labeled_document, true)
-  end
-  
-  def test_advance
-    assert_equal Ariel::Token.new("This", 0, 4), @stream.advance
-  end
-
-  def test_cur_pos
-    assert_equal 0, @stream.cur_pos
-    @stream.advance
-    assert_equal 1, @stream.cur_pos
-  end
-
-  def test_each
-    i=0
-    @stream.each {i=i+1}
-    assert_equal 8, i
-    assert_equal 9, @stream.cur_pos
-  end
-
-  def test_rewind
-    @stream.each {}
-    @stream.rewind
-    assert_equal 0, @stream.cur_pos
-  end
-
-  def test_skip_to
-    assert @stream.skip_to("This")
-    assert_equal 1, @stream.cur_pos  #Test the matched token has been consumed
-    assert_nil @stream.skip_to("Ruby")
-    assert_equal 1, @stream.cur_pos  #Stream's position remains unchanged by a failed match
-    assert @stream.skip_to("See", "below")
-    assert_equal 7, @stream.cur_pos
-    @stream.rewind
-    @stream.skip_to(:anything, "below")
-    assert_equal 7, @stream.cur_pos
-  end
-
-  def test_tokenize
-    assert_equal 8, @stream.tokens.length
-    @stream.each do |token|
-      assert_equal @text[token.start_loc...token.end_loc], token.text
-    end
-    @labeled_stream.each do |token|
-      assert_equal @@labeled_document[token.start_loc...token.end_loc], token.text
-    end
-  end
-
-  def test_set_label_at
-    assert_raise(ArgumentError) {@stream.set_label_at 1}
-    assert_nil @stream.label_index
-    assert(@labeled_stream.set_label_at(16))
-    assert_equal("The", @labeled_stream.tokens[@labeled_stream.label_index].text)
-  end
-
-  def test_raw_text
-    assert_equal @text, @stream.raw_text
-    assert_equal @@labeled_document.chomp, @labeled_stream.raw_text
-  end
-
-  def test_text
-    assert_equal @text, @stream.text
-    assert_equal @@unlabeled_document.chomp, @labeled_stream.text
-  end
-
-  def test_slice_by_token_index
-    assert sliced=@stream.slice_by_token_index(1,3)
-    assert_equal @text[sliced.tokens.first.start_loc...sliced.tokens.last.end_loc], sliced.text
-  end
-end

data/test/test_wildcards.rb

@@ -1,18 +0,0 @@
-require 'ariel'
-require 'ariel_test_case'
-
-class TestWildcards < Ariel::TestCase
-  
-  def test_list
-    assert (wildcards=Ariel::Wildcards.list)
-    assert (wildcards.kind_of? Hash)
-  end
-
-  def test_matching
-    assert matches=Ariel::Wildcards.matching("123")
-    assert (matches.include? :alpha_numeric)
-    assert (matches.include? :numeric)
-    assert (matches.include? :anything)
-    assert_equal 3, matches.size
-  end
-end