ariel 0.0.1 → 0.1.0

data/test/specs/learner_spec.rb

@@ -0,0 +1,39 @@
+require 'ariel'
+require 'fixtures'
+
+include Fixtures
+
+context "A non-exhaustive forward rule learner" do
+  setup do
+    @learner=Ariel::Learner.new(*@@labeled_addresses)
+  end
+
+  specify "set_seed should choose the example with the smallest number of tokens before the label" do
+    @learner.set_seed.should_equal @@labeled_addresses[1]
+  end
+
+  specify "A seed should be set when a Learner instance is initialized and made accessible through #current_seed" do
+    @learner.current_seed.should_equal @@labeled_addresses[1]
+  end
+
+  specify "generate_initial_candidates should generate rule candidates based on the token before the label in the current_seed" do
+    @learner.direction=:forward
+    @learner.generate_initial_candidates
+    c=@learner.candidates
+    c.should_include Ariel::Rule.new([["("]], :forward)
+    c.should_include Ariel::Rule.new([[:anything]], :forward)
+    c.should_include Ariel::Rule.new([[:punctuation]], :forward)
+  end
+
+  specify "learn_rule should return an array of the Rule's learnt" do
+    rules=@learner.learn_rule :forward
+    rules.should_be_a_kind_of Array
+    rules.should_not_be_empty
+  end
+
+  specify "generated rules should be of a :forward type and non-exhaustive" do
+    rule=@learner.learn_rule(:forward).first
+    rule.direction.should_equal :forward
+    rule.should_not_be_exhaustive
+  end
+end

data/test/specs/node_extracted_spec.rb

@@ -0,0 +1,90 @@
+require 'ariel'
+
+context "A new Node::Extracted" do
+  setup do
+    @tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize "This is a test"
+    @structure=Ariel::Node::Structure.new :test
+    @node=Ariel::Node::Extracted.new :test, @tokenstream, @structure
+  end
+
+  specify "Should provide an accessor for its tokenstream" do
+    @node.tokenstream.should_equal @tokenstream
+  end
+
+  specify "Should provide an accessor, structure_node for its structure node" do
+    @node.structure_node.should_equal @structure
+  end
+
+  specify "Should provide a method extracted_text to show the content of the tokenstream" do
+    @node.extracted_text.should_equal @tokenstream.text
+  end
+end
+
+context "A Node::Extracted with children" do
+	setup do
+		@tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize "This is a dummy"
+		@structure=@@labeled_document_with_list_structure
+		@root=Ariel::Node::Extracted.new :root, @tokenstream, @structure
+		[:title, :comment_list].each do |name|
+			@root.add_child(Ariel::Node::Extracted.new(name, @tokenstream, @structure.children[name]))
+		end
+		0.upto 10 do |i|
+			@root.comment_list.add_child Ariel::Node::Extracted.new(i, @tokenstream, @structure.comment_list.comment)
+		end
+	end
+
+	specify "should provide access to the node's children through" do #[] can't be used in a spec name due to a bug
+		@root[:comment_list].node_name.should_equal :comment_list
+		@root.comment_list[2].node_name.should_equal 2
+	end
+
+  specify "square bracket method should return an array when a Range with one member is given" do
+    @root.comment_list[0..0].should_be_a_kind_of Array
+  end
+
+  specify "square bracket should return nil if no matches exist" do
+    @root[:monkey].should_be_nil
+    @root[:monkey, :heaven].should_be_nil
+  end
+
+  specify "square bracket should return those matches that do exist in an array even if some don't" do
+    result=@root[:monkey, :title]
+    result.size.should_equal 1
+    result[0].node_name.should_equal :title
+  end
+
+  specify "#search should return an array of any and all matches to the given query" do
+    result=(@root/'comment_list')
+    result.should_be_a_kind_of Array
+    result.size.should_equal 1
+    result[0].node_name.should_equal :comment_list
+  end
+
+  specify "#search should return an empty array on match failure" do
+    (@root/'monkey').should_equal []
+  end
+
+  specify "#search should accept * as a wildcard" do
+    @root.title.add_child Ariel::Node::Extracted.new(:test, @tokenstream, @structure)
+    result=(@root/'*/*').collect {|r| r.node_name}
+    (((0..10).to_a << :test) - result).should_equal []
+  end
+
+  specify "#search should return numbered children" do
+    result=(@root.comment_list/'0')
+    result.size.should_equal 1
+    result[0].node_name.should_equal 0
+  end
+
+	specify "#search should return sorted results when a wildcard is used" do
+		result=(@root/'comment_list/*').collect {|node| node.node_name}
+		result.should_equal ((0..10).to_a)
+	end
+
+	specify "#at should act like #search, but return only the first result" do
+		@root.at('comment_list/*').node_name.should_equal 0
+	end
+
+end	

data/test/specs/node_spec.rb

@@ -0,0 +1,76 @@
+require 'ariel'
+
+context "A new Node" do
+  setup do
+    @node=Ariel::Node.new(:root)
+  end
+  
+  specify "Should have an empty hash of children" do
+    @node.children.should_equal Hash.new
+  end
+
+  specify "Should give its node_name as a symbol" do
+    @node.node_name.should_equal :root
+  end
+
+  specify "Should not have a parent" do
+    @node.parent.should_be_nil
+  end
+
+  specify "Should yield nothing if each_descendant is called with no arguments" do
+    results=[]
+    @node.each_descendant {|child| results << child}
+    results.should_equal []
+  end
+
+  specify "Should yield only itself if each_descendant is called with true as its argument" do
+    results=[]
+    @node.each_descendant(true) {|child| results << child}
+    results.size.should_equal 1
+    results[0].should_equal @node
+  end
+
+  specify "Should not respond to :id or :type" do
+    @node.should_not_respond_to :id
+    @node.should_not_respond_to :type
+  end
+end
+
+context "Building a tree of Node objects" do
+  setup do
+    @root_parent=Ariel::Node.new :root
+    @child1=Ariel::Node.new :child1
+    @child2=Ariel::Node.new :child2
+    @child1_1=Ariel::Node.new :child1_1
+  end
+
+  specify "Adding a child should add an entry to the parent's children hash with the child's node_name as the key" do
+    @root_parent.add_child @child1
+    @root_parent.children[:child1].should_equal @child1
+  end
+
+  specify "When adding a node as a child, its parent should be set appropriately" do
+    @root_parent.add_child @child1
+    @child1.parent.should_equal @root_parent
+    @child1.add_child @child1_1
+    @child1_1.parent.should_equal @child1
+  end
+
+  specify "When adding a node as a child, an accessor method should be created in the parent with name corresponding to the child's node_name" do
+    @root_parent.add_child @child1
+    @root_parent.should_respond_to :child1
+    @root_parent.child1.should_equal @child1
+  end
+
+  specify "Should yield all children when iterating over each_descendant" do
+    @root_parent.add_child @child1
+    @root_parent.add_child @child2
+    @child1.add_child @child1_1
+    results=[]
+    @root_parent.each_descendant {|child| results << child}
+    results.size.should_equal 3
+    results.should_include @child1
+    results.should_include @child2
+    results.should_include @child1_1
+  end
+end

data/test/specs/node_structure_spec.rb

@@ -0,0 +1,74 @@
+require 'ariel'
+require 'fixtures'
+
+include Fixtures
+
+context "Creating a new Node::Structure tree" do
+  setup do
+    @node=Ariel::Node::Structure.new
+  end
+
+  specify "Should default to :root as the node_name" do
+    @node.node_name=:root
+  end
+
+  specify "Should be possible to create a node with node_type :list_item" do
+    list_node=Ariel::Node::Structure.new :comments, :list_item
+    list_node.node_type.should_equal :list_item
+  end
+
+  specify "Node::Structure#new should yield itself is a block is given" do
+    result=[]
+    new_node=Ariel::Node::Structure.new {|r| result << r}
+    result.should_equal [new_node]
+  end
+
+  specify "Node::Structure#item should create a new child with the given name and of a non-list node_type" do
+    @node.item :test
+    @node.children.keys.should_include :test
+    @node.test.should_be_an_instance_of Ariel::Node::Structure
+    @node.test.node_type.should_not_equal :list_item
+  end
+
+  specify "Node::Structure#list_item should create a new child with the given name and node_type :list_item" do
+    @node.list_item :list
+    @node.children.keys.should_include :list
+    @node.list.should_be_an_instance_of Ariel::Node::Structure
+    @node.list.node_type.should_equal :list_item
+  end
+
+  specify "Should be possible to define a tree by passing blocks to item and list_item" do
+    root=@@labeled_document_with_list_structure
+    root.title.parent.should_equal root
+    root.comment_list.comment.should_respond_to :author
+  end
+
+  specify "#extend_structure should allow new children to be added to an existing Node::Structure" do
+    @node.extend_structure {|r| r.item :test1}
+    @node.extend_structure {|r| r.list_item :test2}
+    @node.children.keys.should_include :test1
+    @node.children.keys.should_include :test2
+  end
+end
+
+context "Applying a tree of Node::Structure objects to extract a tree of Node::Extracted objects, some children don't have rulesets defined" do
+  setup do
+    @structure_root=@@labeled_document_with_list_structure
+    @tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize Ariel::LabelUtils.clean_string(@@labeled_document_with_list)
+    @extracted_root=Ariel::Node::Extracted.new :root, @tokenstream, @structure_root
+  end
+
+  specify "#extract_from should apply the ruleset in the given Node::Structure to extract and return an array new Node::Extracted" do
+    extractions=@structure_root.title.extract_from @extracted_root
+    extractions.size.should_equal 1
+    extractions[0].extracted_text.should_equal "Another example"
+  end
+
+  specify "#apply_extraction_tree_on should apply the RuleSet in every Node::Structure child and add all extracted children to the given Node::Extracted" do
+    @structure_root.apply_extraction_tree_on @extracted_root
+    [:title, :body, :comment_list].each {|node| @extracted_root.children.keys.should_include node}
+    @extracted_root.comment_list.children.size.should_equal 2
+    @extracted_root.comment_list.children[0].children.should_equal({})
+  end
+end

data/test/specs/rule_set_spec.rb

@@ -0,0 +1,85 @@
+require 'ariel'
+require 'fixtures'
+
+include Fixtures
+context "A RuleSet of non-exhaustive rules" do
+  setup do
+    @frule1=Ariel::Rule.new [["1"]], :forward
+    @frule2=Ariel::Rule.new [["2"]], :forward
+    @brule1=Ariel::Rule.new [["a"]], :back
+    @brule2=Ariel::Rule.new [["b"]], :back
+    @ruleset=Ariel::RuleSet.new [@frule1, @frule2], [@brule1, @brule2]
+    @tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize "This is a test. 1 Let's see 2. You know a? what about b"
+  end
+
+  specify "Should return a tokenstream in an array split at the position where the first forward and back rules match" do
+    result=@ruleset.apply_to @tokenstream
+    result.should_be_a_kind_of Array
+    result.size.should_equal 1
+    result[0].should_be_a_kind_of Ariel::TokenStream
+    result[0].tokens.first.text.should_equal "Let"
+    result[0].tokens.last.text.should_equal "know"
+  end
+
+  specify "Should use the next forward or back rule if the first doesn't match" do
+    @tokenstream2=Ariel::TokenStream.new
+    @tokenstream2.tokenize "Only 2 and b in here"
+    result=@ruleset.apply_to @tokenstream2
+    result.size.should_equal 1
+    result[0].tokens.first.text.should_equal "and"
+    result[0].tokens.last.text.should_equal "and"
+  end
+
+  specify "Should return an empty array if there are no matches" do
+    stream=Ariel::TokenStream.new
+    stream.tokenize "Will not match"
+    (@ruleset.apply_to stream).should_equal []
+  end
+end
+
+context "A RuleSet of exhaustive rules" do
+  setup do
+    @tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize <<EOS
+<li>Item number one</li>
+<li>Item number two</li>
+<li>Item number three</li>
+EOS
+    @frule=Ariel::Rule.new [["<li>"]], :forward, true
+    @brule = Ariel::Rule.new [["</li>"]], :back, true
+    @ruleset=Ariel::RuleSet.new [@frule], [@brule]
+  end
+
+  specify "Should return an array of all matches found by applying the rules exhaustively" do
+    result=@ruleset.apply_to @tokenstream
+    result.size.should_equal 3
+    result.each {|tokenstream| tokenstream.tokens.first.text.should_equal "Item"}
+  end
+end
+
+context "A RuleSet of exhaustive rules" do
+  setup do
+    @tokenstream=Ariel::TokenStream.new
+    @tokenstream.tokenize @@unlabeled_restaurant_example
+    @tokenstream=@tokenstream.slice_by_token_index(12, (@tokenstream.tokens.size - 2))
+    @frule=Ariel::Rule.new [["<i>"]], :forward, true
+    @brule=Ariel::Rule.new [["</i>"]], :back, true
+    @ruleset=Ariel::RuleSet.new [@frule], [@brule]
+  end
+
+  specify "Should extract a tokenstream when the first end match is before the first start match" do
+    result=@ruleset.apply_to @tokenstream
+    result.size.should_equal 3
+    result[0].tokens.first.text.should_equal "4000"
+    result[1].tokens.first.text.should_equal "523"
+    result[2].tokens.first.text.should_equal "403"
+  end
+
+  specify "Should return an empty array if there are no matches" do
+    stream=Ariel::TokenStream.new
+    stream.tokenize "Will not match"
+    (@ruleset.apply_to stream).should_equal []
+  end
+end
+

data/test/specs/rule_spec.rb

@@ -0,0 +1,110 @@
+require 'ariel'
+require 'fixtures'
+include Fixtures
+
+tokenstream=Ariel::TokenStream.new
+tokenstream.tokenize @@unlabeled_document
+
+labeled_tokenstream=Ariel::TokenStream.new
+labeled_tokenstream.tokenize @@unlabeled_document
+labeled_tokenstream.label_index=4
+
+context "A forward rule with no landmarks" do
+  setup do
+    @rule=Ariel::Rule.new([], :forward)
+  end
+
+  specify "Should return its direction correctly" do
+    @rule.direction.should_equal :forward
+  end
+
+  specify "Should contain no wildcards" do
+    @rule.wildcard_count.should_equal 0
+  end
+
+  specify "Should match any tokenstream at index 0" do
+    @rule.apply_to(tokenstream).should_equal [0]
+  end
+
+  specify "Should not be exhaustive" do
+    @rule.should_not_be_exhaustive
+  end
+end
+
+context "A back rule with no landmarks" do
+  setup do
+    @rule=Ariel::Rule.new([], :back)
+  end
+
+  specify "Should match any tokenstream at its last token" do
+    match_loc=@rule.apply_to(tokenstream)
+    p match_loc
+    tokenstream.tokens[*@rule.apply_to(tokenstream)].should_equal tokenstream.tokens.last
+  end
+end
+
+context "Creating a new rule" do
+  specify "Should not be possible to create a rule with an invalid direction" do
+    lambda {Ariel::Rule.new([[:anything]], :upward)}.should_raise
+  end
+end
+
+context "Applying a non-exhaustive forward rule" do
+  setup do
+    @rule=Ariel::Rule.new [[:anything]], :forward
+  end
+  specify "apply_to should return an array of match locations" do
+    locs=@rule.apply_to(tokenstream)
+    locs.should_equal [1]
+  end
+
+  specify "apply_to should yield matchdata with type nil for tokenstreams with no label_index" do
+    @rule.apply_to(tokenstream) do |md|
+      md.type.should_be_nil
+      md.token_loc.should_equal 1
+    end
+  end
+
+  specify "apply_to should yield match data with type :early, :late, or :perfect for a labeled tokenstream" do
+    @rule.apply_to(labeled_tokenstream) do |md|
+      md.type.should_equal :early
+    end
+    late_rule=Ariel::Rule.new [["assess"]], :forward
+    late_rule.apply_to(labeled_tokenstream) do |md|
+      md.type.should_equal :late
+    end
+    perfect_rule = Ariel::Rule.new [["test"]], :forward
+    perfect_rule.apply_to(labeled_tokenstream) do |md|
+      md.type.should_equal :perfect
+      md.token_loc.should_equal 4
+    end
+  end
+
+  specify "matches should return true or false if applying the rule to a labeled tokenstream results in a match of one of the given types" do
+    @rule.matches(labeled_tokenstream, :late, :perfect, :fail).should_equal false
+    @rule.matches(labeled_tokenstream, :early, :late, :perfect, :fail).should_equal true
+    failed_rule=Ariel::Rule.new([["bacon"]], :forward)
+    failed_rule.matches(labeled_tokenstream, :fail).should_equal true
+  end
+
+  specify "apply_to should apply the rule in the right direction whether the tokenstream is reversed or not. The returned match should be from the left" do
+    @rule.apply_to(labeled_tokenstream).should_equal [1]
+    @rule.apply_to(labeled_tokenstream.reverse).should_equal [labeled_tokenstream.tokens.size-2]
+    
+  end
+
+end
+
+context "Applying a non-exhaustive back rule" do
+
+end
+
+context "Applying an exhaustive rule" do
+  setup do
+    @rule = Ariel::Rule.new [[:html_tag]], :forward, true
+  end
+
+  specify "apply_to should return an array of multiple matches" do
+    @rule.apply_to(tokenstream).size.should_equal 4
+  end
+end