scrappy 0.3.0 → 0.3.1

data/views/samples.haml

@@ -0,0 +1,28 @@
+#body
+  %h1 Sample pages
+  %p
+    Sample pages are used to build extractors as well as visual patterns that can be applied to retrieve data
+    from other pages.
+  %p
+    -if @samples.empty?
+      Currently, there are no samples.
+    -else
+      %ul.detail
+        -@samples.each_with_index do |sample,i|
+          %li
+            %span.action
+              %a{:href=>"#{settings.base_uri}/samples/#{i}", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete the sample #{sample[:uri]}?"}
+                X
+            %span.short_name
+              -if !sample[:uri].include?('*')
+                %a{:href=>sample[:uri]}=sample[:uri]
+              -else
+                =sample[:uri]
+            -[['Patterns output', :patterns], ['Extractors output', :extractors]].reverse.each do |text, action|
+              %span.format
+                %a{:href=>"#{settings.base_uri}/samples/#{i}/#{action}"}=text
+            %span.format
+              %a{:href=>"#{settings.base_uri}/samples/#{i}/train", :'data-method'=>:post} Train
+            %span.date
+              %a{:href=>"#{settings.base_uri}/samples/#{i}"}
+                =sample[:date].strftime("%Y/%m/%d - %H:%M")

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: scrappy
 version: !ruby/object:Gem::Version 
-  hash: 19
   prerelease: false
   segments: 
   - 0
   - 3
-  - 0
-  version: 0.3.0
+  - 1
+  version: 0.3.1
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -15,18 +14,16 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-11 00:00:00 +01:00
+date: 2011-03-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 9
         segments: 
         - 2
         - 3
@@ -38,11 +35,9 @@ dependencies:
   name: sinatra
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
         segments: 
         - 1
         - 1
@@ -54,11 +49,9 @@ dependencies:
   name: thin
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 17
         segments: 
         - 1
         - 2
@@ -70,11 +63,9 @@ dependencies:
   name: nokogiri
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
         segments: 
         - 1
         - 4
@@ -86,11 +77,9 @@ dependencies:
   name: mechanize
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
         segments: 
         - 1
         - 0
@@ -102,27 +91,23 @@ dependencies:
   name: lightrdf
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 21
         segments: 
         - 0
-        - 2
-        - 1
-        version: 0.2.1
+        - 3
+        - 0
+        version: 0.3.0
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: i18n
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 11
         segments: 
         - 0
         - 4
@@ -134,11 +119,9 @@ dependencies:
   name: rest-client
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 13
         segments: 
         - 1
         - 6
@@ -150,11 +133,9 @@ dependencies:
   name: haml
   prerelease: false
   requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 55
         segments: 
         - 3
         - 0
@@ -162,6 +143,20 @@ dependencies:
         version: 3.0.24
   type: :runtime
   version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: rack-flash
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 1
+        - 1
+        version: 0.1.1
+  type: :runtime
+  version_requirements: *id010
 description: RDF web scraper
 email: joseignacio.fernandez@gmail.com
 executables: 
@@ -176,24 +171,28 @@ extra_rdoc_files:
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
-- lib/scrappy/agent/extractor.rb
-- lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
+- lib/scrappy/extractor/extractor.rb
+- lib/scrappy/extractor/formats.rb
+- lib/scrappy/extractor/fragment.rb
+- lib/scrappy/extractor/selector.rb
+- lib/scrappy/extractor/selectors/base_uri.rb
+- lib/scrappy/extractor/selectors/css.rb
+- lib/scrappy/extractor/selectors/new_uri.rb
+- lib/scrappy/extractor/selectors/root.rb
+- lib/scrappy/extractor/selectors/section.rb
+- lib/scrappy/extractor/selectors/slice.rb
+- lib/scrappy/extractor/selectors/uri.rb
+- lib/scrappy/extractor/selectors/uri_pattern.rb
+- lib/scrappy/extractor/selectors/visual.rb
+- lib/scrappy/extractor/selectors/xpath.rb
 - lib/scrappy/repository.rb
-- lib/scrappy/selectors/base_uri.rb
-- lib/scrappy/selectors/css.rb
-- lib/scrappy/selectors/new_uri.rb
-- lib/scrappy/selectors/root.rb
-- lib/scrappy/selectors/section.rb
-- lib/scrappy/selectors/slice.rb
-- lib/scrappy/selectors/uri.rb
-- lib/scrappy/selectors/uri_pattern.rb
-- lib/scrappy/selectors/xpath.rb
 - lib/scrappy/server/admin.rb
 - lib/scrappy/server/errors.rb
 - lib/scrappy/server/helpers.rb
 - lib/scrappy/server/server.rb
 - lib/scrappy/support.rb
+- lib/scrappy/trainer/trainer.rb
 files: 
 - History.txt
 - Manifest
@@ -206,41 +205,48 @@ files:
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
-- lib/scrappy/agent/extractor.rb
-- lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
+- lib/scrappy/extractor/extractor.rb
+- lib/scrappy/extractor/formats.rb
+- lib/scrappy/extractor/fragment.rb
+- lib/scrappy/extractor/selector.rb
+- lib/scrappy/extractor/selectors/base_uri.rb
+- lib/scrappy/extractor/selectors/css.rb
+- lib/scrappy/extractor/selectors/new_uri.rb
+- lib/scrappy/extractor/selectors/root.rb
+- lib/scrappy/extractor/selectors/section.rb
+- lib/scrappy/extractor/selectors/slice.rb
+- lib/scrappy/extractor/selectors/uri.rb
+- lib/scrappy/extractor/selectors/uri_pattern.rb
+- lib/scrappy/extractor/selectors/visual.rb
+- lib/scrappy/extractor/selectors/xpath.rb
 - lib/scrappy/repository.rb
-- lib/scrappy/selectors/base_uri.rb
-- lib/scrappy/selectors/css.rb
-- lib/scrappy/selectors/new_uri.rb
-- lib/scrappy/selectors/root.rb
-- lib/scrappy/selectors/section.rb
-- lib/scrappy/selectors/slice.rb
-- lib/scrappy/selectors/uri.rb
-- lib/scrappy/selectors/uri_pattern.rb
-- lib/scrappy/selectors/xpath.rb
 - lib/scrappy/server/admin.rb
 - lib/scrappy/server/errors.rb
 - lib/scrappy/server/helpers.rb
 - lib/scrappy/server/server.rb
 - lib/scrappy/support.rb
+- lib/scrappy/trainer/trainer.rb
 - public/favicon.ico
 - public/images/logo.png
 - public/images/logo_tiny.png
-- public/javascripts/scrappy.js
+- public/javascripts/annotator.js
+- public/javascripts/remote.js
 - public/stylesheets/application.css
 - test/test_helper.rb
 - test/test_scrappy.rb
+- views/extractors.haml
 - views/help.haml
 - views/home.haml
-- views/kb.haml
 - views/layout.haml
+- views/patterns.haml
+- views/samples.haml
 - scrappy.gemspec
 has_rdoc: true
 homepage: http://github.com/josei/scrappy
 licenses: []
 
-post_install_message: "**(Optional) Remember to install rbwebkitgtk for visual parsing features**"
+post_install_message: 
 rdoc_options: 
 - --line-numbers
 - --inline-source
@@ -251,20 +257,16 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 11
       segments: 
       - 1
       - 2
@@ -272,10 +274,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: scrappy
-rubygems_version: 1.3.7
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Web scraper that allows producing RDF data out of plain web pages
 test_files: 
-- test/test_helper.rb
 - test/test_scrappy.rb
+- test/test_helper.rb

data/lib/scrappy/agent/extractor.rb

@@ -1,196 +0,0 @@
-require 'digest/md5'
-
-module Scrappy
-  module Extractor
-    def extract uri, html, referenceable=nil
-      if options.debug
-        print "Extracting #{uri}..."; $stdout.flush
-      end
-
-      @selector_pool ||= {}
-      triples = []
-      content = Nokogiri::HTML(html, nil, 'utf-8')
-
-      uri_selectors  = (kb.find(nil, Node('rdf:type'), Node('sc:UriSelector')) + kb.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector'))).flatten.select do |uri_selector|
-        results = selector_pool(uri_selector).filter :content=>content, :uri=>uri
-        !results.empty?
-      end
-
-      fragments = uri_selectors.map { |uri_selector| kb.find(nil, Node('sc:selector'), uri_selector) }.flatten
-
-      fragments.each do |fragment|
-        extract_fragment fragment, :doc=>{:uri=>uri, :content=>content },
-                                   :parent=>uri, :triples=>triples, :referenceable=>!referenceable.nil?
-      end
-
-      add_referenceable_data content, triples, referenceable if referenceable
-
-      puts "done!" if options.debug
-      
-      triples.map do |s,p,o|
-        [ s.is_a?(RDF::Node) ? s.id : s,
-          p.is_a?(RDF::Node) ? p.id : p,
-          o.is_a?(RDF::Node) ? o.id : o ]
-      end 
-    end
-    
-    private
-    def extract_fragment fragment, options={}
-      node = Node(options[:parent])
-      uri = options[:doc][:uri]
-
-      # Select nodes
-      docs = fragment.sc::selector.map { |s| filter s, options[:doc] }.flatten
-
-      # Generate triples
-      docs.each do |doc|
-        # Build URIs if identifier present
-        nodes = fragment.sc::identifier.map { |s| filter s, doc  }.flatten.map do |d|
-          node = Node(parse_uri(uri, d[:value]))
-          if options[:referenceable]
-            # Include the fragment where the URI was built from
-            uri_node = Node(nil)
-            options[:triples] << [ node,     Node("sc:uri"),    uri_node ]
-            options[:triples] << [ uri_node, Node("rdf:value"), node.to_s ]
-            options[:triples] << [ uri_node, Node("sc:source"), Node(node_hash(d[:uri], d[:content].path)) ]
-          end
-          node
-        end
-        nodes << Node(nil) if nodes.empty?
-        
-        nodes.each do |node|
-          # Build the object
-          object = if fragment.sc::type.include?(Node('rdf:Literal'))
-            value = doc[:value].to_s.strip
-            if options[:referenceable]
-              bnode = Node(nil)
-              bnode.rdf::value = value
-              bnode.rdf::type = Node('rdf:Literal')
-              options[:triples].push *bnode.triples
-              bnode
-            else
-              value
-            end
-          else
-            fragment.sc::type.each { |type| options[:triples] << [node, Node('rdf:type'), type] if type != Node('rdf:Resource') }
-            fragment.sc::superclass.each { |superclass| options[:triples] << [node, Node('rdfs:subClassOf'), superclass] }
-            fragment.sc::sameas.each { |samenode| options[:triples] << [node, Node('owl:sameAs'), samenode] }
-            node
-          end
-          fragment.sc::relation.each { |relation| options[:triples] << [options[:parent], relation, object] }
-          
-          # Add referenceable data if requested
-          if options[:referenceable]
-            sources = [doc[:content]].flatten.map { |node| Node(node_hash(doc[:uri], node.path)) }
-            sources.each do |source|
-              options[:triples] << [ object, Node("sc:source"), source ]
-              fragment.sc::type.each { |t| options[:triples] << [ source, Node("sc:type"), t ] }
-              fragment.sc::relation.each { |relation| options[:triples] << [ source, Node("sc:relation"), relation ] }
-            end
-          end
-
-          # Process subfragments
-          fragment.sc::subfragment.each { |subfragment| extract_fragment subfragment, options.merge(:doc=>doc, :parent=>object) }
-        end
-      end
-    end
-
-    def filter selector, doc
-      if selector.sc::debug.first=="true" and options.debug
-        puts '== DEBUG'
-        puts '== Selector:'
-        puts selector.serialize(:yarf, false)
-        puts '== On fragment:'
-        puts "URI: #{doc[:uri]}"
-        puts "Content: #{doc[:content]}"
-        puts "Value: #{doc[:value]}"
-      end
-
-      # Process selector
-      results = selector_pool(selector).filter doc
-
-      if selector.sc::debug.first=="true" and options.debug
-        puts "== No results" if results.empty?
-        results.each_with_index do |result, i|
-          puts "== Result ##{i}:"
-          puts "URI: #{result[:uri]}"
-          puts "Content: #{result[:content]}"
-          puts "Value: #{result[:value].inspect}"
-        end
-        puts
-      end
-      
-      # Return results if no nested selectors
-      return results if selector.sc::selector.empty?
-
-      # Process nested selectors
-      results.map do |result|
-        selector.sc::selector.map { |s| filter s, result }
-      end.flatten
-    end
-    
-    def parse_uri(uri, rel_uri)
-      return ID('*') if rel_uri.nil?
-      begin
-        ID(URI::parse(uri.split('/')[0..3]*'/').merge(rel_uri).to_s)
-      rescue
-        ID('*')
-      end
-    end
-
-    def add_referenceable_data content, triples, referenceable
-      resources = {}; triples.each { |s,p,o| resources[o] = true }
-
-      fragment = Node(node_hash(uri, '/'))
-      selector = Node(nil)
-      presentation = Node(nil)
-
-      selector.rdf::type = Node('sc:UnivocalSelector')
-      selector.sc::path = '/'
-      selector.sc::document = uri
-
-      fragment.sc::selector = selector
-
-      triples.push(*fragment.graph.merge(presentation.graph).merge(selector.graph).triples) if referenceable==:dump or resources.include?(fragment)
-
-      content.search('*').each do |node|
-        next if node.text?
-
-        fragment = Node(node_hash(uri, node.path))
-
-        if referenceable == :dump or resources[fragment]
-          selector     = Node(nil)
-          presentation = Node(nil)
-
-          triples << [selector, ID('rdf:type'), ID('sc:UnivocalSelector')]
-          triples << [selector, ID('sc:path'), node.path.to_s]
-          triples << [selector, ID('sc:tag'), node.name.to_s]
-          triples << [selector, ID('sc:document'), uri]
-
-          triples << [presentation, ID('sc:x'), node[:vx].to_s] if node[:vx]
-          triples << [presentation, ID('sc:y'), node[:vy].to_s] if node[:vy]
-          triples << [presentation, ID('sc:width'), node[:vw].to_s] if node[:vw]
-          triples << [presentation, ID('sc:height'), node[:vh].to_s] if node[:vh]
-          triples << [presentation, ID('sc:font_size'), node[:vsize].gsub("px","").to_s] if node[:vsize]
-          triples << [presentation, ID('sc:font_weight'), node[:vweight].to_s] if node[:vweight]
-          triples << [presentation, ID('sc:color'), node[:vcolor].to_s] if node[:vcolor]
-          triples << [presentation, ID('sc:background_color'), node[:vbcolor].to_s] if node[:vbcolor]
-          triples << [presentation, ID('sc:text'), node.text.strip]
-          triples << [presentation, ID('sc:children_count'), node.children.select{|n| !n.text?}.size.to_s]
-
-          triples << [fragment, ID('sc:selector'), selector]
-          triples << [fragment, ID('sc:presentation'), presentation]
-        end
-      end
-    end
-
-    def node_hash uri, path
-      digest = Digest::MD5.hexdigest("#{uri} #{path}")
-      :"_:bnode#{digest}"
-    end
-    
-    def selector_pool selector
-      @selector_pool[selector.id] ||= kb.node(selector)
-    end
-  end
-end