scrappy 0.2.0 → 0.2.1

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.2.1 2011-03-11
+
+* Added a web admin interface mode
+* Removed proxy support
+
 === 0.2.0 2011-03-09
 
 * Integration with cmoft's improvements: Sesame support, time-aggregated requests and observing URLs

data/Manifest

@@ -4,16 +4,16 @@ README.rdoc
 Rakefile
 bin/scrappy
 kb/elmundo.yarf
-lib/js/annotator.js
 lib/scrappy.rb
 lib/scrappy/agent/agent.rb
 lib/scrappy/agent/blind_agent.rb
 lib/scrappy/agent/cache.rb
 lib/scrappy/agent/dumper.rb
+lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/formats.rb
 lib/scrappy/agent/map_reduce.rb
-lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/visual_agent.rb
+lib/scrappy/repository.rb
 lib/scrappy/selectors/base_uri.rb
 lib/scrappy/selectors/css.rb
 lib/scrappy/selectors/new_uri.rb
@@ -23,16 +23,20 @@ lib/scrappy/selectors/slice.rb
 lib/scrappy/selectors/uri.rb
 lib/scrappy/selectors/uri_pattern.rb
 lib/scrappy/selectors/xpath.rb
-lib/scrappy/server/proxy.rb
+lib/scrappy/server/admin.rb
+lib/scrappy/server/helpers.rb
 lib/scrappy/server/server.rb
-lib/scrappy/server/public/images/logo.png
-lib/scrappy/server/public/images/logo_small.png
-lib/scrappy/server/public/stylesheets/application.css
-lib/scrappy/server/views/home.haml
-lib/scrappy/server/views/help.haml
-lib/scrappy/repository.rb
 lib/scrappy/shell.rb
 lib/scrappy/support.rb
 lib/scrappy/webkit/webkit.rb
+public/favicon.ico
+public/images/logo.png
+public/images/logo_tiny.png
+public/javascripts/scrappy.js
+public/stylesheets/application.css
 test/test_helper.rb
 test/test_scrappy.rb
+views/help.haml
+views/home.haml
+views/kb.haml
+views/layout.haml

data/README.rdoc

@@ -140,37 +140,37 @@ scrappy offers many different interfaces to get RDF data from a web page:
 
 * RDF repository:
 
-Sesame functionality has been included in Scrappy. You can configure
-the repository options by editing the file config.yml placed the folder .scrappy, in your home dir.
-An example of this file can be found at the end of this README.
+  Sesame functionality has been included in Scrappy. You can configure
+  the repository options by editing the file config.yml placed the folder .scrappy, in your home dir.
+  An example of this file can be found at the end of this README.
 
-You can get the data for a certain period of time, by using the time (-t, --time) option:
+  You can get the data for a certain period of time, by using the time (-t, --time) option:
 
-  $ scrappy -g example.org -t 3
+    $ scrappy -g example.org -t 3
 
-This would get all the data stored in Sesame for example.org in the last 3 minutes.
+  This would get all the data stored in Sesame for example.org in the last 3 minutes.
 
 * Sample config.yml
 
-  # This is a sample configuration file, with the options to communicate with Sesame using Scrappy
-  repository:
-    # The host were Sesame is. Do not add the trailing '/'
-    host: http://localhost
-  
-    # The port for the connection
-    port: 8080
-  
-    # The time to consider the data in the repository valid, in minutes
-    time: 15
-  
-    # The name of the repository
-    repository: memory
-  
-    # The format to communicate with the repository
-    format: ntriples
-  
-    # You can use any of the following formats:
-    # rdfxml, ntriples, turtle, n3, trix, trig
+    # This is a sample configuration file, with the options to communicate with Sesame using Scrappy
+    repository:
+      # The host were Sesame is. Do not add the trailing '/'
+      host: http://localhost
+    
+      # The port for the connection
+      port: 8080
+    
+      # The time to consider the data in the repository valid, in minutes
+      time: 15
+    
+      # The name of the repository
+      repository: memory
+    
+      # The format to communicate with the repository
+      format: ntriples
+    
+      # You can use any of the following formats:
+      # rdfxml, ntriples, turtle, n3, trix, trig
 
 
 == INSTALL:
@@ -222,4 +222,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/scrappy

@@ -40,12 +40,12 @@ module Scrappy
         opts.on('-u', '--debug')                { Agent::Options.debug = true }
         opts.on('-i', '--interactive')          { Options.shell = true; Agent::Options.format_header = false }
         opts.on('-s', '--server [BASE_URI]')    { |uri| Options.server = true; Options.base_uri = uri }
-        opts.on('-S', '--proxy-server')         { Options.proxy = true }
+        opts.on('-a', '--admin [BASE_URI]')     { |uri| Options.admin = true; Options.base_uri = uri }
         opts.on('-P P', '--port P')             { |p| Options.port = p }
         opts.on('-c C', '--concurrence C')      { |c| Agent::Options.workers = c.to_i }
         opts.on('-d D', '--delay D')            { |d| Agent::Options.delay = d; Agent::Options.workers = 1 }
         opts.on('-l L', '--levels L')           { |l| Agent::Options.depth = l.to_i }
-        opts.on('-V', '--visual')               { Agent::Options.agent = :visual; Agent::Options.workers = 1 }
+        opts.on('-V', '--visual')               { Agent::Options.agent = :visual }
         opts.on('-r', '--reference')            { Agent::Options.referenceable = :minimum }
         opts.on('-R', '--reference-all')        { Agent::Options.referenceable = :dump }
         opts.on('-w', '--window')               { Agent::Options.window = true }
@@ -63,16 +63,19 @@ module Scrappy
         puts Agent.create.proxy(:http_method=>:get, :uri=>Options.uri).output
       elsif Options.observe
         Agent.create.observe(Options.observe)
-      elsif Options.proxy
-        puts "Launching Scrappy Web Proxy (set http://localhost:#{Options.port} as proxy)..."
-        require 'scrappy/server/proxy'
+      elsif Options.admin
+        puts "Launching Scrappy Admin Web Server (browse http://localhost:#{Options.port})..."
+        require 'scrappy/server/server'
         Thin::Logging.silent = true
-        Scrappy::Proxy.run! :host => 'localhost', :port => Options.port, :environment=>:production
+        Scrappy::Server.register Scrappy::Admin
+        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment => :production,
+                             :base_uri=>Options.base_uri
       elsif Options.server
-        puts "Launching Scrappy Web Server (browse http://localhost:#{Options.port})..."
+        puts "Launching Scrappy Web Server..."
         require 'scrappy/server/server'
         Thin::Logging.silent = true
-        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment=>:production, :base_uri=>Options.base_uri
+        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment => :production,
+                             :base_uri => Options.base_uri
       elsif Options.shell
         puts "Launching Scrappy Shell..."
         require 'scrappy/shell'
@@ -123,7 +126,7 @@ Options
   -w, --window             Shows browser window (requires -v)
 
 Authors
-  José Ignacio Fernández, Jacobo Blasco
+  José Ignacio Fernández, Alberto Mardomingo, Jacobo Blasco
 
 Copyright
   Copyright (c) 2010 José Ignacio Fernández. Licensed under the MIT License:

data/lib/scrappy.rb

@@ -22,7 +22,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -102,10 +102,11 @@ module Scrappy
       end
       
       triples = []; results.each { |result| triples += result }
+      triples.uniq!
       
       puts 'done!'if options.debug
       
-      triples.uniq
+      triples
     end
 
     def request args={}

data/lib/scrappy/agent/visual_agent.rb

@@ -24,6 +24,7 @@ module Scrappy
         @window.show_all
         @visible = true
       end
+      @mechanize = Mechanize.new
     end
 
     def uri
@@ -33,7 +34,7 @@ module Scrappy
     def uri= uri
       # First, check if the requested uri is a valid HTML page
       valid = begin
-        Mechanize.new.get(uri).is_a?(Mechanize::Page)
+        @mechanize.get(uri).is_a?(Mechanize::Page)
       rescue
         false
       end

data/lib/scrappy/server/admin.rb

@@ -0,0 +1,24 @@
+module Scrappy
+  module Admin
+    def self.registered app
+      app.get '/' do
+        if params[:format] and params[:uri]
+          redirect "#{settings.base_uri}/#{params[:format]}/#{simplify_uri(params[:uri])}"
+        else
+          haml :home
+        end
+      end
+      
+      app.get '/help' do
+        haml :help
+      end
+      
+      app.get '/kb' do
+        @uris = ( Agent::Options.kb.find(nil, Node('rdf:type'), Node('sc:UriSelector')) +
+                  Agent::Options.kb.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector')) ).
+                  map { |node| node.rdf::value }.flatten.sort.map(&:to_s)
+        haml :kb
+      end
+    end
+  end
+end

data/lib/scrappy/server/helpers.rb

@@ -0,0 +1,23 @@
+module Scrappy
+  module JavaScriptHelpers
+    def bookmark_js
+      "javascript:(function(){" +
+      "if(!document.getElementById('scrappy')){" +
+        "var e=document.createElement('script');" +
+        "e.src='https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js';" +
+        "e.id='scrappy';" +
+        "document.getElementsByTagName('head')[0].appendChild(e);};" +
+      "if(!window.scrappy_loaded){" +
+        "e=document.createElement('script');" +
+        "e.src='http://localhost:3434/javascripts/scrappy.js?_=#{Time.now.to_i}';" +
+        "e.onerror=function(){alert('Error: Please start Scrappy Server at http://localhost:3434');};" +
+        "document.getElementsByTagName('head')[0].appendChild(e);" +
+      "}"+
+      "})();"
+    end
+    
+    def drag_js
+      "alert(\"Don't click this. Drag it to your bookmarks\"); return false;"
+    end
+  end
+end

data/lib/scrappy/server/server.rb

@@ -1,26 +1,18 @@
 require 'sinatra'
 require 'thin'
 require 'haml'
+require 'scrappy/server/helpers'
+require 'scrappy/server/admin'
 
 module Scrappy
   class Server < Sinatra::Base
+    helpers JavaScriptHelpers
+    
     enable :sessions
-    set    :root,   File.dirname(__FILE__)
+    set    :root,   File.join(File.dirname(__FILE__), '..', '..', '..')
     set    :views,  Proc.new { File.join(root, "views") }
     set    :public, Proc.new { File.join(root, "public") }
 
-    get '/' do
-      if params[:format] and params[:uri]
-        redirect "#{settings.base_uri}/#{params[:format]}/#{simplify_uri(params[:uri])}"
-      else
-        haml :home
-      end
-    end
-    
-    get '/help' do
-      haml :help
-    end
-    
     get '/:format/*' do |format, url|
       process_request :get, format, url, params[:callback]
     end

data/{lib/js/annotator.js → public/javascripts/scrappy.js}

@@ -1,3 +1,22 @@
+add_visual_data = function() {
+  var items = document.documentElement.getElementsByTagName('*');
+  var i=0;
+  for(var i=0; i<items.length; i++) {
+    var item = items[i];
+    item.setAttribute('vx', item.offsetLeft);
+    item.setAttribute('vy', item.offsetTop);
+    item.setAttribute('vw', item.offsetWidth);
+    item.setAttribute('vh', item.offsetHeight);
+    item.setAttribute('vsize', document.defaultView.getComputedStyle(item, null).getPropertyValue('font-size'));
+    var weight = document.defaultView.getComputedStyle(item, null).getPropertyValue('font-weight');
+    if (weight == 'normal') weight = 400;
+    if (weight == 'bold')   weight = 700;
+    item.setAttribute('vweight', weight);
+    item.setAttribute('vcolor', document.defaultView.getComputedStyle(item, null).getPropertyValue('color'));
+    item.setAttribute('vbcolor', document.defaultView.getComputedStyle(item, null).getPropertyValue('background-color'));
+  }
+}
+
 $(document).ready(function(){
   $("body").append("<div id='myTrees'></div>")
 	$("#page > *").bind('mouseover', function(e){
@@ -42,3 +61,5 @@ $(document).ready(function(){
     myTrees.appendChild(li);
   });
 });
+
+window.scrappy_loaded = true

data/public/stylesheets/application.css

@@ -0,0 +1,171 @@
+body {
+  font-family: Arial, sans;
+  margin: 0;
+}
+pre {
+  width: 600px;
+  margin-left: auto;
+  margin-right: auto;
+  border: 1px solid;
+  padding: 10px;
+}
+a:link, a:visited {
+  color: #33f;
+  text-decoration: none;
+}
+a:hover, a:active {
+  color: #33f;
+  text-decoration: underline;
+}
+h1, h2, h3, h4 {
+  color: #777;
+}
+h2 {
+  font-weight: normal;
+  padding-bottom: 3px;
+  padding-top: 5px;
+  border-bottom: 1px solid #aaa;
+}
+img {
+  border: none;
+}
+
+
+#bar {
+  height: 33px;
+  font-size: 13px;
+  border-bottom: 1px solid;
+  background-color: #eee;
+  color: #ddd;
+  margin: 0;
+  padding: 0;
+}
+#bar ul {
+  display: block;
+  margin: 0 5px 0 5px;
+  padding: 0;
+  list-style-type: none;
+}
+#bar li {
+  display: inline-block;
+  margin: 0;
+  padding: 0;
+}
+#bar a {
+  position: relative;
+  margin-left: -4px;
+  height: 14px;
+  padding: 10px;
+  display: block;
+}
+#bar a:hover {
+  background-color: #ddd;
+  text-decoration: none;
+}
+#bar img {
+  position: relative;
+  top: -3px;
+  vertical-align: middle;
+}
+#bar ul.left {
+  float: left;
+}
+#bar ul.right {
+  float: right;
+}
+#column {
+  float:left;
+  margin-top: 20px;
+  height: auto;
+  width: 200px;
+  height: 600px;
+  border-right: 1px solid #ddd;
+  overflow-y: auto;
+}
+#body {
+  margin: auto; width: 800px; padding: 15px;
+  font-size: 14px; color: #333;
+}
+#center {
+  text-align: center;
+  margin-top: 100px;
+  margin-bottom: 100px;
+}
+#search {
+  margin-top: 40px;
+  font-size:20px;
+  margin-bottom: 10px;
+}
+#search input  {
+  width: 700px; height:30px; font-size:16px;
+}
+#buttons {
+  width: 400px;
+  margin: auto;
+}
+#buttons select {
+  width: 100px; height: 30px; font-size:16px;
+  margin-left: 5px;
+}
+#buttons button {
+  width: 100px; height: 30px; font-size:16px;
+  margin-right: 5px;
+}
+#footer {
+  margin-top:30px; text-align: center; font-size:14px; color: #555;
+  height: 50px;
+}
+
+.bookmark,
+a.bookmark:hover,
+a.bookmark:visited,
+a.bookmark:link,
+a.bookmark:active {
+  font-size: 11px;
+  text-decoration: none;
+  border-radius: 7px;
+  padding: 6px;
+  background-color: #ccc;
+  background: -webkit-gradient(linear, left top, left bottom, from(#d7d7d7), to(#ababab));
+  background: -moz-linear-gradient(top, #d7d7d7, #ababab);
+  border-color: #c9c9c9;
+  border-bottom-color: #9a9a9a;
+  border-width: 1px;
+  border-style: solid;
+  margin: 5px;
+  color: #444;
+  -moz-border-radius: 7px;
+}
+
+ul.detail {
+  padding: 0;
+  list-style-type: none;
+}
+ul.detail li {
+  padding: 6px;
+  background-color: #eee;
+  margin: 1px;
+}
+ul.detail li span {
+  display: inline-block;
+}
+ul.detail li span.name {
+  width: 600px;
+  overflow-x: hidden;
+  font-family: monospace;
+  font-size: 12px;
+}
+ul.detail li span.format {
+  float: right;
+  font-size: 12px;
+  font-weight: bold;
+  margin-left: 10px;
+  text-align: center;
+}
+
+ul.detail li span.format a:hover,
+ul.detail li span.format a:link,
+ul.detail li span.format a:visited,
+ul.detail li span.format a:active {
+  color: #900;
+}

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.2.0"
+  s.version = "0.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-03-09}
+  s.date = %q{2011-03-11}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/proxy.rb", "lib/scrappy/server/server.rb", "lib/scrappy/server/public/images/logo.png", "lib/scrappy/server/public/images/logo_small.png", "lib/scrappy/server/public/stylesheets/application.css", "lib/scrappy/server/views/home.haml", "lib/scrappy/server/views/help.haml", "lib/scrappy/repository.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/proxy.rb", "lib/scrappy/server/server.rb", "lib/scrappy/server/public/images/logo.png", "lib/scrappy/server/public/images/logo_small.png", "lib/scrappy/server/public/stylesheets/application.css", "lib/scrappy/server/views/home.haml", "lib/scrappy/server/views/help.haml", "lib/scrappy/repository.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/repository.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/admin.rb", "lib/scrappy/server/helpers.rb", "lib/scrappy/server/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/repository.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/admin.rb", "lib/scrappy/server/helpers.rb", "lib/scrappy/server/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "public/favicon.ico", "public/images/logo.png", "public/images/logo_tiny.png", "public/javascripts/scrappy.js", "public/stylesheets/application.css", "test/test_helper.rb", "test/test_scrappy.rb", "views/help.haml", "views/home.haml", "views/kb.haml", "views/layout.haml", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

data/views/help.haml

@@ -0,0 +1,20 @@
+#body
+  %h1 Help
+  %p
+    Scrappy Web Server is a web interface to Scrappy functionalities.
+    You can use it to get data from a web resource and integrate it with other system.
+  %h2 URL format
+  %p
+    The service uses the following URL format:
+    %pre http://[host]/[format]/[url]
+  %p
+    For example, to retrieve http://example.com/~user/?test=1 with RDF serialization:
+    %pre==#{settings.base_uri || "http://localhost:#{settings.port}"}/rdf/http://example.com/~user/%3Ftest%3D1
+  %p Available serialization formats are rdf, rdfxml, png, yarf, ntriples, turtle, json, jsonrdf, and ejson.
+  %h2 Creating extractors
+  %p
+    Drag this to your bookmarks:
+    %a.bookmark{:href=>bookmark_js, :onclick=>drag_js} Scrappy
+  %p
+    Then visit the web page you want to build a extractor for.
+    Click on your "Scrappy" bookmark and annotate the web page.

data/views/home.haml

@@ -0,0 +1,14 @@
+#center
+  %a{:href=>'http://github.com/josei/scrappy'}
+    %img{:src=>"#{settings.base_uri}/images/logo.png"}
+  %form
+    #search
+      %input{:name=>:uri, :size=>30, :type=>:text, :value=>'Enter URI...', :onclick=>"if(this.value=='Enter URI...') this.value='';"}
+    #buttons
+      %button Scrape
+      %select{:name=>:format}
+        %option{:value=>:rdf} RDF
+        %option{:value=>:png} PNG
+        %option{:value=>:ejson} JSON
+        %option{:value=>:yarf} YARF
+        %option{:value=>:ntriples} nTriples

data/views/kb.haml

@@ -0,0 +1,15 @@
+#body
+  %h1 Knowledge base
+  %p
+    %ul.detail
+      -@uris.each do |uri|
+        %li
+          %span.name
+            -if !uri.include?('*')
+              %a{:href=>uri}=uri
+            -else
+              =uri
+          -if !uri.include?('*')
+            -[['RDF', :rdf], ['JSON', :ejson], ['YARF', :yarf], ['PNG', :png]].reverse.each do |format, format_code|
+              %span.format
+                %a{:href=>"#{settings.base_uri}/#{format_code}/#{uri}"}=format

data/views/layout.haml

@@ -0,0 +1,22 @@
+!!!
+%html
+  %head
+    %title Scrappy
+    %link{:type=>"text/css", :href=>"#{settings.base_uri}/stylesheets/application.css", :rel=>"stylesheet"}
+  %body
+    #bar
+      -if request.fullpath!='/'
+        %ul.left
+          %li
+            %a{:href=>"#{settings.base_uri}/"}
+              %img{:src=>"#{settings.base_uri}/images/logo_tiny.png", :alt=>"Scrappy"}
+      %ul.right
+        %li
+          %a{:href=>"#{settings.base_uri}/kb"} Knowledge base
+        %li
+          %a{:href=>"#{settings.base_uri}/help"} Help
+    =yield
+    #footer
+      %a{:href=>"#{settings.base_uri}/"} Home
+      |
+      %a{:href=>'http://github.com/josei/scrappy'} About

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-09 00:00:00 +01:00
+date: 2011-03-11 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -152,16 +152,16 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 - bin/scrappy
-- lib/js/annotator.js
 - lib/scrappy.rb
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
+- lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
-- lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
+- lib/scrappy/repository.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
 - lib/scrappy/selectors/new_uri.rb
@@ -171,14 +171,9 @@ extra_rdoc_files:
 - lib/scrappy/selectors/uri.rb
 - lib/scrappy/selectors/uri_pattern.rb
 - lib/scrappy/selectors/xpath.rb
-- lib/scrappy/server/proxy.rb
+- lib/scrappy/server/admin.rb
+- lib/scrappy/server/helpers.rb
 - lib/scrappy/server/server.rb
-- lib/scrappy/server/public/images/logo.png
-- lib/scrappy/server/public/images/logo_small.png
-- lib/scrappy/server/public/stylesheets/application.css
-- lib/scrappy/server/views/home.haml
-- lib/scrappy/server/views/help.haml
-- lib/scrappy/repository.rb
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
 - lib/scrappy/webkit/webkit.rb
@@ -189,16 +184,16 @@ files:
 - Rakefile
 - bin/scrappy
 - kb/elmundo.yarf
-- lib/js/annotator.js
 - lib/scrappy.rb
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
+- lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
-- lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
+- lib/scrappy/repository.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
 - lib/scrappy/selectors/new_uri.rb
@@ -208,19 +203,23 @@ files:
 - lib/scrappy/selectors/uri.rb
 - lib/scrappy/selectors/uri_pattern.rb
 - lib/scrappy/selectors/xpath.rb
-- lib/scrappy/server/proxy.rb
+- lib/scrappy/server/admin.rb
+- lib/scrappy/server/helpers.rb
 - lib/scrappy/server/server.rb
-- lib/scrappy/server/public/images/logo.png
-- lib/scrappy/server/public/images/logo_small.png
-- lib/scrappy/server/public/stylesheets/application.css
-- lib/scrappy/server/views/home.haml
-- lib/scrappy/server/views/help.haml
-- lib/scrappy/repository.rb
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
 - lib/scrappy/webkit/webkit.rb
+- public/favicon.ico
+- public/images/logo.png
+- public/images/logo_tiny.png
+- public/javascripts/scrappy.js
+- public/stylesheets/application.css
 - test/test_helper.rb
 - test/test_scrappy.rb
+- views/help.haml
+- views/home.haml
+- views/kb.haml
+- views/layout.haml
 - scrappy.gemspec
 has_rdoc: true
 homepage: http://github.com/josei/scrappy

data/lib/scrappy/server/proxy.rb

@@ -1,34 +0,0 @@
-require 'sinatra'
-require 'thin'
-
-module Scrappy
-  class Proxy < Sinatra::Base
-    get '*' do
-      process_request :get
-    end
-
-    post '*' do
-      process_request :post
-    end
-
-    protected
-    def process_request method
-      response = agent.proxy :method=>method, :uri=>request.env['REQUEST_URI'], :inputs=>params
-
-      case response.status
-      when :redirect
-        redirect response.uri
-      when :ok
-        headers 'Content-Type' => response.content_type
-        response.output
-      else
-        status 500
-        "Internal error"
-      end
-    end
-
-    def agent
-      Scrappy::Agent[request.ip]
-    end
-  end
-end

data/lib/scrappy/server/public/stylesheets/application.css

@@ -1,51 +0,0 @@
-body {
-  font-family: Arial, sans;
-}
-#center {
-  text-align: center;
-  margin-top: 100px;
-  margin-bottom: 100px;
-}
-#search {
-  margin-top: 40px;
-  font-size:20px;
-  margin-bottom: 10px;
-}
-#search input  {
-  width: 700px; height:30px; font-size:16px;
-}
-#buttons {
-  width: 400px;
-  margin: auto;
-}
-#buttons select {
-  width: 100px; height: 30px; font-size:16px;
-  margin-left: 5px;
-}
-#buttons button {
-  width: 100px; height: 30px; font-size:16px;
-  margin-right: 5px;
-}
-
-pre {
-  width: 600px;
-  margin-left: auto;
-  margin-right: auto;
-  border: 1px solid;
-  padding:10px;
-}
-#header {
-  margin: auto; width: 800px; padding: 15px;
-  border-bottom: 1px solid;
-  margin-top: 20px; font-size: 14px; color: #555;
-}
-#body {
-  margin: auto; width: 800px; padding: 15px;
-  font-size: 14px; color: #555;
-}
-#footer {
-  margin-top:30px; text-align: center; font-size:14px; color: #555;
-}
-img {
-  border: none;
-}

data/lib/scrappy/server/views/help.haml

@@ -1,25 +0,0 @@
-!!!
-%html
-  %head
-    %title Help - Scrappy
-    %link{:type=>"text/css", :href=>"#{settings.base_uri}/stylesheets/application.css", :rel=>"stylesheet"}
-  %body
-    %div#header
-      %img{:src=>"#{settings.base_uri}/images/logo_small.png"}
-    %div#body
-      %h1 Help
-      %p
-        Scrappy web server is a web interface to Scrappy functionalities.
-        You can use it to get data from a web resource and integrate it with other system.
-        The service uses the following URL format:
-        %pre http://[host]/[format]/[url]
-      %p
-        For example, to retrieve http://example.com/~user/?test=1 with RDF serialization:
-        %pre==#{settings.base_uri || "http://localhost:#{settings.port}"}/rdf/http://example.com/~user/%3Ftest%3D1
-      %p Available serialization formats are rdf, rdfxml, png, yarf, ntriples, turtle, json, jsonrdf, and ejson.
-    %div#footer
-      %a{:href=>"#{settings.base_uri}/"} Home
-      |
-      %a{:href=>"#{settings.base_uri}/help"} Help
-      |
-      %a{:href=>'http://github.com/josei/scrappy'} About

data/lib/scrappy/server/views/home.haml

@@ -1,26 +0,0 @@
-!!!
-%html
-  %head
-    %title Scrappy
-    %link{:type=>"text/css", :href=>"#{settings.base_uri}/stylesheets/application.css", :rel=>"stylesheet"}
-  %body
-    %div#center
-      %a{:href=>'http://github.com/josei/scrappy'}
-        %img{:src=>"#{settings.base_uri}/images/logo.png"}
-      %form
-        %div#search
-          %input{:name=>:uri, :size=>30, :type=>:text, :value=>'Enter URI...', :onclick=>"if(this.value=='Enter URI...') this.value='';"}
-        %div#buttons
-          %button Scrape
-          %select{:name=>:format}
-            %option{:value=>:rdf} RDF
-            %option{:value=>:png} PNG
-            %option{:value=>:ejson} JSON
-            %option{:value=>:yarf} YARF
-            %option{:value=>:ntriples} nTriples
-    %div#footer
-      %a{:href=>"#{settings.base_uri}/"} Home
-      |
-      %a{:href=>"#{settings.base_uri}/help"} Help
-      |
-      %a{:href=>'http://github.com/josei/scrappy'} About