scrappy 0.1.22 → 0.1.23

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.1.23 2011-03-03
+
+* Prettier web interface
+
 === 0.1.22 2011-02-25
 
 * Support for reverse proxy in web server

data/Manifest

@@ -14,7 +14,6 @@ lib/scrappy/agent/formats.rb
 lib/scrappy/agent/map_reduce.rb
 lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/visual_agent.rb
-lib/scrappy/proxy.rb
 lib/scrappy/selectors/base_uri.rb
 lib/scrappy/selectors/css.rb
 lib/scrappy/selectors/new_uri.rb
@@ -24,10 +23,15 @@ lib/scrappy/selectors/slice.rb
 lib/scrappy/selectors/uri.rb
 lib/scrappy/selectors/uri_pattern.rb
 lib/scrappy/selectors/xpath.rb
-lib/scrappy/server.rb
+lib/scrappy/server/proxy.rb
+lib/scrappy/server/server.rb
+lib/scrappy/server/public/images/logo.png
+lib/scrappy/server/public/images/logo_small.png
+lib/scrappy/server/public/stylesheets/application.css
+lib/scrappy/server/views/home.haml
+lib/scrappy/server/views/help.haml
 lib/scrappy/shell.rb
 lib/scrappy/support.rb
-lib/scrappy/views/home.haml
 lib/scrappy/webkit/webkit.rb
 test/test_helper.rb
 test/test_scrappy.rb

data/bin/scrappy

@@ -36,7 +36,7 @@ module Scrappy
         opts.on('-D', '--dump')                 { Agent::Options.dump = true; Agent::Options.format = :rdf }
         opts.on('-u', '--debug')                { Agent::Options.debug = true }
         opts.on('-i', '--interactive')          { Options.shell = true; Agent::Options.format_header = false }
-        opts.on('-s', '--server [ROOT]')        { |url| Options.server = true; Options.root = url }
+        opts.on('-s', '--server [BASE_URI]')    { |uri| Options.server = true; Options.base_uri = uri }
         opts.on('-S', '--proxy-server')         { Options.proxy = true }
         opts.on('-P P', '--port P')             { |p| Options.port = p }
         opts.on('-c C', '--concurrence C')      { |c| Agent::Options.workers = c.to_i }
@@ -57,15 +57,15 @@ module Scrappy
         Options.quiet = true
         puts Agent.create.proxy(:http_method=>:get, :uri=>Options.url).output
       elsif Options.proxy
-        puts "Launching Scrappy Web Proxy..."
-        require 'scrappy/proxy'
+        puts "Launching Scrappy Web Proxy (set http://localhost:#{Options.port} as proxy)..."
+        require 'scrappy/server/proxy'
         Thin::Logging.silent = true
         Scrappy::Proxy.run! :host => 'localhost', :port => Options.port, :environment=>:production
       elsif Options.server
-        puts "Launching Scrappy Web Server..."
-        require 'scrappy/server'
+        puts "Launching Scrappy Web Server (browse http://localhost:#{Options.port})..."
+        require 'scrappy/server/server'
         Thin::Logging.silent = true
-        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment=>:production, :root=>Options.root
+        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment=>:production, :base_uri=>Options.base_uri
       elsif Options.shell
         puts "Launching Scrappy Shell..."
         require 'scrappy/shell'

data/lib/scrappy/server/public/stylesheets/application.css

@@ -0,0 +1,40 @@
+body {
+  font-family: Arial, sans;
+}
+.center {
+  text-align: center;
+  margin-top: 100px;
+}
+.search {
+  margin-top: 40px;
+  margin-bottom: 100px;
+  font-size:20px;
+}
+.search input  {
+  width: 400px; height:30px; font-size:16px;
+}
+.search select {
+  width: 80px; height: 30px; font-size:16px;
+}
+.search button {
+  width: 80px; height: 30px; font-size:16px;
+}
+pre {
+  width: 600px;
+  margin-left: auto;
+  margin-right: auto;
+  border: 1px solid;
+  padding:10px;
+}
+#header {
+  margin: auto; width: 800px; padding: 15px;
+  border-bottom: 1px solid;
+  margin-top: 20px; font-size: 14px; color: #555;
+}
+#body {
+  margin: auto; width: 800px; padding: 15px;
+  font-size: 14px; color: #555;
+}
+#footer {
+  margin-top:30px; text-align: center; font-size:14px; color: #555;
+}

data/lib/scrappy/{server.rb → server/server.rb}

@@ -5,12 +5,22 @@ require 'haml'
 module Scrappy
   class Server < Sinatra::Base
     enable :sessions
-    set    :views, File.dirname(__FILE__) + '/views'
+    set    :root,   File.dirname(__FILE__)
+    set    :views,  Proc.new { File.join(root, "views") }
+    set    :public, Proc.new { File.join(root, "public") }
 
     get '/' do
-      haml :home
+      if params[:format] and params[:uri]
+        redirect "#{settings.base_uri}/#{params[:format]}/#{simplify_uri(params[:uri])}"
+      else
+        haml :home
+      end
     end
-
+    
+    get '/help' do
+      haml :help
+    end
+    
     get '/:format/*' do |format, url|
       process_request :get, format, url, params[:callback]
     end
@@ -24,7 +34,7 @@ module Scrappy
       response = agent.proxy :method=>method, :uri=>url, :inputs=>inputs, :format=>format.to_sym
       case response.status
       when :redirect
-        redirect "#{settings.root}/#{format}/#{CGI::escape(response.uri).gsub('%2F','/').gsub('%3A',':')}#{textual_inputs}"
+        redirect "#{settings.base_uri}/#{format}/#{simplify_uri(response.uri)}#{textual_inputs}"
       when :ok
         headers 'Content-Type' => response.content_type
         callback ? "#{callback}(#{response.output})" : response.output
@@ -51,5 +61,9 @@ module Scrappy
       return '' if inputs.merge('callback'=>params[:callback]).reject{|k,v| v.nil?}.empty?
       "?" + (inputs.merge('callback'=>params[:callback]).reject{|k,v| v.nil?}.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}*'')
     end
+    
+    def simplify_uri uri
+      CGI::escape(uri).gsub('%2F','/').gsub('%3A',':')
+    end
   end
 end

data/lib/scrappy/server/views/help.haml

@@ -0,0 +1,25 @@
+!!!
+%html
+  %head
+    %title Help - Scrappy
+    %link{:type=>"text/css", :href=>"/stylesheets/application.css", :rel=>"stylesheet"}
+  %body
+    %div#header
+      %img{:src=>'/images/logo_small.png'}
+    %div#body
+      %h1 Help
+      %p
+        Scrappy web server is a web interface to Scrappy functionalities.
+        You can use it to get data from a web resource and integrate it with other system.
+        The service uses the following URL format:
+        %pre http://[host]/[format]/[url]
+      %p
+        For example, to retrieve http://example.com/~user/?test=1 with RDF serialization:
+        %pre http://localhost:3434/rdf/http://example.com/~user/%3Ftest%3D1
+      %p Available serialization formats are rdf, rdfxml, png, yarf, ntriples, turtle, json, jsonrdf, and ejson.
+    %div#footer
+      %a{:href=>"/"} Home
+      |
+      %a{:href=>"/help"} Help
+      |
+      %a{:href=>'http://github.com/josei/scrappy'} About

data/lib/scrappy/server/views/home.haml

@@ -0,0 +1,25 @@
+!!!
+%html
+  %head
+    %title Scrappy
+    %link{:type=>"text/css", :href=>"/stylesheets/application.css", :rel=>"stylesheet"}
+  %body
+    %div.center
+      %a{:href=>'http://github.com/josei/scrappy'}
+        %img{:src=>'/images/logo.png'}
+      %form.search
+        %div
+          %select{:name=>:format}
+            %option{:value=>:rdf} RDF
+            %option{:value=>:png} PNG
+            %option{:value=>:ejson} JSON
+            %option{:value=>:yarf} YARF
+            %option{:value=>:ntriples} nTriples
+          %input{:name=>:uri, :size=>30, :type=>:text, :value=>'Enter URI...', :onclick=>"if(this.value=='Enter URI...') this.value='';"}
+          %button Scrape
+    %div#footer
+      %a{:href=>"/"} Home
+      |
+      %a{:href=>"/help"} Help
+      |
+      %a{:href=>'http://github.com/josei/scrappy'} About

data/lib/scrappy.rb

@@ -21,7 +21,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.22'
+  VERSION = '0.1.23'
 end
 
 # Require selectors

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.22"
+  s.version = "0.1.23"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-02-25}
+  s.date = %q{2011-03-03}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/views/home.haml", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/views/home.haml", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/proxy.rb", "lib/scrappy/server/server.rb", "lib/scrappy/server/public/images/logo.png", "lib/scrappy/server/public/images/logo_small.png", "lib/scrappy/server/public/stylesheets/application.css", "lib/scrappy/server/views/home.haml", "lib/scrappy/server/views/help.haml", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server/proxy.rb", "lib/scrappy/server/server.rb", "lib/scrappy/server/public/images/logo.png", "lib/scrappy/server/public/images/logo_small.png", "lib/scrappy/server/public/stylesheets/application.css", "lib/scrappy/server/views/home.haml", "lib/scrappy/server/views/help.haml", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 22
-  version: 0.1.22
+  - 23
+  version: 0.1.23
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-25 00:00:00 +01:00
+date: 2011-03-03 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -148,7 +148,6 @@ extra_rdoc_files:
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
-- lib/scrappy/proxy.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
 - lib/scrappy/selectors/new_uri.rb
@@ -158,10 +157,15 @@ extra_rdoc_files:
 - lib/scrappy/selectors/uri.rb
 - lib/scrappy/selectors/uri_pattern.rb
 - lib/scrappy/selectors/xpath.rb
-- lib/scrappy/server.rb
+- lib/scrappy/server/proxy.rb
+- lib/scrappy/server/server.rb
+- lib/scrappy/server/public/images/logo.png
+- lib/scrappy/server/public/images/logo_small.png
+- lib/scrappy/server/public/stylesheets/application.css
+- lib/scrappy/server/views/home.haml
+- lib/scrappy/server/views/help.haml
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
-- lib/scrappy/views/home.haml
 - lib/scrappy/webkit/webkit.rb
 files: 
 - History.txt
@@ -180,7 +184,6 @@ files:
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
-- lib/scrappy/proxy.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
 - lib/scrappy/selectors/new_uri.rb
@@ -190,10 +193,15 @@ files:
 - lib/scrappy/selectors/uri.rb
 - lib/scrappy/selectors/uri_pattern.rb
 - lib/scrappy/selectors/xpath.rb
-- lib/scrappy/server.rb
+- lib/scrappy/server/proxy.rb
+- lib/scrappy/server/server.rb
+- lib/scrappy/server/public/images/logo.png
+- lib/scrappy/server/public/images/logo_small.png
+- lib/scrappy/server/public/stylesheets/application.css
+- lib/scrappy/server/views/home.haml
+- lib/scrappy/server/views/help.haml
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
-- lib/scrappy/views/home.haml
 - lib/scrappy/webkit/webkit.rb
 - test/test_helper.rb
 - test/test_scrappy.rb

data/lib/scrappy/views/home.haml

@@ -1,23 +0,0 @@
-!!!
-%html
-  %head
-    %title Scrappy Web Server
-  %body
-    %h1 Scrappy Web Server
-    %p Use following URL format: http://[host]/[format]/[url]
-    %p
-      For example:
-      %a{:href=>"http://localhost:3434/rdfxml/http://www.google.com"}
-        http://localhost:3434/rdfxml/http://www.google.com
-                                 
-      Remember to escape parameters: http://www.example.com/~user/%3Ftest%3D1%26test1%3D2
-      %br
-      or
-      %br
-      http%3A%2F%2Fwww.example.com%2F~user%2F%3Ftest%3D1%26test1%3D2
-      %br
-      instead of
-      %br
-      http://www.example.com/~user/?test=1&test1=2"
-    
-    %p Available formats are png, yarf, rdfxml, ntriples, turtle, json, jsonrdf, ejson