scrappy 0.1.18 → 0.1.19

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.1.19 2011-02-24
+
+* Using Sinatra (in production mode) instead of Camping
+
 === 0.1.18 2011-02-18
 
 * Retry support when opening URLs

data/Manifest

@@ -27,6 +27,7 @@ lib/scrappy/selectors/xpath.rb
 lib/scrappy/server.rb
 lib/scrappy/shell.rb
 lib/scrappy/support.rb
+lib/scrappy/views/home.haml
 lib/scrappy/webkit/webkit.rb
 test/test_helper.rb
 test/test_scrappy.rb

data/Rakefile

@@ -11,7 +11,7 @@ Echoe.new('scrappy', Scrappy::VERSION) do |p|
   p.email          = "joseignacio.fernandez@gmail.com"
   p.install_message = '**(Optional) Remember to install rbwebkitgtk for visual parsing features**'
   p.ignore_pattern = ["pkg/*"]
-  p.dependencies = [['activesupport','>= 2.3.5'], ['markaby', '>= 0.7.1'], ['camping', '= 2.0'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1.9'], ['mongrel', '>= 1.1.5'], ['i18n', '>= 0.4.2']]
+  p.dependencies = [['activesupport','>= 2.3.5'], ['sinatra', '>= 1.1.2'], ['thin', '>= 1.2.7'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1.9'], ['i18n', '>= 0.4.2']]
 end
 
 Rake::RDocTask.new(:rdoc) do |rdoc|

data/bin/scrappy

@@ -11,14 +11,8 @@ module Scrappy
 
   require 'rubygems'
   require 'optparse'
-  require 'logger'
-  require 'readline'
-  gem 'camping', '=2.0'
-  require 'camping'
-  require 'camping/server'
   require "#{Root}/lib/scrappy"
-  require 'scrappy/shell'
-
+  
   SESSION_TOKEN = rand(100000000)
   Options = OpenStruct.new
 
@@ -64,14 +58,20 @@ module Scrappy
         puts Agent.create.proxy(:http_method=>:get, :uri=>Options.url).output
       elsif Options.proxy
         puts "Launching Scrappy Web Proxy..."
-        Camping::Server.new(OpenStruct.new(:host => 'localhost', :port => Options.port, :server=>'mongrel'), ["#{Scrappy::Root}/lib/scrappy/proxy.rb"]).start
+        require 'scrappy/proxy'
+        Thin::Logging.silent = true
+        Scrappy::Proxy.run! :host => 'localhost', :port => Options.port, :environment=>:production
       elsif Options.server
         puts "Launching Scrappy Web Server..."
-        Camping::Server.new(OpenStruct.new(:host => 'localhost', :port => Options.port, :server=>'mongrel'), ["#{Scrappy::Root}/lib/scrappy/server.rb"]).start
+        require 'scrappy/server'
+        Thin::Logging.silent = true
+        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment=>:production
       elsif Options.shell
         puts "Launching Scrappy Shell..."
+        require 'scrappy/shell'
         Shell.new.run
       else
+        require 'scrappy/shell'
         Options.quiet = true
         Shell.new(@file).run
       end

data/lib/scrappy.rb

@@ -21,7 +21,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.18'
+  VERSION = '0.1.19'
 end
 
 # Require selectors

data/lib/scrappy/proxy.rb

@@ -1,41 +1,34 @@
-require 'camping'
-require 'camping/session'
-require 'open3'
-
-Camping.goes :Scrappy
+require 'sinatra'
+require 'thin'
 
 module Scrappy
-  module Controllers
-    class Index < R '.*'
-      include InputEscaping
-
-      def get
-        process_request :get
-      end
+  class Proxy < Sinatra::Base
+    get '*' do
+      process_request :get
+    end
 
-      def post
-        process_request :post
-      end
+    post '*' do
+      process_request :post
+    end
 
-      protected
-      def process_request method
-        response = agent.proxy :method=>method, :uri=>request.env["REQUEST_URI"], :inputs=>@input
+    protected
+    def process_request method
+      response = agent.proxy :method=>method, :uri=>request.env['REQUEST_URI'], :inputs=>params
 
-        case response.status
-        when :redirect
-          redirect response.uri
-        when :ok
-          @headers['Content-Type'] = response.content_type
-          response.output
-        else
-          @status = 500
-          'Error'
-        end
+      case response.status
+      when :redirect
+        redirect response.uri
+      when :ok
+        headers 'Content-Type' => response.content_type
+        response.output
+      else
+        status 500
+        "Internal error"
       end
+    end
 
-      def agent
-        Scrappy::Agent[@request.env["REMOTE_ADDR"]]
-      end
+    def agent
+      Scrappy::Agent[request.ip]
     end
   end
 end

data/lib/scrappy/server.rb

@@ -1,76 +1,55 @@
-require 'camping'
-require 'camping/session'
-require 'open3'
-
-Camping.goes :Scrappy
+require 'sinatra'
+require 'thin'
+require 'haml'
 
 module Scrappy
-  include Camping::Session
-  secret '1a36591bceec49c832079e270d7e8b73'
+  class Server < Sinatra::Base
+    enable :sessions
+    set    :views, File.dirname(__FILE__) + '/views'
 
-  module Controllers
-    class Index
-      def get
-        mab do
-          html do
-            head {}
-            body do
-              h1 "Scrappy Web Server"
-              p  "Use following URL format: http://[host]/[format]/[url]"
-              p  do
-                "For example: " + a("http://localhost:3434/rdfxml/http://www.google.com",
-                             :href=>"http://localhost:3434/rdfxml/http://www.google.com")
-              end
-              p do
-                "Remember to escape parameters: " +
-                "http://www.example.com/~user/%3Ftest%3D1%26test1%3D2<br/> or<br/> " +
-                "http%3A%2F%2Fwww.example.com%2F~user%2F%3Ftest%3D1%26test1%3D2<br/>" +
-                "instead of<br/> http://www.example.com/~user/?test=1&test1=2"
-              end
-              p do
-                "Available formats are png, yarf, rdfxml, ntriples, turtle, json, jsonrdf, ejson"
-              end
-            end
-          end
-        end
-      end
+    get '/' do
+      haml :home
     end
 
-    class Extract < R '/(\w+)/(.+)'
-      include InputEscaping
+    get '/:format/*' do
+      process_request :get, params[:format], params[:splat] * "/", params[:callback]
+    end
 
-      def get format, url
-        process_request :get, format, url
-      end
+    post '/:format/*' do
+      process_request :post, params[:format], params[:splat] * "/", params[:callback]
+    end
 
-      def post format, url
-        process_request :post, format, url
+    protected
+    def process_request method, format, url, callback
+      response = agent.proxy :method=>method, :uri=>url, :inputs=>inputs, :format=>format.to_sym
+      case response.status
+      when :redirect
+        redirect "/#{format}/#{CGI::escape(response.uri).gsub('%2F','/').gsub('%3A',':')}#{textual_inputs}"
+      when :ok
+        headers 'Content-Type' => response.content_type
+        callback ? "#{callback}(#{response.output})" : response.output
+      else
+        status 500
+        "Internal error"
       end
+    end
 
-      protected
-      def process_request method, format, url
-        callback = @input['callback']
-        response = agent.proxy :method=>method, :uri=>url, :inputs=>@input.reject{|k,v| k=='callback'}, :format=>format.to_sym
-        case response.status
-        when :redirect
-          redirect "/#{format}/#{CGI::escape(response.uri).gsub('%2F','/').gsub('%3A',':')}#{inputs}"
-        when :ok
-          @headers['Content-Type'] = response.content_type
-          callback ? "#{callback}(#{response.output})" : response.output
-        else
-          @status = 500
-          'Error'
-        end
+    def agent
+      return @agent if @agent
+      if session[:agent].nil? || session[:token] != SESSION_TOKEN
+        session[:token] = SESSION_TOKEN
+        session[:agent] = Scrappy::Agent.create.id
       end
+      @agent = Scrappy::Agent[session[:agent]]
+    end
 
-      def agent
-        return @agent if @agent
-        if @state[:agent].nil? || @state[:token] != SESSION_TOKEN
-          @state[:token] = SESSION_TOKEN
-          @state[:agent] = Scrappy::Agent.create.id
-        end
-        @agent = Scrappy::Agent[@state[:agent]]
-      end
+    def inputs
+      params.reject{|k,v| ['callback', 'splat', 'format']}
+    end
+    
+    def textual_inputs
+      return '' if inputs.empty?
+      "?" + (inputs.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}*'')
     end
   end
-end
+end

data/lib/scrappy/shell.rb

@@ -1,3 +1,5 @@
+require 'readline'
+
 module Scrappy
   class Shell
     def initialize file=nil

data/lib/scrappy/support.rb

@@ -8,15 +8,6 @@ module URI
   end
 end
 
-module Scrappy
-  module InputEscaping
-    def inputs
-      return '' if @input.empty?
-      "?" + (@input.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}*'')
-    end
-  end
-end
-
 module Nokogiri
   module XML
     class NodeSet
@@ -31,4 +22,4 @@ class String
   def wikify
     gsub(/^[a-z]|\s+[a-z]/) { |a| a.upcase }.gsub(/\s/, '')
   end
-end
+end

data/lib/scrappy/views/home.haml

@@ -0,0 +1,23 @@
+!!!
+%html
+  %head
+    %title Scrappy Web Server
+  %body
+    %h1 Scrappy Web Server
+    %p Use following URL format: http://[host]/[format]/[url]
+    %p
+      For example:
+      %a{:href=>"http://localhost:3434/rdfxml/http://www.google.com"}
+        http://localhost:3434/rdfxml/http://www.google.com
+                                 
+      Remember to escape parameters: http://www.example.com/~user/%3Ftest%3D1%26test1%3D2
+      %br
+      or
+      %br
+      http%3A%2F%2Fwww.example.com%2F~user%2F%3Ftest%3D1%26test1%3D2
+      %br
+      instead of
+      %br
+      http://www.example.com/~user/?test=1&test1=2"
+    
+    %p Available formats are png, yarf, rdfxml, ntriples, turtle, json, jsonrdf, ejson

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.18"
+  s.version = "0.1.19"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-02-18}
+  s.date = %q{2011-02-24}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/views/home.haml", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/views/home.haml", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]
@@ -28,31 +28,28 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.5"])
-      s.add_runtime_dependency(%q<markaby>, [">= 0.7.1"])
-      s.add_runtime_dependency(%q<camping>, ["= 2.0"])
+      s.add_runtime_dependency(%q<sinatra>, [">= 1.1.2"])
+      s.add_runtime_dependency(%q<thin>, [">= 1.2.7"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_runtime_dependency(%q<mechanize>, [">= 1.0.0"])
       s.add_runtime_dependency(%q<lightrdf>, [">= 0.1.9"])
-      s.add_runtime_dependency(%q<mongrel>, [">= 1.1.5"])
       s.add_runtime_dependency(%q<i18n>, [">= 0.4.2"])
     else
       s.add_dependency(%q<activesupport>, [">= 2.3.5"])
-      s.add_dependency(%q<markaby>, [">= 0.7.1"])
-      s.add_dependency(%q<camping>, ["= 2.0"])
+      s.add_dependency(%q<sinatra>, [">= 1.1.2"])
+      s.add_dependency(%q<thin>, [">= 1.2.7"])
       s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_dependency(%q<mechanize>, [">= 1.0.0"])
       s.add_dependency(%q<lightrdf>, [">= 0.1.9"])
-      s.add_dependency(%q<mongrel>, [">= 1.1.5"])
       s.add_dependency(%q<i18n>, [">= 0.4.2"])
     end
   else
     s.add_dependency(%q<activesupport>, [">= 2.3.5"])
-    s.add_dependency(%q<markaby>, [">= 0.7.1"])
-    s.add_dependency(%q<camping>, ["= 2.0"])
+    s.add_dependency(%q<sinatra>, [">= 1.1.2"])
+    s.add_dependency(%q<thin>, [">= 1.2.7"])
     s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
     s.add_dependency(%q<mechanize>, [">= 1.0.0"])
     s.add_dependency(%q<lightrdf>, [">= 0.1.9"])
-    s.add_dependency(%q<mongrel>, [">= 1.1.5"])
     s.add_dependency(%q<i18n>, [">= 0.4.2"])
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 18
-  version: 0.1.18
+  - 19
+  version: 0.1.19
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-18 00:00:00 +01:00
+date: 2011-02-24 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -32,30 +32,31 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: markaby
+  name: sinatra
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
         segments: 
-        - 0
-        - 7
         - 1
-        version: 0.7.1
+        - 1
+        - 2
+        version: 1.1.2
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: camping
+  name: thin
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
         segments: 
+        - 1
         - 2
-        - 0
-        version: "2.0"
+        - 7
+        version: 1.2.7
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -100,24 +101,10 @@ dependencies:
         version: 0.1.9
   type: :runtime
   version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: mongrel
-  prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 1
-        - 5
-        version: 1.1.5
-  type: :runtime
-  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: i18n
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -127,7 +114,7 @@ dependencies:
         - 2
         version: 0.4.2
   type: :runtime
-  version_requirements: *id008
+  version_requirements: *id007
 description: RDF web scraper
 email: joseignacio.fernandez@gmail.com
 executables: 
@@ -160,6 +147,7 @@ extra_rdoc_files:
 - lib/scrappy/server.rb
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
+- lib/scrappy/views/home.haml
 - lib/scrappy/webkit/webkit.rb
 files: 
 - History.txt
@@ -191,6 +179,7 @@ files:
 - lib/scrappy/server.rb
 - lib/scrappy/shell.rb
 - lib/scrappy/support.rb
+- lib/scrappy/views/home.haml
 - lib/scrappy/webkit/webkit.rb
 - test/test_helper.rb
 - test/test_scrappy.rb