sunscraper 1.1.0.beta3 → 1.2.0.beta1

data/ext/standalone/sunscraperrpc.h

@@ -7,7 +7,6 @@
 
 class SunscraperWorker;
 class QLocalSocket;
-class QTimer;
 
 class SunscraperRPC : public QObject
 {
@@ -18,47 +17,45 @@ class SunscraperRPC : public QObject
         StateData,
     };
 
-    struct Header {
-        quint32 queryId;
-        quint32 requestType;
-        quint32 dataLength;
-    };
-
     enum Request {
-        RPC_LOAD_HTML = 1,
-        RPC_LOAD_URL  = 2,
+        RPC_LOAD_URL  = 1,
+        RPC_LOAD_HTML = 2,
         RPC_WAIT      = 3,
         RPC_FETCH     = 4,
-        RPC_DISCARD   = 5,
     };
 
 public:
-    SunscraperRPC(QString socketPath);
+    SunscraperRPC(QLocalSocket *socket);
     ~SunscraperRPC();
 
+signals:
+    void disconnected();
+
 private slots:
     void onInputReadable();
     void onInputDisconnected();
-    void onPageRendered(unsigned queryId, QString data);
-    void onTimeout();
+
+    void onFinish(unsigned queryId);
+    void onTimeout(unsigned queryId);
+    void onFetchDone(unsigned queryId, QString data);
 
 private:
+    static unsigned m_nextQueryId;
+    static SunscraperWorker *m_worker;
+
+    unsigned m_queryId;
     QLocalSocket *m_socket;
 
-    State  m_state;
-    Header m_pendingHeader;
+    State      m_state;
+    unsigned   m_pendingRequest, m_pendingDataLength;
     QByteArray m_buffer;
 
-    SunscraperWorker *m_worker;
-
-    QList<unsigned> m_waitQueue;
-    QMap<unsigned, QTimer*> m_timers;
-    QMap<unsigned, QString> m_results;
+    bool       m_result;
 
     SunscraperRPC();
 
-    void processRequest(Header header, QByteArray data);
-    void sendReply(Header header, QByteArray data);
+    void processRequest(unsigned requestType, QByteArray data);
+    void sendReply(QByteArray data);
 };
 
 #endif // SUNSCRAPERRPC_H

data/ext/standalone/sunscraperrpcserver.cpp

@@ -0,0 +1,26 @@
+#include <QLocalServer>
+#include "sunscraperrpcserver.h"
+#include "sunscraperrpc.h"
+
+SunscraperRPCServer::SunscraperRPCServer(QObject *parent) :
+        QObject(parent)
+{
+    m_localServer = new QLocalServer();
+
+    connect(m_localServer, SIGNAL(newConnection()), this, SLOT(onNewConnection()));
+}
+
+bool SunscraperRPCServer::listen(QString socketPath)
+{
+    return m_localServer->listen(socketPath);
+}
+
+void SunscraperRPCServer::onNewConnection()
+{
+    while(m_localServer->hasPendingConnections()) {
+        QLocalSocket *socket = m_localServer->nextPendingConnection();
+
+        SunscraperRPC *rpc = new SunscraperRPC(socket);
+        connect(rpc, SIGNAL(disconnected()), rpc, SLOT(deleteLater()));
+    }
+}

data/ext/standalone/sunscraperrpcserver.h

@@ -0,0 +1,24 @@
+#ifndef SUNSCRAPERRPCSERVER_H
+#define SUNSCRAPERRPCSERVER_H
+
+#include <QObject>
+
+class QLocalServer;
+
+class SunscraperRPCServer : public QObject
+{
+    Q_OBJECT
+
+public:
+    SunscraperRPCServer(QObject *parent = 0);
+
+    bool listen(QString socketPath);
+
+private slots:
+    void onNewConnection();
+
+private:
+    QLocalServer *m_localServer;
+};
+
+#endif

data/ext/sunscraper-ext.pro

@@ -1,2 +1,2 @@
 TEMPLATE = subdirs
-SUBDIRS  = embed standalone
+SUBDIRS  = common embed standalone

data/lib/sunscraper.rb

@@ -1,5 +1,5 @@
-if !defined?(RUBY_ENGINE) && RUBY_VERSION =~ /^1.8/
-  raise RuntimeError, "Sunscraper does not work on Ruby MRI 1.8.x."
+if RUBY_VERSION =~ /^1.8/
+  raise RuntimeError, "Sunscraper does not work on Ruby 1.8."
 end
 
 # Sunscraper loads an HTML page in a headless browser and waits for `Sunscraper.finish()`
@@ -27,9 +27,9 @@ module Sunscraper
     # If your application depends on base URL being available, use {scrape_url}.
     #
     # @param [Integer] timeout timeout in milliseconds
-    def scrape_html(html, timeout=5000)
+    def scrape_html(html, url="about:blank", timeout=5000)
       scrape(timeout) do |worker, context|
-        worker.load_html context, html
+        worker.load_html context, html, url
       end
     end
 
@@ -47,19 +47,19 @@ module Sunscraper
     def scrape(timeout)
       worker = load_worker
 
-      context = worker.create
-      yield worker, context
-      worker.wait(context, timeout)
+      begin
+        context = worker.create
 
-      data = worker.fetch(context)
+        yield worker, context
 
-      if data == "!SUNSCRAPER_TIMEOUT"
-        raise ScrapeTimeout, "Sunscraper has timed out waiting for the callback"
-      else
-        data
+        if worker.wait(context, timeout)
+          worker.fetch(context)
+        else
+          raise ScrapeTimeout, "Sunscraper has timed out waiting for the callback"
+        end
+      ensure
+        worker.finalize(context) if context
       end
-    ensure
-      worker.discard(context) if context
     end
 
     def load_worker

data/lib/sunscraper/library.rb

@@ -19,23 +19,23 @@ module Sunscraper
     ffi_lib File.join(Gem.loaded_specs['sunscraper'].full_gem_path,
                       'ext', 'embed', "libsunscraper.#{extension}")
 
-    attach_function 'create',    :sunscraper_create,    [],                  :pointer
-    attach_function 'load_html', :sunscraper_load_html, [:pointer, :string], :void
-    attach_function 'load_url',  :sunscraper_load_url,  [:pointer, :string], :void
-    attach_function 'fetch',     :sunscraper_fetch,     [:pointer],          :string
-    attach_function 'discard',   :sunscraper_discard,   [:pointer],          :void
+    attach_function 'create',    :sunscraper_create,    [],                        :uint
+    attach_function 'load_html', :sunscraper_load_html, [:uint, :string, :string], :void
+    attach_function 'load_url',  :sunscraper_load_url,  [:uint, :string],          :void
+    attach_function 'fetch',     :sunscraper_fetch,     [:uint],                   :string
+    attach_function 'finalize',  :sunscraper_finalize,  [:uint],                   :void
 
     if RUBY_ENGINE == 'ruby'
       # MRI uses ffi gem and has GVL. Hence, it needs a rb_thread_blocking_region call.
-      attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void,    :blocking => true
+      attach_function 'wait',    :sunscraper_wait,      [:uint, :uint],            :bool,    :blocking => true
     else
       # Rubinius does not have GVL neither it has options in attach_function.
       # Same for JRuby.
-      attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void
+      attach_function 'wait',    :sunscraper_wait,      [:uint, :uint],            :bool
     end
 
-    attach_function 'finalize',  :sunscraper_finalize,  [],                  :void
+    attach_function 'quit',      :sunscraper_quit,      [],                        :void
 
-    at_exit { finalize }
+    at_exit { quit }
   end
 end

data/lib/sunscraper/standalone.rb

@@ -3,113 +3,65 @@ require 'socket'
 # @private
 module Sunscraper
   module Standalone
-    @last_query_id = 0
+    @rpc_mutex  = Mutex.new
+    @rpc_socket = nil
 
-    @rpc_mutex    = Mutex.new
-    @rpc_waiters  = {}
-    @rpc_results  = {}
-    @rpc_thread   = nil
-
-    RPC_LOAD_HTML = 1
-    RPC_LOAD_URL  = 2
+    RPC_LOAD_URL  = 1
+    RPC_LOAD_HTML = 2
     RPC_WAIT      = 3
     RPC_FETCH     = 4
-    RPC_DISCARD   = 5
 
     class << self
-      attr_reader :rpc_mutex, :rpc_waiters, :rpc_results
-
       def create
-        @rpc_mutex.synchronize do
-          @last_query_id += 1
-          @last_query_id
-        end
-      end
-
-      def load_html(query_id, html)
-        perform_rpc query_id,
-          request:     RPC_LOAD_HTML,
-          data:        html
+        connect_to_worker
       end
 
-      def load_url(query_id, url)
-        perform_rpc query_id,
+      def load_url(socket, url)
+        perform_rpc socket,
           request:     RPC_LOAD_URL,
           data:        url
       end
 
-      def wait(query_id, timeout)
-        perform_rpc query_id,
+      def load_html(socket, html, baseUrl)
+        html, baseUrl = [html, baseUrl].map(&:to_s)
+        perform_rpc socket,
+          request:     RPC_LOAD_HTML,
+          data:        [html.length, html, baseUrl.length, baseUrl].pack("Na*Na*")
+      end
+
+      def wait(socket, timeout)
+        result = perform_rpc socket,
           request:     RPC_WAIT,
           data:        [timeout].pack("N"),
           want_result: true
+        code, = result.unpack("N")
+
+        code == 1 # true
       end
 
-      def fetch(query_id)
-        perform_rpc query_id,
+      def fetch(socket)
+        perform_rpc socket,
           request:     RPC_FETCH,
           want_result: true
       end
 
-      def discard(query_id)
-        perform_rpc query_id,
-          request:     RPC_DISCARD
+      def finalize(socket)
+        socket.close
       end
 
       private
 
-      def perform_rpc(query_id, options={})
+      def perform_rpc(socket, options={})
         data  = options[:data] || ""
-        block = options[:want_result]
-
-        @rpc_mutex.synchronize do
-          if @rpc_thread.nil?
-            @rpc_thread = Standalone::Thread.new(::Thread.current)
-
-            # Some fucko decided not to put any semaphores in Ruby,
-            # _and_ restrict Mutexes to be unlocked only from the thread
-            # which has locked them.
-            #
-            # Please, kill yourself if you're reading this.
-            ::Thread.stop
-          end
-
-          @rpc_thread.perform(query_id, options[:request], data)
-
-          if block
-            @rpc_waiters[query_id] = Thread.current
-          end
-        end
-
-        if block
-          Thread.stop
-          @rpc_results[query_id]
-        end
-      ensure
-        if block
-          @rpc_waiters.delete query_id
-          @rpc_results.delete query_id
-        end
-      end
-    end
-
-    class Thread < ::Thread
-      def initialize(creator)
-        @creator = creator
+        socket.write([options[:request], data.length, data].pack("NNa*"))
 
-        super do
-          @parent = Sunscraper::Standalone
-          work
+        if options[:want_result]
+          data_length, = socket.read(4).unpack("N")
+          socket.read(data_length)
         end
       end
 
-      def perform(query_id, request, data)
-        @socket.write([query_id, request, data.length, data].pack("NNNa*"))
-      end
-
-      private
-
-      def work
+      def spawn_worker
         if ::Sunscraper.os_x?
           # Fuck you, OS X.
           suffix = ".app/Contents/MacOS/sunscraper"
@@ -121,47 +73,41 @@ module Sunscraper
                           'ext', 'standalone', "sunscraper#{suffix}")
 
         server_path = "/tmp/sunscraper.#{Process.pid}.sock"
-        server = UNIXServer.new(server_path)
+        File.unlink server_path if File.exists? server_path
 
         if Kernel.respond_to? :spawn
-          pid = Kernel.spawn "#{executable} #{server_path}"
+          @rpc_pid = Kernel.spawn "#{executable} #{server_path}"
         else
           # rbx does not have Kernel.spawn (yet). Sigh...
-          pid = fork { exec executable, server_path }
+          @rpc_pid = fork { exec executable, server_path }
         end
 
-        Process.detach pid
-
-        @socket = server.accept
+        # Sigh again. Probably no other way.
+        loop do
+          if File.exists? server_path
+            @rpc_socket = server_path
+            break
+          elsif Process.wait(@rpc_pid, Process::WNOHANG)
+            raise RuntimeError, "Cannot start Sunscraper process"
+          end
 
-        server.close
-        FileUtils.rm server_path
+          sleep 0.1
+        end
 
-        # See above.
-        @creator.wakeup
+        Process.detach @rpc_pid
 
-        loop do
-          header = @socket.read(4 * 3)
-          query_id, request, data_length = header.unpack("NNN")
-          data   = @socket.read(data_length) if data_length > 0
-
-          @parent.rpc_mutex.synchronize do
-            if !@parent.rpc_waiters.include?(query_id)
-              $stderr.puts "Sunscraper/standalone: no waiter for #{query_id}"
-            else
-              @parent.rpc_results[query_id] = data
-              @parent.rpc_waiters[query_id].wakeup
-            end
-          end
+        at_exit do
+          Process.kill "KILL", @rpc_pid
+          File.unlink @rpc_socket
         end
-      rescue Exception => e
-        $stderr.puts "Sunscraper error: #{e.class}: #{e.message}"
-        e.backtrace.each do |line|
-          $stderr.puts "  #{line}"
+      end
+
+      def connect_to_worker
+        @rpc_mutex.synchronize do
+          spawn_worker if @rpc_socket.nil?
         end
-      ensure
-        @socket.close
-        Process.kill pid
+
+        UNIXSocket.new(@rpc_socket)
       end
     end
   end

data/spec/sunscraper_spec.rb

@@ -1,15 +1,12 @@
 require 'spec_helper'
-
 require 'webrick'
 
-HTML = <<HTML
+HTML_TEMPLATE = <<HTML
 <html>
 <head>
   <script type="text/javascript">
   document.addEventListener("DOMContentLoaded", function() {
-    document.getElementById('fuga').textContent =
-              ("!skrow tI").split("").reverse().join("");
-    Sunscraper.finish();
+    %code%
   }, true);
   </script>
 </head>
@@ -19,73 +16,118 @@ HTML = <<HTML
 </html>
 HTML
 
-PORT = 45555
+HTML_FUGA = HTML_TEMPLATE.sub("%code%", <<CODE)
+    document.getElementById('fuga').textContent =
+              ("!skrow tI").split("").reverse().join("");
+    Sunscraper.finish();
+CODE
+
+HTML_BASEURL = HTML_TEMPLATE.sub("%code%", <<CODE)
+    var xhr = new XMLHttpRequest();
+    xhr.onreadystatechange = function() {
+      if(xhr.readyState > 3) {
+        document.getElementById('fuga').textContent = xhr.responseText;
+        Sunscraper.finish();
+      }
+    };
+    xhr.open('GET', '/comicstrip', 1);
+    xhr.send();
+CODE
+
+HTML_USERAGENT = HTML_TEMPLATE.sub("%code%", <<CODE)
+    document.getElementById('fuga').textContent =
+              window.navigator.userAgent;
+    Sunscraper.finish();
+CODE
+
+HTML_LOCALSTORAGE = HTML_TEMPLATE.sub("%code%", <<CODE)
+    window.localStorage.setItem("key", ["O", "K"].join(""))
+    document.getElementById('fuga').textContent =
+              window.localStorage.getItem("key");
+    Sunscraper.finish();
+CODE
 
-def with_webserver
-  server = WEBrick::HTTPServer.new :Port => PORT, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
+def with_webserver(html)
+  port = 45555
+  server = WEBrick::HTTPServer.new :Port => port, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
   server.mount_proc '/' do |req, res|
-    res.body = HTML
+    res.body = html
   end
-  Thread.new { server.start }
+  server.mount_proc '/comicstrip' do |req, res|
+    res.body = 'Go Get a Roomie!'
+  end
+  thread = Thread.new { server.start }
 
-  yield PORT
+  yield "http://127.0.0.1:#{port}/"
 ensure
-  server.shutdown if server
+  server.shutdown
+  thread.join
 end
 
-class String
-  def to_v
-    split(".").map(&:to_i).extend Comparable
-  end
-end
+define_tests = lambda do |klass, worker|
+  describe klass do
+    before(:all) do
+      Sunscraper.worker = worker
+    end
 
-unless Sunscraper.os_x?
-  # This part currently crashes on OS X (and will forever).
-  describe "Sunscraper::Library" do
-    before do
-      Sunscraper.worker = :embed
+    after(:all) do
+      sleep(5) # let threads rest in peace
     end
 
     it "can scrape an HTML provided as a string" do
-      Sunscraper.scrape_html(HTML).should include('It works!')
+      Sunscraper.scrape_html(HTML_FUGA).should include('It works!')
     end
 
     it "can scrape an URL" do
-      with_webserver do |port|
-        Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
+      with_webserver(HTML_FUGA) do |url|
+        Sunscraper.scrape_url(url).should include('It works!')
       end
     end
 
     it "should time out if callback is not called" do
-      lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
+      lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", "about:blank", 500) }.
           should raise_exception(Sunscraper::ScrapeTimeout)
     end
-  end
-end
 
-if !(RUBY_ENGINE =~ /rbx/ || RUBY_ENGINE =~ /jruby/) ||
-   ENV['EXPERIMENTAL'] == 'true'
-  # This part currently crashes Rubinius (as of Mar 09, 2012),
-  # and crashes jruby < 1.7.0, and uses Unix sockets which don't
-  # work even on jruby master (as of Mar 09, 2012).
-  describe "Sunscraper::Standalone" do
-    before do
-      Sunscraper.worker = :standalone
+    it "respects baseUrl parameter" do
+      with_webserver("<!-- nothing -->") do |url|
+        Sunscraper.scrape_html(HTML_BASEURL, url).should include('Go Get a Roomie')
+      end
     end
 
-    it "can scrape an HTML provided as a string" do
-      Sunscraper.scrape_html(HTML).should include('It works!')
+    it "should identify itself as Sunscraper" do
+      Sunscraper.scrape_html(HTML_USERAGENT).should include("Sunscraper")
     end
 
-    it "can scrape an URL" do
-      with_webserver do |port|
-        Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
+    it "should work with window.localStorage through webserver" do
+      with_webserver(HTML_LOCALSTORAGE) do |url|
+        Sunscraper.scrape_url(url).should include("OK")
       end
     end
 
-    it "should time out if callback is not called" do
-      lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
-          should raise_exception(Sunscraper::ScrapeTimeout)
+    it "should withstand a lot of concurrent threads" do
+      500.times.map {
+        Thread.new {
+          Sunscraper.scrape_html(HTML_FUGA)
+        }
+      }.each(&:join).
+        map(&:value).
+        each { |result|
+        result.should include('It works!')
+      }
     end
   end
+end
+
+unless Sunscraper.os_x?
+  # This part currently crashes on OS X (and will forever).
+  define_tests.("Sunscraper-Embed", :embed)
+end
+
+if !(RUBY_ENGINE =~ /rbx/ || RUBY_ENGINE =~ /jruby/) ||
+   ENV['EXPERIMENTAL'] == 'true'
+  # This part currently crashes Rubinius (as of Mar 09, 2012),
+  # and crashes jruby < 1.7.0, and uses Unix sockets which don't
+  # work even on jruby master (as of Mar 09, 2012).
+  define_tests.("Sunscraper-Standalone", :standalone)
 end