sunscraper 1.0.0 → 1.1.0.beta1

data/ext/standalone/sunscraperrpc.cpp

@@ -0,0 +1,183 @@
+#include <QLocalSocket>
+#include <QTimer>
+#include <QDataStream>
+#include <QApplication>
+#include <QtDebug>
+#include <arpa/inet.h>
+#include "sunscraperrpc.h"
+#include "sunscraperworker.h"
+
+SunscraperRPC::SunscraperRPC(QString socketPath) :
+        m_state(StateHeader)
+{
+    m_socket = new QLocalSocket(this);
+    m_socket->connectToServer(socketPath);
+    connect(m_socket, SIGNAL(readyRead()), this, SLOT(onInputReadable()));
+    connect(m_socket, SIGNAL(disconnected()), this, SLOT(onInputDisconnected()));
+
+    m_worker = new SunscraperWorker(this);
+    connect(m_worker, SIGNAL(finished(uint,QString)), this, SLOT(onPageRendered(uint,QString)));
+}
+
+SunscraperRPC::~SunscraperRPC()
+{
+    delete m_worker;
+}
+
+void SunscraperRPC::onInputReadable()
+{
+    m_buffer += m_socket->readAll();
+
+    bool moreData = true;
+    while(moreData) {
+        switch(m_state) {
+        case StateHeader:
+            if((unsigned) m_buffer.length() >= sizeof(Header)) {
+                memcpy((void*) &m_pendingHeader, m_buffer.constData(), sizeof(Header));
+                m_buffer.remove(0, sizeof(Header));
+                m_state = StateData;
+            } else {
+                moreData = false;
+            }
+
+            break;
+
+        case StateData:
+            unsigned length = ntohl(m_pendingHeader.dataLength);
+
+            if((unsigned) m_buffer.length() >= length) {
+                QByteArray data = m_buffer.left(length);
+                m_buffer.remove(0, length);
+                processRequest(m_pendingHeader, data);
+                m_state = StateHeader;
+            } else {
+                moreData = false;
+            }
+
+            break;
+        }
+    }
+}
+
+void SunscraperRPC::onInputDisconnected()
+{
+    /* Magic value. */
+    QApplication::exit(42);
+}
+
+void SunscraperRPC::processRequest(Header header, QByteArray data)
+{
+    unsigned queryId, requestType;
+
+    queryId     = ntohl(header.queryId);
+    requestType = ntohl(header.requestType);
+
+    switch(requestType) {
+    case RPC_LOAD_HTML: {
+            m_worker->loadHtml(queryId, data);
+
+            break;
+        }
+
+    case RPC_LOAD_URL: {
+            m_worker->loadUrl(queryId, data);
+
+            break;
+        }
+
+    case RPC_WAIT: {
+            if(m_results.contains(queryId)) {
+                Header reply;
+                reply.queryId     = htonl(queryId);
+                reply.requestType = htonl(RPC_WAIT);
+
+                sendReply(reply, QByteArray());
+            } else {
+                Q_ASSERT(!m_waitQueue.contains(queryId));
+                Q_ASSERT(!m_timers.contains(queryId));
+
+                m_waitQueue.append(queryId);
+
+                unsigned timeout;
+
+                QDataStream stream(data);
+                stream >> timeout;
+
+                QTimer *timer = new QTimer(this);
+                timer->setInterval(timeout);
+                timer->setSingleShot(true);
+                timer->start();
+                connect(timer, SIGNAL(timeout()), this, SLOT(onTimeout()));
+
+                m_timers[queryId] = timer;
+            }
+
+            break;
+        }
+
+    case RPC_FETCH: {
+            Header reply;
+            reply.queryId     = htonl(queryId);
+            reply.requestType = htonl(RPC_FETCH);
+
+            if(m_results.contains(queryId)) {
+                sendReply(reply, m_results[queryId].toLocal8Bit());
+            } else {
+                sendReply(reply, "!SUNSCRAPER_TIMEOUT");
+            }
+
+            break;
+        }
+
+    case RPC_DISCARD: {
+            m_results.remove(queryId);
+            m_waitQueue.removeAll(queryId);
+
+            if(m_timers.contains(queryId)) {
+                QTimer *timer = m_timers[queryId];
+                delete timer;
+
+                m_timers.remove(queryId);
+            }
+
+            m_worker->finalize(queryId);
+
+            break;
+        }
+    }
+}
+
+void SunscraperRPC::onPageRendered(unsigned queryId, QString data)
+{
+    m_results[queryId] = data;
+
+    if(m_waitQueue.contains(queryId)) {
+        Header reply;
+        reply.queryId     = htonl(queryId);
+        reply.requestType = htonl(RPC_WAIT);
+
+        sendReply(reply, QByteArray());
+    }
+}
+
+void SunscraperRPC::onTimeout()
+{
+    QTimer *timer = static_cast<QTimer*>(QObject::sender());
+    unsigned queryId = m_timers.key(timer);
+
+    Header reply;
+    reply.queryId     = htonl(queryId);
+    reply.requestType = htonl(RPC_WAIT);
+
+    sendReply(reply, QByteArray());
+}
+
+void SunscraperRPC::sendReply(Header header, QByteArray data)
+{
+    header.dataLength = htonl(data.length());
+
+    QByteArray serialized((const char*) &header, sizeof(Header));
+    serialized.append(data);
+
+    m_socket->write(serialized);
+}

data/ext/standalone/sunscraperrpc.h

@@ -0,0 +1,64 @@
+#ifndef SUNSCRAPERRPC_H
+#define SUNSCRAPERRPC_H
+
+#include <QObject>
+#include <QVector>
+#include <QMap>
+
+class SunscraperWorker;
+class QLocalSocket;
+class QTimer;
+
+class SunscraperRPC : public QObject
+{
+    Q_OBJECT
+
+    enum State {
+        StateHeader = 0,
+        StateData,
+    };
+
+    struct Header {
+        quint32 queryId;
+        quint32 requestType;
+        quint32 dataLength;
+    };
+
+    enum Request {
+        RPC_LOAD_HTML = 1,
+        RPC_LOAD_URL  = 2,
+        RPC_WAIT      = 3,
+        RPC_FETCH     = 4,
+        RPC_DISCARD   = 5,
+    };
+
+public:
+    SunscraperRPC(QString socketPath);
+    ~SunscraperRPC();
+
+private slots:
+    void onInputReadable();
+    void onInputDisconnected();
+    void onPageRendered(unsigned queryId, QString data);
+    void onTimeout();
+
+private:
+    QLocalSocket *m_socket;
+
+    State  m_state;
+    Header m_pendingHeader;
+    QByteArray m_buffer;
+
+    SunscraperWorker *m_worker;
+
+    QList<unsigned> m_waitQueue;
+    QMap<unsigned, QTimer*> m_timers;
+    QMap<unsigned, QString> m_results;
+
+    SunscraperRPC();
+
+    void processRequest(Header header, QByteArray data);
+    void sendReply(Header header, QByteArray data);
+};
+
+#endif // SUNSCRAPERRPC_H

data/ext/{sunscraperthread.cpp → standalone/sunscraperworker.cpp}

@@ -1,37 +1,28 @@
 #include <QApplication>
 #include <QWebPage>
 #include <QWebFrame>
-#include "sunscraperthread.h"
+#include "sunscraperworker.h"
 #include "sunscraperproxy.h"
+#include <QtDebug>
 
-SunscraperThread::SunscraperThread()
+SunscraperWorker::SunscraperWorker(QObject *parent) :
+        QObject(parent)
 {
 }
 
-void SunscraperThread::run()
-{
-    static int argc;
-    static char **argv = {NULL};
-
-    QApplication app(argc, argv);
-    app.exec();
-
-    qFatal("Sunscraper apartment thread event loop should never end");
-}
-
-void SunscraperThread::loadHtml(unsigned queryId, QString html)
+void SunscraperWorker::loadHtml(unsigned queryId, QString html)
 {
     QWebPage *webPage = initializeWebPage(queryId);
     webPage->mainFrame()->setHtml(html);
 }
 
-void SunscraperThread::loadUrl(unsigned queryId, QString url)
+void SunscraperWorker::loadUrl(unsigned queryId, QString url)
 {
     QWebPage *webPage = initializeWebPage(queryId);
     webPage->mainFrame()->load(url);
 }
 
-void SunscraperThread::finalize(unsigned queryId)
+void SunscraperWorker::finalize(unsigned queryId)
 {
     Q_ASSERT(_webPages[queryId] != NULL);
 
@@ -39,7 +30,7 @@ void SunscraperThread::finalize(unsigned queryId)
     _webPages.remove(queryId);
 }
 
-QWebPage *SunscraperThread::initializeWebPage(unsigned queryId)
+QWebPage *SunscraperWorker::initializeWebPage(unsigned queryId)
 {
     Q_ASSERT(_webPages[queryId] == NULL);
 
@@ -52,7 +43,7 @@ QWebPage *SunscraperThread::initializeWebPage(unsigned queryId)
     return webPage;
 }
 
-void SunscraperThread::attachAPI()
+void SunscraperWorker::attachAPI()
 {
     QWebFrame *origin = static_cast<QWebFrame *>(QObject::sender());
     QWebPage *page = origin->page();

data/ext/{sunscraperthread.h → standalone/sunscraperworker.h}

@@ -1,18 +1,18 @@
-#ifndef SUNSCRAPERTHREAD_H
-#define SUNSCRAPERTHREAD_H
+#ifndef SUNSCRAPERWORKER_H
+#define SUNSCRAPERWORKER_H
 
-#include <QThread>
+#include <QObject>
+#include <QMutex>
 #include <QMap>
 
 class QWebPage;
 
-class SunscraperThread : public QThread
+class SunscraperWorker : public QObject
 {
     Q_OBJECT
-public:
-    SunscraperThread();
 
-    void run();
+public:
+    SunscraperWorker(QObject *parent = 0);
 
 signals:
     void finished(unsigned queryId, QString result);
@@ -31,4 +31,4 @@ private:
     QWebPage *initializeWebPage(unsigned queryId);
 };
 
-#endif // SUNSCRAPERTHREAD_H
+#endif // SUNSCRAPERWORKER_H

data/lib/sunscraper/library.rb

@@ -1,37 +1,41 @@
-if !defined?(RUBY_ENGINE) && RUBY_VERSION =~ /^1.8/
-  raise RuntimeError, "Sunscraper does not work on Ruby MRI 1.8.x."
+if RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
+  raise RuntimeError, "Sunscraper/embed does not work on OS X. Use Sunscraper/standalone."
 end
 
 require 'ffi'
+require 'rbconfig'
 
 # @private
-module Sunscraper::Library
-  extend FFI::Library
-
-  # RbConfig sniffing does not work on JRuby.
-  if Gem.win_platform?
-    extension = 'dll'
-  elsif RUBY_PLATFORM =~ /darwin/i
-    extension = 'dylib'
-  else
-    extension = 'so'
-  end
+module Sunscraper
+  module Library
+    extend FFI::Library
+
+    if Gem.win_platform?
+      extension = 'dll'
+    else
+      extension = 'so'
+    end
+
+    ffi_lib File.join(Gem.loaded_specs['sunscraper'].full_gem_path,
+                      'ext', 'embed', "libsunscraper.#{extension}")
+
+    attach_function 'create',    :sunscraper_create,    [],                  :pointer
+    attach_function 'load_html', :sunscraper_load_html, [:pointer, :string], :void
+    attach_function 'load_url',  :sunscraper_load_url,  [:pointer, :string], :void
+    attach_function 'fetch',     :sunscraper_fetch,     [:pointer],          :string
+    attach_function 'discard',   :sunscraper_discard,   [:pointer],          :void
+
+    if RUBY_ENGINE == 'ruby'
+      # MRI uses ffi gem and has GVL. Hence, it needs a rb_thread_blocking_region call.
+      attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void,    :blocking => true
+    else
+      # Rubinius does not have GVL neither it has options in attach_function.
+      # Same for JRuby.
+      attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void
+    end
+
+    attach_function 'finalize',  :sunscraper_finalize,  [],                  :void
 
-  ffi_lib File.join(Gem.loaded_specs['sunscraper'].full_gem_path,
-                    'ext', "libsunscraper.#{extension}")
-
-  attach_function 'create',    :sunscraper_create,    [],                  :pointer
-  attach_function 'load_html', :sunscraper_load_html, [:pointer, :string], :void
-  attach_function 'load_url',  :sunscraper_load_url,  [:pointer, :string], :void
-  attach_function 'fetch',     :sunscraper_fetch,     [:pointer],          :string
-  attach_function 'discard',   :sunscraper_discard,   [:pointer],          :void
-
-  if RUBY_ENGINE == 'ruby'
-    # MRI uses ffi gem and has GVL. Hence, it needs a rb_thread_blocking_region call.
-    attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void,    :blocking => true
-  else
-    # Rubinius does not have GVL neither it has options in attach_function.
-    # Same for JRuby.
-    attach_function 'wait',    :sunscraper_wait,      [:pointer, :uint],   :void
+    at_exit { finalize }
   end
 end

data/lib/sunscraper/standalone.rb

@@ -0,0 +1,168 @@
+require 'socket'
+
+# @private
+module Sunscraper
+  module Standalone
+    @last_query_id = 0
+
+    @rpc_mutex    = Mutex.new
+    @rpc_waiters  = {}
+    @rpc_results  = {}
+    @rpc_thread   = nil
+
+    RPC_LOAD_HTML = 1
+    RPC_LOAD_URL  = 2
+    RPC_WAIT      = 3
+    RPC_FETCH     = 4
+    RPC_DISCARD   = 5
+
+    class << self
+      attr_reader :rpc_mutex, :rpc_waiters, :rpc_results
+
+      def create
+        @rpc_mutex.synchronize do
+          @last_query_id += 1
+          @last_query_id
+        end
+      end
+
+      def load_html(query_id, html)
+        perform_rpc query_id,
+          request:     RPC_LOAD_HTML,
+          data:        html
+      end
+
+      def load_url(query_id, url)
+        perform_rpc query_id,
+          request:     RPC_LOAD_URL,
+          data:        url
+      end
+
+      def wait(query_id, timeout)
+        perform_rpc query_id,
+          request:     RPC_WAIT,
+          data:        [timeout].pack("N"),
+          want_result: true
+      end
+
+      def fetch(query_id)
+        perform_rpc query_id,
+          request:     RPC_FETCH,
+          want_result: true
+      end
+
+      def discard(query_id)
+        perform_rpc query_id,
+          request:     RPC_DISCARD
+      end
+
+      private
+
+      def perform_rpc(query_id, options={})
+        data  = options[:data] || ""
+        block = options[:want_result]
+
+        @rpc_mutex.synchronize do
+          if @rpc_thread.nil?
+            @rpc_thread = Standalone::Thread.new(::Thread.current)
+
+            # Some fucko decided not to put any semaphores in Ruby,
+            # _and_ restrict Mutexes to be unlocked only from the thread
+            # which has locked them.
+            #
+            # Please, kill yourself if you're reading this.
+            ::Thread.stop
+          end
+
+          @rpc_thread.perform(query_id, options[:request], data)
+
+          if block
+            @rpc_waiters[query_id] = Thread.current
+          end
+        end
+
+        if block
+          Thread.stop
+          @rpc_results[query_id]
+        end
+      ensure
+        if block
+          @rpc_waiters.delete query_id
+          @rpc_results.delete query_id
+        end
+      end
+    end
+
+    class Thread < ::Thread
+      def initialize(creator)
+        @creator = creator
+
+        super do
+          @parent = Sunscraper::Standalone
+          work
+        end
+      end
+
+      def perform(query_id, request, data)
+        @socket.write([query_id, request, data.length, data].pack("NNNa*"))
+      end
+
+      private
+
+      def work
+        if ::Sunscraper.os_x?
+          # Fuck you, OS X.
+          suffix = ".app/Contents/MacOS/sunscraper"
+        else
+          suffix = RbConfig::CONFIG["EXEEXT"]
+        end
+
+        executable = File.join(Gem.loaded_specs['sunscraper'].full_gem_path,
+                          'ext', 'standalone', "sunscraper#{suffix}")
+
+        server_path = "/tmp/sunscraper.#{Process.pid}.sock"
+        server = UNIXServer.new(server_path)
+
+        if Kernel.respond_to? :spawn
+          pid = Kernel.spawn "#{executable} #{server_path}"
+        else
+          # rbx does not have Kernel.spawn (yet). Sigh...
+          pid = fork { exec executable, server_path }
+        end
+
+        Process.detach pid
+
+        @socket = server.accept
+
+        server.close
+        FileUtils.rm server_path
+
+        # See above.
+        @creator.wakeup
+
+        loop do
+          header = @socket.read(4 * 3)
+          query_id, request, data_length = header.unpack("NNN")
+          data   = @socket.read(data_length) if data_length > 0
+
+          @parent.rpc_mutex.synchronize do
+            if !@parent.rpc_waiters.include?(query_id)
+              $stderr.puts "Sunscraper/standalone: no waiter for #{query_id}"
+            else
+              @parent.rpc_results[query_id] = data
+              @parent.rpc_waiters[query_id].wakeup
+            end
+          end
+        end
+      rescue Exception => e
+        $stderr.puts "Sunscraper error: #{e.class}: #{e.message}"
+        e.backtrace.each do |line|
+          $stderr.puts "  #{line}"
+        end
+      ensure
+        @socket.close
+        Process.kill pid
+      end
+    end
+  end
+end

data/lib/sunscraper.rb

@@ -1,4 +1,6 @@
-require 'sunscraper/library'
+if !defined?(RUBY_ENGINE) && RUBY_VERSION =~ /^1.8/
+  raise RuntimeError, "Sunscraper does not work on Ruby MRI 1.8.x."
+end
 
 # Sunscraper loads an HTML page in a headless browser and waits for `Sunscraper.finish()`
 # method to be called. It blocks the calling thread, but is threadsafe, does
@@ -8,13 +10,26 @@ module Sunscraper
   class ScrapeTimeout < StandardError; end
 
   class << self
+    def os_x?
+      RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
+    end
+
+    attr_reader :worker
+    def worker=(worker_type)
+      if [:embed, :standalone].include?(worker_type)
+        @worker = worker_type
+      else
+        raise RuntimeError, "Invalid Sunscraper worker type: #{worker_type.inspect}"
+      end
+    end
+
     # Scrape an inline HTML. The content is loaded without a particular base URL.
     # If your application depends on base URL being available, use {scrape_url}.
     #
     # @param [Integer] timeout timeout in milliseconds
     def scrape_html(html, timeout=5000)
-      scrape(timeout) do |context|
-        Library.load_html context, html
+      scrape(timeout) do |worker, context|
+        worker.load_html context, html
       end
     end
 
@@ -22,21 +37,21 @@ module Sunscraper
     #
     # @param [Integer] timeout timeout in milliseconds
     def scrape_url(url, timeout=5000)
-      scrape(timeout) do |context|
-        Library.load_url context, url
+      scrape(timeout) do |worker, context|
+        worker.load_url context, url
       end
     end
 
     private
 
     def scrape(timeout)
-      context = Library.create
-
-      yield context
+      worker = load_worker
 
-      Library.wait(context, timeout)
+      context = worker.create
+      yield worker, context
+      worker.wait(context, timeout)
 
-      data = Library.fetch(context)
+      data = worker.fetch(context)
 
       if data == "!SUNSCRAPER_TIMEOUT"
         raise ScrapeTimeout, "Sunscraper has timed out waiting for the callback"
@@ -44,7 +59,29 @@ module Sunscraper
         data
       end
     ensure
-      Library.discard(context) if context
+      worker.discard(context) if context
+    end
+
+    def load_worker
+      case @worker
+      when :standalone
+        require 'sunscraper/standalone'
+
+        Sunscraper::Standalone
+
+      when :embed
+        require 'sunscraper/library'
+
+        Sunscraper::Library
+      end
     end
   end
 end
+
+if Sunscraper.os_x?
+  # OS X is braindead
+  Sunscraper.worker = :standalone
+else
+  # ... even Win32 is better.
+  Sunscraper.worker = :embed
+end