sunscraper 1.1.0.beta3 → 1.2.0.beta1

data/ext/embed/sunscraperinterface.h

@@ -0,0 +1,66 @@
+#ifndef SUNSCRAPERINTERFACE_H
+#define SUNSCRAPERINTERFACE_H
+
+#include <QObject>
+#include <QMutex>
+#include <QSemaphore>
+#include <QMap>
+#include <QUrl>
+#include <QByteArray>
+
+class SunscraperWorker;
+
+class SunscraperInterface : public QObject
+{
+    Q_OBJECT
+
+public:
+    static SunscraperInterface *instance();
+
+    unsigned createQuery();
+
+    void loadHtml(unsigned queryId, QString html, QUrl baseUrl);
+    void loadUrl(unsigned queryId, QUrl url);
+
+    bool wait(unsigned queryId, unsigned timeout);
+
+    QByteArray fetch(unsigned queryId);
+
+    void finalize(unsigned queryId);
+
+private slots:
+    void onFinish(unsigned queryId);
+    void onTimeout(unsigned queryId);
+    void onFetchDone(unsigned queryId, QString html);
+
+signals:
+    void requestLoadUrl(unsigned queryId, QUrl url);
+    void requestLoadHtml(unsigned queryId, QString html, QUrl url);
+    void requestTimeout(unsigned queryId, unsigned timeout);
+    void requestFetch(unsigned queryId);
+    void requestFinalize(unsigned queryId);
+
+private:
+    static QMutex m_initializationMutex;
+    static SunscraperInterface *m_instance;
+
+    QMutex m_queryIdMutex;
+    unsigned m_nextQueryId;
+
+    QMutex m_semaphoresMutex;
+    QMap<unsigned, QSemaphore *> m_semaphores;
+
+    QMutex m_resultsMutex;
+    QMap<unsigned, bool> m_results;
+    QMap<unsigned, QByteArray> m_htmlCache;
+
+    SunscraperWorker *m_worker;
+
+    SunscraperInterface();
+
+    void initSemaphore(unsigned queryId);
+    void waitOnSemaphore(unsigned queryId);
+    void signalSemaphore(unsigned queryId);
+};
+
+#endif // SUNSCRAPERINTERFACE_H

data/ext/embed/sunscraperlibrary.cpp

@@ -1,20 +1,10 @@
 #include "sunscraperlibrary.h"
-#include "sunscraperworker.h"
+#include "sunscraperthread.h"
 #include <QtDebug>
 
 SunscraperLibrary SunscraperLibrary::m_instance;
 
 SunscraperLibrary::SunscraperLibrary()
 {
-    SunscraperWorker::invoke();
-}
-
-SunscraperLibrary::~SunscraperLibrary()
-{
-    /* Do nothing. This is on purpose. */
-}
-
-SunscraperLibrary *SunscraperLibrary::instance()
-{
-    return &m_instance;
+    SunscraperThread::invoke();
 }

data/ext/embed/sunscraperlibrary.h

@@ -10,7 +10,6 @@ public:
 private:
     SunscraperLibrary();
     SunscraperLibrary(SunscraperLibrary &);
-    ~SunscraperLibrary();
 
     static SunscraperLibrary m_instance;
 };

data/ext/embed/sunscraperthread.cpp

@@ -0,0 +1,49 @@
+#include <QApplication>
+#include <QtDebug>
+#include "sunscraperthread.h"
+#include "sunscraperworker.h"
+
+#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
+pthread_t SunscraperThread::m_thread;
+#endif
+
+void SunscraperThread::invoke()
+{
+#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
+    pthread_create(&m_thread, NULL, &SunscraperThread::thread_routine, NULL);
+#endif
+}
+
+void *SunscraperThread::thread_routine(void *)
+{
+    /* Better error messages. */
+    int   argc   = 1;
+    char *argv[] = { (char*) "Sunscraper", NULL};
+
+    /* Why (char*)? Because argv can (theoretically) be modified. *
+     * But Qt won't do that with argv[0]. I know, trust me.       */
+
+    QApplication app(argc, argv);
+    app.setApplicationName("Sunscraper-Embed");
+
+    SunscraperWorker::unlock();
+
+    /*
+     * The magic value 42 means we want exit from the loop.
+     * E.g. alerts from within the page may exit the loop with value 0.
+     */
+    while(app.exec() != 42);
+
+    /* Our host application exits. */
+
+    return NULL;
+}
+
+void SunscraperThread::commitSuicide()
+{
+    QApplication::exit(42);
+
+#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
+    pthread_join(m_thread, NULL);
+#endif
+}

data/ext/embed/sunscraperthread.h

@@ -0,0 +1,24 @@
+#ifndef SUNSCRAPERTHREAD_H
+#define SUNSCRAPERTHREAD_H
+
+#include <QThread>
+#include <QSemaphore>
+
+class SunscraperThread : public QThread
+{
+    Q_OBJECT
+public:
+    static void invoke();
+    static void commitSuicide();
+
+private:
+#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
+    static pthread_t m_thread;
+#else
+#error Your platform is unsupported. Implement SunscraperWorker::invoke() and send a pull request.
+#endif
+
+    static void *thread_routine(void *arg);
+};
+
+#endif /* SUNSCRAPERTHREAD_H */

data/ext/extconf.rb

@@ -1,9 +1,11 @@
 # This Makefile will get replaced by qmake.
 
+require 'rbconfig'
+
 if RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
   # Cannot you OS X have a build system like all sane people?
   # Win32 wins again.
-  qmake = %{qmake -spec macx-g++}
+  qmake = %{qmake CONFIG+=debug -spec macx-g++}
 
   File.open("Makefile", "w") do |mf|
     mf.puts <<-ENDM
@@ -16,9 +18,9 @@ install:
   end
 else
   if Gem.win_platform?
-    qmake = %{qmake -spec win32-g++}
+    qmake = %{qmake CONFIG+=debug -spec win32-g++}
   else
-    qmake = %{qmake}
+    qmake = %{qmake CONFIG+=debug}
   end
 
   File.open("Makefile", "w") do |mf|

data/ext/standalone/standalone.pro

@@ -3,11 +3,17 @@ QT += webkit network
 TARGET = sunscraper
 TEMPLATE = app
 
-SOURCES += sunscraperproxy.cpp \
-    sunscraperworker.cpp \
-    sunscraperrpc.cpp \
+SOURCES += sunscraperrpc.cpp \
+    sunscraperrpcserver.cpp \
     sunscrapermain.cpp
 
-HEADERS += sunscraperproxy.h \
-    sunscraperrpc.h \
-    sunscraperworker.h
+HEADERS += sunscraperrpc.h \
+    sunscraperrpcserver.h
+
+INCLUDEPATH += ../common
+
+unix:{
+    LIBS += -L../common -lsunscraper_common
+    POST_TARGETDEPS += ../common/libsunscraper_common.a
+    LDFLAGS += -pthread
+}

data/ext/standalone/sunscrapermain.cpp

@@ -1,13 +1,23 @@
 #include <QApplication>
 #include <QStringList>
 #include "sunscraperworker.h"
-#include "sunscraperrpc.h"
+#include "sunscraperrpcserver.h"
 
 int main(int argc, char **argv)
 {
     QApplication app(argc, argv);
+    app.setApplicationName("Sunscraper-Standalone");
 
-    SunscraperRPC rpc(app.arguments().at(1));
+    SunscraperWorker::unlock();
 
-    return app.exec();
+    SunscraperRPCServer *rpcServer = new SunscraperRPCServer();
+
+    QString socketPath = app.arguments().at(1);
+    if(!rpcServer->listen(socketPath)) {
+        qFatal("Cannot listen on %s", socketPath.toLocal8Bit().constData());
+    }
+
+    app.exec();
+
+    qFatal("finished");
 }

data/ext/standalone/sunscraperrpc.cpp

@@ -4,24 +4,32 @@
 #include <QApplication>
 #include <QtDebug>
 #include <arpa/inet.h>
+#include <sunscraperworker.h>
 #include "sunscraperrpc.h"
-#include "sunscraperworker.h"
 
-SunscraperRPC::SunscraperRPC(QString socketPath) :
-        m_state(StateHeader)
+SunscraperWorker *SunscraperRPC::m_worker;
+unsigned SunscraperRPC::m_nextQueryId;
+
+SunscraperRPC::SunscraperRPC(QLocalSocket *socket) :
+        m_socket(socket), m_state(StateHeader)
 {
-    m_socket = new QLocalSocket(this);
-    m_socket->connectToServer(socketPath);
+    m_nextQueryId += 1;
+    m_queryId = m_nextQueryId;
+
     connect(m_socket, SIGNAL(readyRead()), this, SLOT(onInputReadable()));
     connect(m_socket, SIGNAL(disconnected()), this, SLOT(onInputDisconnected()));
 
-    m_worker = new SunscraperWorker(this);
-    connect(m_worker, SIGNAL(finished(uint,QString)), this, SLOT(onPageRendered(uint,QString)));
+    if(m_worker == NULL)
+        m_worker = new SunscraperWorker();
+
+    connect(m_worker, SIGNAL(finished(uint)), this, SLOT(onFinish(uint)));
+    connect(m_worker, SIGNAL(timedOut(uint)), this, SLOT(onTimeout(uint)));
+    connect(m_worker, SIGNAL(htmlFetched(uint,QString)), this, SLOT(onFetchDone(uint,QString)));
 }
 
 SunscraperRPC::~SunscraperRPC()
 {
-    delete m_worker;
+    delete m_socket;
 }
 
 void SunscraperRPC::onInputReadable()
@@ -32,9 +40,13 @@ void SunscraperRPC::onInputReadable()
     while(moreData) {
         switch(m_state) {
         case StateHeader:
-            if((unsigned) m_buffer.length() >= sizeof(Header)) {
-                memcpy((void*) &m_pendingHeader, m_buffer.constData(), sizeof(Header));
-                m_buffer.remove(0, sizeof(Header));
+            if((unsigned) m_buffer.length() >= sizeof(quint32) * 2) {
+                QDataStream stream(m_buffer);
+                stream >> (quint32&) m_pendingRequest;
+                stream >> (quint32&) m_pendingDataLength;
+
+                m_buffer.remove(0, sizeof(quint32) * 2);
+
                 m_state = StateData;
             } else {
                 moreData = false;
@@ -43,12 +55,12 @@ void SunscraperRPC::onInputReadable()
             break;
 
         case StateData:
-            unsigned length = ntohl(m_pendingHeader.dataLength);
+            if((unsigned) m_buffer.length() >= m_pendingDataLength) {
+                QByteArray data = m_buffer.left(m_pendingDataLength);
+                m_buffer.remove(0, m_pendingDataLength);
+
+                processRequest(m_pendingRequest, data);
 
-            if((unsigned) m_buffer.length() >= length) {
-                QByteArray data = m_buffer.left(length);
-                m_buffer.remove(0, length);
-                processRequest(m_pendingHeader, data);
                 m_state = StateHeader;
             } else {
                 moreData = false;
@@ -61,123 +73,99 @@ void SunscraperRPC::onInputReadable()
 
 void SunscraperRPC::onInputDisconnected()
 {
-    /* Magic value. */
-    QApplication::exit(42);
+    m_worker->finalize(m_queryId);
+
+    emit disconnected();
 }
 
-void SunscraperRPC::processRequest(Header header, QByteArray data)
+void SunscraperRPC::processRequest(unsigned requestType, QByteArray data)
 {
-    unsigned queryId, requestType;
-
-    queryId     = ntohl(header.queryId);
-    requestType = ntohl(header.requestType);
-
     switch(requestType) {
     case RPC_LOAD_HTML: {
-            m_worker->loadHtml(queryId, data);
+            QDataStream stream(data);
+
+            QByteArray html;
+            stream >> html;
+
+            QByteArray baseUrl;
+            stream >> baseUrl;
+
+            m_worker->loadHtml(m_queryId, html, QUrl(baseUrl));
 
             break;
         }
 
     case RPC_LOAD_URL: {
-            m_worker->loadUrl(queryId, data);
+            m_worker->loadUrl(m_queryId, QUrl(data));
 
             break;
         }
 
     case RPC_WAIT: {
-            if(m_results.contains(queryId)) {
-                Header reply;
-                reply.queryId     = htonl(queryId);
-                reply.requestType = htonl(RPC_WAIT);
-
-                sendReply(reply, QByteArray());
-            } else {
-                Q_ASSERT(!m_waitQueue.contains(queryId));
-                Q_ASSERT(!m_timers.contains(queryId));
-
-                m_waitQueue.append(queryId);
+            if(!m_result) {
+                QDataStream stream(data);
 
                 unsigned timeout;
-
-                QDataStream stream(data);
                 stream >> timeout;
 
-                QTimer *timer = new QTimer(this);
-                timer->setInterval(timeout);
-                timer->setSingleShot(true);
-                timer->start();
-                connect(timer, SIGNAL(timeout()), this, SLOT(onTimeout()));
-
-                m_timers[queryId] = timer;
+                m_worker->setTimeout(m_queryId, timeout);
             }
 
             break;
         }
 
     case RPC_FETCH: {
-            Header reply;
-            reply.queryId     = htonl(queryId);
-            reply.requestType = htonl(RPC_FETCH);
-
-            if(m_results.contains(queryId)) {
-                sendReply(reply, m_results[queryId].toLocal8Bit());
-            } else {
-                sendReply(reply, "!SUNSCRAPER_TIMEOUT");
-            }
+            m_worker->fetchHtml(m_queryId);
 
             break;
         }
+    }
+}
 
-    case RPC_DISCARD: {
-            m_results.remove(queryId);
-            m_waitQueue.removeAll(queryId);
+void SunscraperRPC::onFinish(unsigned eventQueryId)
+{
+    if(eventQueryId != m_queryId)
+        return;
 
-            if(m_timers.contains(queryId)) {
-                QTimer *timer = m_timers[queryId];
-                delete timer;
+    QByteArray data;
 
-                m_timers.remove(queryId);
-            }
+    QDataStream stream(&data, QIODevice::WriteOnly);
+    stream << (int) true;
 
-            m_worker->finalize(queryId);
+    sendReply(data);
 
-            break;
-        }
-    }
+    m_result = true;
 }
 
-void SunscraperRPC::onPageRendered(unsigned queryId, QString data)
+void SunscraperRPC::onTimeout(unsigned eventQueryId)
 {
-    m_results[queryId] = data;
+    if(eventQueryId != m_queryId)
+        return;
 
-    if(m_waitQueue.contains(queryId)) {
-        Header reply;
-        reply.queryId     = htonl(queryId);
-        reply.requestType = htonl(RPC_WAIT);
+    QByteArray data;
 
-        sendReply(reply, QByteArray());
-    }
+    QDataStream stream(&data, QIODevice::WriteOnly);
+    stream << (int) false;
+
+    sendReply(data);
+
+    m_result = false;
 }
 
-void SunscraperRPC::onTimeout()
+void SunscraperRPC::onFetchDone(unsigned eventQueryId, QString data)
 {
-    QTimer *timer = static_cast<QTimer*>(QObject::sender());
-    unsigned queryId = m_timers.key(timer);
-
-    Header reply;
-    reply.queryId     = htonl(queryId);
-    reply.requestType = htonl(RPC_WAIT);
+    if(eventQueryId != m_queryId)
+        return;
 
-    sendReply(reply, QByteArray());
+    sendReply(data.toLocal8Bit());
 }
 
-void SunscraperRPC::sendReply(Header header, QByteArray data)
+void SunscraperRPC::sendReply(QByteArray data)
 {
-    header.dataLength = htonl(data.length());
+    QByteArray packet;
 
-    QByteArray serialized((const char*) &header, sizeof(Header));
-    serialized.append(data);
+    QDataStream stream(&packet, QIODevice::WriteOnly);
+    stream << data;
 
-    m_socket->write(serialized);
+    m_socket->write(packet);
 }