sunscraper 1.1.0.beta3 → 1.2.0.beta1

data/ext/common/common.pro

@@ -0,0 +1,13 @@
+TEMPLATE = lib
+TARGET   = sunscraper_common
+
+QT += webkit network
+CONFIG += staticlib
+
+SOURCES += sunscraperproxy.cpp \
+    sunscraperwebpage.cpp \
+    sunscraperworker.cpp
+
+HEADERS += sunscraperproxy.h \
+    sunscraperwebpage.h \
+    sunscraperworker.h

data/ext/common/sunscraperproxy.cpp

@@ -0,0 +1,11 @@
+#include "sunscraperproxy.h"
+
+SunscraperProxy::SunscraperProxy(QObject *parent, unsigned queryId) :
+    QObject(parent), m_queryId(queryId)
+{
+}
+
+void SunscraperProxy::finish()
+{
+    emit finished(m_queryId);
+}

data/ext/{standalone → common}/sunscraperproxy.h

@@ -3,22 +3,22 @@
 
 #include <QObject>
 
-class QWebPage;
-
 class SunscraperProxy : public QObject
 {
     Q_OBJECT
+
 public:
-    SunscraperProxy(QWebPage *parent, unsigned queryId);
+    SunscraperProxy(QObject *parent, unsigned queryId);
 
     Q_INVOKABLE void finish();
 
 signals:
-    void finished(unsigned queryId, QString html);
+    void finished(unsigned queryId);
 
 private:
-    QWebPage *m_webPage;
     unsigned m_queryId;
+
+    SunscraperProxy();
 };
 
 #endif // SUNSCRAPERPROXY_H

data/ext/common/sunscraperworker.cpp

@@ -0,0 +1,124 @@
+#include <QWebPage>
+#include <QWebFrame>
+#include <QTimer>
+#include <QtDebug>
+#include "sunscraperworker.h"
+#include "sunscraperwebpage.h"
+#include "sunscraperproxy.h"
+
+QSemaphore SunscraperWorker::m_initializationLock(0);
+
+SunscraperWorker::SunscraperWorker(QObject *parent) :
+        QObject(parent)
+{
+    m_initializationLock.acquire(1);
+    m_initializationLock.release(1);
+}
+
+void SunscraperWorker::unlock()
+{
+    m_initializationLock.release(1);
+}
+
+void SunscraperWorker::loadHtml(unsigned queryId, QString html, QUrl baseUrl)
+{
+    QWebPage *webPage = initializeWebPage(queryId);
+    webPage->mainFrame()->setHtml(html, baseUrl);
+}
+
+void SunscraperWorker::loadUrl(unsigned queryId, QUrl url)
+{
+    QWebPage *webPage = initializeWebPage(queryId);
+    webPage->mainFrame()->load(url);
+}
+
+void SunscraperWorker::setTimeout(unsigned queryId, unsigned timeout)
+{
+    Q_ASSERT(m_timers[queryId] == NULL);
+
+    QTimer *timer = new QTimer();
+    timer->setInterval(timeout);
+    timer->setSingleShot(true);
+
+    connect(timer, SIGNAL(timeout()), this, SLOT(onTimeout()));
+
+    timer->start();
+    m_timers[queryId] = timer;
+}
+
+void SunscraperWorker::finalize(unsigned queryId)
+{
+    if(m_webPages.contains(queryId)) {
+        m_webPages[queryId]->deleteLater();
+        m_webPages.remove(queryId);
+    }
+
+    if(m_timers.contains(queryId)) {
+        m_timers[queryId]->deleteLater();
+        m_timers.remove(queryId);
+    }
+}
+
+QWebPage *SunscraperWorker::initializeWebPage(unsigned queryId)
+{
+    Q_ASSERT(m_webPages[queryId] == NULL);
+
+    SunscraperWebPage *webPage = new SunscraperWebPage();
+    webPage->settings()->setAttribute(QWebSettings::LocalStorageEnabled, true);
+
+    connect(webPage, SIGNAL(frameCreated(QWebFrame*)), this, SLOT(onFrameCreated(QWebFrame*)));
+    connect(webPage, SIGNAL(consoleMessage(QString)), this, SLOT(onMessage(QString)));
+
+    m_webPages[queryId] = webPage;
+
+    return webPage;
+}
+
+void SunscraperWorker::onFrameCreated(QWebFrame *frame)
+{
+    connect(frame, SIGNAL(javaScriptWindowObjectCleared()),
+        this, SLOT(onJavascriptObjectCleared()));
+}
+
+void SunscraperWorker::onJavascriptObjectCleared()
+{
+    QWebFrame *origin = static_cast<QWebFrame *>(QObject::sender());
+    QWebPage *page = origin->page();
+
+    unsigned queryId = m_webPages.key(page, 0);
+    Q_ASSERT(queryId != 0);
+
+    SunscraperProxy *proxy = new SunscraperProxy(page, queryId);
+    connect(proxy, SIGNAL(finished(uint)), this, SLOT(onFinish(uint)));
+
+    origin->addToJavaScriptWindowObject("Sunscraper", proxy, QScriptEngine::QtOwnership);
+}
+
+void SunscraperWorker::onFinish(unsigned queryId)
+{
+    Q_ASSERT(m_webPages[queryId] != NULL);
+
+    emit finished(queryId);
+}
+
+void SunscraperWorker::onTimeout()
+{
+    QTimer *origin = static_cast<QTimer *>(QObject::sender());
+
+    unsigned queryId = m_timers.key(origin, 0);
+    Q_ASSERT(queryId != 0);
+
+    m_timers[queryId]->deleteLater();
+    m_timers.remove(queryId);
+
+    emit timedOut(queryId);
+}
+
+void SunscraperWorker::onMessage(QString message)
+{
+    qDebug() << "Sunscraper Console:" << message;
+}
+
+void SunscraperWorker::fetchHtml(unsigned queryId) {
+    emit htmlFetched(queryId, m_webPages[queryId]->mainFrame()->toHtml());
+}

data/ext/common/sunscraperworker.h

@@ -0,0 +1,50 @@
+#ifndef SUNSCRAPERWORKER_H
+#define SUNSCRAPERWORKER_H
+
+#include <QObject>
+#include <QSemaphore>
+#include <QMap>
+#include <QUrl>
+
+class QWebPage;
+class QWebFrame;
+class QTimer;
+
+class SunscraperWorker : public QObject
+{
+    Q_OBJECT
+
+public:
+    SunscraperWorker(QObject * parent = 0);
+    
+    static void unlock();
+
+signals:
+    void finished(unsigned queryId);
+    void timedOut(unsigned queryId);
+    void htmlFetched(unsigned queryId, QString data);
+
+public slots:
+    void loadHtml(unsigned queryId, QString html, QUrl baseUrl);
+    void loadUrl(unsigned queryId, QUrl url);
+    void setTimeout(unsigned queryId, unsigned timeout);
+    void fetchHtml(unsigned queryId);
+    void finalize(unsigned queryId);
+
+private slots:
+    void onFrameCreated(QWebFrame *frame);
+    void onJavascriptObjectCleared();
+    void onFinish(unsigned queryId);
+    void onTimeout();
+    void onMessage(QString message);
+
+private:
+    static QSemaphore m_initializationLock;
+    
+    QMap<unsigned, QWebPage *> m_webPages;
+    QMap<unsigned, QTimer *> m_timers;
+
+    QWebPage *initializeWebPage(unsigned queryId);
+};
+
+#endif // SUNSCRAPERWORKER_H

data/ext/embed/embed.pro

@@ -1,19 +1,21 @@
 QT += webkit
 
-TARGET = sunscraper
+TARGET   = sunscraper
 TEMPLATE = lib
 
-SOURCES += sunscraperlibrary.cpp \
-    sunscraperworker.cpp \
-    sunscraperwebpage.cpp \
-    sunscraperexternal.cpp \
-    sunscraper.cpp \
-    sunscraperproxy.cpp
+SOURCES += sunscraperexternal.cpp \
+    sunscraperlibrary.cpp \
+    sunscraperinterface.cpp \
+    sunscraperthread.cpp
 
 HEADERS += sunscraperlibrary.h \
-    sunscraperworker.h \
-    sunscraperwebpage.h \
-    sunscraper.h \
-    sunscraperproxy.h
+    sunscraperinterface.h \
+    sunscraperthread.h
 
-linux:LDFLAGS += -pthread
+INCLUDEPATH += ../common
+
+unix:{
+    LIBS += -L../common -lsunscraper_common
+    POST_TARGETDEPS += ../common/libsunscraper_common.a
+    LDFLAGS += -pthread
+}

data/ext/embed/sunscraperexternal.cpp

@@ -1,39 +1,40 @@
-#include "sunscraper.h"
-#include "sunscraperworker.h"
+#include "sunscraperinterface.h"
+#include "sunscraperthread.h"
 
 extern "C" {
-    Sunscraper *sunscraper_create()
+    unsigned sunscraper_create()
     {
-        return new Sunscraper();
+        return SunscraperInterface::instance()->createQuery();
     }
 
-    void sunscraper_load_html(Sunscraper *sunscraper, const char *html)
+    void sunscraper_load_html(unsigned queryId, const char *html, const char *url)
     {
-        sunscraper->loadHtml(html);
+        SunscraperInterface::instance()->loadHtml(queryId, html, QUrl(url));
     }
 
-    void sunscraper_load_url(Sunscraper *sunscraper, const char *url)
+    void sunscraper_load_url(unsigned queryId, const char *url)
     {
-        sunscraper->loadUrl(url);
+        SunscraperInterface::instance()->loadUrl(queryId, QUrl(url));
     }
 
-    void sunscraper_wait(Sunscraper *sunscraper, unsigned timeout)
+    int sunscraper_wait(unsigned queryId, unsigned timeout)
     {
-        sunscraper->wait(timeout);
+        return SunscraperInterface::instance()->wait(queryId, timeout);
     }
 
-    const char *sunscraper_fetch(Sunscraper *sunscraper)
+    const char *sunscraper_fetch(unsigned queryId)
     {
-        return sunscraper->fetchAsCString();
+        /* VERIFY that the string won't be deleted prematurely */
+        return SunscraperInterface::instance()->fetch(queryId).constData();
     }
 
-    void sunscraper_discard(Sunscraper *sunscraper)
+    void sunscraper_finalize(unsigned queryId)
     {
-        delete sunscraper;
+        SunscraperInterface::instance()->finalize(queryId);
     }
 
-    void sunscraper_finalize()
+    void sunscraper_quit()
     {
-        SunscraperWorker::commitSuicide();
+        SunscraperThread::commitSuicide();
     }
 }

data/ext/embed/sunscraperinterface.cpp

@@ -0,0 +1,206 @@
+#include <QApplication>
+#include <QThread>
+#include <QWebPage>
+#include <QWebFrame>
+#include <QMutexLocker>
+#include <QtDebug>
+#include "sunscraperinterface.h"
+#include "sunscraperlibrary.h"
+#include "sunscraperworker.h"
+
+// #define DEBUG_SUNSCRAPERINTERFACE
+
+QMutex SunscraperInterface::m_initializationMutex;
+SunscraperInterface *SunscraperInterface::m_instance;
+
+SunscraperInterface::SunscraperInterface() :
+        m_nextQueryId(0)
+{
+    m_worker = new SunscraperWorker();
+    m_worker->moveToThread(QApplication::instance()->thread());
+
+    connect(this, SIGNAL(requestLoadHtml(uint,QString,QUrl)),
+            m_worker, SLOT(loadHtml(uint,QString,QUrl)), Qt::QueuedConnection);
+    connect(this, SIGNAL(requestLoadUrl(uint,QUrl)),
+            m_worker, SLOT(loadUrl(uint,QUrl)), Qt::QueuedConnection);
+    connect(this, SIGNAL(requestTimeout(uint,uint)),
+            m_worker, SLOT(setTimeout(uint,uint)), Qt::QueuedConnection);
+    connect(this, SIGNAL(requestFetch(uint)),
+            m_worker, SLOT(fetchHtml(uint)), Qt::QueuedConnection);
+    connect(this, SIGNAL(requestFinalize(uint)),
+            m_worker, SLOT(finalize(uint)), Qt::QueuedConnection);
+
+    connect(m_worker, SIGNAL(finished(uint)),
+            this, SLOT(onFinish(uint)), Qt::DirectConnection);
+    connect(m_worker, SIGNAL(timedOut(uint)),
+            this, SLOT(onTimeout(uint)), Qt::DirectConnection);
+    connect(m_worker, SIGNAL(htmlFetched(uint,QString)),
+            this, SLOT(onFetchDone(uint,QString)), Qt::DirectConnection);
+}
+
+SunscraperInterface *SunscraperInterface::instance()
+{
+    QMutexLocker locker(&m_initializationMutex);
+
+    if(m_instance == NULL)
+        m_instance = new SunscraperInterface();
+
+    return m_instance;
+}
+
+void SunscraperInterface::initSemaphore(unsigned queryId)
+{
+    QMutexLocker locker(&m_semaphoresMutex);
+
+    Q_ASSERT(m_semaphores[queryId] == NULL);
+
+    QSemaphore *semaphore = new QSemaphore(0);
+    m_semaphores[queryId] = semaphore;
+}
+
+void SunscraperInterface::waitOnSemaphore(unsigned queryId)
+{
+    m_semaphoresMutex.lock();
+
+    Q_ASSERT(m_semaphores[queryId] != NULL);
+
+    QSemaphore *semaphore = m_semaphores[queryId];
+
+    m_semaphoresMutex.unlock();
+
+    semaphore->acquire(1);
+
+    m_semaphoresMutex.lock();
+
+    delete semaphore;
+    m_semaphores.remove(queryId);
+
+    m_semaphoresMutex.unlock();
+}
+
+void SunscraperInterface::signalSemaphore(unsigned queryId)
+{
+    QMutexLocker locker(&m_semaphoresMutex);
+
+    Q_ASSERT(m_semaphores[queryId] != NULL);
+
+    m_semaphores[queryId]->release(1);
+}
+
+unsigned SunscraperInterface::createQuery()
+{
+    QMutexLocker locker(&m_queryIdMutex);
+
+    m_nextQueryId += 1;
+
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "createQuery" << m_nextQueryId;
+#endif
+
+    return m_nextQueryId;
+}
+
+void SunscraperInterface::loadHtml(unsigned queryId, QString html, QUrl baseUrl)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "loadHtml" << queryId << html << baseUrl;
+#endif
+
+    emit requestLoadHtml(queryId, html, baseUrl);
+}
+
+void SunscraperInterface::loadUrl(unsigned queryId, QUrl url)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "loadUrl" << queryId << url;
+#endif
+
+    emit requestLoadUrl(queryId, url);
+}
+
+bool SunscraperInterface::wait(unsigned queryId, unsigned timeout)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "wait" << queryId << timeout;
+#endif
+
+    initSemaphore(queryId);
+    emit requestTimeout(queryId, timeout);
+    waitOnSemaphore(queryId);
+
+    /* There was either a finish or timeout */
+
+    {
+        QMutexLocker locker(&m_resultsMutex);
+
+        bool success = m_results[queryId];
+        m_results.remove(queryId);
+
+        return success;
+    }
+}
+
+void SunscraperInterface::onFinish(unsigned queryId)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "onFinish" << queryId;
+#endif
+
+    QMutexLocker locker(&m_resultsMutex);
+    m_results[queryId] = true;
+
+    signalSemaphore(queryId);
+}
+
+void SunscraperInterface::onTimeout(unsigned queryId)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "onTimeout" << queryId;
+#endif
+
+    QMutexLocker locker(&m_resultsMutex);
+    m_results[queryId] = false;
+
+    signalSemaphore(queryId);
+}
+
+void SunscraperInterface::onFetchDone(unsigned queryId, QString html)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "onFetchDone" << queryId;
+#endif
+
+    QMutexLocker locker(&m_resultsMutex);
+    m_htmlCache[queryId] = html.toLocal8Bit();
+
+    signalSemaphore(queryId);
+}
+
+QByteArray SunscraperInterface::fetch(unsigned queryId)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "fetch" << queryId;
+#endif
+
+    initSemaphore(queryId);
+    emit requestFetch(queryId);
+    waitOnSemaphore(queryId);
+
+    {
+        QMutexLocker locker(&m_resultsMutex);
+        return m_htmlCache[queryId];
+    }
+}
+
+void SunscraperInterface::finalize(unsigned queryId)
+{
+#ifdef DEBUG_SUNSCRAPERINTERFACE
+    qDebug() << "finalize" << queryId;
+#endif
+
+    emit requestFinalize(queryId);
+
+    QMutexLocker locker(&m_resultsMutex);
+    m_results.remove(queryId);
+    m_htmlCache.remove(queryId);
+}