sunscraper 1.1.0.beta3 → 1.2.0.beta1

data/sunscraper.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "sunscraper"
-  s.version     = "1.1.0.beta3"
+  s.version     = "1.2.0.beta1"
   s.authors     = ["Peter Zotov"]
   s.email       = ["whitequark@whitequark.org"]
   s.homepage    = "http://github.com/whitequark/sunscraper"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sunscraper
 version: !ruby/object:Gem::Version
-  version: 1.1.0.beta3
+  version: 1.2.0.beta1
   prerelease: 6
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &81880600 !ruby/object:Gem::Requirement
+  requirement: &74877930 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *81880600
+  version_requirements: *74877930
 - !ruby/object:Gem::Dependency
   name: ffi
-  requirement: &81880200 !ruby/object:Gem::Requirement
+  requirement: &74877500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: 1.0.11
   type: :runtime
   prerelease: false
-  version_requirements: *81880200
+  version_requirements: *74877500
 description: A WebKit-based, JavaScript-capable HTML scraper.
 email:
 - whitequark@whitequark.org
@@ -50,27 +50,29 @@ files:
 - README.md
 - Rakefile
 - ext/.gitignore
+- ext/common/common.pro
+- ext/common/libsunscraper_common.a
+- ext/common/sunscraperproxy.cpp
+- ext/common/sunscraperproxy.h
+- ext/common/sunscraperwebpage.cpp
+- ext/common/sunscraperwebpage.h
+- ext/common/sunscraperworker.cpp
+- ext/common/sunscraperworker.h
 - ext/embed/embed.pro
-- ext/embed/sunscraper.cpp
-- ext/embed/sunscraper.h
 - ext/embed/sunscraperexternal.cpp
+- ext/embed/sunscraperinterface.cpp
+- ext/embed/sunscraperinterface.h
 - ext/embed/sunscraperlibrary.cpp
 - ext/embed/sunscraperlibrary.h
-- ext/embed/sunscraperproxy.cpp
-- ext/embed/sunscraperproxy.h
-- ext/embed/sunscraperwebpage.cpp
-- ext/embed/sunscraperwebpage.h
-- ext/embed/sunscraperworker.cpp
-- ext/embed/sunscraperworker.h
+- ext/embed/sunscraperthread.cpp
+- ext/embed/sunscraperthread.h
 - ext/extconf.rb
 - ext/standalone/standalone.pro
 - ext/standalone/sunscrapermain.cpp
-- ext/standalone/sunscraperproxy.cpp
-- ext/standalone/sunscraperproxy.h
 - ext/standalone/sunscraperrpc.cpp
 - ext/standalone/sunscraperrpc.h
-- ext/standalone/sunscraperworker.cpp
-- ext/standalone/sunscraperworker.h
+- ext/standalone/sunscraperrpcserver.cpp
+- ext/standalone/sunscraperrpcserver.h
 - ext/sunscraper-ext.pro
 - lib/sunscraper.rb
 - lib/sunscraper/library.rb

data/ext/embed/sunscraper.cpp

@@ -1,92 +0,0 @@
-#include <QApplication>
-#include <QThread>
-#include <QTimer>
-#include <QWebPage>
-#include <QWebFrame>
-#include <QMutexLocker>
-#include <QEventLoop>
-#include <QtDebug>
-#include "sunscraper.h"
-#include "sunscraperlibrary.h"
-#include "sunscraperworker.h"
-
-unsigned Sunscraper::m_nextQueryId = 1;
-QMutex Sunscraper::m_staticMutex;
-
-Sunscraper::Sunscraper()
-{
-    QMutexLocker locker(&m_staticMutex);
-
-    m_queryId = m_nextQueryId++;
-
-    SunscraperWorker *worker = SunscraperWorker::instance();
-    if(worker == NULL)
-        qFatal("Attempt to run Sunscraper before thread initialization");
-
-    connect(this, SIGNAL(requestLoadHtml(uint,QString)),
-        worker, SLOT(loadHtml(uint,QString)), Qt::QueuedConnection);
-    connect(this, SIGNAL(requestLoadUrl(uint,QString)),
-        worker, SLOT(loadUrl(uint,QString)), Qt::QueuedConnection);
-    connect(this, SIGNAL(requestFinalize(uint)),
-        worker, SLOT(finalize(uint)), Qt::QueuedConnection);
-    connect(this, SIGNAL(requestTimeout(uint,uint)),
-        worker, SLOT(setTimeout(uint, uint)), Qt::QueuedConnection);
-
-    connect(worker, SIGNAL(finished(uint,QString)),
-        this, SLOT(finished(uint,QString)), Qt::QueuedConnection);
-    connect(worker, SIGNAL(timeout(uint)),
-        this, SLOT(timeout(uint)), Qt::QueuedConnection);
-
-    m_eventLoop = new QEventLoop;
-}
-
-void Sunscraper::loadHtml(QString html)
-{
-    emit requestLoadHtml(m_queryId, html);
-}
-
-void Sunscraper::loadUrl(QString url)
-{
-    emit requestLoadUrl(m_queryId, url);
-}
-
-void Sunscraper::wait(unsigned timeout)
-{
-    emit requestTimeout(m_queryId, timeout);
-
-    m_eventLoop->exec();
-}
-
-void Sunscraper::finished(unsigned eventQueryId, QString html)
-{
-    if(eventQueryId != m_queryId)
-        return;
-
-    m_eventLoop->quit();
-
-    m_html = html.toUtf8();
-
-    emit requestFinalize(m_queryId);
-}
-
-void Sunscraper::timeout(unsigned eventQueryId)
-{
-    if(eventQueryId != m_queryId)
-        return;
-
-    m_eventLoop->quit();
-
-    m_html = "!SUNSCRAPER_TIMEOUT";
-
-    emit requestFinalize(m_queryId);
-}
-
-QByteArray Sunscraper::fetch()
-{
-    return m_html;
-}
-
-const char *Sunscraper::fetchAsCString()
-{
-    return m_html.constData();
-}

data/ext/embed/sunscraper.h

@@ -1,47 +0,0 @@
-#ifndef SUNSCRAPER_H
-#define SUNSCRAPER_H
-
-#include <QObject>
-#include <QString>
-#include <QMutex>
-#include <QByteArray>
-
-class QWebPage;
-class QEventLoop;
-
-class Sunscraper : public QObject
-{
-    Q_OBJECT
-
-public:
-    Sunscraper();
-
-    void loadHtml(QString html);
-    void loadUrl(QString url);
-
-    void wait(unsigned timeout);
-
-    QByteArray fetch();
-    const char *fetchAsCString();
-
-private slots:
-    void finished(unsigned queryId, QString html);
-    void timeout(unsigned queryId);
-
-signals:
-    void requestLoadHtml(unsigned queryId, QString html);
-    void requestLoadUrl(unsigned queryId, QString html);
-    void requestTimeout(unsigned queryId, unsigned timeout);
-    void requestFinalize(unsigned queryId);
-
-private:
-    static unsigned m_nextQueryId;
-    static QMutex m_staticMutex;
-
-    QEventLoop *m_eventLoop;
-
-    unsigned m_queryId;
-    QByteArray m_html;
-};
-
-#endif // SUNSCRAPER_H

data/ext/embed/sunscraperproxy.cpp

@@ -1,14 +0,0 @@
-#include <QWebPage>
-#include <QWebFrame>
-#include <QtDebug>
-#include "sunscraperproxy.h"
-
-SunscraperProxy::SunscraperProxy(QWebPage *parent, unsigned queryId) :
-    QObject(parent), m_webPage(parent), m_queryId(queryId)
-{
-}
-
-void SunscraperProxy::finish()
-{
-    emit finished(m_queryId, m_webPage->mainFrame()->toHtml());
-}

data/ext/embed/sunscraperproxy.h

@@ -1,24 +0,0 @@
-#ifndef SUNSCRAPERPROXY_H
-#define SUNSCRAPERPROXY_H
-
-#include <QObject>
-
-class QWebPage;
-
-class SunscraperProxy : public QObject
-{
-    Q_OBJECT
-public:
-    SunscraperProxy(QWebPage *parent, unsigned queryId);
-
-    Q_INVOKABLE void finish();
-
-signals:
-    void finished(unsigned queryId, QString html);
-
-private:
-    QWebPage *m_webPage;
-    unsigned m_queryId;
-};
-
-#endif // SUNSCRAPERPROXY_H

data/ext/embed/sunscraperworker.cpp

@@ -1,163 +0,0 @@
-#include <QApplication>
-#include <QWebPage>
-#include <QWebFrame>
-#include <QTimer>
-#include <QWebView>
-#include "sunscraperworker.h"
-#include "sunscraperwebpage.h"
-#include "sunscraperproxy.h"
-#include <QtDebug>
-#include <time.h>
-
-#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-pthread_t SunscraperWorker::m_thread;
-#endif
-
-SunscraperWorker *SunscraperWorker::m_instance;
-QSemaphore SunscraperWorker::m_initializationLock;
-
-SunscraperWorker::SunscraperWorker()
-{
-}
-
-SunscraperWorker *SunscraperWorker::instance()
-{
-    m_initializationLock.acquire(1);
-    m_initializationLock.release(1);
-
-    return m_instance;
-}
-
-void SunscraperWorker::invoke()
-{
-#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-    pthread_create(&m_thread, NULL, &SunscraperWorker::thread_routine, NULL);
-#endif
-}
-
-void *SunscraperWorker::thread_routine(void *)
-{
-    /* Better error messages. */
-    int   argc   = 1;
-    char *argv[] = { (char*) "Sunscraper", NULL};
-
-    /* Why (char*)? Because argv can (theoretically) be modified. *
-     * But Qt won't do that with argv[0]. I know, trust me.       */
-
-    //qDebug() << "a";
-    //usleep(1000000);
-    //qDebug() << "b";
-
-    QApplication app(argc, argv);
-
-    if(m_instance != NULL)
-        qFatal("Attempt to invoke SunscraperWorker more than once");
-
-    m_instance = new SunscraperWorker();
-    m_initializationLock.release(1);
-
-    /* The magic value 42 means we want exit from the loop. */
-    while(app.exec() != 42);
-
-    /* Our application exits. */
-
-    return NULL;
-}
-
-void SunscraperWorker::commitSuicide()
-{
-    QApplication::exit(42);
-
-#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-    pthread_join(m_thread, NULL);
-#endif
-}
-
-void SunscraperWorker::loadHtml(unsigned queryId, QString html)
-{
-    QWebPage *webPage = initializeWebPage(queryId);
-    webPage->mainFrame()->setHtml(html);
-}
-
-void SunscraperWorker::loadUrl(unsigned queryId, QString url)
-{
-    QWebPage *webPage = initializeWebPage(queryId);
-    webPage->mainFrame()->load(url);
-}
-
-void SunscraperWorker::setTimeout(unsigned queryId, unsigned timeout)
-{
-    Q_ASSERT(m_timers[queryId] == NULL);
-
-    QTimer *timer = new QTimer(this);
-    timer->setInterval(timeout);
-    timer->setSingleShot(true);
-
-    connect(timer, SIGNAL(timeout()), this, SLOT(routeTimeout()));
-
-    timer->start();
-    m_timers[queryId] = timer;
-}
-
-void SunscraperWorker::finalize(unsigned queryId)
-{
-    Q_ASSERT(m_webPages[queryId] != NULL);
-
-    m_webPages[queryId]->deleteLater();
-    m_webPages.remove(queryId);
-
-    if(m_timers.contains(queryId)) {
-        m_timers[queryId]->deleteLater();
-        m_timers.remove(queryId);
-    }
-}
-
-QWebPage *SunscraperWorker::initializeWebPage(unsigned queryId)
-{
-    Q_ASSERT(m_webPages[queryId] == NULL);
-
-    SunscraperWebPage *webPage = new SunscraperWebPage(this);
-    webPage->settings()->setAttribute(QWebSettings::LocalStorageEnabled, true);
-
-    connect(webPage, SIGNAL(frameCreated(QWebFrame*)), this, SLOT(attachFrame(QWebFrame*)));
-    connect(webPage, SIGNAL(consoleMessage(QString)), this, SLOT(routeMessage(QString)));
-
-    m_webPages[queryId] = webPage;
-
-    return webPage;
-}
-
-void SunscraperWorker::attachFrame(QWebFrame *frame)
-{
-    connect(frame, SIGNAL(javaScriptWindowObjectCleared()),
-        this, SLOT(attachAPI()));
-}
-
-void SunscraperWorker::attachAPI()
-{
-    QWebFrame *origin = static_cast<QWebFrame *>(QObject::sender());
-    QWebPage *page = origin->page();
-
-    unsigned queryId = m_webPages.key(page, 0);
-    Q_ASSERT(queryId != 0);
-
-    SunscraperProxy *proxy = new SunscraperProxy(page, queryId);
-    connect(proxy, SIGNAL(finished(uint,QString)), this, SIGNAL(finished(uint,QString)));
-
-    origin->addToJavaScriptWindowObject("Sunscraper", proxy, QScriptEngine::QtOwnership);
-}
-
-void SunscraperWorker::routeTimeout()
-{
-    QTimer *origin = static_cast<QTimer *>(QObject::sender());
-
-    unsigned queryId = m_timers.key(origin, 0);
-    Q_ASSERT(queryId != 0);
-
-    emit timeout(queryId);
-}
-
-void SunscraperWorker::routeMessage(QString message)
-{
-    qDebug() << "Sunscraper Console:" << message;
-}

data/ext/embed/sunscraperworker.h

@@ -1,58 +0,0 @@
-#ifndef SUNSCRAPERWORKER_H
-#define SUNSCRAPERWORKER_H
-
-#include <QObject>
-#include <QSemaphore>
-#include <QMap>
-#include <QUrl>
-
-class QWebPage;
-class QWebFrame;
-class QTimer;
-
-class SunscraperWorker : public QObject
-{
-    Q_OBJECT
-public:
-    static void invoke();
-    static void commitSuicide();
-    static SunscraperWorker *instance();
-
-signals:
-    void finished(unsigned queryId, QString result);
-    void timeout(unsigned queryId);
-
-public slots:
-    void loadHtml(unsigned queryId, QString html);
-    void loadUrl(unsigned queryId, QString url);
-    void setTimeout(unsigned queryId, unsigned timeout);
-    void finalize(unsigned queryId);
-
-private slots:
-    void attachFrame(QWebFrame *frame);
-    void attachAPI();
-    void routeTimeout();
-    void routeMessage(QString message);
-
-private:
-    static SunscraperWorker *m_instance;
-    static QSemaphore m_initializationLock;
-
-#if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-    static pthread_t m_thread;
-#else
-#error Your platform is unsupported. Implement SunscraperWorker::invoke() and send a pull request.
-#endif
-
-    static void *thread_routine(void *arg);
-
-    QMap<unsigned, QWebPage *> m_webPages;
-    QMap<unsigned, QTimer *> m_timers;
-
-    SunscraperWorker();
-    SunscraperWorker(SunscraperWorker &);
-
-    QWebPage *initializeWebPage(unsigned queryId);
-};
-
-#endif // SUNSCRAPERWORKER_H