sunscraper 1.1.0.beta2 → 1.1.0.beta3

data/ext/embed/{sunscraper.pro → embed.pro}

@@ -4,13 +4,15 @@ TARGET = sunscraper
 TEMPLATE = lib
 
 SOURCES += sunscraperlibrary.cpp \
-    sunscraperthread.cpp \
+    sunscraperworker.cpp \
+    sunscraperwebpage.cpp \
     sunscraperexternal.cpp \
     sunscraper.cpp \
     sunscraperproxy.cpp
 
 HEADERS += sunscraperlibrary.h \
-    sunscraperthread.h \
+    sunscraperworker.h \
+    sunscraperwebpage.h \
     sunscraper.h \
     sunscraperproxy.h
 

data/ext/embed/sunscraper.cpp

@@ -8,7 +8,7 @@
 #include <QtDebug>
 #include "sunscraper.h"
 #include "sunscraperlibrary.h"
-#include "sunscraperthread.h"
+#include "sunscraperworker.h"
 
 unsigned Sunscraper::m_nextQueryId = 1;
 QMutex Sunscraper::m_staticMutex;
@@ -19,7 +19,7 @@ Sunscraper::Sunscraper()
 
     m_queryId = m_nextQueryId++;
 
-    SunscraperThread *worker = SunscraperThread::instance();
+    SunscraperWorker *worker = SunscraperWorker::instance();
     if(worker == NULL)
         qFatal("Attempt to run Sunscraper before thread initialization");
 

data/ext/embed/sunscraperexternal.cpp

@@ -1,5 +1,5 @@
 #include "sunscraper.h"
-#include "sunscraperthread.h"
+#include "sunscraperworker.h"
 
 extern "C" {
     Sunscraper *sunscraper_create()
@@ -34,6 +34,6 @@ extern "C" {
 
     void sunscraper_finalize()
     {
-        SunscraperThread::commitSuicide();
+        SunscraperWorker::commitSuicide();
     }
 }

data/ext/embed/sunscraperlibrary.cpp

@@ -1,12 +1,12 @@
 #include "sunscraperlibrary.h"
-#include "sunscraperthread.h"
+#include "sunscraperworker.h"
 #include <QtDebug>
 
 SunscraperLibrary SunscraperLibrary::m_instance;
 
 SunscraperLibrary::SunscraperLibrary()
 {
-    SunscraperThread::invoke();
+    SunscraperWorker::invoke();
 }
 
 SunscraperLibrary::~SunscraperLibrary()

data/ext/embed/sunscraperlibrary.h

@@ -1,7 +1,7 @@
 #ifndef SUNSCRAPERLIBRARY_H
 #define SUNSCRAPERLIBRARY_H
 
-class SunscraperThread;
+class SunscraperWorker;
 
 class SunscraperLibrary {
 public:

data/ext/embed/sunscraperwebpage.cpp

@@ -0,0 +1,12 @@
+#include "sunscraperwebpage.h"
+
+SunscraperWebPage::SunscraperWebPage(QObject *parent) :
+        QWebPage(parent)
+{
+}
+
+void SunscraperWebPage::javaScriptConsoleMessage(const QString &message,
+                                                 int lineNumber, const QString &sourceID)
+{
+    emit consoleMessage(QString("%1:%2> %3").arg(sourceID).arg(lineNumber).arg(message));
+}

data/ext/embed/sunscraperwebpage.h

@@ -0,0 +1,20 @@
+#ifndef SUNSCRAPERWEBPAGE_H
+#define SUNSCRAPERWEBPAGE_H
+
+#include <QWebPage>
+
+class SunscraperWebPage : public QWebPage
+{
+    Q_OBJECT
+
+public:
+    SunscraperWebPage(QObject *parent = 0);
+
+signals:
+    void consoleMessage(QString message);
+
+protected:
+    virtual void javaScriptConsoleMessage(const QString & message, int lineNumber, const QString & sourceID);
+};
+
+#endif /* SUNSCRAPERWEBPAGE_H */

data/ext/embed/{sunscraperthread.cpp → sunscraperworker.cpp}

@@ -2,23 +2,25 @@
 #include <QWebPage>
 #include <QWebFrame>
 #include <QTimer>
-#include "sunscraperthread.h"
+#include <QWebView>
+#include "sunscraperworker.h"
+#include "sunscraperwebpage.h"
 #include "sunscraperproxy.h"
 #include <QtDebug>
 #include <time.h>
 
 #if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-pthread_t SunscraperThread::m_thread;
+pthread_t SunscraperWorker::m_thread;
 #endif
 
-SunscraperThread *SunscraperThread::m_instance;
-QSemaphore SunscraperThread::m_initializationLock;
+SunscraperWorker *SunscraperWorker::m_instance;
+QSemaphore SunscraperWorker::m_initializationLock;
 
-SunscraperThread::SunscraperThread()
+SunscraperWorker::SunscraperWorker()
 {
 }
 
-SunscraperThread *SunscraperThread::instance()
+SunscraperWorker *SunscraperWorker::instance()
 {
     m_initializationLock.acquire(1);
     m_initializationLock.release(1);
@@ -26,14 +28,14 @@ SunscraperThread *SunscraperThread::instance()
     return m_instance;
 }
 
-void SunscraperThread::invoke()
+void SunscraperWorker::invoke()
 {
 #if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
-    pthread_create(&m_thread, NULL, &SunscraperThread::thread_routine, NULL);
+    pthread_create(&m_thread, NULL, &SunscraperWorker::thread_routine, NULL);
 #endif
 }
 
-void *SunscraperThread::thread_routine(void *)
+void *SunscraperWorker::thread_routine(void *)
 {
     /* Better error messages. */
     int   argc   = 1;
@@ -49,9 +51,9 @@ void *SunscraperThread::thread_routine(void *)
     QApplication app(argc, argv);
 
     if(m_instance != NULL)
-        qFatal("Attempt to invoke SunscraperThread more than once");
+        qFatal("Attempt to invoke SunscraperWorker more than once");
 
-    m_instance = new SunscraperThread();
+    m_instance = new SunscraperWorker();
     m_initializationLock.release(1);
 
     /* The magic value 42 means we want exit from the loop. */
@@ -62,7 +64,7 @@ void *SunscraperThread::thread_routine(void *)
     return NULL;
 }
 
-void SunscraperThread::commitSuicide()
+void SunscraperWorker::commitSuicide()
 {
     QApplication::exit(42);
 
@@ -71,19 +73,19 @@ void SunscraperThread::commitSuicide()
 #endif
 }
 
-void SunscraperThread::loadHtml(unsigned queryId, QString html)
+void SunscraperWorker::loadHtml(unsigned queryId, QString html)
 {
     QWebPage *webPage = initializeWebPage(queryId);
     webPage->mainFrame()->setHtml(html);
 }
 
-void SunscraperThread::loadUrl(unsigned queryId, QString url)
+void SunscraperWorker::loadUrl(unsigned queryId, QString url)
 {
     QWebPage *webPage = initializeWebPage(queryId);
     webPage->mainFrame()->load(url);
 }
 
-void SunscraperThread::setTimeout(unsigned queryId, unsigned timeout)
+void SunscraperWorker::setTimeout(unsigned queryId, unsigned timeout)
 {
     Q_ASSERT(m_timers[queryId] == NULL);
 
@@ -97,7 +99,7 @@ void SunscraperThread::setTimeout(unsigned queryId, unsigned timeout)
     m_timers[queryId] = timer;
 }
 
-void SunscraperThread::finalize(unsigned queryId)
+void SunscraperWorker::finalize(unsigned queryId)
 {
     Q_ASSERT(m_webPages[queryId] != NULL);
 
@@ -110,20 +112,28 @@ void SunscraperThread::finalize(unsigned queryId)
     }
 }
 
-QWebPage *SunscraperThread::initializeWebPage(unsigned queryId)
+QWebPage *SunscraperWorker::initializeWebPage(unsigned queryId)
 {
     Q_ASSERT(m_webPages[queryId] == NULL);
 
-    QWebPage *webPage = new QWebPage(this);
-    connect(webPage->mainFrame(), SIGNAL(javaScriptWindowObjectCleared()),
-        this, SLOT(attachAPI()));
+    SunscraperWebPage *webPage = new SunscraperWebPage(this);
+    webPage->settings()->setAttribute(QWebSettings::LocalStorageEnabled, true);
+
+    connect(webPage, SIGNAL(frameCreated(QWebFrame*)), this, SLOT(attachFrame(QWebFrame*)));
+    connect(webPage, SIGNAL(consoleMessage(QString)), this, SLOT(routeMessage(QString)));
 
     m_webPages[queryId] = webPage;
 
     return webPage;
 }
 
-void SunscraperThread::attachAPI()
+void SunscraperWorker::attachFrame(QWebFrame *frame)
+{
+    connect(frame, SIGNAL(javaScriptWindowObjectCleared()),
+        this, SLOT(attachAPI()));
+}
+
+void SunscraperWorker::attachAPI()
 {
     QWebFrame *origin = static_cast<QWebFrame *>(QObject::sender());
     QWebPage *page = origin->page();
@@ -137,7 +147,7 @@ void SunscraperThread::attachAPI()
     origin->addToJavaScriptWindowObject("Sunscraper", proxy, QScriptEngine::QtOwnership);
 }
 
-void SunscraperThread::routeTimeout()
+void SunscraperWorker::routeTimeout()
 {
     QTimer *origin = static_cast<QTimer *>(QObject::sender());
 
@@ -146,3 +156,8 @@ void SunscraperThread::routeTimeout()
 
     emit timeout(queryId);
 }
+
+void SunscraperWorker::routeMessage(QString message)
+{
+    qDebug() << "Sunscraper Console:" << message;
+}

data/ext/embed/{sunscraperthread.h → sunscraperworker.h}

@@ -1,20 +1,22 @@
-#ifndef SUNSCRAPERTHREAD_H
-#define SUNSCRAPERTHREAD_H
+#ifndef SUNSCRAPERWORKER_H
+#define SUNSCRAPERWORKER_H
 
 #include <QObject>
 #include <QSemaphore>
 #include <QMap>
+#include <QUrl>
 
 class QWebPage;
+class QWebFrame;
 class QTimer;
 
-class SunscraperThread : public QObject
+class SunscraperWorker : public QObject
 {
     Q_OBJECT
 public:
     static void invoke();
     static void commitSuicide();
-    static SunscraperThread *instance();
+    static SunscraperWorker *instance();
 
 signals:
     void finished(unsigned queryId, QString result);
@@ -27,17 +29,19 @@ public slots:
     void finalize(unsigned queryId);
 
 private slots:
+    void attachFrame(QWebFrame *frame);
     void attachAPI();
     void routeTimeout();
+    void routeMessage(QString message);
 
 private:
-    static SunscraperThread *m_instance;
+    static SunscraperWorker *m_instance;
     static QSemaphore m_initializationLock;
 
 #if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
     static pthread_t m_thread;
 #else
-#error Your platform is unsupported. Implement SunscraperThread::invoke() and send a pull request.
+#error Your platform is unsupported. Implement SunscraperWorker::invoke() and send a pull request.
 #endif
 
     static void *thread_routine(void *arg);
@@ -45,10 +49,10 @@ private:
     QMap<unsigned, QWebPage *> m_webPages;
     QMap<unsigned, QTimer *> m_timers;
 
-    SunscraperThread();
-    SunscraperThread(SunscraperThread &);
+    SunscraperWorker();
+    SunscraperWorker(SunscraperWorker &);
 
     QWebPage *initializeWebPage(unsigned queryId);
 };
 
-#endif // SUNSCRAPERTHREAD_H
+#endif // SUNSCRAPERWORKER_H

data/ext/extconf.rb

@@ -1,22 +1,31 @@
 # This Makefile will get replaced by qmake.
 
-if Gem.win_platform?
-  qmake = %{qmake -spec win32-g++}
-elsif RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
+if RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
+  # Cannot you OS X have a build system like all sane people?
+  # Win32 wins again.
   qmake = %{qmake -spec macx-g++}
-else
-  qmake = %{qmake}
-end
 
-File.open("Makefile", "w") do |mf|
-  mf.puts <<-ENDM
+  File.open("Makefile", "w") do |mf|
+    mf.puts <<-ENDM
 all:
-	cd embed; #{qmake}
-	cd standalone; #{qmake}
-	make -C embed
-	make -C standalone
-
+	(cd embed && #{qmake}; make)
+	(cd standalone && #{qmake}; make)
 install:
 	# do nothing
-  ENDM
-end
+    ENDM
+  end
+else
+  if Gem.win_platform?
+    qmake = %{qmake -spec win32-g++}
+  else
+    qmake = %{qmake}
+  end
+
+  File.open("Makefile", "w") do |mf|
+    mf.puts <<-ENDM
+all:
+	#{qmake}
+	make
+    ENDM
+  end
+end

data/ext/standalone/sunscraperproxy.cpp

@@ -1,5 +1,6 @@
 #include <QWebPage>
 #include <QWebFrame>
+#include <QtDebug>
 #include "sunscraperproxy.h"
 
 SunscraperProxy::SunscraperProxy(QWebPage *parent, unsigned queryId) :

data/ext/standalone/sunscraperworker.cpp

@@ -35,6 +35,8 @@ QWebPage *SunscraperWorker::initializeWebPage(unsigned queryId)
     Q_ASSERT(_webPages[queryId] == NULL);
 
     QWebPage *webPage = new QWebPage(this);
+    webPage->settings()->setAttribute(QWebSettings::LocalStorageEnabled, true);
+
     connect(webPage->mainFrame(), SIGNAL(javaScriptWindowObjectCleared()),
         this, SLOT(attachAPI()));
 

data/ext/sunscraper-ext.pro

@@ -0,0 +1,2 @@
+TEMPLATE = subdirs
+SUBDIRS  = embed standalone

data/sunscraper.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "sunscraper"
-  s.version     = "1.1.0.beta2"
+  s.version     = "1.1.0.beta3"
   s.authors     = ["Peter Zotov"]
   s.email       = ["whitequark@whitequark.org"]
   s.homepage    = "http://github.com/whitequark/sunscraper"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sunscraper
 version: !ruby/object:Gem::Version
-  version: 1.1.0.beta2
+  version: 1.1.0.beta3
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-09 00:00:00.000000000 Z
+date: 2012-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &76880130 !ruby/object:Gem::Requirement
+  requirement: &81880600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *76880130
+  version_requirements: *81880600
 - !ruby/object:Gem::Dependency
   name: ffi
-  requirement: &76879880 !ruby/object:Gem::Requirement
+  requirement: &81880200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: 1.0.11
   type: :runtime
   prerelease: false
-  version_requirements: *76879880
+  version_requirements: *81880200
 description: A WebKit-based, JavaScript-capable HTML scraper.
 email:
 - whitequark@whitequark.org
@@ -50,18 +50,20 @@ files:
 - README.md
 - Rakefile
 - ext/.gitignore
+- ext/embed/embed.pro
 - ext/embed/sunscraper.cpp
 - ext/embed/sunscraper.h
-- ext/embed/sunscraper.pro
 - ext/embed/sunscraperexternal.cpp
 - ext/embed/sunscraperlibrary.cpp
 - ext/embed/sunscraperlibrary.h
 - ext/embed/sunscraperproxy.cpp
 - ext/embed/sunscraperproxy.h
-- ext/embed/sunscraperthread.cpp
-- ext/embed/sunscraperthread.h
+- ext/embed/sunscraperwebpage.cpp
+- ext/embed/sunscraperwebpage.h
+- ext/embed/sunscraperworker.cpp
+- ext/embed/sunscraperworker.h
 - ext/extconf.rb
-- ext/standalone/sunscraper.pro
+- ext/standalone/standalone.pro
 - ext/standalone/sunscrapermain.cpp
 - ext/standalone/sunscraperproxy.cpp
 - ext/standalone/sunscraperproxy.h
@@ -69,6 +71,7 @@ files:
 - ext/standalone/sunscraperrpc.h
 - ext/standalone/sunscraperworker.cpp
 - ext/standalone/sunscraperworker.h
+- ext/sunscraper-ext.pro
 - lib/sunscraper.rb
 - lib/sunscraper/library.rb
 - lib/sunscraper/standalone.rb
@@ -100,4 +103,3 @@ signing_key:
 specification_version: 3
 summary: A WebKit-based, JavaScript-capable HTML scraper.
 test_files: []
-has_rdoc: