sunscraper 1.2.0.beta1 → 1.2.0.pre1

data/ext/.gitignore

@@ -1,8 +1,10 @@
 *.user
 *.o
+*.a
 *.so*
 *.dylib
 *.dll
+*.app
 moc_*
 Makefile
 standalone/sunscraper

data/ext/common/sunscraperworker.cpp

@@ -86,7 +86,10 @@ void SunscraperWorker::onJavascriptObjectCleared()
     QWebPage *page = origin->page();
 
     unsigned queryId = m_webPages.key(page, 0);
-    Q_ASSERT(queryId != 0);
+
+    /* Called on an already finalized page in the process of finalization. */
+    if(queryId == 0)
+        return;
 
     SunscraperProxy *proxy = new SunscraperProxy(page, queryId);
     connect(proxy, SIGNAL(finished(uint)), this, SLOT(onFinish(uint)));

data/ext/embed/sunscraperinterface.cpp

@@ -82,9 +82,8 @@ void SunscraperInterface::signalSemaphore(unsigned queryId)
 {
     QMutexLocker locker(&m_semaphoresMutex);
 
-    Q_ASSERT(m_semaphores[queryId] != NULL);
-
-    m_semaphores[queryId]->release(1);
+    if(m_semaphores.contains(queryId))
+        m_semaphores[queryId]->release(1);
 }
 
 unsigned SunscraperInterface::createQuery()

data/ext/embed/sunscraperthread.cpp

@@ -1,4 +1,5 @@
 #include <QApplication>
+#include <qnamespace.h>
 #include <QtDebug>
 #include "sunscraperthread.h"
 #include "sunscraperworker.h"
@@ -7,6 +8,8 @@
 pthread_t SunscraperThread::m_thread;
 #endif
 
+extern void qt_set_current_thread_to_main_thread();
+
 void SunscraperThread::invoke()
 {
 #if defined(Q_OS_LINUX) || defined(Q_OS_UNIX)
@@ -23,6 +26,8 @@ void *SunscraperThread::thread_routine(void *)
     /* Why (char*)? Because argv can (theoretically) be modified. *
      * But Qt won't do that with argv[0]. I know, trust me.       */
 
+    QInternal::callFunction(QInternal::SetCurrentThreadToMainThread, NULL);
+
     QApplication app(argc, argv);
     app.setApplicationName("Sunscraper-Embed");
 

data/ext/extconf.rb

@@ -2,32 +2,16 @@
 
 require 'rbconfig'
 
-if RUBY_PLATFORM =~ /darwin/i || RbConfig::CONFIG['target_os'] == 'darwin'
-  # Cannot you OS X have a build system like all sane people?
-  # Win32 wins again.
-  qmake = %{qmake CONFIG+=debug -spec macx-g++}
-
-  File.open("Makefile", "w") do |mf|
-    mf.puts <<-ENDM
-all:
-	(cd embed && #{qmake}; make)
-	(cd standalone && #{qmake}; make)
-install:
-	# do nothing
-    ENDM
-  end
+if Gem.win_platform?
+  qmake = %{qmake CONFIG+=debug -spec win32-g++}
 else
-  if Gem.win_platform?
-    qmake = %{qmake CONFIG+=debug -spec win32-g++}
-  else
-    qmake = %{qmake CONFIG+=debug}
-  end
+  qmake = %{qmake CONFIG+=debug}
+end
 
-  File.open("Makefile", "w") do |mf|
-    mf.puts <<-ENDM
+File.open("Makefile", "w") do |mf|
+  mf.puts <<-ENDM
 all:
 	#{qmake}
 	make
-    ENDM
-  end
+  ENDM
 end

data/ext/standalone/sunscraperrpc.cpp

@@ -7,11 +7,13 @@
 #include <sunscraperworker.h>
 #include "sunscraperrpc.h"
 
+//#define DEBUG_SUNSCRAPERRPC
+
 SunscraperWorker *SunscraperRPC::m_worker;
 unsigned SunscraperRPC::m_nextQueryId;
 
 SunscraperRPC::SunscraperRPC(QLocalSocket *socket) :
-        m_socket(socket), m_state(StateHeader)
+        m_socket(socket), m_state(StateHeader), m_result(false)
 {
     m_nextQueryId += 1;
     m_queryId = m_nextQueryId;
@@ -80,6 +82,10 @@ void SunscraperRPC::onInputDisconnected()
 
 void SunscraperRPC::processRequest(unsigned requestType, QByteArray data)
 {
+#ifdef DEBUG_SUNSCRAPERRPC
+    qDebug() << QString("request(%1)").arg(m_queryId) << requestType << data;
+#endif
+
     switch(requestType) {
     case RPC_LOAD_HTML: {
             QDataStream stream(data);
@@ -127,6 +133,10 @@ void SunscraperRPC::onFinish(unsigned eventQueryId)
     if(eventQueryId != m_queryId)
         return;
 
+#ifdef DEBUG_SUNSCRAPERRPC
+    qDebug() << QString("finish(%1)").arg(m_queryId);
+#endif
+
     QByteArray data;
 
     QDataStream stream(&data, QIODevice::WriteOnly);
@@ -142,6 +152,10 @@ void SunscraperRPC::onTimeout(unsigned eventQueryId)
     if(eventQueryId != m_queryId)
         return;
 
+#ifdef DEBUG_SUNSCRAPERRPC
+    qDebug() << QString("timeout(%1)").arg(m_queryId);
+#endif
+
     QByteArray data;
 
     QDataStream stream(&data, QIODevice::WriteOnly);
@@ -157,6 +171,10 @@ void SunscraperRPC::onFetchDone(unsigned eventQueryId, QString data)
     if(eventQueryId != m_queryId)
         return;
 
+#ifdef DEBUG_SUNSCRAPERRPC
+    qDebug() << QString("fetchDone(%1)").arg(m_queryId);
+#endif
+
     sendReply(data.toLocal8Bit());
 }
 
@@ -164,6 +182,10 @@ void SunscraperRPC::sendReply(QByteArray data)
 {
     QByteArray packet;
 
+#ifdef DEBUG_SUNSCRAPERRPC
+    qDebug() << QString("reply(%1)").arg(m_queryId) << data;
+#endif
+
     QDataStream stream(&packet, QIODevice::WriteOnly);
     stream << data;
 

data/spec/sunscraper_spec.rb

@@ -105,16 +105,18 @@ define_tests = lambda do |klass, worker|
       end
     end
 
-    it "should withstand a lot of concurrent threads" do
-      500.times.map {
-        Thread.new {
-          Sunscraper.scrape_html(HTML_FUGA)
+    if worker == :embed
+      it "should withstand a lot of concurrent threads" do
+        100.times.map {
+          Thread.new {
+            Sunscraper.scrape_html(HTML_FUGA)
+          }
+        }.each(&:join).
+          map(&:value).
+          each { |result|
+          result.should include('It works!')
         }
-      }.each(&:join).
-        map(&:value).
-        each { |result|
-        result.should include('It works!')
-      }
+      end
     end
   end
 end

data/sunscraper.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "sunscraper"
-  s.version     = "1.2.0.beta1"
+  s.version     = "1.2.0.pre1"
   s.authors     = ["Peter Zotov"]
   s.email       = ["whitequark@whitequark.org"]
   s.homepage    = "http://github.com/whitequark/sunscraper"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sunscraper
 version: !ruby/object:Gem::Version
-  version: 1.2.0.beta1
+  version: 1.2.0.pre1
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-10 00:00:00.000000000 Z
+date: 2012-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &74877930 !ruby/object:Gem::Requirement
+  requirement: &85313840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *74877930
+  version_requirements: *85313840
 - !ruby/object:Gem::Dependency
   name: ffi
-  requirement: &74877500 !ruby/object:Gem::Requirement
+  requirement: &85313520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: 1.0.11
   type: :runtime
   prerelease: false
-  version_requirements: *74877500
+  version_requirements: *85313520
 description: A WebKit-based, JavaScript-capable HTML scraper.
 email:
 - whitequark@whitequark.org
@@ -51,7 +51,6 @@ files:
 - Rakefile
 - ext/.gitignore
 - ext/common/common.pro
-- ext/common/libsunscraper_common.a
 - ext/common/sunscraperproxy.cpp
 - ext/common/sunscraperproxy.h
 - ext/common/sunscraperwebpage.cpp