sunscraper 1.0.0 → 1.1.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. data/.travis.yml +28 -0
  2. data/README.md +42 -20
  3. data/ext/.gitignore +5 -1
  4. data/ext/embed/sunscraper.cpp +92 -0
  5. data/ext/{sunscraper.h → embed/sunscraper.h} +9 -7
  6. data/ext/{sunscraper.pro → embed/sunscraper.pro} +2 -0
  7. data/ext/embed/sunscraperexternal.cpp +39 -0
  8. data/ext/{sunscraperlibrary.cpp → embed/sunscraperlibrary.cpp} +4 -9
  9. data/ext/{sunscraperlibrary.h → embed/sunscraperlibrary.h} +1 -5
  10. data/ext/embed/sunscraperproxy.cpp +14 -0
  11. data/ext/{sunscraperproxy.h → embed/sunscraperproxy.h} +3 -3
  12. data/ext/embed/sunscraperthread.cpp +148 -0
  13. data/ext/embed/sunscraperthread.h +54 -0
  14. data/ext/extconf.rb +13 -3
  15. data/ext/standalone/sunscraper.pro +13 -0
  16. data/ext/standalone/sunscrapermain.cpp +13 -0
  17. data/ext/{sunscraperproxy.cpp → standalone/sunscraperproxy.cpp} +2 -2
  18. data/ext/standalone/sunscraperproxy.h +24 -0
  19. data/ext/standalone/sunscraperrpc.cpp +183 -0
  20. data/ext/standalone/sunscraperrpc.h +64 -0
  21. data/ext/{sunscraperthread.cpp → standalone/sunscraperworker.cpp} +9 -18
  22. data/ext/{sunscraperthread.h → standalone/sunscraperworker.h} +8 -8
  23. data/lib/sunscraper/library.rb +33 -29
  24. data/lib/sunscraper/standalone.rb +168 -0
  25. data/lib/sunscraper.rb +48 -11
  26. data/spec/sunscraper_spec.rb +59 -13
  27. data/sunscraper.gemspec +2 -2
  28. metadata +58 -75
  29. data/ext/Makefile +0 -270
  30. data/ext/sunscraper.cpp +0 -86
  31. data/ext/sunscraperexternal.cpp +0 -33
data/spec/sunscraper_spec.rb CHANGED
@@ -21,25 +21,71 @@ HTML
21
21
 
22
22
  PORT = 45555
23
23
 
24
- describe Sunscraper do
25
- it "can scrape an HTML provided as a string" do
26
- Sunscraper.scrape_html(HTML).should include('It works!')
24
+ def with_webserver
25
+ server = WEBrick::HTTPServer.new :Port => PORT, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
26
+ server.mount_proc '/' do |req, res|
27
+ res.body = HTML
27
28
  end
29
+ Thread.new { server.start }
28
30
 
29
- it "can scrape an URL" do
30
- server = WEBrick::HTTPServer.new :Port => PORT, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
31
- server.mount_proc '/' do |req, res|
32
- res.body = HTML
31
+ yield PORT
32
+ ensure
33
+ server.shutdown if server
34
+ end
35
+
36
+ class String
37
+ def to_v
38
+ split(".").map(&:to_i).extend Comparable
39
+ end
40
+ end
41
+
42
+ unless Sunscraper.os_x?
43
+ # This part currently crashes on OS X (and will forever).
44
+ describe "Sunscraper::Library" do
45
+ before do
46
+ Sunscraper.worker = :embed
33
47
  end
34
- Thread.new { server.start }
35
48
 
36
- Sunscraper.scrape_url("http://localhost:#{PORT}/").should include('It works!')
49
+ it "can scrape an HTML provided as a string" do
50
+ Sunscraper.scrape_html(HTML).should include('It works!')
51
+ end
37
52
 
38
- server.stop
53
+ it "can scrape an URL" do
54
+ with_webserver do |port|
55
+ Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
56
+ end
57
+ end
58
+
59
+ it "should time out if callback is not called" do
60
+ lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
61
+ should raise_exception(Sunscraper::ScrapeTimeout)
62
+ end
39
63
  end
64
+ end
40
65
 
41
- it "should time out if callback is not called" do
42
- lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
43
- should raise_exception(Sunscraper::ScrapeTimeout)
66
+ if !(RUBY_ENGINE =~ /rbx/ || RUBY_ENGINE =~ /jruby/) ||
67
+ ENV['EXPERIMENTAL'] == 'true'
68
+ # This part currently crashes Rubinius (as of Mar 09, 2012),
69
+ # and crashes jruby < 1.7.0, and uses Unix sockets which don't
70
+ # work even on jruby master (as of Mar 09, 2012).
71
+ describe "Sunscraper::Standalone" do
72
+ before do
73
+ Sunscraper.worker = :standalone
74
+ end
75
+
76
+ it "can scrape an HTML provided as a string" do
77
+ Sunscraper.scrape_html(HTML).should include('It works!')
78
+ end
79
+
80
+ it "can scrape an URL" do
81
+ with_webserver do |port|
82
+ Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
83
+ end
84
+ end
85
+
86
+ it "should time out if callback is not called" do
87
+ lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
88
+ should raise_exception(Sunscraper::ScrapeTimeout)
89
+ end
44
90
  end
45
91
  end
data/sunscraper.gemspec CHANGED
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "sunscraper"
6
- s.version = "1.0.0"
6
+ s.version = "1.1.0.beta1"
7
7
  s.authors = ["Peter Zotov"]
8
8
  s.email = ["whitequark@whitequark.org"]
9
- s.homepage = "http://github.com/roundlake/sunscraper"
9
+ s.homepage = "http://github.com/whitequark/sunscraper"
10
10
  s.summary = %q{A WebKit-based, JavaScript-capable HTML scraper.}
11
11
  s.description = s.summary
12
12
 
metadata CHANGED
@@ -1,120 +1,103 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: sunscraper
3
- version: !ruby/object:Gem::Version
4
- hash: 540260530
5
- prerelease:
6
- segments:
7
- - 1
8
- - 0
9
- - 0
10
- version: 1.0.0
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.0.beta1
5
+ prerelease: 6
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Peter Zotov
14
9
  autorequire:
15
10
  bindir: bin
16
11
  cert_chain: []
17
-
18
- date: 2012-02-18 00:00:00 Z
19
- dependencies:
20
- - !ruby/object:Gem::Dependency
12
+ date: 2012-03-09 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
21
15
  name: rspec
22
- prerelease: false
23
- requirement: &id001 !ruby/object:Gem::Requirement
16
+ requirement: &80239320 !ruby/object:Gem::Requirement
24
17
  none: false
25
- requirements:
26
- - - ">="
27
- - !ruby/object:Gem::Version
28
- hash: 881230260
29
- segments:
30
- - 0
31
- version: "0"
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
32
22
  type: :development
33
- version_requirements: *id001
34
- - !ruby/object:Gem::Dependency
35
- name: ffi
36
23
  prerelease: false
37
- requirement: &id002 !ruby/object:Gem::Requirement
24
+ version_requirements: *80239320
25
+ - !ruby/object:Gem::Dependency
26
+ name: ffi
27
+ requirement: &80239070 !ruby/object:Gem::Requirement
38
28
  none: false
39
- requirements:
40
- - - ">="
41
- - !ruby/object:Gem::Version
42
- hash: 716237343
43
- segments:
44
- - 1
45
- - 0
46
- - 11
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
47
32
  version: 1.0.11
48
33
  type: :runtime
49
- version_requirements: *id002
34
+ prerelease: false
35
+ version_requirements: *80239070
50
36
  description: A WebKit-based, JavaScript-capable HTML scraper.
51
- email:
37
+ email:
52
38
  - whitequark@whitequark.org
53
39
  executables: []
54
-
55
- extensions:
40
+ extensions:
56
41
  - ext/extconf.rb
57
42
  extra_rdoc_files: []
58
-
59
- files:
43
+ files:
60
44
  - .gitignore
61
45
  - .rspec
46
+ - .travis.yml
62
47
  - .yardopts
63
48
  - Gemfile
64
49
  - LICENSE
65
50
  - README.md
66
51
  - Rakefile
67
52
  - ext/.gitignore
68
- - ext/Makefile
53
+ - ext/embed/sunscraper.cpp
54
+ - ext/embed/sunscraper.h
55
+ - ext/embed/sunscraper.pro
56
+ - ext/embed/sunscraperexternal.cpp
57
+ - ext/embed/sunscraperlibrary.cpp
58
+ - ext/embed/sunscraperlibrary.h
59
+ - ext/embed/sunscraperproxy.cpp
60
+ - ext/embed/sunscraperproxy.h
61
+ - ext/embed/sunscraperthread.cpp
62
+ - ext/embed/sunscraperthread.h
69
63
  - ext/extconf.rb
70
- - ext/sunscraper.cpp
71
- - ext/sunscraper.h
72
- - ext/sunscraper.pro
73
- - ext/sunscraperexternal.cpp
74
- - ext/sunscraperlibrary.cpp
75
- - ext/sunscraperlibrary.h
76
- - ext/sunscraperproxy.cpp
77
- - ext/sunscraperproxy.h
78
- - ext/sunscraperthread.cpp
79
- - ext/sunscraperthread.h
64
+ - ext/standalone/sunscraper.pro
65
+ - ext/standalone/sunscrapermain.cpp
66
+ - ext/standalone/sunscraperproxy.cpp
67
+ - ext/standalone/sunscraperproxy.h
68
+ - ext/standalone/sunscraperrpc.cpp
69
+ - ext/standalone/sunscraperrpc.h
70
+ - ext/standalone/sunscraperworker.cpp
71
+ - ext/standalone/sunscraperworker.h
80
72
  - lib/sunscraper.rb
81
73
  - lib/sunscraper/library.rb
74
+ - lib/sunscraper/standalone.rb
82
75
  - spec/spec_helper.rb
83
76
  - spec/sunscraper_spec.rb
84
77
  - sunscraper.gemspec
85
- homepage: http://github.com/roundlake/sunscraper
78
+ homepage: http://github.com/whitequark/sunscraper
86
79
  licenses: []
87
-
88
80
  post_install_message:
89
81
  rdoc_options: []
90
-
91
- require_paths:
82
+ require_paths:
92
83
  - lib
93
- required_ruby_version: !ruby/object:Gem::Requirement
84
+ required_ruby_version: !ruby/object:Gem::Requirement
94
85
  none: false
95
- requirements:
96
- - - ">="
97
- - !ruby/object:Gem::Version
98
- hash: 881230260
99
- segments:
100
- - 0
101
- version: "0"
102
- required_rubygems_version: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ! '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ required_rubygems_version: !ruby/object:Gem::Requirement
103
91
  none: false
104
- requirements:
105
- - - ">="
106
- - !ruby/object:Gem::Version
107
- hash: 881230260
108
- segments:
109
- - 0
110
- version: "0"
92
+ requirements:
93
+ - - ! '>'
94
+ - !ruby/object:Gem::Version
95
+ version: 1.3.1
111
96
  requirements: []
112
-
113
97
  rubyforge_project: sunscraper
114
- rubygems_version: 1.8.12
98
+ rubygems_version: 1.8.17
115
99
  signing_key:
116
100
  specification_version: 3
117
101
  summary: A WebKit-based, JavaScript-capable HTML scraper.
118
102
  test_files: []
119
-
120
103
  has_rdoc:
data/ext/Makefile DELETED
@@ -1,270 +0,0 @@
1
- #############################################################################
2
- # Makefile for building: libsunscraper.so.1.0.0
3
- # Generated by qmake (2.01a) (Qt 4.7.4) on: Sat Feb 18 05:54:02 2012
4
- # Project: sunscraper.pro
5
- # Template: lib
6
- # Command: /usr/bin/qmake-qt4 -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
7
- #############################################################################
8
-
9
- ####### Compiler, tools and options
10
-
11
- CC = gcc
12
- CXX = g++
13
- DEFINES = -DQT_WEBKIT -DQT_WEBKIT_LIB -DQT_GUI_LIB -DQT_CORE_LIB -DQT_SHARED
14
- CFLAGS = -pipe -g -Wall -W -D_REENTRANT -fPIC $(DEFINES)
15
- CXXFLAGS = -pipe -g -Wall -W -D_REENTRANT -fPIC $(DEFINES)
16
- INCPATH = -I/usr/share/qt4/mkspecs/linux-g++ -I. -I/usr/include/qt4/QtCore -I/usr/include/qt4/QtGui -I/usr/include/qt4/QtWebKit -I/usr/include/qt4 -I.
17
- LINK = g++
18
- LFLAGS = -shared -Wl,-soname,libsunscraper.so.1
19
- LIBS = $(SUBLIBS) -L/usr/lib -lQtWebKit -lQtGui -lQtCore -lpthread
20
- AR = ar cqs
21
- RANLIB =
22
- QMAKE = /usr/bin/qmake-qt4
23
- TAR = tar -cf
24
- COMPRESS = gzip -9f
25
- COPY = cp -f
26
- SED = sed
27
- COPY_FILE = $(COPY)
28
- COPY_DIR = $(COPY) -r
29
- STRIP = strip
30
- INSTALL_FILE = install -m 644 -p
31
- INSTALL_DIR = $(COPY_DIR)
32
- INSTALL_PROGRAM = install -m 755 -p
33
- DEL_FILE = rm -f
34
- SYMLINK = ln -f -s
35
- DEL_DIR = rmdir
36
- MOVE = mv -f
37
- CHK_DIR_EXISTS= test -d
38
- MKDIR = mkdir -p
39
-
40
- ####### Output directory
41
-
42
- OBJECTS_DIR = ./
43
-
44
- ####### Files
45
-
46
- SOURCES = sunscraperlibrary.cpp \
47
- sunscraperthread.cpp \
48
- sunscraperexternal.cpp \
49
- sunscraper.cpp \
50
- sunscraperproxy.cpp moc_sunscraperthread.cpp \
51
- moc_sunscraper.cpp \
52
- moc_sunscraperproxy.cpp
53
- OBJECTS = sunscraperlibrary.o \
54
- sunscraperthread.o \
55
- sunscraperexternal.o \
56
- sunscraper.o \
57
- sunscraperproxy.o \
58
- moc_sunscraperthread.o \
59
- moc_sunscraper.o \
60
- moc_sunscraperproxy.o
61
- DIST = /usr/share/qt4/mkspecs/common/g++.conf \
62
- /usr/share/qt4/mkspecs/common/unix.conf \
63
- /usr/share/qt4/mkspecs/common/linux.conf \
64
- /usr/share/qt4/mkspecs/qconfig.pri \
65
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
66
- /usr/share/qt4/mkspecs/features/qt_functions.prf \
67
- /usr/share/qt4/mkspecs/features/qt_config.prf \
68
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf \
69
- /usr/share/qt4/mkspecs/features/default_pre.prf \
70
- /usr/share/qt4/mkspecs/features/debug.prf \
71
- /usr/share/qt4/mkspecs/features/default_post.prf \
72
- /usr/share/qt4/mkspecs/features/warn_on.prf \
73
- /usr/share/qt4/mkspecs/features/qt.prf \
74
- /usr/share/qt4/mkspecs/features/unix/thread.prf \
75
- /usr/share/qt4/mkspecs/features/moc.prf \
76
- /usr/share/qt4/mkspecs/features/resources.prf \
77
- /usr/share/qt4/mkspecs/features/uic.prf \
78
- /usr/share/qt4/mkspecs/features/yacc.prf \
79
- /usr/share/qt4/mkspecs/features/lex.prf \
80
- /usr/share/qt4/mkspecs/features/include_source_dir.prf \
81
- sunscraper.pro
82
- QMAKE_TARGET = sunscraper
83
- DESTDIR =
84
- TARGET = libsunscraper.so.1.0.0
85
- TARGETA = libsunscraper.a
86
- TARGETD = libsunscraper.so.1.0.0
87
- TARGET0 = libsunscraper.so
88
- TARGET1 = libsunscraper.so.1
89
- TARGET2 = libsunscraper.so.1.0
90
-
91
- first: all
92
- ####### Implicit rules
93
-
94
- .SUFFIXES: .o .c .cpp .cc .cxx .C
95
-
96
- .cpp.o:
97
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
98
-
99
- .cc.o:
100
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
101
-
102
- .cxx.o:
103
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
104
-
105
- .C.o:
106
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
107
-
108
- .c.o:
109
- $(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<"
110
-
111
- ####### Build rules
112
-
113
- all: Makefile $(TARGET)
114
-
115
- $(TARGET): $(OBJECTS) $(SUBLIBS) $(OBJCOMP)
116
- -$(DEL_FILE) $(TARGET) $(TARGET0) $(TARGET1) $(TARGET2)
117
- $(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(LIBS) $(OBJCOMP)
118
- -ln -s $(TARGET) $(TARGET0)
119
- -ln -s $(TARGET) $(TARGET1)
120
- -ln -s $(TARGET) $(TARGET2)
121
-
122
-
123
-
124
- staticlib: $(TARGETA)
125
-
126
- $(TARGETA): $(OBJECTS) $(OBJCOMP)
127
- -$(DEL_FILE) $(TARGETA)
128
- $(AR) $(TARGETA) $(OBJECTS)
129
-
130
- Makefile: sunscraper.pro /usr/share/qt4/mkspecs/linux-g++/qmake.conf /usr/share/qt4/mkspecs/common/g++.conf \
131
- /usr/share/qt4/mkspecs/common/unix.conf \
132
- /usr/share/qt4/mkspecs/common/linux.conf \
133
- /usr/share/qt4/mkspecs/qconfig.pri \
134
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
135
- /usr/share/qt4/mkspecs/features/qt_functions.prf \
136
- /usr/share/qt4/mkspecs/features/qt_config.prf \
137
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf \
138
- /usr/share/qt4/mkspecs/features/default_pre.prf \
139
- /usr/share/qt4/mkspecs/features/debug.prf \
140
- /usr/share/qt4/mkspecs/features/default_post.prf \
141
- /usr/share/qt4/mkspecs/features/warn_on.prf \
142
- /usr/share/qt4/mkspecs/features/qt.prf \
143
- /usr/share/qt4/mkspecs/features/unix/thread.prf \
144
- /usr/share/qt4/mkspecs/features/moc.prf \
145
- /usr/share/qt4/mkspecs/features/resources.prf \
146
- /usr/share/qt4/mkspecs/features/uic.prf \
147
- /usr/share/qt4/mkspecs/features/yacc.prf \
148
- /usr/share/qt4/mkspecs/features/lex.prf \
149
- /usr/share/qt4/mkspecs/features/include_source_dir.prf \
150
- /usr/lib/libQtWebKit.prl \
151
- /usr/lib/libQtGui.prl \
152
- /usr/lib/libQtCore.prl
153
- $(QMAKE) -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
154
- /usr/share/qt4/mkspecs/common/g++.conf:
155
- /usr/share/qt4/mkspecs/common/unix.conf:
156
- /usr/share/qt4/mkspecs/common/linux.conf:
157
- /usr/share/qt4/mkspecs/qconfig.pri:
158
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri:
159
- /usr/share/qt4/mkspecs/features/qt_functions.prf:
160
- /usr/share/qt4/mkspecs/features/qt_config.prf:
161
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf:
162
- /usr/share/qt4/mkspecs/features/default_pre.prf:
163
- /usr/share/qt4/mkspecs/features/debug.prf:
164
- /usr/share/qt4/mkspecs/features/default_post.prf:
165
- /usr/share/qt4/mkspecs/features/warn_on.prf:
166
- /usr/share/qt4/mkspecs/features/qt.prf:
167
- /usr/share/qt4/mkspecs/features/unix/thread.prf:
168
- /usr/share/qt4/mkspecs/features/moc.prf:
169
- /usr/share/qt4/mkspecs/features/resources.prf:
170
- /usr/share/qt4/mkspecs/features/uic.prf:
171
- /usr/share/qt4/mkspecs/features/yacc.prf:
172
- /usr/share/qt4/mkspecs/features/lex.prf:
173
- /usr/share/qt4/mkspecs/features/include_source_dir.prf:
174
- /usr/lib/libQtWebKit.prl:
175
- /usr/lib/libQtGui.prl:
176
- /usr/lib/libQtCore.prl:
177
- qmake: FORCE
178
- @$(QMAKE) -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
179
-
180
- dist:
181
- @$(CHK_DIR_EXISTS) .tmp/sunscraper1.0.0 || $(MKDIR) .tmp/sunscraper1.0.0
182
- $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/sunscraper1.0.0/ && $(COPY_FILE) --parents sunscraperlibrary.h sunscraperthread.h sunscraper.h sunscraperproxy.h .tmp/sunscraper1.0.0/ && $(COPY_FILE) --parents sunscraperlibrary.cpp sunscraperthread.cpp sunscraperexternal.cpp sunscraper.cpp sunscraperproxy.cpp .tmp/sunscraper1.0.0/ && (cd `dirname .tmp/sunscraper1.0.0` && $(TAR) sunscraper1.0.0.tar sunscraper1.0.0 && $(COMPRESS) sunscraper1.0.0.tar) && $(MOVE) `dirname .tmp/sunscraper1.0.0`/sunscraper1.0.0.tar.gz . && $(DEL_FILE) -r .tmp/sunscraper1.0.0
183
-
184
-
185
- clean:compiler_clean
186
- -$(DEL_FILE) $(OBJECTS)
187
- -$(DEL_FILE) *~ core *.core
188
-
189
-
190
- ####### Sub-libraries
191
-
192
- distclean: clean
193
- -$(DEL_FILE) $(TARGET)
194
- -$(DEL_FILE) $(TARGET0) $(TARGET1) $(TARGET2) $(TARGETA)
195
- -$(DEL_FILE) Makefile
196
-
197
-
198
- check: first
199
-
200
- mocclean: compiler_moc_header_clean compiler_moc_source_clean
201
-
202
- mocables: compiler_moc_header_make_all compiler_moc_source_make_all
203
-
204
- compiler_moc_header_make_all: moc_sunscraperthread.cpp moc_sunscraper.cpp moc_sunscraperproxy.cpp
205
- compiler_moc_header_clean:
206
- -$(DEL_FILE) moc_sunscraperthread.cpp moc_sunscraper.cpp moc_sunscraperproxy.cpp
207
- moc_sunscraperthread.cpp: sunscraperthread.h
208
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraperthread.h -o moc_sunscraperthread.cpp
209
-
210
- moc_sunscraper.cpp: sunscraper.h
211
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraper.h -o moc_sunscraper.cpp
212
-
213
- moc_sunscraperproxy.cpp: sunscraperproxy.h
214
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraperproxy.h -o moc_sunscraperproxy.cpp
215
-
216
- compiler_rcc_make_all:
217
- compiler_rcc_clean:
218
- compiler_image_collection_make_all: qmake_image_collection.cpp
219
- compiler_image_collection_clean:
220
- -$(DEL_FILE) qmake_image_collection.cpp
221
- compiler_moc_source_make_all:
222
- compiler_moc_source_clean:
223
- compiler_uic_make_all:
224
- compiler_uic_clean:
225
- compiler_yacc_decl_make_all:
226
- compiler_yacc_decl_clean:
227
- compiler_yacc_impl_make_all:
228
- compiler_yacc_impl_clean:
229
- compiler_lex_make_all:
230
- compiler_lex_clean:
231
- compiler_clean: compiler_moc_header_clean
232
-
233
- ####### Compile
234
-
235
- sunscraperlibrary.o: sunscraperlibrary.cpp sunscraperlibrary.h \
236
- sunscraperthread.h
237
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperlibrary.o sunscraperlibrary.cpp
238
-
239
- sunscraperthread.o: sunscraperthread.cpp sunscraperthread.h \
240
- sunscraperproxy.h
241
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperthread.o sunscraperthread.cpp
242
-
243
- sunscraperexternal.o: sunscraperexternal.cpp sunscraper.h
244
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperexternal.o sunscraperexternal.cpp
245
-
246
- sunscraper.o: sunscraper.cpp sunscraper.h \
247
- sunscraperlibrary.h \
248
- sunscraperthread.h
249
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraper.o sunscraper.cpp
250
-
251
- sunscraperproxy.o: sunscraperproxy.cpp sunscraperproxy.h
252
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperproxy.o sunscraperproxy.cpp
253
-
254
- moc_sunscraperthread.o: moc_sunscraperthread.cpp
255
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraperthread.o moc_sunscraperthread.cpp
256
-
257
- moc_sunscraper.o: moc_sunscraper.cpp
258
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraper.o moc_sunscraper.cpp
259
-
260
- moc_sunscraperproxy.o: moc_sunscraperproxy.cpp
261
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraperproxy.o moc_sunscraperproxy.cpp
262
-
263
- ####### Install
264
-
265
- install: FORCE
266
-
267
- uninstall: FORCE
268
-
269
- FORCE:
270
-
data/ext/sunscraper.cpp DELETED
@@ -1,86 +0,0 @@
1
- #include <QApplication>
2
- #include <QThread>
3
- #include <QTimer>
4
- #include <QWebPage>
5
- #include <QWebFrame>
6
- #include <QMutexLocker>
7
- #include <QtDebug>
8
- #include "sunscraper.h"
9
- #include "sunscraperlibrary.h"
10
- #include "sunscraperthread.h"
11
-
12
- unsigned Sunscraper::_nextQueryId = 1;
13
- QMutex Sunscraper::_staticMutex;
14
-
15
- Sunscraper::Sunscraper()
16
- {
17
- QMutexLocker locker(&_staticMutex);
18
-
19
- _queryId = _nextQueryId++;
20
-
21
- SunscraperThread *worker = SunscraperLibrary::instance()->thread();
22
-
23
- connect(this, SIGNAL(requestLoadHtml(uint,QString)),
24
- worker, SLOT(loadHtml(uint,QString)), Qt::QueuedConnection);
25
- connect(this, SIGNAL(requestLoadUrl(uint,QString)),
26
- worker, SLOT(loadUrl(uint,QString)), Qt::QueuedConnection);
27
- connect(this, SIGNAL(requestFinalize(uint)),
28
- worker, SLOT(finalize(uint)), Qt::QueuedConnection);
29
-
30
- connect(worker, SIGNAL(finished(uint,QString)),
31
- this, SLOT(finished(uint,QString)), Qt::QueuedConnection);
32
- }
33
-
34
- void Sunscraper::loadHtml(QString html)
35
- {
36
- emit requestLoadHtml(_queryId, html);
37
- }
38
-
39
- void Sunscraper::loadUrl(QString url)
40
- {
41
- emit requestLoadUrl(_queryId, url);
42
- }
43
-
44
- void Sunscraper::wait(unsigned timeout)
45
- {
46
- QTimer _timeoutTimer;
47
- connect(&_timeoutTimer, SIGNAL(timeout()), this, SLOT(timeout()));
48
-
49
- _timeoutTimer.setInterval(timeout);
50
- _timeoutTimer.start();
51
-
52
- _eventLoop.exec();
53
-
54
- _timeoutTimer.stop();
55
- }
56
-
57
- void Sunscraper::finished(unsigned eventQueryId, QString html)
58
- {
59
- if(eventQueryId != _queryId)
60
- return;
61
-
62
- _eventLoop.quit();
63
-
64
- _html = html.toUtf8();
65
-
66
- emit requestFinalize(_queryId);
67
- }
68
-
69
- void Sunscraper::timeout()
70
- {
71
- _eventLoop.quit();
72
-
73
- _html = "!SUNSCRAPER_TIMEOUT";
74
-
75
- emit requestFinalize(_queryId);
76
- }
77
-
78
- QByteArray Sunscraper::fetch()
79
- {
80
- return _html;
81
- }
82
-
83
- const char *Sunscraper::fetchAsCString()
84
- {
85
- return _html.constData();
86
- }
data/ext/sunscraperexternal.cpp DELETED
@@ -1,33 +0,0 @@
1
- #include "sunscraper.h"
2
-
3
- extern "C" {
4
- Sunscraper *sunscraper_create()
5
- {
6
- return new Sunscraper();
7
- }
8
-
9
- void sunscraper_load_html(Sunscraper *sunscraper, const char *html)
10
- {
11
- sunscraper->loadHtml(html);
12
- }
13
-
14
- void sunscraper_load_url(Sunscraper *sunscraper, const char *url)
15
- {
16
- sunscraper->loadUrl(url);
17
- }
18
-
19
- void sunscraper_wait(Sunscraper *sunscraper, unsigned timeout)
20
- {
21
- sunscraper->wait(timeout);
22
- }
23
-
24
- const char *sunscraper_fetch(Sunscraper *sunscraper)
25
- {
26
- return sunscraper->fetchAsCString();
27
- }
28
-
29
- void sunscraper_discard(Sunscraper *sunscraper)
30
- {
31
- delete sunscraper;
32
- }
33
- }