sunscraper 1.0.0 → 1.1.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (31) hide show
  1. data/.travis.yml +28 -0
  2. data/README.md +42 -20
  3. data/ext/.gitignore +5 -1
  4. data/ext/embed/sunscraper.cpp +92 -0
  5. data/ext/{sunscraper.h → embed/sunscraper.h} +9 -7
  6. data/ext/{sunscraper.pro → embed/sunscraper.pro} +2 -0
  7. data/ext/embed/sunscraperexternal.cpp +39 -0
  8. data/ext/{sunscraperlibrary.cpp → embed/sunscraperlibrary.cpp} +4 -9
  9. data/ext/{sunscraperlibrary.h → embed/sunscraperlibrary.h} +1 -5
  10. data/ext/embed/sunscraperproxy.cpp +14 -0
  11. data/ext/{sunscraperproxy.h → embed/sunscraperproxy.h} +3 -3
  12. data/ext/embed/sunscraperthread.cpp +148 -0
  13. data/ext/embed/sunscraperthread.h +54 -0
  14. data/ext/extconf.rb +13 -3
  15. data/ext/standalone/sunscraper.pro +13 -0
  16. data/ext/standalone/sunscrapermain.cpp +13 -0
  17. data/ext/{sunscraperproxy.cpp → standalone/sunscraperproxy.cpp} +2 -2
  18. data/ext/standalone/sunscraperproxy.h +24 -0
  19. data/ext/standalone/sunscraperrpc.cpp +183 -0
  20. data/ext/standalone/sunscraperrpc.h +64 -0
  21. data/ext/{sunscraperthread.cpp → standalone/sunscraperworker.cpp} +9 -18
  22. data/ext/{sunscraperthread.h → standalone/sunscraperworker.h} +8 -8
  23. data/lib/sunscraper/library.rb +33 -29
  24. data/lib/sunscraper/standalone.rb +168 -0
  25. data/lib/sunscraper.rb +48 -11
  26. data/spec/sunscraper_spec.rb +59 -13
  27. data/sunscraper.gemspec +2 -2
  28. metadata +58 -75
  29. data/ext/Makefile +0 -270
  30. data/ext/sunscraper.cpp +0 -86
  31. data/ext/sunscraperexternal.cpp +0 -33
data/spec/sunscraper_spec.rb CHANGED
@@ -21,25 +21,71 @@ HTML
21
21
 
22
22
  PORT = 45555
23
23
 
24
- describe Sunscraper do
25
- it "can scrape an HTML provided as a string" do
26
- Sunscraper.scrape_html(HTML).should include('It works!')
24
+ def with_webserver
25
+ server = WEBrick::HTTPServer.new :Port => PORT, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
26
+ server.mount_proc '/' do |req, res|
27
+ res.body = HTML
27
28
  end
29
+ Thread.new { server.start }
28
30
 
29
- it "can scrape an URL" do
30
- server = WEBrick::HTTPServer.new :Port => PORT, :Logger => WEBrick::Log.new('/dev/null'), :AccessLog => []
31
- server.mount_proc '/' do |req, res|
32
- res.body = HTML
31
+ yield PORT
32
+ ensure
33
+ server.shutdown if server
34
+ end
35
+
36
+ class String
37
+ def to_v
38
+ split(".").map(&:to_i).extend Comparable
39
+ end
40
+ end
41
+
42
+ unless Sunscraper.os_x?
43
+ # This part currently crashes on OS X (and will forever).
44
+ describe "Sunscraper::Library" do
45
+ before do
46
+ Sunscraper.worker = :embed
33
47
  end
34
- Thread.new { server.start }
35
48
 
36
- Sunscraper.scrape_url("http://localhost:#{PORT}/").should include('It works!')
49
+ it "can scrape an HTML provided as a string" do
50
+ Sunscraper.scrape_html(HTML).should include('It works!')
51
+ end
37
52
 
38
- server.stop
53
+ it "can scrape an URL" do
54
+ with_webserver do |port|
55
+ Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
56
+ end
57
+ end
58
+
59
+ it "should time out if callback is not called" do
60
+ lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
61
+ should raise_exception(Sunscraper::ScrapeTimeout)
62
+ end
39
63
  end
64
+ end
40
65
 
41
- it "should time out if callback is not called" do
42
- lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
43
- should raise_exception(Sunscraper::ScrapeTimeout)
66
+ if !(RUBY_ENGINE =~ /rbx/ || RUBY_ENGINE =~ /jruby/) ||
67
+ ENV['EXPERIMENTAL'] == 'true'
68
+ # This part currently crashes Rubinius (as of Mar 09, 2012),
69
+ # and crashes jruby < 1.7.0, and uses Unix sockets which don't
70
+ # work even on jruby master (as of Mar 09, 2012).
71
+ describe "Sunscraper::Standalone" do
72
+ before do
73
+ Sunscraper.worker = :standalone
74
+ end
75
+
76
+ it "can scrape an HTML provided as a string" do
77
+ Sunscraper.scrape_html(HTML).should include('It works!')
78
+ end
79
+
80
+ it "can scrape an URL" do
81
+ with_webserver do |port|
82
+ Sunscraper.scrape_url("http://127.0.0.1:#{port}/").should include('It works!')
83
+ end
84
+ end
85
+
86
+ it "should time out if callback is not called" do
87
+ lambda { Sunscraper.scrape_html("<!-- nothing. at least no callbacks -->", 1000) }.
88
+ should raise_exception(Sunscraper::ScrapeTimeout)
89
+ end
44
90
  end
45
91
  end
data/sunscraper.gemspec CHANGED
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "sunscraper"
6
- s.version = "1.0.0"
6
+ s.version = "1.1.0.beta1"
7
7
  s.authors = ["Peter Zotov"]
8
8
  s.email = ["whitequark@whitequark.org"]
9
- s.homepage = "http://github.com/roundlake/sunscraper"
9
+ s.homepage = "http://github.com/whitequark/sunscraper"
10
10
  s.summary = %q{A WebKit-based, JavaScript-capable HTML scraper.}
11
11
  s.description = s.summary
12
12
 
metadata CHANGED
@@ -1,120 +1,103 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: sunscraper
3
- version: !ruby/object:Gem::Version
4
- hash: 540260530
5
- prerelease:
6
- segments:
7
- - 1
8
- - 0
9
- - 0
10
- version: 1.0.0
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.0.beta1
5
+ prerelease: 6
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Peter Zotov
14
9
  autorequire:
15
10
  bindir: bin
16
11
  cert_chain: []
17
-
18
- date: 2012-02-18 00:00:00 Z
19
- dependencies:
20
- - !ruby/object:Gem::Dependency
12
+ date: 2012-03-09 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
21
15
  name: rspec
22
- prerelease: false
23
- requirement: &id001 !ruby/object:Gem::Requirement
16
+ requirement: &80239320 !ruby/object:Gem::Requirement
24
17
  none: false
25
- requirements:
26
- - - ">="
27
- - !ruby/object:Gem::Version
28
- hash: 881230260
29
- segments:
30
- - 0
31
- version: "0"
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
32
22
  type: :development
33
- version_requirements: *id001
34
- - !ruby/object:Gem::Dependency
35
- name: ffi
36
23
  prerelease: false
37
- requirement: &id002 !ruby/object:Gem::Requirement
24
+ version_requirements: *80239320
25
+ - !ruby/object:Gem::Dependency
26
+ name: ffi
27
+ requirement: &80239070 !ruby/object:Gem::Requirement
38
28
  none: false
39
- requirements:
40
- - - ">="
41
- - !ruby/object:Gem::Version
42
- hash: 716237343
43
- segments:
44
- - 1
45
- - 0
46
- - 11
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
47
32
  version: 1.0.11
48
33
  type: :runtime
49
- version_requirements: *id002
34
+ prerelease: false
35
+ version_requirements: *80239070
50
36
  description: A WebKit-based, JavaScript-capable HTML scraper.
51
- email:
37
+ email:
52
38
  - whitequark@whitequark.org
53
39
  executables: []
54
-
55
- extensions:
40
+ extensions:
56
41
  - ext/extconf.rb
57
42
  extra_rdoc_files: []
58
-
59
- files:
43
+ files:
60
44
  - .gitignore
61
45
  - .rspec
46
+ - .travis.yml
62
47
  - .yardopts
63
48
  - Gemfile
64
49
  - LICENSE
65
50
  - README.md
66
51
  - Rakefile
67
52
  - ext/.gitignore
68
- - ext/Makefile
53
+ - ext/embed/sunscraper.cpp
54
+ - ext/embed/sunscraper.h
55
+ - ext/embed/sunscraper.pro
56
+ - ext/embed/sunscraperexternal.cpp
57
+ - ext/embed/sunscraperlibrary.cpp
58
+ - ext/embed/sunscraperlibrary.h
59
+ - ext/embed/sunscraperproxy.cpp
60
+ - ext/embed/sunscraperproxy.h
61
+ - ext/embed/sunscraperthread.cpp
62
+ - ext/embed/sunscraperthread.h
69
63
  - ext/extconf.rb
70
- - ext/sunscraper.cpp
71
- - ext/sunscraper.h
72
- - ext/sunscraper.pro
73
- - ext/sunscraperexternal.cpp
74
- - ext/sunscraperlibrary.cpp
75
- - ext/sunscraperlibrary.h
76
- - ext/sunscraperproxy.cpp
77
- - ext/sunscraperproxy.h
78
- - ext/sunscraperthread.cpp
79
- - ext/sunscraperthread.h
64
+ - ext/standalone/sunscraper.pro
65
+ - ext/standalone/sunscrapermain.cpp
66
+ - ext/standalone/sunscraperproxy.cpp
67
+ - ext/standalone/sunscraperproxy.h
68
+ - ext/standalone/sunscraperrpc.cpp
69
+ - ext/standalone/sunscraperrpc.h
70
+ - ext/standalone/sunscraperworker.cpp
71
+ - ext/standalone/sunscraperworker.h
80
72
  - lib/sunscraper.rb
81
73
  - lib/sunscraper/library.rb
74
+ - lib/sunscraper/standalone.rb
82
75
  - spec/spec_helper.rb
83
76
  - spec/sunscraper_spec.rb
84
77
  - sunscraper.gemspec
85
- homepage: http://github.com/roundlake/sunscraper
78
+ homepage: http://github.com/whitequark/sunscraper
86
79
  licenses: []
87
-
88
80
  post_install_message:
89
81
  rdoc_options: []
90
-
91
- require_paths:
82
+ require_paths:
92
83
  - lib
93
- required_ruby_version: !ruby/object:Gem::Requirement
84
+ required_ruby_version: !ruby/object:Gem::Requirement
94
85
  none: false
95
- requirements:
96
- - - ">="
97
- - !ruby/object:Gem::Version
98
- hash: 881230260
99
- segments:
100
- - 0
101
- version: "0"
102
- required_rubygems_version: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ! '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ required_rubygems_version: !ruby/object:Gem::Requirement
103
91
  none: false
104
- requirements:
105
- - - ">="
106
- - !ruby/object:Gem::Version
107
- hash: 881230260
108
- segments:
109
- - 0
110
- version: "0"
92
+ requirements:
93
+ - - ! '>'
94
+ - !ruby/object:Gem::Version
95
+ version: 1.3.1
111
96
  requirements: []
112
-
113
97
  rubyforge_project: sunscraper
114
- rubygems_version: 1.8.12
98
+ rubygems_version: 1.8.17
115
99
  signing_key:
116
100
  specification_version: 3
117
101
  summary: A WebKit-based, JavaScript-capable HTML scraper.
118
102
  test_files: []
119
-
120
103
  has_rdoc:
data/ext/Makefile DELETED
@@ -1,270 +0,0 @@
1
- #############################################################################
2
- # Makefile for building: libsunscraper.so.1.0.0
3
- # Generated by qmake (2.01a) (Qt 4.7.4) on: Sat Feb 18 05:54:02 2012
4
- # Project: sunscraper.pro
5
- # Template: lib
6
- # Command: /usr/bin/qmake-qt4 -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
7
- #############################################################################
8
-
9
- ####### Compiler, tools and options
10
-
11
- CC = gcc
12
- CXX = g++
13
- DEFINES = -DQT_WEBKIT -DQT_WEBKIT_LIB -DQT_GUI_LIB -DQT_CORE_LIB -DQT_SHARED
14
- CFLAGS = -pipe -g -Wall -W -D_REENTRANT -fPIC $(DEFINES)
15
- CXXFLAGS = -pipe -g -Wall -W -D_REENTRANT -fPIC $(DEFINES)
16
- INCPATH = -I/usr/share/qt4/mkspecs/linux-g++ -I. -I/usr/include/qt4/QtCore -I/usr/include/qt4/QtGui -I/usr/include/qt4/QtWebKit -I/usr/include/qt4 -I.
17
- LINK = g++
18
- LFLAGS = -shared -Wl,-soname,libsunscraper.so.1
19
- LIBS = $(SUBLIBS) -L/usr/lib -lQtWebKit -lQtGui -lQtCore -lpthread
20
- AR = ar cqs
21
- RANLIB =
22
- QMAKE = /usr/bin/qmake-qt4
23
- TAR = tar -cf
24
- COMPRESS = gzip -9f
25
- COPY = cp -f
26
- SED = sed
27
- COPY_FILE = $(COPY)
28
- COPY_DIR = $(COPY) -r
29
- STRIP = strip
30
- INSTALL_FILE = install -m 644 -p
31
- INSTALL_DIR = $(COPY_DIR)
32
- INSTALL_PROGRAM = install -m 755 -p
33
- DEL_FILE = rm -f
34
- SYMLINK = ln -f -s
35
- DEL_DIR = rmdir
36
- MOVE = mv -f
37
- CHK_DIR_EXISTS= test -d
38
- MKDIR = mkdir -p
39
-
40
- ####### Output directory
41
-
42
- OBJECTS_DIR = ./
43
-
44
- ####### Files
45
-
46
- SOURCES = sunscraperlibrary.cpp \
47
- sunscraperthread.cpp \
48
- sunscraperexternal.cpp \
49
- sunscraper.cpp \
50
- sunscraperproxy.cpp moc_sunscraperthread.cpp \
51
- moc_sunscraper.cpp \
52
- moc_sunscraperproxy.cpp
53
- OBJECTS = sunscraperlibrary.o \
54
- sunscraperthread.o \
55
- sunscraperexternal.o \
56
- sunscraper.o \
57
- sunscraperproxy.o \
58
- moc_sunscraperthread.o \
59
- moc_sunscraper.o \
60
- moc_sunscraperproxy.o
61
- DIST = /usr/share/qt4/mkspecs/common/g++.conf \
62
- /usr/share/qt4/mkspecs/common/unix.conf \
63
- /usr/share/qt4/mkspecs/common/linux.conf \
64
- /usr/share/qt4/mkspecs/qconfig.pri \
65
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
66
- /usr/share/qt4/mkspecs/features/qt_functions.prf \
67
- /usr/share/qt4/mkspecs/features/qt_config.prf \
68
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf \
69
- /usr/share/qt4/mkspecs/features/default_pre.prf \
70
- /usr/share/qt4/mkspecs/features/debug.prf \
71
- /usr/share/qt4/mkspecs/features/default_post.prf \
72
- /usr/share/qt4/mkspecs/features/warn_on.prf \
73
- /usr/share/qt4/mkspecs/features/qt.prf \
74
- /usr/share/qt4/mkspecs/features/unix/thread.prf \
75
- /usr/share/qt4/mkspecs/features/moc.prf \
76
- /usr/share/qt4/mkspecs/features/resources.prf \
77
- /usr/share/qt4/mkspecs/features/uic.prf \
78
- /usr/share/qt4/mkspecs/features/yacc.prf \
79
- /usr/share/qt4/mkspecs/features/lex.prf \
80
- /usr/share/qt4/mkspecs/features/include_source_dir.prf \
81
- sunscraper.pro
82
- QMAKE_TARGET = sunscraper
83
- DESTDIR =
84
- TARGET = libsunscraper.so.1.0.0
85
- TARGETA = libsunscraper.a
86
- TARGETD = libsunscraper.so.1.0.0
87
- TARGET0 = libsunscraper.so
88
- TARGET1 = libsunscraper.so.1
89
- TARGET2 = libsunscraper.so.1.0
90
-
91
- first: all
92
- ####### Implicit rules
93
-
94
- .SUFFIXES: .o .c .cpp .cc .cxx .C
95
-
96
- .cpp.o:
97
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
98
-
99
- .cc.o:
100
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
101
-
102
- .cxx.o:
103
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
104
-
105
- .C.o:
106
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
107
-
108
- .c.o:
109
- $(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<"
110
-
111
- ####### Build rules
112
-
113
- all: Makefile $(TARGET)
114
-
115
- $(TARGET): $(OBJECTS) $(SUBLIBS) $(OBJCOMP)
116
- -$(DEL_FILE) $(TARGET) $(TARGET0) $(TARGET1) $(TARGET2)
117
- $(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(LIBS) $(OBJCOMP)
118
- -ln -s $(TARGET) $(TARGET0)
119
- -ln -s $(TARGET) $(TARGET1)
120
- -ln -s $(TARGET) $(TARGET2)
121
-
122
-
123
-
124
- staticlib: $(TARGETA)
125
-
126
- $(TARGETA): $(OBJECTS) $(OBJCOMP)
127
- -$(DEL_FILE) $(TARGETA)
128
- $(AR) $(TARGETA) $(OBJECTS)
129
-
130
- Makefile: sunscraper.pro /usr/share/qt4/mkspecs/linux-g++/qmake.conf /usr/share/qt4/mkspecs/common/g++.conf \
131
- /usr/share/qt4/mkspecs/common/unix.conf \
132
- /usr/share/qt4/mkspecs/common/linux.conf \
133
- /usr/share/qt4/mkspecs/qconfig.pri \
134
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
135
- /usr/share/qt4/mkspecs/features/qt_functions.prf \
136
- /usr/share/qt4/mkspecs/features/qt_config.prf \
137
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf \
138
- /usr/share/qt4/mkspecs/features/default_pre.prf \
139
- /usr/share/qt4/mkspecs/features/debug.prf \
140
- /usr/share/qt4/mkspecs/features/default_post.prf \
141
- /usr/share/qt4/mkspecs/features/warn_on.prf \
142
- /usr/share/qt4/mkspecs/features/qt.prf \
143
- /usr/share/qt4/mkspecs/features/unix/thread.prf \
144
- /usr/share/qt4/mkspecs/features/moc.prf \
145
- /usr/share/qt4/mkspecs/features/resources.prf \
146
- /usr/share/qt4/mkspecs/features/uic.prf \
147
- /usr/share/qt4/mkspecs/features/yacc.prf \
148
- /usr/share/qt4/mkspecs/features/lex.prf \
149
- /usr/share/qt4/mkspecs/features/include_source_dir.prf \
150
- /usr/lib/libQtWebKit.prl \
151
- /usr/lib/libQtGui.prl \
152
- /usr/lib/libQtCore.prl
153
- $(QMAKE) -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
154
- /usr/share/qt4/mkspecs/common/g++.conf:
155
- /usr/share/qt4/mkspecs/common/unix.conf:
156
- /usr/share/qt4/mkspecs/common/linux.conf:
157
- /usr/share/qt4/mkspecs/qconfig.pri:
158
- /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri:
159
- /usr/share/qt4/mkspecs/features/qt_functions.prf:
160
- /usr/share/qt4/mkspecs/features/qt_config.prf:
161
- /usr/share/qt4/mkspecs/features/exclusive_builds.prf:
162
- /usr/share/qt4/mkspecs/features/default_pre.prf:
163
- /usr/share/qt4/mkspecs/features/debug.prf:
164
- /usr/share/qt4/mkspecs/features/default_post.prf:
165
- /usr/share/qt4/mkspecs/features/warn_on.prf:
166
- /usr/share/qt4/mkspecs/features/qt.prf:
167
- /usr/share/qt4/mkspecs/features/unix/thread.prf:
168
- /usr/share/qt4/mkspecs/features/moc.prf:
169
- /usr/share/qt4/mkspecs/features/resources.prf:
170
- /usr/share/qt4/mkspecs/features/uic.prf:
171
- /usr/share/qt4/mkspecs/features/yacc.prf:
172
- /usr/share/qt4/mkspecs/features/lex.prf:
173
- /usr/share/qt4/mkspecs/features/include_source_dir.prf:
174
- /usr/lib/libQtWebKit.prl:
175
- /usr/lib/libQtGui.prl:
176
- /usr/lib/libQtCore.prl:
177
- qmake: FORCE
178
- @$(QMAKE) -spec /usr/share/qt4/mkspecs/linux-g++ CONFIG+=debug -o Makefile sunscraper.pro
179
-
180
- dist:
181
- @$(CHK_DIR_EXISTS) .tmp/sunscraper1.0.0 || $(MKDIR) .tmp/sunscraper1.0.0
182
- $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/sunscraper1.0.0/ && $(COPY_FILE) --parents sunscraperlibrary.h sunscraperthread.h sunscraper.h sunscraperproxy.h .tmp/sunscraper1.0.0/ && $(COPY_FILE) --parents sunscraperlibrary.cpp sunscraperthread.cpp sunscraperexternal.cpp sunscraper.cpp sunscraperproxy.cpp .tmp/sunscraper1.0.0/ && (cd `dirname .tmp/sunscraper1.0.0` && $(TAR) sunscraper1.0.0.tar sunscraper1.0.0 && $(COMPRESS) sunscraper1.0.0.tar) && $(MOVE) `dirname .tmp/sunscraper1.0.0`/sunscraper1.0.0.tar.gz . && $(DEL_FILE) -r .tmp/sunscraper1.0.0
183
-
184
-
185
- clean:compiler_clean
186
- -$(DEL_FILE) $(OBJECTS)
187
- -$(DEL_FILE) *~ core *.core
188
-
189
-
190
- ####### Sub-libraries
191
-
192
- distclean: clean
193
- -$(DEL_FILE) $(TARGET)
194
- -$(DEL_FILE) $(TARGET0) $(TARGET1) $(TARGET2) $(TARGETA)
195
- -$(DEL_FILE) Makefile
196
-
197
-
198
- check: first
199
-
200
- mocclean: compiler_moc_header_clean compiler_moc_source_clean
201
-
202
- mocables: compiler_moc_header_make_all compiler_moc_source_make_all
203
-
204
- compiler_moc_header_make_all: moc_sunscraperthread.cpp moc_sunscraper.cpp moc_sunscraperproxy.cpp
205
- compiler_moc_header_clean:
206
- -$(DEL_FILE) moc_sunscraperthread.cpp moc_sunscraper.cpp moc_sunscraperproxy.cpp
207
- moc_sunscraperthread.cpp: sunscraperthread.h
208
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraperthread.h -o moc_sunscraperthread.cpp
209
-
210
- moc_sunscraper.cpp: sunscraper.h
211
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraper.h -o moc_sunscraper.cpp
212
-
213
- moc_sunscraperproxy.cpp: sunscraperproxy.h
214
- /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) sunscraperproxy.h -o moc_sunscraperproxy.cpp
215
-
216
- compiler_rcc_make_all:
217
- compiler_rcc_clean:
218
- compiler_image_collection_make_all: qmake_image_collection.cpp
219
- compiler_image_collection_clean:
220
- -$(DEL_FILE) qmake_image_collection.cpp
221
- compiler_moc_source_make_all:
222
- compiler_moc_source_clean:
223
- compiler_uic_make_all:
224
- compiler_uic_clean:
225
- compiler_yacc_decl_make_all:
226
- compiler_yacc_decl_clean:
227
- compiler_yacc_impl_make_all:
228
- compiler_yacc_impl_clean:
229
- compiler_lex_make_all:
230
- compiler_lex_clean:
231
- compiler_clean: compiler_moc_header_clean
232
-
233
- ####### Compile
234
-
235
- sunscraperlibrary.o: sunscraperlibrary.cpp sunscraperlibrary.h \
236
- sunscraperthread.h
237
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperlibrary.o sunscraperlibrary.cpp
238
-
239
- sunscraperthread.o: sunscraperthread.cpp sunscraperthread.h \
240
- sunscraperproxy.h
241
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperthread.o sunscraperthread.cpp
242
-
243
- sunscraperexternal.o: sunscraperexternal.cpp sunscraper.h
244
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperexternal.o sunscraperexternal.cpp
245
-
246
- sunscraper.o: sunscraper.cpp sunscraper.h \
247
- sunscraperlibrary.h \
248
- sunscraperthread.h
249
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraper.o sunscraper.cpp
250
-
251
- sunscraperproxy.o: sunscraperproxy.cpp sunscraperproxy.h
252
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o sunscraperproxy.o sunscraperproxy.cpp
253
-
254
- moc_sunscraperthread.o: moc_sunscraperthread.cpp
255
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraperthread.o moc_sunscraperthread.cpp
256
-
257
- moc_sunscraper.o: moc_sunscraper.cpp
258
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraper.o moc_sunscraper.cpp
259
-
260
- moc_sunscraperproxy.o: moc_sunscraperproxy.cpp
261
- $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_sunscraperproxy.o moc_sunscraperproxy.cpp
262
-
263
- ####### Install
264
-
265
- install: FORCE
266
-
267
- uninstall: FORCE
268
-
269
- FORCE:
270
-
data/ext/sunscraper.cpp DELETED
@@ -1,86 +0,0 @@
1
- #include <QApplication>
2
- #include <QThread>
3
- #include <QTimer>
4
- #include <QWebPage>
5
- #include <QWebFrame>
6
- #include <QMutexLocker>
7
- #include <QtDebug>
8
- #include "sunscraper.h"
9
- #include "sunscraperlibrary.h"
10
- #include "sunscraperthread.h"
11
-
12
- unsigned Sunscraper::_nextQueryId = 1;
13
- QMutex Sunscraper::_staticMutex;
14
-
15
- Sunscraper::Sunscraper()
16
- {
17
- QMutexLocker locker(&_staticMutex);
18
-
19
- _queryId = _nextQueryId++;
20
-
21
- SunscraperThread *worker = SunscraperLibrary::instance()->thread();
22
-
23
- connect(this, SIGNAL(requestLoadHtml(uint,QString)),
24
- worker, SLOT(loadHtml(uint,QString)), Qt::QueuedConnection);
25
- connect(this, SIGNAL(requestLoadUrl(uint,QString)),
26
- worker, SLOT(loadUrl(uint,QString)), Qt::QueuedConnection);
27
- connect(this, SIGNAL(requestFinalize(uint)),
28
- worker, SLOT(finalize(uint)), Qt::QueuedConnection);
29
-
30
- connect(worker, SIGNAL(finished(uint,QString)),
31
- this, SLOT(finished(uint,QString)), Qt::QueuedConnection);
32
- }
33
-
34
- void Sunscraper::loadHtml(QString html)
35
- {
36
- emit requestLoadHtml(_queryId, html);
37
- }
38
-
39
- void Sunscraper::loadUrl(QString url)
40
- {
41
- emit requestLoadUrl(_queryId, url);
42
- }
43
-
44
- void Sunscraper::wait(unsigned timeout)
45
- {
46
- QTimer _timeoutTimer;
47
- connect(&_timeoutTimer, SIGNAL(timeout()), this, SLOT(timeout()));
48
-
49
- _timeoutTimer.setInterval(timeout);
50
- _timeoutTimer.start();
51
-
52
- _eventLoop.exec();
53
-
54
- _timeoutTimer.stop();
55
- }
56
-
57
- void Sunscraper::finished(unsigned eventQueryId, QString html)
58
- {
59
- if(eventQueryId != _queryId)
60
- return;
61
-
62
- _eventLoop.quit();
63
-
64
- _html = html.toUtf8();
65
-
66
- emit requestFinalize(_queryId);
67
- }
68
-
69
- void Sunscraper::timeout()
70
- {
71
- _eventLoop.quit();
72
-
73
- _html = "!SUNSCRAPER_TIMEOUT";
74
-
75
- emit requestFinalize(_queryId);
76
- }
77
-
78
- QByteArray Sunscraper::fetch()
79
- {
80
- return _html;
81
- }
82
-
83
- const char *Sunscraper::fetchAsCString()
84
- {
85
- return _html.constData();
86
- }
data/ext/sunscraperexternal.cpp DELETED
@@ -1,33 +0,0 @@
1
- #include "sunscraper.h"
2
-
3
- extern "C" {
4
- Sunscraper *sunscraper_create()
5
- {
6
- return new Sunscraper();
7
- }
8
-
9
- void sunscraper_load_html(Sunscraper *sunscraper, const char *html)
10
- {
11
- sunscraper->loadHtml(html);
12
- }
13
-
14
- void sunscraper_load_url(Sunscraper *sunscraper, const char *url)
15
- {
16
- sunscraper->loadUrl(url);
17
- }
18
-
19
- void sunscraper_wait(Sunscraper *sunscraper, unsigned timeout)
20
- {
21
- sunscraper->wait(timeout);
22
- }
23
-
24
- const char *sunscraper_fetch(Sunscraper *sunscraper)
25
- {
26
- return sunscraper->fetchAsCString();
27
- }
28
-
29
- void sunscraper_discard(Sunscraper *sunscraper)
30
- {
31
- delete sunscraper;
32
- }
33
- }