Dynamised 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/dynamised +1 -1
- data/lib/dynamised/meta.rb +1 -1
- data/lib/dynamised/scraper.rb +8 -3
- data/lib/dynamised.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 85306d1ca4783b76d31d8d80aa9191bab45c89cf
|
4
|
+
data.tar.gz: 379d655d95f1d7785c674b2cdbd06b55e3fdd6c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dfb74739fa85cfd033791dd8058173eff8bfbbe9e4b53ecf168104aa0dbf01f2dd21b88f43bb4d2774b6a72cd22b96dfb7e4b3b133ed867862814b40d90f6380
|
7
|
+
data.tar.gz: f0f0345648923fbb280a858e1fa6b386c549c918dcbdba0f2ee014d56c149761771778496efeef72bac6d4f13f1bb640ab41176f8edb9f519b0d8d1381d49bbe
|
data/bin/dynamised
CHANGED
@@ -21,7 +21,7 @@ module Dynamised
|
|
21
21
|
class_name = get_class_name(args.first)
|
22
22
|
create_temp_class(class_name,File.read(script_path))
|
23
23
|
class_ref = Scraper.fetch(class_name)
|
24
|
-
spinner = TTY::Spinner.new("[:spinner] scraping with %s" %
|
24
|
+
spinner = TTY::Spinner.new("[:spinner] scraping with %s" % class_name)
|
25
25
|
class_ref.new.pull_and_store do
|
26
26
|
spinner.spin
|
27
27
|
end
|
data/lib/dynamised/meta.rb
CHANGED
data/lib/dynamised/scraper.rb
CHANGED
@@ -40,7 +40,7 @@ module Dynamised
|
|
40
40
|
@args = args
|
41
41
|
@tree_pointer = []
|
42
42
|
@use_store = false
|
43
|
-
@scraped_data = DBM_Wrapper.new("%s_scraped_data" % self.class.to_s)
|
43
|
+
@scraped_data = DBM_Wrapper.new("%s_scraped_data" % get_class_name(self.class.to_s))
|
44
44
|
[:inc,:uri,:tree,:tree_pointer,:base_url,:writer].each do |attr|
|
45
45
|
varb_name = "@%s" % attr
|
46
46
|
self.instance_variable_set(varb_name,self.class.instance_variable_get(varb_name))
|
@@ -70,8 +70,12 @@ module Dynamised
|
|
70
70
|
|
71
71
|
private
|
72
72
|
|
73
|
+
def get_class_name(string)
|
74
|
+
string.split('::').last.split('.').first.gsub(/ /,'_').capitalize
|
75
|
+
end
|
76
|
+
|
73
77
|
def scrape_data(&spinner)
|
74
|
-
pull(pull_initial) do |hash|
|
78
|
+
pull(pull_initial,@tree) do |hash|
|
75
79
|
spinner.call
|
76
80
|
end
|
77
81
|
end
|
@@ -88,6 +92,7 @@ module Dynamised
|
|
88
92
|
raise '%s is a non supported writer type'
|
89
93
|
end
|
90
94
|
end
|
95
|
+
@scraped_data.stop
|
91
96
|
end
|
92
97
|
|
93
98
|
|
@@ -128,7 +133,7 @@ module Dynamised
|
|
128
133
|
res_hash[field] = value ? execute_method(data[:meta][:after],value,res_hash) : data[:meta].fetch(:default,nil)
|
129
134
|
end
|
130
135
|
@scraped_data[c_url] = fields.to_json if @use_store
|
131
|
-
block.call(
|
136
|
+
block.call(fields)
|
132
137
|
end
|
133
138
|
end
|
134
139
|
|
data/lib/dynamised.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
%w{tty-spinner nokogiri awesome_print
|
1
|
+
%w{tty-spinner nokogiri awesome_print dbm json}.each {|lib| require lib}
|
2
2
|
%w{meta after_scrape_methods before_scrape_methods curb_dsl helpers node scraper_dsl writers dbm_wrapper scraper}
|
3
3
|
.each do |f|
|
4
4
|
require_relative "dynamised/%s" % f
|