semantic-crawler 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.rdoc +25 -0
- data/README.rdoc +44 -12
- data/Rakefile +9 -1
- data/lib/semantic_crawler.rb +26 -10
- data/lib/semantic_crawler/dbpedia.rb +4 -0
- data/lib/semantic_crawler/factbook/country.rb +13 -9
- data/lib/semantic_crawler/fao.rb +6 -0
- data/lib/semantic_crawler/fao/country.rb +198 -0
- data/lib/semantic_crawler/freebase/country.rb +43 -32
- data/lib/semantic_crawler/gdacs.rb +1 -1
- data/lib/semantic_crawler/gdacs/emergency_feed.rb +67 -0
- data/lib/semantic_crawler/gdacs/emergency_feed_item.rb +55 -0
- data/lib/semantic_crawler/gdacs/feed.rb +2 -1
- data/lib/semantic_crawler/gdacs/resource.rb +0 -4
- data/lib/semantic_crawler/geo_names.rb +4 -0
- data/lib/semantic_crawler/version.rb +1 -1
- data/lib/tasks/semantic_crawler_tasks.rake +4 -4
- data/log/semantic-crawler.log +31 -0
- data/test/dummy/log/test.log +753 -0
- data/test/semantic_crawler_test.rb +0 -19
- metadata +167 -115
- data/lib/semantic_crawler/freebase/crawler.rb +0 -22
- data/lib/semantic_crawler/freebase/entity.rb +0 -4
- data/test/factbook_test.rb +0 -86
- data/test/gdacs_test.rb +0 -80
@@ -5,23 +5,4 @@ class SemanticCrawlerTest < ActiveSupport::TestCase
|
|
5
5
|
test "truth" do
|
6
6
|
assert_kind_of Module, SemanticCrawler
|
7
7
|
end
|
8
|
-
|
9
|
-
test "init austria country information" do
|
10
|
-
austria = SemanticCrawler::Freebase::Crawler.new
|
11
|
-
countryObj = austria.getCountry("Austria")
|
12
|
-
countryNameNormalized = countryObj.input_name
|
13
|
-
assert countryNameNormalized.eql?("austria")
|
14
|
-
end
|
15
|
-
|
16
|
-
test "get austria country information" do
|
17
|
-
austria = SemanticCrawler::Freebase::Crawler.new
|
18
|
-
countryObj = austria.getCountry("Austria")
|
19
|
-
#pp countryObj.get["properties"]["/location/statistical_region/population"]
|
20
|
-
end
|
21
|
-
|
22
|
-
test "get empty crawler object" do
|
23
|
-
empty = SemanticCrawler::Freebase::Crawler.new
|
24
|
-
assert !empty.nil?
|
25
|
-
end
|
26
|
-
|
27
8
|
end
|
metadata
CHANGED
@@ -1,139 +1,195 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: semantic-crawler
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5
|
5
5
|
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 0
|
9
|
-
- 3
|
10
|
-
version: 0.0.3
|
11
6
|
platform: ruby
|
12
|
-
authors:
|
7
|
+
authors:
|
13
8
|
- Alex Oberhauser
|
14
9
|
autorequire:
|
15
10
|
bindir: bin
|
16
11
|
cert_chain: []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
- !ruby/object:Gem::Dependency
|
21
|
-
type: :runtime
|
12
|
+
date: 2012-04-04 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
22
15
|
name: httparty
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
hash: 3
|
30
|
-
segments:
|
31
|
-
- 0
|
32
|
-
version: "0"
|
33
|
-
requirement: *id001
|
34
|
-
- !ruby/object:Gem::Dependency
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
35
22
|
type: :runtime
|
36
|
-
name: json
|
37
23
|
prerelease: false
|
38
|
-
version_requirements:
|
39
|
-
none: false
|
40
|
-
requirements:
|
41
|
-
- -
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: json
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
49
38
|
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
50
47
|
name: nokogiri
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
51
55
|
prerelease: false
|
52
|
-
version_requirements:
|
53
|
-
none: false
|
54
|
-
requirements:
|
55
|
-
- -
|
56
|
-
- !ruby/object:Gem::Version
|
57
|
-
|
58
|
-
|
59
|
-
- 0
|
60
|
-
version: "0"
|
61
|
-
requirement: *id003
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
type: :development
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
64
63
|
name: yard
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
hash: 3
|
72
|
-
segments:
|
73
|
-
- 0
|
74
|
-
version: "0"
|
75
|
-
requirement: *id004
|
76
|
-
- !ruby/object:Gem::Dependency
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
77
70
|
type: :development
|
78
|
-
name: rails
|
79
71
|
prerelease: false
|
80
|
-
version_requirements:
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rails
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
81
|
none: false
|
82
|
-
requirements:
|
82
|
+
requirements:
|
83
83
|
- - ~>
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
hash: 11
|
86
|
-
segments:
|
87
|
-
- 3
|
88
|
-
- 2
|
89
|
-
- 2
|
84
|
+
- !ruby/object:Gem::Version
|
90
85
|
version: 3.2.2
|
91
|
-
requirement: *id005
|
92
|
-
- !ruby/object:Gem::Dependency
|
93
86
|
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 3.2.2
|
94
|
+
- !ruby/object:Gem::Dependency
|
94
95
|
name: sqlite3
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: rspec-rails
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
95
119
|
prerelease: false
|
96
|
-
version_requirements:
|
97
|
-
none: false
|
98
|
-
requirements:
|
99
|
-
- -
|
100
|
-
- !ruby/object:Gem::Version
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: simplecov
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: simplecov-rcov
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
description: SemanticCrawler is a ruby library that encapsulates data gathering from
|
159
|
+
different sources. Currently country information from Factbook and FAO (Food and
|
160
|
+
Agriculture Organization of the United Nations) and crisis information from GDACS.org
|
161
|
+
are supported.
|
162
|
+
email:
|
108
163
|
- alex.oberhauser@sigimera.org
|
109
164
|
executables: []
|
110
|
-
|
111
165
|
extensions: []
|
112
|
-
|
113
166
|
extra_rdoc_files: []
|
114
|
-
|
115
|
-
files:
|
167
|
+
files:
|
116
168
|
- lib/semantic_crawler.rb
|
117
169
|
- lib/tasks/semantic_crawler_tasks.rake
|
118
170
|
- lib/semantic_crawler/freebase/country.rb
|
119
|
-
- lib/semantic_crawler/freebase/crawler.rb
|
120
|
-
- lib/semantic_crawler/freebase/entity.rb
|
121
171
|
- lib/semantic_crawler/factbook/country.rb
|
172
|
+
- lib/semantic_crawler/fao.rb
|
173
|
+
- lib/semantic_crawler/dbpedia.rb
|
174
|
+
- lib/semantic_crawler/fao/country.rb
|
122
175
|
- lib/semantic_crawler/freebase.rb
|
123
176
|
- lib/semantic_crawler/factbook.rb
|
124
177
|
- lib/semantic_crawler/gdacs.rb
|
125
178
|
- lib/semantic_crawler/version.rb
|
126
179
|
- lib/semantic_crawler/gdacs/resource.rb
|
180
|
+
- lib/semantic_crawler/gdacs/emergency_feed.rb
|
127
181
|
- lib/semantic_crawler/gdacs/feed.rb
|
182
|
+
- lib/semantic_crawler/gdacs/emergency_feed_item.rb
|
128
183
|
- lib/semantic_crawler/gdacs/feed_item.rb
|
184
|
+
- lib/semantic_crawler/geo_names.rb
|
129
185
|
- lib/semantic_crawler/linked_geo_data/relevant_node.rb
|
130
186
|
- lib/semantic_crawler/linked_geo_data.rb
|
187
|
+
- log/semantic-crawler.log
|
131
188
|
- MIT-LICENSE
|
132
189
|
- Rakefile
|
133
190
|
- README.rdoc
|
134
|
-
-
|
191
|
+
- CHANGELOG.rdoc
|
135
192
|
- test/test_helper.rb
|
136
|
-
- test/factbook_test.rb
|
137
193
|
- test/semantic_crawler_test.rb
|
138
194
|
- test/dummy/log/test.log
|
139
195
|
- test/dummy/log/development.log
|
@@ -169,41 +225,37 @@ files:
|
|
169
225
|
- test/dummy/app/views/layouts/application.html.erb
|
170
226
|
homepage: https://github.com/obale/semantic_crawler
|
171
227
|
licenses: []
|
172
|
-
|
173
228
|
post_install_message:
|
174
229
|
rdoc_options: []
|
175
|
-
|
176
|
-
require_paths:
|
230
|
+
require_paths:
|
177
231
|
- lib
|
178
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
232
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
179
233
|
none: false
|
180
|
-
requirements:
|
181
|
-
- -
|
182
|
-
- !ruby/object:Gem::Version
|
183
|
-
|
184
|
-
segments:
|
234
|
+
requirements:
|
235
|
+
- - ! '>='
|
236
|
+
- !ruby/object:Gem::Version
|
237
|
+
version: '0'
|
238
|
+
segments:
|
185
239
|
- 0
|
186
|
-
|
187
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
240
|
+
hash: -1761754214109537766
|
241
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
188
242
|
none: false
|
189
|
-
requirements:
|
190
|
-
- -
|
191
|
-
- !ruby/object:Gem::Version
|
192
|
-
|
193
|
-
segments:
|
243
|
+
requirements:
|
244
|
+
- - ! '>='
|
245
|
+
- !ruby/object:Gem::Version
|
246
|
+
version: '0'
|
247
|
+
segments:
|
194
248
|
- 0
|
195
|
-
|
249
|
+
hash: -1761754214109537766
|
196
250
|
requirements: []
|
197
|
-
|
198
251
|
rubyforge_project:
|
199
252
|
rubygems_version: 1.8.21
|
200
253
|
signing_key:
|
201
254
|
specification_version: 3
|
202
|
-
summary: SemanticCrawler is a ruby library that encapsulates data gathering from different
|
203
|
-
|
204
|
-
|
255
|
+
summary: SemanticCrawler is a ruby library that encapsulates data gathering from different
|
256
|
+
sources.
|
257
|
+
test_files:
|
205
258
|
- test/test_helper.rb
|
206
|
-
- test/factbook_test.rb
|
207
259
|
- test/semantic_crawler_test.rb
|
208
260
|
- test/dummy/log/test.log
|
209
261
|
- test/dummy/log/development.log
|
@@ -1,22 +0,0 @@
|
|
1
|
-
module SemanticCrawler
|
2
|
-
module Freebase
|
3
|
-
# [XXX] This class should be deleted. The country object could be called directly.
|
4
|
-
class Crawler
|
5
|
-
# Returns a country object that contains all relevant
|
6
|
-
# information that could be extracted from freebase.com
|
7
|
-
#
|
8
|
-
# Example:
|
9
|
-
# >> freebaseCrawler = SemanticCrawler::Freebase::Crawler.new
|
10
|
-
# >> austria = freebaseCrawler.getCountry("Austria")
|
11
|
-
#
|
12
|
-
# Arguments:
|
13
|
-
# countryName: (String)
|
14
|
-
#
|
15
|
-
def getCountry(countryName)
|
16
|
-
country = SemanticCrawler::Freebase::Country.new
|
17
|
-
country.input_name = countryName.downcase!
|
18
|
-
country
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
data/test/factbook_test.rb
DELETED
@@ -1,86 +0,0 @@
|
|
1
|
-
require 'test_helper'
|
2
|
-
require 'pp'
|
3
|
-
|
4
|
-
class FactbookTest < ActiveSupport::TestCase
|
5
|
-
test "truth" do
|
6
|
-
assert_kind_of Module, SemanticCrawler
|
7
|
-
end
|
8
|
-
|
9
|
-
test "init austria country information" do
|
10
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
11
|
-
assert austria.country_name.eql?("Austria")
|
12
|
-
assert austria.url.eql?("http://www4.wiwiss.fu-berlin.de/factbook/data/austria")
|
13
|
-
end
|
14
|
-
|
15
|
-
test "get austria name" do
|
16
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
17
|
-
value = austria.name
|
18
|
-
assert value.to_s.eql?("Austria")
|
19
|
-
end
|
20
|
-
|
21
|
-
test "get austria background information" do
|
22
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
23
|
-
value = austria.background
|
24
|
-
assert !value.nil?
|
25
|
-
end
|
26
|
-
|
27
|
-
test "get austria population information" do
|
28
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
29
|
-
value = austria.population_total
|
30
|
-
assert !value.nil?
|
31
|
-
end
|
32
|
-
|
33
|
-
test "get austria legalsystem property (not wrapped)" do
|
34
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
35
|
-
value = austria.get_factbook_property("legalsystem")
|
36
|
-
assert !value.nil?
|
37
|
-
end
|
38
|
-
|
39
|
-
test "get austria latitude" do
|
40
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
41
|
-
value = austria.latitude
|
42
|
-
assert value.to_s.eql?("47.3333333333333")
|
43
|
-
end
|
44
|
-
|
45
|
-
test "get austria longitude" do
|
46
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
47
|
-
value = austria.longitude
|
48
|
-
assert value.to_s.eql?("13.3333333333333")
|
49
|
-
end
|
50
|
-
|
51
|
-
test "get austria landboundary" do
|
52
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
53
|
-
value = austria.landboundary
|
54
|
-
assert value.count == 7
|
55
|
-
end
|
56
|
-
|
57
|
-
test "get austria terrain" do
|
58
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
59
|
-
value = austria.terrain
|
60
|
-
assert !value.nil?
|
61
|
-
end
|
62
|
-
|
63
|
-
test "get austria total airports" do
|
64
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
65
|
-
value = austria.airports_total
|
66
|
-
assert value.to_s.eql?("55")
|
67
|
-
end
|
68
|
-
|
69
|
-
test "get austria heliports" do
|
70
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
71
|
-
value = austria.heliports
|
72
|
-
assert value.to_s.eql?("1")
|
73
|
-
end
|
74
|
-
|
75
|
-
test "get austria climate" do
|
76
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
77
|
-
value = austria.climate
|
78
|
-
assert !value.nil?
|
79
|
-
end
|
80
|
-
|
81
|
-
test "get austria location description" do
|
82
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
83
|
-
value = austria.location
|
84
|
-
assert !value.nil?
|
85
|
-
end
|
86
|
-
end
|