semantic-crawler 0.0.3 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.rdoc +25 -0
- data/README.rdoc +44 -12
- data/Rakefile +9 -1
- data/lib/semantic_crawler.rb +26 -10
- data/lib/semantic_crawler/dbpedia.rb +4 -0
- data/lib/semantic_crawler/factbook/country.rb +13 -9
- data/lib/semantic_crawler/fao.rb +6 -0
- data/lib/semantic_crawler/fao/country.rb +198 -0
- data/lib/semantic_crawler/freebase/country.rb +43 -32
- data/lib/semantic_crawler/gdacs.rb +1 -1
- data/lib/semantic_crawler/gdacs/emergency_feed.rb +67 -0
- data/lib/semantic_crawler/gdacs/emergency_feed_item.rb +55 -0
- data/lib/semantic_crawler/gdacs/feed.rb +2 -1
- data/lib/semantic_crawler/gdacs/resource.rb +0 -4
- data/lib/semantic_crawler/geo_names.rb +4 -0
- data/lib/semantic_crawler/version.rb +1 -1
- data/lib/tasks/semantic_crawler_tasks.rake +4 -4
- data/log/semantic-crawler.log +31 -0
- data/test/dummy/log/test.log +753 -0
- data/test/semantic_crawler_test.rb +0 -19
- metadata +167 -115
- data/lib/semantic_crawler/freebase/crawler.rb +0 -22
- data/lib/semantic_crawler/freebase/entity.rb +0 -4
- data/test/factbook_test.rb +0 -86
- data/test/gdacs_test.rb +0 -80
@@ -5,23 +5,4 @@ class SemanticCrawlerTest < ActiveSupport::TestCase
|
|
5
5
|
test "truth" do
|
6
6
|
assert_kind_of Module, SemanticCrawler
|
7
7
|
end
|
8
|
-
|
9
|
-
test "init austria country information" do
|
10
|
-
austria = SemanticCrawler::Freebase::Crawler.new
|
11
|
-
countryObj = austria.getCountry("Austria")
|
12
|
-
countryNameNormalized = countryObj.input_name
|
13
|
-
assert countryNameNormalized.eql?("austria")
|
14
|
-
end
|
15
|
-
|
16
|
-
test "get austria country information" do
|
17
|
-
austria = SemanticCrawler::Freebase::Crawler.new
|
18
|
-
countryObj = austria.getCountry("Austria")
|
19
|
-
#pp countryObj.get["properties"]["/location/statistical_region/population"]
|
20
|
-
end
|
21
|
-
|
22
|
-
test "get empty crawler object" do
|
23
|
-
empty = SemanticCrawler::Freebase::Crawler.new
|
24
|
-
assert !empty.nil?
|
25
|
-
end
|
26
|
-
|
27
8
|
end
|
metadata
CHANGED
@@ -1,139 +1,195 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: semantic-crawler
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5
|
5
5
|
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 0
|
9
|
-
- 3
|
10
|
-
version: 0.0.3
|
11
6
|
platform: ruby
|
12
|
-
authors:
|
7
|
+
authors:
|
13
8
|
- Alex Oberhauser
|
14
9
|
autorequire:
|
15
10
|
bindir: bin
|
16
11
|
cert_chain: []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
- !ruby/object:Gem::Dependency
|
21
|
-
type: :runtime
|
12
|
+
date: 2012-04-04 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
22
15
|
name: httparty
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
hash: 3
|
30
|
-
segments:
|
31
|
-
- 0
|
32
|
-
version: "0"
|
33
|
-
requirement: *id001
|
34
|
-
- !ruby/object:Gem::Dependency
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
35
22
|
type: :runtime
|
36
|
-
name: json
|
37
23
|
prerelease: false
|
38
|
-
version_requirements:
|
39
|
-
none: false
|
40
|
-
requirements:
|
41
|
-
- -
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: json
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
49
38
|
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
50
47
|
name: nokogiri
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
51
55
|
prerelease: false
|
52
|
-
version_requirements:
|
53
|
-
none: false
|
54
|
-
requirements:
|
55
|
-
- -
|
56
|
-
- !ruby/object:Gem::Version
|
57
|
-
|
58
|
-
|
59
|
-
- 0
|
60
|
-
version: "0"
|
61
|
-
requirement: *id003
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
type: :development
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
64
63
|
name: yard
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
hash: 3
|
72
|
-
segments:
|
73
|
-
- 0
|
74
|
-
version: "0"
|
75
|
-
requirement: *id004
|
76
|
-
- !ruby/object:Gem::Dependency
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
77
70
|
type: :development
|
78
|
-
name: rails
|
79
71
|
prerelease: false
|
80
|
-
version_requirements:
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rails
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
81
|
none: false
|
82
|
-
requirements:
|
82
|
+
requirements:
|
83
83
|
- - ~>
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
hash: 11
|
86
|
-
segments:
|
87
|
-
- 3
|
88
|
-
- 2
|
89
|
-
- 2
|
84
|
+
- !ruby/object:Gem::Version
|
90
85
|
version: 3.2.2
|
91
|
-
requirement: *id005
|
92
|
-
- !ruby/object:Gem::Dependency
|
93
86
|
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 3.2.2
|
94
|
+
- !ruby/object:Gem::Dependency
|
94
95
|
name: sqlite3
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: rspec-rails
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
95
119
|
prerelease: false
|
96
|
-
version_requirements:
|
97
|
-
none: false
|
98
|
-
requirements:
|
99
|
-
- -
|
100
|
-
- !ruby/object:Gem::Version
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: simplecov
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: simplecov-rcov
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
description: SemanticCrawler is a ruby library that encapsulates data gathering from
|
159
|
+
different sources. Currently country information from Factbook and FAO (Food and
|
160
|
+
Agriculture Organization of the United Nations) and crisis information from GDACS.org
|
161
|
+
are supported.
|
162
|
+
email:
|
108
163
|
- alex.oberhauser@sigimera.org
|
109
164
|
executables: []
|
110
|
-
|
111
165
|
extensions: []
|
112
|
-
|
113
166
|
extra_rdoc_files: []
|
114
|
-
|
115
|
-
files:
|
167
|
+
files:
|
116
168
|
- lib/semantic_crawler.rb
|
117
169
|
- lib/tasks/semantic_crawler_tasks.rake
|
118
170
|
- lib/semantic_crawler/freebase/country.rb
|
119
|
-
- lib/semantic_crawler/freebase/crawler.rb
|
120
|
-
- lib/semantic_crawler/freebase/entity.rb
|
121
171
|
- lib/semantic_crawler/factbook/country.rb
|
172
|
+
- lib/semantic_crawler/fao.rb
|
173
|
+
- lib/semantic_crawler/dbpedia.rb
|
174
|
+
- lib/semantic_crawler/fao/country.rb
|
122
175
|
- lib/semantic_crawler/freebase.rb
|
123
176
|
- lib/semantic_crawler/factbook.rb
|
124
177
|
- lib/semantic_crawler/gdacs.rb
|
125
178
|
- lib/semantic_crawler/version.rb
|
126
179
|
- lib/semantic_crawler/gdacs/resource.rb
|
180
|
+
- lib/semantic_crawler/gdacs/emergency_feed.rb
|
127
181
|
- lib/semantic_crawler/gdacs/feed.rb
|
182
|
+
- lib/semantic_crawler/gdacs/emergency_feed_item.rb
|
128
183
|
- lib/semantic_crawler/gdacs/feed_item.rb
|
184
|
+
- lib/semantic_crawler/geo_names.rb
|
129
185
|
- lib/semantic_crawler/linked_geo_data/relevant_node.rb
|
130
186
|
- lib/semantic_crawler/linked_geo_data.rb
|
187
|
+
- log/semantic-crawler.log
|
131
188
|
- MIT-LICENSE
|
132
189
|
- Rakefile
|
133
190
|
- README.rdoc
|
134
|
-
-
|
191
|
+
- CHANGELOG.rdoc
|
135
192
|
- test/test_helper.rb
|
136
|
-
- test/factbook_test.rb
|
137
193
|
- test/semantic_crawler_test.rb
|
138
194
|
- test/dummy/log/test.log
|
139
195
|
- test/dummy/log/development.log
|
@@ -169,41 +225,37 @@ files:
|
|
169
225
|
- test/dummy/app/views/layouts/application.html.erb
|
170
226
|
homepage: https://github.com/obale/semantic_crawler
|
171
227
|
licenses: []
|
172
|
-
|
173
228
|
post_install_message:
|
174
229
|
rdoc_options: []
|
175
|
-
|
176
|
-
require_paths:
|
230
|
+
require_paths:
|
177
231
|
- lib
|
178
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
232
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
179
233
|
none: false
|
180
|
-
requirements:
|
181
|
-
- -
|
182
|
-
- !ruby/object:Gem::Version
|
183
|
-
|
184
|
-
segments:
|
234
|
+
requirements:
|
235
|
+
- - ! '>='
|
236
|
+
- !ruby/object:Gem::Version
|
237
|
+
version: '0'
|
238
|
+
segments:
|
185
239
|
- 0
|
186
|
-
|
187
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
240
|
+
hash: -1761754214109537766
|
241
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
188
242
|
none: false
|
189
|
-
requirements:
|
190
|
-
- -
|
191
|
-
- !ruby/object:Gem::Version
|
192
|
-
|
193
|
-
segments:
|
243
|
+
requirements:
|
244
|
+
- - ! '>='
|
245
|
+
- !ruby/object:Gem::Version
|
246
|
+
version: '0'
|
247
|
+
segments:
|
194
248
|
- 0
|
195
|
-
|
249
|
+
hash: -1761754214109537766
|
196
250
|
requirements: []
|
197
|
-
|
198
251
|
rubyforge_project:
|
199
252
|
rubygems_version: 1.8.21
|
200
253
|
signing_key:
|
201
254
|
specification_version: 3
|
202
|
-
summary: SemanticCrawler is a ruby library that encapsulates data gathering from different
|
203
|
-
|
204
|
-
|
255
|
+
summary: SemanticCrawler is a ruby library that encapsulates data gathering from different
|
256
|
+
sources.
|
257
|
+
test_files:
|
205
258
|
- test/test_helper.rb
|
206
|
-
- test/factbook_test.rb
|
207
259
|
- test/semantic_crawler_test.rb
|
208
260
|
- test/dummy/log/test.log
|
209
261
|
- test/dummy/log/development.log
|
@@ -1,22 +0,0 @@
|
|
1
|
-
module SemanticCrawler
|
2
|
-
module Freebase
|
3
|
-
# [XXX] This class should be deleted. The country object could be called directly.
|
4
|
-
class Crawler
|
5
|
-
# Returns a country object that contains all relevant
|
6
|
-
# information that could be extracted from freebase.com
|
7
|
-
#
|
8
|
-
# Example:
|
9
|
-
# >> freebaseCrawler = SemanticCrawler::Freebase::Crawler.new
|
10
|
-
# >> austria = freebaseCrawler.getCountry("Austria")
|
11
|
-
#
|
12
|
-
# Arguments:
|
13
|
-
# countryName: (String)
|
14
|
-
#
|
15
|
-
def getCountry(countryName)
|
16
|
-
country = SemanticCrawler::Freebase::Country.new
|
17
|
-
country.input_name = countryName.downcase!
|
18
|
-
country
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
data/test/factbook_test.rb
DELETED
@@ -1,86 +0,0 @@
|
|
1
|
-
require 'test_helper'
|
2
|
-
require 'pp'
|
3
|
-
|
4
|
-
class FactbookTest < ActiveSupport::TestCase
|
5
|
-
test "truth" do
|
6
|
-
assert_kind_of Module, SemanticCrawler
|
7
|
-
end
|
8
|
-
|
9
|
-
test "init austria country information" do
|
10
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
11
|
-
assert austria.country_name.eql?("Austria")
|
12
|
-
assert austria.url.eql?("http://www4.wiwiss.fu-berlin.de/factbook/data/austria")
|
13
|
-
end
|
14
|
-
|
15
|
-
test "get austria name" do
|
16
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
17
|
-
value = austria.name
|
18
|
-
assert value.to_s.eql?("Austria")
|
19
|
-
end
|
20
|
-
|
21
|
-
test "get austria background information" do
|
22
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
23
|
-
value = austria.background
|
24
|
-
assert !value.nil?
|
25
|
-
end
|
26
|
-
|
27
|
-
test "get austria population information" do
|
28
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
29
|
-
value = austria.population_total
|
30
|
-
assert !value.nil?
|
31
|
-
end
|
32
|
-
|
33
|
-
test "get austria legalsystem property (not wrapped)" do
|
34
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
35
|
-
value = austria.get_factbook_property("legalsystem")
|
36
|
-
assert !value.nil?
|
37
|
-
end
|
38
|
-
|
39
|
-
test "get austria latitude" do
|
40
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
41
|
-
value = austria.latitude
|
42
|
-
assert value.to_s.eql?("47.3333333333333")
|
43
|
-
end
|
44
|
-
|
45
|
-
test "get austria longitude" do
|
46
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
47
|
-
value = austria.longitude
|
48
|
-
assert value.to_s.eql?("13.3333333333333")
|
49
|
-
end
|
50
|
-
|
51
|
-
test "get austria landboundary" do
|
52
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
53
|
-
value = austria.landboundary
|
54
|
-
assert value.count == 7
|
55
|
-
end
|
56
|
-
|
57
|
-
test "get austria terrain" do
|
58
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
59
|
-
value = austria.terrain
|
60
|
-
assert !value.nil?
|
61
|
-
end
|
62
|
-
|
63
|
-
test "get austria total airports" do
|
64
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
65
|
-
value = austria.airports_total
|
66
|
-
assert value.to_s.eql?("55")
|
67
|
-
end
|
68
|
-
|
69
|
-
test "get austria heliports" do
|
70
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
71
|
-
value = austria.heliports
|
72
|
-
assert value.to_s.eql?("1")
|
73
|
-
end
|
74
|
-
|
75
|
-
test "get austria climate" do
|
76
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
77
|
-
value = austria.climate
|
78
|
-
assert !value.nil?
|
79
|
-
end
|
80
|
-
|
81
|
-
test "get austria location description" do
|
82
|
-
austria = SemanticCrawler::Factbook::Country.new("Austria")
|
83
|
-
value = austria.location
|
84
|
-
assert !value.nil?
|
85
|
-
end
|
86
|
-
end
|