worlddb 1.8.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +7 -0
  2. data/{History.md → HISTORY.md} +0 -0
  3. data/Manifest.txt +22 -5
  4. data/README.md +62 -2
  5. data/Rakefile +14 -6
  6. data/lib/worlddb.rb +34 -12
  7. data/lib/worlddb/cli/main.rb +159 -111
  8. data/lib/worlddb/cli/opts.rb +15 -48
  9. data/lib/worlddb/console.rb +6 -5
  10. data/lib/worlddb/deleter.rb +5 -3
  11. data/lib/worlddb/matcher.rb +16 -5
  12. data/lib/worlddb/models/city.rb +66 -30
  13. data/lib/worlddb/models/city_comp.rb +27 -0
  14. data/lib/worlddb/models/continent.rb +30 -8
  15. data/lib/worlddb/models/continent_comp.rb +24 -0
  16. data/lib/worlddb/models/country.rb +60 -36
  17. data/lib/worlddb/models/country_comp.rb +29 -0
  18. data/lib/worlddb/models/forward.rb +53 -0
  19. data/lib/worlddb/models/lang.rb +9 -7
  20. data/lib/worlddb/models/lang_comp.rb +23 -0
  21. data/lib/worlddb/models/name.rb +13 -0
  22. data/lib/worlddb/models/place.rb +16 -0
  23. data/lib/worlddb/models/region.rb +34 -12
  24. data/lib/worlddb/models/region_comp.rb +26 -0
  25. data/lib/worlddb/models/tagdb/tag.rb +16 -0
  26. data/lib/worlddb/models/tagdb/tagging.rb +15 -0
  27. data/lib/worlddb/models/usage.rb +10 -6
  28. data/lib/worlddb/reader.rb +31 -158
  29. data/lib/worlddb/readers/base.rb +41 -0
  30. data/lib/worlddb/readers/city.rb +18 -0
  31. data/lib/worlddb/readers/country.rb +17 -0
  32. data/lib/worlddb/readers/lang.rb +43 -0
  33. data/lib/worlddb/readers/region.rb +17 -0
  34. data/lib/worlddb/readers/usage.rb +35 -0
  35. data/lib/worlddb/schema.rb +100 -65
  36. data/lib/worlddb/stats.rb +9 -3
  37. data/lib/worlddb/utils.rb +3 -0
  38. data/lib/worlddb/version.rb +1 -6
  39. data/test/helper.rb +13 -3
  40. data/test/test_model_city.rb +60 -0
  41. data/test/test_model_comp.rb +48 -0
  42. data/test/test_model_country.rb +43 -0
  43. data/test/test_model_region.rb +50 -0
  44. data/test/test_models.rb +35 -0
  45. metadata +113 -37
  46. data/lib/worlddb/models/prop.rb +0 -32
  47. data/lib/worlddb/models/tag.rb +0 -33
  48. data/lib/worlddb/models/tagging.rb +0 -13
  49. data/test/test_values.rb +0 -114
@@ -0,0 +1,29 @@
1
+ # encoding: utf-8
2
+
3
+ module WorldDb
4
+ module Model
5
+
6
+ #############################################################
7
+ # collect depreciated or methods for future removal here
8
+ # - keep for now for commpatibility (for old code)
9
+
10
+
11
+ class Country
12
+
13
+ def title() name; end
14
+ def title=(value) self.name = value; end
15
+
16
+ scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
17
+
18
+
19
+ def synonyms() alt_names; end
20
+ def synonyms=(value) self.alt_names = value; end
21
+
22
+ def title_w_synonyms( opts={} ) all_names( opts ); end # depreciated: use all_names instead
23
+
24
+
25
+ end # class Country
26
+
27
+ end # module Model
28
+ end # module WorldDb
29
+
@@ -0,0 +1,53 @@
1
+
2
+ ### forward references
3
+ ## require first to resolve circular references
4
+
5
+ module WorldDb
6
+ module Model
7
+
8
+ #############
9
+ # ConfDb
10
+ Prop = ConfDb::Model::Prop
11
+
12
+ ###########
13
+ # TagDb
14
+ Tagging = TagDb::Model::Tagging
15
+ Tag = TagDb::Model::Tag
16
+
17
+
18
+ class Name < ActiveRecord::Base ; end
19
+ class Place < ActiveRecord::Base ; end
20
+ class Continent < ActiveRecord::Base ; end
21
+ class Country < ActiveRecord::Base ; end
22
+ class Region < ActiveRecord::Base ; end
23
+ class City < ActiveRecord::Base ; end
24
+
25
+ class Lang < ActiveRecord::Base ; end
26
+ class Usage < ActiveRecord::Base ; end
27
+
28
+ end
29
+
30
+ # note: convenience alias for Model
31
+ # lets you use include WorldDb::Models
32
+ Models = Model
33
+ end # module # WorldDb
34
+
35
+
36
+ module TagDb
37
+ module Model
38
+
39
+ # add alias? why? why not? # is there a better way?
40
+ # - just include WorldDb::Models - why? why not?
41
+
42
+ Name = WorldDb::Model::Name
43
+ Place = WorldDb::Model::Place
44
+ Continent = WorldDb::Model::Continent
45
+ Country = WorldDb::Model::Country
46
+ Region = WorldDb::Model::Region
47
+ City = WorldDb::Model::City
48
+
49
+ Lang = WorldDb::Model::Lang
50
+ Usage = WorldDb::Model::Usage
51
+
52
+ end
53
+ end
@@ -1,17 +1,19 @@
1
1
  # encoding: utf-8
2
2
 
3
- module WorldDb::Model
3
+ module WorldDb
4
+ module Model
4
5
 
5
- class Lang < ActiveRecord::Base
6
+ class Lang < ActiveRecord::Base
6
7
 
7
- has_many :usages # join table for countries_langs
8
+ has_many :usages # join table for countries_langs
8
9
 
9
- has_many :countries, :through => :usages
10
+ has_many :countries, :through => :usages
10
11
 
11
- validates :key, :format => { :with => /^[a-z]{2}$/, :message => 'expected two lowercase letters a-z' }
12
+ validates :key, format: { with: /^[a-z]{2}$/, message: 'expected two lowercase letters a-z' }
12
13
 
13
- end # class Lang
14
+ end # class Lang
14
15
 
15
- end # module WorldDb::Model
16
+ end # module Model
17
+ end # module WorldDb
16
18
 
17
19
 
@@ -0,0 +1,23 @@
1
+ # encoding: utf-8
2
+
3
+ module WorldDb
4
+ module Model
5
+
6
+ #############################################################
7
+ # collect depreciated or methods for future removal here
8
+ # - keep for now for commpatibility (for old code)
9
+
10
+
11
+ class Lang
12
+
13
+ #####################################################
14
+ # alias for name (remove! add depreciated api call ???)
15
+ def title() name; end
16
+ def title=(value) self.name = value; end
17
+
18
+ scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
19
+
20
+ end # class Lang
21
+
22
+ end # module Model
23
+ end # module WorldDb
@@ -0,0 +1,13 @@
1
+ # encoding: utf-8
2
+
3
+ module WorldDb
4
+ module Model
5
+
6
+ class Name < ActiveRecord::Base
7
+
8
+
9
+ end # class Name
10
+
11
+
12
+ end # module Model
13
+ end # module WorldDb
@@ -0,0 +1,16 @@
1
+ # encoding: utf-8
2
+
3
+ module WorldDb
4
+ module Model
5
+
6
+ class Place < ActiveRecord::Base
7
+
8
+ ## todo: depending on type
9
+ ## has_one continent, country, region, city etc.
10
+
11
+ end # class Place
12
+
13
+
14
+ end # module Model
15
+ end # module WorldDb
16
+
@@ -1,6 +1,8 @@
1
1
  # encoding: UTF-8
2
2
 
3
- module WorldDb::Model
3
+ module WorldDb
4
+ module Model
5
+
4
6
 
5
7
  class Region < ActiveRecord::Base
6
8
 
@@ -10,25 +12,44 @@ class Region < ActiveRecord::Base
10
12
  # self.create_or_update_from_values
11
13
  extend TextUtils::ValueHelper # e.g. is_year?, is_region?, is_address?, is_taglist? etc.
12
14
 
15
+ belongs_to :place, class_name: 'Place', foreign_key: 'place_id'
16
+ belongs_to :country, class_name: 'Country', foreign_key: 'country_id'
17
+
18
+ has_many :cities, class_name: 'City', foreign_key: 'region_id'
19
+
20
+ has_many_tags
21
+
22
+ validates :key, format: { with: /^[a-z]+$/, message: 'expected one or more lowercase letters a-z' }
23
+ validates :code, format: { with: /^[A-Z_]{2,3}$/, message: 'expected two or three uppercase letters A-Z (and _)' }, allow_nil: true
13
24
 
14
- belongs_to :country, :class_name => 'Country', :foreign_key => 'country_id'
25
+ before_create :on_create
26
+ before_update :on_update
15
27
 
16
- has_many :cities, :class_name => 'City', :foreign_key => 'region_id'
28
+ def on_create
29
+ place_rec = Place.create!( name: name, kind: place_kind )
30
+ self.place_id = place_rec.id
31
+ end
32
+
33
+ def on_update
34
+ ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
35
+ place.update_attributes!( name: name, kind: place_kind )
36
+ end
17
37
 
18
- has_many :taggings, :as => :taggable
19
- has_many :tags, :through => :taggings
38
+ def place_kind # use place_kind_of_code ??
39
+ 'ADM1'
40
+ end
20
41
 
21
- validates :key, :format => { :with => /^[a-z]+$/, :message => 'expected one or more lowercase letters a-z' }
22
- validates :code, :format => { :with => /^[A-Z_]{2,3}$/, :message => 'expected two or three uppercase letters A-Z (and _)' }, :allow_nil => true
23
42
 
43
+ def all_names( opts={} )
44
+ ### fix:
45
+ ## allow to passing in sep or separator e.g. | or other
24
46
 
25
- def title_w_synonyms
26
- return title if synonyms.blank?
47
+ return name if alt_names.blank?
27
48
 
28
49
  buf = ''
29
- buf << title
50
+ buf << name
30
51
  buf << ' | '
31
- buf << synonyms.split('|').join(' | ')
52
+ buf << alt_names.split('|').join(' | ')
32
53
  buf
33
54
  end
34
55
 
@@ -150,4 +171,5 @@ class Region < ActiveRecord::Base
150
171
 
151
172
  end # class Region
152
173
 
153
- end # module WorldDb::Model
174
+ end # module Model
175
+ end # module WorldDb
@@ -0,0 +1,26 @@
1
+ # encoding: utf-8
2
+
3
+ module WorldDb
4
+ module Model
5
+
6
+ #############################################################
7
+ # collect depreciated or methods for future removal here
8
+ # - keep for now for commpatibility (for old code)
9
+
10
+ class Region
11
+
12
+ def title() name; end
13
+ def title=(value) self.name = value; end
14
+
15
+ scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
16
+
17
+ def synonyms() alt_names; end
18
+ def synonyms=(value) self.alt_names = value; end
19
+
20
+ def title_w_synonyms( opts={} ) all_names( opts ); end # depreciated: use all_names instead
21
+
22
+
23
+ end # class Region
24
+
25
+ end # module Model
26
+ end # module WorldDb
@@ -0,0 +1,16 @@
1
+ # encoding: utf-8
2
+
3
+ module TagDb
4
+ module Model
5
+
6
+ class Tag
7
+
8
+ has_many :cities, :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::City', class_name: 'WorldDb::Model::City'
9
+ has_many :countries, :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::Country', class_name: 'WorldDb::Model::Country'
10
+ has_many :regions, :through => :taggings, :source => :taggable, source_type: 'WorldDb::Model::Region', class_name: 'WorldDb::Model::Region'
11
+
12
+ end # class Tag
13
+
14
+ end # module Model
15
+ end # module WorldDb
16
+
@@ -0,0 +1,15 @@
1
+ # encoding: utf-8
2
+
3
+ module TagDb
4
+ module Model
5
+
6
+
7
+ class Tagging
8
+
9
+ ## add some code here
10
+
11
+ end # class Tagging
12
+
13
+
14
+ end # module Model
15
+ end # module WorldDb
@@ -1,13 +1,17 @@
1
1
  # encoding: utf-8
2
2
 
3
- module WorldDb::Model
3
+ module WorldDb
4
+ module Model
4
5
 
5
- class Usage < ActiveRecord::Base
6
6
 
7
- belongs_to :country
8
- belongs_to :lang
7
+ class Usage < ActiveRecord::Base
9
8
 
10
- end # class Usage
9
+ belongs_to :country
10
+ belongs_to :lang
11
11
 
12
- end # module WorldDb::Model
12
+ end # class Usage
13
+
14
+
15
+ end # module Model
16
+ end # module WorldDb
13
17
 
@@ -9,24 +9,19 @@ class Reader
9
9
 
10
10
  ## make models available in sportdb module by default with namespace
11
11
  # e.g. lets you use City instead of Models::City
12
- include WorldDb::Models
13
-
12
+ include Models
13
+ include Matcher # e.g. match_cities_for_country, match_regions_for_country, etc.
14
14
 
15
15
  ## value helpers e.g. is_year?, is_taglist? etc.
16
16
  include TextUtils::ValueHelper
17
17
 
18
- include WorldDb::Matcher # e.g. match_cities_for_country, match_regions_for_country, etc.
19
18
 
20
19
 
21
20
  attr_reader :include_path
22
21
 
23
- def skip_tags?
24
- @skip_tags == true
25
- end
22
+ def skip_tags?() @skip_tags == true; end
23
+ def strict?() @strict == true; end
26
24
 
27
- def strict?
28
- @strict == true
29
- end
30
25
 
31
26
  def initialize( include_path, opts = {} )
32
27
 
@@ -67,9 +62,13 @@ class Reader
67
62
  elsif name =~ /\/continents/
68
63
  load_continent_refs( name )
69
64
  elsif name =~ /^lang/
70
- load_langs( name )
65
+ ## todo: pass along opts too
66
+ ## use match_usage( name ) - why? why not?? ???
67
+ LangReader.new( include_path ).read( name )
71
68
  elsif name =~ /\/lang/
72
- load_usages( name )
69
+ ## todo: pass along opts too
70
+ ## use match_usage( name ) - why? why not?? ???
71
+ UsageReader.new( include_path ).read( name )
73
72
  elsif name =~ /\/fifa/
74
73
  load_xxx( 'fifa', name )
75
74
  elsif name =~ /\/iso3/
@@ -78,17 +77,25 @@ class Reader
78
77
  load_xxx( 'net', name )
79
78
  elsif name =~ /\/motor/
80
79
  load_xxx( 'motor', name )
81
- elsif name =~ /^tag.*\.(\d)$/
82
- load_tags( name, :grade => $1.to_i )
80
+ elsif name =~ /^tag.*\.\d$/
81
+ ## todo: pass along opts too
82
+ ## use match_tags( name ) - why? why not?? ???
83
+ TagDb::TagReader.new( include_path ).read( name )
83
84
  elsif match_countries_for_continent( name ) do |continent| # # e.g. africa/countries or america/countries
84
85
  ### NB: continent changed to regions (e.g. middle-east, caribbean, north-america, etc.)
85
86
  ## auto-add continent (from folder structure) as tag
86
87
  ## fix: allow dash/hyphen/minus in tag
87
- load_countries( name, :tags => continent.tr('-', '_') )
88
+
89
+ r = CountryReader.new( include_path )
90
+ r.read( name, tags: continent.tr('-', '_') )
88
91
  end
89
92
  elsif match_cities_for_country( name ) do |country_key| # name =~ /\/([a-z]{2})\/cities/
90
93
  ## auto-add required country code (from folder structure)
91
- load_cities( country_key, name )
94
+ country = Country.find_by_key!( country_key )
95
+ logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
96
+
97
+ r = CityReader.new( include_path )
98
+ r.read( name, country_id: country.id )
92
99
  end
93
100
  elsif match_regions_abbr_for_country( name ) do |country_key| # name =~ /\/([a-z]{2})\/regions\.abbr/
94
101
  load_regions_xxx( country_key, 'abbr', name )
@@ -101,7 +108,11 @@ class Reader
101
108
  end
102
109
  elsif match_regions_for_country( name ) do |country_key| # name =~ /\/([a-z]{2})\/regions/
103
110
  ## auto-add required country code (from folder structure)
104
- load_regions( country_key, name )
111
+ country = Country.find_by_key!( country_key )
112
+ logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
113
+
114
+ r = RegionReader.new( include_path )
115
+ r.read( name, country_id: country.id )
105
116
  end
106
117
  else
107
118
  logger.error "unknown world.db fixture type >#{name}<"
@@ -110,20 +121,7 @@ class Reader
110
121
  end
111
122
 
112
123
 
113
-
114
- def load_countries( name, more_attribs={} )
115
- load_fixtures_for( Country, name, more_attribs )
116
- end
117
-
118
-
119
- def load_regions( country_key, name )
120
- country = Country.find_by_key!( country_key )
121
- logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
122
-
123
- load_fixtures_for( Region, name, country_id: country.id )
124
- end
125
-
126
-
124
+ ### use RegionAttrReader
127
125
  def load_regions_xxx( country_key, xxx, name )
128
126
  country = Country.find_by_key!( country_key )
129
127
  logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
@@ -138,14 +136,7 @@ class Reader
138
136
  end
139
137
 
140
138
 
141
- def load_cities( country_key, name )
142
- country = Country.find_by_key!( country_key )
143
- logger.debug "Country #{country.key} >#{country.title} (#{country.code})<"
144
-
145
- load_fixtures_for( City, name, country_id: country.id )
146
- end
147
-
148
-
139
+ ### use ContinentRefReader
149
140
  def load_continent_refs( name )
150
141
  reader = HashReaderV2.new( name, include_path )
151
142
 
@@ -157,7 +148,7 @@ class Reader
157
148
  end
158
149
  end
159
150
 
160
-
151
+ ### use ContinentDef Reader
161
152
  def load_continent_defs( name, more_attribs={} )
162
153
  reader = ValuesReaderV2.new( name, include_path, more_attribs )
163
154
 
@@ -183,115 +174,7 @@ class Reader
183
174
  end # each lines
184
175
  end # load_continent_defs
185
176
 
186
-
187
- def load_langs( name )
188
-
189
- reader = HashReaderV2.new( name, include_path )
190
-
191
- reader.each do |key, value|
192
-
193
- logger.debug "adding lang >>#{key}<< >>#{value}<<..."
194
-
195
- lang_key = key.strip
196
- lang_title = value.strip
197
-
198
- lang_attribs = {}
199
-
200
- ## check if it exists
201
- lang = Lang.find_by_key( lang_key )
202
- if lang.present?
203
- logger.debug "update lang #{lang.id}-#{lang.key}:"
204
- else
205
- logger.debug "create lang:"
206
- lang = Lang.new
207
- lang_attribs[ :key ] = lang_key
208
- end
209
-
210
- lang_attribs[ :title ] = lang_title
211
-
212
- logger.debug lang_attribs.to_json
213
-
214
- lang.update_attributes!( lang_attribs )
215
- end # each key,value
216
-
217
- end # method load_langs
218
-
219
-
220
- def load_tags( name, more_attribs={} )
221
-
222
- reader = HashReaderV2.new( name, include_path )
223
-
224
- grade = 1
225
-
226
- if more_attribs[:grade].present?
227
- grade = more_attribs[:grade].to_i
228
- end
229
-
230
- reader.each do |key, value|
231
- ### split value by comma (e.g. northern america,southern america, etc.)
232
- logger.debug "adding grade #{grade} tags >>#{key}<< >>#{value}<<..."
233
- tag_pairs = value.split(',')
234
- tag_pairs.each do |pair|
235
- ## split key|title
236
- values = pair.split('|')
237
-
238
- key = values[0]
239
- ### remove (optional comment) from key (e.g. carribean (islands))
240
- key = key.gsub( /\(.+\)/, '' )
241
- ## remove leading n trailing space
242
- key = key.strip
243
-
244
- title = values[1] || '' # nb: title might be empty/missing
245
- title = title.strip
246
-
247
- tag_attribs = {}
248
-
249
- ## check if it exists
250
- ## todo/fix: add country_id for lookup?
251
- tag = Tag.find_by_key( key )
252
- if tag.present?
253
- logger.debug "update tag #{tag.id}-#{tag.key}:"
254
- else
255
- logger.debug "create tag:"
256
- tag = Tag.new
257
- tag_attribs[ :key ] = key
258
- end
259
-
260
- tag_attribs[ :title ] = title
261
- tag_attribs[ :grade ] = grade
262
-
263
- logger.debug tag_attribs.to_json
264
-
265
- tag.update_attributes!( tag_attribs )
266
- end
267
- end # each key,value
268
-
269
- end # method load_tags
270
-
271
-
272
- def load_usages( name )
273
- reader = HashReaderV2.new( name, include_path )
274
-
275
- reader.each do |key, value|
276
- logger.debug " adding langs >>#{value}<<to country >>#{key}<<"
277
-
278
- country = Country.find_by_key!( key )
279
-
280
- lang_keys = value.split(',')
281
- lang_keys.each do |lang_key|
282
-
283
- ### remove (optional comment) from key (e.g. carribean (islands))
284
- lang_key = lang_key.gsub( /\(.+\)/, '' )
285
- ## remove leading n trailing space
286
- lang_key = lang_key.strip
287
-
288
- lang = Lang.find_by_key!( lang_key )
289
- Usage.create!( country_id: country.id, lang_id: lang.id, official: true, minor: false )
290
- end
291
- end
292
- end
293
-
294
-
177
+ ### use CountryAttr Reader
295
178
  def load_xxx( xxx, name )
296
179
  reader = HashReaderV2.new( name, include_path )
297
180
 
@@ -302,15 +185,5 @@ class Reader
302
185
  end
303
186
  end
304
187
 
305
- private
306
- def load_fixtures_for( clazz, name, more_attribs={} )
307
- reader = ValuesReaderV2.new( name, include_path, more_attribs )
308
-
309
- reader.each_line do |attribs, values|
310
- opts = { skip_tags: skip_tags? }
311
- clazz.create_or_update_from_attribs( attribs, values, opts )
312
- end
313
- end
314
-
315
188
  end # class Reader
316
189
  end # module WorldDb