worlddb 2.0.9 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.md +1 -1
  3. data/Manifest.txt +1 -38
  4. data/README.md +0 -67
  5. data/Rakefile +7 -11
  6. data/lib/worlddb.rb +8 -187
  7. data/lib/worlddb/cli/main.rb +3 -8
  8. data/lib/worlddb/{version.rb → cli/version.rb} +9 -7
  9. data/lib/worlddb/console.rb +5 -10
  10. metadata +10 -110
  11. data/.gemtest +0 -0
  12. data/lib/worlddb/deleter.rb +0 -32
  13. data/lib/worlddb/matcher.rb +0 -143
  14. data/lib/worlddb/models/city.rb +0 -240
  15. data/lib/worlddb/models/city_comp.rb +0 -27
  16. data/lib/worlddb/models/continent.rb +0 -41
  17. data/lib/worlddb/models/continent_comp.rb +0 -24
  18. data/lib/worlddb/models/country.rb +0 -328
  19. data/lib/worlddb/models/country_code.rb +0 -41
  20. data/lib/worlddb/models/country_comp.rb +0 -35
  21. data/lib/worlddb/models/forward.rb +0 -57
  22. data/lib/worlddb/models/lang.rb +0 -18
  23. data/lib/worlddb/models/lang_comp.rb +0 -23
  24. data/lib/worlddb/models/name.rb +0 -13
  25. data/lib/worlddb/models/place.rb +0 -16
  26. data/lib/worlddb/models/region.rb +0 -176
  27. data/lib/worlddb/models/region_comp.rb +0 -26
  28. data/lib/worlddb/models/tagdb/tag.rb +0 -16
  29. data/lib/worlddb/models/tagdb/tagging.rb +0 -15
  30. data/lib/worlddb/models/usage.rb +0 -17
  31. data/lib/worlddb/patterns.rb +0 -54
  32. data/lib/worlddb/reader.rb +0 -224
  33. data/lib/worlddb/reader_file.rb +0 -86
  34. data/lib/worlddb/reader_zip.rb +0 -160
  35. data/lib/worlddb/readers/city.rb +0 -81
  36. data/lib/worlddb/readers/country.rb +0 -78
  37. data/lib/worlddb/readers/lang.rb +0 -107
  38. data/lib/worlddb/readers/region.rb +0 -79
  39. data/lib/worlddb/readers/usage.rb +0 -98
  40. data/lib/worlddb/schema.rb +0 -202
  41. data/lib/worlddb/stats.rb +0 -31
  42. data/test/helper.rb +0 -29
  43. data/test/test_fixture_matchers.rb +0 -112
  44. data/test/test_model_city.rb +0 -60
  45. data/test/test_model_comp.rb +0 -48
  46. data/test/test_model_country.rb +0 -53
  47. data/test/test_model_region.rb +0 -50
  48. data/test/test_models.rb +0 -35
@@ -1,27 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module WorldDb
4
- module Model
5
-
6
- #############################################################
7
- # collect depreciated or methods for future removal here
8
- # - keep for now for commpatibility (for old code)
9
-
10
- class City
11
-
12
- def title() name; end
13
- def title=(value) self.name = value; end
14
-
15
- scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
16
-
17
-
18
- def synonyms() alt_names; end
19
- def synonyms=(value) self.alt_names = value; end
20
-
21
- def title_w_synonyms( opts={} ) all_names( opts ); end # depreciated: use all_names instead
22
-
23
- end # class Cities
24
-
25
- end # module Model
26
- end # module WorldDb
27
-
@@ -1,41 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module WorldDb
4
- module Model
5
-
6
- class Continent < ActiveRecord::Base
7
-
8
- belongs_to :place, class_name: 'Place', foreign_key: 'place_id'
9
- has_many :countries
10
-
11
- # NB: allow dots in keys e.g. concacaf.naf etc.
12
-
13
- before_create :on_create
14
- before_update :on_update
15
-
16
- def on_create
17
- place_rec = Place.create!( name: name, kind: place_kind )
18
- self.place_id = place_rec.id
19
-
20
- self.slug = TextUtils.slugify( name ) if slug.blank?
21
- end
22
-
23
- def on_update
24
- ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
25
- place.update_attributes!( name: name, kind: place_kind )
26
-
27
- ## check if name changed -- possible?
28
- ## update slug too??
29
- end
30
-
31
- def place_kind # use place_kind_of_code ??
32
- 'CONT'
33
- end
34
-
35
-
36
- end # class Continent
37
-
38
-
39
- end # module Model
40
- end # module WorldDb
41
-
@@ -1,24 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module WorldDb
4
- module Model
5
-
6
- #############################################################
7
- # collect depreciated or methods for future removal here
8
- # - keep for now for commpatibility (for old code)
9
-
10
- class Continent
11
-
12
- def title() name; end
13
- def title=(value) self.name = value; end
14
-
15
- scope :by_title, ->{ order( 'name asc' ) } # order by title (a-z)
16
-
17
- def synonyms() alt_names; end
18
- def synonyms=(value) self.alt_names = value; end
19
-
20
- end # class Continent
21
-
22
-
23
- end # module Model
24
- end # module WorldDb
@@ -1,328 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module WorldDb
4
- module Model
5
-
6
- ########
7
- # Country / Supra (e.g. European Union) / Territory (e.g. Puerto Rico) or Dependency (e.g. Dependent territory)
8
-
9
- class Country < ActiveRecord::Base
10
-
11
- extend TextUtils::TagHelper # will add self.find_tags, self.find_tags_in_attribs!, etc.
12
-
13
- # NB: use extend - is_<type>? become class methods e.g. self.is_<type>? for use in
14
- # self.create_or_update_from_values
15
- extend TextUtils::ValueHelper # e.g. self.is_year?, self.is_region?, self.is_address?, is_taglist? etc.
16
-
17
-
18
- self.table_name = 'countries'
19
-
20
- belongs_to :place, class_name: 'Place', foreign_key: 'place_id'
21
- belongs_to :continent, class_name: 'Continent', foreign_key: 'continent_id'
22
-
23
- has_many :usages
24
- has_many :langs, :through => :usages # lang(uage)s through usages (that is, countries_langs) join table
25
-
26
- has_many :regions, class_name: 'Region', foreign_key: 'country_id'
27
- has_many :cities, class_name: 'City', foreign_key: 'country_id'
28
-
29
- ## self referencing hierachy within countries e.g. EU > GB > EN
30
- belongs_to :parent, class_name: 'Country', foreign_key: 'country_id'
31
- has_many :countries, class_name: 'Country', foreign_key: 'country_id'
32
-
33
- has_many_tags
34
-
35
- validates :key, format: { with: /#{COUNTRY_KEY_PATTERN}/, message: COUNTRY_KEY_PATTERN_MESSAGE }
36
- validates :code, format: { with: /#{COUNTRY_CODE_PATTERN}/, message: COUNTRY_CODE_PATTERN_MESSAGE }
37
-
38
-
39
- scope :by_key, ->{ order( 'key asc' ) } # order by key (a-z)
40
- scope :by_name, ->{ order( 'name asc' ) } # order by name (a-z)
41
- scope :by_code, ->{ order( 'code asc' ) } # order by code (a-z)
42
- scope :by_pop, ->{ order( 'pop desc' ) } # order by pop(ulation)
43
- scope :by_area, ->{ order( 'area desc') } # order by area (in square km)
44
-
45
- scope :by_num, ->{ order( 'num asc' ) } # order by numeric country code
46
- scope :by_alpha2, ->{ order( 'alpha2 asc' ) }
47
- scope :by_alpha3, ->{ order( 'alpha2 asc' ) }
48
- scope :by_fifa, ->{ order( 'fifa asc' ) } # football
49
- scope :by_ioc, ->{ order( 'ioc asc' ) } # olympics
50
- scope :by_motor, ->{ order( 'motor asc' ) } # designated signs; motor vehicle license plate
51
- scope :by_net, ->{ order( 'net asc' ) } # internet cc top level domain; ccTLD
52
-
53
-
54
- before_create :on_create
55
- before_update :on_update
56
-
57
-
58
- def on_create
59
- place_rec = Place.create!( name: name, kind: place_kind )
60
- self.place_id = place_rec.id
61
-
62
- if slug.blank?
63
- ## todo: change and to n (if en/english) ?? - why? why not?
64
- ## remove subtitles/subnames e.g. () -- why? why not?
65
-
66
- ## remove translations [] e.g. México [Mexico] -> México etc.
67
- self.slug = TextUtils.slugify( name.gsub( /\[[^\]]+\]/, '' ) )
68
- end
69
- end
70
-
71
- def on_update
72
- ## fix/todo: check - if name or kind changed - only update if changed ?? why? why not??
73
- place.update_attributes!( name: name, kind: place_kind )
74
-
75
- ## check if name changed -- possible?
76
- ## update slug too??
77
- end
78
-
79
- def place_kind # use place_kind_of_code ??
80
- if is_supra?
81
- 'SUPR'
82
- elsif is_dependency?
83
- 'TERR'
84
- elsif is_misc? ## misc(ellaneous) country or dependent territory
85
- # todo: use different marker?
86
- # territory w/ shared or disputes claims e.g Antartica/Western Sahara/Paracel Islands pg Spratly Islands/etc.
87
- 'MISC'
88
- else
89
- 'CNTY'
90
- end
91
- end
92
-
93
-
94
- ###
95
- # NB: use is_ for flags to avoid conflict w/ assocs
96
-
97
- def is_supra?() s? == true; end
98
- def is_country?() c? == true; end
99
- def is_dependency?() d? == true; end
100
- def is_misc?() m? == true; end
101
-
102
-
103
- def all_names( opts={} )
104
- ### fix:
105
- ## allow to passing in sep or separator e.g. | or other
106
-
107
- return name if alt_names.blank?
108
-
109
- buf = ''
110
- buf << name
111
- buf << ' | '
112
- buf << alt_names.split('|').join(' | ')
113
- buf
114
- end
115
-
116
-
117
- def to_path( opts={} )
118
- # e.g. europe/at-austria
119
- "#{continent.slug}/#{key}-#{slug}"
120
- end
121
-
122
-
123
- def self.search_by_name( q ) ## todo/check: just use search (rename)? why? why not?
124
-
125
- ## fix: add/configure logger for ActiveRecord!!!
126
- ## logger = LogKernel::Logger.root
127
-
128
- name = q.strip
129
-
130
- ## 1) first try 1:1 (exact) match
131
- cty = Country.find_by_name( name ) # NOTE: assume AR escapes quotes in name ??
132
- if cty.nil?
133
- ## 2) retry: a) remove all (..) enclosed
134
- ## b) remove all extra spaces (e.g. Cocos (Keeling) Islands => Cocos__Islands => Cocos_Islands)
135
- name = name.gsub( /\([^)]+\)/, '' ).strip
136
- name = name.gsub( /[ \t]{2,}/, ' ' )
137
- cty = Country.find_by_name( name )
138
-
139
- ### NOTE: escape ' for sql like clause
140
- ## for now use '' for escapes, that is, double quotes
141
- ## check - working for postgresql n sqlite??
142
- name_esc = name.gsub( /'/, "''" )
143
-
144
- ## 3) retry: use SQL like match
145
- ## % is used to match *zero* or more occurrences of any characters
146
- ## todo: check if it matches zero too
147
- if cty.nil?
148
- cty = Country.where( "name LIKE '%#{name_esc}%'" ).first
149
- end
150
-
151
- ## 4) retry: use SQL like match for alternative names match
152
- if cty.nil?
153
- cty = Country.where( "alt_names LIKE '%#{name_esc}%'" ).first
154
- end
155
-
156
- ## 5) retry: use SQL like match for historic names match (e.g. Burma for Myanmar etc.)
157
- ## todo/check: make it optional (pass in opts hash to configure) - why? why not???
158
- if cty.nil?
159
- cty = Country.where( "hist_names LIKE '%#{name_esc}%'" ).first
160
- end
161
- end
162
-
163
- cty # return cty (country); nil if not found
164
- end
165
-
166
-
167
- def self.create_or_update_from_values( values, more_attribs={} )
168
-
169
- ## key & title
170
- ## NB: three-letter code (.e.g AUT) required - enforce in values? why? why not?
171
- attribs, more_values = find_key_n_title( values )
172
- attribs = attribs.merge( more_attribs )
173
-
174
- Country.create_or_update_from_attribs( attribs, more_values )
175
- end
176
-
177
-
178
- def self.create_or_update_from_attribs( new_attributes, values, opts={} )
179
-
180
- ## opts e.g. :skip_tags true|false
181
-
182
- ## fix: add/configure logger for ActiveRecord!!!
183
- logger = LogKernel::Logger.root
184
-
185
- value_numbers = []
186
- value_tag_keys = []
187
- value_cities = []
188
-
189
- ### check for "default" tags - that is, if present new_attributes[:tags] remove from hash
190
- value_tag_keys += find_tags_in_attribs!( new_attributes )
191
-
192
-
193
- new_attributes[ :c ] = true # assume country type by default (use supra,depend to change)
194
-
195
- ## check for optional values
196
- values.each_with_index do |value,index|
197
- if match_supra_flag( value ) do |_| # supra(national)
198
- new_attributes[ :c ] = false # turn off default c|country flag; make it s|supra only
199
- new_attributes[ :s ] = true
200
- ## auto-add tag supra
201
- value_tag_keys << 'supra'
202
- end
203
- elsif match_supra( value ) do |country| # supra:
204
- new_attributes[ :country_id ] = country.id
205
- end
206
- elsif match_country( value ) do |country| # country:
207
- new_attributes[ :country_id ] = country.id
208
- new_attributes[ :c ] = false # turn off default c|country flag; make it d|depend only
209
- new_attributes[ :d ] = true
210
- ## auto-add tag supra
211
- value_tag_keys << 'territory' # rename tag to dependency? why? why not?
212
- end
213
- elsif match_km_squared( value ) do |num| # allow numbers like 453 km²
214
- value_numbers << num
215
- end
216
- elsif match_number( value ) do |num| # numeric (nb: can use any _ or spaces inside digits e.g. 1_000_000 or 1 000 000)
217
- value_numbers << num
218
- end
219
- elsif value =~ /#{COUNTRY_CODE_PATTERN}/ ## three letter code
220
- new_attributes[ :code ] = value
221
- elsif (values.size==(index+1)) && is_taglist?( value ) # tags must be last entry
222
- logger.debug " found tags: >>#{value}<<"
223
- value_tag_keys += find_tags( value )
224
- else
225
-
226
- ### assume it is the capital city - mark it for auto add
227
- value_cities << value
228
- next
229
-
230
- # issue warning: unknown type for value
231
- # logger.warn "unknown type for value >#{value}<"
232
- end
233
- end # each value
234
-
235
- if value_numbers.size > 0
236
- new_attributes[ :area ] = value_numbers[0]
237
- new_attributes[ :pop ] = value_numbers[1]
238
- end
239
-
240
- =begin
241
- # auto-add tags
242
- area = value_numbers[0]
243
- pop = value_numbers[1]
244
-
245
- # categorize into brackets
246
- if area >= 1_000_000
247
- value_tag_keys << 'area_1_000_000_n_up'
248
- elsif area >= 100_000
249
- value_tag_keys << 'area_100_000_to_1_000_000'
250
- elsif area >= 1000
251
- value_tag_keys << 'area_1_000_to_100_000'
252
- else
253
- value_tag_keys << 'area_1_000_n_less' # microstate
254
- end
255
-
256
- # include all
257
- value_tag_keys << 'area_100_000_n_up' if area >= 100_000
258
- value_tag_keys << 'area_1_000_n_up' if area >= 1_000
259
-
260
-
261
- # categorize into brackets
262
- if pop >= 100_000_000
263
- value_tag_keys << 'pop_100m_n_up'
264
- elsif pop >= 10_000_000
265
- value_tag_keys << 'pop_10m_to_100m'
266
- elsif pop >= 1_000_000
267
- value_tag_keys << 'pop_1m_to_10m'
268
- else
269
- value_tag_keys << 'pop_1m_n_less'
270
- end
271
-
272
- # include all
273
- value_tag_keys << 'pop_10m_n_up' if pop >= 10_000_000
274
- value_tag_keys << 'pop_1m_n_up' if pop >= 1_000_000
275
- =end
276
-
277
-
278
- rec = Country.find_by_key( new_attributes[ :key ] )
279
-
280
- if rec.present?
281
- logger.debug "update Country #{rec.id}-#{rec.key}:"
282
- else
283
- logger.debug "create Country:"
284
- rec = Country.new
285
- end
286
-
287
- logger.debug new_attributes.to_json
288
-
289
- rec.update_attributes!( new_attributes )
290
-
291
- #################
292
- ## auto add capital cities
293
-
294
- City.create_or_update_from_titles( value_cities, country_id: rec.id )
295
-
296
- ##################
297
- ## add taggings
298
-
299
- if value_tag_keys.size > 0
300
-
301
- if opts[:skip_tags].present?
302
- logger.debug " skipping add taggings (flag skip_tag)"
303
- else
304
- value_tag_keys.uniq! # remove duplicates
305
- logger.debug " adding #{value_tag_keys.size} taggings: >>#{value_tag_keys.join('|')}<<..."
306
-
307
- ### fix/todo: check tag_ids and only update diff (add/remove ids)
308
-
309
- value_tag_keys.each do |key|
310
- tag = Tag.find_by_key( key )
311
- if tag.nil? # create tag if it doesn't exit
312
- logger.debug " creating tag >#{key}<"
313
- tag = Tag.create!( key: key )
314
- end
315
- rec.tags << tag
316
- end
317
- end
318
- end
319
-
320
- rec
321
-
322
- end # method create_or_update_from_values
323
-
324
-
325
- end # class Country
326
-
327
- end # module Model
328
- end # module WorldDb
@@ -1,41 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module WorldDb
4
- module Model
5
-
6
- class CountryCode < ActiveRecord::Base
7
-
8
- self.table_name = 'country_codes'
9
-
10
- belongs_to :country, class_name: 'Country', foreign_key: 'country_id'
11
-
12
-
13
- scope :by_name, ->{ order( 'name asc' ) } # order by name (a-z)
14
-
15
-
16
- def self.update!
17
- ## update (auto-create) country codes from existing countries in db
18
-
19
- ## fix: add/configure logger for ActiveRecord!!!
20
- logger = LogKernel::Logger.root
21
-
22
- logger.debug( "delete all (old) country codes" )
23
- CountryCode.delete_all
24
-
25
- Country.order(:id).each do |cty|
26
- logger.debug( "add country #{cty.key} #{cty.name}" )
27
- CountryCode.create!( country_id: cty.id, kind: 'NET', name: cty.net ) unless cty.net.nil?
28
- CountryCode.create!( country_id: cty.id, kind: 'NUM', name: cty.num ) unless cty.num.nil?
29
- CountryCode.create!( country_id: cty.id, kind: 'A2', name: cty.alpha2 ) unless cty.alpha2.nil?
30
- CountryCode.create!( country_id: cty.id, kind: 'A3', name: cty.alpha3 ) unless cty.alpha3.nil?
31
- CountryCode.create!( country_id: cty.id, kind: 'FIFA', name: cty.fifa ) unless cty.fifa.nil?
32
- CountryCode.create!( country_id: cty.id, kind: 'IOC', name: cty.ioc ) unless cty.ioc.nil?
33
- CountryCode.create!( country_id: cty.id, kind: 'FIPS', name: cty.fips ) unless cty.fips.nil?
34
- CountryCode.create!( country_id: cty.id, kind: 'M', name: cty.motor ) unless cty.motor.nil?
35
- end
36
- end
37
-
38
- end # class CountryCode
39
-
40
- end # module Model
41
- end # module WorldDb