worlddb 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Manifest.txt +3 -0
- data/lib/worlddb/console.rb +71 -0
- data/lib/worlddb/reader.rb +14 -61
- data/lib/worlddb/readers/line_reader.rb +45 -0
- data/lib/worlddb/readers/values_reader.rb +84 -0
- data/lib/worlddb/utils.rb +0 -47
- data/lib/worlddb/version.rb +1 -1
- data/lib/worlddb.rb +2 -0
- metadata +7 -4
data/Manifest.txt
CHANGED
@@ -53,6 +53,7 @@ db/tags.yml
|
|
53
53
|
lib/worlddb.rb
|
54
54
|
lib/worlddb/cli/opts.rb
|
55
55
|
lib/worlddb/cli/runner.rb
|
56
|
+
lib/worlddb/console.rb
|
56
57
|
lib/worlddb/loader.rb
|
57
58
|
lib/worlddb/models/city.rb
|
58
59
|
lib/worlddb/models/country.rb
|
@@ -61,6 +62,8 @@ lib/worlddb/models/region.rb
|
|
61
62
|
lib/worlddb/models/tag.rb
|
62
63
|
lib/worlddb/models/tagging.rb
|
63
64
|
lib/worlddb/reader.rb
|
65
|
+
lib/worlddb/readers/line_reader.rb
|
66
|
+
lib/worlddb/readers/values_reader.rb
|
64
67
|
lib/worlddb/schema.rb
|
65
68
|
lib/worlddb/utils.rb
|
66
69
|
lib/worlddb/version.rb
|
@@ -0,0 +1,71 @@
|
|
1
|
+
## for use to run with interactive ruby (irb)
|
2
|
+
## e.g. irb -r worlddb/console
|
3
|
+
|
4
|
+
require 'worlddb'
|
5
|
+
|
6
|
+
# some ruby stdlibs
|
7
|
+
|
8
|
+
require 'logger'
|
9
|
+
require 'pp' # pretty printer
|
10
|
+
require 'uri'
|
11
|
+
require 'json'
|
12
|
+
require 'yaml'
|
13
|
+
|
14
|
+
|
15
|
+
## shortcuts for models
|
16
|
+
|
17
|
+
Tag = WorldDB::Models::Tag
|
18
|
+
Tagging = WorldDB::Models::Tagging
|
19
|
+
Country = WorldDB::Models::Country
|
20
|
+
Region = WorldDB::Models::Region
|
21
|
+
City = WorldDB::Models::City
|
22
|
+
Prop = WorldDB::Models::Prop
|
23
|
+
|
24
|
+
## connect to db
|
25
|
+
|
26
|
+
DB_CONFIG = {
|
27
|
+
adapter: 'sqlite3',
|
28
|
+
database: 'world.db'
|
29
|
+
}
|
30
|
+
|
31
|
+
pp DB_CONFIG
|
32
|
+
ActiveRecord::Base.establish_connection( DB_CONFIG )
|
33
|
+
|
34
|
+
|
35
|
+
|
36
|
+
## test drive
|
37
|
+
|
38
|
+
puts "Welcome to world.db, version #{WorldDB::VERSION}!"
|
39
|
+
puts " #{'%6d' % Country.count} countries"
|
40
|
+
puts " #{'%6d' % Region.count} regions"
|
41
|
+
puts " #{'%6d' % City.count} cities"
|
42
|
+
puts " #{'%6d' % Tagging.count} taggings"
|
43
|
+
puts " #{'%6d' % Tag.count} tags"
|
44
|
+
puts "Ready."
|
45
|
+
|
46
|
+
## some countries
|
47
|
+
|
48
|
+
## add some predefined shortcuts
|
49
|
+
|
50
|
+
##### some countries
|
51
|
+
|
52
|
+
at = Country.find_by_key( 'at' )
|
53
|
+
de = Country.find_by_key( 'de' )
|
54
|
+
en = Country.find_by_key( 'en' )
|
55
|
+
|
56
|
+
us = Country.find_by_key( 'us' )
|
57
|
+
ca = Country.find_by_key( 'ca' )
|
58
|
+
mx = Country.find_by_key( 'mx' )
|
59
|
+
|
60
|
+
### some cities
|
61
|
+
|
62
|
+
nyc = City.find_by_key( 'newyork' )
|
63
|
+
lon = City.find_by_key( 'london' )
|
64
|
+
vie = City.find_by_key( 'wien' )
|
65
|
+
|
66
|
+
## todo: add some predefined tags (e.g. europe, america, g8, euro, etc.)
|
67
|
+
|
68
|
+
|
69
|
+
## turn on activerecord logging to console
|
70
|
+
|
71
|
+
ActiveRecord::Base.logger = Logger.new( STDOUT )
|
data/lib/worlddb/reader.rb
CHANGED
@@ -112,9 +112,11 @@ private
|
|
112
112
|
|
113
113
|
puts "*** parsing data '#{name}' (#{path})..."
|
114
114
|
|
115
|
-
|
115
|
+
reader = ValuesReader.new( logger, path, more_values )
|
116
116
|
|
117
|
-
load_fixtures_worker_for( clazz,
|
117
|
+
load_fixtures_worker_for( clazz, reader )
|
118
|
+
|
119
|
+
Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "file.txt.#{File.mtime(path).strftime('%Y.%m.%d')}" )
|
118
120
|
end
|
119
121
|
|
120
122
|
def load_fixtures_builtin_for( clazz, name, more_values={} ) # load from gem (built-in)
|
@@ -122,72 +124,25 @@ private
|
|
122
124
|
|
123
125
|
puts "*** parsing data '#{name}' (#{path})..."
|
124
126
|
|
125
|
-
|
127
|
+
reader = ValuesReader.new( logger, path, more_values )
|
126
128
|
|
127
|
-
load_fixtures_worker_for( clazz,
|
129
|
+
load_fixtures_worker_for( clazz, reader )
|
130
|
+
|
131
|
+
Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "world.txt.#{WorldDB::VERSION}" )
|
128
132
|
end
|
129
133
|
|
130
|
-
|
134
|
+
|
135
|
+
def load_fixtures_worker_for( clazz, reader )
|
131
136
|
|
132
137
|
## NB: assumes active activerecord db connection
|
133
138
|
##
|
134
139
|
|
135
|
-
|
140
|
+
reader.each_line do |attribs, values|
|
136
141
|
|
137
|
-
if line =~ /^\s*#/
|
138
|
-
# skip komments and do NOT copy to result (keep comments secret!)
|
139
|
-
logger.debug 'skipping comment line'
|
140
|
-
next
|
141
|
-
end
|
142
|
-
|
143
|
-
if line =~ /^\s*$/
|
144
|
-
# kommentar oder leerzeile überspringen
|
145
|
-
logger.debug 'skipping blank line'
|
146
|
-
next
|
147
|
-
end
|
148
|
-
|
149
|
-
# remove leading and trailing whitespace
|
150
|
-
line = line.strip
|
151
|
-
|
152
|
-
puts "line: >>#{line}<<"
|
153
|
-
|
154
|
-
values = line.split(',')
|
155
|
-
|
156
|
-
# remove leading and trailing whitespace for values
|
157
|
-
values = values.map { |value| value.strip }
|
158
|
-
|
159
|
-
## remove comment columns
|
160
|
-
## todo: also removecomments from inside columns ?? why? why not??
|
161
|
-
|
162
|
-
values = values.select do |value|
|
163
|
-
if value =~ /^#/ ## start with # treat it as a comment column; e.g. remove it
|
164
|
-
puts " removing column with value >>#{value}<<"
|
165
|
-
false
|
166
|
-
else
|
167
|
-
true
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
puts " values: >>#{values.join('<< >>')}<<"
|
172
|
-
|
173
|
-
attribs = {
|
174
|
-
key: values[0]
|
175
|
-
}
|
176
|
-
|
177
|
-
## title (split of optional synonyms)
|
178
|
-
# e.g. FC Bayern Muenchen|Bayern Muenchen|Bayern
|
179
|
-
titles = values[1].split('|')
|
180
|
-
|
181
|
-
attribs[ :title ] = titles[0]
|
182
|
-
## add optional synonyms
|
183
|
-
attribs[ :synonyms ] = titles[1..-1].join('|') if titles.size > 1
|
184
|
-
|
185
|
-
attribs = attribs.merge( more_values ) # e.g. merge country_id and other defaults if present
|
186
|
-
|
187
142
|
value_numbers = []
|
188
143
|
|
189
144
|
## check for optional values
|
190
|
-
values
|
145
|
+
values.each_with_index do |value,index|
|
191
146
|
if value =~ /^region:/ ## region:
|
192
147
|
value_region_key = value[7..-1] ## cut off region: prefix
|
193
148
|
value_region = Region.find_by_key!( value_region_key )
|
@@ -227,10 +182,8 @@ private
|
|
227
182
|
|
228
183
|
rec.update_attributes!( attribs )
|
229
184
|
|
230
|
-
end #
|
231
|
-
|
232
|
-
## todo/fix: add Prop.create!( name )
|
233
|
-
|
185
|
+
end # each_line
|
186
|
+
|
234
187
|
end # method load_fixture_worker_for
|
235
188
|
|
236
189
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
class LineReader
|
4
|
+
|
5
|
+
def initialize( logger=nil, path )
|
6
|
+
if logger.nil?
|
7
|
+
@logger = Logger.new(STDOUT)
|
8
|
+
@logger.level = Logger::INFO
|
9
|
+
else
|
10
|
+
@logger = logger
|
11
|
+
end
|
12
|
+
|
13
|
+
@path = path
|
14
|
+
|
15
|
+
## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
|
16
|
+
## - see worlddb/utils.rb
|
17
|
+
@data = File.read_utf8( @path )
|
18
|
+
end
|
19
|
+
|
20
|
+
attr_reader :logger
|
21
|
+
|
22
|
+
|
23
|
+
def each_line
|
24
|
+
@data.each_line do |line|
|
25
|
+
|
26
|
+
if line =~ /^\s*#/
|
27
|
+
# skip komments and do NOT copy to result (keep comments secret!)
|
28
|
+
logger.debug 'skipping comment line'
|
29
|
+
next
|
30
|
+
end
|
31
|
+
|
32
|
+
if line =~ /^\s*$/
|
33
|
+
# kommentar oder leerzeile überspringen
|
34
|
+
logger.debug 'skipping blank line'
|
35
|
+
next
|
36
|
+
end
|
37
|
+
|
38
|
+
# remove leading and trailing whitespace
|
39
|
+
line = line.strip
|
40
|
+
|
41
|
+
yield( line )
|
42
|
+
end # each lines
|
43
|
+
end # method each_line
|
44
|
+
|
45
|
+
end # class LineReader
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
class ValuesReader
|
4
|
+
|
5
|
+
def initialize( logger, path, more_values={} )
|
6
|
+
## todo: check - can we make logger=nil a default arg too?
|
7
|
+
if logger.nil?
|
8
|
+
@logger = Logger.new(STDOUT)
|
9
|
+
@logger.level = Logger::INFO
|
10
|
+
else
|
11
|
+
@logger = logger
|
12
|
+
end
|
13
|
+
|
14
|
+
@path = path
|
15
|
+
|
16
|
+
@more_values = more_values
|
17
|
+
|
18
|
+
@data = File.read_utf8( @path )
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_reader :logger
|
22
|
+
|
23
|
+
def each_line
|
24
|
+
|
25
|
+
@data.each_line do |line|
|
26
|
+
|
27
|
+
if line =~ /^\s*#/
|
28
|
+
# skip komments and do NOT copy to result (keep comments secret!)
|
29
|
+
logger.debug 'skipping comment line'
|
30
|
+
next
|
31
|
+
end
|
32
|
+
|
33
|
+
if line =~ /^\s*$/
|
34
|
+
# kommentar oder leerzeile überspringen
|
35
|
+
logger.debug 'skipping blank line'
|
36
|
+
next
|
37
|
+
end
|
38
|
+
|
39
|
+
# remove leading and trailing whitespace
|
40
|
+
line = line.strip
|
41
|
+
|
42
|
+
puts "line: >>#{line}<<"
|
43
|
+
|
44
|
+
values = line.split(',')
|
45
|
+
|
46
|
+
# remove leading and trailing whitespace for values
|
47
|
+
values = values.map { |value| value.strip }
|
48
|
+
|
49
|
+
## remove comment columns
|
50
|
+
## todo: also removecomments from inside columns ?? why? why not??
|
51
|
+
|
52
|
+
values = values.select do |value|
|
53
|
+
if value =~ /^#/ ## start with # treat it as a comment column; e.g. remove it
|
54
|
+
puts " removing column with value >>#{value}<<"
|
55
|
+
false
|
56
|
+
else
|
57
|
+
true
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
puts " values: >>#{values.join('<< >>')}<<"
|
62
|
+
|
63
|
+
attribs = {
|
64
|
+
key: values[0]
|
65
|
+
}
|
66
|
+
|
67
|
+
## title (split of optional synonyms)
|
68
|
+
# e.g. FC Bayern Muenchen|Bayern Muenchen|Bayern
|
69
|
+
titles = values[1].split('|')
|
70
|
+
|
71
|
+
attribs[ :title ] = titles[0]
|
72
|
+
## add optional synonyms
|
73
|
+
attribs[ :synonyms ] = titles[1..-1].join('|') if titles.size > 1
|
74
|
+
|
75
|
+
attribs = attribs.merge( @more_values ) # e.g. merge country_id and other defaults if present
|
76
|
+
|
77
|
+
yield( attribs, values[2..-1] )
|
78
|
+
|
79
|
+
end # each lines
|
80
|
+
|
81
|
+
end # method each_line
|
82
|
+
|
83
|
+
end # class ValuesReader
|
84
|
+
|
data/lib/worlddb/utils.rb
CHANGED
@@ -91,50 +91,3 @@ end # class File
|
|
91
91
|
title
|
92
92
|
end
|
93
93
|
|
94
|
-
|
95
|
-
class FixtureReader
|
96
|
-
|
97
|
-
def initialize( logger=nil, path )
|
98
|
-
if logger.nil?
|
99
|
-
@logger = Logger.new(STDOUT)
|
100
|
-
@logger.level = Logger::INFO
|
101
|
-
else
|
102
|
-
@logger = logger
|
103
|
-
end
|
104
|
-
|
105
|
-
@path = path
|
106
|
-
|
107
|
-
## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
|
108
|
-
## - see worlddb/utils.rb
|
109
|
-
@data = File.read_utf8( @path )
|
110
|
-
end
|
111
|
-
|
112
|
-
attr_reader :logger
|
113
|
-
|
114
|
-
|
115
|
-
def each_line
|
116
|
-
@data.each_line do |line|
|
117
|
-
|
118
|
-
if line =~ /^\s*#/
|
119
|
-
# skip komments and do NOT copy to result (keep comments secret!)
|
120
|
-
logger.debug 'skipping comment line'
|
121
|
-
next
|
122
|
-
end
|
123
|
-
|
124
|
-
if line =~ /^\s*$/
|
125
|
-
# kommentar oder leerzeile überspringen
|
126
|
-
logger.debug 'skipping blank line'
|
127
|
-
next
|
128
|
-
end
|
129
|
-
|
130
|
-
# remove leading and trailing whitespace
|
131
|
-
line = line.strip
|
132
|
-
|
133
|
-
yield(line)
|
134
|
-
end # lines.each
|
135
|
-
end
|
136
|
-
|
137
|
-
end # class FixtureReader
|
138
|
-
|
139
|
-
|
140
|
-
|
data/lib/worlddb/version.rb
CHANGED
data/lib/worlddb.rb
CHANGED
@@ -30,6 +30,8 @@ require 'worlddb/models/tag'
|
|
30
30
|
require 'worlddb/models/tagging'
|
31
31
|
require 'worlddb/schema' # NB: requires worlddb/models (include WorldDB::Models)
|
32
32
|
require 'worlddb/utils'
|
33
|
+
require 'worlddb/readers/line_reader'
|
34
|
+
require 'worlddb/readers/values_reader'
|
33
35
|
require 'worlddb/reader'
|
34
36
|
require 'worlddb/loader'
|
35
37
|
require 'worlddb/cli/opts'
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: worlddb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 11
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
8
|
+
- 5
|
9
9
|
- 0
|
10
|
-
version: 0.
|
10
|
+
version: 0.5.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Gerald Bauer
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-11-
|
18
|
+
date: 2012-11-21 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: activerecord
|
@@ -164,6 +164,7 @@ files:
|
|
164
164
|
- lib/worlddb.rb
|
165
165
|
- lib/worlddb/cli/opts.rb
|
166
166
|
- lib/worlddb/cli/runner.rb
|
167
|
+
- lib/worlddb/console.rb
|
167
168
|
- lib/worlddb/loader.rb
|
168
169
|
- lib/worlddb/models/city.rb
|
169
170
|
- lib/worlddb/models/country.rb
|
@@ -172,6 +173,8 @@ files:
|
|
172
173
|
- lib/worlddb/models/tag.rb
|
173
174
|
- lib/worlddb/models/tagging.rb
|
174
175
|
- lib/worlddb/reader.rb
|
176
|
+
- lib/worlddb/readers/line_reader.rb
|
177
|
+
- lib/worlddb/readers/values_reader.rb
|
175
178
|
- lib/worlddb/schema.rb
|
176
179
|
- lib/worlddb/utils.rb
|
177
180
|
- lib/worlddb/version.rb
|