throne 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/tasks.rb +328 -0
- data/throne.gemspec +2 -1
- metadata +2 -1
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.5
|
data/lib/tasks.rb
ADDED
@@ -0,0 +1,328 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
# Defines tasks for managing databases and views.
|
4
|
+
#
|
5
|
+
# To reference a file in lib/ from your design doc, use a comment like this
|
6
|
+
#
|
7
|
+
# // !code library.js
|
8
|
+
#
|
9
|
+
# This will include the contents at this location.
|
10
|
+
#
|
11
|
+
# These tasks require an environment variable of SERVER_URL to be set.
|
12
|
+
#
|
13
|
+
# The documents should be stored using the following structure - all items in the
|
14
|
+
# root of base path will be included in all databases, items with a database will be
|
15
|
+
# included into that database only.
|
16
|
+
#
|
17
|
+
# base_path/
|
18
|
+
# |-- lib
|
19
|
+
# | `-- library.js
|
20
|
+
# |-- data
|
21
|
+
# | `-- .json and .yml folders with seed data
|
22
|
+
# |-- design
|
23
|
+
# | `-- <design doc name>
|
24
|
+
# | |-- lists
|
25
|
+
# | | `-- listname.js
|
26
|
+
# | |-- shows
|
27
|
+
# | | `-- showname.js
|
28
|
+
# | |-- validate_doc_update.js
|
29
|
+
# | `-- views
|
30
|
+
# | `-- statuses
|
31
|
+
# | |-- map.js
|
32
|
+
# | `-- reduce.js
|
33
|
+
# |-- <db name>
|
34
|
+
# |-- lib
|
35
|
+
# | `-- library.js
|
36
|
+
# |-- data
|
37
|
+
# | `-- .json and .yml folders with seed data
|
38
|
+
# `-- design
|
39
|
+
# `-- <design doc name>
|
40
|
+
# |-- lists
|
41
|
+
# | `-- listname.js
|
42
|
+
# |-- shows
|
43
|
+
# | `-- showname.js
|
44
|
+
# |-- validate_doc_update.js
|
45
|
+
# `-- views
|
46
|
+
# `-- statuses
|
47
|
+
# |-- map.js
|
48
|
+
# `-- reduce.js
|
49
|
+
#
|
50
|
+
class Throne::Tasks
|
51
|
+
# This will inject Rake tasks for managing the database data. The base path should
|
52
|
+
# be where all the items are stored.
|
53
|
+
#
|
54
|
+
# @param [String] base_path the path where design docs are stored
|
55
|
+
def self.inject_tasks(base_path)
|
56
|
+
namespace :throne do
|
57
|
+
namespace :documents do
|
58
|
+
namespace :load do
|
59
|
+
databases_in_path(base_path).each do |dbname, fulldir|
|
60
|
+
desc "Loads data for database #{dbname}, optionally into db"
|
61
|
+
task dbname.to_sym, :db do |t, args|
|
62
|
+
load_data_for_database(url(dbname, args), base_path, dbname)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
desc "load all dbs"
|
67
|
+
task :all => databases_in_path(base_path)
|
68
|
+
end
|
69
|
+
namespace :dump do
|
70
|
+
desc "Dump data for a database into a YAML file"
|
71
|
+
task :yml, :db do |t, args|
|
72
|
+
dump_docs_from_db(args.db, base_path, :yml)
|
73
|
+
end
|
74
|
+
|
75
|
+
desc "Dump data for a database into a JSON file"
|
76
|
+
task :json, :db do |t, args|
|
77
|
+
dump_docs_from_db(args.db, base_path, :json)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
#design tasks
|
83
|
+
namespace :design do
|
84
|
+
namespace :push do
|
85
|
+
databases_in_path(base_path).each do |dbname, fulldir|
|
86
|
+
desc "pushes designs for database #{dbname}, optionally into db"
|
87
|
+
task dbname.to_sym, :db do |t, args|
|
88
|
+
load_design_documents(url(dbname, args), base_path, dbname)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
desc "push all designs"
|
93
|
+
task :all => databases_in_path(base_path)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# Db Tasks
|
98
|
+
namespace :database do
|
99
|
+
desc "Creates a database if it doesn't exist"
|
100
|
+
task :createdb, :db do |t, args|
|
101
|
+
get_db(args).create_database
|
102
|
+
puts "Database at #{url(args.db)}"
|
103
|
+
end
|
104
|
+
|
105
|
+
desc "Deletes a database"
|
106
|
+
task :deletedb, :db do |t, args|
|
107
|
+
get_db(args).delete_database
|
108
|
+
puts "Database at #{url(args.db)}"
|
109
|
+
end
|
110
|
+
|
111
|
+
desc "Deletes, creates and re-loads a database"
|
112
|
+
task :rebuilddb, :db do |t, args|
|
113
|
+
db = get_db(url(args.db))
|
114
|
+
raise "you must specify the database name (task[db])" unless args.db
|
115
|
+
db.delete_database
|
116
|
+
# re-getting the object will create the DB
|
117
|
+
db = get_db(url(args.db))
|
118
|
+
# load design docs
|
119
|
+
load_design_documents(db.url, base_path, args.db)
|
120
|
+
# load data
|
121
|
+
load_data_for_database(db.url, base_path, args.db)
|
122
|
+
puts "Done."
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
def self.get_db(url)
|
128
|
+
Throne::Database.new(url)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# Loads design documents into the database url, extracted from the source path
|
134
|
+
# The docs should be layed out in the following format:
|
135
|
+
# `-- <design doc name>
|
136
|
+
# |-- lists
|
137
|
+
# | `-- statuses
|
138
|
+
# | `-- list.js
|
139
|
+
# `-- views
|
140
|
+
# `-- statuses
|
141
|
+
# |-- map.js
|
142
|
+
# `-- reduce.js
|
143
|
+
# @param [String] db_url the url of the database to load the data in to
|
144
|
+
# @param [String] source_path the path to search for .yml and .json files
|
145
|
+
def self.load_design_documents(db_url, base_path, database)
|
146
|
+
# for each folder in base path, create a new design doc key
|
147
|
+
# create a lists key
|
148
|
+
# for each path in lists, add a key with the folder name
|
149
|
+
# inside this, there is a key called list, with the contents of the list function
|
150
|
+
# views is the same, except with a map and reduce function
|
151
|
+
paths_for_item(base_path, database, 'design/*').each do |doc_path|
|
152
|
+
doc_name = File.basename(doc_path)
|
153
|
+
doc = {'lists' => {}, 'views' => {}, 'shows' => {}}
|
154
|
+
|
155
|
+
Dir.glob(File.join(doc_path, 'lists', '*.js')) do |list_path|
|
156
|
+
list_name = File.basename(list_path).split('.').first
|
157
|
+
doc['lists'][list_name] = {}
|
158
|
+
doc['lists'][list_name] =
|
159
|
+
inject_code_includes(base_path, database, list_path)
|
160
|
+
end
|
161
|
+
|
162
|
+
Dir.glob(File.join(doc_path, 'shows', '*.js')) do |show_path|
|
163
|
+
show_name = File.basename(show_path).split('.').first
|
164
|
+
doc['shows'][show_name] = {}
|
165
|
+
doc['shows'][show_name] =
|
166
|
+
inject_code_includes(base_path, database, show_path)
|
167
|
+
end
|
168
|
+
|
169
|
+
if File.exists?(vfn = File.join(doc_path, 'validate_doc_update.js'))
|
170
|
+
doc['validate_doc_update'] =
|
171
|
+
inject_code_includes(base_path, database, vfn)
|
172
|
+
end
|
173
|
+
|
174
|
+
Dir.glob(File.join(doc_path, 'views', '*')) do |view_path|
|
175
|
+
view_name = File.basename(view_path)
|
176
|
+
doc['views'][view_name] = {}
|
177
|
+
mapfn = File.join(view_path, 'map.js')
|
178
|
+
reducefn = File.join(view_path, 'reduce.js')
|
179
|
+
doc['views'][view_name]['map'] =
|
180
|
+
inject_code_includes(base_path, database, mapfn) if File.exists?(mapfn)
|
181
|
+
doc['views'][view_name]['reduce'] =
|
182
|
+
inject_code_includes(base_path, database, reducefn) if
|
183
|
+
File.exists?(reducefn)
|
184
|
+
end
|
185
|
+
# try to get the existing doc
|
186
|
+
doc_id = "_design/#{doc_name}"
|
187
|
+
db = Throne::Database.new(db_url)
|
188
|
+
if svr_doc = db.get(doc_id)
|
189
|
+
# merge
|
190
|
+
doc = svr_doc.merge(doc)
|
191
|
+
else
|
192
|
+
doc['_id'] = doc_id
|
193
|
+
end
|
194
|
+
doc['language'] = 'javascript'
|
195
|
+
db.save(doc)
|
196
|
+
puts "Design documents from #{doc_path} loaded"
|
197
|
+
end
|
198
|
+
|
199
|
+
# try and get a document with the design name
|
200
|
+
# if it's there, replace the lists and views keys with above data
|
201
|
+
# otherwise, create a new document, set language to javascript
|
202
|
+
# put document.
|
203
|
+
# WIN
|
204
|
+
end
|
205
|
+
|
206
|
+
# Loads data into the database url from the source path. Picks up .yml and .json
|
207
|
+
# @param [String] db_url the url of the database to load the data in to
|
208
|
+
# @param [String] source_path the path to search for .yml and .json files
|
209
|
+
def self.load_data_for_database(db_url, base_path, database)
|
210
|
+
@db = Throne::Database.new(db_url)
|
211
|
+
items = []
|
212
|
+
doccount = 0
|
213
|
+
# grab and parse all YML files
|
214
|
+
paths_for_item(base_path, database, 'data/**/*.yml').each do |yml|
|
215
|
+
items << YAML::load(File.open(yml))
|
216
|
+
end
|
217
|
+
# and json
|
218
|
+
paths_for_item(base_path, database, 'data/**/*.json').each do |json|
|
219
|
+
items << JSON.parse(File.open(json).read)
|
220
|
+
end
|
221
|
+
# load em up
|
222
|
+
items.each do |item|
|
223
|
+
if item.kind_of? Array
|
224
|
+
item.each do |doc|
|
225
|
+
begin
|
226
|
+
@db.save(doc)
|
227
|
+
rescue RestClient::RequestFailed => e
|
228
|
+
if e.message =~ /409$/
|
229
|
+
puts "Duplicate document - this data has probaby already been loaded"
|
230
|
+
doccount -= 1
|
231
|
+
else
|
232
|
+
raise e
|
233
|
+
end
|
234
|
+
end
|
235
|
+
doccount += 1
|
236
|
+
end
|
237
|
+
elsif item.kind_of? Hash
|
238
|
+
begin
|
239
|
+
@db.save(item)
|
240
|
+
rescue RestClient::RequestFailed => e
|
241
|
+
if e.message =~ /409$/
|
242
|
+
puts "Duplicate document - this data has probaby already been loaded"
|
243
|
+
doccount -= 1
|
244
|
+
else
|
245
|
+
raise e
|
246
|
+
end
|
247
|
+
end
|
248
|
+
doccount += 1
|
249
|
+
else
|
250
|
+
puts "There is something funky with the data for #{source_path}"
|
251
|
+
end
|
252
|
+
end
|
253
|
+
puts "#{doccount} document(s) loaded into database at #{@db.url}"
|
254
|
+
end
|
255
|
+
|
256
|
+
def self.dump_docs_from_db(db, base_path, format)
|
257
|
+
raise "You must specify a DB name to dump task[dbname]" unless db
|
258
|
+
outdir = File.join(base_path, db, "data")
|
259
|
+
FileUtils.mkdir_p(outdir) unless File.exists?(outdir)
|
260
|
+
@db = Throne::Database.new(url(db))
|
261
|
+
docs = []
|
262
|
+
|
263
|
+
@db.function('_all_docs') do |res|
|
264
|
+
docs << @db.get(res['key']) unless res['key'].match(/^_/)
|
265
|
+
end
|
266
|
+
|
267
|
+
outfn = format == :json ? 'dump.json' : 'dump.yml'
|
268
|
+
File.open(File.join(outdir, outfn), 'w') do |f|
|
269
|
+
if format == :json
|
270
|
+
f.puts JSON.pretty_generate(docs)
|
271
|
+
elsif format == :yml
|
272
|
+
f.puts docs.to_yaml
|
273
|
+
else
|
274
|
+
raise("Internal Error - invalid dump format specified")
|
275
|
+
end
|
276
|
+
end
|
277
|
+
puts "Data dumped to #{base_path + '/' + db + '/' + outfn}"
|
278
|
+
end
|
279
|
+
|
280
|
+
private
|
281
|
+
|
282
|
+
def self.inject_code_includes(base_path, database, file)
|
283
|
+
res = ''
|
284
|
+
File.open(file).each do |line|
|
285
|
+
if line =~ /(\/\/|#)\ ?!code (.*)/
|
286
|
+
if File.exists?(inc = File.join(base_path, database, 'lib', $2))
|
287
|
+
res += File.read(inc)
|
288
|
+
elsif File.exists?(inc = File.join(base_path, 'lib', $2))
|
289
|
+
res += File.read(inc)
|
290
|
+
else
|
291
|
+
raise "Include file #{$2} does not exist in lib/ or #{database}/lib"
|
292
|
+
end
|
293
|
+
else
|
294
|
+
res += line
|
295
|
+
end
|
296
|
+
end
|
297
|
+
res
|
298
|
+
end
|
299
|
+
|
300
|
+
# For the gives path, returns a list of database names and their full path.
|
301
|
+
# exludes lib and our other dirs.
|
302
|
+
def self.databases_in_path(path)
|
303
|
+
res = {}
|
304
|
+
Dir.glob(path + '/*').each do |fn|
|
305
|
+
next if %w(lib data design).any? {|i| File.basename(fn) == i}
|
306
|
+
res[File.basename(fn)] = fn if File.directory?(fn)
|
307
|
+
end
|
308
|
+
res
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.url(dbname, args=nil)
|
312
|
+
if svrurl = ENV['SERVER_URL']
|
313
|
+
return svrurl + '/' + args.db if args && args.db
|
314
|
+
return svrurl + '/' + dbname
|
315
|
+
else
|
316
|
+
raise("You must provide a SERVER_URL")
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
# Given the base dir, the DB name and the item path we are after, get a list
|
321
|
+
# of items. This includes global and DB spec.
|
322
|
+
# e.g for item - data , designs, lib.
|
323
|
+
def self.paths_for_item(base, database, item_glob)
|
324
|
+
items = Dir.glob(File.join(base, item_glob))
|
325
|
+
items.concat(Dir.glob(File.join(base, database, item_glob)))
|
326
|
+
end
|
327
|
+
|
328
|
+
end
|
data/throne.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{throne}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.5"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Lincoln Stoll"]
|
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
"Rakefile",
|
26
26
|
"TODO",
|
27
27
|
"VERSION",
|
28
|
+
"lib/tasks.rb",
|
28
29
|
"lib/throne.rb",
|
29
30
|
"lib/throne/database.rb",
|
30
31
|
"lib/throne/document.rb",
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: throne
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lincoln Stoll
|
@@ -70,6 +70,7 @@ files:
|
|
70
70
|
- Rakefile
|
71
71
|
- TODO
|
72
72
|
- VERSION
|
73
|
+
- lib/tasks.rb
|
73
74
|
- lib/throne.rb
|
74
75
|
- lib/throne/database.rb
|
75
76
|
- lib/throne/document.rb
|