wgit 0.5.1 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.yardopts +7 -0
- data/CHANGELOG.md +249 -0
- data/CODE_OF_CONDUCT.md +76 -0
- data/CONTRIBUTING.md +21 -0
- data/LICENSE.txt +21 -0
- data/README.md +232 -0
- data/bin/wgit +39 -0
- data/lib/wgit.rb +3 -1
- data/lib/wgit/assertable.rb +3 -3
- data/lib/wgit/base.rb +30 -0
- data/lib/wgit/core_ext.rb +1 -1
- data/lib/wgit/crawler.rb +304 -148
- data/lib/wgit/database/database.rb +310 -135
- data/lib/wgit/database/model.rb +10 -3
- data/lib/wgit/document.rb +241 -169
- data/lib/wgit/{document_extensions.rb → document_extractors.rb} +20 -10
- data/lib/wgit/dsl.rb +324 -0
- data/lib/wgit/indexer.rb +68 -156
- data/lib/wgit/response.rb +17 -14
- data/lib/wgit/url.rb +213 -73
- data/lib/wgit/utils.rb +32 -20
- data/lib/wgit/version.rb +3 -2
- metadata +38 -19
@@ -9,17 +9,42 @@ require 'logger'
|
|
9
9
|
require 'mongo'
|
10
10
|
|
11
11
|
module Wgit
|
12
|
-
# Class
|
13
|
-
# collections.
|
12
|
+
# Class providing a DB connection and CRUD operations for the Url and
|
13
|
+
# Document collections.
|
14
14
|
class Database
|
15
15
|
include Assertable
|
16
16
|
|
17
|
+
# The default name of the urls collection.
|
18
|
+
URLS_COLLECTION = :urls
|
19
|
+
|
20
|
+
# The default name of the documents collection.
|
21
|
+
DOCUMENTS_COLLECTION = :documents
|
22
|
+
|
23
|
+
# The default name of the documents collection text search index.
|
24
|
+
TEXT_INDEX = 'text_search'
|
25
|
+
|
26
|
+
# The default name of the urls and documents collections unique index.
|
27
|
+
UNIQUE_INDEX = 'unique_url'
|
28
|
+
|
29
|
+
# The documents collection default text search index. Use
|
30
|
+
# `db.text_index = Wgit::Database::DEFAULT_TEXT_INDEX` to revert changes.
|
31
|
+
DEFAULT_TEXT_INDEX = {
|
32
|
+
title: 2,
|
33
|
+
description: 2,
|
34
|
+
keywords: 2,
|
35
|
+
text: 1
|
36
|
+
}.freeze
|
37
|
+
|
17
38
|
# The connection string for the database.
|
18
39
|
attr_reader :connection_string
|
19
40
|
|
20
41
|
# The database client object. Gets set when a connection is established.
|
21
42
|
attr_reader :client
|
22
43
|
|
44
|
+
# The documents collection text index, used to search the DB.
|
45
|
+
# A custom setter method is also provided for changing the search logic.
|
46
|
+
attr_reader :text_index
|
47
|
+
|
23
48
|
# Initializes a connected database client using the provided
|
24
49
|
# connection_string or ENV['WGIT_CONNECTION_STRING'].
|
25
50
|
#
|
@@ -34,6 +59,7 @@ module Wgit
|
|
34
59
|
|
35
60
|
@client = Database.establish_connection(connection_string)
|
36
61
|
@connection_string = connection_string
|
62
|
+
@text_index = DEFAULT_TEXT_INDEX
|
37
63
|
end
|
38
64
|
|
39
65
|
# A class alias for Database.new.
|
@@ -63,31 +89,132 @@ module Wgit
|
|
63
89
|
Mongo::Client.new(connection_string)
|
64
90
|
end
|
65
91
|
|
92
|
+
### DDL ###
|
93
|
+
|
94
|
+
# Creates the urls and documents collections if they don't already exist.
|
95
|
+
# This method is therefore idempotent.
|
96
|
+
#
|
97
|
+
# @return [nil] Always returns nil.
|
98
|
+
def create_collections
|
99
|
+
db.client[URLS_COLLECTION].create rescue nil
|
100
|
+
db.client[DOCUMENTS_COLLECTION].create rescue nil
|
101
|
+
|
102
|
+
nil
|
103
|
+
end
|
104
|
+
|
105
|
+
# Creates the urls and documents unique 'url' indexes if they don't already
|
106
|
+
# exist. This method is therefore idempotent.
|
107
|
+
#
|
108
|
+
# @return [nil] Always returns nil.
|
109
|
+
def create_unique_indexes
|
110
|
+
@client[URLS_COLLECTION].indexes.create_one(
|
111
|
+
{ url: 1 }, name: UNIQUE_INDEX, unique: true
|
112
|
+
) rescue nil
|
113
|
+
|
114
|
+
@client[DOCUMENTS_COLLECTION].indexes.create_one(
|
115
|
+
{ 'url.url' => 1 }, name: UNIQUE_INDEX, unique: true
|
116
|
+
) rescue nil
|
117
|
+
|
118
|
+
nil
|
119
|
+
end
|
120
|
+
|
121
|
+
# Set the documents collection text search index aka the fields to #search.
|
122
|
+
# This is labor intensive on large collections so change little and wisely.
|
123
|
+
# This method is idempotent in that it will remove the index if it already
|
124
|
+
# exists before it creates the new index.
|
125
|
+
#
|
126
|
+
# @param fields [Array<Symbol>, Hash<Symbol, Integer>] The field names or
|
127
|
+
# the field names and their coresponding search weights.
|
128
|
+
# @return [Array<Symbol>, Hash] The passed in value of fields. Use
|
129
|
+
# `#text_index` to get the new index's fields and weights.
|
130
|
+
# @raise [StandardError] If fields is of an incorrect type or an error
|
131
|
+
# occurs with the underlying DB client.
|
132
|
+
def text_index=(fields)
|
133
|
+
# We want to end up with a Hash of fields (Symbols) and their
|
134
|
+
# weights (Integers).
|
135
|
+
case fields
|
136
|
+
when Array # of Strings/Symbols.
|
137
|
+
fields = fields.map { |field| [field.to_sym, 1] }
|
138
|
+
when Hash # of Strings/Symbols and Integers.
|
139
|
+
fields = fields.map { |field, weight| [field.to_sym, weight.to_i] }
|
140
|
+
else
|
141
|
+
raise "fields must be an Array or Hash, not a #{fields.class}"
|
142
|
+
end
|
143
|
+
|
144
|
+
fields = fields.to_h
|
145
|
+
indexes = @client[DOCUMENTS_COLLECTION].indexes
|
146
|
+
|
147
|
+
indexes.drop_one(TEXT_INDEX) if indexes.get(TEXT_INDEX)
|
148
|
+
indexes.create_one(
|
149
|
+
fields.map { |field, _| [field, 'text'] }.to_h,
|
150
|
+
{ name: TEXT_INDEX, weights: fields, background: true }
|
151
|
+
)
|
152
|
+
|
153
|
+
@text_index = fields
|
154
|
+
end
|
155
|
+
|
66
156
|
### Create Data ###
|
67
157
|
|
68
158
|
# Insert one or more Url or Document objects into the DB.
|
69
159
|
#
|
70
160
|
# @param data [Wgit::Url, Wgit::Document, Enumerable<Wgit::Url,
|
71
|
-
# Wgit::Document>]
|
72
|
-
# Wgit::Model.document.
|
161
|
+
# Wgit::Document>] The records to insert/create.
|
73
162
|
# @raise [StandardError] If data isn't valid.
|
74
163
|
def insert(data)
|
75
164
|
data = data.dup # Avoid modifying by reference.
|
76
|
-
|
165
|
+
collection = nil
|
77
166
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
167
|
+
if data.respond_to?(:map!)
|
168
|
+
data.map! do |obj|
|
169
|
+
collection, _, model = get_type_info(obj)
|
170
|
+
model
|
171
|
+
end
|
83
172
|
else
|
84
|
-
|
173
|
+
collection, _, model = get_type_info(data)
|
174
|
+
data = model
|
85
175
|
end
|
176
|
+
|
177
|
+
create(collection, data)
|
178
|
+
end
|
179
|
+
|
180
|
+
# Inserts or updates the object in the database.
|
181
|
+
#
|
182
|
+
# @param obj [Wgit::Url, Wgit::Document] The obj/record to insert/update.
|
183
|
+
# @return [Boolean] True if inserted, false if updated.
|
184
|
+
def upsert(obj)
|
185
|
+
collection, query, model = get_type_info(obj.dup)
|
186
|
+
data_hash = model.merge(Wgit::Model.common_update_data)
|
187
|
+
result = @client[collection].replace_one(query, data_hash, upsert: true)
|
188
|
+
|
189
|
+
result.matched_count.zero?
|
86
190
|
end
|
87
191
|
|
88
192
|
### Retrieve Data ###
|
89
193
|
|
90
|
-
# Returns
|
194
|
+
# Returns all Document records from the DB. Use #search to filter based on
|
195
|
+
# the text_index of the collection.
|
196
|
+
#
|
197
|
+
# All Documents are sorted by date_added ascending, in other words the
|
198
|
+
# first doc returned is the first one that was inserted into the DB.
|
199
|
+
#
|
200
|
+
# @param limit [Integer] The max number of returned records. 0 returns all.
|
201
|
+
# @param skip [Integer] Skip n records.
|
202
|
+
# @yield [doc] Given each Document object (Wgit::Document) returned from
|
203
|
+
# the DB.
|
204
|
+
# @return [Array<Wgit::Document>] The Documents obtained from the DB.
|
205
|
+
def docs(limit: 0, skip: 0)
|
206
|
+
results = retrieve(DOCUMENTS_COLLECTION, {},
|
207
|
+
sort: { date_added: 1 }, limit: limit, skip: skip)
|
208
|
+
return [] if results.count < 1 # results#empty? doesn't exist.
|
209
|
+
|
210
|
+
# results.respond_to? :map! is false so we use map and overwrite the var.
|
211
|
+
results = results.map { |doc_hash| Wgit::Document.new(doc_hash) }
|
212
|
+
results.each { |doc| yield(doc) } if block_given?
|
213
|
+
|
214
|
+
results
|
215
|
+
end
|
216
|
+
|
217
|
+
# Returns all Url records from the DB.
|
91
218
|
#
|
92
219
|
# All Urls are sorted by date_added ascending, in other words the first url
|
93
220
|
# returned is the first one that was inserted into the DB.
|
@@ -101,9 +228,8 @@ module Wgit
|
|
101
228
|
query = crawled.nil? ? {} : { crawled: crawled }
|
102
229
|
sort = { date_added: 1 }
|
103
230
|
|
104
|
-
results = retrieve(
|
105
|
-
sort: sort,
|
106
|
-
limit: limit, skip: skip)
|
231
|
+
results = retrieve(URLS_COLLECTION, query,
|
232
|
+
sort: sort, limit: limit, skip: skip)
|
107
233
|
return [] if results.count < 1 # results#empty? doesn't exist.
|
108
234
|
|
109
235
|
# results.respond_to? :map! is false so we use map and overwrite the var.
|
@@ -154,7 +280,7 @@ module Wgit
|
|
154
280
|
# DB.
|
155
281
|
# @return [Array<Wgit::Document>] The search results obtained from the DB.
|
156
282
|
def search(
|
157
|
-
query, case_sensitive: false, whole_sentence:
|
283
|
+
query, case_sensitive: false, whole_sentence: true, limit: 10, skip: 0
|
158
284
|
)
|
159
285
|
query = query.to_s.strip
|
160
286
|
query.replace('"' + query + '"') if whole_sentence
|
@@ -167,14 +293,59 @@ module Wgit
|
|
167
293
|
:$caseSensitive => case_sensitive
|
168
294
|
} }
|
169
295
|
|
170
|
-
results = retrieve(
|
296
|
+
results = retrieve(DOCUMENTS_COLLECTION, query,
|
171
297
|
sort: sort_proj, projection: sort_proj,
|
172
298
|
limit: limit, skip: skip)
|
173
299
|
return [] if results.count < 1 # respond_to? :empty? == false
|
174
300
|
|
175
301
|
# results.respond_to? :map! is false so we use map and overwrite the var.
|
176
|
-
results = results.map
|
177
|
-
|
302
|
+
results = results.map do |mongo_doc|
|
303
|
+
doc = Wgit::Document.new(mongo_doc)
|
304
|
+
yield(doc) if block_given?
|
305
|
+
doc
|
306
|
+
end
|
307
|
+
|
308
|
+
results
|
309
|
+
end
|
310
|
+
|
311
|
+
# Searches the database's Documents for the given query and then searches
|
312
|
+
# each result in turn using `doc.search!`. This method is therefore the
|
313
|
+
# equivalent of calling `Wgit::Database#search` and then
|
314
|
+
# `Wgit::Document#search!` in turn. See their documentation for more info.
|
315
|
+
#
|
316
|
+
# @param query [String] The text query to search with.
|
317
|
+
# @param case_sensitive [Boolean] Whether character case must match.
|
318
|
+
# @param whole_sentence [Boolean] Whether multiple words should be searched
|
319
|
+
# for separately.
|
320
|
+
# @param limit [Integer] The max number of results to return.
|
321
|
+
# @param skip [Integer] The number of results to skip.
|
322
|
+
# @param sentence_limit [Integer] The max length of each search result
|
323
|
+
# sentence.
|
324
|
+
# @yield [doc] Given each search result (Wgit::Document) returned from the
|
325
|
+
# DB having called `doc.search!(query)`.
|
326
|
+
# @return [Array<Wgit::Document>] The search results obtained from the DB
|
327
|
+
# having called `doc.search!(query)`.
|
328
|
+
def search!(
|
329
|
+
query, case_sensitive: false, whole_sentence: true,
|
330
|
+
limit: 10, skip: 0, sentence_limit: 80
|
331
|
+
)
|
332
|
+
results = search(
|
333
|
+
query,
|
334
|
+
case_sensitive: case_sensitive,
|
335
|
+
whole_sentence: whole_sentence,
|
336
|
+
limit: limit,
|
337
|
+
skip: skip
|
338
|
+
)
|
339
|
+
|
340
|
+
results.each do |doc|
|
341
|
+
doc.search!(
|
342
|
+
query,
|
343
|
+
case_sensitive: case_sensitive,
|
344
|
+
whole_sentence: whole_sentence,
|
345
|
+
sentence_limit: sentence_limit
|
346
|
+
)
|
347
|
+
yield(doc) if block_given?
|
348
|
+
end
|
178
349
|
|
179
350
|
results
|
180
351
|
end
|
@@ -197,14 +368,14 @@ module Wgit
|
|
197
368
|
#
|
198
369
|
# @return [Integer] The current number of URL records.
|
199
370
|
def num_urls
|
200
|
-
@client[
|
371
|
+
@client[URLS_COLLECTION].count
|
201
372
|
end
|
202
373
|
|
203
374
|
# Returns the total number of Document records in the DB.
|
204
375
|
#
|
205
376
|
# @return [Integer] The current number of Document records.
|
206
377
|
def num_docs
|
207
|
-
@client[
|
378
|
+
@client[DOCUMENTS_COLLECTION].count
|
208
379
|
end
|
209
380
|
|
210
381
|
# Returns the total number of records (urls + docs) in the DB.
|
@@ -221,8 +392,8 @@ module Wgit
|
|
221
392
|
# @return [Boolean] True if url exists, otherwise false.
|
222
393
|
def url?(url)
|
223
394
|
assert_type(url, String) # This includes Wgit::Url's.
|
224
|
-
|
225
|
-
|
395
|
+
query = { url: url }
|
396
|
+
retrieve(URLS_COLLECTION, query, limit: 1).any?
|
226
397
|
end
|
227
398
|
|
228
399
|
# Returns whether or not a record with the given doc 'url.url' field
|
@@ -232,114 +403,111 @@ module Wgit
|
|
232
403
|
# @return [Boolean] True if doc exists, otherwise false.
|
233
404
|
def doc?(doc)
|
234
405
|
assert_type(doc, Wgit::Document)
|
235
|
-
|
236
|
-
|
406
|
+
query = { 'url.url' => doc.url }
|
407
|
+
retrieve(DOCUMENTS_COLLECTION, query, limit: 1).any?
|
237
408
|
end
|
238
409
|
|
239
|
-
|
410
|
+
# Returns if a record exists with the given obj's url.
|
411
|
+
#
|
412
|
+
# @param obj [Wgit::Url, Wgit::Document] Object containing the url to
|
413
|
+
# search for.
|
414
|
+
# @return [Boolean] True if a record exists with the url, false otherwise.
|
415
|
+
def exists?(obj)
|
416
|
+
obj.is_a?(String) ? url?(obj) : doc?(obj)
|
417
|
+
end
|
240
418
|
|
241
|
-
#
|
419
|
+
# Returns a record from the database with the matching 'url' field; or nil.
|
420
|
+
# Pass either a Wgit::Url or Wgit::Document instance.
|
242
421
|
#
|
243
|
-
# @param
|
244
|
-
# @
|
245
|
-
|
246
|
-
|
422
|
+
# @param obj [Wgit::Url, Wgit::Document] The record to search the DB for.
|
423
|
+
# @return [Wgit::Url, Wgit::Document, nil] The record with the matching
|
424
|
+
# 'url' field or nil if no results can be found.
|
425
|
+
def get(obj)
|
426
|
+
collection, query = get_type_info(obj)
|
247
427
|
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
update_doc(data)
|
253
|
-
else
|
254
|
-
raise "Unsupported type - #{data.class}: #{data}"
|
255
|
-
end
|
428
|
+
record = retrieve(collection, query, limit: 1).first
|
429
|
+
return nil unless record
|
430
|
+
|
431
|
+
obj.class.new(record)
|
256
432
|
end
|
257
433
|
|
258
|
-
|
434
|
+
### Update Data ###
|
259
435
|
|
260
|
-
#
|
436
|
+
# Update a Url or Document object in the DB.
|
261
437
|
#
|
262
|
-
# @param
|
263
|
-
# @raise [StandardError] If
|
264
|
-
# @return [Integer] The number of
|
265
|
-
def
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
assert_type(data, Wgit::Url)
|
271
|
-
data = Wgit::Model.url(data)
|
272
|
-
end
|
273
|
-
|
274
|
-
create(:urls, data)
|
438
|
+
# @param obj [Wgit::Url, Wgit::Document] The obj/record to update.
|
439
|
+
# @raise [StandardError] If the obj is not valid.
|
440
|
+
# @return [Integer] The number of updated records/objects.
|
441
|
+
def update(obj)
|
442
|
+
collection, query, model = get_type_info(obj.dup)
|
443
|
+
data_hash = model.merge(Wgit::Model.common_update_data)
|
444
|
+
|
445
|
+
mutate(collection, query, { '$set' => data_hash })
|
275
446
|
end
|
276
447
|
|
277
|
-
|
448
|
+
### Delete Data ###
|
449
|
+
|
450
|
+
# Deletes a record from the database with the matching 'url' field.
|
451
|
+
# Pass either a Wgit::Url or Wgit::Document instance.
|
278
452
|
#
|
279
|
-
# @param
|
280
|
-
#
|
281
|
-
# @
|
282
|
-
#
|
283
|
-
def
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
else
|
288
|
-
assert_types(data, Wgit::Document)
|
289
|
-
data = Wgit::Model.document(data)
|
290
|
-
end
|
453
|
+
# @param obj [Wgit::Url, Wgit::Document] The record to search the DB for
|
454
|
+
# and delete.
|
455
|
+
# @return [Integer] The number of records deleted - should always be
|
456
|
+
# 0 or 1 because urls are unique.
|
457
|
+
def delete(obj)
|
458
|
+
collection, query = get_type_info(obj)
|
459
|
+
@client[collection].delete_one(query).n
|
460
|
+
end
|
291
461
|
|
292
|
-
|
462
|
+
# Deletes everything in the urls collection.
|
463
|
+
#
|
464
|
+
# @return [Integer] The number of deleted records.
|
465
|
+
def clear_urls
|
466
|
+
@client[URLS_COLLECTION].delete_many({}).n
|
293
467
|
end
|
294
468
|
|
295
|
-
#
|
469
|
+
# Deletes everything in the documents collection.
|
296
470
|
#
|
297
|
-
# @
|
298
|
-
|
299
|
-
|
300
|
-
assert_type(url, Wgit::Url)
|
301
|
-
selection = { url: url }
|
302
|
-
url_hash = Wgit::Model.url(url).merge(Wgit::Model.common_update_data)
|
303
|
-
update = { '$set' => url_hash }
|
304
|
-
mutate(true, :urls, selection, update)
|
471
|
+
# @return [Integer] The number of deleted records.
|
472
|
+
def clear_docs
|
473
|
+
@client[DOCUMENTS_COLLECTION].delete_many({}).n
|
305
474
|
end
|
306
475
|
|
307
|
-
#
|
476
|
+
# Deletes everything in the urls and documents collections. This will nuke
|
477
|
+
# the entire database so yeah... be careful.
|
308
478
|
#
|
309
|
-
# @
|
310
|
-
|
311
|
-
|
312
|
-
assert_type(doc, Wgit::Document)
|
313
|
-
selection = { 'url.url' => doc.url }
|
314
|
-
doc_hash = Wgit::Model.document(doc).merge(Wgit::Model.common_update_data)
|
315
|
-
update = { '$set' => doc_hash }
|
316
|
-
mutate(true, :documents, selection, update)
|
479
|
+
# @return [Integer] The number of deleted records.
|
480
|
+
def clear_db
|
481
|
+
clear_urls + clear_docs
|
317
482
|
end
|
318
483
|
|
319
484
|
private
|
320
485
|
|
321
|
-
#
|
486
|
+
# Get the database's type info (collection type, query hash, model) for
|
487
|
+
# obj.
|
322
488
|
#
|
323
|
-
#
|
324
|
-
#
|
325
|
-
#
|
326
|
-
# @
|
327
|
-
# @
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
489
|
+
# Raises an error if obj isn't a Wgit::Url or Wgit::Document.
|
490
|
+
# Note, that no database calls are made during this method call.
|
491
|
+
#
|
492
|
+
# @param obj [Wgit::Url, Wgit::Document] The obj to get semantics for.
|
493
|
+
# @raise [StandardError] If obj isn't a Wgit::Url or Wgit::Document.
|
494
|
+
# @return [Array<Symbol, Hash>] The collection type, query to get
|
495
|
+
# the record/obj from the database (if it exists) and the model of obj.
|
496
|
+
def get_type_info(obj)
|
497
|
+
case obj
|
498
|
+
when Wgit::Url
|
499
|
+
collection = URLS_COLLECTION
|
500
|
+
query = { url: obj.to_s }
|
501
|
+
model = Wgit::Model.url(obj)
|
502
|
+
when Wgit::Document
|
503
|
+
collection = DOCUMENTS_COLLECTION
|
504
|
+
query = { 'url.url' => obj.url.to_s }
|
505
|
+
model = Wgit::Model.document(obj)
|
340
506
|
else
|
341
|
-
raise "
|
507
|
+
raise "obj must be a Wgit::Url or Wgit::Document, not: #{obj.class}"
|
342
508
|
end
|
509
|
+
|
510
|
+
[collection, query, model]
|
343
511
|
end
|
344
512
|
|
345
513
|
# Create/insert one or more Url or Document records into the DB.
|
@@ -351,22 +519,20 @@ module Wgit
|
|
351
519
|
def create(collection, data)
|
352
520
|
assert_types(data, [Hash, Array])
|
353
521
|
|
354
|
-
# Single doc.
|
355
522
|
case data
|
356
|
-
when Hash
|
523
|
+
when Hash # Single record.
|
357
524
|
data.merge!(Wgit::Model.common_insert_data)
|
358
525
|
result = @client[collection.to_sym].insert_one(data)
|
359
526
|
raise 'DB write (insert) failed' unless write_succeeded?(result)
|
360
527
|
|
361
528
|
result.n
|
362
|
-
# Multiple
|
363
|
-
|
364
|
-
assert_arr_types(data, Hash)
|
529
|
+
when Array # Multiple records.
|
530
|
+
assert_arr_type(data, Hash)
|
365
531
|
data.map! { |hash| hash.merge(Wgit::Model.common_insert_data) }
|
366
532
|
result = @client[collection.to_sym].insert_many(data)
|
367
|
-
|
368
|
-
|
369
|
-
|
533
|
+
unless write_succeeded?(result, num_writes: data.length)
|
534
|
+
raise 'DB write(s) (insert) failed'
|
535
|
+
end
|
370
536
|
|
371
537
|
result.inserted_count
|
372
538
|
else
|
@@ -374,6 +540,26 @@ module Wgit
|
|
374
540
|
end
|
375
541
|
end
|
376
542
|
|
543
|
+
# Return if the write to the DB succeeded or not.
|
544
|
+
#
|
545
|
+
# @param result [Mongo::Collection::View] The write result.
|
546
|
+
# @param num_writes [Integer] The number of records written to.
|
547
|
+
# @raise [StandardError] If the result type isn't supported.
|
548
|
+
# @return [Boolean] True if the write was successful, false otherwise.
|
549
|
+
def write_succeeded?(result, num_writes: 1)
|
550
|
+
case result
|
551
|
+
when Mongo::Operation::Insert::Result # Single create result.
|
552
|
+
result.documents.first[:err].nil?
|
553
|
+
when Mongo::BulkWrite::Result # Multiple create result.
|
554
|
+
result.inserted_count == num_writes
|
555
|
+
when Mongo::Operation::Update::Result # Single/multiple update result.
|
556
|
+
singleton = (num_writes == 1)
|
557
|
+
singleton ? result.documents.first[:err].nil? : result.n == num_writes
|
558
|
+
else # Class no longer used, have you upgraded the 'mongo' gem?
|
559
|
+
raise "Result class not currently supported: #{result.class}"
|
560
|
+
end
|
561
|
+
end
|
562
|
+
|
377
563
|
# Retrieve Url or Document records from the DB.
|
378
564
|
#
|
379
565
|
# @param collection [Symbol] Either :urls or :documents.
|
@@ -383,7 +569,7 @@ module Wgit
|
|
383
569
|
# @param limit [Integer] The limit to use.
|
384
570
|
# @param skip [Integer] The skip to use.
|
385
571
|
# @raise [StandardError] If query type isn't valid.
|
386
|
-
# @return [Mongo::
|
572
|
+
# @return [Mongo::Collection::View] The retrieval viewset.
|
387
573
|
def retrieve(collection, query,
|
388
574
|
sort: {}, projection: {},
|
389
575
|
limit: 0, skip: 0)
|
@@ -397,32 +583,21 @@ module Wgit
|
|
397
583
|
# This method expects Model.common_update_data to have been merged in
|
398
584
|
# already by the calling method.
|
399
585
|
#
|
400
|
-
# @param single [Boolean] Wether or not a single record is being updated.
|
401
586
|
# @param collection [Symbol] Either :urls or :documents.
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
end
|
587
|
+
# @param query [Hash] The query used for the retrieval before updating.
|
588
|
+
# @param update [Hash] The updated/new object.
|
589
|
+
# @raise [StandardError] If the update fails.
|
590
|
+
# @return [Integer] The number of updated records/objects.
|
591
|
+
def mutate(collection, query, update)
|
592
|
+
assert_arr_type([query, update], Hash)
|
409
593
|
|
410
|
-
result =
|
411
|
-
|
412
|
-
else
|
413
|
-
@client[collection].update_many(selection, update)
|
414
|
-
end
|
415
|
-
raise 'DB write (update) failed' unless write_succeeded?(result)
|
594
|
+
result = @client[collection.to_sym].update_one(query, update)
|
595
|
+
raise 'DB write(s) (update) failed' unless write_succeeded?(result)
|
416
596
|
|
417
597
|
result.n
|
418
598
|
end
|
419
599
|
|
420
|
-
alias
|
421
|
-
alias
|
422
|
-
alias num_documents num_docs
|
423
|
-
alias document? doc?
|
424
|
-
alias insert_url insert_urls
|
425
|
-
alias insert_doc insert_docs
|
426
|
-
alias num_objects num_records
|
600
|
+
alias num_objects num_records
|
601
|
+
alias clear_db! clear_db
|
427
602
|
end
|
428
603
|
end
|