exwiw 0.9.21 → 0.9.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/README.md +101 -10
- data/docs/mongodb.md +21 -1
- data/lib/exwiw/cli.rb +190 -3
- data/lib/exwiw/db_introspector/mysql_introspector.rb +160 -0
- data/lib/exwiw/db_introspector/postgresql_introspector.rb +215 -0
- data/lib/exwiw/db_introspector.rb +166 -0
- data/lib/exwiw/db_schema_generator.rb +298 -0
- data/lib/exwiw/mongodb_collection_config.rb +21 -12
- data/lib/exwiw/mongoid_schema_generator.rb +468 -69
- data/lib/exwiw/schema_check.rb +48 -6
- data/lib/exwiw/table_config.rb +1 -1
- data/lib/exwiw/version.rb +1 -1
- data/lib/exwiw.rb +4 -0
- data/lib/tasks/exwiw.rake +37 -0
- metadata +5 -1
|
@@ -31,21 +31,68 @@ module Exwiw
|
|
|
31
31
|
end
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Every generated collection is keyed by the MongoDB document id. Named so
|
|
35
|
+
# the value the configs carry and the value `DefaultMask` interpolates into a
|
|
36
|
+
# text mask (`masked-{_id}`) provably come from one place.
|
|
37
|
+
PRIMARY_KEY = "_id"
|
|
38
|
+
|
|
39
|
+
# Mongoid field types (`Model.fields[name].type`, a Ruby class) mapped to the
|
|
40
|
+
# type symbols `DefaultMask.for` understands, so safe mode can reuse the very
|
|
41
|
+
# same default masks the ActiveRecord generator emits.
|
|
42
|
+
#
|
|
43
|
+
# Keyed by class NAME rather than by the class itself: this constant is
|
|
44
|
+
# evaluated when exwiw is loaded, which happens in processes that never load
|
|
45
|
+
# Mongoid (the CLI), so naming `Mongoid::Boolean` here would raise. A name
|
|
46
|
+
# comparison also sidesteps `DateTime < Date` — an ancestry test would have to
|
|
47
|
+
# order its branches, while exact names cannot be confused for one another.
|
|
48
|
+
#
|
|
49
|
+
# Everything absent from this map (Hash, Array, Object — which is also what a
|
|
50
|
+
# typeless/dynamic field reports — BSON types, Mongoid::StringifiedSymbol, ...)
|
|
51
|
+
# deliberately gets NO default mask: a constant that does not fit the field
|
|
52
|
+
# would be restored as a value the application cannot read, which is worse
|
|
53
|
+
# than exporting the field while its `needs_mask_decision` flag keeps the
|
|
54
|
+
# change from being merged.
|
|
55
|
+
MASKABLE_FIELD_TYPES = {
|
|
56
|
+
"String" => :string,
|
|
57
|
+
"Integer" => :integer,
|
|
58
|
+
"Float" => :float,
|
|
59
|
+
"BigDecimal" => :decimal,
|
|
60
|
+
"Mongoid::Boolean" => :boolean,
|
|
61
|
+
"Time" => :datetime,
|
|
62
|
+
"DateTime" => :datetime,
|
|
63
|
+
"ActiveSupport::TimeWithZone" => :datetime,
|
|
64
|
+
"Date" => :date,
|
|
65
|
+
}.freeze
|
|
66
|
+
|
|
34
67
|
# `skip_unsupported`: when true, the generator does not abort on a construct
|
|
35
68
|
# it cannot represent. It skips an unresolvable `belongs_to` (keeping the
|
|
36
69
|
# foreign-key field) and emits an unrepresentable embedded collection as an
|
|
37
70
|
# `ignore: true` top-level config annotated with a `comment`, warning to
|
|
38
71
|
# stderr in both cases. Off by default, so the historical fail-loud behavior
|
|
39
72
|
# is unchanged unless a caller opts in.
|
|
40
|
-
|
|
73
|
+
#
|
|
74
|
+
# `safe_new_columns` mirrors `SchemaGenerator`; see #initialize.
|
|
75
|
+
def self.from_rails_application(output_dir:, skip_unsupported: false, safe_new_columns: true)
|
|
41
76
|
Rails.application.eager_load!
|
|
42
|
-
new(
|
|
77
|
+
new(
|
|
78
|
+
models: ::Mongoid.models,
|
|
79
|
+
output_dir: output_dir,
|
|
80
|
+
skip_unsupported: skip_unsupported,
|
|
81
|
+
safe_new_columns: safe_new_columns,
|
|
82
|
+
)
|
|
43
83
|
end
|
|
44
84
|
|
|
45
|
-
|
|
85
|
+
# `safe_new_columns` (the default) emits every field masked — as far as its
|
|
86
|
+
# Mongoid type allows, see MASKABLE_FIELD_TYPES and DefaultMask — and flagged
|
|
87
|
+
# `needs_mask_decision: true`. MongodbCollectionConfig#merge lets an existing
|
|
88
|
+
# entry win, so in practice only fields a model change has just added keep
|
|
89
|
+
# that treatment. Pass false to bootstrap a config, where every field is new
|
|
90
|
+
# and flagging all of them at once is noise.
|
|
91
|
+
def initialize(models:, output_dir:, skip_unsupported: false, safe_new_columns: true)
|
|
46
92
|
@models = models
|
|
47
93
|
@output_dir = output_dir
|
|
48
94
|
@skip_unsupported = skip_unsupported
|
|
95
|
+
@safe_new_columns = safe_new_columns
|
|
49
96
|
end
|
|
50
97
|
|
|
51
98
|
def generate!
|
|
@@ -68,10 +115,118 @@ module Exwiw
|
|
|
68
115
|
# on a construct the user has deliberately ignored. Empty (the default) when
|
|
69
116
|
# called directly without an output dir, in which case nothing is honored.
|
|
70
117
|
def build_collections(existing_by_name = {})
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
118
|
+
models_by_collection_name.map do |collection_name, group|
|
|
119
|
+
build_collection_for(collection_name, group, existing_by_name[collection_name])
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Reconcile the config files already on disk against the models, deleting the
|
|
124
|
+
# config of a collection no model stores into any more. This is the
|
|
125
|
+
# counterpart of `generate!`, which adds and updates config files but can
|
|
126
|
+
# never delete one: a collection whose model was removed leaves a config that
|
|
127
|
+
# would otherwise be dumped forever.
|
|
128
|
+
#
|
|
129
|
+
# Fields are deliberately NOT touched: `generate!`'s #merge already drives the
|
|
130
|
+
# field list from the model, so a field the model lost is dropped there.
|
|
131
|
+
#
|
|
132
|
+
# Unlike `SchemaGenerator#tidy!`, the source of truth is the model set, not a
|
|
133
|
+
# live connection. MongoDB has no schema to read: a collection exists only
|
|
134
|
+
# once a document is written to it, so "still in the database" is not a
|
|
135
|
+
# question that can be answered before a dump, and introspection here is
|
|
136
|
+
# class-level (see this class's preamble) and needs no connection at all. The
|
|
137
|
+
# name set therefore comes from exactly the grouping `generate!` writes files
|
|
138
|
+
# from (`models_by_collection_name`, descendants expanded and embedded models
|
|
139
|
+
# included), so the two can never disagree about which files are live.
|
|
140
|
+
#
|
|
141
|
+
# A config declaring `embedded_in` is never deleted, whatever its name — see
|
|
142
|
+
# #hand_written_embedded_config?.
|
|
143
|
+
#
|
|
144
|
+
# Returns a `SchemaGenerator::TidyResult` — reused rather than duplicated,
|
|
145
|
+
# since a removal report is the same shape whatever the ORM — describing the
|
|
146
|
+
# removals so callers (e.g. the rake task) can report them. Its
|
|
147
|
+
# `removed_columns` stays empty, for the reason above.
|
|
148
|
+
def tidy!
|
|
149
|
+
result = SchemaGenerator::TidyResult.new
|
|
150
|
+
return result unless @output_dir && File.directory?(@output_dir)
|
|
151
|
+
|
|
152
|
+
live_names = models_by_collection_name.keys
|
|
153
|
+
|
|
154
|
+
Dir[File.join(@output_dir, "*.json")].sort.each do |path|
|
|
155
|
+
config = read_raw_config(path)
|
|
156
|
+
|
|
157
|
+
# A file that does not parse says nothing about what it describes: its
|
|
158
|
+
# basename would stand in for the collection name, and for a hand-written
|
|
159
|
+
# embedded config that basename matches no collection by construction —
|
|
160
|
+
# so the guard below would be bypassed for exactly the files it exists to
|
|
161
|
+
# protect, and a stray merge-conflict marker would be enough to delete a
|
|
162
|
+
# collection's masking rules. Broken JSON already fails loudly wherever
|
|
163
|
+
# the config is loaded, so leave it in place and say so.
|
|
164
|
+
if config.nil?
|
|
165
|
+
warn("exwiw: skipping '#{path}' while tidying: it is not valid JSON, so what it describes cannot be determined.")
|
|
166
|
+
next
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
name = declared_name(config, path)
|
|
170
|
+
next if live_names.include?(name)
|
|
171
|
+
next if hand_written_embedded_config?(config)
|
|
172
|
+
|
|
173
|
+
File.delete(path)
|
|
174
|
+
result.add_removed_table(name)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
result
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Whether a config on disk describes an embedded collection this generator
|
|
181
|
+
# could not have written, and whose liveness it therefore cannot judge.
|
|
182
|
+
#
|
|
183
|
+
# `generate!` writes one config per *collection name*, so when two embedded
|
|
184
|
+
# classes share a collection name — which they routinely do, since an embedded
|
|
185
|
+
# class's collection name is derived from the class name alone (`Address`
|
|
186
|
+
# embedded under two different parents is `addresses` both times) — only one
|
|
187
|
+
# of them is emitted. The other can only be expressed by hand, under a name
|
|
188
|
+
# that says where it lives (`orders_deliveries_addresses`) rather than naming
|
|
189
|
+
# a collection. Such a name matches no model by construction, so "no model
|
|
190
|
+
# stores into it" is not evidence that it is dead: it is evidence that the
|
|
191
|
+
# config exists precisely because generation cannot reach it.
|
|
192
|
+
#
|
|
193
|
+
# Deleting one is silent and expensive — an embedded config carries the
|
|
194
|
+
# masking rules applied to those subdocuments, so removing it exports them
|
|
195
|
+
# raw, and nothing else in the pipeline notices. So an `embedded_in` config
|
|
196
|
+
# is kept whatever its name; a genuinely dead one is removed by hand, which is
|
|
197
|
+
# also how it arrived.
|
|
198
|
+
private def hand_written_embedded_config?(config)
|
|
199
|
+
config.is_a?(Hash) && !config["embedded_in"].nil?
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# The collection -> models grouping both `build_collections` and `tidy!` work
|
|
203
|
+
# from: every model that stores into a collection, keyed by that collection's
|
|
204
|
+
# name. Models are grouped by `collection_name` so an STI hierarchy collapses
|
|
205
|
+
# into one entry (see `build_collections`), and `expand_with_descendants`
|
|
206
|
+
# supplies the subclasses Mongoid does not register.
|
|
207
|
+
#
|
|
208
|
+
# Shared so the two operations can never disagree about which collections
|
|
209
|
+
# exist: `tidy!` deletes exactly the config files `generate!` would not write.
|
|
210
|
+
private def models_by_collection_name
|
|
211
|
+
expand_with_descendants(concrete_models).group_by { |model| model.collection_name.to_s }
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# A config file on disk as plain JSON, rather than through
|
|
215
|
+
# `MongodbCollectionConfig.from` on purpose: a stale file is exactly the one
|
|
216
|
+
# that may no longer satisfy the current validations (an unknown key, a
|
|
217
|
+
# belongs_to whose target is gone), and tidy has to be able to delete such a
|
|
218
|
+
# file rather than abort on it. Unparseable JSON reads as nil, which the
|
|
219
|
+
# caller treats as "cannot judge this file" rather than as an empty config.
|
|
220
|
+
private def read_raw_config(path)
|
|
221
|
+
JSON.parse(File.read(path))
|
|
222
|
+
rescue JSON::ParserError
|
|
223
|
+
nil
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# The collection a config declares itself to be. Without a readable `name` it
|
|
227
|
+
# falls back to the file's basename, which is the name `write_files` gives it.
|
|
228
|
+
private def declared_name(config, path)
|
|
229
|
+
(config.is_a?(Hash) && config["name"]) || File.basename(path, ".json")
|
|
75
230
|
end
|
|
76
231
|
|
|
77
232
|
# Loads the configs already on disk so the generator can honor an explicit
|
|
@@ -120,50 +275,108 @@ module Exwiw
|
|
|
120
275
|
return existing if existing&.ignore
|
|
121
276
|
|
|
122
277
|
ordered = models.sort_by { |model| [model.fields.size, model.name] }
|
|
278
|
+
embedded_models, non_embedded_models = ordered.partition(&:embedded?)
|
|
279
|
+
|
|
280
|
+
return top_level_collection(collection_name, ordered, existing) if embedded_models.empty?
|
|
281
|
+
|
|
282
|
+
# `embedded?` means "declares an `embedded_in`", not "has root documents":
|
|
283
|
+
# a plain base class of embedded documents declares none but stores nothing
|
|
284
|
+
# at the root either. Reading it as a root model would flip the family to a
|
|
285
|
+
# top-level config and delete the `embedded_in` that masks its subdocuments,
|
|
286
|
+
# so a non-embedded model with embedded descendants in the group counts as
|
|
287
|
+
# their base — even if root documents were also stored under it (keeps the
|
|
288
|
+
# generator's historical behavior rather than risking a masking regression).
|
|
289
|
+
root_models = non_embedded_models.reject do |model|
|
|
290
|
+
embedded_models.any? { |embedded| embedded < model }
|
|
291
|
+
end
|
|
123
292
|
|
|
124
|
-
|
|
293
|
+
# A root model left after that is a genuine name collision (an embedded
|
|
294
|
+
# class derives its collection name from the class name alone). The
|
|
295
|
+
# collection has root documents that must keep being dumped, so generate it
|
|
296
|
+
# as top-level from the root models only — emitting `embedded_in` here would
|
|
297
|
+
# make `MongodbAdapter#dumpable?` skip it silently, and #merge would force
|
|
298
|
+
# that shape back onto a hand-maintained config on every run. The embedded
|
|
299
|
+
# namesakes are masked by a hand-written `embedded_in` config under a
|
|
300
|
+
# synthetic name (which generation and tidy leave alone); the warning says
|
|
301
|
+
# so instead of a generated `comment`, which would overwrite the user's own
|
|
302
|
+
# note via #merge.
|
|
303
|
+
if root_models.any?
|
|
304
|
+
warn_mixed_embedding(collection_name, embedded_models)
|
|
305
|
+
return top_level_collection(collection_name, root_models, existing)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Purely embedded (possibly under a plain base, whose fields keep being
|
|
309
|
+
# unioned in as before).
|
|
310
|
+
embedded_collection(collection_name, ordered)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# The config for a collection dumped in its own right: its models' fields and
|
|
314
|
+
# the belongs_tos its extraction follows.
|
|
315
|
+
private def top_level_collection(collection_name, models, existing)
|
|
316
|
+
MongodbCollectionConfig.from_symbol_keys(
|
|
125
317
|
name: collection_name,
|
|
126
|
-
primary_key:
|
|
127
|
-
|
|
128
|
-
|
|
318
|
+
primary_key: PRIMARY_KEY,
|
|
319
|
+
belongs_tos: aggregate_belongs_tos(models, existing),
|
|
320
|
+
fields: aggregate_fields(models),
|
|
321
|
+
)
|
|
322
|
+
end
|
|
129
323
|
|
|
130
|
-
|
|
324
|
+
# The config for a collection that exists only inside another's documents.
|
|
325
|
+
# `models` is the whole group, which may include the embedded documents' plain
|
|
326
|
+
# base class — its fields belong in the union, but only a class that declares
|
|
327
|
+
# an `embedded_in` can say where the collection lives, hence the `find`.
|
|
328
|
+
private def embedded_collection(collection_name, models)
|
|
329
|
+
attrs = {
|
|
330
|
+
name: collection_name,
|
|
331
|
+
primary_key: PRIMARY_KEY,
|
|
332
|
+
fields: aggregate_fields(models),
|
|
131
333
|
# Cross-collection references from inside an embedded array are not
|
|
132
334
|
# supported (MongodbCollectionConfig rejects them), so embedded configs
|
|
133
335
|
# always carry an empty belongs_tos and instead declare where they live.
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
attrs[:belongs_tos] = aggregate_belongs_tos(ordered, existing)
|
|
336
|
+
belongs_tos: [],
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
begin
|
|
340
|
+
attrs[:embedded_in] = embedded_in_for(models.find(&:embedded?))
|
|
341
|
+
rescue => e
|
|
342
|
+
# Known-unrepresentable shapes arrive as UnsupportedEmbedding (with a
|
|
343
|
+
# concise reason). Without skip_unsupported, re-raise so the historical
|
|
344
|
+
# fail-loud behavior is preserved. The broad rescue is a deliberate
|
|
345
|
+
# safety net for skip_unsupported (a best-effort bootstrapping mode):
|
|
346
|
+
# any other error while deriving the embedding is turned into an
|
|
347
|
+
# `ignore: true` config too, so a single odd model never aborts the run.
|
|
348
|
+
raise e unless @skip_unsupported
|
|
349
|
+
|
|
350
|
+
reason =
|
|
351
|
+
if e.is_a?(UnsupportedEmbedding)
|
|
352
|
+
e.reason
|
|
353
|
+
else
|
|
354
|
+
"raised #{e.class} while deriving embedded_in (#{e.message.lines.first&.strip})"
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Emit the collection as a top-level config marked `ignore: true` so it
|
|
358
|
+
# is NOT (wrongly) dumped as its own collection, and record why. The
|
|
359
|
+
# user can hand-write its embedded_in config later to dump/mask it.
|
|
360
|
+
warn("exwiw: skip_unsupported: '#{collection_name}' #{reason}; emitting ignore:true (define embedded_in by hand to dump/mask it).")
|
|
361
|
+
attrs[:ignore] = true
|
|
362
|
+
attrs[:comment] = "exwiw could not derive embedded_in (#{reason}); marked ignore:true. Define this collection's embedded_in config by hand to dump/mask it."
|
|
162
363
|
end
|
|
163
364
|
|
|
164
365
|
MongodbCollectionConfig.from_symbol_keys(attrs)
|
|
165
366
|
end
|
|
166
367
|
|
|
368
|
+
# Names the collision on stderr, once per affected collection.
|
|
369
|
+
private def warn_mixed_embedding(collection_name, embedded_models)
|
|
370
|
+
warn(
|
|
371
|
+
"exwiw: collection '#{collection_name}' is stored into by both top-level and embedded models " \
|
|
372
|
+
"(embedded: #{embedded_models.map(&:name).sort.join(', ')}); generating it as a TOP-LEVEL collection " \
|
|
373
|
+
"from its non-embedded model(s) only, so its root documents keep being dumped. The embedded " \
|
|
374
|
+
"documents of the same name are NOT covered by this config: to mask them, add a config by hand " \
|
|
375
|
+
"with an `embedded_in` and a distinct `name`/file name of your choosing (generation and tidy leave " \
|
|
376
|
+
"embedded configs alone)."
|
|
377
|
+
)
|
|
378
|
+
end
|
|
379
|
+
|
|
167
380
|
# Mongoid registers only the base class of an inheritance hierarchy in
|
|
168
381
|
# `Mongoid.models`; subclasses that store into the base's collection
|
|
169
382
|
# (STI-style, distinguished by the auto-added `_type` discriminator) are
|
|
@@ -184,11 +397,20 @@ module Exwiw
|
|
|
184
397
|
concrete(@models)
|
|
185
398
|
end
|
|
186
399
|
|
|
400
|
+
# The single place a model is judged fit to describe (used by generate! and
|
|
401
|
+
# tidy! alike): Mongoid's internal helper classes are dropped, and so is any
|
|
402
|
+
# class with an empty `collection_name` — `store_in collection: nil` is the
|
|
403
|
+
# never-persisted-model idiom, and a config for the nameless group such
|
|
404
|
+
# classes collapse into describes nothing (as top-level it would even
|
|
405
|
+
# instruct the dump to read a collection with no name). The emptiness test
|
|
406
|
+
# stays last: the preceding conditions establish that calling
|
|
407
|
+
# `collection_name` is meaningful.
|
|
187
408
|
private def concrete(models)
|
|
188
409
|
models.select do |model|
|
|
189
410
|
model.respond_to?(:collection_name) &&
|
|
190
411
|
model.name &&
|
|
191
|
-
!model.name.start_with?("Mongoid::")
|
|
412
|
+
!model.name.start_with?("Mongoid::") &&
|
|
413
|
+
!model.collection_name.to_s.empty?
|
|
192
414
|
end
|
|
193
415
|
end
|
|
194
416
|
|
|
@@ -196,10 +418,13 @@ module Exwiw
|
|
|
196
418
|
# order. A subclass's `fields` already includes everything it inherits, so
|
|
197
419
|
# the base's fields lead and each subclass appends only its own.
|
|
198
420
|
private def aggregate_fields(models)
|
|
421
|
+
structural = structural_field_names(models)
|
|
422
|
+
unique = unique_field_names(models)
|
|
423
|
+
|
|
199
424
|
seen = {}
|
|
200
425
|
models.each_with_object([]) do |model, fields|
|
|
201
426
|
accessor_by_storage = aliased_field_accessors(model)
|
|
202
|
-
model.fields.
|
|
427
|
+
model.fields.each do |name, definition|
|
|
203
428
|
next if seen[name]
|
|
204
429
|
|
|
205
430
|
seen[name] = true
|
|
@@ -208,9 +433,129 @@ module Exwiw
|
|
|
208
433
|
# Ruby accessor so the short key is not cryptic in the config.
|
|
209
434
|
accessor = accessor_by_storage[name]
|
|
210
435
|
field[:mongoid_field_name] = accessor if accessor
|
|
211
|
-
fields << field
|
|
436
|
+
fields << field.merge(safe_mode_attributes(name, definition, structural, unique))
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
# The safe-mode additions to a field entry: the `needs_mask_decision` flag,
|
|
442
|
+
# plus a default `replace_with` where a safe one exists. Empty when safe mode
|
|
443
|
+
# is off, so the emitted config is byte-identical to the historical output.
|
|
444
|
+
#
|
|
445
|
+
# A structural field is flagged but never masked: masking it would break the
|
|
446
|
+
# very lookups the dump is assembled from (see #structural_field_names).
|
|
447
|
+
private def safe_mode_attributes(name, definition, structural, unique)
|
|
448
|
+
return {} unless @safe_new_columns
|
|
449
|
+
|
|
450
|
+
attrs = { needs_mask_decision: true }
|
|
451
|
+
return attrs if structural.include?(name)
|
|
452
|
+
|
|
453
|
+
mask = default_mask_for(name, definition, unique.include?(name))
|
|
454
|
+
mask.nil? ? attrs : attrs.merge(replace_with: mask)
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
# The default mask for one field, or nil when no safe constant fits its
|
|
458
|
+
# Mongoid type (see MASKABLE_FIELD_TYPES).
|
|
459
|
+
#
|
|
460
|
+
# `limit: nil` because MongoDB stores no per-field length: the length checks
|
|
461
|
+
# DefaultMask applies to a `varchar(n)` have nothing to read here, and any
|
|
462
|
+
# string the mask renders is storable. `unique` comes from the model's index
|
|
463
|
+
# declarations, so a constant is never emitted for a field a unique index
|
|
464
|
+
# would then collide on across every restored document.
|
|
465
|
+
#
|
|
466
|
+
# NOTE the mask a date/time field gets is DefaultMask's fixed *string*
|
|
467
|
+
# constant, which replaces a BSON Date with a String and so changes the
|
|
468
|
+
# field's BSON type. That is deliberate at this stage: the value is only a
|
|
469
|
+
# proposal, and it rides on `needs_mask_decision` precisely so a human either
|
|
470
|
+
# keeps it knowingly, replaces it, or drops it before the config is merged.
|
|
471
|
+
private def default_mask_for(name, definition, unique)
|
|
472
|
+
type = MASKABLE_FIELD_TYPES[mongoid_type_name(definition)]
|
|
473
|
+
return nil if type.nil?
|
|
474
|
+
|
|
475
|
+
DefaultMask.for(
|
|
476
|
+
name: name,
|
|
477
|
+
type: type,
|
|
478
|
+
limit: nil,
|
|
479
|
+
primary_key: PRIMARY_KEY,
|
|
480
|
+
unique: unique,
|
|
481
|
+
column_default: field_default(definition),
|
|
482
|
+
)
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
# `Model.fields[name].type` is a Ruby class; an anonymous one (or a field
|
|
486
|
+
# object that does not expose a type at all) has no name to look up, which
|
|
487
|
+
# MASKABLE_FIELD_TYPES answers with "no default mask".
|
|
488
|
+
private def mongoid_type_name(definition)
|
|
489
|
+
type = definition.type if definition.respond_to?(:type)
|
|
490
|
+
type.name if type.is_a?(Module)
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
# The field's declared `default:`, handed to DefaultMask so a field with a
|
|
494
|
+
# default of its own is masked with that value rather than the per-type
|
|
495
|
+
# constant (see DefaultMask.constant_mask). Mongoid also accepts a *lambda*
|
|
496
|
+
# default (`default: -> { Time.now }`), which is not a constant at all;
|
|
497
|
+
# DefaultMask's scalar_default answers such a value with nil, so it falls
|
|
498
|
+
# through to the per-type constant on its own.
|
|
499
|
+
private def field_default(definition)
|
|
500
|
+
definition.default_val if definition.respond_to?(:default_val)
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
# The fields safe mode flags but must never mask, because the dump is
|
|
504
|
+
# assembled by looking documents up through them:
|
|
505
|
+
#
|
|
506
|
+
# - the primary key `_id`, which every belongs_to and every `--ids` filter
|
|
507
|
+
# resolves against (and which the text masks interpolate to stay unique),
|
|
508
|
+
# - the STI discriminator (`Model.discriminator_key`, `_type` by default),
|
|
509
|
+
# which is what tells one class's documents from another's inside the
|
|
510
|
+
# shared collection, and
|
|
511
|
+
# - every `belongs_to` foreign key declared by any model of this collection.
|
|
512
|
+
#
|
|
513
|
+
# The foreign keys are taken from the MODELS, not from the emitted
|
|
514
|
+
# `belongs_tos`, so the ones the generator deliberately drops are covered
|
|
515
|
+
# too: a polymorphic belongs_to (excluded because it has no single target
|
|
516
|
+
# collection) and a referenced belongs_to on an embedded document (excluded
|
|
517
|
+
# because an embedded config carries none) still store a reference some other
|
|
518
|
+
# collection's documents are found by, and masking it would silently rewrite
|
|
519
|
+
# that reference. A polymorphic relation's type field (`reviewable_type`)
|
|
520
|
+
# is exempt for the same reason, mirroring how the ActiveRecord generator
|
|
521
|
+
# treats a belongs_to's `foreign_type`.
|
|
522
|
+
private def structural_field_names(models)
|
|
523
|
+
names = [PRIMARY_KEY]
|
|
524
|
+
|
|
525
|
+
models.each do |model|
|
|
526
|
+
names << model.discriminator_key.to_s if model.respond_to?(:discriminator_key)
|
|
527
|
+
model.relations.each_value do |assoc|
|
|
528
|
+
next unless assoc.is_a?(::Mongoid::Association::Referenced::BelongsTo)
|
|
529
|
+
|
|
530
|
+
names << assoc.foreign_key.to_s
|
|
531
|
+
names << assoc.inverse_type.to_s if assoc.polymorphic? && assoc.inverse_type
|
|
212
532
|
end
|
|
213
533
|
end
|
|
534
|
+
|
|
535
|
+
names.uniq
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
# The fields covered by a unique index declared on any model of this
|
|
539
|
+
# collection (`index({ email: 1 }, unique: true)`), so DefaultMask withholds
|
|
540
|
+
# any mask that does not vary per document — a constant would collapse every
|
|
541
|
+
# document onto one value and break the restore with a duplicate key. Every
|
|
542
|
+
# field of a compound unique index counts, mirroring the ActiveRecord
|
|
543
|
+
# generator's "any column of a unique index".
|
|
544
|
+
#
|
|
545
|
+
# Index declarations are class-level Mongoid metadata (they are what
|
|
546
|
+
# `db/mongoid.rake`'s create_indexes would build), so this needs no
|
|
547
|
+
# connection — unlike the ActiveRecord generator, which reads the live
|
|
548
|
+
# database and has to cope with that failing. An index created out of band
|
|
549
|
+
# and never declared on the model is therefore invisible here; declaring it
|
|
550
|
+
# is what makes exwiw (and Mongoid itself) aware of it.
|
|
551
|
+
private def unique_field_names(models)
|
|
552
|
+
models.flat_map do |model|
|
|
553
|
+
next [] unless model.respond_to?(:index_specifications)
|
|
554
|
+
|
|
555
|
+
model.index_specifications
|
|
556
|
+
.select { |spec| spec.options[:unique] }
|
|
557
|
+
.flat_map { |spec| spec.key.keys.map(&:to_s) }
|
|
558
|
+
end.uniq
|
|
214
559
|
end
|
|
215
560
|
|
|
216
561
|
# Maps a stored document key -> its Mongoid Ruby accessor, but ONLY for
|
|
@@ -231,7 +576,7 @@ module Exwiw
|
|
|
231
576
|
end
|
|
232
577
|
|
|
233
578
|
private def aggregate_belongs_tos(models, existing = nil)
|
|
234
|
-
|
|
579
|
+
ignored = ignored_belongs_tos(existing)
|
|
235
580
|
|
|
236
581
|
belongs_to_assocs = models.flat_map do |model|
|
|
237
582
|
model.relations.values.select do |assoc|
|
|
@@ -248,20 +593,68 @@ module Exwiw
|
|
|
248
593
|
# same belongs_to twice, so uniq them.
|
|
249
594
|
belongs_to_assocs
|
|
250
595
|
.reject(&:polymorphic?)
|
|
251
|
-
.filter_map { |assoc| belongs_to_for(assoc,
|
|
596
|
+
.filter_map { |assoc| belongs_to_for(assoc, ignored) }
|
|
252
597
|
.uniq
|
|
253
598
|
end
|
|
254
599
|
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
|
|
260
|
-
|
|
600
|
+
# The on-disk `ignore: true` belongs_to entries, so a relation the user has
|
|
601
|
+
# explicitly ignored is preserved verbatim instead of re-resolved (which, for
|
|
602
|
+
# a stale relation whose target class is gone, would otherwise abort the run).
|
|
603
|
+
#
|
|
604
|
+
# Kept as a list rather than indexed by foreign key: a foreign key does not
|
|
605
|
+
# identify a relation on its own (see #ignored_belongs_to_for).
|
|
606
|
+
private def ignored_belongs_tos(existing)
|
|
607
|
+
return [] unless existing
|
|
261
608
|
|
|
262
|
-
existing.belongs_tos.select(&:ignore)
|
|
263
|
-
|
|
264
|
-
|
|
609
|
+
existing.belongs_tos.select(&:ignore)
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
# The on-disk ignored entry that stands for `assoc`, or nil when none does.
|
|
613
|
+
#
|
|
614
|
+
# `target_collection` is the association's resolved target, or nil when it
|
|
615
|
+
# could not be resolved, and that distinction is what the match is keyed on:
|
|
616
|
+
#
|
|
617
|
+
# - Resolved: the entry has to agree on BOTH the target collection and the
|
|
618
|
+
# foreign key. Two belongs_tos of one collection can legitimately share a
|
|
619
|
+
# foreign key — a relation to one collection plus a second relation to
|
|
620
|
+
# another declared with `primary_key:`, scoping by the same stored value —
|
|
621
|
+
# and matching on the foreign key alone let one ignored entry stand for both
|
|
622
|
+
# of them. The second relation was then re-emitted as a copy of that entry,
|
|
623
|
+
# `uniq` collapsed the two, and its edge silently vanished from the config.
|
|
624
|
+
# An entry naming a different target must therefore not swallow this
|
|
625
|
+
# relation; with none matching, the relation is generated normally.
|
|
626
|
+
# - Unresolved: there is no target to compare, which is precisely the case
|
|
627
|
+
# this mechanism exists for (a stale relation whose class is gone, whose
|
|
628
|
+
# entry usually carries no `table_name` either), so fall back to matching on
|
|
629
|
+
# the foreign key alone.
|
|
630
|
+
#
|
|
631
|
+
# An entry with no `table_name` still matches a resolved relation on the
|
|
632
|
+
# foreign key alone, as a fallback after the exact match: omitting the target
|
|
633
|
+
# is allowed only on an ignored entry, and dropping the user's decision
|
|
634
|
+
# because they wrote the minimal form would resurrect an edge they had
|
|
635
|
+
# deliberately cut. Two foreign-key-sharing relations with one such entry
|
|
636
|
+
# between them remain genuinely indistinguishable — writing the `table_name`
|
|
637
|
+
# is what tells exwiw which of them is meant.
|
|
638
|
+
private def ignored_belongs_to_for(assoc, target_collection, ignored)
|
|
639
|
+
foreign_key = assoc.foreign_key
|
|
640
|
+
return ignored.find { |bt| bt.foreign_key == foreign_key } if target_collection.nil?
|
|
641
|
+
|
|
642
|
+
ignored.find { |bt| bt.foreign_key == foreign_key && bt.table_name == target_collection } ||
|
|
643
|
+
ignored.find { |bt| bt.foreign_key == foreign_key && bt.table_name.nil? }
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
# The collection a referenced belongs_to targets, as `[collection_name, nil]`
|
|
647
|
+
# — or `[nil, error]` when the association's target class no longer exists (a
|
|
648
|
+
# stale/legacy `belongs_to`, e.g. pointing at a model removed years ago), so
|
|
649
|
+
# `assoc.klass` raised. The error is carried rather than raised because
|
|
650
|
+
# resolution is attempted BEFORE the on-disk ignored entries are consulted (to
|
|
651
|
+
# match one against the right relation), and a relation the user has already
|
|
652
|
+
# triaged must not abort the run; `belongs_to_for` raises it only once no
|
|
653
|
+
# entry stands for the relation.
|
|
654
|
+
private def resolve_target_collection(assoc)
|
|
655
|
+
[assoc.klass.collection_name.to_s, nil]
|
|
656
|
+
rescue NameError, ::Mongoid::Errors::MongoidError => e
|
|
657
|
+
[nil, e]
|
|
265
658
|
end
|
|
266
659
|
|
|
267
660
|
# Resolves a referenced belongs_to to a `{ table_name, foreign_key }` pair
|
|
@@ -272,18 +665,29 @@ module Exwiw
|
|
|
272
665
|
# its foreign-key column is still tracked as an ordinary field by
|
|
273
666
|
# `aggregate_fields`, mirroring how polymorphic / HABTM relations are dropped.
|
|
274
667
|
#
|
|
275
|
-
# `
|
|
276
|
-
#
|
|
277
|
-
# it, so preserve their entry verbatim (its `ignore_type` /
|
|
278
|
-
#
|
|
279
|
-
# extraction at load (`#reject_ignored_members!`) while its FK
|
|
280
|
-
# field, and the run never aborts on a relation already
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
668
|
+
# `ignored` carries the on-disk `ignore: true` belongs_to entries: when one of
|
|
669
|
+
# them stands for this relation (see #ignored_belongs_to_for), the user has
|
|
670
|
+
# explicitly ignored it, so preserve their entry verbatim (its `ignore_type` /
|
|
671
|
+
# `comment`) rather than emitting a freshly derived one. The relation is
|
|
672
|
+
# dropped from extraction at load (`#reject_ignored_members!`) while its FK
|
|
673
|
+
# column stays a field, and the run never aborts on a relation already
|
|
674
|
+
# triaged: resolving the — possibly gone — target is attempted first, but its
|
|
675
|
+
# failure is only raised once no entry stands for the relation.
|
|
676
|
+
private def belongs_to_for(assoc, ignored = [])
|
|
677
|
+
target_collection, resolution_error = resolve_target_collection(assoc)
|
|
678
|
+
|
|
679
|
+
if (entry = ignored_belongs_to_for(assoc, target_collection, ignored))
|
|
680
|
+
return preserve_ignored_belongs_to(entry)
|
|
284
681
|
end
|
|
285
682
|
|
|
286
|
-
|
|
683
|
+
if resolution_error
|
|
684
|
+
raise resolution_error unless @skip_unsupported
|
|
685
|
+
|
|
686
|
+
warn("exwiw: skip_unsupported: skipping belongs_to ':#{assoc.name}' that could not be resolved (#{resolution_error.class}: #{resolution_error.message.lines.first&.strip}); its foreign key '#{assoc.foreign_key}' is still kept as a field.")
|
|
687
|
+
return nil
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
result = { table_name: target_collection, foreign_key: assoc.foreign_key }
|
|
287
691
|
# Mongoid's `belongs_to ..., primary_key: :uuid` makes the child's foreign
|
|
288
692
|
# key reference that parent field rather than the parent's `_id`. Surface
|
|
289
693
|
# it as `references` so MongodbAdapter constrains children by the right
|
|
@@ -293,11 +697,6 @@ module Exwiw
|
|
|
293
697
|
reference_field = assoc.primary_key.to_s
|
|
294
698
|
result[:references] = reference_field unless reference_field == "_id"
|
|
295
699
|
result
|
|
296
|
-
rescue NameError, ::Mongoid::Errors::MongoidError => e
|
|
297
|
-
raise e unless @skip_unsupported
|
|
298
|
-
|
|
299
|
-
warn("exwiw: skip_unsupported: skipping belongs_to ':#{assoc.name}' that could not be resolved (#{e.class}: #{e.message.lines.first&.strip}); its foreign key '#{assoc.foreign_key}' is still kept as a field.")
|
|
300
|
-
nil
|
|
301
700
|
end
|
|
302
701
|
|
|
303
702
|
# Re-emits a user's on-disk ignored belongs_to as a symbol-keyed hash (the
|