exwiw 0.9.7 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +90 -27
- data/lib/exwiw/adapter/mongodb_adapter.rb +118 -2
- data/lib/exwiw/determine_table_processing_order.rb +109 -31
- data/lib/exwiw/explain_runner.rb +13 -2
- data/lib/exwiw/japanese_names.rb +315 -0
- data/lib/exwiw/mongodb_collection_config.rb +31 -0
- data/lib/exwiw/row_transformer.rb +161 -19
- data/lib/exwiw/runner.rb +27 -4
- data/lib/exwiw/strict_keys.rb +79 -0
- data/lib/exwiw/table_config.rb +12 -2
- data/lib/exwiw/version.rb +1 -1
- data/lib/exwiw.rb +1 -0
- metadata +3 -1
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Exwiw
|
|
4
|
+
# Paired (kanji, katakana) Japanese name data for the `ja` locale of the
|
|
5
|
+
# person-family `replace_with_fake_data` types (see {RowTransformer}).
|
|
6
|
+
#
|
|
7
|
+
# faker's `ja` locale ships kanji names only, with no reading, so it cannot
|
|
8
|
+
# produce kana (フリガナ) at all, and its last_name / first_name pools are
|
|
9
|
+
# unrelated to each other. exwiw bundles this paired dataset instead so that:
|
|
10
|
+
#
|
|
11
|
+
# - a fake person carries a kana reading that actually matches its kanji, and
|
|
12
|
+
# - last_name / first_name / full name (and their kana) for a single seed all
|
|
13
|
+
# come from the same person (a coherent identity).
|
|
14
|
+
#
|
|
15
|
+
# The kanji sets extend faker's `ja` name list (its top 100 surnames / 100
|
|
16
|
+
# given names) with common additions; exwiw supplies the readings. 142 x 142
|
|
17
|
+
# = 20,164 combinations exceeds the person pool size
|
|
18
|
+
# (RowTransformer::PERSON_POOL_SIZE = 20,000), so the ja pool is filled with
|
|
19
|
+
# that many DISTINCT people rather than repeating a small subset.
|
|
20
|
+
#
|
|
21
|
+
# Readings are katakana (matching the katakana convention of typical `*_kana`
|
|
22
|
+
# columns). They are plausible common readings, not the only possible reading
|
|
23
|
+
# of each kanji — fake data only needs to look real, not be canonical.
|
|
24
|
+
module JapaneseNames
|
|
25
|
+
SURNAMES = [
|
|
26
|
+
%w[佐藤 サトウ],
|
|
27
|
+
%w[鈴木 スズキ],
|
|
28
|
+
%w[高橋 タカハシ],
|
|
29
|
+
%w[田中 タナカ],
|
|
30
|
+
%w[伊藤 イトウ],
|
|
31
|
+
%w[山本 ヤマモト],
|
|
32
|
+
%w[渡辺 ワタナベ],
|
|
33
|
+
%w[中村 ナカムラ],
|
|
34
|
+
%w[小林 コバヤシ],
|
|
35
|
+
%w[加藤 カトウ],
|
|
36
|
+
%w[吉田 ヨシダ],
|
|
37
|
+
%w[山田 ヤマダ],
|
|
38
|
+
%w[佐々木 ササキ],
|
|
39
|
+
%w[山口 ヤマグチ],
|
|
40
|
+
%w[松本 マツモト],
|
|
41
|
+
%w[井上 イノウエ],
|
|
42
|
+
%w[木村 キムラ],
|
|
43
|
+
%w[林 ハヤシ],
|
|
44
|
+
%w[斎藤 サイトウ],
|
|
45
|
+
%w[清水 シミズ],
|
|
46
|
+
%w[山崎 ヤマザキ],
|
|
47
|
+
%w[阿部 アベ],
|
|
48
|
+
%w[森 モリ],
|
|
49
|
+
%w[池田 イケダ],
|
|
50
|
+
%w[橋本 ハシモト],
|
|
51
|
+
%w[山下 ヤマシタ],
|
|
52
|
+
%w[石川 イシカワ],
|
|
53
|
+
%w[中島 ナカジマ],
|
|
54
|
+
%w[前田 マエダ],
|
|
55
|
+
%w[藤田 フジタ],
|
|
56
|
+
%w[後藤 ゴトウ],
|
|
57
|
+
%w[小川 オガワ],
|
|
58
|
+
%w[岡田 オカダ],
|
|
59
|
+
%w[村上 ムラカミ],
|
|
60
|
+
%w[長谷川 ハセガワ],
|
|
61
|
+
%w[近藤 コンドウ],
|
|
62
|
+
%w[石井 イシイ],
|
|
63
|
+
%w[斉藤 サイトウ],
|
|
64
|
+
%w[坂本 サカモト],
|
|
65
|
+
%w[遠藤 エンドウ],
|
|
66
|
+
%w[藤井 フジイ],
|
|
67
|
+
%w[青木 アオキ],
|
|
68
|
+
%w[福田 フクダ],
|
|
69
|
+
%w[三浦 ミウラ],
|
|
70
|
+
%w[西村 ニシムラ],
|
|
71
|
+
%w[藤原 フジワラ],
|
|
72
|
+
%w[太田 オオタ],
|
|
73
|
+
%w[松田 マツダ],
|
|
74
|
+
%w[原田 ハラダ],
|
|
75
|
+
%w[岡本 オカモト],
|
|
76
|
+
%w[中野 ナカノ],
|
|
77
|
+
%w[中川 ナカガワ],
|
|
78
|
+
%w[小野 オノ],
|
|
79
|
+
%w[田村 タムラ],
|
|
80
|
+
%w[竹内 タケウチ],
|
|
81
|
+
%w[金子 カネコ],
|
|
82
|
+
%w[中山 ナカヤマ],
|
|
83
|
+
%w[和田 ワダ],
|
|
84
|
+
%w[石田 イシダ],
|
|
85
|
+
%w[工藤 クドウ],
|
|
86
|
+
%w[上田 ウエダ],
|
|
87
|
+
%w[原 ハラ],
|
|
88
|
+
%w[森田 モリタ],
|
|
89
|
+
%w[酒井 サカイ],
|
|
90
|
+
%w[横山 ヨコヤマ],
|
|
91
|
+
%w[柴田 シバタ],
|
|
92
|
+
%w[宮崎 ミヤザキ],
|
|
93
|
+
%w[宮本 ミヤモト],
|
|
94
|
+
%w[内田 ウチダ],
|
|
95
|
+
%w[高木 タカギ],
|
|
96
|
+
%w[谷口 タニグチ],
|
|
97
|
+
%w[安藤 アンドウ],
|
|
98
|
+
%w[丸山 マルヤマ],
|
|
99
|
+
%w[今井 イマイ],
|
|
100
|
+
%w[大野 オオノ],
|
|
101
|
+
%w[高田 タカダ],
|
|
102
|
+
%w[菅原 スガワラ],
|
|
103
|
+
%w[河野 コウノ],
|
|
104
|
+
%w[武田 タケダ],
|
|
105
|
+
%w[藤本 フジモト],
|
|
106
|
+
%w[上野 ウエノ],
|
|
107
|
+
%w[杉山 スギヤマ],
|
|
108
|
+
%w[千葉 チバ],
|
|
109
|
+
%w[村田 ムラタ],
|
|
110
|
+
%w[増田 マスダ],
|
|
111
|
+
%w[小島 コジマ],
|
|
112
|
+
%w[小山 コヤマ],
|
|
113
|
+
%w[大塚 オオツカ],
|
|
114
|
+
%w[平野 ヒラノ],
|
|
115
|
+
%w[久保 クボ],
|
|
116
|
+
%w[渡部 ワタナベ],
|
|
117
|
+
%w[松井 マツイ],
|
|
118
|
+
%w[菊地 キクチ],
|
|
119
|
+
%w[岩崎 イワサキ],
|
|
120
|
+
%w[松尾 マツオ],
|
|
121
|
+
%w[佐野 サノ],
|
|
122
|
+
%w[木下 キノシタ],
|
|
123
|
+
%w[野口 ノグチ],
|
|
124
|
+
%w[野村 ノムラ],
|
|
125
|
+
%w[新井 アライ],
|
|
126
|
+
%w[北村 キタムラ],
|
|
127
|
+
%w[平田 ヒラタ],
|
|
128
|
+
%w[森本 モリモト],
|
|
129
|
+
%w[荒木 アラキ],
|
|
130
|
+
%w[大西 オオニシ],
|
|
131
|
+
%w[菅野 カンノ],
|
|
132
|
+
%w[小池 コイケ],
|
|
133
|
+
%w[桜井 サクライ],
|
|
134
|
+
%w[高野 タカノ],
|
|
135
|
+
%w[川口 カワグチ],
|
|
136
|
+
%w[大久保 オオクボ],
|
|
137
|
+
%w[中田 ナカタ],
|
|
138
|
+
%w[川崎 カワサキ],
|
|
139
|
+
%w[飯田 イイダ],
|
|
140
|
+
%w[星野 ホシノ],
|
|
141
|
+
%w[岩田 イワタ],
|
|
142
|
+
%w[大森 オオモリ],
|
|
143
|
+
%w[松岡 マツオカ],
|
|
144
|
+
%w[沢田 サワダ],
|
|
145
|
+
%w[内藤 ナイトウ],
|
|
146
|
+
%w[松浦 マツウラ],
|
|
147
|
+
%w[西田 ニシダ],
|
|
148
|
+
%w[白石 シライシ],
|
|
149
|
+
%w[浜田 ハマダ],
|
|
150
|
+
%w[早川 ハヤカワ],
|
|
151
|
+
%w[永井 ナガイ],
|
|
152
|
+
%w[松村 マツムラ],
|
|
153
|
+
%w[三宅 ミヤケ],
|
|
154
|
+
%w[福島 フクシマ],
|
|
155
|
+
%w[大橋 オオハシ],
|
|
156
|
+
%w[中西 ナカニシ],
|
|
157
|
+
%w[吉川 ヨシカワ],
|
|
158
|
+
%w[服部 ハットリ],
|
|
159
|
+
%w[尾崎 オザキ],
|
|
160
|
+
%w[水野 ミズノ],
|
|
161
|
+
%w[香川 カガワ],
|
|
162
|
+
%w[荒井 アライ],
|
|
163
|
+
%w[田口 タグチ],
|
|
164
|
+
%w[江口 エグチ],
|
|
165
|
+
%w[秋山 アキヤマ],
|
|
166
|
+
%w[黒田 クロダ],
|
|
167
|
+
%w[今村 イマムラ],
|
|
168
|
+
].freeze
|
|
169
|
+
|
|
170
|
+
GIVEN_NAMES = [
|
|
171
|
+
%w[翔太 ショウタ],
|
|
172
|
+
%w[蓮 レン],
|
|
173
|
+
%w[翔 ショウ],
|
|
174
|
+
%w[陸 リク],
|
|
175
|
+
%w[颯太 ソウタ],
|
|
176
|
+
%w[悠斗 ユウト],
|
|
177
|
+
%w[大翔 ヒロト],
|
|
178
|
+
%w[翼 ツバサ],
|
|
179
|
+
%w[樹 イツキ],
|
|
180
|
+
%w[奏太 ソウタ],
|
|
181
|
+
%w[大和 ヤマト],
|
|
182
|
+
%w[大輝 ダイキ],
|
|
183
|
+
%w[悠 ユウ],
|
|
184
|
+
%w[隼人 ハヤト],
|
|
185
|
+
%w[健太 ケンタ],
|
|
186
|
+
%w[大輔 ダイスケ],
|
|
187
|
+
%w[駿 シュン],
|
|
188
|
+
%w[陽斗 ハルト],
|
|
189
|
+
%w[優 ユウ],
|
|
190
|
+
%w[陽 ヨウ],
|
|
191
|
+
%w[悠人 ユウト],
|
|
192
|
+
%w[誠 マコト],
|
|
193
|
+
%w[拓海 タクミ],
|
|
194
|
+
%w[仁 ジン],
|
|
195
|
+
%w[悠太 ユウタ],
|
|
196
|
+
%w[悠真 ユウマ],
|
|
197
|
+
%w[大地 ダイチ],
|
|
198
|
+
%w[健 ケン],
|
|
199
|
+
%w[遼 リョウ],
|
|
200
|
+
%w[大樹 ダイキ],
|
|
201
|
+
%w[諒 リョウ],
|
|
202
|
+
%w[響 ヒビキ],
|
|
203
|
+
%w[太一 タイチ],
|
|
204
|
+
%w[一郎 イチロウ],
|
|
205
|
+
%w[優斗 ユウト],
|
|
206
|
+
%w[亮 リョウ],
|
|
207
|
+
%w[海斗 カイト],
|
|
208
|
+
%w[颯 ソウ],
|
|
209
|
+
%w[亮太 リョウタ],
|
|
210
|
+
%w[匠 タクミ],
|
|
211
|
+
%w[陽太 ヨウタ],
|
|
212
|
+
%w[航 ワタル],
|
|
213
|
+
%w[瑛太 エイタ],
|
|
214
|
+
%w[直樹 ナオキ],
|
|
215
|
+
%w[空 ソラ],
|
|
216
|
+
%w[光 ヒカル],
|
|
217
|
+
%w[太郎 タロウ],
|
|
218
|
+
%w[輝 ヒカル],
|
|
219
|
+
%w[一輝 カズキ],
|
|
220
|
+
%w[蒼 アオイ],
|
|
221
|
+
%w[葵 アオイ],
|
|
222
|
+
%w[優那 ユウナ],
|
|
223
|
+
%w[優奈 ユウナ],
|
|
224
|
+
%w[凛 リン],
|
|
225
|
+
%w[陽菜 ヒナ],
|
|
226
|
+
%w[愛 アイ],
|
|
227
|
+
%w[結衣 ユイ],
|
|
228
|
+
%w[美咲 ミサキ],
|
|
229
|
+
%w[楓 カエデ],
|
|
230
|
+
%w[さくら サクラ],
|
|
231
|
+
%w[遥 ハルカ],
|
|
232
|
+
%w[美優 ミユ],
|
|
233
|
+
%w[莉子 リコ],
|
|
234
|
+
%w[七海 ナナミ],
|
|
235
|
+
%w[美月 ミヅキ],
|
|
236
|
+
%w[結菜 ユナ],
|
|
237
|
+
%w[真央 マオ],
|
|
238
|
+
%w[花音 カノン],
|
|
239
|
+
%w[陽子 ヨウコ],
|
|
240
|
+
%w[舞 マイ],
|
|
241
|
+
%w[美羽 ミウ],
|
|
242
|
+
%w[優衣 ユイ],
|
|
243
|
+
%w[未来 ミク],
|
|
244
|
+
%w[彩 アヤ],
|
|
245
|
+
%w[彩乃 アヤノ],
|
|
246
|
+
%w[彩花 アヤカ],
|
|
247
|
+
%w[智子 トモコ],
|
|
248
|
+
%w[奈々 ナナ],
|
|
249
|
+
%w[千尋 チヒロ],
|
|
250
|
+
%w[愛美 マナミ],
|
|
251
|
+
%w[優菜 ユウナ],
|
|
252
|
+
%w[杏 アン],
|
|
253
|
+
%w[裕子 ユウコ],
|
|
254
|
+
%w[芽衣 メイ],
|
|
255
|
+
%w[綾乃 アヤノ],
|
|
256
|
+
%w[琴音 コトネ],
|
|
257
|
+
%w[桜 サクラ],
|
|
258
|
+
%w[恵 メグミ],
|
|
259
|
+
%w[杏奈 アンナ],
|
|
260
|
+
%w[美桜 ミオ],
|
|
261
|
+
%w[優花 ユウカ],
|
|
262
|
+
%w[玲奈 レナ],
|
|
263
|
+
%w[結 ユイ],
|
|
264
|
+
%w[茜 アカネ],
|
|
265
|
+
%w[美穂 ミホ],
|
|
266
|
+
%w[明日香 アスカ],
|
|
267
|
+
%w[愛子 アイコ],
|
|
268
|
+
%w[美緒 ミオ],
|
|
269
|
+
%w[千夏 チナツ],
|
|
270
|
+
%w[真希 マキ],
|
|
271
|
+
%w[陽向 ヒナタ],
|
|
272
|
+
%w[大介 ダイスケ],
|
|
273
|
+
%w[涼介 リョウスケ],
|
|
274
|
+
%w[悠介 ユウスケ],
|
|
275
|
+
%w[拓也 タクヤ],
|
|
276
|
+
%w[和也 カズヤ],
|
|
277
|
+
%w[直人 ナオト],
|
|
278
|
+
%w[雄太 ユウタ],
|
|
279
|
+
%w[翔平 ショウヘイ],
|
|
280
|
+
%w[健太郎 ケンタロウ],
|
|
281
|
+
%w[慎太郎 シンタロウ],
|
|
282
|
+
%w[竜也 タツヤ],
|
|
283
|
+
%w[圭 ケイ],
|
|
284
|
+
%w[樹里 ジュリ],
|
|
285
|
+
%w[花子 ハナコ],
|
|
286
|
+
%w[由美 ユミ],
|
|
287
|
+
%w[由美子 ユミコ],
|
|
288
|
+
%w[美穂子 ミホコ],
|
|
289
|
+
%w[直子 ナオコ],
|
|
290
|
+
%w[京子 キョウコ],
|
|
291
|
+
%w[順子 ジュンコ],
|
|
292
|
+
%w[洋子 ヨウコ],
|
|
293
|
+
%w[幸子 サチコ],
|
|
294
|
+
%w[美和 ミワ],
|
|
295
|
+
%w[理沙 リサ],
|
|
296
|
+
%w[麻衣 マイ],
|
|
297
|
+
%w[早紀 サキ],
|
|
298
|
+
%w[春香 ハルカ],
|
|
299
|
+
%w[沙織 サオリ],
|
|
300
|
+
%w[香織 カオリ],
|
|
301
|
+
%w[友美 トモミ],
|
|
302
|
+
%w[恵子 ケイコ],
|
|
303
|
+
%w[真由美 マユミ],
|
|
304
|
+
%w[里奈 リナ],
|
|
305
|
+
%w[彩香 アヤカ],
|
|
306
|
+
%w[亜美 アミ],
|
|
307
|
+
%w[春菜 ハルナ],
|
|
308
|
+
%w[千秋 チアキ],
|
|
309
|
+
%w[桃子 モモコ],
|
|
310
|
+
%w[美里 ミサト],
|
|
311
|
+
%w[結子 ユウコ],
|
|
312
|
+
%w[陽介 ヨウスケ],
|
|
313
|
+
].freeze
|
|
314
|
+
end
|
|
315
|
+
end
|
|
@@ -39,7 +39,30 @@ module Exwiw
|
|
|
39
39
|
# `path`.
|
|
40
40
|
attribute :embedded_in, optional(EmbeddedIn), skip_serializing_if_nil: true
|
|
41
41
|
|
|
42
|
+
# `reverse_scope` opts a collection into multi-referencer reverse scoping,
|
|
43
|
+
# mirroring the SQL TableConfig key (see Exwiw::ReverseScope): a
|
|
44
|
+
# global-identity collection referenced by many scoped collections is
|
|
45
|
+
# constrained to the union of the ids those referencers actually point at,
|
|
46
|
+
# instead of being dumped in full. Unlike the SQL adapters (which emit a
|
|
47
|
+
# UNION subquery), the mongodb adapter captures each `via` arm's column
|
|
48
|
+
# values at runtime while the referencer collection streams, so the
|
|
49
|
+
# reverse-scoped collection must be processed AFTER its referencers — see
|
|
50
|
+
# DetermineTableProcessingOrder (runtime_reverse_scope) and
|
|
51
|
+
# MongodbAdapter#reverse_scope_filter. User-configured and never emitted by
|
|
52
|
+
# MongoidSchemaGenerator; preserved across regeneration (see #merge).
|
|
53
|
+
attribute :reverse_scope, Serdes::OptionalType.new(ReverseScope), skip_serializing_if_nil: true
|
|
54
|
+
|
|
42
55
|
def self.from(obj)
|
|
56
|
+
# Reject unknown keys before deserializing: Serdes silently drops them,
|
|
57
|
+
# which would turn a typo'd key — or a key only the SQL adapters support,
|
|
58
|
+
# like a field-level `raw_sql` — into a silent no-op (see
|
|
59
|
+
# Exwiw::StrictKeys). `comment` is a declared attribute here and on the
|
|
60
|
+
# nested belongs_to/field entries, so free-form notes stay accepted.
|
|
61
|
+
if obj.is_a?(Hash)
|
|
62
|
+
collection_name = obj["name"] || obj[:name]
|
|
63
|
+
StrictKeys.validate!(self, obj, owner: "collection '#{collection_name}'")
|
|
64
|
+
end
|
|
65
|
+
|
|
43
66
|
instance = super
|
|
44
67
|
instance.__send__(:validate_embedded!)
|
|
45
68
|
instance.__send__(:validate_belongs_tos!)
|
|
@@ -89,6 +112,8 @@ module Exwiw
|
|
|
89
112
|
# is kept.
|
|
90
113
|
merged.comment = passed.comment || comment
|
|
91
114
|
merged.embedded_in = passed.embedded_in
|
|
115
|
+
# User-owned, never regenerated: carry over from the existing config.
|
|
116
|
+
merged.reverse_scope = reverse_scope
|
|
92
117
|
|
|
93
118
|
# Structural facts of each belongs_to come from the freshly generated
|
|
94
119
|
# config (including a generator-derived `references`), but the user-owned
|
|
@@ -124,6 +149,12 @@ module Exwiw
|
|
|
124
149
|
|
|
125
150
|
private def validate_embedded!
|
|
126
151
|
return unless embedded?
|
|
152
|
+
|
|
153
|
+
if reverse_scope
|
|
154
|
+
raise ArgumentError,
|
|
155
|
+
"MongodbCollectionConfig '#{name}' is embedded_in '#{embedded_in.collection_name}'; " \
|
|
156
|
+
"reverse_scope must not be defined (an embedded config is never dumped on its own)."
|
|
157
|
+
end
|
|
127
158
|
return if belongs_tos.empty?
|
|
128
159
|
|
|
129
160
|
raise ArgumentError,
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
4
|
|
|
5
|
+
require_relative "japanese_names"
|
|
6
|
+
|
|
5
7
|
module Exwiw
|
|
6
8
|
# Applies the Ruby-process-side masking modes — `map` and
|
|
7
9
|
# `replace_with_fake_data` — to the rows streamed out of a SQL adapter's
|
|
@@ -31,16 +33,46 @@ module Exwiw
|
|
|
31
33
|
# given faker gem version + locale.
|
|
32
34
|
POOL_RANDOM_SEED = 715_517
|
|
33
35
|
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
|
|
36
|
+
# The person pool (PERSON_TYPES) is sized independently of POOL_SIZE — at
|
|
37
|
+
# twice the size — so the coherent-identity name space is larger without
|
|
38
|
+
# disturbing the independent types' seed->value mappings. The ja pool is
|
|
39
|
+
# filled with distinct people (see .build_japanese_person_pool), so
|
|
40
|
+
# JapaneseNames must supply at least this many (surname, given) combinations.
|
|
41
|
+
PERSON_POOL_SIZE = POOL_SIZE * 2
|
|
42
|
+
|
|
43
|
+
# A coherent fake identity. All person-family types (PERSON_TYPES) draw
|
|
44
|
+
# from one shared pool of these per locale, and a seed picks the same pool
|
|
45
|
+
# index for every such column — so last_name / first_name / full name (and
|
|
46
|
+
# their kana) for one seed all belong to the same person. kana fields are
|
|
47
|
+
# only populated for the `ja` locale (faker has no readings; exwiw bundles
|
|
48
|
+
# a paired dataset — see JapaneseNames); they are nil otherwise.
|
|
49
|
+
Person = Struct.new(:last_name, :first_name, :last_name_kana, :first_name_kana)
|
|
50
|
+
|
|
51
|
+
# Person-family types: they share the per-locale person pool (built by
|
|
52
|
+
# .person_pool), and each extractor derives its value from the picked
|
|
53
|
+
# Person. Full-name ordering is locale-aware ("姓 名" for ja, "First Last"
|
|
54
|
+
# elsewhere). This is what keeps a single seed's name columns consistent.
|
|
55
|
+
PERSON_TYPES = {
|
|
56
|
+
"human_name" => ->(p, locale) { join_full_name(p.last_name, p.first_name, locale) },
|
|
57
|
+
"last_name" => ->(p, _locale) { p.last_name },
|
|
58
|
+
"first_name" => ->(p, _locale) { p.first_name },
|
|
59
|
+
"human_name_kana" => ->(p, locale) { join_full_name(p.last_name_kana, p.first_name_kana, locale) },
|
|
60
|
+
"last_name_kana" => ->(p, _locale) { p.last_name_kana },
|
|
61
|
+
"first_name_kana" => ->(p, _locale) { p.first_name_kana },
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
# Person-family types whose value is a kana reading, only available for the
|
|
65
|
+
# `ja` locale (the person pool carries nil kana for other locales, so these
|
|
66
|
+
# are rejected at build time with a clear error).
|
|
67
|
+
KANA_TYPES = %w[human_name_kana last_name_kana first_name_kana].freeze
|
|
68
|
+
|
|
69
|
+
# Independent (non-person) types. `pool` builds one candidate value (called
|
|
70
|
+
# POOL_SIZE times under a seeded Faker random). Types with `compose` are
|
|
71
|
+
# uniqueness-sensitive: the pooled base alone would collide under a unique
|
|
72
|
+
# index at scale, so the final value composes the base with a 64-bit hex
|
|
73
|
+
# token derived from the same seed digest (collision probability at 5M
|
|
74
|
+
# distinct seeds ≈ 7e-7).
|
|
40
75
|
FAKE_TYPES = {
|
|
41
|
-
"human_name" => { pool: -> { Faker::Name.name } },
|
|
42
|
-
"first_name" => { pool: -> { Faker::Name.first_name } },
|
|
43
|
-
"last_name" => { pool: -> { Faker::Name.last_name } },
|
|
44
76
|
"phone_number" => { pool: -> { Faker::PhoneNumber.phone_number } },
|
|
45
77
|
"address" => { pool: -> { Faker::Address.full_address } },
|
|
46
78
|
"company_name" => { pool: -> { Faker::Company.name } },
|
|
@@ -50,6 +82,13 @@ module Exwiw
|
|
|
50
82
|
compose: ->(base, token) { "#{base}_#{token}" } },
|
|
51
83
|
}.freeze
|
|
52
84
|
|
|
85
|
+
# "姓 名" for Japanese, "First Last" otherwise. A nil part (e.g. missing
|
|
86
|
+
# kana) collapses so the result never has a dangling separator.
|
|
87
|
+
def self.join_full_name(last, first, locale)
|
|
88
|
+
parts = locale.to_s == "ja" ? [last, first] : [first, last]
|
|
89
|
+
parts.compact.join(" ")
|
|
90
|
+
end
|
|
91
|
+
|
|
53
92
|
# The `r` a map proc receives: read-only access to the current row's
|
|
54
93
|
# values by column name (`r['id']`). One instance is reused across all
|
|
55
94
|
# rows (zero per-row allocation) — do not retain it outside the proc.
|
|
@@ -118,6 +157,15 @@ module Exwiw
|
|
|
118
157
|
"Add `gem \"faker\"` to your Gemfile (it is not a runtime dependency of exwiw)."
|
|
119
158
|
end
|
|
120
159
|
|
|
160
|
+
# Whether a (type, locale) draws any value from faker. The `ja` person pool
|
|
161
|
+
# is built entirely from exwiw's bundled dataset, so a config that only
|
|
162
|
+
# uses `ja` person types needs no faker at all.
|
|
163
|
+
def self.type_needs_faker?(type, locale)
|
|
164
|
+
return true if FAKE_TYPES.key?(type)
|
|
165
|
+
|
|
166
|
+
PERSON_TYPES.key?(type) && locale.to_s != "ja"
|
|
167
|
+
end
|
|
168
|
+
|
|
121
169
|
# Pools are memoized per (type, locale): every column sharing a type+locale
|
|
122
170
|
# sees the same pool, which is what makes equal seed values map to equal
|
|
123
171
|
# fake values across tables and runs.
|
|
@@ -138,6 +186,51 @@ module Exwiw
|
|
|
138
186
|
Faker::Config.random = previous_random
|
|
139
187
|
end
|
|
140
188
|
|
|
189
|
+
# One pool of coherent Person records per locale, shared by every
|
|
190
|
+
# person-family type. Memoized so equal seeds map to equal people across
|
|
191
|
+
# tables and runs. `ja` is built from exwiw's bundled (kanji, kana) dataset
|
|
192
|
+
# so kana matches kanji; other locales use faker (no kana).
|
|
193
|
+
def self.person_pool(locale)
|
|
194
|
+
@person_pools ||= {}
|
|
195
|
+
@person_pools[locale] ||= build_person_pool(locale)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def self.build_person_pool(locale)
|
|
199
|
+
random = Random.new(POOL_RANDOM_SEED)
|
|
200
|
+
return build_japanese_person_pool(random) if locale.to_s == "ja"
|
|
201
|
+
|
|
202
|
+
previous_locale = Faker::Config.locale
|
|
203
|
+
previous_random = Faker::Config.random
|
|
204
|
+
Faker::Config.locale = locale if locale
|
|
205
|
+
Faker::Config.random = random
|
|
206
|
+
Array.new(PERSON_POOL_SIZE) { Person.new(Faker::Name.last_name, Faker::Name.first_name, nil, nil).freeze }.freeze
|
|
207
|
+
ensure
|
|
208
|
+
unless locale.to_s == "ja"
|
|
209
|
+
Faker::Config.locale = previous_locale
|
|
210
|
+
Faker::Config.random = previous_random
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Fill the ja pool with DISTINCT people: enumerate every (surname, given)
|
|
215
|
+
# combination, shuffle deterministically, and take PERSON_POOL_SIZE. Unlike
|
|
216
|
+
# sampling with replacement (which wastes ~37% of slots to duplicates), this
|
|
217
|
+
# gives PERSON_POOL_SIZE distinct identities — so JapaneseNames must supply
|
|
218
|
+
# at least that many combinations.
|
|
219
|
+
def self.build_japanese_person_pool(random)
|
|
220
|
+
surnames = JapaneseNames::SURNAMES
|
|
221
|
+
given_names = JapaneseNames::GIVEN_NAMES
|
|
222
|
+
combinations = surnames.size * given_names.size
|
|
223
|
+
if combinations < PERSON_POOL_SIZE
|
|
224
|
+
raise "JapaneseNames has #{combinations} (surname, given) combinations, " \
|
|
225
|
+
"need at least PERSON_POOL_SIZE=#{PERSON_POOL_SIZE}"
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
people = surnames.flat_map do |last|
|
|
229
|
+
given_names.map { |first| Person.new(last[0], first[0], last[1], first[1]).freeze }
|
|
230
|
+
end
|
|
231
|
+
people.shuffle(random: random).first(PERSON_POOL_SIZE).freeze
|
|
232
|
+
end
|
|
233
|
+
|
|
141
234
|
def initialize(table)
|
|
142
235
|
@table_name = table.name
|
|
143
236
|
@name_to_index = {}
|
|
@@ -145,7 +238,11 @@ module Exwiw
|
|
|
145
238
|
@name_to_index.freeze
|
|
146
239
|
@row_accessor = Row.new(@table_name, @name_to_index)
|
|
147
240
|
|
|
148
|
-
|
|
241
|
+
needs_faker = table.columns.any? do |column|
|
|
242
|
+
fd = column.replace_with_fake_data
|
|
243
|
+
fd && self.class.type_needs_faker?(fd.type, fd.locale)
|
|
244
|
+
end
|
|
245
|
+
self.class.require_faker! if needs_faker
|
|
149
246
|
|
|
150
247
|
@transforms = table.columns.each_with_index.filter_map do |column, index|
|
|
151
248
|
if column.map
|
|
@@ -210,11 +307,26 @@ module Exwiw
|
|
|
210
307
|
|
|
211
308
|
private def compile_fake(column, column_index)
|
|
212
309
|
fake_data = column.replace_with_fake_data
|
|
213
|
-
|
|
310
|
+
type = fake_data.type
|
|
311
|
+
unless PERSON_TYPES.key?(type) || FAKE_TYPES.key?(type)
|
|
312
|
+
raise ArgumentError,
|
|
313
|
+
"replace_with_fake_data for column '#{@table_name}.#{column.name}': " \
|
|
314
|
+
"unknown type '#{type}'. Supported: #{(PERSON_TYPES.keys + FAKE_TYPES.keys).join(', ')}"
|
|
315
|
+
end
|
|
214
316
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
317
|
+
seed_index = resolve_seed_index(fake_data, column)
|
|
318
|
+
|
|
319
|
+
if PERSON_TYPES.key?(type)
|
|
320
|
+
compile_person_fake(fake_data, column, column_index, seed_index)
|
|
321
|
+
else
|
|
322
|
+
compile_independent_fake(fake_data, column_index, seed_index)
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# Re-resolve the seed here, against the effective (post-ignore) columns:
|
|
327
|
+
# load-time validation sees the full column list, so a seed column that was
|
|
328
|
+
# ignore:true is only caught at dump time.
|
|
329
|
+
private def resolve_seed_index(fake_data, column)
|
|
218
330
|
seed_column = fake_data.seed.delete_prefix("#{@table_name}.")
|
|
219
331
|
seed_index = @name_to_index[seed_column]
|
|
220
332
|
unless seed_index
|
|
@@ -223,14 +335,44 @@ module Exwiw
|
|
|
223
335
|
"seed '#{fake_data.seed}' does not resolve to an extracted column " \
|
|
224
336
|
"(is it ignore:true?)"
|
|
225
337
|
end
|
|
338
|
+
seed_index
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Person-family type: pick a coherent Person from the per-locale pool. The
|
|
342
|
+
# digest[0,8] index is shared with every other person column, so one seed's
|
|
343
|
+
# name columns all belong to the same person.
|
|
344
|
+
private def compile_person_fake(fake_data, column, column_index, seed_index)
|
|
345
|
+
type = fake_data.type
|
|
346
|
+
locale = fake_data.locale
|
|
347
|
+
pool = self.class.person_pool(locale)
|
|
348
|
+
extractor = PERSON_TYPES.fetch(type)
|
|
349
|
+
|
|
350
|
+
if KANA_TYPES.include?(type) && pool.first.last_name_kana.nil?
|
|
351
|
+
raise ArgumentError,
|
|
352
|
+
"replace_with_fake_data for column '#{@table_name}.#{column.name}': " \
|
|
353
|
+
"type '#{type}' needs kana readings, which are only available with locale: ja " \
|
|
354
|
+
"(got locale: #{locale.inspect})"
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# NULL-preserving like replace_with: a NULL target stays NULL.
|
|
358
|
+
lambda do |row|
|
|
359
|
+
next nil if row[column_index].nil?
|
|
360
|
+
|
|
361
|
+
digest = Digest::SHA256.digest(row[seed_index].to_s)
|
|
362
|
+
person = pool[digest[0, 8].unpack1("Q>") % PERSON_POOL_SIZE]
|
|
363
|
+
extractor.call(person, locale)
|
|
364
|
+
end
|
|
365
|
+
end
|
|
226
366
|
|
|
367
|
+
# Independent type: pick a value from its own (type, locale) pool.
|
|
368
|
+
private def compile_independent_fake(fake_data, column_index, seed_index)
|
|
227
369
|
pool = self.class.fake_pool(fake_data.type, fake_data.locale)
|
|
228
|
-
compose =
|
|
370
|
+
compose = FAKE_TYPES.fetch(fake_data.type)[:compose]
|
|
229
371
|
|
|
230
|
-
# NULL-preserving like replace_with: a NULL target stays NULL. A nil
|
|
231
|
-
#
|
|
232
|
-
#
|
|
233
|
-
#
|
|
372
|
+
# NULL-preserving like replace_with: a NULL target stays NULL. A nil seed
|
|
373
|
+
# value hashes "" (deterministic). Seed values are normalized with to_s so
|
|
374
|
+
# sqlite's native Integer 123 and pg/mysql's string "123" pick the same
|
|
375
|
+
# fake value.
|
|
234
376
|
lambda do |row|
|
|
235
377
|
next nil if row[column_index].nil?
|
|
236
378
|
|
data/lib/exwiw/runner.rb
CHANGED
|
@@ -47,7 +47,16 @@ module Exwiw
|
|
|
47
47
|
QueryAstBuilder.validate_scope!(dumpable_configs, table_by_name, @dump_target, @logger)
|
|
48
48
|
|
|
49
49
|
@logger.info("Determining table processing order...")
|
|
50
|
-
|
|
50
|
+
# runtime_reverse_scope: the mongodb adapter builds a reverse-scoped
|
|
51
|
+
# collection's filter from ids captured while its `via` referencers were
|
|
52
|
+
# dumped, so those referencers must be processed first. The SQL adapters
|
|
53
|
+
# scope via subqueries and keep the historical belongs_to-only order
|
|
54
|
+
# (which also keeps their INSERT output loadable in foreign-key order).
|
|
55
|
+
ordered_table_names = DetermineTableProcessingOrder.run(
|
|
56
|
+
dumpable_configs,
|
|
57
|
+
logger: @logger,
|
|
58
|
+
runtime_reverse_scope: adapter.is_a?(Adapter::MongodbAdapter),
|
|
59
|
+
)
|
|
51
60
|
|
|
52
61
|
clean_output_dir!
|
|
53
62
|
|
|
@@ -58,7 +67,7 @@ module Exwiw
|
|
|
58
67
|
# replacement for the whole schema+inserts pass, after which the common
|
|
59
68
|
# after-insert hook still runs. Everything before this point (validation,
|
|
60
69
|
# scope check, ordering, output-dir clean) applies to both paths.
|
|
61
|
-
if use_mongodb_parallel?(adapter)
|
|
70
|
+
if use_mongodb_parallel?(adapter, configs)
|
|
62
71
|
dump_mongodb_parallel(configs, table_by_name)
|
|
63
72
|
run_after_insert_hook(adapter, ordered_table_names.size)
|
|
64
73
|
return
|
|
@@ -209,7 +218,7 @@ module Exwiw
|
|
|
209
218
|
# target (the schedule is built around the scoped DAG), and a runtime that can
|
|
210
219
|
# fork. Anything else falls back to the serial path (warning when the user
|
|
211
220
|
# explicitly asked for parallelism but it cannot apply).
|
|
212
|
-
private def use_mongodb_parallel?(adapter)
|
|
221
|
+
private def use_mongodb_parallel?(adapter, configs)
|
|
213
222
|
return false unless adapter.is_a?(Adapter::MongodbAdapter)
|
|
214
223
|
return false unless @parallel_workers && @parallel_workers > 1
|
|
215
224
|
|
|
@@ -218,6 +227,15 @@ module Exwiw
|
|
|
218
227
|
return false
|
|
219
228
|
end
|
|
220
229
|
|
|
230
|
+
# A reverse-scoped collection consumes @state captured from its `via`
|
|
231
|
+
# referencers, an ordering constraint the parallel schedule (built around
|
|
232
|
+
# the belongs_to DAG only) does not express yet — a worker could dump the
|
|
233
|
+
# collection before its arms and silently drop rows. Run serially instead.
|
|
234
|
+
if configs.any? { |c| c.respond_to?(:reverse_scope) && c.reverse_scope&.via&.any? }
|
|
235
|
+
@logger.warn("--parallel-workers ignored: reverse_scope collections require the serial processing order; running serially.")
|
|
236
|
+
return false
|
|
237
|
+
end
|
|
238
|
+
|
|
221
239
|
unless MongodbParallelDumper.available?
|
|
222
240
|
@logger.warn("--parallel-workers ignored: fork is unavailable on this runtime; running serially.")
|
|
223
241
|
return false
|
|
@@ -275,7 +293,12 @@ module Exwiw
|
|
|
275
293
|
# considered during extraction. Done here (after loading from file)
|
|
276
294
|
# rather than in `.from` so the schema generators keep the full config
|
|
277
295
|
# and can preserve the ignored entries on regeneration.
|
|
278
|
-
|
|
296
|
+
begin
|
|
297
|
+
klass.from(json).reject_ignored_members!
|
|
298
|
+
rescue UnknownConfigKeyError => e
|
|
299
|
+
# `.from` knows the table, not the file; point at the offending file.
|
|
300
|
+
raise UnknownConfigKeyError, "#{file}: #{e.message}", e.backtrace
|
|
301
|
+
end
|
|
279
302
|
end
|
|
280
303
|
end
|
|
281
304
|
|