idata 0.1.31 → 0.1.32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/bin/iload +9 -3
  3. data/full-pg.sh +680 -0
  4. data/full.sh +52 -38
  5. data/idata.gemspec +1 -1
  6. data/lib/idata/version.rb +1 -1
  7. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 22656f3fc7154695b9099ba5619509c58d7bf3b8
4
- data.tar.gz: d7acaa17bf908edff29e1102476c673a75a008f4
3
+ metadata.gz: ef83b06323363e84794a14ef18b23903141f2cd8
4
+ data.tar.gz: 21cf263eafd81be05628474f6f80c4a0eac76192
5
5
  SHA512:
6
- metadata.gz: bc405e9e818193c229fc31fa9e427a5704de38809ad0454a4ce1f48117bf7ca471660d507781651eab1df2783ce6dd23a4a4c5bff5434de88ce6f14b983a3055
7
- data.tar.gz: 377964f122054009547008e8414578d019d1ecac27cffee3be364643a4f25cb75f39f3857759b3c223c3844a28e98be08407ade3ac1de5d4294b5beee23db7ae
6
+ metadata.gz: 9470f905dee46dc8d14a6cdabca4b018ba95a14d967e7ae2836fe0827f6c1613e5b38a28786c0e9d5feb8dfceae5d13d1d1c1df42a803a7d7917d46618ac5b0c
7
+ data.tar.gz: 29c38860bc9d90146d67f07865c3846c3a9c3f476447a53fd378a0211dc329387f1746e15219fa945733ac45d3484697408a04b678fbb0bbaaf0ccb122f3f810
data/bin/iload CHANGED
@@ -176,14 +176,20 @@ ActiveRecord::Base.establish_connection(
176
176
  $csv_converters = [:stripper]
177
177
 
178
178
  CSV::Converters[:stripper] = lambda{ |s|
179
- return s.strip if s.is_a?(String)
179
+ if s.is_a?(String)
180
+ r = s.strip
181
+ return nil if r == ""
182
+ return r
183
+ else
184
+ return s
185
+ end
180
186
  }
181
187
 
182
188
  $csv_converters << :null_converter if $options[:null]
183
189
 
184
190
  CSV::Converters[:null_converter] = lambda{ |s|
185
- return nil if s == $options[:null] or s == "" or s.nil?
186
- return s.strip
191
+ return nil if s == $options[:null]
192
+ return s
187
193
  }
188
194
 
189
195
  class String
data/full-pg.sh ADDED
@@ -0,0 +1,680 @@
1
+ #!/bin/bash
2
+
3
+ # SHARED VARIABLES
4
+ # ---------------------------------------------------------------------------------
5
+ # Database to store data tables
6
+ ORGNAME="ne20140729"
7
+
8
+ # ENV variables used by the validation command
9
+ export HOST="localhost"
10
+ export USERNAME="postgres"
11
+ export PASSWORD="postgres"
12
+ export DATABASE="$ORGNAME"
13
+ export LISTEN=7432
14
+
15
+ # Create temporary output folder
16
+ OUTPUT_DIR="/tmp/$ORGNAME" && test -e $OUTPUT_DIR || mkdir $OUTPUT_DIR
17
+
18
+ # Input files and correspondings names
19
+ FCONTRACTO="Contract.csv"
20
+ FVENDOR="VendorMasterUpdate.csv"
21
+ FITEM="ItemMaster.csv"
22
+ FINVOICE="InvoiceHistory.csv"
23
+ FMFR="MfrMaster.csv"
24
+ FPO="20140708_150920_s2_PoHistory_Report2_NEGA_modified.csv"
25
+ FPO2="PurchaseOrder.csv"
26
+ FUSER="User.csv"
27
+ FLOCATION="Location.csv"
28
+ FITEMCOST="ItemCostCenterAcctExceptions.csv"
29
+ FREQ="ReqHistoryLoad.csv"
30
+ FGL="GLAccount.csv"
31
+ FINVENTORY="Inventory.csv"
32
+ FULPR="UserAndLocationProfile.csv"
33
+ FLPR="LocationProfile.csv"
34
+
35
+ CONTRACTO="contracts"
36
+ VENDOR="vendors"
37
+ ITEM="items"
38
+ INVOICE="invoices"
39
+ MFR="manufacturers"
40
+ PO="purchase_orders"
41
+ PO2="second_purchase_orders"
42
+ USER="users"
43
+ LOCATION="locations"
44
+ ITEMCOST="item_costs"
45
+ REQ="reqs"
46
+ GL="gls"
47
+ INVENTORY="inventory"
48
+ ULPR="user_location_profiles"
49
+ LPR="location_profiles"
50
+
51
+ # Standard UOM for reference
52
+ FUOMSTD="stduom.txt"
53
+ UOMSTD="uomstd"
54
+
55
+ # ---------------------------------------------------------------------------------
56
+ # LOAD
57
+ # ---------------------------------------------------------------------------------
58
+ # Sanitize input files
59
+ #isanitize --strip-newline --remove=65279 "$FCONTRACTO" "$FVENDOR" "$FITEM" "$FINVOICE" "$FMFR" "$FPO" \
60
+ # "$FTMPL" "$FUSER" "$FITEMEXPO" "$FUSEREQ" "$FLOCATION" "$FITEMCOST" "$FREQ" \
61
+ # "$FGL" "$FUSERCOST" "$FINVENTORY" "$FUSERCREATEREQ"
62
+
63
+ # Load
64
+ iload -i "$FCONTRACTO" -t "$CONTRACTO" -f csv
65
+ iload -i "$FVENDOR" -t "$VENDOR" -f csv
66
+ iload -i "$FITEM" -t "$ITEM" -f csv
67
+ #iload -i "$FINVOICE" -t "$INVOICE" -f csv
68
+ iload -i "$FMFR" -t "$MFR" -f csv
69
+ iload -i "$FPO" -t "$PO" -f csv
70
+ iload -i "$FPO2" -t "$PO2" -f csv
71
+ iload -i "$FUSER" -t "$USER" -f csv
72
+ iload -i "$FLOCATION" -t "$LOCATION" -f csv
73
+ #iload -i "$FITEMCOST" -t "$ITEMCOST" -f csv
74
+ iload -i "$FREQ" -t "$REQ" -f csv
75
+ iload -i "$FGL" -t "$GL" -f csv
76
+ iload -i "$FINVENTORY" -t "$INVENTORY" -f csv
77
+ iload -i "$FULPR" -t "$ULPR" -f csv
78
+ iload -i "$FLPR" -t "$LPR" -f csv
79
+
80
+ # Load UOM STANDARD for reference
81
+ iload -i "$FUOMSTD" -t "$UOMSTD" -f csv
82
+
83
+ # ---------------------------------------------------------------------------------
84
+ # INDEXING (for speeding up validation queries)
85
+ # ---------------------------------------------------------------------------------
86
+ ipatch -q "drop index if exists items_item_id_index; create index items_item_id_index on $ITEM(item_id)"
87
+ ipatch -q "drop index if exists pos_item_id_index; create index pos_item_id_index on $PO(item_id)"
88
+ ipatch -q "drop index if exists gls_corp_acct_no_index; create index gls_corp_acct_no_index on $GL(corp_acct_no)"
89
+ ipatch -q "drop index if exists gls_corp_acct_name_index; create index gls_corp_acct_name_index on $GL(corp_acct_name)"
90
+ ipatch -q "drop index if exists gls_cc_acct_no_index; create index gls_cc_acct_no_index on $GL(cc_acct_no)"
91
+ ipatch -q "drop index if exists gls_cc_acct_name_index; create index gls_cc_acct_name_index on $GL(cc_acct_name)"
92
+ ipatch -q "drop index if exists locations_name_index; create index locations_name_index on $LOCATION(name)"
93
+
94
+ # ---------------------------------------------------------------------------------
95
+ # Some adjustment
96
+ # ---------------------------------------------------------------------------------
97
+ ipatch -q "
98
+ -- Normalize values
99
+ UPDATE $ITEM set active = '1' where active = 'Y';
100
+ UPDATE items set active = '3' where active = 'N';
101
+ "
102
+
103
+ ipatch -q "
104
+ -- Extract FIRST_NAME, LAST_NAME from NAME
105
+ --ALTER TABLE $USER ADD COLUMN first_name varchar;
106
+ --ALTER TABLE $USER ADD COLUMN last_name varchar;
107
+ --UPDATE $USER SET username = trim(regexp_replace(username, '\s+', ' ', 'g'));
108
+ --UPDATE $USER SET first_name = username, last_name = username
109
+ --WHERE array_length(string_to_array(username, ' '), 1) = 1;
110
+ --UPDATE $USER SET first_name = substring(username, '^[^\s]+'),
111
+ -- last_name = regexp_replace(username, '^[^\s]+\s', '', 'g')
112
+ --WHERE array_length(string_to_array(username, ' '), 1) >= 2;
113
+ "
114
+
115
+ ipatch -q "
116
+ -- ADD EMAIL COLUMN
117
+ --ALTER TABLE $USER ADD COLUMN email VARCHAR;
118
+ --UPDATE $USER SET email = userlogin;
119
+ "
120
+
121
+ ipatch -q "
122
+ -- FIX ISSUE WITH linebreak
123
+ update items set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\r\n', ' '), '\s+', ' ', 'g'), ' '))[1]
124
+ where vendor_item_id like E'%\r\n%';
125
+ update purchase_orders set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\r\n', ' '), '\s+', ' ', 'g'), ' '))[1]
126
+ where vendor_item_id like E'%\r\n%';
127
+ update purchase_orders set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\n', ' '), '\s+', ' ', 'g'), ' '))[1]
128
+ where vendor_item_id like E'%\n%';
129
+ "
130
+
131
+ ipatch -q "
132
+ -- Normalize DATE
133
+ UPDATE $PO SET po_date = to_char(to_date(po_date, 'MM/DD/YYYY'), 'YYYY-MM-DD')
134
+ WHERE po_date IS NOT NULL AND po_date != '';
135
+ UPDATE $CONTRACTO SET contract_start = to_char(to_date(contract_start, 'MM/DD/YYYY'), 'YYYY-MM-DD')
136
+ WHERE contract_start like '%/%/%';
137
+ UPDATE $CONTRACTO SET contract_end = to_char(to_date(contract_end, 'MM/DD/YYYY'), 'YYYY-MM-DD')
138
+ WHERE contract_end like '%/%/%';
139
+
140
+ -- ADD MORE EMPTY FIELD (for validation)
141
+ -- ALTER TABLE $LOCATION ADD COLUMN corp_id varchar;
142
+
143
+ -- FILL EMPTY FIELDS WITH DEFAULT VALUE
144
+ UPDATE $LOCATION SET ship_to_ind = 'N' WHERE ship_to_ind IS NULL OR LENGTH(trim(ship_to_ind)) = 0;
145
+ UPDATE $LOCATION SET bill_to_ind = 'N' WHERE bill_to_ind IS NULL OR LENGTH(trim(bill_to_ind)) = 0;
146
+ UPDATE $LOCATION SET stockless_ind = 'N' WHERE stockless_ind IS NULL OR LENGTH(trim(stockless_ind)) = 0;
147
+ "
148
+
149
+ # Convert A2A2A2 to A2-A2-A2
150
+ # Manual check and comment out the following to speed up validating
151
+ # @todo IEVAL performance is very poor, avoid using it (use IPATCH instead) unless there is no better way
152
+ ieval -t $GL --eval="
153
+ if item.corp_acct_fmt && item.corp_acct_fmt[/^[A-Z][0-9][A-Z][0-9][A-Z][0-9]$/]
154
+ item.corp_acct_fmt = item.corp_acct_fmt.unpack('a2a2a2').join('-')
155
+ end
156
+ "
157
+
158
+ # ---------------------------------------------------------------------------------
159
+ # VALIDATE
160
+ # ---------------------------------------------------------------------------------
161
+ # validate VENDORS
162
+ ivalidate --case-insensitive --pretty -t $VENDOR \
163
+ --log-to=validation_errors \
164
+ --not-null="vendor_code" \
165
+ --not-null="vendor_name" \
166
+ --unique="vendor_code,vendor_name" \
167
+ --match="vendor_code/[a-zA-Z0-9]/" \
168
+ --match="vendor_name/[a-zA-Z0-9]/" \
169
+ --consistent-by="vendor_code|vendor_name" \
170
+ --consistent-by="vendor_name|vendor_code" \
171
+ --consistent-by="country_code|country_name" \
172
+ --consistent-by="country_name|country_code"
173
+
174
+ # validate MANUFACTURERS
175
+ ivalidate --case-insensitive --pretty -t $MFR \
176
+ --log-to=validation_errors \
177
+ --not-null="mfr_number" \
178
+ --not-null="mfr_name" \
179
+ --match="mfr_number/[a-zA-Z0-9]/" \
180
+ --match="mfr_name/[a-zA-Z0-9]/" \
181
+ --consistent-by="mfr_name|mfr_number" \
182
+ --consistent-by="country_code|country_name" \
183
+ --consistent-by="country_name|country_code"
184
+
185
+ # validate GL
186
+ ivalidate --case-insensitive --pretty -t $GL \
187
+ --log-to=validation_errors \
188
+ --not-null=corp_acct_no \
189
+ --match="corp_acct_no/[a-zA-Z0-9]/" \
190
+ --not-null=corp_acct_name \
191
+ --match="corp_acct_name/[a-zA-Z0-9]/" \
192
+ --not-null=corp_acct_fmt \
193
+ --match="corp_acct_fmt/^[A-Z][0-9]-[A-Z][0-9]-[A-Z][0-9]$/" \
194
+ --not-null=cc_acct_no \
195
+ --match="cc_acct_no/[a-zA-Z0-9]/" \
196
+ --not-null=cc_acct_name \
197
+ --match="cc_acct_name/[a-zA-Z0-9]/" \
198
+ --not-null=cc_acct_type \
199
+ --match="cc_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/" \
200
+ --not-null=exp_acct_no \
201
+ --match="exp_acct_no/[a-zA-Z0-9]/" \
202
+ --not-null=exp_acct_name \
203
+ --match="exp_acct_name/[a-zA-Z0-9]/" \
204
+ --consistent-by="corp_acct_no|corp_acct_name" \
205
+ --consistent-by="corp_acct_name|corp_acct_no" \
206
+ --consistent-by="exp_acct_no|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_name" \
207
+ --consistent-by="exp_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_no" \
208
+ --consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name" \
209
+ --consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no" \
210
+ --not-null=exp_acct_type \
211
+ --match="exp_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/"
212
+
213
+ # validate LOCATION
214
+ ivalidate --case-insensitive --pretty -t $LOCATION \
215
+ --log-to=validation_errors \
216
+ --not-null=loc_id \
217
+ --match="loc_id/[a-zA-Z0-9]/" \
218
+ --not-null="name" \
219
+ --match="name/[a-zA-Z0-9]/" \
220
+ --not-null=facility_code \
221
+ --match="facility_code/[a-zA-Z0-9]/" \
222
+ --not-null=facility_desc \
223
+ --match="facility_desc/[a-zA-Z0-9]/" \
224
+ --match="ship_to_ind/^(Y|N)$/" \
225
+ --match="bill_to_ind/^(Y|N)$/" \
226
+ --match="stockless_ind/^(Y|N)$/" \
227
+ --not-null=loc_type \
228
+ --match="loc_type/^(C|S|LOC_TYPE_SUPPLY|LOC_TYPE_CONSUME)$/" \
229
+ --not-null=inventory_path_name \
230
+ --not-null=inventory_location_name \
231
+ --query="(route_no is null or route_no = '' or regexp_replace(route_no, '[,\.]', '', 'g') ~ '^[1-9][0-9]+$') -- invalid route_no" \
232
+ --rquery="(loc_type ~* '^(LOC_TYPE_SUPPLY|S)$' and (corp_acct_no is null or corp_name is null or corp_id is null)) -- either corp id/name or corp_acct_no is null" \
233
+ --not-null=active \
234
+ --match="active/^(Y|N)$/" \
235
+ --not-null=corp_acct_no \
236
+ --match="corp_acct_no/[a-zA-Z0-9]/" \
237
+ --not-null=inventory_loc_seq_no \
238
+ --match="inventory_loc_seq_no/^[1-9][0-9]*$/" \
239
+ --match="route_no/^[1-9][0-9]*$/" \
240
+ --match="route_name/[a-zA-Z0-9]/" \
241
+ --match="corp_name/[a-zA-Z0-9]/" \
242
+ --consistent-by="corp_name|corp_id" \
243
+ --consistent-by="corp_id|corp_name" \
244
+ --consistent-by="name|facility_code, loc_id" \
245
+ --consistent-by="loc_id|facility_code, name" \
246
+ --cross-reference="inventory_path_name|$LOCATION.name" \
247
+ --cross-reference="inventory_location_name|$LOCATION.name" \
248
+ --cross-reference="corp_id|$GL.corp_acct_no" \
249
+ --cross-reference="corp_name|$GL.corp_acct_name"
250
+
251
+
252
+ # validate CONTRACTS ORIGINAL
253
+ ivalidate --case-insensitive --pretty -t $CONTRACTO \
254
+ --log-to=validation_errors \
255
+ --not-null=contract_number \
256
+ --not-null=contract_start \
257
+ --not-null=contract_end \
258
+ --not-null=vendor_name \
259
+ --not-null=mfr_item_id \
260
+ --not-null=mfr_name \
261
+ --not-null=item_uom \
262
+ --not-null=item_qoe \
263
+ --not-null=contract_price \
264
+ --not-null=contract_gpo_name \
265
+ --match="contract_number/[a-zA-Z0-9]/" \
266
+ --match="corp_name/[a-zA-Z0-9]/" \
267
+ --match="vendor_item_id/[a-zA-Z0-9]/" \
268
+ --match="vendor_name/[a-zA-Z0-9]/" \
269
+ --match="mfr_item_id/[a-zA-Z0-9]/" \
270
+ --match="mfr_name/[a-zA-Z0-9]/" \
271
+ --query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- Contract end-date goes before start-date" \
272
+ --match="contract_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
273
+ --match="item_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
274
+ --consistent-by="corp_id|corp_name" \
275
+ --consistent-by="corp_name|corp_id" \
276
+ --consistent-by="mfr_name|mfr_number" \
277
+ --consistent-by="vendor_code|vendor_name" \
278
+ --consistent-by="vendor_name|vendor_code" \
279
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
280
+ --cross-reference="mfr_name|$MFR.mfr_name" \
281
+ --cross-reference="corp_id|$GL.corp_acct_no" \
282
+ --cross-reference="corp_name|$GL.corp_acct_name" \
283
+ --match="contract_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
284
+ --match="item_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
285
+ --rquery="(item_uom NOT IN (SELECT code FROM uomstd) AND item_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid item_uom" \
286
+ --unique="contract_gpo_name, contract_number, contract_start, contract_end, vendor_name, mfr_item_id, mfr_name, item_uom, corp_id" \
287
+
288
+ # validate ITEM
289
+ # Accepted:
290
+ # --rquery="mfr_name IN (SELECT mfr_name FROM $ITEM WHERE mfr_name IS NOT NULL GROUP BY mfr_name HAVING count(DISTINCT mfr_number) > 1) -- same mfr_name but with different mfr_number" \
291
+ ivalidate --case-insensitive --pretty -t $ITEM \
292
+ --log-to=validation_errors \
293
+ --not-null="item_id" \
294
+ --match="item_id/[a-zA-Z0-9]/" \
295
+ --not-null="item_descr" \
296
+ --match="item_descr/[a-zA-Z0-9]/" \
297
+ --not-null="item_uom" \
298
+ --not-null="default_uom" \
299
+ --not-null="item_price" \
300
+ --not-null="item_qoe" \
301
+ --not-null="corp_id" \
302
+ --not-null="corp_name" \
303
+ --not-null="vendor_code" \
304
+ --not-null="vendor_name" \
305
+ --not-null="mfr_number" \
306
+ --not-null="mfr_name" \
307
+ --not-null="active" \
308
+ --match="corp_id/[a-zA-Z0-9]/" \
309
+ --match="corp_name/[a-zA-Z0-9]/" \
310
+ --match="vendor_code/[a-zA-Z0-9]/" \
311
+ --match="vendor_name/[a-zA-Z0-9]/" \
312
+ --match="mfr_number/[a-zA-Z0-9]/" \
313
+ --match="mfr_name/[a-zA-Z0-9]/" \
314
+ --match="active/^(1|2|3|A|I)$/" \
315
+ --cross-reference="vendor_code|$VENDOR.vendor_code" \
316
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
317
+ --cross-reference="mfr_number|$MFR.mfr_number" \
318
+ --cross-reference="mfr_name|$MFR.mfr_name" \
319
+ --cross-reference="corp_id|$GL.corp_acct_no" \
320
+ --cross-reference="corp_name|$GL.corp_acct_name" \
321
+ --consistent-by="corp_id|corp_name" \
322
+ --consistent-by="corp_name|corp_id" \
323
+ --consistent-by="mfr_name|mfr_number" \
324
+ --consistent-by="vendor_code|vendor_name" \
325
+ --consistent-by="vendor_name|vendor_code" \
326
+ --rquery="(item_uom NOT IN (SELECT code FROM uomstd) AND item_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid item_uom" \
327
+ --rquery="(default_uom NOT IN (SELECT code FROM uomstd) AND default_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid default_uom" \
328
+ --match="item_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
329
+ --match="item_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
330
+
331
+ # validate PO
332
+ ivalidate --case-insensitive --pretty -t $PO \
333
+ --log-to=validation_errors \
334
+ --not-null=po_no \
335
+ --match="po_no/[a-zA-Z0-9]/" \
336
+ --not-null=po_date \
337
+ --not-null=corp_id \
338
+ --match="corp_id/[a-zA-Z0-9]/" \
339
+ --not-null=corp_name \
340
+ --match="corp_name/[a-zA-Z0-9]/" \
341
+ --not-null=Paragon_Cost_Center_Acct \
342
+ --match="Paragon_Cost_Center_Acct/[a-zA-Z0-9]/" \
343
+ --not-null=cost_center_name \
344
+ --match="cost_center_name/[a-zA-Z0-9]/" \
345
+ --not-null=PO_Line_No \
346
+ --match="PO_Line_No/^[1-9][0-9]*$/" \
347
+ --not-null=item_id \
348
+ --match="item_id/[a-zA-Z0-9]/" \
349
+ --not-null=vendor_name \
350
+ --match="vendor_name/[a-zA-Z0-9]/" \
351
+ --not-null=Paragon_Vendor_Code \
352
+ --match="Paragon_Vendor_Code/[a-zA-Z0-9]/" \
353
+ --match="mfr_name/[a-zA-Z0-9]/" \
354
+ --not-null=mfr_number \
355
+ --match="mfr_number/[a-zA-Z0-9]/" \
356
+ --not-null=item_descr \
357
+ --consistent-by="corp_id|corp_name" \
358
+ --consistent-by="corp_name|corp_id" \
359
+ --consistent-by="Paragon_Vendor_Code|vendor_name" \
360
+ --consistent-by="vendor_name|Paragon_Vendor_Code" \
361
+ --consistent-by="mfr_name|mfr_number" \
362
+ --unique="po_no, PO_Line_No" \
363
+ --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
364
+ --cross-reference="Paragon_Vendor_Code|$VENDOR.vendor_code" \
365
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
366
+ --cross-reference="mfr_number|$MFR.mfr_number" \
367
+ --cross-reference="mfr_name|$MFR.mfr_name" \
368
+ --cross-reference="corp_id|$GL.corp_acct_no" \
369
+ --cross-reference="corp_name|$GL.corp_acct_name" \
370
+ --cross-reference="Paragon_Cost_Center_Acct|$GL.cc_acct_no" \
371
+ --cross-reference="cost_center_name|$GL.cc_acct_name" \
372
+ --rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
373
+ --rquery="(item_id IS NOT NULL AND (Paragon_Vendor_Code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
374
+ --match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
375
+ --match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
376
+
377
+ ivalidate --case-insensitive --pretty -t $PO2 \
378
+ --log-to=validation_errors \
379
+ --not-null=po_no \
380
+ --match="po_no/[a-zA-Z0-9]/" \
381
+ --not-null=po_date \
382
+ --not-null=corp_id \
383
+ --match="corp_id/[a-zA-Z0-9]/" \
384
+ --not-null=corp_name \
385
+ --match="corp_name/[a-zA-Z0-9]/" \
386
+ --not-null=cost_center_id \
387
+ --match="cost_center_id/[a-zA-Z0-9]/" \
388
+ --not-null=cost_center_name \
389
+ --match="cost_center_name/[a-zA-Z0-9]/" \
390
+ --not-null=PO_Line_Number \
391
+ --match="PO_Line_Number/^[1-9][0-9]*$/" \
392
+ --not-null=item_id \
393
+ --match="item_id/[a-zA-Z0-9]/" \
394
+ --not-null=vendor_name \
395
+ --match="vendor_name/[a-zA-Z0-9]/" \
396
+ --not-null=vendor_code \
397
+ --match="vendor_code/[a-zA-Z0-9]/" \
398
+ --not-null=mfr_name \
399
+ --match="mfr_name/[a-zA-Z0-9]/" \
400
+ --not-null=mfr_number \
401
+ --match="mfr_number/[a-zA-Z0-9]/" \
402
+ --not-null=item_descr \
403
+ --consistent-by="corp_id|corp_name" \
404
+ --consistent-by="corp_name|corp_id" \
405
+ --consistent-by="vendor_code|vendor_name" \
406
+ --consistent-by="vendor_name|vendor_code" \
407
+ --consistent-by="mfr_name|mfr_number" \
408
+ --unique="po_no, PO_Line_Number" \
409
+ --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
410
+ --cross-reference="vendor_code|$VENDOR.vendor_code" \
411
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
412
+ --cross-reference="mfr_number|$MFR.mfr_number" \
413
+ --cross-reference="mfr_name|$MFR.mfr_name" \
414
+ --cross-reference="corp_id|$GL.corp_acct_no" \
415
+ --cross-reference="corp_name|$GL.corp_acct_name" \
416
+ --cross-reference="cost_center_id|$GL.cc_acct_no" \
417
+ --cross-reference="cost_center_name|$GL.cc_acct_name" \
418
+ --rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
419
+ --rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
420
+ --match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
421
+ --match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
422
+
423
+ # validate Req
424
+ ivalidate --case-insensitive --pretty -t $REQ \
425
+ --log-to=validation_errors \
426
+ --not-null="req_no" \
427
+ --match="req_no/[a-zA-Z0-9]/" \
428
+ --not-null="req_date" \
429
+ --match="req_date/[a-zA-Z0-9]/" \
430
+ --not-null="corp_id" \
431
+ --match="corp_id/[a-zA-Z0-9]/" \
432
+ --not-null="corp_name" \
433
+ --match="corp_name/[a-zA-Z0-9]/" \
434
+ --not-null="req_line_number" \
435
+ --match="req_line_number/^[1-9][0-9]*$/" \
436
+ --not-null="item_id" \
437
+ --match="item_id/[a-zA-Z0-9]/" \
438
+ --not-null="vendor_name" \
439
+ --match="vendor_name/[a-zA-Z0-9]/" \
440
+ --not-null="vendor_code" \
441
+ --match="vendor_code/[a-zA-Z0-9]/" \
442
+ --not-null="mfr_name" \
443
+ --match="mfr_name/[a-zA-Z0-9]/" \
444
+ --not-null="item_descr" \
445
+ --match="item_descr/[a-zA-Z0-9]/" \
446
+ --consistent-by="corp_id|corp_name" \
447
+ --consistent-by="corp_name|corp_id" \
448
+ --rquery="(item_id not like '%~[%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
449
+ --cross-reference="corp_id|$GL.corp_acct_no" \
450
+ --cross-reference="corp_name|$GL.corp_acct_name" \
451
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
452
+ --cross-reference="vendor_code|$VENDOR.vendor_code" \
453
+ --cross-reference="mfr_name|$MFR.mfr_name" \
454
+ --cross-reference="costcenter_id|$GL.cc_acct_no" \
455
+ --cross-reference="costcenter_name|$GL.cc_acct_name" \
456
+ --unique="req_no, req_line_number" \
457
+
458
+ # validate USERS
459
+ ivalidate --case-insensitive --pretty -t $USER \
460
+ --log-to=validation_errors \
461
+ --not-null="email" \
462
+ --unique="email" \
463
+ --query="lower(email) ~* '[a-z0-9][a-z0-9_\.]+@[a-z0-9][a-z0-9_\.\-]+\.[a-z0-9_\.\-]+' -- invalid email address"
464
+
465
+ # validate
466
+ #ivalidate --case-insensitive --pretty -t $ITEMCOST \
467
+ # --log-to=validation_errors \
468
+ # --not-null="corp_acct_no" \
469
+ # --match="corp_acct_no/[a-zA-Z0-9]/" \
470
+ # --not-null="item_no" \
471
+ # --match="item_no/[a-zA-Z0-9]/" \
472
+ # --not-null="excp_type" \
473
+ # --match="excp_type/^(A|R)$/" \
474
+ # --cross-reference="corp_acct_no|$GL.corp_acct_no" \
475
+ # --cross-reference="item_no|$ITEM.item_id" \
476
+ # --cross-reference="exp_acct_no|$GL.exp_acct_no"
477
+
478
+ # validate INVENTORY
479
+ ivalidate --case-insensitive --pretty -t $INVENTORY \
480
+ --log-to=validation_errors \
481
+ --not-null="item_id" \
482
+ --match="item_id/[a-zA-Z0-9]/" \
483
+ --not-null="loc_id" \
484
+ --match="loc_id/[a-zA-Z0-9]/" \
485
+ --not-null="vendor_code" \
486
+ --match="vendor_code/[a-zA-Z0-9]/" \
487
+ --not-null="vendor_name" \
488
+ --match="vendor_name/[a-zA-Z0-9]/" \
489
+ --not-null="corp_id" \
490
+ --match="corp_name/[a-zA-Z0-9]/" \
491
+ --not-null="location_name" \
492
+ --match="location_name/[a-zA-Z0-9]/" \
493
+ --not-null="item_id" \
494
+ --match="item_id/[a-zA-Z0-9]/" \
495
+ --match="inventory_status/^(Active|Pending Inactive|Inactive)$/" \
496
+ --cross-reference="item_id|$ITEM.item_id" \
497
+ --cross-reference="location_name|$LOCATION.name" \
498
+ --cross-reference="vendor_code|$VENDOR.vendor_code" \
499
+ --cross-reference="vendor_name|$VENDOR.vendor_name" \
500
+ --cross-reference="corp_id|$GL.corp_acct_no" \
501
+ --cross-reference="corp_name|$GL.corp_acct_name"
502
+
503
+ ivalidate --case-insensitive --pretty -t $ULPR \
504
+ --log-to=validation_errors \
505
+ --match="Default_Indicator/^(Y|N|y|n)$/" \
506
+ --not-null="email" \
507
+ --cross-reference="email|$USER.email" \
508
+ --cross-reference="loc_id|$LOCATION.loc_id" \
509
+ --cross-reference="loc_name|$LOCATION.name" \
510
+ --cross-reference="corp_no|$GL.corp_acct_no" \
511
+ --cross-reference="corp_name|$GL.corp_acct_name" \
512
+
513
+ ivalidate --case-insensitive --pretty -t $LPR \
514
+ --log-to=validation_errors \
515
+ --not-null="Default_Inventory_Location_Name" \
516
+ --cross-reference="Default_Inventory_Location_Name|$LOCATION.Inventory_Location_Name" \
517
+ --match="loc_type/^(C|LOC_TYPE_CONSUME)$/" \
518
+ --match="active/^(1|2|3|A|I)$/" \
519
+ --cross-reference="item_id|$ITEM.item_id" \
520
+ --cross-reference="corp_acct_no|$GL.corp_acct_no" \
521
+ --cross-reference="corp_name|$GL.corp_acct_name" \
522
+ --cross-reference="cc_acct_no|$GL.cc_acct_no" \
523
+ --cross-reference="cc_acct_name|$GL.cc_acct_name" \
524
+ --cross-reference="loc_id|$LOCATION.loc_id" \
525
+ --cross-reference="Location_Name|$LOCATION.name"
526
+
527
+ ####################################################
528
+ # Create report file for every table (extract 1000 records for every error)
529
+ # These file will then be used for the Validation Report
530
+ ####################################################
531
+
532
+ #iexport -t $ITEMCOST \
533
+ # -o "$OUTPUT_DIR/$ITEMCOST.csv" -f csv --no-quote-empty --quotes --headers \
534
+ # --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
535
+ # FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
536
+ # $ITEMCOST order by id ) as main) as tmp
537
+ # where group_index <= 1000" \
538
+ # --exclude="id, validation_errors, group_index"
539
+
540
+ iexport -t $CONTRACTO \
541
+ -o "$OUTPUT_DIR/$CONTRACTO.csv" -f csv --no-quote-empty --quotes --headers \
542
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
543
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
544
+ $CONTRACTO order by id ) as main) as tmp
545
+ where group_index <= 1000" \
546
+ --exclude="id, validation_errors, group_index"
547
+
548
+ iexport -t $VENDOR \
549
+ -o "$OUTPUT_DIR/$VENDOR.csv" -f csv --no-quote-empty --quotes --headers \
550
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
551
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
552
+ $VENDOR order by id ) as main) as tmp
553
+ where group_index <= 1000" \
554
+ --exclude="id, validation_errors, group_index"
555
+
556
+ iexport -t $MFR \
557
+ -o "$OUTPUT_DIR/$MFR.csv" -f csv --no-quote-empty --quotes --headers \
558
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
559
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
560
+ $MFR order by id ) as main) as tmp
561
+ where group_index <= 1000" \
562
+ --exclude="id, validation_errors, group_index"
563
+
564
+ iexport -t $GL \
565
+ -o "$OUTPUT_DIR/$GL.csv" -f csv --no-quote-empty --quotes --headers \
566
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
567
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
568
+ $GL order by id ) as main) as tmp
569
+ where group_index <= 1000" \
570
+ --exclude="id, validation_errors, group_index"
571
+
572
+ iexport -t $PO \
573
+ -o "$OUTPUT_DIR/$PO.csv" -f csv --no-quote-empty --quotes --headers \
574
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
575
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
576
+ $PO order by id ) as main) as tmp
577
+ where group_index <= 1000" \
578
+ --exclude="id, validation_errors, group_index"
579
+
580
+ iexport -t $PO2 \
581
+ -o "$OUTPUT_DIR/$PO2.csv" -f csv --no-quote-empty --quotes --headers \
582
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
583
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
584
+ $PO2 order by id ) as main) as tmp
585
+ where group_index <= 1000" \
586
+ --exclude="id, validation_errors, group_index"
587
+
588
+ iexport -t $INVENTORY \
589
+ -o "$OUTPUT_DIR/$INVENTORY.csv" -f csv --no-quote-empty --quotes --headers \
590
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
591
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
592
+ $INVENTORY order by id ) as main) as tmp
593
+ where group_index <= 1000" \
594
+ --exclude="id, validation_errors, group_index"
595
+
596
+ iexport -t $REQ \
597
+ -o "$OUTPUT_DIR/$REQ.csv" -f csv --no-quote-empty --quotes --headers \
598
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
599
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
600
+ $REQ order by id ) as main) as tmp
601
+ where group_index <= 1000" \
602
+ --exclude="id, validation_errors, group_index"
603
+
604
+ iexport -t $ITEM \
605
+ -o "$OUTPUT_DIR/$ITEM.csv" -f csv --no-quote-empty --quotes --headers \
606
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
607
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
608
+ $ITEM order by id ) as main) as tmp
609
+ where group_index <= 1000" \
610
+ --exclude="id, validation_errors, group_index"
611
+
612
+ iexport -t $USER \
613
+ -o "$OUTPUT_DIR/$USER.csv" -f csv --no-quote-empty --quotes --headers \
614
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
615
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
616
+ $USER order by id ) as main) as tmp
617
+ where group_index <= 1000" \
618
+ --exclude="id, validation_errors, group_index"
619
+
620
+ iexport -t $LOCATION \
621
+ -o "$OUTPUT_DIR/$LOCATION.csv" -f csv --no-quote-empty --quotes --headers \
622
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
623
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
624
+ $LOCATION order by id ) as main) as tmp
625
+ where group_index <= 1000" \
626
+ --exclude="id, validation_errors, group_index"
627
+
628
+ iexport -t $ULPR \
629
+ -o "$OUTPUT_DIR/$ULPR.csv" -f csv --no-quote-empty --quotes --headers \
630
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
631
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
632
+ $ULPR order by id ) as main) as tmp
633
+ where group_index <= 1000" \
634
+ --exclude="id, validation_errors, group_index"
635
+
636
+ iexport -t $LPR \
637
+ -o "$OUTPUT_DIR/$LPR.csv" -f csv --no-quote-empty --quotes --headers \
638
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
639
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
640
+ $LPR order by id ) as main) as tmp
641
+ where group_index <= 1000" \
642
+ --exclude="id, validation_errors, group_index"
643
+
644
+
645
+ # Use SQL to compute the summary, write the outputs to summary.csv
646
+ # --------------------------------------------------------------
647
+ iexport --output="$OUTPUT_DIR/summary.csv" -f csv --no-quote-empty --quotes --headers \
648
+ --query="(select 'ContractMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $CONTRACTO), 2)::varchar || '%' as percentage from $CONTRACTO group by error order by error) union
649
+ (select 'VendorMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $VENDOR), 2)::varchar || '%' as percentage from $VENDOR group by error order by error) union
650
+ (select 'ItemMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEM), 2)::varchar || '%' as percentage from $ITEM group by error order by error) union
651
+ (select 'ReqHistoryLoad' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $REQ), 2)::varchar || '%' as percentage from $REQ group by error order by error) union
652
+ (select 'MfrMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $MFR), 2)::varchar || '%' as percentage from $MFR group by error order by error) union
653
+ (select 'PurchaseOrder' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $PO), 2)::varchar || '%' as percentage from $PO group by error order by error) union
654
+ (select 'PurchaseOrder2' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $PO2), 2)::varchar || '%' as percentage from $PO2 group by error order by error) union
655
+ (select 'Inventory' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $INVENTORY), 2)::varchar || '%' as percentage from $INVENTORY group by error order by error) union
656
+ (select 'User' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $USER), 2)::varchar || '%' as percentage from $USER group by error order by error) union
657
+ (select 'Location' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LOCATION), 2)::varchar || '%' as percentage from $LOCATION group by error order by error) union
658
+ --(select 'ItemCost' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMCOST), 2)::varchar || '%' as percentage from $ITEMCOST group by error order by error) union
659
+ (select 'GLAccount' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $GL), 2)::varchar || '%' as percentage from $GL group by error order by error) union
660
+ (select 'UserLocationProfile' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ULPR), 2)::varchar || '%' as percentage from $ULPR group by error order by error) union
661
+ (select 'LocationProfile' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LPR), 2)::varchar || '%' as percentage from $LPR group by error order by error)"
662
+
663
+ # Merge summary.xls and report files into one single file with several tabs
664
+ # --input="ItemCostCenterAcctExceptions:$OUTPUT_DIR/$ITEMCOST.csv" \
665
+ imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
666
+ --input="Summary:$OUTPUT_DIR/summary.csv" \
667
+ --input="ContractMaster:$OUTPUT_DIR/$CONTRACTO.csv" \
668
+ --input="ItemMaster:$OUTPUT_DIR/$ITEM.csv" \
669
+ --input="MfrMaster:$OUTPUT_DIR/$MFR.csv" \
670
+ --input="PurchaseOrder:$OUTPUT_DIR/$PO.csv" \
671
+ --input="PurchaseOrder2:$OUTPUT_DIR/$PO2.csv" \
672
+ --input="User:$OUTPUT_DIR/$USER.csv" \
673
+ --input="Location:$OUTPUT_DIR/$LOCATION.csv" \
674
+ --input="ReqHistoryLoad:$OUTPUT_DIR/$REQ.csv" \
675
+ --input="GLAccount:$OUTPUT_DIR/$GL.csv" \
676
+ --input="Inventory:$OUTPUT_DIR/$INVENTORY.csv" \
677
+ --input="$ULPR:$OUTPUT_DIR/$ULPR.csv" \
678
+ --input="$LPR:$OUTPUT_DIR/$LPR.csv"
679
+
680
+ exit
data/full.sh CHANGED
@@ -28,6 +28,7 @@ FITEMCOST="ItemCostCenterAcctExceptions.csv"
28
28
  FREQ="ReqHistoryLoad.csv"
29
29
  FGL="GLAccount.csv"
30
30
  FINVENTORY="Inventory.csv"
31
+ FITEMDEFAULTEXP="ItemAndDefaultExpenseAccount.csv"
31
32
 
32
33
  CONTRACTO="contracts"
33
34
  VENDOR="vendors"
@@ -41,6 +42,7 @@ ITEMCOST="item_costs"
41
42
  REQ="reqs"
42
43
  GL="gls"
43
44
  INVENTORY="inventory"
45
+ ITEMDEFAULTEXP="ItemAndDefaultExpenseAccount"
44
46
 
45
47
  # Standard UOM for reference
46
48
  FUOMSTD="stduom.txt"
@@ -67,6 +69,7 @@ iload -i "$FITEMCOST" -t "$ITEMCOST" -f csv
67
69
  iload -i "$FREQ" -t "$REQ" -f csv
68
70
  iload -i "$FGL" -t "$GL" -f csv
69
71
  iload -i "$FINVENTORY" -t "$INVENTORY" -f csv
72
+ iload -i "$FITEMDEFAULTEXP" -t "$ITEMDEFAULTEXP" -f csv
70
73
 
71
74
  # Load UOM STANDARD for reference
72
75
  iload -i "$FUOMSTD" -t "$UOMSTD" -f csv
@@ -127,7 +130,9 @@ UPDATE $CONTRACTO SET contract_start = to_char(to_date(contract_start, 'MM/DD/YY
127
130
  WHERE contract_start like '%/%/%';
128
131
  UPDATE $CONTRACTO SET contract_end = to_char(to_date(contract_end, 'MM/DD/YYYY'), 'YYYY-MM-DD')
129
132
  WHERE contract_end like '%/%/%';
133
+ "
130
134
 
135
+ ipatch -q "
131
136
  -- ADD MORE EMPTY FIELD (for validation)
132
137
  -- ALTER TABLE $LOCATION ADD COLUMN corp_id varchar;
133
138
 
@@ -139,7 +144,7 @@ UPDATE $LOCATION SET stockless_ind = 'N' WHERE stockless_ind IS NULL OR LENGTH(t
139
144
 
140
145
  # Convert A2A2A2 to A2-A2-A2
141
146
  # Manual check and comment out the following to speed up validating
142
- # @todo IEVAL performance is very poor, avoid using it (use IPATCH instead) unless there is no better way
147
+ # @todo ieval (evaluate Ruby code) performance is very poor, avoid using it (use IPATCH instead) unless there is no better way
143
148
  ieval -t $GL --eval="
144
149
  if item.corp_acct_fmt && item.corp_acct_fmt[/^[A-Z][0-9][A-Z][0-9][A-Z][0-9]$/]
145
150
  item.corp_acct_fmt = item.corp_acct_fmt.unpack('a2a2a2').join('-')
@@ -154,8 +159,7 @@ ivalidate --case-insensitive --pretty -t $VENDOR \
154
159
  --log-to=validation_errors \
155
160
  --not-null="vendor_code" \
156
161
  --not-null="vendor_name" \
157
- --unique="vendor_code" \
158
- --unique="vendor_name" \
162
+ --unique="vendor_code, vendor_name" \
159
163
  --match="vendor_code/[a-zA-Z0-9]/" \
160
164
  --match="vendor_name/[a-zA-Z0-9]/" \
161
165
  --consistent-by="vendor_code|vendor_name" \
@@ -168,10 +172,11 @@ ivalidate --case-insensitive --pretty -t $MFR \
168
172
  --log-to=validation_errors \
169
173
  --not-null="mfr_number" \
170
174
  --not-null="mfr_name" \
175
+ --unique="mfr_number, mfr_name" \
171
176
  --match="mfr_number/[a-zA-Z0-9]/" \
172
177
  --match="mfr_name/[a-zA-Z0-9]/" \
173
- --consistent-by="mfr_number|mfr_name" \
174
178
  --consistent-by="mfr_name|mfr_number" \
179
+ --consistent-by="mfr_number|mfr_name" \
175
180
  --consistent-by="country_code|country_name" \
176
181
  --consistent-by="country_name|country_code"
177
182
 
@@ -198,15 +203,15 @@ ivalidate --case-insensitive --pretty -t $GL \
198
203
  --consistent-by="corp_acct_name|corp_acct_no" \
199
204
  --consistent-by="exp_acct_no|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_name" \
200
205
  --consistent-by="exp_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_no" \
201
- --consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name" \
202
- --consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no" \
206
+ --consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name, exp_acct_no, exp_acct_name" \
207
+ --consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, exp_acct_no, exp_acct_name" \
203
208
  --not-null=exp_acct_type \
204
209
  --match="exp_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/"
205
210
 
206
211
  # validate LOCATION
207
212
  ivalidate --case-insensitive --pretty -t $LOCATION \
208
213
  --log-to=validation_errors \
209
- --not-null=loc_id \
214
+ --not-null="loc_id" \
210
215
  --match="loc_id/[a-zA-Z0-9]/" \
211
216
  --not-null="name" \
212
217
  --match="name/[a-zA-Z0-9]/" \
@@ -214,22 +219,19 @@ ivalidate --case-insensitive --pretty -t $LOCATION \
214
219
  --match="facility_code/[a-zA-Z0-9]/" \
215
220
  --not-null=facility_desc \
216
221
  --match="facility_desc/[a-zA-Z0-9]/" \
217
- --match="ship_to_ind/^(Y|N)$/" \
218
- --match="bill_to_ind/^(Y|N)$/" \
219
- --match="stockless_ind/^(Y|N)$/" \
220
- --not-null=loc_type \
222
+ --match="ship_to_ind/^(Y|N|y|n)$/" \
223
+ --match="bill_to_ind/^(Y|N|y|n)$/" \
224
+ --match="stockless_ind/^(Y|N|y|n)$/" \
225
+ --not-null="loc_type" \
221
226
  --match="loc_type/^(C|S|LOC_TYPE_SUPPLY|LOC_TYPE_CONSUME)$/" \
222
- --not-null=inventory_path_name \
223
- --not-null=inventory_location_name \
224
- --query="(route_no is null or route_no = '' or regexp_replace(route_no, '[,\.]', '', 'g') ~ '^[1-9][0-9]+$') -- invalid route_no" \
225
227
  --rquery="(loc_type ~* '^(LOC_TYPE_SUPPLY|S)$' and (corp_acct_no is null or corp_name is null or corp_id is null)) -- either corp id/name or corp_acct_no is null" \
226
- --not-null=active \
227
- --match="active/^(Y|N)$/" \
228
- --not-null=corp_acct_no \
228
+ --not-null="active" \
229
+ --match="active/^(Y|N|1|2|3)$/" \
230
+ --not-null="corp_acct_no" \
229
231
  --match="corp_acct_no/[a-zA-Z0-9]/" \
230
- --not-null=inventory_loc_seq_no \
231
- --match="inventory_loc_seq_no/^[1-9][0-9]*$/" \
232
- --match="route_no/^[1-9][0-9]*$/" \
232
+ --rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_loc_seq_no IS NULL OR inventory_loc_seq_no = '')) -- [inventory_loc_seq_no] is null" \
233
+ --rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_location_name IS NULL OR inventory_location_name = '')) -- [inventory_location_name] is null" \
234
+ --match="route_no/[a-zA-Z0-9]/" \
233
235
  --match="route_name/[a-zA-Z0-9]/" \
234
236
  --match="corp_name/[a-zA-Z0-9]/" \
235
237
  --consistent-by="corp_name|corp_id" \
@@ -252,21 +254,25 @@ ivalidate --case-insensitive --pretty -t $CONTRACTO \
252
254
  --not-null=mfr_item_id \
253
255
  --not-null=mfr_name \
254
256
  --not-null=item_uom \
257
+ --not-null=corp_id \
258
+ --not-null=item_descr \
255
259
  --not-null=item_qoe \
256
260
  --not-null=contract_price \
257
261
  --not-null=contract_gpo_name \
262
+ --not-null=contract_gpo_id \
258
263
  --match="contract_number/[a-zA-Z0-9]/" \
264
+ --match="contract_gpo_name/[a-zA-Z0-9]/" \
265
+ --match="corp_id/[a-zA-Z0-9]/" \
259
266
  --match="corp_name/[a-zA-Z0-9]/" \
260
267
  --match="vendor_item_id/[a-zA-Z0-9]/" \
261
268
  --match="vendor_name/[a-zA-Z0-9]/" \
262
269
  --match="mfr_item_id/[a-zA-Z0-9]/" \
263
270
  --match="mfr_name/[a-zA-Z0-9]/" \
264
- --query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- Contract end-date goes before start-date" \
271
+ --query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- [contract_end] comes before [contract_start]" \
265
272
  --match="contract_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
266
273
  --match="item_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
267
274
  --consistent-by="corp_id|corp_name" \
268
275
  --consistent-by="corp_name|corp_id" \
269
- --consistent-by="mfr_number|mfr_name" \
270
276
  --consistent-by="mfr_name|mfr_number" \
271
277
  --consistent-by="vendor_code|vendor_name" \
272
278
  --consistent-by="vendor_name|vendor_code" \
@@ -306,6 +312,7 @@ ivalidate --case-insensitive --pretty -t $ITEM \
306
312
  --match="mfr_number/[a-zA-Z0-9]/" \
307
313
  --match="mfr_name/[a-zA-Z0-9]/" \
308
314
  --match="active/^(1|2|3|A|I)$/" \
315
+ --unique="item_id, corp_id, vendor_code, item_uom" \
309
316
  --cross-reference="vendor_code|$VENDOR.vendor_code" \
310
317
  --cross-reference="vendor_name|$VENDOR.vendor_name" \
311
318
  --cross-reference="mfr_number|$MFR.mfr_number" \
@@ -314,7 +321,6 @@ ivalidate --case-insensitive --pretty -t $ITEM \
314
321
  --cross-reference="corp_name|$GL.corp_acct_name" \
315
322
  --consistent-by="corp_id|corp_name" \
316
323
  --consistent-by="corp_name|corp_id" \
317
- --consistent-by="mfr_number|mfr_name" \
318
324
  --consistent-by="mfr_name|mfr_number" \
319
325
  --consistent-by="vendor_code|vendor_name" \
320
326
  --consistent-by="vendor_name|vendor_code" \
@@ -356,7 +362,7 @@ ivalidate --case-insensitive --pretty -t $PO \
356
362
  --consistent-by="vendor_name|vendor_code" \
357
363
  --consistent-by="mfr_name|mfr_number" \
358
364
  --unique="po_no, po_line_number" \
359
- --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
365
+ --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- [item_id] does not reference [items.item_id]" \
360
366
  --cross-reference="vendor_code|$VENDOR.vendor_code" \
361
367
  --cross-reference="vendor_name|$VENDOR.vendor_name" \
362
368
  --cross-reference="mfr_number|$MFR.mfr_number" \
@@ -365,8 +371,8 @@ ivalidate --case-insensitive --pretty -t $PO \
365
371
  --cross-reference="corp_name|$GL.corp_acct_name" \
366
372
  --cross-reference="cost_center_id|$GL.cc_acct_no" \
367
373
  --cross-reference="cost_center_name|$GL.cc_acct_name" \
368
- --rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
369
- --rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
374
+ --rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid [purchase_uom]" \
375
+ --rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- [vendor_item_id] is null" \
370
376
  --match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
371
377
  --match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
372
378
 
@@ -391,13 +397,9 @@ ivalidate --case-insensitive --pretty -t $REQ \
391
397
  --match="vendor_name/[a-zA-Z0-9]/" \
392
398
  --not-null="vendor_code" \
393
399
  --match="vendor_code/[a-zA-Z0-9]/" \
394
- --not-null="mfr_name" \
395
- --match="mfr_name/[a-zA-Z0-9]/" \
396
- --not-null="item_descr" \
397
- --match="item_descr/[a-zA-Z0-9]/" \
398
400
  --consistent-by="corp_id|corp_name" \
399
401
  --consistent-by="corp_name|corp_id" \
400
- --rquery="(item_id not like '%~[%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
402
+ --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
401
403
  --cross-reference="corp_id|$GL.corp_acct_no" \
402
404
  --cross-reference="corp_name|$GL.corp_acct_name" \
403
405
  --cross-reference="vendor_name|$VENDOR.vendor_name" \
@@ -412,7 +414,7 @@ ivalidate --case-insensitive --pretty -t $USER \
412
414
  --log-to=validation_errors \
413
415
  --not-null="email" \
414
416
  --unique="email" \
415
- --query="lower(email) ~* '[a-z0-9][a-z0-9_\.]+@[a-z0-9][a-z0-9_\.\-]+\.[a-z0-9_\.\-]+' -- invalid email address"
417
+ --match="lower(email)/[a-z0-9][a-z0-9_\.]+@[a-z0-9][a-z0-9_\.\-]+\.[a-z0-9_\.\-]+/"
416
418
 
417
419
  # validate INVENTORY
418
420
  ivalidate --case-insensitive --pretty -t $INVENTORY \
@@ -423,15 +425,11 @@ ivalidate --case-insensitive --pretty -t $INVENTORY \
423
425
  --match="loc_id/[a-zA-Z0-9]/" \
424
426
  --not-null="vendor_code" \
425
427
  --match="vendor_code/[a-zA-Z0-9]/" \
426
- --not-null="vendor_name" \
427
- --match="vendor_name/[a-zA-Z0-9]/" \
428
428
  --not-null="corp_id" \
429
- --match="corp_name/[a-zA-Z0-9]/" \
430
- --not-null="location_name" \
429
+ --match="corp_id/[a-zA-Z0-9]/" \
431
430
  --match="location_name/[a-zA-Z0-9]/" \
432
431
  --not-null="item_id" \
433
- --match="item_id/[a-zA-Z0-9]/" \
434
- --match="inventory_status/^(Active|Pending Inactive|Inactive)$/" \
432
+ --match="inventory_status/^(Active|Pending Inactive|Inactive|1|2|3)$/" \
435
433
  --cross-reference="item_id|$ITEM.item_id" \
436
434
  --cross-reference="location_name|$LOCATION.name" \
437
435
  --cross-reference="vendor_code|$VENDOR.vendor_code" \
@@ -452,6 +450,12 @@ ivalidate --case-insensitive --pretty -t $ITEMCOST \
452
450
  --cross-reference="item_no|$ITEM.item_id" \
453
451
  --cross-reference="exp_acct_no|$GL.exp_acct_no"
454
452
 
453
+ ivalidate --case-insensitive --pretty -t $ITEMDEFAULTEXP \
454
+ --log-to=validation_errors \
455
+ --cross-reference="corp_acct_no|$GL.corp_acct_no" \
456
+ --cross-reference="item_no|$ITEM.item_id" \
457
+ --cross-reference="exp_acct_no|$GL.exp_acct_no"
458
+
455
459
 
456
460
 
457
461
  ####################################################
@@ -466,6 +470,14 @@ iexport -t $ITEMCOST \
466
470
  $ITEMCOST order by id ) as main) as tmp
467
471
  where group_index <= 1000" \
468
472
  --exclude="id, validation_errors, group_index"
473
+
474
+ iexport -t $ITEMDEFAULTEXP \
475
+ -o "$OUTPUT_DIR/$ITEMDEFAULTEXP.csv" -f csv --no-quote-empty --quotes --headers \
476
+ --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
477
+ FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
478
+ $ITEMDEFAULTEXP order by id ) as main) as tmp
479
+ where group_index <= 1000" \
480
+ --exclude="id, validation_errors, group_index"
469
481
 
470
482
  iexport -t $CONTRACTO \
471
483
  -o "$OUTPUT_DIR/$CONTRACTO.csv" -f csv --no-quote-empty --quotes --headers \
@@ -561,12 +573,14 @@ iexport --output="$OUTPUT_DIR/summary.csv" -f csv --no-quote-empty --quotes --he
561
573
  (select 'User' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $USER), 2)::varchar || '%' as percentage from $USER group by error order by error) union
562
574
  (select 'Location' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LOCATION), 2)::varchar || '%' as percentage from $LOCATION group by error order by error) union
563
575
  (select 'ItemCost' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMCOST), 2)::varchar || '%' as percentage from $ITEMCOST group by error order by error) union
576
+ (select 'ItemAndDefaultEXP' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMDEFAULTEXP), 2)::varchar || '%' as percentage from $ITEMDEFAULTEXP group by error order by error) union
564
577
  (select 'GLAccount' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $GL), 2)::varchar || '%' as percentage from $GL group by error order by error)"
565
578
 
566
579
  # Merge summary.xls and report files into one single file with several tabs
567
580
  imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
568
581
  --input="Summary:$OUTPUT_DIR/summary.csv" \
569
582
  --input="ItemCostCenterAcctExceptions:$OUTPUT_DIR/$ITEMCOST.csv" \
583
+ --input="ItemAndDefaultEXP:$OUTPUT_DIR/$ITEMDEFAULTEXP.csv" \
570
584
  --input="ContractMaster:$OUTPUT_DIR/$CONTRACTO.csv" \
571
585
  --input="ItemMaster:$OUTPUT_DIR/$ITEM.csv" \
572
586
  --input="MfrMaster:$OUTPUT_DIR/$MFR.csv" \
data/idata.gemspec CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
10
10
  spec.email = ["minhnghivn@gmail.com"]
11
11
  spec.description = %q{Tools for importing data from raw files}
12
12
  spec.summary = %q{Tools include: iload, ivalidate, isanitize, ipatch, ieval, iexport}
13
- spec.homepage = "http://bolero.vn"
13
+ spec.homepage = "https://github.com/minhnghivn/idata"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files`.split($/)
data/lib/idata/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Idata
2
- VERSION = "0.1.31"
2
+ VERSION = "0.1.32"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: idata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.31
4
+ version: 0.1.32
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nghi Pham
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-07-29 00:00:00.000000000 Z
11
+ date: 2014-07-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -92,12 +92,13 @@ files:
92
92
  - bin/ipatch
93
93
  - bin/isanitize
94
94
  - bin/ivalidate
95
+ - full-pg.sh
95
96
  - full.sh
96
97
  - idata.gemspec
97
98
  - lib/idata.rb
98
99
  - lib/idata/version.rb
99
100
  - sample.sh
100
- homepage: http://bolero.vn
101
+ homepage: https://github.com/minhnghivn/idata
101
102
  licenses:
102
103
  - MIT
103
104
  metadata: {}