idata 0.1.31 → 0.1.32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/iload +9 -3
- data/full-pg.sh +680 -0
- data/full.sh +52 -38
- data/idata.gemspec +1 -1
- data/lib/idata/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef83b06323363e84794a14ef18b23903141f2cd8
|
4
|
+
data.tar.gz: 21cf263eafd81be05628474f6f80c4a0eac76192
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9470f905dee46dc8d14a6cdabca4b018ba95a14d967e7ae2836fe0827f6c1613e5b38a28786c0e9d5feb8dfceae5d13d1d1c1df42a803a7d7917d46618ac5b0c
|
7
|
+
data.tar.gz: 29c38860bc9d90146d67f07865c3846c3a9c3f476447a53fd378a0211dc329387f1746e15219fa945733ac45d3484697408a04b678fbb0bbaaf0ccb122f3f810
|
data/bin/iload
CHANGED
@@ -176,14 +176,20 @@ ActiveRecord::Base.establish_connection(
|
|
176
176
|
$csv_converters = [:stripper]
|
177
177
|
|
178
178
|
CSV::Converters[:stripper] = lambda{ |s|
|
179
|
-
|
179
|
+
if s.is_a?(String)
|
180
|
+
r = s.strip
|
181
|
+
return nil if r == ""
|
182
|
+
return r
|
183
|
+
else
|
184
|
+
return s
|
185
|
+
end
|
180
186
|
}
|
181
187
|
|
182
188
|
$csv_converters << :null_converter if $options[:null]
|
183
189
|
|
184
190
|
CSV::Converters[:null_converter] = lambda{ |s|
|
185
|
-
return nil if s == $options[:null]
|
186
|
-
return s
|
191
|
+
return nil if s == $options[:null]
|
192
|
+
return s
|
187
193
|
}
|
188
194
|
|
189
195
|
class String
|
data/full-pg.sh
ADDED
@@ -0,0 +1,680 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# SHARED VARIABLES
|
4
|
+
# ---------------------------------------------------------------------------------
|
5
|
+
# Database to store data tables
|
6
|
+
ORGNAME="ne20140729"
|
7
|
+
|
8
|
+
# ENV variables used by the validation command
|
9
|
+
export HOST="localhost"
|
10
|
+
export USERNAME="postgres"
|
11
|
+
export PASSWORD="postgres"
|
12
|
+
export DATABASE="$ORGNAME"
|
13
|
+
export LISTEN=7432
|
14
|
+
|
15
|
+
# Create temporary output folder
|
16
|
+
OUTPUT_DIR="/tmp/$ORGNAME" && test -e $OUTPUT_DIR || mkdir $OUTPUT_DIR
|
17
|
+
|
18
|
+
# Input files and correspondings names
|
19
|
+
FCONTRACTO="Contract.csv"
|
20
|
+
FVENDOR="VendorMasterUpdate.csv"
|
21
|
+
FITEM="ItemMaster.csv"
|
22
|
+
FINVOICE="InvoiceHistory.csv"
|
23
|
+
FMFR="MfrMaster.csv"
|
24
|
+
FPO="20140708_150920_s2_PoHistory_Report2_NEGA_modified.csv"
|
25
|
+
FPO2="PurchaseOrder.csv"
|
26
|
+
FUSER="User.csv"
|
27
|
+
FLOCATION="Location.csv"
|
28
|
+
FITEMCOST="ItemCostCenterAcctExceptions.csv"
|
29
|
+
FREQ="ReqHistoryLoad.csv"
|
30
|
+
FGL="GLAccount.csv"
|
31
|
+
FINVENTORY="Inventory.csv"
|
32
|
+
FULPR="UserAndLocationProfile.csv"
|
33
|
+
FLPR="LocationProfile.csv"
|
34
|
+
|
35
|
+
CONTRACTO="contracts"
|
36
|
+
VENDOR="vendors"
|
37
|
+
ITEM="items"
|
38
|
+
INVOICE="invoices"
|
39
|
+
MFR="manufacturers"
|
40
|
+
PO="purchase_orders"
|
41
|
+
PO2="second_purchase_orders"
|
42
|
+
USER="users"
|
43
|
+
LOCATION="locations"
|
44
|
+
ITEMCOST="item_costs"
|
45
|
+
REQ="reqs"
|
46
|
+
GL="gls"
|
47
|
+
INVENTORY="inventory"
|
48
|
+
ULPR="user_location_profiles"
|
49
|
+
LPR="location_profiles"
|
50
|
+
|
51
|
+
# Standard UOM for reference
|
52
|
+
FUOMSTD="stduom.txt"
|
53
|
+
UOMSTD="uomstd"
|
54
|
+
|
55
|
+
# ---------------------------------------------------------------------------------
|
56
|
+
# LOAD
|
57
|
+
# ---------------------------------------------------------------------------------
|
58
|
+
# Sanitize input files
|
59
|
+
#isanitize --strip-newline --remove=65279 "$FCONTRACTO" "$FVENDOR" "$FITEM" "$FINVOICE" "$FMFR" "$FPO" \
|
60
|
+
# "$FTMPL" "$FUSER" "$FITEMEXPO" "$FUSEREQ" "$FLOCATION" "$FITEMCOST" "$FREQ" \
|
61
|
+
# "$FGL" "$FUSERCOST" "$FINVENTORY" "$FUSERCREATEREQ"
|
62
|
+
|
63
|
+
# Load
|
64
|
+
iload -i "$FCONTRACTO" -t "$CONTRACTO" -f csv
|
65
|
+
iload -i "$FVENDOR" -t "$VENDOR" -f csv
|
66
|
+
iload -i "$FITEM" -t "$ITEM" -f csv
|
67
|
+
#iload -i "$FINVOICE" -t "$INVOICE" -f csv
|
68
|
+
iload -i "$FMFR" -t "$MFR" -f csv
|
69
|
+
iload -i "$FPO" -t "$PO" -f csv
|
70
|
+
iload -i "$FPO2" -t "$PO2" -f csv
|
71
|
+
iload -i "$FUSER" -t "$USER" -f csv
|
72
|
+
iload -i "$FLOCATION" -t "$LOCATION" -f csv
|
73
|
+
#iload -i "$FITEMCOST" -t "$ITEMCOST" -f csv
|
74
|
+
iload -i "$FREQ" -t "$REQ" -f csv
|
75
|
+
iload -i "$FGL" -t "$GL" -f csv
|
76
|
+
iload -i "$FINVENTORY" -t "$INVENTORY" -f csv
|
77
|
+
iload -i "$FULPR" -t "$ULPR" -f csv
|
78
|
+
iload -i "$FLPR" -t "$LPR" -f csv
|
79
|
+
|
80
|
+
# Load UOM STANDARD for reference
|
81
|
+
iload -i "$FUOMSTD" -t "$UOMSTD" -f csv
|
82
|
+
|
83
|
+
# ---------------------------------------------------------------------------------
|
84
|
+
# INDEXING (for speeding up validation queries)
|
85
|
+
# ---------------------------------------------------------------------------------
|
86
|
+
ipatch -q "drop index if exists items_item_id_index; create index items_item_id_index on $ITEM(item_id)"
|
87
|
+
ipatch -q "drop index if exists pos_item_id_index; create index pos_item_id_index on $PO(item_id)"
|
88
|
+
ipatch -q "drop index if exists gls_corp_acct_no_index; create index gls_corp_acct_no_index on $GL(corp_acct_no)"
|
89
|
+
ipatch -q "drop index if exists gls_corp_acct_name_index; create index gls_corp_acct_name_index on $GL(corp_acct_name)"
|
90
|
+
ipatch -q "drop index if exists gls_cc_acct_no_index; create index gls_cc_acct_no_index on $GL(cc_acct_no)"
|
91
|
+
ipatch -q "drop index if exists gls_cc_acct_name_index; create index gls_cc_acct_name_index on $GL(cc_acct_name)"
|
92
|
+
ipatch -q "drop index if exists locations_name_index; create index locations_name_index on $LOCATION(name)"
|
93
|
+
|
94
|
+
# ---------------------------------------------------------------------------------
|
95
|
+
# Some adjustment
|
96
|
+
# ---------------------------------------------------------------------------------
|
97
|
+
ipatch -q "
|
98
|
+
-- Normalize values
|
99
|
+
UPDATE $ITEM set active = '1' where active = 'Y';
|
100
|
+
UPDATE items set active = '3' where active = 'N';
|
101
|
+
"
|
102
|
+
|
103
|
+
ipatch -q "
|
104
|
+
-- Extract FIRST_NAME, LAST_NAME from NAME
|
105
|
+
--ALTER TABLE $USER ADD COLUMN first_name varchar;
|
106
|
+
--ALTER TABLE $USER ADD COLUMN last_name varchar;
|
107
|
+
--UPDATE $USER SET username = trim(regexp_replace(username, '\s+', ' ', 'g'));
|
108
|
+
--UPDATE $USER SET first_name = username, last_name = username
|
109
|
+
--WHERE array_length(string_to_array(username, ' '), 1) = 1;
|
110
|
+
--UPDATE $USER SET first_name = substring(username, '^[^\s]+'),
|
111
|
+
-- last_name = regexp_replace(username, '^[^\s]+\s', '', 'g')
|
112
|
+
--WHERE array_length(string_to_array(username, ' '), 1) >= 2;
|
113
|
+
"
|
114
|
+
|
115
|
+
ipatch -q "
|
116
|
+
-- ADD EMAIL COLUMN
|
117
|
+
--ALTER TABLE $USER ADD COLUMN email VARCHAR;
|
118
|
+
--UPDATE $USER SET email = userlogin;
|
119
|
+
"
|
120
|
+
|
121
|
+
ipatch -q "
|
122
|
+
-- FIX ISSUE WITH linebreak
|
123
|
+
update items set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\r\n', ' '), '\s+', ' ', 'g'), ' '))[1]
|
124
|
+
where vendor_item_id like E'%\r\n%';
|
125
|
+
update purchase_orders set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\r\n', ' '), '\s+', ' ', 'g'), ' '))[1]
|
126
|
+
where vendor_item_id like E'%\r\n%';
|
127
|
+
update purchase_orders set vendor_item_id = (string_to_array(regexp_replace(replace(vendor_item_id, E'\n', ' '), '\s+', ' ', 'g'), ' '))[1]
|
128
|
+
where vendor_item_id like E'%\n%';
|
129
|
+
"
|
130
|
+
|
131
|
+
ipatch -q "
|
132
|
+
-- Normalize DATE
|
133
|
+
UPDATE $PO SET po_date = to_char(to_date(po_date, 'MM/DD/YYYY'), 'YYYY-MM-DD')
|
134
|
+
WHERE po_date IS NOT NULL AND po_date != '';
|
135
|
+
UPDATE $CONTRACTO SET contract_start = to_char(to_date(contract_start, 'MM/DD/YYYY'), 'YYYY-MM-DD')
|
136
|
+
WHERE contract_start like '%/%/%';
|
137
|
+
UPDATE $CONTRACTO SET contract_end = to_char(to_date(contract_end, 'MM/DD/YYYY'), 'YYYY-MM-DD')
|
138
|
+
WHERE contract_end like '%/%/%';
|
139
|
+
|
140
|
+
-- ADD MORE EMPTY FIELD (for validation)
|
141
|
+
-- ALTER TABLE $LOCATION ADD COLUMN corp_id varchar;
|
142
|
+
|
143
|
+
-- FILL EMPTY FIELDS WITH DEFAULT VALUE
|
144
|
+
UPDATE $LOCATION SET ship_to_ind = 'N' WHERE ship_to_ind IS NULL OR LENGTH(trim(ship_to_ind)) = 0;
|
145
|
+
UPDATE $LOCATION SET bill_to_ind = 'N' WHERE bill_to_ind IS NULL OR LENGTH(trim(bill_to_ind)) = 0;
|
146
|
+
UPDATE $LOCATION SET stockless_ind = 'N' WHERE stockless_ind IS NULL OR LENGTH(trim(stockless_ind)) = 0;
|
147
|
+
"
|
148
|
+
|
149
|
+
# Convert A2A2A2 to A2-A2-A2
|
150
|
+
# Manual check and comment out the following to speed up validating
|
151
|
+
# @todo IEVAL performance is very poor, avoid using it (use IPATCH instead) unless there is no better way
|
152
|
+
ieval -t $GL --eval="
|
153
|
+
if item.corp_acct_fmt && item.corp_acct_fmt[/^[A-Z][0-9][A-Z][0-9][A-Z][0-9]$/]
|
154
|
+
item.corp_acct_fmt = item.corp_acct_fmt.unpack('a2a2a2').join('-')
|
155
|
+
end
|
156
|
+
"
|
157
|
+
|
158
|
+
# ---------------------------------------------------------------------------------
|
159
|
+
# VALIDATE
|
160
|
+
# ---------------------------------------------------------------------------------
|
161
|
+
# validate VENDORS
|
162
|
+
ivalidate --case-insensitive --pretty -t $VENDOR \
|
163
|
+
--log-to=validation_errors \
|
164
|
+
--not-null="vendor_code" \
|
165
|
+
--not-null="vendor_name" \
|
166
|
+
--unique="vendor_code,vendor_name" \
|
167
|
+
--match="vendor_code/[a-zA-Z0-9]/" \
|
168
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
169
|
+
--consistent-by="vendor_code|vendor_name" \
|
170
|
+
--consistent-by="vendor_name|vendor_code" \
|
171
|
+
--consistent-by="country_code|country_name" \
|
172
|
+
--consistent-by="country_name|country_code"
|
173
|
+
|
174
|
+
# validate MANUFACTURERS
|
175
|
+
ivalidate --case-insensitive --pretty -t $MFR \
|
176
|
+
--log-to=validation_errors \
|
177
|
+
--not-null="mfr_number" \
|
178
|
+
--not-null="mfr_name" \
|
179
|
+
--match="mfr_number/[a-zA-Z0-9]/" \
|
180
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
181
|
+
--consistent-by="mfr_name|mfr_number" \
|
182
|
+
--consistent-by="country_code|country_name" \
|
183
|
+
--consistent-by="country_name|country_code"
|
184
|
+
|
185
|
+
# validate GL
|
186
|
+
ivalidate --case-insensitive --pretty -t $GL \
|
187
|
+
--log-to=validation_errors \
|
188
|
+
--not-null=corp_acct_no \
|
189
|
+
--match="corp_acct_no/[a-zA-Z0-9]/" \
|
190
|
+
--not-null=corp_acct_name \
|
191
|
+
--match="corp_acct_name/[a-zA-Z0-9]/" \
|
192
|
+
--not-null=corp_acct_fmt \
|
193
|
+
--match="corp_acct_fmt/^[A-Z][0-9]-[A-Z][0-9]-[A-Z][0-9]$/" \
|
194
|
+
--not-null=cc_acct_no \
|
195
|
+
--match="cc_acct_no/[a-zA-Z0-9]/" \
|
196
|
+
--not-null=cc_acct_name \
|
197
|
+
--match="cc_acct_name/[a-zA-Z0-9]/" \
|
198
|
+
--not-null=cc_acct_type \
|
199
|
+
--match="cc_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/" \
|
200
|
+
--not-null=exp_acct_no \
|
201
|
+
--match="exp_acct_no/[a-zA-Z0-9]/" \
|
202
|
+
--not-null=exp_acct_name \
|
203
|
+
--match="exp_acct_name/[a-zA-Z0-9]/" \
|
204
|
+
--consistent-by="corp_acct_no|corp_acct_name" \
|
205
|
+
--consistent-by="corp_acct_name|corp_acct_no" \
|
206
|
+
--consistent-by="exp_acct_no|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_name" \
|
207
|
+
--consistent-by="exp_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_no" \
|
208
|
+
--consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name" \
|
209
|
+
--consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no" \
|
210
|
+
--not-null=exp_acct_type \
|
211
|
+
--match="exp_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/"
|
212
|
+
|
213
|
+
# validate LOCATION
|
214
|
+
ivalidate --case-insensitive --pretty -t $LOCATION \
|
215
|
+
--log-to=validation_errors \
|
216
|
+
--not-null=loc_id \
|
217
|
+
--match="loc_id/[a-zA-Z0-9]/" \
|
218
|
+
--not-null="name" \
|
219
|
+
--match="name/[a-zA-Z0-9]/" \
|
220
|
+
--not-null=facility_code \
|
221
|
+
--match="facility_code/[a-zA-Z0-9]/" \
|
222
|
+
--not-null=facility_desc \
|
223
|
+
--match="facility_desc/[a-zA-Z0-9]/" \
|
224
|
+
--match="ship_to_ind/^(Y|N)$/" \
|
225
|
+
--match="bill_to_ind/^(Y|N)$/" \
|
226
|
+
--match="stockless_ind/^(Y|N)$/" \
|
227
|
+
--not-null=loc_type \
|
228
|
+
--match="loc_type/^(C|S|LOC_TYPE_SUPPLY|LOC_TYPE_CONSUME)$/" \
|
229
|
+
--not-null=inventory_path_name \
|
230
|
+
--not-null=inventory_location_name \
|
231
|
+
--query="(route_no is null or route_no = '' or regexp_replace(route_no, '[,\.]', '', 'g') ~ '^[1-9][0-9]+$') -- invalid route_no" \
|
232
|
+
--rquery="(loc_type ~* '^(LOC_TYPE_SUPPLY|S)$' and (corp_acct_no is null or corp_name is null or corp_id is null)) -- either corp id/name or corp_acct_no is null" \
|
233
|
+
--not-null=active \
|
234
|
+
--match="active/^(Y|N)$/" \
|
235
|
+
--not-null=corp_acct_no \
|
236
|
+
--match="corp_acct_no/[a-zA-Z0-9]/" \
|
237
|
+
--not-null=inventory_loc_seq_no \
|
238
|
+
--match="inventory_loc_seq_no/^[1-9][0-9]*$/" \
|
239
|
+
--match="route_no/^[1-9][0-9]*$/" \
|
240
|
+
--match="route_name/[a-zA-Z0-9]/" \
|
241
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
242
|
+
--consistent-by="corp_name|corp_id" \
|
243
|
+
--consistent-by="corp_id|corp_name" \
|
244
|
+
--consistent-by="name|facility_code, loc_id" \
|
245
|
+
--consistent-by="loc_id|facility_code, name" \
|
246
|
+
--cross-reference="inventory_path_name|$LOCATION.name" \
|
247
|
+
--cross-reference="inventory_location_name|$LOCATION.name" \
|
248
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
249
|
+
--cross-reference="corp_name|$GL.corp_acct_name"
|
250
|
+
|
251
|
+
|
252
|
+
# validate CONTRACTS ORIGINAL
|
253
|
+
ivalidate --case-insensitive --pretty -t $CONTRACTO \
|
254
|
+
--log-to=validation_errors \
|
255
|
+
--not-null=contract_number \
|
256
|
+
--not-null=contract_start \
|
257
|
+
--not-null=contract_end \
|
258
|
+
--not-null=vendor_name \
|
259
|
+
--not-null=mfr_item_id \
|
260
|
+
--not-null=mfr_name \
|
261
|
+
--not-null=item_uom \
|
262
|
+
--not-null=item_qoe \
|
263
|
+
--not-null=contract_price \
|
264
|
+
--not-null=contract_gpo_name \
|
265
|
+
--match="contract_number/[a-zA-Z0-9]/" \
|
266
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
267
|
+
--match="vendor_item_id/[a-zA-Z0-9]/" \
|
268
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
269
|
+
--match="mfr_item_id/[a-zA-Z0-9]/" \
|
270
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
271
|
+
--query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- Contract end-date goes before start-date" \
|
272
|
+
--match="contract_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
|
273
|
+
--match="item_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
|
274
|
+
--consistent-by="corp_id|corp_name" \
|
275
|
+
--consistent-by="corp_name|corp_id" \
|
276
|
+
--consistent-by="mfr_name|mfr_number" \
|
277
|
+
--consistent-by="vendor_code|vendor_name" \
|
278
|
+
--consistent-by="vendor_name|vendor_code" \
|
279
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
280
|
+
--cross-reference="mfr_name|$MFR.mfr_name" \
|
281
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
282
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
283
|
+
--match="contract_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
284
|
+
--match="item_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
285
|
+
--rquery="(item_uom NOT IN (SELECT code FROM uomstd) AND item_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid item_uom" \
|
286
|
+
--unique="contract_gpo_name, contract_number, contract_start, contract_end, vendor_name, mfr_item_id, mfr_name, item_uom, corp_id" \
|
287
|
+
|
288
|
+
# validate ITEM
|
289
|
+
# Accepted:
|
290
|
+
# --rquery="mfr_name IN (SELECT mfr_name FROM $ITEM WHERE mfr_name IS NOT NULL GROUP BY mfr_name HAVING count(DISTINCT mfr_number) > 1) -- same mfr_name but with different mfr_number" \
|
291
|
+
ivalidate --case-insensitive --pretty -t $ITEM \
|
292
|
+
--log-to=validation_errors \
|
293
|
+
--not-null="item_id" \
|
294
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
295
|
+
--not-null="item_descr" \
|
296
|
+
--match="item_descr/[a-zA-Z0-9]/" \
|
297
|
+
--not-null="item_uom" \
|
298
|
+
--not-null="default_uom" \
|
299
|
+
--not-null="item_price" \
|
300
|
+
--not-null="item_qoe" \
|
301
|
+
--not-null="corp_id" \
|
302
|
+
--not-null="corp_name" \
|
303
|
+
--not-null="vendor_code" \
|
304
|
+
--not-null="vendor_name" \
|
305
|
+
--not-null="mfr_number" \
|
306
|
+
--not-null="mfr_name" \
|
307
|
+
--not-null="active" \
|
308
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
309
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
310
|
+
--match="vendor_code/[a-zA-Z0-9]/" \
|
311
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
312
|
+
--match="mfr_number/[a-zA-Z0-9]/" \
|
313
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
314
|
+
--match="active/^(1|2|3|A|I)$/" \
|
315
|
+
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
316
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
317
|
+
--cross-reference="mfr_number|$MFR.mfr_number" \
|
318
|
+
--cross-reference="mfr_name|$MFR.mfr_name" \
|
319
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
320
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
321
|
+
--consistent-by="corp_id|corp_name" \
|
322
|
+
--consistent-by="corp_name|corp_id" \
|
323
|
+
--consistent-by="mfr_name|mfr_number" \
|
324
|
+
--consistent-by="vendor_code|vendor_name" \
|
325
|
+
--consistent-by="vendor_name|vendor_code" \
|
326
|
+
--rquery="(item_uom NOT IN (SELECT code FROM uomstd) AND item_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid item_uom" \
|
327
|
+
--rquery="(default_uom NOT IN (SELECT code FROM uomstd) AND default_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid default_uom" \
|
328
|
+
--match="item_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
329
|
+
--match="item_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
|
330
|
+
|
331
|
+
# validate PO
|
332
|
+
ivalidate --case-insensitive --pretty -t $PO \
|
333
|
+
--log-to=validation_errors \
|
334
|
+
--not-null=po_no \
|
335
|
+
--match="po_no/[a-zA-Z0-9]/" \
|
336
|
+
--not-null=po_date \
|
337
|
+
--not-null=corp_id \
|
338
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
339
|
+
--not-null=corp_name \
|
340
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
341
|
+
--not-null=Paragon_Cost_Center_Acct \
|
342
|
+
--match="Paragon_Cost_Center_Acct/[a-zA-Z0-9]/" \
|
343
|
+
--not-null=cost_center_name \
|
344
|
+
--match="cost_center_name/[a-zA-Z0-9]/" \
|
345
|
+
--not-null=PO_Line_No \
|
346
|
+
--match="PO_Line_No/^[1-9][0-9]*$/" \
|
347
|
+
--not-null=item_id \
|
348
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
349
|
+
--not-null=vendor_name \
|
350
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
351
|
+
--not-null=Paragon_Vendor_Code \
|
352
|
+
--match="Paragon_Vendor_Code/[a-zA-Z0-9]/" \
|
353
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
354
|
+
--not-null=mfr_number \
|
355
|
+
--match="mfr_number/[a-zA-Z0-9]/" \
|
356
|
+
--not-null=item_descr \
|
357
|
+
--consistent-by="corp_id|corp_name" \
|
358
|
+
--consistent-by="corp_name|corp_id" \
|
359
|
+
--consistent-by="Paragon_Vendor_Code|vendor_name" \
|
360
|
+
--consistent-by="vendor_name|Paragon_Vendor_Code" \
|
361
|
+
--consistent-by="mfr_name|mfr_number" \
|
362
|
+
--unique="po_no, PO_Line_No" \
|
363
|
+
--rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
|
364
|
+
--cross-reference="Paragon_Vendor_Code|$VENDOR.vendor_code" \
|
365
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
366
|
+
--cross-reference="mfr_number|$MFR.mfr_number" \
|
367
|
+
--cross-reference="mfr_name|$MFR.mfr_name" \
|
368
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
369
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
370
|
+
--cross-reference="Paragon_Cost_Center_Acct|$GL.cc_acct_no" \
|
371
|
+
--cross-reference="cost_center_name|$GL.cc_acct_name" \
|
372
|
+
--rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
|
373
|
+
--rquery="(item_id IS NOT NULL AND (Paragon_Vendor_Code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
|
374
|
+
--match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
375
|
+
--match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
|
376
|
+
|
377
|
+
ivalidate --case-insensitive --pretty -t $PO2 \
|
378
|
+
--log-to=validation_errors \
|
379
|
+
--not-null=po_no \
|
380
|
+
--match="po_no/[a-zA-Z0-9]/" \
|
381
|
+
--not-null=po_date \
|
382
|
+
--not-null=corp_id \
|
383
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
384
|
+
--not-null=corp_name \
|
385
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
386
|
+
--not-null=cost_center_id \
|
387
|
+
--match="cost_center_id/[a-zA-Z0-9]/" \
|
388
|
+
--not-null=cost_center_name \
|
389
|
+
--match="cost_center_name/[a-zA-Z0-9]/" \
|
390
|
+
--not-null=PO_Line_Number \
|
391
|
+
--match="PO_Line_Number/^[1-9][0-9]*$/" \
|
392
|
+
--not-null=item_id \
|
393
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
394
|
+
--not-null=vendor_name \
|
395
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
396
|
+
--not-null=vendor_code \
|
397
|
+
--match="vendor_code/[a-zA-Z0-9]/" \
|
398
|
+
--not-null=mfr_name \
|
399
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
400
|
+
--not-null=mfr_number \
|
401
|
+
--match="mfr_number/[a-zA-Z0-9]/" \
|
402
|
+
--not-null=item_descr \
|
403
|
+
--consistent-by="corp_id|corp_name" \
|
404
|
+
--consistent-by="corp_name|corp_id" \
|
405
|
+
--consistent-by="vendor_code|vendor_name" \
|
406
|
+
--consistent-by="vendor_name|vendor_code" \
|
407
|
+
--consistent-by="mfr_name|mfr_number" \
|
408
|
+
--unique="po_no, PO_Line_Number" \
|
409
|
+
--rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
|
410
|
+
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
411
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
412
|
+
--cross-reference="mfr_number|$MFR.mfr_number" \
|
413
|
+
--cross-reference="mfr_name|$MFR.mfr_name" \
|
414
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
415
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
416
|
+
--cross-reference="cost_center_id|$GL.cc_acct_no" \
|
417
|
+
--cross-reference="cost_center_name|$GL.cc_acct_name" \
|
418
|
+
--rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
|
419
|
+
--rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
|
420
|
+
--match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
421
|
+
--match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
|
422
|
+
|
423
|
+
# validate Req
|
424
|
+
ivalidate --case-insensitive --pretty -t $REQ \
|
425
|
+
--log-to=validation_errors \
|
426
|
+
--not-null="req_no" \
|
427
|
+
--match="req_no/[a-zA-Z0-9]/" \
|
428
|
+
--not-null="req_date" \
|
429
|
+
--match="req_date/[a-zA-Z0-9]/" \
|
430
|
+
--not-null="corp_id" \
|
431
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
432
|
+
--not-null="corp_name" \
|
433
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
434
|
+
--not-null="req_line_number" \
|
435
|
+
--match="req_line_number/^[1-9][0-9]*$/" \
|
436
|
+
--not-null="item_id" \
|
437
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
438
|
+
--not-null="vendor_name" \
|
439
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
440
|
+
--not-null="vendor_code" \
|
441
|
+
--match="vendor_code/[a-zA-Z0-9]/" \
|
442
|
+
--not-null="mfr_name" \
|
443
|
+
--match="mfr_name/[a-zA-Z0-9]/" \
|
444
|
+
--not-null="item_descr" \
|
445
|
+
--match="item_descr/[a-zA-Z0-9]/" \
|
446
|
+
--consistent-by="corp_id|corp_name" \
|
447
|
+
--consistent-by="corp_name|corp_id" \
|
448
|
+
--rquery="(item_id not like '%~[%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
|
449
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
450
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
451
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
452
|
+
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
453
|
+
--cross-reference="mfr_name|$MFR.mfr_name" \
|
454
|
+
--cross-reference="costcenter_id|$GL.cc_acct_no" \
|
455
|
+
--cross-reference="costcenter_name|$GL.cc_acct_name" \
|
456
|
+
--unique="req_no, req_line_number" \
|
457
|
+
|
458
|
+
# validate USERS
|
459
|
+
ivalidate --case-insensitive --pretty -t $USER \
|
460
|
+
--log-to=validation_errors \
|
461
|
+
--not-null="email" \
|
462
|
+
--unique="email" \
|
463
|
+
--query="lower(email) ~* '[a-z0-9][a-z0-9_\.]+@[a-z0-9][a-z0-9_\.\-]+\.[a-z0-9_\.\-]+' -- invalid email address"
|
464
|
+
|
465
|
+
# validate
|
466
|
+
#ivalidate --case-insensitive --pretty -t $ITEMCOST \
|
467
|
+
# --log-to=validation_errors \
|
468
|
+
# --not-null="corp_acct_no" \
|
469
|
+
# --match="corp_acct_no/[a-zA-Z0-9]/" \
|
470
|
+
# --not-null="item_no" \
|
471
|
+
# --match="item_no/[a-zA-Z0-9]/" \
|
472
|
+
# --not-null="excp_type" \
|
473
|
+
# --match="excp_type/^(A|R)$/" \
|
474
|
+
# --cross-reference="corp_acct_no|$GL.corp_acct_no" \
|
475
|
+
# --cross-reference="item_no|$ITEM.item_id" \
|
476
|
+
# --cross-reference="exp_acct_no|$GL.exp_acct_no"
|
477
|
+
|
478
|
+
# validate INVENTORY
|
479
|
+
ivalidate --case-insensitive --pretty -t $INVENTORY \
|
480
|
+
--log-to=validation_errors \
|
481
|
+
--not-null="item_id" \
|
482
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
483
|
+
--not-null="loc_id" \
|
484
|
+
--match="loc_id/[a-zA-Z0-9]/" \
|
485
|
+
--not-null="vendor_code" \
|
486
|
+
--match="vendor_code/[a-zA-Z0-9]/" \
|
487
|
+
--not-null="vendor_name" \
|
488
|
+
--match="vendor_name/[a-zA-Z0-9]/" \
|
489
|
+
--not-null="corp_id" \
|
490
|
+
--match="corp_name/[a-zA-Z0-9]/" \
|
491
|
+
--not-null="location_name" \
|
492
|
+
--match="location_name/[a-zA-Z0-9]/" \
|
493
|
+
--not-null="item_id" \
|
494
|
+
--match="item_id/[a-zA-Z0-9]/" \
|
495
|
+
--match="inventory_status/^(Active|Pending Inactive|Inactive)$/" \
|
496
|
+
--cross-reference="item_id|$ITEM.item_id" \
|
497
|
+
--cross-reference="location_name|$LOCATION.name" \
|
498
|
+
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
499
|
+
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
500
|
+
--cross-reference="corp_id|$GL.corp_acct_no" \
|
501
|
+
--cross-reference="corp_name|$GL.corp_acct_name"
|
502
|
+
|
503
|
+
ivalidate --case-insensitive --pretty -t $ULPR \
|
504
|
+
--log-to=validation_errors \
|
505
|
+
--match="Default_Indicator/^(Y|N|y|n)$/" \
|
506
|
+
--not-null="email" \
|
507
|
+
--cross-reference="email|$USER.email" \
|
508
|
+
--cross-reference="loc_id|$LOCATION.loc_id" \
|
509
|
+
--cross-reference="loc_name|$LOCATION.name" \
|
510
|
+
--cross-reference="corp_no|$GL.corp_acct_no" \
|
511
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
512
|
+
|
513
|
+
ivalidate --case-insensitive --pretty -t $LPR \
|
514
|
+
--log-to=validation_errors \
|
515
|
+
--not-null="Default_Inventory_Location_Name" \
|
516
|
+
--cross-reference="Default_Inventory_Location_Name|$LOCATION.Inventory_Location_Name" \
|
517
|
+
--match="loc_type/^(C|LOC_TYPE_CONSUME)$/" \
|
518
|
+
--match="active/^(1|2|3|A|I)$/" \
|
519
|
+
--cross-reference="item_id|$ITEM.item_id" \
|
520
|
+
--cross-reference="corp_acct_no|$GL.corp_acct_no" \
|
521
|
+
--cross-reference="corp_name|$GL.corp_acct_name" \
|
522
|
+
--cross-reference="cc_acct_no|$GL.cc_acct_no" \
|
523
|
+
--cross-reference="cc_acct_name|$GL.cc_acct_name" \
|
524
|
+
--cross-reference="loc_id|$LOCATION.loc_id" \
|
525
|
+
--cross-reference="Location_Name|$LOCATION.name"
|
526
|
+
|
527
|
+
####################################################
|
528
|
+
# Create report file for every table (extract 1000 records for every error)
|
529
|
+
# These file will then be used for the Validation Report
|
530
|
+
####################################################
|
531
|
+
|
532
|
+
#iexport -t $ITEMCOST \
|
533
|
+
# -o "$OUTPUT_DIR/$ITEMCOST.csv" -f csv --no-quote-empty --quotes --headers \
|
534
|
+
# --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
535
|
+
# FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
536
|
+
# $ITEMCOST order by id ) as main) as tmp
|
537
|
+
# where group_index <= 1000" \
|
538
|
+
# --exclude="id, validation_errors, group_index"
|
539
|
+
|
540
|
+
iexport -t $CONTRACTO \
|
541
|
+
-o "$OUTPUT_DIR/$CONTRACTO.csv" -f csv --no-quote-empty --quotes --headers \
|
542
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
543
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
544
|
+
$CONTRACTO order by id ) as main) as tmp
|
545
|
+
where group_index <= 1000" \
|
546
|
+
--exclude="id, validation_errors, group_index"
|
547
|
+
|
548
|
+
iexport -t $VENDOR \
|
549
|
+
-o "$OUTPUT_DIR/$VENDOR.csv" -f csv --no-quote-empty --quotes --headers \
|
550
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
551
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
552
|
+
$VENDOR order by id ) as main) as tmp
|
553
|
+
where group_index <= 1000" \
|
554
|
+
--exclude="id, validation_errors, group_index"
|
555
|
+
|
556
|
+
iexport -t $MFR \
|
557
|
+
-o "$OUTPUT_DIR/$MFR.csv" -f csv --no-quote-empty --quotes --headers \
|
558
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
559
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
560
|
+
$MFR order by id ) as main) as tmp
|
561
|
+
where group_index <= 1000" \
|
562
|
+
--exclude="id, validation_errors, group_index"
|
563
|
+
|
564
|
+
iexport -t $GL \
|
565
|
+
-o "$OUTPUT_DIR/$GL.csv" -f csv --no-quote-empty --quotes --headers \
|
566
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
567
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
568
|
+
$GL order by id ) as main) as tmp
|
569
|
+
where group_index <= 1000" \
|
570
|
+
--exclude="id, validation_errors, group_index"
|
571
|
+
|
572
|
+
iexport -t $PO \
|
573
|
+
-o "$OUTPUT_DIR/$PO.csv" -f csv --no-quote-empty --quotes --headers \
|
574
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
575
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
576
|
+
$PO order by id ) as main) as tmp
|
577
|
+
where group_index <= 1000" \
|
578
|
+
--exclude="id, validation_errors, group_index"
|
579
|
+
|
580
|
+
iexport -t $PO2 \
|
581
|
+
-o "$OUTPUT_DIR/$PO2.csv" -f csv --no-quote-empty --quotes --headers \
|
582
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
583
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
584
|
+
$PO2 order by id ) as main) as tmp
|
585
|
+
where group_index <= 1000" \
|
586
|
+
--exclude="id, validation_errors, group_index"
|
587
|
+
|
588
|
+
iexport -t $INVENTORY \
|
589
|
+
-o "$OUTPUT_DIR/$INVENTORY.csv" -f csv --no-quote-empty --quotes --headers \
|
590
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
591
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
592
|
+
$INVENTORY order by id ) as main) as tmp
|
593
|
+
where group_index <= 1000" \
|
594
|
+
--exclude="id, validation_errors, group_index"
|
595
|
+
|
596
|
+
iexport -t $REQ \
|
597
|
+
-o "$OUTPUT_DIR/$REQ.csv" -f csv --no-quote-empty --quotes --headers \
|
598
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
599
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
600
|
+
$REQ order by id ) as main) as tmp
|
601
|
+
where group_index <= 1000" \
|
602
|
+
--exclude="id, validation_errors, group_index"
|
603
|
+
|
604
|
+
iexport -t $ITEM \
|
605
|
+
-o "$OUTPUT_DIR/$ITEM.csv" -f csv --no-quote-empty --quotes --headers \
|
606
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
607
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
608
|
+
$ITEM order by id ) as main) as tmp
|
609
|
+
where group_index <= 1000" \
|
610
|
+
--exclude="id, validation_errors, group_index"
|
611
|
+
|
612
|
+
iexport -t $USER \
|
613
|
+
-o "$OUTPUT_DIR/$USER.csv" -f csv --no-quote-empty --quotes --headers \
|
614
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
615
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
616
|
+
$USER order by id ) as main) as tmp
|
617
|
+
where group_index <= 1000" \
|
618
|
+
--exclude="id, validation_errors, group_index"
|
619
|
+
|
620
|
+
iexport -t $LOCATION \
|
621
|
+
-o "$OUTPUT_DIR/$LOCATION.csv" -f csv --no-quote-empty --quotes --headers \
|
622
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
623
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
624
|
+
$LOCATION order by id ) as main) as tmp
|
625
|
+
where group_index <= 1000" \
|
626
|
+
--exclude="id, validation_errors, group_index"
|
627
|
+
|
628
|
+
iexport -t $ULPR \
|
629
|
+
-o "$OUTPUT_DIR/$ULPR.csv" -f csv --no-quote-empty --quotes --headers \
|
630
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
631
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
632
|
+
$ULPR order by id ) as main) as tmp
|
633
|
+
where group_index <= 1000" \
|
634
|
+
--exclude="id, validation_errors, group_index"
|
635
|
+
|
636
|
+
iexport -t $LPR \
|
637
|
+
-o "$OUTPUT_DIR/$LPR.csv" -f csv --no-quote-empty --quotes --headers \
|
638
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
639
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
640
|
+
$LPR order by id ) as main) as tmp
|
641
|
+
where group_index <= 1000" \
|
642
|
+
--exclude="id, validation_errors, group_index"
|
643
|
+
|
644
|
+
|
645
|
+
# Use SQL to compute the summary, write the outputs to summary.csv
|
646
|
+
# --------------------------------------------------------------
|
647
|
+
iexport --output="$OUTPUT_DIR/summary.csv" -f csv --no-quote-empty --quotes --headers \
|
648
|
+
--query="(select 'ContractMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $CONTRACTO), 2)::varchar || '%' as percentage from $CONTRACTO group by error order by error) union
|
649
|
+
(select 'VendorMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $VENDOR), 2)::varchar || '%' as percentage from $VENDOR group by error order by error) union
|
650
|
+
(select 'ItemMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEM), 2)::varchar || '%' as percentage from $ITEM group by error order by error) union
|
651
|
+
(select 'ReqHistoryLoad' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $REQ), 2)::varchar || '%' as percentage from $REQ group by error order by error) union
|
652
|
+
(select 'MfrMaster' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $MFR), 2)::varchar || '%' as percentage from $MFR group by error order by error) union
|
653
|
+
(select 'PurchaseOrder' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $PO), 2)::varchar || '%' as percentage from $PO group by error order by error) union
|
654
|
+
(select 'PurchaseOrder2' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $PO2), 2)::varchar || '%' as percentage from $PO2 group by error order by error) union
|
655
|
+
(select 'Inventory' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $INVENTORY), 2)::varchar || '%' as percentage from $INVENTORY group by error order by error) union
|
656
|
+
(select 'User' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $USER), 2)::varchar || '%' as percentage from $USER group by error order by error) union
|
657
|
+
(select 'Location' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LOCATION), 2)::varchar || '%' as percentage from $LOCATION group by error order by error) union
|
658
|
+
--(select 'ItemCost' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMCOST), 2)::varchar || '%' as percentage from $ITEMCOST group by error order by error) union
|
659
|
+
(select 'GLAccount' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $GL), 2)::varchar || '%' as percentage from $GL group by error order by error) union
|
660
|
+
(select 'UserLocationProfile' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ULPR), 2)::varchar || '%' as percentage from $ULPR group by error order by error) union
|
661
|
+
(select 'LocationProfile' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LPR), 2)::varchar || '%' as percentage from $LPR group by error order by error)"
|
662
|
+
|
663
|
+
# Merge summary.xls and report files into one single file with several tabs
|
664
|
+
# --input="ItemCostCenterAcctExceptions:$OUTPUT_DIR/$ITEMCOST.csv" \
|
665
|
+
imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
|
666
|
+
--input="Summary:$OUTPUT_DIR/summary.csv" \
|
667
|
+
--input="ContractMaster:$OUTPUT_DIR/$CONTRACTO.csv" \
|
668
|
+
--input="ItemMaster:$OUTPUT_DIR/$ITEM.csv" \
|
669
|
+
--input="MfrMaster:$OUTPUT_DIR/$MFR.csv" \
|
670
|
+
--input="PurchaseOrder:$OUTPUT_DIR/$PO.csv" \
|
671
|
+
--input="PurchaseOrder2:$OUTPUT_DIR/$PO2.csv" \
|
672
|
+
--input="User:$OUTPUT_DIR/$USER.csv" \
|
673
|
+
--input="Location:$OUTPUT_DIR/$LOCATION.csv" \
|
674
|
+
--input="ReqHistoryLoad:$OUTPUT_DIR/$REQ.csv" \
|
675
|
+
--input="GLAccount:$OUTPUT_DIR/$GL.csv" \
|
676
|
+
--input="Inventory:$OUTPUT_DIR/$INVENTORY.csv" \
|
677
|
+
--input="$ULPR:$OUTPUT_DIR/$ULPR.csv" \
|
678
|
+
--input="$LPR:$OUTPUT_DIR/$LPR.csv"
|
679
|
+
|
680
|
+
exit
|
data/full.sh
CHANGED
@@ -28,6 +28,7 @@ FITEMCOST="ItemCostCenterAcctExceptions.csv"
|
|
28
28
|
FREQ="ReqHistoryLoad.csv"
|
29
29
|
FGL="GLAccount.csv"
|
30
30
|
FINVENTORY="Inventory.csv"
|
31
|
+
FITEMDEFAULTEXP="ItemAndDefaultExpenseAccount.csv"
|
31
32
|
|
32
33
|
CONTRACTO="contracts"
|
33
34
|
VENDOR="vendors"
|
@@ -41,6 +42,7 @@ ITEMCOST="item_costs"
|
|
41
42
|
REQ="reqs"
|
42
43
|
GL="gls"
|
43
44
|
INVENTORY="inventory"
|
45
|
+
ITEMDEFAULTEXP="ItemAndDefaultExpenseAccount"
|
44
46
|
|
45
47
|
# Standard UOM for reference
|
46
48
|
FUOMSTD="stduom.txt"
|
@@ -67,6 +69,7 @@ iload -i "$FITEMCOST" -t "$ITEMCOST" -f csv
|
|
67
69
|
iload -i "$FREQ" -t "$REQ" -f csv
|
68
70
|
iload -i "$FGL" -t "$GL" -f csv
|
69
71
|
iload -i "$FINVENTORY" -t "$INVENTORY" -f csv
|
72
|
+
iload -i "$FITEMDEFAULTEXP" -t "$ITEMDEFAULTEXP" -f csv
|
70
73
|
|
71
74
|
# Load UOM STANDARD for reference
|
72
75
|
iload -i "$FUOMSTD" -t "$UOMSTD" -f csv
|
@@ -127,7 +130,9 @@ UPDATE $CONTRACTO SET contract_start = to_char(to_date(contract_start, 'MM/DD/YY
|
|
127
130
|
WHERE contract_start like '%/%/%';
|
128
131
|
UPDATE $CONTRACTO SET contract_end = to_char(to_date(contract_end, 'MM/DD/YYYY'), 'YYYY-MM-DD')
|
129
132
|
WHERE contract_end like '%/%/%';
|
133
|
+
"
|
130
134
|
|
135
|
+
ipatch -q "
|
131
136
|
-- ADD MORE EMPTY FIELD (for validation)
|
132
137
|
-- ALTER TABLE $LOCATION ADD COLUMN corp_id varchar;
|
133
138
|
|
@@ -139,7 +144,7 @@ UPDATE $LOCATION SET stockless_ind = 'N' WHERE stockless_ind IS NULL OR LENGTH(t
|
|
139
144
|
|
140
145
|
# Convert A2A2A2 to A2-A2-A2
|
141
146
|
# Manual check and comment out the following to speed up validating
|
142
|
-
# @todo
|
147
|
+
# @todo ieval (evaluate Ruby code) performance is very poor, avoid using it (use IPATCH instead) unless there is no better way
|
143
148
|
ieval -t $GL --eval="
|
144
149
|
if item.corp_acct_fmt && item.corp_acct_fmt[/^[A-Z][0-9][A-Z][0-9][A-Z][0-9]$/]
|
145
150
|
item.corp_acct_fmt = item.corp_acct_fmt.unpack('a2a2a2').join('-')
|
@@ -154,8 +159,7 @@ ivalidate --case-insensitive --pretty -t $VENDOR \
|
|
154
159
|
--log-to=validation_errors \
|
155
160
|
--not-null="vendor_code" \
|
156
161
|
--not-null="vendor_name" \
|
157
|
-
--unique="vendor_code" \
|
158
|
-
--unique="vendor_name" \
|
162
|
+
--unique="vendor_code, vendor_name" \
|
159
163
|
--match="vendor_code/[a-zA-Z0-9]/" \
|
160
164
|
--match="vendor_name/[a-zA-Z0-9]/" \
|
161
165
|
--consistent-by="vendor_code|vendor_name" \
|
@@ -168,10 +172,11 @@ ivalidate --case-insensitive --pretty -t $MFR \
|
|
168
172
|
--log-to=validation_errors \
|
169
173
|
--not-null="mfr_number" \
|
170
174
|
--not-null="mfr_name" \
|
175
|
+
--unique="mfr_number, mfr_name" \
|
171
176
|
--match="mfr_number/[a-zA-Z0-9]/" \
|
172
177
|
--match="mfr_name/[a-zA-Z0-9]/" \
|
173
|
-
--consistent-by="mfr_number|mfr_name" \
|
174
178
|
--consistent-by="mfr_name|mfr_number" \
|
179
|
+
--consistent-by="mfr_number|mfr_name" \
|
175
180
|
--consistent-by="country_code|country_name" \
|
176
181
|
--consistent-by="country_name|country_code"
|
177
182
|
|
@@ -198,15 +203,15 @@ ivalidate --case-insensitive --pretty -t $GL \
|
|
198
203
|
--consistent-by="corp_acct_name|corp_acct_no" \
|
199
204
|
--consistent-by="exp_acct_no|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_name" \
|
200
205
|
--consistent-by="exp_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_no" \
|
201
|
-
--consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name" \
|
202
|
-
--consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no" \
|
206
|
+
--consistent-by="cc_acct_no|corp_acct_no, corp_acct_name, cc_acct_name, exp_acct_no, exp_acct_name" \
|
207
|
+
--consistent-by="cc_acct_name|corp_acct_no, corp_acct_name, cc_acct_no, exp_acct_no, exp_acct_name" \
|
203
208
|
--not-null=exp_acct_type \
|
204
209
|
--match="exp_acct_type/^(1|2|3|4|5|Asset|Liability|Equity|Income\sStatement|Expense|Income)$/"
|
205
210
|
|
206
211
|
# validate LOCATION
|
207
212
|
ivalidate --case-insensitive --pretty -t $LOCATION \
|
208
213
|
--log-to=validation_errors \
|
209
|
-
--not-null=loc_id \
|
214
|
+
--not-null="loc_id" \
|
210
215
|
--match="loc_id/[a-zA-Z0-9]/" \
|
211
216
|
--not-null="name" \
|
212
217
|
--match="name/[a-zA-Z0-9]/" \
|
@@ -214,22 +219,19 @@ ivalidate --case-insensitive --pretty -t $LOCATION \
|
|
214
219
|
--match="facility_code/[a-zA-Z0-9]/" \
|
215
220
|
--not-null=facility_desc \
|
216
221
|
--match="facility_desc/[a-zA-Z0-9]/" \
|
217
|
-
--match="ship_to_ind/^(Y|N)$/" \
|
218
|
-
--match="bill_to_ind/^(Y|N)$/" \
|
219
|
-
--match="stockless_ind/^(Y|N)$/" \
|
220
|
-
--not-null=loc_type \
|
222
|
+
--match="ship_to_ind/^(Y|N|y|n)$/" \
|
223
|
+
--match="bill_to_ind/^(Y|N|y|n)$/" \
|
224
|
+
--match="stockless_ind/^(Y|N|y|n)$/" \
|
225
|
+
--not-null="loc_type" \
|
221
226
|
--match="loc_type/^(C|S|LOC_TYPE_SUPPLY|LOC_TYPE_CONSUME)$/" \
|
222
|
-
--not-null=inventory_path_name \
|
223
|
-
--not-null=inventory_location_name \
|
224
|
-
--query="(route_no is null or route_no = '' or regexp_replace(route_no, '[,\.]', '', 'g') ~ '^[1-9][0-9]+$') -- invalid route_no" \
|
225
227
|
--rquery="(loc_type ~* '^(LOC_TYPE_SUPPLY|S)$' and (corp_acct_no is null or corp_name is null or corp_id is null)) -- either corp id/name or corp_acct_no is null" \
|
226
|
-
--not-null=active \
|
227
|
-
--match="active/^(Y|N)$/" \
|
228
|
-
--not-null=corp_acct_no \
|
228
|
+
--not-null="active" \
|
229
|
+
--match="active/^(Y|N|1|2|3)$/" \
|
230
|
+
--not-null="corp_acct_no" \
|
229
231
|
--match="corp_acct_no/[a-zA-Z0-9]/" \
|
230
|
-
--
|
231
|
-
--
|
232
|
-
--match="route_no
|
232
|
+
--rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_loc_seq_no IS NULL OR inventory_loc_seq_no = '')) -- [inventory_loc_seq_no] is null" \
|
233
|
+
--rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_location_name IS NULL OR inventory_location_name = '')) -- [inventory_location_name] is null" \
|
234
|
+
--match="route_no/[a-zA-Z0-9]/" \
|
233
235
|
--match="route_name/[a-zA-Z0-9]/" \
|
234
236
|
--match="corp_name/[a-zA-Z0-9]/" \
|
235
237
|
--consistent-by="corp_name|corp_id" \
|
@@ -252,21 +254,25 @@ ivalidate --case-insensitive --pretty -t $CONTRACTO \
|
|
252
254
|
--not-null=mfr_item_id \
|
253
255
|
--not-null=mfr_name \
|
254
256
|
--not-null=item_uom \
|
257
|
+
--not-null=corp_id \
|
258
|
+
--not-null=item_descr \
|
255
259
|
--not-null=item_qoe \
|
256
260
|
--not-null=contract_price \
|
257
261
|
--not-null=contract_gpo_name \
|
262
|
+
--not-null=contract_gpo_id \
|
258
263
|
--match="contract_number/[a-zA-Z0-9]/" \
|
264
|
+
--match="contract_gpo_name/[a-zA-Z0-9]/" \
|
265
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
259
266
|
--match="corp_name/[a-zA-Z0-9]/" \
|
260
267
|
--match="vendor_item_id/[a-zA-Z0-9]/" \
|
261
268
|
--match="vendor_name/[a-zA-Z0-9]/" \
|
262
269
|
--match="mfr_item_id/[a-zA-Z0-9]/" \
|
263
270
|
--match="mfr_name/[a-zA-Z0-9]/" \
|
264
|
-
--query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') --
|
271
|
+
--query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- [contract_end] comes before [contract_start]" \
|
265
272
|
--match="contract_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
|
266
273
|
--match="item_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
|
267
274
|
--consistent-by="corp_id|corp_name" \
|
268
275
|
--consistent-by="corp_name|corp_id" \
|
269
|
-
--consistent-by="mfr_number|mfr_name" \
|
270
276
|
--consistent-by="mfr_name|mfr_number" \
|
271
277
|
--consistent-by="vendor_code|vendor_name" \
|
272
278
|
--consistent-by="vendor_name|vendor_code" \
|
@@ -306,6 +312,7 @@ ivalidate --case-insensitive --pretty -t $ITEM \
|
|
306
312
|
--match="mfr_number/[a-zA-Z0-9]/" \
|
307
313
|
--match="mfr_name/[a-zA-Z0-9]/" \
|
308
314
|
--match="active/^(1|2|3|A|I)$/" \
|
315
|
+
--unique="item_id, corp_id, vendor_code, item_uom" \
|
309
316
|
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
310
317
|
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
311
318
|
--cross-reference="mfr_number|$MFR.mfr_number" \
|
@@ -314,7 +321,6 @@ ivalidate --case-insensitive --pretty -t $ITEM \
|
|
314
321
|
--cross-reference="corp_name|$GL.corp_acct_name" \
|
315
322
|
--consistent-by="corp_id|corp_name" \
|
316
323
|
--consistent-by="corp_name|corp_id" \
|
317
|
-
--consistent-by="mfr_number|mfr_name" \
|
318
324
|
--consistent-by="mfr_name|mfr_number" \
|
319
325
|
--consistent-by="vendor_code|vendor_name" \
|
320
326
|
--consistent-by="vendor_name|vendor_code" \
|
@@ -356,7 +362,7 @@ ivalidate --case-insensitive --pretty -t $PO \
|
|
356
362
|
--consistent-by="vendor_name|vendor_code" \
|
357
363
|
--consistent-by="mfr_name|mfr_number" \
|
358
364
|
--unique="po_no, po_line_number" \
|
359
|
-
--rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
|
365
|
+
--rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- [item_id] does not reference [items.item_id]" \
|
360
366
|
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
361
367
|
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
362
368
|
--cross-reference="mfr_number|$MFR.mfr_number" \
|
@@ -365,8 +371,8 @@ ivalidate --case-insensitive --pretty -t $PO \
|
|
365
371
|
--cross-reference="corp_name|$GL.corp_acct_name" \
|
366
372
|
--cross-reference="cost_center_id|$GL.cc_acct_no" \
|
367
373
|
--cross-reference="cost_center_name|$GL.cc_acct_name" \
|
368
|
-
--rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid purchase_uom" \
|
369
|
-
--rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- vendor_item_id is null" \
|
374
|
+
--rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid [purchase_uom]" \
|
375
|
+
--rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- [vendor_item_id] is null" \
|
370
376
|
--match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
|
371
377
|
--match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
|
372
378
|
|
@@ -391,13 +397,9 @@ ivalidate --case-insensitive --pretty -t $REQ \
|
|
391
397
|
--match="vendor_name/[a-zA-Z0-9]/" \
|
392
398
|
--not-null="vendor_code" \
|
393
399
|
--match="vendor_code/[a-zA-Z0-9]/" \
|
394
|
-
--not-null="mfr_name" \
|
395
|
-
--match="mfr_name/[a-zA-Z0-9]/" \
|
396
|
-
--not-null="item_descr" \
|
397
|
-
--match="item_descr/[a-zA-Z0-9]/" \
|
398
400
|
--consistent-by="corp_id|corp_name" \
|
399
401
|
--consistent-by="corp_name|corp_id" \
|
400
|
-
--rquery="(item_id not like '
|
402
|
+
--rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- item_id does not reference items.item_id" \
|
401
403
|
--cross-reference="corp_id|$GL.corp_acct_no" \
|
402
404
|
--cross-reference="corp_name|$GL.corp_acct_name" \
|
403
405
|
--cross-reference="vendor_name|$VENDOR.vendor_name" \
|
@@ -412,7 +414,7 @@ ivalidate --case-insensitive --pretty -t $USER \
|
|
412
414
|
--log-to=validation_errors \
|
413
415
|
--not-null="email" \
|
414
416
|
--unique="email" \
|
415
|
-
--
|
417
|
+
--match="lower(email)/[a-z0-9][a-z0-9_\.]+@[a-z0-9][a-z0-9_\.\-]+\.[a-z0-9_\.\-]+/"
|
416
418
|
|
417
419
|
# validate INVENTORY
|
418
420
|
ivalidate --case-insensitive --pretty -t $INVENTORY \
|
@@ -423,15 +425,11 @@ ivalidate --case-insensitive --pretty -t $INVENTORY \
|
|
423
425
|
--match="loc_id/[a-zA-Z0-9]/" \
|
424
426
|
--not-null="vendor_code" \
|
425
427
|
--match="vendor_code/[a-zA-Z0-9]/" \
|
426
|
-
--not-null="vendor_name" \
|
427
|
-
--match="vendor_name/[a-zA-Z0-9]/" \
|
428
428
|
--not-null="corp_id" \
|
429
|
-
--match="
|
430
|
-
--not-null="location_name" \
|
429
|
+
--match="corp_id/[a-zA-Z0-9]/" \
|
431
430
|
--match="location_name/[a-zA-Z0-9]/" \
|
432
431
|
--not-null="item_id" \
|
433
|
-
--match="
|
434
|
-
--match="inventory_status/^(Active|Pending Inactive|Inactive)$/" \
|
432
|
+
--match="inventory_status/^(Active|Pending Inactive|Inactive|1|2|3)$/" \
|
435
433
|
--cross-reference="item_id|$ITEM.item_id" \
|
436
434
|
--cross-reference="location_name|$LOCATION.name" \
|
437
435
|
--cross-reference="vendor_code|$VENDOR.vendor_code" \
|
@@ -452,6 +450,12 @@ ivalidate --case-insensitive --pretty -t $ITEMCOST \
|
|
452
450
|
--cross-reference="item_no|$ITEM.item_id" \
|
453
451
|
--cross-reference="exp_acct_no|$GL.exp_acct_no"
|
454
452
|
|
453
|
+
ivalidate --case-insensitive --pretty -t $ITEMDEFAULTEXP \
|
454
|
+
--log-to=validation_errors \
|
455
|
+
--cross-reference="corp_acct_no|$GL.corp_acct_no" \
|
456
|
+
--cross-reference="item_no|$ITEM.item_id" \
|
457
|
+
--cross-reference="exp_acct_no|$GL.exp_acct_no"
|
458
|
+
|
455
459
|
|
456
460
|
|
457
461
|
####################################################
|
@@ -466,6 +470,14 @@ iexport -t $ITEMCOST \
|
|
466
470
|
$ITEMCOST order by id ) as main) as tmp
|
467
471
|
where group_index <= 1000" \
|
468
472
|
--exclude="id, validation_errors, group_index"
|
473
|
+
|
474
|
+
iexport -t $ITEMDEFAULTEXP \
|
475
|
+
-o "$OUTPUT_DIR/$ITEMDEFAULTEXP.csv" -f csv --no-quote-empty --quotes --headers \
|
476
|
+
--query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, *
|
477
|
+
FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
|
478
|
+
$ITEMDEFAULTEXP order by id ) as main) as tmp
|
479
|
+
where group_index <= 1000" \
|
480
|
+
--exclude="id, validation_errors, group_index"
|
469
481
|
|
470
482
|
iexport -t $CONTRACTO \
|
471
483
|
-o "$OUTPUT_DIR/$CONTRACTO.csv" -f csv --no-quote-empty --quotes --headers \
|
@@ -561,12 +573,14 @@ iexport --output="$OUTPUT_DIR/summary.csv" -f csv --no-quote-empty --quotes --he
|
|
561
573
|
(select 'User' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $USER), 2)::varchar || '%' as percentage from $USER group by error order by error) union
|
562
574
|
(select 'Location' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $LOCATION), 2)::varchar || '%' as percentage from $LOCATION group by error order by error) union
|
563
575
|
(select 'ItemCost' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMCOST), 2)::varchar || '%' as percentage from $ITEMCOST group by error order by error) union
|
576
|
+
(select 'ItemAndDefaultEXP' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEMDEFAULTEXP), 2)::varchar || '%' as percentage from $ITEMDEFAULTEXP group by error order by error) union
|
564
577
|
(select 'GLAccount' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $GL), 2)::varchar || '%' as percentage from $GL group by error order by error)"
|
565
578
|
|
566
579
|
# Merge summary.xls and report files into one single file with several tabs
|
567
580
|
imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
|
568
581
|
--input="Summary:$OUTPUT_DIR/summary.csv" \
|
569
582
|
--input="ItemCostCenterAcctExceptions:$OUTPUT_DIR/$ITEMCOST.csv" \
|
583
|
+
--input="ItemAndDefaultEXP:$OUTPUT_DIR/$ITEMDEFAULTEXP.csv" \
|
570
584
|
--input="ContractMaster:$OUTPUT_DIR/$CONTRACTO.csv" \
|
571
585
|
--input="ItemMaster:$OUTPUT_DIR/$ITEM.csv" \
|
572
586
|
--input="MfrMaster:$OUTPUT_DIR/$MFR.csv" \
|
data/idata.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ["minhnghivn@gmail.com"]
|
11
11
|
spec.description = %q{Tools for importing data from raw files}
|
12
12
|
spec.summary = %q{Tools include: iload, ivalidate, isanitize, ipatch, ieval, iexport}
|
13
|
-
spec.homepage = "
|
13
|
+
spec.homepage = "https://github.com/minhnghivn/idata"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files`.split($/)
|
data/lib/idata/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: idata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.32
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nghi Pham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -92,12 +92,13 @@ files:
|
|
92
92
|
- bin/ipatch
|
93
93
|
- bin/isanitize
|
94
94
|
- bin/ivalidate
|
95
|
+
- full-pg.sh
|
95
96
|
- full.sh
|
96
97
|
- idata.gemspec
|
97
98
|
- lib/idata.rb
|
98
99
|
- lib/idata/version.rb
|
99
100
|
- sample.sh
|
100
|
-
homepage:
|
101
|
+
homepage: https://github.com/minhnghivn/idata
|
101
102
|
licenses:
|
102
103
|
- MIT
|
103
104
|
metadata: {}
|