idata 0.1.33 → 0.2.1

data/full.sh

@@ -199,6 +199,9 @@ ivalidate --case-insensitive --pretty -t $GL \
        --match="exp_acct_no/[a-zA-Z0-9]/" \
        --not-null=exp_acct_name \
        --match="exp_acct_name/[a-zA-Z0-9]/" \
+       --not-null=fq_acct_no \
+       --unique=fq_acct_no \
+       --rquery="(fq_acct_no IS NOT NULL AND corp_acct_fmt IS NOT NULL AND fq_acct_no !~ ('^' || regexp_replace(replace(corp_acct_fmt, '-', '}-'), '(?=[abcABC])[A-Z]', '[0-9]{', 'g') || '}$')) -- Invalid fq_acct_no" \
        --consistent-by="corp_acct_no|corp_acct_name" \
        --consistent-by="corp_acct_name|corp_acct_no" \
        --consistent-by="exp_acct_no|corp_acct_no, corp_acct_name, cc_acct_no, cc_acct_name, exp_acct_name" \
@@ -227,7 +230,6 @@ ivalidate --case-insensitive --pretty -t $LOCATION \
        --rquery="(loc_type ~* '^(LOC_TYPE_SUPPLY|S)$' and (corp_acct_no is null or corp_name is null or corp_id is null)) -- either corp id/name or corp_acct_no is null" \
        --not-null="active" \
        --match="active/^(Y|N|1|2|3)$/" \
-       --not-null="corp_acct_no" \
        --match="corp_acct_no/[a-zA-Z0-9]/" \
        --rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_loc_seq_no IS NULL OR inventory_loc_seq_no = '')) -- [inventory_loc_seq_no] is null" \
        --rquery="((inventory_path_name != '' AND inventory_path_name IS NOT NULL AND lower(inventory_path_name) != 'default') AND (inventory_location_name IS NULL OR inventory_location_name = '')) -- [inventory_location_name] is null" \
@@ -238,13 +240,15 @@ ivalidate --case-insensitive --pretty -t $LOCATION \
        --consistent-by="corp_id|corp_name" \
        --consistent-by="name|facility_code, loc_id" \
        --consistent-by="loc_id|facility_code, name" \
-       --cross-reference="inventory_path_name|$LOCATION.name" \
        --cross-reference="inventory_location_name|$LOCATION.name" \
        --cross-reference="corp_id|$GL.corp_acct_no" \
        --cross-reference="corp_name|$GL.corp_acct_name"
 
 
 # validate CONTRACTS ORIGINAL
+# @note Check unique keyset with item_id included for MSCM only
+# Accepted:
+#   --not-null=contract_gpo_name \
 ivalidate --case-insensitive --pretty -t $CONTRACTO \
        --log-to=validation_errors \
        --not-null=contract_number \
@@ -258,7 +262,6 @@ ivalidate --case-insensitive --pretty -t $CONTRACTO \
        --not-null=item_descr \
        --not-null=item_qoe \
        --not-null=contract_price \
-       --not-null=contract_gpo_name \
        --not-null=contract_gpo_id \
        --match="contract_number/[a-zA-Z0-9]/" \
        --match="contract_gpo_name/[a-zA-Z0-9]/" \
@@ -268,6 +271,7 @@ ivalidate --case-insensitive --pretty -t $CONTRACTO \
        --match="vendor_name/[a-zA-Z0-9]/" \
        --match="mfr_item_id/[a-zA-Z0-9]/" \
        --match="mfr_name/[a-zA-Z0-9]/" \
+       --rquery="(lower(mfr_name) = 'unknown' AND mfr_number IS NULL) -- Unknown mfr_name" \
        --query="to_date(contract_end, 'YYYY-MM-DD') >= to_date(contract_start, 'YYYY-MM-DD') -- [contract_end] comes before [contract_start]" \
        --match="contract_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
        --match="item_status/^(1|2|3|A|I|Inactive|Active|Y)$/" \
@@ -283,7 +287,7 @@ ivalidate --case-insensitive --pretty -t $CONTRACTO \
        --match="contract_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
        --match="item_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
        --rquery="(item_uom NOT IN (SELECT code FROM uomstd) AND item_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid item_uom" \
-       --unique="contract_gpo_name, contract_number, contract_start, contract_end, vendor_name, mfr_item_id, mfr_name, item_uom, corp_id" \
+       --unique="contract_gpo_name, contract_number, contract_start, contract_end, vendor_name, mfr_item_id, mfr_name, item_uom, corp_id, item_id" \
 
 # validate ITEM
 # Accepted:
@@ -305,6 +309,7 @@ ivalidate --case-insensitive --pretty -t $ITEM \
        --not-null="mfr_number" \
        --not-null="mfr_name" \
        --not-null="active" \
+       --rquery="(lower(mfr_name) = 'unknown' AND mfr_number IS NULL) -- Unknown mfr_name" \
        --match="corp_id/[a-zA-Z0-9]/" \
        --match="corp_name/[a-zA-Z0-9]/" \
        --match="vendor_code/[a-zA-Z0-9]/" \
@@ -362,6 +367,7 @@ ivalidate --case-insensitive --pretty -t $PO \
        --consistent-by="vendor_name|vendor_code" \
        --consistent-by="mfr_name|mfr_number" \
        --unique="po_no, po_line_number" \
+       --rquery="(lower(mfr_name) = 'unknown' AND mfr_number IS NULL) -- Unknown mfr_name" \
        --rquery="(item_id not like '%~%' and item_id not in (select item_id from items)) -- [item_id] does not reference [items.item_id]" \
        --cross-reference="vendor_code|$VENDOR.vendor_code" \
        --cross-reference="vendor_name|$VENDOR.vendor_name" \
@@ -372,7 +378,7 @@ ivalidate --case-insensitive --pretty -t $PO \
        --cross-reference="cost_center_id|$GL.cc_acct_no" \
        --cross-reference="cost_center_name|$GL.cc_acct_name" \
        --rquery="(purchase_uom NOT IN (SELECT code FROM uomstd) AND purchase_uom !~ '^[a-zA-Z0-9]{1,3}$') -- invalid [purchase_uom]" \
-       --rquery="(item_id IS NOT NULL AND (vendor_code IS NOT NULL OR vendor_name IS NOT NULL) AND vendor_item_id IS NULL) -- [vendor_item_id] is null" \
+       --rquery="((item_id IS NULL OR item_id !~ '[a-zA-Z0-9]') AND (vendor_item_id IS NULL OR vendor_item_id !~ '[a-zA-Z0-9]')) -- [vendor_item_id] is either null or invalid" \
        --match="purchase_price/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/" \
        --match="purchase_qoe/^[0-9]+(\.{0,1}[0-9]+|[0-9]*)$/"
        
@@ -586,6 +592,7 @@ imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
         --input="ContractMaster:$OUTPUT_DIR/$CONTRACTO.csv" \
         --input="ItemMaster:$OUTPUT_DIR/$ITEM.csv" \
         --input="MfrMaster:$OUTPUT_DIR/$MFR.csv" \
+        --input="VendorMaster:$OUTPUT_DIR/$VENDOR.csv" \
         --input="PurchaseOrder:$OUTPUT_DIR/$PO.csv" \
         --input="User:$OUTPUT_DIR/$USER.csv" \
         --input="Location:$OUTPUT_DIR/$LOCATION.csv" \
@@ -594,3 +601,66 @@ imerge --output=$OUTPUT_DIR/$ORGNAME.xls \
         --input="Inventory:$OUTPUT_DIR/$INVENTORY.csv"
 
 exit
+
+####################################################
+# EXPORT FOR UPLOADING
+####################################################
+iexport -t $ITEMCOST \
+        -o "$OUTPUT_DIR/$ITEMCOST.csv" -f csv --no-quote-empty --no-quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+        
+iexport -t $CONTRACTO \
+        -o "$OUTPUT_DIR/$CONTRACTO.csv" -f csv --no-quote-empty --no-quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $VENDOR \
+        -o "$OUTPUT_DIR/$VENDOR.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $MFR \
+        -o "$OUTPUT_DIR/$MFR.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $GL \
+        -o "$OUTPUT_DIR/$GL.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $PO \
+        -o "$OUTPUT_DIR/$PO.csv" -f csv --no-quote-empty --no-quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $INVENTORY \
+        -o "$OUTPUT_DIR/$INVENTORY.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $REQ \
+        -o "$OUTPUT_DIR/$REQ.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+iexport -t $ITEM \
+        -o "$OUTPUT_DIR/$ITEM.csv" -f csv --no-quote-empty --no-quotes --headers --delim=$'\t' \
+        --query="select item_id, item_descr,vendor_name,vendor_code,vendor_item_id,mfr_name,mfr_number,mfr_item_id,corp_id,corp_name, active, array_to_string(array_agg(item_uom), ',') item_uom, array_to_string(array_agg(item_qoe),',') item_qoe,array_to_string(array_agg(item_price),',') item_price
+          from 
+          (
+           select * from items order by item_id, item_descr,vendor_name,vendor_code,vendor_item_id,mfr_name,mfr_number,mfr_item_id,corp_id,corp_name, active, item_qoe::float desc
+          ) abc
+          group by item_id, item_descr,vendor_name,vendor_code,vendor_item_id,mfr_name,mfr_number,mfr_item_id,corp_id,corp_name, active
+          " \
+        --exclude="id, validation_errors, group_index"
+
+
+ipatch -q "
+  update users set phone = regexp_replace(phone, '[^0123456789]', '', 'g');
+  update users set phone = '1234567890' where phone is null or length(phone) < 10;
+  update users set first_name = username where length(first_name) < 2;
+  update users set last_name = username where length(last_name) < 2;
+"
+iexport -t $USER \
+        -o "$OUTPUT_DIR/$USER.csv" -f csv --no-quote-empty --no-quotes --no-headers --delim=',' \
+        --query="select first_name, last_name, phone, 0 as tmp1, -1 as tmp2, -1 as tmp3, -1 as tmp4, -1 as tmp5, email, '12345678' as passwd, 'Analyst' as tmp6 from users WHERE email IS NOT NULL AND length(email) > 0"
+
+iexport -t $LOCATION \
+        -o "$OUTPUT_DIR/$LOCATION.csv" -f csv --no-quote-empty --quotes --headers --delim=$'\t' \
+        --exclude="id, validation_errors"
+
+

data/lib/idata/version.rb

@@ -1,3 +1,3 @@
 module Idata
-  VERSION = "0.1.33"
+  VERSION = "0.2.1"
 end

data/sample.sh

@@ -13,11 +13,11 @@
 # SET UP ENVIRONMENT VARIABLES
 ###################################################################################
 # Instead of passing PostgreSQL credentials as parameters to every validation command,
-# you can set the corresponding environment variables which can be used by the those commands
+# you can set the corresponding environment variables which can be used by those commands
 export HOST="localhost"
 export USERNAME="postgres"
-export PASSWORD="postgres"
-export DATABASE="northeast_georgia"
+export PASSWORD="t0p!Secret"
+export DATABASE="sampledb"
 export LISTEN=5432
 
 # Input file paths and corresponding table names
@@ -39,6 +39,7 @@ REPORT="/tmp/report.xls"
 # Load data from VendorMaster.csv to the corresponding vendors table
 # and from ItemMaster.csv to items table.
 # Note: instead of using iload utility,you can use the PSQL COPY of PostgreSQL
+#       in such case, make sure an AUTO ID (unique) field is added to every table
 iload -i "$FVENDOR" -t "$VENDOR" -f csv
 iload -i "$FITEM" -t "$ITEM" -f csv
 
@@ -76,6 +77,8 @@ ivalidate --table=$ITEM \
        --not-null="mfr_number" \
        --not-null="mfr_name" \
        --not-null="active" \
+       --rquery="cast(item_id as integer) > 1000 -- invalid item_id" \
+       --rquery="purchase_date < mfr_date -- purchase_date comes before mfr_date" \
        --match="corp_id/[a-zA-Z0-9]/" \
        --match="corp_name/[a-zA-Z0-9]/" \
        --match="vendor_code/[a-zA-Z0-9]/" \
@@ -102,7 +105,7 @@ ivalidate --table=$ITEM \
 # Just to make a MORE comprehensive report, we can:
 #    1 Create a summary table which tells us how many errors found, how many records associated with each...
 #    2 Extract the first 1000 sample records for every error
-#    3 Put all together into one single Excel report
+#    3 Put them all together into one single Excel report
 
 # 1) Create error summary report table and write to /tmp/summary.csv
 # This can be done using the iexport utility which can generate a CSV file from a data table or from a custom query
@@ -111,8 +114,8 @@ iexport --output="$TMP/summary.csv" -f csv --no-quote-empty --quotes --headers \
         --query="(select '$FVENDOR' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $VENDOR), 2)::varchar || '%' as percentage from $VENDOR group by error order by error) union
                  (select '$FITEM' as input_file, unnest(string_to_array(validation_errors, ' || ')) as error, count(*), round((count(*) * 100)::numeric / (select count(*) from $ITEM), 2)::varchar || '%' as percentage from $ITEM group by error order by error)"
 
-# Export the first 1000 records of every error in the items table
-# Write the results to /tmp/items.csv
+# Export the first 1000 sample records of every error in the vendors table
+# Write the results to /tmp/vendors.csv
 iexport --table=$VENDOR --output="$TMP/$VENDOR.csv" -f csv --no-quote-empty --quotes --headers \
         --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, * 
                  FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
@@ -120,8 +123,8 @@ iexport --table=$VENDOR --output="$TMP/$VENDOR.csv" -f csv --no-quote-empty --qu
                  where group_index <= 1000" \
         --exclude="id, validation_errors, group_index"
 
-# 2) Export the first 1000 records of every error in the vendors table
-# Write the results to /tmp/vendors.csv
+# 2) Export the first 1000 sample records for every error in the items table
+# Write the results to /tmp/items.csv
 iexport --table=$ITEM --output="$TMP/$ITEM.csv" -f csv --no-quote-empty --quotes --headers \
         --query="select * from (select ROW_NUMBER() OVER (PARTITION BY error) AS group_index, * 
                  FROM ( select unnest(string_to_array(validation_errors, ' || ')) as error, * from
@@ -130,11 +133,11 @@ iexport --table=$ITEM --output="$TMP/$ITEM.csv" -f csv --no-quote-empty --quotes
         --exclude="id, validation_errors, group_index"
 
 # 3) Put the above 3 CSV files into one Excel file /tmp/report.xls
-# This can be done using imerge which takes a list of CSV files put them to corresponding sheets
+# This can be done using the imerge utility which takes a list of CSV files and put them into corresponding sheets
 # of one single Excel file
 imerge --output=$REPORT \
        --input="Summary:$TMP/summary.csv" \
-       --input="$FVENDOR:$TMP/$VENDOR.csv" \
+       --input="VendorMaster:$TMP/$VENDOR.csv" \
        --input="ItemMaster:$TMP/$ITEM.csv"
 
 # CLEANUP

metadata

@@ -1,69 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: idata
 version: !ruby/object:Gem::Version
-  version: 0.1.33
+  version: 0.2.1
 platform: ruby
 authors:
 - Nghi Pham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-31 00:00:00.000000000 Z
+date: 2014-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.16'
 description: 'Included: iload, ivalidate, isanitize, ipatch, ieval, iexpor'
@@ -80,10 +80,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
+- README2.md
 - Rakefile
 - bin/ieval
 - bin/iexport
@@ -92,6 +93,8 @@ files:
 - bin/ipatch
 - bin/isanitize
 - bin/ivalidate
+- bin/ivalidate2
+- full-pg-lawson.sh
 - full-pg.sh
 - full.sh
 - idata.gemspec
@@ -108,18 +111,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Data validation utilities
 test_files: []
+has_rdoc: