utf8_sanitizer 0.0.2.pre.rc.01 → 0.0.2.pre.rc.02

data/lib/utf8_sanitizer/csv/seeds_mini.csv

@@ -0,0 +1,100 @@
+url,act_name,street,city,state,zip,phone
+stanleykaufman.net,Stanley Chevrolet Kaufman,825 E Fair St,Kaufman,TX,75142,(888) 457-4391
+leepartyka.com,Lee Partyka Chevrolet Mazda Isuzu Truck,200 Skiff St,Hamden,CT,6518,(203) 288-7761
+burienhonda.com,"Honda of Burien
+cortlandchryslerdodgejeep.com,Cortland Chrysler Dodge Jeep RAM,3878 West Rd,Cortland,NY,13045,(877) 279-3113
+imperialmotors.net,Imperial Motors,4839 Virginia Beach Blvd,Virginia Beach,VA,23462,(757) 490-3651
+liatoyotaofnorthampton.com,"Lia Toyota of Northampton
+nelsonhallchevrolet.com,Nelson Hall Chevrolet,1811 S Frontage Rd,Meridian,MS,39301,(601) 621-4593
+marshallfordco.com,Marshall Ford Co Inc.,14843 MS-16,Philadelphia,MS,39350,(888) 461-7643
+warrentontoyota.com,Warrenton Toyota,6449 Lee Hwy,Warrenton,VA,20187,(540) 878-4100
+toyotacertifiedatcentralcity.com,"Toyota Certified
 Central City",4800 Chestnut St,Philadelphia,PA,19139,(888) 379-1155
+rockcitychrysler.com,Rock City Chrysler Jeep Dodge,520 Rock City St,Little Valley,NY,14755,(866) 414-1024
+eddinsford.com,Eddins Ford Inc.,2895 S Seminole Trail,Madison,VA,22727,(888) 348-0661
+lithiasubarufresno.com,Lithia Subaru of Fresno,5499 N Blackstone Ave,Fresno,CA,93710,(888) 453-5359
+nnchevrolet.com,Northern Neck Chevrolet,18175 Kings Hwy,Montross,VA,22520,(804) 368-6547
+brewsterhonda.com,"Lia Honda Brewster NY
+tegelerchevrolet.com,Tegeler Chevrolet,17114 Fordtran Blvd,Industry,TX,78944,(979) 357-2575
+rochesterhillschryslerjeep.net,Rochester Hills Chrysler Jeep Dodge,1301 S Rochester Rd,Rochester,MI,48307,(248) 841-4094
+1stkia.com,"First Kia
+audijax.com,Audi Jacksonville,7230 Blanding Blvd,Jacksonville,FL,32244,(888) 451-7572
+kalsauto.com,Kal's Auto Sales II,5130 W 8 Mile Rd,Detroit,MI,48234,(313) 891-0000
+carwashcarsinc.com,Car Wash Cars Inc,462 Route 9W,Glenmont,NY,12077,(518) 729-4317
+autonationnissanorangepark.com,AutoNation Nissan Orange Park,7447 Blanding Blvd,Jacksonville,FL,32244,(904) 270-9954
+cfschwartztoyota.com,C. F. Schwartz Toyota,1536 N Dupont Hwy,Dover,DE,19901,(877) 769-6001
+lhmlexus.com,"Directions to Larry H. Miller Lexus Murray near Salt Lake City, UT
+agchevy.com,Arroyo Grande Chevrolet,303 Traffic Way,Arroyo Grande,CA,93420,(805) 202-4636
+billowenscars.com,Bill Owens Auto Sales,705 U.S. 27 SOUTH,Avon Park,FL,33825,(863) 452-5599
+damascusmotors.com,Damascus Motors,26100 Woodfield Rd,Damascus,MD,20872,(301) 253-2151
+dennisdillonchryslerjeepdodge.com,Dennis Dillon Chrysler Jeep Dodge,4025 Cleveland Blvd,Caldwell,ID,83605,(888) 506-8191
+bobhowardhonda.com,Bob Howard Honda,13201 N Kelley Ave,Oklahoma City,OK,73131,(405) 753-8700
+humboldtdodgechryslerjeep.com,Lonnie Cobb's Humboldt Chrysler Dodge Jeep RAM,3301 Eastend Dr,Humboldt,TN,38343,(888) 554-6047
+livingstonvw.com,Livingston Volkswagen,21141 Ventura Blvd,Woodland Hills,CA,91364,(855) 419-9290
+beckmastennorth.com,Beck and Masten North,11300 FM 1960,Houston,TX,77065,(281) 469-5222
+fairbanksnissan.com,Fairbanks Nissan,2610 S Cushman St,Fairbanks,AK,99701,(866) 467-8690
+feyerlincoln.net,Feyer Lincoln Inc.,1677 US-17 BUS,Williamston,NC,27892,(866) 493-1351
+forrestersdodgecity.com,Forrester's Dodge City,204 S Walnut St,Arnold,NE,69120,(888) 463-0311
+victorchevrolet.com,Victor Chevrolet - A Rochester Area  Dealer,7200 Pittsford Victor Rd,Victor,NY,14564,(585) 433-2500
+landroverpalmbeach.com,Land Rover Palm Beach,7550 Okeechobee Blvd,West Palm Beach,FL,33411,(561) 209-7000
+goudyhonda.com,Goudy Honda,1400 W Main St,Alhambra,CA,91801,(626) 576-1114
+davis-olaughlin.com,Davis O'Laughlin Buick GMC Cadillac,4580 US-219,Great Valley,NY,14741,(716) 945-5050
+lutherhondaofstcloud.com,Luther St. Cloud Honda,1500 Hwy 23 West,Waite Park,MN,56387,(888) 453-5650
+titanautosales.com,Titan Auto Sales LLC,1037 Central Ave,Albany,NY,12205,(518) 438-0101
+hixsonbmw.com,Hixson BMW Monroe,1201 Louisville Ave,Monroe,LA,71201,(855) 519-8884
+antwerpenautoworld.com,Antwerpen Auto World,9400 Liberty Road,Randallstown,MD,21133,(888) 437-9880
+lakelandcdj.com,Lakeland Chrysler Dodge Jeep,2875 Mall Hill Drive,Lakeland,FL,33810,(888) 420-1648
+blakegreenfieldchevroletbuick.com,Blake Greenfield Chevrolet Buick,150 3rd St NE,Wells,MN,56097,(507) 774-0365
+carltonmb.com,Carlton Motorcars,2446 Laurens Rd,Greenville,SC,29607,(855) 336-9760
+silkohonda.com,Silko Honda,1580 New State Hwy,Raynham,MA,2767,(508) 880-5500
+lahabramotors.com,La Habra Motors,980 E Whittier Blvd,La Habra,CA,90631,(562) 697-2400
+hugoautosales.com,Hugo Auto Sales,2583 Cameron Langston Rd,Grifton,NC,28530,(252) 522-4140
+baxterfordsouth.com,Baxter Ford South,9203 S 145th St,Omaha,NE,68138,(855) 985-0071
+roemotorsny.com,Roe Motors Ltd,662 Montauk Hwy,Shirley,NY,11967,(631) 395-2277
+kingmontclair.com,King Auto Sales,5053 Mission Blvd,Montclair,CA,91763,(909) 627-1211
+douglasinfiniti.com,Douglas Infiniti,430 Morris Ave,Summit,NJ,7901,(908) 522-7300
+plazainfiniti.com,Plaza Infiniti,755 N New Ballas Rd,Creve Coeur,MO,63141,(888) 593-1881
+coughlinmarysvillechrysler.com,Coughlin Chrysler Jeep Dodge RAM,15777 Watkins Rd,Marysville,OH,43040,(888) 378-6128
+dillonlakemtrs.com,Dillon Lake Motors,3350 Maple Ave,Zanesville,OH,43701,(740) 297-7265
+chapmanspeedway.com,Chapman Used Cars on Speedway,6001 E Speedway Blvd,Tucson,AZ,85712,(520) 829-0032
+beckchevroletonline.com,"Beck Chevrolet Co., Inc.",561 Central Park Ave,Yonkers,NY,10704,(914) 595-1463
+infinitimv.com,INFINITI of Mission Viejo,28471 Marguerite Pkwy,Mission Viejo,CA,92692,(855) 410-3445
+hondaofserramonte.com,Honda of Serramonte,485 Serramonte Blvd,Colma,CA,94014,(888) 892-5461
+douglassnissanofwaco.com,Douglass Nissan of Waco,5605 Legend Lake Pkwy,Waco,TX,76712,(877) 302-5270
+saturnofknoxville.com,McManus Auto Sales,6404 Maynardville Pike,Knoxville,TN,37918,(865) 281-2278
+avchevy.com,Antelope Valley Chevrolet,1160 Motor Ln,Lancaster,CA,93534,(661) 952-2300
+skylineford.net,Skyline Ford,2510 Commercial St SE,Salem,OR,97302,(888) 873-7584
+moseshuntingtongm.com,Moses Buick GMC Cadillac,5200 Route 60 East,Huntington,WV,25705,(304) 736-5291
+hatchtoyota.com,2017 Hatch Toyota.,1051 Automall Pkwy,Show Low,AZ,85901,(928) 242-9007
+malouffordinc.com,Malouf Ford,2210 US-1,North Brunswick Township,NJ,8902,(866) 951-8097
+seminolefordok.com,Seminole Ford,2222 N Milt Phillips Ave,Seminole,OK,74868,(888) 903-5643
+flagshipmotorcars.com,Flagship Motorcars of Lynnfield,385 Broadway,Lynnfield,MA,1940,(877) 907-1967
+capitolhyundai.com,Capitol Hyundai Columbia,101 Newland Rd,Columbia,SC,29229,(800) 258-1436
+searsimports.com,"Sears Imported Autos, Inc.",13500 Wayzata Blvd,Minnetonka,MN,55305,(888) 580-8791
+videon.com,Videon Chrysler Dodge Jeep RAM,4951 West Chester Pike,Newtown Square,PA,19073,(610) 356-7000
+towncountrychryslerdodge.com,Town & Country Chrysler Dodge Ram,703 Crim Ave,Belington,WV,26250,(877) 830-9359
+palmharborhonda.com,Courtesy Palm Harbor Honda,31200 US Hwy 19 N,Palm Harbor,FL,34684,(866) 423-6155
+bravolascruces.com,Bravo Chevrolet Cadillac,1601 S Main St,Las Cruces,NM,88005,(575) 527-3800
+ricksautosite.com,Rick's Auto Sales Used Cars,1536 E 5th St,Metropolis,IL,62960,(618) 524-7366
+audiofhuntington.com,Audi of Huntington,363 E Jericho Turnpike,Huntington Station,NY,11746,(866) 455-0268
+discountautocorner.com,Discount Auto,2908 Lincoln Hwy,Langhorne,PA,19047,(215) 757-5170
+audipetoskey.com,Audi Petoskey,825 Charlevoix Ave,Petoskey,MI,49770,(888) 478-1060
+vwoforchardpark.com,Volkswagen of Orchard Park,3524 Southwestern Blvd,Orchard Park,NY,14127,(716) 216-0256
+pietroske.com,Pietroske Chevrolet,4000 Grand Avenue,Manitowoc,WI,54221,(920) 684-0224
+hendrickhyundainorth.com,Hendrick Hyundai North Charleston,8485 Rivers Ave,North Charleston,SC,29406,(866) 542-1320
+johnsgreatcars.com,John's Great Cars,1133 Lancaster Ave,Reading,PA,19607,(610) 603-9100
+beckmotors.net,Beck Motors,305 US Hwy 63,Freeburg,MO,65035,(888) 638-4505
+parksmotorsales.com,Parks Motor Sales Inc.,919 Nashville Hwy,Columbia,TN,38401,(931) 388-5843
+josephbuickgmc.com,Joseph Buick GMC,8700 Colerain Ave,Groesbeck,OH,45251,(513) 741-1000
+centralgmcnorwood.com,Central Buick GMC of Norwood,70 Boston Providence Hwy,Norwood,MA,2062,(781) 473-0908
+cruisinclassicsinc.com,Cruisin Classics Auto Sales,3575 Fisher Rd,Columbus,OH,43228,(614) 276-7355
+everettchryslerjeepdodgeram.com,Everett Chrysler Dodge Jeep Ram,3709 S Thompson St,Springdale,AR,72764,(888) 257-0465
+garavelsubaru.com,Garavel Subaru,10 Tindall Ave,Norwalk,CT,6851,(877) 223-0913
+jansenchevrolet.com,Jansen Chevrolet,7801 IL-161,Germantown,IL,62245,(618) 613-4215
+bramanbmwjupiter.com,Braman BMW of Jupiter,1555 W Indiantown Rd,Jupiter,FL,33458,(561) 609-0144
+mildenbergermotors.com,Mildenberger Motors,1717 N 1st St,Hamilton,MT,59840,(406) 626-3050
+autonationfordarlington.com,AutoNation Ford Arlington,1400 W Interstate 20,Arlington,TX,76017,(817) 200-4503
+rampmotors.com,Ramp Motors,4869 Nesconset Hwy,Port Jefferson Station,NY,11776,(631) 473-1550
+millerchevy.com,Miller & Sons Chevrolet Buick,3107 Green Garden Rd,Aliquippa,PA,15001,(724) 378-0541
+cjsautosales.com,CJS Auto Sales,2509 S Houston Rd,Pasadena,TX,77502,(713) 947-6405
+redwingchev.net,Red Wing Chevrolet Buick Cadillac,2500 US Hwy 61,Red Wing,MN,55066,(651) 388-4777
+sunnyvalevw.com,Sunnyvale Volkswagen,1025 E El Camino Real,Sunnyvale,CA,94087,(877) 785-8252

data/lib/utf8_sanitizer/csv/seeds_mini_10.csv

@@ -0,0 +1,11 @@
+url,act_name,street,city,state,zip,phone
+stanleykaufman.net,Stanley Chevrolet Kaufman,825 E Fair St,Kaufman,TX,75142,(888) 457-4391
+leepartyka.com,Lee Partyka Chevrolet Mazda Isuzu Truck,200 Skiff St,Hamden,CT,6518,(203) 288-7761
+burienhonda.com,"Honda of Burien
+cortlandchryslerdodgejeep.com,Cortland Chrysler Dodge Jeep RAM,3878 West Rd,Cortland,NY,13045,(877) 279-3113
+imperialmotors.net,Imperial Motors,4839 Virginia Beach Blvd,Virginia Beach,VA,23462,(757) 490-3651
+liatoyotaofnorthampton.com,"Lia Toyota of Northampton
+nelsonhallchevrolet.com,Nelson Hall Chevrolet,1811 S Frontage Rd,Meridian,MS,39301,(601) 621-4593
+marshallfordco.com,Marshall Ford Co Inc.,14843 MS-16,Philadelphia,MS,39350,(888) 461-7643
+warrentontoyota.com,Warrenton Toyota,6449 Lee Hwy,Warrenton,VA,20187,(540) 878-4100
+toyotacertifiedatcentralcity.com,"Toyota Certified
 Central City",4800 Chestnut St,Philadelphia,PA,19139,(888) 379-1155

data/lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv

@@ -0,0 +1,3 @@
+url,act_name,street,city,state,zip,phone
+stanleykaufman.net.com.gov,Stanley Chevrolet Kaufman,825 E Fair St,Kaufman,TX,75142,(888) 457-4391
+leepartyka,Lee Partyka Chevrolet Mazda Isuzu Truck,200 Skiff St,Hamden,CT,6518,(203) 288-7761

data/lib/utf8_sanitizer/run.rb

@@ -0,0 +1,31 @@
+
+module Utf8Sanitizer
+  class Run
+
+    def initialize
+      @crm_data = {}
+    end
+
+
+    def import(args={})
+      @crm_data = { stats: nil, data: nil, file_path: nil, criteria: nil }
+      @crm_data.merge!(args)
+      keys = args.compact.keys
+
+      unless (keys & [:data, :file_path]).any?
+        @crm_data[:file_path] = Seed.new.grab_seed_file_path
+        # @crm_data[:data] = Seed.new.grab_seed_hashes
+        @crm_data[:pollute_seeds] = true
+        unless keys.include?(:criteria)
+          @crm_data[:criteria] = Seed.new.grab_seed_web_criteria
+        end
+      end
+
+      utf_result = Utf8Sanitizer::UTF.new.validate_data(@crm_data)
+      @crm_data.merge!(utf_result)
+    end
+
+
+
+  end
+end

data/lib/utf8_sanitizer/seed.rb

@@ -0,0 +1,97 @@
+require 'csv'
+
+module Utf8Sanitizer
+  class Seed
+    def initialize(args={})
+      # @pollute_seeds = args.fetch(:pollute_seeds, false)
+      # @seed_hashes = args.fetch(:seed_hashes, false)
+      # @seed_csv = args.fetch(:seed_csv, false)
+    end
+
+    def pollute_seeds(text)
+      list = ['h∑', 'lÔ', "\x92", "\x98", "\x99", "\xC0", "\xC1", "\xC2", "\xCC", "\xDD", "\xE5", "\xF8"]
+      index = text.length / 2
+      var = "#{list.sample}_#{list.sample}"
+      text.insert(index, var)
+      text.insert(-1, "\r\n")
+      text
+    end
+
+    def grab_seed_file_path
+      # "./lib/utf8_sanitizer/csv/seeds_clean.csv"
+      # "./lib/utf8_sanitizer/csv/seeds_dirty.csv"
+      # "./lib/utf8_sanitizer/csv/seeds_mega.csv"
+      # "./lib/utf8_sanitizer/csv/seeds_mini.csv"
+      # "./lib/utf8_sanitizer/csv/seeds_mini_10.csv"
+      './lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv'
+    end
+
+    ### Sample Hashes for validate_data
+    def grab_seed_hashes
+      [{ row_id: 1,
+         url: 'stanleykaufman.com',
+         act_name: 'Stanley Chevrolet Kaufman',
+         street: '825 E Fair St',
+         city: 'Kaufman',
+         state: 'TX',
+         zip: '75142',
+         phone: '(888) 457-4391' },
+       { row_id: 2,
+         url: 'leepartyka',
+         act_name: 'Lee Partyka Chevrolet Mazda Isuzu Truck',
+         street: '200 Skiff St',
+         city: 'Hamden',
+         state: 'CT',
+         zip: '6518',
+         phone: '(203) 288-7761' },
+       { row_id: 3,
+         url: 'burienhonda.fake.not.net.com',
+         act_name: 'Honda of Burien 15026 1st Avenue South, Burien, WA 98148',
+         street: '15026 1st Avenue South',
+         city: 'Burien',
+         state: 'WA',
+         zip: '98148',
+         phone: '(206) 246-9700' },
+       { row_id: 4,
+         url: 'cortlandchryslerdodgejeep.com',
+         act_name: 'Cortland Chrysler Dodge Jeep RAM',
+         street: '3878 West Rd',
+         city: 'Cortland',
+         state: 'NY',
+         zip: '13045',
+         phone: '(877) 279-3113' },
+       { row_id: 5,
+         url: 'imperialmotors.net',
+         act_name: 'Imperial Motors',
+         street: '4839 Virginia Beach Blvd',
+         city: 'Virginia Beach',
+         state: 'VA',
+         zip: '23462',
+         phone: '(757) 490-3651' }]
+    end
+
+
+
+    def grab_seed_web_criteria
+      pos_urls, neg_urls, neg_links, neg_hrefs, neg_exts = [], [], [], [], []
+
+      neg_urls = %w[approv avis budget collis eat enterprise facebook financ food google gourmet hertz hotel hyatt insur invest loan lube mobility motel motorola parts quick rent repair restaur rv ryder service softwar travel twitter webhost yellowpages yelp youtube]
+
+      pos_urls = ['acura', 'alfa romeo', 'aston martin', 'audi', 'bmw', 'bentley', 'bugatti', 'buick', 'cdjr', 'cadillac', 'chevrolet', 'chrysler', 'dodge', 'ferrari', 'fiat', 'ford', 'gmc', 'group', 'group', 'honda', 'hummer', 'hyundai', 'infiniti', 'isuzu', 'jaguar', 'jeep', 'kia', 'lamborghini', 'lexus', 'lincoln', 'lotus', 'mini', 'maserati', 'mazda', 'mclaren', 'mercedes-benz', 'mitsubishi', 'nissan', 'porsche', 'ram', 'rolls-royce', 'saab', 'scion', 'smart', 'subaru', 'suzuki', 'toyota', 'volkswagen', 'volvo']
+
+      # neg_links = %w(: .biz .co .edu .gov .jpg .net // afri anounc book business buy bye call cash cheap click collis cont distrib download drop event face feature feed financ find fleet form gas generat graphic hello home hospi hour hours http info insta inventory item join login mail mailto mobile movie museu music news none offer part phone policy priva pump rate regist review schedul school service shop site test ticket tire tv twitter watch www yelp youth)
+
+      # neg_hrefs = %w(? .com .jpg @ * afri after anounc apply approved blog book business buy call care career cash charit cheap check click collis commerc cont contrib deal distrib download employ event face feature feed financ find fleet form gas generat golf here holiday hospi hour info insta inventory join later light login mail mobile movie museu music news none now oil part pay phone policy priva pump quick quote rate regist review saving schedul service shop sign site speci ticket tire today transla travel truck tv twitter watch youth)
+
+      neg_exts = %w[au ca edu es gov in ru uk us]
+
+      oa_args = { pos_urls: pos_urls, neg_urls: neg_urls, neg_links: neg_links, neg_hrefs: neg_hrefs, neg_exts: neg_exts }
+      oa_args.compact
+    end
+
+
+
+
+
+  end
+end

data/lib/utf8_sanitizer/utf.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: false
+# require 'csv'
+
+module Utf8Sanitizer
+  class UTF
+    def initialize(args={})
+      @utf_result = { stats: {}, data: {} }
+      @valid_rows = []
+      @encoded_rows = []
+      @defective_rows = []
+      @error_rows = []
+      @headers = []
+      @row_id = 0
+      @data_hash = {}
+    end
+
+    #################### * VALIDATE DATA * ####################
+    def validate_data(args={})
+      args = args.slice(:file_path, :data, :pollute_seeds)
+      args = args.compact
+
+      @seed = Seed.new if args.fetch(:pollute_seeds)
+      file_path = args[:file_path]
+      data = args[:data]
+
+      utf_result = validate_csv(file_path) if file_path
+      utf_result = validate_hashes(data) if data
+      utf_result
+    end
+
+    #################### * COMPILE RESULTS * ####################
+    def compile_results
+      utf_status = @valid_rows.map { |hsh| hsh[:utf_status] }
+      mapped_details = utf_status.map { |str| str.split(', ') }.flatten.compact
+      groups = make_groups_from_array(mapped_details)
+      wchar = groups['wchar']
+      perfect = groups['perfect']
+
+      header_row_count = @headers.any? ? 1 : 0
+      stats = { total_rows: @row_id, header_row: header_row_count, valid_rows: @valid_rows.count, error_rows: @error_rows.count, defective_rows: @defective_rows.count, perfect_rows: perfect, encoded_rows: @encoded_rows.count, wchar_rows: wchar }
+      data = { valid_data: @valid_rows, encoded_data: @encoded_rows, defective_data: @defective_rows, error_data: @error_rows }
+      @utf_result = { stats: stats, data: data }
+      utf_result = @utf_result
+      initialize
+      utf_result
+    end
+
+    #################### * VALIDATE CSV * ####################
+    def validate_csv(file_path)
+      return unless file_path.present?
+      File.open(file_path).each do |file_line|
+        validated_line = utf_filter(check_utf(file_line))
+        @row_id += 1
+        if validated_line
+          CSV.parse(validated_line) do |row|
+            if @headers.empty?
+              @headers = row
+            else
+              @data_hash.merge!(row_to_hsh(row))
+              @valid_rows << @data_hash
+            end
+          end
+        end
+      rescue StandardError => error
+        @error_rows << { row_id: @row_id, text: error.message }
+      end
+      compile_results
+    end
+
+    #################### * VALIDATE HASHES * ####################
+    def validate_hashes(orig_hashes)
+      return unless orig_hashes.present?
+      begin
+        process_hash_row(orig_hashes.first) ## re keys for headers.
+        orig_hashes.each { |hsh| process_hash_row(hsh) } ## re values
+      rescue StandardError => error
+        @error_rows << { row_id: @row_id, text: error.message }
+      end
+      compile_results ## handles returns.
+    end
+
+    ### process_hash_row - helper VALIDATE HASHES ###
+    ### Converts hash keys and vals into parsed line.
+    def process_hash_row(hsh)
+      if @headers.any?
+        keys_or_values = hsh.values
+        @row_id = hsh[:row_id]
+      else
+        keys_or_values = hsh.keys.map(&:to_s)
+      end
+
+      file_line = keys_or_values.join(',')
+      line_parse(utf_filter(check_utf(file_line)))
+    end
+
+    ### line_parse - helper VALIDATE HASHES ###
+    ### Parses line to row, then updates final results.
+    def line_parse(validated_line)
+      return unless validated_line
+      row = validated_line.split(',')
+      return unless row.any?
+      if @headers.empty?
+        @headers = row
+      else
+        @data_hash.merge!(row_to_hsh(row))
+        @valid_rows << @data_hash
+      end
+    end
+
+    #################### * CHECK UTF * ####################
+    def check_utf(text)
+      return unless text.present?
+      text = @seed.pollute_seeds(text) if @seed && @headers.any?
+      results = { text: text, encoded: nil, wchar: nil, error: nil }
+      begin
+        if !text.valid_encoding?
+          encoded = text.chars.select(&:valid_encoding?).join
+          encoded.delete!('_')
+          encoded = encoded.delete("^\u{0000}-\u{007F}")
+        else
+          encoded = text.delete("^\u{0000}-\u{007F}")
+        end
+        wchar = encoded&.gsub(/\s+/, ' ')&.strip
+        results[:encoded] = encoded if text != encoded
+        results[:wchar] = wchar if encoded != wchar
+      rescue StandardError => error
+        results[:error] = error.message if error
+      end
+      results
+    end
+
+    #################### * UTF FILTER * ####################
+    def utf_filter(utf)
+      return unless utf.present?
+      puts utf.inspect
+      utf_status = utf.except(:text).compact.keys
+      utf_status = utf_status&.map(&:to_s)&.join(', ')
+      utf_status = 'perfect' if utf_status.blank?
+
+      encoded = utf[:text] if utf[:encoded]
+      error = utf[:error]
+      line = utf.except(:error).compact.values.last unless error
+      data_hash = { row_id: @row_id, utf_status: utf_status }
+
+      @encoded_rows << { row_id: @row_id, text: encoded } if encoded
+      @error_rows << { row_id: @row_id, text: error } if error
+      @defective_rows << filt_utf_hsh[:text] if error
+      @data_hash = data_hash if @data_hash[:row_id] != @row_id
+      line
+    end
+
+    ############# !! HELPERS BELOW !! #############
+    ############# KEY VALUE CONVERTERS #############
+    def row_to_hsh(row)
+      h = Hash[@headers.zip(row)]
+      h.symbolize_keys
+    end
+
+    def val_hsh(cols, hsh)
+      keys = hsh.keys
+      keys.each { |key| hsh.delete(key) unless cols.include?(key) }
+      hsh
+    end
+
+    def make_groups_from_array(array)
+      array.each_with_object(Hash.new(0)) { |e, h| h[e] += 1; }
+    end
+
+  end
+end

data/lib/utf8_sanitizer/version.rb

@@ -1,4 +1,4 @@
 module Utf8Sanitizer
   # VERSION = "0.0.1-rc.1"
-  VERSION = "0.0.2.pre.rc.01"
+  VERSION = "0.0.2.pre.rc.02"
 end

data/lib/utf8_sanitizer.rb

@@ -1,11 +1,14 @@
 require "utf8_sanitizer/version"
+require 'utf8_sanitizer/run'
+require 'utf8_sanitizer/seed'
+require 'utf8_sanitizer/utf'
+require 'pry'
 
 module Utf8Sanitizer
 
   def self.run_wrap
-    puts "Sample the Wrap!"
-    # wrap = self::Wrap.new
-    # wrap.run_wrap ## returns formatted urls.
+    run = self::Run.new
+    result = run.import ## returns formatted urls.
   end
 
 end

data/utf8_sanitizer.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/4rlm/utf8_sanitizer'
   spec.license       = 'MIT'
 
-  spec.summary       = "Still in Development: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings."
-  spec.description   = "Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.\n Example: ABC Au\\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\\n\\r\\n \n=> ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234"
+  spec.summary       = "Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings."
+  spec.description   = "Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.\n Example: ABC Au\\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\\n\\r\\n  =>  ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234"
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = "https://rubygems.org"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 0.0.2.pre.rc.01
+  version: 0.0.2.pre.rc.02
 platform: ruby
 authors:
 - Adam Booth
@@ -180,10 +180,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.3
-description: "Removes invalid UTF8 characters, and extra whitespace (carriage returns,
-  new lines, tabs, spaces, etc.) from csv, or strings.\n Example: ABC Au\\xC1tos,123
-  E Main St,Anytown,TX,75142,(888) 555-1234\\n\\r\\n \n=> ABC Autos,123 E Main St,Anytown,TX,75142,(888)
-  555-1234"
+description: |-
+  Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.
+   Example: ABC Au\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\n\r\n  =>  ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234
 email:
 - 4rlm@protonmail.ch
 executables: []
@@ -201,6 +200,16 @@ files:
 - bin/console
 - bin/setup
 - lib/utf8_sanitizer.rb
+- lib/utf8_sanitizer/csv/extensions.csv
+- lib/utf8_sanitizer/csv/seeds_clean.csv
+- lib/utf8_sanitizer/csv/seeds_dirty.csv
+- lib/utf8_sanitizer/csv/seeds_mega.csv
+- lib/utf8_sanitizer/csv/seeds_mini.csv
+- lib/utf8_sanitizer/csv/seeds_mini_10.csv
+- lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv
+- lib/utf8_sanitizer/run.rb
+- lib/utf8_sanitizer/seed.rb
+- lib/utf8_sanitizer/utf.rb
 - lib/utf8_sanitizer/version.rb
 - utf8_sanitizer.gemspec
 - utf8_sanitizer_gemspec_orig.txt
@@ -228,6 +237,6 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: 'Still in Development: Removes invalid UTF8 characters, and extra whitespace
-  (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.'
+summary: 'Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage
+  returns, new lines, tabs, spaces, etc.) from csv, or strings.'
 test_files: []