utf8_sanitizer 0.0.2.pre.rc.02 → 0.0.2.pre.rc.03

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a38bc8d107aceca249f57e024c44ac5a29a29d83b6caa6afbfe1b3e5bddc3092
-  data.tar.gz: 8475996137f588b1a18f335770a08d19db0a5291076abdf0714ca487ed87b8f0
+  metadata.gz: 1128d1d63dfb289df84978713ef4fc7f8efc1bfb7d6784ab7accd85684824bff
+  data.tar.gz: 54111bdd0d5e3e1c6f97892c5404d68cbaf283068f77a3372e3bc6facb7cd9d3
 SHA512:
-  metadata.gz: f1272a9c28b5e4420992a04089ef241a3c9c299aa2643a39b29ee3a411b1dbe18ccb7d0658c9e77a0ead3ae388f046a01e05d024b7558aef562022f3ebf703e9
-  data.tar.gz: 4df59b0146fb3516bc1f1ce81c8ece8ed1104094681bb2e0ff70dfa1741af8f0cb13117218e2fb488b58f4c94484bca76470728b6fde56b52153f352cec48f0b
+  metadata.gz: 0e64c64ab3dafaa19fdc1cac3fae947b8aafe909bde3d747b8a23b8516ad5224769ae690605074a22e85082275e799173e866d58963ef951b62d7183d9503ae7
+  data.tar.gz: '0813f8565e88f697799fd3882ed883595c89d5af7a6042a792b1918fe4e97524ed0ba6da5ac0e1e21b0c91dc7ef89766122000cdeffaaa08505fbfbb300a63c5'

data/Rakefile

@@ -13,6 +13,7 @@ task :console do
   require 'utf8_sanitizer' # You know what to do.
   require "active_support/all"
   ARGV.clear
-  Utf8Sanitizer.run_wrap
+  # sanitized_data = Utf8Sanitizer.sanitize(file_path: "./lib/utf8_sanitizer/csv/seeds_mini.csv")
+  sanitized_data = Utf8Sanitizer.sanitize
   IRB.start
 end

data/lib/utf8_sanitizer/seed.rb

@@ -19,11 +19,11 @@ module Utf8Sanitizer
 
     def grab_seed_file_path
       # "./lib/utf8_sanitizer/csv/seeds_clean.csv"
-      # "./lib/utf8_sanitizer/csv/seeds_dirty.csv"
+      "./lib/utf8_sanitizer/csv/seeds_dirty.csv"
       # "./lib/utf8_sanitizer/csv/seeds_mega.csv"
       # "./lib/utf8_sanitizer/csv/seeds_mini.csv"
       # "./lib/utf8_sanitizer/csv/seeds_mini_10.csv"
-      './lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv'
+      # './lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv'
     end
 
     ### Sample Hashes for validate_data
@@ -70,28 +70,5 @@ module Utf8Sanitizer
          phone: '(757) 490-3651' }]
     end
 
-
-
-    def grab_seed_web_criteria
-      pos_urls, neg_urls, neg_links, neg_hrefs, neg_exts = [], [], [], [], []
-
-      neg_urls = %w[approv avis budget collis eat enterprise facebook financ food google gourmet hertz hotel hyatt insur invest loan lube mobility motel motorola parts quick rent repair restaur rv ryder service softwar travel twitter webhost yellowpages yelp youtube]
-
-      pos_urls = ['acura', 'alfa romeo', 'aston martin', 'audi', 'bmw', 'bentley', 'bugatti', 'buick', 'cdjr', 'cadillac', 'chevrolet', 'chrysler', 'dodge', 'ferrari', 'fiat', 'ford', 'gmc', 'group', 'group', 'honda', 'hummer', 'hyundai', 'infiniti', 'isuzu', 'jaguar', 'jeep', 'kia', 'lamborghini', 'lexus', 'lincoln', 'lotus', 'mini', 'maserati', 'mazda', 'mclaren', 'mercedes-benz', 'mitsubishi', 'nissan', 'porsche', 'ram', 'rolls-royce', 'saab', 'scion', 'smart', 'subaru', 'suzuki', 'toyota', 'volkswagen', 'volvo']
-
-      # neg_links = %w(: .biz .co .edu .gov .jpg .net // afri anounc book business buy bye call cash cheap click collis cont distrib download drop event face feature feed financ find fleet form gas generat graphic hello home hospi hour hours http info insta inventory item join login mail mailto mobile movie museu music news none offer part phone policy priva pump rate regist review schedul school service shop site test ticket tire tv twitter watch www yelp youth)
-
-      # neg_hrefs = %w(? .com .jpg @ * afri after anounc apply approved blog book business buy call care career cash charit cheap check click collis commerc cont contrib deal distrib download employ event face feature feed financ find fleet form gas generat golf here holiday hospi hour info insta inventory join later light login mail mobile movie museu music news none now oil part pay phone policy priva pump quick quote rate regist review saving schedul service shop sign site speci ticket tire today transla travel truck tv twitter watch youth)
-
-      neg_exts = %w[au ca edu es gov in ru uk us]
-
-      oa_args = { pos_urls: pos_urls, neg_urls: neg_urls, neg_links: neg_links, neg_hrefs: neg_hrefs, neg_exts: neg_exts }
-      oa_args.compact
-    end
-
-
-
-
-
   end
 end

data/lib/utf8_sanitizer/utf.rb

@@ -4,7 +4,6 @@
 module Utf8Sanitizer
   class UTF
     def initialize(args={})
-      @utf_result = { stats: {}, data: {} }
       @valid_rows = []
       @encoded_rows = []
       @defective_rows = []
@@ -18,8 +17,7 @@ module Utf8Sanitizer
     def validate_data(args={})
       args = args.slice(:file_path, :data, :pollute_seeds)
       args = args.compact
-
-      @seed = Seed.new if args.fetch(:pollute_seeds)
+      @seed = Seed.new if args[:pollute_seeds]
       file_path = args[:file_path]
       data = args[:data]
 
@@ -37,12 +35,10 @@ module Utf8Sanitizer
       perfect = groups['perfect']
 
       header_row_count = @headers.any? ? 1 : 0
-      stats = { total_rows: @row_id, header_row: header_row_count, valid_rows: @valid_rows.count, error_rows: @error_rows.count, defective_rows: @defective_rows.count, perfect_rows: perfect, encoded_rows: @encoded_rows.count, wchar_rows: wchar }
-      data = { valid_data: @valid_rows, encoded_data: @encoded_rows, defective_data: @defective_rows, error_data: @error_rows }
-      @utf_result = { stats: stats, data: data }
-      utf_result = @utf_result
-      initialize
-      utf_result
+      utf_result = {
+        stats: { total_rows: @row_id, header_row: header_row_count, valid_rows: @valid_rows.count, error_rows: @error_rows.count, defective_rows: @defective_rows.count, perfect_rows: perfect, encoded_rows: @encoded_rows.count, wchar_rows: wchar },
+        data: { valid_data: @valid_rows, encoded_data: @encoded_rows, defective_data: @defective_rows, error_data: @error_rows }
+      }
     end
 
     #################### * VALIDATE CSV * ####################

data/lib/utf8_sanitizer/version.rb

@@ -1,4 +1,4 @@
 module Utf8Sanitizer
   # VERSION = "0.0.1-rc.1"
-  VERSION = "0.0.2.pre.rc.02"
+  VERSION = "0.0.2.pre.rc.03"
 end

data/lib/utf8_sanitizer.rb

@@ -1,14 +1,28 @@
 require "utf8_sanitizer/version"
-require 'utf8_sanitizer/run'
 require 'utf8_sanitizer/seed'
 require 'utf8_sanitizer/utf'
 require 'pry'
 
 module Utf8Sanitizer
 
-  def self.run_wrap
-    run = self::Run.new
-    result = run.import ## returns formatted urls.
+  ## Args must include :data or :file_path, else seeds will run by default.
+  def self.sanitize(args={})
+    keys = args.compact.keys
+    input = { stats: nil, file_path: nil, data: nil }.merge(args)
+
+    ## Grabs seeds if :data or :file_path empty.
+    unless (keys & [:data, :file_path]).any?
+      ## Toggle data[:file_path] & data[:data] to test csv parsing or data hashes.
+      # input[:file_path] = Seed.new.grab_seed_file_path
+      input[:data] = Seed.new.grab_seed_hashes
+
+      ## For Testing: Pollute_seeds adds non-utf8 chars to each line.
+      input[:pollute_seeds] = true
+    end
+
+    ## Sanitizes input hash, then merges results to original input hash, and returns as sanitized_data.
+    input.merge!(Utf8Sanitizer::UTF.new.validate_data(input))
   end
 
+
 end

data/utf8_sanitizer.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/4rlm/utf8_sanitizer'
   spec.license       = 'MIT'
 
-  spec.summary       = "Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings."
-  spec.description   = "Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.\n Example: ABC Au\\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\\n\\r\\n  =>  ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234"
+  spec.summary       = "Removes invalid UTF8 characters & extra whitespace from csv or strings."
+  spec.description   = "Removes invalid UTF8 characters & extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.\n Example: ABC Au\\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\\n\\r\\n  =>  ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234"
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = "https://rubygems.org"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 0.0.2.pre.rc.02
+  version: 0.0.2.pre.rc.03
 platform: ruby
 authors:
 - Adam Booth
@@ -181,7 +181,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.11.3
 description: |-
-  Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.
+  Removes invalid UTF8 characters & extra whitespace (carriage returns, new lines, tabs, spaces, etc.) from csv, or strings.
    Example: ABC Au\xC1tos,123 E Main St,Anytown,TX,75142,(888) 555-1234\n\r\n  =>  ABC Autos,123 E Main St,Anytown,TX,75142,(888) 555-1234
 email:
 - 4rlm@protonmail.ch
@@ -207,12 +207,10 @@ files:
 - lib/utf8_sanitizer/csv/seeds_mini.csv
 - lib/utf8_sanitizer/csv/seeds_mini_10.csv
 - lib/utf8_sanitizer/csv/seeds_mini_2_bug.csv
-- lib/utf8_sanitizer/run.rb
 - lib/utf8_sanitizer/seed.rb
 - lib/utf8_sanitizer/utf.rb
 - lib/utf8_sanitizer/version.rb
 - utf8_sanitizer.gemspec
-- utf8_sanitizer_gemspec_orig.txt
 homepage: https://github.com/4rlm/utf8_sanitizer
 licenses:
 - MIT
@@ -237,6 +235,5 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: 'Still in BETA: Removes invalid UTF8 characters, and extra whitespace (carriage
-  returns, new lines, tabs, spaces, etc.) from csv, or strings.'
+summary: Removes invalid UTF8 characters & extra whitespace from csv or strings.
 test_files: []

data/lib/utf8_sanitizer/run.rb

@@ -1,31 +0,0 @@
-
-module Utf8Sanitizer
-  class Run
-
-    def initialize
-      @crm_data = {}
-    end
-
-
-    def import(args={})
-      @crm_data = { stats: nil, data: nil, file_path: nil, criteria: nil }
-      @crm_data.merge!(args)
-      keys = args.compact.keys
-
-      unless (keys & [:data, :file_path]).any?
-        @crm_data[:file_path] = Seed.new.grab_seed_file_path
-        # @crm_data[:data] = Seed.new.grab_seed_hashes
-        @crm_data[:pollute_seeds] = true
-        unless keys.include?(:criteria)
-          @crm_data[:criteria] = Seed.new.grab_seed_web_criteria
-        end
-      end
-
-      utf_result = Utf8Sanitizer::UTF.new.validate_data(@crm_data)
-      @crm_data.merge!(utf_result)
-    end
-
-
-
-  end
-end

data/utf8_sanitizer_gemspec_orig.txt

@@ -1,36 +0,0 @@
-
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "utf8_sanitizer/version"
-
-Gem::Specification.new do |spec|
-  spec.name          = "utf8_sanitizer"
-  spec.version       = Utf8Sanitizer::VERSION
-  spec.authors       = ["Adam Booth"]
-  spec.email         = ["4rlm@protonmail.ch"]
-
-  spec.summary       = %q{TODO: Write a short summary, because RubyGems requires one.}
-  spec.description   = %q{TODO: Write a longer description or delete this line.}
-  spec.homepage      = "TODO: Put your gem's website or public repo URL here."
-  spec.license       = "MIT"
-
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  if spec.respond_to?(:metadata)
-    spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
-  end
-
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-end