crm_formatter 1.0.7.pre.rc.1 → 2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1a9e605ef7ec90b0c88e80a32cd22fa604a9cc810fe559eb19c389c72c0e494
-  data.tar.gz: 07b4436cf1d31125e44a0bd71424b7c7c21a50954faa8e14b2792f08fd7391d9
+  metadata.gz: 96ed8a01bb47d8aac9c3bb7b95a6e8d261ecc107d6d47c8813088157978239c4
+  data.tar.gz: 88125eff101ca1ab5e5fcec3015d8cdb3f5f45571e644cfd54547ab8259f268c
 SHA512:
-  metadata.gz: 26357b48e1933ed4a421c9fc8ce1bd01a097d553e7d1810910bb7b891496f341ce456a45cff6313189adf1321e7fe95aad25bf9d4bf6471301dfa61bf4c637c3
-  data.tar.gz: bb86e8cdcc5ea5526e16d7963687e4dd058e3097bdc2b88f6b65284f5cd282ab7f6890304abf90477e1f7503c0a0023b89f48b41c994467bc0f52feffc8509b5
+  metadata.gz: 27ba92aa172d3de3813b6338ac77281f840ce7d158a29b54ffa329edf6eac4f2c6394f3608522ec78bb7fe8efaeb9fcd2f034b47e6676c2f275a61e3c39aa2b9
+  data.tar.gz: 7b497e726c925b6cf402c3efb20c0baae87adad9bfd5e18d4a024915bc2127533356aa331804282e9325e4b0055db55eb911aa5cb14ec85eb68870d2a78631e5

data/.gitignore

@@ -10,6 +10,5 @@
 crm_formatter-*.gem
 .DS_Store
 .idea/
+.xlsx
 .txt
-.csv
-!extensions.csv

data/.rspec_status

@@ -0,0 +1,7 @@
+example_id                        | status  | run_time        |
+--------------------------------- | ------- | --------------- |
+./spec/crm_formatter_spec.rb[1:1] | passed  | 0.00103 seconds |
+./spec/crm_formatter_spec.rb[1:2] | failed  | 0.0207 seconds  |
+./spec/crm_formatter_spec.rb[1:3] | passed  | 0.00086 seconds |
+./spec/crm_formatter_spec.rb[1:4] | passed  | 0.00009 seconds |
+./spec/crm_formatter_spec.rb[1:5] | unknown |                 |

data/.rubocop.yml

@@ -0,0 +1,10 @@
+inherit_from: .rubocop_todo.yml
+
+Metrics/LineLength:
+  Enabled: false
+
+AllCops:
+  DisabledByDefault: false
+
+AllCops:
+  TargetRubyVersion: 2.5.1

data/.rubocop_todo.yml

@@ -0,0 +1,188 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-05-22 12:28:19 -0500 using RuboCop version 0.56.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'crm_formatter.gemspec'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Layout/CommentIndentation:
+  Exclude:
+    - 'Rakefile'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: AllowAdjacentOneLineDefs, NumberOfEmptyLines.
+Layout/EmptyLineBetweenDefs:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 25
+# Cop supports --auto-correct.
+Layout/EmptyLines:
+  Exclude:
+    - 'Rakefile'
+    - 'lib/crm_formatter.rb'
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: empty_lines, no_empty_lines
+Layout/EmptyLinesAroundBlockBody:
+  Exclude:
+    - 'spec/crm_formatter_spec.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
+Layout/EmptyLinesAroundClassBody:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines
+Layout/EmptyLinesAroundModuleBody:
+  Exclude:
+    - 'lib/crm_formatter.rb'
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: .
+# SupportedStyles: space, no_space
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: no_space
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment.
+Layout/SpaceAroundOperators:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
+# SupportedStyles: space, no_space
+# SupportedStylesForEmptyBraces: space, no_space
+Layout/SpaceInsideBlockBraces:
+  Exclude:
+    - 'Gemfile'
+
+# Offense count: 6
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces.
+# SupportedStyles: space, no_space, compact
+# SupportedStylesForEmptyBraces: space, no_space
+Layout/SpaceInsideHashLiteralBraces:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 2
+Lint/Debugger:
+  Exclude:
+    - 'lib/crm_formatter.rb'
+
+# Offense count: 1
+# Configuration parameters: IgnoreImplicitReferences.
+Lint/ShadowedArgument:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods.
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 4
+Lint/UselessAssignment:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/BlockLength:
+  Max: 30
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 12
+
+# Offense count: 1
+Naming/AccessorMethodName:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 3
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/crm_formatter.rb'
+    - 'lib/crm_formatter/dictionary.rb'
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 1
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
+# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
+Style/HashSyntax:
+  Exclude:
+    - 'Rakefile'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/MutableConstant:
+  Exclude:
+    - 'lib/crm_formatter/version.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Exclude:
+    - 'lib/crm_formatter/wrap.rb'
+
+# Offense count: 67
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'bin/console'
+    - 'lib/crm_formatter.rb'
+    - 'lib/crm_formatter/version.rb'
+    - 'lib/crm_formatter/wrap.rb'
+    - 'menu.rb'
+    - 'spec/crm_formatter_spec.rb'
+    - 'spec/spec_helper.rb'
+
+# Offense count: 7
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 549

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at 4rlm@protonmail.ch. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: false
+
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in crm_formatter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Adam Booth
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -1,172 +1,238 @@
 
-# **CRM Formatter**
+# CRM Wrap
+#### Efficiently Reformat, Normalize, and Scrub CRM Contact Data, such as Addresses, Phones and URLs.
 
-#### Reformat and Normalize CRM Contact Data, such as Addresses, Phones and URLs.
+CRM Wrap is perfect for curating high-volume enterprise-scale web scraping, and integrates well with Nokogiri, Mechanize, and asynchronous jobs via Delayed_job or SideKick, to name a few.  Web Scraping and Harvesting often gathers a lot of junk to sift through; presenting unexpected edge cases around each corner.  CRM Wrap has been developed and refined during the past few years to focus on improving that task.
 
-**CRM Formatter** was originally designed to curate high-volume enterprise-scale asynchronous web scraping via Nokogiri, Mechanize, and Delayed_job.  Web Scraping *aka Web Harvesting / Data Mining* is notoriously unreliable *sticky* work with endless edge-cases to overcome.  Accurately, yet efficiently curating such data is a constant and evolving task, and will continue to be the core functionality of **CRM Formatter**.
-However, it also plays an integral role in routine functions of apps, like formatting, normalizing, and even scrubbing existing databases, and submitted form data before saving to the database; via model callbacks, such as `before_validation` or `before_save`.
+It's also perfect for processing API data, Web Forms, and routine DB normalizing and scrubbing processes.  Not only does it reformat Address, Phone, and Web data, it can also accept lists to scrub against, then providing detailed reports about how each piece of data compares with your criteria lists.
 
-###### The **CRM Formatter** Gem is currently in `--pre` versioning, aka **beta mode** with frequent updates. Formal tests in the gem environment are still on the way.
-However, **CRM Formatter** has been developed continuously for several years and is a reliable and integral part of a production CRM data verification app.  The process of isolating the various modules into a consolidated open source gem has just recently begun, so documentation is still limited, but is frequently being added and refined.
+The CRM Wrap Gem is currently in '--pre versioning', or 'beta mode' as the process of reorganizing these proprietary, production environment processes from their native app environment into this newly created open source gem.  Formal tests in the gem environment are still on the way, as is the documentation.  But the processes themselves have been very reliable and an integral part of a very large app dedicated to such services.  
 
 ## Getting Started
-**CRM Formatter** is compatible with Rails 4.2 and 5.0, 5.1 and 5.2 on Ruby 2.2 and later.
+CRM Wrap is compatible with Rails 4.2 and 5.0, 5.1 and 5.2 on Ruby 2.2 and later.
 
 In your Gemfile add:
-
 ```
-gem 'crm_formatter', '~> 1.0.6.pre.rc.1'
+gem 'crm_formatter', '~> 1.0.8.pre.rc.1'
 ```
-
 Or to install locally:
-
 ```
 gem install crm_formatter --pre
 ```
-
 ## Usage
+Using CRM Wrap in your app is very simple, and could be accessed from your app's concerns, , helpers, lib, models, or services, but depends on the scope, location, and size of your application and server. For simple form submission validations the model callback is typically ideal.  For database normalizing tasks the concerns, helpers, or lib is typically ideal.  For long running processes like web scraping or high volume APIs calls, like Google Linkedin, or Twitter  the lib or services might be ideal (asynchronous multithreaded even better)
 
-##### Usage is organized into three sections, Overview, Methods and Examples.
-
-### I. Overview
-
-#### 1. Access and Integration
-##### Using **CRM Formatter** in your app is very simple, and could be accessed from your app's concerns, controllers, helpers, lib, models, or services, but depends on the scope, location, and size of your application and server.    
-  * Simple form submission validations: model callback typically ideal.
-  * Database normalizing tasks: wrapper method in concerns, helpers, or lib typically ideal.
-  * Long running processes like web scraping or high volume APIs calls, like Google Linkedin, or Twitter: the lib or services might be ideal (multithreaded asynchronously even better)
-
-#### 2. Hash Response
-##### Formatted data will always be returned as a hash datatype the following key-value pairs:
-  * The originally submitted data as the first pair.
-  * Formatted data in the remaining pairs.
-  * A T/F boolean indicator pair regarding if the original and formatted data are different.
-
-#### 3. Optional Arguments *OA*
-##### A class can be instantiated with optional arguments *OA*.
-  * OA house the criteria by which you'd like to scrub your data.  
-  * Each is either 'Pos' or 'Neg', for more accurate reporting of your scrubbing results.
-  * List of available Web OA is below, and each accepts data in the hash datatype, aka 'keyword-args'.
-  * For example, you might want to know which URLs contain 'twitter', 'facebook', or 'linkedin' either to focus on developing a list of business social media links, or perhaps you want to use such a list to better avoid such links.
-  * *OA is currently only available for the Web class.*
-  * *Address OA & Phone OA will be available in a future release.*
-
-### II. Methods
-##### CRM Formatter**'s top level module is `CRMFormatter` and contains the following three classes:
-1. Address:  `CRMFormatter::Address.new`
-2. Phone:  `CRMFormatter::Address.new`
-3. Web:  `CRMFormatter::Address.new`
-
-###### Then assign the above to a variable name of your choosing.
-`addr_formatter = CRMFormatter::Address.new`
-`@addr_formatter = CRMFormatter::Address.new`
-
-###### Web accepts optional arguments *OA* as a Hash (with Key-Value pairs)
-Without OA: Instantiate normally if not using OA.
-`web_formatter = CRMFormatter::Web.new`
-
-With OA: Follow the steps to use Web OA:
-1. Available Web OA and the required Key-Value naming and datatypes.  
-* Only list the OA K-V Pairs you're using.  No need to list empty values.  It's not all or nothing. These are empty to illustrate the expected datatypes.
-
-Below is how the OA are received in the Web class at initialization.
-**3. Web Examples at the very bottom has a very detailed example including how OA can be used.**
-```
-def initialize(args={})
-  @empty_oa = args.empty?
-  @pos_urls = args.fetch(:pos_urls, [])
-  @neg_urls = args.fetch(:neg_urls, [])
-  @pos_links = args.fetch(:pos_links, [])
-  @neg_links = args.fetch(:neg_links, [])
-  @pos_hrefs = args.fetch(:pos_hrefs, [])
-  @neg_hrefs = args.fetch(:neg_hrefs, [])
-  @pos_exts = args.fetch(:pos_exts, [])
-  @neg_exts = args.fetch(:neg_exts, [])
-  @min_length = args.fetch(:min_length, 2)
-  @max_length = args.fetch(:max_length, 100)
-end
+### Class Names
+CrmFormatter contains three classes, which can be accessed like below with local or instance variables; you can name them anything you like.
 ```
+adr_formatter = CrmFormatter::Address.new
+@adr_formatter = CrmFormatter::Address.new
 
-Example: Below is the syntax for how to use OA.
-There are both Positive and Negative.  They work the same and could just be included in the same array if you prefer.  But they are intended to help you scrub data against negative criteria and for positive criteria.
-```
-oa_args = { neg_urls: %w(approv insur invest loan quick rent repair),
-            neg_links: %w(buy call cash cheap click gas insta),
-            neg_hrefs: %w(after anounc apply approved blog buy call click),
-            neg_exts: %w(au ca edu es gov in ru uk us),
-            min_length: 0,
-            max_length: 30
-          }
+ph_formatter = CrmFormatter::Phone.new
+@ph_formatter = CrmFormatter::Phone.new
 
-@web_formatter = CRMFormatter::Web.new(oa_args)
+web_formatter = CrmFormatter::Web.new
+@web_formatter = CrmFormatter::Web.new
 ```
 
-#### 1. Address Methods
-
-`get_full_address()` takes a hash of address parts then runs each through their respective formatters, then also adds an additional feature of combining them into a long full address string, and indicates if there were any changes from the original version and newly formatted.
+### Available Methods in Each Class
 
+## Address Methods
+These are the methods available to you.  You can use them a la cart, for example if you just wanted to wrap all your states, or you could combine the entire address into `get_full_address()` which will run each of the related methods for you.  It also adds an additional hash pair containing the full address as a single string.  There is also an indicator pair to report if there were any changes from the original version to the newly formatted.
 ```
-  addr_formatter = CRMFormatter::Address.new
-
+  addr_formatter = CrmFormatter::Address.new
   full_address_hash = {street: street, city: city, state: state, zip: zip}
-
   addr_formatter.get_full_address(full_address_hash)
-
   addr_formatter.format_street(street_string)
-
   addr_formatter.format_city(city_string)
-
   addr_formatter.format_state(state_string)
-
   addr_formatter.format_zip(zip)
-
   addr_formatter.format_full_address(adr = {})
-
   addr_formatter.compare_versions(original, formatted)
-
 ```
 
 #### Phone Methods
+Phone only has two methods, with a subtle but important distinction between them.  For simply formatting a known phone, use `format_phone` to convert to the normalized (555) 123-4567 wrap.  Use `validate_phone` if either your phone data has a bunch of text and special characters to remove, or if you aren't even sure that it is a phone, as it will help determine if the phone number seem legitimate.  If so, it then passes it along to `format_phone`.
+```
+  ph_formatter = CrmFormatter::Phone.new
+  ph_formatter.validate_phone(phone)
+  ph_formatter.format_phone(phone)
+```
+
+#### Web Methods
+The examples on this README are from `format_url` method.  The others are for web scraping, which will be documented in the near future.
+```
+  web_formatter = CrmFormatter::Web.new
+  web_formatter.format_url(url)
+  web_formatter.extract_path(url_path)
+  web_formatter.remove_invalid_links(link)
+  web_formatter.remove_invalid_hrefs(href)
+  web_formatter.convert_to_scheme_host(url)
+```
 
-Subtle but important distinction between 'format_phone' which simply puts a phone in any format, like 555-123-4567 into normalized (555) 123-4567, and 'validate_phone' which also uses 'format_phone' to normalize its output, but is mainly tasked with determining if the phone number seem legitimate. If you know for sure that it is a phone number, but just want to normalize then first try format_phone.  If you are doing web scraping or throwing in strings of text mixed with phones, then validate_phone might work better.
+## Examples
+#### Below are two examples using the Web `format_url(url)` method:
 
+### Example 1: 6 Example URLs Submitted:
+Custom Method to Query URLs
+```
+def self.get_urls
+  urls = %w(website.com website.business.site website website.fake website.fake.com website.com.fake)
+end
+```
+Custom Wrapper Method
+```
+def self.run_webs
+  web = CrmFormatter::Web.new
+  formatted_url_hashes = get_urls.map do |url|
+    url_hash = web.format_url(url)
+  end
+end
+```
+Results as Hash: 3/6 Reformatted due to invalid or no url extensions. 3 Reformatted and Normalized with `http://www.`  
+URL Extensions, **.com, .net, .fake** cross referenced with official IANA list.
+```
+[ {:reformatted=>true, :url_path=>"website.com", :formatted_url=>"http://www.website.com", :neg=>[], :pos=>[]},
+  {:reformatted=>false, :url_path=>"website.business.site", :formatted_url=>nil, :neg=>["error: ext.valid > 1 [business, site]"], :pos=>[]}, {:reformatted=>false, :url_path=>"website", :formatted_url=>nil, :neg=>["error: ext.none"], :pos=>[]},
+  {:reformatted=>false, :url_path=>"website.fake", :formatted_url=>nil, :neg=>["error: ext.invalid [fake]"], :pos=>[]},
+  {:reformatted=>true, :url_path=>"website.fake.com", :formatted_url=>"http://www.website.com", :neg=>[], :pos=>[]},
+  {:reformatted=>true, :url_path=>"website.com.fake", :formatted_url=>"http://www.website.com", :neg=>[], :pos=>[]}
+]
 ```
-  ph_formatter = CRMFormatter::Phone.new
 
-  ph_formatter.validate_phone(phone)
+### Example 2: 6 Real URLs with Scrubbing Feature, but same configuration as above:
+**Intentionally partially obfuscated**
+```
+urls = %w(approvXXXutosales.org autXXXartfinance.com leXXXummitautorepair.net melXXXtoyota.com norXXXastacura.com XXXmazda.com)
+```
+These results list 'neg' and 'pos', which are the criteria I was scrubbing against.  I wanted to find the URLs of franchise auto dealers and exclude ancillary URLs.
+```
+[{:reformatted=>true, :url_path=>"approvXXXutosales.org", :formatted_url=>"http://www.approvXXXutosales.org", :neg=>["neg_urls: approv"], :pos=>[]},
+ {:reformatted=>true, :url_path=>"autXXXartfinance.com", :formatted_url=>"http://www.autXXXartfinance.com", :neg=>["neg_urls: financ"], :pos=>["pos_urls: smart"]},
+ {:reformatted=>true, :url_path=>"leXXXummitautorepair.net", :formatted_url=>"http://www.leXXXummitautorepair.net", :neg=>["neg_urls: repair"], :pos=>[]},
+ {:reformatted=>true, :url_path=>"melXXXtoyota.com", :formatted_url=>"http://www.melXXXtoyota.com", :neg=>[], :pos=>["pos_urls: toyota"]},
+ {:reformatted=>true, :url_path=>"norXXXastacura.com", :formatted_url=>"http://www.norXXXastacura.com", :neg=>[], :pos=>["pos_urls: acura"]},
+ {:reformatted=>true, :url_path=>"XXXmazda.com", :formatted_url=>"http://www.XXXmazda.com", :neg=>[], :pos=>["pos_urls: mazda"]}
+ ]
+```
 
-  ph_formatter.format_phone(phone)
+## Quick Setup Guide
 
+#### Create a Wrapper with a custom Class and Method(s)
+This is just one of several ways to configure.  If you only need the gem for formatting form data, you could just create a callback method in your model, but to scrub a database or process API and Harvested data, you'll want a dedicated process so you can manage the queue, criteria, and results.  If you don't already have one, this example will show you how. Concerns, Helpers and Models might be fine for smaller tasks, but for heavier tasks Lib and Services are ideal, but depends on your specifications.
+```
+# /app/lib/start_crm.rb
 ```
+```
+class StartCrm
+  def initialize
+    @web = CrmFormatter::Web.new
+  end
 
-#### Web Methods
+  def run_webs
+    formatted_url_hashes = urls.map do |url|
+      url_hash = @web.format_url(url)
+    end
+  end
+end
+```
+You may need to edit your application config file to recognize your new class.
+```
+#/app/config/application.rb
 
+config.eager_load_paths << Rails.root.join('lib/**')
+config.eager_load_paths += Dir["#{config.root}/lib/**/"]
+```
+#### Run in Rails Console
+In this example, we'll run it in Rails Console like below, but you could also create a Rake Task and integrate it with a scheduled Cron Job.  You could also run the process through your contoller actions in a GUI. If accessing through the front end, you might want to do it asynchronously with gems like Delayed_job or SideKick so you can free-up your controllers and prevent your front end from freezing while waiting for the job to complete; if running very large tasks.
+```
+2.5.1 :001 > StartCrm.new.run_webs
 ```
-  web_formatter = CRMFormatter::Web.new
+#### Instance vs Class Methods in your Wrapper
+In the above example, `run_webs` is an instance method, but a class method `self.run_webs` could work well too, like the example below.  At lease in the early stages, this is a little easier if you keep running it in Rails C, because not requiring initializing means less to type to call it.  Next you could setup your class with various methods to assist your process, like so:
+```
+class StartCrm
+  def self.run_webs
+    web = CrmFormatter::Web.new
 
-  web_formatter.format_url(url)
+    formatted_url_hashes = query_accounts.map do |act|
+      url_hsh = web.format_url(act.url)
 
-  web_formatter.extract_link(url_path)
+      if url_hash[:reformatted]
 
-  web_formatter.remove_invalid_links(link)
+        act_hsh = { url: url_hsh[:formatted_url],
+                    url_sts: url_hsh[:formatted_url],
+                    scrub_date: Time.now
+                  }
+      else
+        act_hsh = { scrub_date: Time.now }
+      end
 
-  web_formatter.remove_invalid_hrefs(href)
+      act.update(act_hsh)
+    end
+  end
 
-  web_formatter.convert_to_scheme_host(url)
+  def self.query_accounts
+    accounts = Account.where(url_sts: 'Invalid').limit(50)
+  end
+end
+```
 
+#### Data Response in a Hash
+CRM Wrap returns data as a hash, which includes your original unaltered data you submitted, the formatted data, a T/F boolean indicator regarding if the original and formatted data are different, and for some methods, negs and pos regarding your criteria to scrub against.  In the above example, the returned data from each submitted url would resemble the one below.
+```
+# format_url method returns data like below this example...
+# url_hash = {:reformatted=>false,
+    :url_path=>"https://www.steXXXXXXmitsubishiserviceandpartscenter.com",
+    :formatted_url=>"https://www.steXXXXXXmitsubishiserviceandpartscenter.com",
+    :neg=>["neg_urls: parts, rv, service"],
+    :pos=>["pos_urls: mitsubishi"]
+  }
 ```
 
-### III. Examples
+#### Optional Arguments OA
+A class can be instantiated with optional arguments 'OA', to load your criteria to scrub against. Only list the OA K-V Pairs you're using.  No need to list empty values.  It's not all or nothing. These are empty to illustrate the expected datatypes.
+**OA is currently only available for the Web class, but will soon be available in the Address & Phone classes.**
+
+Below is how the OA are received in the Web class at initialization.
+```
+def initialize(args={})
+  @empty_oa = args.empty?
+  @pos_urls = args.fetch(:pos_urls, [])
+  @neg_urls = args.fetch(:neg_urls, [])
+  @pos_links = args.fetch(:pos_links, [])
+  @neg_links = args.fetch(:neg_links, [])
+  @pos_hrefs = args.fetch(:pos_hrefs, [])
+  @neg_hrefs = args.fetch(:neg_hrefs, [])
+  @pos_exts = args.fetch(:pos_exts, [])
+  @neg_exts = args.fetch(:neg_exts, [])
+  @min_length = args.fetch(:min_length, 2)
+  @max_length = args.fetch(:max_length, 100)
+end
+```
+
+Below is the syntax for how to use OA. Positive and Negative options available, and essentially function the same, but allow additional options for scrubbing data.
+```
+oa_args = { neg_urls: %w(approv insur invest loan quick rent repair),
+            neg_links: %w(buy call cash cheap click gas insta),
+            neg_hrefs: %w(after anounc apply approved blog buy call click),
+            neg_exts: %w(au ca edu es gov in ru uk us),
+            min_length: 0,
+            max_length: 30
+          }
+@web_formatter = CrmFormatter::Web.new(oa_args)
+```
+
+### III. Detailed Examples
 Some of the examples are excessively verbose to help illustrate the datatypes and processes.  Here are a few guidelines and tips:
-**3. Web Examples at the very bottom is the most detailed and recent.  It might be a good place to start.**
-*These are just examples below, not strict usage guides ...*
 
-#### 1. Address Examples
+*These are just examples, not strict usage guides ...*
 
+#### 1. Address Examples
 ```
 def self.run_adrs
 
-  crm_address_formatter = CRMFormatter::Address.new
+  crm_address_formatter = CrmFormatter::Address.new
 
   contacts = Contact.where.not(full_address: nil)
 
@@ -184,11 +250,9 @@ end
 ```
 
 #### 2. Phone Examples
-
-In the phone example, format_all_phone_in_my_db could be a custom wrapper method, which when called by Rails C or from a front end GUI process, could grab all phones in db meeting certain criteria to be scrubbed. The results will always be in hash format, such as below.... phone_hash  
-
+In the phone example, format_all_phone_in_my_db could be a custom wrapper method, which when called by Rails C or from a front end GUI process, could grab all phones in db meeting certain criteria to be scrubbed. The results will always be in hash wrap, such as below.... phone_hash  
 ```
-@crm_phone = CRMFormatter::Phone.new
+@crm_phone = CrmFormatter::Phone.new
 
 def self.format_all_phone_in_my_db
   phones_from_contacts = Contacts.where.not(phone: nil)
@@ -199,15 +263,11 @@ def self.format_all_phone_in_my_db
 
 end
 
-phone_hash = { phone: 555-123-4567, valid_phone: (555) 123-4567, phone_edit: true }
+phone_hash = { phone: 555-123-4567, phone_f: (555) 123-4567, phone_status: true }
 ```
 
 #### 3. Web Examples
-
-The steps below will show you an option for how you could integrate larger processes in your app.  
-1. Create a wrapper method you can call from an action or Rails C.  In this example, a new class was also created in Lib for that purpose, as there could be related methods to create.
-* These examples only include `CRMFormatter::Web.new.format_url(url)` method.  There are several additional methods available to you.  Documentation is on the way, but in the mean time, try out the below example, then play around with the others too.   
-
+The steps below will show you an option for how you could integrate larger processes in your app.  Create a wrapper method you can call from an action or Rails C. In this example, a new class was also created in Lib for that purpose, as there could be related methods to create.  
 ```
 # /app/lib/start_crm.rb
 
@@ -216,7 +276,7 @@ class StartCrm
   ##Rails C: StartCrm.run_webs
   def self.run_webs
     oa_args = get_args
-    web = CRMFormatter::Web.new(oa_args)
+    web = CrmFormatter::Web.new(oa_args)
 
     formatted_url_hashes = get_urls.map do |url|
       url_hash = web.format_url(url)
@@ -227,15 +287,14 @@ class StartCrm
 
 end
 ```
-2. Make sure to modify your application config file to recognize your new class.
-
+Application Config
 ```
 #/app/config/application.rb
 
 config.eager_load_paths << Rails.root.join('lib/**')
 config.eager_load_paths += Dir["#{config.root}/lib/**/"]
 ```
-3. Create your db query or put together a list of URLs to process, along with any OA to include.  The below example is very verbose, but designed to be helpful. In reality, you might have various criteria saved in the db rather than writing it out.
+Create your db query or put together a list of URLs to process, along with any OA to include.  The below example is very verbose, but designed to be helpful. In reality, you might have various criteria saved in the db rather than writing it out.
 In this example, we have auto dealer URLs.  In this process, we're focusing on franchise dealers.
 ```
 def self.get_args
@@ -251,48 +310,46 @@ def self.get_urls
   urls = ["https://www.stevXXXXXXmitsubishiserviceandpartscenter.com", "https://www.perXXXXXXchryslerjeepcenterville.com", "http://www.peXXXXXXchryslerjeepcenterville.com", "http://www.colXXXXXXchryslerdodgejeepram.com"]
 end
 ```
-4. Run your class and wrapper method in Rails C.  By creating the wrapper method, you have set up the entire process to run like a runner.  In reality, you might have several different criteria accessible from a GUI or even running in Cron Jobs.
-
-`2.5.1 :001 > StartCrm.run_webs`
-
-5. Results are always in a Hash, like below.  The URLs are slightly obfuscated out of respect (it's not a bug).  These are examples from a large DB that runs on a loop 24/7 and gets to each organization about once a week, so it's already pretty well up to date, so there aren't any big changes below, but there are still a few things to point out.
-
-* `:is_reformatted` indicates T/F if url_path and `:formatted_url` differ.  If False, then it means they are the same, or the `:url_path` had significant errors which prevented it from being formatted, thus `:formatted_url` would be nil in such a case.  The reality is that you might have some URLs that are so far off that, that they can't be reliably reformatted, so better to only let them pass if we are confident that they are reliable.
-
-* `:url_path` is the url originally submitted by the client.  It can include directory links on the end too, '/careers/, '/about-us/', etc.
-
-* `:formatted_url` is the formatted version of `:url_path`.  It will be stripped of additional paths, '/deals/', '/staff/', etc.  Also, often times people ommit 'http://:' and 'www' in CRMs.  This can sometimes cause errors for users or Mechanized Web Scrapers.  So, those will always be included to ensure consistency.  In our production app we follow up the formatting with url redirect following, which our configurations require the entire path, so it will always be included.  The redirect following gem is already being worked on and will be released as an additional gem shortly.
-
-* `:neg` is an array of all the errors and negative, undesirable criteria to scrub against.  If you include the criteria in OA `neg_urls:`, like above, it will automatically scrub and report.  Regardless, any errors will also be included in there.  So, if the url was not ultimately formatted, there will be details regarding why in `:neg`.
-
-* `:pos` is the opposite, which highlights positive criteria you might be looking for.  It too is available in OA via `pos_urls:`, like above.
-
+Run your class and wrapper method in Rails C.  By creating the wrapper method, you have set up the entire process to run like a runner.  In reality, you might have several different criteria accessible from a GUI or even running in Cron Jobs.
+```
+2.5.1 :001 > StartCrm.run_webs
+```
+Results are always in a Hash, like below.  The URLs are slightly obfuscated out of respect (it's not a bug).  These are examples from a large DB that runs on a loop 24/7 and gets to each organization about once a week, so it's already pretty well up to date, so there aren't any big changes below, but there are still a few things to point out below the code example.
 ```
-[ {:is_reformatted=>false,
+[ {:reformatted=>false,
     :url_path=>"https://www.steXXXXXXmitsubishiserviceandpartscenter.com",
     :formatted_url=>"https://www.steXXXXXXmitsubishiserviceandpartscenter.com",
     :neg=>["neg_urls: parts, rv, service"],
     :pos=>["pos_urls: mitsubishi"]},
 
-   {:is_reformatted=>false,
+   {:reformatted=>false,
     :url_path=>"https://www.perXXXXXXchryslerjeepcenterville.com",
     :formatted_url=>"https://www.perXXXXXXchryslerjeepcenterville.com",
     :neg=>["neg_urls: rv"],
     :pos=>["pos_urls: chrysler, jeep"]},
 
-   {:is_reformatted=>false,
+   {:reformatted=>false,
     :url_path=>"http://www.pXXXXXXchryslerjeepcenterville.com",
     :formatted_url=>"http://www.XXXXXXechryslerjeepcenterville.com",
     :neg=>["neg_urls: rv"],
     :pos=>["pos_urls: chrysler, jeep"]},
 
-   {:is_reformatted=>false,
+   {:reformatted=>false,
     :url_path=>"http://www.colXXXXXXchryslerdodgejeepram.com",
     :formatted_url=>"http://www.colXXXXXXchryslerdodgejeepram.com",
     :neg=>["neg_urls: rv"],
     :pos=>["pos_urls: chrysler, dodge, jeep, ram"]}
   ]
 ```
+`:reformatted` indicates T/F if url_path and `:formatted_url` differ.  If False, then it means they are the same, or the `:url_path` had significant errors which prevented it from being formatted, thus `:formatted_url` would be nil in such a case.  The reality is that you might have some URLs that are so far off that, that they can't be reliably reformatted, so better to only let them pass if we are confident that they are reliable.
+
+`:url_path` is the url originally submitted by the client.  It can include directory links on the end too, '/careers/, '/about-us/', etc.
+
+`:formatted_url` is the formatted version of `:url_path`.  It will be stripped of additional paths, '/deals/', '/staff/', etc.  Also, often times people ommit 'http://:' and 'www' in CRMs.  This can sometimes cause errors for users or Mechanized Web Scrapers.  So, those will always be included to ensure consistency.  In our production app we follow up the formatting with url redirect following, which our configurations require the entire path, so it will always be included.  The redirect following gem is already being worked on and will be released as an additional gem shortly.
+
+`:neg` is an array of all the errors and negative, undesirable criteria to scrub against.  If you include the criteria in OA `neg_urls:`, like above, it will automatically scrub and report.  Regardless, any errors will also be included in there.  So, if the url was not ultimately formatted, there will be details regarding why in `:neg`.
+
+`:pos` is the opposite, which highlights positive criteria you might be looking for.  It too is available in OA via `pos_urls:`, like above.
 
 
 ## Author