fluffix 0.0.0

data/features/bank.feature

@@ -0,0 +1,13 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have bank's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Bank |
+    | Banking |
+    | Bankers |
+    And I combine them
+    Then the cleansed version should match the name

data/features/company.feature

@@ -0,0 +1,19 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have bank's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | and Company |
+    | and Co |
+    | & Company |
+    | & Co |
+    | + Company |
+    | + Co |
+    | Trust Company |
+    | Trust Co |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/cooperative.feature

@@ -0,0 +1,15 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have Coop
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Cooperative |
+    | Co-operative |
+    | Coop |
+    | Co-op |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/corporation.feature

@@ -0,0 +1,47 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have Corporations
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Corporation |
+    | Corp |
+    | A Corp |
+    | A Professional Corp |
+    | A Pro Corp |
+    | A Pro. Corp |
+    | A Pro. Corporation |
+    | A Chartered Corp |
+    | A Chartered Corporation |
+    | A Service Corp |
+    | A Service Corporation |
+    | A Svc Corp |
+    | A Svc. Corp |
+    | A Svc Corporation |
+    | A Svc. Corporation |
+    | A Nonprofit Corp |
+    | A Non Profit Corp |
+    | A Not for Profit Corp |
+    | Professional Corp |
+    | Pro Corp |
+    | Pro. Corp |
+    | Pro. Corporation |
+    | Chartered Corp |
+    | Chartered Corporation |
+    | Service Corp |
+    | Service Corporation |
+    | Svc Corp |
+    | Svc. Corp |
+    | Svc Corporation |
+    | Svc. Corporation |
+    | Nonprofit Corp |
+    | Non Profit Corp |
+    | Not for Profit Corp |
+    | Nonprofit Corporation |
+    | Non Profit Corporation |
+    | Not for Profit Corporation |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/inc.feature

@@ -0,0 +1,12 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have Inc's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Incorporated |
+    | Inc |
+    And I combine them
+    Then the cleansed version should match the name

data/features/l3c.feature

@@ -0,0 +1,15 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have an L3C
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | L3C |
+    | L.3.C |
+    | 13C |
+    | 1.3.C |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/ltd.feature

@@ -0,0 +1,15 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have Ltd
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Limited |
+    | LTD |
+    | Limited Liability co |
+    | Limited Liability company |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/pac.feature

@@ -0,0 +1,14 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have P.A.C's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | PA |
+    | P.A |
+    | PC |
+    | P.C |
+    And I combine them
+    Then the cleansed version should match the name

data/features/plc.feature

@@ -0,0 +1,17 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have an PLC
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | P.L.C |
+    | PLC |
+    | P.L.L.C |
+    | PLLC |
+    | L.L.C |
+    | LLC |
+
+    And I combine them
+    Then the cleansed version should match the name

data/features/sc.feature

@@ -0,0 +1,14 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have S.C's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    | prototype |
+    | prototype interactive laboratories (dead in '99) |
+    And I have these suffixes
+    | SC |
+    | S.C |
+    And I combine them
+    Then the cleansed version should match the name

data/features/simple.feature

@@ -0,0 +1,26 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have Inc's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | Deposit |
+    | Credit Union |
+    | Societa Per Azioni |
+    | Club |
+    | Foundation |
+    | Fund |
+    | Institute |
+    | Society |
+    | Union |
+    | Syndicate |
+    | Church |
+    | College |
+    | University |
+    | Chartered |
+    | League |
+    | Committee |
+    And I combine them
+    Then the cleansed version should match the name

data/features/spa.feature

@@ -0,0 +1,12 @@
+Feature: cleanse a company name of suffix
+  In order to have clean company names that we could compare
+
+  Scenario: I have SPA's
+    Given I have these names
+    | Prototype |
+    | Prototype Interactive Laboratories (dead in '99) |
+    And I have these suffixes
+    | S.P.A |
+    | SPA |
+    And I combine them
+    Then the cleansed version should match the name

data/features/step_definitions/sluffix_steps.rb

@@ -0,0 +1,24 @@
+require "fluffix"
+require "awesome_print"
+
+# generics
+Given(/^I have these names$/i) do |names|
+  names = names.raw.flatten
+  @names = names + names.map(&:downcase)
+end
+
+Given(/^I have these suffixes$/i) do |suffixes|
+  suffixes = suffixes.raw.flatten
+  @suffixes = suffixes + suffixes.map(&:downcase)
+  @suffixes = @suffixes + @suffixes.map{ |s| "#{s}." }
+end
+
+Given(/^I combine them$/i) do
+  @examples = @names.product(@suffixes).map{ |i| [i.join(" "), i.first] }
+end
+
+Then(/^The cleansed version should match the name$/i) do
+  @examples.each do |example|
+    Fluffix::US.cleanse(example.first).should eq(example.last)
+  end
+end

data/features/support/env.rb

@@ -0,0 +1,29 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'fluffix'
+
+require 'rspec/expectations'

data/lib/fluffix.rb

@@ -0,0 +1,21 @@
+require "yaml"
+require "awesome_print"
+
+module Fluffix
+  
+  RULES = YAML.load(File.open("entity_types_edited.yml"))[0].flatten.uniq.sort{ |a, b| b.size <=> a.size }.map{ |r| Regexp::new(" #{r}\.?$", Regexp::IGNORECASE) }
+  
+  class US
+    def self.cleanse(text)
+      raise("Must quack like a string") unless text.respond_to?(:to_s)
+      RULES.each do |rule|
+        # should return the text in the original form and should handle / ignore punctuation
+        if text =~ rule
+          result = text.sub(rule, '')
+          return result
+        end
+      end
+      return text
+    end
+  end
+end

data/lib/tasks/get.rb

@@ -0,0 +1,86 @@
+# https://en.wikipedia.org/wiki/Types_of_business_entity
+
+# pull the data, and through a series of techniques, tip out the likely business suffixes
+require "nokogiri"
+require "open-uri"
+require "awesome_print"
+require "yaml"
+
+task :basic_get do
+  html = open("https://en.wikipedia.org/wiki/Types_of_business_entity")
+  dom = Nokogiri::HTML.parse(html)
+
+  out_data = []
+  dom.css("ul li").each do |tag|
+    parts = tag.text.split("≈")
+    if parts.length > 1
+      p2 = parts.inject([]) do |r, i|
+        i.split(/[\n]/).each do |j|
+          j.strip!
+          j.downcase!
+          j.sub!(/\(\s?i.e.\s[^\)]+\)/, "")
+          if j.end_with?(":")
+            j.gsub!(/\(([^\)]+)\)/, '~~~\1')
+            j = j.split("~~~").map{ |item| item.sub(/[\s\:]+$/, '') }
+            r << j
+          end
+        end
+        r
+      end
+      out_data << p2
+    end
+  end
+
+  File.open("entity_types.yml", "w") do |f|
+    f.write(out_data.to_yaml)
+  end
+end
+
+task :TLD do 
+  # https://github.com/alexrabarts/tld
+  # get a list of all TLD's if that one wont work
+end
+
+task :open_company_data do
+  # http://index.okfn.org/dataset/companies/
+  # http://download.companieshouse.gov.uk/en_output.html
+  # pull a whole bunch of data in the hopes that it gives us clues about this
+
+  # GET http://download.companieshouse.gov.uk/BasicCompanyData-2015-12-01-part1_5.zip
+
+  # unzip it
+
+  # 
+  require "csv"
+  output = Hash.new(0)
+  CSV.foreach("/Users/johnjansen/Downloads/BasicCompanyData-2015-12-01-part1_5.csv", headers:true) do |row|
+    parts = row["CompanyName"].split(" ").map(&:strip)
+    while not parts.empty?
+      output[parts.join(" ")] += 1
+      parts.shift
+    end
+  end
+  ap output.map{ |(a, b)| [a, b * b] }.sort{ |a,b| b[1] <=> a[1] }[0..100]
+end
+
+task :edgar do
+  # https://www.sec.gov/edgar/searchedgar/ftpusers.htm
+  # pull all of edgar and figure out the highest frequency suffixes
+  # i.e. for "something incorporated"
+  # 1) reverse it => "incorporated something"
+  # 2) split it => ["incorporated", "something"]
+  # 3) ngram it from longest to shortest =>
+  #     ["incorporated", "something"]
+  #     ["incorporated"]
+  # 4) go through the entire db and count instances of ngrams =>
+  #     ["incorporated", "something"] => 1
+  #     ["incorporated"] => 1
+  # 5) suqare the frequencies and trim the long tail
+  # 6) manually clean the result
+end
+
+task :dump do
+  File.open("entity_types_edited.yml", "r") do |f|
+    ap YAML.load(f)[0].flatten.uniq.map(&:downcase).sort{ |a, b| b.size <=> a.size }
+  end
+end

data/spec/fluffix_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Fluffix" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'fluffix'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end