store_list_scraper 0.1.0

data/lib/storeListScraper/cli.rb

@@ -0,0 +1,96 @@
+
+class ListScraper::CLI
+
+    def start #greet user with menu
+        puts "\nHello user! ヽ(‘ ∇‘ )ノ"
+        menu
+    end
+
+    def menu
+        puts "\nWhat are we interested in accomplishing today?\n1) New Scrape\n2) View Business List\n3) Update Business List\n4) Exit Program\n"
+        case gets.strip.to_i
+        when 1
+            new_scrape
+        when 2
+            list_view
+        when 3
+            list_update
+        when 4
+            puts "\n(⌐■_■)ノ\nGoodbye friend!\n"
+            exit
+        else
+            puts "\n(ರ_ರ)\nThat is not a valid input.."
+            menu
+        end
+    end
+
+    def new_scrape
+        puts "\n(∪ ◡ ∪)\nSo you want to run a scrape?\nPlease type a company name as found in my Business List.\nOR type 'list' to view the Business List\nOr 'menu' to return to the main menu"
+        name_check
+    end
+
+    def name_check
+        input = gets.strip
+        case input
+        when "list"
+            list_view
+        when "menu"
+            menu
+        else  
+            #check business list for name
+            link = ListScraper::CSVmanager.business_list_check(input)
+            if link != nil
+                puts "\n( ﾟヮﾟ)\nFound the business! Want me to scrape a list? (y/n)"
+                confimation = gets.strip
+                confimation == 'y' ? scrape(link) : exit
+            else
+                puts "\n(●_●)\nSorry, but it doesn't look like that's on the list.\nPlease enter another name\nOR type 'list' to view my Business List\nOR type 'menu' to return to the main menu"
+                name_check
+            end
+        end
+    end
+
+    def list_view
+        puts "\nThis is a list of 27,000+ businesses,\nHow would you like to view it?\n1) select a letter group\n2) search a keyword\n3) return to main menu"
+        case gets.strip.to_i
+        when 1
+            puts "Please type letter(s) you want at the START of the business name:"
+            ListScraper::CSVmanager.list_view_by_letter(gets.strip)
+            menu
+        when 2
+            puts "Please type the keyword as you expect to see it in the business name.\n(example: 'Jimmy's Pizza' contains 'Pizza'"
+            ListScraper::CSVmanager.list_view_by_search(gets.strip)
+            menu
+        when 3
+            menu
+        else
+            puts "\n(ರ_ರ)\nThat is not a valid input.."
+            list_view
+        end
+    end
+
+    def scrape(link)
+        a = ListScraper::LocationScraper.new("#{ListScraper::LocationScraper.base}#{link}")
+        a.page_scrape(a.link)
+        a.clean_out
+        puts "\n( ◕‿◕)\nI found #{a.loc_pages.length} locations across #{a.state_pages.length}for this business.\nWould you like to export? (y/n)"
+        confirmation = gets.strip
+        if confirmation == 'y'
+            puts "\n(°ロ°)☝\nWhat would you like to name the file?"
+            fileName = gets.strip
+            puts "...\n( ◉_◉)\nExporting now, this can take awhile. I will alert when done."
+            a.create_stores
+            ListScraper::CSVmanager.locations_export(fileName)
+            puts "Export Completed Successfully!"
+        else
+            puts "\n(⌐■_■)ノ\nI hope you have a great day!\nGoodbye"
+            exit
+        end
+    end
+
+    def list_update
+        puts "\n⊙﹏⊙\nRunning update, this could take several minutes.."
+        ListScraper::UpdateScraper.new
+    end
+
+end

data/lib/storeListScraper/csv_manage.rb

@@ -0,0 +1,28 @@
+
+class ListScraper::CSVmanager
+
+    def self.locations_export(name)
+        c = CSV.open("#{name}.csv", "w")
+        c << ["IDnum", "Address", "City", "State", "ZIP"] #headers
+        ListScraper::Store.all.each do |loc|
+            c << [loc.idnum, loc.address, loc.city, loc.state, loc.zip]
+        end
+        c.close()
+    end
+
+    def self.business_list_check(company)
+        #check for business existance on file
+        h = CSV.read("./lib/storeListScraper/business_list.csv").find {|row| row[0] == "#{company}"}
+        h != nil ? h[1] : nil
+    end
+
+    def self.list_view_by_letter(letter)
+        g = CSV.read("./lib/storeListScraper/business_list.csv").select {|row| row[0].downcase.start_with?("#{letter.downcase}")}
+        g.each {|item| puts item[0]}
+    end
+
+    def self.list_view_by_search(word)
+        g = CSV.read("./lib/storeListScraper/business_list.csv").select {|row| row[0].downcase.include?("#{word.downcase}")}
+        g.each {|item| puts item[0]}
+    end
+end

data/lib/storeListScraper/location_scraper.rb

@@ -0,0 +1,103 @@
+
+class ListScraper::LocationScraper 
+    attr_accessor :link, :state_pages, :city_pages, :loc_pages
+    @@base = 'https://storefound.org'
+
+    def initialize(link)
+        @link = link
+        @state_pages = []
+        @city_pages = []
+        @loc_pages = []
+    end
+
+    def self.base
+        @@base
+    end
+
+    def page_scrape(page, type = 'all')
+        begin
+            doc = Nokogiri::HTML5(URI.open(page))
+            doc.css(".main-block a").each do |lk| #pull all links from main body 
+                j = lk.attribute("href").text #look at only the url text
+                case type
+                when 'all'
+                    case j.split("/").length
+                    when 3 
+                        @state_pages << j
+                    when 4
+                        @city_pages << j
+                    when 5
+                        @loc_pages << j
+                    end
+                when 'State'
+                    case j.split("/").length
+                    when 4
+                        @city_pages << j
+                    when 5
+                        @loc_pages << j
+                    end
+                when 'City'
+                    case j.split("/").length
+                    when 5
+                        @loc_pages << j
+                    end
+                end
+            end
+        rescue #OpenURI::HTTPError => e
+        end
+      end
+
+    #when running a store, the first table of links will
+    #only ever be states, cities, or locations. On state pages,
+    #cities will be picked up at the bottom and need to be removed
+    #clean_out will determine next steps and de-duplicate arrays
+    #clean_out can only be used after the first pass
+      def clean_out
+        @state_pages.uniq!
+        @city_pages.uniq!
+        @loc_pages.uniq!
+        if @state_pages.length > 0 #if state links avaliable, clean other arrays and scrape each state
+            @city_pages.clear
+            @loc_pages.clear
+            linked_page_scrape(@state_pages,'State')
+            linked_page_scrape(@city_pages,'City')
+        elsif @city_pages.length > 0
+            linked_page_scrape(@city_pages,'City')
+        end
+      end
+
+      def linked_page_scrape(array,type)
+        total = array.length
+        i = 0
+        array.each do |page|
+            i += 1
+            page_scrape("#{@@base}#{page}","#{type}")
+            print "#{((i.to_f/total.to_f)*100).round(2)}% | #{type} Progress: #{i}/#{total}\r"
+        end
+        puts "#{((i.to_f/total.to_f)*100).round(2)}% | #{type} Progress: #{i}/#{total}\r"
+      end
+
+      def create_stores
+        total = @loc_pages.length
+        i = 1
+        @loc_pages.each do |loc|
+            begin
+                st = Nokogiri::HTML5(URI.open("#{@@base}#{loc}"))
+                j = st.css("li span")
+                info = {
+                    idnum: i,
+                    address: j[0].text,
+                    city: j[1].text,
+                    state: j[2].text,
+                    zip: j[3].text
+                }
+                ListScraper::Store.new(info)
+                print "#{((i.to_f/total.to_f)*100).round(2)}% | Progress: #{i}/#{total}\r"
+                i += 1
+            rescue
+                next
+            end
+        end
+      end
+
+end

data/lib/storeListScraper/store.rb

@@ -0,0 +1,19 @@
+
+class ListScraper::Store  
+    attr_accessor :idnum, :address, :city, :state, :zip 
+    @@all = []
+
+    def initialize(idnum:,address:,city:,state:,zip:)
+        @idnum = idnum
+        @address = address
+        @city = city
+        @state = state
+        @zip = zip
+        @@all << self
+    end
+
+    def self.all
+        @@all
+    end
+
+end

data/lib/storeListScraper/update_scraper.rb

@@ -0,0 +1,51 @@
+
+class ListScraper::UpdateScraper 
+    attr_accessor :letters, :pages, :list
+    attr_reader :base
+
+    def initialize
+        @base = 'https://storefound.org/'
+        @letters = [] #array of links for each letter group
+        @pages = [] #array to store pages for each letter
+        File.delete('./lib/storeListScrapper/business_list.csv') if File.exist?('./lib/storeListScrapper/business_list.csv')
+        @list = CSV.open("./lib/storeListScrapper/business_list.csv", "w")
+        @list << ["Company Name", "link"] #headers
+        update
+    end
+
+    def update
+        letters_scrape
+        @letters.each do |letter|
+            pages_scrape(letter)
+            update_business_list
+        end
+        puts "Successfully updated!!"
+    end
+
+    def letters_scrape
+        #scrape all main links for letters group
+        doc = Nokogiri::HTML5(URI.open('https://storefound.org/store/starts-a/page-1'))
+        doc.css('.letter-block a').each do |lk|
+            @letters << lk.attribute('href').text
+        end
+    end
+
+    def pages_scrape(letter_link)
+        #scrape all page links for each letter group
+        @pages.clear
+        doc = Nokogiri::HTML5(URI.open("#{@base}#{letter_link}"))
+        doc.css('.pagination a').each do |lk|
+            @pages << lk.attribute('href').text
+        end
+    end
+
+    def update_business_list
+        #scrape all business names and corresponding links
+        @pages.each do |lk|
+            doc = Nokogiri::HTML5(URI.open("#{@base}#{lk}"))
+            j = doc.css('.main-block .col-half a').each do |biz|
+                @list << [biz.text, biz.attribute('href').text]
+            end
+        end
+    end
+end

data/lib/storeListScraper/version.rb

@@ -0,0 +1,4 @@
+
+module ListScraper
+    VERSION = "0.1.0"
+  end

data/sig/store_list_scraper.rbs

@@ -0,0 +1,4 @@
+module ListScraper
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/store_list_scraper.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "lib/storeListScraper/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "store_list_scraper"
+  spec.version = ListScraper::VERSION
+  spec.authors = ["itsmattpaw"]
+  spec.email = ["itsmattpaw@gmail.com"]
+
+  spec.summary = "Scrape Store Address lists from StoreFound.org."
+  spec.description = "Scrape Store Address lists from StoreFound.org and export into a CSV file for distribution or use withe other softwares such as ESRI ARC GIS."
+  spec.homepage = "https://github.com/itsmattpaw/store_list_scraper"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  #spec.metadata["allowed_push_host"] = "https://example.com"
+
+  #spec.metadata["homepage_uri"] = spec.homepage
+  #spec.metadata["source_code_uri"] = "http://www.bob.com"
+  #spec.metadata["changelog_uri"] = "http://www.bob.com"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "pry"
+  spec.add_dependency 'nokogiri', '~> 1.13'
+  spec.add_dependency "open-uri"
+  spec.add_dependency "csv"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: store_list_scraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- itsmattpaw
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-03-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: open-uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scrape Store Address lists from StoreFound.org and export into a CSV
+  file for distribution or use withe other softwares such as ESRI ARC GIS.
+email:
+- itsmattpaw@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/List_Scraper.rb
+- lib/storeListScraper/business_list.csv
+- lib/storeListScraper/cli.rb
+- lib/storeListScraper/csv_manage.rb
+- lib/storeListScraper/location_scraper.rb
+- lib/storeListScraper/store.rb
+- lib/storeListScraper/update_scraper.rb
+- lib/storeListScraper/version.rb
+- sig/store_list_scraper.rbs
+- store_list_scraper.gemspec
+homepage: https://github.com/itsmattpaw/store_list_scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.9
+signing_key:
+specification_version: 4
+summary: Scrape Store Address lists from StoreFound.org.
+test_files: []