ish_models 0.0.33.75 → 0.0.33.76

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d6a34177992d8327e400da5ecd0b1761060f3ac
-  data.tar.gz: 6c3bd3d89889af78955cdc3787d2ea211a45ab39
+  metadata.gz: deeec6fc835a3c8501c2b2441ae52be690fc05f9
+  data.tar.gz: a5ca96a086af904a497f264181d868e48cf68070
 SHA512:
-  metadata.gz: 46fcdaf6bf4511d92f65d7dfffce1530840766ddf7af63c447f79965de07aeb6aad3b2ec097cb35c51333008acaafaee7bed166a5e4ca5e0166810fff5404899
-  data.tar.gz: 9ffbb1d290ec5315f53b87ebde19cad6f23be90c8bf945f1e0675d8d5d1ab24d5bdcadec7bd62a5243d417a77734f248268afe0c41d6868a7f4b8a2d0cf565fd
+  metadata.gz: f9332c01b89e1df6d74d85ad8452cd42ca3e36db82521b9451553246158b0622ec6ef26007f19208fc0c6f27c55d5d8c32d9690e2eaa59838df152584e2befbd
+  data.tar.gz: d5cea6d952ed121551506aa9b210a07d041b48a8296cf17ef48a2c97aa2efdd8fa7d54808f2f5537f8e4919a6280e79dc3c460308c3ee585c204fc40551b4fdd

data/lib/ish/crawler.rb

@@ -0,0 +1,35 @@
+
+def puts! a, b=''
+    puts "+++ #{b}"
+    puts a.inspect
+end
+
+module Ish
+  class Crawler
+
+    def self.google_first_result text
+      result = HTTParty.get( "https://www.google.com/search?q=#{text}", :verify => false )
+      r = Nokogiri::HTML(result.body)
+      website = r.css('cite')[0].text
+      website = "https://#{website}" unless website[0..3] == 'http'
+
+      puts! website, 'website'
+
+      begin
+        r = HTTParty.get( website, :verify => false )
+      rescue OpenSSL::SSL::SSLError => e
+        puts! e, 'e'
+        return { :url => website }
+      end
+
+      return { :url => website, :html => r.body }
+    end
+
+    def self.look_for_emails text
+      email_regex = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
+      result = text.scan( email_regex )
+      return result.length > 0 ? result.join(',') : nil
+    end
+
+  end
+end

data/lib/ish/lead.rb

@@ -17,6 +17,7 @@ class Ish::Lead
   field :email
   field :job_url
   field :company_url
+  field :website_html
   field :yelp_url
 
   field :company
@@ -42,5 +43,9 @@ class Ish::Lead
     write_attribute :phone, which.gsub(/\D/, '').to_i
   end
   field :phone, :type => Integer
+  field :address, :type => String
+
+  # If I crawl on 20180724, I add "20180724" here, so I don't crawl in the same way again.
+  field :extra, :type => Array, :default => []
 
 end

data/lib/ish_models.rb

@@ -26,6 +26,7 @@ require 'co_tailors/order.rb'
 # require 'co_tailors/order_item.rb' # this is required from within order.rb
 require 'co_tailors/address.rb'
 
+require 'ish/crawler.rb'
 require 'ish/gallery_name.rb'
 require 'ish/payment.rb'
 require 'ish/stock_action.rb'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ish_models
 version: !ruby/object:Gem::Version
-  version: 0.0.33.75
+  version: 0.0.33.76
 platform: ruby
 authors:
 - piousbox
@@ -122,6 +122,7 @@ files:
 - lib/gallery.rb
 - lib/gallery2.rb
 - lib/ish/campaign.rb
+- lib/ish/crawler.rb
 - lib/ish/gallery_name.rb
 - lib/ish/invoice.rb
 - lib/ish/issue.rb