bots 1.0.3 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a8018e0d8575a415699c41dcba236e3c4f400e8132111093e421ac02e792548
-  data.tar.gz: 4b876044081e94743d1b719c53424331d44bb200a8bdcfddd7d78562209eeed3
+  metadata.gz: d58802b035984822506024bb8745293a9c6edf666c9ed4b69311282f226b079a
+  data.tar.gz: 250faa62467ee442ed198b36247f4d5a271dfc03f9614e0021357873bdf749a4
 SHA512:
-  metadata.gz: 326e82a582132f2d267e906df73aad0d812f9fc4fe00c8af2ba9ef6cd93a174ae2004719ec76f5cc6b018da0eda1b2cd891dd51f7df7adaad6044da773f207ec
-  data.tar.gz: 2a4944c21854faee39f81b81004fb63baf3f270b2374f081b98ca25b4f7695af80184b1449dff38207d4c21b7f34b524d59c997931f2db2743df42c56a714875
+  metadata.gz: ddb39e422226e1b490e18d364ae7ab15b6bb8c20425091f41168b32c59ef59809c769e7cf2d8628998676799a21b2cb7957aeb807a9bd1995e1658217822be65
+  data.tar.gz: f8431a7d719c30326d3c1dd69e8b80dd4f916b004c7a44b464c22d83fa5acce26cd59f0b94709b1955ff1dda6450ccd8bc7121b33368b91f92380e88a1f01e54

data/lib/scraper.rb

@@ -106,8 +106,8 @@ module BlackStack
                 self.get_links_from_sitemap(l)
             end # def get_links
         
-            def find_keywords(a, stop_at=50, l=nil)
-                ret = []
+            def find_keywords(a, stop_at=50, stop_on_first_link_found=false, l=nil)
+                pages = []
                 l = BlackStack::DummyLogger.new(nil) if l.nil?
                 # iterate the links
                 j = 0
@@ -119,25 +119,33 @@ module BlackStack
                         # get the page
                         page = self.get(link)
                         # get page body content in plain text
-                        s = Timeout::timeout(5) { Nokogiri::HTML(page.body).text }
+                        title = page.title
+                        s = Timeout::timeout(5) { page.search('body').text }
+                        # add the link to the results of no-keyword
+                        hpage = { 'page_url' => link.downcase, 'page_title' => title, 'page_html' => page.body, 'keywords' => [] }
+                        pages << hpage
                         # iterate the keywords
                         i = 0
+                        match = false
                         a.each { |k|
                             # find the keyword
-                            if s =~ /#{Regexp.escape(k)}/i
+                            match = ( s =~ /#{Regexp.escape(k)}/i )
+                            hpage['keywords'] << k if match
+                            # count the number of links with match
+                            # break if only 1 link is needed
+                            if match 
                                 i += 1
-                                ret << link if ret.select { |link| link == link }.empty?
-                                break
+                                break if stop_on_first_link_found
                             end # if
                         } # each
-                        break if ret.size > 0
+                        break if match && stop_on_first_link_found
                         l.logf i == 0 ? 'no keywords found'.yellow : "#{i} keywords found".green # find_keywords
                     rescue => e
                         l.logf "Error: #{e.message.split("\n").first[0..100]}".red # get_links
                     end # begin
                 } # each
                 # return
-                ret
+                pages
             end
         
         end # class Scraper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bots
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.5
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-16 00:00:00.000000000 Z
+date: 2023-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simple_cloud_logging