bots 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a8018e0d8575a415699c41dcba236e3c4f400e8132111093e421ac02e792548
-  data.tar.gz: 4b876044081e94743d1b719c53424331d44bb200a8bdcfddd7d78562209eeed3
+  metadata.gz: ef6ff9fa026dd9e1f1c750b5963c957a4deacf5413f92a3dc8fd1f0584305898
+  data.tar.gz: e652c088971c2fd9c97c12902d09ce7ed4933da36da6e010f2f3ffae1abdb292
 SHA512:
-  metadata.gz: 326e82a582132f2d267e906df73aad0d812f9fc4fe00c8af2ba9ef6cd93a174ae2004719ec76f5cc6b018da0eda1b2cd891dd51f7df7adaad6044da773f207ec
-  data.tar.gz: 2a4944c21854faee39f81b81004fb63baf3f270b2374f081b98ca25b4f7695af80184b1449dff38207d4c21b7f34b524d59c997931f2db2743df42c56a714875
+  metadata.gz: 1c8fce3accf4fc84701706355880bd59cc1cd5a16dfbc4a05bbdb30faa492c7f212f4a22ef917b81e7b8ba5f0c6caaa911f23bef1df4f4f23868ab225673adff
+  data.tar.gz: 3942240b8cff2bdbfdb003912af8da5c26f6b117a3744becbff6a781f77f9eb32d2c5e0aef1439edc6b288ec0ff359f5e45fec68e21db8d6cca78c8d36cf04dc

data/lib/scraper.rb

@@ -106,8 +106,8 @@ module BlackStack
                 self.get_links_from_sitemap(l)
             end # def get_links
         
-            def find_keywords(a, stop_at=50, l=nil)
-                ret = []
+            def find_keywords(a, stop_at=50, stop_on_first_link_found=false, l=nil)
+                pages = []
                 l = BlackStack::DummyLogger.new(nil) if l.nil?
                 # iterate the links
                 j = 0
@@ -119,25 +119,33 @@ module BlackStack
                         # get the page
                         page = self.get(link)
                         # get page body content in plain text
-                        s = Timeout::timeout(5) { Nokogiri::HTML(page.body).text }
+                        title = page.title
+                        s = Timeout::timeout(5) { page.search('body').text }
+                        # add the link to the results of no-keyword
+                        hpage = { :url => link, :title => title, :html => page.body, :keywords => [] }
+                        pages << hpage
                         # iterate the keywords
                         i = 0
+                        match = false
                         a.each { |k|
                             # find the keyword
-                            if s =~ /#{Regexp.escape(k)}/i
+                            match = ( s =~ /#{Regexp.escape(k)}/i )
+                            hpage[:keywords] << k if match
+                            # count the number of links with match
+                            # break if only 1 link is needed
+                            if match 
                                 i += 1
-                                ret << link if ret.select { |link| link == link }.empty?
-                                break
+                                break if stop_on_first_link_found
                             end # if
                         } # each
-                        break if ret.size > 0
+                        break if match && stop_on_first_link_found
                         l.logf i == 0 ? 'no keywords found'.yellow : "#{i} keywords found".green # find_keywords
                     rescue => e
                         l.logf "Error: #{e.message.split("\n").first[0..100]}".red # get_links
                     end # begin
                 } # each
                 # return
-                ret
+                pages
             end
         
         end # class Scraper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bots
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-16 00:00:00.000000000 Z
+date: 2023-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simple_cloud_logging