google-local-results-ai-parser 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f8076de3793dabc268bc5705443e28202655e26c5a1503498767e1b8a62ff16
-  data.tar.gz: 7e3cf96add3d90d2e6174f9acda7f6cd8357f2f06792d46fc308aa80cc7fa93f
+  metadata.gz: 6935f20f427cc87b991af4c49d4db7d52004303837e075f20c73e933c5f77d90
+  data.tar.gz: 5be934605c20bcfea761f148ae3574315f56d3d2e9ea19307f891e8bc0ab7938
 SHA512:
-  metadata.gz: 91c997a71f5765a0d43cc1c4418cecc1a83b00eb3b418f4aed6802a1c5fa78391b337384433147cf5c7eb8af5455ad07289f8440c31e58585493978d4673394a
-  data.tar.gz: b0d1a462cba2a8097c0bc54ec0142e78b5d7daffe5c5ed4c663ddb7522c35391ee4ba9587f803a711f2e0fbe7395ab7d353254eda6ecab6212bbeed64c563b4d
+  metadata.gz: d34c7e63db0ed905053b98c68888f2830fdbdc1e2d2117d7353df8df2a704b66beb72dd8940f44224ceaebf6d643d8df1dce67c56a5864ed56f99d2ba6c13f78
+  data.tar.gz: 10453d1c6f06571a0d81f3c6b8bbd96659764c230d44cec898b8f8832f317d22bfa661b04c0cf4cc091f16033f68f44beb56a20505419738e1653d9d57d59a90

data/lib/google-local-results-ai-parser.rb

@@ -24,13 +24,13 @@ module GoogleLocalResultsAiParser
   end
 
   class << self
-    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       response_bodies = Parallel.map(html_parts, in_threads: html_parts.size) do |html|
-        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: broken_css, iteration: iteration, debug: debug, no_cache: no_cache)
       end
     end
 
-    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       doc = Nokolexbor::HTML(html)
 
       # Rejecting title, buttons, and label
@@ -46,12 +46,20 @@ module GoogleLocalResultsAiParser
       cleaned_text = split_text.map(&:strip).reject(&:empty?).flatten
     
       # Making parallel requests to server for classification
-      results = parallel_post_requests(server, bearer_token, cleaned_text)
+      time_start = Time.now
+      results = parallel_post_requests(server, bearer_token, cleaned_text, no_cache)
+      time_end = Time.now
       
       # After-fix and sorting of results
       results = sort_results(results, extracted_text, unsplit_text, iteration, doc)
       final_results = transform_hash(results, unsplit_text)
-      final_results
+
+      unless debug
+        final_results # Default output
+      else
+        time_taken = time_end - time_start # Time taken to make requests for debugging purpurses
+        return final_results, time_taken
+      end
     end
 
     def transform_hash(results, unsplit_text)
@@ -74,6 +82,17 @@ module GoogleLocalResultsAiParser
     end
 
     def sort_results(results, extracted_text, unsplit_text, iteration, doc)
+      # Some endpoints load array of hashes whereas some of them
+      # load a wrapped version of this. The Free Inference API
+      # should be taken as reference since most people will
+      # prototype there.
+      results.map! do |item|
+        if item[:result][0].is_a?(Hash)
+          item[:result] = [item[:result]]
+        end
+        item
+      end
+
       # Make at most 2 iterations for after-corrections
       (0..iteration).each do |i|
         begin
@@ -464,19 +483,17 @@ module GoogleLocalResultsAiParser
         "Online estimates not available",
         "Takeaway"
       ]
-      caught_results_indices = results.map.with_index {|result, index| index if known_errors.include?(result[:input])}.compact
+      caught_results_indices = results.map.with_index {|result, index| index if known_errors.include?(result[:input]) && result[:result][0][0]["label"] != "service options"}.compact
 
       not_service_option_duplicates = []
       caught_results_indices.each do |caught_index|
         duplicates.each.with_index do |duplicate, duplicate_index|
-          if duplicate.include?(caught_index) && results[caught_index][:result][0][0]["label"] != "service_options"
+          if duplicate.include?(caught_index)
             not_service_option_duplicates << duplicate_index
           end
         end
       end
 
-      already_a_service_option = caught_results_indices.all? {|index| results[index][:result][0][0]["label"] == "service options"}
-      return results, label_order, duplicates if already_a_service_option
       # Zero out the `type` or `description`, and put it to last position
       caught_results_indices.each do |caught_index|
         service_options_hash = results[caught_index][:result][0].find {|hash| hash["label"] == "service options" }
@@ -487,10 +504,11 @@ module GoogleLocalResultsAiParser
         old_result_hash["score"] = 0.0
         results[caught_index][:result][0] << old_result_hash
       end
-
+      
       # Rearranging `label_order`
-      caught_results_indices.each {|caught_index| label_order[caught_index] = "service_options"}
+      caught_results_indices.each {|caught_index| label_order[caught_index] = "service options"}
 
+      return results, label_order, duplicates if not_service_option_duplicates == []
       # Rearranging duplicates
       not_service_option_duplicates.each do |not_service_option_duplicate|
         last_item = duplicates[not_service_option_duplicate][-1]
@@ -540,18 +558,23 @@ module GoogleLocalResultsAiParser
 
     private
 
-    def parallel_post_requests(server, bearer_token, inputs)
+    def parallel_post_requests(server, bearer_token, inputs, no_cache)
       response_bodies = Parallel.map(inputs, in_threads: inputs.size) do |input|
-        post_request(server, bearer_token, input)
+        post_request(server, bearer_token, input, no_cache)
       end
 
       response_bodies
     end
 
-    def post_request(server, bearer_token, input)
+    def post_request(server, bearer_token, input, no_cache)
       url = URI.parse(server)
-      headers = { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
-      body = { inputs: input }.to_json
+      headers = unless no_cache
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
+      else
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json', 'Cache-Control' => 'no-cache' } # To benchmark initial load of the model
+      end
+
+      body = { inputs: input, parameters: {top_k: 11}}.to_json # 11 represents the number of labels the model has
     
       response = HTTP.headers(headers).post(url, body: body)
       response_body = JSON.parse(response.body)

metadata

@@ -1,15 +1,77 @@
 --- !ruby/object:Gem::Specification
 name: google-local-results-ai-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - Emirhan Akdeniz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-21 00:00:00.000000000 Z
-dependencies: []
+date: 2023-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokolexbor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A gem to be used with serpapi/bert-base-local-results model to predict
   different parts of Google Local Listings. This gem uses BERT model at https://huggingface.co/serpapi/bert-base-local-results
   in the background. For serving private servers, head to https://github.com/serpapi/google-local-results-ai-server