google-local-results-ai-parser 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f8076de3793dabc268bc5705443e28202655e26c5a1503498767e1b8a62ff16
-  data.tar.gz: 7e3cf96add3d90d2e6174f9acda7f6cd8357f2f06792d46fc308aa80cc7fa93f
+  metadata.gz: 23d5956da19fd752805805c70fbd41c9b0c4b19a8b080d81a399e9efbced322c
+  data.tar.gz: 2ae85f484532bf47836efe3b354cb342d2672526adec07d86e8e294a2206f856
 SHA512:
-  metadata.gz: 91c997a71f5765a0d43cc1c4418cecc1a83b00eb3b418f4aed6802a1c5fa78391b337384433147cf5c7eb8af5455ad07289f8440c31e58585493978d4673394a
-  data.tar.gz: b0d1a462cba2a8097c0bc54ec0142e78b5d7daffe5c5ed4c663ddb7522c35391ee4ba9587f803a711f2e0fbe7395ab7d353254eda6ecab6212bbeed64c563b4d
+  metadata.gz: 77d0a15beddddc3e9324cc56c71f16393d1222cc3bab8dcb6df5becf0a0ed3d8a35bb08e4a25da0f463411ee154af1807be279046d551381a595a583f5a661d9
+  data.tar.gz: c2a9844133da6f5a60cfb2f499e23b265043c41ec9a0d9216cba21ce5f4f5bb1e6766e40e1f1bd6eec41d9143174d47b0e468d1c0d280470c03d2cc7a0bbfb88

data/lib/google-local-results-ai-parser.rb

@@ -24,13 +24,13 @@ module GoogleLocalResultsAiParser
   end
 
   class << self
-    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       response_bodies = Parallel.map(html_parts, in_threads: html_parts.size) do |html|
-        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: broken_css, iteration: iteration, debug: debug, no_cache: no_cache)
       end
     end
 
-    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       doc = Nokolexbor::HTML(html)
 
       # Rejecting title, buttons, and label
@@ -46,12 +46,20 @@ module GoogleLocalResultsAiParser
       cleaned_text = split_text.map(&:strip).reject(&:empty?).flatten
     
       # Making parallel requests to server for classification
-      results = parallel_post_requests(server, bearer_token, cleaned_text)
+      time_start = Time.now
+      results = parallel_post_requests(server, bearer_token, cleaned_text, no_cache)
+      time_end = Time.now
       
       # After-fix and sorting of results
       results = sort_results(results, extracted_text, unsplit_text, iteration, doc)
       final_results = transform_hash(results, unsplit_text)
-      final_results
+
+      unless debug
+        final_results # Default output
+      else
+        time_taken = time_end - time_start # Time taken to make requests for debugging purpurses
+        return final_results, time_taken
+      end
     end
 
     def transform_hash(results, unsplit_text)
@@ -74,6 +82,17 @@ module GoogleLocalResultsAiParser
     end
 
     def sort_results(results, extracted_text, unsplit_text, iteration, doc)
+      # Some endpoints load array of hashes whereas some of them
+      # load a wrapped version of this. The Free Inference API
+      # should be taken as reference since most people will
+      # prototype there.
+      results.map! do |item|
+        if item[:result][0].is_a?(Hash)
+          item[:result] = [item[:result]]
+        end
+        item
+      end
+
       # Make at most 2 iterations for after-corrections
       (0..iteration).each do |i|
         begin
@@ -475,8 +494,7 @@ module GoogleLocalResultsAiParser
         end
       end
 
-      already_a_service_option = caught_results_indices.all? {|index| results[index][:result][0][0]["label"] == "service options"}
-      return results, label_order, duplicates if already_a_service_option
+      return results, label_order, duplicates if not_service_option_duplicates == []
       # Zero out the `type` or `description`, and put it to last position
       caught_results_indices.each do |caught_index|
         service_options_hash = results[caught_index][:result][0].find {|hash| hash["label"] == "service options" }
@@ -540,18 +558,23 @@ module GoogleLocalResultsAiParser
 
     private
 
-    def parallel_post_requests(server, bearer_token, inputs)
+    def parallel_post_requests(server, bearer_token, inputs, no_cache)
       response_bodies = Parallel.map(inputs, in_threads: inputs.size) do |input|
-        post_request(server, bearer_token, input)
+        post_request(server, bearer_token, input, no_cache)
       end
 
       response_bodies
     end
 
-    def post_request(server, bearer_token, input)
+    def post_request(server, bearer_token, input, no_cache)
       url = URI.parse(server)
-      headers = { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
-      body = { inputs: input }.to_json
+      headers = unless no_cache
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
+      else
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json', 'Cache-Control' => 'no-cache' } # To benchmark initial load of the model
+      end
+
+      body = { inputs: input, parameters: {top_k: 11}}.to_json # 11 represents the number of labels the model has
     
       response = HTTP.headers(headers).post(url, body: body)
       response_body = JSON.parse(response.body)

metadata

@@ -1,15 +1,77 @@
 --- !ruby/object:Gem::Specification
 name: google-local-results-ai-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Emirhan Akdeniz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-21 00:00:00.000000000 Z
-dependencies: []
+date: 2023-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokolexbor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A gem to be used with serpapi/bert-base-local-results model to predict
   different parts of Google Local Listings. This gem uses BERT model at https://huggingface.co/serpapi/bert-base-local-results
   in the background. For serving private servers, head to https://github.com/serpapi/google-local-results-ai-server