google-local-results-ai-parser 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62925be13b2ff224585862444f2e77bdea9f7975316b2f66253e7e2fa7a6fe34
-  data.tar.gz: e2ec29030b6c442f955b3d4c54b21473a23f214d3db0944e2006c6ac3c7c9446
+  metadata.gz: 23d5956da19fd752805805c70fbd41c9b0c4b19a8b080d81a399e9efbced322c
+  data.tar.gz: 2ae85f484532bf47836efe3b354cb342d2672526adec07d86e8e294a2206f856
 SHA512:
-  metadata.gz: 48054b17769730be43150502b909a82666adca5f35b2bf6fa235cf4b96da37040338eb41c62382da0320729158b3d70898efb6d595ad6cef7363d212aa4414f1
-  data.tar.gz: af7d2196995a5991c017940af28de5816e1fc9535b1424c070670db65cf4f6df546d76d2c77ed0f64281810dda649f769e7ead071e5c99c68f43a18c4dbf89ee
+  metadata.gz: 77d0a15beddddc3e9324cc56c71f16393d1222cc3bab8dcb6df5becf0a0ed3d8a35bb08e4a25da0f463411ee154af1807be279046d551381a595a583f5a661d9
+  data.tar.gz: c2a9844133da6f5a60cfb2f499e23b265043c41ec9a0d9216cba21ce5f4f5bb1e6766e40e1f1bd6eec41d9143174d47b0e468d1c0d280470c03d2cc7a0bbfb88

data/lib/google-local-results-ai-parser.rb

@@ -2,7 +2,6 @@ require 'nokolexbor'
 require 'http'
 require 'parallel'
 require 'json'
-require 'pry'
 
 module GoogleLocalResultsAiParser
   DEFAULT_SERVER = 'https://api-inference.huggingface.co/models/serpapi/bert-base-local-results'.freeze
@@ -25,13 +24,13 @@ module GoogleLocalResultsAiParser
   end
 
   class << self
-    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse_multiple(html_parts: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       response_bodies = Parallel.map(html_parts, in_threads: html_parts.size) do |html|
-        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+        parse(html: html, bearer_token: bearer_token, server: server, separator_regex: separator_regex, rejected_css: rejected_css, broken_css: broken_css, iteration: iteration, debug: debug, no_cache: no_cache)
       end
     end
 
-    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION)
+    def parse(html: nil, bearer_token: nil, server: DEFAULT_SERVER, separator_regex: DEFAULT_SEPARATOR_REGEX, rejected_css: DEFAULT_REJECTED_CSS, broken_css: DEFAULT_BROKEN_CSS, iteration: DEFAULT_MAX_ITERATION, debug: false, no_cache: false)
       doc = Nokolexbor::HTML(html)
 
       # Rejecting title, buttons, and label
@@ -47,12 +46,20 @@ module GoogleLocalResultsAiParser
       cleaned_text = split_text.map(&:strip).reject(&:empty?).flatten
     
       # Making parallel requests to server for classification
-      results = parallel_post_requests(server, bearer_token, cleaned_text)
+      time_start = Time.now
+      results = parallel_post_requests(server, bearer_token, cleaned_text, no_cache)
+      time_end = Time.now
       
       # After-fix and sorting of results
       results = sort_results(results, extracted_text, unsplit_text, iteration, doc)
       final_results = transform_hash(results, unsplit_text)
-      final_results
+
+      unless debug
+        final_results # Default output
+      else
+        time_taken = time_end - time_start # Time taken to make requests for debugging purpurses
+        return final_results, time_taken
+      end
     end
 
     def transform_hash(results, unsplit_text)
@@ -75,6 +82,17 @@ module GoogleLocalResultsAiParser
     end
 
     def sort_results(results, extracted_text, unsplit_text, iteration, doc)
+      # Some endpoints load array of hashes whereas some of them
+      # load a wrapped version of this. The Free Inference API
+      # should be taken as reference since most people will
+      # prototype there.
+      results.map! do |item|
+        if item[:result][0].is_a?(Hash)
+          item[:result] = [item[:result]]
+        end
+        item
+      end
+
       # Make at most 2 iterations for after-corrections
       (0..iteration).each do |i|
         begin
@@ -540,18 +558,23 @@ module GoogleLocalResultsAiParser
 
     private
 
-    def parallel_post_requests(server, bearer_token, inputs)
+    def parallel_post_requests(server, bearer_token, inputs, no_cache)
       response_bodies = Parallel.map(inputs, in_threads: inputs.size) do |input|
-        post_request(server, bearer_token, input)
+        post_request(server, bearer_token, input, no_cache)
       end
 
       response_bodies
     end
 
-    def post_request(server, bearer_token, input)
+    def post_request(server, bearer_token, input, no_cache)
       url = URI.parse(server)
-      headers = { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
-      body = { inputs: input }.to_json
+      headers = unless no_cache
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json' }
+      else
+        { 'Authorization' => "Bearer #{bearer_token}", 'Content-Type' => 'application/json', 'Cache-Control' => 'no-cache' } # To benchmark initial load of the model
+      end
+
+      body = { inputs: input, parameters: {top_k: 11}}.to_json # 11 represents the number of labels the model has
     
       response = HTTP.headers(headers).post(url, body: body)
       response_body = JSON.parse(response.body)

metadata

@@ -1,15 +1,77 @@
 --- !ruby/object:Gem::Specification
 name: google-local-results-ai-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.5
 platform: ruby
 authors:
 - Emirhan Akdeniz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-21 00:00:00.000000000 Z
-dependencies: []
+date: 2023-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokolexbor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.20.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A gem to be used with serpapi/bert-base-local-results model to predict
   different parts of Google Local Listings. This gem uses BERT model at https://huggingface.co/serpapi/bert-base-local-results
   in the background. For serving private servers, head to https://github.com/serpapi/google-local-results-ai-server