testai_classifier 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c3ea7ee65e1ed0cd0115c7f376bdd3d1246408649e40eddd778fe8c6b876cdeb
+  data.tar.gz: 605c5c9f9a448cc28e0b5818722da66976f062c81aa835d1b4728dcad070514e
+SHA512:
+  metadata.gz: 146b9275636a14fb6a4b0c2598d3a6dba53eca1a8ddf1dd810aaf993d71a624b8c085245e1dbb819288b76f127a5bd8e8a4f09b9300de3516f669074b2e5a271
+  data.tar.gz: 47e9a22da0eb1442e770395a02d7b4308d1d8d7195287a5f93272233a4518cea42682e825956924c886ebd52d1352232ebe3c0d995ba2fda07e6b4cdff6710df

data/lib/testai_classifier/classifier_pb.rb

@@ -0,0 +1,27 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: classifier.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("classifier.proto", :syntax => :proto3) do
+    add_message "ElementClassificationRequest" do
+      optional :labelHint, :string, 1
+      map :elementImages, :string, :bytes, 2
+      optional :confidenceThreshold, :double, 3
+      optional :allowWeakerMatches, :bool, 4
+    end
+    add_message "ElementClassificationResult" do
+      optional :label, :string, 1
+      optional :confidence, :double, 2
+      optional :confidenceForHint, :double, 3
+    end
+    add_message "ClassifiedElements" do
+      map :classifications, :string, :message, 1, "ElementClassificationResult"
+    end
+  end
+end
+
+ElementClassificationRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("ElementClassificationRequest").msgclass
+ElementClassificationResult = Google::Protobuf::DescriptorPool.generated_pool.lookup("ElementClassificationResult").msgclass
+ClassifiedElements = Google::Protobuf::DescriptorPool.generated_pool.lookup("ClassifiedElements").msgclass

data/lib/testai_classifier/classifier_services_pb.rb

@@ -0,0 +1,20 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: classifier.proto for package ''
+
+require 'grpc'
+require_relative 'classifier_pb'
+
+module Classifier
+  class Service
+
+    include GRPC::GenericService
+
+    self.marshal_class_method = :encode
+    self.unmarshal_class_method = :decode
+    self.service_name = 'Classifier'
+
+    rpc :ClassifyElements, ElementClassificationRequest, ClassifiedElements
+  end
+
+  Stub = Service.rpc_stub_class
+end

data/lib/testai_classifier/extensions.rb

@@ -0,0 +1,14 @@
+module WebDriverExtensions
+    module Element
+        def screenshot()
+            data = bridge.take_element_screenshot @id
+            data.unpack1 'm'
+        end
+    end
+
+    module Bridge
+        def take_element_screenshot(element)
+            execute :take_element_screenshot, {id: element}
+        end
+    end
+end

data/lib/testai_classifier.rb

@@ -0,0 +1,53 @@
+require 'selenium-webdriver'
+require 'testai_classifier/classifier_services_pb'
+require 'testai_classifier/extensions'
+
+# Selenium Ruby client doesn't currently implement the 'take element
+# screenshot' command, so we have to patch it in
+Selenium::WebDriver::Element.include WebDriverExtensions::Element
+Selenium::WebDriver::Remote::Bridge.include WebDriverExtensions::Bridge
+
+DEF_CONFIDENCE = 0.2
+QUERY = "//body//*[not(self::script) and not(self::style) and not(child::*)]"
+
+class ClassifierClient
+    def initialize(host, port)
+        @stub = Classifier::Stub.new("#{host}:#{port}", :this_channel_is_insecure)
+    end
+
+    def classify_images(label, element_images, confidence=DEF_CONFIDENCE,
+                        allow_weaker_matches=false)
+        ecr = ElementClassificationRequest
+        req = ecr.new(labelHint: label, 
+                      elementImages: element_images,
+                      confidenceThreshold: confidence, 
+                      allowWeakerMatches: allow_weaker_matches)
+        resp = @stub.classify_elements(req)
+        ret = {}
+        resp.classifications.each do |id, cls|
+            ret[id] = {
+                :label => cls.label, 
+                :confidence => cls.confidence,
+                :confidence_for_hint => cls.confidenceForHint
+            }
+        end
+        return ret
+    end
+
+    def find_elements_matching_label(driver, label, confidence=DEF_CONFIDENCE, allow_weaker_matches=false)
+        els = driver.find_elements(:xpath, QUERY)
+        element_images = {}
+        els.each do |el|
+            begin
+                element_images[el.ref] = el.screenshot
+            rescue
+            end
+        end
+        if element_images.size < 1
+            raise "Could not find any screenshots for leaf node elements"
+        end
+        matched = self.classify_images(label, element_images, confidence,
+                                       allow_weaker_matches)
+        return els.select {|el| matched.has_key? el.ref}
+    end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: testai_classifier
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jonathan Lipps
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: grpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.142'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.142'
+- !ruby/object:Gem::Dependency
+  name: grpc-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.13'
+description: The test.ai image classification RPC server can listen for requests from
+  this client in order to classify images and even use a Selenium client object to
+  find elements in a webpage matching a certain label
+email: jlipps@cloudgrey.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/testai_classifier.rb
+- lib/testai_classifier/classifier_pb.rb
+- lib/testai_classifier/classifier_services_pb.rb
+- lib/testai_classifier/extensions.rb
+homepage: https://github.com/testdotai/appium-classifier-plugin
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A client to the Test.ai + Appium Classifier server
+test_files: []