fingerprinter 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1d85553bfd3f1487b0185188933d1e409f12cc8e8a96762e5f2e8e5a2ab7f7d1
+  data.tar.gz: bd5fc3fd49ceac7667bcaca1eaf4cb3e0fb7be4dc68d62bc58bbf15f6ff8751d
+SHA512:
+  metadata.gz: e598f2f6025e1db35ca2089aace62176b048cc6e432868086199d5af999138257dcd6a3f8134c6b9c303c189e565d15ffb8e0e1e6736eae8b3be9f45c16f360b
+  data.tar.gz: c9829ca7d4288d7f096b4c3dd7dea07369bebbee12e9b308c743bd749176f03c9898a3de1b937bf30f1ea33b5ef675d317ccf1546aeb3feba22a57f0909fdf91

data/lib/fingerprinter/configs/scan_options.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This class builds and maintains the list of options provided for the scan to make it available
+# when needed
+class ScanOptions
+  def self.build(options)
+    @proxy = options[:proxy]
+    @user_agent = options[:ua]
+    @timeout = options[:timeout]
+    @concurrency = options[:concurrency]
+    @silent = options[:silent]
+  end
+
+  def self.proxy?
+    !!@proxy
+  end
+
+  def self.proxy_host
+    uri = URI(@proxy)
+    return '' unless uri
+
+    uri.host.to_s
+  end
+
+  def self.proxy_url
+    @proxy
+  end
+
+  def self.proxy_port
+    uri = URI(@proxy)
+    return nil unless uri
+
+    uri.port.to_s
+  end
+
+  def self.user_agent
+    @user_agent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
+  end
+
+  def self.timeout
+    @timeout ? @timeout.to_i : 10
+  end
+
+  def self.http_concurrency
+    @concurrency || 10
+  end
+
+  def self.silent?
+    !!@silent
+  end
+end

data/lib/fingerprinter/core/detector.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Fingerprinter : Entry point
+module Fingerprinter
+  # Technologies : Groups the different detection methods
+  class Technologies
+    def self.response_headers_check(response, regexes)
+      response.headers&.each do |header, value|
+        regex = regexes[header.downcase]
+        next unless regex
+    
+        if value.is_a?(Array)
+          return true if value.any? { |v| regex.match?(v) }
+        else
+          return true if regex.match?(value)
+        end
+      end
+    
+      false
+    end
+
+    def self.whole_body_check(response, regexes)
+      return false if response.body.nil?
+
+      regexes.each do |regex|
+        return true if response.body.match?(regex)
+      end
+
+      false
+    end
+
+    def self.meta_detection(doc, regexes, type = 'generator')
+      nodes = doc.xpath("//meta[@name='#{type}']/@content")
+      nodes&.each do |node|
+        return true if regexes.any? { |regex| node.value.match?(regex) }
+      end
+
+      false
+    end
+
+    def self.title_detection(doc, title)
+      doc.title&.downcase == title.downcase
+    end
+  end
+end

data/lib/fingerprinter/core/http_client.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'typhoeus'
+
+# HTTP Client used to perform some requests (check for some content for example) or as a fallback in some
+# browser processing (less resource intensive than using a Chrome instance but won't provide access to the DOM
+# for example)
+class HttpClient
+  def initialize
+    Typhoeus::Config.user_agent = ScanOptions.user_agent
+    @hydra = Typhoeus::Hydra.new(max_concurrency: ScanOptions.http_concurrency)
+  end
+
+  def request_options(method, options, body = nil)
+    req_options = {
+      ssl_verifypeer: false,
+      ssl_verifyhost: 0,
+      followlocation: options[:follow_location] || false,
+      method: method,
+      headers: options[:headers] || {},
+      body: body
+    }
+
+    req_options[:headers].merge!({
+      'Priority' => 'u=0, i',
+      'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8'
+    })
+
+    req_options[:params] = options[:params] if options[:params]
+
+    req_options[:proxy] = ScanOptions.proxy_url if ScanOptions.proxy?
+    req_options[:timeout] = ScanOptions.timeout
+
+    req_options
+  end
+
+  def get(urls, options = {})
+    responses = {}
+
+    urls = [urls] if urls.is_a?(String)
+    urls.each do |url|
+      http_request = Typhoeus::Request.new(url, request_options(:get, options))
+
+      http_request.on_complete { |response| responses[url] = response }
+
+      @hydra.queue(http_request)
+    end
+
+    @hydra.run
+
+    responses
+  end
+
+  def post(urls, body, options = {})
+    responses = {}
+
+    urls = [urls] if urls.is_a?(String)
+    urls.each do |url|
+      http_request = Typhoeus::Request.new(url, request_options(:post, options, body))
+
+      http_request.on_complete { |response| responses[url] = response }
+
+      @hydra.queue(http_request)
+    end
+
+    @hydra.run
+
+    responses
+  end
+end

data/lib/fingerprinter/technologies/cms/magento.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Magento Detection
+class Magento < Fingerprinter::Technologies
+  META_CONTENT_REGEX = [
+    'Magento'
+  ].freeze
+
+  HEADERS_REGEX = {
+    'x-magento-debug' => /\d/,
+    'x-magento-cache-control' => /\w/
+  }.freeze
+
+  BODY_CONTENT_REGEX = [
+    /Magento_PageCache/,
+    /Mage\.Cookies\.path/,
+    /data-requiremodule="(mage|Magento_)/,
+    /mage\/cookies/,
+    /MAGENTO_/,
+    /Magento Security Scan/,
+    /js\/mage\//,
+    /x-magento-init/
+  ].freeze
+
+  def self.get_graphql(url)
+    url = File.join(Utilities::Urls.up_to_port(url), '/graphql?query=+{customerDownloadableProducts+{+items+{+date+download_url}}+}')
+    return if Utilities::Kb.inspected?(self, url)
+
+    Utilities::Kb.inspected(self, url)
+    Fingerprinter.http_client.get(url)[url]
+  end
+
+  def self.run(data)
+    detected = meta_detection(data[:doc], META_CONTENT_REGEX) || 
+                whole_body_check(data[:response], BODY_CONTENT_REGEX)
+
+    if detected
+      'Magento'
+    else
+      response = get_graphql(data[:url])
+      return unless response&.code == 200 && ['The current customer', 'graphql-authorization'].all? { |pattern| response.body.include?(pattern) }
+
+      'Magento'
+    end
+  end
+end

data/lib/fingerprinter/technologies/cms/wordpress.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# Wordpress Detection
+class Wordpress < Fingerprinter::Technologies
+  HEADERS_REGEX = {
+    'link' => %r{rel="https//api\.w\.org/"},
+    'x-pingback' => %r{/xmlrpc.php}
+  }.freeze
+
+  META_CONTENT_REGEX = [
+    'WordPress'
+  ].freeze
+
+  BODY_CONTENT_REGEX = [
+    %r{<script src=['"]https?://[\w./-]+wp-embed\.min\.js},
+    %r{<link rel=['"]stylesheet['"] id=['"][\w-]+['"]\s+href=['"]https?://[\w.-]+/wp-(?:content|includes)/},
+    %r{<script src=['"]https?://[\w./-]+wp-(?:content|includes)}
+  ].freeze
+
+  def self.run(data)
+    return unless response_headers_check(data[:response], HEADERS_REGEX) ||
+                  meta_detection(data[:doc], META_CONTENT_REGEX) ||
+                  whole_body_check(data[:response], BODY_CONTENT_REGEX)
+
+    'WordPress'
+  end
+end

data/lib/fingerprinter/technologies/softwares/apache_ofbiz.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Apache Ofbiz Detection
+class ApacheOfbiz < Fingerprinter::Technologies
+  META_CONTENT_REGEX = [
+    'Apache OFBiz'
+  ].freeze
+
+  HEADERS_REGEX = {
+    'set-cookie' => /OFBiz\.Visitor/
+  }.freeze
+
+  BODY_CONTENT_REGEX = [
+    /Powered by.*OFBiz/
+  ].freeze
+
+  def self.get_xmlrpc(url)
+    url = File.join(Utilities::Urls.up_to_port(url), '/webtools/control/xmlrpc')
+    return if Utilities::Kb.inspected?(self, url)
+
+    Utilities::Kb.inspected(self, url)
+    Fingerprinter.http_client.get(url)[url]
+  end
+
+  def self.run(data)
+    response = data[:response]
+    if response.code == 404
+      response = get_xmlrpc(data[:url])
+      return unless response&.code == 200
+
+      doc = Utilities::Parser.doc(response.body)
+    end
+
+    return unless response_headers_check(response, HEADERS_REGEX) ||
+                  meta_detection(data[:doc], META_CONTENT_REGEX) ||
+                  whole_body_check(response, BODY_CONTENT_REGEX)
+
+    'Apache Ofbiz'
+  end
+end

data/lib/fingerprinter/technologies/softwares/f5_next_central_manager.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# F5 Next Central Manager Detection
+class F5NextCentralManager < Fingerprinter::Technologies
+  def self.run(data)
+    return unless title_detection(data[:doc], 'BIG-IP Next | Central Manager')
+
+    'F5 Big-IP Next Central manager'
+  end
+end

data/lib/fingerprinter/technologies/softwares/nexus_repository.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# Nexus Repository Detection
+class NexusRepository < Fingerprinter::Technologies
+  META_CONTENT_REGEX = [
+    'Nexus Repository'
+  ].freeze
+
+  def self.check_path(url)
+    url = File.join(Utilities::Urls.up_to_port(url), '/nexus/')
+    return if Utilities::Kb.inspected?(self, url)
+
+    Utilities::Kb.inspected(self, url)
+    Fingerprinter.http_client.get(url, { follow_location: true })[url]
+  end
+
+  def self.run(data)
+    detected = title_detection(data[:doc], 'Sonatype Nexus Repository') || meta_detection(data[:doc], META_CONTENT_REGEX, 'description')
+    unless detected
+      response = check_path(data[:url])
+      return unless response&.code == 200
+
+      doc = Utilities::Parser.doc(response.body)
+      detected = title_detection(doc, 'Sonatype Nexus Repository') || meta_detection(doc, META_CONTENT_REGEX, 'description')
+    end
+    return unless detected
+
+    'Nexus Repository'
+  end
+end

data/lib/fingerprinter/technologies/softwares/tinyproxy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# TinyProxy Detection
+class TinyProxy < Fingerprinter::Technologies
+  HEADERS_REGEX = {
+    'server' => /tinyproxy/
+  }.freeze
+
+  def self.run(data)
+    return unless response_headers_check(data[:response], HEADERS_REGEX)
+
+    'TinyProxy'
+  end
+end

data/lib/fingerprinter/utilities/kb.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Utilities
+  # Files : Utilities related to files
+  class Kb 
+    def self.coverage_id(plugin, url)
+      "#{plugin}:#{url}"
+    end
+
+    def self.inspected(plugin, url)
+      Fingerprinter::Technologies.kb[:inspected] << coverage_id(plugin, url)
+    end
+
+    def self.inspected?(plugin, url)
+      Fingerprinter::Technologies.kb[:inspected].include?(coverage_id(plugin, url))
+    end
+  end
+end

data/lib/fingerprinter/utilities/parser.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class Utilities
+  # Parser : Utilities related to parsing
+  class Parser
+    require 'nokogiri'
+
+    def self.doc(body)
+      Nokogiri::HTML(body)
+    end
+  end
+end

data/lib/fingerprinter/utilities/urls.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class Utilities
+  # Urls : Utilities related to urls
+  class Urls
+    require 'uri'
+
+    def self.uri_parse(url)
+      URI(url)
+    rescue ArgumentError, URI::InvalidURIError
+      nil
+    end
+
+    def self.up_to_port(url)
+      uri = url.is_a?(URI) ? url : uri_parse(url)
+      return unless uri
+
+      url = "#{uri.scheme}://#{uri.host}"
+      url += ":#{uri.port}" unless [80, 443].include?(uri.port)
+
+      url
+    end
+
+    def self.without_query(url)
+      uri = url.is_a?(URI) ? url : uri_parse(url)
+      return unless uri
+
+      uri.to_s.split('?', 2).first.to_s
+    end
+  end
+end

data/lib/fingerprinter.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+# Fingerprinter : Entry point
+module Fingerprinter
+  Dir[File.join(__dir__, 'fingerprinter', 'configs/*.rb')].sort.each { |file| require file }
+  Dir[File.join(__dir__, 'fingerprinter', 'core/*.rb')].sort.each { |file| require file }
+  Dir[File.join(__dir__, 'fingerprinter', 'utilities/*.rb')].sort.each { |file| require file }
+
+  def self.http_client
+    @http_client ||= HttpClient.new
+  end
+
+  # Technologies : Groups the different subcategories of technologies
+  class Technologies
+    Dir[File.join(__dir__, 'fingerprinter', 'technologies', '**/*.rb')].sort.each { |file| require file }
+
+    attr_accessor :results
+    attr_reader :http_client
+
+    def initialize(options = {})
+      ScanOptions.build(options)
+      @results ||= Concurrent::Hash.new
+    end
+
+    def self.kb
+      @kb ||= {
+        inspected: Concurrent::Array.new
+      }
+    end
+
+    def run(urls)
+      urls.each do |url|
+        response = get_response(url)
+        next unless response
+
+        url = Utilities::Urls.up_to_port(response.effective_url)
+  
+        responses = response.redirections
+        responses << response
+
+        responses.each do |response|
+          doc = Utilities::Parser.doc(response.body)
+
+          results[url] = Concurrent::Array.new
+          data = { response:, doc:, url: }
+          Technologies.subclasses.each { |technology| results[url] << technology.run(data) }
+        end
+      end
+
+      results.transform_values! { |v| v.compact.uniq }
+      results
+    end
+
+    private
+
+    def get_response(url)
+      response = Fingerprinter.http_client.get(url, { follow_location: true })[url]
+      return if response&.code&.zero?
+      return response if same_scope?(url, response)
+
+      Fingerprinter.http_client.get(url, { follow_location: false })[url]
+    end
+
+    def same_scope?(url, response)
+      url = "https://#{url}" unless url.start_with?('http')
+
+      Utilities::Urls.uri_parse(url)&.host == Utilities::Urls.uri_parse(response&.effective_url)&.host
+    end
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: fingerprinter
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Joshua MARTINELLE
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-06-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- contact@jomar.fr
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fingerprinter.rb
+- lib/fingerprinter/configs/scan_options.rb
+- lib/fingerprinter/core/detector.rb
+- lib/fingerprinter/core/http_client.rb
+- lib/fingerprinter/technologies/cms/magento.rb
+- lib/fingerprinter/technologies/cms/wordpress.rb
+- lib/fingerprinter/technologies/softwares/apache_ofbiz.rb
+- lib/fingerprinter/technologies/softwares/f5_next_central_manager.rb
+- lib/fingerprinter/technologies/softwares/nexus_repository.rb
+- lib/fingerprinter/technologies/softwares/tinyproxy.rb
+- lib/fingerprinter/utilities/kb.rb
+- lib/fingerprinter/utilities/parser.rb
+- lib/fingerprinter/utilities/urls.rb
+homepage:
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: Lorem Ipsum
+test_files: []