intrigue-ident 0.2 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d05d96fbb44b3e9ea2ed428a0c63873fcfe5679a74882faa3534051ffe51338
-  data.tar.gz: 56e52215528ae59c96cdeb9a832ea2e9a7655a607da986499dc714b9a90fa74d
+  metadata.gz: 32cb17663810844bd445fbd30fc87ed296e432dee65c809c55c0103c2018223f
+  data.tar.gz: ab582bae1548877adda260b27b3cc36c3c399f5dfac4bc2120df58ff11c028c5
 SHA512:
-  metadata.gz: 17b3fe532217c204bd8ac15f42e395268ea010f409f9b37d623d115f4fa24b5a245b77a0b79e9fd5d1a76fd9558443ad13280245d15e012e3f0180f23e6f4b47
-  data.tar.gz: 664d65996160c1ce2ab55569e76b9342f4574a34a6e2aec92f20810590f31056ec568191b14828a35f6745892f6820e24ab863fff50a62b44aae3ad1e8e794ae
+  metadata.gz: cdb9095cd2c3b4eef1515260b929f9685293d9a888a50cf30942c875557e736ddf328e13fc60a14302f1a92489ca5f1658839ac1294e29c2e2d6a5567d58226d
+  data.tar.gz: '09a8a0beb4ccca2821c8543f8939491dbbec030b873e7211cde9966f5367a7cd599b67cd77908ebf69a59c85d8f26611adb6b1dcc5fdc41c433ef074c7ce6b54'

data/.circleci/config.yml

@@ -0,0 +1,21 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2 
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.5-stretch-node
+        environment:
+          BUNDLER_VERSION: 2.1.4
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies & run specs 
+          command:  | 
+            gem update --system
+            gem install bundler:2.1.4
+            bundle install
+            bundle exec rspec spec/*   
+    

data/.gitignore

@@ -0,0 +1,3 @@
+.rbenv-gemsets
+requests.txt
+.DS_Store

data/.ruby-version

@@ -0,0 +1 @@
+2.6.5

data/Dockerfile

@@ -0,0 +1,39 @@
+FROM debian:bullseye-slim
+
+ENV CHROME_BIN=/usr/bin/chromium \
+    CHROME_PATH=/usr/lib/chromium/ \
+    GEM_HOME="/home/ident/.gem"
+
+RUN adduser --disabled-password --gecos "" ident \
+    && apt-get update \
+    && apt-get install -yq apt-utils build-essential curl gcc \
+       libbison-dev libcurl4-openssl-dev libgdbm-compat-dev libgdbm-dev \
+       libgmp-dev libharfbuzz-dev libssl-dev libxml2-dev libxslt1-dev openssl \
+       readline-common  \
+    && mkdir -p /src/ruby  \
+    && cd /src/ruby \
+    && curl -O https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.gz \
+    && tar -xvzf ruby-2.6.5.tar.gz \
+    && cd ruby-2.6.5 \
+    && ./configure --disable-install-rdoc \
+    && make && make install \
+    && rm -rf /var/cache/apt/* \
+    && cd / \
+    && rm -rf /src/ruby/  \
+       /usr/local/share/{doc,man}
+
+# if Chrome support is desired:     
+# apt-get install 'xvfb chromium chromium-driver zlib1g-dev zlib1g'
+
+# Save some space if needed: 
+# apt-get remove -yq gcc build-essential \
+  #&& apt-get autoremove  -yq \
+  #&& bundle config build.nokogiri --use-system-libraries 
+
+ADD . /home/ident
+RUN chown -R ident:ident /home/ident
+USER ident
+WORKDIR /home/ident
+RUN gem install bundler:2.0.2 \
+    && bundle install
+ENTRYPOINT ["/home/ident/util/docker.sh"]

data/Gemfile

@@ -1,5 +1,12 @@
 source 'https://rubygems.org'
-ruby '2.5.1'
+ruby '2.6.5'
 
-gem "rspec"
-gem "pry"
+gem 'pry'
+gem 'socketry'
+gem 'recog-intrigue',         :git => 'https://github.com/intrigueio/recog.git'
+#gem 'recog-intrigue', :path => "/Users/jcran/work/intrigue/local/recog"
+gem 'slop'
+gem 'snmp'
+gem 'json'
+gem 'rspec-core'
+gem 'rspec-expectations'

data/Gemfile.lock

@@ -1,35 +1,50 @@
+GIT
+  remote: https://github.com/intrigueio/recog.git
+  revision: d6be4b8008add1cc90c69b276c223cf7345e31f1
+  specs:
+    recog-intrigue (2.3.7)
+      nokogiri
+
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
+    coderay (1.1.3)
     diff-lcs (1.3)
-    method_source (0.9.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+    hitimes (1.3.1)
+    json (2.3.0)
+    method_source (1.0.0)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    slop (4.8.1)
+    snmp (1.3.2)
+    socketry (0.5.1)
+      hitimes (~> 1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  json
   pry
-  rspec
+  recog-intrigue!
+  rspec-core
+  rspec-expectations
+  slop
+  snmp
+  socketry
 
 RUBY VERSION
-   ruby 2.5.1p57
+   ruby 2.6.5p114
 
 BUNDLED WITH
-   1.16.1
+   2.1.4

data/LICENSE.md

@@ -0,0 +1,12 @@
+Copyright (c) 2020, Intrigue Corp
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,79 @@
+Ident is an application and service fingerprinting library used within Intrigue Core. 
+
+The Ident project has a few stated goals: 
+ - To deeply identify network service and app software while remain extremely fast - limiting unnecessary requests
+ - To be the most complete library for modern application and server software identification 
+ - To integrate well as a library, for easy use in other projects
+ - To be commercially viable through use of the BSD license
+ - To maintain a robust command line interface for use as a standalone tool 
+
+ Ident is written in Ruby and uses core Ruby (HTTP|Socket|etc) libraries to make requests. 
+
+ Below, find instruction on how to get stated:
+
+Using ident via Docker image:
+=============================
+1) Pull and run the docker image! It'll download directly from [DockerHub](https://cloud.docker.com/u/intrigueio/repository/docker/intrigueio/intrigue-ident) if the image doesnt exist locally
+  `docker pull intrigueio/intrigue-ident && docker run -t intrigueio/intrigue-ident --url https://intrigue.io`
+
+Installing directly on OSX:
+===========================
+1) [Install brew](https://brew.sh/) (if you don't already have it)
+2) Install ruby [preferrably using rbenv](https://github.com/rbenv/rbenv#installation)
+3) Install bundler `gem install bundler`
+4) Clone the ident repository `git clone https://github.com/intrigueio/intrigue-ident.git`
+5) Install the ident gem dependencies `bundle install`
+6) Now use the check utility to test. Example below:
+```
+$ bundle exec ruby ./util/ident.rb -v -u https://xyz.com:443
+Checking... https://xyz.com:443
+Fingerprint:
+ - PHP PHP 5.4.16  - x-powered-by header (CPE: cpe:2.3:a:php:php:5.4.16:) (Tags: ["Web Framework"])
+   - Vuln: CVE-2016-5768 (CVSSv3: 9.8)
+   - Vuln: CVE-2016-4538 (CVSSv3: 9.8)
+ - PHP PHP   - PHPSESSID cookie (CPE: cpe:2.3:a:php:php::) (Tags: ["Web Framework"])
+ - Apache HTTP Server 2.4.6  - Apache web server - server header - with versions (CPE: cpe:2.3:a:apache:http_server:2.4.6:) (Tags: ["Web Server"])
+   - Vuln: CVE-2018-1312 (CVSSv3: 9.8)
+   - Vuln: CVE-2017-15715 (CVSSv3: 8.1)
+ - Wordpress Wordpress   - common link (CPE: cpe:2.3:a:wordpress:wordpress::) (Tags: ["CMS"])
+ - Yoast Wordpress SEO   - unique string (CPE: cpe:2.3:a:yoast:wordpress_seo::) (Tags: ["Marketing", "Wordpress Plugin"])
+ - Bootstrap Bootstrap   - boostrap css (CPE: cpe:2.3:a:bootstrap:bootstrap::) (Tags: ["Web Framework"])
+ - OpenSSL OpenSSL 1.0.2k-fips  - server header (CPE: cpe:2.3:a:openssl:openssl:1.0.2k-fips:) (Tags: ["Library"])
+   - Vuln: CVE-2018-0732 (CVSSv3: 7.5)
+Content Checks:
+ - Authentication - HTTP: false
+ - Authentication - Session Identifier: true
+ - Google Analytics Account Detected: false
+ - Location Header: 
+ - Directory Listing Detected: false
+ - Form Detected: false
+ - File Upload Form Detected: false
+ - Email Addresses Detected: []
+ - Access-Control-Allow-Origin Header: false
+ - P3P Header: false
+ - X-Frame-Options Header: true
+ - X-XSS-Protection Header: false
+```
+
+For Check Writers: 
+===================
+```
+HTTP Check types are in the following categories. See each check's 'match_content': 
+ - content_body: checks should be run against body
+ - content_headers: checks should be run against response headers (assumes one string of text, each header '\n' delimited)
+ - content_title: checks should be run against text inside the <title> tag (do not include the tag when writing checks)
+ - content_generator:  checks should be run against text inside the <meta generator> tag (do not include the tag when writing checks)
+ - content_cookies; checks should be run against the set-cookie: header (do not include the header name, just the content when writing checks)
+ ```
+
+
+Contributors:
+=============
+
+A special thanks to the following contributors who help make ident awesome!
+ - @duartemateus: Checks!
+ - @jen140: Checks
+ - @bensalah_anas: Checks
+ - @bcoles: Checks, bugfixes, JSON output
+ - @bmcdevitt: Checks
+ - @retornam: Dockerization

data/checks/ftp/base.rb

@@ -0,0 +1,15 @@
+module Intrigue
+module Ident
+module FtpCheck
+class Base
+
+  include Intrigue::Ident::BannerHelpers
+
+  def self.inherited(base)
+    Intrigue::Ident::Ftp::CheckFactory.register(base)
+  end
+
+end
+end
+end
+end

data/checks/ftp/filezilla.rb

@@ -0,0 +1,28 @@
+module Intrigue
+module Ident
+module FtpCheck
+class Filezilla < Intrigue::Ident::FtpCheck::Base
+
+  def generate_checks
+    [
+      {
+        :type => "fingerprint",
+        :category => "application",
+        :tags => ["FTP Server"],
+        :vendor => "Filezilla",
+        :product => "Server",
+        :references => [],
+        :version => nil,
+        :match_type => :content_banner,
+        :match_content => /FileZilla Server/i,
+        :dynamic_version => lambda { |x| _first_banner_capture(x, /FileZilla Server version ([\d\.]+)/i)},
+        :match_details => "banner",
+        :hide => false,
+        :inference => true
+      }
+    ]
+  end
+end
+end
+end
+end

data/checks/ftp/microsoft.rb

@@ -0,0 +1,27 @@
+module Intrigue
+module Ident
+module FtpCheck
+class Microsoft < Intrigue::Ident::FtpCheck::Base
+
+  def generate_checks
+    [
+      {
+        :type => "fingerprint",
+        :category => "application",
+        :tags => ["FTP Server"],
+        :vendor => "Microsoft",
+        :product => "FTP Service",
+        :references => [],
+        :version => nil,
+        :match_type => :content_banner,
+        :match_content => /Microsoft FTP Service/i,
+        :match_details => "banner",
+        :hide => false,
+        :inference => false
+      }
+    ]
+  end
+end
+end
+end
+end

data/checks/ftp/proftp.rb

@@ -0,0 +1,28 @@
+module Intrigue
+module Ident
+module FtpCheck
+class ProFtp < Intrigue::Ident::FtpCheck::Base
+
+  def generate_checks
+    [
+      {
+        :type => "fingerprint",
+        :category => "application",
+        :tags => ["FTP Server"],
+        :vendor => "ProFTP",
+        :product => "ProFTPd",
+        :references => [],
+        :version => nil,
+        :match_type => :content_banner,
+        :match_content => /ProFTPD ([\d\w\.]+) Server/i,
+        :dynamic_version => lambda { |x| _first_banner_capture(x, /ProFTPD ([\d\w\.]+) Server/i)},
+        :match_details => "banner",
+        :hide => false,
+        :inference => true
+      }
+    ]
+  end
+end
+end
+end
+end

data/checks/ftp/pureftpd.rb

@@ -0,0 +1,27 @@
+module Intrigue
+module Ident
+module FtpCheck
+class PureFtpd < Intrigue::Ident::FtpCheck::Base
+
+  def generate_checks
+    [
+      {
+        :type => "fingerprint",
+        :category => "application",
+        :tags => ["FTP Server"],
+        :vendor => "PureFtpd",
+        :product => "PureFtpd",
+        :references => [],
+        :version => nil,
+        :match_type => :content_banner,
+        :match_content => /Welcome to Pure-FTPd/i,
+        :match_details => "banner",
+        :hide => false,
+        :inference => false
+      }
+    ]
+  end
+end
+end
+end
+end

data/checks/ftp/vsftp.rb

@@ -0,0 +1,28 @@
+module Intrigue
+module Ident
+module FtpCheck
+class Vsftp < Intrigue::Ident::FtpCheck::Base
+
+  def generate_checks
+    [
+      {
+        :type => "fingerprint",
+        :category => "application",
+        :tags => ["FTP Server"],
+        :vendor => "vsFTPd",
+        :product => "vsFTPd",
+        :references => [],
+        :version => nil,
+        :match_type => :content_banner,
+        :match_content => /\(vsFTPd\s[\d\.]+\)/i,
+        :dynamic_version => lambda { |x| _first_banner_capture(x, /\(vsFTPd\s([\d\.]+)\)/i)},
+        :match_details => "banner",
+        :hide => false,
+        :inference => true
+      }
+    ]
+  end
+end
+end
+end
+end

data/checks/http/123reg.rb

@@ -0,0 +1,31 @@
+module Intrigue
+  module Ident
+  module Check
+  class OneTwoThreeReg < Intrigue::Ident::Check::Base
+  
+    def generate_checks(url)
+      [
+        {
+          :type => "fingerprint",
+          :category => "service",
+          :tags => ["Parked"],
+          :vendor => "123Reg",
+          :product => "123Reg",
+          :website => "https://www.123-reg.co.uk/",
+          :references => [],
+          :version => nil,
+          :match_type => :content_title,
+          :match_content => /Want your own website\? \| 123 Reg/i,
+          :match_details => "Want your own website\? \| 123 Reg",
+          :hide => false,
+          :paths => ["#{url}"],
+          :inference => false
+        }
+      ]
+    end
+  
+  end
+  end
+  end
+  end
+