recog 2.3.7 → 2.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bce6ea617950159feebc525962a70eb2d04340cef05c75e522897c9c5bf780e
-  data.tar.gz: d3fa168beab209826c82a45147d149c939e8a37bc0f2cf9fad31a35a0d2ec2df
+  metadata.gz: be02bd17e124bbded970024eb6634e60ce5a3764da67faecb0da21157179d6ed
+  data.tar.gz: 04eacbfe28e565b359b4b798f0ef171c97274f9d5a43f273b8f09972b999ad46
 SHA512:
-  metadata.gz: 4b051ce5e7bb403b9367851befcc365052812d45a9d1702759241e01ccd34ea1fffaf077f5ab30e14bcb5b8dae4612df5e1c2d1b0e270e0d0a04d8ecb8368801
-  data.tar.gz: d85bf09c0fa22d54ef00c66710794cfb5bebe3eca569f07236de6d11e9bf0f08373e8f2da8f21aeb12c213373f7c49e94c55470b0ab900d6cc06621db8100f0f
+  metadata.gz: bc69e881e5a68c16227bff868480d9f68760ddb72ea8203e89c5ce2bef06a5558ba5a0a730f44e1976f4481b9c7e543b1ec2685a4dc9755b6cfa967d69fe8b66
+  data.tar.gz: 87a4cc900949a643cb89c7c2058939fb5e49a2d875b72687dc7cb739b3969fdbc6e5dafc7dceace9e4c89e16b0e94ac0f9b0327ef563a74ac7f247c7173f58ff

data/.gitignore

@@ -1,6 +1,4 @@
 # Ruby and tooling specific
-.ruby-version
-.ruby-gemset
 .yardoc
 coverage/
 doc/
@@ -8,6 +6,9 @@ pkg/
 
 /Gemfile.lock
 
+#Python specific
+venv
+
 # IDE specific
 .vscode/
 .idea

data/.ruby-gemset

@@ -0,0 +1 @@
+recog

data/.ruby-version

@@ -0,0 +1 @@
+2.6.6

data/.travis.yml

@@ -2,10 +2,8 @@ language: ruby
 sudo: false
 cache: bundler
 rvm:
-  - '2.3.8'
-  - '2.4.5'
-  - '2.5.3'
-  - '2.6.1'
+  - '2.5.8'
+  - '2.6.6'
   - 'jruby-9.1.9.0'
 jdk:
   - openjdk8

data/Gemfile

@@ -1,13 +1,10 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec name: 'recog'
 
 gem 'nokogiri'
 
 group :test do
   gem 'rake'
-  gem 'rspec', '>= 2.99'
-  gem 'cucumber', '~> 1.3.8'
-  gem 'aruba', '~> 0.5.3'
-  gem 'regexp_parser', '~> 0.2.0'
+  gem 'regexp_parser'
 end

data/bin/recog_standardize

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+
+def load_identifiers(path)
+  res = {}
+  File.readlines(path).map{|line| line.strip}.each do |ident|
+    res[ident] = true
+  end
+  return res
+end
+
+def write_identifiers(vals, path)
+  res = []
+  vals.each_pair do |k,v|
+    res = res.push(k)
+  end
+  res = res.sort.uniq
+  File.write(path, res.join("\n") + "\n")
+end
+
+bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
+
+options = OpenStruct.new(write: false)
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
+  opts.separator "Verifies that each fingerprint asserts known identifiers."
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-w", "--write") do
+      options.write = true
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.empty?
+  $stderr.puts 'Missing XML fingerprint files'
+  puts option_parser
+  exit(1)
+end
+
+# Load the unique identifiers
+vendors = load_identifiers(File.join(bdir, "vendor.txt"))
+os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
+os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
+os_family = load_identifiers(File.join(bdir, "os_family.txt"))
+os_device = load_identifiers(File.join(bdir, "os_device.txt"))
+svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
+svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
+
+ARGV.each do |arg|
+  Dir.glob(arg).each do |file|
+    ndb = Recog::DB.new(file)
+    ndb.fingerprints.each do |f|
+      f.params.each do |k,v|
+        paramIndex, val = v
+        next if paramIndex != 0
+        case k
+        when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
+          if ! vendors[val]
+            puts "VENDOR MISSING: #{val}"
+            vendors[val] = true
+          end
+        when "os.product"
+          if ! os_prod[val]
+            puts "OS PRODUCT MISSING: #{val}"
+            os_prod[val] = true
+          end
+        when "os.arch"
+          if ! os_arch[val]
+            puts "OS ARCH MISSING: #{val}"
+            os_arch[val] = true
+          end
+        when "os.family"
+          if ! os_family[val]
+            puts "OS FAMILY MISSING: #{val}"
+            os_family[val] = true
+          end
+        when "os.device"
+          if ! os_device[val]
+            puts "OS DEVICE MISSING: #{val}"
+            os_device[val] = true
+          end
+        when "service.product"
+          if ! svc_prod[val]
+            puts "SERVICE PRODUCT MISSING: #{val}"
+            svc_prod[val] = true
+          end            
+        when "service.family"
+          if ! svc_family[val]
+            puts "SERVICE FAMILY MISSING: #{val}"
+            svc_family[val] = true
+          end          
+        end
+      end
+    end
+  end
+end
+
+exit if ! options.write
+
+# Write back the unique identifiers
+write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+write_identifiers(svc_family, File.join(bdir, "service_family.txt"))

data/cpe-remap.yaml

@@ -1,4 +1,6 @@
 mappings:
+  alpine:
+    vendor: alpinelinux
   apache:
     vendor: apache
     products:
@@ -45,10 +47,17 @@ mappings:
     vendor: ibm
     products:
       lotus_domino: lotus_domino_server
+      os/400: os_400
+      z/os: z\/os
+  jamf:
+    products:
+      jamf_pro: jamf
   juniper:
     vendor: juniper
     products:
       junos_os: junos
+  kibana:
+    vendor: elasticsearch
   linux:
     vendor: linux
     products:
@@ -94,6 +103,11 @@ mappings:
     vendor: paloaltonetworks
     products:
       pa_firewall: pan-os
+  parallels:
+    products:
+      plesk: parallels_plesk_panel
+  plesk:
+    vendor: parallels
   proftpd_project:
     vendor: proftpd
   realvnc_ltd.:
@@ -113,6 +127,13 @@ mappings:
     vendor: sun
     products:
       solaris: sunos
+  tandberg:
+    vendor: cisco
+  tightvnc:
+    products:
+      desktop: tightvnc
+  ubiquiti:
+    vendor: ui
   ubuntu:
     vendor: canonical
     products:

data/features/match.feature

@@ -1,4 +1,5 @@
 Feature: Match
+  @no-clobber
   Scenario: Finds matches
     When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
     Then it should pass with:
@@ -7,6 +8,7 @@ Feature: Match
       MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
       """
 
+  @no-clobber
   Scenario: Fails at finding matches
     When I run `recog_match failing_banners_fingerprints.xml sample_banner.txt`
     Then it should pass with:
@@ -15,6 +17,7 @@ Feature: Match
       FAIL: polaris FTP server (SunOS 5.8) ready
       """
 
+  @no-clobber
   Scenario: Finds multiple matches
     When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
     Then it should pass with:
@@ -23,6 +26,7 @@ Feature: Match
       MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
       """
 
+  @no-clobber
   Scenario: Finds first matches using no-multi-match flag
     When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
     Then it should pass with:

data/features/support/aruba.rb

@@ -0,0 +1,3 @@
+Aruba.configure do |config|
+  config.working_directory = 'features/data'
+end

data/features/verify.feature

@@ -1,4 +1,5 @@
 Feature: Verify
+  @no-clobber
   Scenario: No tests
     When I run `recog_verify no_tests.xml`
     Then it should pass with:
@@ -6,6 +7,7 @@ Feature: Verify
       SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Successful tests
     When I run `recog_verify successful_tests.xml`
     Then it should pass with:
@@ -13,6 +15,7 @@ Feature: Verify
       SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Tests with warnings, warnings enabled
     When I run `recog_verify tests_with_warnings.xml`
     Then it should fail with:
@@ -23,6 +26,7 @@ Feature: Verify
       """
     And the exit status should be 2
 
+  @no-clobber
   Scenario: Tests with warnings, warnings disabled
     When I run `recog_verify --no-warnings tests_with_warnings.xml`
     Then it should pass with:
@@ -30,6 +34,7 @@ Feature: Verify
       SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Tests with failures
     When I run `recog_verify tests_with_failures.xml`
     Then it should fail with:

data/identifiers/README.md

@@ -0,0 +1,47 @@
+# Recog: Identifiers
+
+This directory contains lists of standard identifiers for mapping Recog matches. The goal is define a standard set of constants to represent known software, hardware, vendors, and categories.
+
+This is currently incomplete and will be updated as standardization work moves forward.
+
+Fingerprints should use these identifiers whenever possible; if a different name or syntax for a given identifier is preferred, this should be implemented in the application through a mapping function.
+
+## Lists
+
+### Vendors
+
+`vendor.txt` defines known vendor names, covering services, operating systems, and hardware.
+
+### Operating Systems
+
+`os_architecture.txt` defines known CPU types.
+
+`os_product.txt` defines known operating system names.
+
+`os_family.txt` defines known operating system families.
+
+`os_device.txt` defines known types of devices by function or purpose.
+
+### Services
+
+`service_product.txt` defines known service product names.
+
+`service_family.txt` defines known service product families.
+
+### Software
+
+`software_product.txt` defines known software product names.
+
+`software_family.txt` defines known software product families.
+
+`software_class.txt` defines known types of software by function or purpose.
+
+## Pending Work
+
+  * All existing fingerprints should be correlated against these lists to identify mismatches and updated accordingly.
+
+  * All net new identifiers from the existing fingerprints should be merged into these lists.
+
+  * All fingerprint assertions should be enumerated, documented, and standardized where possible (`host.mac`, etc).
+
+  * Hardware identifiers should be enumerated, consolidated, and standardized.

data/identifiers/os_architecture.txt

@@ -0,0 +1,20 @@
+680xx
+880xx
+Alpha
+ARM
+ARM64
+ia64
+iSeries
+MIPS
+MIPS64
+MPC
+PA
+PowerPC
+pSeries
+Risc
+s390
+s390x
+Sparc
+System/6000
+x86
+x86_64

data/identifiers/os_device.txt

@@ -0,0 +1,52 @@
+BBS
+Bridge
+Broadband router
+Console server
+CSU/DSU
+Domain controller
+DSLAM
+Encryption accelerator
+Fax server
+File server
+Firewall
+Game console
+General
+Hub
+IPS
+KVM
+Lights Out Management
+Load balancer
+Mainframe
+Management
+Monitoring
+Multifunction Device
+Multiplexer
+NAC
+Network management device
+PBX
+PDA
+Point of sale
+Power device
+Print server
+Printer
+Remote access server
+Router
+Scanner
+Server
+Specialized
+Storage
+Switch
+Tablet
+Tape library
+Telecom
+Terminal server
+UPS
+Virtualization host
+VoIP
+VPN
+WAP
+Web cam
+Web proxy
+Web server
+Workstation
+X terminal

data/identifiers/os_family.txt

@@ -0,0 +1,160 @@
+A/UX
+Adaptive Security Appliance
+Aficio
+AirPort
+AIX
+AmigaOS
+AMOS
+AOS
+AOS/VS
+APC
+Atari
+AtheOS
+AuspexOS
+BeOS
+BIG-IP
+Brocade
+BSD
+BSDi
+CacheOS
+CatOS
+CBOS
+CentOS
+Check Point
+Clix
+ComOS
+ConnectUPS
+Content Networking System
+ConvexOS
+Cyras
+CyROS
+DART
+Data ONTAP
+Dell Remote Access Controller
+DG/UX
+Digital UNIX
+Domain/OS
+DOS
+Dynix
+Embedded
+ES
+ExtremeWare
+Firewall-1
+Fortinet
+FreeBSD
+GAiA
+GigaVUE HD
+GigaVUE TA
+HI-UX
+HP-UX
+Hurd
+iLO
+IM Series
+Imagistics
+Integrated Dell Remote Access Controller
+IOS
+IPS
+IPSO
+Irix
+Ironware
+JetDirect
+Junos
+KA9Q
+LaserJet
+Linux
+lwIP
+LynxOS
+Mac OS
+Mac OS X
+Mach
+Madge CrossFire
+MAXserver
+MedNet
+Minix
+MPE/iX
+MT
+MVS
+NC Series
+NetBSD
+NetCache
+Netopia
+NetOS
+NetStation
+NetVanta
+NetWare
+NewsOS
+Newton OS
+Nexpose
+NEXTSTEP
+NmpSW
+NX-OS
+OpenBSD
+OpenROUTE
+OpenServer
+OpenVMS
+OS/2
+OS/390
+OS/400
+OS-9
+PacketShaper pSOS
+PalmOS
+Palo Alto
+PAN-OS
+PIX
+Plan9
+ProCurve
+ProLiant
+QNX
+Raptor
+Reliant UNIX
+RISC OS
+RouterOS
+RS
+RT
+SAN-OS
+SCO UNIX
+ScreenOS
+SHARP AR Series
+SHARP MX Series
+SINIX
+Solaris
+SpeedTouch
+SPP-UX
+SSL-VPN
+StackTOS
+SunOS
+SVR4
+Tahoe OS
+Tandem NSK
+Taos
+ThreadX
+TINIOS
+TiOS
+TOPS-20
+Tru64 UNIX
+Ubuntu
+UCOS
+UCS
+Ultrasound Device
+Ultrix
+UnicOS
+Unisys
+UnixWare
+UX/4800
+VG200
+VirtuOS
+VM
+VM/CMS
+VM/ESA
+VMS
+VMware ESX/ESXi
+VOS
+VRP
+VxWorks
+WAAS
+Wide Format Printer
+Windows
+Worldgroup
+xMach
+z/OS
+ZyNOS