recog 2.3.7 → 2.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -2
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +2 -4
  6. data/Gemfile +2 -5
  7. data/bin/recog_standardize +118 -0
  8. data/cpe-remap.yaml +21 -0
  9. data/features/match.feature +4 -0
  10. data/features/support/aruba.rb +3 -0
  11. data/features/verify.feature +5 -0
  12. data/identifiers/README.md +47 -0
  13. data/identifiers/os_architecture.txt +20 -0
  14. data/identifiers/os_device.txt +52 -0
  15. data/identifiers/os_family.txt +160 -0
  16. data/identifiers/os_product.txt +199 -0
  17. data/identifiers/service_family.txt +185 -0
  18. data/identifiers/service_product.txt +255 -0
  19. data/identifiers/software_class.txt +26 -0
  20. data/identifiers/software_family.txt +91 -0
  21. data/identifiers/software_product.txt +333 -0
  22. data/identifiers/vendor.txt +405 -0
  23. data/lib/recog/version.rb +1 -1
  24. data/spec/lib/fingerprint_self_test_spec.rb +1 -1
  25. data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +1 -1
  26. data/update_cpes.py +1 -1
  27. data/xml/apache_os.xml +9 -0
  28. data/xml/architecture.xml +9 -5
  29. data/xml/dns_versionbind.xml +9 -11
  30. data/xml/ftp_banners.xml +47 -5
  31. data/xml/html_title.xml +310 -196
  32. data/xml/http_cookies.xml +13 -1
  33. data/xml/http_servers.xml +145 -7
  34. data/xml/http_wwwauth.xml +13 -1
  35. data/xml/mdns_device-info_txt.xml +20 -16
  36. data/xml/mysql_banners.xml +420 -38
  37. data/xml/ntp_banners.xml +16 -8
  38. data/xml/operating_system.xml +1 -0
  39. data/xml/rtsp_servers.xml +21 -21
  40. data/xml/sip_banners.xml +5 -41
  41. data/xml/sip_user_agents.xml +40 -1
  42. data/xml/smb_native_os.xml +1 -0
  43. data/xml/smtp_banners.xml +3 -4
  44. data/xml/snmp_sysdescr.xml +44 -28
  45. data/xml/ssh_banners.xml +78 -73
  46. data/xml/telnet_banners.xml +30 -9
  47. data/xml/x11_banners.xml +1 -1
  48. data/xml/x509_issuers.xml +7 -7
  49. data/xml/x509_subjects.xml +329 -28
  50. metadata +23 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8bce6ea617950159feebc525962a70eb2d04340cef05c75e522897c9c5bf780e
4
- data.tar.gz: d3fa168beab209826c82a45147d149c939e8a37bc0f2cf9fad31a35a0d2ec2df
3
+ metadata.gz: be02bd17e124bbded970024eb6634e60ce5a3764da67faecb0da21157179d6ed
4
+ data.tar.gz: 04eacbfe28e565b359b4b798f0ef171c97274f9d5a43f273b8f09972b999ad46
5
5
  SHA512:
6
- metadata.gz: 4b051ce5e7bb403b9367851befcc365052812d45a9d1702759241e01ccd34ea1fffaf077f5ab30e14bcb5b8dae4612df5e1c2d1b0e270e0d0a04d8ecb8368801
7
- data.tar.gz: d85bf09c0fa22d54ef00c66710794cfb5bebe3eca569f07236de6d11e9bf0f08373e8f2da8f21aeb12c213373f7c49e94c55470b0ab900d6cc06621db8100f0f
6
+ metadata.gz: bc69e881e5a68c16227bff868480d9f68760ddb72ea8203e89c5ce2bef06a5558ba5a0a730f44e1976f4481b9c7e543b1ec2685a4dc9755b6cfa967d69fe8b66
7
+ data.tar.gz: 87a4cc900949a643cb89c7c2058939fb5e49a2d875b72687dc7cb739b3969fdbc6e5dafc7dceace9e4c89e16b0e94ac0f9b0327ef563a74ac7f247c7173f58ff
data/.gitignore CHANGED
@@ -1,6 +1,4 @@
1
1
  # Ruby and tooling specific
2
- .ruby-version
3
- .ruby-gemset
4
2
  .yardoc
5
3
  coverage/
6
4
  doc/
@@ -8,6 +6,9 @@ pkg/
8
6
 
9
7
  /Gemfile.lock
10
8
 
9
+ #Python specific
10
+ venv
11
+
11
12
  # IDE specific
12
13
  .vscode/
13
14
  .idea
@@ -0,0 +1 @@
1
+ recog
@@ -0,0 +1 @@
1
+ 2.6.6
@@ -2,10 +2,8 @@ language: ruby
2
2
  sudo: false
3
3
  cache: bundler
4
4
  rvm:
5
- - '2.3.8'
6
- - '2.4.5'
7
- - '2.5.3'
8
- - '2.6.1'
5
+ - '2.5.8'
6
+ - '2.6.6'
9
7
  - 'jruby-9.1.9.0'
10
8
  jdk:
11
9
  - openjdk8
data/Gemfile CHANGED
@@ -1,13 +1,10 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
- gemspec
3
+ gemspec name: 'recog'
4
4
 
5
5
  gem 'nokogiri'
6
6
 
7
7
  group :test do
8
8
  gem 'rake'
9
- gem 'rspec', '>= 2.99'
10
- gem 'cucumber', '~> 1.3.8'
11
- gem 'aruba', '~> 0.5.3'
12
- gem 'regexp_parser', '~> 0.2.0'
9
+ gem 'regexp_parser'
13
10
  end
@@ -0,0 +1,118 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
4
+ require 'optparse'
5
+ require 'ostruct'
6
+ require 'recog'
7
+
8
+ def load_identifiers(path)
9
+ res = {}
10
+ File.readlines(path).map{|line| line.strip}.each do |ident|
11
+ res[ident] = true
12
+ end
13
+ return res
14
+ end
15
+
16
+ def write_identifiers(vals, path)
17
+ res = []
18
+ vals.each_pair do |k,v|
19
+ res = res.push(k)
20
+ end
21
+ res = res.sort.uniq
22
+ File.write(path, res.join("\n") + "\n")
23
+ end
24
+
25
+ bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
26
+
27
+ options = OpenStruct.new(write: false)
28
+ option_parser = OptionParser.new do |opts|
29
+ opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
30
+ opts.separator "Verifies that each fingerprint asserts known identifiers."
31
+ opts.separator ""
32
+ opts.separator "Options"
33
+
34
+ opts.on("-w", "--write") do
35
+ options.write = true
36
+ end
37
+
38
+ opts.on("-h", "--help", "Show this message.") do
39
+ puts opts
40
+ exit
41
+ end
42
+ end
43
+ option_parser.parse!(ARGV)
44
+
45
+ if ARGV.empty?
46
+ $stderr.puts 'Missing XML fingerprint files'
47
+ puts option_parser
48
+ exit(1)
49
+ end
50
+
51
+ # Load the unique identifiers
52
+ vendors = load_identifiers(File.join(bdir, "vendor.txt"))
53
+ os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
54
+ os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
55
+ os_family = load_identifiers(File.join(bdir, "os_family.txt"))
56
+ os_device = load_identifiers(File.join(bdir, "os_device.txt"))
57
+ svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
58
+ svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
59
+
60
+ ARGV.each do |arg|
61
+ Dir.glob(arg).each do |file|
62
+ ndb = Recog::DB.new(file)
63
+ ndb.fingerprints.each do |f|
64
+ f.params.each do |k,v|
65
+ paramIndex, val = v
66
+ next if paramIndex != 0
67
+ case k
68
+ when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
69
+ if ! vendors[val]
70
+ puts "VENDOR MISSING: #{val}"
71
+ vendors[val] = true
72
+ end
73
+ when "os.product"
74
+ if ! os_prod[val]
75
+ puts "OS PRODUCT MISSING: #{val}"
76
+ os_prod[val] = true
77
+ end
78
+ when "os.arch"
79
+ if ! os_arch[val]
80
+ puts "OS ARCH MISSING: #{val}"
81
+ os_arch[val] = true
82
+ end
83
+ when "os.family"
84
+ if ! os_family[val]
85
+ puts "OS FAMILY MISSING: #{val}"
86
+ os_family[val] = true
87
+ end
88
+ when "os.device"
89
+ if ! os_device[val]
90
+ puts "OS DEVICE MISSING: #{val}"
91
+ os_device[val] = true
92
+ end
93
+ when "service.product"
94
+ if ! svc_prod[val]
95
+ puts "SERVICE PRODUCT MISSING: #{val}"
96
+ svc_prod[val] = true
97
+ end
98
+ when "service.family"
99
+ if ! svc_family[val]
100
+ puts "SERVICE FAMILY MISSING: #{val}"
101
+ svc_family[val] = true
102
+ end
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ exit if ! options.write
110
+
111
+ # Write back the unique identifiers
112
+ write_identifiers(vendors, File.join(bdir, "vendor.txt"))
113
+ write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
114
+ write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
115
+ write_identifiers(os_family, File.join(bdir, "os_family.txt"))
116
+ write_identifiers(os_device, File.join(bdir, "os_device.txt"))
117
+ write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
118
+ write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
@@ -1,4 +1,6 @@
1
1
  mappings:
2
+ alpine:
3
+ vendor: alpinelinux
2
4
  apache:
3
5
  vendor: apache
4
6
  products:
@@ -45,10 +47,17 @@ mappings:
45
47
  vendor: ibm
46
48
  products:
47
49
  lotus_domino: lotus_domino_server
50
+ os/400: os_400
51
+ z/os: z\/os
52
+ jamf:
53
+ products:
54
+ jamf_pro: jamf
48
55
  juniper:
49
56
  vendor: juniper
50
57
  products:
51
58
  junos_os: junos
59
+ kibana:
60
+ vendor: elasticsearch
52
61
  linux:
53
62
  vendor: linux
54
63
  products:
@@ -94,6 +103,11 @@ mappings:
94
103
  vendor: paloaltonetworks
95
104
  products:
96
105
  pa_firewall: pan-os
106
+ parallels:
107
+ products:
108
+ plesk: parallels_plesk_panel
109
+ plesk:
110
+ vendor: parallels
97
111
  proftpd_project:
98
112
  vendor: proftpd
99
113
  realvnc_ltd.:
@@ -113,6 +127,13 @@ mappings:
113
127
  vendor: sun
114
128
  products:
115
129
  solaris: sunos
130
+ tandberg:
131
+ vendor: cisco
132
+ tightvnc:
133
+ products:
134
+ desktop: tightvnc
135
+ ubiquiti:
136
+ vendor: ui
116
137
  ubuntu:
117
138
  vendor: canonical
118
139
  products:
@@ -1,4 +1,5 @@
1
1
  Feature: Match
2
+ @no-clobber
2
3
  Scenario: Finds matches
3
4
  When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
4
5
  Then it should pass with:
@@ -7,6 +8,7 @@ Feature: Match
7
8
  MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
8
9
  """
9
10
 
11
+ @no-clobber
10
12
  Scenario: Fails at finding matches
11
13
  When I run `recog_match failing_banners_fingerprints.xml sample_banner.txt`
12
14
  Then it should pass with:
@@ -15,6 +17,7 @@ Feature: Match
15
17
  FAIL: polaris FTP server (SunOS 5.8) ready
16
18
  """
17
19
 
20
+ @no-clobber
18
21
  Scenario: Finds multiple matches
19
22
  When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
20
23
  Then it should pass with:
@@ -23,6 +26,7 @@ Feature: Match
23
26
  MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
24
27
  """
25
28
 
29
+ @no-clobber
26
30
  Scenario: Finds first matches using no-multi-match flag
27
31
  When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
28
32
  Then it should pass with:
@@ -0,0 +1,3 @@
1
+ Aruba.configure do |config|
2
+ config.working_directory = 'features/data'
3
+ end
@@ -1,4 +1,5 @@
1
1
  Feature: Verify
2
+ @no-clobber
2
3
  Scenario: No tests
3
4
  When I run `recog_verify no_tests.xml`
4
5
  Then it should pass with:
@@ -6,6 +7,7 @@ Feature: Verify
6
7
  SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
7
8
  """
8
9
 
10
+ @no-clobber
9
11
  Scenario: Successful tests
10
12
  When I run `recog_verify successful_tests.xml`
11
13
  Then it should pass with:
@@ -13,6 +15,7 @@ Feature: Verify
13
15
  SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
14
16
  """
15
17
 
18
+ @no-clobber
16
19
  Scenario: Tests with warnings, warnings enabled
17
20
  When I run `recog_verify tests_with_warnings.xml`
18
21
  Then it should fail with:
@@ -23,6 +26,7 @@ Feature: Verify
23
26
  """
24
27
  And the exit status should be 2
25
28
 
29
+ @no-clobber
26
30
  Scenario: Tests with warnings, warnings disabled
27
31
  When I run `recog_verify --no-warnings tests_with_warnings.xml`
28
32
  Then it should pass with:
@@ -30,6 +34,7 @@ Feature: Verify
30
34
  SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
31
35
  """
32
36
 
37
+ @no-clobber
33
38
  Scenario: Tests with failures
34
39
  When I run `recog_verify tests_with_failures.xml`
35
40
  Then it should fail with:
@@ -0,0 +1,47 @@
1
+ # Recog: Identifiers
2
+
3
+ This directory contains lists of standard identifiers for mapping Recog matches. The goal is define a standard set of constants to represent known software, hardware, vendors, and categories.
4
+
5
+ This is currently incomplete and will be updated as standardization work moves forward.
6
+
7
+ Fingerprints should use these identifiers whenever possible; if a different name or syntax for a given identifier is preferred, this should be implemented in the application through a mapping function.
8
+
9
+ ## Lists
10
+
11
+ ### Vendors
12
+
13
+ `vendor.txt` defines known vendor names, covering services, operating systems, and hardware.
14
+
15
+ ### Operating Systems
16
+
17
+ `os_architecture.txt` defines known CPU types.
18
+
19
+ `os_product.txt` defines known operating system names.
20
+
21
+ `os_family.txt` defines known operating system families.
22
+
23
+ `os_device.txt` defines known types of devices by function or purpose.
24
+
25
+ ### Services
26
+
27
+ `service_product.txt` defines known service product names.
28
+
29
+ `service_family.txt` defines known service product families.
30
+
31
+ ### Software
32
+
33
+ `software_product.txt` defines known software product names.
34
+
35
+ `software_family.txt` defines known software product families.
36
+
37
+ `software_class.txt` defines known types of software by function or purpose.
38
+
39
+ ## Pending Work
40
+
41
+ * All existing fingerprints should be correlated against these lists to identify mismatches and updated accordingly.
42
+
43
+ * All net new identifiers from the existing fingerprints should be merged into these lists.
44
+
45
+ * All fingerprint assertions should be enumerated, documented, and standardized where possible (`host.mac`, etc).
46
+
47
+ * Hardware identifiers should be enumerated, consolidated, and standardized.
@@ -0,0 +1,20 @@
1
+ 680xx
2
+ 880xx
3
+ Alpha
4
+ ARM
5
+ ARM64
6
+ ia64
7
+ iSeries
8
+ MIPS
9
+ MIPS64
10
+ MPC
11
+ PA
12
+ PowerPC
13
+ pSeries
14
+ Risc
15
+ s390
16
+ s390x
17
+ Sparc
18
+ System/6000
19
+ x86
20
+ x86_64
@@ -0,0 +1,52 @@
1
+ BBS
2
+ Bridge
3
+ Broadband router
4
+ Console server
5
+ CSU/DSU
6
+ Domain controller
7
+ DSLAM
8
+ Encryption accelerator
9
+ Fax server
10
+ File server
11
+ Firewall
12
+ Game console
13
+ General
14
+ Hub
15
+ IPS
16
+ KVM
17
+ Lights Out Management
18
+ Load balancer
19
+ Mainframe
20
+ Management
21
+ Monitoring
22
+ Multifunction Device
23
+ Multiplexer
24
+ NAC
25
+ Network management device
26
+ PBX
27
+ PDA
28
+ Point of sale
29
+ Power device
30
+ Print server
31
+ Printer
32
+ Remote access server
33
+ Router
34
+ Scanner
35
+ Server
36
+ Specialized
37
+ Storage
38
+ Switch
39
+ Tablet
40
+ Tape library
41
+ Telecom
42
+ Terminal server
43
+ UPS
44
+ Virtualization host
45
+ VoIP
46
+ VPN
47
+ WAP
48
+ Web cam
49
+ Web proxy
50
+ Web server
51
+ Workstation
52
+ X terminal
@@ -0,0 +1,160 @@
1
+ A/UX
2
+ Adaptive Security Appliance
3
+ Aficio
4
+ AirPort
5
+ AIX
6
+ AmigaOS
7
+ AMOS
8
+ AOS
9
+ AOS/VS
10
+ APC
11
+ Atari
12
+ AtheOS
13
+ AuspexOS
14
+ BeOS
15
+ BIG-IP
16
+ Brocade
17
+ BSD
18
+ BSDi
19
+ CacheOS
20
+ CatOS
21
+ CBOS
22
+ CentOS
23
+ Check Point
24
+ Clix
25
+ ComOS
26
+ ConnectUPS
27
+ Content Networking System
28
+ ConvexOS
29
+ Cyras
30
+ CyROS
31
+ DART
32
+ Data ONTAP
33
+ Dell Remote Access Controller
34
+ DG/UX
35
+ Digital UNIX
36
+ Domain/OS
37
+ DOS
38
+ Dynix
39
+ Embedded
40
+ ES
41
+ ExtremeWare
42
+ Firewall-1
43
+ Fortinet
44
+ FreeBSD
45
+ GAiA
46
+ GigaVUE HD
47
+ GigaVUE TA
48
+ HI-UX
49
+ HP-UX
50
+ Hurd
51
+ iLO
52
+ IM Series
53
+ Imagistics
54
+ Integrated Dell Remote Access Controller
55
+ IOS
56
+ IPS
57
+ IPSO
58
+ Irix
59
+ Ironware
60
+ JetDirect
61
+ Junos
62
+ KA9Q
63
+ LaserJet
64
+ Linux
65
+ lwIP
66
+ LynxOS
67
+ Mac OS
68
+ Mac OS X
69
+ Mach
70
+ Madge CrossFire
71
+ MAXserver
72
+ MedNet
73
+ Minix
74
+ MPE/iX
75
+ MT
76
+ MVS
77
+ NC Series
78
+ NetBSD
79
+ NetCache
80
+ Netopia
81
+ NetOS
82
+ NetStation
83
+ NetVanta
84
+ NetWare
85
+ NewsOS
86
+ Newton OS
87
+ Nexpose
88
+ NEXTSTEP
89
+ NmpSW
90
+ NX-OS
91
+ OpenBSD
92
+ OpenROUTE
93
+ OpenServer
94
+ OpenVMS
95
+ OS/2
96
+ OS/390
97
+ OS/400
98
+ OS-9
99
+ PacketShaper pSOS
100
+ PalmOS
101
+ Palo Alto
102
+ PAN-OS
103
+ PIX
104
+ Plan9
105
+ ProCurve
106
+ ProLiant
107
+ QNX
108
+ Raptor
109
+ Reliant UNIX
110
+ RISC OS
111
+ RouterOS
112
+ RS
113
+ RT
114
+ SAN-OS
115
+ SCO UNIX
116
+ ScreenOS
117
+ SHARP AR Series
118
+ SHARP MX Series
119
+ SINIX
120
+ Solaris
121
+ SpeedTouch
122
+ SPP-UX
123
+ SSL-VPN
124
+ StackTOS
125
+ SunOS
126
+ SVR4
127
+ Tahoe OS
128
+ Tandem NSK
129
+ Taos
130
+ ThreadX
131
+ TINIOS
132
+ TiOS
133
+ TOPS-20
134
+ Tru64 UNIX
135
+ Ubuntu
136
+ UCOS
137
+ UCS
138
+ Ultrasound Device
139
+ Ultrix
140
+ UnicOS
141
+ Unisys
142
+ UnixWare
143
+ UX/4800
144
+ VG200
145
+ VirtuOS
146
+ VM
147
+ VM/CMS
148
+ VM/ESA
149
+ VMS
150
+ VMware ESX/ESXi
151
+ VOS
152
+ VRP
153
+ VxWorks
154
+ WAAS
155
+ Wide Format Printer
156
+ Windows
157
+ Worldgroup
158
+ xMach
159
+ z/OS
160
+ ZyNOS