recog 2.3.6 → 2.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +17 -5
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +2 -4
  6. data/CONTRIBUTING.md +136 -37
  7. data/Gemfile +2 -5
  8. data/README.md +34 -29
  9. data/bin/recog_cleanup +16 -0
  10. data/bin/recog_standardize +142 -0
  11. data/cpe-remap.yaml +20 -0
  12. data/features/match.feature +4 -0
  13. data/features/support/aruba.rb +3 -0
  14. data/features/verify.feature +5 -0
  15. data/identifiers/README.md +56 -0
  16. data/identifiers/hw_device.txt +77 -0
  17. data/identifiers/hw_family.txt +96 -0
  18. data/identifiers/hw_product.txt +328 -0
  19. data/identifiers/os_architecture.txt +20 -0
  20. data/identifiers/os_device.txt +94 -0
  21. data/identifiers/os_family.txt +325 -0
  22. data/identifiers/os_product.txt +420 -0
  23. data/identifiers/service_family.txt +272 -0
  24. data/identifiers/service_product.txt +557 -0
  25. data/identifiers/software_class.txt +26 -0
  26. data/identifiers/software_family.txt +91 -0
  27. data/identifiers/software_product.txt +333 -0
  28. data/identifiers/vendor.txt +891 -0
  29. data/lib/recog/version.rb +1 -1
  30. data/requirements.txt +1 -1
  31. data/spec/lib/fingerprint_self_test_spec.rb +1 -1
  32. data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +1 -1
  33. data/update_cpes.py +4 -1
  34. data/xml/apache_modules.xml +292 -5
  35. data/xml/apache_os.xml +50 -2
  36. data/xml/architecture.xml +19 -7
  37. data/xml/dns_versionbind.xml +215 -11
  38. data/xml/favicons.xml +1701 -0
  39. data/xml/ftp_banners.xml +225 -12
  40. data/xml/h323_callresp.xml +112 -12
  41. data/xml/hp_pjl_id.xml +47 -5
  42. data/xml/html_title.xml +2371 -17
  43. data/xml/http_cookies.xml +82 -7
  44. data/xml/http_servers.xml +863 -43
  45. data/xml/http_wwwauth.xml +154 -27
  46. data/xml/imap_banners.xml +19 -13
  47. data/xml/ldap_searchresult.xml +81 -9
  48. data/xml/mdns_device-info_txt.xml +194 -17
  49. data/xml/mdns_workstation_txt.xml +4 -2
  50. data/xml/mysql_banners.xml +554 -45
  51. data/xml/mysql_error.xml +113 -6
  52. data/xml/nntp_banners.xml +10 -2
  53. data/xml/ntp_banners.xml +95 -11
  54. data/xml/operating_system.xml +90 -3
  55. data/xml/pop_banners.xml +30 -31
  56. data/xml/rsh_resp.xml +11 -2
  57. data/xml/rtsp_servers.xml +96 -0
  58. data/xml/sip_banners.xml +192 -17
  59. data/xml/sip_user_agents.xml +69 -3
  60. data/xml/smb_native_lm.xml +10 -2
  61. data/xml/smb_native_os.xml +80 -2
  62. data/xml/smtp_banners.xml +166 -9
  63. data/xml/smtp_debug.xml +6 -4
  64. data/xml/smtp_ehlo.xml +7 -5
  65. data/xml/smtp_expn.xml +13 -4
  66. data/xml/smtp_help.xml +23 -4
  67. data/xml/smtp_mailfrom.xml +5 -2
  68. data/xml/smtp_noop.xml +6 -5
  69. data/xml/smtp_quit.xml +5 -4
  70. data/xml/smtp_rcptto.xml +5 -2
  71. data/xml/smtp_rset.xml +4 -4
  72. data/xml/smtp_turn.xml +4 -4
  73. data/xml/smtp_vrfy.xml +14 -4
  74. data/xml/snmp_sysdescr.xml +863 -122
  75. data/xml/snmp_sysobjid.xml +47 -2
  76. data/xml/ssh_banners.xml +253 -78
  77. data/xml/telnet_banners.xml +419 -14
  78. data/xml/x11_banners.xml +27 -4
  79. data/xml/x509_issuers.xml +39 -15
  80. data/xml/x509_subjects.xml +545 -64
  81. metadata +30 -6
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
4
+ require 'optparse'
5
+ require 'ostruct'
6
+ require 'recog'
7
+
8
+ # Cleanup trailing whitespace around fingerprints
9
+ Dir[ File.expand_path(File.join(File.dirname(__FILE__), "..", "xml")) + "/*.xml" ].each do |f|
10
+ data = File.read(f).
11
+ gsub(/\s+$/, ''). # Trailing whitespace and empty lines
12
+ gsub("</fingerprint>", "</fingerprint>\n"). # Every fingerprint should have an empty line after it
13
+ gsub("-->", "-->\n") # Every comment should have an empty line after it
14
+
15
+ File.write(f, data)
16
+ end
@@ -0,0 +1,142 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
4
+ require 'optparse'
5
+ require 'ostruct'
6
+ require 'recog'
7
+
8
+ def load_identifiers(path)
9
+ res = {}
10
+ File.readlines(path).map{|line| line.strip}.each do |ident|
11
+ res[ident] = true
12
+ end
13
+ return res
14
+ end
15
+
16
+ def write_identifiers(vals, path)
17
+ res = []
18
+ vals.each_pair do |k,v|
19
+ res = res.push(k)
20
+ end
21
+ res = res.map{|x| x.strip}.select{|x| x.length > 0}.sort.uniq
22
+ File.write(path, res.join("\n") + "\n")
23
+ end
24
+
25
+ bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
26
+
27
+ options = OpenStruct.new(write: false)
28
+ option_parser = OptionParser.new do |opts|
29
+ opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
30
+ opts.separator "Verifies that each fingerprint asserts known identifiers."
31
+ opts.separator ""
32
+ opts.separator "Options"
33
+
34
+ opts.on("-w", "--write") do
35
+ options.write = true
36
+ end
37
+
38
+ opts.on("-h", "--help", "Show this message.") do
39
+ puts opts
40
+ exit
41
+ end
42
+ end
43
+ option_parser.parse!(ARGV)
44
+
45
+ if ARGV.empty?
46
+ $stderr.puts 'Missing XML fingerprint files'
47
+ puts option_parser
48
+ exit(1)
49
+ end
50
+
51
+ # Load the unique identifiers
52
+ vendors = load_identifiers(File.join(bdir, "vendor.txt"))
53
+ os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
54
+ os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
55
+ os_family = load_identifiers(File.join(bdir, "os_family.txt"))
56
+ os_device = load_identifiers(File.join(bdir, "os_device.txt"))
57
+ hw_prod = load_identifiers(File.join(bdir, "hw_product.txt"))
58
+ hw_family = load_identifiers(File.join(bdir, "hw_family.txt"))
59
+ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
60
+ svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
61
+ svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
62
+
63
+
64
+ ARGV.each do |arg|
65
+ Dir.glob(arg).each do |file|
66
+ ndb = Recog::DB.new(file)
67
+ ndb.fingerprints.each do |f|
68
+ f.params.each do |k,v|
69
+ paramIndex, val = v
70
+ next if paramIndex != 0
71
+ next if val.index("{") != -1
72
+ next if val.strip == ""
73
+ case k
74
+ when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
75
+ if ! vendors[val]
76
+ puts "VENDOR MISSING: #{val}"
77
+ vendors[val] = true
78
+ end
79
+ when "os.arch"
80
+ if ! os_arch[val]
81
+ puts "OS ARCH MISSING: #{val}"
82
+ os_arch[val] = true
83
+ end
84
+ when "os.product"
85
+ if ! os_prod[val]
86
+ puts "OS PRODUCT MISSING: #{val}"
87
+ os_prod[val] = true
88
+ end
89
+ when "os.family"
90
+ if ! os_family[val]
91
+ puts "OS FAMILY MISSING: #{val}"
92
+ os_family[val] = true
93
+ end
94
+ when "os.device"
95
+ if ! os_device[val]
96
+ puts "OS DEVICE MISSING: #{val}"
97
+ os_device[val] = true
98
+ end
99
+ when "hw.product"
100
+ if ! hw_prod[val]
101
+ puts "HW PRODUCT MISSING: #{val}"
102
+ hw_prod[val] = true
103
+ end
104
+ when "hw.family"
105
+ if ! hw_family[val]
106
+ puts "HW FAMILY MISSING: #{val}"
107
+ hw_family[val] = true
108
+ end
109
+ when "hw.device"
110
+ if ! hw_device[val]
111
+ puts "HW DEVICE MISSING: #{val}"
112
+ hw_device[val] = true
113
+ end
114
+ when "service.product"
115
+ if ! svc_prod[val]
116
+ puts "SERVICE PRODUCT MISSING: #{val}"
117
+ svc_prod[val] = true
118
+ end
119
+ when "service.family"
120
+ if ! svc_family[val]
121
+ puts "SERVICE FAMILY MISSING: #{val}"
122
+ svc_family[val] = true
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
129
+
130
+ exit if ! options.write
131
+
132
+ # Write back the unique identifiers
133
+ write_identifiers(vendors, File.join(bdir, "vendor.txt"))
134
+ write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
135
+ write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
136
+ write_identifiers(os_family, File.join(bdir, "os_family.txt"))
137
+ write_identifiers(os_device, File.join(bdir, "os_device.txt"))
138
+ write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
139
+ write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
140
+ write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
141
+ write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
142
+ write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
@@ -1,4 +1,6 @@
1
1
  mappings:
2
+ alpine:
3
+ vendor: alpinelinux
2
4
  apache:
3
5
  vendor: apache
4
6
  products:
@@ -45,10 +47,16 @@ mappings:
45
47
  vendor: ibm
46
48
  products:
47
49
  lotus_domino: lotus_domino_server
50
+ os/400: os_400
51
+ jamf:
52
+ products:
53
+ jamf_pro: jamf
48
54
  juniper:
49
55
  vendor: juniper
50
56
  products:
51
57
  junos_os: junos
58
+ kibana:
59
+ vendor: elasticsearch
52
60
  linux:
53
61
  vendor: linux
54
62
  products:
@@ -94,6 +102,11 @@ mappings:
94
102
  vendor: paloaltonetworks
95
103
  products:
96
104
  pa_firewall: pan-os
105
+ parallels:
106
+ products:
107
+ plesk: parallels_plesk_panel
108
+ plesk:
109
+ vendor: parallels
97
110
  proftpd_project:
98
111
  vendor: proftpd
99
112
  realvnc_ltd.:
@@ -113,6 +126,13 @@ mappings:
113
126
  vendor: sun
114
127
  products:
115
128
  solaris: sunos
129
+ tandberg:
130
+ vendor: cisco
131
+ tightvnc:
132
+ products:
133
+ desktop: tightvnc
134
+ ubiquiti:
135
+ vendor: ui
116
136
  ubuntu:
117
137
  vendor: canonical
118
138
  products:
@@ -1,4 +1,5 @@
1
1
  Feature: Match
2
+ @no-clobber
2
3
  Scenario: Finds matches
3
4
  When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
4
5
  Then it should pass with:
@@ -7,6 +8,7 @@ Feature: Match
7
8
  MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
8
9
  """
9
10
 
11
+ @no-clobber
10
12
  Scenario: Fails at finding matches
11
13
  When I run `recog_match failing_banners_fingerprints.xml sample_banner.txt`
12
14
  Then it should pass with:
@@ -15,6 +17,7 @@ Feature: Match
15
17
  FAIL: polaris FTP server (SunOS 5.8) ready
16
18
  """
17
19
 
20
+ @no-clobber
18
21
  Scenario: Finds multiple matches
19
22
  When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
20
23
  Then it should pass with:
@@ -23,6 +26,7 @@ Feature: Match
23
26
  MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
24
27
  """
25
28
 
29
+ @no-clobber
26
30
  Scenario: Finds first matches using no-multi-match flag
27
31
  When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
28
32
  Then it should pass with:
@@ -0,0 +1,3 @@
1
+ Aruba.configure do |config|
2
+ config.working_directory = 'features/data'
3
+ end
@@ -1,4 +1,5 @@
1
1
  Feature: Verify
2
+ @no-clobber
2
3
  Scenario: No tests
3
4
  When I run `recog_verify no_tests.xml`
4
5
  Then it should pass with:
@@ -6,6 +7,7 @@ Feature: Verify
6
7
  SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
7
8
  """
8
9
 
10
+ @no-clobber
9
11
  Scenario: Successful tests
10
12
  When I run `recog_verify successful_tests.xml`
11
13
  Then it should pass with:
@@ -13,6 +15,7 @@ Feature: Verify
13
15
  SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
14
16
  """
15
17
 
18
+ @no-clobber
16
19
  Scenario: Tests with warnings, warnings enabled
17
20
  When I run `recog_verify tests_with_warnings.xml`
18
21
  Then it should fail with:
@@ -23,6 +26,7 @@ Feature: Verify
23
26
  """
24
27
  And the exit status should be 2
25
28
 
29
+ @no-clobber
26
30
  Scenario: Tests with warnings, warnings disabled
27
31
  When I run `recog_verify --no-warnings tests_with_warnings.xml`
28
32
  Then it should pass with:
@@ -30,6 +34,7 @@ Feature: Verify
30
34
  SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
31
35
  """
32
36
 
37
+ @no-clobber
33
38
  Scenario: Tests with failures
34
39
  When I run `recog_verify tests_with_failures.xml`
35
40
  Then it should fail with:
@@ -0,0 +1,56 @@
1
+ # Recog: Identifiers
2
+
3
+ This directory contains lists of standard identifiers for mapping Recog matches. The goal is define a standard set of constants to represent known software, hardware, vendors, and categories.
4
+
5
+ This is currently incomplete and will be updated as standardization work moves forward.
6
+
7
+ Fingerprints should use these identifiers whenever possible; if a different name or syntax for a given identifier is preferred, this should be implemented in the application through a mapping function.
8
+
9
+ ## Lists
10
+
11
+ ### Vendors
12
+
13
+ `vendor.txt` defines known vendor names, covering services, operating systems, and hardware.
14
+
15
+ ### Operating Systems
16
+
17
+ `os_architecture.txt` defines known CPU types.
18
+
19
+ `os_product.txt` defines known operating system names.
20
+
21
+ `os_family.txt` defines known operating system families.
22
+
23
+ `os_device.txt` defines known types of devices by function or purpose.
24
+
25
+
26
+ ### Hardware
27
+
28
+ `hw_product.txt` defines known hardware product names.
29
+
30
+ `hw_family.txt` defines known hardware product families.
31
+
32
+ `hw_device.txt` defines known types of devices by function or purpose (overlaps with `os_device.txt`).
33
+
34
+ ### Services
35
+
36
+ `service_product.txt` defines known service product names.
37
+
38
+ `service_family.txt` defines known service product families.
39
+
40
+ ### Software
41
+
42
+ `software_product.txt` defines known software product names.
43
+
44
+ `software_family.txt` defines known software product families.
45
+
46
+ `software_class.txt` defines known types of software by function or purpose.
47
+
48
+ ## Pending Work
49
+
50
+ * All existing fingerprints should be correlated against these lists to identify mismatches and updated accordingly.
51
+
52
+ * All net new identifiers from the existing fingerprints should be merged into these lists.
53
+
54
+ * All fingerprint assertions should be enumerated, documented, and standardized where possible (`host.mac`, etc).
55
+
56
+ * Hardware identifiers should be enumerated, consolidated, and standardized.
@@ -0,0 +1,77 @@
1
+ ADSL Modem
2
+ AV Receiver
3
+ Access Control
4
+ Alarm Panel
5
+ Appliance
6
+ Audio Encoder
7
+ Broadband router
8
+ Building Automation
9
+ Cable Modem
10
+ Check Scanner
11
+ DOCSIS Cable Modem
12
+ DSL Modem
13
+ DVR
14
+ Data Terminal
15
+ Desktop
16
+ Device
17
+ Device Hub
18
+ Device Server
19
+ Display Controller
20
+ Environment Control
21
+ Ethernet Adapter
22
+ Firewall
23
+ HMI Controller
24
+ Handheld Scanner
25
+ IP Camera
26
+ IPS
27
+ Industrial Control
28
+ JTAG Adapter
29
+ KVM
30
+ Laptop
31
+ Light Bulb
32
+ Lights Out Management
33
+ Media Receiver
34
+ Media Server
35
+ Mobile Phone
36
+ Monitoring
37
+ Multifunction Device
38
+ NAS
39
+ Network Appliance
40
+ Network Audio
41
+ Network Management Device
42
+ PLC
43
+ Power Relay
44
+ Power device
45
+ Powerline
46
+ Print server
47
+ Printer
48
+ Router
49
+ SD-WAN Appliance
50
+ SIP Device
51
+ SIP Gateway
52
+ Scanner
53
+ Security Appliance
54
+ Smart TV
55
+ Storage
56
+ Storage Appliance
57
+ Support Appliance
58
+ Switch
59
+ Tablet
60
+ Tape Library
61
+ Telecom
62
+ Test Instrument
63
+ VPN
64
+ Video Conference
65
+ Video Conferencing
66
+ Video Decoder
67
+ Video Encoder
68
+ VoIP
69
+ VoIP Server
70
+ VoIP Switch
71
+ Voice Appliance
72
+ WAP
73
+ WLAN Repeater
74
+ Web cam
75
+ Whiteboard
76
+ Wireless Controller
77
+ Wireless Presenter
@@ -0,0 +1,96 @@
1
+ AR Series
2
+ Adaptive Security Appliance
3
+ Aficio
4
+ AirPort
5
+ Apple TV
6
+ Communication Manager
7
+ DVR
8
+ DiskStation
9
+ Document Centre
10
+ Extended Systems ExtendNet
11
+ FRITZ!Box
12
+ FRITZ!Powerline
13
+ FRITZ!WLAN Repeater
14
+ FS
15
+ Firewall-1
16
+ Forms Printer
17
+ FortiGate
18
+ GW25
19
+ GXV
20
+ GXW
21
+ HDX
22
+ HandyTone
23
+ HomePod
24
+ Honeywell
25
+ ILOM
26
+ IMDVR
27
+ ION
28
+ JetDirect
29
+ LaserJet
30
+ LinkCom Xpress
31
+ MGate
32
+ MPEG4 DVR
33
+ MT
34
+ MX Series
35
+ Mac mini
36
+ MacBook
37
+ MacBook Pro
38
+ MegaRAC
39
+ Multifunction
40
+ My Book
41
+ NE
42
+ NPort
43
+ NetVanta
44
+ Network Audio
45
+ Network Video Door Station
46
+ Optra
47
+ Orbi
48
+ POWER System
49
+ Phaser
50
+ Primergy
51
+ Pro Series
52
+ RMX
53
+ ReadyNAS
54
+ RealPresence
55
+ RealPresence Group
56
+ Router
57
+ S500 Range
58
+ SIP Device
59
+ SIP Gateway
60
+ Service Access Switch
61
+ Service Router
62
+ SoundPoint
63
+ SoundTouch
64
+ SpeedTouch
65
+ Speedport
66
+ Storage
67
+ Sun Fire
68
+ Sunny
69
+ Switch
70
+ System X
71
+ TASKalfa
72
+ TelePresence
73
+ Time Capsule
74
+ TippingPoint
75
+ Turbo Station
76
+ UniFi
77
+ Unified Security Gateway
78
+ VSX
79
+ VoIP
80
+ WD2GO
81
+ WiMax
82
+ Wide Format
83
+ Wide Format Printer
84
+ WorkCentre
85
+ WorkCentre Pro
86
+ Xserve
87
+ ZXDSL
88
+ ZXHN
89
+ ZXV
90
+ iLO
91
+ iMac
92
+ iPad
93
+ iPad Air
94
+ iPad Pro
95
+ iPad mini
96
+ iPhone