recog 2.3.21 → 2.3.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 972b7cc1ae69526b61f221eeefce61d192ccf4b1603342f94195bf6cd2ddba95
-  data.tar.gz: 807831da5cdfd3160bca893367c92f4b817514758b996968829253f548d19709
+  metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
+  data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
 SHA512:
-  metadata.gz: a923e57f1f34fb74358756372fc3d3d08c20f0a0b9b1088905f57f4b09a2a56b2d9d4940d39e609ff50c164d15e285c9e1707032864d55f969a7ed4d72e68de5
-  data.tar.gz: 62d17cd2cdf9c3a6d35b36e4ace9c20744d42cfd99a2e90f65e463540fceb551f35f7572179af261c8881116aeadc51986e371631dc451df31081d52d79a58c1
+  metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
+  data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "11:00"
+  open-pull-requests-limit: 10

data/.github/workflows/verify.yml

@@ -0,0 +1,89 @@
+name: Verify
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'xml/**.xml'
+  pull_request:
+    paths:
+      - 'xml/**.xml'
+
+jobs:
+  standardize:
+    name: 'Standardize'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog standardize
+        run: bundle exec bin/recog_standardize xml/*.xml
+  ruby-verify:
+    name: 'Ruby Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Ruby implementation
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog verify
+        run: bundle exec recog_verify --no-warnings xml/*.xml
+  java-verify:
+    name: 'Java Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Java implementation
+        uses: actions/checkout@v2
+        with:
+          repository: rapid7/recog-java
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: '17'
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
+      - name: Run recog verify
+        run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
+  go-verify:
+    name: 'Go Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Go implementation
+        uses: actions/checkout@v2
+        with:
+          repository: RumbleDiscovery/recog-go
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.1'
+      - name: Run recog verify
+        run: go run cmd/recog_verify/main.go recog-content/xml/

data/CONTRIBUTING.md

@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
     git fetch --all
     ```
 
+1. Set up git hooks to help identify potential issues with your contributions:
+
+    ```bash
+    ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
+    ```
+
 [^back to top](#contributing-to-recog)
 
 ### Branch and Improve

data/README.md

@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
 </example>
 ````
 
+Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
+
+```
+xml/services.xml
+xml/services/file1
+xml/services/file2
+...
+```
+
+They can then be loaded using the `_filename` attribute:
+
+```xml
+<example _filename="file1"/>
+```
+
+This is useful for long examples.
+
 [^back to top](#recog-a-recognition-framework)
 
 ## Contributing

data/bin/recog_standardize

@@ -61,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
 svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
 svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
 
+missing_count = 0
 
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
@@ -70,6 +71,7 @@ ARGV.each do |arg|
         paramIndex, val = v
         if ! fields[k]
           puts "FIELD MISSING: #{k}"
+          missing_count += 1
           fields[k] = true
         end
         next if paramIndex != 0
@@ -79,51 +81,61 @@ ARGV.each do |arg|
         when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
           if ! vendors[val]
             puts "VENDOR MISSING: #{val}"
+            missing_count += 1
             vendors[val] = true
           end
         when "os.arch"
           if ! os_arch[val]
             puts "OS ARCH MISSING: #{val}"
+            missing_count += 1
             os_arch[val] = true
           end          
         when "os.product"
           if ! os_prod[val]
             puts "OS PRODUCT MISSING: #{val}"
+            missing_count += 1
             os_prod[val] = true
           end
         when "os.family"
           if ! os_family[val]
             puts "OS FAMILY MISSING: #{val}"
+            missing_count += 1
             os_family[val] = true
           end
         when "os.device"
           if ! os_device[val]
             puts "OS DEVICE MISSING: #{val}"
+            missing_count += 1
             os_device[val] = true
           end
         when "hw.product"
           if ! hw_prod[val]
             puts "HW PRODUCT MISSING: #{val}"
+            missing_count += 1
             hw_prod[val] = true
           end
         when "hw.family"
           if ! hw_family[val]
             puts "HW FAMILY MISSING: #{val}"
+            missing_count += 1
             hw_family[val] = true
           end
         when "hw.device"
           if ! hw_device[val]
             puts "HW DEVICE MISSING: #{val}"
+            missing_count += 1
             hw_device[val] = true
           end          
         when "service.product", "service.component.product"
           if ! svc_prod[val]
             puts "SERVICE PRODUCT MISSING: #{val}"
+            missing_count += 1
             svc_prod[val] = true
           end            
         when "service.family"
           if ! svc_family[val]
             puts "SERVICE FAMILY MISSING: #{val}"
+            missing_count += 1
             svc_family[val] = true
           end          
         end
@@ -132,17 +144,20 @@ ARGV.each do |arg|
   end
 end
 
-exit if ! options.write
+if options.write
+  # Write back the unique identifiers
+  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+  write_identifiers(fields, File.join(bdir, "fields.txt"))
+  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+  write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+  write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
+  write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
+  write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
+  write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+  write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+end
 
-# Write back the unique identifiers
-write_identifiers(vendors, File.join(bdir, "vendor.txt"))
-write_identifiers(fields, File.join(bdir, "fields.txt"))
-write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
-write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
-write_identifiers(os_family, File.join(bdir, "os_family.txt"))
-write_identifiers(os_device, File.join(bdir, "os_device.txt"))
-write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
-write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
-write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
-write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
-write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+exit_code = (missing_count > 0 ? 1 : 0)
+exit(exit_code)

data/bin/recog_verify

@@ -53,8 +53,7 @@ failures = 0
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
     ndb = Recog::DB.new(file)
-    options.fingerprints = ndb.fingerprints
-    verifier = Recog::VerifierFactory.build(options)
+    verifier = Recog::VerifierFactory.build(options, ndb)
     verified = verifier.verify
     failures += verifier.reporter.failure_count
     warnings += verifier.reporter.warning_count

data/cpe-remap.yaml

@@ -140,6 +140,8 @@ mappings:
     parallels:
       products:
         plesk: parallels_plesk_panel
+    phoenix_contact:
+      vendor: phoenixcontact
     plesk:
       vendor: parallels
     proftpd_project:
@@ -159,6 +161,8 @@ mappings:
         jboss_eap: jboss_enterprise_application_platform
         jbossweb: jboss_web_framework_kit
         red_hat_directory_server: directory_server
+    rundeck:
+      vendor: pagerduty
     serv-u:
       vendor: solarwinds
     squid_cache:
@@ -208,6 +212,7 @@ mappings:
     apple:
       products:
         ios: iphone_os
+        mac_os: macos
     brocade:
       vendor: broadcom
       products:
@@ -277,10 +282,14 @@ mappings:
         ilom: integrated_lights_out_manager_firmware
     palo_alto_networks:
       vendor: paloaltonetworks
+    phoenix_contact:
+      vendor: phoenixcontact
     red_hat:
       vendor: redhat
       products:
         fedora_core_linux: fedora_core
+    software_house:
+      vendor: swhouse
     sun:
       products:
         solaris: sunos
@@ -337,6 +346,10 @@ mappings:
       vendor: dell
       products:
         k1000: kace_k1000_systems_management_appliance
+    phoenix_contact:
+      vendor: phoenixcontact
+    software_house:
+      vendor: swhouse
     tandberg:
       vendor: cisco
     ubiquiti:

data/features/verify.feature

@@ -2,17 +2,17 @@ Feature: Verify
   @no-clobber
   Scenario: No tests
     When I run `recog_verify no_tests.xml`
-    Then it should pass with:
+    Then it should pass with exactly:
       """
-      SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
+      no_tests.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
       """
 
   @no-clobber
   Scenario: Successful tests
     When I run `recog_verify successful_tests.xml`
-    Then it should pass with:
+    Then it should pass with exactly:
       """
-      SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
+      successful_tests.xml: SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
       """
 
   @no-clobber
@@ -20,18 +20,18 @@ Feature: Verify
     When I run `recog_verify tests_with_warnings.xml`
     Then it should fail with:
       """
-      WARN: 'Pure-FTPd' has no test cases
-      WARN: 'Pure-FTPd' is missing an example that checks for parameter 'pureftpd.config' messsage which is derived from a capture group
-      SUMMARY: Test completed with 1 successful, 2 warnings, and 0 failures
+      tests_with_warnings.xml: WARN: 'Pure-FTPd' has no test cases
+      tests_with_warnings.xml: WARN: 'Pure-FTPd' is missing an example that checks for parameter 'pureftpd.config' which is derived from a capture group
+      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 2 warnings, and 0 failures
       """
     And the exit status should be 2
 
   @no-clobber
   Scenario: Tests with warnings, warnings disabled
     When I run `recog_verify --no-warnings tests_with_warnings.xml`
-    Then it should pass with:
+    Then it should pass with exactly:
       """
-      SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
+      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
       """
 
   @no-clobber
@@ -39,10 +39,10 @@ Feature: Verify
     When I run `recog_verify tests_with_failures.xml`
     Then it should fail with:
       """
-      FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
-      FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
-      FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
-      FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
-      SUMMARY: Test completed with 0 successful, 0 warnings, and 4 failures
+      tests_with_failures.xml: FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
+      tests_with_failures.xml: FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
+      tests_with_failures.xml: FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
+      tests_with_failures.xml: FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
+      tests_with_failures.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 4 failures
       """
     And the exit status should be 4

data/identifiers/fields.txt

@@ -2,15 +2,15 @@ agilent.serial
 apache.info
 apache.variant
 apache.variant.version
+aptinex.model
 chromecast.capabilities
 chromecast.generation
-chromecast.serial_number
 cisco.imc_model
-cisco.serial_number
+cisco.model
 cookie
 dell.service_tag
+digi.serial_number
 extron.model
-fortinet.serial_number
 host.domain
 host.ip
 host.mac
@@ -32,6 +32,7 @@ imail.eval
 jetty.info
 junction.cookie
 junction.name
+lantronix.serial_number
 lenovo.machine_model
 lenovo.machine_type
 linux.kernel.version
@@ -67,7 +68,6 @@ proftpd.server.name
 pureftpd.config
 python.version
 qpopper.version
-ruckus.serial_number
 securetransport.build
 sendmail.config.version
 sendmail.hpux.phne.version
@@ -100,5 +100,6 @@ system.time.millis
 tandberg.model
 thttpd.mx-patch
 tomcat.info
+unify.model
 wd2go.device_id
 zmailer.ident

data/identifiers/hw_device.txt

@@ -31,6 +31,7 @@ KVM
 Laptop
 Light Bulb
 Lights Out Management
+Media Player
 Media Receiver
 Media Server
 Mobile Phone
@@ -42,6 +43,7 @@ Network Audio
 Network Management Device
 PLC
 Power Device
+Power Meter
 Power Relay
 Powerline
 Print Server
@@ -53,6 +55,7 @@ SIP Device
 SIP Gateway
 Scanner
 Security Appliance
+Sensor
 Smart TV
 Storage
 Storage Appliance
@@ -62,6 +65,8 @@ Tablet
 Tape Library
 Telecom
 Test Instrument
+Thin Client
+UPS
 VPN
 Video Conference
 Video Conferencing
@@ -73,6 +78,7 @@ VoIP Switch
 Voice Appliance
 WAP
 WLAN Repeater
+Web Cam
 Whiteboard
 Wireless Controller
 Wireless Presenter

data/identifiers/hw_family.txt

@@ -9,6 +9,7 @@ Data ONTAP
 DiskStation
 Document Centre
 EDR
+Eurotherm
 Extended Systems ExtendNet
 FRITZ!Box
 FRITZ!Fon
@@ -30,6 +31,7 @@ ILOM
 IMDVR
 ION
 JetDirect
+LX Series
 LaserJet
 LinkCom Xpress
 MGate
@@ -51,6 +53,7 @@ Netscaler
 Network Audio
 Network Security Appliance
 Network Video Door Station
+OpenScape Desk Phone
 Optra
 Orbi
 POWER System
@@ -91,6 +94,7 @@ UniFi
 Unified Security Gateway
 VDX
 VSX
+VVX
 Vigor
 VoIP
 WD2GO
@@ -111,3 +115,7 @@ iPad Air
 iPad Pro
 iPad mini
 iPhone
+iPod
+iSTAR Door Controllers
+imageClass
+imageRunner