recog 2.3.18 → 2.3.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8494784d36925a1996a74aeca8ffd899bf049dd9c03a959c067caadc4e9ecea4
-  data.tar.gz: '050028f6d74692c1c14d7b4e3f8a8d7a2ef5a258f916b6bb50b911ceb132aa7e'
+  metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
+  data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
 SHA512:
-  metadata.gz: 640f3433ec61e3c320a380a761e569c4db81dcc29899a7f5232e5b800b2209c35bcfcc5fa990d56a42d4867908866a5cdaff44417bc4b81258127cc451795e55
-  data.tar.gz: c4ab3c69bcca4c2b1b47724ced99e7308b62dab88df94e3b031f3f45ad7debc56a4a8e5642aec66e370663419ae3c6e9d186ff2be42f41ed22e67c9ac00dac6e
+  metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
+  data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "11:00"
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: 'Ruby: ${{ matrix.ruby-version }}'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: |
+        bundle exec rake --version
+        bundle exec rake tests
+      env:
+        JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G

data/.github/workflows/verify.yml

@@ -0,0 +1,89 @@
+name: Verify
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'xml/**.xml'
+  pull_request:
+    paths:
+      - 'xml/**.xml'
+
+jobs:
+  standardize:
+    name: 'Standardize'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog standardize
+        run: bundle exec bin/recog_standardize xml/*.xml
+  ruby-verify:
+    name: 'Ruby Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Ruby implementation
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog verify
+        run: bundle exec recog_verify --no-warnings xml/*.xml
+  java-verify:
+    name: 'Java Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Java implementation
+        uses: actions/checkout@v2
+        with:
+          repository: rapid7/recog-java
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: '17'
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
+      - name: Run recog verify
+        run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
+  go-verify:
+    name: 'Go Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Go implementation
+        uses: actions/checkout@v2
+        with:
+          repository: RumbleDiscovery/recog-go
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.1'
+      - name: Run recog verify
+        run: go run cmd/recog_verify/main.go recog-content/xml/

data/CONTRIBUTING.md

@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
     git fetch --all
     ```
 
+1. Set up git hooks to help identify potential issues with your contributions:
+
+    ```bash
+    ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
+    ```
+
 [^back to top](#contributing-to-recog)
 
 ### Branch and Improve

data/README.md

@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
 </example>
 ````
 
+Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
+
+```
+xml/services.xml
+xml/services/file1
+xml/services/file2
+...
+```
+
+They can then be loaded using the `_filename` attribute:
+
+```xml
+<example _filename="file1"/>
+```
+
+This is useful for long examples.
+
 [^back to top](#recog-a-recognition-framework)
 
 ## Contributing

data/bin/recog_standardize

@@ -50,6 +50,7 @@ end
 
 # Load the unique identifiers
 vendors = load_identifiers(File.join(bdir, "vendor.txt"))
+fields = load_identifiers(File.join(bdir, "fields.txt"))
 os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
 os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
 os_family = load_identifiers(File.join(bdir, "os_family.txt"))
@@ -60,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
 svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
 svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
 
+missing_count = 0
 
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
@@ -67,6 +69,11 @@ ARGV.each do |arg|
     ndb.fingerprints.each do |f|
       f.params.each do |k,v|
         paramIndex, val = v
+        if ! fields[k]
+          puts "FIELD MISSING: #{k}"
+          missing_count += 1
+          fields[k] = true
+        end
         next if paramIndex != 0
         next if val.index("{") != nil
         next if val.strip == ""
@@ -74,51 +81,61 @@ ARGV.each do |arg|
         when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
           if ! vendors[val]
             puts "VENDOR MISSING: #{val}"
+            missing_count += 1
             vendors[val] = true
           end
         when "os.arch"
           if ! os_arch[val]
             puts "OS ARCH MISSING: #{val}"
+            missing_count += 1
             os_arch[val] = true
           end          
         when "os.product"
           if ! os_prod[val]
             puts "OS PRODUCT MISSING: #{val}"
+            missing_count += 1
             os_prod[val] = true
           end
         when "os.family"
           if ! os_family[val]
             puts "OS FAMILY MISSING: #{val}"
+            missing_count += 1
             os_family[val] = true
           end
         when "os.device"
           if ! os_device[val]
             puts "OS DEVICE MISSING: #{val}"
+            missing_count += 1
             os_device[val] = true
           end
         when "hw.product"
           if ! hw_prod[val]
             puts "HW PRODUCT MISSING: #{val}"
+            missing_count += 1
             hw_prod[val] = true
           end
         when "hw.family"
           if ! hw_family[val]
             puts "HW FAMILY MISSING: #{val}"
+            missing_count += 1
             hw_family[val] = true
           end
         when "hw.device"
           if ! hw_device[val]
             puts "HW DEVICE MISSING: #{val}"
+            missing_count += 1
             hw_device[val] = true
           end          
         when "service.product", "service.component.product"
           if ! svc_prod[val]
             puts "SERVICE PRODUCT MISSING: #{val}"
+            missing_count += 1
             svc_prod[val] = true
           end            
         when "service.family"
           if ! svc_family[val]
             puts "SERVICE FAMILY MISSING: #{val}"
+            missing_count += 1
             svc_family[val] = true
           end          
         end
@@ -127,16 +144,20 @@ ARGV.each do |arg|
   end
 end
 
-exit if ! options.write
+if options.write
+  # Write back the unique identifiers
+  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+  write_identifiers(fields, File.join(bdir, "fields.txt"))
+  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+  write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+  write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
+  write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
+  write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
+  write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+  write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+end
 
-# Write back the unique identifiers
-write_identifiers(vendors, File.join(bdir, "vendor.txt"))
-write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
-write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
-write_identifiers(os_family, File.join(bdir, "os_family.txt"))
-write_identifiers(os_device, File.join(bdir, "os_device.txt"))
-write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
-write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
-write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
-write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
-write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+exit_code = (missing_count > 0 ? 1 : 0)
+exit(exit_code)

data/bin/recog_verify

@@ -53,8 +53,7 @@ failures = 0
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
     ndb = Recog::DB.new(file)
-    options.fingerprints = ndb.fingerprints
-    verifier = Recog::VerifierFactory.build(options)
+    verifier = Recog::VerifierFactory.build(options, ndb)
     verified = verifier.verify
     failures += verifier.reporter.failure_count
     warnings += verifier.reporter.warning_count