recog 2.3.20 → 2.3.23

data/lib/recog/nizer.rb

@@ -8,13 +8,13 @@ class Nizer
   # Non-weighted host attributes that can be extracted from fingerprint matches
   HOST_ATTRIBUTES = %W{
     host.domain
-    host.id
     host.ip
     host.mac
     host.name
     host.time
     hw.device
     hw.family
+    hw.serial_number
     hw.product
     hw.vendor
   }
@@ -264,84 +264,3 @@ class Nizer
 
 end
 end
-
-=begin
-
-Current key names:
-
-  apache.info
-  apache.variant
-  apache.variant.version
-  cookie
-  host.domain
-  host.id
-  host.ip
-  host.mac
-  host.name
-  host.time
-  hw.device
-  hw.family
-  hw.product
-  hw.vendor
-  imail.eval
-  jetty.info
-  junction.cookie
-  junction.name
-  linux.kernel.version
-  loadbalancer.poolname
-  mdaemon.unregistered
-  mercur.os.info
-  metainfo.version
-  metainfo.version.version
-  ms.nttp.version
-  notes.build.version
-  notes.intl
-  ntmail.id
-  openssh.comment
-  openssh.cvepatch
-  os.arch
-  os.build
-  os.certainty
-  os.device
-  os.edition
-  os.family
-  os.product
-  os.vendor
-  os.version
-  os.version.version
-  os.version.version.version
-  postfix.os.info
-  postoffice.build
-  postoffice.id
-  proftpd.server.name
-  pureftpd.config
-  qpopper.version
-  sendmail.config.version
-  sendmail.hpux.phne.version
-  sendmail.vendor.version
-  service.certainty
-  service.component.family
-  service.component.product
-  service.component.vendor
-  service.component.version
-  service.family
-  service.product
-  service.vendor
-  service.version
-  service.version.version
-  service.version.version.version
-  service.version.version.version.version
-  service.version.version.version.version.version
-  siemens.model
-  snmp.fpmib.oid.1
-  snmp.fpmib.oid.2
-  system.time
-  system.time.format
-  system.time.micros
-  system.time.millis
-  thttpd.mx-patch
-  timeout
-  tomcat.info
-  zmailer.ident
-
-=end

data/lib/recog/verifier.rb

@@ -1,23 +1,23 @@
 module Recog
 class Verifier
-  attr_reader :fingerprints, :reporter
+  attr_reader :db, :reporter
 
-  def initialize(fingerprints, reporter)
-    @fingerprints = fingerprints
+  def initialize(db, reporter)
+    @db = db
     @reporter = reporter
   end
 
   def verify
-    reporter.report(fingerprints.count) do
-      fingerprints.each do |fp|
+    reporter.report(db.fingerprints.count) do
+      db.fingerprints.each do |fp|
         reporter.print_name fp
 
         fp.verify_params do |status, message|
           case status
           when :warn
-            reporter.warning "WARN: #{message}"
+            reporter.warning message, fp.line
           when :fail
-            reporter.failure "FAIL: #{message}"
+            reporter.failure message, fp.line
           when :success
             reporter.success(message)
           end
@@ -25,9 +25,9 @@ class Verifier
         fp.verify_tests do |status, message|
           case status
           when :warn
-            reporter.warning "WARN: #{message}"
+            reporter.warning message, fp.line
           when :fail
-            reporter.failure "FAIL: #{message}"
+            reporter.failure message, fp.line
           when :success
             reporter.success(message)
           end

data/lib/recog/verify_reporter.rb

@@ -3,14 +3,18 @@ class VerifyReporter
   attr_reader :formatter
   attr_reader :success_count, :warning_count, :failure_count
 
-  def initialize(options, formatter)
+  def initialize(options, formatter, path=nil)
     @options = options
     @formatter = formatter
+    @path = path
     reset_counts
   end
 
   def report(fingerprint_count)
     reset_counts
+    if detail? and !@path.to_s.empty?
+      formatter.status_message("\n#{@path}:\n")
+    end
     yield self
     summarize(fingerprint_count) unless @options.quiet
   end
@@ -20,15 +24,15 @@ class VerifyReporter
     formatter.success_message("#{padding}#{text}") if detail?
   end
 
-  def warning(text)
+  def warning(text, line=nil)
     return unless @options.warnings
     @warning_count += 1
-    formatter.warning_message("#{padding}#{text}")
+    formatter.warning_message("#{path_label(line)}#{padding}WARN: #{text}")
   end
 
-  def failure(text)
+  def failure(text, line=nil)
     @failure_count += 1
-    formatter.failure_message("#{padding}#{text}")
+    formatter.failure_message("#{path_label(line)}#{padding}FAIL: #{text}")
   end
 
   def print_name(fingerprint)
@@ -61,12 +65,19 @@ class VerifyReporter
     @options.detail
   end
 
+  def path_label(line=nil)
+    unless detail?
+      line_label = line ? line.to_s + ":" : ""
+      @path.to_s.empty? ? "" : "#{@path}:#{line_label} "
+    end
+  end
+
   def padding
     '   ' if @options.detail
   end
 
   def summary_line
-    summary = "SUMMARY: Test completed with "
+    summary = "#{path_label}SUMMARY: Test completed with "
     summary << "#{@success_count} successful"
     summary << ", #{@warning_count} warnings"
     summary << ", and #{@failure_count} failures"

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = '2.3.20'
+  VERSION = '2.3.23'
 end

data/requirements.txt

@@ -1,2 +1,2 @@
-lxml==4.6.3
+lxml==4.6.5
 pyyaml

data/spec/data/external_example_fingerprint/hp_printer_ex_01.txt

@@ -0,0 +1 @@
+HP LaserJet 4100 Series

data/spec/data/external_example_fingerprint/hp_printer_ex_02.txt

@@ -0,0 +1 @@
+HP LaserJet 2200

data/spec/data/external_example_fingerprint.xml

@@ -0,0 +1,8 @@
+<fingerprints>
+  <fingerprint pattern="laserjet (.*)(?: series)?" flags="REG_ICASE">
+    <description>HP JetDirect Printer</description>
+    <example _filename="hp_printer_ex_01.txt"/>
+    <example _filename="hp_printer_ex_02.txt"/>
+    <param pos="0" name="service.vendor" value="HP"/>
+  </fingerprint>
+</fingerprints>

data/spec/data/external_example_illegal_path_fingerprint.xml

@@ -0,0 +1,7 @@
+<fingerprints>
+  <fingerprint pattern="laserjet (.*)(?: series)?" flags="REG_ICASE">
+    <description>HP JetDirect Printer</description>
+    <example _filename="../bad_path.txt"/>
+    <param pos="0" name="service.vendor" value="HP"/>
+  </fingerprint>
+</fingerprints>

data/spec/lib/fingerprint_self_test_spec.rb

@@ -151,6 +151,7 @@ describe Recog::DB do
               # test any extractions specified in the example
               example.attributes.each_pair do |k,v|
                 next if k == '_encoding'
+                next if k == '_filename'
                 expect(match[k]).to eq(v), "Regex didn't extract expected value for fingerprint attribute #{k} -- got #{match[k]} instead of #{v}"
               end
             end

data/spec/lib/recog/db_spec.rb

@@ -1,97 +1,120 @@
 require 'recog/db'
 
 describe Recog::DB do
-  let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
-  subject { Recog::DB.new(xml_file) }
 
   describe "#fingerprints" do
-    subject(:fingerprints) { described_class.new(xml_file).fingerprints }
+    context "with inline example content" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-    it { is_expected.to be_a(Enumerable) }
+      subject(:fingerprints) { described_class.new(xml_file).fingerprints }
 
-    context "with only a pattern" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[0] }
+      it { is_expected.to be_a(Enumerable) }
 
-      it "has a blank name with no description" do
-        expect(entry.name).to be_empty
-      end
+      context "with only a pattern" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[0] }
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq(".*\\(iSeries\\).*")
-      end
+        it "has a blank name with no description" do
+          expect(entry.name).to be_empty
+        end
 
-      it "has no params" do
-        expect(entry.params).to be_empty
-      end
+        it "has a pattern" do
+          expect(entry.regex.source).to eq(".*\\(iSeries\\).*")
+        end
+
+        it "has no params" do
+          expect(entry.params).to be_empty
+        end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
-    end
 
-    context "with params" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[1] }
+      context "with params" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[1] }
 
-      it "has a name" do
-        expect(entry.name).to eq('PalmOS')
-      end
+        it "has a name" do
+          expect(entry.name).to eq('PalmOS')
+        end
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq(".*\\(PalmOS\\).*")
-      end
+        it "has a pattern" do
+          expect(entry.regex.source).to eq(".*\\(PalmOS\\).*")
+        end
 
-      it "has params" do
-        expect(entry.params).to eq({"os.vendor"=>[1, "Palm"], "os.device"=>[2, "General"]})
-      end
+        it "has params" do
+          expect(entry.params).to eq({"os.vendor"=>[1, "Palm"], "os.device"=>[2, "General"]})
+        end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
-    end
 
-    context "with pattern flags" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[2] }
+      context "with pattern flags" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[2] }
 
-      it "has a name and only uses the first value" do
-        expect(entry.name).to eq('HP Designjet printer')
-      end
+        it "has a name and only uses the first value" do
+          expect(entry.name).to eq('HP Designjet printer')
+        end
 
-      it 'creates a Regexp with expected flags' do
-        expect(entry.regex).to be_a(Regexp)
-        expect(entry.regex.options).to eq(Recog::Fingerprint::RegexpFactory::DEFAULT_FLAGS | Regexp::IGNORECASE)
-      end
+        it 'creates a Regexp with expected flags' do
+          expect(entry.regex).to be_a(Regexp)
+          expect(entry.regex.options).to eq(Recog::Fingerprint::RegexpFactory::DEFAULT_FLAGS | Regexp::IGNORECASE)
+        end
 
-      it "has a pattern" do
-        expect(entry.regex).to be_a(Regexp)
-        expect(entry.regex.source).to eq("(designjet \\S+)")
-      end
+        it "has a pattern" do
+          expect(entry.regex).to be_a(Regexp)
+          expect(entry.regex.source).to eq("(designjet \\S+)")
+        end
+
+        it "has params" do
+          expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        end
 
-      it "has params" do
-        expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+      context "with test" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[3] }
+
+        it "has a name" do
+          expect(entry.name).to eq('HP JetDirect Printer')
+        end
+
+        it "has a pattern" do
+          expect(entry.regex.source).to eq("laserjet (.*)(?: series)?")
+        end
+
+        it "has params" do
+          expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        end
+
+        it "has tests" do
+          expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
+        end
       end
     end
 
-    context "with test" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[3] }
+    context "with external example content" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'external_example_fingerprint.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-      it "has a name" do
-        expect(entry.name).to eq('HP JetDirect Printer')
-      end
+      subject(:entry) { described_class.new(xml_file).fingerprints[0] }
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq("laserjet (.*)(?: series)?")
+      it "has tests" do
+        expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
       end
+    end
 
-      it "has params" do
-        expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
-      end
+    context "with external example content illegal path" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'external_example_illegal_path_fingerprint.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-      it "has no tests" do
-        expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
+      it "raises an illegal file path error" do
+        expect { subject }.to raise_error(/an example specifies an illegal file path '.+'/)
       end
     end
   end

data/spec/lib/recog/fingerprint_spec.rb

@@ -75,7 +75,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[6]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -83,7 +83,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[7]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 
@@ -92,7 +92,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[8]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -100,7 +100,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[9]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 

data/spec/lib/recog/verify_reporter_spec.rb

@@ -7,6 +7,7 @@ describe Recog::VerifyReporter do
   let(:summary_line) do
     "SUMMARY: Test completed with 1 successful, 1 warnings, and 1 failures"
   end
+  let(:path) { "fingerprint.xml" }
 
   subject { Recog::VerifyReporter.new(double(detail: false, quiet: false, warnings: true), formatter) }
 
@@ -21,12 +22,12 @@ describe Recog::VerifyReporter do
 
   describe "#report" do
     it "prints warnings" do
-      expect(formatter).to receive(:warning_message).with('a warning')
+      expect(formatter).to receive(:warning_message).with('WARN: a warning')
       run_report
     end
 
     it "prints failures" do
-      expect(formatter).to receive(:failure_message).with('a failure')
+      expect(formatter).to receive(:failure_message).with('FAIL: a failure')
       run_report
     end
 
@@ -49,12 +50,80 @@ describe Recog::VerifyReporter do
       end
 
       it "prints warnings" do
-        expect(formatter).to receive(:warning_message).with('   a warning')
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
         run_report
       end
 
       it "prints failures" do
-        expect(formatter).to receive(:failure_message).with('   a failure')
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
+        run_report
+      end
+
+      it "prints the fingerprint count" do
+        expect(formatter).to receive(:status_message).with("\nVerified 1 fingerprints:")
+        run_report
+      end
+
+      it "prints summary" do
+        expect(formatter).to receive(:failure_message).with(summary_line)
+        run_report
+      end
+
+      context "with no fingerprint tests" do
+        let(:tests) { [] }
+
+        it "does not print the name" do
+          expect(formatter).not_to receive(:status_message).with("\na name")
+          run_report
+        end
+      end
+    end
+
+    context "with fingerprint path" do
+
+      subject { Recog::VerifyReporter.new(double(detail: false, quiet: false, warnings: true), formatter, path) }
+
+      it "prints warnings" do
+        expect(formatter).to receive(:warning_message).with("#{path}: WARN: a warning")
+        run_report
+      end
+
+      it "prints failures" do
+        expect(formatter).to receive(:failure_message).with("#{path}: FAIL: a failure")
+        run_report
+      end
+
+      it "prints summary" do
+        expect(formatter).to receive(:failure_message).with("#{path}: #{summary_line}")
+        run_report
+      end
+    end
+
+    context "with fingerprint path and detail" do
+      subject { Recog::VerifyReporter.new(double(detail: true, quiet: false, warnings: true), formatter, path) }
+
+      it "prints the fingerprint path" do
+        expect(formatter).to receive(:status_message).with("\n#{path}:\n")
+        run_report
+      end
+
+      it "prints the fingerprint name" do
+        expect(formatter).to receive(:status_message).with("\na name")
+        run_report
+      end
+
+      it "prints successes" do
+        expect(formatter).to receive(:success_message).with('   passed')
+        run_report
+      end
+
+      it "prints warnings" do
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
+        run_report
+      end
+
+      it "prints failures" do
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
         run_report
       end
 

data/tools/dev/hooks/pre-commit

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Hook script to verify changes about to be committed.
+# The hook should exit with non-zero status after issuing an appropriate
+# message if it wants to stop the commit.
+
+# Verify that each fingerprint asserts known identifiers.
+git diff --cached --name-only --diff-filter=ACM -z xml/*.xml | xargs -0 ./bin/recog_standardize --write
+
+# get status
+status=$?
+
+if [ $status -ne 0 ]; then
+    echo "Please review any new additions to the text files under 'identifiers/'."
+    echo "If any of these names are close to an existing name, update the offending"
+    echo "fingerprint to use the existing name instead. Once the fingerprints are fixed,"
+    echo "remove the 'extra' names from the identifiers files, and run the tool again."
+    exit 1
+fi
+
+exit 0