recog 2.3.22 → 2.3.23

data/identifiers/service_product.txt

@@ -20,6 +20,7 @@ Active Directory Controller
 Active Intelligence Engine
 ActiveMQ
 AdGuard Home
+Adminer
 AirTunes
 Airflow
 Alteon Web Switch
@@ -30,6 +31,7 @@ Apache Tomcat HTTP Connector
 AppleShare IP Mail Server
 Application Load Balancer
 Application Protection System, Enterprise
+Application Server
 Application Server Portal
 Application Server Web Cache
 Appweb
@@ -50,6 +52,7 @@ BlackJumboDog
 BladeSystems
 Boa
 Bugzilla
+CALDERA
 CCProxy
 CMS
 CMS400.NET
@@ -90,6 +93,7 @@ CouchDB
 Couchbase Server
 Courier IMAP
 Courier POP
+Covenant
 Cowboy
 Coyote
 Cross Web Server
@@ -194,6 +198,7 @@ HttpProxy
 Hummingbird Exceed X server
 IBM Domino
 IIS
+ILOM
 IMail Server
 IOS
 IPVA
@@ -451,7 +456,9 @@ SmartDNS
 SmartSense Tool
 SmbFTPD
 Snowball
+Solr
 SonarQube
+Spark
 SpeedTouch
 Splunk
 Squeezebox
@@ -473,6 +480,7 @@ TCPIP POP server
 TUX Web Server
 Tableau Server
 TeamCity
+TeamSpeak
 Tengine
 TestCenter IQ
 Thin
@@ -531,6 +539,7 @@ WebGUI
 WebLogic
 WebServer
 WebShield
+WebSocket++
 WebSphere
 WebSphere Load Balancer
 WebTrends
@@ -581,6 +590,7 @@ gSOAP
 gdnsd
 httpd
 iLO
+iPlanet Web Server
 iScale
 inetutils ftpd
 ipGENADevice
@@ -590,6 +600,7 @@ libupnp
 lighttpd
 micro_httpd
 mini_httpd
+mitmproxy
 mod_access
 mod_access_compat
 mod_actions
@@ -737,17 +748,20 @@ mod_vhost_alias
 mod_watchdog
 mod_wsgi
 mod_xml2enc
+mongo-express
 nginx
 noVNC
 ntopng
 ownCloud Server
 perl
 pfSense
+phpMyAdmin
 qmail
 qpopper-mysql
 raptor
 rbldnsd
 sfcb
+sofia-sip
 sshlib
 thttpd
 tnftpd

data/identifiers/vendor.txt

@@ -36,6 +36,7 @@ Accelerated Technology
 AdGuard
 Adaptec
 AdminDroid
+Adminer
 Adobe
 Adtran
 Aerohive
@@ -227,6 +228,7 @@ Eclipse
 Ektron
 Elastic
 Eltek
+Eltex
 EmbedThis
 Embedthis
 Emby
@@ -410,6 +412,7 @@ Lucent
 Lynx Technology
 Lyris
 MBP Kommunikationssysteme GmbH
+MITRE
 MPI Technologies
 MPS Software
 MRV Communications
@@ -478,6 +481,7 @@ NetData
 NetIQ
 NetSarang Computer, Inc.
 NetWin
+Netgate
 Netgear
 Netia
 Netopia
@@ -645,6 +649,7 @@ Segger
 Sendmail
 Sequent
 Sercomm
+SerenityOS
 Serome Technology, Inc.
 Serv-U
 ServerTech
@@ -710,6 +715,7 @@ Tableau
 Tandberg
 Taobao
 Tasman Networks
+TeamSpeak
 Technicolor
 Tektronix
 Teldat H. Kruszynski, M. Cichocki Sp. J.
@@ -732,6 +738,8 @@ Tinyproxy Project
 Tivo
 Tobit Software
 Tokutek
+Tomato
+TomatoUSB
 Tor Project
 TornadoWeb
 Toshiba
@@ -797,7 +805,7 @@ XAMPP
 XFree86
 XRoads
 Xerox
-Xiongmai Tech
+Xiongmai Technology
 Xiph
 Xiph.org
 Xitami
@@ -812,6 +820,7 @@ Youngzsoft
 ZMailer
 ZTE
 Zabbix
+Zaphoyd Studios
 ZebraNet
 Zed Shaw
 Zyxel
@@ -833,14 +842,17 @@ iXsystems
 innovaphone
 libssh
 lighttpd
+mitmproxy
 mod_ssl
 mod_wsgi
+mongo-express Project
 nginx
 ninenines
 noVNC
 ntop
 ownCloud
 pfSense
+phpMyAdmin
 port25
 qmail
 rPath

data/lib/recog/fingerprint.rb

@@ -5,6 +5,7 @@ module Recog
 class Fingerprint
   require 'set'
 
+  require 'recog/fingerprint_parse_error'
   require 'recog/fingerprint/regexp_factory'
   require 'recog/fingerprint/test'
 
@@ -28,20 +29,27 @@ class Fingerprint
   # @return (see #parse_examples)
   attr_reader :tests
 
+  # The line number of the XML entity in the source file for this
+  # fingerprint.
+  #
+  # @return [Integer] The line number of this entity.
+  attr_reader :line
+
   # @param xml [Nokogiri::XML::Element]
   # @param match_key [String] See Recog::DB
   # @param protocol [String] Protocol such as ftp, mssql, http, etc.
-  # @param filepath [String] Directory path for fingerprint example files
-  def initialize(xml, match_key=nil, protocol=nil, filepath=nil)
+  # @param example_path [String] Directory path for fingerprint example files
+  def initialize(xml, match_key=nil, protocol=nil, example_path=nil)
     @match_key = match_key
     @protocol = protocol
     @name   = parse_description(xml)
     @regex  = create_regexp(xml)
+    @line   = xml.line
     @params = {}
     @tests = []
 
     @protocol.downcase! if @protocol
-    parse_examples(xml, filepath)
+    parse_examples(xml, example_path)
     parse_params(xml)
   end
 
@@ -161,6 +169,7 @@ class Fingerprint
     # look for the presence of test cases
     if tests.size == 0
       yield :warn, "'#{@name}' has no test cases"
+      return
     end
 
     # make sure each test case passes
@@ -226,7 +235,7 @@ class Fingerprint
       if !param_used
         message = "'#{@name}' is missing an example that checks for parameter '#{param_name}' " +
                   "which is derived from a capture group"
-        yield :warn, message
+        yield :fail, message
       end
     end
   end
@@ -249,9 +258,9 @@ class Fingerprint
   end
 
   # @param xml [Nokogiri::XML::Element]
-  # @param filepath [String] Directory path for fingerprint example files
+  # @param example_path [String] Directory path for fingerprint example files
   # @return [void]
-  def parse_examples(xml, filepath)
+  def parse_examples(xml, example_path)
     elements = xml.xpath('example')
 
     elements.each do |elem|
@@ -259,7 +268,12 @@ class Fingerprint
       attrs = elem.attributes.values.reduce({}) { |a,e| a.merge(e.name => e.value) }
       if attrs["_filename"]
         contents = ""
-        fn = File.join(filepath, attrs["_filename"])
+        filename = attrs["_filename"]
+        fn = File.expand_path(File.join(example_path, filename))
+        unless fn.start_with?(File.expand_path(example_path) + File::Separator)
+          raise FingerprintParseError.new("an example specifies an illegal file path '#{filename}'", line_number = @line)
+        end
+
         File.open(fn, "rb") do |file|
           contents = file.read
           contents.force_encoding(Encoding::ASCII_8BIT)

data/lib/recog/fingerprint_parse_error.rb

@@ -0,0 +1,10 @@
+module Recog
+  class FingerprintParseError < StandardError
+    attr_reader :line_number
+
+    def initialize(msg, line_number=nil)
+      @line_number = line_number
+      super(msg)
+    end
+  end
+end

data/lib/recog/verifier.rb

@@ -15,9 +15,9 @@ class Verifier
         fp.verify_params do |status, message|
           case status
           when :warn
-            reporter.warning "WARN: #{message}"
+            reporter.warning message, fp.line
           when :fail
-            reporter.failure "FAIL: #{message}"
+            reporter.failure message, fp.line
           when :success
             reporter.success(message)
           end
@@ -25,9 +25,9 @@ class Verifier
         fp.verify_tests do |status, message|
           case status
           when :warn
-            reporter.warning "WARN: #{message}"
+            reporter.warning message, fp.line
           when :fail
-            reporter.failure "FAIL: #{message}"
+            reporter.failure message, fp.line
           when :success
             reporter.success(message)
           end

data/lib/recog/verify_reporter.rb

@@ -24,15 +24,15 @@ class VerifyReporter
     formatter.success_message("#{padding}#{text}") if detail?
   end
 
-  def warning(text)
+  def warning(text, line=nil)
     return unless @options.warnings
     @warning_count += 1
-    formatter.warning_message("#{path_label}#{padding}#{text}")
+    formatter.warning_message("#{path_label(line)}#{padding}WARN: #{text}")
   end
 
-  def failure(text)
+  def failure(text, line=nil)
     @failure_count += 1
-    formatter.failure_message("#{path_label}#{padding}#{text}")
+    formatter.failure_message("#{path_label(line)}#{padding}FAIL: #{text}")
   end
 
   def print_name(fingerprint)
@@ -65,9 +65,10 @@ class VerifyReporter
     @options.detail
   end
 
-  def path_label
+  def path_label(line=nil)
     unless detail?
-      @path.to_s.empty? ? "" : "#{@path}: "
+      line_label = line ? line.to_s + ":" : ""
+      @path.to_s.empty? ? "" : "#{@path}:#{line_label} "
     end
   end
 

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = '2.3.22'
+  VERSION = '2.3.23'
 end

data/requirements.txt

@@ -1,2 +1,2 @@
-lxml==4.6.3
+lxml==4.6.5
 pyyaml

data/spec/data/external_example_fingerprint/hp_printer_ex_01.txt

@@ -0,0 +1 @@
+HP LaserJet 4100 Series

data/spec/data/external_example_fingerprint/hp_printer_ex_02.txt

@@ -0,0 +1 @@
+HP LaserJet 2200

data/spec/data/external_example_fingerprint.xml

@@ -0,0 +1,8 @@
+<fingerprints>
+  <fingerprint pattern="laserjet (.*)(?: series)?" flags="REG_ICASE">
+    <description>HP JetDirect Printer</description>
+    <example _filename="hp_printer_ex_01.txt"/>
+    <example _filename="hp_printer_ex_02.txt"/>
+    <param pos="0" name="service.vendor" value="HP"/>
+  </fingerprint>
+</fingerprints>

data/spec/data/external_example_illegal_path_fingerprint.xml

@@ -0,0 +1,7 @@
+<fingerprints>
+  <fingerprint pattern="laserjet (.*)(?: series)?" flags="REG_ICASE">
+    <description>HP JetDirect Printer</description>
+    <example _filename="../bad_path.txt"/>
+    <param pos="0" name="service.vendor" value="HP"/>
+  </fingerprint>
+</fingerprints>

data/spec/lib/recog/db_spec.rb

@@ -1,97 +1,120 @@
 require 'recog/db'
 
 describe Recog::DB do
-  let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
-  subject { Recog::DB.new(xml_file) }
 
   describe "#fingerprints" do
-    subject(:fingerprints) { described_class.new(xml_file).fingerprints }
+    context "with inline example content" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-    it { is_expected.to be_a(Enumerable) }
+      subject(:fingerprints) { described_class.new(xml_file).fingerprints }
 
-    context "with only a pattern" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[0] }
+      it { is_expected.to be_a(Enumerable) }
 
-      it "has a blank name with no description" do
-        expect(entry.name).to be_empty
-      end
+      context "with only a pattern" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[0] }
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq(".*\\(iSeries\\).*")
-      end
+        it "has a blank name with no description" do
+          expect(entry.name).to be_empty
+        end
 
-      it "has no params" do
-        expect(entry.params).to be_empty
-      end
+        it "has a pattern" do
+          expect(entry.regex.source).to eq(".*\\(iSeries\\).*")
+        end
+
+        it "has no params" do
+          expect(entry.params).to be_empty
+        end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
-    end
 
-    context "with params" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[1] }
+      context "with params" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[1] }
 
-      it "has a name" do
-        expect(entry.name).to eq('PalmOS')
-      end
+        it "has a name" do
+          expect(entry.name).to eq('PalmOS')
+        end
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq(".*\\(PalmOS\\).*")
-      end
+        it "has a pattern" do
+          expect(entry.regex.source).to eq(".*\\(PalmOS\\).*")
+        end
 
-      it "has params" do
-        expect(entry.params).to eq({"os.vendor"=>[1, "Palm"], "os.device"=>[2, "General"]})
-      end
+        it "has params" do
+          expect(entry.params).to eq({"os.vendor"=>[1, "Palm"], "os.device"=>[2, "General"]})
+        end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
-    end
 
-    context "with pattern flags" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[2] }
+      context "with pattern flags" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[2] }
 
-      it "has a name and only uses the first value" do
-        expect(entry.name).to eq('HP Designjet printer')
-      end
+        it "has a name and only uses the first value" do
+          expect(entry.name).to eq('HP Designjet printer')
+        end
 
-      it 'creates a Regexp with expected flags' do
-        expect(entry.regex).to be_a(Regexp)
-        expect(entry.regex.options).to eq(Recog::Fingerprint::RegexpFactory::DEFAULT_FLAGS | Regexp::IGNORECASE)
-      end
+        it 'creates a Regexp with expected flags' do
+          expect(entry.regex).to be_a(Regexp)
+          expect(entry.regex.options).to eq(Recog::Fingerprint::RegexpFactory::DEFAULT_FLAGS | Regexp::IGNORECASE)
+        end
 
-      it "has a pattern" do
-        expect(entry.regex).to be_a(Regexp)
-        expect(entry.regex.source).to eq("(designjet \\S+)")
-      end
+        it "has a pattern" do
+          expect(entry.regex).to be_a(Regexp)
+          expect(entry.regex.source).to eq("(designjet \\S+)")
+        end
+
+        it "has params" do
+          expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        end
 
-      it "has params" do
-        expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        it "has no tests" do
+          expect(entry.tests).to be_empty
+        end
       end
 
-      it "has no tests" do
-        expect(entry.tests).to be_empty
+      context "with test" do
+        subject(:entry) { described_class.new(xml_file).fingerprints[3] }
+
+        it "has a name" do
+          expect(entry.name).to eq('HP JetDirect Printer')
+        end
+
+        it "has a pattern" do
+          expect(entry.regex.source).to eq("laserjet (.*)(?: series)?")
+        end
+
+        it "has params" do
+          expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
+        end
+
+        it "has tests" do
+          expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
+        end
       end
     end
 
-    context "with test" do
-      subject(:entry) { described_class.new(xml_file).fingerprints[3] }
+    context "with external example content" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'external_example_fingerprint.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-      it "has a name" do
-        expect(entry.name).to eq('HP JetDirect Printer')
-      end
+      subject(:entry) { described_class.new(xml_file).fingerprints[0] }
 
-      it "has a pattern" do
-        expect(entry.regex.source).to eq("laserjet (.*)(?: series)?")
+      it "has tests" do
+        expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
       end
+    end
 
-      it "has params" do
-        expect(entry.params).to eq({"service.vendor"=>[0, "HP"]})
-      end
+    context "with external example content illegal path" do
+      let(:xml_file) { File.expand_path File.join('spec', 'data', 'external_example_illegal_path_fingerprint.xml') }
+      subject { Recog::DB.new(xml_file) }
 
-      it "has no tests" do
-        expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
+      it "raises an illegal file path error" do
+        expect { subject }.to raise_error(/an example specifies an illegal file path '.+'/)
       end
     end
   end

data/spec/lib/recog/fingerprint_spec.rb

@@ -75,7 +75,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[6]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -83,7 +83,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[7]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 
@@ -92,7 +92,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[8]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -100,7 +100,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[9]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 

data/spec/lib/recog/verify_reporter_spec.rb

@@ -22,12 +22,12 @@ describe Recog::VerifyReporter do
 
   describe "#report" do
     it "prints warnings" do
-      expect(formatter).to receive(:warning_message).with('a warning')
+      expect(formatter).to receive(:warning_message).with('WARN: a warning')
       run_report
     end
 
     it "prints failures" do
-      expect(formatter).to receive(:failure_message).with('a failure')
+      expect(formatter).to receive(:failure_message).with('FAIL: a failure')
       run_report
     end
 
@@ -50,12 +50,12 @@ describe Recog::VerifyReporter do
       end
 
       it "prints warnings" do
-        expect(formatter).to receive(:warning_message).with('   a warning')
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
         run_report
       end
 
       it "prints failures" do
-        expect(formatter).to receive(:failure_message).with('   a failure')
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
         run_report
       end
 
@@ -84,12 +84,12 @@ describe Recog::VerifyReporter do
       subject { Recog::VerifyReporter.new(double(detail: false, quiet: false, warnings: true), formatter, path) }
 
       it "prints warnings" do
-        expect(formatter).to receive(:warning_message).with("#{path}: a warning")
+        expect(formatter).to receive(:warning_message).with("#{path}: WARN: a warning")
         run_report
       end
 
       it "prints failures" do
-        expect(formatter).to receive(:failure_message).with("#{path}: a failure")
+        expect(formatter).to receive(:failure_message).with("#{path}: FAIL: a failure")
         run_report
       end
 
@@ -118,12 +118,12 @@ describe Recog::VerifyReporter do
       end
 
       it "prints warnings" do
-        expect(formatter).to receive(:warning_message).with('   a warning')
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
         run_report
       end
 
       it "prints failures" do
-        expect(formatter).to receive(:failure_message).with('   a failure')
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
         run_report
       end