fluent-plugin-shodan 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2fe44cee9c9cd48566c10962f21e41a17e533e64922ab9310279e4e04c0cc8f8
-  data.tar.gz: 54775456dd2156f19b1f77944438302f9ec4bce33040478e4521aad3028c4d2f
+  metadata.gz: 6433f3162ef921f1b577f5489f802a1a6d88d561026a4274440732d326e55e88
+  data.tar.gz: 6d97984422e0d0bcaada2ac521a47d8fcd4049e437327fd03e3c39aac11858e5
 SHA512:
-  metadata.gz: 7a81fffe6a3a5266936f8372ca518ca8a67e36dc9d72290b0c6aea97ceb411d6d22eef7e4e67b4a45dfa1a5bf30e61db318ff0e056399fd0fde5c67884d038d2
-  data.tar.gz: c895e61fa305082e4b98c206bb9886352a8c23257a2fa58cb37326493dfa343094fd768dbb2fb492c8660977b793a50a34a683813842bcda46e8f90c101f57fa
+  metadata.gz: 775543570f5613db0515f258ca8b88b034f87e6265d954b82f59b21e09befee57e0f162f79d8ab6fd8b6e932427bd8d4e242c2dbec8b96a57477c0535d97fd06
+  data.tar.gz: 82e80191ff18df2b6f393e6b2330f61a273d1a786863f58041c0d0aeef14c79e16f41bbd36febe625d9f5b7005b7ffd1dbd3bf3a792d464b89126d4715eaef6b

data/README.md

@@ -83,9 +83,11 @@ Default value: `3600`.
 
 The tag to apply to each shodan entries.
 
-#### query (string) (required)
+#### query (string) (optional)
 
-The Shodan query to execute.
+The Shodan query to execute. The query can be empty if at least one filter is set.
+
+Default: `nil`
 
 #### max_pages (integer) (optional)
 
@@ -93,6 +95,16 @@ The maximum amount of pages to crawl. A 0 or negative value means to crawl all p
 
 Default value: `1`.
 
+#### filter (optional) (multi)
+
+##### name (string) (required)
+
+The name of the filter to be added to the query. Full filters list is available on the [Shodan filter reference page](https://www.shodan.io/search/filters). The filter can be negated by prepending `-` to the filter name (ex: `name -port`).
+
+##### value (string) (required)
+
+The value to be passed to the filter.
+
 ## Shodan Stream
 
 WIP
@@ -103,7 +115,7 @@ WIP
 
 ```
 <source>
-  @type shodan_search
+  @type shodan_alert
   interval 15m
   alert_id GA3FRJ1HJNDPORHV
   api_key 1234567890AZERTYUIOP

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/lib/fluent/plugin/in_shodan_search.rb

@@ -7,6 +7,25 @@ module Fluent::Plugin
 
     helpers :timer
 
+    SUPPORTED_FILTERS = [
+      'asn','city','country','cpe','device','geo','has_ipv6','has_screenshot',
+      'has_ssl','has_vuln','hash','hostname','ip','isp','link','net','org','os',
+      'port','postal','product','region','scan','shodan.module','state',
+      'version','screenshot.label','cloud.provider','cloud.region',
+      'cloud.service','http.component','http.component_category',
+      'http.favicon.hash','http.html','http.html_hash','http.robots_hash',
+      'http.securitytxt','http.status','http.title','http.waf','bitcoin.ip',
+      'bitcoin.ip_count','bitcoin.port','bitcoin.version','snmp.contact',
+      'snmp.location','snmp.name','ssl','ssl.alpn','ssl.cert.alg',
+      'ssl.cert.expired','ssl.cert.extension','ssl.cert.fingerprint',
+      'ssl.cert.issuer.cn','ssl.cert.pubkey.bits','ssl.cert.pubkey.type',
+      'ssl.cert.serial','ssl.cert.subject.cn','ssl.chain_count',
+      'ssl.cipher.bits','ssl.cipher.name','ssl.cipher.version','ssl.ja3s',
+      'ssl.jarm','ssl.version','ntp.ip','ntp.ip_count','ntp.more','ntp.port',
+      'telnet.do','telnet.dont','telnet.option','telnet.will','telnet.wont',
+      'ssh.hassh','ssh.type', 'tag', 'vuln'
+    ]
+
     desc "The API key to connect to the Shodan API."
     config_param :api_key, :string, secret: true
     desc "The interval time between running queries."
@@ -14,9 +33,16 @@ module Fluent::Plugin
     desc "The tag to apply to each shodan entries."
     config_param :tag, :string, default: nil
     desc "The Shodan query to execute."
-    config_param :query, :string
+    config_param :query, :string, default: ''
     desc "The maximum amount of pages to crawl. A 0 or negative value means to crawl all pages."
     config_param :max_pages, :integer, default: 1
+    desc "Search filters configuration."
+    config_section :filter, param_name: 'filters', required: false, multi: true do
+      desc "Name of the filter. See https://www.shodan.io/search/filters for a list of supported filters"
+      config_param :name, :enum, list: (SUPPORTED_FILTERS + SUPPORTED_FILTERS.map {|filter| "-#{filter}"}).map { |filter| filter.to_sym }
+      desc "Value to be given to the filter"
+      config_param :value, :string
+    end
 
     def configure(conf)
       super
@@ -27,6 +53,13 @@ module Fluent::Plugin
       rescue RuntimeError => exception
         raise Fluent::ConfigError.new "Invalid Shodan API key"
       end
+
+      raise Fluent::ConfigError.new("At least a query or one filter should be configured") if @query.empty? and @filters.empty?
+
+      @search_filters = {}
+      @filters.each do |filter|
+        @search_filters[filter.name] = filter.value
+      end
     end
 
     def multi_workers_ready?
@@ -44,20 +77,21 @@ module Fluent::Plugin
     def run
       log.debug "Starting Shodan search", query: @query, max_pages: @max_pages
       es_time = Fluent::EventTime.now
-      current_page = 0
+      opts = @search_filters.merge({page: 0})
       read_entries = 0
       loop do
-        current_page += 1
-        result = @client.host_search(@query, page: current_page)
+        opts[:page] += 1
+        log.trace query: @query, opts: opts
+        result = @client.host_search(@query.dup, **opts)
         result['matches'].each do |rec|
           router.emit(@tag, es_time, rec)
         end
         read_entries += result['matches'].length
-        break if (@max_pages >= 0 && current_page >= @max_pages) || read_entries >= result['total']
+        break if (@max_pages >= 0 && opts[:page] >= @max_pages) || read_entries >= result['total']
       end
-      log.debug "Shodan search ending", query: @query, total_read: read_entries
+      log.debug "Shodan search ending", query: @query, filters: @search_filters, total_read: read_entries
     rescue RuntimeError => re
-      log.error "Unable to execute Shodan query", query: @query, page: current_page, error: re
+      log.error "Unable to execute Shodan query", query: @query, filters: @search_filters, page: current_page, error: re
     rescue => exception
       log.error "Error executing Shodan query", error: exception
     end

data/test/test_in_shodan_search.rb

@@ -9,6 +9,13 @@ class ShodanSearchInputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
 
   API_KEY = ENV['SHODAN_TEST_API_KEY']
+  CONFIG = %[
+    api_key #{API_KEY}
+    query 8.8.8.8
+  ]
+
+  private_constant :API_KEY
+  private_constant :CONFIG
 
   def setup
     Fluent::Test.setup
@@ -30,10 +37,21 @@ class ShodanSearchInputTest < Test::Unit::TestCase
         ])
       end
     end
-    test 'is invalid without a query' do
+    test 'is invalid without a query or a filter' do
+      assert_raise Fluent::ConfigError do
+        create_driver(%[
+          api_key #{API_KEY}
+        ])
+      end
+    end
+    test 'is invalid with an unsupported filter' do
       assert_raise Fluent::ConfigError do
         create_driver(%[
           api_key #{API_KEY}
+          <filter>
+            name some.dumb.filter
+            value 42
+          </filter>
         ])
       end
     end
@@ -41,18 +59,100 @@ class ShodanSearchInputTest < Test::Unit::TestCase
       d = create_driver
       assert_equal '8.8.8.8', d.instance.query
     end
+    test 'is valid with one filter and no query' do
+      assert_nothing_raised do
+        d = create_driver(%[
+          api_key #{API_KEY}
+          <filter>
+            name product
+            value ssh
+          </filter>
+        ])
+      end
+    end
   end
-
+  
   sub_test_case 'Plugin emission' do
+    test 'with simple query' do
+      expected_tag = 'test_shodan'
+      d = create_driver(CONFIG + "\ninterval 5\ntag #{expected_tag}")
+      d.run(expect_emits: 10, timeout: 30)
+      events = d.events
+      assert_not_empty(events)
+      assert_all(events, "Events are not properly tagged") {|evt| expected_tag == evt[0]}
+      assert_all(events, "Events do not match query") {|evt| evt[2]['data'].downcase.include?('8.8.8.8')}
+      assert_all(events, "Events do not have a '_shodan' key") {|evt| evt[2].has_key?('_shodan')}
+    end
+
+    test 'with a single filter' do
+      config = %[
+        api_key #{API_KEY}
+        query ssh
+        interval 5
+        tag shodan_test
+        <filter>
+          name product
+          value ssh
+        </filter>
+      ]
+      d = create_driver(config)
+      d.run(expect_emits: 10, timeout: 30)
+      events = d.events
+      assert_not_empty(events)
+      assert_all(events) {|evt| evt[2]['product'].downcase.include?('ssh')}
+    end
+
+    test 'with multiple filters' do
+      config = %[
+        api_key #{API_KEY}
+        query ssh
+        interval 5
+        tag shodan_test
+        <filter>
+          name product
+          value ssh
+        </filter>
+        <filter>
+          name -port
+          value 22
+        </filter>
+      ]
+      d = create_driver(config)
+      d.run(expect_emits: 10, timeout: 30)
+      events = d.events
+      assert_not_empty(events)
+      assert_all(events) {|evt| evt[2]['product'].downcase.include?('ssh')}
+      assert_all(events) {|evt| evt[2]['port'] != 22}
+    end
+
+    test 'combine query and filters' do
+      config = %[
+        api_key #{API_KEY}
+        query ssh
+        interval 5
+        tag shodan_test
+        query plex
+        <filter>
+          name country
+          value FR
+        </filter>
+        <filter>
+          name -port
+          value 32400
+        </filter>
+      ]
+      d = create_driver(config)
+      d.run(expect_emits: 10, timeout: 30)
+      events = d.events
+      assert_not_empty(events)
+      assert_all(events) {|evt| evt[2]['data'].downcase.include?('plex')}
+      assert_all(events) {|evt| evt[2]['location']['country_code'] == 'FR'}
+      assert_all(events) {|evt| evt[2]['port'] != 32400}
+    end
   end
 
   private
 
-  CONFIG = %[
-    api_key #{API_KEY}
-    query 8.8.8.8
-  ]
-
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::ShodanSearch).configure(conf)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-shodan
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - srilumpa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-07 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd