fluent-plugin-obsolete-plugins 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 969a6a02d586a34539327fcecf6eff5c9bc72a8a9683e9b75d0176797f9480c1
-  data.tar.gz: 9e1c70429456286de80d8a3f44b4a7d8ca15a66caf2f6e2dc461e676b5b384a6
+  metadata.gz: 64c35dab27cb19ba6b8f33edd023c6dbcc100256b0f372e7f51f8d4dc99a3197
+  data.tar.gz: aedff69e17e8d8ce189920ee3249e5385b3dbde50157b8b83630e40e276ad8cf
 SHA512:
-  metadata.gz: 8f6bb96d9a1080d508adb9cbefc1432b2a246691129deadf0a4a42b1e422da8b11a39c76b8c6a6d06b7ad5276ab91812165347fce04b141d29bb329edef6373b
-  data.tar.gz: 2982233e3a6f4f3add4b5bf223edfe1d3c3e552611339b7afd14534d2b350c8771f0c7f2e6897aa7b390d2ebae44c54f8622ca7d5ad318caee65c72aa8bc1d1f
+  metadata.gz: '028c5a09fc99fac1c8be4a36c4ab1d2bdbd7c88178336771696d6e6e22393b6e2de14d42c7cfc2509a1e12b01b2204845ab1b342f0007badd75e46b96c7cd1b2'
+  data.tar.gz: 1440e86c8bc39de6bcd6a68624254a624d4d2d235460430c22dc0ae1ded7cf54eb985b1662101f3ab7ebd25b0faf3ffd9e101de8909ef1df8ec1f79784a7ad2d

data/README.md

@@ -30,13 +30,49 @@ $ bundle
 
 ## Configuration
 
-### obsolete_plugins_yml (string) (optional)
+### Filter plugin
 
-Path to obsolete-plugins.yml
+#### plugins_json (string) (optional)
 
-Default value: `https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/obsolete-plugins.yml`.
+Path to `plugins.json`.
 
-### raise_error (bool) (optional)
+Default value: `https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json`.
+
+#### Deprecated: obsolete_plugins_yml (string) (optional)
+
+Path to `obsolete-plugins.yml`. This parameter is deprecated. Please use `plugins_json` parameter instead.
+
+Default value: `nil`
+
+#### timeout (integer) (optional)
+
+Timeout to read data of obsolete plugins.
+If it occurs timeout, it just skips to detect obsolete plugins.
+
+Default value: `5`
+
+#### raise_error (bool) (optional)
+
+Raise error if obsolete plugins are detected
+
+Default value: `false`.
+
+### Input plugin
+
+#### plugins_json (string) (optional)
+
+Path to `plugins.json`.
+
+Default value: `https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json`.
+
+#### timeout (integer) (optional)
+
+Timeout to read data of obsolete plugins.
+If it occurs timeout, it just skips to detect obsolete plugins.
+
+Default value: `5`
+
+#### raise_error (bool) (optional)
 
 Raise error if obsolete plugins are detected
 

data/fluent-plugin-obsolete-plugins.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-obsolete-plugins"
-  spec.version = "0.1.1"
+  spec.version = "0.2.0"
   spec.authors = ["okkez"]
   spec.email   = ["okkez000@gmail.com"]
 

data/lib/fluent/plugin/filter_obsolete_plugins.rb

@@ -14,35 +14,39 @@
 # limitations under the License.
 
 require "fluent/plugin/filter"
-require "open-uri"
-require "yaml"
+require "fluent/plugin/obsolete_plugins_utils"
 
 module Fluent
   module Plugin
     class ObsoletePluginsFilter < Fluent::Plugin::Filter
       Fluent::Plugin.register_filter("obsolete_plugins", self)
 
-      OBSOLETE_PLUGINS_URL = "https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/obsolete-plugins.yml"
+      PLUGINS_JSON_URL = "https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json"
 
       desc "Path to obsolete-plugins.yml"
-      config_param :obsolete_plugins_yml, :string, default: OBSOLETE_PLUGINS_URL
+      config_param :obsolete_plugins_yml, :string, default: nil, deprecated: "use plugins_json parameter instead"
+      desc "Path to plugins.json"
+      config_param :plugins_json, :string, default: PLUGINS_JSON_URL
+      desc "Timeout value to read data of obsolete plugins"
+      config_param :timeout, :integer, default: 5
       desc "Raise error if obsolete plugins are detected"
       config_param :raise_error, :bool, default: false
 
       def configure(conf)
         super
 
-        @obsolete_plugins = open(@obsolete_plugins_yml) do |io|
-          YAML.safe_load(io.read)
-        end
-
-        obsolete_plugins = Gem.loaded_specs.keys & @obsolete_plugins.keys
-        obsolete_plugins.each do |name|
-          log.warn("#{name} is obsolete: #{@obsolete_plugins[name].chomp}")
-        end
-        if @raise_error && !obsolete_plugins.empty?
-          raise Fluent::ConfigError, "Detected obsolete plugins"
-        end
+        obsolete_plugins =
+          if @obsolete_plugins_yml
+            ObsoletePluginsUtils.obsolete_plugins_from_yaml(@obsolete_plugins_yml, timeout: @timeout)
+          else
+            ObsoletePluginsUtils.obsolete_plugins_from_json(@plugins_json, timeout: @timeout)
+          end
+
+        ObsoletePluginsUtils.notify(log, obsolete_plugins, raise_error: @raise_error)
+      rescue Fluent::ConfigError
+        raise
+      rescue => e
+        log.info("Failed to notify obsolete plugins", error: e)
       end
 
       def filter(tag, time, record)

data/lib/fluent/plugin/in_obsolete_plugins.rb

@@ -0,0 +1,30 @@
+require "fluent/plugin/input"
+require "fluent/plugin/obsolete_plugins_utils"
+
+module Fluent
+  module Plugin
+    class ObsoletePluginsInput < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("obsolete_plugins", self)
+
+      PLUGINS_JSON_URL = "https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json"
+
+      desc "Path to plugins.json"
+      config_param :plugins_json, :string, default: PLUGINS_JSON_URL
+      desc "Timeout value to read data of obsolete plugins"
+      config_param :timeout, :integer, default: 5
+      desc "Raise error if obsolete plugins are detected"
+      config_param :raise_error, :bool, default: false
+
+      def configure(conf)
+        super
+
+        obsolete_plugins = ObsoletePluginsUtils.obsolete_plugins_from_json(@plugins_json, timeout: @timeout)
+        ObsoletePluginsUtils.notify(log, obsolete_plugins, raise_error: @raise_error)
+      rescue Fluent::ConfigError
+        raise
+      rescue => e
+        log.info("Failed to notify obsolete plugins", error: e)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/obsolete_plugins_utils.rb

@@ -0,0 +1,40 @@
+require "fluent/config/error"
+require "open-uri"
+require "yaml"
+require "json"
+require "timeout"
+
+class Fluent::Plugin::ObsoletePluginsUtils
+  def self.obsolete_plugins_from_yaml(url, timeout: 5)
+    Timeout.timeout(timeout) do
+      URI.open(url) do |io|
+        YAML.safe_load(io.read)
+      end
+    end
+  end
+
+  def self.obsolete_plugins_from_json(url, timeout: 5)
+    plugins = Timeout.timeout(timeout) do
+      URI.open(url) do |io|
+        # io.read causes Encoding::UndefinedConversionError with UTF-8 data when Ruby is started with "-Eascii-8bit:ascii-8bit".
+        # It set the proper encoding to avoid the error.
+        io.set_encoding("UTF-8", "UTF-8")
+        JSON.parse(io.read)
+      end
+    end
+    plugins.select { |plugin| plugin["obsolete"] }.reduce({}) do |result, plugin|
+      result[plugin["name"]] = plugin["note"]
+      result
+    end
+  end
+
+  def self.notify(logger, obsolete_plugins, raise_error: false)
+    plugins = Gem.loaded_specs.keys & obsolete_plugins.keys
+    plugins.each do |name|
+      logger.warn("#{name} is obsolete: #{obsolete_plugins[name].chomp}")
+    end
+    if raise_error && !plugins.empty?
+      raise Fluent::ConfigError, "Detected obsolete plugins"
+    end
+  end
+end

data/test/fixtures/invalid.json

@@ -0,0 +1,11 @@
+[
+  {
+    "obsolete": null,
+    "note": null,
+    "name": "fluent-plugin-s3",
+    "info": "Amazon S3 output plugin for Fluentd event collector",
+    "authors": "Sadayuki Furuhashi, Masahiro Nakagawa",
+    "version": "1.8.3",
+    "downloads": 134522742,
+    "homepage_uri": "https://github.com/fluent/fluent-plugin-s3",
+    "source_code_uri": null