logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/lib/logstash/modules/file_reader.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash module Modules class FileReader
+  # stub these methods for testing
+  include LogStash::Util::Loggable
+
+  def self.read(path)
+    begin
+      ::File.read(path)
+    rescue => e
+      logger.error("Error when reading file from path", :path => path)
+      ""
+    end
+  end
+
+  def self.read_json(path)
+    json = read(path)
+    begin
+      LogStash::Json.load(json)
+    rescue => e
+      logger.error("Error when parsing json from path", :path => path)
+      return {}
+    end
+  end
+
+  def self.glob(path)
+    files = Dir.glob(path)
+    if files.empty?
+      logger.warn("No files found for glob", :pattern => path)
+    end
+    files
+  end
+end end end

data/lib/logstash/modules/kibana_base.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/json"
+
+module LogStash module Modules class KibanaBase
+  attr_reader :import_path, :content
+
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    raise NotImplementedError, "#{self.class.name} needs to implement `#import`"
+  end
+
+  def to_s
+    import_path
+  end
+
+  def content_as_object
+    return content unless content.is_a?(String)
+    LogStash::Json.load(content) rescue nil
+  end
+end end end

data/lib/logstash/modules/kibana_client.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+require "manticore/client"
+
+module LogStash module Modules class KibanaClient
+  include LogStash::Util::Loggable
+
+  class Response
+    # to create a custom response with body as an Object (Hash or Array)
+    attr_reader :status, :body, :headers
+    def initialize(status, body, headers={})
+      @status, @body, @headers = status, body, headers
+      @body = body.is_a?(String) ? LogStash::Json.load(body) : body
+    end
+
+    def succeeded?
+      @status >= 200 && @status < 300
+    end
+
+    def failed?
+      !succeeded?
+    end
+  end
+
+  attr_reader :version
+
+  def initialize(settings)
+    @settings = settings
+
+    client_options = {
+      request_timeout: 5,
+      connect_timeout: 5,
+      socket_timeout: 5,
+      pool_max: 10,
+      pool_max_per_route: 2
+    }
+
+    ssl_options = {}
+
+    if @settings["var.kibana.ssl.enabled"] == "true"
+      ssl_options[:verify] = @settings.fetch("var.kibana.ssl.verification_mode", "strict").to_sym
+      ssl_options[:ca_file] = @settings.fetch("var.kibana.ssl.certificate_authority", nil)
+      ssl_options[:client_cert] = @settings.fetch("var.kibana.ssl.certificate", nil)
+      ssl_options[:client_key] = @settings.fetch("var.kibana.ssl.key", nil)
+    end
+
+    client_options[:ssl] = ssl_options
+
+    @client = Manticore::Client.new(client_options)
+    @host = @settings.fetch("var.kibana.host", "localhost:5601")
+    username = @settings["var.kibana.username"]
+    password = @settings["var.kibana.password"]
+
+    @scheme = @settings.fetch("var.kibana.scheme", "http")
+    @http_options = {:headers => {'Content-Type' => 'application/json'}}
+    if username
+      @http_options[:headers]['Authorization'] = 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp
+    end
+
+    # e.g. {"name":"Elastics-MacBook-Pro.local","version":{"number":"6.0.0-beta1","build_hash":"41e69","build_number":15613,"build_snapshot":true}..}
+    @version = "0.0.0"
+    response = get("api/status")
+    if response.succeeded?
+      status = response.body
+      if status["version"].is_a?(Hash)
+        @version = status["version"]["number"]
+        if status["version"]["build_snapshot"]
+          @version.concat("-SNAPSHOT")
+        end
+      end
+    end
+    @http_options[:headers]['kbn-version'] = @version
+  end
+
+  def version_parts
+    @version.split(/\.|\-/)
+  end
+
+  def host_settings
+    "[\"#{@host}\"]"
+  end
+
+  def get(relative_path)
+    # e.g. api/kibana/settings
+    safely(:get, relative_path, @http_options)
+  end
+
+  # content will be converted to a json string
+  def post(relative_path, content, headers = nil)
+
+    body = content.is_a?(String) ? content : LogStash::Json.dump(content)
+    options = {:body => body}.merge(headers || @http_options)
+    safely(:post, relative_path, options)
+  end
+
+  def head(relative_path)
+    safely(:head, relative_path, @http_options)
+  end
+
+  def can_connect?
+    head("api/status").succeeded?
+  end
+
+  private
+
+  def safely(method_sym, relative_path, options = {})
+    begin
+      resp = @client.http(method_sym, full_url(relative_path), options).call
+      Response.new(resp.code, resp.body, resp.headers)
+    rescue Manticore::ManticoreException => e
+      logger.error("Error when executing Kibana client request", :error => e)
+      body = {"statusCode" => 0, "error" => e.message}
+      Response.new(0, body, {})
+    end
+  end
+
+  def full_url(relative)
+    "#{@scheme}://#{@host}/#{relative}"
+  end
+end end end

data/lib/logstash/modules/kibana_config.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require_relative "file_reader"
+require_relative "kibana_settings"
+require_relative "kibana_dashboards"
+require_relative "kibana_resource"
+
+module LogStash module Modules class KibanaConfig
+  include LogStash::Util::Loggable
+
+  ALLOWED_DIRECTORIES = ["search", "visualization"]
+  METRICS_MAX_BUCKETS = (24 * 60 * 60).freeze # 24 hours of events/sec buckets.
+  attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
+
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    build_versioned_directory(modul)
+    @name = modul.module_name
+    @settings = settings
+    @index_name = "kibana"
+    @pattern_name = "#{@name}-*"
+    @metrics_max_buckets = @settings.fetch("dashboards.metrics_max_buckets", METRICS_MAX_BUCKETS).to_i
+    @kibana_settings = [
+      KibanaSettings::Setting.new("defaultIndex", @pattern_name),
+      KibanaSettings::Setting.new("metrics:max_buckets", @metrics_max_buckets)
+    ]
+  end
+
+  def dashboards
+    # there can be more than one dashboard to load
+    filenames = FileReader.read_json(dynamic("dashboard"))
+    filenames.map do |filename|
+      KibanaResource.new(@index_name, "dashboard", dynamic("dashboard", filename))
+    end
+  end
+
+  def index_pattern
+    [KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"),nil, @pattern_name)]
+  end
+
+  def resources
+    list = index_pattern
+    dashboards.each do |board|
+      list << board
+      extract_panels_into(board, list)
+    end
+    list.concat(extract_saved_searches_into(list))
+    [
+      KibanaSettings.new("api/kibana/settings", @kibana_settings),
+      KibanaDashboards.new("api/kibana/dashboards/import", list)
+    ]
+  end
+
+  private
+
+  def build_versioned_directory(modul)
+    # try to detect which directory holds the config for the kibana version
+    base_dir = ::File.join(modul.directory, "kibana")
+    maj, min, patch = modul.kibana_version_parts
+    version_dir = "#{maj}.#{min}.#{patch}"
+    @directory = ::File.join(base_dir, version_dir)
+    return if ::File.directory?(@directory)
+    version_dir = "#{maj}.#{min}.x"
+    @directory = ::File.join(base_dir, version_dir)
+    return if ::File.directory?(@directory)
+    version_dir = "#{maj}.x"
+    @directory = ::File.join(base_dir, version_dir)
+    unless ::File.directory?(@directory)
+      logger.error("Cannot find kibana version sub-directory", :module => @name, :base_directory => base_dir)
+    end
+  end
+
+  def dynamic(dynamic_folder, filename = @name)
+    ::File.join(@directory, dynamic_folder, "#{filename}.json")
+  end
+
+  def extract_panels_into(dashboard, list)
+    dash = dashboard.content_as_object
+
+    if !dash.is_a?(Hash)
+      logger.warn("Kibana dashboard JSON is not an Object", :module => @name)
+      return
+    end
+
+    panelsjson = dash["panelsJSON"]
+
+    if panelsjson.nil?
+      logger.info("No panelJSON key found in kibana dashboard", :module => @name)
+      return
+    end
+
+    begin
+      panels = LogStash::Json.load(panelsjson)
+    rescue => e
+      logger.error("JSON parse error when reading kibana panelsJSON", :module => @name)
+      return
+    end
+
+    panels.each do |panel|
+      panel_type = panel["type"]
+      if ALLOWED_DIRECTORIES.member?(panel_type)
+        list << KibanaResource.new(@index_name, panel_type, dynamic(panel_type, panel["id"]))
+      else
+        logger.warn("panelJSON contained unknown type", :type => panel_type)
+      end
+    end
+  end
+
+  def extract_saved_searches_into(list)
+    result = [] # must not add to list while iterating
+    list.each do |resource|
+      content = resource.content_as_object
+      next if content.nil?
+      next unless content.keys.include?("savedSearchId")
+      saved_search = content["savedSearchId"]
+      next if saved_search.nil?
+      ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search))
+      next if list.member?(ss_resource) || result.member?(ss_resource)
+      result << ss_resource
+    end
+    result
+  end
+end end end

data/lib/logstash/modules/kibana_dashboards.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require_relative "kibana_base"
+
+module LogStash module Modules class KibanaDashboards < KibanaBase
+  include LogStash::Util::Loggable
+
+  attr_reader :import_path, :content
+
+  # content is a list of kibana file resources
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    # e.g. curl "http://localhost:5601/api/kibana/dashboards/import"
+    # extract and prepare all objects
+    objects = []
+    content.each do |resource|
+      hash = {
+        "id" => resource.content_id,
+        "type" => resource.content_type,
+        "version" => 1,
+        "attributes" => resource.content_as_object
+      }
+      objects << hash
+    end
+    body = {"version": client.version, "objects": objects}
+    response = client.post(import_path, body)
+    if response.failed?
+      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
+    end
+    response
+  end
+end end end

data/lib/logstash/modules/kibana_importer.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+module LogStash module Modules class KibanaImporter
+  include LogStash::Util::Loggable
+
+  def initialize(client)
+    @client = client
+  end
+
+  def put(via_kibana)
+    path = via_kibana.import_path
+    logger.debug("Attempting POST", :url_path => path, :content => via_kibana.content)
+    via_kibana.import(@client)
+  end
+end end end

data/lib/logstash/modules/kibana_resource.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "resource_base"
+
+module LogStash module Modules class KibanaResource
+  include ResourceBase
+  def import_path
+    base + "/" + content_type + "/" + content_id
+  end
+end end end

data/lib/logstash/modules/kibana_settings.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require_relative "kibana_base"
+
+module LogStash module Modules class KibanaSettings < KibanaBase
+  include LogStash::Util::Loggable
+
+  class Setting
+    attr_reader :name, :value
+    def initialize(name, value)
+      @name, @value = name, value
+    end
+  end
+
+  attr_reader :import_path, :content
+
+  # content is an array of Setting required for this module
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    # e.g. curl "http://localhost:5601/api/kibana/settings"
+    # 6.0.0-beta1 -> {"settings":{"buildNum":{"userValue":15613},"defaultIndex":{"userValue":"arcsight-*"}}}
+    # 5.4 -> {"settings":{"defaultIndex":{"userValue":"cef-*"},"metrics:max_buckets":{"userValue":"600000"}}}
+    # array of Setting objects
+    # The POST api body { "changes": { "defaultIndex": "arcsight-*", "metrics:max_buckets": "400" } }
+    settings = {}
+    content.each do |setting|
+      settings[setting.name] = "#{setting.value}"
+    end
+    body = {"changes" => settings}
+    response = client.post(import_path, body)
+    if response.failed?
+      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
+    end
+    response
+  end
+end end end

data/lib/logstash/modules/logstash_config.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "file_reader"
+require "logstash/settings"
+
+module LogStash module Modules class LogStashConfig
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    @directory = ::File.join(modul.directory, "logstash")
+    @name = modul.module_name
+    @settings = settings
+  end
+
+  def template
+    ::File.join(@directory, "#{@name}.conf.erb")
+  end
+
+  def configured_inputs(default = [], aliases = {})
+    name = "var.inputs"
+    values = get_setting(LogStash::Setting::SplittableStringArray.new(name, String, default))
+
+    aliases.each { |k,v| values << v if values.include?(k) }
+    aliases.invert.each { |k,v| values << v if values.include?(k) }
+    values.flatten.uniq
+  end
+
+  def alias_settings_keys!(aliases)
+    aliased_settings = alias_matching_keys(aliases, @settings)
+    @settings = alias_matching_keys(aliases.invert, aliased_settings)
+  end
+
+  def array_to_string(array)
+    "[#{array.collect { |i| "'#{i}'" }.join(", ")}]"
+  end
+
+  def csv_string(array)
+    "'#{array.join(',')}'"
+  end
+
+  def get_setting(setting_class)
+    raw_value = @settings[setting_class.name]
+    # If we dont check for NIL, the Settings class will try to coerce the value
+    # and most of the it will fails when a NIL value is explicitely set.
+    # This will be fixed once we wrap the plugins settings into a Settings class
+    setting_class.set(raw_value) unless raw_value.nil?
+    setting_class.value
+  end
+
+  def setting(name, default)
+    # by default we use the more permissive setting which is a `NullableString`
+    # This is fine because the end format of the logstash configuration is a string representation
+    # of the pipeline. There is a good reason why I think we should use the settings classes, we
+    # can `preprocess` a template and generate a configuration from the defined settings
+    # validate the values and replace them in the template.
+    case default
+      when String
+        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+      when Numeric
+        get_setting(LogStash::Setting::Numeric.new(name, default))
+      else
+        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+      end
+  end
+
+  def elasticsearch_output_config(type_string = nil)
+    hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])))
+    index = "#{@name}-#{setting("var.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"
+    user = @settings["var.elasticsearch.username"]
+    password = @settings["var.elasticsearch.password"]
+    lines = ["hosts => #{hosts}", "index => \"#{index}\""]
+    lines.push(user ? "user => \"#{user}\"" : nil)
+    lines.push(password ? "password => \"#{password}\"" : nil)
+    lines.push(type_string ? "document_type => #{type_string}" : nil)
+    lines.push("ssl => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
+    if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
+      lines.push("cacert => \"#{cacert}\"") if cacert
+    end
+    # NOTE: the first line should be indented in the conf.erb
+    <<-CONF
+elasticsearch {
+    #{lines.compact.join("\n    ")}
+    manage_template => false
+  }
+CONF
+  end
+
+  def config_string
+    # process the template and settings
+    # send back as a string
+    renderer = ERB.new(FileReader.read(template))
+    renderer.result(binding)
+  end
+
+  private
+  # For a first version we are copying the values of the original hash,
+  # this might become problematic if we users changes the values of the
+  # settings in the template, which could result in an inconsistent view of the original data
+  #
+  # For v1 of the feature I think its an OK compromise, v2 we have a more advanced hash that
+  # support alias.
+  def alias_matching_keys(aliases, target)
+    aliased_target = target.dup
+
+    aliases.each do |matching_key_prefix, new_key_prefix|
+      target.each do |k, v|
+        re = /^#{matching_key_prefix}\./
+
+        if k =~ re
+          alias_key = k.gsub(re, "#{new_key_prefix}.")
+
+          # If the user setup the same values twices with different values lets just halt.
+          raise "Cannot create an alias, the destination key has already a value set: original key: #{k}, alias key: #{alias_key}" if (!aliased_target[alias_key].nil? && aliased_target[alias_key] != v)
+          aliased_target[alias_key] = v unless v.nil?
+        end
+      end
+    end
+
+    aliased_target
+  end
+end end end