fluentd-hubspot 0.14.14.1

data/fluentd.gemspec

@@ -0,0 +1,51 @@
+require File.expand_path('../lib/fluent/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluentd-hubspot"
+  gem.version       = Fluent::VERSION # see lib/fluent/version.rb
+
+  gem.authors       = ["Sadayuki Furuhashi"]
+  gem.email         = ["frsyuki@gmail.com"]
+  gem.description   = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
+  gem.summary       = %q{Fluentd event collector}
+  gem.homepage      = "http://fluentd.org/"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc = false
+  gem.license = "Apache-2.0"
+
+  gem.required_ruby_version = '>= 2.1'
+
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
+  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
+  gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
+  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
+  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
+  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
+  gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+
+  # build gem for a certain platform. see also Rakefile
+  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
+  gem.platform = fake_platform unless fake_platform.empty?
+  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
+    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
+  end
+
+  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
+  gem.add_development_dependency("oj", ["~> 2.14"])
+end

data/lib/fluent/agent.rb

@@ -0,0 +1,163 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+require 'fluent/plugin'
+require 'fluent/output'
+require 'fluent/match'
+
+module Fluent
+  #
+  # Agent is a resource unit who manages emittable plugins
+  #
+  # Next step: `fluentd/root_agent.rb`
+  # Next step: `fluentd/label.rb`
+  #
+  class Agent
+    include Configurable
+
+    def initialize(log:)
+      super()
+
+      @context = nil
+      @outputs = []
+      @filters = []
+
+      @lifecycle_control_list = nil
+      # lifecycle_control_list is the list of plugins in this agent, and ordered
+      # from plugins which DOES emit, then DOESN'T emit
+      # (input -> output w/ router -> filter -> output w/o router)
+      # for start: use this order DESC
+      #   (because plugins which appears later in configurations will receive events from plugins which appears ealier)
+      # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
+      @lifecycle_cache = nil
+
+      @log = log
+      @event_router = EventRouter.new(NoMatchMatch.new(log), self)
+      @error_collector = nil
+    end
+
+    attr_reader :log
+    attr_reader :outputs
+    attr_reader :filters
+    attr_reader :context
+    attr_reader :event_router
+    attr_reader :error_collector
+
+    def configure(conf)
+      super
+
+      # initialize <match> and <filter> elements
+      conf.elements('filter', 'match').each { |e|
+        pattern = e.arg.empty? ? '**' : e.arg
+        type = e['@type']
+        raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
+        if e.name == 'filter'
+          add_filter(type, pattern, e)
+        else
+          add_match(type, pattern, e)
+        end
+      }
+    end
+
+    def lifecycle_control_list
+      return @lifecycle_control_list if @lifecycle_control_list
+
+      lifecycle_control_list = {
+        input: [],
+        output_with_router: [],
+        filter: [],
+        output: [],
+      }
+      if self.respond_to?(:inputs)
+        inputs.each do |i|
+          lifecycle_control_list[:input] << i
+        end
+      end
+      outputs.each do |o|
+        if o.has_router?
+          lifecycle_control_list[:output_with_router] << o
+        else
+          lifecycle_control_list[:output] << o
+        end
+      end
+      filters.each do |f|
+        lifecycle_control_list[:filter] << f
+      end
+
+      @lifecycle_control_list = lifecycle_control_list
+    end
+
+    def lifecycle(desc: false)
+      kind_list = if desc
+                    [:output, :filter, :output_with_router]
+                  else
+                    [:output_with_router, :filter, :output]
+                  end
+      kind_list.each do |kind|
+        list = if desc
+                 lifecycle_control_list[kind].reverse
+               else
+                 lifecycle_control_list[kind]
+               end
+        display_kind = (kind == :output_with_router ? :output : kind)
+        list.each do |instance|
+          yield instance, display_kind
+        end
+      end
+    end
+
+    def add_match(type, pattern, conf)
+      log.info :worker0, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      output = Plugin.new_output(type)
+      output.context_router = @event_router
+      output.configure(conf)
+      @outputs << output
+      if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
+        # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
+        outputs = if output.respond_to?(:static_outputs)
+                    output.static_outputs
+                  else
+                    output.outputs
+                  end
+        @outputs.push(*outputs)
+      end
+      @event_router.add_rule(pattern, output)
+
+      output
+    end
+
+    def add_filter(type, pattern, conf)
+      log.info :worker0, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      filter = Plugin.new_filter(type)
+      filter.context_router = @event_router
+      filter.configure(conf)
+      @filters << filter
+      @event_router.add_rule(pattern, filter)
+
+      filter
+    end
+
+    # For handling invalid record
+    def emit_error_event(tag, time, record, error)
+    end
+
+    def handle_emits_error(tag, es, error)
+    end
+  end
+end

data/lib/fluent/clock.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Clock
+    CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
+    @@block_level = 0
+    @@frozen_clock = nil
+
+    def self.now
+      @@frozen_clock || now_raw
+    end
+
+    def self.freeze(dst = nil, &block)
+      return freeze_block(dst, &block) if block_given?
+
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      @@frozen_clock = dst || now_raw
+    end
+
+    def self.return
+      raise "invalid return while running code in blocks" if @@block_level > 0
+      @@frozen_clock = nil
+    end
+
+    # internal use
+
+    def self.now_raw
+      Process.clock_gettime(CLOCK_ID)
+    end
+
+    def self.dst_clock_from_time(time)
+      diff_sec = Time.now - time
+      now_raw - diff_sec
+    end
+
+    def self.freeze_block(dst)
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      pre_frozen_clock = @@frozen_clock
+      @@frozen_clock = dst || now_raw
+      @@block_level += 1
+      yield
+    ensure
+      @@block_level -= 1
+      @@frozen_clock = pre_frozen_clock
+    end
+  end
+end

data/lib/fluent/command/binlog_reader.rb

@@ -0,0 +1,234 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'msgpack'
+
+require 'fluent/msgpack_factory'
+require 'fluent/formatter'
+require 'fluent/plugin'
+require 'fluent/config/element'
+
+class FluentBinlogReader
+  SUBCOMMAND = %w(cat head formats)
+  HELP_TEXT = <<HELP
+Usage: fluent-binlog-reader <command> [<args>]
+
+Commands of fluent-binlog-reader:
+   cat     :     Read files sequentially, writing them to standard output.
+   head    :     Display the beginning of a text file.
+   format  :     Display plugins that you can use.
+
+See 'fluent-binlog-reader <command> --help' for more information on a specific command.
+HELP
+
+  def initialize(argv = ARGV)
+    @argv = argv
+  end
+
+  def call
+    command_class = BinlogReaderCommand.const_get(command)
+    command_class.new(@argv).call
+  end
+
+  private
+
+  def command
+    command = @argv.shift
+    if !command || !SUBCOMMAND.include?(command)
+      usage "Required subcommand : #{SUBCOMMAND.join(' | ')}"
+    end
+
+    command.split('_').map(&:capitalize).join('')
+  end
+
+  def usage(msg = nil)
+    puts HELP_TEXT
+    puts "Error: #{msg}" if msg
+    exit 1
+  end
+end
+
+module BinlogReaderCommand
+  class Base
+    def initialize(argv = ARGV)
+      @argv = argv
+
+      @options = { plugin: [] }
+      @opt_parser = OptionParser.new do |opt|
+        opt.separator 'Options:'
+
+        opt.on('-p DIR', '--plugin', 'add library directory path') do |v|
+          @options[:plugin] << v
+        end
+      end
+    end
+
+    def call
+      raise NotImplementedError, 'BUG: command  MUST implement this method'
+    end
+
+    private
+
+    def usage(msg = nil)
+      puts @opt_parser.to_s
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+
+      unless @options[:plugin].empty?
+        if dir = @options[:plugin].find { |d| !Dir.exist?(d) }
+          usage "Directory #{dir} doesn't exist"
+        else
+          @options[:plugin].each do |d|
+            Fluent::Plugin.add_plugin_dir(d)
+          end
+        end
+      end
+    rescue => e
+      usage e
+    end
+  end
+
+  module Formattable
+    DEFAULT_OPTIONS = {
+      format: :out_file
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(DEFAULT_OPTIONS)
+      configure_option_parser
+    end
+
+    private
+
+    def configure_option_parser
+      @options.merge!(config_params: {})
+
+      @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
+
+      @opt_parser.on('-f TYPE', '--format', 'configure output format') do |v|
+        @options[:format] = v.to_sym
+      end
+
+      @opt_parser.on('-e KEY=VALUE', 'configure formatter config params') do |v|
+        key, value = v.split('=')
+        usage "#{v} is invalid. valid format is like `key=value`" unless value
+        @options[:config_params].merge!(key => value)
+      end
+    end
+
+    def lookup_formatter(format, params)
+      conf = Fluent::Config::Element.new('ROOT', '', params, [])
+      formatter = Fluent::Plugin.new_formatter(format)
+
+      if formatter.respond_to?(:configure)
+        formatter.configure(conf)
+      end
+      formatter
+    rescue => e
+      usage e
+    end
+  end
+
+  class Head < Base
+    include Formattable
+
+    DEFAULT_HEAD_OPTIONS = {
+      count: 5
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(default_options)
+      parse_options!
+    end
+
+    def call
+      @formatter = lookup_formatter(@options[:format], @options[:config_params])
+
+      File.open(@path, 'rb') do |io|
+        i = 1
+        Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
+          print @formatter.format(@path, time, record) # path is used for tag
+          break if @options[:count] && i == @options[:count]
+          i += 1
+        end
+      end
+    end
+
+    private
+
+    def default_options
+      DEFAULT_HEAD_OPTIONS
+    end
+
+    def parse_options!
+      @opt_parser.on('-n COUNT', 'Set the number of lines to display') do |v|
+        @options[:count] = v.to_i
+        usage "illegal line count -- #{@options[:count]}" if @options[:count] < 1
+      end
+
+      super
+
+      usage 'Path is required' if @argv.empty?
+      @path = @argv.first
+      usage "#{@path} is not found" unless File.exist?(@path)
+    end
+  end
+
+  class Cat < Head
+    DEFAULT_CAT_OPTIONS = {
+      count: nil                # Overwrite DEFAULT_HEAD_OPTIONS[:count]
+    }
+
+    def default_options
+      DEFAULT_CAT_OPTIONS
+    end
+  end
+
+  class Formats < Base
+    def initialize(argv = ARGV)
+      super
+      parse_options!
+    end
+
+    def call
+      prefix = Fluent::Plugin::FORMATTER_REGISTRY.dir_search_prefix || 'formatter_'
+
+      plugin_dirs = @options[:plugin]
+      unless plugin_dirs.empty?
+        plugin_dirs.each do |d|
+          Dir.glob("#{d}/#{prefix}*.rb").each do |path|
+            require File.absolute_path(path)
+          end
+        end
+      end
+
+      $LOAD_PATH.map do |lp|
+        Dir.glob("#{lp}/#{prefix}*.rb").each do |path|
+          require path
+        end
+      end
+
+      puts Fluent::Plugin::FORMATTER_REGISTRY.map.keys
+    end
+  end
+end