logstash-lib 1.3.2

data/docs/docgen.rb

@@ -0,0 +1,242 @@
+require "rubygems"
+require "erb"
+require "optparse"
+require "bluecloth" # for markdown parsing
+
+$: << Dir.pwd
+$: << File.join(File.dirname(__FILE__), "..", "lib")
+
+require "logstash/config/mixin"
+require "logstash/inputs/base"
+require "logstash/codecs/base"
+require "logstash/filters/base"
+require "logstash/outputs/base"
+require "logstash/version"
+
+class LogStashConfigDocGenerator
+  COMMENT_RE = /^ *#(?: (.*)| *$)/
+
+  def initialize
+    @rules = {
+      COMMENT_RE => lambda { |m| add_comment(m[1]) },
+      /^ *class.*< *LogStash::(Outputs|Filters|Inputs|Codecs)::(Base|Threadable)/ => \
+        lambda { |m| set_class_description },
+      /^ *config +[^=].*/ => lambda { |m| add_config(m[0]) },
+      /^ *milestone .*/ => lambda { |m| set_milestone(m[0]) },
+      /^ *config_name .*/ => lambda { |m| set_config_name(m[0]) },
+      /^ *flag[( ].*/ => lambda { |m| add_flag(m[0]) },
+      /^ *(class|def|module) / => lambda { |m| clear_comments },
+    }
+  end
+
+  def parse(string)
+    clear_comments
+    buffer = ""
+    string.split(/\r\n|\n/).each do |line|
+      # Join long lines
+      if line =~ COMMENT_RE
+        # nothing
+      else
+        # Join extended lines
+        if line =~ /(, *$)|(\\$)|(\[ *$)/
+          buffer += line.gsub(/\\$/, "")
+          next
+        end
+      end
+
+      line = buffer + line
+      buffer = ""
+
+      @rules.each do |re, action|
+        m = re.match(line)
+        if m
+          action.call(m)
+        end
+      end # RULES.each
+    end # string.split("\n").each
+  end # def parse
+
+  def set_class_description
+    @class_description = @comments.join("\n")
+    clear_comments
+  end # def set_class_description
+ 
+  def add_comment(comment)
+    return if comment == "encoding: utf-8"
+    @comments << comment
+  end # def add_comment
+
+  def add_config(code)
+    # I just care about the 'config :name' part
+    code = code.sub(/,.*/, "")
+
+    # call the code, which calls 'config' in this class.
+    # This will let us align comments with config options.
+    name, opts = eval(code)
+
+    # TODO(sissel): This hack is only required until regexp configs
+    # are gone from logstash.
+    name = name.to_s unless name.is_a?(Regexp)
+
+    description = BlueCloth.new(@comments.join("\n")).to_html
+    @attributes[name][:description] = description
+    clear_comments
+  end # def add_config
+
+  def add_flag(code)
+    # call the code, which calls 'config' in this class.
+    # This will let us align comments with config options.
+    #p :code => code
+    fixed_code = code.gsub(/ do .*/, "")
+    #p :fixedcode => fixed_code
+    name, description = eval(fixed_code)
+    @flags[name] = description
+    clear_comments
+  end # def add_flag
+
+  def set_config_name(code)
+    name = eval(code)
+    @name = name
+  end # def set_config_name
+
+  def set_milestone(code)
+    @milestone = eval(code)
+  end
+
+  # pretend to be the config DSL and just get the name
+  def config(name, opts={})
+    return name, opts
+  end # def config
+
+  # Pretend to support the flag DSL
+  def flag(*args, &block)
+    name = args.first
+    description = args.last
+    return name, description
+  end # def config
+
+  # pretend to be the config dsl's 'config_name' method
+  def config_name(name)
+    return name
+  end # def config_name
+
+  # pretend to be the config dsl's 'milestone' method
+  def milestone(m)
+    return m
+  end # def milestone
+
+  def clear_comments
+    @comments.clear
+  end # def clear_comments
+
+  def generate(file, settings)
+    @class_description = ""
+    @milestone = ""
+    @comments = []
+    @attributes = Hash.new { |h,k| h[k] = {} }
+    @flags = {}
+
+    # local scoping for the monkeypatch belowg
+    attributes = @attributes
+    # Monkeypatch the 'config' method to capture
+    # Note, this monkeypatch requires us do the config processing
+    # one at a time.
+    #LogStash::Config::Mixin::DSL.instance_eval do
+      #define_method(:config) do |name, opts={}|
+        #p name => opts
+        #attributes[name].merge!(opts)
+      #end
+    #end
+
+    # Loading the file will trigger the config dsl which should
+    # collect all the config settings.
+    load file
+
+    # parse base first
+    parse(File.new(File.join(File.dirname(file), "base.rb"), "r").read)
+
+    # Now parse the real library
+    code = File.new(file).read
+
+    # inputs either inherit from Base or Threadable.
+    if code =~ /\< LogStash::Inputs::Threadable/
+      parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read)
+    end
+
+    if code =~ /include LogStash::PluginMixins/
+      mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1')
+      mixin.gsub!(/(.)([A-Z])/, '\1_\2')
+      mixin.downcase!
+      parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read)
+    end
+    
+    parse(code)
+
+    puts "Generating docs for #{file}"
+
+    if @name.nil?
+      $stderr.puts "Missing 'config_name' setting in #{file}?"
+      return nil
+    end
+
+    klass = LogStash::Config::Registry.registry[@name]
+    if klass.ancestors.include?(LogStash::Inputs::Base)
+      section = "input"
+    elsif klass.ancestors.include?(LogStash::Filters::Base)
+      section = "filter"
+    elsif klass.ancestors.include?(LogStash::Outputs::Base)
+      section = "output"
+    elsif klass.ancestors.include?(LogStash::Codecs::Base)
+      section = "codec"
+    end
+
+    template_file = File.join(File.dirname(__FILE__), "plugin-doc.html.erb")
+    template = ERB.new(File.new(template_file).read, nil, "-")
+
+    # descriptions are assumed to be markdown
+    description = BlueCloth.new(@class_description).to_html
+
+    klass.get_config.each do |name, settings|
+      @attributes[name].merge!(settings)
+    end
+    sorted_attributes = @attributes.sort { |a,b| a.first.to_s <=> b.first.to_s }
+    klassname = LogStash::Config::Registry.registry[@name].to_s
+    name = @name
+
+    synopsis_file = File.join(File.dirname(__FILE__), "plugin-synopsis.html.erb")
+    synopsis = ERB.new(File.new(synopsis_file).read, nil, "-").result(binding)
+
+    if settings[:output]
+      dir = File.join(settings[:output], section + "s")
+      path = File.join(dir, "#{name}.html")
+      Dir.mkdir(settings[:output]) if !File.directory?(settings[:output])
+      Dir.mkdir(dir) if !File.directory?(dir)
+      File.open(path, "w") do |out|
+        html = template.result(binding)
+        html.gsub!("%VERSION%", LOGSTASH_VERSION)
+        html.gsub!("%PLUGIN%", @name)
+        out.puts(html)
+      end
+    else 
+      puts template.result(binding)
+    end
+  end # def generate
+
+end # class LogStashConfigDocGenerator
+
+if __FILE__ == $0
+  opts = OptionParser.new
+  settings = {}
+  opts.on("-o DIR", "--output DIR", 
+          "Directory to output to; optional. If not specified,"\
+          "we write to stdout.") do |val|
+    settings[:output] = val
+  end
+
+  args = opts.parse(ARGV)
+
+  args.each do |arg|
+    gen = LogStashConfigDocGenerator.new
+    gen.generate(arg, settings)
+  end
+end

data/docs/extending/example-add-a-new-filter.md

@@ -0,0 +1,121 @@
+---
+title: How to extend - logstash
+layout: content_right
+---
+# Add a new filter
+
+This document shows you how to add a new filter to logstash.
+
+For a general overview of how to add a new plugin, see [the extending
+logstash](.) overview.
+
+## Write code.
+
+Let's write a 'hello world' filter. This filter will replace the 'message' in
+the event with "Hello world!"
+
+First, logstash expects plugins in a certain directory structure: `logstash/TYPE/PLUGIN_NAME.rb`
+
+Since we're creating a filter, let's mkdir this:
+
+    mkdir -p logstash/filters/
+    cd logstash/filters
+
+Now add the code:
+
+    # Call this file 'foo.rb' (in logstash/filters, as above)
+    require "logstash/filters/base"
+    require "logstash/namespace"
+
+    class LogStash::Filters::Foo < LogStash::Filters::Base
+
+      # Setting the config_name here is required. This is how you
+      # configure this filter from your logstash config.
+      #
+      # filter {
+      #   foo { ... }
+      # }
+      config_name "foo"
+
+      # New plugins should start life at milestone 1.
+      milestone 1
+
+      # Replace the message with this value.
+      config :message, :validate => :string
+
+      public
+      def register
+        # nothing to do
+      end # def register
+
+      public
+      def filter(event)
+        # return nothing unless there's an actual filter event
+        return unless filter?(event)
+        if @message
+          # Replace the event message with our message as configured in the
+          # config file.
+          event["message"] = @message
+        end
+        # filter_matched should go in the last line of our successful code 
+        filter_matched(event)
+      end # def filter
+    end # class LogStash::Filters::Foo
+
+## Add it to your configuration
+
+For this simple example, let's just use stdin input and stdout output.
+The config file looks like this:
+
+    input { 
+      stdin { type => "foo" } 
+    }
+    filter {
+      if [type] == "foo" {
+        foo {
+          message => "Hello world!"
+        }
+      }
+    }
+    output {
+      stdout { }
+    }
+
+Call this file 'example.conf'
+
+## Tell logstash about it.
+
+Depending on how you installed logstash, you have a few ways of including this
+plugin.
+
+You can use the agent flag --pluginpath flag to specify where the root of your
+plugin tree is. In our case, it's the current directory.
+
+    % logstash --pluginpath . -f example.conf
+
+If you use the jar release of logstash, you have an additional option - you can
+include the plugin right in the jar file.
+
+    # This command will take your 'logstash/filters/foo.rb' file
+    # and add it into the jar file.
+    % jar -uf logstash-%VERSION%-flatjar.jar logstash/filters/foo.rb
+
+    # Verify it's in the right location in the jar!
+    % jar tf logstash-%VERSION%-flatjar.jar | grep foo.rb
+    logstash/filters/foo.rb
+
+    % java -jar logstash-%VERSION%-flatjar.jar agent -f example.conf
+
+## Example running
+
+In the example below, I typed in "the quick brown fox" after running the java
+command.
+
+    % java -jar logstash-%VERSION%-flatjar.jar agent -f example.conf
+    the quick brown fox   
+    2011-05-12T01:05:09.495000Z stdin://snack.home/: Hello world!
+
+The output is the standard logstash stdout output, but in this case our "the
+quick brown fox" message was replaced with "Hello world!"
+
+All done! :)

data/docs/extending/index.md

@@ -0,0 +1,91 @@
+---
+title: How to extend - logstash
+layout: content_right
+---
+# Extending logstash
+
+You can add your own input, output, or filter plugins to logstash.
+
+If you're looking to extend logstash today, please look at the existing plugins.
+
+## Good examples of plugins
+
+* [inputs/tcp](https://github.com/logstash/logstash/blob/master/lib/logstash/inputs/tcp.rb)
+* [filters/multiline](https://github.com/logstash/logstash/blob/master/lib/logstash/filters/multiline.rb)
+* [outputs/mongodb](https://github.com/logstash/logstash/blob/master/lib/logstash/outputs/mongodb.rb)
+
+## Common concepts
+
+* The `config_name` sets the name used in the config file.
+* The `milestone` sets the milestone number of the plugin. See <../plugin-milestones> for more info.
+* The `config` lines define config options.
+* The `register` method is called per plugin instantiation. Do any of your initialization here.
+
+### Required modules
+
+All plugins should require the Logstash module.
+
+    require 'logstash/namespace'
+
+### Plugin name
+
+Every plugin must have a name set with the `config_name` method. If this
+is not specified plugins will fail to load with an error.
+
+### Milestones
+
+Every plugin needs a milestone set using `milestone`. See
+<../plugin-milestones> for more info.
+
+### Config lines
+
+The `config` lines define configuration options and are constructed like
+so:
+
+    config :host, :validate => :string, :default => "0.0.0.0"
+
+The name of the option is specified, here `:host` and then the
+attributes of the option. They can include `:validate`, `:default`,
+`:required` (a Boolean `true` or `false`), and `:deprecated` (also a
+Boolean).  
+ 
+## Inputs
+
+All inputs require the LogStash::Inputs::Base class:
+
+    require 'logstash/inputs/base'
+ 
+Inputs have two methods: `register` and `run`.
+
+* Each input runs as its own thread.
+* The `run` method is expected to run-forever.
+
+## Filters
+
+All filters require the LogStash::Filters::Base class:
+
+    require 'logstash/filters/base'
+ 
+Filters have two methods: `register` and `filter`.
+
+* The `filter` method gets an event. 
+* Call `event.cancel` to drop the event.
+* To modify an event, simply make changes to the event you are given.
+* The return value is ignored.
+
+## Outputs
+
+All outputs require the LogStash::Outputs::Base class:
+
+    require 'logstash/outputs/base'
+ 
+Outputs have two methods: `register` and `receive`.
+
+* The `register` method is called per plugin instantiation. Do any of your initialization here.
+* The `receive` method is called when an event gets pushed to your output
+
+## Example: a new filter
+
+Learn by example how to [add a new filter to logstash](example-add-a-new-filter)
+
+