fluentd-server 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 23543f023f2b14f82abde08cceba1337e730c494
+  data.tar.gz: bb43d8d9a3686cc3e1cba3da52bf532438ba1cc9
+SHA512:
+  metadata.gz: b430c589ae0aaa6f102d36c98a61dc6230510ea590f528922f26d30a3e4b1511b07b323d7107b82ed417c00fab6734cb657ff7e4ceca60ef771d9a0766a0be64
+  data.tar.gz: 512e455f5e280674c272a992c8ea676323e207c7693eb597d460a9a6d3c88135b394d21b95872724a28c13f340d53538e5832f982b34132cf34644bc1dde0376

data/.env

@@ -0,0 +1,5 @@
+PORT=5126
+HOST=0.0.0.0
+# DATABASE_URL=sqlite3:data/fluentd_server.db
+# LOG_PATH=log/application.log
+# LOG_LEVEL=warn

data/.gitignore

@@ -0,0 +1,17 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+coverage
+.yardoc
+.ruby-version
+pkg/*
+data/*
+log/*
+fluentd-server/*

data/.travis.yml

@@ -0,0 +1,5 @@
+rvm:
+  - 2.0.0
+  - 2.1
+gemfile:
+  - Gemfile

data/API.md

@@ -0,0 +1,21 @@
+## API
+
+### GET /api/:name
+
+Get the contents of Fluentd config whose name is :name. 
+Query parameters are replaced with variables in erb. 
+
+Supported query parameter formats are:
+
+* var=value
+
+  * The variable `var` is replaced with its value in erb.
+
+* var[]=value1&var[]=value2
+
+  * Array. The variable `var[idx]` such as `var[0]` and `var[1]` is replaced with its value in erb.
+
+* var[key1]=value1&var[key2]=value2
+
+  * Hash. The variable `var[key]` such as `var['key1']` and `var['key2']` is replaced with its value in erb.
+

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 0.1.0
+
+First version

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gemspec
+gem 'sqlite3'
+
+group :development, :test do
+  gem 'coveralls', :require => false
+  gem 'parallel'
+end

data/Procfile

@@ -0,0 +1 @@
+web: bundle exec unicorn -E production -p $PORT -o $HOST -c config/unicorn.conf

data/README.md

@@ -0,0 +1,127 @@
+# Fluentd Server
+
+[![Build Status](https://secure.travis-ci.org/sonots/fluentd-server.png?branch=master)](http://travis-ci.org/sonots/fluentd-server)
+[![Coverage Status](https://coveralls.io/repos/sonots/fluentd-server/badge.png?branch=master)](https://coveralls.io/r/sonots/fluentd-server?branch=master)
+
+A Fluentd config distribution server
+
+Demo: [http://fluentd-server.herokuapp.com](http://fluentd-server.herokuapp.com)
+
+## What You Can Do
+
+With Fluentd Server, you can manage fluentd configuration files centrally with `erb`. 
+
+For example, you may create a config post whose name is `worker` as:
+
+```
+<source>
+  type forward
+  port <%= port %>
+</source>
+
+<match **>
+  type stdout
+</match>
+```
+
+Then you can download the config via an API whose uri is like `/api/worker?port=24224` where its query parameters are replaced with variables in the erb. 
+The downloaded contents should become as follows:
+
+```
+<source>
+  type forward
+  port 24224
+</source>
+
+<match **>
+  type stdout
+</match>
+```
+
+## How to Use
+
+The `include` directive of fluentd config supports `http`, so write just one line on your fluentd.conf as:
+
+```
+# /etc/fluentd.conf
+include http://fqdn.to.fluentd-server/api/:name?port=24224
+```
+
+where :name is the name of your config post, so that it will download the real configuration from the Fluentd Server.
+
+## Installation
+
+Prerequisites
+
+* SQLite
+* Ruby 2.0 or later
+
+### From Gem package
+
+Easy steps on installation with gem and SQLite.
+
+```bash
+$ gem install fluentd-server
+$ gem install sqlite3
+$ fluentd-server new
+$ cd fluentd-server
+$ fluentd-server init # creates database scheme on SQLite
+$ fluentd-server start
+```
+
+Then see `http://localhost:5126/`.
+
+### From Git repository
+
+Install from git repository. 
+
+```bash
+$ git clone https://github.com/sonots/fluentd-server.git
+$ cd fluentd-server
+$ bundle
+$ bundle exec fluentd-server init # creates database scheme on SQLite
+$ bundle exec fluentd-server start
+```
+
+Then see `http://localhost:5126/`. 
+
+## Configuration
+
+To configure fluentd-server, edit the `.env` file in the project root directory.
+
+The default configuration is as follows:
+
+```
+PORT=5126
+HOST=0.0.0.0
+# DATABASE_URL=sqlite3:data/fluentd_server.db
+# LOG_PATH=log/application.log
+# LOG_LEVEL=warn
+```
+
+## HTTP API
+
+See [API.md](API.md).
+
+## ToDo
+
+* Local file storage
+
+  * Saving and loading conf from localfiles rather than DB would be nice because it makes possible to manage conf with git
+  * Fluentd Server should cache them on memory, and refresh caches by detecting files are updated
+
+## ChangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Copyright
+
+Copyright (c) 2014 Naotoshi Seo. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,26 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["-c", "-f progress"] # '--format specdoc'
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dotenv/tasks'
+require 'fluentd_server/config'
+
+require 'sinatra/activerecord/rake'
+
+task :console => :dotenv do
+  require "fluentd_server"
+  require 'irb'
+  # require 'irb/completion'
+  ARGV.clear
+  IRB.start
+end
+task :c => :console
+
+task :test => :spec
+task :default => :spec

data/bench/bench.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+ 
+require 'parallel'
+require 'net/http'
+
+def main
+  requests = 20000  # number of requests to perform
+  concurrency = 126 # number of multiple requests to make
+  name = 'worker'
+  puts "requests = #{requests}"
+  puts "concurrency = #{concurrency}"
+
+  client = Client.new("http://localhost:5126")
+
+  duration = elapsed_time do 
+    Parallel.each_with_index([name]*requests, :in_processes => concurrency) do |name, i|
+      puts "processing #{i}" if i % 1000 == 0
+      res = client.get(name)
+      puts 'error' unless res.code == '200'
+    end
+  end
+
+  req_per_sec = ( duration > 0 ) ? requests/duration : 0
+  puts "req/sec = #{req_per_sec}"
+end
+
+def elapsed_time(&block)
+  s = Time.now
+  yield
+  Time.now - s 
+end 
+
+class Client
+  attr_reader   :base_uri
+  attr_reader   :host
+  attr_reader   :port
+  attr_accessor :debug_dev
+  attr_accessor :open_timeout
+  attr_accessor :read_timeout
+  attr_accessor :verify_ssl
+  attr_accessor :keepalive
+
+  def initialize(base_uri = 'http://127.0.0.1:5126', opts = {})
+    @base_uri = base_uri
+
+    URI.parse(base_uri).tap {|uri|
+      @host = uri.host
+      @port = uri.port
+      @use_ssl = uri.scheme == 'https'
+    }
+    @debug_dev    = opts['debug_dev'] # IO object such as STDOUT
+    @open_timeout = opts['open_timeout'] # 60
+    @read_timeout = opts['read_timeout'] # 60
+    @verify_ssl   = opts['verify_ssl']
+    @keepalive    = opts['keepalive']
+  end
+
+  def http_connection
+    Net::HTTP.new(@host, @port).tap {|http|
+      http.use_ssl      = @use_ssl
+      http.open_timeout = @open_timeout if @open_timeout
+      http.read_timeout = @read_timeout if @read_timeout
+      http.verify_mode  = OpenSSL::SSL::VERIFY_NONE unless @verify_ssl
+      http.set_debug_output(@debug_dev) if @debug_dev
+    }
+  end
+
+  def get_request(path, extheader = {})
+    Net::HTTP::Get.new(path).tap {|req|
+      req['Host'] = @host
+      req['Connection'] = 'Keep-Alive' if @keepalive
+      extheader.each {|key, value| req[key] = value }
+    }
+  end
+
+  def get(name)
+    path = "/api/#{name}"
+    req  = get_request(path)
+    @res = http_connection.start {|http| http.request(req) }
+  end
+end
+
+main

data/bench/result.md

@@ -0,0 +1,17 @@
+The conf body as followings, would be too small to evaluate:
+
+```
+<source>
+  type forward
+  port <%= port %>
+</source>
+```
+
+| # of requests | concurrency | server  | # of workers | req/sec |
+|---------------|-------------|---------|--------------|---------|
+|20000          |126          | unicorn | 1            |865.07   |
+|20000          |126          | unicorn | 12           |4116.78  |
+|20000          |126          | puma    | 1            |634.20|
+|20000          |126          | puma    | 12           |3765.66  |
+
+Let's use unicorn

data/bin/fluentd-server

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path("../../lib", __FILE__)
+
+require "fluentd_server/cli"
+
+FluentdServer::CLI.start

data/config.ru

@@ -0,0 +1,15 @@
+require "rubygems"
+require "sinatra"
+require "fluentd_server"
+require "fluentd_server/web"
+
+## Unicorn self-process killer
+#require 'unicorn/worker_killer'
+#
+## Max requests per worker
+#use Unicorn::WorkerKiller::MaxRequests, 3072, 4096
+#
+## Max memory size (RSS) per worker
+#use Unicorn::WorkerKiller::Oom, (192*(1024**2)), (256*(1024**2))
+
+run FluentdServer::Web

data/config/unicorn.conf

@@ -0,0 +1,20 @@
+worker_processes 1
+# timeout 6000
+# preload_app true
+
+GC.respond_to?(:copy_on_write_friendly=) and GC.copy_on_write_friendly = true
+
+before_fork do |server, worker|
+  defined?(ActiveRecord::Base) and ActiveRecord::Base.connection.disconnect!
+
+  old_pid = "#{server.config[:pid]}.oldbin"
+  if old_pid != server.pid
+    begin
+      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+      Process.kill(sig, File.read(old_pid).to_i)
+    rescue Errno::ENOENT, Errno::ESRCH
+    end
+  end
+
+  sleep 1
+end

data/db/migrate/20140512203133_create_posts.rb

@@ -0,0 +1,14 @@
+class CreatePosts < ActiveRecord::Migration
+  def self.up
+    create_table :posts do |t|
+      t.string :name
+      t.text :body
+      t.timestamps
+    end
+    add_index :posts, :name, length: 255, unique: true # explicit length is required for MySQL
+  end
+ 
+  def self.down
+    drop_table :posts
+  end
+end

data/db/schema.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20140512203133) do
+
+  create_table "posts", force: true do |t|
+    t.string   "name"
+    t.text     "body"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "posts", ["name"], name: "index_posts_on_name", unique: true
+
+end

data/fluentd-server.gemspec

@@ -0,0 +1,43 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluentd_server/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluentd-server"
+  spec.version       = FluentdServer::VERSION
+  spec.authors       = ["Naotoshi Seo"]
+  spec.email         = ["sonots@gmail.com"]
+  spec.description   = %q{Fluentd config distribution server}
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/sonots/fluentd-server"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "capybara"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-nav"
+  spec.add_development_dependency "rake"
+
+  spec.add_runtime_dependency "dotenv"
+  spec.add_runtime_dependency "foreman"
+  spec.add_runtime_dependency "thor"
+
+  spec.add_runtime_dependency "sinatra"
+  spec.add_runtime_dependency "activerecord"
+  spec.add_runtime_dependency "sinatra-contrib"
+  spec.add_runtime_dependency "sinatra-activerecord"
+  spec.add_runtime_dependency 'sinatra-flash'
+  spec.add_runtime_dependency 'sinatra-redirect-with-flash'
+  spec.add_runtime_dependency 'sinatra-decorator'
+  spec.add_runtime_dependency 'slim'
+  spec.add_runtime_dependency "unicorn"
+  spec.add_runtime_dependency "unicorn-worker-killer"
+  # spec.add_runtime_dependency 'sqlite3'
+end