fluentd-server 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23543f023f2b14f82abde08cceba1337e730c494
-  data.tar.gz: bb43d8d9a3686cc3e1cba3da52bf532438ba1cc9
+  metadata.gz: 27858b7cd48db1e7cad5f4b2d1f10724b863f103
+  data.tar.gz: 0d0ed73cd1cc881632cafdcefb546b9a9071f3b0
 SHA512:
-  metadata.gz: b430c589ae0aaa6f102d36c98a61dc6230510ea590f528922f26d30a3e4b1511b07b323d7107b82ed417c00fab6734cb657ff7e4ceca60ef771d9a0766a0be64
-  data.tar.gz: 512e455f5e280674c272a992c8ea676323e207c7693eb597d460a9a6d3c88135b394d21b95872724a28c13f340d53538e5832f982b34132cf34644bc1dde0376
+  metadata.gz: bfb1a7297aa0c1f929966fa330060ba52c0ddc207418abf9cf6de97c2b6ac00ad2fb7ef449c833531706f4008755c1fbca69466460d19131defbc9e1f99e9140
+  data.tar.gz: dc2dc999877f2f215d3ab65ee29823697e6429690a150b89ad8da5e12b273ce72519dff4299130f0aad8d93e94fe556a3328e00867e18aa98a5c0ec7b4fdbd36

data/.env

@@ -1,5 +1,8 @@
 PORT=5126
 HOST=0.0.0.0
 # DATABASE_URL=sqlite3:data/fluentd_server.db
-# LOG_PATH=log/application.log
-# LOG_LEVEL=warn
+# JOB_DIR=jobs
+# LOG_PATH=STDOUT
+# LOG_LEVEL=debug
+# LOG_SHIFT_AGE=0
+# LOG_SHIFT_SIZE=1048576

data/.gitignore

@@ -15,3 +15,5 @@ pkg/*
 data/*
 log/*
 fluentd-server/*
+spec/tmp
+jobs/*

data/.travis.yml

@@ -3,3 +3,6 @@ rvm:
   - 2.1
 gemfile:
   - Gemfile
+env:
+  - DATA_DIR=
+  - DATA_DIR=spec/tmp

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.2.0
+
+Enhancements:
+
+* Support restarting td-agent via serf
+  * Created serf-td-agent gem
+  * Introduce Resque
+  * Ajax reload of command result
+* Save/load the content of configuration into/from a file (experimental)
+  * Created acts_as_file gem
+
 # 0.1.0
 
 First version
+

data/Procfile

@@ -1 +1,3 @@
 web: bundle exec unicorn -E production -p $PORT -o $HOST -c config/unicorn.conf
+job: bundle exec bin/fluentd-server job
+serf: $(bundle exec gem path serf-td-agent)/bin/serf agent

data/README.md

@@ -47,7 +47,7 @@ The `include` directive of fluentd config supports `http`, so write just one lin
 include http://fqdn.to.fluentd-server/api/:name?port=24224
 ```
 
-where :name is the name of your config post, so that it will download the real configuration from the Fluentd Server.
+so that it will download the real configuration from the Fluentd Server where :name is the name of your post. 
 
 ## Installation
 
@@ -95,20 +95,32 @@ The default configuration is as follows:
 PORT=5126
 HOST=0.0.0.0
 # DATABASE_URL=sqlite3:data/fluentd_server.db
-# LOG_PATH=log/application.log
-# LOG_LEVEL=warn
+# JOB_DIR=jobs
+# LOG_PATH=STDOUT
+# LOG_LEVEL=debug
+# LOG_SHIFT_AGE=0
+# LOG_SHIFT_SIZE=1048576
 ```
 
+### DATA_DIR (experimental)
+
+Configure `DATA_DIR` in `.env` file as:
+
+```
+DATA_DIR=data
+```
+
+to store and load Fluentd config contents not from DB, but from local files located at the directory `DATA_DIR`.
+This would be useful when you want to manage your config files with git. 
+
 ## HTTP API
 
 See [API.md](API.md).
 
 ## ToDo
 
-* Local file storage
-
-  * Saving and loading conf from localfiles rather than DB would be nice because it makes possible to manage conf with git
-  * Fluentd Server should cache them on memory, and refresh caches by detecting files are updated
+* Create a sync worker to synchronize local file existences with db (delete and create entries on db). 
+* Automatic deployment (restart) support like the one of chef-server
 
 ## ChangeLog
 

data/Rakefile

@@ -1,17 +1,16 @@
 require "bundler/gem_tasks"
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dotenv/tasks'
+require 'fluentd_server/environment'
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec) do |t|
   t.rspec_opts = ["-c", "-f progress"] # '--format specdoc'
   t.pattern = 'spec/**/*_spec.rb'
 end
-
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'dotenv/tasks'
-require 'fluentd_server/config'
-
-require 'sinatra/activerecord/rake'
+task :test => :spec
+task :default => :spec
 
 task :console => :dotenv do
   require "fluentd_server"
@@ -22,5 +21,4 @@ task :console => :dotenv do
 end
 task :c => :console
 
-task :test => :spec
-task :default => :spec
+require 'sinatra/activerecord/rake'

data/db/migrate/0002_create_delayed_jobs.rb

@@ -0,0 +1,22 @@
+class CreateDelayedJobs < ActiveRecord::Migration
+  def self.up
+    create_table :delayed_jobs do |table|
+      table.integer  :priority, :default => 0, :null => false # Allows some jobs to jump to the front of the queue
+      table.integer  :attempts, :default => 0, :null => false # Provides for retries, but still fail eventually.
+      table.text     :handler, :null => false                 # YAML-encoded string of the object that will do work
+      table.text     :last_error                              # reason for last failure (See Note below)
+      table.datetime :run_at                                  # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :locked_at                               # Set when a client is working on this object
+      table.datetime :failed_at                               # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.string   :locked_by                               # Who is working on this object (if locked)
+      table.string   :queue                                   # The name of the queue this job is in
+      table.timestamps
+    end
+
+    add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+  end
+
+  def self.down
+    drop_table :delayed_jobs
+  end
+end

data/db/migrate/0003_create_tasks.rb

@@ -0,0 +1,14 @@
+class CreateTasks < ActiveRecord::Migration
+  def self.up
+    create_table :tasks do |t|
+      t.string :name
+      t.text :body
+      t.integer :exit_code
+      t.timestamps
+    end
+  end
+ 
+  def self.down
+    drop_table :tasks
+  end
+end

data/db/schema.rb

@@ -11,7 +11,23 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20140512203133) do
+ActiveRecord::Schema.define(version: 3) do
+
+  create_table "delayed_jobs", force: true do |t|
+    t.integer  "priority",   default: 0, null: false
+    t.integer  "attempts",   default: 0, null: false
+    t.text     "handler",                null: false
+    t.text     "last_error"
+    t.datetime "run_at"
+    t.datetime "locked_at"
+    t.datetime "failed_at"
+    t.string   "locked_by"
+    t.string   "queue"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority"
 
   create_table "posts", force: true do |t|
     t.string   "name"
@@ -22,4 +38,12 @@ ActiveRecord::Schema.define(version: 20140512203133) do
 
   add_index "posts", ["name"], name: "index_posts_on_name", unique: true
 
+  create_table "tasks", force: true do |t|
+    t.string   "name"
+    t.text     "body"
+    t.integer  "exit_code"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
 end

data/fluentd-server.gemspec

@@ -39,5 +39,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'slim'
   spec.add_runtime_dependency "unicorn"
   spec.add_runtime_dependency "unicorn-worker-killer"
+  spec.add_runtime_dependency "delayed_job_active_record"
+  spec.add_runtime_dependency "daemons"
+  spec.add_runtime_dependency "serf-td-agent"
+  spec.add_runtime_dependency "acts_as_file"
   # spec.add_runtime_dependency 'sqlite3'
 end

data/lib/fluentd_server/cli.rb

@@ -7,54 +7,104 @@ class FluentdServer::CLI < Thor
   BASE_DIR = File.join(Dir.pwd, "fluentd-server")
   DATA_DIR = File.join(BASE_DIR, "data")
   LOG_DIR = File.join(BASE_DIR, "log")
+  JOB_DIR = File.join(BASE_DIR, "jobs")
   LOG_FILE = File.join(LOG_DIR, "application.log")
   ENV_FILE = File.join(BASE_DIR, ".env")
   PROCFILE = File.join(BASE_DIR, "Procfile")
   CONFIGRU_FILE = File.join(BASE_DIR, "config.ru")
-  DB_DIR = File.join(BASE_DIR, "db")
   CONFIG_DIR= File.join(BASE_DIR, "config")
 
   DEFAULT_DOTENV =<<-EOS
 PORT=5126
 HOST=0.0.0.0
 DATABASE_URL=sqlite3:#{DATA_DIR}/fluentd_server.db
+JOB_DIR=#{JOB_DIR}
 LOG_PATH=#{LOG_FILE}
 LOG_LEVEL=warn
+LOG_SHIFT_AGE=0
+LOG_SHIFT_SIZE=1048576
 EOS
 
   DEFAULT_PROCFILE =<<-EOS
 web: unicorn -E production -p $PORT -o $HOST -c config/unicorn.conf
+job: fluentd-server job
+serf: $(gem path serf-td-agent)/bin/serf agent
 EOS
 
   default_command :start
 
+  def initialize(args = [], opts = [], config = {})
+    super(args, opts, config)
+  end
+
   desc "new", "Creates fluentd-server resource directory"
   def new
     FileUtils.mkdir_p(LOG_DIR)
+    FileUtils.mkdir_p(JOB_DIR)
     File.write ENV_FILE, DEFAULT_DOTENV
     File.write PROCFILE, DEFAULT_PROCFILE
     FileUtils.cp(File.expand_path("../../../config.ru", __FILE__), CONFIGRU_FILE)
-    FileUtils.cp_r(File.expand_path("../../../db", __FILE__), DB_DIR)
     FileUtils.cp_r(File.expand_path("../../../config", __FILE__), CONFIG_DIR)
     puts 'fluentd-server new finished.'
   end
 
   desc "init", "Creates database schema"
   def init
-    require 'fluentd_server/environments'
+    Dotenv.load
+    require 'fluentd_server/environment'
     require 'rake'
     require 'sinatra/activerecord/rake'
-    Rake::Task['db:migrate'].invoke
+    ActiveRecord::Tasks::DatabaseTasks.db_dir = File.expand_path("../../../db", __FILE__)
+    ActiveRecord::Tasks::DatabaseTasks.migrations_paths = [File.expand_path("../../../db/migrate", __FILE__)]
+    # ToDo: Fix that db:migrate raises an error in the case of sqlite3 like
+    # SQLite3::SQLException: database schema has changed: INSERT INTO "schema_migrations" ("version") VALUES (?)
+    # Rake::Task['db:migrate'].invoke
+    # Use db:schema:load after generating db/schema.rb by executing db:migrate several times for now
+    Rake::Task['db:schema:load'].invoke
     puts 'fluentd-server init finished.'
   end
 
+  desc "migrate", "Migrate database schema"
+  def migrate
+    Dotenv.load
+    require 'fluentd_server/environment'
+    require 'rake'
+    require 'sinatra/activerecord/rake'
+    ActiveRecord::Tasks::DatabaseTasks.db_dir = File.expand_path("../../../db", __FILE__)
+    ActiveRecord::Tasks::DatabaseTasks.migrations_paths = [File.expand_path("../../../db/migrate", __FILE__)]
+    Rake::Task['db:migrate'].invoke
+  end
+
   desc "start", "Sartup fluentd_server"
   def start
-    Dotenv.load
+    self.migrate # do migration before start not to forget on updation
     require "foreman/cli"
     Foreman::CLI.new.invoke(:start, [], {})
   end
 
+  # reference: https://gist.github.com/robhurring/732327
+  desc "job", "Sartup fluentd_server job worker"
+  def job
+    Dotenv.load
+    require 'delayed_job'
+    require 'fluentd_server/model'
+    worker_options = {
+      :min_priority => ENV['MIN_PRIORITY'],
+      :max_priority => ENV['MAX_PRIORITY'],
+      :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','),
+      :quiet => false
+    }
+    Delayed::Worker.new(worker_options).start
+  end
+
+  desc "job_clear", "Clear fluentd_server delayed_job queue"
+  def job_clear
+    Dotenv.load
+    require 'delayed_job'
+    require 'fluentd_server/model'
+    Delayed::Job.delete_all
+  end
+
   no_tasks do
     def abort(msg)
       $stderr.puts msg

data/lib/fluentd_server/config.rb

@@ -5,10 +5,18 @@ require 'dotenv'
 Dotenv.load
 
 module FluentdServer::Config
+  def self.data_dir
+    ENV['DATA_DIR'] == "" ? nil : ENV['DATA_DIR']
+  end
+
   def self.database_url
     ENV.fetch('DATABASE_URL', 'sqlite3:data/fluentd_server.db')
   end
 
+  def self.job_dir
+    ENV.fetch('JOB_DIR', 'jobs')
+  end
+
   def self.log_path
     ENV.fetch('LOG_PATH', 'STDOUT')
   end
@@ -24,4 +32,8 @@ module FluentdServer::Config
   def self.log_shift_size
     ENV.fetch('LOG_SHIFT_SIZE', '1048576')
   end
+
+  def self.task_max_num
+    ENV.fetch('TASK_MAX_NUM', '20').to_i
+  end
 end

data/lib/fluentd_server/decorator.rb

@@ -1,7 +1,8 @@
 require 'sinatra/decorator'
+require 'fluentd_server/web_helper'
 
 class PostDecorator < Sinatra::Decorator::Base
-  include Rack::Utils
+  include FluentdServer::WebHelper
 
   def success_message
     'Success!'
@@ -17,5 +18,26 @@ class PostDecorator < Sinatra::Decorator::Base
     namespace = OpenStruct.new(locals)
     ERB.new(self.body, nil, "-").result(namespace.instance_eval { binding })
   end
+
+  def link_to
+    %Q[<a href="#{h("/posts/#{self.id}/edit")}">
+      <span class="label label-success">&nbsp;</span> ##{h(self.id)} #{h(self.name)}
+    </a>]
+  end
+
+  def create_button
+    %Q[<div style="padding: 0 0 10px 10px;font-variant:small-caps;">
+    <a class="btn btn-default #{active_if(self.new?)}" role="button"
+    href="/posts/create">Create Config</a></div>]
+  end
 end
 
+class TaskDecorator < Sinatra::Decorator::Base
+  include FluentdServer::WebHelper
+
+  def link_to
+    %Q[<a href="#{h("/tasks/#{self.id}")}">
+      <span class="label label-success">&nbsp;</span> ##{h(self.id)} #{h(self.name)}
+    </a>]
+  end
+end

data/lib/fluentd_server/{environments.rb → environment.rb}

@@ -3,6 +3,8 @@ require 'sinatra/activerecord'
 require 'fluentd_server/config'
 require 'fluentd_server/logger'
 
+ROOT = File.expand_path('../../..', __FILE__)
+
 configure do
   set :show_exceptions, true
   ActiveRecord::Base.logger = FluentdServer.logger
@@ -29,6 +31,20 @@ end
 configure :test do
   ActiveRecord::Base.establish_connection(
     :adapter  => 'sqlite3',
-    :database => ':memory'
+    :database => ':memory:'
   )
 end
+
+# Configure DelayedJob
+require 'delayed_job'
+configure do
+  Delayed::Worker.backend = :active_record # This defines Delayed::Job model
+  Delayed::Worker.logger = FluentdServer.logger
+end
+
+configure :development, :test do
+  Delayed::Worker.destroy_failed_jobs = true
+  Delayed::Worker.sleep_delay = 5
+  Delayed::Worker.max_attempts = 5
+  Delayed::Worker.max_run_time = 5.minutes
+end

data/lib/fluentd_server/model.rb

@@ -1,10 +1,68 @@
 require 'sinatra/activerecord'
 require 'sinatra/decorator'
-require 'fluentd_server/environments'
+require 'fluentd_server/environment'
+require 'fluentd_server/task_runner'
+require 'acts_as_file'
+
+class Delayed::Job < ActiveRecord::Base; end
 
 class Post < ActiveRecord::Base
   include Sinatra::Decorator::Decoratable
+  include FluentdServer::Logger
 
   validates :name, presence: true
   validates :body, presence: true
+
+  if FluentdServer::Config.data_dir
+    include ActsAsFile
+
+    def filename
+      File.join(FluentdServer::Config.data_dir, "#{self.name}.erb") if self.name
+    end
+
+    acts_as_file :body => self.instance_method(:filename)
+  end
+
+  def new?
+    self.id.nil?
+  end
+end
+
+class Task < ActiveRecord::Base
+  include Sinatra::Decorator::Decoratable
+  include FluentdServer::Logger
+  include ActsAsFile
+  include TaskRunner # task runnable codes are here
+
+  def filename
+    prefix = "#{self.id.to_s.rjust(4, '0')}" if self.id
+    File.join(FluentdServer::Config.job_dir, "#{prefix}_result.txt") if prefix
+  end
+
+  acts_as_file :body => self.instance_method(:filename)
+
+  def finished?
+    !self.exit_code.nil?
+  end
+
+  def successful?
+    self.finished? and self.exit_code == 0
+  end
+
+  def error?
+    self.finished? and self.exit_code != 0
+  end
+
+  def new?
+    self.id.nil?
+  end
+
+  def self.create_and_delete(*args)
+    created = self.create(*args)
+    if self.count > FluentdServer::Config.task_max_num
+      oldest = self.first
+      oldest.destroy_with_file
+    end
+    created
+  end
 end

data/lib/fluentd_server/task_runner.rb

@@ -0,0 +1,88 @@
+# included by class Task
+module TaskRunner
+  def self.included(klass)
+    require 'fileutils'
+    klass.extend(ClassMethods)
+  end
+
+  def restart
+    system(write_event_header('restart'))
+    cmd = serf_event('restart')
+    logger.debug "run #{cmd}"
+    self.exit_code = system(cmd)
+    self.save!
+  end
+ 
+  def status
+    system(write_query_header('status'))
+    self.delay.delayed_status
+  end
+
+  def delayed_status
+    cmd = serf_query('status')
+    logger.debug "run #{cmd}"
+    self.exit_code = system(cmd)
+    self.save!
+  end
+
+  def configtest
+    system(write_query_header('configtest'))
+    self.delay.delayed_configtest
+  end
+
+  def delayed_configtest
+    cmd = serf_query('configtest')
+    logger.debug "run #{cmd}"
+    self.exit_code = system(cmd)
+    self.save!
+  end
+
+  ## delayed_job hooks
+
+  def before(job)
+    @job = job
+  end
+
+  def failure
+    logger.warn "job #{@job.attributes} failed"
+  end
+
+  ## helpers
+
+  def write_event_header(cmd)
+    "echo '$ serf event td-agent-#{cmd}' > #{self.filename}"
+  end
+
+  def write_query_header(cmd)
+    "echo '$ serf query td-agent-#{cmd}' > #{self.filename}"
+  end
+
+  # serf event works asynchronously, so it does not take time
+  def serf_event(cmd)
+    "#{self.class.serf_path} event td-agent-#{cmd} >> #{self.filename} 2>&1"
+  end
+
+  # serf query works synchronously, so it takes time
+  def serf_query(cmd)
+    "#{self.class.serf_path} query td-agent-#{cmd} >> #{self.filename} 2>&1"
+  end
+
+  module ClassMethods
+    def serf_path
+      @serf_path ||= "#{find_path_gem('serf-td-agent')}/bin/serf"
+    end
+
+    # from gem-path gem
+    def find_path_gem name
+      path_gem = Gem.path.find do |base|
+        path_gem = $LOAD_PATH.find do |path|
+          path_gem = path[%r{#{base}/gems/#{name}\-[^/-]+/}]
+          break path_gem if path_gem
+        end
+        break path_gem if path_gem
+      end
+      path_gem[0...-1] if path_gem
+    end
+  end
+end
+

data/lib/fluentd_server/version.rb

@@ -1,3 +1,3 @@
 module FluentdServer
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end