fluentd-server 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27858b7cd48db1e7cad5f4b2d1f10724b863f103
-  data.tar.gz: 0d0ed73cd1cc881632cafdcefb546b9a9071f3b0
+  metadata.gz: 29880280e87c05f7108661ac476c81546ad0f5b6
+  data.tar.gz: 9d375e9b51d467ffadebf388991dab55a267d943
 SHA512:
-  metadata.gz: bfb1a7297aa0c1f929966fa330060ba52c0ddc207418abf9cf6de97c2b6ac00ad2fb7ef449c833531706f4008755c1fbca69466460d19131defbc9e1f99e9140
-  data.tar.gz: dc2dc999877f2f215d3ab65ee29823697e6429690a150b89ad8da5e12b273ce72519dff4299130f0aad8d93e94fe556a3328e00867e18aa98a5c0ec7b4fdbd36
+  metadata.gz: 94c767cc074eaf54d61b60b21bd472ab20545bb39e35c41bbbc9742f5c148d9a8a2bfc0a1e68be936a28a6dddddf9d653087c71ceeef31c83262b67c4c7319bf
+  data.tar.gz: b7c46262f055be84c81a4205ec496022188bbc4923b1bd49a1d5919c65d30fb3b931a7194464c3668e35fbca60db03eddeeb15315d62a2d88dcf0a63c142f10b

data/.env

@@ -6,3 +6,6 @@ HOST=0.0.0.0
 # LOG_LEVEL=debug
 # LOG_SHIFT_AGE=0
 # LOG_SHIFT_SIZE=1048576
+# LOCAL_STORAGE=false
+# DATA_DIR=data
+# SYNC_INTERVAL=60

data/.travis.yml

@@ -4,5 +4,5 @@ rvm:
 gemfile:
   - Gemfile
 env:
-  - DATA_DIR=
-  - DATA_DIR=spec/tmp
+  - LOCAL_STORAGE=false
+  - LOCAL_STORAGE=true DATA_DIR=spec/tmp

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 0.3.0
+
+Enhancements:
+
+* Add sync_worker
+
+Changes:
+
+* Use env LOCAL_STORAGE rather than DATA_DIR to enable local file storage feature
+
 # 0.2.0
 
 Enhancements:

data/Procfile

@@ -1,3 +1,4 @@
 web: bundle exec unicorn -E production -p $PORT -o $HOST -c config/unicorn.conf
 job: bundle exec bin/fluentd-server job
+sync: bundle exec bin/fluentd-server sync
 serf: $(bundle exec gem path serf-td-agent)/bin/serf agent

data/README.md

@@ -100,18 +100,28 @@ HOST=0.0.0.0
 # LOG_LEVEL=debug
 # LOG_SHIFT_AGE=0
 # LOG_SHIFT_SIZE=1048576
+# LOCAL_STORAGE=false
+# DATA_DIR=data
+# SYNC_INTERVAL=60
 ```
 
-### DATA_DIR (experimental)
+### LOCAL FILE STORAGE
 
-Configure `DATA_DIR` in `.env` file as:
+Fluentd Server also supports to edit Fluentd configuration files as local files for the case using `git` (or any VCSs) to manage Fluentd configuration revisions. 
+
+To use this feature, enable `LOCAL_STORAGE` in `.env` file:
 
 ```
+LOCAL_STORAGE=true
 DATA_DIR=data
+SYNC_INTERVAL=60
 ```
 
-to store and load Fluentd config contents not from DB, but from local files located at the directory `DATA_DIR`.
-This would be useful when you want to manage your config files with git. 
+The `DATA_DIR` is the location to place your configuration files locally, and the `SYNC_INTERVAL` is the interval where a synchronization worker works.
+
+Putting any files whose name ends with `.erb` in `DATA_DIR` automatically synchronizes with DB by the `sync` worker. Removing `.erb` files is also synchronized with DB. 
+
+NOTE: Enabling this feature disables to edit the Fluentd configuration from the Web UI. 
 
 ## HTTP API
 
@@ -119,7 +129,6 @@ See [API.md](API.md).
 
 ## ToDo
 
-* Create a sync worker to synchronize local file existences with db (delete and create entries on db). 
 * Automatic deployment (restart) support like the one of chef-server
 
 ## ChangeLog

data/fluentd-server.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec", "~> 3"
   spec.add_development_dependency "capybara"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-nav"

data/lib/fluentd_server/cli.rb

@@ -23,11 +23,15 @@ LOG_PATH=#{LOG_FILE}
 LOG_LEVEL=warn
 LOG_SHIFT_AGE=0
 LOG_SHIFT_SIZE=1048576
+LOCAL_STORAGE=false
+DATA_DIR=#{DATA_DIR}
+SYNC_INTERVAL=60
 EOS
 
   DEFAULT_PROCFILE =<<-EOS
 web: unicorn -E production -p $PORT -o $HOST -c config/unicorn.conf
 job: fluentd-server job
+sync: fluentd-server sync
 serf: $(gem path serf-td-agent)/bin/serf agent
 EOS
 
@@ -105,6 +109,13 @@ EOS
     Delayed::Job.delete_all
   end
 
+  desc "sync", "Sartup fluentd_server sync worker"
+  def sync
+    Dotenv.load
+    require 'fluentd_server/sync_worker'
+    FluentdServer::SyncWorker.start
+  end
+
   no_tasks do
     def abort(msg)
       $stderr.puts msg

data/lib/fluentd_server/config.rb

@@ -5,10 +5,6 @@ require 'dotenv'
 Dotenv.load
 
 module FluentdServer::Config
-  def self.data_dir
-    ENV['DATA_DIR'] == "" ? nil : ENV['DATA_DIR']
-  end
-
   def self.database_url
     ENV.fetch('DATABASE_URL', 'sqlite3:data/fluentd_server.db')
   end
@@ -36,4 +32,16 @@ module FluentdServer::Config
   def self.task_max_num
     ENV.fetch('TASK_MAX_NUM', '20').to_i
   end
+
+  def self.local_storage
+    ENV.fetch('LOCAL_STORAGE', 'false') == 'true' ? true : false
+  end
+
+  def self.data_dir
+    ENV.fetch('DATA_DIR', 'data')
+  end
+
+  def self.sync_interval
+    ENV.fetch('SYNC_INTERVAL', '60').to_i
+  end
 end

data/lib/fluentd_server/model.rb

@@ -11,9 +11,8 @@ class Post < ActiveRecord::Base
   include FluentdServer::Logger
 
   validates :name, presence: true
-  validates :body, presence: true
 
-  if FluentdServer::Config.data_dir
+  if FluentdServer::Config.local_storage
     include ActsAsFile
 
     def filename

data/lib/fluentd_server/sync_runner.rb

@@ -0,0 +1,70 @@
+require 'fluentd_server/model'
+require 'fluentd_server/logger'
+
+class FluentdServer::SyncRunner
+  include FluentdServer::Logger
+
+  def self.run(opts = {})
+    self.new(opts).run
+  end
+
+  def initialize(opts = {})
+  end
+
+  def run
+    return nil unless FluentdServer::Config.local_storage
+    plus, minus = find_diff
+    create(plus)
+    delete(minus)
+  end
+
+  def find_locals
+    return [] unless FluentdServer::Config.local_storage
+    names = []
+    Dir.chdir(FluentdServer::Config.data_dir) do
+      Dir.glob("*.erb") do |filename|
+        names << filename.chomp('.erb')
+      end
+    end
+    names
+  end
+
+  def create(names)
+    # ToDo: bulk insert with sqlite, postgresql? use activerecord-import for mysql2
+    logger.debug "[sync] create #{names}"
+    names.each do |name|
+      begin
+        Post.create(name: name)
+      rescue ActiveRecord::RecordNotUnique => e
+        logger.debug "#{e.class} #{e.message} #{name}"
+      rescue => e
+        logger.warn "#{e.class} #{e.message} #{name}"
+      end
+    end
+  end
+
+  def delete(names)
+    logger.debug "[sync] remove #{names}"
+    begin
+      Post.where(:name => names).delete_all
+    rescue => e
+      logger.warn "#{e.class} #{e.message} #{names}"
+    end
+  end
+
+  # Find difference between given array of paths and paths stored in DB
+  #
+  # @param [Integer] batch_size The batch size of a select query
+  # @return [Array] Plus (array) and minus (array) differences
+  def find_diff(batch_size: 1000)
+    names = find_locals
+    plus  = names
+    minus = []
+    Post.select('id, name').find_in_batches(batch_size: batch_size) do |batches|
+      batches = batches.map(&:name)
+      plus   -= batches
+      minus  += (batches - names)
+    end
+    [plus, minus]
+  end
+end

data/lib/fluentd_server/sync_worker.rb

@@ -0,0 +1,81 @@
+require "fluentd_server/config"
+require "fluentd_server/logger"
+require "fluentd_server/sync_runner"
+
+# reference: https://github.com/focuslight/focuslight/blob/master/lib/focuslight/worker.rb
+# thanks!
+class FluentdServer::SyncWorker
+  include FluentdServer::Logger
+
+  DEFAULT_INTERVAL = 60
+  attr_reader :interval
+
+  def self.start(opts = {})
+    self.new(opts).start
+  end
+
+  def initialize(opts = {})
+    @opts = opts
+    @interval = opts[:interval] || FluentdServer::Config.sync_interval || DEFAULT_INTERVAL
+    @signals = []
+  end
+
+  def update_next!
+    now = Time.now
+    @next_time = now - ( now.to_i % @interval ) + @interval
+  end
+
+  def start
+    Signal.trap(:INT){  @signals << :INT }
+    Signal.trap(:HUP){  @signals << :HUP }
+    Signal.trap(:TERM){ @signals << :TERM }
+    Signal.trap(:PIPE, "IGNORE")
+
+    update_next!
+    logger.info("[sync] first updater start in #{@next_time}")
+
+    childpid = nil
+    while sleep(0.5) do
+      if childpid
+        begin
+          if Process.waitpid(childpid, Process::WNOHANG)
+            #TODO: $? (Process::Status object)
+            logger.debug("[sync] update finished pid: #{childpid}, code: #{$? >> 8}")
+            logger.debug("[sync] next updater start in #{@next_time}")
+            childpid = nil
+          end
+        rescue Errno::ECHILD
+          logger.warn("[sync] no child process");
+          childpid = nil
+        end
+      end
+
+      unless @signals.empty?
+        logger.warn("[sync] signals_received: #{@signals.join(',')}")
+        break
+      end
+
+      next if Time.now < @next_time
+      update_next!
+      logger.debug("[sync] (#{@next_time}) updater start")
+
+      if childpid
+        logger.warn("[sync] previous updater exists, skipping this time")
+        next
+      end
+
+      childpid = fork do
+        FluentdServer::SyncRunner.run(@opts)
+      end
+    end
+
+    if childpid
+      logger.warn("[sync] waiting for updater process finishing")
+      begin
+        waitpid childpid
+      rescue Errno::ECHILD
+        # ignore
+      end
+    end
+  end
+end

data/lib/fluentd_server/version.rb

@@ -1,3 +1,3 @@
 module FluentdServer
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/model_spec.rb

@@ -11,8 +11,8 @@ describe Task do
   after { Task.delete_all }
 
   context '#new?' do
-    it { expect(Task.new.new?).to be_true }
-    it { expect(Task.create.new?).to be_false }
+    it { expect(Task.new.new?).to be_truthy }
+    it { expect(Task.create.new?).to be_falsey }
   end
 
   context '#filename' do
@@ -21,7 +21,7 @@ describe Task do
   end
 
   context '#create_and_delete' do
-    before { FluentdServer::Config.stub(:task_max_num).and_return(1) }
+    before { allow(FluentdServer::Config).to receive(:task_max_num).and_return(1) }
     before { @oldest = Task.create(name: 'Restart') }
     it {
       Task.create_and_delete(name: 'Restart')

data/spec/sync_runner_spec.rb

@@ -0,0 +1,77 @@
+require_relative 'spec_helper'
+require 'fluentd_server/sync_runner'
+
+if FluentdServer::Config.local_storage
+  describe 'SyncRunner' do
+    around {
+      filenames = File.join(FluentdServer::Config.data_dir, '*.erb') 
+      Dir.glob(filenames).each { |f| File.delete(f) rescue nil }
+      Post.delete_all
+    }
+    let(:runner) { FluentdServer::SyncRunner.new }
+
+    context '#find_locals' do
+      before { Post.create(name: 'post1') }
+      before { Post.create(name: 'post2') }
+      let(:subject) { runner.find_locals }
+      it { should =~ ['post1', 'post2' ] }
+    end
+
+    context '#find_diff' do
+      before { Post.new(name: 'post1').save_without_file }
+      before { Post.create(name: 'post2') }
+      before { File.open(Post.new(name: 'post3').filename, "w") {} }
+      it {
+        plus, minus = runner.find_diff
+        expect(minus).to eql(['post1'])
+        expect(plus).to eql(['post3'])
+      }
+    end
+
+    context '#create' do
+      before { Post.create(name: 'post1') }
+      before { runner.create(%w[post1 post2]) }
+      it {
+        expect(Post.find_by(name: 'post1').body).not_to be_nil
+        expect(Post.find_by(name: 'post2').body).to be_nil
+      }
+    end
+
+    context '#delete' do
+      before {
+        post1 = Post.create(name: 'post1')
+        post2 = Post.create(name: 'post2')
+        runner.delete(%w[post1])
+      }
+      it {
+        expect(Post.find_by(name: 'post1')).to be_nil
+        expect(Post.find_by(name: 'post2')).not_to be_nil
+      }
+    end
+
+    context '#run' do
+      before { Post.new(name: 'post1').save_without_file }
+      before { Post.create(name: 'post2') }
+      before { File.open(Post.new(name: 'post3').filename, "w") {} }
+      it {
+        runner.run
+        expect(Post.find_by(name: 'post1')).to be_nil
+        expect(Post.find_by(name: 'post2')).not_to be_nil
+        expect(Post.find_by(name: 'post3')).not_to be_nil
+      }
+    end
+  end
+else
+  describe 'SyncRunner' do
+    context '#run' do
+      let(:subject) { FluentdServer::SyncRunner.new.run }
+      it { should be_nil }
+    end
+    context '#find_locals' do
+      before { Post.create(name: 'post1') }
+      before { Post.create(name: 'post2') }
+      let(:subject) { FluentdServer::SyncRunner.new.find_locals }
+      it { should == [] }
+    end
+  end
+end

data/spec/task_runner_spec.rb

@@ -4,11 +4,11 @@ require 'fluentd_server/model'
 
 describe 'TaskRunner' do
   let(:task) { Task.create }
-  before { task.stub(:delay).and_return { task } }
+  before { allow(task).to receive(:delay).and_return(task) }
   after { Task.delete_all }
 
   context '.serf_path' do
-    it { expect(File.executable?(Task.serf_path)).to be_true }
+    it { expect(File.executable?(Task.serf_path)).to be_truthy }
   end
 
   # ToDo: Test whether the serf command is executed correctly
@@ -20,7 +20,7 @@ describe 'TaskRunner' do
     it { expect { task.status }.not_to raise_error }
   end
 
-  context '#configrest' do
+  context '#configtest' do
     it { expect { task.configtest }.not_to raise_error }
   end
 end

data/spec/web_spec.rb

@@ -12,7 +12,7 @@ describe 'Post' do
   context 'get ALL posts' do
     it 'visit' do
       visit '/'
-      page.status_code.should == 200
+      expect(page.status_code).to be == 200
     end
   end
 
@@ -20,7 +20,7 @@ describe 'Post' do
     before { visit '/posts/create' }
 
     it 'visit' do
-      page.status_code.should == 200
+      expect(page.status_code).to be == 200
     end
 
     it 'create' do
@@ -38,7 +38,7 @@ describe 'Post' do
     before { visit "/posts/#{post.id}/edit" }
 
     it 'visit' do
-      page.status_code.should == 200
+      expect(page.status_code).to be == 200
     end
 
     it 'edit' do
@@ -81,7 +81,7 @@ describe 'Task' do
   context 'list tasks' do
     it 'visit' do
       visit '/tasks'
-      page.status_code.should == 200
+      expect(page.status_code).to be == 200
     end
   end
 
@@ -89,7 +89,7 @@ describe 'Task' do
     before { @task = Task.create }
     it 'visit' do
       visit "/tasks/#{@task.id}"
-      page.status_code.should == 200
+      expect(page.status_code).to be == 200
     end
   end
 

data/views/_js.slim

@@ -28,3 +28,9 @@ javascript:
     form.submit();
     return false;
   });
+- if FluentdServer::Config.local_storage
+  javascript:
+    $('#post_name').attr("disabled", "disabled");
+    $('#post_body').attr("disabled", "disabled");
+    $('#post_submit').addClass('disabled');
+    $('#post_delete').addClass('disabled');

data/views/posts/create.slim

@@ -2,7 +2,7 @@ h2[style="margin-top:0px;"]
   | Create Config
 br
 == bootstrap_flash
-form[action=url_for("/posts") method="post" role="form"]
+form[id="post_form" action=url_for("/posts") method="post" role="form"]
   .form-group
     label[for="post_name"]
       | Name:
@@ -15,6 +15,6 @@ form[action=url_for("/posts") method="post" role="form"]
     textarea#post_body.form-control[name="post[body]" rows="10"]
       = @post.body
     br
-  button.btn.btn-success[type="submit"]
+  button.btn.btn-success[id="post_submit" type="submit"]
     | Submit
   br

data/views/posts/edit.slim

@@ -2,7 +2,7 @@ h2[style="margin-top:0px;"]
   | Edit Config
 br
 == bootstrap_flash
-form action=url_for("/posts/#{@post.id}") method="post"
+form id="post_form" action=url_for("/posts/#{@post.id}") method="post"
   .form-group
     label[for="post_name"]
       | Name:
@@ -17,8 +17,8 @@ form action=url_for("/posts/#{@post.id}") method="post"
     br
     a href=url_for("/api/#{@post.name}")
       | Render
-  button.btn.btn-success[type="submit"]
+  button.btn.btn-success[id="post_submit" type="submit"]
     | Submit
   |  
-  a.btn.btn-danger data-method="post" data-confirm="true" href=url_for("/posts/#{@post.id}/delete")
+  a.btn.btn-danger id="post_delete" data-method="post" data-confirm="true" href=url_for("/posts/#{@post.id}/delete")
     | Delete

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd-server
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-26 00:00:00.000000000 Z
+date: 2014-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
@@ -367,6 +367,8 @@ files:
 - lib/fluentd_server/environment.rb
 - lib/fluentd_server/logger.rb
 - lib/fluentd_server/model.rb
+- lib/fluentd_server/sync_runner.rb
+- lib/fluentd_server/sync_worker.rb
 - lib/fluentd_server/task_runner.rb
 - lib/fluentd_server/version.rb
 - lib/fluentd_server/web.rb
@@ -385,6 +387,7 @@ files:
 - spec/bench/result.md
 - spec/model_spec.rb
 - spec/spec_helper.rb
+- spec/sync_runner_spec.rb
 - spec/task_runner_spec.rb
 - spec/tmp/.gitkeep
 - spec/web_helper_spec.rb
@@ -428,6 +431,7 @@ test_files:
 - spec/bench/result.md
 - spec/model_spec.rb
 - spec/spec_helper.rb
+- spec/sync_runner_spec.rb
 - spec/task_runner_spec.rb
 - spec/tmp/.gitkeep
 - spec/web_helper_spec.rb