fluentd-ui 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2210220da165c45be2831a62cc96474a80a9a795
-  data.tar.gz: 6261e50fe33359ce5938a6379e5d686972c89858
+  metadata.gz: e8a878149540a61f5a1c3332f91c788ba07658a0
+  data.tar.gz: b3ffe587af20b8d6feeb79ea48b2ab684a851fe2
 SHA512:
-  metadata.gz: af189cc9da3708a2d1aa7870a0fbd9ed6e9af0b2eaf27a764f6a325b8c786a951fb83648b5432168d7024f0c1c56fa0528ac5f2009149416ce27fc0f38e3802e
-  data.tar.gz: ea8050d54e079e2963348aba49bb1d48606a348efa5b34f546fea42489e98e93543cf0500ef9626375e74bdf2e16993f9215b061ad2591926a7d8e448356d001
+  metadata.gz: 9e467bd3ae13cb3c600a63c9a5bd7e76d2d1be168ddfc1bd39a9fb3e529b04f9650a140f535ced690630363745f6f7543c559e87acbc7fa58d528ff24e8bfa73
+  data.tar.gz: 77f4eb293166a4511e38076e46047326040c08537668a37c6f13ce137ffa9a631452beeaff001026f5f474d8260412064977ce044a124b72a17964d845efab65

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.3.8 - 2014/11/19
+
+* [improve] Recommended plugins are updated.
+* [fixed] Plugin installation on td-agent-ui.
+
+
 Release 0.3.7 - 2014/11/13
 
 * [maintenance] Change httpclient gem as stable version.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluentd-ui (0.3.7)
+    fluentd-ui (0.3.8)
       addressable
       bundler
       draper (~> 1.3)

data/app/models/fluent_gem.rb

@@ -0,0 +1,66 @@
+module FluentGem
+  class GemError < StandardError; end
+
+  class << self
+    LIST_CACHE_KEY = "gem_list".freeze
+
+    def install(*args)
+      run("install", *args)
+    end
+
+    def uninstall(*args)
+      run("uninstall", *args)
+    end
+
+    def list
+      # NOTE: gem list is heavyly used from anywhere in 1 request, if not caching, user experience to be bad
+      #       but long living caching causes mismatch with actual status e.g. user install plugin from console (without fluentd-ui)
+      #       So our decision is that cache `gem list` in 3 seconds
+      Rails.cache.fetch(LIST_CACHE_KEY, expires_in: 3.seconds) do
+        output = `#{gem} list`
+        unless $?.exitstatus.zero?
+          raise GemError, "failed command: `#{gem} list`"
+        end
+        output.lines
+      end
+    end
+
+    def run(*args)
+      # NOTE: use `fluent-gem` instead of `gem`
+      Bundler.with_clean_env do
+        # NOTE: this app is under the Bundler, so call `system` in with_clean_env is Bundler jail breaking
+        cmd = [gem, *args].compact
+        unless system(*cmd)
+          raise GemError, "failed command: `#{cmd.join(" ")}`"
+        end
+        Rails.cache.delete(LIST_CACHE_KEY)
+      end
+      true
+    end
+
+    def gem
+      # Not yet setup any fluentd/td-agent
+      return "fluent-gem" unless Fluentd.instance
+
+      # On installed both td-agent and fluentd system, decide which fluent-gem command should be used depend on setup(Fluentd.instance)
+      if Fluentd.instance && Fluentd.instance.fluentd?
+        "fluent-gem" # maybe `fluent-gem` command is in the $PATH
+      else
+        detect_td_agent_gem
+      end
+    end
+
+    def detect_td_agent_gem
+      # NOTE: td-agent has a command under the /usr/lib{,64}, td-agent2 has under /opt/td-agent
+      %W(
+        /usr/sbin/td-agent-gem
+        /opt/td-agent/embedded/bin/fluent-gem
+        /usr/lib/fluent/ruby/bin/fluent-gem
+        /usr/lib64/fluent/ruby/bin/fluent-gem
+        fluent-gem
+      ).find do |path|
+        system("which #{path}", out: File::NULL, err: File::NULL)
+      end
+    end
+  end
+end

data/app/models/plugin.rb

@@ -87,17 +87,12 @@ class Plugin
   end
 
   def self.installed
-    Rails.cache.fetch("installed_gems", expires_in: 3.seconds) do
-      Bundler.with_clean_env do
-        fluent_gem = fluent_gem_path
-        return [] unless fluent_gem
-        gems = `#{fluent_gem} list`.try(:lines)
-        return [] unless gems
-        gems.grep(/fluent-plugin/).map do |gem|
-          name, versions_str = gem.strip.split(" ")
-          version = versions_str[/[^(), ]+/]
-          new(gem_name: name, version: version)
-        end
+    Bundler.with_clean_env do
+      gems = FluentGem.list
+      gems.grep(/fluent-plugin/).map do |gem|
+        name, versions_str = gem.strip.split(" ")
+        version = versions_str[/[^(), ]+/]
+        new(gem_name: name, version: version)
       end
     end
   end
@@ -141,23 +136,6 @@ class Plugin
     "https://rubygems.org/api/v1/versions/#{gem_name}.json"
   end
 
-  def self.fluent_gem_path
-    # On installed both td-agent and fluentd system, decide which fluent-gem command should be used depend on setup(Fluentd.instance)
-    if Fluentd.instance && Fluentd.instance.fluentd?
-      return "fluent-gem" # maybe `fluent-gem` command is in the $PATH
-    end
-
-    # NOTE: td-agent has a command under the /usr/lib{,64}, td-agent2 has under /opt/td-agent
-    %W(
-      /opt/td-agent/embedded/bin/fluent-gem
-      /usr/lib/fluent/ruby/bin/fluent-gem
-      /usr/lib64/fluent/ruby/bin/fluent-gem
-      fluent-gem
-    ).find do |path|
-      system("which #{path}", out: File::NULL, err: File::NULL)
-    end
-  end
-
   private
 
   def gem_install
@@ -165,7 +143,7 @@ class Plugin
     return if processing?
     return if installed?
     WORKING.push(data)
-    fluent_gem("install", gem_name, "--no-document", "-v", version)
+    FluentGem.install(gem_name, "--no-ri", "--no-rdoc", "-v", version)
   ensure
     WORKING.delete(data)
   end
@@ -175,23 +153,8 @@ class Plugin
     return if processing?
     return unless installed?
     WORKING.push(data)
-    fluent_gem("uninstall", gem_name, "-x", "-a")
+    FluentGem.uninstall(gem_name, "-x", "-a")
   ensure
     WORKING.delete(data)
   end
-
-  def fluent_gem(*commands)
-    # NOTE: use `fluent-gem` instead of `gem`
-    Bundler.with_clean_env do
-      # NOTE: this app is under the Bundler, so call `system` in with_clean_env is Bundler jail breaking
-      unless system(* [fluent_gem_path, *commands])
-        raise GemError, "failed command #{commands.join(" ")}"
-      end
-    end
-    true
-  end
-
-  def fluent_gem_path
-    self.class.fluent_gem_path
-  end
 end

data/config/application.yml

@@ -34,8 +34,6 @@ defaults: &defaults
       name: "numeric-counter"
     - category: storage
       name: "webhdfs"
-    - category: data_source
-      name: "tail-multiline"
     - category: filter
       name: "parser"
     - category: storage
@@ -44,14 +42,18 @@ defaults: &defaults
       name: "dstat"
     - category: filter
       name: "record-reformer"
-    - category: data_source
-      name: "tail-ex"
     - category: storage
       name: "zabbix-simple-bufferd"
     - category: processing
       name: "numeric-monitor"
     - category: notification
       name: "ping-message"
+    - category: processing
+      name: "extract_query_params"
+    - category: processing
+      name: "norikra"
+    - category: notification
+      name: "cloudwatch"
     - category: storage
       name: "pghstore"
     - category: filter
@@ -62,44 +64,30 @@ defaults: &defaults
       name: "grep"
     - category: processing
       name: "grepcounter"
-    - category: data_source
-      name: "mysqlslowquery"
+    - category: processing
+      name: "groupcounter"
     - category: routing
       name: "keep-forward"
     - category: notification
       name: "mail"
     - category: filter
       name: "record-modifier"
-    - category: storage
-      name: "mysql"
     - category: notification
       name: "sns"
     - category: data_source
       name: "munin"
-    - category: routing
-      name: "flume"
-    - category: data_source
-      name: "tail-asis"
     - category: monitoring
       name: "metricsense"
     - category: storage
       name: "sqs"
     - category: processing
       name: "datacalculator"
-    - category: search
-      name: "splunkapi"
     - category: filter
       name: "amplifier-filter"
     - category: filter
       name: "sampling-filter"
-    - category: notification
-      name: "growl"
-    - category: storage
-      name: "dynamodb"
     - category: processing
       name: "flatten"
-    - category: storage
-      name: "hoop"
     - category: processing
       name: "anomalydetect"
     - category: notification
@@ -110,34 +98,28 @@ defaults: &defaults
       name: "redis-counter"
     - category: storage
       name: "kestrel"
-    - category: routing
-      name: "udp"
     - category: storage
       name: "forward-aws"
-    - category: processing
-      name: "extract_query_params"
-    - category: processing
-      name: "norikra"
-    - category: notification
-      name: "cloudwatch"
     - category: data_source
       name: "snmp"
-    - category: data_source
-      name: "http-enhanced"
     - category: processing
-      name: "calc"
+      name: "stats"
+    - category: storage
+      name: "dynamodb"
+    - category: storage
+      name: "cassandra"
     - category: storage
       name: "cassandra-cql"
-    - category: filter
-      name: "hostname"
     - category: data_source
       name: "cloudstack"
+    - category: storage
+      name: "mysql"
     - category: data_source
       name: "mysql-query"
+    - category: data_source
+      name: "mysqlslowquery"
     - category: storage
       name: "redshift"
-    - category: processing
-      name: "groupcounter"
     - category: data_source
       name: "twitter"
     - category: data_source
@@ -148,8 +130,6 @@ defaults: &defaults
       name: "histogram"
     - category: routing
       name: "hash-forward"
-    - category: notification
-      name: "yohoushi"
     - category: storage
       name: "zmq"
     - category: monitoring
@@ -164,6 +144,8 @@ defaults: &defaults
       name: "suppress"
     - category: filter
       name: "filter"
+    - category: filter
+      name: "select"
     - category: storage
       name: "resque"
     - category: data_source
@@ -180,26 +162,24 @@ defaults: &defaults
       name: "reassemble"
     - category: data_source
       name: "jvmwatcher"
-    - category: storage
-      name: "redshift-kwarter"
     - category: data_source
       name: "http-status"
     - category: data_source
       name: "df"
     - category: processing
       name: "typecast"
-    - category: filter
-      name: "select"
     - category: data_source
       name: "rds-slowlog"
     - category: security
       name: "anonymizer"
     - category: search
       name: "splunk"
-    - category: storage
-      name: "cassandra"
+    - category: search
+      name: "splunkapi"
     - category: notification
       name: "boundio"
+    - category: notification
+      name: "growl"
 
 development:
   <<: *defaults

data/lib/fluentd-ui/version.rb

@@ -1,3 +1,3 @@
 module FluentdUI
-  VERSION = "0.3.7"
+  VERSION = "0.3.8"
 end

data/lib/tasks/dep.rake

@@ -2,6 +2,7 @@ namespace :dep do
   desc "list dependency gems order by less referenced"
   task :list do
     require "set"
+    require "fileutils"
     deps = Set.new
     context = false
     current_parent  = false
@@ -9,9 +10,14 @@ namespace :dep do
       "bundler" => "1.7.4" # bundler version does not appear in Gemfile.lock
     }
     skip_gems = %w(fluentd)
+    ignore_gems_at_dump = %w(bundler rake json httpclient fluentd-ui) # these gems are installed by td-agent
     lock_file = "Gemfile.production.lock"
     unless ENV["SKIP_BUNDLE_INSTALL"]
-      system("bundle install --gemfile Gemfile.production", out: STDERR) # ensure lock file is up to date
+      # ensure Gemfile.production.lock file is up to date
+      Bundler.with_clean_env do
+        FileUtils.cp "Gemfile.lock", "Gemfile.production.lock"
+        system("bundle install --no-deployment --gemfile Gemfile.production")
+      end
     end
 
     File.open(lock_file).each_line do |line|
@@ -53,6 +59,7 @@ namespace :dep do
       rank[parent] += 1
     end
     rank.to_a.sort_by {|(name, score)| score }.each do |(name, score)|
+      next if ignore_gems_at_dump.include?(name)
       puts %Q|download "#{name}", "#{versions[name]}"|
     end
   end

data/spec/lib/file_reverse_reader_spec.rb

@@ -39,12 +39,12 @@ describe FileReverseReader do
 
     context "contain ascii only" do
       let(:content) { "ABCDE" }
-      it { should be_false }
+      it { should == false }
     end
 
     context "contain non-ascii" do
       let(:content) { "\x89NG" }
-      it { should be_true }
+      it { should == true }
     end
   end
 

data/spec/models/fluent_gem_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+
+describe FluentGem do
+  describe "#install" do
+    let(:gem) { FluentGem.gem }
+
+    context "no argument" do
+      after { FluentGem.install }
+      it { FluentGem.should_receive(:run).with("install") }
+    end
+
+    context "with arguments" do
+      after { FluentGem.install(*args) }
+
+      context "1" do
+        let(:args) { ["plugin-foo"] }
+        it { FluentGem.should_receive(:run).with("install", *args) }
+      end
+
+      context "2" do
+        let(:args) { ["plugin-foo", "--no-document"] }
+        it { FluentGem.should_receive(:run).with("install", *args) }
+      end
+    end
+  end
+
+  describe "#uninstall" do
+    let(:gem) { FluentGem.gem }
+
+    context "no argument" do
+      after { FluentGem.uninstall }
+      it { FluentGem.should_receive(:run).with("uninstall") }
+    end
+
+    context "with arguments" do
+      after { FluentGem.uninstall(*args) }
+
+      context "1" do
+        let(:args) { ["plugin-foo"] }
+        it { FluentGem.should_receive(:run).with("uninstall", *args) }
+      end
+
+      context "2" do
+        let(:args) { ["plugin-foo", "--no-document"] }
+        it { FluentGem.should_receive(:run).with("uninstall", *args) }
+      end
+    end
+  end
+
+  describe "#list" do
+    before { FluentGem.stub(:`).and_return(gem_list) }
+    subject { FluentGem.list }
+
+    context "no list" do
+      let(:gem_list) { "" }
+      it { subject.to_a.should == [] }
+    end
+
+    context "some lines" do
+      let(:gem_list) { <<-GEM.strip_heredoc }
+        dummy (3.3.3)
+        fluent-plugin-foo (0.1.2)
+        more_dummy (0.0.1)
+      GEM
+      it { subject.to_a.should == gem_list.lines.to_a }
+    end
+
+    context "failed" do
+      let(:gem_list) { "" }
+      before { $?.stub(:exitstatus).and_return(128) }
+      it { expect{ subject }.to raise_error(FluentGem::GemError) }
+    end
+  end
+
+  describe "#run" do
+    before { FluentGem.stub(:system).and_return(ret) }
+    let(:args) { ["install", "foobar"] }
+
+    describe "success" do
+      let(:ret) { true }
+      after { FluentGem.run(*args) }
+      it { FluentGem.should_receive(:system) }
+    end
+
+    describe "failed" do
+      let(:ret) { false }
+      it { expect{ FluentGem.run(*args) }.to raise_error(FluentGem::GemError) }
+    end
+  end
+
+  describe "#gem" do
+    before { Fluentd.stub(:instance).and_return(instance) }
+    subject { FluentGem.gem }
+
+    context "any instance not setup yet" do
+      let(:instance) { nil }
+      it { should == "fluent-gem" }
+    end
+
+    context "fluentd setup" do
+      let(:instance) { Fluentd.new(id: nil, variant: "fluentd_gem", log_file: "dummy.log", pid_file: "dummy.pid", config_file: "dummy.conf") }
+      it { should == "fluent-gem" }
+    end
+
+    context "td-agent 2 setup" do
+      let(:instance) { Fluentd.new(id: nil, variant: "td_agent", log_file: "dummy.log", pid_file: "dummy.pid", config_file: "dummy.conf") }
+      it { should == FluentGem.detect_td_agent_gem }
+    end
+  end
+end

data/spec/models/plugin_spec.rb

@@ -4,7 +4,7 @@ describe Plugin do
   let(:plugin) { build(:plugin) }
 
   describe ".installed" do
-    before { Plugin.stub(:"`").and_return(gem_list) }
+    before { FluentGem.stub(:"`").and_return(gem_list) }
 
     context "fluent-plugin-foo 0.1.2" do
       let(:target) { Plugin.new(gem_name: "fluent-plugin-foo", version: "0.1.2") }
@@ -73,12 +73,12 @@ describe Plugin do
 
         context "installed" do
           let(:installed) { true }
-          it { plugin.should_not_receive(:fluent_gem) }
+          it { FluentGem.should_not_receive(:install) }
         end
 
         context "not installed" do
           let(:installed) { false }
-          it { plugin.should_receive(:fluent_gem) }
+          it { FluentGem.should_receive(:install) }
         end
       end
 
@@ -87,22 +87,22 @@ describe Plugin do
 
         context "installed" do
           let(:installed) { true }
-          it { plugin.should_not_receive(:fluent_gem) }
+          it { FluentGem.should_not_receive(:install) }
         end
 
         context "not installed" do
           let(:installed) { false }
-          it { plugin.should_not_receive(:fluent_gem) }
+          it { FluentGem.should_not_receive(:installed) }
         end
       end
     end
 
     context "system command error" do
-      before { plugin.should_receive(:system).at_least(1).and_return(false) }
+      before { FluentGem.should_receive(:system).at_least(1).and_return(false) }
       subject { expect { plugin.install! } }
 
       it "raise GemError" do
-        subject.to raise_error(Plugin::GemError)
+        subject.to raise_error(FluentGem::GemError)
       end
 
       it "error message contains gem name" do
@@ -139,10 +139,10 @@ describe Plugin do
     before do
       # NOTE: not `plugin.stub` because upgrade! creates new Plugin instance internally
       installed_plugin.stub(:installed?).and_return(true)
-      Plugin.any_instance.stub(:fluent_gem).and_return(true)
+      FluentGem.stub(:run).and_return(true)
 
       installed_plugin.should_receive(:uninstall!)
-      Plugin.any_instance.should_receive(:install!)
+      FluentGem.should_receive(:install)
     end
 
     it { installed_plugin.upgrade!(target_version) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd-ui
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors:
 - Masahiro Nakagawa
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-13 00:00:00.000000000 Z
+date: 2014-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -323,6 +323,7 @@ files:
 - app/mailers/.keep
 - app/models/.keep
 - app/models/concerns/.keep
+- app/models/fluent_gem.rb
 - app/models/fluentd.rb
 - app/models/fluentd/agent.rb
 - app/models/fluentd/agent/common.rb
@@ -473,6 +474,7 @@ files:
 - spec/grok_converter_spec.rb
 - spec/lib/file_reverse_reader_spec.rb
 - spec/lib/fluentd-ui_spec.rb
+- spec/models/fluent_gem_spec.rb
 - spec/models/fluentd/agent_spec.rb
 - spec/models/fluentd/setting/common_spec.rb
 - spec/models/fluentd/setting/in_syslog_spec.rb
@@ -638,6 +640,7 @@ test_files:
 - spec/grok_converter_spec.rb
 - spec/lib/file_reverse_reader_spec.rb
 - spec/lib/fluentd-ui_spec.rb
+- spec/models/fluent_gem_spec.rb
 - spec/models/fluentd/agent_spec.rb
 - spec/models/fluentd/setting/common_spec.rb
 - spec/models/fluentd/setting/in_syslog_spec.rb