fluentd-ui 0.1.0

data/config/locales/tutorial_en.yml

@@ -0,0 +1,117 @@
+en:
+  tutorials:
+    index:
+      step1: "Setup fluentd"
+      step2: "Start fluentd"
+      page_title: Tutorial
+      start_tutorial: Start tutorial
+    chapter1:
+      page_title: "Chapter 1 | Try to send data"
+      description: You can send an arbitrary JSON data via HTTP. URL path will be tag name.
+      send: Send
+    chapter2:
+      page_title: "Chapter 2 | in_http and out_stdout"
+      lesson_markdown: |
+        You can see the log when fluentd started.
+
+            2014-06-05 14:43:14 +0900 [info]: adding source type="http"
+            2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout"
+
+        Line 1 enable http plugin that allows to receive HTTP requests.
+
+        Line 2 enable stdout plugin that process the data with matched `debug.*` tag.
+
+        These settings are defined as following fluent.conf:
+
+            <source>
+              type http
+              port %{http_port}
+            </source>
+
+            <match debug.*>
+              type stdout
+            </match>
+    chapter3:
+      page_title: "Chapter 3 | Build your fluentd!"
+      lesson_markdown: |
+        fluentd can receive from [syslog protocol](http://docs.fluentd.org/articles/in_syslog), [file](http://docs.fluentd.org/articles/in_tail), etc.
+
+        Also fluentd can output to [MongoDB](http://docs.fluentd.org/articles/out_mongo), [AWS S3](http://docs.fluentd.org/articles/out_s3), etc.
+
+        ![fluentd](/fluentd.png)
+
+        These input/output are provided as plugin. Install them and write a setting, then restart fluentd, you can get the power!
+
+        [Many plugins](/plugins/recommended) are available. And you can [edit config file from here](%{edit_config_url}).
+    chapter4:
+      page_title: "Chapter 4 | Use case"
+      lesson_markdown: |
+        ### Monitoring Apache 5xx response and email it
+
+        **Required plugins**
+
+        - fluent-plugin-grepcounter
+        - fluent-plugin-mail
+
+        **config file example**
+
+            <source>
+              type tail
+              format apache2
+              path /var/log/apache2/access.log #This is the location of your Apache log
+              tag apache.access
+            </source>
+
+            <match apache.access>
+              type grepcounter
+              count_interval 3         # Time window to grep and count the # of events
+              input_key code           # We look at the (http status) "code" field
+              regexp ^5\d\d$           # This regexp matches 5xx status codes
+              threshold 1              # The # of events to trigger emitting an output
+              add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access
+            </match>
+        
+            <match error_5xx.apache.access>
+              # The event that comes here looks like
+              # {
+              #   "count":1,
+              #   "input_tag":"error_5xx.apache.access",
+              #   "input_tag_last":"access",
+              #   "message":[500]
+              # }
+
+              type mail
+              host smtp.gmail.com       # This is for Gmail and Google Apps. Any SMTP server should work
+              port 587                  # port for smtp.gmail.com
+              user example@gmail.com    # your Gmail here for login
+              password XXXXXX           # Gmail password
+              enable_starttls_auto true # Gmail required this
+
+              from YOUR_SENDER_EMAIL_HERE
+              to   YOUR_RECIPIENT_EMAIL_HERE
+              subject [URGENT] APACHE 5XX ERROR
+              message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP
+              message_out_keys count # The value of 'count' will be substituted into %s above.
+            </match>
+
+        **process flow**
+
+            [log file] -> 
+              (in_tail) ->
+                Capturing file content with tagged as apache.access ->
+              (match apache.access) ->
+                "grepcounter" re-send data with appending prefix ->
+              (match error_5xx.apache.access) ->
+                "mail" send a mail
+    chapter5:
+      page_title: "Chapter 5 | Finish!"
+      lesson_markdown: |
+        Tutorial is over. congratulation!
+
+        Other resources:
+
+        - [Quick start](http://docs.fluentd.org/articles/quickstart)
+        - [Forum](https://groups.google.com/forum/?fromgroups#!forum/fluentd)
+        - [Source code(GitHub)](https://github.com/fluent/fluentd)
+        - [Twitter @fluentd](https://twitter.com/fluentd)
+

data/config/locales/tutorial_ja.yml

@@ -0,0 +1,120 @@
+ja:
+  tutorials:
+    index:
+      page_title: チュートリアル
+      step1: "fluentdをセットアップ"
+      step2: "fluentdを起動"
+      start_tutorial: "チュートリアルを始める"
+    chapter1:
+      page_title: "Chapter 1 | データを渡してみる"
+      description: fluentdに任意のデータをJSONで送ることができます。URLのパスがタグの名前になります。
+      learn_more: |
+        他にもin_tail, in_syslogなどのinputプラグインがあります。
+        <a href="http://docs.fluentd.org/ja/articles/input-plugin-overview">Learn More</a>
+      send: 送信
+    chapter2:
+      page_title: "Chapter 2 | in_httpとout_stdout"
+      lesson_markdown: |
+        fluentdの起動時にこのようなログがあるかと思います。
+
+            2014-06-05 14:43:14 +0900 [info]: adding source type="http"
+            2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout"
+
+        この1行目でhttpが有効化されています。これでHTTPリクエストを受け付けるようになります。
+
+        2行目でstdoutが有効化されています。受け取ったデータのうち、タグが`debug.*`にマッチするものはstdoutへと渡されます。
+
+        この2つはfluent.confでそれぞれ次のように設定されています。
+
+            <source>
+              type http
+              port %{http_port}
+            </source>
+
+            <match debug.*>
+              type stdout
+            </match>
+    chapter3:
+      page_title: "Chapter 3 | fluentdを構築しよう！"
+      lesson_markdown: |
+        fluentdはHTTP以外にも[syslogプロトコル](http://docs.fluentd.org/ja/articles/in_syslog)や[ファイル](http://docs.fluentd.org/ja/articles/in_tail)を入力として受け取ることができます。
+
+        また出力についても、stdout以外に[MongoDB](http://docs.fluentd.org/ja/articles/out_mongo)や[AWS S3](http://docs.fluentd.org/ja/articles/out_s3)などを出力先として指定できます。
+
+        ![fluentd](/fluentd.png)
+
+        これらはプラグインとして提供されています。プラグインをインストールし、設定ファイルに追記してfluentdを再起動すると使用可能となります。
+
+        [数多くのプラグイン](/plugins/recommended)がありますので、用途にあったものを探して使いましょう！ 設定ファイルは[ここから編集できます。](%{edit_config_url})
+    chapter4:
+      page_title: "Chapter 4 | 設定事例"
+      lesson_markdown: |
+        ### 例：Apacheの5xxレスポンスを検知してメールを送る
+
+        **必要なプラグイン**
+
+        - fluent-plugin-grepcounter
+        - fluent-plugin-mail
+
+        **設定ファイル例**
+
+            <source>
+              type tail
+              format apache2
+              path /var/log/apache2/access.log #This is the location of your Apache log
+              tag apache.access
+            </source>
+
+            <match apache.access>
+              type grepcounter
+              count_interval 3         # Time window to grep and count the # of events
+              input_key code           # We look at the (http status) "code" field
+              regexp ^5\d\d$           # This regexp matches 5xx status codes
+              threshold 1              # The # of events to trigger emitting an output
+              add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access
+            </match>
+        
+            <match error_5xx.apache.access>
+              # The event that comes here looks like
+              # {
+              #   "count":1,
+              #   "input_tag":"error_5xx.apache.access",
+              #   "input_tag_last":"access",
+              #   "message":[500]
+              # }
+
+              type mail
+              host smtp.gmail.com       # This is for Gmail and Google Apps. Any SMTP server should work
+              port 587                  # port for smtp.gmail.com
+              user example@gmail.com    # your Gmail here for login
+              password XXXXXX           # Gmail password
+              enable_starttls_auto true # Gmail required this
+
+              from YOUR_SENDER_EMAIL_HERE
+              to   YOUR_RECIPIENT_EMAIL_HERE
+              subject [URGENT] APACHE 5XX ERROR
+              message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP
+              message_out_keys count # The value of 'count' will be substituted into %s above.
+            </match>
+
+        **処理の流れ**
+
+            [log file] -> 
+              (in_tail) ->
+                apache.accessタグでfluentdに取り込む ->
+              (apache.accessにマッチ) ->
+                grepcounterがタグにprefixを追加して再送 ->
+              (error_5xx.apache.accessにマッチ) ->
+                mailがメール送信
+    chapter5:
+      page_title: "Chapter 5 | チュートリアル完了"
+      lesson_markdown: |
+        以上でチュートリアルは終了です。お疲れさまでした！
+
+        関連リソース：
+
+        - [クイックスタートガイド](http://docs.fluentd.org/ja/articles/quickstart)
+        - [メーリングリスト](https://groups.google.com/forum/?fromgroups#!forum/fluentd)
+        - [ソースコード(GitHub)](https://github.com/fluent/fluentd)
+        - [Twitter @fluentd](https://twitter.com/fluentd)
+

data/config/routes.rb

@@ -0,0 +1,93 @@
+Rails.application.routes.draw do
+  root "welcome#home"
+
+  resource :daemon, controller: :fluentd do
+    get "log"
+    get "raw_log"
+    get "errors"
+
+    scope module: :fluentd do
+      resource :agent, only: [] do
+        put "start"
+        put "stop"
+        put "restart"
+        get "log_tail"
+      end
+
+      resource :setting, only: [:show, :edit, :update] do
+        get "source_and_output"
+
+        resource :in_tail, only: [:show], module: :settings, controller: :in_tail do
+          post "after_file_choose"
+          post "after_format"
+          post "confirm"
+          post "finish"
+        end
+
+        resource :in_syslog, only: [:show], module: :settings, controller: :in_syslog do
+          post "finish"
+        end
+
+        resource :out_mongo, only: [:show], module: :settings, controller: :out_mongo do
+          post "finish"
+        end
+
+        resource :out_td, only: [:show], module: :settings, controller: :out_td do
+          post "finish"
+        end
+
+        resource :out_s3, only: [:show], module: :settings, controller: :out_s3 do
+          post "finish"
+        end
+
+        resource :out_forward, only: [:show], module: :settings, controller: :out_forward do
+          post "finish"
+        end
+      end
+    end
+  end
+
+  resource :sessions
+
+  resources :plugins do
+    collection do
+      get :installed
+      get :recommended
+      get :updated
+      patch :install
+      patch :uninstall
+      patch :upgrade
+      patch :bulk_upgrade
+    end
+  end
+
+  resource :user, only: [:show, :edit, :update]
+
+  get "misc" => "misc#show"
+  get "misc/information"
+  post "misc/update_fluentd_ui"
+  get "misc/upgrading_status"
+  get "misc/download_info"
+
+  namespace :polling do
+    get "alerts"
+  end
+
+  namespace :tutorials do
+    get "/" => :index
+    get "chapter1"
+    get "chapter2"
+    get "chapter3"
+    get "chapter4"
+    get "chapter5"
+    get "log_tail"
+    post "request_fluentd"
+  end
+
+  namespace :api do
+    get "tree"
+    get "file_preview"
+    post "regexp_preview"
+    post "grok_to_regexp"
+  end
+end

data/config/secrets.yml

@@ -0,0 +1,22 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rake secret` to generate a secure secret key.
+
+# Make sure the secrets in this file are kept private
+# if you're sharing your code publicly.
+
+development:
+  secret_key_base: 79a259b8c70133d47d6d667fb932021e6b2e1a6be3cbd378f3a1105eaabbfbcc9b7804473c94f60c0f599f8a885fd831425e79b3bf35b3cf2f06ad4979de6048
+
+test:
+  secret_key_base: fe23b4a81c05fab9380d5a5214f33055b6cf2cc75dcc5f3d0cd5128f2893b31b8dd47b707829ef0dcefe3146e3dc10e57552b717ab47c1c56022a20fc806ba67
+
+# Do not keep production secrets in the repository,
+# instead read values from the environment.
+production:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/db/seeds.rb

@@ -0,0 +1,11 @@
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+#
+# Examples:
+#
+#   cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }])
+#   Mayor.create(name: 'Emanuel', city: cities.first)
+
+unless User.exists?
+  User.create!(name: "admin", password: Settings.default_password)
+end

data/fluentd-ui.gemspec

@@ -0,0 +1,46 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluentd-ui/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluentd-ui"
+  spec.version       = FluentdUI::VERSION
+  spec.authors       = ["Masahiro Nakagawa", "uu59"]
+  spec.email         = ["repeatedly@gmail.com", "k@uu59.org"]
+  spec.summary       = %q{Web UI for Fluentd}
+  spec.description   = %q{Web UI for Fluentd}
+  spec.homepage      = "https://github.com/fluent/fluentd-ui"
+  spec.license       = "MIT"
+
+  unless (ARGV & %w(rake build)).length.zero?
+    # NOTE: `fluentd-ui start` will run `bundle exec ...`. so this gemspec file evaluated by bundler then exec `git ls-files`
+    #       but `git ls-files` would be warn if $PWD is not git dir, and unnecessary this step for to do it.
+    #       And spec.files required for building a .gem only.
+    #       Thus git ls-files only invoked with `rake build` command
+    spec.files         = `git ls-files`.split($/)
+  end
+  spec.executables   = ["fluentd-ui"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", "~> 0.10.51"
+  spec.add_dependency 'rails', '4.1.4'
+  spec.add_dependency 'sucker_punch', "~> 1.0.5"
+  spec.add_dependency 'i18n_generators', '1.2.1'
+  spec.add_dependency 'bcrypt', '~> 3.1.5'
+  spec.add_dependency 'addressable'
+  spec.add_dependency "font-awesome-rails"
+  spec.add_dependency 'sass-rails', '~> 4.0.3'
+  spec.add_dependency "haml-rails", "~> 0.5.3"
+  spec.add_dependency 'jquery-rails', "~> 3.1.0"
+  spec.add_dependency 'jbuilder', '~> 2.0'
+  spec.add_dependency "bundler"
+  spec.add_dependency "httpclient"
+  spec.add_dependency "settingslogic"
+  spec.add_dependency "puma"
+  spec.add_dependency "thor"
+  spec.add_dependency "kramdown", "> 1.0.0"
+  spec.add_dependency "kramdown-haml"
+  spec.add_dependency "rubyzip", "~> 1.1" # API changed as Zip::ZipFile -> Zip::File since v1.0.0
+end

data/lib/file_reverse_reader.rb

@@ -0,0 +1,56 @@
+class FileReverseReader
+  attr_reader :io, :step
+
+  def initialize(io, step = 1024 * 1024 * 1)
+    @io = io
+    @step = step
+  end
+
+  def each_line(&block)
+    io.seek(0, IO::SEEK_END)
+    buf = ""
+    loop do
+      if reach_start_of_file?
+        last_pos = io.pos
+        io.seek(0, IO::SEEK_SET)
+        buf.insert(0, io.read(last_pos))
+        split_each_line(buf, &block)
+        break
+      end
+
+      io.seek(-1 * step, IO::SEEK_CUR)
+      buf.insert(0, io.read(step))
+      io.seek(-1 * step, IO::SEEK_CUR)
+      next if buf[$/].nil?
+      gap = buf.index($/)
+      buf.gsub!(/\A.*?\n/, "")
+      split_each_line(buf, &block)
+      buf = ""
+      io.seek(gap, IO::SEEK_CUR)
+    end
+  end
+
+  def tail(limit = 10)
+    enum_for(:each_line).first(limit).reverse
+  end
+
+  def binary_file?
+    sample = io.read(1024) || ""
+    sample2 = sample.force_encoding('ascii-8bit').encode('us-ascii', :undef => :replace, :invalid => :replace, :replace => "")
+    sample != sample2 # maybe binary file
+  ensure
+    io.rewind
+  end
+
+  private
+
+  def split_each_line(buf, &block)
+    buf.split($/).reverse.each do |line|
+      block.call(line)
+    end
+  end
+
+  def reach_start_of_file?
+    step >= io.pos
+  end
+end