magellan-log-funnel 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f3a179d70b7b8be45d9247c5e3216336d17f7d34
4
+ data.tar.gz: be0d89ab8429793a2d38d99b6f95b375573c6868
5
+ SHA512:
6
+ metadata.gz: 154b0f07392802f3a2cacd02ec2e623ebfc3b3f19d0dfaaad07b182c915ded0eb89da43f8900023ce17e89d3519a9edd451541ad7b1a6fe3bd43eced8b829865
7
+ data.tar.gz: 398bd67a2cce0efa416c9c9dc6364f22adb4d4f8d90b93d760d727203fca24ad15749fbe19c8dd8aa9fcefbb4e0db028037414f263ec61cb314e4c3fb4207e46
data/.gitignore ADDED
@@ -0,0 +1,2 @@
1
+ /.bundle/
2
+ /vendor/bundle
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source "https://rubygems.org/"
2
+
3
+ group :development, :test do
4
+ gem "rspec", "~> 3.0.0"
5
+ end
data/Gemfile.lock ADDED
@@ -0,0 +1,22 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ diff-lcs (1.2.5)
5
+ rspec (3.0.0)
6
+ rspec-core (~> 3.0.0)
7
+ rspec-expectations (~> 3.0.0)
8
+ rspec-mocks (~> 3.0.0)
9
+ rspec-core (3.0.4)
10
+ rspec-support (~> 3.0.0)
11
+ rspec-expectations (3.0.4)
12
+ diff-lcs (>= 1.2.0, < 2.0)
13
+ rspec-support (~> 3.0.0)
14
+ rspec-mocks (3.0.4)
15
+ rspec-support (~> 3.0.0)
16
+ rspec-support (3.0.4)
17
+
18
+ PLATFORMS
19
+ ruby
20
+
21
+ DEPENDENCIES
22
+ rspec (~> 3.0.0)
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 akima
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,27 @@
1
+ # Magellan::Cli
2
+
3
+ magellan-log-funnel:
4
+
5
+ Collect stdout/stderr from sub-process and pass it to fluentd.
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'magellan-log-funnel'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install magellan-log-funnel
22
+
23
+ ## Usage
24
+
25
+ ```
26
+ bundle exec magellan-log-funnel --project PROJECT1 --stage STAGE1 --worker-version 1.0 --container CONTAINER1 [command ...]
27
+ ```
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ begin
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+ rescue LoadError
8
+ end
9
+
10
+ task :default => :spec
11
+
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'magellan/log_funnel'
4
+
5
+ begin
6
+ ret = Magellan::LogFunnel.start(ARGV)
7
+ exit(ret)
8
+ rescue Magellan::LogFunnel::Error
9
+ $stderr.puts("\e[31m#{$!.message}\e[0m")
10
+ exit(1)
11
+ rescue
12
+ $stderr.puts("\e[31m[#{$!.class}] #{$!.message}\e[0m\n " << $!.backtrace.join("\n "))
13
+ exit(1)
14
+ end
@@ -0,0 +1,35 @@
1
+ ## built-in TCP input
2
+ ## $ echo <json> | fluent-cat <tag>
3
+ <source>
4
+ type forward
5
+ </source>
6
+
7
+ # HTTP input
8
+ # http://localhost:8888/<tag>?json=<json>
9
+ <source>
10
+ type http
11
+ port 8888
12
+ </source>
13
+
14
+ <match worker_logs.*.*>
15
+ type forest
16
+ subtype copy
17
+ <template>
18
+ <store>
19
+ type bigquery
20
+ method insert
21
+ auth_method compute_engine
22
+ #auth_method private_key
23
+ #email xxxxxxxx@developer.gserviceaccount.com
24
+ #private_key_path /path/to/private/key.p12
25
+ project myproject
26
+ dataset worker_logs
27
+ time_format %s
28
+ time_field time
29
+ table ${tag_parts[1]}_${tag_parts[2]}_%Y_%m
30
+ auto_create_table true
31
+ field_integer time,bytes
32
+ field_string project,stage,version,container,message
33
+ </store>
34
+ </template>
35
+ </match>
@@ -0,0 +1,111 @@
1
+ # coding: utf-8
2
+
3
+ require "json"
4
+ require "optparse"
5
+ require "net/http"
6
+
7
+ module Magellan
8
+ class LogFunnel
9
+
10
+ class Error < StandardError
11
+ end
12
+
13
+ def self.start(argv)
14
+ self.new(argv).run
15
+ end
16
+
17
+ def initialize(argv)
18
+ @argv = argv
19
+ end
20
+
21
+ def parse_options(argv)
22
+ @conf = {}
23
+ OptionParser.new do |o|
24
+ o.on("--fluentd URI") do |uri|
25
+ @conf[:fluentd] = URI(uri)
26
+ end
27
+ o.on("--project PROJECT") do |proj|
28
+ @conf[:project] = proj
29
+ end
30
+ o.on("--stage STAGE") do |stage|
31
+ @conf[:stage] = stage
32
+ end
33
+ o.on("--worker-version VERSION") do |ver|
34
+ @conf[:version] = ver
35
+ end
36
+ o.on("--container NAME") do |cname|
37
+ @conf[:container] = cname
38
+ end
39
+ o.on("-d", "--[no-]daemon") do |daemon|
40
+ @conf[:daemon] = daemon
41
+ end
42
+ o.order!(argv)
43
+ end
44
+ end
45
+
46
+ def now
47
+ Time.now
48
+ end
49
+
50
+ def post_msg(msg)
51
+ Net::HTTP.post_form(@conf[:fluentd], json: JSON.generate({
52
+ time: self.now.to_i,
53
+ project: @conf[:project],
54
+ stage: @conf[:stage],
55
+ version: @conf[:version],
56
+ container: @conf[:container],
57
+ message: msg
58
+ }))
59
+ end
60
+
61
+ def run
62
+ parse_options(@argv)
63
+ @conf[:fluentd] ||= URI("http://localhost:8888")
64
+ @conf[:fluentd].path = "/worker_logs.#{@conf[:project]}.#{@conf[:stage]}"
65
+
66
+ if @conf[:daemon]
67
+ Process.daemon
68
+ end
69
+
70
+ begin
71
+ IO.pipe do |out|
72
+ IO.pipe do |err|
73
+ @pid = Process.spawn(*@argv, out: out[1], err: err[1])
74
+ out[1].close
75
+ err[1].close
76
+
77
+ th_out = Thread.start do
78
+ begin
79
+ while l = out[0].gets
80
+ post_msg(l.chomp)
81
+ end
82
+ rescue IOError
83
+ end
84
+ end
85
+ th_err = Thread.start do
86
+ begin
87
+ while l = err[0].gets
88
+ post_msg(l.chomp)
89
+ end
90
+ rescue IOError
91
+ end
92
+ end
93
+ th_out.join
94
+ th_err.join
95
+ Process.waitpid(@pid)
96
+ @child_status = $?
97
+ return (@child_status.exitstatus || 127)
98
+ end
99
+ end
100
+ end
101
+ ensure
102
+ unless @child_status
103
+ # interrupted by signal or so.
104
+ # stop child process
105
+ if @pid
106
+ Process.kill(:TERM, @pid)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,5 @@
1
+ module Magellan
2
+ class LogFunnel
3
+ VERSION = "1.0.0"
4
+ end
5
+ end
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+
3
+ lib = File.expand_path('../lib', __FILE__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require 'magellan/log_funnel/version'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = "magellan-log-funnel"
9
+ spec.version = Magellan::LogFunnel::VERSION
10
+ spec.authors = ["t-chikanaga"]
11
+ spec.email = ["t-chikanaga@groovenauts.jp"]
12
+ spec.summary = %q{log collector for magellanic cloud service.}
13
+ spec.description = %q{log collector for magellanic cloud service.}
14
+ spec.homepage = ""
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0")
18
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
19
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.6"
23
+ spec.add_development_dependency "rake", "~> 10.0"
24
+ spec.add_development_dependency "rspec"
25
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+
3
+ describe Magellan::LogFunnel do
4
+ before do
5
+ @command = [ "/bin/echo", "foo" ]
6
+ @argv = [
7
+ "--fluentd", "http://fluentd.example:8888",
8
+ "--project", "PROJECT1",
9
+ "--stage", "STAGE1",
10
+ "--worker-version", "1.0",
11
+ "--container", "CONTAINER1",
12
+ ] + @command
13
+ @funnel = Magellan::LogFunnel.new(@argv)
14
+ # stub Net::HTTP.post_form
15
+ @now = Time.now
16
+ allow(@funnel).to receive(:now).and_return(@now)
17
+ expect(Net::HTTP).to receive(:post_form).with(URI("http://fluentd.example:8888/worker_logs.PROJECT1.STAGE1"),
18
+ json: JSON.generate({
19
+ time: @now.to_i,
20
+ project: "PROJECT1",
21
+ stage: "STAGE1",
22
+ version: "1.0",
23
+ container: "CONTAINER1",
24
+ message: "foo"
25
+ }))
26
+ end
27
+ it 'can handle stdout' do
28
+ expect(@funnel.run).to eql(0)
29
+ end
30
+ end
@@ -0,0 +1,2 @@
1
+ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
+ require 'magellan/log_funnel'
metadata ADDED
@@ -0,0 +1,102 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: magellan-log-funnel
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - t-chikanaga
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-12-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: log collector for magellanic cloud service.
56
+ email:
57
+ - t-chikanaga@groovenauts.jp
58
+ executables:
59
+ - magellan-log-funnel
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - Gemfile
65
+ - Gemfile.lock
66
+ - LICENSE.txt
67
+ - README.md
68
+ - Rakefile
69
+ - bin/magellan-log-funnel
70
+ - config/fluent.conf.example
71
+ - lib/magellan/log_funnel.rb
72
+ - lib/magellan/log_funnel/version.rb
73
+ - magellan-log-funnel.gemspec
74
+ - spec/magellan/log_funnel_spec.rb
75
+ - spec/spec_helper.rb
76
+ homepage: ''
77
+ licenses:
78
+ - MIT
79
+ metadata: {}
80
+ post_install_message:
81
+ rdoc_options: []
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - ">="
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project:
96
+ rubygems_version: 2.2.2
97
+ signing_key:
98
+ specification_version: 4
99
+ summary: log collector for magellanic cloud service.
100
+ test_files:
101
+ - spec/magellan/log_funnel_spec.rb
102
+ - spec/spec_helper.rb