fluent-plugin-macos-log 0.0.1.beta.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +143 -0
- data/.gitignore +6 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +52 -0
- data/Makefile +17 -0
- data/README.md +176 -0
- data/Rakefile +9 -0
- data/fluent-plugin-macos-log.gemspec +32 -0
- data/fluent-plugin-macos-log.gemspec.lock +10 -0
- data/lib/fluent/plugin/in_exec/iterative_process.rb +71 -0
- data/lib/fluent/plugin/in_macoslog.rb +199 -0
- data/version.txt +1 -0
- metadata +145 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 57016186f913c0af39bac56db61c9c053bed21a1f91230bbd47e24c9636264e2
|
4
|
+
data.tar.gz: 41b000bade26ccbdcca84d65e7c624a74a144d5845ad3814935ef4151c14857c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 04d5a09d8d5ea8181fe8d697090af30c304c1a80181d28389eba85c0293abe41e0a84d47f2f8ab62d0f274c39adb08ecbe7b4c357dac92cfe112cc50b5f60deb
|
7
|
+
data.tar.gz: 8080c62ca068e40444f8aa3c3aa265b21e428fb47f9c18bf4b464955d87bf32b754caac11ed808e6f1d9db8d1232534d73cb4514fb935f156ca4d188a45f204e
|
@@ -0,0 +1,143 @@
|
|
1
|
+
# Ruby CircleCI 2.0 configuration file
|
2
|
+
#
|
3
|
+
# Check https://circleci.com/docs/2.0/language-ruby/ for more details
|
4
|
+
#
|
5
|
+
version: 2.1
|
6
|
+
|
7
|
+
executors:
|
8
|
+
ruby:
|
9
|
+
docker:
|
10
|
+
- image: circleci/ruby:2.6.6-node
|
11
|
+
environment:
|
12
|
+
BUNDLER_VERSION: 2.1.4
|
13
|
+
REPO_NAME: "fluent-plugin-macos-log"
|
14
|
+
BUNDLE_PATH: "vendor/bundle"
|
15
|
+
VERSION_FILE: "version.txt"
|
16
|
+
|
17
|
+
references:
|
18
|
+
depends_cache_key: &depends_cache_key
|
19
|
+
v1-dependencies-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
|
20
|
+
fallback_repo_cache_key: &fallback_repo_cache_key
|
21
|
+
v1-dependencies-
|
22
|
+
restore_deps: &restore_deps
|
23
|
+
restore_cache:
|
24
|
+
keys:
|
25
|
+
- *depends_cache_key
|
26
|
+
- *fallback_repo_cache_key
|
27
|
+
save_deps: &save_deps
|
28
|
+
save_cache:
|
29
|
+
paths:
|
30
|
+
- ./vendor/bundle
|
31
|
+
key: *depends_cache_key
|
32
|
+
install_bundler: &install_bundler
|
33
|
+
run:
|
34
|
+
name: Install bundler
|
35
|
+
command: gem install bundler:$BUNDLER_VERSION
|
36
|
+
|
37
|
+
jobs:
|
38
|
+
build:
|
39
|
+
executor: ruby
|
40
|
+
working_directory: ~/repo
|
41
|
+
steps:
|
42
|
+
- checkout
|
43
|
+
|
44
|
+
# Download and cache dependencies
|
45
|
+
- *restore_deps
|
46
|
+
- *install_bundler
|
47
|
+
- run:
|
48
|
+
name: Install dependencies
|
49
|
+
command: bundle install --jobs=4 --retry=3
|
50
|
+
|
51
|
+
- *save_deps
|
52
|
+
|
53
|
+
# run tests!
|
54
|
+
- run:
|
55
|
+
name: run tests
|
56
|
+
command: |
|
57
|
+
mkdir /tmp/test-results
|
58
|
+
bundle exec rake test
|
59
|
+
|
60
|
+
# collect reports
|
61
|
+
- store_test_results:
|
62
|
+
path: /tmp/test-results
|
63
|
+
- store_artifacts:
|
64
|
+
path: /tmp/test-results
|
65
|
+
destination: test-results
|
66
|
+
|
67
|
+
release:
|
68
|
+
executor: ruby
|
69
|
+
working_directory: ~/repo
|
70
|
+
steps:
|
71
|
+
- add_ssh_keys:
|
72
|
+
fingerprints:
|
73
|
+
- "a5:d3:86:70:21:30:78:71:87:a7:45:34:0a:47:f6:5c"
|
74
|
+
- checkout
|
75
|
+
- *restore_deps
|
76
|
+
- *install_bundler
|
77
|
+
- run:
|
78
|
+
name: Ruby version
|
79
|
+
command: |
|
80
|
+
ruby --version
|
81
|
+
gem env version
|
82
|
+
bundle --version
|
83
|
+
- run:
|
84
|
+
name: Configure git for release
|
85
|
+
command: |
|
86
|
+
git config user.name "librato-ci"
|
87
|
+
git config user.email "tools+librato-ci-githublibrato.com"
|
88
|
+
# - run:
|
89
|
+
# name: Prepare release
|
90
|
+
# command: |
|
91
|
+
# sed -i'' 's/\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1.\2.\3/g' $VERSION_FILE
|
92
|
+
# GITTAG=$(cat $VERSION_FILE)
|
93
|
+
# git add $VERSION_FILE
|
94
|
+
# git commit -m "[ci skip] prepare release $GITTAG"
|
95
|
+
# git push --set-upstream origin $CIRCLE_BRANCH
|
96
|
+
# git tag $GITTAG
|
97
|
+
# git push origin $GITTAG
|
98
|
+
- run:
|
99
|
+
name: Generate gem credentials
|
100
|
+
command: |
|
101
|
+
mkdir -p ~/.gem
|
102
|
+
echo -e "---\r\n:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
|
103
|
+
chmod 0600 ~/.gem/credentials
|
104
|
+
- run:
|
105
|
+
name: Perform build
|
106
|
+
command: |
|
107
|
+
rm -rf ${REPO_NAME}-*.gem
|
108
|
+
bundle exec gem build ${REPO_NAME}.gemspec
|
109
|
+
- run:
|
110
|
+
name: Perform publish
|
111
|
+
command: |
|
112
|
+
cat ~/.gem/credentials
|
113
|
+
bundle exec gem push ${REPO_NAME}-*.gem
|
114
|
+
- run:
|
115
|
+
name: Prepare next release version
|
116
|
+
command: |
|
117
|
+
VERSION=$(cat $VERSION_FILE | xargs)
|
118
|
+
if [[ $VERSION =~ ^(.*)(([0-9]+)\.([0-9]+)\.([0-9]+))(-.*)? ]]; then
|
119
|
+
SEMVERMAJOR=${BASH_REMATCH[3]}
|
120
|
+
SEMVERMINOR=${BASH_REMATCH[4]}
|
121
|
+
SEMVERPATCH=$(expr ${BASH_REMATCH[5]} + 1)
|
122
|
+
NEWVERSION="${SEMVERMAJOR}.${SEMVERMINOR}.${SEMVERPATCH}.beta.1"
|
123
|
+
echo "$NEWVERSION" > $VERSION_FILE
|
124
|
+
git add $VERSION_FILE Gemfile.lock
|
125
|
+
git commit -m "[ci skip] prepare for next development iteration"
|
126
|
+
git push --set-upstream origin $CIRCLE_BRANCH
|
127
|
+
fi
|
128
|
+
- run:
|
129
|
+
name: Clean credentials
|
130
|
+
command: shred -u ~/.gem/credentials
|
131
|
+
when: always
|
132
|
+
|
133
|
+
workflows:
|
134
|
+
build_test_release:
|
135
|
+
jobs:
|
136
|
+
- build
|
137
|
+
- release:
|
138
|
+
filters:
|
139
|
+
branches:
|
140
|
+
only:
|
141
|
+
- develop
|
142
|
+
requires:
|
143
|
+
- build
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-macos-log (0.0.1.beta.1)
|
5
|
+
fluentd (>= 1.2, < 2)
|
6
|
+
yajl-ruby (~> 1.3)
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: https://rubygems.org/
|
10
|
+
specs:
|
11
|
+
concurrent-ruby (1.1.8)
|
12
|
+
cool.io (1.7.1)
|
13
|
+
fluentd (1.12.1)
|
14
|
+
bundler
|
15
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
16
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
17
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
18
|
+
serverengine (>= 2.2.2, < 3.0.0)
|
19
|
+
sigdump (~> 0.2.2)
|
20
|
+
strptime (>= 0.2.2, < 1.0.0)
|
21
|
+
tzinfo (>= 1.0, < 3.0)
|
22
|
+
tzinfo-data (~> 1.0)
|
23
|
+
yajl-ruby (~> 1.0)
|
24
|
+
http_parser.rb (0.6.0)
|
25
|
+
minitest (5.14.2)
|
26
|
+
msgpack (1.4.2)
|
27
|
+
power_assert (1.2.0)
|
28
|
+
rake (10.5.0)
|
29
|
+
serverengine (2.2.3)
|
30
|
+
sigdump (~> 0.2.2)
|
31
|
+
sigdump (0.2.4)
|
32
|
+
strptime (0.2.5)
|
33
|
+
test-unit (3.3.7)
|
34
|
+
power_assert
|
35
|
+
tzinfo (2.0.4)
|
36
|
+
concurrent-ruby (~> 1.0)
|
37
|
+
tzinfo-data (1.2021.1)
|
38
|
+
tzinfo (>= 1.0.0)
|
39
|
+
yajl-ruby (1.4.1)
|
40
|
+
|
41
|
+
PLATFORMS
|
42
|
+
ruby
|
43
|
+
|
44
|
+
DEPENDENCIES
|
45
|
+
bundler (~> 2.0)
|
46
|
+
fluent-plugin-macos-log!
|
47
|
+
minitest (~> 5.0)
|
48
|
+
rake (~> 10.0)
|
49
|
+
test-unit (~> 3.2)
|
50
|
+
|
51
|
+
BUNDLED WITH
|
52
|
+
2.1.4
|
data/Makefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
REPO_NAME=fluent-plugin-macos-log
|
2
|
+
|
3
|
+
bundle:
|
4
|
+
bundle install
|
5
|
+
|
6
|
+
test: bundle
|
7
|
+
bundle exec rake test
|
8
|
+
|
9
|
+
package: bundle
|
10
|
+
rm -rf ${REPO_NAME}-*.gem
|
11
|
+
bundle exec gem build ${REPO_NAME}.gemspec
|
12
|
+
|
13
|
+
deploy: package
|
14
|
+
gem install ${REPO_NAME}
|
15
|
+
|
16
|
+
release: package
|
17
|
+
bundle exec gem push ${REPO_NAME}-*.gem
|
data/README.md
ADDED
@@ -0,0 +1,176 @@
|
|
1
|
+
# Fluent::Plugin::MacOsLogInput
|
2
|
+
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-macos-log.svg)](https://badge.fury.io/rb/fluent-plugin-macos-log) [![CircleCI](https://circleci.com/gh/loggly/fluent-plugin-macos-log/tree/master.svg?style=shield)](https://circleci.com/gh/loggly/fluent-plugin-macos-log/tree/master)
|
4
|
+
|
5
|
+
## Description
|
6
|
+
|
7
|
+
This repository contains the Fluentd MacOs unified logs input Plugin.
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Install this gem when setting up fluentd:
|
12
|
+
```ruby
|
13
|
+
gem install fluent-plugin-macos-log
|
14
|
+
```
|
15
|
+
|
16
|
+
## Usage
|
17
|
+
|
18
|
+
### Setup
|
19
|
+
|
20
|
+
This is a process execution input plugin for Fluentd that periodically executes external `log show` command and parses log events into Fluentd's core system.
|
21
|
+
Each execution alters `start` and `end` time input parameters of log utility to slowly iterates over log data. The iteration
|
22
|
+
interval can be configured by user, but cannot be lower than 1s.
|
23
|
+
|
24
|
+
There are multiple configurations one can use:
|
25
|
+
|
26
|
+
#### Simplified Output
|
27
|
+
Uses human-readable output of the command. The process output is parsed using `regexp` parser
|
28
|
+
and logic, which combines multiple lines together. The parameter `log_line_start` defines regular expresion, which matches the
|
29
|
+
beginning of line. Anything in between will be merged into single log entry. Although the parser is `regexp`, user can select any other supported parser.
|
30
|
+
To configure this in fluentd:
|
31
|
+
```xml
|
32
|
+
<source>
|
33
|
+
@type macoslog
|
34
|
+
tag macos
|
35
|
+
pos_file /path/to/position/file
|
36
|
+
run_interval 10s
|
37
|
+
</source>
|
38
|
+
```
|
39
|
+
|
40
|
+
#### Detail Output
|
41
|
+
Use when more detailed output is required. It uses `ndjson` style of `log` command, which is then parsed by json parser.
|
42
|
+
To configure this in fluentd:
|
43
|
+
```xml
|
44
|
+
<source>
|
45
|
+
@type macoslog
|
46
|
+
style ndjson
|
47
|
+
tag macos
|
48
|
+
pos_file last-starttime.log
|
49
|
+
run_interval 10s
|
50
|
+
<parse>
|
51
|
+
@type json
|
52
|
+
time_type string
|
53
|
+
time_key timestamp
|
54
|
+
time_format %Y-%m-%d %H:%M:%S.%L%z
|
55
|
+
</parse>
|
56
|
+
</source>
|
57
|
+
```
|
58
|
+
|
59
|
+
### Advanced Configuration
|
60
|
+
This plugin inherits Fluentd's standard input parameters.
|
61
|
+
|
62
|
+
The command used by default is `log show --style default --start @%s --end @%s` in order to combine multiple lines and iterate over
|
63
|
+
each period of time. Notice the `start` and `end` parameters use `@`, which is notation for unix timestamp format, used by plugin.
|
64
|
+
|
65
|
+
Optionally the plugin uses position file, where it records last processed timestamp. Whenever the `fluentd` process
|
66
|
+
restarts the plugin picks up from the last position. When no position files is used the plugin starts from current time
|
67
|
+
and keeps last position only in memory.
|
68
|
+
|
69
|
+
Optionally one can configure any `predicate` to filter required logs.
|
70
|
+
|
71
|
+
* `command` - external command to be executed for each interval. The command's first parameter noted ruby's `%s` as start
|
72
|
+
unix timestamp and the second `%s` for end timestamp. Default: `log show --style default --start @%s --end @%s`
|
73
|
+
* `predicate` - log filter predicate as per Apple's documentation. Default: `nil`
|
74
|
+
* `levels` - Controls what logging levels will be shown. Supported by `log` command:
|
75
|
+
* [no-]backtrace Control whether backtraces are shown
|
76
|
+
* [no-]debug Control whether "Debug" events are shown
|
77
|
+
* [no-]info Control whether "Info" events are shown
|
78
|
+
* [no-]loss Control whether message loss events are shown
|
79
|
+
* [no-]signpost Control whether signposts are shown
|
80
|
+
* `style` - Controls style of logging tool output.
|
81
|
+
* `ndjson` - single lined json format output. When used, the json parser must be configured.
|
82
|
+
* `connect_mode` - Control target IO:
|
83
|
+
* `read`: Read logs from stdio
|
84
|
+
* `read_with_stderr`: Read logs from stdio and stderr (mainly for debug).
|
85
|
+
* `parser` section - Refer these for more details about parse section. Default `regexp`
|
86
|
+
* `tag` - The tag of the output events.
|
87
|
+
* `run_interval` - The interval time between periodic program runs.
|
88
|
+
* `max_age` - The time base max age of logs to process. Default `3d`
|
89
|
+
* `pos_file` - Fluentd will record the position it last read from external command.
|
90
|
+
Don't share pos_file between in_macoslog configurations. It causes unexpected behavior e.g. corrupt pos_file content.
|
91
|
+
* `log_line_start` - Regexp of start of the log to combine multiline logs. Default: `\d+-\d+-\d+\s+\d+:\d+:\d+[^ ]+`
|
92
|
+
* `log_header_lines` - Number of header lines to skip when parsing. When `ndjson` style used the parameter refers
|
93
|
+
to number of footer lines to be skipped. Default: `1`
|
94
|
+
|
95
|
+
One can configure own parser:
|
96
|
+
```xml
|
97
|
+
<source>
|
98
|
+
@type macoslog
|
99
|
+
tag macos
|
100
|
+
pos_file /path/to/position/file
|
101
|
+
run_interval 10s
|
102
|
+
<parse>
|
103
|
+
@type tsv
|
104
|
+
keys avg1,avg5,avg15
|
105
|
+
delimiter " "
|
106
|
+
</parse>
|
107
|
+
</source>
|
108
|
+
```
|
109
|
+
|
110
|
+
### Example
|
111
|
+
Example configuration for sending logs over to Loggly. The input plugin collects unified logs with filter `process == "sharingd"`
|
112
|
+
every `10s` while recording position in file `/path/to/position/file`.
|
113
|
+
|
114
|
+
It uses output [fluent-plugin-loggly](https://github.com/patant/fluent-plugin-loggly) configured in buffer mode.
|
115
|
+
|
116
|
+
```xml
|
117
|
+
<source>
|
118
|
+
@type macoslog
|
119
|
+
predicate process == "sharingd"
|
120
|
+
tag macos
|
121
|
+
pos_file /path/to/position/file
|
122
|
+
run_interval 10s
|
123
|
+
</source>
|
124
|
+
|
125
|
+
<match macos>
|
126
|
+
type loggly_buffered
|
127
|
+
loggly_url https://logs-01.loggly.com/bulk/xxx-xxxx-xxxx-xxxxx-xxxxxxxxxx
|
128
|
+
output_include_time true
|
129
|
+
time_precision_digits 3
|
130
|
+
buffer_type file
|
131
|
+
buffer_path /path/to/buffer/file
|
132
|
+
flush_interval 10s
|
133
|
+
</match>
|
134
|
+
```
|
135
|
+
|
136
|
+
## Development
|
137
|
+
|
138
|
+
This plugin is targeting Ruby 2.6 and Fluentd v1.0, although it should work with older versions of both.
|
139
|
+
|
140
|
+
We have a [Makefile](Makefile) to wrap common functions and make life easier.
|
141
|
+
|
142
|
+
### Prepare development
|
143
|
+
To install fluentd on MacOs use following ruby environment.
|
144
|
+
```shell script
|
145
|
+
brew install rbenv ruby-build
|
146
|
+
echo 'if which rbenv > /dev/null; then eval "$(rbenv init -)"; fi' >> ~/.zshrc
|
147
|
+
source ~/.zshrc
|
148
|
+
rbenv install 2.6.3
|
149
|
+
rbenv global 2.6.3
|
150
|
+
|
151
|
+
gem install fluentd --no-doc
|
152
|
+
```
|
153
|
+
|
154
|
+
Install latest bundler
|
155
|
+
```shell script
|
156
|
+
gem install bundler
|
157
|
+
```
|
158
|
+
|
159
|
+
### Install Dependencies
|
160
|
+
`make bundle`
|
161
|
+
|
162
|
+
### Test
|
163
|
+
`make test`
|
164
|
+
|
165
|
+
### Release in [RubyGems](https://rubygems.org/gems/fluent-plugin-macos-log)
|
166
|
+
To release a new version, update the version number in the [GemSpec](fluent-plugin-macos-log.gemspec) and then, run:
|
167
|
+
|
168
|
+
`make release`
|
169
|
+
|
170
|
+
## Contributing
|
171
|
+
|
172
|
+
Bug reports and pull requests are welcome on GitHub at: https://github.com/loggly/fluent-plugin-macos-log
|
173
|
+
|
174
|
+
# Questions/Comments?
|
175
|
+
|
176
|
+
Please [open an issue](https://github.com/loggly/fluent-plugin-macos-log/issues/new), we'd love to hear from you.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
version = File.read('version.txt').strip
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = "fluent-plugin-macos-log"
|
9
|
+
spec.version = version
|
10
|
+
spec.authors = ["Petr Langr"]
|
11
|
+
spec.email = ["petr.langr@solarwinds.com"]
|
12
|
+
|
13
|
+
spec.summary = %q{Fluentd input plugin for MacOS unified log}
|
14
|
+
spec.description = %q{Fluentd input from MacOS unified log}
|
15
|
+
spec.homepage = "https://github.com/loggly/fluent-plugin-macos-log"
|
16
|
+
spec.license = "MIT"
|
17
|
+
|
18
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
19
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
20
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
21
|
+
spec.bindir = "exe"
|
22
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_dependency "fluentd", '>= 1.2', '< 2'
|
26
|
+
spec.add_dependency "yajl-ruby", '~> 1.3'
|
27
|
+
|
28
|
+
spec.add_development_dependency "bundler", "~> 2.0"
|
29
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
30
|
+
spec.add_development_dependency "minitest", "~> 5.0"
|
31
|
+
spec.add_development_dependency "test-unit", "~> 3.2"
|
32
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
|
2
|
+
require 'fluent/plugin_helper/child_process'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module PluginHelper
|
6
|
+
module ChildProcess
|
7
|
+
include Fluent::PluginHelper::Thread
|
8
|
+
include Fluent::PluginHelper::Timer
|
9
|
+
|
10
|
+
def timer_process_execute(
|
11
|
+
title, command, start_timestamp, interval, time_callback,
|
12
|
+
arguments: nil, subprocess_name: nil, immediate: false, parallel: false,
|
13
|
+
mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,
|
14
|
+
internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,
|
15
|
+
wait_timeout: nil, on_exit_callback: nil, delay_seconds: 0,
|
16
|
+
&block
|
17
|
+
)
|
18
|
+
raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
|
19
|
+
raise ArgumentError, "BUG: arguments required if subprocess name is replaced" if subprocess_name && !arguments
|
20
|
+
|
21
|
+
mode ||= []
|
22
|
+
mode = [] unless block
|
23
|
+
raise ArgumentError, "BUG: invalid mode specification" unless mode.all?{|m| MODE_PARAMS.include?(m) }
|
24
|
+
raise ArgumentError, "BUG: read_with_stderr is exclusive with :read and :stderr" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))
|
25
|
+
raise ArgumentError, "BUG: invalid stderr handling specification" unless STDERR_OPTIONS.include?(stderr)
|
26
|
+
|
27
|
+
raise ArgumentError, "BUG: number of block arguments are different from size of mode" if block && block.arity != mode.size
|
28
|
+
|
29
|
+
running = false
|
30
|
+
callback = ->(*args) {
|
31
|
+
running = true
|
32
|
+
begin
|
33
|
+
block && block.call(*args)
|
34
|
+
ensure
|
35
|
+
running = false
|
36
|
+
end
|
37
|
+
}
|
38
|
+
|
39
|
+
execute_child_process = ->(cmd) {
|
40
|
+
child_process_execute_once(
|
41
|
+
title, cmd, arguments,
|
42
|
+
subprocess_name, mode, stderr, env, unsetenv, chdir,
|
43
|
+
internal_encoding, external_encoding, scrub, replace_string,
|
44
|
+
wait_timeout, on_exit_callback,
|
45
|
+
&callback
|
46
|
+
)
|
47
|
+
}
|
48
|
+
|
49
|
+
now = Fluent::EventTime.now.to_int - delay_seconds
|
50
|
+
if immediate && start_timestamp.to_i < now
|
51
|
+
execute_child_process.call(command % [start_timestamp, now])
|
52
|
+
start_timestamp = now
|
53
|
+
time_callback.call(start_timestamp)
|
54
|
+
end
|
55
|
+
|
56
|
+
timer_execute(:child_process_execute, interval, repeat: true) do
|
57
|
+
if !parallel && running
|
58
|
+
log.warn "previous child process is still running. skipped.", title: title, command: command, arguments: arguments, interval: interval
|
59
|
+
else
|
60
|
+
end_timestamp = Fluent::EventTime.now.to_int - delay_seconds
|
61
|
+
execute_child_process.call(command % [start_timestamp, end_timestamp])
|
62
|
+
start_timestamp = end_timestamp
|
63
|
+
time_callback.call(start_timestamp)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,199 @@
|
|
1
|
+
|
2
|
+
require 'fluent/plugin/input'
|
3
|
+
require 'fluent/plugin/in_exec/iterative_process'
|
4
|
+
require 'yajl'
|
5
|
+
|
6
|
+
module Fluent::Plugin
|
7
|
+
class MacOsLogInput < Fluent::Plugin::Input
|
8
|
+
Fluent::Plugin.register_input('macoslog', self)
|
9
|
+
|
10
|
+
helpers :compat_parameters, :extract, :parser, :child_process
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
super
|
14
|
+
@pf_file = nil
|
15
|
+
@log_start_regex = nil
|
16
|
+
@compiled_command = nil
|
17
|
+
end
|
18
|
+
|
19
|
+
desc 'The command (program) to execute.'
|
20
|
+
config_param :command, :string, default: 'log show --start @%s --end @%s'
|
21
|
+
desc 'The unified log filter predicate as per Apple\'s documentation'
|
22
|
+
config_param :predicate, :string, default: nil
|
23
|
+
desc 'Specify connect mode to executed process'
|
24
|
+
config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
|
25
|
+
desc 'Logging levels supported by Unified Logging ([no-]backtrace, [no-]debug, [no-]info, [no-]loss, [no-]signpost)'
|
26
|
+
config_param :levels, :array, default: [], value_type: :string
|
27
|
+
desc 'Output formatting of events from logging tool'
|
28
|
+
config_param :style, :enum, list: [:default, :syslog, :ndjson, :compact], default: nil
|
29
|
+
|
30
|
+
config_section :parse do
|
31
|
+
config_set_default :@type, 'regexp'
|
32
|
+
config_set_default :expression, /^(?<logtime>[\d\-]+\s*[\d\.:\+]+)\s+(?<thread>[^ ]*)\s+(?<level>[^ ]+)\s+(?<activity>[^ ]*)\s+(?<pid>[0-9]+)\s+(?<ttl>[0-9]+)\s+(?<process>[^:]*)(?:[^\:]*\:)\s*(?<message>.*)$/m
|
33
|
+
config_set_default :time_key, 'logtime'
|
34
|
+
config_set_default :time_format, '%Y-%m-%d %H:%M:%S.%L%z'
|
35
|
+
end
|
36
|
+
|
37
|
+
desc 'Tag of the output events.'
|
38
|
+
config_param :tag, :string, default: nil
|
39
|
+
desc 'The interval time between periodic program runs.'
|
40
|
+
config_param :run_interval, :time, default: nil
|
41
|
+
desc 'Fluentd will record the time it last read into this file.'
|
42
|
+
config_param :pos_file, :string, default: nil
|
43
|
+
desc 'The identifier of log line beginning used to split output by.'
|
44
|
+
config_param :log_line_start, :string, default: '\d+-\d+-\d+\s+\d+:\d+:\d+[^ ]+'
|
45
|
+
desc 'Number of header lines to be skipped. Use negative value if no header'
|
46
|
+
config_param :log_header_lines, :integer, default: 1
|
47
|
+
desc 'Max age of logs to be inputted'
|
48
|
+
config_param :max_age, :time, default: 259200 #3d
|
49
|
+
|
50
|
+
attr_reader :parser
|
51
|
+
|
52
|
+
def configure(conf)
|
53
|
+
compat_parameters_convert(conf, :parser)
|
54
|
+
|
55
|
+
super
|
56
|
+
|
57
|
+
unless @tag
|
58
|
+
raise Fluent::ConfigError, "'tag' option is required on macoslog input"
|
59
|
+
end
|
60
|
+
|
61
|
+
@compiled_command = compile_command
|
62
|
+
|
63
|
+
$log.info "MacOs log command '#{@compiled_command}'"
|
64
|
+
|
65
|
+
@parser = parser_create
|
66
|
+
|
67
|
+
unless @pos_file
|
68
|
+
$log.warn "'pos_file PATH' parameter is not set to a 'macoslog' source."
|
69
|
+
$log.warn "this parameter is highly recommended to save the position to resume from."
|
70
|
+
end
|
71
|
+
|
72
|
+
@log_start_regex = Regexp.compile("\\A#{@log_line_start}")
|
73
|
+
end
|
74
|
+
|
75
|
+
def compile_command
|
76
|
+
result = @command
|
77
|
+
|
78
|
+
if @style
|
79
|
+
result += " --style #{@style}"
|
80
|
+
elsif not result =~ /"--style"/
|
81
|
+
result += " --style default"
|
82
|
+
end
|
83
|
+
|
84
|
+
if @predicate
|
85
|
+
result += " --predicate '#{@predicate}'"
|
86
|
+
end
|
87
|
+
|
88
|
+
if @levels.length > 0
|
89
|
+
compiled_level = @levels.map { |level| "--#{level}" }.join(" ")
|
90
|
+
result += " #{compiled_level}"
|
91
|
+
end
|
92
|
+
|
93
|
+
result
|
94
|
+
end
|
95
|
+
|
96
|
+
def multi_workers_ready?
|
97
|
+
true
|
98
|
+
end
|
99
|
+
|
100
|
+
def start
|
101
|
+
super
|
102
|
+
|
103
|
+
if @pos_file
|
104
|
+
pos_file_dir = File.dirname(@pos_file)
|
105
|
+
FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)
|
106
|
+
@pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
|
107
|
+
@pf_file.sync = true
|
108
|
+
|
109
|
+
start = @pf_file.read.to_i
|
110
|
+
if start == 0
|
111
|
+
start = Fluent::EventTime.now.to_int
|
112
|
+
@pf_file.write(start)
|
113
|
+
end
|
114
|
+
else
|
115
|
+
start = Fluent::EventTime.now.to_int
|
116
|
+
@pf_file.write(start) if @pf_file
|
117
|
+
end
|
118
|
+
|
119
|
+
oldest = Fluent::EventTime.now.to_int - @max_age.to_i
|
120
|
+
if start < oldest
|
121
|
+
log.info "Start timestamp over max_age ", start: start, oldest: oldest
|
122
|
+
start = oldest
|
123
|
+
end
|
124
|
+
|
125
|
+
time_callback = -> (timestamp) {
|
126
|
+
if @pf_file
|
127
|
+
@pf_file.rewind
|
128
|
+
@pf_file.write(timestamp)
|
129
|
+
end
|
130
|
+
}
|
131
|
+
|
132
|
+
timer_process_execute(:exec_input,
|
133
|
+
@compiled_command,
|
134
|
+
start, @run_interval,
|
135
|
+
time_callback,
|
136
|
+
delay_seconds: 1,
|
137
|
+
immediate: true, mode: [@connect_mode], &method(:run))
|
138
|
+
end
|
139
|
+
|
140
|
+
def shutdown
|
141
|
+
@pf_file.close if @pf_file
|
142
|
+
|
143
|
+
super
|
144
|
+
end
|
145
|
+
|
146
|
+
def run(io)
|
147
|
+
unless io.eof
|
148
|
+
if @style == :ndjson
|
149
|
+
parse_line_json(io)
|
150
|
+
else
|
151
|
+
parse_timestamp_base(io)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def parse_line_json(io)
|
157
|
+
logs = Queue.new
|
158
|
+
io.each_line.with_index do |line,index|
|
159
|
+
logs.push(line.chomp("\n"))
|
160
|
+
if index >= @log_header_lines
|
161
|
+
@parser.parse(logs.pop, &method(:on_record))
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
def parse_timestamp_base(io)
|
167
|
+
log = ""
|
168
|
+
io.each_line.with_index do |line,index|
|
169
|
+
# Skips log header
|
170
|
+
if index >= @log_header_lines
|
171
|
+
if line =~ @log_start_regex
|
172
|
+
if log.empty?
|
173
|
+
log = line
|
174
|
+
else
|
175
|
+
@parser.parse(log.chomp("\n"), &method(:on_record))
|
176
|
+
log = line
|
177
|
+
end
|
178
|
+
else
|
179
|
+
log += line
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
unless log.empty?
|
185
|
+
@parser.parse(log.chomp("\n"), &method(:on_record))
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
def on_record(time, record)
|
190
|
+
tag = extract_tag_from_record(record)
|
191
|
+
tag ||= @tag
|
192
|
+
time ||= extract_time_from_record(record) || Fluent::EventTime.now
|
193
|
+
router.emit(tag, time, record)
|
194
|
+
rescue => e
|
195
|
+
log.error "macoslog failed to emit", tag: tag, record: Yajl.dump(record), error: e
|
196
|
+
router.emit_error_event(tag, time, record, e) if tag && time && record
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
data/version.txt
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1.beta.1
|
metadata
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-macos-log
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.beta.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Petr Langr
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-04-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.2'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.2'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: yajl-ruby
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '1.3'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1.3'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bundler
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '2.0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '2.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rake
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '10.0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '10.0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: minitest
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '5.0'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '5.0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: test-unit
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - "~>"
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '3.2'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '3.2'
|
103
|
+
description: Fluentd input from MacOS unified log
|
104
|
+
email:
|
105
|
+
- petr.langr@solarwinds.com
|
106
|
+
executables: []
|
107
|
+
extensions: []
|
108
|
+
extra_rdoc_files: []
|
109
|
+
files:
|
110
|
+
- ".circleci/config.yml"
|
111
|
+
- ".gitignore"
|
112
|
+
- Gemfile
|
113
|
+
- Gemfile.lock
|
114
|
+
- Makefile
|
115
|
+
- README.md
|
116
|
+
- Rakefile
|
117
|
+
- fluent-plugin-macos-log.gemspec
|
118
|
+
- fluent-plugin-macos-log.gemspec.lock
|
119
|
+
- lib/fluent/plugin/in_exec/iterative_process.rb
|
120
|
+
- lib/fluent/plugin/in_macoslog.rb
|
121
|
+
- version.txt
|
122
|
+
homepage: https://github.com/loggly/fluent-plugin-macos-log
|
123
|
+
licenses:
|
124
|
+
- MIT
|
125
|
+
metadata: {}
|
126
|
+
post_install_message:
|
127
|
+
rdoc_options: []
|
128
|
+
require_paths:
|
129
|
+
- lib
|
130
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
131
|
+
requirements:
|
132
|
+
- - ">="
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: '0'
|
135
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">"
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: 1.3.1
|
140
|
+
requirements: []
|
141
|
+
rubygems_version: 3.0.3
|
142
|
+
signing_key:
|
143
|
+
specification_version: 4
|
144
|
+
summary: Fluentd input plugin for MacOS unified log
|
145
|
+
test_files: []
|