lstash 0.2.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.devcontainer/Aptfile +6 -0
- data/.devcontainer/Dockerfile +44 -0
- data/.devcontainer/devcontainer.json +36 -0
- data/.github/workflows/test.yml +67 -0
- data/CHANGELOG.md +22 -3
- data/README.md +86 -86
- data/bin/lstash +1 -3
- data/dip.yml +48 -0
- data/docker-compose.yml +28 -0
- data/lib/lstash/cli.rb +42 -29
- data/lib/lstash/client.rb +58 -37
- data/lib/lstash/query.rb +54 -77
- data/lib/lstash/version.rb +1 -1
- data/lib/lstash.rb +4 -4
- data/lstash.gemspec +14 -19
- data/spec/lstash/cli_spec.rb +21 -23
- data/spec/lstash/client_spec.rb +29 -33
- data/spec/lstash/query_spec.rb +62 -60
- data/spec/lstash_spec.rb +3 -3
- data/spec/spec_helper.rb +28 -13
- metadata +17 -58
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b595d887abdc37f07322ccb509d627d477b6e4897bc7768c512ba55f45ab4029
|
|
4
|
+
data.tar.gz: 814170a66956839f128b52a31bc56d99578d434a73e1194c0fa45ecd686f9b40
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 308cb925f02b6dc9ae7c67ca9ac3099615c72b82254e3d9dfd8d0fad33d2b1aa2ead526b35023cf4a20c9497960010ea84cadd50d12d8664813fd7a5f96bb06c
|
|
7
|
+
data.tar.gz: 6e3b63eb59aa2c45bd5b1de469dc79c84bfbed514f2ec4a6b4c12bf92475d1c37daae6a07ef900c6c286fe4ac5ef2e8c0efc8898f88efa11ce5c7d16e47dcd73
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
ARG RUBY_VERSION=2.4.6
|
|
2
|
+
ARG BUNDLER_VERSION=1.17.3
|
|
3
|
+
ARG DISTRO_NAME=buster
|
|
4
|
+
|
|
5
|
+
FROM ruby:$RUBY_VERSION-slim-$DISTRO_NAME
|
|
6
|
+
|
|
7
|
+
LABEL maintainer="k.j.wierenga@kerkdienstgemist.nl"
|
|
8
|
+
|
|
9
|
+
# Install dependencies specified in Aptfile
|
|
10
|
+
COPY Aptfile /tmp/
|
|
11
|
+
RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade \
|
|
12
|
+
&& mkdir -p /usr/share/man/man1 /usr/share/man/man7 \
|
|
13
|
+
&& DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
|
|
14
|
+
$(grep -Evh '^\s*#' /tmp/Aptfile /tmp/Aptfile.dev | xargs) \
|
|
15
|
+
&& apt-get autoremove -y \
|
|
16
|
+
&& apt-get clean \
|
|
17
|
+
&& rm -rf /var/cache/apt/archives/* \
|
|
18
|
+
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
|
|
19
|
+
&& truncate -s 0 /var/log/*log
|
|
20
|
+
|
|
21
|
+
# Configure bundler
|
|
22
|
+
# ENV LANG=C.UTF-8 \
|
|
23
|
+
# BUNDLE_JOBS=4 \
|
|
24
|
+
# BUNDLE_RETRY=3 \
|
|
25
|
+
# TZ=Europe/Amsterdam
|
|
26
|
+
|
|
27
|
+
# Store Bundler settings in the project's root
|
|
28
|
+
# ENV BUNDLE_APP_CONFIG=.bundle
|
|
29
|
+
|
|
30
|
+
# Uncomment this line if you want to run binstubs without prefixing with `bin/` or `bundle exec`
|
|
31
|
+
# ENV PATH /gem/bin:$PATH
|
|
32
|
+
|
|
33
|
+
# Upgrade RubyGems and install the latest Bundler version
|
|
34
|
+
ARG BUNDLER_VERSION
|
|
35
|
+
# gem update --system &&
|
|
36
|
+
RUN gem install bundler:$BUNDLER_VERSION
|
|
37
|
+
|
|
38
|
+
ENV TZ="Europe/Amsterdam"
|
|
39
|
+
|
|
40
|
+
# Configure the main working directory. This is the base
|
|
41
|
+
# directory used in any further RUN, COPY, and ENTRYPOINT commands.
|
|
42
|
+
ENV WORKDIR /gem
|
|
43
|
+
RUN mkdir -p $WORKDIR
|
|
44
|
+
WORKDIR $WORKDIR
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
|
|
2
|
+
// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/docker-existing-docker-compose
|
|
3
|
+
// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
|
|
4
|
+
{
|
|
5
|
+
"name": "lstash",
|
|
6
|
+
|
|
7
|
+
// Update the 'dockerComposeFile' list if you have more compose files or use different names.
|
|
8
|
+
// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
|
|
9
|
+
"dockerComposeFile": [
|
|
10
|
+
"../docker-compose.yml"
|
|
11
|
+
// "../docker-compose.override.yml"
|
|
12
|
+
],
|
|
13
|
+
|
|
14
|
+
// The 'service' property is the name of the service for the container that VS Code should
|
|
15
|
+
// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
|
|
16
|
+
"service": "runner",
|
|
17
|
+
|
|
18
|
+
// The optional 'workspaceFolder' property is the path VS Code should open by default when
|
|
19
|
+
// connected. This is typically a file mount in .devcontainer/docker-compose.yml
|
|
20
|
+
"workspaceFolder": "/gem"
|
|
21
|
+
|
|
22
|
+
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
|
23
|
+
// "forwardPorts": [],
|
|
24
|
+
|
|
25
|
+
// Uncomment the next line if you want start specific services in your Docker Compose config.
|
|
26
|
+
// "runServices": [],
|
|
27
|
+
|
|
28
|
+
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
|
|
29
|
+
// "shutdownAction": "none",
|
|
30
|
+
|
|
31
|
+
// Uncomment the next line to run commands after the container is created - for example installing curl.
|
|
32
|
+
// "postCreateCommand": "apt-get update && apt-get install -y curl",
|
|
33
|
+
|
|
34
|
+
// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
|
|
35
|
+
// "remoteUser": "vscode"
|
|
36
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: Run tests
|
|
2
|
+
on: [push]
|
|
3
|
+
|
|
4
|
+
jobs:
|
|
5
|
+
# This job uses buildx layer caching
|
|
6
|
+
# See https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching#the-cache-dance-off
|
|
7
|
+
test:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
env:
|
|
10
|
+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
|
11
|
+
steps:
|
|
12
|
+
- name: Checkout code
|
|
13
|
+
uses: actions/checkout@v2
|
|
14
|
+
|
|
15
|
+
- name: Set up Docker Buildx
|
|
16
|
+
id: buildx
|
|
17
|
+
uses: docker/setup-buildx-action@v2
|
|
18
|
+
with:
|
|
19
|
+
install: true # needed to ensure docker compose uses the build cache too
|
|
20
|
+
|
|
21
|
+
- name: Cache Docker layers
|
|
22
|
+
uses: actions/cache@v2
|
|
23
|
+
with:
|
|
24
|
+
path: /tmp/.buildx-cache
|
|
25
|
+
key: ${{ runner.os }}-single-buildx-${{ github.sha }}
|
|
26
|
+
restore-keys: |
|
|
27
|
+
${{ runner.os }}-single-buildx
|
|
28
|
+
|
|
29
|
+
- name: Build runner image
|
|
30
|
+
uses: docker/build-push-action@v2
|
|
31
|
+
with:
|
|
32
|
+
context: .devcontainer
|
|
33
|
+
builder: ${{ steps.buildx.outputs.name }}
|
|
34
|
+
push: false # This would be set to true in a real world deployment scenario.
|
|
35
|
+
load: true # Needed to ensure image is used in the "Run tests" step
|
|
36
|
+
tags: lstash_runner
|
|
37
|
+
cache-from: type=local,src=/tmp/.buildx-cache
|
|
38
|
+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
|
|
39
|
+
|
|
40
|
+
- name: Run tests
|
|
41
|
+
id: test
|
|
42
|
+
env:
|
|
43
|
+
DOCKER_BUILDKIT: 1
|
|
44
|
+
COMPOSE_DOCKER_CLI_BUILD: 1
|
|
45
|
+
run: |
|
|
46
|
+
docker compose run runner bundle install
|
|
47
|
+
docker compose run runner bundle exec rspec
|
|
48
|
+
|
|
49
|
+
# Temp fix
|
|
50
|
+
# https://github.com/docker/build-push-action/issues/252
|
|
51
|
+
# https://github.com/moby/buildkit/issues/1896
|
|
52
|
+
- name: Move cache
|
|
53
|
+
run: |
|
|
54
|
+
rm -rf /tmp/.buildx-cache
|
|
55
|
+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
|
|
56
|
+
|
|
57
|
+
# Notify via Slack when workflow is not successful
|
|
58
|
+
- uses: act10ns/slack@v1
|
|
59
|
+
with:
|
|
60
|
+
status: ${{ job.status }}
|
|
61
|
+
steps: ${{ toJson(steps) }}
|
|
62
|
+
if: always() # ${{ !success() }}
|
|
63
|
+
|
|
64
|
+
# cancel in progress workflows for the same github ref (e.g. branch)
|
|
65
|
+
concurrency:
|
|
66
|
+
group: ci-tests-${{ github.ref }}-1
|
|
67
|
+
cancel-in-progress: true
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,27 @@
|
|
|
1
|
+
## Release 1.0.0
|
|
2
|
+
|
|
3
|
+
* BREAKING CHANGE: default options changed
|
|
4
|
+
* Default --from today changed to --from yesterday
|
|
5
|
+
* Default --to now changed to --to today
|
|
6
|
+
* This ensures that by default lstash counts or greps in yesterdays logging.
|
|
7
|
+
* Fixed bug which caused empty range (e.g. --from today --to today) to incorrectly return non-zero count and logging.
|
|
8
|
+
* Upgrade elasticsearch gem from version ~> 0.4 to ~> 7.17.7.
|
|
9
|
+
* Update queries and field selectors to be compatible with Elasticsearch version 7.
|
|
10
|
+
* Increase scroll step size for grep from 2 minutes to 1 hour (current Elasticsearch can handle it).
|
|
11
|
+
* Add --wildcard / --no-wildcard option to use logstash-* wildcard instead of iterating over indices directly.
|
|
12
|
+
* For the count command --wildcard is faster so that's the default for count.
|
|
13
|
+
* For the grep command --no-wildcard is faster so that's the default for grep.
|
|
14
|
+
* Dockerize development and add GitHub action for testing.
|
|
15
|
+
* Moved repo from kdgm/lstash to kdgm/lstash.
|
|
16
|
+
* Rubocop fixes
|
|
17
|
+
|
|
1
18
|
## Release 0.2.0
|
|
19
|
+
|
|
2
20
|
Merge branch 'feature/fix/hashie-warnings' into develop
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
21
|
+
|
|
22
|
+
* [fix] pin faraday to a compatible version
|
|
23
|
+
* [enh] upgrade to ruby 2.4(.6)
|
|
24
|
+
* [fix] suppress Hashie warnings; it would generate a warning for each log line (on stdout) leading to very large output
|
|
6
25
|
|
|
7
26
|
### 0.1.4 / 2015-05-29
|
|
8
27
|
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# lstash
|
|
2
2
|
|
|
3
|
-
[](https://github.com/kdgm/lstash/actions/workflows/test.yml)
|
|
4
4
|
|
|
5
5
|
Lstash is a gem and command line utility to count or grep log messages in a certain time frame from a Logstash Elasticsearch server.
|
|
6
6
|
|
|
@@ -8,48 +8,48 @@ Lstash is a gem and command line utility to count or grep log messages in a cert
|
|
|
8
8
|
|
|
9
9
|
Or install it yourself as:
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
gem install lstash
|
|
12
12
|
|
|
13
13
|
## Running lstash from the command line
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
$ lstash
|
|
16
|
+
Commands:
|
|
17
|
+
lstash count QUERY # count number of log messages matching the QUERY
|
|
18
|
+
lstash grep QUERY # grep log messages from Logstash
|
|
19
|
+
lstash help [COMMAND] # Describe available commands or one specific command
|
|
20
20
|
|
|
21
21
|
## The `count` command
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
Usage:
|
|
24
|
+
lstash count QUERY
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
Description:
|
|
27
|
+
Count log messages matching the QUERY from Logstash and output this count to stdout. QUERY can use Apache Lucene query
|
|
28
|
+
parser syntax.
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
Example to count the number of HAProxy log messages in yesterdays month.
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
|
|
33
33
|
|
|
34
34
|
## The `grep` command
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
Usage:
|
|
37
|
+
lstash grep QUERY
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
Description:
|
|
40
|
+
Grep log messages matching the QUERY from Logstash in ascending timestamp order and output to stdout. QUERY can use Apache Lucene query parser syntax.
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
Example to grep HAProxy log messages from the beginning of this month upto now
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
lstash grep 'program:haproxy' --from firstday --to now
|
|
45
45
|
|
|
46
46
|
## Command line options
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
Options:
|
|
49
|
+
-f, [--from=start of time range] # date/time, 'now', 'today', 'yesterday', or 'firstday'
|
|
50
|
+
-t, [--to=end of time range] # date/time, 'now', 'today', 'yesterday', or 'firstday'
|
|
51
|
+
-a, [--anchor=anchor date/time] # used as reference date for firstday
|
|
52
|
+
-e, [--es-url=Elasticsearch endpoint for Logstash] # or ES_URL environment variable
|
|
53
53
|
|
|
54
54
|
All times will be relative to the timezone of the machine on which you are running lstash.
|
|
55
55
|
|
|
@@ -66,7 +66,7 @@ Example
|
|
|
66
66
|
|
|
67
67
|
Or
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
lstash count program:haproxy --es-url log.mydomain.com
|
|
70
70
|
|
|
71
71
|
## Examples
|
|
72
72
|
|
|
@@ -80,41 +80,41 @@ Grep all haproxy log messages using for one day (Aug 24 1 0:00 am upto and inclu
|
|
|
80
80
|
|
|
81
81
|
Assuming today is Sep 1 2014. Count all haproxy log messages in the previous month.
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
83
|
+
lstash count program:haproxy --anchor yesterday --from firstday --to today -d
|
|
84
|
+
time range: [2014-08-01 00:00:00 +0200..2014-09-01 00:00:00 +0200]
|
|
85
|
+
logstash-2014.07.31: 1
|
|
86
|
+
logstash-2014.08.01: 13
|
|
87
|
+
logstash-2014.08.02: 14
|
|
88
|
+
logstash-2014.08.03: 1654
|
|
89
|
+
logstash-2014.08.04: 6
|
|
90
|
+
logstash-2014.08.05: 20
|
|
91
|
+
logstash-2014.08.06: 219
|
|
92
|
+
logstash-2014.08.07: 32
|
|
93
|
+
logstash-2014.08.08: 14
|
|
94
|
+
logstash-2014.08.09: 28
|
|
95
|
+
logstash-2014.08.10: 799
|
|
96
|
+
logstash-2014.08.11: 18
|
|
97
|
+
logstash-2014.08.12: 8
|
|
98
|
+
logstash-2014.08.13: 23
|
|
99
|
+
logstash-2014.08.14: 25
|
|
100
|
+
logstash-2014.08.15: 69
|
|
101
|
+
logstash-2014.08.16: 19
|
|
102
|
+
logstash-2014.08.17: 1160
|
|
103
|
+
logstash-2014.08.18: 284
|
|
104
|
+
logstash-2014.08.19: 61
|
|
105
|
+
logstash-2014.08.20: 26
|
|
106
|
+
logstash-2014.08.21: 16
|
|
107
|
+
logstash-2014.08.22: 145
|
|
108
|
+
logstash-2014.08.23: 72
|
|
109
|
+
logstash-2014.08.24: 792
|
|
110
|
+
logstash-2014.08.25: 31
|
|
111
|
+
logstash-2014.08.26: 33
|
|
112
|
+
logstash-2014.08.27: 51
|
|
113
|
+
logstash-2014.08.28: 8
|
|
114
|
+
logstash-2014.08.29: 23
|
|
115
|
+
logstash-2014.08.30: 25
|
|
116
|
+
logstash-2014.08.31: 69
|
|
117
|
+
5633
|
|
118
118
|
|
|
119
119
|
## Using lstash as a gem in your project
|
|
120
120
|
|
|
@@ -124,46 +124,46 @@ Add this line to your application's Gemfile:
|
|
|
124
124
|
|
|
125
125
|
And then execute:
|
|
126
126
|
|
|
127
|
-
|
|
127
|
+
bundle
|
|
128
128
|
|
|
129
129
|
Usage:
|
|
130
130
|
|
|
131
|
-
|
|
131
|
+
bundle console
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
elasticsearch = Elasticsearch::Client.new(url: 'log.mydomain.com')
|
|
135
|
-
client = Lstash::Client.new(elasticsearch)
|
|
133
|
+
Connect to elasticsearch and create the Lstash client
|
|
136
134
|
|
|
137
|
-
|
|
138
|
-
|
|
135
|
+
elasticsearch = Elasticsearch::Client.new(url: ENV['ES_URL'])
|
|
136
|
+
client = Lstash::Client.new(elasticsearch, debug: true)
|
|
139
137
|
|
|
140
|
-
|
|
141
|
-
client.count(query)
|
|
138
|
+
Create the query
|
|
142
139
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
query = Lstash::Query.new('program:haproxy', from: 'today', to: 'now')
|
|
141
|
+
|
|
142
|
+
Count example
|
|
143
|
+
|
|
144
|
+
client.count(query)
|
|
145
|
+
|
|
146
|
+
Grep example
|
|
147
|
+
|
|
148
|
+
client.grep(query) do |message|
|
|
149
|
+
puts message
|
|
150
|
+
end
|
|
147
151
|
|
|
148
152
|
## Publishing the gem to RubyGems.org
|
|
149
153
|
|
|
150
154
|
1. Build the gem
|
|
151
155
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
File: lstash-0.2.0.gem
|
|
158
|
-
```
|
|
156
|
+
$ gem build lstash
|
|
157
|
+
Successfully built RubyGem
|
|
158
|
+
Name: lstash
|
|
159
|
+
Version: 0.2.0
|
|
160
|
+
File: lstash-0.2.0.gem
|
|
159
161
|
|
|
160
162
|
2. Pushing your gem to RubyGems.org
|
|
161
163
|
|
|
162
|
-
|
|
163
|
-
gem
|
|
164
|
-
|
|
165
|
-
Successfully registered gem: lstash (0.2.0)
|
|
166
|
-
```
|
|
164
|
+
$ gem push lstash-0.2.0.gem
|
|
165
|
+
Pushing gem to RubyGems.org...
|
|
166
|
+
Successfully registered gem: lstash (0.2.0)
|
|
167
167
|
|
|
168
168
|
See [RubyGems.org documention](https://guides.rubygems.org/) for more info.
|
|
169
169
|
|
data/bin/lstash
CHANGED
data/dip.yml
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Required minimum dip version
|
|
2
|
+
version: '6.1'
|
|
3
|
+
|
|
4
|
+
compose:
|
|
5
|
+
files:
|
|
6
|
+
- docker-compose.yml
|
|
7
|
+
- docker-compose.override.yml # platform specific overrides
|
|
8
|
+
|
|
9
|
+
# Specify project name explicitly to avoid name collisions:
|
|
10
|
+
# docker-compose uses the compose file's folder name as the project name by default,
|
|
11
|
+
# which could be the same for different projects (e.g., if you store docker-compose.yml
|
|
12
|
+
# in the .devcontainer/ folder)
|
|
13
|
+
|
|
14
|
+
# NOTE: We deliberately do not set project_name to prevent collissions
|
|
15
|
+
# between git work trees.
|
|
16
|
+
# Let the name be determined automatically from the base directory name.
|
|
17
|
+
# project_name: UNSET DELIBERATELY
|
|
18
|
+
|
|
19
|
+
interaction:
|
|
20
|
+
sh:
|
|
21
|
+
description: Start a Bash shell in the container
|
|
22
|
+
service: runner
|
|
23
|
+
command: /bin/bash
|
|
24
|
+
|
|
25
|
+
bundle:
|
|
26
|
+
description: Run bundler commands
|
|
27
|
+
service: runner
|
|
28
|
+
command: bundle
|
|
29
|
+
|
|
30
|
+
rake:
|
|
31
|
+
description: Run rake commands
|
|
32
|
+
service: runner
|
|
33
|
+
command: bundle exec rake
|
|
34
|
+
|
|
35
|
+
rspec:
|
|
36
|
+
description: Run specs
|
|
37
|
+
service: runner
|
|
38
|
+
command: bundle exec rspec
|
|
39
|
+
|
|
40
|
+
provision:
|
|
41
|
+
# Remove old containers and volumes.
|
|
42
|
+
- dip compose down --volumes --remove-orphans
|
|
43
|
+
|
|
44
|
+
# Build the development container (based on .devcontainer/Dockerfile)
|
|
45
|
+
- dip compose build
|
|
46
|
+
|
|
47
|
+
# Install gem dependencies
|
|
48
|
+
- dip bundle install
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
x-base: &base
|
|
2
|
+
# NOTE: We deliberately do not set the image name to prevent
|
|
3
|
+
# collissions between images built in different git work trees (directories)
|
|
4
|
+
# image: LEFT UNSET DELIBERATELY
|
|
5
|
+
build:
|
|
6
|
+
context: .devcontainer
|
|
7
|
+
tmpfs:
|
|
8
|
+
- /tmp
|
|
9
|
+
volumes:
|
|
10
|
+
- .:/gem:cached
|
|
11
|
+
- bundle:/usr/local/bundle
|
|
12
|
+
- history:/usr/local/hist
|
|
13
|
+
stdin_open: true
|
|
14
|
+
tty: true
|
|
15
|
+
environment:
|
|
16
|
+
HISTFILE: /usr/local/hist/.bash_history
|
|
17
|
+
IRB_HISTFILE: /usr/local/hist/.irb_history
|
|
18
|
+
EDITOR: ${EDITOR:-vi}
|
|
19
|
+
ES_URL: http://host.docker.internal:9200
|
|
20
|
+
|
|
21
|
+
services:
|
|
22
|
+
runner:
|
|
23
|
+
<<: *base
|
|
24
|
+
command: /bin/bash
|
|
25
|
+
|
|
26
|
+
volumes:
|
|
27
|
+
bundle:
|
|
28
|
+
history:
|
data/lib/lstash/cli.rb
CHANGED
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
# external dependencies
|
|
2
|
-
require
|
|
3
|
-
require
|
|
4
|
-
require
|
|
2
|
+
require "thor"
|
|
3
|
+
require "uri"
|
|
4
|
+
require "elasticsearch"
|
|
5
5
|
|
|
6
6
|
# local files we need
|
|
7
|
-
require
|
|
8
|
-
require
|
|
7
|
+
require "lstash/query"
|
|
8
|
+
require "lstash/client"
|
|
9
|
+
require "lstash/version"
|
|
9
10
|
|
|
10
11
|
module Lstash
|
|
12
|
+
TRANSPORT_REQUEST_TIMEOUT = 120 # 2 minute request timeout
|
|
13
|
+
|
|
14
|
+
class CLIBase < Thor
|
|
15
|
+
class << self
|
|
16
|
+
def shared_options
|
|
17
|
+
method_option :anchor, banner: "YYYY-mm-dd", aliases: "-a", desc: "The 'firstday' is relative to this anchor date", default: "today"
|
|
18
|
+
method_option :from, banner: "YYYY-mm-dd [HH:MM:SS]", aliases: "-f", desc: "Start date/time, 'now', 'today', 'yesterday', or 'firstday'", default: "yesterday"
|
|
19
|
+
method_option :to, banner: "YYYY-mm-dd [HH:MM:SS]", aliases: "-t", desc: "End date/time, 'now', 'today', 'yesterday', or 'firstday'", default: "today"
|
|
20
|
+
method_option :es_url, banner: "http://localhost:9200", aliases: "-e", desc: "Elasticsearch URL or set ES_URL environment variable"
|
|
21
|
+
method_option :debug, desc: "Log debugging info to stderr", aliases: "-d", type: :boolean, default: false
|
|
22
|
+
method_option :wildcard, desc: "Use index wildcard to query all logstash-* indices (fast for count, slow for grep)", type: :boolean
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
11
26
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class CLI < Thor
|
|
15
|
-
|
|
16
|
-
class_option :from, :banner => 'start of time range', :aliases => '-f', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
|
|
17
|
-
class_option :to, :banner => 'end of time range', :aliases => '-t', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
|
|
18
|
-
class_option :anchor, :banner => 'anchor date/time', :aliases => '-a', :desc => "used as reference date for firstday"
|
|
19
|
-
class_option :es_url, :banner => 'Elasticsearch endpoint for Logstash', :aliases => '-e', :desc => "or ES_URL environment variable"
|
|
20
|
-
class_option :debug, :banner => 'debug log to stderr', :aliases => '-d', :type => :boolean
|
|
21
|
-
|
|
27
|
+
class CLI < CLIBase
|
|
22
28
|
long_desc <<-LONGDESC
|
|
23
29
|
Grep log messages matching the QUERY from Logstash in ascending timestamp order
|
|
24
30
|
and output to stdout. QUERY can use Apache Lucene query parser syntax.
|
|
@@ -27,7 +33,8 @@ module Lstash
|
|
|
27
33
|
|
|
28
34
|
lstash grep 'program:haproxy' --from firstday --to now
|
|
29
35
|
LONGDESC
|
|
30
|
-
desc "grep QUERY", "
|
|
36
|
+
desc "grep QUERY", "Grep log messages from Logstash"
|
|
37
|
+
shared_options
|
|
31
38
|
def grep(query_string)
|
|
32
39
|
run_command(query_string) do |es_client, query|
|
|
33
40
|
Lstash::Client.new(es_client, options).grep(query) do |message|
|
|
@@ -44,7 +51,8 @@ module Lstash
|
|
|
44
51
|
|
|
45
52
|
lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
|
|
46
53
|
LONGDESC
|
|
47
|
-
desc "count QUERY", "
|
|
54
|
+
desc "count QUERY", "Count number of log messages matching the QUERY"
|
|
55
|
+
shared_options
|
|
48
56
|
def count(query_string)
|
|
49
57
|
run_command(query_string) do |es_client, query|
|
|
50
58
|
count = Lstash::Client.new(es_client, options).count(query)
|
|
@@ -52,29 +60,34 @@ module Lstash
|
|
|
52
60
|
end
|
|
53
61
|
end
|
|
54
62
|
|
|
63
|
+
long_desc "Print the lstash version"
|
|
64
|
+
desc "version", "print lstash version"
|
|
65
|
+
def version
|
|
66
|
+
puts Lstash::VERSION
|
|
67
|
+
end
|
|
68
|
+
|
|
55
69
|
private
|
|
56
70
|
|
|
57
71
|
def run_command(query_string)
|
|
58
72
|
es_client = ::Elasticsearch::Client.new(
|
|
59
|
-
url: options[:es_url] || ENV[
|
|
60
|
-
log:
|
|
61
|
-
transport_options: {
|
|
73
|
+
url: options[:es_url] || ENV["ES_URL"] || "http://localhost:9200",
|
|
74
|
+
log: ENV["DEBUG"] == "true",
|
|
75
|
+
transport_options: {request: {timeout: TRANSPORT_REQUEST_TIMEOUT}}
|
|
62
76
|
)
|
|
63
|
-
query
|
|
77
|
+
query = Lstash::Query.new(query_string, options)
|
|
64
78
|
|
|
65
79
|
yield es_client, query
|
|
66
|
-
|
|
67
|
-
rescue Exception => e
|
|
80
|
+
rescue => e
|
|
68
81
|
options[:debug] ? raise(e) : raise(Thor::Error.new(e.message))
|
|
69
82
|
end
|
|
70
83
|
|
|
71
|
-
protected
|
|
72
|
-
|
|
73
84
|
# Make sure we exit on failure with an error code
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
end
|
|
85
|
+
class << self
|
|
86
|
+
protected
|
|
77
87
|
|
|
88
|
+
def exit_on_failure?
|
|
89
|
+
true
|
|
90
|
+
end
|
|
91
|
+
end
|
|
78
92
|
end
|
|
79
|
-
|
|
80
93
|
end
|