lstash 0.2.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.devcontainer/Aptfile +6 -0
- data/.devcontainer/Dockerfile +44 -0
- data/.devcontainer/devcontainer.json +36 -0
- data/.github/workflows/test.yml +67 -0
- data/CHANGELOG.md +22 -3
- data/README.md +86 -86
- data/bin/lstash +1 -3
- data/dip.yml +48 -0
- data/docker-compose.yml +28 -0
- data/lib/lstash/cli.rb +42 -29
- data/lib/lstash/client.rb +58 -37
- data/lib/lstash/query.rb +54 -77
- data/lib/lstash/version.rb +1 -1
- data/lib/lstash.rb +4 -4
- data/lstash.gemspec +14 -19
- data/spec/lstash/cli_spec.rb +21 -23
- data/spec/lstash/client_spec.rb +29 -33
- data/spec/lstash/query_spec.rb +62 -60
- data/spec/lstash_spec.rb +3 -3
- data/spec/spec_helper.rb +28 -13
- metadata +17 -58
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b595d887abdc37f07322ccb509d627d477b6e4897bc7768c512ba55f45ab4029
|
|
4
|
+
data.tar.gz: 814170a66956839f128b52a31bc56d99578d434a73e1194c0fa45ecd686f9b40
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 308cb925f02b6dc9ae7c67ca9ac3099615c72b82254e3d9dfd8d0fad33d2b1aa2ead526b35023cf4a20c9497960010ea84cadd50d12d8664813fd7a5f96bb06c
|
|
7
|
+
data.tar.gz: 6e3b63eb59aa2c45bd5b1de469dc79c84bfbed514f2ec4a6b4c12bf92475d1c37daae6a07ef900c6c286fe4ac5ef2e8c0efc8898f88efa11ce5c7d16e47dcd73
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
ARG RUBY_VERSION=2.4.6
|
|
2
|
+
ARG BUNDLER_VERSION=1.17.3
|
|
3
|
+
ARG DISTRO_NAME=buster
|
|
4
|
+
|
|
5
|
+
FROM ruby:$RUBY_VERSION-slim-$DISTRO_NAME
|
|
6
|
+
|
|
7
|
+
LABEL maintainer="k.j.wierenga@kerkdienstgemist.nl"
|
|
8
|
+
|
|
9
|
+
# Install dependencies specified in Aptfile
|
|
10
|
+
COPY Aptfile /tmp/
|
|
11
|
+
RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade \
|
|
12
|
+
&& mkdir -p /usr/share/man/man1 /usr/share/man/man7 \
|
|
13
|
+
&& DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
|
|
14
|
+
$(grep -Evh '^\s*#' /tmp/Aptfile /tmp/Aptfile.dev | xargs) \
|
|
15
|
+
&& apt-get autoremove -y \
|
|
16
|
+
&& apt-get clean \
|
|
17
|
+
&& rm -rf /var/cache/apt/archives/* \
|
|
18
|
+
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
|
|
19
|
+
&& truncate -s 0 /var/log/*log
|
|
20
|
+
|
|
21
|
+
# Configure bundler
|
|
22
|
+
# ENV LANG=C.UTF-8 \
|
|
23
|
+
# BUNDLE_JOBS=4 \
|
|
24
|
+
# BUNDLE_RETRY=3 \
|
|
25
|
+
# TZ=Europe/Amsterdam
|
|
26
|
+
|
|
27
|
+
# Store Bundler settings in the project's root
|
|
28
|
+
# ENV BUNDLE_APP_CONFIG=.bundle
|
|
29
|
+
|
|
30
|
+
# Uncomment this line if you want to run binstubs without prefixing with `bin/` or `bundle exec`
|
|
31
|
+
# ENV PATH /gem/bin:$PATH
|
|
32
|
+
|
|
33
|
+
# Upgrade RubyGems and install the latest Bundler version
|
|
34
|
+
ARG BUNDLER_VERSION
|
|
35
|
+
# gem update --system &&
|
|
36
|
+
RUN gem install bundler:$BUNDLER_VERSION
|
|
37
|
+
|
|
38
|
+
ENV TZ="Europe/Amsterdam"
|
|
39
|
+
|
|
40
|
+
# Configure the main working directory. This is the base
|
|
41
|
+
# directory used in any further RUN, COPY, and ENTRYPOINT commands.
|
|
42
|
+
ENV WORKDIR /gem
|
|
43
|
+
RUN mkdir -p $WORKDIR
|
|
44
|
+
WORKDIR $WORKDIR
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
|
|
2
|
+
// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/docker-existing-docker-compose
|
|
3
|
+
// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
|
|
4
|
+
{
|
|
5
|
+
"name": "lstash",
|
|
6
|
+
|
|
7
|
+
// Update the 'dockerComposeFile' list if you have more compose files or use different names.
|
|
8
|
+
// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
|
|
9
|
+
"dockerComposeFile": [
|
|
10
|
+
"../docker-compose.yml"
|
|
11
|
+
// "../docker-compose.override.yml"
|
|
12
|
+
],
|
|
13
|
+
|
|
14
|
+
// The 'service' property is the name of the service for the container that VS Code should
|
|
15
|
+
// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
|
|
16
|
+
"service": "runner",
|
|
17
|
+
|
|
18
|
+
// The optional 'workspaceFolder' property is the path VS Code should open by default when
|
|
19
|
+
// connected. This is typically a file mount in .devcontainer/docker-compose.yml
|
|
20
|
+
"workspaceFolder": "/gem"
|
|
21
|
+
|
|
22
|
+
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
|
23
|
+
// "forwardPorts": [],
|
|
24
|
+
|
|
25
|
+
// Uncomment the next line if you want start specific services in your Docker Compose config.
|
|
26
|
+
// "runServices": [],
|
|
27
|
+
|
|
28
|
+
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
|
|
29
|
+
// "shutdownAction": "none",
|
|
30
|
+
|
|
31
|
+
// Uncomment the next line to run commands after the container is created - for example installing curl.
|
|
32
|
+
// "postCreateCommand": "apt-get update && apt-get install -y curl",
|
|
33
|
+
|
|
34
|
+
// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
|
|
35
|
+
// "remoteUser": "vscode"
|
|
36
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: Run tests
|
|
2
|
+
on: [push]
|
|
3
|
+
|
|
4
|
+
jobs:
|
|
5
|
+
# This job uses buildx layer caching
|
|
6
|
+
# See https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching#the-cache-dance-off
|
|
7
|
+
test:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
env:
|
|
10
|
+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
|
11
|
+
steps:
|
|
12
|
+
- name: Checkout code
|
|
13
|
+
uses: actions/checkout@v2
|
|
14
|
+
|
|
15
|
+
- name: Set up Docker Buildx
|
|
16
|
+
id: buildx
|
|
17
|
+
uses: docker/setup-buildx-action@v2
|
|
18
|
+
with:
|
|
19
|
+
install: true # needed to ensure docker compose uses the build cache too
|
|
20
|
+
|
|
21
|
+
- name: Cache Docker layers
|
|
22
|
+
uses: actions/cache@v2
|
|
23
|
+
with:
|
|
24
|
+
path: /tmp/.buildx-cache
|
|
25
|
+
key: ${{ runner.os }}-single-buildx-${{ github.sha }}
|
|
26
|
+
restore-keys: |
|
|
27
|
+
${{ runner.os }}-single-buildx
|
|
28
|
+
|
|
29
|
+
- name: Build runner image
|
|
30
|
+
uses: docker/build-push-action@v2
|
|
31
|
+
with:
|
|
32
|
+
context: .devcontainer
|
|
33
|
+
builder: ${{ steps.buildx.outputs.name }}
|
|
34
|
+
push: false # This would be set to true in a real world deployment scenario.
|
|
35
|
+
load: true # Needed to ensure image is used in the "Run tests" step
|
|
36
|
+
tags: lstash_runner
|
|
37
|
+
cache-from: type=local,src=/tmp/.buildx-cache
|
|
38
|
+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
|
|
39
|
+
|
|
40
|
+
- name: Run tests
|
|
41
|
+
id: test
|
|
42
|
+
env:
|
|
43
|
+
DOCKER_BUILDKIT: 1
|
|
44
|
+
COMPOSE_DOCKER_CLI_BUILD: 1
|
|
45
|
+
run: |
|
|
46
|
+
docker compose run runner bundle install
|
|
47
|
+
docker compose run runner bundle exec rspec
|
|
48
|
+
|
|
49
|
+
# Temp fix
|
|
50
|
+
# https://github.com/docker/build-push-action/issues/252
|
|
51
|
+
# https://github.com/moby/buildkit/issues/1896
|
|
52
|
+
- name: Move cache
|
|
53
|
+
run: |
|
|
54
|
+
rm -rf /tmp/.buildx-cache
|
|
55
|
+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
|
|
56
|
+
|
|
57
|
+
# Notify via Slack when workflow is not successful
|
|
58
|
+
- uses: act10ns/slack@v1
|
|
59
|
+
with:
|
|
60
|
+
status: ${{ job.status }}
|
|
61
|
+
steps: ${{ toJson(steps) }}
|
|
62
|
+
if: always() # ${{ !success() }}
|
|
63
|
+
|
|
64
|
+
# cancel in progress workflows for the same github ref (e.g. branch)
|
|
65
|
+
concurrency:
|
|
66
|
+
group: ci-tests-${{ github.ref }}-1
|
|
67
|
+
cancel-in-progress: true
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,27 @@
|
|
|
1
|
+
## Release 1.0.0
|
|
2
|
+
|
|
3
|
+
* BREAKING CHANGE: default options changed
|
|
4
|
+
* Default --from today changed to --from yesterday
|
|
5
|
+
* Default --to now changed to --to today
|
|
6
|
+
* This ensures that by default lstash counts or greps in yesterdays logging.
|
|
7
|
+
* Fixed bug which caused empty range (e.g. --from today --to today) to incorrectly return non-zero count and logging.
|
|
8
|
+
* Upgrade elasticsearch gem from version ~> 0.4 to ~> 7.17.7.
|
|
9
|
+
* Update queries and field selectors to be compatible with Elasticsearch version 7.
|
|
10
|
+
* Increase scroll step size for grep from 2 minutes to 1 hour (current Elasticsearch can handle it).
|
|
11
|
+
* Add --wildcard / --no-wildcard option to use logstash-* wildcard instead of iterating over indices directly.
|
|
12
|
+
* For the count command --wildcard is faster so that's the default for count.
|
|
13
|
+
* For the grep command --no-wildcard is faster so that's the default for grep.
|
|
14
|
+
* Dockerize development and add GitHub action for testing.
|
|
15
|
+
* Moved repo from kdgm/lstash to kdgm/lstash.
|
|
16
|
+
* Rubocop fixes
|
|
17
|
+
|
|
1
18
|
## Release 0.2.0
|
|
19
|
+
|
|
2
20
|
Merge branch 'feature/fix/hashie-warnings' into develop
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
21
|
+
|
|
22
|
+
* [fix] pin faraday to a compatible version
|
|
23
|
+
* [enh] upgrade to ruby 2.4(.6)
|
|
24
|
+
* [fix] suppress Hashie warnings; it would generate a warning for each log line (on stdout) leading to very large output
|
|
6
25
|
|
|
7
26
|
### 0.1.4 / 2015-05-29
|
|
8
27
|
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# lstash
|
|
2
2
|
|
|
3
|
-
[](https://github.com/kdgm/lstash/actions/workflows/test.yml)
|
|
4
4
|
|
|
5
5
|
Lstash is a gem and command line utility to count or grep log messages in a certain time frame from a Logstash Elasticsearch server.
|
|
6
6
|
|
|
@@ -8,48 +8,48 @@ Lstash is a gem and command line utility to count or grep log messages in a cert
|
|
|
8
8
|
|
|
9
9
|
Or install it yourself as:
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
gem install lstash
|
|
12
12
|
|
|
13
13
|
## Running lstash from the command line
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
$ lstash
|
|
16
|
+
Commands:
|
|
17
|
+
lstash count QUERY # count number of log messages matching the QUERY
|
|
18
|
+
lstash grep QUERY # grep log messages from Logstash
|
|
19
|
+
lstash help [COMMAND] # Describe available commands or one specific command
|
|
20
20
|
|
|
21
21
|
## The `count` command
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
Usage:
|
|
24
|
+
lstash count QUERY
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
Description:
|
|
27
|
+
Count log messages matching the QUERY from Logstash and output this count to stdout. QUERY can use Apache Lucene query
|
|
28
|
+
parser syntax.
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
Example to count the number of HAProxy log messages in yesterdays month.
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
|
|
33
33
|
|
|
34
34
|
## The `grep` command
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
Usage:
|
|
37
|
+
lstash grep QUERY
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
Description:
|
|
40
|
+
Grep log messages matching the QUERY from Logstash in ascending timestamp order and output to stdout. QUERY can use Apache Lucene query parser syntax.
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
Example to grep HAProxy log messages from the beginning of this month upto now
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
lstash grep 'program:haproxy' --from firstday --to now
|
|
45
45
|
|
|
46
46
|
## Command line options
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
Options:
|
|
49
|
+
-f, [--from=start of time range] # date/time, 'now', 'today', 'yesterday', or 'firstday'
|
|
50
|
+
-t, [--to=end of time range] # date/time, 'now', 'today', 'yesterday', or 'firstday'
|
|
51
|
+
-a, [--anchor=anchor date/time] # used as reference date for firstday
|
|
52
|
+
-e, [--es-url=Elasticsearch endpoint for Logstash] # or ES_URL environment variable
|
|
53
53
|
|
|
54
54
|
All times will be relative to the timezone of the machine on which you are running lstash.
|
|
55
55
|
|
|
@@ -66,7 +66,7 @@ Example
|
|
|
66
66
|
|
|
67
67
|
Or
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
lstash count program:haproxy --es-url log.mydomain.com
|
|
70
70
|
|
|
71
71
|
## Examples
|
|
72
72
|
|
|
@@ -80,41 +80,41 @@ Grep all haproxy log messages using for one day (Aug 24 1 0:00 am upto and inclu
|
|
|
80
80
|
|
|
81
81
|
Assuming today is Sep 1 2014. Count all haproxy log messages in the previous month.
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
83
|
+
lstash count program:haproxy --anchor yesterday --from firstday --to today -d
|
|
84
|
+
time range: [2014-08-01 00:00:00 +0200..2014-09-01 00:00:00 +0200]
|
|
85
|
+
logstash-2014.07.31: 1
|
|
86
|
+
logstash-2014.08.01: 13
|
|
87
|
+
logstash-2014.08.02: 14
|
|
88
|
+
logstash-2014.08.03: 1654
|
|
89
|
+
logstash-2014.08.04: 6
|
|
90
|
+
logstash-2014.08.05: 20
|
|
91
|
+
logstash-2014.08.06: 219
|
|
92
|
+
logstash-2014.08.07: 32
|
|
93
|
+
logstash-2014.08.08: 14
|
|
94
|
+
logstash-2014.08.09: 28
|
|
95
|
+
logstash-2014.08.10: 799
|
|
96
|
+
logstash-2014.08.11: 18
|
|
97
|
+
logstash-2014.08.12: 8
|
|
98
|
+
logstash-2014.08.13: 23
|
|
99
|
+
logstash-2014.08.14: 25
|
|
100
|
+
logstash-2014.08.15: 69
|
|
101
|
+
logstash-2014.08.16: 19
|
|
102
|
+
logstash-2014.08.17: 1160
|
|
103
|
+
logstash-2014.08.18: 284
|
|
104
|
+
logstash-2014.08.19: 61
|
|
105
|
+
logstash-2014.08.20: 26
|
|
106
|
+
logstash-2014.08.21: 16
|
|
107
|
+
logstash-2014.08.22: 145
|
|
108
|
+
logstash-2014.08.23: 72
|
|
109
|
+
logstash-2014.08.24: 792
|
|
110
|
+
logstash-2014.08.25: 31
|
|
111
|
+
logstash-2014.08.26: 33
|
|
112
|
+
logstash-2014.08.27: 51
|
|
113
|
+
logstash-2014.08.28: 8
|
|
114
|
+
logstash-2014.08.29: 23
|
|
115
|
+
logstash-2014.08.30: 25
|
|
116
|
+
logstash-2014.08.31: 69
|
|
117
|
+
5633
|
|
118
118
|
|
|
119
119
|
## Using lstash as a gem in your project
|
|
120
120
|
|
|
@@ -124,46 +124,46 @@ Add this line to your application's Gemfile:
|
|
|
124
124
|
|
|
125
125
|
And then execute:
|
|
126
126
|
|
|
127
|
-
|
|
127
|
+
bundle
|
|
128
128
|
|
|
129
129
|
Usage:
|
|
130
130
|
|
|
131
|
-
|
|
131
|
+
bundle console
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
elasticsearch = Elasticsearch::Client.new(url: 'log.mydomain.com')
|
|
135
|
-
client = Lstash::Client.new(elasticsearch)
|
|
133
|
+
Connect to elasticsearch and create the Lstash client
|
|
136
134
|
|
|
137
|
-
|
|
138
|
-
|
|
135
|
+
elasticsearch = Elasticsearch::Client.new(url: ENV['ES_URL'])
|
|
136
|
+
client = Lstash::Client.new(elasticsearch, debug: true)
|
|
139
137
|
|
|
140
|
-
|
|
141
|
-
client.count(query)
|
|
138
|
+
Create the query
|
|
142
139
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
query = Lstash::Query.new('program:haproxy', from: 'today', to: 'now')
|
|
141
|
+
|
|
142
|
+
Count example
|
|
143
|
+
|
|
144
|
+
client.count(query)
|
|
145
|
+
|
|
146
|
+
Grep example
|
|
147
|
+
|
|
148
|
+
client.grep(query) do |message|
|
|
149
|
+
puts message
|
|
150
|
+
end
|
|
147
151
|
|
|
148
152
|
## Publishing the gem to RubyGems.org
|
|
149
153
|
|
|
150
154
|
1. Build the gem
|
|
151
155
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
File: lstash-0.2.0.gem
|
|
158
|
-
```
|
|
156
|
+
$ gem build lstash
|
|
157
|
+
Successfully built RubyGem
|
|
158
|
+
Name: lstash
|
|
159
|
+
Version: 0.2.0
|
|
160
|
+
File: lstash-0.2.0.gem
|
|
159
161
|
|
|
160
162
|
2. Pushing your gem to RubyGems.org
|
|
161
163
|
|
|
162
|
-
|
|
163
|
-
gem
|
|
164
|
-
|
|
165
|
-
Successfully registered gem: lstash (0.2.0)
|
|
166
|
-
```
|
|
164
|
+
$ gem push lstash-0.2.0.gem
|
|
165
|
+
Pushing gem to RubyGems.org...
|
|
166
|
+
Successfully registered gem: lstash (0.2.0)
|
|
167
167
|
|
|
168
168
|
See [RubyGems.org documention](https://guides.rubygems.org/) for more info.
|
|
169
169
|
|
data/bin/lstash
CHANGED
data/dip.yml
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Required minimum dip version
|
|
2
|
+
version: '6.1'
|
|
3
|
+
|
|
4
|
+
compose:
|
|
5
|
+
files:
|
|
6
|
+
- docker-compose.yml
|
|
7
|
+
- docker-compose.override.yml # platform specific overrides
|
|
8
|
+
|
|
9
|
+
# Specify project name explicitly to avoid name collisions:
|
|
10
|
+
# docker-compose uses the compose file's folder name as the project name by default,
|
|
11
|
+
# which could be the same for different projects (e.g., if you store docker-compose.yml
|
|
12
|
+
# in the .devcontainer/ folder)
|
|
13
|
+
|
|
14
|
+
# NOTE: We deliberately do not set project_name to prevent collissions
|
|
15
|
+
# between git work trees.
|
|
16
|
+
# Let the name be determined automatically from the base directory name.
|
|
17
|
+
# project_name: UNSET DELIBERATELY
|
|
18
|
+
|
|
19
|
+
interaction:
|
|
20
|
+
sh:
|
|
21
|
+
description: Start a Bash shell in the container
|
|
22
|
+
service: runner
|
|
23
|
+
command: /bin/bash
|
|
24
|
+
|
|
25
|
+
bundle:
|
|
26
|
+
description: Run bundler commands
|
|
27
|
+
service: runner
|
|
28
|
+
command: bundle
|
|
29
|
+
|
|
30
|
+
rake:
|
|
31
|
+
description: Run rake commands
|
|
32
|
+
service: runner
|
|
33
|
+
command: bundle exec rake
|
|
34
|
+
|
|
35
|
+
rspec:
|
|
36
|
+
description: Run specs
|
|
37
|
+
service: runner
|
|
38
|
+
command: bundle exec rspec
|
|
39
|
+
|
|
40
|
+
provision:
|
|
41
|
+
# Remove old containers and volumes.
|
|
42
|
+
- dip compose down --volumes --remove-orphans
|
|
43
|
+
|
|
44
|
+
# Build the development container (based on .devcontainer/Dockerfile)
|
|
45
|
+
- dip compose build
|
|
46
|
+
|
|
47
|
+
# Install gem dependencies
|
|
48
|
+
- dip bundle install
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
x-base: &base
|
|
2
|
+
# NOTE: We deliberately do not set the image name to prevent
|
|
3
|
+
# collissions between images built in different git work trees (directories)
|
|
4
|
+
# image: LEFT UNSET DELIBERATELY
|
|
5
|
+
build:
|
|
6
|
+
context: .devcontainer
|
|
7
|
+
tmpfs:
|
|
8
|
+
- /tmp
|
|
9
|
+
volumes:
|
|
10
|
+
- .:/gem:cached
|
|
11
|
+
- bundle:/usr/local/bundle
|
|
12
|
+
- history:/usr/local/hist
|
|
13
|
+
stdin_open: true
|
|
14
|
+
tty: true
|
|
15
|
+
environment:
|
|
16
|
+
HISTFILE: /usr/local/hist/.bash_history
|
|
17
|
+
IRB_HISTFILE: /usr/local/hist/.irb_history
|
|
18
|
+
EDITOR: ${EDITOR:-vi}
|
|
19
|
+
ES_URL: http://host.docker.internal:9200
|
|
20
|
+
|
|
21
|
+
services:
|
|
22
|
+
runner:
|
|
23
|
+
<<: *base
|
|
24
|
+
command: /bin/bash
|
|
25
|
+
|
|
26
|
+
volumes:
|
|
27
|
+
bundle:
|
|
28
|
+
history:
|
data/lib/lstash/cli.rb
CHANGED
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
# external dependencies
|
|
2
|
-
require
|
|
3
|
-
require
|
|
4
|
-
require
|
|
2
|
+
require "thor"
|
|
3
|
+
require "uri"
|
|
4
|
+
require "elasticsearch"
|
|
5
5
|
|
|
6
6
|
# local files we need
|
|
7
|
-
require
|
|
8
|
-
require
|
|
7
|
+
require "lstash/query"
|
|
8
|
+
require "lstash/client"
|
|
9
|
+
require "lstash/version"
|
|
9
10
|
|
|
10
11
|
module Lstash
|
|
12
|
+
TRANSPORT_REQUEST_TIMEOUT = 120 # 2 minute request timeout
|
|
13
|
+
|
|
14
|
+
class CLIBase < Thor
|
|
15
|
+
class << self
|
|
16
|
+
def shared_options
|
|
17
|
+
method_option :anchor, banner: "YYYY-mm-dd", aliases: "-a", desc: "The 'firstday' is relative to this anchor date", default: "today"
|
|
18
|
+
method_option :from, banner: "YYYY-mm-dd [HH:MM:SS]", aliases: "-f", desc: "Start date/time, 'now', 'today', 'yesterday', or 'firstday'", default: "yesterday"
|
|
19
|
+
method_option :to, banner: "YYYY-mm-dd [HH:MM:SS]", aliases: "-t", desc: "End date/time, 'now', 'today', 'yesterday', or 'firstday'", default: "today"
|
|
20
|
+
method_option :es_url, banner: "http://localhost:9200", aliases: "-e", desc: "Elasticsearch URL or set ES_URL environment variable"
|
|
21
|
+
method_option :debug, desc: "Log debugging info to stderr", aliases: "-d", type: :boolean, default: false
|
|
22
|
+
method_option :wildcard, desc: "Use index wildcard to query all logstash-* indices (fast for count, slow for grep)", type: :boolean
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
11
26
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class CLI < Thor
|
|
15
|
-
|
|
16
|
-
class_option :from, :banner => 'start of time range', :aliases => '-f', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
|
|
17
|
-
class_option :to, :banner => 'end of time range', :aliases => '-t', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
|
|
18
|
-
class_option :anchor, :banner => 'anchor date/time', :aliases => '-a', :desc => "used as reference date for firstday"
|
|
19
|
-
class_option :es_url, :banner => 'Elasticsearch endpoint for Logstash', :aliases => '-e', :desc => "or ES_URL environment variable"
|
|
20
|
-
class_option :debug, :banner => 'debug log to stderr', :aliases => '-d', :type => :boolean
|
|
21
|
-
|
|
27
|
+
class CLI < CLIBase
|
|
22
28
|
long_desc <<-LONGDESC
|
|
23
29
|
Grep log messages matching the QUERY from Logstash in ascending timestamp order
|
|
24
30
|
and output to stdout. QUERY can use Apache Lucene query parser syntax.
|
|
@@ -27,7 +33,8 @@ module Lstash
|
|
|
27
33
|
|
|
28
34
|
lstash grep 'program:haproxy' --from firstday --to now
|
|
29
35
|
LONGDESC
|
|
30
|
-
desc "grep QUERY", "
|
|
36
|
+
desc "grep QUERY", "Grep log messages from Logstash"
|
|
37
|
+
shared_options
|
|
31
38
|
def grep(query_string)
|
|
32
39
|
run_command(query_string) do |es_client, query|
|
|
33
40
|
Lstash::Client.new(es_client, options).grep(query) do |message|
|
|
@@ -44,7 +51,8 @@ module Lstash
|
|
|
44
51
|
|
|
45
52
|
lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
|
|
46
53
|
LONGDESC
|
|
47
|
-
desc "count QUERY", "
|
|
54
|
+
desc "count QUERY", "Count number of log messages matching the QUERY"
|
|
55
|
+
shared_options
|
|
48
56
|
def count(query_string)
|
|
49
57
|
run_command(query_string) do |es_client, query|
|
|
50
58
|
count = Lstash::Client.new(es_client, options).count(query)
|
|
@@ -52,29 +60,34 @@ module Lstash
|
|
|
52
60
|
end
|
|
53
61
|
end
|
|
54
62
|
|
|
63
|
+
long_desc "Print the lstash version"
|
|
64
|
+
desc "version", "print lstash version"
|
|
65
|
+
def version
|
|
66
|
+
puts Lstash::VERSION
|
|
67
|
+
end
|
|
68
|
+
|
|
55
69
|
private
|
|
56
70
|
|
|
57
71
|
def run_command(query_string)
|
|
58
72
|
es_client = ::Elasticsearch::Client.new(
|
|
59
|
-
url: options[:es_url] || ENV[
|
|
60
|
-
log:
|
|
61
|
-
transport_options: {
|
|
73
|
+
url: options[:es_url] || ENV["ES_URL"] || "http://localhost:9200",
|
|
74
|
+
log: ENV["DEBUG"] == "true",
|
|
75
|
+
transport_options: {request: {timeout: TRANSPORT_REQUEST_TIMEOUT}}
|
|
62
76
|
)
|
|
63
|
-
query
|
|
77
|
+
query = Lstash::Query.new(query_string, options)
|
|
64
78
|
|
|
65
79
|
yield es_client, query
|
|
66
|
-
|
|
67
|
-
rescue Exception => e
|
|
80
|
+
rescue => e
|
|
68
81
|
options[:debug] ? raise(e) : raise(Thor::Error.new(e.message))
|
|
69
82
|
end
|
|
70
83
|
|
|
71
|
-
protected
|
|
72
|
-
|
|
73
84
|
# Make sure we exit on failure with an error code
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
end
|
|
85
|
+
class << self
|
|
86
|
+
protected
|
|
77
87
|
|
|
88
|
+
def exit_on_failure?
|
|
89
|
+
true
|
|
90
|
+
end
|
|
91
|
+
end
|
|
78
92
|
end
|
|
79
|
-
|
|
80
93
|
end
|