logcli 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/Gemfile.lock +15 -2
- data/README.md +19 -20
- data/lib/cli.rb +12 -1
- data/lib/logcli.rb +2 -0
- data/lib/logcli/actions/elasticsearch.rb +63 -0
- data/lib/logcli/params/elasticsearch.rb +22 -0
- data/lib/logcli/version.rb +1 -1
- data/logcli.gemspec +1 -0
- metadata +27 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ac147cb720b80c5f8a5bcd351700b9973f839491ab66908d023824dce30b18b1
|
4
|
+
data.tar.gz: 86e4319fa339699d9de38cbe3c8f5640753a984897a8fd2e43b4b2814fca9b8b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ddee66f38f61ff9c9b28207767ab15f9e39e8284a81d9e46c4514bf751fa5b36fdde7362f31362d2af57886f4537e7d77474a7ec39153fbff09f81662530ebd
|
7
|
+
data.tar.gz: e7a4ee7a860eed6532f671d6a260f4050b97d07168a2432a23614788da31c4c9f5b07b5df59b6e67b843ff071373a940b0c43bee3a632938b1a6586cd19a52e6
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
logcli (0.1.
|
4
|
+
logcli (0.1.1)
|
5
|
+
elasticsearch (~> 7.0)
|
5
6
|
net-scp (~> 1.2.1, >= 1.2.1)
|
6
7
|
net-ssh (~> 5.0.2, >= 5.0.2)
|
7
8
|
thor (~> 0.20.3)
|
@@ -11,7 +12,19 @@ GEM
|
|
11
12
|
specs:
|
12
13
|
coderay (1.1.2)
|
13
14
|
diff-lcs (1.3)
|
15
|
+
elasticsearch (7.0.0)
|
16
|
+
elasticsearch-api (= 7.0.0)
|
17
|
+
elasticsearch-transport (= 7.0.0)
|
18
|
+
elasticsearch-api (7.0.0)
|
19
|
+
multi_json
|
20
|
+
elasticsearch-transport (7.0.0)
|
21
|
+
faraday
|
22
|
+
multi_json
|
23
|
+
faraday (0.15.4)
|
24
|
+
multipart-post (>= 1.2, < 3)
|
14
25
|
method_source (0.9.0)
|
26
|
+
multi_json (1.13.1)
|
27
|
+
multipart-post (2.1.0)
|
15
28
|
net-scp (1.2.1)
|
16
29
|
net-ssh (>= 2.6.5)
|
17
30
|
net-ssh (5.0.2)
|
@@ -45,4 +58,4 @@ DEPENDENCIES
|
|
45
58
|
rspec (~> 3.0)
|
46
59
|
|
47
60
|
BUNDLED WITH
|
48
|
-
1.16.
|
61
|
+
1.16.6
|
data/README.md
CHANGED
@@ -1,34 +1,33 @@
|
|
1
1
|
# Logcli
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
logcli is tool for easy working with logs but its opionated my work flow to working with logs looks like
|
4
|
+
fetch json logs from lines and than push it to elasticsearch for future analytics my logs stored
|
5
|
+
as JSON objects. Example how my logs look like
|
6
|
+
```text
|
7
|
+
I, [2019-05-09T17:00:16.049805 #57671] INFO -- : {"evt_type":"FileUpload":"info","time":"2019-05-09T15:00:16+0000","payload":{"time_elapsed_human":"00:00:00.874","time_elapsed":0.874553},"trace_id":"cd5916a6-14b7-4628-bdda-334e4285a4e0"}
|
8
|
+
```
|
9
|
+
as you can see there a log prefixer like `I, [2019-05-09T17:00:16.049805 #57671] INFO -- : `
|
10
|
+
that's why its not a JSON but than goes a JSON object that I want to extract
|
11
|
+
I'm using this command to extract JSON from log line
|
12
|
+
```bash
|
13
|
+
logcli extract_json --filenames=log1.log log2.log
|
13
14
|
```
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
$ bundle
|
16
|
+
as result tool create in the SAME directory files with names `log1_json.log, log2_json.log`
|
18
17
|
|
19
|
-
|
18
|
+
than I'm push data to ES for analytics using ES + Kibana
|
20
19
|
|
21
|
-
|
20
|
+
```bash
|
21
|
+
logcli elasticsearch --filenames=events_json.log --elasticsearch_url=http://localhost:9201
|
22
|
+
```
|
22
23
|
|
23
|
-
|
24
|
+
tool using batch API and push 100 records and the time
|
24
25
|
|
25
|
-
|
26
|
+
Pretty dummy stuff hah
|
26
27
|
|
27
28
|
## Development
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
30
|
+
In active - passive development
|
32
31
|
|
33
32
|
## Contributing
|
34
33
|
|
data/lib/cli.rb
CHANGED
@@ -29,5 +29,16 @@ module Logcli
|
|
29
29
|
json = Logcli::Actions::ExtractJson.new(params.parse_params)
|
30
30
|
json.call
|
31
31
|
end
|
32
|
+
|
33
|
+
desc "elasticsearch", "push json file to elasticsearch instance"
|
34
|
+
option :filenames, type: :array, required: true, banner: 'example1.json example2.json'
|
35
|
+
option :elasticsearch_url, type: :string, required: true, banner: 'http://localhost:9200'
|
36
|
+
option :mapping_file, type: :string, required: false, banner: 'example-mapping.json'
|
37
|
+
|
38
|
+
def elasticsearch
|
39
|
+
params = Logcli::Params::Elasticsearch.new options
|
40
|
+
json = Logcli::Actions::Elasticsearch.new(params.parse_params)
|
41
|
+
json.call
|
42
|
+
end
|
32
43
|
end
|
33
|
-
end
|
44
|
+
end
|
data/lib/logcli.rb
CHANGED
@@ -2,12 +2,14 @@ require "logcli/version"
|
|
2
2
|
require "logcli/version"
|
3
3
|
require "logcli/params/session"
|
4
4
|
require "logcli/params/json"
|
5
|
+
require "logcli/params/elasticsearch"
|
5
6
|
require "logcli/ssh"
|
6
7
|
require "logcli/scp"
|
7
8
|
require "logcli/helper"
|
8
9
|
require "logcli/actions/download"
|
9
10
|
require "logcli/actions/grep"
|
10
11
|
require "logcli/actions/extract_json"
|
12
|
+
require "logcli/actions/elasticsearch"
|
11
13
|
require 'cli'
|
12
14
|
require 'net/ssh'
|
13
15
|
require 'net/scp'
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'elasticsearch'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Logcli
|
5
|
+
module Actions
|
6
|
+
class Elasticsearch
|
7
|
+
attr_reader :filenames, :elasticsearch_url, :mapping_file, :buffer, :id, :total_records
|
8
|
+
BULK_SIZE = 100
|
9
|
+
INDEX_NAME = 'log-index'.freeze
|
10
|
+
|
11
|
+
def initialize opts
|
12
|
+
@filenames = opts.fetch(:filenames)
|
13
|
+
@mapping_file = opts.fetch(:mapping_file)
|
14
|
+
@elasticsearch_url = opts.fetch(:elasticsearch_url)
|
15
|
+
@buffer = []
|
16
|
+
@id = 1
|
17
|
+
@total_records = 0
|
18
|
+
end
|
19
|
+
|
20
|
+
def call
|
21
|
+
create_mapping if mapping_file
|
22
|
+
|
23
|
+
filenames.each do |filename|
|
24
|
+
File.open(filename).each do |line|
|
25
|
+
@buffer << line
|
26
|
+
if @buffer.count > BULK_SIZE
|
27
|
+
flush
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
if @buffer.count > 0
|
32
|
+
flush
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def flush
|
39
|
+
payload = @buffer.flat_map do |item|
|
40
|
+
hash = JSON.parse(item)
|
41
|
+
@id += 1
|
42
|
+
@total_records += 1
|
43
|
+
[
|
44
|
+
{ index: { _index: INDEX_NAME, _id: @id } },
|
45
|
+
hash
|
46
|
+
]
|
47
|
+
end
|
48
|
+
client.bulk body: payload
|
49
|
+
puts "records uploaded #{@total_records}"
|
50
|
+
@buffer = []
|
51
|
+
end
|
52
|
+
|
53
|
+
def create_mapping
|
54
|
+
payload = JSON.parse(File.read(mapping_file))
|
55
|
+
client.indices.create index: INDEX_NAME, body: payload
|
56
|
+
end
|
57
|
+
|
58
|
+
def client
|
59
|
+
@client ||= ::Elasticsearch::Client.new url: elasticsearch_url, log: true
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Logcli
|
2
|
+
module Params
|
3
|
+
class Elasticsearch
|
4
|
+
FILENAMES = :filenames
|
5
|
+
MAPPING_FILE = :mapping_file
|
6
|
+
ELASTICSEARCH_URL = :elasticsearch_url
|
7
|
+
OPTS = [FILENAMES, MAPPING_FILE, ELASTICSEARCH_URL].freeze
|
8
|
+
attr_accessor(*OPTS)
|
9
|
+
|
10
|
+
def initialize opts
|
11
|
+
OPTS.each do |opt|
|
12
|
+
val = opts.fetch opt, nil
|
13
|
+
instance_variable_set "@#{opt}", val
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse_params
|
18
|
+
{ filenames: filenames, mapping_file: mapping_file, elasticsearch_url: elasticsearch_url }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/logcli/version.rb
CHANGED
data/logcli.gemspec
CHANGED
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
30
|
+
spec.add_runtime_dependency 'elasticsearch', '~> 7.0'
|
30
31
|
spec.add_runtime_dependency 'net-ssh', '~> 5.0.2', '>= 5.0.2'
|
31
32
|
spec.add_runtime_dependency 'net-scp', '~> 1.2.1', '>= 1.2.1'
|
32
33
|
spec.add_runtime_dependency 'thor', '~> 0.20.3'
|
metadata
CHANGED
@@ -1,53 +1,67 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logcli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- gingray
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-05-
|
11
|
+
date: 2019-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: elasticsearch
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '7.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '7.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: net-ssh
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
20
31
|
- - ">="
|
21
32
|
- !ruby/object:Gem::Version
|
22
33
|
version: 5.0.2
|
34
|
+
- - "~>"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 5.0.2
|
23
37
|
type: :runtime
|
24
38
|
prerelease: false
|
25
39
|
version_requirements: !ruby/object:Gem::Requirement
|
26
40
|
requirements:
|
27
|
-
- - "
|
41
|
+
- - ">="
|
28
42
|
- !ruby/object:Gem::Version
|
29
43
|
version: 5.0.2
|
30
|
-
- - "
|
44
|
+
- - "~>"
|
31
45
|
- !ruby/object:Gem::Version
|
32
46
|
version: 5.0.2
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: net-scp
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
36
50
|
requirements:
|
37
|
-
- - "
|
51
|
+
- - ">="
|
38
52
|
- !ruby/object:Gem::Version
|
39
53
|
version: 1.2.1
|
40
|
-
- - "
|
54
|
+
- - "~>"
|
41
55
|
- !ruby/object:Gem::Version
|
42
56
|
version: 1.2.1
|
43
57
|
type: :runtime
|
44
58
|
prerelease: false
|
45
59
|
version_requirements: !ruby/object:Gem::Requirement
|
46
60
|
requirements:
|
47
|
-
- - "
|
61
|
+
- - ">="
|
48
62
|
- !ruby/object:Gem::Version
|
49
63
|
version: 1.2.1
|
50
|
-
- - "
|
64
|
+
- - "~>"
|
51
65
|
- !ruby/object:Gem::Version
|
52
66
|
version: 1.2.1
|
53
67
|
- !ruby/object:Gem::Dependency
|
@@ -129,9 +143,11 @@ files:
|
|
129
143
|
- lib/cli.rb
|
130
144
|
- lib/logcli.rb
|
131
145
|
- lib/logcli/actions/download.rb
|
146
|
+
- lib/logcli/actions/elasticsearch.rb
|
132
147
|
- lib/logcli/actions/extract_json.rb
|
133
148
|
- lib/logcli/actions/grep.rb
|
134
149
|
- lib/logcli/helper.rb
|
150
|
+
- lib/logcli/params/elasticsearch.rb
|
135
151
|
- lib/logcli/params/json.rb
|
136
152
|
- lib/logcli/params/session.rb
|
137
153
|
- lib/logcli/scp.rb
|
@@ -159,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
175
|
version: '0'
|
160
176
|
requirements: []
|
161
177
|
rubyforge_project:
|
162
|
-
rubygems_version: 2.
|
178
|
+
rubygems_version: 2.7.8
|
163
179
|
signing_key:
|
164
180
|
specification_version: 4
|
165
181
|
summary: log fetcher and remote retriveve files
|