logcli 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/Gemfile.lock +15 -2
- data/README.md +19 -20
- data/lib/cli.rb +12 -1
- data/lib/logcli.rb +2 -0
- data/lib/logcli/actions/elasticsearch.rb +63 -0
- data/lib/logcli/params/elasticsearch.rb +22 -0
- data/lib/logcli/version.rb +1 -1
- data/logcli.gemspec +1 -0
- metadata +27 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ac147cb720b80c5f8a5bcd351700b9973f839491ab66908d023824dce30b18b1
|
4
|
+
data.tar.gz: 86e4319fa339699d9de38cbe3c8f5640753a984897a8fd2e43b4b2814fca9b8b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ddee66f38f61ff9c9b28207767ab15f9e39e8284a81d9e46c4514bf751fa5b36fdde7362f31362d2af57886f4537e7d77474a7ec39153fbff09f81662530ebd
|
7
|
+
data.tar.gz: e7a4ee7a860eed6532f671d6a260f4050b97d07168a2432a23614788da31c4c9f5b07b5df59b6e67b843ff071373a940b0c43bee3a632938b1a6586cd19a52e6
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
logcli (0.1.
|
4
|
+
logcli (0.1.1)
|
5
|
+
elasticsearch (~> 7.0)
|
5
6
|
net-scp (~> 1.2.1, >= 1.2.1)
|
6
7
|
net-ssh (~> 5.0.2, >= 5.0.2)
|
7
8
|
thor (~> 0.20.3)
|
@@ -11,7 +12,19 @@ GEM
|
|
11
12
|
specs:
|
12
13
|
coderay (1.1.2)
|
13
14
|
diff-lcs (1.3)
|
15
|
+
elasticsearch (7.0.0)
|
16
|
+
elasticsearch-api (= 7.0.0)
|
17
|
+
elasticsearch-transport (= 7.0.0)
|
18
|
+
elasticsearch-api (7.0.0)
|
19
|
+
multi_json
|
20
|
+
elasticsearch-transport (7.0.0)
|
21
|
+
faraday
|
22
|
+
multi_json
|
23
|
+
faraday (0.15.4)
|
24
|
+
multipart-post (>= 1.2, < 3)
|
14
25
|
method_source (0.9.0)
|
26
|
+
multi_json (1.13.1)
|
27
|
+
multipart-post (2.1.0)
|
15
28
|
net-scp (1.2.1)
|
16
29
|
net-ssh (>= 2.6.5)
|
17
30
|
net-ssh (5.0.2)
|
@@ -45,4 +58,4 @@ DEPENDENCIES
|
|
45
58
|
rspec (~> 3.0)
|
46
59
|
|
47
60
|
BUNDLED WITH
|
48
|
-
1.16.
|
61
|
+
1.16.6
|
data/README.md
CHANGED
@@ -1,34 +1,33 @@
|
|
1
1
|
# Logcli
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
logcli is tool for easy working with logs but its opionated my work flow to working with logs looks like
|
4
|
+
fetch json logs from lines and than push it to elasticsearch for future analytics my logs stored
|
5
|
+
as JSON objects. Example how my logs look like
|
6
|
+
```text
|
7
|
+
I, [2019-05-09T17:00:16.049805 #57671] INFO -- : {"evt_type":"FileUpload":"info","time":"2019-05-09T15:00:16+0000","payload":{"time_elapsed_human":"00:00:00.874","time_elapsed":0.874553},"trace_id":"cd5916a6-14b7-4628-bdda-334e4285a4e0"}
|
8
|
+
```
|
9
|
+
as you can see there a log prefixer like `I, [2019-05-09T17:00:16.049805 #57671] INFO -- : `
|
10
|
+
that's why its not a JSON but than goes a JSON object that I want to extract
|
11
|
+
I'm using this command to extract JSON from log line
|
12
|
+
```bash
|
13
|
+
logcli extract_json --filenames=log1.log log2.log
|
13
14
|
```
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
$ bundle
|
16
|
+
as result tool create in the SAME directory files with names `log1_json.log, log2_json.log`
|
18
17
|
|
19
|
-
|
18
|
+
than I'm push data to ES for analytics using ES + Kibana
|
20
19
|
|
21
|
-
|
20
|
+
```bash
|
21
|
+
logcli elasticsearch --filenames=events_json.log --elasticsearch_url=http://localhost:9201
|
22
|
+
```
|
22
23
|
|
23
|
-
|
24
|
+
tool using batch API and push 100 records and the time
|
24
25
|
|
25
|
-
|
26
|
+
Pretty dummy stuff hah
|
26
27
|
|
27
28
|
## Development
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
30
|
+
In active - passive development
|
32
31
|
|
33
32
|
## Contributing
|
34
33
|
|
data/lib/cli.rb
CHANGED
@@ -29,5 +29,16 @@ module Logcli
|
|
29
29
|
json = Logcli::Actions::ExtractJson.new(params.parse_params)
|
30
30
|
json.call
|
31
31
|
end
|
32
|
+
|
33
|
+
desc "elasticsearch", "push json file to elasticsearch instance"
|
34
|
+
option :filenames, type: :array, required: true, banner: 'example1.json example2.json'
|
35
|
+
option :elasticsearch_url, type: :string, required: true, banner: 'http://localhost:9200'
|
36
|
+
option :mapping_file, type: :string, required: false, banner: 'example-mapping.json'
|
37
|
+
|
38
|
+
def elasticsearch
|
39
|
+
params = Logcli::Params::Elasticsearch.new options
|
40
|
+
json = Logcli::Actions::Elasticsearch.new(params.parse_params)
|
41
|
+
json.call
|
42
|
+
end
|
32
43
|
end
|
33
|
-
end
|
44
|
+
end
|
data/lib/logcli.rb
CHANGED
@@ -2,12 +2,14 @@ require "logcli/version"
|
|
2
2
|
require "logcli/version"
|
3
3
|
require "logcli/params/session"
|
4
4
|
require "logcli/params/json"
|
5
|
+
require "logcli/params/elasticsearch"
|
5
6
|
require "logcli/ssh"
|
6
7
|
require "logcli/scp"
|
7
8
|
require "logcli/helper"
|
8
9
|
require "logcli/actions/download"
|
9
10
|
require "logcli/actions/grep"
|
10
11
|
require "logcli/actions/extract_json"
|
12
|
+
require "logcli/actions/elasticsearch"
|
11
13
|
require 'cli'
|
12
14
|
require 'net/ssh'
|
13
15
|
require 'net/scp'
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'elasticsearch'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Logcli
|
5
|
+
module Actions
|
6
|
+
class Elasticsearch
|
7
|
+
attr_reader :filenames, :elasticsearch_url, :mapping_file, :buffer, :id, :total_records
|
8
|
+
BULK_SIZE = 100
|
9
|
+
INDEX_NAME = 'log-index'.freeze
|
10
|
+
|
11
|
+
def initialize opts
|
12
|
+
@filenames = opts.fetch(:filenames)
|
13
|
+
@mapping_file = opts.fetch(:mapping_file)
|
14
|
+
@elasticsearch_url = opts.fetch(:elasticsearch_url)
|
15
|
+
@buffer = []
|
16
|
+
@id = 1
|
17
|
+
@total_records = 0
|
18
|
+
end
|
19
|
+
|
20
|
+
def call
|
21
|
+
create_mapping if mapping_file
|
22
|
+
|
23
|
+
filenames.each do |filename|
|
24
|
+
File.open(filename).each do |line|
|
25
|
+
@buffer << line
|
26
|
+
if @buffer.count > BULK_SIZE
|
27
|
+
flush
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
if @buffer.count > 0
|
32
|
+
flush
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def flush
|
39
|
+
payload = @buffer.flat_map do |item|
|
40
|
+
hash = JSON.parse(item)
|
41
|
+
@id += 1
|
42
|
+
@total_records += 1
|
43
|
+
[
|
44
|
+
{ index: { _index: INDEX_NAME, _id: @id } },
|
45
|
+
hash
|
46
|
+
]
|
47
|
+
end
|
48
|
+
client.bulk body: payload
|
49
|
+
puts "records uploaded #{@total_records}"
|
50
|
+
@buffer = []
|
51
|
+
end
|
52
|
+
|
53
|
+
def create_mapping
|
54
|
+
payload = JSON.parse(File.read(mapping_file))
|
55
|
+
client.indices.create index: INDEX_NAME, body: payload
|
56
|
+
end
|
57
|
+
|
58
|
+
def client
|
59
|
+
@client ||= ::Elasticsearch::Client.new url: elasticsearch_url, log: true
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Logcli
|
2
|
+
module Params
|
3
|
+
class Elasticsearch
|
4
|
+
FILENAMES = :filenames
|
5
|
+
MAPPING_FILE = :mapping_file
|
6
|
+
ELASTICSEARCH_URL = :elasticsearch_url
|
7
|
+
OPTS = [FILENAMES, MAPPING_FILE, ELASTICSEARCH_URL].freeze
|
8
|
+
attr_accessor(*OPTS)
|
9
|
+
|
10
|
+
def initialize opts
|
11
|
+
OPTS.each do |opt|
|
12
|
+
val = opts.fetch opt, nil
|
13
|
+
instance_variable_set "@#{opt}", val
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse_params
|
18
|
+
{ filenames: filenames, mapping_file: mapping_file, elasticsearch_url: elasticsearch_url }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/logcli/version.rb
CHANGED
data/logcli.gemspec
CHANGED
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
30
|
+
spec.add_runtime_dependency 'elasticsearch', '~> 7.0'
|
30
31
|
spec.add_runtime_dependency 'net-ssh', '~> 5.0.2', '>= 5.0.2'
|
31
32
|
spec.add_runtime_dependency 'net-scp', '~> 1.2.1', '>= 1.2.1'
|
32
33
|
spec.add_runtime_dependency 'thor', '~> 0.20.3'
|
metadata
CHANGED
@@ -1,53 +1,67 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logcli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- gingray
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-05-
|
11
|
+
date: 2019-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: elasticsearch
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '7.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '7.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: net-ssh
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
20
31
|
- - ">="
|
21
32
|
- !ruby/object:Gem::Version
|
22
33
|
version: 5.0.2
|
34
|
+
- - "~>"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 5.0.2
|
23
37
|
type: :runtime
|
24
38
|
prerelease: false
|
25
39
|
version_requirements: !ruby/object:Gem::Requirement
|
26
40
|
requirements:
|
27
|
-
- - "
|
41
|
+
- - ">="
|
28
42
|
- !ruby/object:Gem::Version
|
29
43
|
version: 5.0.2
|
30
|
-
- - "
|
44
|
+
- - "~>"
|
31
45
|
- !ruby/object:Gem::Version
|
32
46
|
version: 5.0.2
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: net-scp
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
36
50
|
requirements:
|
37
|
-
- - "
|
51
|
+
- - ">="
|
38
52
|
- !ruby/object:Gem::Version
|
39
53
|
version: 1.2.1
|
40
|
-
- - "
|
54
|
+
- - "~>"
|
41
55
|
- !ruby/object:Gem::Version
|
42
56
|
version: 1.2.1
|
43
57
|
type: :runtime
|
44
58
|
prerelease: false
|
45
59
|
version_requirements: !ruby/object:Gem::Requirement
|
46
60
|
requirements:
|
47
|
-
- - "
|
61
|
+
- - ">="
|
48
62
|
- !ruby/object:Gem::Version
|
49
63
|
version: 1.2.1
|
50
|
-
- - "
|
64
|
+
- - "~>"
|
51
65
|
- !ruby/object:Gem::Version
|
52
66
|
version: 1.2.1
|
53
67
|
- !ruby/object:Gem::Dependency
|
@@ -129,9 +143,11 @@ files:
|
|
129
143
|
- lib/cli.rb
|
130
144
|
- lib/logcli.rb
|
131
145
|
- lib/logcli/actions/download.rb
|
146
|
+
- lib/logcli/actions/elasticsearch.rb
|
132
147
|
- lib/logcli/actions/extract_json.rb
|
133
148
|
- lib/logcli/actions/grep.rb
|
134
149
|
- lib/logcli/helper.rb
|
150
|
+
- lib/logcli/params/elasticsearch.rb
|
135
151
|
- lib/logcli/params/json.rb
|
136
152
|
- lib/logcli/params/session.rb
|
137
153
|
- lib/logcli/scp.rb
|
@@ -159,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
175
|
version: '0'
|
160
176
|
requirements: []
|
161
177
|
rubyforge_project:
|
162
|
-
rubygems_version: 2.
|
178
|
+
rubygems_version: 2.7.8
|
163
179
|
signing_key:
|
164
180
|
specification_version: 4
|
165
181
|
summary: log fetcher and remote retriveve files
|