logcli 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fd1b6872420f9e477da09618e3df36c1c40b55b6
-  data.tar.gz: 175049a04cbee70b8d9623fbeb9a1dfefc8bf57e
+SHA256:
+  metadata.gz: ac147cb720b80c5f8a5bcd351700b9973f839491ab66908d023824dce30b18b1
+  data.tar.gz: 86e4319fa339699d9de38cbe3c8f5640753a984897a8fd2e43b4b2814fca9b8b
 SHA512:
-  metadata.gz: 2da758a6aeb05ddf92bdd74594ecdcb0259d903922c051a5cdf4e6b4915f4b7474d4bb38a3223be036d044a5331c48e494a737c28d60e8265bd236538f67e42c
-  data.tar.gz: 1836136670258bc44f22b2cf560e42e28a65efbe82f13927a698e9db8b23951d580efe40b11c50f7c0380cf73192c255e4c9168c89425da13d7c8dfa1237f535
+  metadata.gz: 4ddee66f38f61ff9c9b28207767ab15f9e39e8284a81d9e46c4514bf751fa5b36fdde7362f31362d2af57886f4537e7d77474a7ec39153fbff09f81662530ebd
+  data.tar.gz: e7a4ee7a860eed6532f671d6a260f4050b97d07168a2432a23614788da31c4c9f5b07b5df59b6e67b843ff071373a940b0c43bee3a632938b1a6586cd19a52e6

data/.gitignore

@@ -11,3 +11,4 @@
 .rspec_status
 .DS_Store
 .idea/*
+*.gem

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    logcli (0.1.0)
+    logcli (0.1.1)
+      elasticsearch (~> 7.0)
       net-scp (~> 1.2.1, >= 1.2.1)
       net-ssh (~> 5.0.2, >= 5.0.2)
       thor (~> 0.20.3)
@@ -11,7 +12,19 @@ GEM
   specs:
     coderay (1.1.2)
     diff-lcs (1.3)
+    elasticsearch (7.0.0)
+      elasticsearch-api (= 7.0.0)
+      elasticsearch-transport (= 7.0.0)
+    elasticsearch-api (7.0.0)
+      multi_json
+    elasticsearch-transport (7.0.0)
+      faraday
+      multi_json
+    faraday (0.15.4)
+      multipart-post (>= 1.2, < 3)
     method_source (0.9.0)
+    multi_json (1.13.1)
+    multipart-post (2.1.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (5.0.2)
@@ -45,4 +58,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.0
+   1.16.6

data/README.md

@@ -1,34 +1,33 @@
 # Logcli
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/logcli`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'logcli'
+logcli is tool for easy working with logs but its opionated my work flow to working with logs looks like
+fetch json logs from lines and than push it to elasticsearch for future analytics my logs stored
+as JSON objects. Example how my logs look like
+```text
+I, [2019-05-09T17:00:16.049805 #57671]  INFO -- : {"evt_type":"FileUpload":"info","time":"2019-05-09T15:00:16+0000","payload":{"time_elapsed_human":"00:00:00.874","time_elapsed":0.874553},"trace_id":"cd5916a6-14b7-4628-bdda-334e4285a4e0"}
+```
+as you can see there a log prefixer like `I, [2019-05-09T17:00:16.049805 #57671]  INFO -- : `
+that's why its not a JSON but than goes a JSON object that I want to extract
+I'm using this command to extract JSON from log line
+```bash
+logcli extract_json --filenames=log1.log log2.log
 ```
 
-And then execute:
-
-    $ bundle
+as result tool create in the SAME directory files with names `log1_json.log, log2_json.log`
 
-Or install it yourself as:
+than I'm push data to ES for analytics using ES + Kibana
 
-    $ gem install logcli
+```bash
+logcli elasticsearch --filenames=events_json.log --elasticsearch_url=http://localhost:9201
+```
 
-## Usage
+tool using batch API and push 100 records and the time
 
-TODO: Write usage instructions here
+Pretty dummy stuff hah
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+In active - passive development
 
 ## Contributing
 

data/lib/cli.rb

@@ -29,5 +29,16 @@ module Logcli
       json = Logcli::Actions::ExtractJson.new(params.parse_params)
       json.call
     end
+
+    desc "elasticsearch", "push json file to elasticsearch instance"
+    option :filenames, type: :array, required: true, banner: 'example1.json example2.json'
+    option :elasticsearch_url, type: :string, required: true, banner: 'http://localhost:9200'
+    option :mapping_file, type: :string, required: false, banner: 'example-mapping.json'
+
+    def elasticsearch
+      params = Logcli::Params::Elasticsearch.new options
+      json = Logcli::Actions::Elasticsearch.new(params.parse_params)
+      json.call
+    end
   end
-end
+end

data/lib/logcli.rb

@@ -2,12 +2,14 @@ require "logcli/version"
 require "logcli/version"
 require "logcli/params/session"
 require "logcli/params/json"
+require "logcli/params/elasticsearch"
 require "logcli/ssh"
 require "logcli/scp"
 require "logcli/helper"
 require "logcli/actions/download"
 require "logcli/actions/grep"
 require "logcli/actions/extract_json"
+require "logcli/actions/elasticsearch"
 require 'cli'
 require 'net/ssh'
 require 'net/scp'

data/lib/logcli/actions/elasticsearch.rb

@@ -0,0 +1,63 @@
+require 'elasticsearch'
+require 'json'
+
+module Logcli
+  module Actions
+    class Elasticsearch
+      attr_reader :filenames, :elasticsearch_url, :mapping_file, :buffer, :id, :total_records
+      BULK_SIZE = 100
+      INDEX_NAME = 'log-index'.freeze
+
+      def initialize opts
+        @filenames = opts.fetch(:filenames)
+        @mapping_file = opts.fetch(:mapping_file)
+        @elasticsearch_url = opts.fetch(:elasticsearch_url)
+        @buffer = []
+        @id = 1
+        @total_records = 0
+      end
+
+      def call
+        create_mapping if mapping_file
+
+        filenames.each do |filename|
+          File.open(filename).each do |line|
+            @buffer << line
+            if @buffer.count > BULK_SIZE
+              flush
+            end
+          end
+        end
+        if @buffer.count > 0
+          flush
+        end
+      end
+
+      private
+
+      def flush
+        payload = @buffer.flat_map do |item|
+          hash = JSON.parse(item)
+          @id += 1
+          @total_records += 1
+          [
+              { index: { _index: INDEX_NAME, _id: @id } },
+              hash
+          ]
+        end
+        client.bulk body: payload
+        puts "records uploaded #{@total_records}"
+        @buffer = []
+      end
+
+      def create_mapping
+        payload = JSON.parse(File.read(mapping_file))
+        client.indices.create index: INDEX_NAME, body: payload
+      end
+
+      def client
+        @client ||= ::Elasticsearch::Client.new url: elasticsearch_url, log: true
+      end
+    end
+  end
+end

data/lib/logcli/params/elasticsearch.rb

@@ -0,0 +1,22 @@
+module Logcli
+  module Params
+    class Elasticsearch
+      FILENAMES = :filenames
+      MAPPING_FILE = :mapping_file
+      ELASTICSEARCH_URL = :elasticsearch_url
+      OPTS = [FILENAMES, MAPPING_FILE, ELASTICSEARCH_URL].freeze
+      attr_accessor(*OPTS)
+
+      def initialize opts
+        OPTS.each do |opt|
+          val = opts.fetch opt, nil
+          instance_variable_set "@#{opt}", val
+        end
+      end
+
+      def parse_params
+        { filenames: filenames, mapping_file: mapping_file, elasticsearch_url: elasticsearch_url }
+      end
+    end
+  end
+end

data/lib/logcli/version.rb

@@ -1,3 +1,3 @@
 module Logcli
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/logcli.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency 'elasticsearch', '~> 7.0'
   spec.add_runtime_dependency 'net-ssh', '~> 5.0.2', '>= 5.0.2'
   spec.add_runtime_dependency 'net-scp', '~> 1.2.1', '>= 1.2.1'
   spec.add_runtime_dependency 'thor', '~> 0.20.3'

metadata

@@ -1,53 +1,67 @@
 --- !ruby/object:Gem::Specification
 name: logcli
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - gingray
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: net-ssh
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.2
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.2
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.2
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 5.0.2
 - !ruby/object:Gem::Dependency
   name: net-scp
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.1
 - !ruby/object:Gem::Dependency
@@ -129,9 +143,11 @@ files:
 - lib/cli.rb
 - lib/logcli.rb
 - lib/logcli/actions/download.rb
+- lib/logcli/actions/elasticsearch.rb
 - lib/logcli/actions/extract_json.rb
 - lib/logcli/actions/grep.rb
 - lib/logcli/helper.rb
+- lib/logcli/params/elasticsearch.rb
 - lib/logcli/params/json.rb
 - lib/logcli/params/session.rb
 - lib/logcli/scp.rb
@@ -159,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: log fetcher and remote retriveve files