loggie 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +32 -0
- data/bin/console +0 -3
- data/bin/loggie +0 -4
- data/lib/loggie/configuration.rb +35 -0
- data/lib/loggie/extract.rb +2 -6
- data/lib/loggie/logentries/retry.rb +13 -7
- data/lib/loggie/logentries/search.rb +1 -6
- data/lib/loggie/logging.rb +1 -3
- data/lib/loggie/request.rb +1 -2
- data/lib/loggie/version.rb +1 -1
- data/lib/loggie.rb +1 -0
- data/loggie.gemspec +1 -2
- metadata +5 -20
- data/.env.example +0 -13
- data/.env.test +0 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1f5379f3de4e3589a9e4932702b79244c6f26a1a
|
4
|
+
data.tar.gz: a80e9996174e3a85fdfad3f5ac0aad08a3626404
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05f8aeb73e4ae741e9bb791bd32ccb602b54c0600ab0e0e743b28d3bebe98aad998532425ea0aac87071f7cb4edc00eadf64d16dd06514300470bed65ca254ee
|
7
|
+
data.tar.gz: 1e97433ff0b40817b2eb9360be901018adb250019f4cd1ee89ff30e876506bbe0809aa1c5cdb43ccd377dc4e9aacd6ab1dfefac312481e90145f853ae9e5ba93
|
data/README.md
CHANGED
@@ -34,6 +34,38 @@ Or, use from the command line with:
|
|
34
34
|
|
35
35
|
`loggie foobar`
|
36
36
|
|
37
|
+
env is required for command line usage, and can be prefixed to the command, eg:
|
38
|
+
|
39
|
+
`READ_TOKEN=abc LOG_FILES=x,y,z loggie foobar`
|
40
|
+
|
41
|
+
Or the create a `.loggie` file in the current path.
|
42
|
+
|
43
|
+
## Configuring Loggie
|
44
|
+
|
45
|
+
```
|
46
|
+
Loggie.configure do |config|
|
47
|
+
# from https://logentries.com/app/<app>#/user-account/apikey
|
48
|
+
config.read_token = 'key'
|
49
|
+
|
50
|
+
# A comma separated list of log file ids
|
51
|
+
config.log_files = ['e20bd6af', 'c83c7cd7', '6fb426fd', '776dfea9']
|
52
|
+
|
53
|
+
# Depending on the size of the underlying dataset of the complexity of the query,
|
54
|
+
# a request may not yield a value straight away. In this case this gem will request
|
55
|
+
# the results up until this retry count
|
56
|
+
config.max_retry = 50
|
57
|
+
|
58
|
+
# Time to sleep before each retry, smaller value will poll more
|
59
|
+
# and could get get result quicker, but will eat into rate limit
|
60
|
+
# of request count
|
61
|
+
config.sleep_before_retry_seconds = 0.5
|
62
|
+
|
63
|
+
# If the log message is JSON parsable, then it will slice off all but
|
64
|
+
# the ones listed here. Whole log line is returned otherwise.
|
65
|
+
config.default_fields_included = ["keys", "from", "log", "output"]
|
66
|
+
end
|
67
|
+
```
|
68
|
+
|
37
69
|
## Development
|
38
70
|
|
39
71
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/bin/console
CHANGED
data/bin/loggie
CHANGED
@@ -0,0 +1,35 @@
|
|
1
|
+
module Loggie
|
2
|
+
class << self
|
3
|
+
attr_accessor :configuration
|
4
|
+
end
|
5
|
+
|
6
|
+
def self.configure
|
7
|
+
self.configuration ||= Configuration.new
|
8
|
+
yield(configuration)
|
9
|
+
end
|
10
|
+
|
11
|
+
class Configuration
|
12
|
+
MAX_RETRY_DELEY_SECONDS = 20.0 # max 20 seconds, or it will expire
|
13
|
+
|
14
|
+
attr_accessor :read_token
|
15
|
+
attr_accessor :log_files
|
16
|
+
attr_accessor :max_retry
|
17
|
+
attr_accessor :log_level
|
18
|
+
attr_accessor :sleep_before_retry_seconds
|
19
|
+
attr_accessor :default_fields_included
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@max_retry = 50
|
23
|
+
@log_level = :info
|
24
|
+
@sleep_before_retry_seconds = 0.5
|
25
|
+
@default_fields_included = [
|
26
|
+
"request_method", "path_info", "query_string", "agent",
|
27
|
+
"authorization", "response_body", "request_params"
|
28
|
+
]
|
29
|
+
end
|
30
|
+
|
31
|
+
def valid?
|
32
|
+
sleep_before_retry_seconds.to_f < MAX_RETRY_DELEY_SECONDS
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/loggie/extract.rb
CHANGED
@@ -36,12 +36,8 @@ module Loggie
|
|
36
36
|
# "request_params" ....
|
37
37
|
# "duration"
|
38
38
|
|
39
|
-
DEFAULT_FIELDS_INCLUDED = [
|
40
|
-
"request_method", "path_info", "query_string", "agent", "authorization", "response_body", "request_params"
|
41
|
-
]
|
42
|
-
|
43
39
|
def initialize
|
44
|
-
@keep_fields =
|
40
|
+
@keep_fields = Loggie.configuration.default_fields_included
|
45
41
|
end
|
46
42
|
|
47
43
|
##
|
@@ -72,7 +68,7 @@ module Loggie
|
|
72
68
|
m = remove_rails_timestamp(m)
|
73
69
|
m = safe_parse(m)
|
74
70
|
m = remove_empty_fields(m)
|
75
|
-
m.
|
71
|
+
m.slice(*keep_fields)
|
76
72
|
end
|
77
73
|
|
78
74
|
def remove_rails_timestamp(message)
|
@@ -5,16 +5,18 @@ module Loggie
|
|
5
5
|
# It checks the response and extracts the polling URI for progress
|
6
6
|
class Retry
|
7
7
|
include Logging
|
8
|
-
MAX_RETRY = ENV.fetch('MAX_RETRY', 5).to_i
|
9
|
-
RETRY_DELAY_SECONDS = ENV.fetch('RETRY_DELAY_SECONDS', 0.2).to_f
|
10
|
-
MAX_RETRY_DELEY_SECONDS = 20 # max 20 seconds, or it will expire
|
11
8
|
|
12
9
|
class RetryError < RuntimeError; end
|
13
10
|
class RetryCountExceededError < RetryError; end
|
14
11
|
class RetryResponseError < RetryError; end
|
15
12
|
|
13
|
+
def initialize
|
14
|
+
@retry_count = 0
|
15
|
+
@max_retry = Loggie.configuration.max_retry
|
16
|
+
@sleep_before_retry_seconds = Loggie.configuration.sleep_before_retry_seconds
|
17
|
+
end
|
18
|
+
|
16
19
|
def call(url, method, options, &block)
|
17
|
-
# TODO: ensure retry_count is reset or if that matters
|
18
20
|
@retry_count ||= 0
|
19
21
|
response = block.call(url, method, options)
|
20
22
|
logger.debug "#{self.class} retry:#{@retry_count}, response:#{response.body}"
|
@@ -28,11 +30,11 @@ module Loggie
|
|
28
30
|
logger.info "Logentries returned progress:#{res.progress}"
|
29
31
|
|
30
32
|
@retry_count += 1
|
31
|
-
if @retry_count >
|
32
|
-
raise RetryCountExceededError, "Retry count of #{
|
33
|
+
if @retry_count > max_retry
|
34
|
+
raise RetryCountExceededError, "Retry count of #{max_retry} reached"
|
33
35
|
end
|
34
36
|
|
35
|
-
sleep
|
37
|
+
sleep sleep_before_retry_seconds
|
36
38
|
|
37
39
|
self.call(res.next_url, :get, nil, &block)
|
38
40
|
|
@@ -40,6 +42,10 @@ module Loggie
|
|
40
42
|
logger.error e.message
|
41
43
|
nil
|
42
44
|
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
attr_reader :max_retry, :sleep_before_retry_seconds
|
43
49
|
end
|
44
50
|
end
|
45
51
|
end
|
@@ -5,7 +5,6 @@ module Loggie
|
|
5
5
|
include Logging
|
6
6
|
BASE_URI = "https://rest.logentries.com"
|
7
7
|
QUERY_PATH = "query/logs"
|
8
|
-
LOG_FILES = ENV['LOG_FILES']
|
9
8
|
|
10
9
|
##
|
11
10
|
# @param [String] query: to perform
|
@@ -15,7 +14,7 @@ module Loggie
|
|
15
14
|
#
|
16
15
|
def initialize(query: nil, from: nil, to: nil, log_files: nil)
|
17
16
|
@query, @from, @to = query, from, to
|
18
|
-
@log_files = log_files ||
|
17
|
+
@log_files = log_files || Loggie.configuration.log_files
|
19
18
|
@extract = Extract.new
|
20
19
|
@request = Request.new(retry_mechanism: Retry.new)
|
21
20
|
end
|
@@ -52,10 +51,6 @@ module Loggie
|
|
52
51
|
def convert(time)
|
53
52
|
(time.to_f * 1000).floor
|
54
53
|
end
|
55
|
-
|
56
|
-
def log_files_from_env
|
57
|
-
LOG_FILES&.split(",")
|
58
|
-
end
|
59
54
|
end
|
60
55
|
end
|
61
56
|
end
|
data/lib/loggie/logging.rb
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
require 'logger'
|
2
2
|
|
3
3
|
module Logging
|
4
|
-
LOG_LEVEL = ENV.fetch('LOG_LEVEL', :warn).to_sym
|
5
|
-
|
6
4
|
class << self
|
7
5
|
def logger
|
8
|
-
@logger ||= ::Logger.new(STDOUT).tap { |l| l.level =
|
6
|
+
@logger ||= ::Logger.new(STDOUT).tap { |l| l.level = Loggie.configuration.log_level }
|
9
7
|
end
|
10
8
|
|
11
9
|
def logger=(logger)
|
data/lib/loggie/request.rb
CHANGED
@@ -6,7 +6,6 @@ module Loggie
|
|
6
6
|
# polling long running queries on remote server
|
7
7
|
class Request
|
8
8
|
include Logging
|
9
|
-
READ_TOKEN = ENV['READ_TOKEN']
|
10
9
|
|
11
10
|
def initialize(retry_mechanism: )
|
12
11
|
@retry_mechanism = retry_mechanism || Retry.new
|
@@ -40,7 +39,7 @@ module Loggie
|
|
40
39
|
else
|
41
40
|
Net::HTTP::Post.new(url)
|
42
41
|
end
|
43
|
-
request["x-api-key"] =
|
42
|
+
request["x-api-key"] = Loggie.configuration.read_token
|
44
43
|
|
45
44
|
if method == :post
|
46
45
|
request["content-type"] = 'application/json'
|
data/lib/loggie/version.rb
CHANGED
data/lib/loggie.rb
CHANGED
data/loggie.gemspec
CHANGED
@@ -18,8 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables << "loggie"
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
-
spec.add_dependency "
|
22
|
-
spec.add_dependency "activesupport", "~> 5.0"
|
21
|
+
spec.add_dependency "activesupport", "~> 4.0"
|
23
22
|
|
24
23
|
spec.add_development_dependency "bundler", "~> 1.12"
|
25
24
|
spec.add_development_dependency "rake", "~> 10.0"
|
metadata
CHANGED
@@ -1,43 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: loggie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ian Vaughan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-01-
|
11
|
+
date: 2017-01-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: dotenv
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '2.1'
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '2.1'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: activesupport
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
30
16
|
requirements:
|
31
17
|
- - "~>"
|
32
18
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
19
|
+
version: '4.0'
|
34
20
|
type: :runtime
|
35
21
|
prerelease: false
|
36
22
|
version_requirements: !ruby/object:Gem::Requirement
|
37
23
|
requirements:
|
38
24
|
- - "~>"
|
39
25
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
26
|
+
version: '4.0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: bundler
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -144,8 +130,6 @@ executables:
|
|
144
130
|
extensions: []
|
145
131
|
extra_rdoc_files: []
|
146
132
|
files:
|
147
|
-
- ".env.example"
|
148
|
-
- ".env.test"
|
149
133
|
- ".gitignore"
|
150
134
|
- ".rspec"
|
151
135
|
- ".ruby-version"
|
@@ -158,6 +142,7 @@ files:
|
|
158
142
|
- bin/loggie
|
159
143
|
- bin/setup
|
160
144
|
- lib/loggie.rb
|
145
|
+
- lib/loggie/configuration.rb
|
161
146
|
- lib/loggie/extract.rb
|
162
147
|
- lib/loggie/logentries/response.rb
|
163
148
|
- lib/loggie/logentries/retry.rb
|
data/.env.example
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
# from https://logentries.com/app/<app>#/user-account/apikey
|
2
|
-
READ_TOKEN=key
|
3
|
-
|
4
|
-
# A comma separated list of log file ids
|
5
|
-
LOG_FILES=e20bd6af, c83c7cd7, 6fb426fd, 776dfea9
|
6
|
-
|
7
|
-
# Depending on the size of the underlying dataset of the complexity of the query,
|
8
|
-
# a request may not yield a value straight away. In this case this gem will request
|
9
|
-
# the results up until this retry count
|
10
|
-
MAX_RETRY=50
|
11
|
-
|
12
|
-
# Internal logging level. default: info
|
13
|
-
LOG_LEVEL=info
|
data/.env.test
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
READ_TOKEN=bf3b4d32
|
2
|
-
|
3
|
-
# A comma separated list of log file ids
|
4
|
-
LOG_FILES=e20bd6af,c83c7cd7,6fb426fd,776dfea9
|
5
|
-
|
6
|
-
# Depending on the size of the underlying dataset of the complexity of the query,
|
7
|
-
# a request may not yield a value straight away. In this case this gem will request
|
8
|
-
# the results up until this retry count
|
9
|
-
MAX_RETRY=50
|
10
|
-
|
11
|
-
# LOG_LEVEL=info
|
12
|
-
LOG_LEVEL=debug
|