milemarker 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +27 -0
- data/.gitignore +2 -0
- data/LICENSE.txt +1 -1
- data/lib/milemarker/structured.rb +6 -6
- data/lib/milemarker/version.rb +1 -1
- data/lib/milemarker.rb +18 -18
- data/lib/ppnum.rb +8 -8
- data/milemarker.gemspec +16 -16
- metadata +25 -25
- data/.github/workflows/main.yml +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c802759bd48442b9a33c53d949c98786840cb78daa62b08eb9f2bc9ac4f6689
|
4
|
+
data.tar.gz: 37507e1e5d185967ab224df2bb79a7f8f45b0edf66a26a2acdee0c470dec146f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 046b4f34129f9ac46d99f03f44b174617f37ab468436d2f27dfece59c3ce190c23492037e60ac5285a17b05c774b18032bafb9f607e8c40e463a4f373a1fa619
|
7
|
+
data.tar.gz: 4b36594579c0e79d03d9b9d827f5dda74ccb118b2bfe8f1e294d8fd87d62c74233c3d78386bc3dc90a82346d04fb36fcb2d1d30905f6e5c58b569646ff779adb
|
@@ -0,0 +1,27 @@
|
|
1
|
+
name: Run Tests
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
push:
|
6
|
+
branches:
|
7
|
+
- main
|
8
|
+
|
9
|
+
env:
|
10
|
+
# See https://github.com/jruby/jruby/issues/5509
|
11
|
+
JAVA_OPTS: "--add-opens java.xml/com.sun.org.apache.xerces.internal.impl=org.jruby.dist"
|
12
|
+
|
13
|
+
jobs:
|
14
|
+
tests:
|
15
|
+
runs-on: ubuntu-latest
|
16
|
+
strategy:
|
17
|
+
matrix:
|
18
|
+
ruby: [3.2, 3.3, 3.4, jruby]
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v4
|
21
|
+
- name: Set up Ruby
|
22
|
+
uses: ruby/setup-ruby@v1
|
23
|
+
with:
|
24
|
+
ruby-version: ${{ matrix.ruby }}
|
25
|
+
bundler-cache: true
|
26
|
+
- name: Run tests
|
27
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/LICENSE.txt
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
The MIT License (MIT)
|
2
2
|
|
3
|
-
Copyright (c) 2021
|
3
|
+
Copyright (c) 2021 The Regents of the University of Michigan
|
4
4
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
@@ -27,12 +27,12 @@ class Milemarker
|
|
27
27
|
when Hash
|
28
28
|
msg
|
29
29
|
when String
|
30
|
-
{
|
30
|
+
{msg: msg}
|
31
31
|
when Exception
|
32
32
|
exception_message_hash(msg)
|
33
33
|
else
|
34
34
|
other_message_hash(msg)
|
35
|
-
end.merge({
|
35
|
+
end.merge({level: severity, time: datetime}).to_json
|
36
36
|
end
|
37
37
|
self
|
38
38
|
end
|
@@ -50,7 +50,7 @@ class Milemarker
|
|
50
50
|
}
|
51
51
|
end
|
52
52
|
|
53
|
-
|
53
|
+
alias_method :batch_data, :batch_line
|
54
54
|
|
55
55
|
# @return [Hash] hash with information about the last batch
|
56
56
|
def final_line
|
@@ -63,17 +63,17 @@ class Milemarker
|
|
63
63
|
}
|
64
64
|
end
|
65
65
|
|
66
|
-
|
66
|
+
alias_method :final_data, :final_line
|
67
67
|
|
68
68
|
def exception_message_hash(msg)
|
69
|
-
{
|
69
|
+
{msg: msg.message, error: msg.class, at: msg.backtrace&.first, hostname: Socket.gethostname}
|
70
70
|
end
|
71
71
|
|
72
72
|
def other_message_hash(msg)
|
73
73
|
if msg.respond_to? :to_h
|
74
74
|
msg.to_h
|
75
75
|
else
|
76
|
-
{
|
76
|
+
{msg: msg.inspect}
|
77
77
|
end
|
78
78
|
end
|
79
79
|
end
|
data/lib/milemarker/version.rb
CHANGED
data/lib/milemarker.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative "ppnum"
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
4
|
+
require "logger"
|
5
|
+
require "socket"
|
6
|
+
require "json"
|
7
|
+
require "milemarker/structured"
|
8
8
|
|
9
9
|
# milemarker class, to keep track of progress over time for long-running
|
10
10
|
# iterating processes
|
@@ -51,18 +51,18 @@ class Milemarker
|
|
51
51
|
# @param [Logger, #info, #warn] Optional logger that responds to the normal #info, #warn, etc.
|
52
52
|
def initialize(batch_size: 1000, name: nil, logger: nil)
|
53
53
|
@batch_size = batch_size
|
54
|
-
@name
|
55
|
-
@logger
|
54
|
+
@name = name
|
55
|
+
@logger = logger
|
56
56
|
|
57
57
|
@batch_number = 0
|
58
|
-
@last_batch_size
|
58
|
+
@last_batch_size = 0
|
59
59
|
@last_batch_seconds = 0
|
60
60
|
|
61
|
-
@start_time
|
61
|
+
@start_time = Time.now
|
62
62
|
@batch_start_time = @start_time
|
63
|
-
@batch_end_time
|
63
|
+
@batch_end_time = @start_time
|
64
64
|
|
65
|
-
@count
|
65
|
+
@count = 0
|
66
66
|
@prev_count = 0
|
67
67
|
end
|
68
68
|
|
@@ -84,7 +84,7 @@ class Milemarker
|
|
84
84
|
self
|
85
85
|
end
|
86
86
|
|
87
|
-
|
87
|
+
alias_method :increment, :incr
|
88
88
|
|
89
89
|
# Create a logger for use in logging milemaker information
|
90
90
|
# @example mm.create_logger!(STDOUT)
|
@@ -108,7 +108,7 @@ class Milemarker
|
|
108
108
|
incr.on_batch(&blk)
|
109
109
|
end
|
110
110
|
|
111
|
-
|
111
|
+
alias_method :increment_and_on_batch, :_increment_and_on_batch
|
112
112
|
|
113
113
|
# Threadsafe version of #increment_and_on_batch, doing the whole thing as a single atomic action
|
114
114
|
def threadsafe_increment_and_on_batch(&blk)
|
@@ -154,7 +154,7 @@ class Milemarker
|
|
154
154
|
count - prev_count
|
155
155
|
end
|
156
156
|
|
157
|
-
|
157
|
+
alias_method :batch_count_so_far, :final_batch_size
|
158
158
|
|
159
159
|
# A line describing the entire run, suitable for logging, of the form
|
160
160
|
# load records.ndj FINISHED. 27_138_118 total records in 00h 12m 39s. Overall 35_718 r/s.
|
@@ -208,8 +208,8 @@ class Milemarker
|
|
208
208
|
# Set/reset all the internal state. Called by #on_batch when necessary;
|
209
209
|
# should probably not be called manually
|
210
210
|
def set_milemarker!
|
211
|
-
@batch_end_time
|
212
|
-
@last_batch_size
|
211
|
+
@batch_end_time = Time.now
|
212
|
+
@last_batch_size = @count - @prev_count
|
213
213
|
@last_batch_seconds = @batch_end_time - @batch_start_time
|
214
214
|
|
215
215
|
reset_for_next_batch!
|
@@ -218,8 +218,8 @@ class Milemarker
|
|
218
218
|
# Reset the internal counters/timers at the end of a batch. Taken care of
|
219
219
|
# by #on_batch; should probably not be called manually.
|
220
220
|
def reset_for_next_batch!
|
221
|
-
@batch_start_time
|
222
|
-
@prev_count
|
221
|
+
@batch_start_time = batch_end_time
|
222
|
+
@prev_count = count
|
223
223
|
@batch_number = batch_divisor
|
224
224
|
end
|
225
225
|
|
@@ -242,7 +242,7 @@ class Milemarker
|
|
242
242
|
|
243
243
|
def seconds_to_time_string(sec)
|
244
244
|
hours, leftover = sec.divmod(3600)
|
245
|
-
minutes, secs
|
245
|
+
minutes, secs = leftover.divmod(60)
|
246
246
|
format("%02dh %02dm %02ds", hours, minutes, secs)
|
247
247
|
end
|
248
248
|
end
|
data/lib/ppnum.rb
CHANGED
@@ -20,15 +20,15 @@
|
|
20
20
|
def ppnum(num, width = 0, decimals = 0)
|
21
21
|
num = num.round(decimals)
|
22
22
|
dec_str = if decimals.zero?
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
""
|
24
|
+
else
|
25
|
+
".#{format("%.#{decimals}f", num).split(".").last}"
|
26
|
+
end
|
27
27
|
numstr = num.floor.to_s.reverse.split(/(...)/)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
.reject(&:empty?)
|
29
|
+
.map(&:reverse)
|
30
|
+
.reverse
|
31
|
+
.join("_") + dec_str
|
32
32
|
if width.zero?
|
33
33
|
numstr
|
34
34
|
else
|
data/milemarker.gemspec
CHANGED
@@ -3,27 +3,27 @@
|
|
3
3
|
require_relative "lib/milemarker/version"
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name
|
7
|
-
spec.version
|
8
|
-
spec.authors
|
9
|
-
spec.email
|
10
|
-
|
11
|
-
spec.summary
|
12
|
-
spec.homepage
|
13
|
-
spec.license
|
6
|
+
spec.name = "milemarker"
|
7
|
+
spec.version = Milemarker::VERSION
|
8
|
+
spec.authors = ["Bill Dueber"]
|
9
|
+
spec.email = ["bill@dueber.com"]
|
10
|
+
|
11
|
+
spec.summary = "Track and produce loglines for batch processing progress."
|
12
|
+
spec.homepage = "https://github.com/billdueber/milemarker"
|
13
|
+
spec.license = "MIT"
|
14
14
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
15
15
|
|
16
16
|
spec.metadata["homepage_uri"] = spec.homepage
|
17
17
|
spec.metadata["source_code_uri"] = spec.homepage
|
18
|
-
spec.metadata["changelog_uri"] = spec.homepage +
|
18
|
+
spec.metadata["changelog_uri"] = spec.homepage + "/CHANGELOG.md"
|
19
19
|
|
20
20
|
# Specify which files should be added to the gem when it is released.
|
21
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
22
22
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
23
23
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
24
24
|
end
|
25
|
-
spec.bindir
|
26
|
-
spec.executables
|
25
|
+
spec.bindir = "exe"
|
26
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
27
27
|
spec.require_paths = ["lib"]
|
28
28
|
|
29
29
|
# Uncomment to register a new dependency of your gem
|
@@ -33,9 +33,9 @@ Gem::Specification.new do |spec|
|
|
33
33
|
# guide at: https://bundler.io/guides/creating_gem.html
|
34
34
|
#
|
35
35
|
|
36
|
-
spec.add_development_dependency
|
37
|
-
spec.add_development_dependency
|
38
|
-
spec.add_development_dependency
|
39
|
-
spec.add_development_dependency
|
40
|
-
spec.add_development_dependency
|
36
|
+
spec.add_development_dependency "bundler"
|
37
|
+
spec.add_development_dependency "pry"
|
38
|
+
spec.add_development_dependency "rake"
|
39
|
+
spec.add_development_dependency "rspec"
|
40
|
+
spec.add_development_dependency "standardrb"
|
41
41
|
end
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: milemarker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bill Dueber
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-02-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '0'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: pry
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -42,52 +42,52 @@ dependencies:
|
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rspec
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: standardrb
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
83
|
-
description:
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
84
|
email:
|
85
85
|
- bill@dueber.com
|
86
86
|
executables: []
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
|
-
- ".github/workflows/
|
90
|
+
- ".github/workflows/tests.yml"
|
91
91
|
- ".gitignore"
|
92
92
|
- ".rspec"
|
93
93
|
- ".rubocop.yml"
|
@@ -110,7 +110,7 @@ metadata:
|
|
110
110
|
homepage_uri: https://github.com/billdueber/milemarker
|
111
111
|
source_code_uri: https://github.com/billdueber/milemarker
|
112
112
|
changelog_uri: https://github.com/billdueber/milemarker/CHANGELOG.md
|
113
|
-
post_install_message:
|
113
|
+
post_install_message:
|
114
114
|
rdoc_options: []
|
115
115
|
require_paths:
|
116
116
|
- lib
|
@@ -125,8 +125,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
125
|
- !ruby/object:Gem::Version
|
126
126
|
version: '0'
|
127
127
|
requirements: []
|
128
|
-
rubygems_version: 3.
|
129
|
-
signing_key:
|
128
|
+
rubygems_version: 3.3.5
|
129
|
+
signing_key:
|
130
130
|
specification_version: 4
|
131
131
|
summary: Track and produce loglines for batch processing progress.
|
132
132
|
test_files: []
|
data/.github/workflows/main.yml
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
name: Ruby
|
2
|
-
|
3
|
-
on: [push,pull_request]
|
4
|
-
|
5
|
-
jobs:
|
6
|
-
build:
|
7
|
-
runs-on: ubuntu-latest
|
8
|
-
steps:
|
9
|
-
- uses: actions/checkout@v2
|
10
|
-
- name: Set up Ruby
|
11
|
-
uses: ruby/setup-ruby@v1
|
12
|
-
with:
|
13
|
-
ruby-version: 2.6.6
|
14
|
-
bundler-cache: true
|
15
|
-
- name: Run the default task
|
16
|
-
run: bundle exec rake
|