milemarker 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +27 -0
- data/.gitignore +2 -0
- data/LICENSE.txt +1 -1
- data/lib/milemarker/structured.rb +6 -6
- data/lib/milemarker/version.rb +1 -1
- data/lib/milemarker.rb +18 -18
- data/lib/ppnum.rb +8 -8
- data/milemarker.gemspec +16 -16
- metadata +25 -25
- data/.github/workflows/main.yml +0 -16
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7c802759bd48442b9a33c53d949c98786840cb78daa62b08eb9f2bc9ac4f6689
|
|
4
|
+
data.tar.gz: 37507e1e5d185967ab224df2bb79a7f8f45b0edf66a26a2acdee0c470dec146f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 046b4f34129f9ac46d99f03f44b174617f37ab468436d2f27dfece59c3ce190c23492037e60ac5285a17b05c774b18032bafb9f607e8c40e463a4f373a1fa619
|
|
7
|
+
data.tar.gz: 4b36594579c0e79d03d9b9d827f5dda74ccb118b2bfe8f1e294d8fd87d62c74233c3d78386bc3dc90a82346d04fb36fcb2d1d30905f6e5c58b569646ff779adb
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Run Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
push:
|
|
6
|
+
branches:
|
|
7
|
+
- main
|
|
8
|
+
|
|
9
|
+
env:
|
|
10
|
+
# See https://github.com/jruby/jruby/issues/5509
|
|
11
|
+
JAVA_OPTS: "--add-opens java.xml/com.sun.org.apache.xerces.internal.impl=org.jruby.dist"
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
tests:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
strategy:
|
|
17
|
+
matrix:
|
|
18
|
+
ruby: [3.2, 3.3, 3.4, jruby]
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- name: Set up Ruby
|
|
22
|
+
uses: ruby/setup-ruby@v1
|
|
23
|
+
with:
|
|
24
|
+
ruby-version: ${{ matrix.ruby }}
|
|
25
|
+
bundler-cache: true
|
|
26
|
+
- name: Run tests
|
|
27
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/LICENSE.txt
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
The MIT License (MIT)
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2021
|
|
3
|
+
Copyright (c) 2021 The Regents of the University of Michigan
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -27,12 +27,12 @@ class Milemarker
|
|
|
27
27
|
when Hash
|
|
28
28
|
msg
|
|
29
29
|
when String
|
|
30
|
-
{
|
|
30
|
+
{msg: msg}
|
|
31
31
|
when Exception
|
|
32
32
|
exception_message_hash(msg)
|
|
33
33
|
else
|
|
34
34
|
other_message_hash(msg)
|
|
35
|
-
end.merge({
|
|
35
|
+
end.merge({level: severity, time: datetime}).to_json
|
|
36
36
|
end
|
|
37
37
|
self
|
|
38
38
|
end
|
|
@@ -50,7 +50,7 @@ class Milemarker
|
|
|
50
50
|
}
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
alias_method :batch_data, :batch_line
|
|
54
54
|
|
|
55
55
|
# @return [Hash] hash with information about the last batch
|
|
56
56
|
def final_line
|
|
@@ -63,17 +63,17 @@ class Milemarker
|
|
|
63
63
|
}
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
alias_method :final_data, :final_line
|
|
67
67
|
|
|
68
68
|
def exception_message_hash(msg)
|
|
69
|
-
{
|
|
69
|
+
{msg: msg.message, error: msg.class, at: msg.backtrace&.first, hostname: Socket.gethostname}
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
def other_message_hash(msg)
|
|
73
73
|
if msg.respond_to? :to_h
|
|
74
74
|
msg.to_h
|
|
75
75
|
else
|
|
76
|
-
{
|
|
76
|
+
{msg: msg.inspect}
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|
data/lib/milemarker/version.rb
CHANGED
data/lib/milemarker.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "ppnum"
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
-
require
|
|
7
|
-
require
|
|
4
|
+
require "logger"
|
|
5
|
+
require "socket"
|
|
6
|
+
require "json"
|
|
7
|
+
require "milemarker/structured"
|
|
8
8
|
|
|
9
9
|
# milemarker class, to keep track of progress over time for long-running
|
|
10
10
|
# iterating processes
|
|
@@ -51,18 +51,18 @@ class Milemarker
|
|
|
51
51
|
# @param [Logger, #info, #warn] Optional logger that responds to the normal #info, #warn, etc.
|
|
52
52
|
def initialize(batch_size: 1000, name: nil, logger: nil)
|
|
53
53
|
@batch_size = batch_size
|
|
54
|
-
@name
|
|
55
|
-
@logger
|
|
54
|
+
@name = name
|
|
55
|
+
@logger = logger
|
|
56
56
|
|
|
57
57
|
@batch_number = 0
|
|
58
|
-
@last_batch_size
|
|
58
|
+
@last_batch_size = 0
|
|
59
59
|
@last_batch_seconds = 0
|
|
60
60
|
|
|
61
|
-
@start_time
|
|
61
|
+
@start_time = Time.now
|
|
62
62
|
@batch_start_time = @start_time
|
|
63
|
-
@batch_end_time
|
|
63
|
+
@batch_end_time = @start_time
|
|
64
64
|
|
|
65
|
-
@count
|
|
65
|
+
@count = 0
|
|
66
66
|
@prev_count = 0
|
|
67
67
|
end
|
|
68
68
|
|
|
@@ -84,7 +84,7 @@ class Milemarker
|
|
|
84
84
|
self
|
|
85
85
|
end
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
alias_method :increment, :incr
|
|
88
88
|
|
|
89
89
|
# Create a logger for use in logging milemaker information
|
|
90
90
|
# @example mm.create_logger!(STDOUT)
|
|
@@ -108,7 +108,7 @@ class Milemarker
|
|
|
108
108
|
incr.on_batch(&blk)
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
|
|
111
|
+
alias_method :increment_and_on_batch, :_increment_and_on_batch
|
|
112
112
|
|
|
113
113
|
# Threadsafe version of #increment_and_on_batch, doing the whole thing as a single atomic action
|
|
114
114
|
def threadsafe_increment_and_on_batch(&blk)
|
|
@@ -154,7 +154,7 @@ class Milemarker
|
|
|
154
154
|
count - prev_count
|
|
155
155
|
end
|
|
156
156
|
|
|
157
|
-
|
|
157
|
+
alias_method :batch_count_so_far, :final_batch_size
|
|
158
158
|
|
|
159
159
|
# A line describing the entire run, suitable for logging, of the form
|
|
160
160
|
# load records.ndj FINISHED. 27_138_118 total records in 00h 12m 39s. Overall 35_718 r/s.
|
|
@@ -208,8 +208,8 @@ class Milemarker
|
|
|
208
208
|
# Set/reset all the internal state. Called by #on_batch when necessary;
|
|
209
209
|
# should probably not be called manually
|
|
210
210
|
def set_milemarker!
|
|
211
|
-
@batch_end_time
|
|
212
|
-
@last_batch_size
|
|
211
|
+
@batch_end_time = Time.now
|
|
212
|
+
@last_batch_size = @count - @prev_count
|
|
213
213
|
@last_batch_seconds = @batch_end_time - @batch_start_time
|
|
214
214
|
|
|
215
215
|
reset_for_next_batch!
|
|
@@ -218,8 +218,8 @@ class Milemarker
|
|
|
218
218
|
# Reset the internal counters/timers at the end of a batch. Taken care of
|
|
219
219
|
# by #on_batch; should probably not be called manually.
|
|
220
220
|
def reset_for_next_batch!
|
|
221
|
-
@batch_start_time
|
|
222
|
-
@prev_count
|
|
221
|
+
@batch_start_time = batch_end_time
|
|
222
|
+
@prev_count = count
|
|
223
223
|
@batch_number = batch_divisor
|
|
224
224
|
end
|
|
225
225
|
|
|
@@ -242,7 +242,7 @@ class Milemarker
|
|
|
242
242
|
|
|
243
243
|
def seconds_to_time_string(sec)
|
|
244
244
|
hours, leftover = sec.divmod(3600)
|
|
245
|
-
minutes, secs
|
|
245
|
+
minutes, secs = leftover.divmod(60)
|
|
246
246
|
format("%02dh %02dm %02ds", hours, minutes, secs)
|
|
247
247
|
end
|
|
248
248
|
end
|
data/lib/ppnum.rb
CHANGED
|
@@ -20,15 +20,15 @@
|
|
|
20
20
|
def ppnum(num, width = 0, decimals = 0)
|
|
21
21
|
num = num.round(decimals)
|
|
22
22
|
dec_str = if decimals.zero?
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
""
|
|
24
|
+
else
|
|
25
|
+
".#{format("%.#{decimals}f", num).split(".").last}"
|
|
26
|
+
end
|
|
27
27
|
numstr = num.floor.to_s.reverse.split(/(...)/)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
.reject(&:empty?)
|
|
29
|
+
.map(&:reverse)
|
|
30
|
+
.reverse
|
|
31
|
+
.join("_") + dec_str
|
|
32
32
|
if width.zero?
|
|
33
33
|
numstr
|
|
34
34
|
else
|
data/milemarker.gemspec
CHANGED
|
@@ -3,27 +3,27 @@
|
|
|
3
3
|
require_relative "lib/milemarker/version"
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
|
-
spec.name
|
|
7
|
-
spec.version
|
|
8
|
-
spec.authors
|
|
9
|
-
spec.email
|
|
10
|
-
|
|
11
|
-
spec.summary
|
|
12
|
-
spec.homepage
|
|
13
|
-
spec.license
|
|
6
|
+
spec.name = "milemarker"
|
|
7
|
+
spec.version = Milemarker::VERSION
|
|
8
|
+
spec.authors = ["Bill Dueber"]
|
|
9
|
+
spec.email = ["bill@dueber.com"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "Track and produce loglines for batch processing progress."
|
|
12
|
+
spec.homepage = "https://github.com/billdueber/milemarker"
|
|
13
|
+
spec.license = "MIT"
|
|
14
14
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
|
15
15
|
|
|
16
16
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
17
17
|
spec.metadata["source_code_uri"] = spec.homepage
|
|
18
|
-
spec.metadata["changelog_uri"] = spec.homepage +
|
|
18
|
+
spec.metadata["changelog_uri"] = spec.homepage + "/CHANGELOG.md"
|
|
19
19
|
|
|
20
20
|
# Specify which files should be added to the gem when it is released.
|
|
21
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
22
22
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
23
23
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
24
24
|
end
|
|
25
|
-
spec.bindir
|
|
26
|
-
spec.executables
|
|
25
|
+
spec.bindir = "exe"
|
|
26
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
27
27
|
spec.require_paths = ["lib"]
|
|
28
28
|
|
|
29
29
|
# Uncomment to register a new dependency of your gem
|
|
@@ -33,9 +33,9 @@ Gem::Specification.new do |spec|
|
|
|
33
33
|
# guide at: https://bundler.io/guides/creating_gem.html
|
|
34
34
|
#
|
|
35
35
|
|
|
36
|
-
spec.add_development_dependency
|
|
37
|
-
spec.add_development_dependency
|
|
38
|
-
spec.add_development_dependency
|
|
39
|
-
spec.add_development_dependency
|
|
40
|
-
spec.add_development_dependency
|
|
36
|
+
spec.add_development_dependency "bundler"
|
|
37
|
+
spec.add_development_dependency "pry"
|
|
38
|
+
spec.add_development_dependency "rake"
|
|
39
|
+
spec.add_development_dependency "rspec"
|
|
40
|
+
spec.add_development_dependency "standardrb"
|
|
41
41
|
end
|
metadata
CHANGED
|
@@ -1,29 +1,29 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: milemarker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Bill Dueber
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2025-02-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "
|
|
17
|
+
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '
|
|
19
|
+
version: '0'
|
|
20
20
|
type: :development
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
|
-
- - "
|
|
24
|
+
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '
|
|
26
|
+
version: '0'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: pry
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -42,52 +42,52 @@ dependencies:
|
|
|
42
42
|
name: rake
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
|
-
- - "
|
|
45
|
+
- - ">="
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '
|
|
47
|
+
version: '0'
|
|
48
48
|
type: :development
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
|
-
- - "
|
|
52
|
+
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '
|
|
54
|
+
version: '0'
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: rspec
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
|
-
- - "
|
|
59
|
+
- - ">="
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '
|
|
61
|
+
version: '0'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
|
-
- - "
|
|
66
|
+
- - ">="
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '
|
|
68
|
+
version: '0'
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
70
|
+
name: standardrb
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
|
72
72
|
requirements:
|
|
73
|
-
- - "
|
|
73
|
+
- - ">="
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '
|
|
75
|
+
version: '0'
|
|
76
76
|
type: :development
|
|
77
77
|
prerelease: false
|
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
79
|
requirements:
|
|
80
|
-
- - "
|
|
80
|
+
- - ">="
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: '
|
|
83
|
-
description:
|
|
82
|
+
version: '0'
|
|
83
|
+
description:
|
|
84
84
|
email:
|
|
85
85
|
- bill@dueber.com
|
|
86
86
|
executables: []
|
|
87
87
|
extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
|
89
89
|
files:
|
|
90
|
-
- ".github/workflows/
|
|
90
|
+
- ".github/workflows/tests.yml"
|
|
91
91
|
- ".gitignore"
|
|
92
92
|
- ".rspec"
|
|
93
93
|
- ".rubocop.yml"
|
|
@@ -110,7 +110,7 @@ metadata:
|
|
|
110
110
|
homepage_uri: https://github.com/billdueber/milemarker
|
|
111
111
|
source_code_uri: https://github.com/billdueber/milemarker
|
|
112
112
|
changelog_uri: https://github.com/billdueber/milemarker/CHANGELOG.md
|
|
113
|
-
post_install_message:
|
|
113
|
+
post_install_message:
|
|
114
114
|
rdoc_options: []
|
|
115
115
|
require_paths:
|
|
116
116
|
- lib
|
|
@@ -125,8 +125,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
125
125
|
- !ruby/object:Gem::Version
|
|
126
126
|
version: '0'
|
|
127
127
|
requirements: []
|
|
128
|
-
rubygems_version: 3.
|
|
129
|
-
signing_key:
|
|
128
|
+
rubygems_version: 3.3.5
|
|
129
|
+
signing_key:
|
|
130
130
|
specification_version: 4
|
|
131
131
|
summary: Track and produce loglines for batch processing progress.
|
|
132
132
|
test_files: []
|
data/.github/workflows/main.yml
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
name: Ruby
|
|
2
|
-
|
|
3
|
-
on: [push,pull_request]
|
|
4
|
-
|
|
5
|
-
jobs:
|
|
6
|
-
build:
|
|
7
|
-
runs-on: ubuntu-latest
|
|
8
|
-
steps:
|
|
9
|
-
- uses: actions/checkout@v2
|
|
10
|
-
- name: Set up Ruby
|
|
11
|
-
uses: ruby/setup-ruby@v1
|
|
12
|
-
with:
|
|
13
|
-
ruby-version: 2.6.6
|
|
14
|
-
bundler-cache: true
|
|
15
|
-
- name: Run the default task
|
|
16
|
-
run: bundle exec rake
|