lemans 1.0.0.pre.1 → 1.0.0.rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -5
- data/exe/lemans-remote +1 -1
- data/lib/lemans/cli.rb +1 -1
- data/lib/lemans/secrets_filter.rb +25 -0
- data/lib/lemans/stores/fs.rb +17 -4
- data/lib/lemans/trial/verifier/assets/eport-lemans.rb +9 -1
- data/lib/lemans/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 811f501d983a415eea28f7e987bf9d30f79537bba9f5adae25902dd9d252d253
|
|
4
|
+
data.tar.gz: 256e0bc81791b6b8dc35401985662e6c2ae52749704a31391158ac4b62c7b05f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6babf67abbe45e381a40cf4df59b3b8287cb9e14356de332513fab807ae9e52e38df248905036af2257b7b9e8792b436ee6bd0ce16df4abacfb023d445ccfc2b
|
|
7
|
+
data.tar.gz: 28cf1ca435238966d829b6057867a32c3a2f23165f1341bd8499d3d393c82461c7116ea4f410e6252035e1e7f1a3194be50673a00e410207af452da2f3c73d42
|
data/CHANGELOG.md
CHANGED
data/exe/lemans-remote
CHANGED
|
@@ -743,7 +743,7 @@ module LemansRemote # :nodoc: all
|
|
|
743
743
|
option :concurrency, type: :numeric, default: 4, aliases: "-C",
|
|
744
744
|
desc: "Async mode: how many sandboxes to launch in parallel (1 to serialize)"
|
|
745
745
|
def run_bench
|
|
746
|
-
bench = Lemans::
|
|
746
|
+
bench = Lemans::Config.load_file(options[:bench])
|
|
747
747
|
tasks = options[:task] || []
|
|
748
748
|
unknown = tasks - bench.tasks.map(&:name)
|
|
749
749
|
raise Thor::Error, "lemans-remote: no such task(s): #{unknown.join(", ")}" if unknown.any?
|
data/lib/lemans/cli.rb
CHANGED
|
@@ -54,7 +54,7 @@ module Lemans
|
|
|
54
54
|
|
|
55
55
|
tasks = filter_tasks(config.tasks, tags: options[:tag], name: options[:task])
|
|
56
56
|
|
|
57
|
-
store = Stores::FS.new(options[:runs_dir])
|
|
57
|
+
store = Stores::FS.new(options[:runs_dir], filterer: SecretsFilter.default)
|
|
58
58
|
|
|
59
59
|
runner = Runner.new(config, tasks, store:, resume: options[:resume])
|
|
60
60
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lemans
|
|
4
|
+
# Redacts known secret values (API credentials) from everything the store persists.
|
|
5
|
+
class SecretsFilter
|
|
6
|
+
# Credentials follow the <PROVIDER>_<KIND> environment convention
|
|
7
|
+
# (DAYTONA_API_KEY, OPENROUTER_API_KEY, DAYTONA_TOKEN, ...).
|
|
8
|
+
def self.default = new(ENV.filter_map { |name, value| value if name.match?(/_(API_KEY|TOKEN|SECRET|JWT)\z/) })
|
|
9
|
+
|
|
10
|
+
def initialize(secrets, replacement: "<filtered>", min_length: 8)
|
|
11
|
+
@replacement = replacement
|
|
12
|
+
|
|
13
|
+
secrets = secrets.compact.uniq.select { it.length >= min_length }
|
|
14
|
+
# Match bytes, not characters, so artifacts with non-UTF-8 content
|
|
15
|
+
# (binary patch hunks) don't raise on scanning.
|
|
16
|
+
@pattern = Regexp.union(secrets.map(&:b)) unless secrets.empty?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def filter(text)
|
|
20
|
+
return text unless @pattern
|
|
21
|
+
|
|
22
|
+
text.b.gsub(@pattern, @replacement)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
data/lib/lemans/stores/fs.rb
CHANGED
|
@@ -11,11 +11,12 @@ module Lemans
|
|
|
11
11
|
class FS < Store
|
|
12
12
|
FILENAME = "result.json"
|
|
13
13
|
|
|
14
|
-
private attr_reader :root
|
|
14
|
+
private attr_reader :root, :filterer
|
|
15
15
|
|
|
16
|
-
def initialize(root)
|
|
16
|
+
def initialize(root, filterer: nil)
|
|
17
17
|
super()
|
|
18
18
|
@root = Pathname(root)
|
|
19
|
+
@filterer = filterer
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def setup
|
|
@@ -62,7 +63,7 @@ module Lemans
|
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
def save(result)
|
|
65
|
-
atomic_write(result_dir(result).join(FILENAME), "#{JSON.pretty_generate(result.as_json)}\n")
|
|
66
|
+
atomic_write(result_dir(result).join(FILENAME), filtered("#{JSON.pretty_generate(result.as_json)}\n"))
|
|
66
67
|
rescue SystemCallError, JSON::GeneratorError => e
|
|
67
68
|
# runs_dir unwritable, disk full
|
|
68
69
|
raise ConfigError, "cannot record trial #{result.id}: #{e.message}"
|
|
@@ -76,7 +77,7 @@ module Lemans
|
|
|
76
77
|
end
|
|
77
78
|
|
|
78
79
|
destination.dirname.mkpath
|
|
79
|
-
contents.is_a?(String) ? destination.write(contents) :
|
|
80
|
+
contents.is_a?(String) ? destination.write(filtered(contents)) : copy_filtered(contents, destination)
|
|
80
81
|
destination
|
|
81
82
|
rescue SystemCallError => e
|
|
82
83
|
warn "lemans: could not save artifact #{path} for #{result.id}: #{e.message}"
|
|
@@ -85,6 +86,18 @@ module Lemans
|
|
|
85
86
|
|
|
86
87
|
private
|
|
87
88
|
|
|
89
|
+
def filtered(text) = filterer ? filterer.filter(text) : text
|
|
90
|
+
|
|
91
|
+
# Secrets never span lines, so filtering line by line keeps memory
|
|
92
|
+
# constant without a plain-text copy ever touching the disk.
|
|
93
|
+
def copy_filtered(contents, destination)
|
|
94
|
+
return IO.copy_stream(contents, destination) unless filterer
|
|
95
|
+
|
|
96
|
+
destination.open("wb") do |file|
|
|
97
|
+
contents.each_line { file.write(filterer.filter(it)) }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
88
101
|
def prune_empty_parents(dir)
|
|
89
102
|
while dir != root && dir.children.empty?
|
|
90
103
|
dir.rmdir
|
|
@@ -22,12 +22,20 @@ end
|
|
|
22
22
|
if defined?(Minitest)
|
|
23
23
|
LemansReport.register
|
|
24
24
|
else
|
|
25
|
+
module LemansReport # :nodoc:
|
|
26
|
+
class << self
|
|
27
|
+
attr_accessor :name_method
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
self.name_method = Module.instance_method(:name)
|
|
31
|
+
end
|
|
32
|
+
|
|
25
33
|
# `-r` runs before bundler picks the app's minitest, so requiring minitest
|
|
26
34
|
# here would activate the wrong version. Instead watch class definitions and
|
|
27
35
|
# register the moment minitest's own module body closes; the probe disarms
|
|
28
36
|
# itself and nothing foreign is patched.
|
|
29
37
|
trace = TracePoint.new(:end) do |event|
|
|
30
|
-
next unless event.self.is_a?(Module) && event.self
|
|
38
|
+
next unless event.self.is_a?(Module) && LemansReport.name_method.bind_call(event.self) == "Minitest"
|
|
31
39
|
|
|
32
40
|
LemansReport.register
|
|
33
41
|
trace.disable if LemansReport.registered?
|
data/lib/lemans/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lemans
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.0.
|
|
4
|
+
version: 1.0.0.rc.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Svyatoslav Kryukov
|
|
@@ -186,6 +186,7 @@ files:
|
|
|
186
186
|
- lib/lemans/runner.rb
|
|
187
187
|
- lib/lemans/runner/executor.rb
|
|
188
188
|
- lib/lemans/runner/task.rb
|
|
189
|
+
- lib/lemans/secrets_filter.rb
|
|
189
190
|
- lib/lemans/store.rb
|
|
190
191
|
- lib/lemans/stores/fs.rb
|
|
191
192
|
- lib/lemans/task_definition.rb
|