specwrk 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -6
- data/docker/pitchfork.conf +2 -2
- data/lib/specwrk/cli.rb +6 -4
- data/lib/specwrk/client.rb +12 -2
- data/lib/specwrk/store/base_adapter.rb +62 -0
- data/lib/specwrk/store/file_adapter.rb +44 -25
- data/lib/specwrk/store/memory_adapter.rb +59 -0
- data/lib/specwrk/store.rb +45 -11
- data/lib/specwrk/version.rb +1 -1
- data/lib/specwrk/web/endpoints.rb +57 -53
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f5fa0464c4d73ad78593cd2ce5626dac594bb74721d18303f1fee4d33e28615
|
4
|
+
data.tar.gz: 262e478389d7dd07375fc5a864012c992709199838a2f32f7e72e09bbb412759
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21fda87f804947653887bcd897c45c97ebdc9227a63bc19019b137e01955dc96701f202f1ebe8a472ed3b871e799a52f54ab9b2872f4f76a2ebefc4e411801df
|
7
|
+
data.tar.gz: 7a6a62993e9a1262a50c291021fdb1ec6662ed36453bc0ec3855aa7a01735647943500e354a6705c1b6e1f190e3bd0f297bd2317cc639d9fa576a115c4061e30
|
data/README.md
CHANGED
@@ -51,14 +51,15 @@ Options:
|
|
51
51
|
--key=VALUE, -k VALUE # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
|
52
52
|
--run=VALUE, -r VALUE # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
|
53
53
|
--timeout=VALUE, -t VALUE # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
|
54
|
-
--id=VALUE # The identifier for this worker.
|
54
|
+
--id=VALUE # The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used
|
55
55
|
--count=VALUE, -c VALUE # The number of worker processes you want to start, default: 1
|
56
56
|
--output=VALUE, -o VALUE # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
|
57
57
|
--seed-waits=VALUE, -w VALUE # Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS, default: "10"
|
58
58
|
--port=VALUE, -p VALUE # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
|
59
59
|
--bind=VALUE, -b VALUE # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
|
60
|
+
--store-uri=VALUE # Directory where server state is stored. Required for multi-node or multi-process servers.
|
60
61
|
--group-by=VALUE # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
|
61
|
-
--[no-]verbose # Run in verbose mode
|
62
|
+
--[no-]verbose # Run in verbose mode, default: false
|
62
63
|
--help, -h # Print this help
|
63
64
|
```
|
64
65
|
|
@@ -80,10 +81,11 @@ Options:
|
|
80
81
|
--port=VALUE, -p VALUE # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
|
81
82
|
--bind=VALUE, -b VALUE # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
|
82
83
|
--key=VALUE, -k VALUE # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
|
83
|
-
--output=VALUE, -o VALUE # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
|
84
|
+
--output=VALUE, -o VALUE # Directory where worker or server output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
|
85
|
+
--store-uri=VALUE # Directory where server state is stored. Required for multi-node or multi-process servers.
|
84
86
|
--group-by=VALUE # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
|
85
|
-
--[no-]verbose # Run in verbose mode
|
86
|
-
--[no-]single-run # Act on shutdown requests from clients
|
87
|
+
--[no-]verbose # Run in verbose mode, default: false
|
88
|
+
--[no-]single-run # Act on shutdown requests from clients, default: false
|
87
89
|
--help, -h # Print this help
|
88
90
|
```
|
89
91
|
|
@@ -196,7 +198,7 @@ Start a persistent Queue Server given one of the following methods
|
|
196
198
|
|
197
199
|
### Configuring your Queue Server
|
198
200
|
- Secure your server with a key either with the `SPECWRK_SRV_KEY` environment variable or `--key` CLI option
|
199
|
-
- Configure the server output to be a persisted volume so your timings survive between restarts with
|
201
|
+
- Configure the server output to be a persisted volume so your timings survive between system restarts with the `SPECWRK_SRV_STORE_URI` environment variable or `--store-uri` CLI option. By default, `memory:///` will be used for the run's data stores (so run data will no survive server restarts) while `file://#{Dir.tmpdir}` will be used for run timings. Pass `--store-uri file:///whatever/absolute/path` to store all data on disk (required for multiple server processes).
|
200
202
|
|
201
203
|
See [specwrk serve --help](#specwrk-serve) for all possible configuration options.
|
202
204
|
|
data/docker/pitchfork.conf
CHANGED
data/lib/specwrk/cli.rb
CHANGED
@@ -74,13 +74,15 @@ module Specwrk
|
|
74
74
|
base.unique_option :port, type: :integer, default: ENV.fetch("SPECWRK_SRV_PORT", "5138"), aliases: ["-p"], desc: "Server port. Overrides SPECWRK_SRV_PORT"
|
75
75
|
base.unique_option :bind, type: :string, default: ENV.fetch("SPECWRK_SRV_BIND", "127.0.0.1"), aliases: ["-b"], desc: "Server bind address. Overrides SPECWRK_SRV_BIND"
|
76
76
|
base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
|
77
|
-
base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
|
77
|
+
base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker or server output is stored. Overrides SPECWRK_OUT"
|
78
|
+
base.unique_option :store_uri, type: :string, desc: "Directory where server state is stored. Required for multi-node or multi-process servers."
|
78
79
|
base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
|
79
|
-
base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode
|
80
|
+
base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode"
|
80
81
|
end
|
81
82
|
|
82
|
-
on_setup do |port:, bind:, output:, key:, group_by:, verbose:,
|
83
|
+
on_setup do |port:, bind:, output:, key:, group_by:, verbose:, **opts|
|
83
84
|
ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
|
85
|
+
ENV["SPECWRK_SRV_STORE_URI"] = opts[:store_uri] if opts.key? :store_uri
|
84
86
|
ENV["SPECWRK_SRV_VERBOSE"] = "1" if verbose
|
85
87
|
|
86
88
|
ENV["SPECWRK_SRV_PORT"] = port
|
@@ -158,7 +160,7 @@ module Specwrk
|
|
158
160
|
include Servable
|
159
161
|
|
160
162
|
desc "Start a queue server"
|
161
|
-
option :single_run, type: :boolean, default: false, desc: "Act on shutdown requests from clients
|
163
|
+
option :single_run, type: :boolean, default: false, desc: "Act on shutdown requests from clients"
|
162
164
|
|
163
165
|
def call(single_run:, **args)
|
164
166
|
ENV["SPECWRK_SRV_SINGLE_RUN"] = "1" if single_run
|
data/lib/specwrk/client.rb
CHANGED
@@ -44,7 +44,7 @@ module Specwrk
|
|
44
44
|
raise Errno::ECONNREFUSED unless connected
|
45
45
|
end
|
46
46
|
|
47
|
-
attr_reader :last_request_at
|
47
|
+
attr_reader :last_request_at, :retry_count
|
48
48
|
|
49
49
|
def initialize
|
50
50
|
@mutex = Mutex.new
|
@@ -120,6 +120,16 @@ module Specwrk
|
|
120
120
|
else
|
121
121
|
raise UnhandledResponseError.new("#{response.code}: #{response.body}")
|
122
122
|
end
|
123
|
+
rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout => e
|
124
|
+
@retry_count ||= 0
|
125
|
+
@retry_count += 1
|
126
|
+
|
127
|
+
raise e if @retry_count == 5
|
128
|
+
|
129
|
+
warn e
|
130
|
+
sleep @retry_count
|
131
|
+
|
132
|
+
retry
|
123
133
|
end
|
124
134
|
|
125
135
|
def seed(examples)
|
@@ -161,7 +171,7 @@ module Specwrk
|
|
161
171
|
def make_request(request)
|
162
172
|
@mutex.synchronize do
|
163
173
|
@last_request_at = Time.now
|
164
|
-
@http.request(request)
|
174
|
+
@http.request(request).tap { @retry_count = 0 }
|
165
175
|
end
|
166
176
|
end
|
167
177
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "uri"
|
4
|
+
|
5
|
+
require "specwrk/store"
|
6
|
+
|
7
|
+
module Specwrk
|
8
|
+
class Store
|
9
|
+
class BaseAdapter
|
10
|
+
class << self
|
11
|
+
def with_lock(_uri, _key)
|
12
|
+
yield
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(uri, scope)
|
17
|
+
@uri = uri
|
18
|
+
@scope = scope
|
19
|
+
end
|
20
|
+
|
21
|
+
def [](key)
|
22
|
+
raise "Not implemented"
|
23
|
+
end
|
24
|
+
|
25
|
+
def []=(key, value)
|
26
|
+
raise "Not implemented"
|
27
|
+
end
|
28
|
+
|
29
|
+
def keys
|
30
|
+
raise "Not implemented"
|
31
|
+
end
|
32
|
+
|
33
|
+
def clear
|
34
|
+
raise "Not implemented"
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete(*keys)
|
38
|
+
raise "Not implemented"
|
39
|
+
end
|
40
|
+
|
41
|
+
def merge!(h2)
|
42
|
+
raise "Not implemented"
|
43
|
+
end
|
44
|
+
|
45
|
+
def multi_read(*read_keys)
|
46
|
+
raise "Not implemented"
|
47
|
+
end
|
48
|
+
|
49
|
+
def multi_write(hash)
|
50
|
+
raise "Not implemented"
|
51
|
+
end
|
52
|
+
|
53
|
+
def empty?
|
54
|
+
raise "Not implemented"
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
attr_reader :uri, :scope
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -3,34 +3,47 @@
|
|
3
3
|
require "json"
|
4
4
|
require "base64"
|
5
5
|
|
6
|
-
require "specwrk/store"
|
6
|
+
require "specwrk/store/base_adapter"
|
7
7
|
|
8
8
|
module Specwrk
|
9
9
|
class Store
|
10
|
-
class FileAdapter
|
10
|
+
class FileAdapter < BaseAdapter
|
11
11
|
EXT = ".wrk.json"
|
12
12
|
|
13
|
-
|
14
|
-
|
13
|
+
@work_queue = Queue.new
|
14
|
+
@threads = []
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
16
|
+
class << self
|
17
|
+
def with_lock(uri, key)
|
18
|
+
lock_file_path = File.join(uri.path, "#{key}.lock").tap do |path|
|
19
|
+
FileUtils.mkdir_p(uri.path)
|
21
20
|
end
|
21
|
+
|
22
|
+
lock_file = File.open(lock_file_path, "a")
|
23
|
+
|
24
|
+
Thread.pass until lock_file.flock(File::LOCK_EX)
|
25
|
+
|
26
|
+
yield
|
27
|
+
ensure
|
28
|
+
lock_file.flock(File::LOCK_UN)
|
22
29
|
end
|
23
30
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
end
|
31
|
+
def schedule_work(&blk)
|
32
|
+
start_threads!
|
33
|
+
@work_queue.push blk
|
28
34
|
end
|
29
|
-
end
|
30
35
|
|
31
|
-
|
32
|
-
|
33
|
-
|
36
|
+
def start_threads!
|
37
|
+
return if @threads.length.positive?
|
38
|
+
|
39
|
+
Array.new(ENV.fetch("SPECWRK_THREAD_COUNT", "4").to_i) do
|
40
|
+
@threads << Thread.new do
|
41
|
+
loop do
|
42
|
+
@work_queue.pop.call
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
34
47
|
end
|
35
48
|
|
36
49
|
def [](key)
|
@@ -56,8 +69,8 @@ module Specwrk
|
|
56
69
|
end
|
57
70
|
|
58
71
|
def clear
|
59
|
-
FileUtils.rm_rf(
|
60
|
-
FileUtils.mkdir_p(
|
72
|
+
FileUtils.rm_rf(path)
|
73
|
+
FileUtils.mkdir_p(path)
|
61
74
|
|
62
75
|
@known_key_pairs = nil
|
63
76
|
end
|
@@ -80,7 +93,7 @@ module Specwrk
|
|
80
93
|
result_queue = Queue.new
|
81
94
|
|
82
95
|
read_keys.each do |key|
|
83
|
-
|
96
|
+
self.class.schedule_work do
|
84
97
|
result_queue.push([key.to_s, read(key)])
|
85
98
|
end
|
86
99
|
end
|
@@ -107,7 +120,7 @@ module Specwrk
|
|
107
120
|
hash_with_filenames.each do |key, (filename, value)|
|
108
121
|
content = JSON.generate(value)
|
109
122
|
|
110
|
-
|
123
|
+
self.class.schedule_work do
|
111
124
|
result_queue << write(filename, content)
|
112
125
|
end
|
113
126
|
end
|
@@ -117,7 +130,7 @@ module Specwrk
|
|
117
130
|
end
|
118
131
|
|
119
132
|
def empty?
|
120
|
-
Dir.empty?
|
133
|
+
Dir.empty? path
|
121
134
|
end
|
122
135
|
|
123
136
|
private
|
@@ -137,7 +150,7 @@ module Specwrk
|
|
137
150
|
|
138
151
|
def filename_for_key(key)
|
139
152
|
File.join(
|
140
|
-
|
153
|
+
path,
|
141
154
|
[
|
142
155
|
counter_prefix(key),
|
143
156
|
encode_key(key)
|
@@ -155,6 +168,12 @@ module Specwrk
|
|
155
168
|
@counter ||= keys.length
|
156
169
|
end
|
157
170
|
|
171
|
+
def path
|
172
|
+
@path ||= File.join(uri.path, scope).tap do |full_path|
|
173
|
+
FileUtils.mkdir_p(full_path)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
158
177
|
def encode_key(key)
|
159
178
|
Base64.urlsafe_encode64(key).delete("=")
|
160
179
|
end
|
@@ -167,11 +186,11 @@ module Specwrk
|
|
167
186
|
end
|
168
187
|
|
169
188
|
def known_key_pairs
|
170
|
-
@known_key_pairs ||= Dir.entries(
|
189
|
+
@known_key_pairs ||= Dir.entries(path).sort.map do |filename|
|
171
190
|
next if filename.start_with? "."
|
172
191
|
next unless filename.end_with? EXT
|
173
192
|
|
174
|
-
file_path = File.join(
|
193
|
+
file_path = File.join(path, filename)
|
175
194
|
[decode_key(filename), file_path]
|
176
195
|
end.compact.to_h
|
177
196
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "specwrk/store/base_adapter"
|
4
|
+
|
5
|
+
module Specwrk
|
6
|
+
class Store
|
7
|
+
class MemoryAdapter < BaseAdapter
|
8
|
+
@@stores = Hash.new { |hash, key| hash[key] = {} }
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def clear
|
12
|
+
@@stores.values.each(&:clear)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def [](key)
|
17
|
+
store[key]
|
18
|
+
end
|
19
|
+
|
20
|
+
def []=(key, value)
|
21
|
+
store[key] = value
|
22
|
+
end
|
23
|
+
|
24
|
+
def keys
|
25
|
+
store.keys
|
26
|
+
end
|
27
|
+
|
28
|
+
def clear
|
29
|
+
store.clear
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete(*keys)
|
33
|
+
keys.each { |key| store.delete(key) }
|
34
|
+
end
|
35
|
+
|
36
|
+
def merge!(h2)
|
37
|
+
store.merge!(h2)
|
38
|
+
end
|
39
|
+
|
40
|
+
def multi_read(*read_keys)
|
41
|
+
store.slice(*read_keys)
|
42
|
+
end
|
43
|
+
|
44
|
+
def multi_write(hash)
|
45
|
+
merge!(hash)
|
46
|
+
end
|
47
|
+
|
48
|
+
def empty?
|
49
|
+
store.keys.length.zero?
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def store
|
55
|
+
@store ||= @@stores[scope]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/specwrk/store.rb
CHANGED
@@ -1,14 +1,31 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "time"
|
4
|
-
require "json"
|
5
|
-
|
6
|
-
require "specwrk/store/file_adapter"
|
7
4
|
|
8
5
|
module Specwrk
|
9
6
|
class Store
|
10
|
-
|
11
|
-
|
7
|
+
class << self
|
8
|
+
def with_lock(uri, key)
|
9
|
+
adapter_klass(uri).with_lock(uri, key) { yield }
|
10
|
+
end
|
11
|
+
|
12
|
+
def adapter_klass(uri)
|
13
|
+
case uri.scheme
|
14
|
+
when "memory"
|
15
|
+
require "specwrk/store/memory_adapter" unless defined?(MemoryAdapter)
|
16
|
+
|
17
|
+
MemoryAdapter
|
18
|
+
when "file"
|
19
|
+
require "specwrk/store/file_adapter" unless defined?(FileAdapter)
|
20
|
+
|
21
|
+
FileAdapter
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(uri_string, scope)
|
27
|
+
@uri = URI(uri_string)
|
28
|
+
@scope = scope
|
12
29
|
end
|
13
30
|
|
14
31
|
def [](key)
|
@@ -72,12 +89,10 @@ module Specwrk
|
|
72
89
|
|
73
90
|
private
|
74
91
|
|
75
|
-
|
76
|
-
@adapter ||= FileAdapter.new(@path)
|
77
|
-
end
|
92
|
+
attr_reader :uri, :scope
|
78
93
|
|
79
|
-
def
|
80
|
-
@
|
94
|
+
def adapter
|
95
|
+
@adapter ||= self.class.adapter_klass(uri).new uri, scope
|
81
96
|
end
|
82
97
|
end
|
83
98
|
|
@@ -142,7 +157,7 @@ module Specwrk
|
|
142
157
|
estimated_run_time_total = 0
|
143
158
|
|
144
159
|
catch(:full) do
|
145
|
-
keys.each_slice(
|
160
|
+
keys.each_slice(24).each do |key_group|
|
146
161
|
examples = multi_read(*key_group)
|
147
162
|
|
148
163
|
examples.each do |key, example|
|
@@ -160,6 +175,25 @@ module Specwrk
|
|
160
175
|
end
|
161
176
|
end
|
162
177
|
|
178
|
+
class ProcessingStore < Store
|
179
|
+
def expired
|
180
|
+
@expired ||= begin
|
181
|
+
bucket = []
|
182
|
+
|
183
|
+
keys.each_slice(24).each do |key_group|
|
184
|
+
examples = multi_read(*key_group)
|
185
|
+
examples.each do |id, example|
|
186
|
+
next if example[:completion_threshold].nil?
|
187
|
+
|
188
|
+
bucket << [id, example] if example[:completion_threshold] < Time.now.to_i
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
bucket.to_h
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
163
197
|
class CompletedStore < Store
|
164
198
|
def dump
|
165
199
|
@run_times = []
|
data/lib/specwrk/version.rb
CHANGED
@@ -51,11 +51,19 @@ module Specwrk
|
|
51
51
|
end
|
52
52
|
|
53
53
|
def not_found
|
54
|
-
|
54
|
+
if request.head?
|
55
|
+
[404, {}, []]
|
56
|
+
else
|
57
|
+
[404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
|
58
|
+
end
|
55
59
|
end
|
56
60
|
|
57
61
|
def ok
|
58
|
-
|
62
|
+
if request.head?
|
63
|
+
[200, {}, []]
|
64
|
+
else
|
65
|
+
[200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
|
66
|
+
end
|
59
67
|
end
|
60
68
|
|
61
69
|
def payload
|
@@ -71,52 +79,35 @@ module Specwrk
|
|
71
79
|
end
|
72
80
|
|
73
81
|
def pending
|
74
|
-
@pending ||= PendingStore.new(File.join(
|
82
|
+
@pending ||= PendingStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "pending"))
|
75
83
|
end
|
76
84
|
|
77
85
|
def processing
|
78
|
-
@processing ||=
|
86
|
+
@processing ||= ProcessingStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "processing"))
|
79
87
|
end
|
80
88
|
|
81
89
|
def completed
|
82
|
-
@completed ||= CompletedStore.new(File.join(
|
90
|
+
@completed ||= CompletedStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "completed"))
|
83
91
|
end
|
84
92
|
|
85
93
|
def metadata
|
86
|
-
@metadata ||= Store.new(File.join(
|
94
|
+
@metadata ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "metadata"))
|
87
95
|
end
|
88
96
|
|
89
97
|
def run_times
|
90
|
-
@run_times ||= Store.new(File.join(
|
98
|
+
@run_times ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "file://#{File.join(Dir.tmpdir, "specwrk")}"), "run_times")
|
91
99
|
end
|
92
100
|
|
93
101
|
def worker
|
94
|
-
@worker ||= Store.new(File.join(
|
102
|
+
@worker ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
|
95
103
|
end
|
96
104
|
|
97
105
|
def run_id
|
98
106
|
request.get_header("HTTP_X_SPECWRK_RUN")
|
99
107
|
end
|
100
108
|
|
101
|
-
def run_report_file_path
|
102
|
-
@run_report_file_path ||= File.join(datastore_path, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
|
103
|
-
end
|
104
|
-
|
105
|
-
def datastore_path
|
106
|
-
@datastore_path ||= File.join(ENV["SPECWRK_OUT"], run_id).to_s.tap do |path|
|
107
|
-
FileUtils.mkdir_p(path) unless File.directory?(path)
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
109
|
def with_lock
|
112
|
-
|
113
|
-
yield
|
114
|
-
ensure
|
115
|
-
lock_file.flock(File::LOCK_UN)
|
116
|
-
end
|
117
|
-
|
118
|
-
def lock_file
|
119
|
-
@lock_file ||= File.open(File.join(datastore_path, "lock"), "a")
|
110
|
+
Store.with_lock(URI(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///")), "server") { yield }
|
120
111
|
end
|
121
112
|
end
|
122
113
|
|
@@ -125,7 +116,7 @@ module Specwrk
|
|
125
116
|
|
126
117
|
class Health < Base
|
127
118
|
def with_response
|
128
|
-
|
119
|
+
ok
|
129
120
|
end
|
130
121
|
end
|
131
122
|
|
@@ -222,45 +213,59 @@ module Specwrk
|
|
222
213
|
end
|
223
214
|
end
|
224
215
|
|
225
|
-
class
|
226
|
-
|
227
|
-
@examples = pending.shift_bucket
|
228
|
-
|
229
|
-
processing_data = @examples.map { |example| [example[:id], example] }.to_h
|
230
|
-
processing.merge!(processing_data)
|
216
|
+
class Popable < Base
|
217
|
+
private
|
231
218
|
|
232
|
-
|
233
|
-
|
219
|
+
def with_pop_response
|
220
|
+
if examples.any?
|
221
|
+
[200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
|
234
222
|
elsif pending.empty? && processing.empty? && completed.empty?
|
235
223
|
[204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
|
236
224
|
elsif completed.any? && processing.empty?
|
237
225
|
[410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
|
226
|
+
elsif processing.any? && processing.expired.keys.any?
|
227
|
+
pending.merge!(processing.expired)
|
228
|
+
processing.delete(*processing.expired.keys)
|
229
|
+
@examples = nil
|
230
|
+
|
231
|
+
[200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
|
238
232
|
else
|
239
233
|
not_found
|
240
234
|
end
|
241
235
|
end
|
236
|
+
|
237
|
+
def examples
|
238
|
+
@examples ||= begin
|
239
|
+
examples = pending.shift_bucket
|
240
|
+
maximum_completion_threshold = (Time.now + ((pending.run_time_bucket_maximum || 30) * 2)).to_i
|
241
|
+
|
242
|
+
processing_data = examples.map do |example|
|
243
|
+
example_run_time_completion_threshold = (Time.now + example[:expected_run_time].to_f * 2).to_i
|
244
|
+
|
245
|
+
[
|
246
|
+
example[:id], example.merge(completion_threshold: [maximum_completion_threshold, example_run_time_completion_threshold].compact.max)
|
247
|
+
]
|
248
|
+
end
|
249
|
+
|
250
|
+
processing.merge!(processing_data.to_h)
|
251
|
+
|
252
|
+
examples
|
253
|
+
end
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
class Pop < Popable
|
258
|
+
def with_response
|
259
|
+
with_pop_response
|
260
|
+
end
|
242
261
|
end
|
243
262
|
|
244
|
-
class CompleteAndPop <
|
263
|
+
class CompleteAndPop < Popable
|
245
264
|
def with_response
|
246
265
|
completed.merge!(completed_examples)
|
247
|
-
run_times.merge! run_time_data
|
248
266
|
processing.delete(*completed_examples.keys)
|
249
267
|
|
250
|
-
|
251
|
-
|
252
|
-
processing_data = @examples.map { |example| [example[:id], example] }.to_h
|
253
|
-
processing.merge!(processing_data)
|
254
|
-
|
255
|
-
if @examples.any?
|
256
|
-
[200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
|
257
|
-
elsif pending.empty? && processing.empty? && completed.empty?
|
258
|
-
[204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
|
259
|
-
elsif completed.any? && processing.empty?
|
260
|
-
[410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
|
261
|
-
else
|
262
|
-
not_found
|
263
|
-
end
|
268
|
+
with_pop_response
|
264
269
|
end
|
265
270
|
|
266
271
|
private
|
@@ -277,8 +282,7 @@ module Specwrk
|
|
277
282
|
# We don't care about exact values here, just approximate run times are fine
|
278
283
|
# So if we overwrite run times from another process it is nbd
|
279
284
|
def after_lock
|
280
|
-
|
281
|
-
# run_times.merge! run_time_data
|
285
|
+
run_times.merge! run_time_data
|
282
286
|
end
|
283
287
|
|
284
288
|
def run_time_data
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: specwrk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Westendorf
|
@@ -192,7 +192,9 @@ files:
|
|
192
192
|
- lib/specwrk/hookable.rb
|
193
193
|
- lib/specwrk/list_examples.rb
|
194
194
|
- lib/specwrk/store.rb
|
195
|
+
- lib/specwrk/store/base_adapter.rb
|
195
196
|
- lib/specwrk/store/file_adapter.rb
|
197
|
+
- lib/specwrk/store/memory_adapter.rb
|
196
198
|
- lib/specwrk/version.rb
|
197
199
|
- lib/specwrk/web.rb
|
198
200
|
- lib/specwrk/web/app.rb
|