asgard 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +272 -99
- data/README.md +147 -46
- data/bin/asgard +10 -0
- data/lib/asgard/base/dependency_graph.rb +19 -11
- data/lib/asgard/base/dispatch.rb +272 -32
- data/lib/asgard/base/registry.rb +7 -3
- data/lib/asgard/base/task_dsl.rb +6 -0
- data/lib/asgard/base.rb +147 -9
- data/lib/asgard/declarations.rb +21 -0
- data/lib/asgard/doctor/report.rb +5 -1
- data/lib/asgard/doctor/task_sections.rb +11 -3
- data/lib/asgard/doctor.rb +41 -31
- data/lib/asgard/jobs/commands.rb +183 -0
- data/lib/asgard/jobs/declaration.rb +149 -0
- data/lib/asgard/jobs/launcher.rb +109 -0
- data/lib/asgard/jobs/state.rb +195 -0
- data/lib/asgard/jobs/stopper.rb +141 -0
- data/lib/asgard/jobs.rb +25 -0
- data/lib/asgard/kernel_methods.rb +25 -30
- data/lib/asgard/leading_flag.rb +26 -0
- data/lib/asgard/log.rb +90 -0
- data/lib/asgard/pid.rb +113 -0
- data/lib/asgard/schedule/commands.rb +68 -18
- data/lib/asgard/schedule/declaration.rb +158 -26
- data/lib/asgard/schedule/job_files.rb +24 -0
- data/lib/asgard/schedule/launchd.rb +57 -21
- data/lib/asgard/schedule/systemd.rb +72 -28
- data/lib/asgard/schedule/table.rb +48 -26
- data/lib/asgard/schedule.rb +49 -9
- data/lib/asgard/shell.rb +366 -44
- data/lib/asgard/state_dir.rb +90 -0
- data/lib/asgard/tasks.rb +8 -0
- data/lib/asgard/version.rb +1 -1
- data/lib/asgard/watch/changes.rb +26 -1
- data/lib/asgard/watch/commands.rb +63 -13
- data/lib/asgard/watch/daemon.rb +347 -52
- data/lib/asgard/watch/declaration.rb +135 -40
- data/lib/asgard/watch/dispatcher.rb +61 -10
- data/lib/asgard/watch/mailbox.rb +50 -0
- data/lib/asgard/watch/reporter.rb +54 -32
- data/lib/asgard/watch/spawn.rb +43 -18
- data/lib/asgard/watch/state.rb +93 -27
- data/lib/asgard/watch/worker.rb +44 -9
- data/lib/asgard/watch.rb +4 -11
- data/lib/asgard.rb +100 -16
- metadata +170 -186
- data/.envrc +0 -6
- data/.github/workflows/deploy-github-pages.yml +0 -52
- data/.loki +0 -39
- data/.reek.yml +0 -166
- data/.rubocop.yml +0 -163
- data/Archspec.rb +0 -24
- data/CLAUDE.md +0 -139
- data/COMMITS.md +0 -196
- data/bin/console +0 -11
- data/bin/setup +0 -8
- data/doc_tasks.loki +0 -21
- data/docs/api.md +0 -368
- data/docs/assets/css/custom.css +0 -93
- data/docs/assets/images/asgard.jpg +0 -0
- data/docs/changelog.md +0 -1
- data/docs/dependencies.md +0 -313
- data/docs/environment.md +0 -129
- data/docs/examples.md +0 -194
- data/docs/getting-started.md +0 -181
- data/docs/helpers.md +0 -262
- data/docs/index.md +0 -87
- data/docs/options.md +0 -224
- data/docs/schedule.md +0 -106
- data/docs/shell.md +0 -237
- data/docs/subcommands.md +0 -181
- data/docs/task-files.md +0 -407
- data/docs/tasks.md +0 -326
- data/docs/variables.md +0 -338
- data/docs/watch.md +0 -123
- data/examples/.env +0 -4
- data/examples/.envrc +0 -3
- data/examples/.loki +0 -24
- data/examples/bad.loki +0 -63
- data/examples/concurrent.loki +0 -58
- data/examples/confirm.loki +0 -48
- data/examples/db_subcommands.loki +0 -74
- data/examples/depends_on_block/bad/.loki +0 -51
- data/examples/depends_on_block/good/.loki +0 -83
- data/examples/do.loki +0 -45
- data/examples/env_usage.loki +0 -27
- data/examples/kitchen_sink.loki +0 -210
- data/examples/server_subcommands.loki +0 -65
- data/examples/subdir/.loki +0 -12
- data/examples/subdir/import_demo.loki +0 -14
- data/examples/subdir/import_up_demo.loki +0 -18
- data/examples/watcher_examples/.envrc +0 -3
- data/examples/watcher_examples/.loki +0 -54
- data/examples/watcher_examples/README.md +0 -112
- data/examples/watcher_examples/bin/hello +0 -3
- data/examples/watcher_examples/data/records.json +0 -1
- data/examples/watcher_examples/docs/drafts/ideas.md +0 -5
- data/examples/watcher_examples/docs/index.md +0 -4
- data/examples/watcher_examples/lib/greeter.rb +0 -7
- data/gem_tasks.loki +0 -51
- data/git.loki +0 -13
- data/mkdocs.yml +0 -166
- data/quality.loki +0 -266
- data/quality_rails.loki +0 -47
- data/xyzzy.loki +0 -12
|
@@ -7,6 +7,14 @@ module Asgard
|
|
|
7
7
|
# more than one recorded location (the each/each_gem class of bug) —
|
|
8
8
|
# earlier ones OVERRIDDEN (dead), the last one the active redefinition.
|
|
9
9
|
module TaskSections
|
|
10
|
+
# A hidden task (one defined without a desc) says so, after any
|
|
11
|
+
# override note.
|
|
12
|
+
def self.with_hidden(status, hidden)
|
|
13
|
+
return status unless hidden
|
|
14
|
+
|
|
15
|
+
[status, "hidden (no desc; not shown by help)"].compact.join("; ")
|
|
16
|
+
end
|
|
17
|
+
|
|
10
18
|
private
|
|
11
19
|
|
|
12
20
|
def build_task_sections
|
|
@@ -20,12 +28,12 @@ module Asgard
|
|
|
20
28
|
file_map = Hash.new { |h, k| h[k] = [] }
|
|
21
29
|
@loaded_files.each { |f| file_map[f] }
|
|
22
30
|
|
|
23
|
-
commands = klass.all_commands
|
|
31
|
+
commands = klass.all_commands
|
|
24
32
|
klass._method_log.each do |name, locations|
|
|
25
|
-
|
|
33
|
+
command = commands[name.to_s] or next
|
|
26
34
|
|
|
27
35
|
locations.each_with_index do |(file, line), idx|
|
|
28
|
-
file_map[file] << [name, line, task_status(locations, idx)]
|
|
36
|
+
file_map[file] << [name, line, TaskSections.with_hidden(task_status(locations, idx), command.hidden?)]
|
|
29
37
|
end
|
|
30
38
|
end
|
|
31
39
|
file_map.each_value { |tasks| tasks.sort_by! { |t| t[1] } }
|
data/lib/asgard/doctor.rb
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "pathname"
|
|
4
|
-
|
|
5
3
|
require_relative "doctor/task_sections"
|
|
6
4
|
require_relative "doctor/report"
|
|
7
5
|
|
|
@@ -31,23 +29,14 @@ module Asgard
|
|
|
31
29
|
build_task_sections
|
|
32
30
|
report_dependencies
|
|
33
31
|
print_report
|
|
32
|
+
problem_count.zero? ? 0 : 1
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
# Every ".loki" marker from +dir+ up to the filesystem root, nearest
|
|
37
|
-
# first —
|
|
36
|
+
# first — the same walk as loki_up, but collects every match instead of
|
|
38
37
|
# stopping at the first (so shadowed ancestor markers can be reported).
|
|
39
38
|
def self.ancestor_markers(dir)
|
|
40
|
-
|
|
41
|
-
current = Pathname.new(dir)
|
|
42
|
-
loop do
|
|
43
|
-
candidate = current + ".loki"
|
|
44
|
-
markers << candidate.to_s if candidate.exist?
|
|
45
|
-
parent = current.parent
|
|
46
|
-
break if parent == current
|
|
47
|
-
|
|
48
|
-
current = parent
|
|
49
|
-
end
|
|
50
|
-
markers
|
|
39
|
+
Pathname.new(dir).ascend.map { it + ".loki" }.select(&:file?).map(&:to_s)
|
|
51
40
|
end
|
|
52
41
|
|
|
53
42
|
# method_name => [[file, line], ...] entries defined more than once —
|
|
@@ -59,6 +48,23 @@ module Asgard
|
|
|
59
48
|
end
|
|
60
49
|
end
|
|
61
50
|
|
|
51
|
+
# The "load failed" line for whatever stopped the .loki chain loading.
|
|
52
|
+
# An exit or abort raises SystemExit; a bare exit (or raise SystemExit)
|
|
53
|
+
# carries only a default message, so only the status is worth showing then.
|
|
54
|
+
def self.load_failure(error)
|
|
55
|
+
return "load failed: #{error.class}: #{error.message}" unless error.is_a?(SystemExit)
|
|
56
|
+
|
|
57
|
+
detail = ": #{error.message}" unless ["exit", error.class.to_s].include?(error.message)
|
|
58
|
+
"load failed: #{error.class} (status #{error.status})#{detail}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# What ImportTracer saw one import or import_up do: the line the report
|
|
62
|
+
# prints for it, and the .loki files it loaded.
|
|
63
|
+
def record_import(message, added = [])
|
|
64
|
+
@imports << message
|
|
65
|
+
@loaded_files.concat(added)
|
|
66
|
+
end
|
|
67
|
+
|
|
62
68
|
private
|
|
63
69
|
|
|
64
70
|
def report_markers
|
|
@@ -79,16 +85,16 @@ module Asgard
|
|
|
79
85
|
return unless @marker
|
|
80
86
|
|
|
81
87
|
Kernel.prepend(ImportTracer)
|
|
82
|
-
ImportTracer.
|
|
88
|
+
ImportTracer.observe(self)
|
|
83
89
|
@loaded_files << @marker
|
|
84
90
|
|
|
85
|
-
before = Asgard::Base.
|
|
86
|
-
|
|
87
|
-
@new_subclasses = (Asgard::Base.
|
|
88
|
-
rescue StandardError, ScriptError => e
|
|
89
|
-
@findings << Finding.new(level: :error, message:
|
|
91
|
+
before = Asgard::Base._asgard_subclasses.dup
|
|
92
|
+
Asgard.load_root(@marker)
|
|
93
|
+
@new_subclasses = (Asgard::Base._asgard_subclasses - before + [Tasks]).uniq
|
|
94
|
+
rescue StandardError, ScriptError, SystemExit => e
|
|
95
|
+
@findings << Finding.new(level: :error, message: self.class.load_failure(e))
|
|
90
96
|
ensure
|
|
91
|
-
ImportTracer.
|
|
97
|
+
ImportTracer.observe(nil)
|
|
92
98
|
end
|
|
93
99
|
|
|
94
100
|
def report_imports
|
|
@@ -105,26 +111,24 @@ module Asgard
|
|
|
105
111
|
end
|
|
106
112
|
end
|
|
107
113
|
|
|
108
|
-
def record_import(message, added = [])
|
|
109
|
-
@imports << message
|
|
110
|
-
@loaded_files.concat(added)
|
|
111
|
-
end
|
|
112
|
-
|
|
113
114
|
# Prepended onto Kernel for the process lifetime of `asgard --doctor`
|
|
114
115
|
# (Asgard.run! exits right after Doctor#run, so it's never unprepended)
|
|
115
116
|
# to observe every import/import_up call, whether typed at the top
|
|
116
117
|
# level of a .loki file or reached indirectly through another import.
|
|
117
118
|
module ImportTracer
|
|
118
119
|
class << self
|
|
119
|
-
|
|
120
|
+
# The Doctor that records each import, or nil when none is running.
|
|
121
|
+
attr_reader :doctor
|
|
122
|
+
|
|
123
|
+
def observe(doctor) = @doctor = doctor
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
def import(path)
|
|
123
127
|
loc = caller_locations(1, 1).first
|
|
124
128
|
before = $LOADED_FEATURES.dup
|
|
125
129
|
result = super(path, from: loc)
|
|
126
|
-
added = $LOADED_FEATURES - before
|
|
127
|
-
ImportTracer.doctor&.
|
|
130
|
+
added = ImportTracer.loki_files($LOADED_FEATURES - before)
|
|
131
|
+
ImportTracer.doctor&.record_import(ImportTracer.trace_message("import", path, added), added) unless
|
|
128
132
|
loc&.absolute_path&.end_with?("kernel_methods.rb")
|
|
129
133
|
result
|
|
130
134
|
end
|
|
@@ -132,11 +136,17 @@ module Asgard
|
|
|
132
136
|
def import_up(name = ".loki")
|
|
133
137
|
before = $LOADED_FEATURES.dup
|
|
134
138
|
result = super
|
|
135
|
-
added = $LOADED_FEATURES - before
|
|
136
|
-
ImportTracer.doctor&.
|
|
139
|
+
added = ImportTracer.loki_files($LOADED_FEATURES - before)
|
|
140
|
+
ImportTracer.doctor&.record_import(ImportTracer.trace_message("import_up", name, added), added)
|
|
137
141
|
result
|
|
138
142
|
end
|
|
139
143
|
|
|
144
|
+
# $LOADED_FEATURES also grows with every `require` a .loki file runs
|
|
145
|
+
# (net/http, uri, ...); only .loki paths were brought in by import.
|
|
146
|
+
def self.loki_files(added)
|
|
147
|
+
added.select { |f| f.end_with?(".loki") }
|
|
148
|
+
end
|
|
149
|
+
|
|
140
150
|
def self.trace_message(verb, arg, added)
|
|
141
151
|
inspected = arg.inspect
|
|
142
152
|
return "#{verb} #{inspected} -- nothing new (not found or already loaded)" if added.empty?
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
require_relative "../log"
|
|
5
|
+
|
|
6
|
+
# warn is silent when $VERBOSE is nil and bypasses $stderr in Ruby 4.0 (see
|
|
7
|
+
# base/task_dsl.rb); these messages are for the user, so they always print.
|
|
8
|
+
# rubocop:disable Style/StderrPuts
|
|
9
|
+
|
|
10
|
+
module Asgard
|
|
11
|
+
module Jobs
|
|
12
|
+
# `asgard jobs SUBCOMMAND` — lists, stops and restarts the background
|
|
13
|
+
# jobs that `sh ..., background:` started. Subclasses Asgard::Base
|
|
14
|
+
# rather than Tasks so `asgard jobs help` lists only these
|
|
15
|
+
# subcommands, not every project task.
|
|
16
|
+
class Commands < Asgard::Base
|
|
17
|
+
namespace "jobs"
|
|
18
|
+
default_command :list
|
|
19
|
+
|
|
20
|
+
HEADERS = %w[JOB TASK PID STATUS STARTED COMMAND].freeze
|
|
21
|
+
KEYS = %i[name task pid state started command].freeze
|
|
22
|
+
FLEXIBLE = [5].freeze
|
|
23
|
+
|
|
24
|
+
class << self
|
|
25
|
+
# One `list` row per record, with what the job is doing now.
|
|
26
|
+
def rows(state) = state.records.map { |record| row(record, state) }
|
|
27
|
+
|
|
28
|
+
# verified? goes first: a job that ends between the reads shows
|
|
29
|
+
# its exit, never "gone", and a code read means its time is there.
|
|
30
|
+
def row(record, state)
|
|
31
|
+
name = record[:name]
|
|
32
|
+
verified = state.verified?(record)
|
|
33
|
+
code = state.exit_code(name)
|
|
34
|
+
exited = state.exit_time(name)
|
|
35
|
+
status = Jobs.status(exit_code: code, verified:, leftovers: State.leftovers?(record, exited_at: exited))
|
|
36
|
+
seconds = (exited - Time.iso8601(record[:started])).round(1) if status.is_a?(Array)
|
|
37
|
+
{ name:, task: record[:task], pid: record[:pid], state: Jobs.describe_status(status, seconds:),
|
|
38
|
+
started: Time.iso8601(record[:started]).strftime("%Y-%m-%d %H:%M:%S"), command: record[:command] }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# One line per record file +states+ left out of the listing.
|
|
42
|
+
def skipped_lines(states) = states.flat_map(&:invalid_records).map { |path| "jobs: skipped #{path}: not a job record" }
|
|
43
|
+
|
|
44
|
+
# The argv that follows a log the way `tail -f` does, from the top.
|
|
45
|
+
def tail_argv(path) = ["tail", "-n", "+1", "-f", path]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
desc "tree", "Print a tree of the jobs subcommands", hide: true
|
|
49
|
+
# Redefined only to re-register tree as hidden in this namespace.
|
|
50
|
+
def tree = super # rubocop:disable Lint/UselessMethodDefinition
|
|
51
|
+
|
|
52
|
+
no_commands do
|
|
53
|
+
def jobs_root = @jobs_root ||= Asgard.project_root
|
|
54
|
+
|
|
55
|
+
def jobs_project = @jobs_project ||= Schedule.project_slug(jobs_root)
|
|
56
|
+
|
|
57
|
+
def state = @state ||= State.new(project: jobs_project, root: jobs_root)
|
|
58
|
+
|
|
59
|
+
# True when a command should cover every project: asked for with
|
|
60
|
+
# --all, or there is no .loki here, so no single project to scope to.
|
|
61
|
+
def every_project? = options[:all] || Asgard.find_task_file.nil?
|
|
62
|
+
|
|
63
|
+
# Every project's state, keyed by "Project: slug (/root)".
|
|
64
|
+
def all_states
|
|
65
|
+
State.all.to_h do |slug, root|
|
|
66
|
+
title = ["Project: #{slug} (#{root})", ("[root missing]" unless File.directory?(root))].compact.join(" ")
|
|
67
|
+
[title, State.new(project: slug, root:)]
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def record_for!(name) = state.read_record(name) || abort("jobs: no such job: #{name} (see `asgard jobs list`)")
|
|
72
|
+
|
|
73
|
+
# Whether +path+ is NAME's log to truncate. A job on record that has
|
|
74
|
+
# not written a log yet says so and is skipped; any other name aborts.
|
|
75
|
+
def log_to_truncate?(name, path)
|
|
76
|
+
found = File.file?(path)
|
|
77
|
+
abort "jobs: no such job log: #{name}" if name.include?("/") || !(found || state.read_record(name))
|
|
78
|
+
return true if found
|
|
79
|
+
|
|
80
|
+
$stderr.puts "#{name} has no log yet (#{path}); nothing to truncate"
|
|
81
|
+
false
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def stopper = Stopper.new(timeout: options[:timeout] || Stopper::TIMEOUT)
|
|
85
|
+
|
|
86
|
+
# The records `stop` ends, newest start first, and the names that
|
|
87
|
+
# matched nothing. A task name reaches the jobs of every task it
|
|
88
|
+
# depends on, which needs this project's .loki loaded.
|
|
89
|
+
def stop_targets(names)
|
|
90
|
+
records, unknown =
|
|
91
|
+
if names.empty? || options[:all] then [listed_states.values.flat_map(&:records), []]
|
|
92
|
+
else Jobs.resolve(names, state.records, closure: ->(task) { Jobs.closure(task, Tasks._deps) })
|
|
93
|
+
end
|
|
94
|
+
[records.sort_by { |record| [record[:started].to_s, record[:name].to_s] }.reverse, unknown]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The states `list` and a bare `stop` cover, by title: every
|
|
98
|
+
# project's, or this one's under no title.
|
|
99
|
+
def listed_states = every_project? ? all_states : { nil => state }
|
|
100
|
+
|
|
101
|
+
def no_jobs_message
|
|
102
|
+
return "No jobs have been started on this machine." if every_project?
|
|
103
|
+
|
|
104
|
+
"No jobs recorded for #{jobs_project}. Start one with `sh \"...\", background: true` inside a task.\n" \
|
|
105
|
+
"Use `asgard jobs list --all` to see every project's jobs."
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
desc "list", "Show this project's jobs: running, exited with a status, or gone (--all: every project's)"
|
|
110
|
+
method_option :all, aliases: "-a", type: :boolean, desc: "List every project's jobs, not just this one's"
|
|
111
|
+
def list
|
|
112
|
+
states = listed_states
|
|
113
|
+
Commands.skipped_lines(states.values).each { |line| $stderr.puts line }
|
|
114
|
+
groups = states.transform_values { |each_state| Commands.rows(each_state) }.reject { |_, rows| rows.empty? }
|
|
115
|
+
return puts(no_jobs_message) if groups.empty?
|
|
116
|
+
|
|
117
|
+
puts groups.map { |title, rows| Schedule::Table.draw(rows, title:, headers: HEADERS, keys: KEYS, flexible: FLEXIBLE) }.join("\n\n")
|
|
118
|
+
puts "\nState: #{State.base_dir}"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
desc "stop [NAME...]", "Stop jobs by job name or by task (and the tasks it depends on); no name stops every job of this project"
|
|
122
|
+
method_option :all, aliases: "-a", type: :boolean, desc: "Stop every project's jobs"
|
|
123
|
+
method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
|
|
124
|
+
def stop(*names)
|
|
125
|
+
abort "jobs: stop --all takes no job names; name the jobs without --all" if options[:all] && names.any?
|
|
126
|
+
records, unmatched = stop_targets(names)
|
|
127
|
+
idle, unknown = unmatched.partition { |name| Tasks.all_commands.key?(name) }
|
|
128
|
+
idle.each { |name| puts "#{name} has no running jobs" }
|
|
129
|
+
unknown.each { |name| $stderr.puts "jobs: no such job or task: #{name}" }
|
|
130
|
+
puts "No jobs recorded for #{jobs_project}." if records.empty? && unmatched.empty?
|
|
131
|
+
outcomes = records.map { |record| stopper.stop_and_report(record) }
|
|
132
|
+
exit 1 if unknown.any? || outcomes.include?(:killed)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
desc "log NAME", "Print one job's log (stdout and stderr of its latest run; each start empties it)"
|
|
136
|
+
method_option :follow, aliases: "-f", type: :boolean, desc: "Keep printing new output as it arrives (tail -f)"
|
|
137
|
+
def log(name)
|
|
138
|
+
path = record_for!(name)[:log]
|
|
139
|
+
return $stderr.puts "#{name} has no log yet (#{path})." unless File.exist?(path)
|
|
140
|
+
return exec(*Commands.tail_argv(path)) if options[:follow]
|
|
141
|
+
|
|
142
|
+
IO.copy_stream(path, $stdout)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
desc "truncate_log [NAME]", "Keep the last M lines of a job log; no NAME truncates every job log of this project"
|
|
146
|
+
method_option :lines, type: :string, default: "1000", banner: "M", desc: "Lines to keep per log; 0 empties it"
|
|
147
|
+
method_option :dry_run, aliases: "-n", type: :boolean, desc: "Report what would be removed without changing logs"
|
|
148
|
+
def truncate_log(name = nil)
|
|
149
|
+
Log.line_count(options[:lines]) # reject a bad --lines even when there is no log to truncate
|
|
150
|
+
paths = name ? [state.log_path(name)] : Dir.glob(File.join(Asgard.glob_escape(state.dir), "*.log"))
|
|
151
|
+
return if name && !log_to_truncate?(name, paths.first)
|
|
152
|
+
|
|
153
|
+
Log.truncate_logs(paths, lines: options[:lines], dry_run: options[:dry_run])
|
|
154
|
+
rescue ArgumentError, SystemCallError, IOError => e
|
|
155
|
+
abort "jobs: #{e.message}"
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
desc "restart NAME", "Stop the job, then run its owning task again"
|
|
159
|
+
method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
|
|
160
|
+
def restart(name)
|
|
161
|
+
record = record_for!(name)
|
|
162
|
+
task = record[:task].to_s
|
|
163
|
+
abort "jobs: #{name} was not started from a task, so there is nothing to run again" if task.empty?
|
|
164
|
+
stopper.stop_and_report(record)
|
|
165
|
+
Tasks._reset_ran!
|
|
166
|
+
Tasks.start([task])
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
desc "clean", "Forget every job that is no longer running: its record, script, status and log"
|
|
170
|
+
# Each removal re-checks under the job's start lock (see
|
|
171
|
+
# State#forget_unless_active), so a start that ran meanwhile wins.
|
|
172
|
+
def clean
|
|
173
|
+
invalid = state.invalid_records.map { |path| [File.basename(path, ".json"), " (not a job record)"] }
|
|
174
|
+
gone = state.records.reject { |record| state.active?(record) }.map { |record| [record[:name], ""] }
|
|
175
|
+
removed = (invalid + gone).select { |name, _| state.forget_unless_active(name) }
|
|
176
|
+
return puts("Nothing to clean for #{jobs_project}.") if removed.empty?
|
|
177
|
+
|
|
178
|
+
removed.each { |name, note| puts "removed #{name}#{note}" }
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
# rubocop:enable Style/StderrPuts
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Asgard
|
|
6
|
+
# Background jobs: a process a task starts on purpose, by name, that
|
|
7
|
+
# outlives asgard and that a later `asgard jobs` can list and stop.
|
|
8
|
+
# `sh "mkdocs serve", background: true` starts one; see
|
|
9
|
+
# Jobs::Launcher for how, Jobs::State for what is kept on disk and
|
|
10
|
+
# Jobs::Commands for the management commands. This file holds the pure
|
|
11
|
+
# rules: naming, records, name resolution and status.
|
|
12
|
+
module Jobs
|
|
13
|
+
# A job name a user may choose: letters, digits and underscore, so it
|
|
14
|
+
# is safe as a file name and can never look like an auto-generated
|
|
15
|
+
# "<task>-<N>" name.
|
|
16
|
+
NAME = /\A\w+\z/
|
|
17
|
+
|
|
18
|
+
# One started job, as `sh ..., background:` returns it.
|
|
19
|
+
Job = Data.define(:name, :task, :pid, :log)
|
|
20
|
+
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
# The job name a `background:` value asks for, or nil for the
|
|
24
|
+
# foreground. +task+ is the task whose body made the call and
|
|
25
|
+
# +sequence+ its next job number, consulted only for `true`: a named
|
|
26
|
+
# job never consumes a number, so adding or removing one never
|
|
27
|
+
# renumbers the rest.
|
|
28
|
+
def job_name(background, task:, sequence: nil)
|
|
29
|
+
case background
|
|
30
|
+
when nil, false then nil
|
|
31
|
+
when true then sequenced_name(task, sequence)
|
|
32
|
+
when Symbol, String then chosen_name(background)
|
|
33
|
+
else raise ArgumentError, "background: must be true, a Symbol or a String, not #{background.inspect}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# How a task name's characters outside \w are spelled in its job name,
|
|
38
|
+
# which keeps to [\w-] because a record's file name must for `jobs
|
|
39
|
+
# list` to find it. Every spelling starts with "-" and none with
|
|
40
|
+
# "-<digit>", so no two task names share a job name and none reads as
|
|
41
|
+
# the "-<N>" counter: ready? is ready-q-1, never the ready_-1 of ready_.
|
|
42
|
+
NAME_ESCAPES = { "?" => "-q", "!" => "-b", "=" => "-e", "-" => "--" }.freeze
|
|
43
|
+
|
|
44
|
+
def sequenced_name(task, sequence)
|
|
45
|
+
raise ArgumentError, "sh background: true must be called from a task body" unless task
|
|
46
|
+
|
|
47
|
+
:"#{escape_task(task)}-#{sequence}"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Any other character (a non-ASCII letter) is "-u" and six hex digits.
|
|
51
|
+
def escape_task(task) = task.to_s.gsub(/\W/) { |char| NAME_ESCAPES.fetch(char) { format("-u%06x", char.ord) } }
|
|
52
|
+
|
|
53
|
+
def chosen_name(value)
|
|
54
|
+
name = value.to_s.strip
|
|
55
|
+
raise ArgumentError, "background: #{value.inspect} is not a job name (letters, digits and _ only)" unless name.match?(NAME)
|
|
56
|
+
|
|
57
|
+
name.to_sym
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Whether +record+ is one a started job wrote: a Hash with a name, a
|
|
61
|
+
# pid and a group id above 1 (kill(2) reads -1 and 0 as "everyone"
|
|
62
|
+
# and "us"), a script path to verify against and a start time. A hand
|
|
63
|
+
# edit or another tool's JSON fails this and stays out of listings.
|
|
64
|
+
def valid_record?(record)
|
|
65
|
+
return false unless record.is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
name, pid, pgid, script, started = record.values_at(:name, :pid, :pgid, :script, :started)
|
|
68
|
+
text?(name) && text?(script) && process_id?(pid) && process_id?(pgid) && time?(started)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def text?(value) = value.is_a?(String) && !value.empty?
|
|
72
|
+
|
|
73
|
+
def process_id?(id) = id.is_a?(Integer) && id > 1
|
|
74
|
+
|
|
75
|
+
def time?(value)
|
|
76
|
+
value.is_a?(String) && Time.iso8601(value).is_a?(Time)
|
|
77
|
+
rescue ArgumentError
|
|
78
|
+
false
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The first non-blank line of a script, for the listing.
|
|
82
|
+
def summary(script) = script.to_s.each_line.map(&:strip).find { |line| !line.empty? }.to_s
|
|
83
|
+
|
|
84
|
+
# The record written to disk for a started job.
|
|
85
|
+
def record(name:, task:, pid:, pgid:, cwd:, script:, log:, command:, started:)
|
|
86
|
+
{ name: name.to_s, task: task.to_s, pid:, pgid:, started:, cwd:, script:, log:, command: }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# +task+ plus every task reachable through +deps+ (a `_deps` Hash of
|
|
90
|
+
# stage lists), in discovery order. `stop servers` reaches the `db-1`
|
|
91
|
+
# a dependency started.
|
|
92
|
+
def closure(task, deps)
|
|
93
|
+
seen = []
|
|
94
|
+
queue = [task.to_sym]
|
|
95
|
+
until queue.empty?
|
|
96
|
+
current = queue.shift
|
|
97
|
+
next if seen.include?(current)
|
|
98
|
+
|
|
99
|
+
seen << current
|
|
100
|
+
queue.concat(Array(deps[current]).flatten.map(&:to_sym))
|
|
101
|
+
end
|
|
102
|
+
seen
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The records +names+ pick out: an exact job name first, else every
|
|
106
|
+
# record whose task is the name or, through +closure+ (a callable from
|
|
107
|
+
# task name to task names), one it depends on. Returns [records,
|
|
108
|
+
# unknown names].
|
|
109
|
+
def resolve(names, records, closure: nil)
|
|
110
|
+
found = []
|
|
111
|
+
unknown = []
|
|
112
|
+
names.each do |name|
|
|
113
|
+
matches = matching_records(name.to_s, records, closure)
|
|
114
|
+
matches.empty? ? unknown << name.to_s : found.concat(matches)
|
|
115
|
+
end
|
|
116
|
+
[found.uniq, unknown]
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def matching_records(name, records, closure)
|
|
120
|
+
exact = records.select { |record| record[:name].to_s == name }
|
|
121
|
+
return exact unless exact.empty?
|
|
122
|
+
|
|
123
|
+
tasks = (closure ? closure.call(name) : [name]).map(&:to_s)
|
|
124
|
+
records.select { |record| tasks.include?(record[:task].to_s) }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# What a record's job is doing now: :running while its pid verifies,
|
|
128
|
+
# [:exited, code] once the trampoline wrote a status (with
|
|
129
|
+
# :processes_left while processes the script started remain in its
|
|
130
|
+
# group), :gone when neither (the trampoline died, or the machine
|
|
131
|
+
# rebooted).
|
|
132
|
+
def status(exit_code:, verified:, leftovers: false)
|
|
133
|
+
return [:exited, exit_code, (:processes_left if leftovers)].compact if exit_code
|
|
134
|
+
return :running if verified
|
|
135
|
+
|
|
136
|
+
:gone
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The STATUS column: "running", "exited 3 (2.5s)", "exited 0 (2.5s,
|
|
140
|
+
# processes left)" or "gone".
|
|
141
|
+
def describe_status(status, seconds: nil)
|
|
142
|
+
return status.to_s unless status.is_a?(Array)
|
|
143
|
+
|
|
144
|
+
_, code, left = status
|
|
145
|
+
notes = [(format("%.1fs", seconds) if seconds), ("processes left" if left)].compact
|
|
146
|
+
notes.empty? ? "exited #{code}" : "exited #{code} (#{notes.join(', ')})"
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
module Asgard
|
|
7
|
+
module Jobs
|
|
8
|
+
# Starts a job: a new session with stdio detached, running the job's
|
|
9
|
+
# shell under a POSIX /bin/sh trampoline that writes the exit status
|
|
10
|
+
# to a file once nobody is left to wait on it. Everything but `launch`
|
|
11
|
+
# is pure, and `launch` is three lines.
|
|
12
|
+
module Launcher
|
|
13
|
+
# The trampoline body. The trap is load-bearing: a group TERM reaches
|
|
14
|
+
# this shell too, and an untrapped shell would die at once, before
|
|
15
|
+
# the status is written. With a handler it finishes waiting for the
|
|
16
|
+
# child, then runs on to the echo; the child still gets its own TERM
|
|
17
|
+
# with the default disposition, since handled traps reset on exec.
|
|
18
|
+
# Never `trap ''`: an ignored signal is inherited across exec. Two
|
|
19
|
+
# commands, not one, so no shell execs the body away and vanishes.
|
|
20
|
+
TRAMPOLINE = 'trap ":" TERM INT HUP; "$@"; echo $? > "$ASGARD_JOB_STATUS"'
|
|
21
|
+
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
# /bin/sh by absolute path: never bash and never a registry entry,
|
|
25
|
+
# so the user's default_shell and the presence of bash do not
|
|
26
|
+
# matter. The status path travels in the environment, so nothing
|
|
27
|
+
# here is quoted.
|
|
28
|
+
def argv(shell_argv) = ["/bin/sh", "-c", TRAMPOLINE, "asgard-job", *shell_argv]
|
|
29
|
+
|
|
30
|
+
# The variables added to the task's environment. Python fully
|
|
31
|
+
# buffers stdout when it is a file, so a print loop would show
|
|
32
|
+
# nothing in the log until exit without PYTHONUNBUFFERED.
|
|
33
|
+
def environment(name, status_path, env = ENV)
|
|
34
|
+
added = { "ASGARD_JOB" => name.to_s, "ASGARD_JOB_STATUS" => status_path }
|
|
35
|
+
added["PYTHONUNBUFFERED"] = "1" unless env.key?("PYTHONUNBUFFERED")
|
|
36
|
+
added
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# stdin from /dev/null, stdout and stderr appended to the log, so
|
|
40
|
+
# the job never takes SIGTTIN or writes into a terminal that
|
|
41
|
+
# happens to be attached.
|
|
42
|
+
# close_others: a descriptor asgard inherited (a CI runner's, a
|
|
43
|
+
# `3>file`) has no business staying open for the job's lifetime.
|
|
44
|
+
def spawn_options(log_path) = { in: File::NULL, %i[out err] => [log_path, "a"], close_others: true }
|
|
45
|
+
|
|
46
|
+
# Forks, starts a new session (pid = pgid = sid, no controlling
|
|
47
|
+
# terminal) and execs; returns the child's pid, detached. The parent
|
|
48
|
+
# waits on a pipe whose write end the child holds until exec closes
|
|
49
|
+
# it (pipes are close-on-exec), so by the time the pid is recorded it
|
|
50
|
+
# already leads its own group, and already runs the trampoline: a
|
|
51
|
+
# `stop` issued at once signals the job, never the parent's group,
|
|
52
|
+
# and a start issued at once sees the job's own command line.
|
|
53
|
+
def launch(env, argv, options)
|
|
54
|
+
reader, writer = IO.pipe
|
|
55
|
+
pid = fork do
|
|
56
|
+
reader.close
|
|
57
|
+
Process.setsid
|
|
58
|
+
exec(env, *argv, **options)
|
|
59
|
+
rescue SystemCallError => e
|
|
60
|
+
# exit!, not raise: no at_exit handler of the parent may run here.
|
|
61
|
+
$stderr.puts "asgard: could not start job: #{e.message}" # rubocop:disable Style/StderrPuts
|
|
62
|
+
$stderr.flush # exit! skips Ruby's own flush
|
|
63
|
+
exit!(127)
|
|
64
|
+
end
|
|
65
|
+
writer.close
|
|
66
|
+
reader.read
|
|
67
|
+
reader.close
|
|
68
|
+
Process.detach(pid)
|
|
69
|
+
pid
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Starts +run+ as the job +name+ owned by +task+, or returns the job
|
|
73
|
+
# already running under that name. A missing shell exits 127 up
|
|
74
|
+
# front, as foreground `sh` does; the trampoline would otherwise
|
|
75
|
+
# hide it as a status nobody is watching for. A ps that cannot
|
|
76
|
+
# verify jobs raises Pid::Unusable first: a job nothing could find
|
|
77
|
+
# again would be started twice.
|
|
78
|
+
def start(run, name:, task:, state:, silent: false, out: $stdout)
|
|
79
|
+
Pid.require_ps!
|
|
80
|
+
state.with_lock(name) do
|
|
81
|
+
existing = state.read_record(name)
|
|
82
|
+
return already_running(existing, out:) if existing && state.verified?(existing)
|
|
83
|
+
|
|
84
|
+
run.missing! unless Shell.resolve(run.shell[:path])
|
|
85
|
+
job = spawn_job(run, name:, task:, state:)
|
|
86
|
+
out.puts "started #{job.name} (pid #{job.pid}), log: #{job.log}" unless silent
|
|
87
|
+
job
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def already_running(record, out:)
|
|
92
|
+
out.puts "#{record[:name]} already running (pid #{record[:pid]}, started by #{record[:task]}), log: #{record[:log]}"
|
|
93
|
+
Job.new(name: record[:name].to_sym, task: record[:task].to_sym, pid: record[:pid], log: record[:log])
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def spawn_job(run, name:, task:, state:)
|
|
97
|
+
state.prepare.remove(name)
|
|
98
|
+
script = state.script_path(name, run.ext)
|
|
99
|
+
log = state.log_path(name)
|
|
100
|
+
File.write(script, "#{run.script}\n")
|
|
101
|
+
FileUtils.touch(log) # so `jobs log` and a reader never race the child's open
|
|
102
|
+
pid = launch(environment(name, state.status_path(name)), argv(run.argv(script)), spawn_options(log))
|
|
103
|
+
state.write_record(Jobs.record(name:, task:, pid:, pgid: pid, cwd: Dir.pwd, script:, log:,
|
|
104
|
+
command: Jobs.summary(run.script), started: Time.now.iso8601(6)))
|
|
105
|
+
Job.new(name: name.to_sym, task: task&.to_sym, pid:, log:)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|