asgard 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +272 -99
  3. data/README.md +147 -46
  4. data/bin/asgard +10 -0
  5. data/lib/asgard/base/dependency_graph.rb +19 -11
  6. data/lib/asgard/base/dispatch.rb +272 -32
  7. data/lib/asgard/base/registry.rb +7 -3
  8. data/lib/asgard/base/task_dsl.rb +6 -0
  9. data/lib/asgard/base.rb +147 -9
  10. data/lib/asgard/declarations.rb +21 -0
  11. data/lib/asgard/doctor/report.rb +5 -1
  12. data/lib/asgard/doctor/task_sections.rb +11 -3
  13. data/lib/asgard/doctor.rb +41 -31
  14. data/lib/asgard/jobs/commands.rb +183 -0
  15. data/lib/asgard/jobs/declaration.rb +149 -0
  16. data/lib/asgard/jobs/launcher.rb +109 -0
  17. data/lib/asgard/jobs/state.rb +195 -0
  18. data/lib/asgard/jobs/stopper.rb +141 -0
  19. data/lib/asgard/jobs.rb +25 -0
  20. data/lib/asgard/kernel_methods.rb +25 -30
  21. data/lib/asgard/leading_flag.rb +26 -0
  22. data/lib/asgard/log.rb +90 -0
  23. data/lib/asgard/pid.rb +113 -0
  24. data/lib/asgard/schedule/commands.rb +68 -18
  25. data/lib/asgard/schedule/declaration.rb +158 -26
  26. data/lib/asgard/schedule/job_files.rb +24 -0
  27. data/lib/asgard/schedule/launchd.rb +57 -21
  28. data/lib/asgard/schedule/systemd.rb +72 -28
  29. data/lib/asgard/schedule/table.rb +48 -26
  30. data/lib/asgard/schedule.rb +49 -9
  31. data/lib/asgard/shell.rb +366 -44
  32. data/lib/asgard/state_dir.rb +90 -0
  33. data/lib/asgard/tasks.rb +8 -0
  34. data/lib/asgard/version.rb +1 -1
  35. data/lib/asgard/watch/changes.rb +26 -1
  36. data/lib/asgard/watch/commands.rb +63 -13
  37. data/lib/asgard/watch/daemon.rb +347 -52
  38. data/lib/asgard/watch/declaration.rb +135 -40
  39. data/lib/asgard/watch/dispatcher.rb +61 -10
  40. data/lib/asgard/watch/mailbox.rb +50 -0
  41. data/lib/asgard/watch/reporter.rb +54 -32
  42. data/lib/asgard/watch/spawn.rb +43 -18
  43. data/lib/asgard/watch/state.rb +93 -27
  44. data/lib/asgard/watch/worker.rb +44 -9
  45. data/lib/asgard/watch.rb +4 -11
  46. data/lib/asgard.rb +100 -16
  47. metadata +170 -186
  48. data/.envrc +0 -6
  49. data/.github/workflows/deploy-github-pages.yml +0 -52
  50. data/.loki +0 -39
  51. data/.reek.yml +0 -166
  52. data/.rubocop.yml +0 -163
  53. data/Archspec.rb +0 -24
  54. data/CLAUDE.md +0 -139
  55. data/COMMITS.md +0 -196
  56. data/bin/console +0 -11
  57. data/bin/setup +0 -8
  58. data/doc_tasks.loki +0 -21
  59. data/docs/api.md +0 -368
  60. data/docs/assets/css/custom.css +0 -93
  61. data/docs/assets/images/asgard.jpg +0 -0
  62. data/docs/changelog.md +0 -1
  63. data/docs/dependencies.md +0 -313
  64. data/docs/environment.md +0 -129
  65. data/docs/examples.md +0 -194
  66. data/docs/getting-started.md +0 -181
  67. data/docs/helpers.md +0 -262
  68. data/docs/index.md +0 -87
  69. data/docs/options.md +0 -224
  70. data/docs/schedule.md +0 -106
  71. data/docs/shell.md +0 -237
  72. data/docs/subcommands.md +0 -181
  73. data/docs/task-files.md +0 -407
  74. data/docs/tasks.md +0 -326
  75. data/docs/variables.md +0 -338
  76. data/docs/watch.md +0 -123
  77. data/examples/.env +0 -4
  78. data/examples/.envrc +0 -3
  79. data/examples/.loki +0 -24
  80. data/examples/bad.loki +0 -63
  81. data/examples/concurrent.loki +0 -58
  82. data/examples/confirm.loki +0 -48
  83. data/examples/db_subcommands.loki +0 -74
  84. data/examples/depends_on_block/bad/.loki +0 -51
  85. data/examples/depends_on_block/good/.loki +0 -83
  86. data/examples/do.loki +0 -45
  87. data/examples/env_usage.loki +0 -27
  88. data/examples/kitchen_sink.loki +0 -210
  89. data/examples/server_subcommands.loki +0 -65
  90. data/examples/subdir/.loki +0 -12
  91. data/examples/subdir/import_demo.loki +0 -14
  92. data/examples/subdir/import_up_demo.loki +0 -18
  93. data/examples/watcher_examples/.envrc +0 -3
  94. data/examples/watcher_examples/.loki +0 -54
  95. data/examples/watcher_examples/README.md +0 -112
  96. data/examples/watcher_examples/bin/hello +0 -3
  97. data/examples/watcher_examples/data/records.json +0 -1
  98. data/examples/watcher_examples/docs/drafts/ideas.md +0 -5
  99. data/examples/watcher_examples/docs/index.md +0 -4
  100. data/examples/watcher_examples/lib/greeter.rb +0 -7
  101. data/gem_tasks.loki +0 -51
  102. data/git.loki +0 -13
  103. data/mkdocs.yml +0 -166
  104. data/quality.loki +0 -266
  105. data/quality_rails.loki +0 -47
  106. data/xyzzy.loki +0 -12
@@ -7,6 +7,14 @@ module Asgard
7
7
  # more than one recorded location (the each/each_gem class of bug) —
8
8
  # earlier ones OVERRIDDEN (dead), the last one the active redefinition.
9
9
  module TaskSections
10
+ # A hidden task (one defined without a desc) says so, after any
11
+ # override note.
12
+ def self.with_hidden(status, hidden)
13
+ return status unless hidden
14
+
15
+ [status, "hidden (no desc; not shown by help)"].compact.join("; ")
16
+ end
17
+
10
18
  private
11
19
 
12
20
  def build_task_sections
@@ -20,12 +28,12 @@ module Asgard
20
28
  file_map = Hash.new { |h, k| h[k] = [] }
21
29
  @loaded_files.each { |f| file_map[f] }
22
30
 
23
- commands = klass.all_commands.keys
31
+ commands = klass.all_commands
24
32
  klass._method_log.each do |name, locations|
25
- next unless commands.include?(name.to_s)
33
+ command = commands[name.to_s] or next
26
34
 
27
35
  locations.each_with_index do |(file, line), idx|
28
- file_map[file] << [name, line, task_status(locations, idx)]
36
+ file_map[file] << [name, line, TaskSections.with_hidden(task_status(locations, idx), command.hidden?)]
29
37
  end
30
38
  end
31
39
  file_map.each_value { |tasks| tasks.sort_by! { |t| t[1] } }
data/lib/asgard/doctor.rb CHANGED
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "pathname"
4
-
5
3
  require_relative "doctor/task_sections"
6
4
  require_relative "doctor/report"
7
5
 
@@ -31,23 +29,14 @@ module Asgard
31
29
  build_task_sections
32
30
  report_dependencies
33
31
  print_report
32
+ problem_count.zero? ? 0 : 1
34
33
  end
35
34
 
36
35
  # Every ".loki" marker from +dir+ up to the filesystem root, nearest
37
- # first — mirrors loki_up's walk, but collects every match instead of
36
+ # first — the same walk as loki_up, but collects every match instead of
38
37
  # stopping at the first (so shadowed ancestor markers can be reported).
39
38
  def self.ancestor_markers(dir)
40
- markers = []
41
- current = Pathname.new(dir)
42
- loop do
43
- candidate = current + ".loki"
44
- markers << candidate.to_s if candidate.exist?
45
- parent = current.parent
46
- break if parent == current
47
-
48
- current = parent
49
- end
50
- markers
39
+ Pathname.new(dir).ascend.map { it + ".loki" }.select(&:file?).map(&:to_s)
51
40
  end
52
41
 
53
42
  # method_name => [[file, line], ...] entries defined more than once —
@@ -59,6 +48,23 @@ module Asgard
59
48
  end
60
49
  end
61
50
 
51
+ # The "load failed" line for whatever stopped the .loki chain loading.
52
+ # An exit or abort raises SystemExit; a bare exit (or raise SystemExit)
53
+ # carries only a default message, so only the status is worth showing then.
54
+ def self.load_failure(error)
55
+ return "load failed: #{error.class}: #{error.message}" unless error.is_a?(SystemExit)
56
+
57
+ detail = ": #{error.message}" unless ["exit", error.class.to_s].include?(error.message)
58
+ "load failed: #{error.class} (status #{error.status})#{detail}"
59
+ end
60
+
61
+ # What ImportTracer saw one import or import_up do: the line the report
62
+ # prints for it, and the .loki files it loaded.
63
+ def record_import(message, added = [])
64
+ @imports << message
65
+ @loaded_files.concat(added)
66
+ end
67
+
62
68
  private
63
69
 
64
70
  def report_markers
@@ -79,16 +85,16 @@ module Asgard
79
85
  return unless @marker
80
86
 
81
87
  Kernel.prepend(ImportTracer)
82
- ImportTracer.doctor = self
88
+ ImportTracer.observe(self)
83
89
  @loaded_files << @marker
84
90
 
85
- before = Asgard::Base.subclasses.dup
86
- load @marker
87
- @new_subclasses = (Asgard::Base.subclasses - before + [Tasks]).uniq
88
- rescue StandardError, ScriptError => e
89
- @findings << Finding.new(level: :error, message: "load failed: #{e.class}: #{e.message}")
91
+ before = Asgard::Base._asgard_subclasses.dup
92
+ Asgard.load_root(@marker)
93
+ @new_subclasses = (Asgard::Base._asgard_subclasses - before + [Tasks]).uniq
94
+ rescue StandardError, ScriptError, SystemExit => e
95
+ @findings << Finding.new(level: :error, message: self.class.load_failure(e))
90
96
  ensure
91
- ImportTracer.doctor = nil
97
+ ImportTracer.observe(nil)
92
98
  end
93
99
 
94
100
  def report_imports
@@ -105,26 +111,24 @@ module Asgard
105
111
  end
106
112
  end
107
113
 
108
- def record_import(message, added = [])
109
- @imports << message
110
- @loaded_files.concat(added)
111
- end
112
-
113
114
  # Prepended onto Kernel for the process lifetime of `asgard --doctor`
114
115
  # (Asgard.run! exits right after Doctor#run, so it's never unprepended)
115
116
  # to observe every import/import_up call, whether typed at the top
116
117
  # level of a .loki file or reached indirectly through another import.
117
118
  module ImportTracer
118
119
  class << self
119
- attr_accessor :doctor
120
+ # The Doctor that records each import, or nil when none is running.
121
+ attr_reader :doctor
122
+
123
+ def observe(doctor) = @doctor = doctor
120
124
  end
121
125
 
122
126
  def import(path)
123
127
  loc = caller_locations(1, 1).first
124
128
  before = $LOADED_FEATURES.dup
125
129
  result = super(path, from: loc)
126
- added = $LOADED_FEATURES - before
127
- ImportTracer.doctor&.send(:record_import, ImportTracer.trace_message("import", path, added), added) unless
130
+ added = ImportTracer.loki_files($LOADED_FEATURES - before)
131
+ ImportTracer.doctor&.record_import(ImportTracer.trace_message("import", path, added), added) unless
128
132
  loc&.absolute_path&.end_with?("kernel_methods.rb")
129
133
  result
130
134
  end
@@ -132,11 +136,17 @@ module Asgard
132
136
  def import_up(name = ".loki")
133
137
  before = $LOADED_FEATURES.dup
134
138
  result = super
135
- added = $LOADED_FEATURES - before
136
- ImportTracer.doctor&.send(:record_import, ImportTracer.trace_message("import_up", name, added), added)
139
+ added = ImportTracer.loki_files($LOADED_FEATURES - before)
140
+ ImportTracer.doctor&.record_import(ImportTracer.trace_message("import_up", name, added), added)
137
141
  result
138
142
  end
139
143
 
144
+ # $LOADED_FEATURES also grows with every `require` a .loki file runs
145
+ # (net/http, uri, ...); only .loki paths were brought in by import.
146
+ def self.loki_files(added)
147
+ added.select { |f| f.end_with?(".loki") }
148
+ end
149
+
140
150
  def self.trace_message(verb, arg, added)
141
151
  inspected = arg.inspect
142
152
  return "#{verb} #{inspected} -- nothing new (not found or already loaded)" if added.empty?
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+ require_relative "../log"
5
+
6
+ # warn is silent when $VERBOSE is nil and bypasses $stderr in Ruby 4.0 (see
7
+ # base/task_dsl.rb); these messages are for the user, so they always print.
8
+ # rubocop:disable Style/StderrPuts
9
+
10
+ module Asgard
11
+ module Jobs
12
+ # `asgard jobs SUBCOMMAND` — lists, stops and restarts the background
13
+ # jobs that `sh ..., background:` started. Subclasses Asgard::Base
14
+ # rather than Tasks so `asgard jobs help` lists only these
15
+ # subcommands, not every project task.
16
+ class Commands < Asgard::Base
17
+ namespace "jobs"
18
+ default_command :list
19
+
20
+ HEADERS = %w[JOB TASK PID STATUS STARTED COMMAND].freeze
21
+ KEYS = %i[name task pid state started command].freeze
22
+ FLEXIBLE = [5].freeze
23
+
24
+ class << self
25
+ # One `list` row per record, with what the job is doing now.
26
+ def rows(state) = state.records.map { |record| row(record, state) }
27
+
28
+ # verified? goes first: a job that ends between the reads shows
29
+ # its exit, never "gone", and a code read means its time is there.
30
+ def row(record, state)
31
+ name = record[:name]
32
+ verified = state.verified?(record)
33
+ code = state.exit_code(name)
34
+ exited = state.exit_time(name)
35
+ status = Jobs.status(exit_code: code, verified:, leftovers: State.leftovers?(record, exited_at: exited))
36
+ seconds = (exited - Time.iso8601(record[:started])).round(1) if status.is_a?(Array)
37
+ { name:, task: record[:task], pid: record[:pid], state: Jobs.describe_status(status, seconds:),
38
+ started: Time.iso8601(record[:started]).strftime("%Y-%m-%d %H:%M:%S"), command: record[:command] }
39
+ end
40
+
41
+ # One line per record file +states+ left out of the listing.
42
+ def skipped_lines(states) = states.flat_map(&:invalid_records).map { |path| "jobs: skipped #{path}: not a job record" }
43
+
44
+ # The argv that follows a log the way `tail -f` does, from the top.
45
+ def tail_argv(path) = ["tail", "-n", "+1", "-f", path]
46
+ end
47
+
48
+ desc "tree", "Print a tree of the jobs subcommands", hide: true
49
+ # Redefined only to re-register tree as hidden in this namespace.
50
+ def tree = super # rubocop:disable Lint/UselessMethodDefinition
51
+
52
+ no_commands do
53
+ def jobs_root = @jobs_root ||= Asgard.project_root
54
+
55
+ def jobs_project = @jobs_project ||= Schedule.project_slug(jobs_root)
56
+
57
+ def state = @state ||= State.new(project: jobs_project, root: jobs_root)
58
+
59
+ # True when a command should cover every project: asked for with
60
+ # --all, or there is no .loki here, so no single project to scope to.
61
+ def every_project? = options[:all] || Asgard.find_task_file.nil?
62
+
63
+ # Every project's state, keyed by "Project: slug (/root)".
64
+ def all_states
65
+ State.all.to_h do |slug, root|
66
+ title = ["Project: #{slug} (#{root})", ("[root missing]" unless File.directory?(root))].compact.join(" ")
67
+ [title, State.new(project: slug, root:)]
68
+ end
69
+ end
70
+
71
+ def record_for!(name) = state.read_record(name) || abort("jobs: no such job: #{name} (see `asgard jobs list`)")
72
+
73
+ # Whether +path+ is NAME's log to truncate. A job on record that has
74
+ # not written a log yet says so and is skipped; any other name aborts.
75
+ def log_to_truncate?(name, path)
76
+ found = File.file?(path)
77
+ abort "jobs: no such job log: #{name}" if name.include?("/") || !(found || state.read_record(name))
78
+ return true if found
79
+
80
+ $stderr.puts "#{name} has no log yet (#{path}); nothing to truncate"
81
+ false
82
+ end
83
+
84
+ def stopper = Stopper.new(timeout: options[:timeout] || Stopper::TIMEOUT)
85
+
86
+ # The records `stop` ends, newest start first, and the names that
87
+ # matched nothing. A task name reaches the jobs of every task it
88
+ # depends on, which needs this project's .loki loaded.
89
+ def stop_targets(names)
90
+ records, unknown =
91
+ if names.empty? || options[:all] then [listed_states.values.flat_map(&:records), []]
92
+ else Jobs.resolve(names, state.records, closure: ->(task) { Jobs.closure(task, Tasks._deps) })
93
+ end
94
+ [records.sort_by { |record| [record[:started].to_s, record[:name].to_s] }.reverse, unknown]
95
+ end
96
+
97
+ # The states `list` and a bare `stop` cover, by title: every
98
+ # project's, or this one's under no title.
99
+ def listed_states = every_project? ? all_states : { nil => state }
100
+
101
+ def no_jobs_message
102
+ return "No jobs have been started on this machine." if every_project?
103
+
104
+ "No jobs recorded for #{jobs_project}. Start one with `sh \"...\", background: true` inside a task.\n" \
105
+ "Use `asgard jobs list --all` to see every project's jobs."
106
+ end
107
+ end
108
+
109
+ desc "list", "Show this project's jobs: running, exited with a status, or gone (--all: every project's)"
110
+ method_option :all, aliases: "-a", type: :boolean, desc: "List every project's jobs, not just this one's"
111
+ def list
112
+ states = listed_states
113
+ Commands.skipped_lines(states.values).each { |line| $stderr.puts line }
114
+ groups = states.transform_values { |each_state| Commands.rows(each_state) }.reject { |_, rows| rows.empty? }
115
+ return puts(no_jobs_message) if groups.empty?
116
+
117
+ puts groups.map { |title, rows| Schedule::Table.draw(rows, title:, headers: HEADERS, keys: KEYS, flexible: FLEXIBLE) }.join("\n\n")
118
+ puts "\nState: #{State.base_dir}"
119
+ end
120
+
121
+ desc "stop [NAME...]", "Stop jobs by job name or by task (and the tasks it depends on); no name stops every job of this project"
122
+ method_option :all, aliases: "-a", type: :boolean, desc: "Stop every project's jobs"
123
+ method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
124
+ def stop(*names)
125
+ abort "jobs: stop --all takes no job names; name the jobs without --all" if options[:all] && names.any?
126
+ records, unmatched = stop_targets(names)
127
+ idle, unknown = unmatched.partition { |name| Tasks.all_commands.key?(name) }
128
+ idle.each { |name| puts "#{name} has no running jobs" }
129
+ unknown.each { |name| $stderr.puts "jobs: no such job or task: #{name}" }
130
+ puts "No jobs recorded for #{jobs_project}." if records.empty? && unmatched.empty?
131
+ outcomes = records.map { |record| stopper.stop_and_report(record) }
132
+ exit 1 if unknown.any? || outcomes.include?(:killed)
133
+ end
134
+
135
+ desc "log NAME", "Print one job's log (stdout and stderr of its latest run; each start empties it)"
136
+ method_option :follow, aliases: "-f", type: :boolean, desc: "Keep printing new output as it arrives (tail -f)"
137
+ def log(name)
138
+ path = record_for!(name)[:log]
139
+ return $stderr.puts "#{name} has no log yet (#{path})." unless File.exist?(path)
140
+ return exec(*Commands.tail_argv(path)) if options[:follow]
141
+
142
+ IO.copy_stream(path, $stdout)
143
+ end
144
+
145
+ desc "truncate_log [NAME]", "Keep the last M lines of a job log; no NAME truncates every job log of this project"
146
+ method_option :lines, type: :string, default: "1000", banner: "M", desc: "Lines to keep per log; 0 empties it"
147
+ method_option :dry_run, aliases: "-n", type: :boolean, desc: "Report what would be removed without changing logs"
148
+ def truncate_log(name = nil)
149
+ Log.line_count(options[:lines]) # reject a bad --lines even when there is no log to truncate
150
+ paths = name ? [state.log_path(name)] : Dir.glob(File.join(Asgard.glob_escape(state.dir), "*.log"))
151
+ return if name && !log_to_truncate?(name, paths.first)
152
+
153
+ Log.truncate_logs(paths, lines: options[:lines], dry_run: options[:dry_run])
154
+ rescue ArgumentError, SystemCallError, IOError => e
155
+ abort "jobs: #{e.message}"
156
+ end
157
+
158
+ desc "restart NAME", "Stop the job, then run its owning task again"
159
+ method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
160
+ def restart(name)
161
+ record = record_for!(name)
162
+ task = record[:task].to_s
163
+ abort "jobs: #{name} was not started from a task, so there is nothing to run again" if task.empty?
164
+ stopper.stop_and_report(record)
165
+ Tasks._reset_ran!
166
+ Tasks.start([task])
167
+ end
168
+
169
+ desc "clean", "Forget every job that is no longer running: its record, script, status and log"
170
+ # Each removal re-checks under the job's start lock (see
171
+ # State#forget_unless_active), so a start that ran meanwhile wins.
172
+ def clean
173
+ invalid = state.invalid_records.map { |path| [File.basename(path, ".json"), " (not a job record)"] }
174
+ gone = state.records.reject { |record| state.active?(record) }.map { |record| [record[:name], ""] }
175
+ removed = (invalid + gone).select { |name, _| state.forget_unless_active(name) }
176
+ return puts("Nothing to clean for #{jobs_project}.") if removed.empty?
177
+
178
+ removed.each { |name, note| puts "removed #{name}#{note}" }
179
+ end
180
+ end
181
+ end
182
+ end
183
+ # rubocop:enable Style/StderrPuts
@@ -0,0 +1,149 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Asgard
6
+ # Background jobs: a process a task starts on purpose, by name, that
7
+ # outlives asgard and that a later `asgard jobs` can list and stop.
8
+ # `sh "mkdocs serve", background: true` starts one; see
9
+ # Jobs::Launcher for how, Jobs::State for what is kept on disk and
10
+ # Jobs::Commands for the management commands. This file holds the pure
11
+ # rules: naming, records, name resolution and status.
12
+ module Jobs
13
+ # A job name a user may choose: letters, digits and underscore, so it
14
+ # is safe as a file name and can never look like an auto-generated
15
+ # "<task>-<N>" name.
16
+ NAME = /\A\w+\z/
17
+
18
+ # One started job, as `sh ..., background:` returns it.
19
+ Job = Data.define(:name, :task, :pid, :log)
20
+
21
+ module_function
22
+
23
+ # The job name a `background:` value asks for, or nil for the
24
+ # foreground. +task+ is the task whose body made the call and
25
+ # +sequence+ its next job number, consulted only for `true`: a named
26
+ # job never consumes a number, so adding or removing one never
27
+ # renumbers the rest.
28
+ def job_name(background, task:, sequence: nil)
29
+ case background
30
+ when nil, false then nil
31
+ when true then sequenced_name(task, sequence)
32
+ when Symbol, String then chosen_name(background)
33
+ else raise ArgumentError, "background: must be true, a Symbol or a String, not #{background.inspect}"
34
+ end
35
+ end
36
+
37
+ # How a task name's characters outside \w are spelled in its job name,
38
+ # which keeps to [\w-] because a record's file name must for `jobs
39
+ # list` to find it. Every spelling starts with "-" and none with
40
+ # "-<digit>", so no two task names share a job name and none reads as
41
+ # the "-<N>" counter: ready? is ready-q-1, never the ready_-1 of ready_.
42
+ NAME_ESCAPES = { "?" => "-q", "!" => "-b", "=" => "-e", "-" => "--" }.freeze
43
+
44
+ def sequenced_name(task, sequence)
45
+ raise ArgumentError, "sh background: true must be called from a task body" unless task
46
+
47
+ :"#{escape_task(task)}-#{sequence}"
48
+ end
49
+
50
+ # Any other character (a non-ASCII letter) is "-u" and six hex digits.
51
+ def escape_task(task) = task.to_s.gsub(/\W/) { |char| NAME_ESCAPES.fetch(char) { format("-u%06x", char.ord) } }
52
+
53
+ def chosen_name(value)
54
+ name = value.to_s.strip
55
+ raise ArgumentError, "background: #{value.inspect} is not a job name (letters, digits and _ only)" unless name.match?(NAME)
56
+
57
+ name.to_sym
58
+ end
59
+
60
+ # Whether +record+ is one a started job wrote: a Hash with a name, a
61
+ # pid and a group id above 1 (kill(2) reads -1 and 0 as "everyone"
62
+ # and "us"), a script path to verify against and a start time. A hand
63
+ # edit or another tool's JSON fails this and stays out of listings.
64
+ def valid_record?(record)
65
+ return false unless record.is_a?(Hash)
66
+
67
+ name, pid, pgid, script, started = record.values_at(:name, :pid, :pgid, :script, :started)
68
+ text?(name) && text?(script) && process_id?(pid) && process_id?(pgid) && time?(started)
69
+ end
70
+
71
+ def text?(value) = value.is_a?(String) && !value.empty?
72
+
73
+ def process_id?(id) = id.is_a?(Integer) && id > 1
74
+
75
+ def time?(value)
76
+ value.is_a?(String) && Time.iso8601(value).is_a?(Time)
77
+ rescue ArgumentError
78
+ false
79
+ end
80
+
81
+ # The first non-blank line of a script, for the listing.
82
+ def summary(script) = script.to_s.each_line.map(&:strip).find { |line| !line.empty? }.to_s
83
+
84
+ # The record written to disk for a started job.
85
+ def record(name:, task:, pid:, pgid:, cwd:, script:, log:, command:, started:)
86
+ { name: name.to_s, task: task.to_s, pid:, pgid:, started:, cwd:, script:, log:, command: }
87
+ end
88
+
89
+ # +task+ plus every task reachable through +deps+ (a `_deps` Hash of
90
+ # stage lists), in discovery order. `stop servers` reaches the `db-1`
91
+ # a dependency started.
92
+ def closure(task, deps)
93
+ seen = []
94
+ queue = [task.to_sym]
95
+ until queue.empty?
96
+ current = queue.shift
97
+ next if seen.include?(current)
98
+
99
+ seen << current
100
+ queue.concat(Array(deps[current]).flatten.map(&:to_sym))
101
+ end
102
+ seen
103
+ end
104
+
105
+ # The records +names+ pick out: an exact job name first, else every
106
+ # record whose task is the name or, through +closure+ (a callable from
107
+ # task name to task names), one it depends on. Returns [records,
108
+ # unknown names].
109
+ def resolve(names, records, closure: nil)
110
+ found = []
111
+ unknown = []
112
+ names.each do |name|
113
+ matches = matching_records(name.to_s, records, closure)
114
+ matches.empty? ? unknown << name.to_s : found.concat(matches)
115
+ end
116
+ [found.uniq, unknown]
117
+ end
118
+
119
+ def matching_records(name, records, closure)
120
+ exact = records.select { |record| record[:name].to_s == name }
121
+ return exact unless exact.empty?
122
+
123
+ tasks = (closure ? closure.call(name) : [name]).map(&:to_s)
124
+ records.select { |record| tasks.include?(record[:task].to_s) }
125
+ end
126
+
127
+ # What a record's job is doing now: :running while its pid verifies,
128
+ # [:exited, code] once the trampoline wrote a status (with
129
+ # :processes_left while processes the script started remain in its
130
+ # group), :gone when neither (the trampoline died, or the machine
131
+ # rebooted).
132
+ def status(exit_code:, verified:, leftovers: false)
133
+ return [:exited, exit_code, (:processes_left if leftovers)].compact if exit_code
134
+ return :running if verified
135
+
136
+ :gone
137
+ end
138
+
139
+ # The STATUS column: "running", "exited 3 (2.5s)", "exited 0 (2.5s,
140
+ # processes left)" or "gone".
141
+ def describe_status(status, seconds: nil)
142
+ return status.to_s unless status.is_a?(Array)
143
+
144
+ _, code, left = status
145
+ notes = [(format("%.1fs", seconds) if seconds), ("processes left" if left)].compact
146
+ notes.empty? ? "exited #{code}" : "exited #{code} (#{notes.join(', ')})"
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "time"
5
+
6
+ module Asgard
7
+ module Jobs
8
+ # Starts a job: a new session with stdio detached, running the job's
9
+ # shell under a POSIX /bin/sh trampoline that writes the exit status
10
+ # to a file once nobody is left to wait on it. Everything but `launch`
11
+ # is pure, and `launch` is three lines.
12
+ module Launcher
13
+ # The trampoline body. The trap is load-bearing: a group TERM reaches
14
+ # this shell too, and an untrapped shell would die at once, before
15
+ # the status is written. With a handler it finishes waiting for the
16
+ # child, then runs on to the echo; the child still gets its own TERM
17
+ # with the default disposition, since handled traps reset on exec.
18
+ # Never `trap ''`: an ignored signal is inherited across exec. Two
19
+ # commands, not one, so no shell execs the body away and vanishes.
20
+ TRAMPOLINE = 'trap ":" TERM INT HUP; "$@"; echo $? > "$ASGARD_JOB_STATUS"'
21
+
22
+ module_function
23
+
24
+ # /bin/sh by absolute path: never bash and never a registry entry,
25
+ # so the user's default_shell and the presence of bash do not
26
+ # matter. The status path travels in the environment, so nothing
27
+ # here is quoted.
28
+ def argv(shell_argv) = ["/bin/sh", "-c", TRAMPOLINE, "asgard-job", *shell_argv]
29
+
30
+ # The variables added to the task's environment. Python fully
31
+ # buffers stdout when it is a file, so a print loop would show
32
+ # nothing in the log until exit without PYTHONUNBUFFERED.
33
+ def environment(name, status_path, env = ENV)
34
+ added = { "ASGARD_JOB" => name.to_s, "ASGARD_JOB_STATUS" => status_path }
35
+ added["PYTHONUNBUFFERED"] = "1" unless env.key?("PYTHONUNBUFFERED")
36
+ added
37
+ end
38
+
39
+ # stdin from /dev/null, stdout and stderr appended to the log, so
40
+ # the job never takes SIGTTIN or writes into a terminal that
41
+ # happens to be attached.
42
+ # close_others: a descriptor asgard inherited (a CI runner's, a
43
+ # `3>file`) has no business staying open for the job's lifetime.
44
+ def spawn_options(log_path) = { in: File::NULL, %i[out err] => [log_path, "a"], close_others: true }
45
+
46
+ # Forks, starts a new session (pid = pgid = sid, no controlling
47
+ # terminal) and execs; returns the child's pid, detached. The parent
48
+ # waits on a pipe whose write end the child holds until exec closes
49
+ # it (pipes are close-on-exec), so by the time the pid is recorded it
50
+ # already leads its own group, and already runs the trampoline: a
51
+ # `stop` issued at once signals the job, never the parent's group,
52
+ # and a start issued at once sees the job's own command line.
53
+ def launch(env, argv, options)
54
+ reader, writer = IO.pipe
55
+ pid = fork do
56
+ reader.close
57
+ Process.setsid
58
+ exec(env, *argv, **options)
59
+ rescue SystemCallError => e
60
+ # exit!, not raise: no at_exit handler of the parent may run here.
61
+ $stderr.puts "asgard: could not start job: #{e.message}" # rubocop:disable Style/StderrPuts
62
+ $stderr.flush # exit! skips Ruby's own flush
63
+ exit!(127)
64
+ end
65
+ writer.close
66
+ reader.read
67
+ reader.close
68
+ Process.detach(pid)
69
+ pid
70
+ end
71
+
72
+ # Starts +run+ as the job +name+ owned by +task+, or returns the job
73
+ # already running under that name. A missing shell exits 127 up
74
+ # front, as foreground `sh` does; the trampoline would otherwise
75
+ # hide it as a status nobody is watching for. A ps that cannot
76
+ # verify jobs raises Pid::Unusable first: a job nothing could find
77
+ # again would be started twice.
78
+ def start(run, name:, task:, state:, silent: false, out: $stdout)
79
+ Pid.require_ps!
80
+ state.with_lock(name) do
81
+ existing = state.read_record(name)
82
+ return already_running(existing, out:) if existing && state.verified?(existing)
83
+
84
+ run.missing! unless Shell.resolve(run.shell[:path])
85
+ job = spawn_job(run, name:, task:, state:)
86
+ out.puts "started #{job.name} (pid #{job.pid}), log: #{job.log}" unless silent
87
+ job
88
+ end
89
+ end
90
+
91
+ def already_running(record, out:)
92
+ out.puts "#{record[:name]} already running (pid #{record[:pid]}, started by #{record[:task]}), log: #{record[:log]}"
93
+ Job.new(name: record[:name].to_sym, task: record[:task].to_sym, pid: record[:pid], log: record[:log])
94
+ end
95
+
96
+ def spawn_job(run, name:, task:, state:)
97
+ state.prepare.remove(name)
98
+ script = state.script_path(name, run.ext)
99
+ log = state.log_path(name)
100
+ File.write(script, "#{run.script}\n")
101
+ FileUtils.touch(log) # so `jobs log` and a reader never race the child's open
102
+ pid = launch(environment(name, state.status_path(name)), argv(run.argv(script)), spawn_options(log))
103
+ state.write_record(Jobs.record(name:, task:, pid:, pgid: pid, cwd: Dir.pwd, script:, log:,
104
+ command: Jobs.summary(run.script), started: Time.now.iso8601(6)))
105
+ Job.new(name: name.to_sym, task: task&.to_sym, pid:, log:)
106
+ end
107
+ end
108
+ end
109
+ end