fun_ci 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +241 -0
  3. data/README.md +180 -59
  4. data/lib/fun_ci/agent/age.rb +15 -0
  5. data/lib/fun_ci/agent/command_support.rb +27 -0
  6. data/lib/fun_ci/agent/commands.rb +27 -0
  7. data/lib/fun_ci/agent/context.rb +10 -0
  8. data/lib/fun_ci/agent/digest.rb +49 -0
  9. data/lib/fun_ci/agent/events.rb +47 -0
  10. data/lib/fun_ci/agent/events_command.rb +61 -0
  11. data/lib/fun_ci/agent/evidence_text.rb +46 -0
  12. data/lib/fun_ci/agent/exit_code.rb +11 -0
  13. data/lib/fun_ci/agent/git.rb +31 -0
  14. data/lib/fun_ci/agent/live_pipeline.rb +35 -0
  15. data/lib/fun_ci/agent/options.rb +75 -0
  16. data/lib/fun_ci/agent/output.rb +81 -0
  17. data/lib/fun_ci/agent/report_stage.rb +73 -0
  18. data/lib/fun_ci/agent/reports.rb +47 -0
  19. data/lib/fun_ci/agent/run_report.rb +30 -0
  20. data/lib/fun_ci/agent/runs_command.rb +34 -0
  21. data/lib/fun_ci/agent/runs_text.rb +27 -0
  22. data/lib/fun_ci/agent/snapshots.rb +39 -0
  23. data/lib/fun_ci/agent/stage_summary.rb +30 -0
  24. data/lib/fun_ci/agent/status_command.rb +36 -0
  25. data/lib/fun_ci/agent/status_json.rb +27 -0
  26. data/lib/fun_ci/agent/status_text.rb +37 -0
  27. data/lib/fun_ci/agent/system_clock.rb +11 -0
  28. data/lib/fun_ci/agent/verdict.rb +32 -0
  29. data/lib/fun_ci/agent/wait_command.rb +71 -0
  30. data/lib/fun_ci/agent/waiting.rb +32 -0
  31. data/lib/fun_ci/agent/why_command.rb +50 -0
  32. data/lib/fun_ci/agent/why_json.rb +35 -0
  33. data/lib/fun_ci/agent/why_text.rb +43 -0
  34. data/lib/fun_ci/cli.rb +72 -94
  35. data/lib/fun_ci/cli_help.rb +65 -0
  36. data/lib/fun_ci/console/board_data.rb +66 -0
  37. data/lib/fun_ci/console/board_heartbeat.rb +40 -0
  38. data/lib/fun_ci/console/console_events.rb +20 -0
  39. data/lib/fun_ci/console/console_log.rb +26 -0
  40. data/lib/fun_ci/console/console_loop.rb +41 -0
  41. data/lib/fun_ci/console/console_session.rb +82 -0
  42. data/lib/fun_ci/console/console_state.rb +42 -0
  43. data/lib/fun_ci/console/key_handler.rb +73 -0
  44. data/lib/fun_ci/console/launcher.rb +43 -0
  45. data/lib/fun_ci/console/milestone_events.rb +38 -0
  46. data/lib/fun_ci/console/milestones.rb +25 -0
  47. data/lib/fun_ci/console/renderer_lookup.rb +51 -0
  48. data/lib/fun_ci/console/renderer_process.rb +60 -0
  49. data/lib/fun_ci/console/run_message.rb +33 -0
  50. data/lib/fun_ci/console/stage_change_detector.rb +32 -0
  51. data/lib/fun_ci/console/stage_events.rb +31 -0
  52. data/lib/fun_ci/console/stop_report.rb +39 -0
  53. data/lib/fun_ci/console/streak_counter.rb +19 -0
  54. data/lib/fun_ci/console/view.rb +47 -0
  55. data/lib/fun_ci/evidence/about.rb +12 -0
  56. data/lib/fun_ci/evidence/about_stage.rb +29 -0
  57. data/lib/fun_ci/evidence/caps.rb +77 -0
  58. data/lib/fun_ci/evidence/catalog.rb +83 -0
  59. data/lib/fun_ci/evidence/collector.rb +95 -0
  60. data/lib/fun_ci/evidence/command_output.rb +45 -0
  61. data/lib/fun_ci/evidence/command_reading.rb +38 -0
  62. data/lib/fun_ci/evidence/command_runner.rb +75 -0
  63. data/lib/fun_ci/evidence/context.rb +21 -0
  64. data/lib/fun_ci/evidence/context_document.rb +29 -0
  65. data/lib/fun_ci/evidence/deadline.rb +15 -0
  66. data/lib/fun_ci/evidence/detection.rb +82 -0
  67. data/lib/fun_ci/evidence/document.rb +59 -0
  68. data/lib/fun_ci/evidence/extract_command.rb +83 -0
  69. data/lib/fun_ci/evidence/extract_options.rb +45 -0
  70. data/lib/fun_ci/evidence/extraction.rb +54 -0
  71. data/lib/fun_ci/evidence/extractors/command.rb +52 -0
  72. data/lib/fun_ci/evidence/extractors/grep.rb +49 -0
  73. data/lib/fun_ci/evidence/extractors/json_log.rb +45 -0
  74. data/lib/fun_ci/evidence/extractors/junit_files.rb +32 -0
  75. data/lib/fun_ci/evidence/extractors/last_lines.rb +25 -0
  76. data/lib/fun_ci/evidence/extractors/log_file.rb +62 -0
  77. data/lib/fun_ci/evidence/extractors/process_tree.rb +51 -0
  78. data/lib/fun_ci/evidence/extractors/section.rb +49 -0
  79. data/lib/fun_ci/evidence/findings.rb +14 -0
  80. data/lib/fun_ci/evidence/line_ranges.rb +21 -0
  81. data/lib/fun_ci/evidence/line_scan.rb +22 -0
  82. data/lib/fun_ci/evidence/log_record.rb +58 -0
  83. data/lib/fun_ci/evidence/masking.rb +55 -0
  84. data/lib/fun_ci/evidence/option_check.rb +53 -0
  85. data/lib/fun_ci/evidence/outcome.rb +15 -0
  86. data/lib/fun_ci/evidence/patterns.rb +22 -0
  87. data/lib/fun_ci/evidence/presets/README.md +7 -0
  88. data/lib/fun_ci/evidence/presets/bun-test.yml +6 -0
  89. data/lib/fun_ci/evidence/presets/cargo-test.yml +6 -0
  90. data/lib/fun_ci/evidence/presets/dart-test.yml +6 -0
  91. data/lib/fun_ci/evidence/presets/deno-test.yml +6 -0
  92. data/lib/fun_ci/evidence/presets/dotnet-test.yml +6 -0
  93. data/lib/fun_ci/evidence/presets/ecs.yml +3 -0
  94. data/lib/fun_ci/evidence/presets/eslint.yml +6 -0
  95. data/lib/fun_ci/evidence/presets/exunit.yml +6 -0
  96. data/lib/fun_ci/evidence/presets/gcc.yml +6 -0
  97. data/lib/fun_ci/evidence/presets/go-build.yml +5 -0
  98. data/lib/fun_ci/evidence/presets/go-test.yml +6 -0
  99. data/lib/fun_ci/evidence/presets/gradle.yml +6 -0
  100. data/lib/fun_ci/evidence/presets/gtest.yml +6 -0
  101. data/lib/fun_ci/evidence/presets/jest.yml +6 -0
  102. data/lib/fun_ci/evidence/presets/logstash.yml +3 -0
  103. data/lib/fun_ci/evidence/presets/maven.yml +6 -0
  104. data/lib/fun_ci/evidence/presets/minitest.yml +6 -0
  105. data/lib/fun_ci/evidence/presets/mocha.yml +6 -0
  106. data/lib/fun_ci/evidence/presets/mypy.yml +6 -0
  107. data/lib/fun_ci/evidence/presets/node-test.yml +6 -0
  108. data/lib/fun_ci/evidence/presets/phpunit.yml +6 -0
  109. data/lib/fun_ci/evidence/presets/pino.yml +5 -0
  110. data/lib/fun_ci/evidence/presets/prove.yml +6 -0
  111. data/lib/fun_ci/evidence/presets/pytest.yml +6 -0
  112. data/lib/fun_ci/evidence/presets/rspec.yml +6 -0
  113. data/lib/fun_ci/evidence/presets/rubocop.yml +6 -0
  114. data/lib/fun_ci/evidence/presets/ruff.yml +6 -0
  115. data/lib/fun_ci/evidence/presets/rustc.yml +6 -0
  116. data/lib/fun_ci/evidence/presets/shellcheck.yml +5 -0
  117. data/lib/fun_ci/evidence/presets/structlog.yml +4 -0
  118. data/lib/fun_ci/evidence/presets/swift-test.yml +5 -0
  119. data/lib/fun_ci/evidence/presets/tsc.yml +6 -0
  120. data/lib/fun_ci/evidence/presets/unittest.yml +6 -0
  121. data/lib/fun_ci/evidence/presets/vitest.yml +6 -0
  122. data/lib/fun_ci/evidence/presets.rb +30 -0
  123. data/lib/fun_ci/evidence/problem.rb +9 -0
  124. data/lib/fun_ci/evidence/process_table.rb +40 -0
  125. data/lib/fun_ci/evidence/saved_run.rb +22 -0
  126. data/lib/fun_ci/evidence/section_walk.rb +49 -0
  127. data/lib/fun_ci/evidence/settings.rb +64 -0
  128. data/lib/fun_ci/evidence/settings_check.rb +78 -0
  129. data/lib/fun_ci/evidence/source.rb +34 -0
  130. data/lib/fun_ci/evidence/sources.rb +21 -0
  131. data/lib/fun_ci/evidence/stage_context.rb +30 -0
  132. data/lib/fun_ci/evidence/stamp.rb +10 -0
  133. data/lib/fun_ci/evidence/start.rb +36 -0
  134. data/lib/fun_ci/evidence/worktree.rb +62 -0
  135. data/lib/fun_ci/persistence/active_runs.rb +69 -0
  136. data/lib/fun_ci/persistence/database.rb +59 -30
  137. data/lib/fun_ci/persistence/null_recorder.rb +25 -0
  138. data/lib/fun_ci/persistence/output_tail.rb +24 -0
  139. data/lib/fun_ci/persistence/pipeline_recorder.rb +78 -32
  140. data/lib/fun_ci/persistence/pipeline_run.rb +26 -17
  141. data/lib/fun_ci/persistence/project_runs.rb +52 -0
  142. data/lib/fun_ci/persistence/raw_outputs.rb +46 -0
  143. data/lib/fun_ci/persistence/retention.rb +25 -0
  144. data/lib/fun_ci/persistence/run_status.rb +31 -0
  145. data/lib/fun_ci/persistence/stage_job.rb +58 -13
  146. data/lib/fun_ci/persistence/state_dir.rb +26 -0
  147. data/lib/fun_ci/persistence/write_trouble.rb +46 -0
  148. data/lib/fun_ci/pipeline/background_fork.rb +42 -0
  149. data/lib/fun_ci/pipeline/command_executor.rb +58 -0
  150. data/lib/fun_ci/pipeline/gate.rb +35 -0
  151. data/lib/fun_ci/pipeline/git_environment.rb +16 -0
  152. data/lib/fun_ci/pipeline/head_part.rb +40 -0
  153. data/lib/fun_ci/pipeline/in_place.rb +17 -0
  154. data/lib/fun_ci/pipeline/output_window.rb +73 -0
  155. data/lib/fun_ci/pipeline/pipeline_forker.rb +18 -16
  156. data/lib/fun_ci/pipeline/process_runner.rb +90 -17
  157. data/lib/fun_ci/pipeline/run_canceller.rb +75 -0
  158. data/lib/fun_ci/pipeline/slot.rb +30 -0
  159. data/lib/fun_ci/pipeline/slot_lock.rb +13 -0
  160. data/lib/fun_ci/pipeline/slot_run.rb +86 -0
  161. data/lib/fun_ci/pipeline/stage_dir.rb +50 -0
  162. data/lib/fun_ci/pipeline/stage_end.rb +70 -0
  163. data/lib/fun_ci/pipeline/stage_execution.rb +67 -0
  164. data/lib/fun_ci/pipeline/stage_runner.rb +42 -64
  165. data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +16 -33
  166. data/lib/fun_ci/pipeline/tail_segments.rb +60 -0
  167. data/lib/fun_ci/pipeline/test_report.rb +48 -0
  168. data/lib/fun_ci/pipeline/trigger.rb +55 -115
  169. data/lib/fun_ci/pipeline/trigger_command.rb +62 -0
  170. data/lib/fun_ci/pipeline/trigger_params.rb +48 -0
  171. data/lib/fun_ci/pipeline/workspaces.rb +22 -0
  172. data/lib/fun_ci/pipeline/worktree_pool.rb +66 -0
  173. data/lib/fun_ci/pipeline/worktree_prune.rb +49 -0
  174. data/lib/fun_ci/pipeline/worktrees.rb +45 -0
  175. data/lib/fun_ci/setup/agent_instructions.rb +31 -0
  176. data/lib/fun_ci/setup/commands.rb +49 -0
  177. data/lib/fun_ci/setup/git_hooks.rb +20 -0
  178. data/lib/fun_ci/setup/hook_script.rb +47 -0
  179. data/lib/fun_ci/setup/hook_writer.rb +15 -33
  180. data/lib/fun_ci/setup/installer.rb +39 -23
  181. data/lib/fun_ci/setup/legacy_hooks.rb +38 -0
  182. data/lib/fun_ci/setup/project_config.rb +43 -17
  183. data/lib/fun_ci/setup/project_detector.rb +39 -7
  184. data/lib/fun_ci/setup/settings.rb +36 -0
  185. data/lib/fun_ci/setup/setup_checker.rb +18 -12
  186. data/lib/fun_ci/setup/stage_templates/javascript.rb +18 -0
  187. data/lib/fun_ci/setup/stage_templates/managed.rb +24 -0
  188. data/lib/fun_ci/setup/stage_templates/native.rb +25 -0
  189. data/lib/fun_ci/setup/stage_templates/scripting.rb +27 -0
  190. data/lib/fun_ci/setup/stage_templates/shell.rb +16 -0
  191. data/lib/fun_ci/setup/stage_templates.rb +25 -0
  192. data/lib/fun_ci/setup/template_writer.rb +11 -36
  193. data/lib/fun_ci/version.rb +6 -0
  194. data/lib/fun_ci.rb +1 -3
  195. metadata +212 -43
  196. data/exe/fun-ci-trigger +0 -19
  197. data/exe/fun-ci-tui +0 -23
  198. data/lib/fun_ci/animations/celebrate.rb +0 -210
  199. data/lib/fun_ci/animations/explosion.rb +0 -138
  200. data/lib/fun_ci/animations/flash.rb +0 -205
  201. data/lib/fun_ci/animations/idle.rb +0 -127
  202. data/lib/fun_ci/animations/leprechauns.rb +0 -240
  203. data/lib/fun_ci/animations/running.rb +0 -129
  204. data/lib/fun_ci/animations/success.rb +0 -281
  205. data/lib/fun_ci/animations/yay.rb +0 -169
  206. data/lib/fun_ci/persistence/state_machine.rb +0 -50
  207. data/lib/fun_ci/pipeline/background_wrapper.rb +0 -27
  208. data/lib/fun_ci/tui/admin_tui.rb +0 -90
  209. data/lib/fun_ci/tui/animation.rb +0 -49
  210. data/lib/fun_ci/tui/animation_compositor.rb +0 -107
  211. data/lib/fun_ci/tui/animation_frames.rb +0 -112
  212. data/lib/fun_ci/tui/animation_library.rb +0 -46
  213. data/lib/fun_ci/tui/animation_renderer.rb +0 -144
  214. data/lib/fun_ci/tui/ansi.rb +0 -34
  215. data/lib/fun_ci/tui/board_data.rb +0 -53
  216. data/lib/fun_ci/tui/board_renderer.rb +0 -105
  217. data/lib/fun_ci/tui/duration_formatter.rb +0 -24
  218. data/lib/fun_ci/tui/header_animation_manager.rb +0 -71
  219. data/lib/fun_ci/tui/header_animation_player.rb +0 -45
  220. data/lib/fun_ci/tui/key_handler.rb +0 -86
  221. data/lib/fun_ci/tui/looping_animation_player.rb +0 -45
  222. data/lib/fun_ci/tui/relative_time.rb +0 -22
  223. data/lib/fun_ci/tui/row_formatter.rb +0 -108
  224. data/lib/fun_ci/tui/screen.rb +0 -103
  225. data/lib/fun_ci/tui/spinner.rb +0 -24
  226. data/lib/fun_ci/tui/stage_change_detector.rb +0 -58
  227. data/lib/fun_ci/tui/streak_counter.rb +0 -29
  228. data/lib/fun_ci/tui/terminal_input.rb +0 -69
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Evidence
5
+ # What an extractor is given about a failed stage (architecture.md, "Evidence of a failed stage"):
6
+ # the stage, its output as the window kept it, the worktree it ran in
7
+ # (read through #read, #exist?, #glob, #stamp and #lines_before, by path
8
+ # relative to it), the deadline it checks as it goes, the Stamp of each
9
+ # watched file when the stage started, by path, what is known About the
10
+ # stage, `commands`, which runs a project's own extractors, and, for a
11
+ # stage that ran over budget, before the kill, its process group (`pgid`)
12
+ # and `processes`, which answers what ps lists.
13
+ Context = Data.define(:stage, :output, :worktree, :deadline, :watched, :about, :commands, :pgid, :processes)
14
+
15
+ class Context
16
+ UNKNOWN = { watched: {}, about: nil, commands: nil, pgid: nil, processes: -> { [] } }.freeze
17
+
18
+ def initialize(**given) = super(**UNKNOWN, **given)
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "about"
4
+
5
+ module FunCi
6
+ module Evidence
7
+ # The context a project's command reads on stdin (architecture.md, "Evidence
8
+ # of a failed stage"). Fields once published keep their names; a change that
9
+ # isn't additive bumps SCHEMA.
10
+ module ContextDocument
11
+ SCHEMA = 1
12
+
13
+ # watch: the entry's globs; options: whatever else the entry said.
14
+ def self.for(about, context, watch:, options:)
15
+ paths = watch.flat_map { |glob| context.worktree.glob(glob) }.uniq.sort
16
+ { schema: SCHEMA, **about.to_h, changed: changed(paths, context), watched: watched(paths, context),
17
+ options: options, pgid: context.pgid }
18
+ end
19
+
20
+ def self.changed(paths, context) = paths.reject { |path| context.watched[path] == context.worktree.stamp(path) }
21
+
22
+ def self.watched(paths, context)
23
+ existed = paths.select { |path| context.watched.key?(path) }
24
+ existed.map { |path| { path: path, offset: context.watched[path].size } }
25
+ end
26
+ private_class_method :changed, :watched
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Evidence
5
+ # When the evidence budget runs out. `clock` answers the time in seconds.
6
+ Deadline = Data.define(:clock, :at)
7
+
8
+ class Deadline
9
+ def self.after(clock, seconds) = new(clock: clock, at: clock.call + seconds)
10
+
11
+ def passed? = clock.call >= at
12
+ def left = [at - clock.call, 0].max
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "presets"
4
+ require_relative "line_scan"
5
+ require_relative "patterns"
6
+ require_relative "source"
7
+
8
+ module FunCi
9
+ module Evidence
10
+ # Which presets run when a stage fails (architecture.md, "Evidence of a failed stage").
11
+ # When it starts, the candidates: presets any of whose marker files the
12
+ # worktree has (a marker may also name a text the file must hold), and
13
+ # presets without markers. When it fails, of the candidates with an
14
+ # output signature, those whose signature matched a line of the output's
15
+ # last megabyte, where tools print their summaries: each signature costs
16
+ # a read, and over the whole window with every preset a candidate that
17
+ # was most of the budget (script/bench_detection.rb). The part is read
18
+ # once, against the signatures joined.
19
+ module Detection
20
+ # because: why it was chosen, such as "file Gemfile", or nil.
21
+ Found = Data.define(:preset, :because)
22
+ SCAN_BYTES = 1_048_576
23
+
24
+ def self.candidates(worktree, presets)
25
+ presets.filter_map do |preset|
26
+ next Found.new(preset: preset, because: nil) if preset.markers.empty?
27
+
28
+ file = preset.markers.lazy.filter_map { |marker| marked(worktree, marker) }.first
29
+ file && Found.new(preset: preset, because: "file #{file}")
30
+ end
31
+ end
32
+
33
+ def self.chosen(output, candidates, deadline, scan_bytes: SCAN_BYTES)
34
+ last_part = output.byteslice([output.bytesize - scan_bytes, 0].max..)
35
+ seen = SignatureScan.new(candidates.map(&:preset)).seen(Source.of("output", last_part).lines, deadline)
36
+ candidates.filter_map { |found| chose(found, seen) }
37
+ end
38
+
39
+ def self.chose(found, seen)
40
+ return found unless found.preset.signature
41
+
42
+ matched = seen[found.preset.name]
43
+ matched && found.with(because: [found.because, "output matched #{matched.inspect}"].compact.join(", "))
44
+ end
45
+
46
+ # The file that bears the marker out, or nil. A marker is a path or a
47
+ # glob, or { path, contains } for a file that must hold a text.
48
+ def self.marked(worktree, marker)
49
+ return worktree.glob(marker).first if marker.is_a?(String)
50
+
51
+ path = marker["path"]
52
+ path if worktree.exist?(path) && worktree.read(path).include?(marker["contains"])
53
+ end
54
+ private_class_method :chose, :marked
55
+ end
56
+
57
+ # Reads lines once against a set of presets' signatures joined into one
58
+ # pattern; a line the joined pattern matches is tried against each
59
+ # signature, since a joined pattern reports one of those that match.
60
+ class SignatureScan
61
+ def initialize(presets)
62
+ @signatures = presets.select(&:signature).to_h { |preset| [preset.name, Regexp.new(preset.signature)] }
63
+ @joined = Patterns.compile(@signatures.values.map { |pattern| "(?:#{pattern.source})" }.join("|")).first
64
+ end
65
+
66
+ # The first text each signature matched, by preset name.
67
+ def seen(lines, deadline)
68
+ seen = {}
69
+ return seen if @signatures.empty?
70
+
71
+ LineScan.cut_short?(lines, deadline) { |line, _| note(seen, line) if @joined.match?(line) }
72
+ seen
73
+ end
74
+
75
+ private
76
+
77
+ def note(seen, line)
78
+ @signatures.each { |name, pattern| seen[name] ||= pattern.match(line)&.[](0) }
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module FunCi
6
+ module Evidence
7
+ # What fun-ci kept about why a stage failed (architecture.md, "Evidence of a failed stage"): the extractors
8
+ # chosen and why, facts, failures, excerpts, and the problems extractors
9
+ # had. Each item is a hash naming the extractor that found it.
10
+ Document = Data.define(:chosen, :facts, :failures, :excerpts, :problems)
11
+
12
+ class Document
13
+ TAIL_TITLE = "The output's last lines"
14
+
15
+ # A document as fun-ci kept it; a list it doesn't have is empty.
16
+ def self.from_json(text)
17
+ parsed = JSON.parse(text, symbolize_names: true)
18
+ new(**members.to_h { |member| [member, parsed.fetch(member, [])] })
19
+ end
20
+
21
+ # The evidence from each extractor's findings, [name, findings] in the
22
+ # order they are shown, each item crediting the extractor that found
23
+ # it; the same failure found twice is kept where it was found first.
24
+ def self.assemble(parts, problems:, chosen:)
25
+ failures = credited(parts, :failures).uniq { |failure| failure.values_at(:test, :file, :line) }
26
+ new(chosen: chosen, facts: credited(parts, :facts), failures: failures,
27
+ excerpts: credited(parts, :excerpts), problems: problems)
28
+ end
29
+
30
+ def self.credited(parts, list)
31
+ parts.flat_map { |name, found| found.public_send(list).map { |item| item.merge(extractor: name) } }
32
+ end
33
+
34
+ # The evidence of a stage whose collecting went wrong: what went wrong.
35
+ def self.broken(message)
36
+ new(chosen: [], facts: [], failures: [], excerpts: [],
37
+ problems: [{ extractor: "fun-ci", message: "couldn't collect the evidence: #{message}" }])
38
+ end
39
+
40
+ # The evidence of a row an older fun-ci kept: the output's tail, and the
41
+ # failures its stage's reports named.
42
+ def self.legacy(tail:, failures:)
43
+ new(chosen: [], facts: [], failures: failures.map { |failure| failure.merge(extractor: "test-reports") },
44
+ excerpts: tail ? [tail_excerpt(tail)] : [], problems: [])
45
+ end
46
+
47
+ # The output's last lines as output-tail kept them, or nil.
48
+ def tail
49
+ excerpt = excerpts.find { |candidate| candidate[:extractor] == "output-tail" }
50
+ excerpt && excerpt[:lines].map { |line| "#{line}\n" }.join
51
+ end
52
+
53
+ def self.tail_excerpt(tail)
54
+ { title: TAIL_TITLE, location: "output", lines: tail.lines(chomp: true), extractor: "output-tail" }
55
+ end
56
+ private_class_method :tail_excerpt, :credited
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "tmpdir"
5
+ require_relative "extract_options"
6
+ require_relative "saved_run"
7
+ require_relative "collector"
8
+ require_relative "command_runner"
9
+ require_relative "worktree"
10
+ require_relative "../agent/stage_summary"
11
+ require_relative "../agent/evidence_text"
12
+ require_relative "../agent/report_stage"
13
+
14
+ module FunCi
15
+ module Evidence
16
+ # `fun-ci extract STAGE --output FILE` (acceptance-tests.md, AT-10.16):
17
+ # runs a stage's extractors against a saved output, with the current
18
+ # directory as the worktree, and prints what `why` would print, touching
19
+ # no database, so a project can try an extractor without a commit.
20
+ class ExtractCommand
21
+ NO_START = { name: "start sizes",
22
+ value: "none, so files are read whole and every watched file counts as changed" }.freeze
23
+ NO_GROUP = { name: "process group", value: "none, so entries with on: overrun don't run" }.freeze
24
+
25
+ def initialize(dir, io)
26
+ @dir = dir
27
+ @io = io
28
+ end
29
+
30
+ def run(args)
31
+ options = ExtractOptions.parse(args)
32
+ document = Dir.mktmpdir("fun-ci-extract") { |scratch| collect(options, scratch) }
33
+ options.json ? print_json(options, document) : print_text(options, document)
34
+ 0
35
+ rescue ExtractOptions::Invalid => e
36
+ @io.stderr.puts "fun-ci extract: #{e.message}"
37
+ 64
38
+ end
39
+
40
+ private
41
+
42
+ def collect(options, scratch)
43
+ sources = Sources.new(stage: options.stage, worktree: @dir, stage_dir: SavedRun.new(scratch),
44
+ environment: ENV.to_h)
45
+ collector = Collector.new(sources, settings: Settings.load(File.join(@dir, ".fun-ci", "config")),
46
+ commands: CommandRunner.new(dir: @dir, scratch: scratch))
47
+ noted(collector.collect(output(options), outcome(options)), options)
48
+ end
49
+
50
+ # The saved output, a path from the project or an absolute one.
51
+ def output(options)
52
+ path = File.expand_path(options.output, @dir)
53
+ Worktree.new(File.dirname(path)).read(File.basename(path))
54
+ rescue SystemCallError => e
55
+ raise ExtractOptions::Invalid, "can't read #{options.output}: #{e.message.sub(/ @ .*/, "")}"
56
+ end
57
+
58
+ def state(options) = options.timed_out ? "over_budget" : "failed"
59
+
60
+ def outcome(options)
61
+ Outcome.new(state: state(options), exit_status: options.timed_out ? nil : options.exit_status)
62
+ end
63
+
64
+ def noted(document, options)
65
+ notes = [NO_START, *(options.timed_out ? [NO_GROUP] : [])]
66
+ notes = notes.map { |fact| fact.merge(extractor: "fun-ci extract") }
67
+ document.with(facts: notes + document.facts)
68
+ end
69
+
70
+ def print_text(options, document)
71
+ stage = Agent::RunReport::Stage.new(name: options.stage, state: state(options), seconds: nil,
72
+ exit: Agent::RunReport::Exit.new(exit_status: outcome(options).exit_status,
73
+ signal: nil, budget: nil))
74
+ [Agent::StageSummary.line(stage), *Agent::EvidenceText.lines(document)].each { |line| @io.stdout.puts line }
75
+ end
76
+
77
+ def print_json(options, document)
78
+ @io.stdout.puts JSON.generate({ schema: 1, stage: options.stage, state: state(options),
79
+ exit_status: outcome(options).exit_status, evidence: document.to_h })
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+
5
+ module FunCi
6
+ module Evidence
7
+ # The arguments of `fun-ci extract STAGE --output FILE [--exit N |
8
+ # --timed-out] [--json]`.
9
+ module ExtractOptions
10
+ class Invalid < StandardError; end
11
+
12
+ Parsed = Data.define(:stage, :output, :exit_status, :timed_out, :json)
13
+ STAGES = %w[lint build fast slow].freeze
14
+ DEFAULTS = { output: nil, exit_status: 1, timed_out: false, json: false }.freeze
15
+
16
+ SWITCHES = { output: ["--output FILE"], exit_status: ["--exit N", Integer], timed_out: ["--timed-out"],
17
+ json: ["--json"] }.freeze
18
+
19
+ def self.parse(args)
20
+ values = DEFAULTS.dup
21
+ stages = parser(values).parse(args)
22
+ raise Invalid, "--output FILE names the saved output to read" unless values[:output]
23
+
24
+ Parsed.new(stage: one_stage(stages), **values)
25
+ rescue OptionParser::ParseError => e
26
+ raise Invalid, e.message
27
+ end
28
+
29
+ def self.one_stage(stages)
30
+ return stages.first if stages.size == 1 && STAGES.include?(stages.first)
31
+
32
+ raise Invalid, "name one stage of #{STAGES.join(", ")}, not #{stages.join(" ")}"
33
+ end
34
+
35
+ # A switch without an argument sets its value to true.
36
+ def self.parser(values)
37
+ parser = OptionParser.new
38
+ parser.base.long.clear
39
+ SWITCHES.each { |key, switch| parser.on(*switch) { |value| values[key] = value } }
40
+ parser
41
+ end
42
+ private_class_method :one_stage, :parser
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "catalog"
4
+ require_relative "problem"
5
+
6
+ module FunCi
7
+ module Evidence
8
+ # Runs a stage's entries in order within the budget: an entry that is
9
+ # mistaken, raises, or comes after the deadline is a problem, and the
10
+ # others still run.
11
+ class Extraction
12
+ # parts: [[name, findings], ...] in order; chosen: [{ extractor:, because: }].
13
+ Result = Data.define(:parts, :problems, :chosen)
14
+ # An entry that ran; one that didn't is a problem, { extractor:, message: }.
15
+ Ran = Data.define(:name, :findings, :because)
16
+
17
+ def initialize(context, budget)
18
+ @context = context
19
+ @budget = budget
20
+ end
21
+
22
+ # raw_entries: [[raw entry, why it was chosen], ...]
23
+ def run(raw_entries)
24
+ ran, problems = raw_entries.map { |raw, because| attempt(raw, because) }.partition { |done| done.is_a?(Ran) }
25
+ Result.new(parts: ran.map { |done| [done.name, done.findings] }, problems: problems,
26
+ chosen: ran.map { |done| { extractor: done.name, because: done.because } })
27
+ end
28
+
29
+ private
30
+
31
+ def attempt(raw, because)
32
+ entry = Catalog.entry(raw)
33
+ return failed(entry.name, "not run: the evidence budget of #{@budget}s ran out") if @context.deadline.passed?
34
+
35
+ ran(entry, because)
36
+ rescue Catalog::Refused => e
37
+ failed(label(raw), e.message)
38
+ end
39
+
40
+ # Whatever an extractor raises is a problem, never the end of the evidence.
41
+ def ran(entry, because)
42
+ Ran.new(name: entry.name, findings: entry.extractor.extract(@context), because: because)
43
+ rescue Problem => e
44
+ failed(entry.name, e.message)
45
+ rescue StandardError => e
46
+ failed(entry.name, "#{e.class}: #{e.message}")
47
+ end
48
+
49
+ def failed(name, message) = { extractor: name, message: message }
50
+
51
+ def label(raw) = raw.is_a?(Hash) ? raw["use"] || "run:#{raw["run"]}" : raw.to_s
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require_relative "../command_output"
5
+ require_relative "../context_document"
6
+ require_relative "../problem"
7
+
8
+ module FunCi
9
+ module Evidence
10
+ module Extractors
11
+ # A project's own extractor, a `run:` entry: the command, run in the
12
+ # worktree with what is left of the budget, reads the context on stdin
13
+ # and prints evidence as text or, with `format: json`, as a document
14
+ # (architecture.md, "Evidence of a failed stage"). `watch` names the files it is told
15
+ # about; whatever else the entry says is passed on as `options`.
16
+ class Command
17
+ OWN = %w[run format watch].freeze
18
+ STDERR_LINES = 20
19
+
20
+ def initialize(options)
21
+ @options = options
22
+ end
23
+
24
+ def extract(context)
25
+ ran = context.commands.call(@options["run"], stdin: JSON.generate(document(context)),
26
+ seconds: context.deadline.left, env: group(context))
27
+ refuse(ran)
28
+ return CommandOutput.json(ran.stdout) if @options["format"] == "json"
29
+
30
+ CommandOutput.text(@options["run"], ran.stdout)
31
+ end
32
+
33
+ private
34
+
35
+ # Run before the kill of a stage over budget, it is told the stage's group.
36
+ def group(context) = context.pgid ? { "FUN_CI_PGID" => context.pgid.to_s } : {}
37
+
38
+ def document(context)
39
+ ContextDocument.for(context.about, context, watch: Array(@options["watch"]), options: @options.except(*OWN))
40
+ end
41
+
42
+ def refuse(ran)
43
+ raise Problem, "killed: still running when the evidence budget ran out" if ran.killed == :budget
44
+ raise Problem, "killed: printed more than 256 KB" if ran.killed == :overflow
45
+ raise Problem, "exited #{ran.exit_status}: #{last_lines(ran.stderr)}" unless ran.exit_status.zero?
46
+ end
47
+
48
+ def last_lines(stderr) = stderr.lines(chomp: true).last(STDERR_LINES).join("\n")
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../findings"
4
+ require_relative "../source"
5
+ require_relative "../line_scan"
6
+ require_relative "../patterns"
7
+ require_relative "../line_ranges"
8
+
9
+ module FunCi
10
+ module Evidence
11
+ module Extractors
12
+ # `grep`: the lines of the output, or of a file, that match any of
13
+ # `patterns`, with `context` lines around each; matches whose context
14
+ # meets are one excerpt.
15
+ class Grep
16
+ OPTIONS = { "patterns" => :patterns, "context" => :count, "path" => :path, "title" => :word }.freeze
17
+ REQUIRED = %w[patterns].freeze
18
+
19
+ def initialize(options)
20
+ @options = options
21
+ end
22
+
23
+ def extract(context) = found_in(Source.for(context, @options["path"]), context.deadline)
24
+
25
+ def found_in(source, deadline)
26
+ hits, truncated = matches(source.lines, deadline)
27
+ ranges = LineRanges.around(hits, @options.fetch("context", 0), source.lines.size)
28
+ Findings.new(excerpts: ranges.map { |range| excerpt(source, range, truncated) })
29
+ end
30
+
31
+ private
32
+
33
+ def matches(lines, deadline)
34
+ patterns = Patterns.compile(@options["patterns"])
35
+ hits = []
36
+ truncated = LineScan.cut_short?(lines, deadline) do |line, index|
37
+ hits << index if patterns.any? { |pattern| pattern.match?(line) }
38
+ end
39
+ [hits, truncated]
40
+ end
41
+
42
+ def excerpt(source, range, truncated)
43
+ { title: @options.fetch("title") { "Lines matching #{Array(@options["patterns"]).join(" or ")}" },
44
+ location: source.location(range.first, range.last), lines: source.lines[range], truncated: truncated }
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../findings"
4
+ require_relative "../source"
5
+ require_relative "../line_scan"
6
+ require_relative "../log_record"
7
+
8
+ module FunCi
9
+ module Evidence
10
+ module Extractors
11
+ # `json-log`: the JSON log lines of the output, or of a file, at or
12
+ # above `level` (warn by default), each as one line (time, level,
13
+ # logger, message) followed by its stack trace. A preset names a
14
+ # logging setup's `fields`, such as logstash's.
15
+ class JsonLog
16
+ OPTIONS = { "path" => :path, "level" => :level, "fields" => :fields, "levels" => :levels }.freeze
17
+ REQUIRED = %w[fields].freeze
18
+
19
+ def initialize(options)
20
+ @options = options
21
+ end
22
+
23
+ def extract(context)
24
+ source = Source.for(context, @options["path"])
25
+ kept = []
26
+ truncated = LineScan.cut_short?(source.lines, context.deadline) { |line, index| keep(kept, line, index) }
27
+ Findings.new(excerpts: kept.empty? ? [] : [excerpt(source, kept, truncated)])
28
+ end
29
+
30
+ private
31
+
32
+ def keep(kept, line, index)
33
+ record = LogRecord.parse(line, @options["fields"], @options.fetch("levels", {}))
34
+ kept << [index, record.lines] if record&.at_least?(LogRecord.rank(@options.fetch("level", "warn")))
35
+ end
36
+
37
+ def excerpt(source, kept, truncated)
38
+ { title: "Log records at #{@options.fetch("level", "warn")} or above",
39
+ location: source.location(kept.first.first, kept.last.first), lines: kept.flat_map(&:last),
40
+ truncated: truncated }
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../findings"
4
+ require_relative "../../pipeline/test_report"
5
+
6
+ module FunCi
7
+ module Evidence
8
+ module Extractors
9
+ # `junit-files`: the failures in the JUnit XML under `paths` that the
10
+ # stage wrote, where a build tool keeps its own reports
11
+ # (target/surefire-reports/*.xml, build/test-results/**/*.xml).
12
+ class JunitFiles
13
+ OPTIONS = { "paths" => :globs }.freeze
14
+ REQUIRED = %w[paths].freeze
15
+
16
+ def initialize(options)
17
+ @options = options
18
+ end
19
+
20
+ def extract(context)
21
+ written = globs.flat_map { |glob| context.worktree.glob(glob) }.uniq
22
+ .reject { |path| context.watched[path] == context.worktree.stamp(path) }
23
+ Findings.new(failures: written.flat_map do |path|
24
+ Pipeline::TestReport.junit(context.worktree.read(path)) || []
25
+ end)
26
+ end
27
+
28
+ def globs = Array(@options["paths"])
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../findings"
4
+
5
+ module FunCi
6
+ module Evidence
7
+ module Extractors
8
+ # The last `count` lines of a source, as one excerpt.
9
+ class LastLines
10
+ def initialize(count)
11
+ @count = count
12
+ end
13
+
14
+ def found_in(source, _deadline)
15
+ return Findings.new if source.lines.empty?
16
+
17
+ first = [source.lines.size - @count, 0].max
18
+ Findings.new(excerpts: [{ title: "What the stage wrote to #{source.name}",
19
+ location: source.location(first, source.lines.size - 1),
20
+ lines: source.lines[first..], truncated: first.positive? }])
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../findings"
4
+ require_relative "../source"
5
+ require_relative "grep"
6
+ require_relative "section"
7
+ require_relative "last_lines"
8
+
9
+ module FunCi
10
+ module Evidence
11
+ module Extractors
12
+ # `log-file`: what the stage wrote to the files under `path`, read from
13
+ # the size each had when it started; whole, for one it created, or one
14
+ # rewritten (no longer, or a new inode), which the fact `truncated`
15
+ # names. Filtered by `grep` or by a section from `start`, when given,
16
+ # else its last `lines`.
17
+ class LogFile
18
+ OPTIONS = { "path" => :path, "grep" => :patterns, "context" => :count, "start" => :patterns,
19
+ "end" => :patterns, "lines" => :count }.freeze
20
+ REQUIRED = %w[path].freeze
21
+ LINES = 200
22
+
23
+ def initialize(options)
24
+ @options = options
25
+ end
26
+
27
+ def extract(context)
28
+ changed = context.worktree.glob(@options["path"]).reject { |path| unchanged?(context, path) }
29
+ found = changed.map { |path| written(context, path) }
30
+ Findings.new(facts: changed.select { |path| rewritten?(context, path) }.map { |path| truncated(path) },
31
+ excerpts: found.flat_map(&:excerpts))
32
+ end
33
+
34
+ private
35
+
36
+ def truncated(path) = { name: "truncated", value: path }
37
+ def unchanged?(context, path) = context.watched[path] == context.worktree.stamp(path)
38
+
39
+ # Changed without growing, or replaced by another file.
40
+ def rewritten?(context, path)
41
+ before = context.watched[path]
42
+ now = context.worktree.stamp(path)
43
+ !before.nil? && (now.size <= before.size || now.inode != before.inode)
44
+ end
45
+
46
+ def written(context, path)
47
+ from = rewritten?(context, path) ? 0 : context.watched[path]&.size.to_i
48
+ source = Source.of(path, context.worktree.read(path, from: from),
49
+ skipped: context.worktree.lines_before(path, from))
50
+ filter.found_in(source, context.deadline)
51
+ end
52
+
53
+ def filter
54
+ return Grep.new(@options.slice("context").merge("patterns" => @options["grep"])) if @options["grep"]
55
+ return Section.new(@options.slice("start", "end", "lines")) if @options["start"]
56
+
57
+ LastLines.new(@options.fetch("lines", LINES))
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end