fun_ci 2.0.0-arm64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +294 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +230 -0
  5. data/exe/fun-ci +7 -0
  6. data/lib/fun_ci/agent/age.rb +15 -0
  7. data/lib/fun_ci/agent/command_support.rb +27 -0
  8. data/lib/fun_ci/agent/commands.rb +27 -0
  9. data/lib/fun_ci/agent/context.rb +10 -0
  10. data/lib/fun_ci/agent/digest.rb +49 -0
  11. data/lib/fun_ci/agent/events.rb +47 -0
  12. data/lib/fun_ci/agent/events_command.rb +61 -0
  13. data/lib/fun_ci/agent/evidence_text.rb +46 -0
  14. data/lib/fun_ci/agent/exit_code.rb +11 -0
  15. data/lib/fun_ci/agent/git.rb +31 -0
  16. data/lib/fun_ci/agent/live_pipeline.rb +35 -0
  17. data/lib/fun_ci/agent/options.rb +75 -0
  18. data/lib/fun_ci/agent/output.rb +81 -0
  19. data/lib/fun_ci/agent/report_stage.rb +73 -0
  20. data/lib/fun_ci/agent/reports.rb +47 -0
  21. data/lib/fun_ci/agent/run_report.rb +30 -0
  22. data/lib/fun_ci/agent/runs_command.rb +34 -0
  23. data/lib/fun_ci/agent/runs_text.rb +27 -0
  24. data/lib/fun_ci/agent/snapshots.rb +39 -0
  25. data/lib/fun_ci/agent/stage_summary.rb +30 -0
  26. data/lib/fun_ci/agent/status_command.rb +36 -0
  27. data/lib/fun_ci/agent/status_json.rb +27 -0
  28. data/lib/fun_ci/agent/status_text.rb +37 -0
  29. data/lib/fun_ci/agent/system_clock.rb +11 -0
  30. data/lib/fun_ci/agent/verdict.rb +32 -0
  31. data/lib/fun_ci/agent/wait_command.rb +71 -0
  32. data/lib/fun_ci/agent/waiting.rb +32 -0
  33. data/lib/fun_ci/agent/why_command.rb +50 -0
  34. data/lib/fun_ci/agent/why_json.rb +35 -0
  35. data/lib/fun_ci/agent/why_text.rb +43 -0
  36. data/lib/fun_ci/cli.rb +128 -0
  37. data/lib/fun_ci/cli_help.rb +65 -0
  38. data/lib/fun_ci/console/board_data.rb +66 -0
  39. data/lib/fun_ci/console/board_heartbeat.rb +40 -0
  40. data/lib/fun_ci/console/console_events.rb +20 -0
  41. data/lib/fun_ci/console/console_log.rb +26 -0
  42. data/lib/fun_ci/console/console_loop.rb +41 -0
  43. data/lib/fun_ci/console/console_session.rb +82 -0
  44. data/lib/fun_ci/console/console_state.rb +42 -0
  45. data/lib/fun_ci/console/key_handler.rb +73 -0
  46. data/lib/fun_ci/console/launcher.rb +43 -0
  47. data/lib/fun_ci/console/milestone_events.rb +38 -0
  48. data/lib/fun_ci/console/milestones.rb +25 -0
  49. data/lib/fun_ci/console/renderer_lookup.rb +51 -0
  50. data/lib/fun_ci/console/renderer_process.rb +60 -0
  51. data/lib/fun_ci/console/run_message.rb +33 -0
  52. data/lib/fun_ci/console/stage_change_detector.rb +32 -0
  53. data/lib/fun_ci/console/stage_events.rb +31 -0
  54. data/lib/fun_ci/console/stop_report.rb +39 -0
  55. data/lib/fun_ci/console/streak_counter.rb +19 -0
  56. data/lib/fun_ci/console/view.rb +47 -0
  57. data/lib/fun_ci/evidence/about.rb +12 -0
  58. data/lib/fun_ci/evidence/about_stage.rb +29 -0
  59. data/lib/fun_ci/evidence/caps.rb +77 -0
  60. data/lib/fun_ci/evidence/catalog.rb +83 -0
  61. data/lib/fun_ci/evidence/collector.rb +95 -0
  62. data/lib/fun_ci/evidence/command_output.rb +45 -0
  63. data/lib/fun_ci/evidence/command_reading.rb +38 -0
  64. data/lib/fun_ci/evidence/command_runner.rb +75 -0
  65. data/lib/fun_ci/evidence/context.rb +21 -0
  66. data/lib/fun_ci/evidence/context_document.rb +29 -0
  67. data/lib/fun_ci/evidence/deadline.rb +15 -0
  68. data/lib/fun_ci/evidence/detection.rb +82 -0
  69. data/lib/fun_ci/evidence/document.rb +59 -0
  70. data/lib/fun_ci/evidence/extract_command.rb +83 -0
  71. data/lib/fun_ci/evidence/extract_options.rb +45 -0
  72. data/lib/fun_ci/evidence/extraction.rb +54 -0
  73. data/lib/fun_ci/evidence/extractors/command.rb +52 -0
  74. data/lib/fun_ci/evidence/extractors/grep.rb +49 -0
  75. data/lib/fun_ci/evidence/extractors/json_log.rb +45 -0
  76. data/lib/fun_ci/evidence/extractors/junit_files.rb +32 -0
  77. data/lib/fun_ci/evidence/extractors/last_lines.rb +25 -0
  78. data/lib/fun_ci/evidence/extractors/log_file.rb +62 -0
  79. data/lib/fun_ci/evidence/extractors/process_tree.rb +51 -0
  80. data/lib/fun_ci/evidence/extractors/section.rb +49 -0
  81. data/lib/fun_ci/evidence/findings.rb +14 -0
  82. data/lib/fun_ci/evidence/line_ranges.rb +21 -0
  83. data/lib/fun_ci/evidence/line_scan.rb +22 -0
  84. data/lib/fun_ci/evidence/log_record.rb +58 -0
  85. data/lib/fun_ci/evidence/masking.rb +55 -0
  86. data/lib/fun_ci/evidence/option_check.rb +53 -0
  87. data/lib/fun_ci/evidence/outcome.rb +15 -0
  88. data/lib/fun_ci/evidence/patterns.rb +22 -0
  89. data/lib/fun_ci/evidence/presets/README.md +7 -0
  90. data/lib/fun_ci/evidence/presets/bun-test.yml +6 -0
  91. data/lib/fun_ci/evidence/presets/cargo-test.yml +6 -0
  92. data/lib/fun_ci/evidence/presets/dart-test.yml +6 -0
  93. data/lib/fun_ci/evidence/presets/deno-test.yml +6 -0
  94. data/lib/fun_ci/evidence/presets/dotnet-test.yml +6 -0
  95. data/lib/fun_ci/evidence/presets/ecs.yml +3 -0
  96. data/lib/fun_ci/evidence/presets/eslint.yml +6 -0
  97. data/lib/fun_ci/evidence/presets/exunit.yml +6 -0
  98. data/lib/fun_ci/evidence/presets/gcc.yml +6 -0
  99. data/lib/fun_ci/evidence/presets/go-build.yml +5 -0
  100. data/lib/fun_ci/evidence/presets/go-test.yml +6 -0
  101. data/lib/fun_ci/evidence/presets/gradle.yml +6 -0
  102. data/lib/fun_ci/evidence/presets/gtest.yml +6 -0
  103. data/lib/fun_ci/evidence/presets/jest.yml +6 -0
  104. data/lib/fun_ci/evidence/presets/logstash.yml +3 -0
  105. data/lib/fun_ci/evidence/presets/maven.yml +6 -0
  106. data/lib/fun_ci/evidence/presets/minitest.yml +6 -0
  107. data/lib/fun_ci/evidence/presets/mocha.yml +6 -0
  108. data/lib/fun_ci/evidence/presets/mypy.yml +6 -0
  109. data/lib/fun_ci/evidence/presets/node-test.yml +6 -0
  110. data/lib/fun_ci/evidence/presets/phpunit.yml +6 -0
  111. data/lib/fun_ci/evidence/presets/pino.yml +5 -0
  112. data/lib/fun_ci/evidence/presets/prove.yml +6 -0
  113. data/lib/fun_ci/evidence/presets/pytest.yml +6 -0
  114. data/lib/fun_ci/evidence/presets/rspec.yml +6 -0
  115. data/lib/fun_ci/evidence/presets/rubocop.yml +6 -0
  116. data/lib/fun_ci/evidence/presets/ruff.yml +6 -0
  117. data/lib/fun_ci/evidence/presets/rustc.yml +6 -0
  118. data/lib/fun_ci/evidence/presets/shellcheck.yml +5 -0
  119. data/lib/fun_ci/evidence/presets/structlog.yml +4 -0
  120. data/lib/fun_ci/evidence/presets/swift-test.yml +5 -0
  121. data/lib/fun_ci/evidence/presets/tsc.yml +6 -0
  122. data/lib/fun_ci/evidence/presets/unittest.yml +6 -0
  123. data/lib/fun_ci/evidence/presets/vitest.yml +6 -0
  124. data/lib/fun_ci/evidence/presets.rb +30 -0
  125. data/lib/fun_ci/evidence/problem.rb +9 -0
  126. data/lib/fun_ci/evidence/process_table.rb +40 -0
  127. data/lib/fun_ci/evidence/saved_run.rb +22 -0
  128. data/lib/fun_ci/evidence/section_walk.rb +49 -0
  129. data/lib/fun_ci/evidence/settings.rb +64 -0
  130. data/lib/fun_ci/evidence/settings_check.rb +78 -0
  131. data/lib/fun_ci/evidence/source.rb +34 -0
  132. data/lib/fun_ci/evidence/sources.rb +21 -0
  133. data/lib/fun_ci/evidence/stage_context.rb +30 -0
  134. data/lib/fun_ci/evidence/stamp.rb +10 -0
  135. data/lib/fun_ci/evidence/start.rb +36 -0
  136. data/lib/fun_ci/evidence/worktree.rb +62 -0
  137. data/lib/fun_ci/persistence/active_runs.rb +69 -0
  138. data/lib/fun_ci/persistence/database.rb +81 -0
  139. data/lib/fun_ci/persistence/null_recorder.rb +25 -0
  140. data/lib/fun_ci/persistence/output_tail.rb +24 -0
  141. data/lib/fun_ci/persistence/pipeline_recorder.rb +120 -0
  142. data/lib/fun_ci/persistence/pipeline_run.rb +68 -0
  143. data/lib/fun_ci/persistence/project_runs.rb +52 -0
  144. data/lib/fun_ci/persistence/raw_outputs.rb +46 -0
  145. data/lib/fun_ci/persistence/retention.rb +25 -0
  146. data/lib/fun_ci/persistence/run_status.rb +31 -0
  147. data/lib/fun_ci/persistence/stage_job.rb +91 -0
  148. data/lib/fun_ci/persistence/state_dir.rb +26 -0
  149. data/lib/fun_ci/persistence/write_trouble.rb +46 -0
  150. data/lib/fun_ci/pipeline/background_fork.rb +42 -0
  151. data/lib/fun_ci/pipeline/command_executor.rb +58 -0
  152. data/lib/fun_ci/pipeline/gate.rb +35 -0
  153. data/lib/fun_ci/pipeline/git_environment.rb +16 -0
  154. data/lib/fun_ci/pipeline/head_part.rb +40 -0
  155. data/lib/fun_ci/pipeline/in_place.rb +17 -0
  156. data/lib/fun_ci/pipeline/output_window.rb +73 -0
  157. data/lib/fun_ci/pipeline/pipeline_forker.rb +40 -0
  158. data/lib/fun_ci/pipeline/process_runner.rb +101 -0
  159. data/lib/fun_ci/pipeline/progress_reporter.rb +31 -0
  160. data/lib/fun_ci/pipeline/run_canceller.rb +75 -0
  161. data/lib/fun_ci/pipeline/slot.rb +30 -0
  162. data/lib/fun_ci/pipeline/slot_lock.rb +13 -0
  163. data/lib/fun_ci/pipeline/slot_run.rb +86 -0
  164. data/lib/fun_ci/pipeline/stage_dir.rb +50 -0
  165. data/lib/fun_ci/pipeline/stage_end.rb +70 -0
  166. data/lib/fun_ci/pipeline/stage_execution.rb +67 -0
  167. data/lib/fun_ci/pipeline/stage_runner.rb +63 -0
  168. data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +36 -0
  169. data/lib/fun_ci/pipeline/tail_segments.rb +60 -0
  170. data/lib/fun_ci/pipeline/test_report.rb +48 -0
  171. data/lib/fun_ci/pipeline/trigger.rb +93 -0
  172. data/lib/fun_ci/pipeline/trigger_command.rb +62 -0
  173. data/lib/fun_ci/pipeline/trigger_params.rb +48 -0
  174. data/lib/fun_ci/pipeline/workspaces.rb +22 -0
  175. data/lib/fun_ci/pipeline/worktree_pool.rb +66 -0
  176. data/lib/fun_ci/pipeline/worktree_prune.rb +49 -0
  177. data/lib/fun_ci/pipeline/worktrees.rb +45 -0
  178. data/lib/fun_ci/setup/agent_instructions.rb +31 -0
  179. data/lib/fun_ci/setup/commands.rb +49 -0
  180. data/lib/fun_ci/setup/git_hooks.rb +20 -0
  181. data/lib/fun_ci/setup/hook_script.rb +47 -0
  182. data/lib/fun_ci/setup/hook_writer.rb +57 -0
  183. data/lib/fun_ci/setup/installer.rb +71 -0
  184. data/lib/fun_ci/setup/legacy_hooks.rb +38 -0
  185. data/lib/fun_ci/setup/maven_linter_detector.rb +26 -0
  186. data/lib/fun_ci/setup/project_config.rb +68 -0
  187. data/lib/fun_ci/setup/project_detector.rb +54 -0
  188. data/lib/fun_ci/setup/settings.rb +36 -0
  189. data/lib/fun_ci/setup/setup_checker.rb +36 -0
  190. data/lib/fun_ci/setup/stage_templates/javascript.rb +18 -0
  191. data/lib/fun_ci/setup/stage_templates/managed.rb +24 -0
  192. data/lib/fun_ci/setup/stage_templates/native.rb +25 -0
  193. data/lib/fun_ci/setup/stage_templates/scripting.rb +27 -0
  194. data/lib/fun_ci/setup/stage_templates/shell.rb +16 -0
  195. data/lib/fun_ci/setup/stage_templates.rb +25 -0
  196. data/lib/fun_ci/setup/template_writer.rb +28 -0
  197. data/lib/fun_ci/version.rb +6 -0
  198. data/lib/fun_ci.rb +13 -0
  199. data/libexec/fun-ci-renderer +0 -0
  200. metadata +286 -0
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "pipeline_run"
4
+ require_relative "stage_job"
5
+ require_relative "run_status"
6
+ require_relative "write_trouble"
7
+ require "json"
8
+ require_relative "null_recorder"
9
+ require_relative "project_runs"
10
+ require_relative "retention"
11
+
12
+ module FunCi
13
+ module Persistence
14
+ class DbRecorder
15
+ KEPT_RUNS = Retention::KEPT_RUNS
16
+
17
+ attr_reader :db, :db_path, :pipeline_run_id, :trouble
18
+
19
+ def self.for_background(db_path, pipeline_run_id, trouble: WriteTrouble.silent(db_path))
20
+ db = Database.connection(db_path)
21
+ new(db, pipeline_run_id: pipeline_run_id, trouble: trouble)
22
+ end
23
+
24
+ # `trouble` decides what a refused write does; by default, nothing is said.
25
+ def initialize(db, pipeline_run_id: nil, trouble: nil)
26
+ @db = db
27
+ @db_path = db.filename("main")
28
+ @pipeline_run_id = pipeline_run_id
29
+ @trouble = trouble || WriteTrouble.silent(@db_path)
30
+ end
31
+
32
+ # Says what goes wrong writing to the database on `out` from now on.
33
+ def report_trouble_to(out)
34
+ @trouble = WriteTrouble.new(out, @db_path)
35
+ end
36
+
37
+ # The block's value, or nil when the database refused a write in it.
38
+ def tolerating(&) = @trouble.guard(&)
39
+
40
+ def close
41
+ @db.close
42
+ rescue StandardError
43
+ nil
44
+ end
45
+
46
+ # Notes this process as the one running the pipeline, so cancelling can stop it.
47
+ def create_run(commit_hash:, branch:, project_path: nil)
48
+ tolerating do
49
+ @pipeline_run_id = PipelineRun.create(@db, commit_hash: commit_hash, branch: branch,
50
+ project_path: project_path)
51
+ PipelineRun.store_trigger_pid(@db, @pipeline_run_id, Process.pid)
52
+ Retention.apply(@db, project_path) if project_path
53
+ @pipeline_run_id
54
+ end
55
+ end
56
+
57
+ def stage_process(job_id, pid)
58
+ tolerating { StageJob.store_pid(@db, job_id, pid) }
59
+ end
60
+
61
+ # Once it has exited its pid may be reused, so a cancel must not signal it.
62
+ def foreground_done
63
+ tolerating { PipelineRun.store_trigger_pid(@db, @pipeline_run_id, nil) }
64
+ end
65
+
66
+ def slot_taken(lock_file)
67
+ tolerating { PipelineRun.store_slot_lock(@db, @pipeline_run_id, lock_file) }
68
+ end
69
+
70
+ def start_stage(stage, budget: nil)
71
+ return nil unless @pipeline_run_id
72
+
73
+ tolerating do
74
+ ensure_running
75
+ job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
76
+ StageJob.update_status(@db, job_id, "running")
77
+ job_id
78
+ end
79
+ end
80
+
81
+ # Keeps a failed stage's evidence, and the output's tail where an older
82
+ # fun-ci sharing the database reads it.
83
+ def keep_evidence(job_id, document)
84
+ tolerating do
85
+ StageJob.keep_evidence(@db, job_id, JSON.generate(document.to_h))
86
+ StageJob.keep_output(@db, job_id, document.tail)
87
+ end
88
+ end
89
+
90
+ # Keeps what a failed stage printed, masked and cut to its window.
91
+ def keep_raw(job_id, text)
92
+ tolerating { RawOutputs.beside(@db_path).write(job_id, text) }
93
+ end
94
+
95
+ # The stages that shared the slot with this one (architecture.md, "Evidence of a failed stage").
96
+ def alongside(job_id) = tolerating { StageJob.alongside(@db, job_id) } || []
97
+
98
+ def keep_exit(job_id, exit_status, signal)
99
+ tolerating { StageJob.keep_exit(@db, job_id, exit_status, signal) }
100
+ end
101
+
102
+ # Records the stage's outcome, then settles the run's status from its stages.
103
+ def end_stage(job_id, status)
104
+ tolerating do
105
+ StageJob.update_status(@db, job_id, status)
106
+ RunStatus.settle(@db, @pipeline_run_id)
107
+ end
108
+ end
109
+
110
+ private
111
+
112
+ def ensure_running
113
+ run = PipelineRun.find(@db, @pipeline_run_id)
114
+ return unless run && run[:status] == "scheduled"
115
+
116
+ PipelineRun.update_status(@db, @pipeline_run_id, "running")
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ module PipelineRun
8
+ COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at].freeze
9
+ SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
10
+
11
+ def self.create(db, commit_hash:, branch:, project_path: nil)
12
+ now = Time.now.utc.iso8601
13
+ db.execute(
14
+ "INSERT INTO pipeline_runs (commit_hash, branch, project_path, status, created_at, updated_at) " \
15
+ "VALUES (?, ?, ?, 'scheduled', ?, ?)",
16
+ [commit_hash, branch, project_path, now, now]
17
+ )
18
+ db.last_insert_row_id
19
+ end
20
+
21
+ def self.find(db, id)
22
+ query(db, "WHERE id = ?", id).first
23
+ end
24
+
25
+ def self.find_by_branch(db, branch)
26
+ query(db, "WHERE branch = ? ORDER BY id DESC", branch)
27
+ end
28
+
29
+ def self.find_by_commit(db, commit_hash)
30
+ query(db, "WHERE commit_hash = ? ORDER BY id DESC", commit_hash)
31
+ end
32
+
33
+ # The forked slow suite's process.
34
+ def self.store_pid(db, id, pid)
35
+ db.execute("UPDATE pipeline_runs SET pid = ? WHERE id = ?", [pid, id])
36
+ end
37
+
38
+ # The lock file of the worktree slot the run holds.
39
+ def self.store_slot_lock(db, id, path)
40
+ db.execute("UPDATE pipeline_runs SET slot_lock = ? WHERE id = ?", [path, id])
41
+ end
42
+
43
+ # The process running the pipeline itself.
44
+ def self.store_trigger_pid(db, id, pid)
45
+ db.execute("UPDATE pipeline_runs SET trigger_pid = ? WHERE id = ?", [pid, id])
46
+ end
47
+
48
+ # When an agent last polled the run, waiting on it (acceptance-tests.md, AT-9.12).
49
+ def self.mark_waited(db, id, at)
50
+ db.execute("UPDATE pipeline_runs SET waited_at = ? WHERE id = ?", [at, id])
51
+ end
52
+
53
+ def self.update_status(db, id, new_status)
54
+ now = Time.now.utc.iso8601
55
+ db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, now, id])
56
+ end
57
+
58
+ def self.recent(db, limit: 20)
59
+ query(db, "ORDER BY id DESC LIMIT ?", limit)
60
+ end
61
+
62
+ def self.query(db, clause, param)
63
+ db.execute("#{SELECT} #{clause}", [param]).map { |row| COLUMNS.zip(row).to_h }
64
+ end
65
+ private_class_method :query
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "pipeline_run"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # The runs of one project. Every project records its runs in the same
8
+ # database, so each query is scoped to the project's path.
9
+ class ProjectRuns
10
+ def initialize(db, project)
11
+ @db = db
12
+ @project = project
13
+ end
14
+
15
+ def latest_of_commit(sha) = query("AND commit_hash = ? ORDER BY id DESC LIMIT 1", sha).first
16
+
17
+ # The commit of the first run on the same branch after this one, if any.
18
+ def superseded_by(run)
19
+ query("AND branch = ? AND id > ? ORDER BY id LIMIT 1", run[:branch], run[:id]).first&.fetch(:commit_hash)
20
+ end
21
+
22
+ # The newest runs first, on one branch when one is named.
23
+ def recent(limit:, branch: nil)
24
+ return query("ORDER BY id DESC LIMIT ?", limit) unless branch
25
+
26
+ query("AND branch = ? ORDER BY id DESC LIMIT ?", branch, limit)
27
+ end
28
+
29
+ # Drops the kept output and an older fun-ci's failures of every run but the
30
+ # project's newest `keep`, and marks their stages pruned.
31
+ def forget_output(keep:)
32
+ @db.execute("UPDATE stage_jobs SET output_tail = NULL, failures = NULL, pruned = 1 WHERE pipeline_run_id IN " \
33
+ "(SELECT id FROM pipeline_runs WHERE project_path = ? ORDER BY id DESC LIMIT -1 OFFSET ?)",
34
+ [@project, keep])
35
+ end
36
+
37
+ # The stages of every run but the project's newest `keep`.
38
+ def stage_ids_beyond(keep:)
39
+ @db.execute("SELECT id FROM stage_jobs WHERE pipeline_run_id IN " \
40
+ "(SELECT id FROM pipeline_runs WHERE project_path = ? ORDER BY id DESC LIMIT -1 OFFSET ?)",
41
+ [@project, keep]).flatten
42
+ end
43
+
44
+ private
45
+
46
+ def query(clause, *params)
47
+ @db.execute("#{PipelineRun::SELECT} WHERE project_path = ? #{clause}", [@project, *params])
48
+ .map { |row| PipelineRun::COLUMNS.zip(row).to_h }
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "zlib"
5
+
6
+ module FunCi
7
+ module Persistence
8
+ # What each failed stage printed, masked and cut to its window, kept as a
9
+ # gzip file per stage in a directory beside the database: SQLite would
10
+ # not give a deleted blob's space back without a VACUUM (architecture.md, "Evidence of a failed stage").
11
+ class RawOutputs
12
+ def self.beside(db_path) = new(File.join(File.dirname(db_path), "raw"))
13
+
14
+ def initialize(dir)
15
+ @dir = dir
16
+ end
17
+
18
+ def write(job_id, text)
19
+ FileUtils.mkdir_p(@dir)
20
+ Zlib::GzipWriter.open(path(job_id)) { |gzip| gzip.write(text) }
21
+ end
22
+
23
+ def read(job_id)
24
+ File.exist?(path(job_id)) ? Zlib::GzipReader.open(path(job_id), &:read) : nil
25
+ end
26
+
27
+ # How many bytes it kept, from the size gzip records at the file's end.
28
+ def bytes(job_id)
29
+ File.exist?(path(job_id)) ? File.binread(path(job_id), 4, File.size(path(job_id)) - 4).unpack1("V") : nil
30
+ end
31
+
32
+ def delete(job_ids) = job_ids.each { |job_id| FileUtils.rm_f(path(job_id)) }
33
+
34
+ # Deletes what it kept of every stage but those named.
35
+ def keep_only(job_ids)
36
+ kept = job_ids.to_set(&:to_s)
37
+ Dir.glob("*.gz", base: @dir).reject { |name| kept.include?(File.basename(name, ".gz")) }
38
+ .each { |name| FileUtils.rm_f(File.join(@dir, name)) }
39
+ end
40
+
41
+ private
42
+
43
+ def path(job_id) = File.join(@dir, "#{job_id}.gz")
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "project_runs"
4
+ require_relative "raw_outputs"
5
+ require_relative "stage_job"
6
+
7
+ module FunCi
8
+ module Persistence
9
+ # How long fun-ci keeps what failed stages left, applied when a run
10
+ # starts: the evidence for a project's 50 newest runs, the raw output for
11
+ # its 10 newest, and no raw output whose stage row is gone.
12
+ module Retention
13
+ KEPT_RUNS = 50
14
+ KEPT_RAW = 10
15
+
16
+ def self.apply(db, project)
17
+ runs = ProjectRuns.new(db, project)
18
+ runs.forget_output(keep: KEPT_RUNS)
19
+ raw = RawOutputs.beside(db.filename("main"))
20
+ raw.delete(runs.stage_ids_beyond(keep: KEPT_RAW))
21
+ raw.keep_only(StageJob.ids(db))
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # A run's status, worked out from its stages alone (acceptance-tests.md,
8
+ # AT-7.1): failed once any stage failed or timed out, completed once all
9
+ # four passed, and unchanged until then. One statement reads the stage rows
10
+ # and writes the run, so the foreground pipeline and the forked slow suite
11
+ # can't overwrite each other's verdict; failed, completed and cancelled
12
+ # runs are never touched again.
13
+ module RunStatus
14
+ OF_RUN = "pipeline_runs.id = stage_jobs.pipeline_run_id"
15
+ DERIVED = <<~SQL.freeze
16
+ CASE
17
+ WHEN EXISTS (SELECT 1 FROM stage_jobs WHERE #{OF_RUN} AND status IN ('failed', 'timed_out')) THEN 'failed'
18
+ WHEN (SELECT COUNT(DISTINCT stage) FROM stage_jobs WHERE #{OF_RUN} AND status = 'completed'
19
+ AND stage IN ('lint', 'build', 'fast', 'slow')) = 4 THEN 'completed'
20
+ ELSE status
21
+ END
22
+ SQL
23
+ SETTLE = "UPDATE pipeline_runs SET status = #{DERIVED}, updated_at = ? " \
24
+ "WHERE id = ? AND status IN ('scheduled', 'running') AND status != #{DERIVED}".freeze
25
+
26
+ def self.settle(db, id)
27
+ db.execute(SETTLE, [Time.now.utc.iso8601, id])
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ module StageJob
8
+ TERMINAL_STATUSES = %w[completed failed timed_out cancelled].freeze
9
+ FIELDS = %i[id pipeline_run_id stage status started_at completed_at finished_order output_tail failures
10
+ exit_status signal budget pruned evidence].freeze
11
+ COLUMNS = FIELDS.join(", ")
12
+ NEXT_IN_RUN = "(SELECT COALESCE(MAX(others.finished_order), 0) + 1 FROM stage_jobs AS others " \
13
+ "WHERE others.pipeline_run_id = stage_jobs.pipeline_run_id)"
14
+ TIMESTAMP_COLUMNS = TERMINAL_STATUSES.to_h { |status| [status, "completed_at"] }
15
+ .merge("running" => "started_at").freeze
16
+
17
+ def self.create(db, pipeline_run_id:, stage:, budget: nil)
18
+ db.execute(
19
+ "INSERT INTO stage_jobs (pipeline_run_id, stage, status, budget) VALUES (?, ?, 'scheduled', ?)",
20
+ [pipeline_run_id, stage, budget]
21
+ )
22
+ db.last_insert_row_id
23
+ end
24
+
25
+ # The stage script's process, which leads a process group of its own.
26
+ def self.store_pid(db, id, pid)
27
+ db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
28
+ end
29
+
30
+ # The end of what the stage printed, kept when it failed (acceptance-tests.md, AT-9.5).
31
+ def self.keep_output(db, id, tail)
32
+ db.execute("UPDATE stage_jobs SET output_tail = ? WHERE id = ?", [tail, id])
33
+ end
34
+
35
+ # What was kept about why the stage failed, as JSON (architecture.md, "Evidence of a failed stage").
36
+ def self.keep_evidence(db, id, evidence_json)
37
+ db.execute("UPDATE stage_jobs SET evidence = ? WHERE id = ?", [evidence_json, id])
38
+ end
39
+
40
+ # How the stage's process exited: its status, or the signal that ended it.
41
+ def self.keep_exit(db, id, exit_status, signal)
42
+ db.execute("UPDATE stage_jobs SET exit_status = ?, signal = ? WHERE id = ?", [exit_status, signal, id])
43
+ end
44
+
45
+ # The other stages of its run that were running at any moment since it started.
46
+ def self.alongside(db, id)
47
+ db.execute("SELECT others.stage FROM stage_jobs AS others JOIN stage_jobs AS me ON me.id = ? " \
48
+ "WHERE others.pipeline_run_id = me.pipeline_run_id AND others.id != me.id " \
49
+ "AND others.started_at IS NOT NULL " \
50
+ "AND (others.completed_at IS NULL OR others.completed_at > me.started_at) ORDER BY others.id",
51
+ [id]).flatten
52
+ end
53
+
54
+ def self.ids(db) = db.execute("SELECT id FROM stage_jobs").flatten
55
+
56
+ def self.find(db, id)
57
+ row = db.execute("SELECT #{COLUMNS} FROM stage_jobs WHERE id = ?", [id]).first
58
+ return nil unless row
59
+
60
+ row_to_hash(row)
61
+ end
62
+
63
+ # The run's jobs, in the order they were created.
64
+ def self.for_run(db, pipeline_run_id)
65
+ db.execute("SELECT #{COLUMNS} FROM stage_jobs WHERE pipeline_run_id = ? ORDER BY id", [pipeline_run_id])
66
+ .map { |row| row_to_hash(row) }
67
+ end
68
+
69
+ # A stage that finishes is numbered after every other of its run that
70
+ # finished before it, in the same statement, so the order holds however
71
+ # close together they finished.
72
+ def self.update_status(db, id, new_status)
73
+ column = TIMESTAMP_COLUMNS.fetch(new_status)
74
+ order = TERMINAL_STATUSES.include?(new_status) ? NEXT_IN_RUN : "finished_order"
75
+ db.execute("UPDATE stage_jobs SET status = ?, #{column} = ?, finished_order = #{order} WHERE id = ?",
76
+ [new_status, Time.now.utc.iso8601(3), id])
77
+ end
78
+
79
+ def self.elapsed_duration(job)
80
+ return nil unless job[:started_at] && job[:completed_at]
81
+
82
+ Time.parse(job[:completed_at]) - Time.parse(job[:started_at])
83
+ end
84
+
85
+ def self.row_to_hash(row)
86
+ FIELDS.zip(row).to_h
87
+ end
88
+ private_class_method :row_to_hash
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # Where fun-ci keeps its database: the user's state directory, which every
8
+ # process of that user agrees on (acceptance-tests.md, AT-9.1). Only that
9
+ # user can read it, since what fun-ci keeps can hold a secret masking missed.
10
+ module StateDir
11
+ MODE = 0o700
12
+
13
+ # Creates the directory if need be, and makes it the user's alone.
14
+ def self.prepare(path)
15
+ FileUtils.mkdir_p(path)
16
+ File.chmod(MODE, path)
17
+ end
18
+
19
+ def self.path(env)
20
+ base = env["XDG_STATE_HOME"].to_s
21
+ base = File.join(env.fetch("HOME"), ".local", "state") if base.empty?
22
+ File.join(base, "fun-ci")
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sqlite3"
4
+ require "stringio"
5
+
6
+ module FunCi
7
+ module Persistence
8
+ # What happens when the database won't take a write (acceptance-tests.md,
9
+ # AT-8.2, AT-8.4): the pipeline carries on without recording it, and the
10
+ # developer is told once, with what to do about it.
11
+ class WriteTrouble
12
+ # SystemCallError: the files kept beside the database, such as raw output.
13
+ UNWRITABLE = [SQLite3::FullException, SQLite3::ReadOnlyException, SQLite3::IOException,
14
+ SQLite3::CantOpenException, SystemCallError].freeze
15
+ BUSY = "fun-ci: couldn't record this run: the database stayed busy. Your stages still ran. " \
16
+ "Trigger it again to record it."
17
+ STILL_RAN = "Check the disk has space. Your stages still ran."
18
+
19
+ # Says nothing, for a process whose output nobody reads.
20
+ def self.silent(db_path) = new(StringIO.new, db_path)
21
+
22
+ def initialize(out, db_path)
23
+ @out = out
24
+ @db_path = db_path
25
+ @told = false
26
+ end
27
+
28
+ # The block's value, or nil when the database refused the write.
29
+ def guard
30
+ yield
31
+ rescue SQLite3::BusyException
32
+ tell(BUSY)
33
+ rescue *UNWRITABLE
34
+ tell("fun-ci: can't write to the database at #{@db_path}. #{STILL_RAN}")
35
+ end
36
+
37
+ private
38
+
39
+ def tell(message)
40
+ @out.puts(message) unless @told
41
+ @told = true
42
+ nil
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../persistence/pipeline_recorder"
4
+ require_relative "../persistence/pipeline_run"
5
+
6
+ module FunCi
7
+ module Pipeline
8
+ # Runs the slow suite in a forked child. A child must not inherit an open
9
+ # SQLite connection, so the parent's recorder is closed before the fork and
10
+ # #launch answers a fresh one for the parent to carry on with. The slot was
11
+ # shared for the slow suite before the fork; after it, each process drops
12
+ # the other's share and keeps its own.
13
+ class BackgroundFork
14
+ def initialize(recorder, slot)
15
+ @recorder = recorder
16
+ @slot = slot
17
+ end
18
+
19
+ def launch(db_path:, pipeline_run_id:, job_id:, executor:)
20
+ @recorder.close
21
+ pid = fork { run_in_child(db_path, pipeline_run_id, job_id, executor) }
22
+ @slot.release
23
+ Process.detach(pid)
24
+ reopened = Persistence::DbRecorder.for_background(db_path, pipeline_run_id, trouble: @recorder.trouble)
25
+ reopened.tolerating { Persistence::PipelineRun.store_pid(reopened.db, pipeline_run_id, pid) }
26
+ reopened
27
+ end
28
+
29
+ private
30
+
31
+ # Records its own pid before the slow suite starts, so a cancel can't
32
+ # miss it; the parent records it too, for its callers.
33
+ def run_in_child(db_path, pipeline_run_id, job_id, executor)
34
+ @slot.release
35
+ recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
36
+ recorder.tolerating { Persistence::PipelineRun.store_pid(recorder.db, pipeline_run_id, Process.pid) }
37
+ executor.call(recorder, job_id)
38
+ recorder.close
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "timeout"
4
+ require_relative "process_runner"
5
+
6
+ module FunCi
7
+ module Pipeline
8
+ # Runs a stage command within its budget and answers [output, status, timed_out].
9
+ # An injected command runner replaces the real process and signals a blown
10
+ # budget by raising Timeout::Error. A real process inherits fun-ci's own
11
+ # environment; an injected runner sees `environment` in its place.
12
+ class CommandExecutor
13
+ include ProcessRunner
14
+
15
+ def initialize(command_runner, dir = Dir.pwd, environment = {})
16
+ @command_runner = command_runner
17
+ @dir = dir
18
+ @environment = environment
19
+ end
20
+
21
+ # Yields the pid of the process the command runs in, when there is one.
22
+ # The launch's `env` is added to the command's environment; an injected
23
+ # runner that takes a second argument is given the whole of it. What
24
+ # the command prints is written to the launch's `output`, and answered
25
+ # as the window keeps it; its `before_kill` is called with the pid of
26
+ # a command over budget, while it still runs.
27
+ def call(cmd, budget, launch = Launch.new, &)
28
+ launch = launch.with(chdir: @dir)
29
+ return run_process_with_timeout(cmd, budget, launch: launch, &) unless @command_runner
30
+
31
+ injected(cmd, launch, &)
32
+ end
33
+
34
+ private
35
+
36
+ # An injected runner signals a blown budget by raising Timeout::Error,
37
+ # which is its moment before the kill.
38
+ def injected(cmd, launch)
39
+ started = nil
40
+ printed, status = run_injected(cmd, launch.env) { |pid| (started = pid) && (yield pid if block_given?) }
41
+ [text(launch.output << printed), status, false]
42
+ rescue Timeout::Error
43
+ launch.before_kill&.call(started)
44
+ ["", nil, true]
45
+ end
46
+
47
+ # A lambda's own, or those of an object's #call.
48
+ def parameters
49
+ @command_runner.respond_to?(:parameters) ? @command_runner.parameters : @command_runner.method(:call).parameters
50
+ end
51
+
52
+ def run_injected(cmd, env, &)
53
+ takes_env = parameters.count { |kind, _| %i[req opt].include?(kind) } > 1
54
+ takes_env ? @command_runner.call(cmd, @environment.merge(env), &) : @command_runner.call(cmd, &)
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "io/nonblock"
4
+
5
+ module FunCi
6
+ module Pipeline
7
+ # A pipe a child process waits on until the caller opens it. Ruby makes
8
+ # pipes non-blocking, and a child shares that setting with the parent, so
9
+ # the child's end is made blocking: otherwise a child reading before the
10
+ # caller has opened the gate fails instead of waiting.
11
+ class Gate
12
+ attr_reader :child_end
13
+
14
+ def self.create
15
+ child_end, opener = IO.pipe
16
+ child_end.nonblock = false
17
+ new(child_end, opener)
18
+ end
19
+
20
+ def initialize(child_end, opener)
21
+ @child_end = child_end
22
+ @opener = opener
23
+ end
24
+
25
+ # Lets the child through. A child that has gone already needs nothing.
26
+ def open
27
+ @opener.puts
28
+ rescue Errno::EPIPE
29
+ nil
30
+ ensure
31
+ @opener.close
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Pipeline
5
+ # Git runs hooks with variables naming the repository and index that ran
6
+ # them. Passed on, they would point fun-ci's git commands and the stage
7
+ # scripts in a worktree at the checkout's index instead of their own.
8
+ module GitEnvironment
9
+ REPOSITORY = %w[GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_PREFIX GIT_COMMON_DIR GIT_OBJECT_DIRECTORY
10
+ GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_NAMESPACE].freeze
11
+
12
+ # An environment for spawn that unsets each of them.
13
+ CLEAN = REPOSITORY.to_h { |name| [name, nil] }.freeze
14
+ end
15
+ end
16
+ end